diff --git a/.claude/commands/mendix/diff-local.md b/.claude/commands/mendix/diff-local.md index b99ce06e8a..d62f7a2aa8 100644 --- a/.claude/commands/mendix/diff-local.md +++ b/.claude/commands/mendix/diff-local.md @@ -79,7 +79,7 @@ Entity: MyModule.Customer ## How It Works -1. **Finds changed files**: Uses `git diff --name-status` to identify modified `.mxunit` files in the `mprcontents/` folder +1. **Finds changed files**: Uses `git diff --name-status` to identify modified `.mxunit` files in the `mprcontents/` folder. Git runs in the project's own directory, so `-p` may point into a repository from anywhere. A unit file moved to another name is another unit ID, so it is reported as one unit deleted and one added 2. **Reads both versions**: Current version from disk, git version using `git show` 3. **Parses BSON**: Extracts the unit type and content from the binary BSON format 4. **Generates MDL**: Converts both versions to MDL for human-readable comparison @@ -89,16 +89,34 @@ Entity: MyModule.Customer The command generates MDL representations for: -- **Entities**: Shows attributes, constraints, and entity type -- **Microflows**: Shows parameters and return type -- **Nanoflows**: Shows parameters and return type -- **Enumerations**: Shows values and captions -- **Pages**: Shows title and layout reference -- **Snippets**: Shows basic structure -- **Layouts**: Shows basic structure -- **Modules**: Shows module name +- **Domain models**: each entity's attributes, constraints and entity type, its access rules as `grant` statements (as `describe entity` prints them), and the associations +- **Microflows / Nanoflows**: the whole body, as `describe` prints it, `grant execute` included +- **Pages / Snippets / Layouts**: the header properties and the widget tree, as `describe` prints them +- **Enumerations**: values and captions +- **Project settings**: as `describe settings` prints them, with configuration constant values masked +- **Modules**: module name -Other unit types show a generic representation with the type name. +Configuration constant values often hold secrets, so they are never printed. A +changed value still shows, as a masked pair: + +```diff +-alter settings constant 'MyModule.ApiKey' value ++alter settings constant 'MyModule.ApiKey' value + in configuration 'Default'; +``` + +### Changes the MDL does not show + +A unit can change only in properties its rendering leaves out — navigation, +project or module security, a page's allowed roles, an entity index, a +constant's default value. Such a unit is printed as one line, in every format, +naming the properties that changed (never their values): + +``` +Modified: ModuleSecurity MyModule: changed in properties the MDL rendering does not show: ModuleRoles (1 -> 2 items) +``` + +Run `describe` on the element before and after to see the change itself. ## Summary Output @@ -122,6 +140,7 @@ Summary: 2 new, 3 modified, 1 deleted - Use `--format side` when comparing large objects with subtle differences - Use `--ref HEAD~5` to see what changed in the last 5 commits - Use `--ref feature-branch` to compare against a different branch +- Use `--ref main...feature-branch` (three dots) to review only what a branch changed: the old side is the merge base, so changes made on `main` since do not show - After restoring a `.mxunit` with `git checkout`/`git restore`, run `mxcli fix hashes -p app.mpr --repair`: git does not update the `.mpr`'s ContentsHash index, and `mx check` does not notice the stale entry diff --git a/.claude/lint-rules/orphaned_elements.star b/.claude/lint-rules/orphaned_elements.star index b9b22f4dbf..bbaf9115af 100644 --- a/.claude/lint-rules/orphaned_elements.star +++ b/.claude/lint-rules/orphaned_elements.star @@ -7,9 +7,12 @@ # - Work in progress that was forgotten # # Checks: -# - Microflows with no callers (except ACT_ which are UI entry points) -# - Pages with no show_page references (except entry pages) -# - Entities with no references at all +# - Microflows, nanoflows and rules nothing references (except entry-point +# prefixes such as ACT_) +# - Pages nothing references (except likely entry pages) +# +# "Referenced" means what `mxcli refs` and CATALOG.GRAPH_DEAD_ASSETS mean: any +# edge in the catalog's reference graph that points at the document. # # This rule requires REFRESH CATALOG FULL to populate the refs table. # @@ -34,13 +37,23 @@ ENTRY_POINT_PREFIXES = ["ACT_", "SCH_", "WS_", "REST_", "OData_"] # Page name patterns that are likely entry points ENTRY_PAGE_PATTERNS = ["Home", "Login", "Index", "Dashboard"] -# Reference kinds that mean "something causes this microflow to run". These are -# catalog RefKind values (mdl/catalog/builder_references.go); a kind missing here -# turns a live document into a false "not called from anywhere" finding. -MICROFLOW_ENTRY_KINDS = ["call", "schedule", "publish", "datasource", "action", "calculate", "settings", "event"] +# There is deliberately no list of the reference kinds that "count". Every edge +# into a microflow or a page is something that runs or opens it — a call, a +# widget, a calculated attribute, a scheduled event, a published service, an +# event handler, a project setting, the navigation — and the catalog keeps +# adding ways. A hand-kept list of them fell behind: it had the page kinds for a +# home page and a menu item but not the microflow ones, so a microflow started +# only from the navigation was reported with "Remove if unused" while +# `mxcli refs` listed the reference (ako/mxcli#1484). With no list, a new kind +# counts the day the catalog emits it. +# +# TestQUAL004CountsEveryRefKind (mdl/linter) drives every kind the catalog +# declares at a microflow, a nanoflow and a page, and fails on one that does not +# count. -# Reference kinds that mean "something opens this page". -PAGE_ENTRY_KINDS = ["show_page", "home_page", "login_page", "menu_item", "action"] +def is_referenced(qualified_name): + """True when anything in the catalog's reference graph points at the document.""" + return len(refs_to(qualified_name)) > 0 def is_entry_point_microflow(name, prefixes): """Check if a microflow name suggests it's a UI/scheduled entry point.""" @@ -70,34 +83,14 @@ def check(): if is_entry_point_microflow(mf.name, entry_prefixes): continue - # Get references to this microflow - refs = refs_to(mf.qualified_name) - # Anything that causes the microflow to run counts as a caller, not just - # a literal "call" edge. A microflow reached only through one of the other - # kinds was reported as orphaned with the suggestion "Remove if unused": - # - # schedule a scheduled event runs it (Mendix's cron) - # datasource a page or widget uses it as a data source - # action a widget button calls it - # calculate a calculated attribute computes with it - # settings a project setting names it (after-startup, before-shutdown, - # health-check) — the RUNTIME calls it, and nothing in the - # model does - # - # The banking-app report hit the 'datasource' case: DS_CurrentCustomer and - # DS_MyAccounts are both page data sources and both were flagged. The - # CapTrackV4 report hit 'settings': the project's own AfterStartupMicroflow - # was reported as "not called from anywhere. Remove if unused", and taking - # that advice left a dangling name mx check did not catch either — only the - # runtime refused to start. - has_callers = False - for ref in refs: - if ref.ref_kind in MICROFLOW_ENTRY_KINDS: - has_callers = True - break - - if not has_callers: + # a literal "call" edge. Each of these once reached a live microflow + # that this rule then advised deleting: a page data source (DS_ microflows in + # the banking-app report), the project's AfterStartupMicroflow (whose + # removal mx check did not catch — the runtime refused to start), a + # scheduled event, a published REST operation, an entity event handler, + # and a home or menu-item microflow (ako/mxcli#1484). + if not is_referenced(mf.qualified_name): loc = location( module=mf.module_name, document_type=mf.document_noun_title, @@ -116,21 +109,10 @@ def check(): if is_entry_point_page(page.name): continue - # Get references to this page - refs = refs_to(page.qualified_name) - # A page is reachable if anything opens it. Navigation counts: a page that # is only a home page, a login page or a menu item is reached by the - # client, not by a microflow. Counting only 'show_page' reported those as - # orphaned — masked until now by ENTRY_PAGE_PATTERNS, which happens to - # cover the pages most likely to be navigation targets. - is_shown = False - for ref in refs: - if ref.ref_kind in PAGE_ENTRY_KINDS: - is_shown = True - break - - if not is_shown: + # client, not by a microflow. + if not is_referenced(page.qualified_name): loc = location( module=page.module_name, document_type="Page", diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-ensure-db-empty-cluster-over-stopped-system-cluster.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-ensure-db-empty-cluster-over-stopped-system-cluster.json new file mode 100644 index 0000000000..1cdca4a622 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-ensure-db-empty-cluster-over-stopped-system-cluster.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-27","symptom":"ako/mxcli#866: \"In a container restarted with the system cluster down, the `mxcli init` SessionStart hook runs `--ensure-db`, provisions a fresh **empty** cluster, and both apps boot healthy against it. Every captured row (7,966 laps here) is absent, and no error says so. An empty database is indistinguishable from a new one.\"","cause":"`startLocalPostgres` fell back to the #823 user-owned initdb cluster whenever no service manager made the port ready, without asking whether a system cluster for that port existed; once #984 made that fallback actually work on Debian/Ubuntu, 'PostgreSQL is down' became 'PostgreSQL is empty'. Separately, the bootstrap hook's banner named the LAST log line containing 'error', which behind a service-manager diagnostic is pg_ctlcluster's own 'Error:' line, not mxcli's","file":"`cmd/mxcli/docker/ensuredb.go` (`findSystemClusters`, `conflictingSystemCluster`, `systemClusterDownError`, `MXCLI_ENSURE_DB_NEW_CLUSTER`, the NEW-EMPTY warning in `startUserCluster`); `cmd/mxcli/init_hook.go` (banner takes the first `^Error:` line)","insight":"The fallback's success was the defect: an empty database passes every check above it (role/db creation, boot, health), so the only place the difference is knowable is the moment before initdb — decide there, and announce every initdb loudly. Detect by data directory, not by service: Debian's /etc/postgresql///postgresql.conf is world-readable and carries the port, while the data dir is 0700 postgres, so a non-root stat of PG_VERSION returns EACCES — evidence OF a cluster; a probe that counts only err==nil misses exactly the devcontainer user. Match on port so an unrelated cluster never blocks, and treat an existing user-owned cluster as a restart (note the stopped system one, do not refuse). Measured on Ubuntu/PG16 with 16/main down, via a harness calling EnsureDatabase as a non-root user without sudo (privileges dropped in-process, since su was unavailable): fixed -> refusal naming /var/lib/postgresql/16/main; guard stubbed -> 'Database ready' on a new empty ~/.mxcli/postgres cluster, err nil. Trap that cost a cleanup: the first control binary predated the setuid step and ran as ROOT, whose `service postgresql start` succeeded and provisioned into the real cluster — print the uid in any privilege-sensitive harness before believing it. Hermeticity: the stub-PATH test helper must pin the probe root too; dev hosts (this one) have a stopped cluster on 5432","refs":["ako/mxcli#866","#823","mendixlabs/mxcli#984"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-mxcli-test-local-ce0114-generated-registerendpoint-entity-access.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-mxcli-test-local-ce0114-generated-registerendpoint-entity-access.json new file mode 100644 index 0000000000..f7633311a4 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-mxcli-test-local-ce0114-generated-registerendpoint-entity-access.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-10-27", "symptom": "`mxcli test --local` fails the build with `[CE0114] A microflow that does not apply entity access can only call microflows that also do not apply entity access. — at MxTest / Microflow 'RegisterEndpoint' / Call microflow activity ''` whenever the project's after-startup microflow applies entity access; no test runs, and the error says it is \"in the project, not in this run's tests\"", "cause": "`GenerateEndpointMDL` wrote `MxTest.RegisterEndpoint` without `@applyentityaccess`, so it did not apply entity access, and since the 2026-08 chaining fix it calls the project's after-startup microflow. `buildFailureHint` called every error not in a `Test_*` flow \"in the project\", including ones in the two documents the run itself generates", "file": "`cmd/mxcli/testrunner/endpoint.go` (`GenerateEndpointMDL`), `cmd/mxcli/testrunner/build_attribution.go` (`buildFailureHint`, `splitRunnerDocuments`)", "insight": "Measure the rule's DIRECTION before choosing between mirroring the callee and a constant: with mx check on 11.13.0 (security Off and Production), non-applying -> applying is CE0114 while applying -> non-applying and applying -> applying are 0 errors (MDL-SEC22's table says the same for 11.14.0). So `@applyentityaccess` on the generated caller is compatible with every startup microflow, and needs no scrape of the user's flow — which the mirror fix would have added beside the DESCRIBE SETTINGS scrape that already broke once. The setting is neutral at runtime: it governs the caller's own activities only (a Java action ignores it), each callee runs with its own. Prove the chain still runs with a marker the test CONSUMES (`@cleanup none` + delete): a plain count passed on an earlier boot's marker under --skip-app-startup, i.e. the control could not fail. Generated documents with fixed names (RegisterEndpoint, RegisterTestEndpoint) are attributable exactly like the Test_* flows; say so and offer --skip-app-startup as the escape hatch", "refs": ["ako/mxcli#1654", "mendixlabs/mxcli#1645", "mendixlabs/mxcli#1500"], "ce": ["CE0114"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-mxcli-test-stops-parse-error-call-has-own-on-error-clause.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-mxcli-test-stops-parse-error-call-has-own-on-error-clause.json new file mode 100644 index 0000000000..c03158c332 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-mxcli-test-stops-parse-error-call-has-own-on-error-clause.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-10-27", "symptom": "`mxcli test` stops at injection with `Parse error: line 8:75 mismatched input 'ON' expecting ';'` (and a misleading `'Error' is a keyword` hint) when a test body's call has its own `on error rollback` / `on error continue` / custom handler — no test runs", "cause": "`attachOnError` appended the runner's `ON ERROR BEGIN … END ERROR` to every line `startsErrorRaisingCall` matched, unconditionally, so a call with its own clause got a second one. The line-based join also broke on a custom handler (its body's first `;` ended the 'statement') and on a trailing `-- comment`. Both runners had their own copy of the join", "file": "`cmd/mxcli/testrunner/generator_endpoint.go` (`attachOnError`, now shared by both runners), `mdl/visitor/flow_statements.go` (`FlowStatements`), `cmd/mxcli/testrunner/results.go` (`ParseLogResults`)", "insight": "Find statements by PARSING the body (in the generator's own wrapper and language version) and read each call's own onErrorClause from the tree: a regex cannot tell a clause on the next line from `on error` in a string, a comment or a nested handler. Decide per clause from what Mendix stores: `on error rollback` is the default spelled out (same ErrorHandlingType as no clause), so it gets the runner's handler — keeping it made the same exception an ERROR, failed a matching @throws, and aborted the legacy suite; continue/custom are kept verbatim. Keeping a custom handler opened a FALSE GREEN only a runtime run showed: a handler that ends normally rejoins PAST every statement reading the failed call's output (the flow builder's skip-var rule), i.e. past the @expect checks, and the test returned PASS with nothing evaluated. Fixed fail-closed: each assertion counts itself in both branches and a guard after the last fails a short count — an assertion on what the handler set still passes. The legacy log parser had the twin: a RUN line with no verdict read as PASS (StatusPass is the zero value); a re-raising handler did exactly that. Measured verdict table on 11.13.0 in mdl-examples/bug-tests/test-runner-1448-own-error-handler.test.mdl", "refs": ["ako/mxcli#1448", "mendixlabs/mxcli#1550"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-403-reported-as-sign-in.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-403-reported-as-sign-in.json new file mode 100644 index 0000000000..14c08b7cf1 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-403-reported-as-sign-in.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-27","symptom":"`mxcli playwright check /p/secret /p/items -p app.mpr --role User`, where User may not open /p/secret, printed `url=/login.html?messageCode=403#/secret LOGIN PAGE` and `FAIL the sign-in form is showing, not the page (pass --user/--password or --role)` for a user it had just signed in; every later run printed `login demo_user` again while the runtime's session count rose 1, 2, 3, 4, and /p/secret checked alone had its session signed out after every run","cause":"LoginForm (#usernameInput present) was the whole sign-in test, and the client puts a refused page on the same form: the runtime answers POST /xas/ get_session_data (referrer /secret) with 403 and the client goes to /login.html?messageCode=403#/secret. NeedsLogin() read the refusal as a forgotten session, so the stale-session retry signed in beside the live session (a new one per run), and the no-page-past-the-form rule signed a valid session out","file":"cmd/mxcli/docker/pagecheck_assert.go (AccessDenied, NeedsLogin, EvaluatePage), cmd/mxcli/docker/pagecheck.go (ACCESS DENIED in formatPageVerdict, wrongPage), cmd/mxcli/docker/pagecheck_check.go (credentials.roles/signedInAs; checkOrigin sets SignedInAs)","insight":"Record the network before choosing a signal: a 30-line Playwright script logging navigations and /xas/ statuses settled it in one run per case. Signed in and refused: 403, then /login.html?messageCode=403#/. Anonymous, and with a session the runtime forgot: 401, then /login.html#/, guest access on as well (11.15) — the runtime checks authentication before page access, so a 403 never means a stale session. The DOM cannot tell them apart: login.js reads the code as messageCode.split('=')[1], which is undefined, so #loginMessage stays empty and hidden on both and its http403 text (translated anyway) never shows; title, heading and text=34 are identical. The one predicate drove all three symptoms — the wrong advice, the per-run session leak and the needless sign-out — because the retry and sign-out decisions hang on 'needs a login'; check what else reads a predicate before rewording only its message. Count sessions with M2EE runtime_statistics rather than trusting the verdict. Aside: without401 matched the chunk hash 85ce401445201 in the 11.15 client's [Startup] error, which is why 11.15 printed one FAIL line fewer than 11.13 — substring-matching '401' is luckier than it looks","refs":["mdl-examples/bug-tests/pagecheck-403-role-denied.mdl","cmd/mxcli/docker/pagecheck_denied_test.go","ako/mxcli#1216"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-explicit-url-other-host-sign-in-form.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-explicit-url-other-host-sign-in-form.json new file mode 100644 index 0000000000..efc77ee7e0 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-explicit-url-other-host-sign-in-form.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-27","symptom":"`mxcli playwright check http://frontend.local:8180/ -p app.mpr --role R` failed every run with `FAIL the sign-in form is showing, not the page (pass --user/--password or --role)` after printing `login …`, while the same command without the URL passed; the login demonstrably happened (LastLogin advanced). When the session cap was then hit, `login failed` printed no explanation, because `playwright check` never passed RuntimeLogPath to the login","cause":"RunPageCheck signed in on opts.BaseURL — with -p that is the run handshake's `http://localhost:` — and then loaded the page on whatever host the URL named. The session cookies were set for localhost and the browser sent none to frontend.local, so the client started anonymous (`[Startup] … 401`) and drew the sign-in form. With a saved session the renewal path signed in again on localhost and retried, so every run added one live session","file":"cmd/mxcli/docker/pagecheck_check.go (RunPageCheck → groupByOrigin/checkOrigin, RuntimeLogPath default), cmd/mxcli/docker/screenshot_login.go (SignOut), cmd/mxcli/cmd_playwright.go (--sign-out)","insight":"The tell was in the two commands' only difference: project mode probes the base URL itself, an explicit URL may name another host for the same app. Cookies belong to a host, not to an app, so 'signed in' is a property of (host, session), and any check that signs in on one URL and probes another must derive the sign-in origin from the page. Cheapest repro needs no /etc/hosts: `run --local` prints `http://127.0.0.1:/` while the handshake says localhost, so pasting the boot URL into `playwright check -p` fails the same way. Count sessions rather than trusting the verdict: the M2EE `runtime_statistics` action lists `user_sessions` per user — three failing runs, three sessions; four fixed runs, one; at six the trial cap refused the sign-in. Wrong turn to skip: the `[Startup] … 401` line is not the cause (it is the client's anonymous start, present on passing runs too), and the cap was not the cause either (reporter ruled it out) — it was the consequence. Sibling not fixed: a /p/ page the role may not open lands on `/login.html?messageCode=403#/…` and is reported as 'the sign-in form is showing (pass --user/--password or --role)' although the user is signed in; such a session is now signed out rather than re-signed-in beside every run","refs":["ako/mxcli#1216","#16"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-unknown-page-url-passes-as-home.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-unknown-page-url-passes-as-home.json new file mode 100644 index 0000000000..0880982a21 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-unknown-page-url-passes-as-home.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-27","symptom":"`mxcli playwright check / /p/doesnotexist -p app.mpr` printed the home page's verdict for both (`title=\"Mendix - Homepage\" h=\"Welcome to your homepage\"`) and `OK 2 page(s)`, exit 0 — a page URL that no page has passed the check","cause":"The probe already captured the final location (finalUrl) but nothing compared it with the URL asked for, and nothing printed it. The runtime answers every /p/ with `301 Location: /#/` whether or not a page has that URL; the client then replaces the location with /p/ for a known page and with / for an unknown one, rendering the home page plus an information dialog, with HTTP 200 throughout — so status, title, text and console were all clean","file":"cmd/mxcli/docker/pagecheck.go (ProbePages sets RequestedURL; landedAt/wrongPage; url= in formatPageVerdict; .mx-dialog-info in alerts), cmd/mxcli/docker/pagecheck_assert.go (EvaluatePage)","insight":"Measure the routing before choosing the signal: one curl (`curl -sD- /p/x` → 301 to /#/x, identical for known and unknown pages) and a 20ms poll of location.pathname in Playwright settled it on 11.13.0 and 11.15.0 in minutes — the client redirects (history.replaceState) rather than rendering in place, so the final path is a language-independent signal and needs no second load of the home page to compare titles. The not-found dialog ('The page you requested was not found. You are redirected to the home page.', .mx-dialog-info) is translated, so it is reported, not matched. Matching is exact: /p/ITEMS and /p/items/ also land on / on 11.13, so a case or trailing-slash typo now fails instead of silently checking the home page. Restrict the rule to /p/ requests: / and /#/x legitimately end elsewhere. The anonymous case (/login.html#/items) is the sign-in failure's, and must not also count as a wrong page. `run --page-check` gets the url= field for free because ProbePages sets RequestedURL; its verdict has no pass/fail","refs":["ako/mxcli#998"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-waits-out-wait-on-sign-in-form.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-waits-out-wait-on-sign-in-form.json new file mode 100644 index 0000000000..585aeb481a --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-playwright-check-waits-out-wait-on-sign-in-form.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-10-27", "symptom": "`mxcli playwright check ` with no session waited out the whole --wait (33s at --wait 30s, 8s at 5s) and printed `WAIT not settled … still waiting for POST /xas/` and `FAIL still loading … (raise --wait if it is only slow)` on top of `FAIL the sign-in form is showing`", "cause": "The probe's settle loop requires no same-origin request open for 500ms (#845), and the client keeps a POST /xas/ open behind the sign-in form (measured on 11.13.0 and 11.15.0), so the page never settled; EvaluatePage then added the still-loading failure for an Unsettled page whatever it showed", "file": "cmd/mxcli/docker/pagecheck.go (pageProbeJS settle loop), cmd/mxcli/docker/pagecheck_assert.go (EvaluatePage)", "insight": "A settle condition added for one kind of page (#845's list view whose only loading sign is an open POST /xas/) becomes a wait on every page that holds a request open for another reason — and the sign-in form does, indefinitely. Exempt the state that is a verdict in itself: once #usernameInput is up, settle on text and busy indicators alone. And keep remedies apart in the verdict: 'raise --wait' next to 'sign in' sends the user to the wrong knob. The fake-server harness (probeOrSkip) reproduces it in 20s against 2s fixed, so the test is cheap; the live runs on 11.13 and 11.15 (33s → 3s, exit 1 both ways) are the control for the real client.", "refs": ["mdl-examples/bug-tests/pagecheck-signin-form-waits-out-wait.mdl", "cmd/mxcli/docker/pagecheck_signin_wait_test.go"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-run-local-json-removed-under-live-run-no-stop-reason.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-run-local-json-removed-under-live-run-no-stop-reason.json new file mode 100644 index 0000000000..c1860b5a48 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-run-local-json-removed-under-live-run-no-stop-reason.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-27","symptom":"ako/mxcli#1542 (mendixlabs/mxcli#1614): \"run --local: .mxcli/run-local.json disappears while the app is believed up (once after a trial-licence stop), and callers get no reason\" — twice in one day the file was gone while the session treated the app as running; callers got HTTP 403 from the admin API and blamed credentials; `run status` gave no reason","cause":"Three gaps on one path. (1) cmd_run.go deferred an UNCONDITIONAL removal of run-local.json in every `run --local` process, including `--setup`, which never publishes one and which the SessionStart hook runs on every session start, resume and compaction — so the hook deleted the live detached run's handshake, and oql/log fell back to default/.docker/.env credentials against a runtime that was up (403). (2) runtimeStoppedError read the stop reason from LocalRuntime.Log(), the JVM's stdout, but the licence's 'Maximum run time exceeded' is an application log line that only the file subscriber writes, to .mxcli/runtime.log — so the §60 reason detector never fired in a real run. (3) `run status` printed only the first line of the recorded error, and consumers turned a missing handshake into a credentials error","file":"`cmd/mxcli/devloop_handshake.go` (`withdrawDevLoopHandshake` owner-pid check, `runStatusNote`); `cmd/mxcli/cmd_run.go` (state.Exit then withdraw); `cmd/mxcli/docker/runtime_stop_reason.go` (`RuntimeStoppedError`, `currentRuntimeLog`, `RuntimeLogStopReason`); `cmd/mxcli/docker/runstate.go` (`Reason`); `cmd/mxcli/cmd_run_lifecycle.go` (`statusLine` probes, missing-handshake warning); `cmd/mxcli/devloop_admin.go` (`devLoopAdminHint`); `cmd/mxcli/docker/m2ee.go` (`ErrAdminAuth`)","insight":"A 403 is the discriminating detail: it needs a runtime that is UP, so 'the file is gone' could not have meant 'the run ended' — something else deleted it while the run lived, and the only other writer of that path was every other mxcli run process's deferred cleanup. Grep for every remover of a shared file and ask which processes reach it; the culprit here ran on a schedule nobody associated with the dev loop (SessionStart fires on resume and compaction too). Fix by ownership (remove only if the file's pid is yours), not by special-casing --setup, so a refused second run is covered as well. The second lesson: a reason detector tested on a string proves nothing about where the string lives — check which stream the runtime writes it to (application log -> file subscriber only) and read the newest boot's section of runtime.log (it is appended across restarts; an earlier boot's licence stop must not be blamed). Status must not turn 'handshake missing' into 'stopped' while the pid is alive and the app serves: `run status || run --detach` would then collide with the live run; report running and name the missing file. Controls: unconditional removal, first-line-only status and memory-only reason each make their test fail with the reported text","refs":["ako/mxcli#1542","mendixlabs/mxcli#1614","FINDINGS §60","ako/mxcli#982"]} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-sync-java-deps-silent-hang-stale-gradle-daemon.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-sync-java-deps-silent-hang-stale-gradle-daemon.json new file mode 100644 index 0000000000..617f03f9d7 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-27-sync-java-deps-silent-hang-stale-gradle-daemon.json @@ -0,0 +1 @@ +{"area":"cmd/mxcli","date":"2026-10-27","symptom":"ako/mxcli#1340 (mendixlabs/mxcli#1483): \"`mxcli run --local` sometimes stops for more than 10 minutes, without any output, at: `Resolving 4 managed Java dependency/dependencies (...)...` ... The same happened in the first build of `mxcli new` (step 6).\" Each `mx sync-java-dependencies` left a Gradle 8.5 daemon running for hours; the hang coincided with an idle daemon from an earlier run, and `GRADLE_OPTS=-Dorg.gradle.daemon=false` made the sync finish in ~12s with nothing left behind (Windows 11, Mendix 11.12.4)","cause":"`SyncJavaDependencies` ran `cmd.Run()` with no bound and no progress (the #1396 WaitDelay only covers the pipe AFTER mx exits, not an mx that never does), and every one-shot mx/mxbuild run (sync, `mxcli new`'s settle build, docker build) used Gradle's default daemon, which outlives the run and is what the next run attached to","file":"`cmd/mxcli/docker/mxenv.go` (`prepareOneShotMxCommand`, `withGradleNoDaemon`); `cmd/mxcli/docker/javadeps.go` (`syncJavaDepsCommand`, `syncJavaDepsLimits`, `MXCLI_SYNC_JAVA_DEPS_TIMEOUT`); `cmd/mxcli/docker/mxbuild_watchdog.go` (Progress/Step/Hint); `settle.go` (`settleCommand`), `build.go` (`mxbuildCmd`)","insight":"Three Gradle-daemon findings now (#1396 pipe held after exit, the 2026-10-11 unbounded mxbuild, this one) are one lesson: a one-shot that runs Gradle must neither leave a daemon nor wait on one unbounded — bound the tree with the existing watchdog and turn the daemon off, rather than patching each symptom. The cheap tell in the report was 'Mendix's own mx sync on the same project took 6-12s': same tool, different state, so the state (a daemon) was the cause, not mxcli's arguments. APPEND to GRADLE_OPTS — clobbering it drops a proxy, which on a proxied host turns into mx's misleading 'Failed to find some hosts' (sibling finding #1450) — and leave an explicit org.gradle.daemon alone; on Windows compare the key case-insensitively, or a second GRADLE_OPTS beside 'Gradle_Opts' replaces the user's value. Measured with real Gradle 8.14.3 here: GRADLE_OPTS='-Xmx256m -Dorg.gradle.daemon=false' gradle help -> 'No Gradle daemons are running'; plain gradle help -> one IDLE daemon left. No Mendix mx on this host, so the mx/mxbuild side rests on the reporter's measurement. `mxbuild --serve` deliberately keeps its daemon (it builds per change). Tests are fast with a fake mx that prints then `sleep 30` and an env-injected 700ms bound; revert proof: bare cmd.Run() blocks past the 15s guard with empty output, and the env tests show GRADLE_OPTS unchanged","refs":["ako/mxcli#1340","mendixlabs/mxcli#1483","mendixlabs/mxcli#1396"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-catalog-query-strips-catalog-prefix-inside-string-literals.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-catalog-query-strips-catalog-prefix-inside-string-literals.json new file mode 100644 index 0000000000..d68544506d --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-catalog-query-strips-catalog-prefix-inside-string-literals.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-27", "symptom": "A catalog query removes the text `Catalog.` even inside a string literal, so a module named Catalog cannot be filtered on: `select QualifiedName, 'Catalog.Product' as lit from CATALOG.ENTITIES` returns lit = 'Product', and `... where QualifiedName = 'Catalog.Product'` returns 0 rows although the entity exists. Reported for `mxcli -c`; the same on exec scripts, the REPL and the LSP (which shells out to -c). Siblings of the same rewrite: a quoted identifier `\"Catalog.Name\"` and comments were rewritten too, and `CATALOG . ENTITIES` (whitespace at the dot, which the grammar accepts) failed with `no such table: CATALOG.ENTITIES`.", "cause": "execCatalogQuery stripped the prefix with a case-insensitive regex `\\bcatalog\\.([a-z_][a-z0-9_]*)` over the raw query text, which cannot tell a table reference from data. The mode warning beside it had the same flaw in a second textual scan: it took the word after the first `from ` in the text, so a literal containing 'from' hid the real table and only the first table of a join was ever checked.", "file": "`mdl/executor/catalog_query_rewrite.go` (`rewriteCatalogTableRefs`, a SQLite-shaped token scanner), `mdl/executor/cmd_catalog.go` (`execCatalogQuery`, `warnIfCatalogModeInsufficient` now takes the scanner's table list; `extractTableFromQuery` removed); tests `mdl/executor/cmd_catalog_prefix_rewrite_test.go`; repro `mdl-examples/bug-tests/catalog-1450-prefix-in-string-literals.mdl`", "insight": "There is exactly one place user SQL meets the catalog (MDL SELECT -> visitor extractOriginalText -> execCatalogQuery); -c, scripts, the REPL and the LSP all reach it, and Starlark lint exposes builtins, not raw SQL, so one fix covers every path. Rewriting SQL text needs a lexer, not a regex: skip single-quoted strings (doubled-quote escape), double-quoted/backtick/bracket identifiers and both comment forms, and treat `catalog` as the prefix only when it is a whole identifier token followed by a dot and a name, not preceded by a dot. Follow SQLite's lexing, not MDL's: the text goes to SQLite verbatim. An ATTACHed `catalog` schema would have avoided rewriting altogether, but the catalog is an in-memory SQLite DB, which cannot be attached to itself without changing how it is opened. Let the warning consume the tables the scanner found, so the rewrite and the warning cannot disagree about which tables a query reads. Revert proof: 19 subtests fail on the old regex (the issue's two queries, literals, quoted identifiers, comments, whitespace at the dot, the warning cases); the controls (upper/lower/mixed prefix, joins, subqueries, ProductCatalog.Item / my_catalog aliases) pass both ways.", "refs": ["ako/mxcli#1450", "mendixlabs/mxcli#1553"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-describe-layout-drops-scroll-container-layoutmode-sidebar-copy-headline.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-describe-layout-drops-scroll-container-layoutmode-sidebar-copy-headline.json new file mode 100644 index 0000000000..bb58133d40 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-describe-layout-drops-scroll-container-layoutmode-sidebar-copy-headline.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-27", "symptom": "ako/mxcli#1703 (mendixlabs/mxcli#1659): `describe layout Atlas_Core.Atlas_SideBar` printed `scrollcontainer layoutContainer {` with no LayoutMode and `create layout` could not write one, so the documented describe → rename → exec copy of Atlas_SideBar (stored LayoutMode Sidebar) was a Headline layout — a different frame; `(LayoutMode: 'Sidebar')` passed check and exec and was dropped; mx check 0 errors either way.", "cause": "Forms$ScrollContainer.LayoutMode (Headline | Sidebar) had no field on pages.ScrollContainer, no read in buildScrollContainerV3, no SetLayoutMode in the modelsdk writer, no parse in describe, and no ALTER SET case; the copies came out Headline only because canon.CompletePropertySets fills the absent key with Studio Pro's measured default. The key passed check because the MDL-WIDGET07 vocabulary was a union, and layouts were walked by no widget rule.", "file": "sdk/pages/pages_widgets_container.go (ScrollContainer.LayoutMode, CanonicalScrollContainerLayoutMode); mdl/executor/cmd_pages_builder_v3.go (buildScrollContainerV3); mdl/backend/modelsdk/widget_write.go; mdl/executor/cmd_pages_describe_parse.go + cmd_pages_describe_output.go; mdl/backend/pagemutator/mutator.go (setScrollContainerLayoutModeMut)", "insight": "Measured the stored documents before choosing the describe policy: of the 17 Atlas_Core layouts with a scroll container on 11.13, 8 are Sidebar and 9 Headline, and canon's measured defaults say Studio Pro fills an absent key with Headline — so describe prints only Sidebar and the writer writes the key only when stated, which leaves every layout that never set it byte-identical. Both 'every widget and every key matches' copy tests (TestCopyEveryAtlasLayoutKeepsEveryWidget compared $Types and key SETS) were green while the value was wrong: a key-set comparison cannot see a defaulted value — add the value. With the per-widget verdict of ako/mxcli#1698 in place the same script was refused (MDL-WIDGET52) instead of dropped before the support existed, which is the class fix doing its job on the next instance. ALTER SET writes only a stored key, never invents one (overlay rule). Verified on a fresh 11.13 app: copy stores Sidebar, describe of the copy matches, alter set flips Headline → Sidebar, mx check 0 errors.", "refs": ["ako/mxcli#1703", "mendixlabs/mxcli#1659", "ako/mxcli#1698", "mendixlabs/mxcli#1220"], "test": "mdl/executor/layout_mode_1703_test.go (TestScrollContainerLayoutMode_*, TestCopyAtlasSideBarKeepsItsLayoutMode on PedApp); mdl/executor/sidebartoggle_copy_layout_pedapp_test.go (TestCopyEveryAtlasLayoutKeepsEveryWidget now compares LayoutMode); mdl/backend/modelsdk/widget_layout_write_test.go (TestScrollContainerToGen_LayoutMode); mdl/backend/pagemutator/scrollcontainer_layoutmode_test.go; sdk/pages/scrollcontainer_layoutmode_test.go; repro mdl-examples/bug-tests/layout-1703-scrollcontainer-layoutmode.mdl"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-domain-model-access-rules-not-rendered.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-domain-model-access-rules-not-rendered.json new file mode 100644 index 0000000000..36729fd777 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-domain-model-access-rules-not-rendered.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-27", "symptom": "`mxcli diff-local` counted a domain model as modified after `grant read *, write * on entity Shop.Customer to Shop.User` (or any grant/revoke on an entity) and printed no difference — `describe entity` showed the new rule", "cause": "domainModelBsonToMDL printed each entity's attributes (from the raw map) and the associations, but never the entity's AccessRules, so an access-only change rendered identically at both revisions", "file": "`mdl/executor/cmd_diff_local.go` (domainModelBsonToMDL), `mdl/backend/modelsdk/parse_raw.go` (ParseDomainModelFromRaw), `cmd_entities_access.go` (outputEntityAccessGrants, the writer DESCRIBE ENTITY uses)", "insight": "diff-local's domain-model renderer reads the raw BSON map, while DESCRIBE ENTITY's grant writer takes the semantic `*domainmodel.Entity` — the temptation is a third grant renderer over the raw map. Instead parse the historical document through the backend (`Parse…FromRaw`, the same gen→semantic conversion GetDomainModel uses) and call the describer's writer, then pin it with a test that every `grant` line `describe entity` prints appears verbatim on diff-local's new side (a where-clause and member lists included). With the class-level note from ako/mxcli#1512 in place, the reverted renderer degrades to `Modified: DomainModel M: changed in properties the MDL rendering does not show: Entities[Item].AccessRules[0]…` rather than silence — which is how to tell the note is working: revert a renderer and the change must still be named", "refs": ["ako/mxcli#1562", "mendixlabs/mxcli#1630"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-git-runs-in-cwd.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-git-runs-in-cwd.json new file mode 100644 index 0000000000..ae8ac0a3a1 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-git-runs-in-cwd.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-27", "symptom": "`mxcli diff-local -p App/App.mpr` run from outside the project's git repository failed with `Error: failed to find changed files: failed to git diff: exit status 1`; run from a subdirectory of the repo it warned `failed to read current file App/mprcontents/…` and reported `0 modified`", "cause": "findChangedMxunitFiles and the `git show` calls used execCommand with no Dir, so git ran in the process's working directory; `git diff --name-status` reports paths relative to the repo root, which readFile and `git show rev:path` then resolved against the wrong directory", "file": "mdl/executor/cmd_diff_local.go (gitInProject, gitShowArg)", "insight": "Two separate wrong directories hide behind one 'works for me': the cwd git runs in, and the base the reported paths are relative to. Both only coincide when cwd is the repo root and the project sits at it, which is how every earlier repro ran. Run git with Dir = the project directory, ask for `--relative` paths, and read revisions as `rev:./path` — './' makes git resolve against the working directory instead of the root, so a project below the root needs no rev-parse. Test it with a real temp repo and t.Chdir to both an unrelated dir and a subdirectory; the execCommand stubs cannot see a wrong Dir.", "refs": ["mendixlabs/mxcli#1602", "ako/mxcli#1512"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-renamed-unit-file-counted-nowhere.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-renamed-unit-file-counted-nowhere.json new file mode 100644 index 0000000000..2f6e1904b5 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-renamed-unit-file-counted-nowhere.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-27", "symptom": "`diff-local --ref HEAD~1..HEAD` after a unit's .mxunit moved to another name (another unit ID) printed `Warning: failed to read HEAD version of mprcontents/.mxunit: exit status 128` and `Summary: 0 new, 0 modified, 0 deleted` — the change counted nowhere", "cause": "`git diff --name-status` detects renames by default (diff.renames), and reported the move as `R100\\told\\tnew`; findChangedMxunitFiles took the second field (the OLD path) and a status that is neither A nor D, so the unit was read as modified at a path the new revision does not have", "file": "mdl/executor/cmd_diff_local.go (findChangedMxunitFiles: --no-renames)", "insight": "An .mxunit's file name IS its unit ID, so a git 'rename' is never a rename to Mendix — it is one unit deleted and another added, and that is what diff-local must report. Git's similarity detection makes this reachable without anyone moving a file: a document duplicated and the original deleted can pair up as R. Ask git for exactly the semantics you parse (--no-renames) instead of parsing a format with optional third fields. Real-git test: `git mv` a unit to a new UUID name, commit, diff HEAD~1..HEAD → want 1 new, 1 deleted.", "refs": ["mdl/executor/cmd_diff_local_refrange_test.go"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-settings-and-unrendered-changes-counted-silent.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-settings-and-unrendered-changes-counted-silent.json new file mode 100644 index 0000000000..0ad3090aaf --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-settings-and-unrendered-changes-counted-silent.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-27", "symptom": "`mxcli diff-local` printed only `Summary: 0 new, 1 modified, 0 deleted` (and `--format struct` said \"(no changes)\") after `alter settings constant … in configuration …` or `alter settings runtime (BcryptCost: 11)`; the same silence for navigation, project/module security, a page's or published REST service's allowed roles, an entity index, a constant's default value", "cause": "bsonToMDL had no case for `Settings$ProjectSettings`, so the unit fell to the generic rendering: two comment lines naming the type, identical at every revision. More generally, diff-local's renderers are lossy per doctype, and a unit whose change lies outside what its renderer prints rendered the same on both sides; the output loop then printed nothing (unified), the identical table (side) or \"(no changes)\" (struct) while the summary counted it", "file": "`mdl/executor/cmd_diff_local.go` (bsonToMDL settings case, projectSettingsBsonToMDL, constantValueMasks, unrenderedChangeNote; output via WriteDiffResults), `cmd_settings.go` (writeProjectSettingsMDL split out of describeSettings), `cmd_diff.go` (ChangedPaths, moved from mdl/scriptdiff and shared), `mdl/backend/modelsdk/parse_raw.go` (ParseProjectSettingsFromRaw)", "insight": "**Fix the class at the comparison, not per renderer**: git already said the bytes differ, so `Current == Proposed` on a modified unit is never \"no change\" — it is \"the renderer is lossy here\". Print what changed (property paths from the raw BSON, never values) instead of nothing; `mxcli diff` had this exact mechanism (`DiffResult.Writes`, `changedPaths`) and diff-local had simply never used it — moving the path walker into the executor gave both commands one answer. Probe before believing the reported doctype is the only one: one script per statement kind against a git-initialised app copy found nine silent unit kinds behind the two reported. The walker counted Mendix's leading int32 typed-array marker as a list item (a first grant read \"1 -> 2 items\"), in `mxcli diff` too. Masking a secret must keep the change visible: a constant `` on both sides diffs to nothing, so each side is rendered knowing the other's value (`` / ``), and the mask is deliberately not a string literal so pasted output fails to parse rather than storing \"masked\"", "refs": ["ako/mxcli#1512", "mendixlabs/mxcli#1602"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-three-dot-range-reads-dot-target.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-three-dot-range-reads-dot-target.json new file mode 100644 index 0000000000..52dd6f3101 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-diff-local-three-dot-range-reads-dot-target.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-27", "symptom": "`mxcli diff-local --ref main...feature-branch` (documented as \"changes since common ancestor\") printed `Warning: failed to read .feature version of mprcontents/…: exit status 128` for every unit and `Summary: 0 new, 0 modified, 0 deleted`", "cause": "parseRefRange split on the first \"..\", so \"a...b\" became base \"a\", target \".b\"; `git diff --name-status a...b` listed the right files, but every `git show .b:./path` failed. It also read the old side at a, where `git diff a...b` compares against the merge base", "file": "mdl/executor/cmd_diff_local.go (parseRefRange, resolveRefSpec)", "insight": "Two independent defects hide behind one symptom: the split (\".b\") fails loudly, and the obvious fix — split on \"...\" and read the old side at a — would pass any test whose branches did not diverge, then show main's own later changes reversed in every hunk. Only a repo where BOTH branches change the same unit after the split tells merge-base from a: assert the old side's value — an enumeration renders its values, so Base → Feat at the merge base and Mainval → Feat at a are told apart. Resolve the range once (git merge-base, in the project directory) rather than per unit, and treat an empty side as HEAD the way git does (`a..` read the index through `git show :./path`). Repro: the pedapp git copy with a feature branch and a diverging master.", "refs": ["mdl/executor/cmd_diff_local_refrange_test.go"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-grant-published-rest-service-created-in-new-folder-not-found.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-grant-published-rest-service-created-in-new-folder-not-found.json new file mode 100644 index 0000000000..c799e2f662 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-grant-published-rest-service-created-in-new-folder-not-found.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-27", "symptom": "`grant access on published rest service M.Svc to M.Role` fails with \"published rest service not found: M.Svc\" when the service was created with a `folder '…'` clause earlier in the same script, and exec rolls the whole script back; `check -p --references` passes. Without the folder, with a folder that already existed, or with the grant in a second exec it resolves. Same for describe/drop of a scheduled event, task queue, regular expression or consumed REST service created in a new folder — and a re-run `create or modify` of any of the five in the same script silently created a SECOND document of the same name", "cause": "Exec caches the container hierarchy once (ctx.Cache.hierarchy) and resolves every document's module by walking up from its ContainerID. The CREATE's own existence check built that cache, then resolveFolder → createFolder made the folder, and nothing recorded it: the walk stopped at an unknown folder, GetModuleName returned \"\", and the name comparison missed. Every other container-changing write (create module, move document, move/drop folder, applyDocumentFolder, rename module) already invalidated the cache; folder creation was the gap. It stayed hidden on most doctypes because their CREATE invalidates the hierarchy afterwards (microflow, page, workflow, enumeration, constant, image collection, OData, …) or their lookup does not use it (menu: GetMenuDocumentByQualifiedName); published REST, consumed REST, scheduled event, task queue and regular expression do neither", "file": "`mdl/executor/helpers.go` (createFolder now records the folder via ContainerHierarchy.addFolder in hierarchy.go); `cmd_pages_builder.go` (pageBuilder.createFolder delegates to it); guard and tests in `mdl/executor/hierarchy_created_folder_test.go`", "insight": "Pin the trigger before reading code: three exec variants (no folder / folder pre-existing from a previous exec / grant in a second exec) all passed, so the defect was a per-exec cache going stale on a write, not a folder-scoped lookup — the lookup was already folder-aware. Then audit by DOCTYPE, not by statement: 19 doctypes × create-in-new-folder + lookup, warm and cold cache, split the failures exactly along 'does this CREATE call invalidateHierarchy at the end', which is the tell that the fix belongs in the folder creator rather than in five handlers. Fixture trap that hid it: document_placement_test.go's mock CreateFolder registers the new folder into the cached hierarchy itself — the very bookkeeping production lacked; the new fixture's CreateFolder deliberately does not. The silent duplicate on re-run is the more damaging half and is what a stale existence check always becomes in a CREATE OR MODIFY. check had nothing to predict: its grant-ref validation tracks script-created documents correctly; exec's cache was wrong. Guard: TestFolderCreationHasOneCallSite fails if a second Backend.CreateFolder call site appears in the executor", "fix": "createFolder records each new folder in the cached hierarchy (addFolder); the page builder's folder creator delegates to it, leaving one call site", "refs": ["ako/mxcli#1486", "mendixlabs/mxcli#1581"], "ce": ["CE0338"]} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-onclick-not-an-action-refused-as-unknown-container-property-mdl1.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-onclick-not-an-action-refused-as-unknown-container-property-mdl1.json new file mode 100644 index 0000000000..7e11065fde --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-onclick-not-an-action-refused-as-unknown-container-property-mdl1.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-27", "symptom": "Follow-up to ako/mxcli#1698: the upgrade property test failed on mdl-examples/bug-tests/action-slot-not-an-action-1062.fail.mdl — `container cCard (OnClick: TOTALLY_MADE_UP)` exec'd under mdl 0 and was refused under `mdl 1;` with MDL-WIDGET52 \"`OnClick` is not a property mxcli writes on a container\", although a container's OnClick IS its click action.", "cause": "The visitor stores `OnClick:` under \"Action\" only when the value parses as an action (#603); a malformed value falls through to the generic `key: value` branch and keeps its key AS WRITTEN. The per-widget verdict therefore saw an unknown `OnClick` key. Separately, MDL-WIDGET28 (the rule that owns a non-action value in an action slot) looked its slots up by exact key, so the same malformed value written `onClick:` — the casing the skills teach — or `action:` / `onchange:` was reported by nothing and written as a dead control.", "file": "mdl/executor/widget_native_props.go (nativeWidgetKeyAliases: onclick -> Action, used by nativeWidgetAccepts and the click-action ownership); mdl/executor/validate_widget_action_slot.go (actionSlotSpellings: every case on a built-in widget, exact on a pluggable one)", "insight": "A rule keyed on the AST's property names inherits every place the visitor renames a key conditionally: the well-formed value is under the new name, the malformed one under the old. Grep the visitor for dedicated branches that store under a different key than their token (here only ONCLICK -> \"Action\"; Visible/Editable lower to keys judged as a group, everything else keeps its name) and model each as an alias, not as a builder read. The sweep that missed it excluded .fail scripts — but the upgrade test executes a .fail script's mdl-0 original without check, so a gated refusal in a negative test is still a differential; sweep the .fail scripts too. Measured after: 0 MDL-WIDGET52 / MDL-V1-WIDGETPROP across all 1129 mdl-examples scripts other than the two intentional .fail tests (both already `mdl 1;`).", "refs": ["ako/mxcli#1698", "mendixlabs/mxcli#1062", "#603"], "rules": ["MDL-WIDGET52", "MDL-WIDGET28", "MDL-V1-WIDGETPROP"], "test": "mdl/executor/widget_native_props_test.go (TestNativeWidgetProps_MalformedActionIsTheActionSlotRules, _OnClickAliasControls, _AliasesFollowTheVisitor); mdl/roundtrip TestUpgradeExecutesToTheSameModel"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-textbox-caption-passes-check-exec-silently-not-written-per-widget.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-textbox-caption-passes-check-exec-silently-not-written-per-widget.json new file mode 100644 index 0000000000..a6d94e65c7 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-27-textbox-caption-passes-check-exec-silently-not-written-per-widget.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-27", "symptom": "ako/mxcli#1698 (mendixlabs/mxcli#1651): `textbox tb (Attribute: Name, Caption: 'Item name')` passed check and exec and the text box had no label — describe read `textbox tb (Attribute: Name)`, mx check 0 errors. Inputs take `Label:`; `Caption:` is a button's. The class: a property a built-in widget does not take (#1307 Tooltip, #1465 tab page CaptionParams, #1582 RenderMode, #1643) was never refused.", "cause": "MDL-WIDGET07 judged every built-in widget against staticWidgetKnownProps, the UNION of all built-in widgets' vocabularies, so a key valid on any widget passed on all of them. Measured on the per-kind sets: Visible: on a snippet call, scroll container, placeholder, layout header or menu widget is also dropped (their stored types have no ConditionalVisibilitySettings, so applyWidgetBase silently skips it), a scroll-container region writes only Class of its Appearance, and layouts were checked by no widget rule at all (documentWidgets has no CreateLayoutStmt case).", "file": "mdl/executor/widget_native_props.go (nativeWidgetKinds, nativeWidgetPropRefusals — the one verdict; validateNativeWidgetProps for check, refuseNativeWidgetProps in buildWidgetV3 and in each parent that builds a child itself); mdl/executor/validate_widgets.go (wired in validateWidgetTreeIn in place of MDL-WIDGET07 for built-in kinds, and a layout walk); mdl/executor/widget_native_props_source_test.go", "insight": "A per-type allow-list became safe to make a REFUSAL once it stopped being a hand-typed promise: TestNativeWidgetProps_MatchBuilderReads taint-walks the builders' source (go/parser, following the *ast.WidgetV3 argument through helper calls, literal keys incl. `range []string{…}`) and requires each kind's `reads` to equal what its builder reads, both ways; TestNativeWidgetProps_TailMatchesGen ties the common tail's Visible to whether the gen type the writer encodes has a SetConditionalVisibilitySettings setter; TestNativeWidgetProps_DescribeEmitsOnlyAccepted holds describe's per-$Type cases to the same sets. Two traps the scan needed: a helper that reads a key only to NAME the widget in an error (widgetRefForMessage reads Caption, so every input 'read' Caption — the very key of the bug), and positional keywords (`column`, `template`, `row`, `item`) whose builder depends on the parent — under a pluggable widget or at an ALTER INSERT root they are not built natively and must not be judged. Blast radius measured before choosing refuse over warn: 0 refusals across every non-.fail doctype/bug-test/deprecated-alias script, 0 across all skill and docs-site MDL blocks, and 0 in describe → check of every module of a blank 11.13 app, ako/TestApp and PedApp (~200 documents) and of 19 doctype page scripts exec'd into a fresh app. A new rejection, so gated per ADR-0011 like MDL-V1-TABPAGEPROP: MDL-WIDGET52 under mdl 1, MDL-V1-WIDGETPROP warning without the header.", "refs": ["ako/mxcli#1698", "mendixlabs/mxcli#1651", "mendixlabs/mxcli#1307", "mendixlabs/mxcli#1465", "mendixlabs/mxcli#1582", "mendixlabs/mxcli#1643", "docs-wiki/bug-patterns/silent-property-drop.md"], "rules": ["MDL-WIDGET52", "MDL-V1-WIDGETPROP", "MDL-WIDGET07"], "test": "mdl/executor/widget_native_props_test.go (TestNativeWidgetProps_CaptionOnTextboxIsRefused, _HeaderlessWarns, _ExecRefuses, _PositionalKeywords, _VisibleWhereNotStored, _LayoutRegions); mdl/executor/widget_native_props_source_test.go (_MatchBuilderReads, _TailMatchesGen, _DescribeEmitsOnlyAccepted); repro mdl-examples/bug-tests/widget-props-1698-caption-on-textbox.fail.mdl"} diff --git a/.claude/skills/fix-issue/findings/mdl-grammar/2026-10-27-widget-action-empty-argument-list-syntax-error.json b/.claude/skills/fix-issue/findings/mdl-grammar/2026-10-27-widget-action-empty-argument-list-syntax-error.json new file mode 100644 index 0000000000..5181d0c8d2 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-grammar/2026-10-27-widget-action-empty-argument-list-syntax-error.json @@ -0,0 +1 @@ +{"area": "mdl/grammar", "date": "2026-10-27", "symptom": "ako/mxcli#1516 (mendixlabs/mxcli#1606): `actionbutton b (Caption: 'Ping', Action: call microflow M.Ping())` failed `mxcli check` with `mismatched input ')' expecting the start of a statement`, while `call microflow M.Ping();` parses in a flow body and the widget action parsed without the brackets. Same error for call nanoflow, show page, the deprecated microflow/nanoflow/show_page spellings, a microflow or nanoflow DataSource, OnClick, OnChange, a pluggable widget's action property, `alter page set Action`/`set DataSource`, and a navigation or menu-document item's OnClick. `use fragment Hdr()` was worse: no error at all without a project", "cause": "microflowArgsV3 (MDLPage.g4) required at least one argument, `LPAREN microflowArgV3 (COMMA microflowArgV3)* RPAREN`, and every one of those sites reaches it through actionExprV3 or dataSourceExprV3. The optional rule failed on `( )`, the widget's property list then met `(` and recovery blamed the `)`. fragmentArgs had the same shape, but there the page body's generic widget alternative (`widgetTypeV3` ends in IDENTIFIER, and widgetPropertiesV3 MAY be empty) re-read `use fragment Hdr()` as a widget of type `use` named `fragment` plus a widget of type `Hdr` with `()` as its properties, so check without -p passed and check -p said '`use` is not a widget in this project' (MDL-WIDGET25)", "file": "mdl/grammar/domains/MDLPage.g4 (microflowArgsV3, fragmentArgs)", "fix": "Both rules take an optional list inside the brackets, like workflowCallArguments and the flow-body callArgumentList already did. No visitor change: buildMicroflowArgsV3 and buildFragmentArgs return nil for no children, so `X()` builds the statement `X` builds. Purely additive (a refused script now parses), so no mdl-version gate. describe still writes no brackets", "test": "mdl/visitor/visitor_empty_call_args_test.go: 23 sites, each X() vs X compared as whole statements plus deprecation codes, with a non-empty argument list as the control that must differ. Reverting the grammar (patch out, make grammar) fails all 23: 22 as parse errors and use fragment on statement equality, since it parsed as two widgets. Real 11.13.0 app (mdl-examples/bug-tests/widget-action-1516-empty-call-args.mdl): a page written with () and its bracket-less twin store identical BSON once $ID and the page name are set aside, describe identically, describe -> exec reports Unchanged, mx check 0 errors", "insight": "One shared rule carried eight alternatives (six actions, two data sources) into every site that takes an action or a data source, which is the argument for auditing by rule rather than by the reported spelling: grep for the rule name, not the keyword. Compare WHOLE statements between X() and X, not the Args field: an empty-but-present mapping list, or an alternative that silently matched something else, both show up there, and a control (a real argument list must change the statement) proves the comparison can see a difference. A rule that cannot match does not always produce a parse error: when a generic fall-back alternative exists (here an unknown widget type with an optional empty property list) the input is re-read as something else and `check` without a project passes, so a site that 'parses' at baseline still needs the AST checked. `use building block X()` and `snippetcall (Params: ())` stay errors on purpose: they are an override list and a property map, not an argument list, and the messages already name what is expected", "refs": ["ako/mxcli#1516", "mendixlabs/mxcli#1606"], "rules": ["MDL-WIDGET25"]} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-10-27-lint-location-repeats-module-name.json b/.claude/skills/fix-issue/findings/mdl-other/2026-10-27-lint-location-repeats-module-name.json new file mode 100644 index 0000000000..9272676c7b --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-10-27-lint-location-repeats-module-name.json @@ -0,0 +1 @@ +{"area": "mdl/linter", "date": "2026-10-27", "symptom": "Every QUAL004 finding (and CONV001-010, the SEC rules, most bundled Starlark rules, and missing_translations) prints its location with the module twice: `at M.M.NAV_Home`, `at Sales.Sales.Order.Status`; CONV018's module-level finding prints `at Sales.Sales`", "cause": "`Location.DocumentName` is documented as the document's own name and `QualifiedName()` prefixes the module, but ~30 bundled Starlark rules pass `document_name=.qualified_name`, as does the Go rule missing_translations. CONV018 passes the module as the document of a module-level finding", "file": "`mdl/linter/linter.go` (`Location.QualifiedName`); test `mdl/linter/location_test.go`", "insight": "Fix the one place the name is assembled, not the rules: text, Markdown, SARIF and `lint -d` all go through `QualifiedName()`, and fixing ~30 rule files would leave every project's stale `mxcli init` copies and every user-written rule still doubling. It is safe to strip on prefix because a Mendix document name cannot contain a dot, so a leading `.` is always the module; the controls are a document named like its module (`Sales.Sales` stays) and a module name that is a prefix of the document's (`M` + `MX_Home`). The JSON `document` field still carries whichever spelling the rule passed — `lint -d` already accepts both. Found while reproducing ako/mxcli#1484", "refs": ["ako/mxcli#1484"], "rules": ["QUAL004", "CONV018"]} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-10-27-published-odata-service-microflows-no-refs-edge.json b/.claude/skills/fix-issue/findings/mdl-other/2026-10-27-published-odata-service-microflows-no-refs-edge.json new file mode 100644 index 0000000000..b4c3138ce1 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-10-27-published-odata-service-microflows-no-refs-edge.json @@ -0,0 +1 @@ +{"area": "mdl/catalog", "date": "2026-10-27", "symptom": "`mxcli refs ` says \"no references found\" for a microflow that is a published OData resource's `ReadMode: microflow`, and likewise for the service's `Authentication: (microflow X)`; `impact` agrees and QUAL004 says \"Remove if unused\" — 35 of one backend's 43 QUAL004 findings were the read microflows and the authentication microflow of six published services (ako/mxcli#1639)", "cause": "`buildODataServices` wrote only service-level columns and emitted no edge for any microflow the service names: entity-set read/change modes (`CallMicroflow:` from the reader), published OData actions (`svc.Microflows`), `AuthMicroflow`. Same shape on three siblings the reader already populates: a published REST service's `AuthenticationMicroflow`, a business event service operation's `Microflow` (stored in business_events_data and dropped), a consumed OData service's configuration / headers / error-handling microflows", "file": "`mdl/catalog/builder_service_refs.go` (new: `serviceMicroflowRef`, `addServiceMicroflowRefs`, `publishedODataMicroflows`, `extractServiceMicroflowRefs`), collection in `builder_modules.go` (`buildODataServices`, `buildODataClients`, `buildBusinessEventServices`) and `builder_rest.go`, emission in `builder_references.go`, `RefObject*Service` source types, `CatalogSchemaVersion` 28->29; tests `mdl/catalog/builder_service_refs_test.go`", "insight": "Fifth instance of the class after widget actions, scheduled events, settings, published REST operations and entity events — **audit the whole document family when one pointer is missing**: the reported two (ReadMode, Authentication) sat beside the change modes, the OData actions and three sibling services with the identical gap, found by grepping modelsdk/gen for `*Microflow*QualifiedName()` and checking each against the builders. **Reuse a kind whose consumers already treat it as an entry point** rather than minting one: `publish` (a published service runs it), `event` (an event handler — entity or business event; detail stays in CATALOG.BUSINESS_EVENTS), `call` (the consumed service calls its configuration microflow, as a workflow calls its handlers). A new kind would have needed `callerRefKinds`, `graphRefKinds`, the skill's ref_kind row and the docs; reusing one needed none. The source is the service with its catalog id in SourceId, which joins to the service's own table and to `objects` (the RefObject values equal the objects ObjectType values). Bumped CatalogSchemaVersion: no column changed, but a cached catalog would keep the old edge set until the .mpr changed. Controls: stubbing the collection hooks empties every edge test and GRAPH_DEAD_ASSETS lists Read_Rows again. Real 11.13 app, same model, baseline vs fixed binary: refs for the read, authentication and action microflows, the REST authentication microflow and a business event handler go from \"no references\" to one row each; QUAL004 drops them, an unreferenced microflow is still reported. Studio Pro-authored ako/TestApp: its OData actions (GetOrder, GetCustomerOrders, SaveOrder) gain publish edges; no Studio Pro document with a read microflow or authentication microflow was available, so those two read paths are verified on mxcli-written documents (keys match generated/metamodel). Not fixed, reported: published SOAP web service operations and header-auth microflow, import/export mapping microflows, ProjectSecurity.SignInMicroflow, Microflow.ConcurrencyErrorMicroflow, the navigation not-found page, document-template data sources, app services", "refs": ["ako/mxcli#1639", "ako/mxcli#1484"], "rules": ["QUAL004"]} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-10-27-qual004-navigation-microflow-reported-unused.json b/.claude/skills/fix-issue/findings/mdl-other/2026-10-27-qual004-navigation-microflow-reported-unused.json new file mode 100644 index 0000000000..a2a769bdf5 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-10-27-qual004-navigation-microflow-reported-unused.json @@ -0,0 +1 @@ +{"area": "mdl/linter", "date": "2026-10-27", "symptom": "QUAL004 reports a microflow started only from the navigation — a profile's home microflow, or a menu item's `call microflow` — as \"is not called from anywhere. Remove if unused\", while `mxcli refs` lists its `home_page` / `menu_item` edge (ako/mxcli#1484, upstream mendixlabs/mxcli#1579; the 4 NAV_ findings left in ako/mxcli#1639's Formula1Frontend)", "cause": "The rule decided \"has a caller\" from its own copy of the reference vocabulary, MICROFLOW_ENTRY_KINDS. The page list (PAGE_ENTRY_KINDS) had `home_page` and `menu_item`; the microflow list did not, though the catalog emits both kinds into a MICROFLOW too (a home microflow, a menu item that calls one)", "file": "`.claude/lint-rules/orphaned_elements.star` (`is_referenced`), `mdl/catalog/builder_references.go` (`RefKinds`); tests `mdl/linter/starlark_qual004_refs_test.go`, `mdl/catalog/lint_rule_doc_vocabulary_test.go` (`RefKind` row of `TestEveryObjectTypeConstantIsInAPublishedList`)", "insight": "This was the fourth time the same list fell behind (datasource/action/calculate, schedule, settings/publish, event, now the navigation pair) — **a consumer that keeps a copy of an allowlist of edge kinds will fall behind the producer again**, and the upstream one-line fix (add the two kinds) would have reset the clock. The rule now counts any edge into the document, i.e. the graph `refs` and GRAPH_DEAD_ASSETS already read; no kind targeting a microflow or page means anything but \"this runs/opens it\", so no exclusion list was needed. Drift is guarded behaviourally rather than by a second list: `catalog.RefKinds` is held complete against the RefKind constants (AST scan), and TestQUAL004CountsEveryRefKind drives every one of them at a microflow, a nanoflow and a page through the real rule, with an unreferenced microflow as the per-kind control — so a new kind is tested the moment it is declared, and a reintroduced allowlist fails by kind name. Verified on a real 11.13 app with the BASELINE binary and only the rule swapped: NAV_Home/NAV_Orders drop out, an unreferenced microflow and page are still reported. The OData microflows in the same report are a catalog gap, not a rule gap: `refs` was empty for them too (ako/mxcli#1639)", "refs": ["ako/mxcli#1484", "mendixlabs/mxcli#1579", "ako/mxcli#1639"], "rules": ["QUAL004"]} diff --git a/.claude/skills/fix-issue/findings/mdl-visitor/2026-10-27-alter-app-security-admin-password-ce0129.json b/.claude/skills/fix-issue/findings/mdl-visitor/2026-10-27-alter-app-security-admin-password-ce0129.json new file mode 100644 index 0000000000..633d909ef9 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-visitor/2026-10-27-alter-app-security-admin-password-ce0129.json @@ -0,0 +1 @@ +{"area": "mdl/visitor", "date": "2026-10-27", "symptom": "CE0129 \"Administrator password has not been set\" could not be resolved from MDL: `alter app security ( AdminPassword: '…' )` failed `alter app security has no property \"AdminPassword\"`", "cause": "Left out on purpose when AdminUserName was added (#1314), pending this issue; the backend had no writer for Security$ProjectSecurity.AdminPassword although gen offered SetAdminPassword", "file": "mdl/visitor/visitor_security.go (appSecurityKeys), mdl/executor/cmd_security_write.go, mdl/backend/modelsdk/security_write.go (SetProjectAdminPassword), mdl/executor/cmd_security.go (show: set / not set)", "insight": "Measure what raises the error before deciding what to guard: on 11.13.0 a blank AdminPassword is CE0129 only at prototype/production (off: no error), and a password violating the project's password policy ('1' vs a 12-char minimum) raises nothing at any level — so mxcli refuses '' and adds no policy check of its own. Studio Pro's starter apps store '1'. Patch the BSON directly (top-level string + document length) on fixture copies to get the blank-password control, since MDL cannot write the fault; PedApp's ~1000 theme errors drown a summary, so compare the CE code multiset against the untouched copy. Never echo the value: exec says 'Admin password set', show says set / not set.", "refs": ["mendixlabs/mxcli#624", "ako/mxcli#1259", "mdl-examples/bug-tests/624-alter-app-security-admin-password.mdl"]} diff --git a/.claude/skills/fix-issue/findings/modelsdk/2026-10-27-show-version-reports-v2-for-mpr-v1-project.json b/.claude/skills/fix-issue/findings/modelsdk/2026-10-27-show-version-reports-v2-for-mpr-v1-project.json new file mode 100644 index 0000000000..dbdc9e3bd1 --- /dev/null +++ b/.claude/skills/fix-issue/findings/modelsdk/2026-10-27-show-version-reports-v2-for-mpr-v1-project.json @@ -0,0 +1 @@ +{"area": "modelsdk/mpr/version", "date": "2026-10-27", "symptom": "`SHOW VERSION` prints `MPR Format: v2` for an MPR v1 project (single .mpr, no mprcontents/); `STATUS` and the session log's `mpr_format` (shipped by `diag --bundle`) say 2 as well, while `diff-local`, `diag --check-units` and `fix hashes` on the same project correctly treat it as v1", "cause": "Two answers to one question. The reader decides v1/v2 from disk (mprcontents/ beside the .mpr, reconciled by the Unit.Contents column) and reads units with that; `ProjectVersion.FormatVersion` came separately from `version.DetectFromDB`, which reads `_MetaData._FormatVersion` and, when the column is missing, assumed 'a newer schema (11.6.2+), so 2'. The column is in fact missing because the schema is v1: absent in a 9.24.30 and an 11.13.0 v1 project, present and 2 in 11.6.6/11.13.0/11.14.0 v2 ones", "file": "`modelsdk/mpr/version/version.go` (`DetectFromDB` no longer sets FormatVersion), `modelsdk/mpr/reader.go` (`reportDetectedFormat`, called in `OpenWithOptions` after the reconciliation and in `OpenWithDB`)", "insight": "Before trusting a field that reports a fact, find the code that *acts* on the fact and check the two agree: every consumer that branches on the format (`diff-local` guard, harvest, check-units, fix hashes) used `Reader.Version()`, and only the printers used the metadata copy, so the defect was invisible to anything but a human reading the output. The comment justifying the default ('Mendix 11.6.2+') was a guess written as a measurement; one `pragma table_info(_MetaData)` on a v1 file refutes it. Stamp after the Unit.Contents reconciliation, not before it: a v2-schema .mpr copied away from its folder is v1 by the folder test and v2 after reconciliation, and `TestProjectVersionReportsTheDetectedStorageFormat` has a case that fails if the order is swapped. The v2 project is the control: it printed v2 before the fix too (ako/mxcli#1553, upstream mendixlabs/mxcli#1621)", "refs": ["ako/mxcli#1553", "mendixlabs/mxcli#1621"]} diff --git a/.claude/skills/maintain-wiki/pages.md b/.claude/skills/maintain-wiki/pages.md index 5cb4e2a8eb..81135e5c32 100644 --- a/.claude/skills/maintain-wiki/pages.md +++ b/.claude/skills/maintain-wiki/pages.md @@ -21,6 +21,7 @@ | `bug-patterns/silent-property-drop.md` | bug-pattern | a property parses, passes every check, and never reaches the model | | `bug-patterns/check-mxbuild-drift.md` | bug-pattern | `mxcli check` as a model of mxbuild, drifting in both directions | | `bug-patterns/platform-semantics-gaps.md` | bug-pattern | legal MDL, illegal Mendix — expression / XPath / variable-scope rules the grammar cannot carry | +| `bug-patterns/diffs-narrower-than-the-change.md` | bug-pattern | `diff-local` compares rendered MDL, so it shows only what its renderer prints: "N modified" with no hunk, and the field-level fallback that closes the class | | `bug-patterns/duplicate-resolver-drift.md` | bug-pattern | one question answered in two places, and the two disagree | | `bug-patterns/rewrite-drops-unauthored-state.md` | bug-pattern | CREATE OR REPLACE losing what the statement did not mention | | `bug-patterns/flow-graph-geometry.md` | bug-pattern | generated microflow coordinates and sequence-flow wiring | diff --git a/.claude/skills/mendix/create-page/SKILL.md b/.claude/skills/mendix/create-page/SKILL.md index 3d5fdb32f5..c7eb687e76 100644 --- a/.claude/skills/mendix/create-page/SKILL.md +++ b/.claude/skills/mendix/create-page/SKILL.md @@ -12,15 +12,15 @@ description: "CREATE PAGE syntax reference — parameters, variables, layouts, a |---|---| | 28-43 | Reference files | | 44-46 | Overview | -| 47-206 | Syntax | -| 207-277 | Basic Examples | -| 278-299 | Modifying Existing Pages | -| 300-403 | Conditional Visibility and Editability | -| 404-568 | Known Limitations | -| 569-577 | Tips | -| 578-604 | Related Commands | -| 605-629 | PLUGGABLEWIDGET Escape Hatch | -| 630-633 | See Also | +| 47-217 | Syntax | +| 218-288 | Basic Examples | +| 289-310 | Modifying Existing Pages | +| 311-414 | Conditional Visibility and Editability | +| 415-579 | Known Limitations | +| 580-588 | Tips | +| 589-615 | Related Commands | +| 616-640 | PLUGGABLEWIDGET Escape Hatch | +| 641-644 | See Also | # CREATE PAGE - MDL Syntax Guide @@ -199,10 +199,21 @@ container ctnHero ( } ``` -> `mxcli check` **warns** (MDL-WIDGET07) when a built-in widget carries an -> unrecognized property (a typo, or a property mxcli doesn't persist) — it would -> otherwise be silently dropped on write. It is a warning, not an error, so the -> check still passes; fix the spelling or remove the property. +> **Each built-in widget takes its own properties, not every widget's.** An +> input widget's label is `Label:`; `Caption:` is a button's (and a data grid +> column's). A property the widget does not take — `Caption:` on a `textbox`, +> `Label:` on an `actionbutton`, `Visible:` on a `snippetcall`, a misspelt key — +> is **refused** under `mdl 1;` (MDL-WIDGET52, by `check` and by `exec`, with +> nothing written) and warned about without the header (MDL-V1-WIDGETPROP); it +> used to be silently dropped. The message names what was most likely meant and +> lists the properties that widget takes. +> +> ```sql +> -- ❌ WRONG: a text box has no Caption — refused (MDL-WIDGET52) +> textbox tbName (Attribute: Name, Caption: 'Name') +> -- ✅ an input's label is Label: +> textbox tbName (Attribute: Name, Label: 'Name') +> ``` ## Basic Examples diff --git a/.claude/skills/mendix/create-page/reference/widgets.md b/.claude/skills/mendix/create-page/reference/widgets.md index 20f3a43a3a..3e4752edc3 100644 --- a/.claude/skills/mendix/create-page/reference/widgets.md +++ b/.claude/skills/mendix/create-page/reference/widgets.md @@ -543,8 +543,8 @@ picker with `alter page … { set (DateFormat: DateTime) on dpStart; }`. ```sql textbox tbAmount (label: 'Amount', attribute: Amount, DecimalPrecision: 2, GroupDigits: true) ``` -A text box binds no dates (CE2421), so `DateFormat` on it is dropped and warned about -(MDL-WIDGET07), as on any widget other than a date picker. +A text box binds no dates (CE2421), so it does not take `DateFormat`: under `mdl 1;` +it is refused (MDL-WIDGET52), as on any widget other than a date picker. **COMBOBOX** - Combo box (pluggable widget): ```sql diff --git a/.claude/skills/mendix/database-connections/SKILL.md b/.claude/skills/mendix/database-connections/SKILL.md index 03af9bb6c6..e8c9c4210c 100644 --- a/.claude/skills/mendix/database-connections/SKILL.md +++ b/.claude/skills/mendix/database-connections/SKILL.md @@ -13,16 +13,16 @@ description: "Connect a Mendix app to an external database over JDBC with the Ex | 30-32 | Purpose | | 33-49 | When to Use This Skill | | 50-84 | Prerequisites | -| 85-193 | Database Connection Syntax | -| 194-255 | Query Definition Syntax | -| 256-353 | Complete Examples | -| 354-369 | Viewing Connections | -| 370-390 | Best Practices | -| 391-404 | Troubleshooting | -| 405-416 | Related Commands | -| 417-542 | Executing Queries from Microflows | -| 543-557 | Importing Data from External Databases | -| 558-561 | References | +| 85-199 | Database Connection Syntax | +| 200-261 | Query Definition Syntax | +| 262-359 | Complete Examples | +| 360-375 | Viewing Connections | +| 376-396 | Best Practices | +| 397-410 | Troubleshooting | +| 411-422 | Related Commands | +| 423-548 | Executing Queries from Microflows | +| 549-563 | Importing Data from External Databases | +| 564-567 | References | # Skill: Create External Database Connections @@ -180,7 +180,13 @@ mxcli sync-java-deps -p app.mpr --check # report what is missing, exit 1 (build ``` `mxcli run --local` does this automatically for anything not already in -`vendorlib/`, so the warm loop works from a fresh clone. Dropping the jar into +`vendorlib/`, so the warm loop works from a fresh clone. The sync runs Gradle +**without a daemon** (`-Dorg.gradle.daemon=false` appended to `GRADLE_OPTS`): a +daemon an earlier `mx`/`mxbuild` left behind used to hang it for ten minutes and +more with no output (ako/mxcli#1340). A sync still running prints `still resolving +Java dependencies... (30s)` every 30s and is killed after 10 minutes with the +likely cause and its last output; `MXCLI_SYNC_JAVA_DEPS_TIMEOUT=30m` (or `0`) +raises the bound for a genuinely slow download. Dropping the jar into `userlib/` by hand works too — it is the same classpath — but then the model and the file system disagree about where the dependency comes from. diff --git a/.claude/skills/mendix/manage-security/SKILL.md b/.claude/skills/mendix/manage-security/SKILL.md index e1bce19e8e..9f294a4333 100644 --- a/.claude/skills/mendix/manage-security/SKILL.md +++ b/.claude/skills/mendix/manage-security/SKILL.md @@ -13,11 +13,11 @@ description: "Configure Mendix security in MDL — module and user roles, access | 27-35 | When to Use This Skill | | 36-43 | Security Concepts | | 44-120 | The System-module ceiling — decide the role model around this FIRST | -| 121-553 | Syntax Reference | -| 554-568 | Starlark Lint Rule APIs | -| 569-621 | Common Workflow: Setting Up Module Security | -| 622-631 | Common Mistakes | -| 632-641 | Validation | +| 121-558 | Syntax Reference | +| 559-573 | Starlark Lint Rule APIs | +| 574-626 | Common Workflow: Setting Up Module Security | +| 627-636 | Common Mistakes | +| 637-646 | Validation | # Security Management Skill @@ -491,9 +491,14 @@ alter app security ( SecurityLevel: production ); alter app security ( EnableDemoUsers: true ); alter app security ( EnableDemoUsers: false ); --- Rename the built-in administrator (default MxAdmin). The password is not --- settable from MDL. +-- Rename the built-in administrator (default MxAdmin). alter app security ( AdminUserName: 'appadmin' ); + +-- Set its password. An empty one is CE0129 "Administrator password has not +-- been set" at prototype/production level; '' is refused. Stored in the model +-- as written (as Studio Pro stores it), so keep real secrets out of committed +-- scripts. `describe app security` says whether one is set, never what. +alter app security ( AdminPassword: 'N3w-Admin-Passw0rd' ); ``` ### Guest (Anonymous) Access diff --git a/.claude/skills/mendix/run-local/SKILL.md b/.claude/skills/mendix/run-local/SKILL.md index d7ee14d1fd..d2b5c66760 100644 --- a/.claude/skills/mendix/run-local/SKILL.md +++ b/.claude/skills/mendix/run-local/SKILL.md @@ -12,27 +12,27 @@ description: "The warm Docker-free dev loop, `mxcli run --local` — model chang |---|---| | 40-47 | Overview | | 48-60 | When to Use This Skill | -| 61-118 | Usage | -| 119-131 | How apply is chosen | -| 132-201 | Prerequisites | -| 202-219 | The intended loop | -| 220-247 | Editing themes (SCSS) — rebuild, never clear caches | -| 248-274 | `--watch` waits for a write to finish before it builds | -| 275-303 | "My edit didn't show up" — stale process, not stale cache | -| 304-334 | Flags | -| 335-388 | Metrics and OpenTelemetry | -| 389-446 | Pages render in the browser | -| 447-464 | Pixel-perfect page loop | -| 465-485 | Debugging a server-side error | -| 486-506 | "Sign in failed" that is not about the password | -| 507-539 | The same licence also stops the app after a few hours | -| 540-619 | External browser preview (`--hub`) | -| 620-628 | Validation checklist | -| 629-685 | Constant values come from a configuration | -| 686-689 | Warm local dev loop (`mxcli run --local [--watch] [--screenshot]`) | -| 690-693 | External browser preview (`mxcli run --hub ` + `mxcli tunnel-hub`) | -| 694-697 | Tunnel-hub GitHub authentication (opt-in, gated on `--github-oauth-client-id`; absent = today's open hub) | -| 698-700 | Runtime metrics + settings passthrough (`mxcli run --local --metrics` / `--runtime-setting Key=Value`) | +| 61-126 | Usage | +| 127-139 | How apply is chosen | +| 140-230 | Prerequisites | +| 231-248 | The intended loop | +| 249-276 | Editing themes (SCSS) — rebuild, never clear caches | +| 277-303 | `--watch` waits for a write to finish before it builds | +| 304-332 | "My edit didn't show up" — stale process, not stale cache | +| 333-363 | Flags | +| 364-417 | Metrics and OpenTelemetry | +| 418-475 | Pages render in the browser | +| 476-493 | Pixel-perfect page loop | +| 494-514 | Debugging a server-side error | +| 515-538 | "Sign in failed" that is not about the password | +| 539-571 | The same licence also stops the app after a few hours | +| 572-651 | External browser preview (`--hub`) | +| 652-660 | Validation checklist | +| 661-717 | Constant values come from a configuration | +| 718-721 | Warm local dev loop (`mxcli run --local [--watch] [--screenshot]`) | +| 722-725 | External browser preview (`mxcli run --hub ` + `mxcli tunnel-hub`) | +| 726-729 | Tunnel-hub GitHub authentication (opt-in, gated on `--github-oauth-client-id`; absent = today's open hub) | +| 730-732 | Runtime metrics + settings passthrough (`mxcli run --local --metrics` / `--runtime-setting Key=Value`) | # Warm Local Dev Loop — `mxcli run --local` @@ -116,6 +116,14 @@ What each one does, precisely: - **`run restart`** reuses the arguments recorded in the state file; to change a flag, `run stop` and `--detach` again. +- **`.mxcli/run-local.json`** (ports, admin password — how `oql`, `log` and `constant set + --apply` reach the app) is written when the app serves and removed only by the run + that wrote it, right after `run-state.json` records `phase: stopped` with a `reason`. + **Missing while you believed the app was up? Run `mxcli run status` before + suspecting credentials** — it names the reason (`stopped: trial licence time limit + reached at 18:43:16 (runtime.log); restart with …`) or a live run whose file is gone. + Lifecycle and what each answer means: [reference/run-local-json.md](reference/run-local-json.md). + ## How apply is chosen Every warm rebuild reports whether a restart is required; `run --local` applies the @@ -153,6 +161,19 @@ association catalog only at startup; behavioural changes are hot-reloaded. `~/.mxcli/postgres/server.log`. Stop it with `pg_ctl -D "$HOME/.mxcli/postgres/data" stop`. To remove it, stop it first and then delete `~/.mxcli/postgres` (this permanently deletes its databases). + **A stopped system cluster is never replaced silently.** When a package-made + cluster for the same port exists (`/var/lib/postgresql//`, + `/var/lib/pgsql/data`, `/var/lib/postgres/data`, Homebrew's) and the service + manager did not bring it up, `--ensure-db` **stops** with *refusing to create a + new, empty PostgreSQL cluster: the cluster in … is not running* and the + command that starts it (`sudo pg_ctlcluster 16 main start`). A new cluster on + that port would answer every probe while holding none of the app's data — an + empty database looks exactly like a new one (ako/mxcli#866). Start it and rerun; + if it is not this app's and cannot be started, `MXCLI_ENSURE_DB_NEW_CLUSTER=1` + creates the user-owned cluster anyway. Whenever `initdb` does run, the output + says **WARNING: created a NEW, EMPTY PostgreSQL cluster** — if you see that and + expected your data, stop: the data is in another server. An existing user-owned + cluster is simply restarted (with a note naming any stopped system cluster). - Without `--ensure-db`, create it once and the command errors if it's unreachable: ```bash @@ -198,6 +219,14 @@ mxbuild from the Mendix CDN is a Linux binary and cannot run natively on darwin install directory (usually `C:\Program Files\Mendix`). A "Gradle not found" from a local run therefore points at an incomplete or foreign mxbuild bundle — check which mxbuild was resolved before looking for a system Gradle. + The **one-shot** runs — `Resolving N managed Java dependency…` (`mx + sync-java-dependencies`), `mxcli new`'s first build, `docker build` — run Gradle + **without a daemon** (`-Dorg.gradle.daemon=false` appended to your + `GRADLE_OPTS`, which is otherwise kept). An idle daemon an earlier run left + behind made the dependency step hang 10+ minutes in silence (ako/mxcli#1340); it + now prints `still resolving…` every 30s and gives up after 10 minutes + (`MXCLI_SYNC_JAVA_DEPS_TIMEOUT`) naming that cause. `mxbuild --serve` keeps its + daemon: it builds on every change, which is what a daemon is for. ## The intended loop @@ -496,13 +525,16 @@ Maximum number of sessions exceeded! (You are currently using a trial license) So: when a login you know is correct starts failing, `grep -c "Maximum number of sessions" .mxcli/runtime.log` before touching the credentials or the user's -password in the model. `--screenshot-user` does this for you — a rejected sign-in -now reads the log and says so instead of quietly screenshotting the login page. +password in the model. `--screenshot-user` and `mxcli playwright check` do this +for you — a rejected sign-in reads the log and says so instead of quietly +screenshotting or checking the login page. Sessions are held until they expire; restarting `run --local` clears them all. A script that drives the app through a browser should **sign out at the end**, otherwise each run leaks a slot and the fifth or sixth run is the one that fails — which makes it look like a change you just made broke authentication. +`mxcli playwright check` keeps one saved session per host and user and reuses it, +so repeated checks hold one slot; `--sign-out` releases it at the end of a run. ## The same licence also stops the app after a few hours diff --git a/.claude/skills/mendix/run-local/reference/run-local-json.md b/.claude/skills/mendix/run-local/reference/run-local-json.md new file mode 100644 index 0000000000..27f2d69a81 --- /dev/null +++ b/.claude/skills/mendix/run-local/reference/run-local-json.md @@ -0,0 +1,34 @@ +# `.mxcli/run-local.json` — the handshake, and what its absence means + +`run-state.json` says what the run is doing; **`run-local.json`** (0600) is how other +commands *reach* the app: its app and admin ports, the admin password and the boot +configuration. `mxcli oql`, `mxcli log` and `constant set --apply` read it; tools that +call the admin API themselves should too. + +| When | What happens to it | +|---|---| +| the app serves (boot done) | **written** by the run, with the run's pid | +| the run ends — `Ctrl-C`, `run stop`, or the **runtime stopped on its own** (trial-licence time limit, crash, kill) | **removed** by that run, right after `run-state.json` records `phase: stopped`, `ended` and the `reason` | +| `run stop` on a run killed with `-9` | removed by `run stop` | +| any *other* mxcli process exits (`run --local --setup` from the SessionStart hook, a refused second run) | **untouched** — only the run whose pid is in it may remove it | + +## Missing while you believed the app was up? + +Run `mxcli run status -p app.mpr` before suspecting credentials. It says one of: + +- `stopped: trial licence time limit reached at 18:43:16 (runtime.log); restart with + mxcli run restart -p …` — the reason comes from `.mxcli/runtime.log` (the newest + boot's section) when it gives one: the licence limit, an `OutOfMemoryError`, else + the last `ERROR`/`CRITICAL` line, reported as exactly that; +- `stopped: pid N exited without cleaning up …; its runtime log says: …` — the run + was killed outright, so it could not record its end; +- `running: … but .mxcli/run-local.json is MISSING` — the app is up and only the + file is gone; `mxcli run restart` publishes it again. + +A 401/403 from the admin API with no handshake means the connection fell back to +default settings (environment, `.docker/.env`, `mxcli-local-dev`), not that a +password is wrong; `oql` and `log` say so and quote `run status`. + +Before ako/mxcli#1542, every `run --local` process removed the file on exit whether +or not it had written it — so the SessionStart hook's `--setup` run deleted a live +run's handshake on every session start, resume or compaction. diff --git a/.claude/skills/mendix/test-app/SKILL.md b/.claude/skills/mendix/test-app/SKILL.md index d996b98c6e..4dfc0ff5c8 100644 --- a/.claude/skills/mendix/test-app/SKILL.md +++ b/.claude/skills/mendix/test-app/SKILL.md @@ -11,18 +11,18 @@ description: "Verify a running Mendix app in a browser: `mxcli playwright check` | lines | section | |---|---| | 32-42 | When to Use This Skill | -| 43-90 | Start Here: `mxcli playwright check` (one call, text, no screenshot) | -| 91-141 | Prerequisites | -| 142-166 | Quick Start | -| 167-188 | Widget Name Selectors | -| 189-304 | Verification Patterns | -| 305-378 | CI/CD: Test Scripts | -| 379-422 | Session Management | -| 423-450 | Debugging | -| 451-483 | Selector Rules | -| 484-500 | Known Gotchas | -| 501-534 | Feedback Loop Workflow | -| 535-542 | Related Skills | +| 43-104 | Start Here: `mxcli playwright check` (one call, text, no screenshot) | +| 105-155 | Prerequisites | +| 156-180 | Quick Start | +| 181-202 | Widget Name Selectors | +| 203-318 | Verification Patterns | +| 319-392 | CI/CD: Test Scripts | +| 393-436 | Session Management | +| 437-464 | Debugging | +| 465-497 | Selector Rules | +| 498-514 | Known Gotchas | +| 515-548 | Feedback Loop Workflow | +| 549-556 | Related Skills | # Test App Skill @@ -61,8 +61,18 @@ FAIL 1 of 2 page(s) ``` - **Exit status:** 0 all pages passed; 1 a page failed (error banner/dialog, console - error, failed request, HTTP error, sign-in form instead of the page, failed - assertion, or `STILL LOADING`); 2 the check could not run. + error, failed request, HTTP error, access denied, sign-in form instead of the page, + a `/p/` URL that ended elsewhere, failed assertion, or `STILL LOADING`); 2 the + check could not run. +- **`ACCESS DENIED` is a role problem, not a sign-in one.** A page the signed-in + user's roles may not open lands on the sign-in form with `messageCode=403` in the + URL and fails as `access denied: signed in as demo_user (user role User), the + runtime refused /p/secret (403)`. Compare `show access on page Module.Page` with + `describe user role R`; changing credentials or `--fresh-login` will not help. +- **`url=` is where the browser ended**, printed when that is not where it was + sent; title and heading then describe that page. A `/p/` URL no page has lands on + the home page with HTTP 200 (`url=/`) and fails — the match is exact, so check the + page's URL spelling and case before suspecting the page. - **It waits for the data.** A page is measured once no request to the app has been open for half a second and its text holds still — a list view shows no spinner while its data source runs, so the text alone would hold still on an empty list. @@ -71,8 +81,12 @@ FAIL 1 of 2 page(s) the answer. A page with a list or grid always prints `rows=`, `rows=0` included. - **Login is handled:** `--user U --password P`, or `--role R` (the project's demo user holding user role R), or nothing but `-p` (a demo user, an administrator first). - The session is saved under `.mxcli/playwright-check/` and reused by the next check; - after a runtime restart it is renewed automatically. `--fresh-login` forces it. + It signs in on the host of the pages checked (a session for `localhost` is not + sent to `127.0.0.1`), saves the session per host under `.mxcli/playwright-check/` + and reuses it — one session however many checks run; after a runtime restart it + is renewed automatically. `--fresh-login` signs the saved session out and signs + in again; `--sign-out` ends the run signed out. A refused sign-in names the + unlicensed runtime's session cap when `.mxcli/runtime.log` reports it. - **Assertions:** `--assert-text 'Customer overview'` (body text contains), `--assert-count '.mx-name-dgCustomers [role=row]>=2'` (ops `>= <= == != > <`; a bare selector means at least one). Both repeatable, applied to every page. diff --git a/.claude/skills/mendix/test-microflows/SKILL.md b/.claude/skills/mendix/test-microflows/SKILL.md index ff1b9e6e75..aa2e181621 100644 --- a/.claude/skills/mendix/test-microflows/SKILL.md +++ b/.claude/skills/mendix/test-microflows/SKILL.md @@ -12,18 +12,18 @@ description: "Write and run MDL-based microflow tests with `mxcli test` — anno |---|---| | 33-39 | Reference files | | 40-50 | When to Use This Skill | -| 51-179 | Prerequisites | -| 180-222 | Test File Formats | -| 223-238 | string Operations | -| 239-262 | Running Tests | -| 263-450 | How It Works | -| 451-489 | Check a test file before you run it | -| 490-596 | Writing Good Tests | -| 597-611 | Test File Organization | -| 612-624 | Interpreting Failures | -| 625-646 | CI Integration | -| 647-653 | Related Skills | -| 654-656 | Warm test loop (`mxcli test --local [--watch]`, `--attach`, `run --local --test-endpoint`) | +| 51-180 | Prerequisites | +| 181-223 | Test File Formats | +| 224-239 | string Operations | +| 240-263 | Running Tests | +| 264-457 | How It Works | +| 458-496 | Check a test file before you run it | +| 497-618 | Writing Good Tests | +| 619-633 | Test File Organization | +| 634-646 | Interpreting Failures | +| 647-668 | CI Integration | +| 669-675 | Related Skills | +| 676-678 | Warm test loop (`mxcli test --local [--watch]`, `--attach`, `run --local --test-endpoint`) | # Test Microflows Skill @@ -103,7 +103,8 @@ app looks half-working. Only Java-backed resources are affected, which is why it does not look like the test run did it. mxcli warns when it sees a dev loop serving the same project (`run --local` -records itself in `.mxcli/run-local.json`, removed on exit). **If something the +records itself in `.mxcli/run-local.json`, removed when that run ends — see the +run-local skill for its lifecycle). **If something the app serves stops returning data after a test run, restart that app.** That is the whole remedy. @@ -315,6 +316,12 @@ chained, `--local` ran with the app's startup logic suppressed, and a suite that depended on startup state passed under `--attach` and failed under `--local` for reasons unrelated to the code. +`MxTest.RegisterEndpoint` applies entity access, so it can call a startup +microflow that does or does not (CE0114 only forbids a non-applying microflow +calling an applying one); yours still runs with its own setting. A build error +located in `MxTest.RegisterEndpoint` or `MxTest.RegisterTestEndpoint` is reported +as an mxcli defect, not as an error in your project. + One thing rollback does **not** cover: whatever the startup microflow writes happens at boot, outside any test's transaction, so `@cleanup rollback` does not undo it. Under `--local` that lands in the scratch `_test` database; @@ -585,6 +592,21 @@ call microflow Sales.ValidateOrder(Total = -1); Anything else after the tag is an error, not a message: `@throws not found` is refused rather than read as an unquoted one. +A call that already has its own `on error` clause keeps it — the runner adds +its handler only to calls without one, so the test runs the statement exactly as +production does: + +- `on error rollback` is Mendix's default spelled out and behaves like no clause: + FAIL `exception during execution: …`, or PASS under a matching `@throws`. +- `on error continue` swallows the error; the assertions decide. `@throws` FAILs + (`expected an exception but none was thrown`). +- A custom handler that ends normally skips every statement reading the failed + call's result — the assertions — so the test FAILs with `not every assertion + was evaluated`. Assert on something the handler sets instead. +- A custom handler that re-raises lets the error leave the test: ERROR with the + message, also under `@throws`. Under Docker / `--legacy-runner` it ends the + whole run. + `@throws` and `@expect` cannot be combined. A `@throws` test compiles to a different shape — the verdict starts as a failure and only the error handler clears it — and the `@expect` checks are not emitted into it at all, so the diff --git a/.claude/skills/mendix/write-layouts/SKILL.md b/.claude/skills/mendix/write-layouts/SKILL.md index d6f2c20669..7bfad531db 100644 --- a/.claude/skills/mendix/write-layouts/SKILL.md +++ b/.claude/skills/mendix/write-layouts/SKILL.md @@ -196,8 +196,8 @@ create or modify layout MyModule.Layout_Framed ( | Element | MDL | Notes | |---------|-----|-------| -| Scroll container | `scrollcontainer name { … }` | The layout's root. Its children are **regions**, never widgets | -| Region | `region top \| right \| bottom \| left \| center` | Five **named slots**, not a list. One region per slot; a repeat is refused | +| Scroll container | `scrollcontainer name [(layoutmode: 'Sidebar')] { … }` | The layout's root. Its children are **regions**, never widgets. `layoutmode` is the frame: `'Headline'` (the default — the top region runs the full width, the left one starts under it) or `'Sidebar'` (the left region runs the full height, the top bar beside it; Atlas_SideBar and Atlas_TopBar). `describe` prints it only when it is Sidebar | +| Region | `region top \| right \| bottom \| left \| center` | Five **named slots**, not a list. One region per slot; a repeat is refused. Takes `size`, `sizemode`, `togglemode` and `class` — no other appearance key is written for a region, so `style:` on one is refused (MDL-WIDGET52) | | Placeholder | `placeholder Main` | The hole a page's content goes into. No properties, no body | | Navigation tree | `navigationtree name (profile: 'Responsive')` | The sidebar menu — vertical. The profile is a navigation profile name | | Menu bar | `menubar name (profile: 'Responsive')` | The topbar menu — horizontal. Same stored shape as a navigation tree | diff --git a/.claude/skills/mendix/write-lint-rules/SKILL.md b/.claude/skills/mendix/write-lint-rules/SKILL.md index a32c87198a..e695d17ed6 100644 --- a/.claude/skills/mendix/write-lint-rules/SKILL.md +++ b/.claude/skills/mendix/write-lint-rules/SKILL.md @@ -561,13 +561,13 @@ Returned by `permissions()` (all types) or `permissions_for()` (entity-specific) ### reference | Property | Type | Example | |----------|------|---------| -| `source_type` | string | The document the edge comes FROM, upper-case: `"MICROFLOW"`, `"NANOFLOW"`, `"RULE"`, `"PAGE"`, `"SNIPPET"`, `"ENTITY"`, `"ASSOCIATION"`, `"WORKFLOW"`, `"NAVIGATION"`, `"SCHEDULED_EVENT"`, `"PUBLISHED_REST_OPERATION"`, `"PROJECT_SETTINGS"`, `"IMPORT_MAPPING"`, `"EXPORT_MAPPING"` | +| `source_type` | string | The document the edge comes FROM, upper-case: `"MICROFLOW"`, `"NANOFLOW"`, `"RULE"`, `"PAGE"`, `"SNIPPET"`, `"ENTITY"`, `"ASSOCIATION"`, `"WORKFLOW"`, `"NAVIGATION"`, `"SCHEDULED_EVENT"`, `"PUBLISHED_REST_OPERATION"`, `"PUBLISHED_REST_SERVICE"`, `"PUBLISHED_ODATA_SERVICE"`, `"CONSUMED_ODATA_SERVICE"`, `"BUSINESS_EVENT_SERVICE"`, `"PROJECT_SETTINGS"`, `"IMPORT_MAPPING"`, `"EXPORT_MAPPING"` | | `source_id` | string | Source UUID | | `source_name` | string | `"Sales.ACT_Customer_Create"` | -| `target_type` | string | What it points AT, upper-case: `"ENTITY"`, `"ASSOCIATION"`, `"MICROFLOW"`, `"NANOFLOW"`, `"RULE"`, `"PAGE"`, `"LAYOUT"`, `"WORKFLOW"`, `"WIDGET"`, `"JAVA_ACTION"`, `"JAVASCRIPT_ACTION"`, `"REST_OPERATION"`, `"REGULAR_EXPRESSION"`, `"ATTRIBUTE"`, `"ENUMERATION"`, `"ENUMERATION_VALUE"`. `LAYOUT`, `WIDGET`, `ATTRIBUTE`, `ENUMERATION` and `ENUMERATION_VALUE` are only ever targets; `SCHEDULED_EVENT` and `PROJECT_SETTINGS` only ever sources | +| `target_type` | string | What it points AT, upper-case: `"ENTITY"`, `"ASSOCIATION"`, `"MICROFLOW"`, `"NANOFLOW"`, `"RULE"`, `"PAGE"`, `"LAYOUT"`, `"WORKFLOW"`, `"WIDGET"`, `"JAVA_ACTION"`, `"JAVASCRIPT_ACTION"`, `"REST_OPERATION"`, `"REGULAR_EXPRESSION"`, `"ATTRIBUTE"`, `"ENUMERATION"`, `"ENUMERATION_VALUE"`. `LAYOUT`, `WIDGET`, `ATTRIBUTE`, `ENUMERATION` and `ENUMERATION_VALUE` are only ever targets; `SCHEDULED_EVENT`, `PROJECT_SETTINGS` and the service types only ever sources | | `target_id` | string | Target UUID | | `target_name` | string | `"Sales.Customer"`; three-part for an attribute or an enumeration value: `"Sales.Order.Total"`, `"Sales.OrderStatus.Open"` | -| `ref_kind` | string | How it references: `"call"`, `"create"`, `"retrieve"`, `"change"`, `"delete"`, `"commit"` (a commit action, or a create/change that commits — beside its `"create"`/`"change"` edge; a commit of a variable whose entity the flow cannot tell has no edge), `"show_page"`, `"datasource"`, `"action"`, `"layout"`, `"parameter"`, `"return"`, `"generalize"`, `"associate"`, `"home_page"`, `"login_page"`, `"menu_item"`, `"calculate"`, `"schedule"`, `"validate"`, `"settings"`, `"widget"`, `"sync"`, `"publish"`, `"event"`, `"member"` (binds/reads/writes an attribute or navigates an association), `"xpath"` (an XPath constraint names it), `"type"` (typed as an enumeration), `"value"` (an expression names an enumeration value), `"mapping"` (an import/export mapping maps the entity) — lower-case, unlike the types above. Attribute names used only through a variable in a free-text expression (`$Order/Total`) have no edge | +| `ref_kind` | string | How it references: `"call"`, `"create"`, `"retrieve"`, `"change"`, `"delete"`, `"commit"` (a commit action, or a create/change that commits — beside its `"create"`/`"change"` edge; a commit of a variable whose entity the flow cannot tell has no edge), `"show_page"`, `"datasource"`, `"action"`, `"layout"`, `"parameter"`, `"return"`, `"generalize"`, `"associate"`, `"home_page"`, `"login_page"`, `"menu_item"`, `"calculate"`, `"schedule"`, `"validate"`, `"settings"`, `"widget"`, `"sync"`, `"publish"` (a published REST operation, OData resource or action, or a service's authentication microflow), `"event"` (an entity event or a business event handler), `"member"` (binds/reads/writes an attribute or navigates an association), `"xpath"` (an XPath constraint names it), `"type"` (typed as an enumeration), `"value"` (an expression names an enumeration value), `"mapping"` (an import/export mapping maps the entity) — lower-case, unlike the types above. Attribute names used only through a variable in a free-text expression (`$Order/Total`) have no edge | | `module_name` | string | Source module | ### project_security diff --git a/.claude/skills/verify-in-runtime.md b/.claude/skills/verify-in-runtime.md index 899e8dd93a..7c8e733d31 100644 --- a/.claude/skills/verify-in-runtime.md +++ b/.claude/skills/verify-in-runtime.md @@ -70,8 +70,11 @@ So: - **Default to `mxcli playwright check`** (app already running) **or `run --page-check`** (while booting it). They answer the question and leave the PNG unread. The exit status is the verdict: 0 every page passed; 1 a page showed an error banner or - dialog, a console error, a failed request, an HTTP error, the sign-in form, or a - failed `--assert-*`; 2 the check could not run. + dialog, a console error, a failed request, an HTTP error, the sign-in form, a + page the signed-in role may not open (`ACCESS DENIED`, `messageCode=403` — a + page-access problem, not a sign-in one), a `/p/` URL that ended elsewhere + (`url=/`: no page has that URL, so the home page rendered), or a failed + `--assert-*`; 2 the check could not run. - **Never hand-roll a playwright-cli login/goto/sleep sequence** to look at a page (`playwright-cli open …/login.html; fill …; click …; goto …; sleep 6; eval …; screenshot`). `playwright check` does all of it in one call: it signs in with diff --git a/cmd/mxcli/cmd_diff.go b/cmd/mxcli/cmd_diff.go index 419913c0ab..3e7de74ae8 100644 --- a/cmd/mxcli/cmd_diff.go +++ b/cmd/mxcli/cmd_diff.go @@ -131,6 +131,11 @@ var diffLocalCmd = &cobra.Command{ This command finds modified mxunit files in the mprcontents/ folder and shows the differences as MDL. Only works with MPR v2 format (Mendix 10.18+). +A unit that changed only in properties the MDL does not show (navigation, +security, an index, …) is listed on one line naming those properties. +Configuration constant values are masked: a changed one shows as + against . + The --ref flag accepts any git ref or range (e.g., HEAD, main, main..feature-branch). Examples: diff --git a/cmd/mxcli/cmd_log.go b/cmd/mxcli/cmd_log.go index a8dcd116e3..13e847b792 100644 --- a/cmd/mxcli/cmd_log.go +++ b/cmd/mxcli/cmd_log.go @@ -189,25 +189,34 @@ func logAdminOptions(cmd *cobra.Command) docker.M2EEOptions { pass, _ := cmd.Flags().GetString("admin-pass") projectPath, _ := cmd.Flags().GetString("project") opts := docker.M2EEOptions{Host: host, Port: port, Token: pass, Direct: true} - return devLoopAdminOptions(projectPath, opts, adminFlagsSet{ + return devLoopAdminOptions(projectPath, opts, logAdminFlagsSet(cmd)) +} + +// logAdminFlagsSet records which admin connection settings `mxcli log` was +// given explicitly. +func logAdminFlagsSet(cmd *cobra.Command) adminFlagsSet { + return adminFlagsSet{ host: cmd.Flags().Changed("admin-host"), port: cmd.Flags().Changed("admin-port"), token: cmd.Flags().Changed("admin-pass") || os.Getenv("MXCLI_ADMIN_PASS") != "", - }) + } } // logConnectionHint names the most likely cause when nothing is listening — but // ONLY then. The runtime rejecting a request (an unknown node, a bad level) is // not a connection problem, and telling someone to start an app they clearly -// already have running buries the sentence that matters. +// already have running buries the sentence that matters. When -p names a +// project whose run --local did not supply the connection, it says that too: +// a 403 is then a stopped run, not a bad password (ako/mxcli#1542). func logConnectionHint(cmd *cobra.Command, err error) string { if !errors.Is(err, docker.ErrAdminUnreachable) { return "" } opts := logAdminOptions(cmd) + projectPath, _ := cmd.Flags().GetString("project") return fmt.Sprintf(" Log levels come from a RUNNING app's admin API (%s:%d).\n"+ " Start one with 'mxcli run --local -p ' (and pass the same -p here), or point at another with --admin-host/--admin-port/--admin-pass.\n", - opts.Host, opts.Port) + opts.Host, opts.Port) + devLoopAdminHint(projectPath, logAdminFlagsSet(cmd), err) } func init() { diff --git a/cmd/mxcli/cmd_playwright.go b/cmd/mxcli/cmd_playwright.go index 80873fd1f0..f92b17bc98 100644 --- a/cmd/mxcli/cmd_playwright.go +++ b/cmd/mxcli/cmd_playwright.go @@ -177,9 +177,19 @@ for -p, else APP_PORT in .docker/.env, else http://localhost:8080. Login: --user/--password, or --role R to sign in as the project's demo user with that user role (needs -p). With only -p, a secured app is signed into as one of -the project's demo users (an administrator first). The session is saved under -/.mxcli/playwright-check/ and reused by later checks; it is renewed -automatically when the runtime has forgotten it (after a restart). +the project's demo users (an administrator first). The sign-in happens on the +host of the pages checked — a session signed in on localhost is not sent to +127.0.0.1 or to another name for the same app — and is saved per host and user +under /.mxcli/playwright-check/, where later checks reuse it: one +session however many checks run. It is renewed automatically when the runtime +has forgotten it (after a restart). --fresh-login signs the saved session out +before signing in again; --sign-out ends the run signed out, keeping nothing. +A session that got no page past the sign-in form is signed out, not kept. A page +the signed-in user's roles may not open fails as ACCESS DENIED (the client lands +on /login.html?messageCode=403); its session is valid, so it is kept. When +a sign-in is refused, the runtime log (/.mxcli/runtime.log) is read for +the cause: the unlicensed local runtime caps concurrent sessions, and its sign-in +page reports that as a wrong password. A page is measured once it has settled: no request to the app open for half a second, no progress indicator, and its text no longer changing. Waiting for the @@ -189,10 +199,15 @@ page that has not settled within --wait is reported STILL LOADING, naming the requests it was waiting for, and fails. A page with a list view or grid always reports its rows, rows=0 included. +When the browser ends somewhere other than the URL it was sent to, the verdict +says where (url=...), and the title and heading describe that page. A /p/ URL +that ends elsewhere fails: the client sends a /p/ URL it has no page for to the +home page, with HTTP 200, so without this an unknown page passed as the home page. + Exit status: 0 every page passed; 1 a page failed (still loading when measured, -HTTP error, sign-in form instead of the page, error banner or dialog, console -error, failed same-origin request, or a failed assertion); 2 the check could -not run. +HTTP error, access denied to the signed-in user, sign-in form instead of the +page, a /p/ URL that ended elsewhere, error banner or dialog, console error, +failed same-origin request, or a failed assertion); 2 the check could not run. Examples: mxcli playwright check /p/customers /p/orders -p app.mpr @@ -212,6 +227,7 @@ Examples: shot, _ := cmd.Flags().GetString("screenshot") wait, _ := cmd.Flags().GetDuration("wait") fresh, _ := cmd.Flags().GetBool("fresh-login") + signOut, _ := cmd.Flags().GetBool("sign-out") allowConsole, _ := cmd.Flags().GetBool("allow-console-errors") if user != "" && role != "" { @@ -236,6 +252,7 @@ Examples: Password: password, Role: role, FreshLogin: fresh, + SignOut: signOut, Spec: spec, Screenshot: shot, WaitMs: int(wait.Milliseconds()), @@ -292,7 +309,8 @@ func init() { playwrightCheckCmd.Flags().StringArray("assert-count", nil, "Fail unless SELECTOR OP N holds, e.g. '.mx-listview li>=1' (ops >= <= == != > <; bare selector = >=1; repeatable)") playwrightCheckCmd.Flags().String("screenshot", "", "Also write a full-page PNG here (per-page suffix for several pages); only the path is printed") playwrightCheckCmd.Flags().Duration("wait", time.Duration(docker.DefaultPageWaitMs)*time.Millisecond, "Longest a page may keep loading after it mounts (open requests, changing text) before it is measured; a page still loading then fails as STILL LOADING") - playwrightCheckCmd.Flags().Bool("fresh-login", false, "Ignore the saved session and sign in again") + playwrightCheckCmd.Flags().Bool("fresh-login", false, "Sign the saved session out and sign in again") + playwrightCheckCmd.Flags().Bool("sign-out", false, "Sign out when the check is done instead of keeping the session for the next check") playwrightCheckCmd.Flags().Bool("allow-console-errors", false, "Report console errors but do not fail on them") playwrightCmd.AddCommand(playwrightCheckCmd) diff --git a/cmd/mxcli/cmd_playwright_check_test.go b/cmd/mxcli/cmd_playwright_check_test.go index eea3ae46d1..e007b2cd12 100644 --- a/cmd/mxcli/cmd_playwright_check_test.go +++ b/cmd/mxcli/cmd_playwright_check_test.go @@ -25,7 +25,7 @@ func TestPlaywrightCheckFlags(t *testing.T) { if err != nil || cmd != playwrightCheckCmd { t.Fatalf("playwright check not registered: %v", err) } - for _, f := range []string{"user", "password", "role", "assert-text", "assert-count", "screenshot", "base-url", "fresh-login", "wait"} { + for _, f := range []string{"user", "password", "role", "assert-text", "assert-count", "screenshot", "base-url", "fresh-login", "sign-out", "wait"} { if cmd.Flags().Lookup(f) == nil { t.Errorf("missing flag --%s", f) } diff --git a/cmd/mxcli/cmd_run.go b/cmd/mxcli/cmd_run.go index 32018d5b74..a4fe094279 100644 --- a/cmd/mxcli/cmd_run.go +++ b/cmd/mxcli/cmd_run.go @@ -342,7 +342,12 @@ Examples: fmt.Fprintf(os.Stderr, "Warning: could not publish the dev-loop handshake: %v\n", err) } } - defer removeDevLoopHandshake(projectPath) + // Withdrawn only by this process, and only if it published one: a + // `--setup` run (the SessionStart hook, on every resume and compaction) + // used to delete the handshake of the detached run serving the app + // (ako/mxcli#1542). The deferred call covers a panic; the normal paths + // withdraw right after recording the end in the run state, below. + defer withdrawDevLoopHandshake(projectPath, os.Getpid()) // Publish the run's state for `mxcli run status/wait/stop` — for a // foreground run too, so another terminal (or an agent) can manage it. @@ -363,12 +368,16 @@ Examples: } err = docker.RunLocal(opts) + // One moment, two files: the run state says stopped (with the reason, when + // the runtime stopped on its own) before the handshake disappears, so a + // caller that finds run-local.json gone finds the explanation in `run + // status` (ako/mxcli#1542). state.Exit(err) + withdrawDevLoopHandshake(projectPath, os.Getpid()) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) // os.Exit skips deferred calls, so remove explicitly here. hosted.Remove() - removeDevLoopHandshake(projectPath) os.Exit(1) } }, diff --git a/cmd/mxcli/cmd_run_lifecycle.go b/cmd/mxcli/cmd_run_lifecycle.go index 2b31e7b562..66ffe4e236 100644 --- a/cmd/mxcli/cmd_run_lifecycle.go +++ b/cmd/mxcli/cmd_run_lifecycle.go @@ -64,7 +64,16 @@ stopped, its URL, pid and uptime, the last build generation and how it was applied (reload or restart) or why it failed, and whether a change on disk is still waiting to be applied. -A state file whose process is gone is reported as stopped, and says so. +A state file whose process is gone is reported as stopped, and says so. A run +whose runtime stopped on its own (the trial licence's time limit, a crash) says +why when its runtime log does: + + stopped: trial licence time limit reached at 18:43:16 (runtime.log); restart with mxcli run restart -p app.mpr + +Check this first when .mxcli/run-local.json is missing or the admin API answers +401/403: that file is removed when the run ends, and a command that cannot find +it falls back to default credentials. A live run whose run-local.json is gone is +reported as running, with the missing file named. Exit codes: 0 running, 3 not running, 4 still starting.`, Example: ` mxcli run status -p app.mpr @@ -368,11 +377,77 @@ func runStatus(projectPath string, out io.Writer) int { fmt.Fprintf(out, "unknown: %v\n", err) return runExitFailed } - line, code := statusLine(st, docker.PidAlive, docker.SourceMTime(projectPath), time.Now(), leftoverCount) + line, code := statusLine(st, liveStatusProbes(), docker.SourceMTime(projectPath), time.Now()) fmt.Fprintln(out, line) return code } +// runStatusSummary is the line `mxcli run status -p projectPath` would print, +// for an error elsewhere to quote; "" when no run of the project is recorded. +func runStatusSummary(projectPath string) string { + st, err := docker.ReadRunState(projectPath) + if err != nil || st == nil { + return "" + } + line, _ := statusLine(st, liveStatusProbes(), docker.SourceMTime(projectPath), time.Now()) + return line +} + +// statusProbes are what statusLine asks of the system, injected so its rules +// are testable without processes. A nil probe has nothing to say. +type statusProbes struct { + alive func(int) bool + leftovers func(*docker.RunState) int + // logReason reads why the runtime stopped from the run's runtime log. + logReason func(*docker.RunState) string + // handshake reports whether the run's .mxcli/run-local.json exists. + handshake func(*docker.RunState) bool +} + +// liveStatusProbes asks the real system. +func liveStatusProbes() statusProbes { + return statusProbes{ + alive: docker.PidAlive, + leftovers: leftoverCount, + logReason: func(st *docker.RunState) string { return docker.RuntimeLogStopReason(st.RuntimeLog) }, + handshake: func(st *docker.RunState) bool { + if st.Project == "" { + return true // cannot tell; do not cry wolf + } + _, err := os.Stat(devLoopHandshakePath(st.Project)) + return err == nil + }, + } +} + +// probesWithAlive is liveStatusProbes with liveness already decided by the +// caller, so one snapshot gives one answer. +func probesWithAlive(alive bool) statusProbes { + p := liveStatusProbes() + p.alive = func(int) bool { return alive } + return p +} + +func (p statusProbes) reasonFromLog(st *docker.RunState) string { + if p.logReason == nil || st.RuntimeLog == "" { + return "" + } + return p.logReason(st) +} + +func (p statusProbes) handshakePublished(st *docker.RunState) bool { + return p.handshake == nil || p.handshake(st) +} + +// restartHint is the command that brings a stopped run back with the same +// arguments. +func restartHint(st *docker.RunState) string { + if st.Project == "" { + return "mxcli run restart" + } + return "mxcli run restart -p " + st.Project +} + // leftoverCount is how many processes of a dead detached run are still alive. func leftoverCount(st *docker.RunState) int { if st == nil || !st.Detached { @@ -383,19 +458,29 @@ func leftoverCount(st *docker.RunState) int { // statusLine renders the one status line and its exit code. Its inputs are // injected so the staleness rules are testable without processes. -func statusLine(st *docker.RunState, alive func(int) bool, source, now time.Time, leftovers func(*docker.RunState) int) (string, int) { +// +// A run that stopped on its own says why when anything recorded it — the run +// state's reason, else the runtime log — because a caller that only learns +// "stopped" (or, worse, finds run-local.json gone) goes looking at its +// credentials instead (ako/mxcli#1542). +func statusLine(st *docker.RunState, p statusProbes, source, now time.Time) (string, int) { if st == nil { return "stopped: no run recorded for this project", runExitNotRunning } - live := alive(st.PID) + live := p.alive(st.PID) if !live && (st.Phase == docker.PhaseStarting || st.Phase == docker.PhaseRunning) { line := fmt.Sprintf("stopped: pid %d exited without cleaning up (it was %s", st.PID, st.Phase) if st.URL != "" { line += " at " + st.URL } line += ")" - if n := leftovers(st); n > 0 { - line += fmt.Sprintf("; %d leftover process(es) still running — mxcli run stop reaps them", n) + if reason := p.reasonFromLog(st); reason != "" { + line += "; its runtime log says: " + reason + } + if p.leftovers != nil { + if n := p.leftovers(st); n > 0 { + line += fmt.Sprintf("; %d leftover process(es) still running — mxcli run stop reaps them", n) + } } return line, runExitNotRunning } @@ -403,6 +488,15 @@ func statusLine(st *docker.RunState, alive func(int) bool, source, now time.Time case docker.PhaseFailed: return fmt.Sprintf("stopped: boot failed — %s (log %s)", firstLineOf(st.Error), displayPath(st.Log)), runExitNotRunning case docker.PhaseStopped: + reason := st.Reason + if reason == "" && st.Error != "" { + // A run recorded by an older mxcli, or one that ended on another + // error: the runtime log may still know. + reason = p.reasonFromLog(st) + } + if reason != "" { + return "stopped: " + reason + "; restart with " + restartHint(st), runExitNotRunning + } if st.Error != "" { return "stopped: " + firstLineOf(st.Error), runExitNotRunning } @@ -441,7 +535,14 @@ func statusLine(st *docker.RunState, alive func(int) bool, source, now time.Time case source.After(covered): parts = append(parts, "changes on disk NOT applied (no --watch): mxcli run restart applies them") } - return fmt.Sprintf("running: %s (%s)", st.URL, strings.Join(parts, ", ")), runExitOK + line := fmt.Sprintf("running: %s (%s)", st.URL, strings.Join(parts, ", ")) + if !p.handshakePublished(st) { + // The app is up, but the file other commands find it by is gone — so + // they fall back to default credentials and meet a 403 (#1542). + line += "; but .mxcli/run-local.json is MISSING, so oql, log and constant --apply cannot " + + "reach its admin API — " + restartHint(st) + " publishes it again" + } + return line, runExitOK } // --- wait -------------------------------------------------------------- @@ -472,7 +573,7 @@ func runWait(projectPath string, timeout time.Duration, since int, out io.Writer printBootFailure(out, "boot failed: "+st.Error, st.Log) return runExitFailed case docker.WaitNotRunning: - line, _ := statusLine(st, func(int) bool { return alive }, target, time.Now(), leftoverCount) + line, _ := statusLine(st, probesWithAlive(alive), target, time.Now()) fmt.Fprintf(out, "not running: %s\n", strings.TrimPrefix(line, "stopped: ")) return runExitNotRunning case docker.WaitNotWatching: @@ -509,7 +610,7 @@ func runWaitReady(projectPath string, timeout time.Duration, out io.Writer) int printBootFailure(out, st.Error, st.Log) return runExitFailed case docker.WaitNotRunning: - line, _ := statusLine(st, func(int) bool { return alive }, time.Time{}, time.Now(), leftoverCount) + line, _ := statusLine(st, probesWithAlive(alive), time.Time{}, time.Now()) fmt.Fprintf(out, "not running: %s\n", strings.TrimPrefix(line, "stopped: ")) return runExitNotRunning } @@ -554,6 +655,8 @@ func runStop(projectPath string, out io.Writer) int { cur.Phase, cur.Ended, cur.Building = docker.PhaseStopped, time.Now(), false _ = docker.WriteRunState(projectPath, cur) } + // And its handshake, which only it could withdraw (ako/mxcli#1542). + withdrawDevLoopHandshake(projectPath, st.PID) } if left := remaining(st); len(left) > 0 { diff --git a/cmd/mxcli/cmd_run_lifecycle_test.go b/cmd/mxcli/cmd_run_lifecycle_test.go index 7f43254630..dd9d6eda3d 100644 --- a/cmd/mxcli/cmd_run_lifecycle_test.go +++ b/cmd/mxcli/cmd_run_lifecycle_test.go @@ -79,7 +79,7 @@ func TestStatusLine(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - line, code := statusLine(c.st, c.alive, c.source, now, c.leftovers) + line, code := statusLine(c.st, statusProbes{alive: c.alive, leftovers: c.leftovers}, c.source, now) if code != c.wantCode { t.Errorf("exit %d, want %d (%s)", code, c.wantCode, line) } diff --git a/cmd/mxcli/cmd_run_status_reason_test.go b/cmd/mxcli/cmd_run_status_reason_test.go new file mode 100644 index 0000000000..49104a99b0 --- /dev/null +++ b/cmd/mxcli/cmd_run_status_reason_test.go @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/mendixlabs/mxcli/cmd/mxcli/docker" +) + +// ako/mxcli#1542, expected behaviour 1, verbatim: "When the runtime stops +// (licence limit, crash, kill), `run status` says so, with the reason when the +// runtime log gives one: e.g. `stopped: trial licence time limit reached at +// 18:43:16 (runtime.log); restart with mxcli run restart`." + +const licenceStoppedRuntimeLog = "\n=== runtime start 2026-10-26T14:51:02Z ===\n" + + "2026-10-26 14:51:20.101 WARNING - LicenseService: The runtime has been started using a trial licence,\n" + + " the framework will be terminated when the maximum time is exceeded!\n" + + "2026-10-26 18:43:16.412 CRITICAL - LicenseService: Maximum run time exceeded, framework is now terminating\n" + + "\tat com.mendix.basis.license.LicenseUtil.checkRuntimeDuration(LicenseUtil.java:212)\n" + + "2026-10-26 18:43:16.530 INFO - Core: Mendix Runtime is shutting down\n" + +// stoppedRunFixture writes the runtime.log and the run state a licence stop +// leaves behind, and returns the project. +func stoppedRunFixture(t *testing.T, mutate func(*docker.RunState)) string { + t.Helper() + mpr := scratchProject(t) + logPath := filepath.Join(filepath.Dir(mpr), ".mxcli", "runtime.log") + if err := os.MkdirAll(filepath.Dir(logPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(logPath, []byte(licenceStoppedRuntimeLog), 0o644); err != nil { + t.Fatal(err) + } + now := time.Now() + st := &docker.RunState{ + Project: mpr, PID: 1 << 30, Phase: docker.PhaseStopped, Watch: true, + URL: "http://127.0.0.1:8080/", RuntimeLog: logPath, Args: []string{"run", "--local", "--watch", "-p", mpr}, + Started: now.Add(-4 * time.Hour), Ready: now.Add(-4 * time.Hour), Ended: now.Add(-time.Minute), + // Exactly what a licence stop recorded: the reason was never captured, + // so only the generic first line remained. + Error: "the app stopped: the Mendix runtime is no longer running\n" + + " mxcli is exiting rather than staying up over a dead app", + } + if mutate != nil { + mutate(st) + } + if err := docker.WriteRunState(mpr, st); err != nil { + t.Fatal(err) + } + return mpr +} + +func TestRunStatus_RuntimeStopNamesTheReason(t *testing.T) { + mpr := stoppedRunFixture(t, nil) + var out bytes.Buffer + code := runStatus(mpr, &out) + line := strings.TrimSpace(out.String()) + if code != runExitNotRunning { + t.Errorf("exit %d, want %d (%s)", code, runExitNotRunning, line) + } + for _, want := range []string{ + "stopped: trial licence time limit reached at 18:43:16 (runtime.log)", + "mxcli run restart", + } { + if !strings.Contains(line, want) { + t.Errorf("run status does not say why the app stopped — callers blamed their "+ + "credentials instead (ako/mxcli#1542)\n got: %s\nwant: %s", line, want) + } + } +} + +// A run killed outright never records its end; the pid is gone and the state +// still says running. The runtime log can still say why. +func TestRunStatus_KilledRunStillNamesTheLogReason(t *testing.T) { + mpr := stoppedRunFixture(t, func(st *docker.RunState) { + st.Phase, st.Error, st.Ended = docker.PhaseRunning, "", time.Time{} + }) + var out bytes.Buffer + code := runStatus(mpr, &out) + line := strings.TrimSpace(out.String()) + if code != runExitNotRunning { + t.Errorf("exit %d, want %d (%s)", code, runExitNotRunning, line) + } + for _, want := range []string{"stopped: pid", "exited without cleaning up", "trial licence time limit reached at 18:43:16"} { + if !strings.Contains(line, want) { + t.Errorf("%q lacks %q", line, want) + } + } +} + +// The reason the run recorded wins over re-reading the log, and a plain +// requested stop still says only "stopped". +func TestStatusLine_StoppedReason(t *testing.T) { + now := time.Now() + st := &docker.RunState{Project: "/p/App.mpr", PID: 7, Phase: docker.PhaseStopped, Ended: now, + Error: "the app stopped: the Mendix runtime is no longer running", Reason: "last runtime error at 12:30:02: Core: db gone (runtime.log)"} + line, code := statusLine(st, statusProbes{alive: neverAlive}, now, now) + if code != runExitNotRunning || line != "stopped: last runtime error at 12:30:02: Core: db gone (runtime.log); restart with mxcli run restart -p /p/App.mpr" { + t.Errorf("exit %d, line %q", code, line) + } + plain := &docker.RunState{Project: "/p/App.mpr", PID: 7, Phase: docker.PhaseStopped, Ended: now, RuntimeLog: "/p/.mxcli/runtime.log"} + probes := statusProbes{alive: neverAlive, logReason: func(*docker.RunState) string { return "should not be asked" }} + if line, _ := statusLine(plain, probes, now, now); line != "stopped" { + t.Errorf("a requested stop must not borrow a reason from the log: %q", line) + } +} + +// Expected behaviour, and the state #1542 was found in: the app is up, the +// handshake is gone. Saying "stopped" would be false (and `run status || +// run --detach` would then collide with the live run); saying nothing leaves +// the 403 unexplained. Running, exit 0, with the missing file named. +func TestStatusLine_RunningButHandshakeMissing(t *testing.T) { + now := time.Now() + st := &docker.RunState{Project: "/p/App.mpr", PID: 7, Phase: docker.PhaseRunning, Watch: true, + URL: "http://127.0.0.1:8080/", Ready: now.Add(-time.Hour)} + missing := statusProbes{alive: alwaysAlive, handshake: func(*docker.RunState) bool { return false }} + line, code := statusLine(st, missing, time.Time{}, now) + if code != runExitOK || !strings.HasPrefix(line, "running: ") { + t.Errorf("exit %d, line %q — the app is still up", code, line) + } + for _, want := range []string{"run-local.json is MISSING", "oql, log and constant --apply", "mxcli run restart -p /p/App.mpr"} { + if !strings.Contains(line, want) { + t.Errorf("%q lacks %q", line, want) + } + } + present := statusProbes{alive: alwaysAlive, handshake: func(*docker.RunState) bool { return true }} + if line, _ := statusLine(st, present, time.Time{}, now); strings.Contains(line, "MISSING") { + t.Errorf("CONTROL: a published handshake must not be reported missing: %q", line) + } +} + +// Expected behaviour 3's caller side: "what a caller should do when it is +// missing (check `run status`, not credentials)". The fake admin API stands in +// for a runtime that is up but was booted with another password. +func TestDevLoopAdminHint_ForbiddenWithoutHandshakePointsAtRunStatus(t *testing.T) { + t.Setenv("ADMIN_PORT", "") + t.Setenv("M2EE_ADMIN_PASS", "") + p := stoppedRunFixture(t, func(st *docker.RunState) { + st.Reason = "trial licence time limit reached at 18:43:16 (runtime.log)" + }) + port := fakeAdminAPI(t, "loop-secret") + set := adminFlagsSet{port: true} + opts := devLoopAdminOptions(p, docker.M2EEOptions{ProjectPath: p, Port: port, Direct: true}, set) + _, err := docker.ExecuteOQL(docker.OQLOptions{Host: "127.0.0.1", Port: opts.Port, Token: opts.Token, + ProjectPath: p, Direct: true}, "SELECT 1") + if err == nil || !strings.Contains(err.Error(), "authentication failed") { + t.Fatalf("expected the admin API to refuse the default password, got %v", err) + } + hint := devLoopAdminHint(p, set, err) + t.Logf("what `mxcli oql` prints:\nError: %v\n%s", err, hint) + for _, want := range []string{ + "run-local.json is missing", + "is NOT a\n wrong password", + "mxcli run status -p " + p, + "stopped: trial licence time limit reached at 18:43:16 (runtime.log)", + } { + if !strings.Contains(hint, want) { + t.Errorf("hint lacks %q:\n%s", want, hint) + } + } + + // A stale handshake is named as such. + if err := writeDevLoopHandshake(p, devLoopHandshake{Project: p, PID: 1 << 30, AdminPort: port}); err != nil { + t.Fatal(err) + } + if hint := devLoopAdminHint(p, set, err); !strings.Contains(hint, "names pid 1073741824, which is no longer running") { + t.Errorf("stale handshake hint:\n%s", hint) + } + + // CONTROL: with the live run's handshake in hand there is nothing to explain. + publishFakeLoop(t, p, port, "loop-secret") + if hint := devLoopAdminHint(p, set, err); hint != "" { + t.Errorf("a live handshake was used; no hint expected, got:\n%s", hint) + } + // Nor when the user named the endpoint themselves. + if hint := devLoopAdminHint(p, adminFlagsSet{host: true}, err); hint != "" { + t.Errorf("explicit --host: no hint expected, got:\n%s", hint) + } +} + +// `mxcli log` gets the same explanation; its admin errors arrive wrapped as +// unreachable, so the auth failure must survive the wrapping. +func TestLogConnectionHint_ExplainsAMissingHandshake(t *testing.T) { + p := stoppedRunFixture(t, func(st *docker.RunState) { + st.Reason = "trial licence time limit reached at 18:43:16 (runtime.log)" + }) + port := fakeAdminAPI(t, "loop-secret") + cmd := newLogFlagsCmd() + if err := cmd.ParseFlags([]string{"-p", p, "--admin-port", strconv.Itoa(port)}); err != nil { + t.Fatal(err) + } + _, err := docker.GetLogSettings(logAdminOptions(cmd)) + if err == nil || !errors.Is(err, docker.ErrAdminAuth) { + t.Fatalf("expected an auth failure that errors.Is can still see, got %v", err) + } + hint := logConnectionHint(cmd, err) + if !strings.Contains(hint, "run-local.json is missing") || !strings.Contains(hint, "trial licence time limit reached") { + t.Errorf("log hint:\n%s", hint) + } +} + +// `constant set --apply` reads the handshake itself; its refusal quotes run +// status too. +func TestReadDevLoopHandshake_MissingQuotesRunStatus(t *testing.T) { + p := stoppedRunFixture(t, func(st *docker.RunState) { + st.Reason = "trial licence time limit reached at 18:43:16 (runtime.log)" + }) + _, err := readDevLoopHandshake(p) + if err == nil || !errors.Is(err, errNoDevLoopHandshake) || + !strings.Contains(err.Error(), "mxcli run status: stopped: trial licence time limit reached at 18:43:16") { + t.Fatalf("got %v", err) + } +} diff --git a/cmd/mxcli/devloop_admin.go b/cmd/mxcli/devloop_admin.go index 2c4f2ade3d..e72d63aa74 100644 --- a/cmd/mxcli/devloop_admin.go +++ b/cmd/mxcli/devloop_admin.go @@ -2,7 +2,12 @@ package main -import "github.com/mendixlabs/mxcli/cmd/mxcli/docker" +import ( + "errors" + "fmt" + + "github.com/mendixlabs/mxcli/cmd/mxcli/docker" +) // adminFlagsSet records which admin connection settings the user gave // explicitly. Those always win; only the rest are taken from a dev loop. @@ -42,3 +47,39 @@ func devLoopAdminOptions(projectPath string, opts docker.M2EEOptions, set adminF opts.Direct = true return opts } + +// devLoopAdminHint explains an admin-API failure for a project whose `mxcli run +// --local` did not supply the connection — its .mxcli/run-local.json missing, +// or naming a process that is gone — so defaults went to whatever answers on +// that port. From a runtime that is up, that is HTTP 403, which reads as a +// wrong password; twice in one day it was a run that had stopped (or a +// handshake deleted under a live one), and the caller blamed the credentials +// (ako/mxcli#1542). The hint says so and quotes `run status`. +// +// It is "" when the handshake was used, when the user named the endpoint +// themselves, and for a failure that is neither reaching nor authenticating. +func devLoopAdminHint(projectPath string, set adminFlagsSet, err error) string { + if err == nil || projectPath == "" || set.host || (set.port && set.token) { + return "" + } + if !errors.Is(err, docker.ErrAdminAuth) && !errors.Is(err, docker.ErrAdminUnreachable) { + return "" + } + hs, herr := readDevLoopHandshake(projectPath) + if herr == nil { + return "" + } + why := "is missing" + if errors.Is(herr, errStaleDevLoopHandshake) { + why = fmt.Sprintf("names pid %d, which is no longer running", hs.PID) + } else if !errors.Is(herr, errNoDevLoopHandshake) { + why = "could not be read" + } + hint := fmt.Sprintf(" .mxcli/run-local.json %s, so these were default connection settings, not those of\n"+ + " this project's 'mxcli run --local'. When that run has stopped, a 401/403 here is NOT a\n"+ + " wrong password. Check the run before the credentials:\n", why) + if line := runStatusSummary(projectPath); line != "" { + return hint + " mxcli run status -p " + projectPath + "\n → " + line + "\n" + } + return hint + " mxcli run status -p " + projectPath + "\n" +} diff --git a/cmd/mxcli/devloop_handshake.go b/cmd/mxcli/devloop_handshake.go index d3b0e4ac59..f0dc17d094 100644 --- a/cmd/mxcli/devloop_handshake.go +++ b/cmd/mxcli/devloop_handshake.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -72,8 +73,27 @@ func writeDevLoopHandshake(projectPath string, h devLoopHandshake) error { return nil } -func removeDevLoopHandshake(projectPath string) { - _ = os.Remove(devLoopHandshakePath(projectPath)) +// withdrawDevLoopHandshake removes the handshake the run with pid published, +// and nothing else. +// +// Removal used to be unconditional, and every `mxcli run --local` deferred it — +// including `--setup`, which never publishes one and which the SessionStart +// hook runs on every session start, resume and compaction included. So while a +// detached run was serving the app, the hook deleted ITS handshake: oql, log +// and constant --apply fell back to default credentials, met HTTP 403 from a +// runtime that was up, and the caller blamed the password (ako/mxcli#1542). +// A file that cannot be read is left alone too: it is not provably ours. +func withdrawDevLoopHandshake(projectPath string, pid int) { + path := devLoopHandshakePath(projectPath) + body, err := os.ReadFile(path) + if err != nil { + return + } + var h devLoopHandshake + if json.Unmarshal(body, &h) != nil || h.PID != pid { + return + } + _ = os.Remove(path) } // readDevLoopHandshake returns the handshake for a project, or an error a user @@ -82,13 +102,18 @@ func removeDevLoopHandshake(projectPath string) { // A handshake whose process is gone is refused rather than used: the ports in it // may since have been taken by something else, and sending a configuration // change to the wrong process is worse than saying no dev loop is running. +// +// Both refusals quote `run status`, which knows what became of the run — the +// licence limit, a crash — where the missing file alone reads like a +// credentials problem (ako/mxcli#1542). func readDevLoopHandshake(projectPath string) (devLoopHandshake, error) { var h devLoopHandshake path := devLoopHandshakePath(projectPath) body, err := os.ReadFile(path) if os.IsNotExist(err) { - return h, fmt.Errorf("no 'mxcli run --local' is serving this project\n" + - " start one in another terminal, or drop --apply to only record the value") + return h, fmt.Errorf("%w\n"+ + " start one in another terminal, or drop --apply to only record the value%s", + errNoDevLoopHandshake, runStatusNote(projectPath)) } if err != nil { return h, fmt.Errorf("reading %s: %w", path, err) @@ -97,8 +122,26 @@ func readDevLoopHandshake(projectPath string) (devLoopHandshake, error) { return h, fmt.Errorf("%s is not valid JSON: %w", path, err) } if !procalive.Alive(h.PID) { - return h, fmt.Errorf("%s refers to process %d, which is no longer running\n"+ - " (the dev loop was stopped without cleaning up; start a new one)", path, h.PID) + return h, fmt.Errorf("%s refers to process %d, which is %w\n"+ + " (the dev loop was stopped without cleaning up; start a new one)%s", + path, h.PID, errStaleDevLoopHandshake, runStatusNote(projectPath)) } return h, nil } + +var ( + // errNoDevLoopHandshake: no run-local.json — no run is serving, or it has + // ended and withdrawn it. + errNoDevLoopHandshake = errors.New("no 'mxcli run --local' is serving this project") + // errStaleDevLoopHandshake: run-local.json names a process that is gone. + errStaleDevLoopHandshake = errors.New("no longer running") +) + +// runStatusNote is "\n mxcli run status: " for a project with a +// recorded run, or "". +func runStatusNote(projectPath string) string { + if line := runStatusSummary(projectPath); line != "" { + return "\n mxcli run status: " + line + } + return "" +} diff --git a/cmd/mxcli/devloop_handshake_test.go b/cmd/mxcli/devloop_handshake_test.go index 161e09831b..b14b35ae07 100644 --- a/cmd/mxcli/devloop_handshake_test.go +++ b/cmd/mxcli/devloop_handshake_test.go @@ -109,14 +109,44 @@ func TestReadDevLoopHandshake_CorruptFileIsNamed(t *testing.T) { } } -func TestRemoveDevLoopHandshake(t *testing.T) { +func TestWithdrawDevLoopHandshake(t *testing.T) { p := handshakeProject(t) if err := writeDevLoopHandshake(p, devLoopHandshake{PID: os.Getpid()}); err != nil { t.Fatal(err) } - removeDevLoopHandshake(p) + withdrawDevLoopHandshake(p, os.Getpid()) if _, err := os.Stat(devLoopHandshakePath(p)); !os.IsNotExist(err) { - t.Errorf("the handshake survived removal: %v", err) + t.Errorf("the handshake survived removal by the run that wrote it: %v", err) + } + withdrawDevLoopHandshake(p, os.Getpid()) // idempotent +} + +// ako/mxcli#1542: ".mxcli/run-local.json disappears while the app is believed +// up ... Callers then got HTTP 403 from the admin API and blamed credentials." +// +// The SessionStart hook runs `mxcli run --local --setup --ensure-db` on every +// session start — resume and compaction included — while a detached run is +// serving the app. That setup-only process never publishes a handshake, yet on +// exit it withdrew one: the live run's. Every other `run --local` that failed +// early did the same. Only the run that wrote the file may remove it. +func TestWithdrawDevLoopHandshake_LeavesAnotherRunsHandshake(t *testing.T) { + p := handshakeProject(t) + // The parent (the test driver) stands in for the live detached run: alive + // for the whole test, and not this process. + live := os.Getppid() + if err := writeDevLoopHandshake(p, devLoopHandshake{Project: p, PID: live, AdminPort: 8091, AdminPass: "loop-secret"}); err != nil { + t.Fatal(err) + } + + withdrawDevLoopHandshake(p, os.Getpid()) // a `run --local --setup` exiting + + hs, err := readDevLoopHandshake(p) + if err != nil { + t.Fatalf("the live run's handshake was removed by a process that never wrote it (a "+ + "`run --local --setup` exiting) — oql/log/constant --apply now fall back to default "+ + "credentials while the app is up (ako/mxcli#1542): %v", err) + } + if hs.PID != live || hs.AdminPass != "loop-secret" { + t.Fatalf("handshake changed: %+v", hs) } - removeDevLoopHandshake(p) // idempotent } diff --git a/cmd/mxcli/docker/build.go b/cmd/mxcli/docker/build.go index 6ba68869bb..da8762c777 100644 --- a/cmd/mxcli/docker/build.go +++ b/cmd/mxcli/docker/build.go @@ -65,7 +65,7 @@ var mxbuildCmd = func(mxbuildPath, javaHome, outputDir, mprPath string, w, stder ) cmd.Stdout = w cmd.Stderr = stderr - PrepareMxCommand(cmd) + prepareOneShotMxCommand(cmd) // one build, so no Gradle daemon (ako/mxcli#1340) return runWithWatchdog(cmd, mxbuildWatchdogLimits(), javaHome) } diff --git a/cmd/mxcli/docker/ensuredb.go b/cmd/mxcli/docker/ensuredb.go index d4324a9d59..1bd3954dad 100644 --- a/cmd/mxcli/docker/ensuredb.go +++ b/cmd/mxcli/docker/ensuredb.go @@ -6,11 +6,13 @@ import ( "errors" "fmt" "io" + "io/fs" "net" "os" "os/exec" "path/filepath" "regexp" + "runtime" "strconv" "strings" "time" @@ -20,7 +22,8 @@ import ( // fresh session comes up testable without a manual createdb (slice 2 of the // warm-loop proposal). It is best-effort: it starts the local Postgres service // if the port is down — via a service manager, or a user-owned initdb/pg_ctl -// cluster when no service becomes ready (e.g. Arch, #823) — then ensures the app +// cluster when no service becomes ready (e.g. Arch, #823), but never in place of +// a stopped system cluster for the same port (ako/mxcli#866) — then ensures the app // role and database exist via a superuser (a trust connection over the // user-owned cluster's private socket, or `sudo -u postgres psql` for a system // cluster). For a non-local DB host it does nothing but verify reachability — @@ -55,6 +58,197 @@ var currentEUID = os.Geteuid // (#984). A variable so a stubbed PATH also gets a hermetic tool lookup. var postgresServerBinGlobs = []string{"/usr/lib/postgresql/*/bin"} +// systemClusterRoot prefixes every well-known location probed for a system +// cluster (see findSystemClusters), so a test can point the probe at a +// temporary tree. Empty means the real filesystem root. +var systemClusterRoot = "" + +// clusterProbeStat is the stat behind the system-cluster probe; a variable so a +// test running as root can still exercise a permission-denied data directory. +var clusterProbeStat = os.Stat + +// newClusterOptInEnv, set to 1, lets the user-owned fallback initialize a new +// cluster although a stopped system cluster is configured for the same port — +// the deliberate way past the ako/mxcli#866 refusal when that cluster is not +// this app's and cannot be started (no sudo). +const newClusterOptInEnv = "MXCLI_ENSURE_DB_NEW_CLUSTER" + +// systemClusterLayouts are the data directories a package or service manager +// creates, beyond Debian's postgresql-common (read from /etc/postgresql). Paths +// are slash-separated and taken relative to systemClusterRoot. +var systemClusterLayouts = []struct { + glob string + kind clusterKind +}{ + {"/var/lib/postgresql/*/*", clusterDebian}, // a data directory whose /etc config is gone + {"/var/lib/pgsql/data", clusterSystemd}, // RHEL, Fedora, CentOS + {"/var/lib/pgsql/*/data", clusterSystemd}, // PGDG packages on RHEL + {"/var/lib/postgres/data", clusterSystemd}, // Arch + {"/opt/homebrew/var/postgres*", clusterHomebrew}, + {"/usr/local/var/postgres*", clusterHomebrew}, + {"C:/Program Files/PostgreSQL/*/data", clusterWindows}, +} + +type clusterKind int + +const ( + clusterDebian clusterKind = iota + clusterSystemd + clusterHomebrew + clusterWindows +) + +// systemCluster is a PostgreSQL cluster mxcli did not create: an initialized +// data directory in a location a package or service manager uses. +type systemCluster struct { + dataDir string + // port is the configured port, or PostgreSQL's default 5432 when the + // configuration cannot be read (a 0700 data directory, as a non-root user). + port string + kind clusterKind + // version and name identify a postgresql-common cluster, which + // pg_ctlcluster addresses by exactly those two. + version, name string +} + +// startCommand is how a user starts this cluster by hand. +func (c systemCluster) startCommand() string { + switch c.kind { + case clusterDebian: + if c.version != "" && c.name != "" { + return fmt.Sprintf("sudo pg_ctlcluster %s %s start (or: sudo service postgresql start)", c.version, c.name) + } + return fmt.Sprintf("sudo service postgresql start (or: sudo -u postgres pg_ctl -D %s start)", c.dataDir) + case clusterHomebrew: + return fmt.Sprintf("brew services start %s (or: pg_ctl -D %s start)", filepath.Base(c.dataDir), c.dataDir) + case clusterWindows: + return fmt.Sprintf("net start postgresql-x64-%s (as Administrator; or start it in services.msc)", + filepath.Base(filepath.Dir(c.dataDir))) + } + return fmt.Sprintf("sudo systemctl start postgresql (or: sudo -u postgres pg_ctl -D %s start)", c.dataDir) +} + +// clusterDataDirPresent reports whether dir is a data directory initdb has +// populated. A PG_VERSION the probe may not read counts: as a non-root user, +// /var/lib/postgresql/16/main is 0700 and owned by postgres, so stat is refused +// rather than answered — and that user is exactly who #866 happened to. +func clusterDataDirPresent(dir string) bool { + info, err := clusterProbeStat(dir) + if err != nil || !info.IsDir() { + return false + } + _, err = clusterProbeStat(filepath.Join(dir, "PG_VERSION")) + return err == nil || errors.Is(err, fs.ErrPermission) +} + +var ( + pgConfPortRe = regexp.MustCompile(`^\s*port\s*=?\s*'?(\d+)'?`) + pgConfDataDirRe = regexp.MustCompile(`^\s*data_directory\s*=?\s*'([^']*)'`) +) + +// readClusterConf returns the port and data_directory a postgresql.conf sets; +// either is "" when the file is unreadable or leaves it at the default. +func readClusterConf(path string) (port, dataDir string) { + body, err := os.ReadFile(path) + if err != nil { + return "", "" + } + for _, line := range strings.Split(string(body), "\n") { + if m := pgConfPortRe.FindStringSubmatch(line); m != nil { + port = m[1] + } + if m := pgConfDataDirRe.FindStringSubmatch(line); m != nil { + dataDir = m[1] + } + } + return port, dataDir +} + +// findSystemClusters lists the initialized PostgreSQL clusters a package or +// service manager made on this machine, whether or not they are running. +func findSystemClusters() []systemCluster { + at := func(p string) string { return filepath.Join(systemClusterRoot, filepath.FromSlash(p)) } + var out []systemCluster + seen := map[string]bool{} + add := func(c systemCluster) { + c.dataDir = filepath.Clean(c.dataDir) + if seen[c.dataDir] || !clusterDataDirPresent(c.dataDir) { + return + } + seen[c.dataDir] = true + if c.port == "" { + c.port = "5432" + } + out = append(out, c) + } + + // Debian/Ubuntu postgresql-common: the configuration, port included, lives + // in /etc/postgresql/// and stays readable to every user. + confs, _ := filepath.Glob(at("/etc/postgresql/*/*/postgresql.conf")) + for _, conf := range confs { + clusterDir := filepath.Dir(conf) + name, version := filepath.Base(clusterDir), filepath.Base(filepath.Dir(clusterDir)) + port, dataDir := readClusterConf(conf) + if dataDir == "" { + dataDir = "/var/lib/postgresql/" + version + "/" + name + } + add(systemCluster{dataDir: at(dataDir), port: port, kind: clusterDebian, version: version, name: name}) + } + + for _, layout := range systemClusterLayouts { + // A drive-letter glob is a RELATIVE path off Windows, resolved against + // the working directory; a /var glob means nothing on Windows. + if (layout.kind == clusterWindows) != (runtime.GOOS == "windows") { + continue + } + dirs, _ := filepath.Glob(at(layout.glob)) + for _, dir := range dirs { + port, _ := readClusterConf(filepath.Join(dir, "postgresql.conf")) + add(systemCluster{dataDir: dir, port: port, kind: layout.kind}) + } + } + return out +} + +// conflictingSystemCluster returns the system cluster configured for port, if +// one exists. The caller has already established that nothing listens there, +// so a match is a cluster that is down. +func conflictingSystemCluster(port string) (systemCluster, bool) { + for _, c := range findSystemClusters() { + if c.port == port { + return c, true + } + } + return systemCluster{}, false +} + +// systemClusterDownError is the refusal to initialize a new cluster over a +// stopped one. Its first line carries the data directory, because that is the +// line a hook banner shows; nothing after it may read as a second error. +func systemClusterDownError(c systemCluster) error { + return fmt.Errorf("refusing to create a new, empty PostgreSQL cluster: the cluster in %s "+ + "(port %s) exists but is not running\n"+ + " The app's data, if it lives there, would be absent from a new cluster, and an empty\n"+ + " database looks exactly like a new one, so nothing would say so (ako/mxcli#866).\n"+ + " Start it, then rerun:\n"+ + " %s\n"+ + " If that cluster does not hold this app's data, create a new, empty one deliberately:\n"+ + " %s=1 mxcli run --local --ensure-db ...\n"+ + " or boot on the built-in database instead: --db-type hsqldb", + c.dataDir, c.port, c.startCommand(), newClusterOptInEnv) +} + +// userClusterInitialized reports whether the user-owned cluster under +// ~/.mxcli/postgres already exists — a restart of it is not a new database. +func userClusterInitialized() bool { + _, dataDir, _, err := userClusterDirs() + if err != nil { + return false + } + _, err = os.Stat(filepath.Join(dataDir, "PG_VERSION")) + return err == nil +} + // onPath reports whether a command is resolvable through PATH. func onPath(name string) bool { _, err := exec.LookPath(name) @@ -304,7 +498,8 @@ func canConnectDB(db DBConfig) bool { // startLocalPostgres starts the local PostgreSQL service, trying the common // service managers in turn. When none is present — e.g. on Arch — or they do // not produce a ready server, it falls back to a user-owned cluster started with -// the portable initdb/pg_ctl tools (#823). +// the portable initdb/pg_ctl tools (#823) — unless a system cluster configured +// for the port exists and is down, which it refuses to paper over (#866). // It returns whatever the service managers reported, so a caller whose own // readiness wait then fails can say what this package already saw. func startLocalPostgres(host, port string, w io.Writer) ([]string, error) { @@ -343,6 +538,27 @@ func startLocalPostgres(host, port string, w io.Writer) ([]string, error) { return serviceDiag, nil } + // A system cluster configured for this port exists and is down — the + // service managers above did not bring it up. A new cluster here would + // answer every probe the app makes while holding none of its data, so the + // run would boot healthy against an empty database with nothing saying so + // (ako/mxcli#866). Refuse, unless the user-owned cluster already exists (a + // restart of it is not a new database) or the user opted in. + if sys, found := conflictingSystemCluster(port); found { + switch { + case userClusterInitialized(): + _, dataDir, _, _ := userClusterDirs() + fmt.Fprintf(w, " Note: using the user-owned cluster in %s; the system cluster in %s (port %s) "+ + "is not running.\n If the app's data is in that one, stop this cluster (pg_ctl -D %s stop) and start it:\n %s\n", + dataDir, sys.dataDir, sys.port, dataDir, sys.startCommand()) + case os.Getenv(newClusterOptInEnv) == "1": + fmt.Fprintf(w, " %s=1: not starting the stopped cluster in %s; creating a new one instead.\n", + newClusterOptInEnv, sys.dataDir) + default: + return serviceDiag, withServiceDiag(systemClusterDownError(sys), serviceDiag) + } + } + // No service manager made PostgreSQL ready: start a user-owned cluster with // the portable tools. This needs neither a `postgres` OS account nor sudo. if err := startUserCluster(host, port, w); err != nil { @@ -545,6 +761,12 @@ func startUserCluster(host, port string, w io.Writer) error { return fmt.Errorf("initializing PostgreSQL cluster in %s: %w\n%s", dataDir, err, strings.TrimSpace(string(out))) } + // An empty database is indistinguishable from a new one, so the one + // moment that knows the difference has to say it (ako/mxcli#866). + fmt.Fprintf(w, " WARNING: created a NEW, EMPTY PostgreSQL cluster in %s.\n"+ + " The app's database in it starts empty: no data from any other PostgreSQL\n"+ + " installation is in it. If the app had data elsewhere, stop here and start that server.\n", + dataDir) } else if err != nil { return fmt.Errorf("checking PostgreSQL cluster in %s: %w", dataDir, err) } diff --git a/cmd/mxcli/docker/ensuredb_systemcluster_probe_test.go b/cmd/mxcli/docker/ensuredb_systemcluster_probe_test.go new file mode 100644 index 0000000000..b6c40fceed --- /dev/null +++ b/cmd/mxcli/docker/ensuredb_systemcluster_probe_test.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +// The probe behind the ako/mxcli#866 refusal: which data directories count as +// an existing cluster, on which port, and how to start each. + +// The layouts other distributions use. Port matching is the guard's scope: a +// configuration it can read decides; one it cannot read (a 0700 data +// directory, as a non-root user) is PostgreSQL's default, 5432. +func TestFindSystemClusters_Layouts(t *testing.T) { + cases := []struct { + name, dir string + conf string // postgresql.conf in the data directory, if any + wantPort string + hint string + }{ + {name: "RHEL/Fedora", dir: "var/lib/pgsql/data", wantPort: "5432", hint: "systemctl start postgresql"}, + {name: "PGDG", dir: "var/lib/pgsql/16/data", conf: "port = 5433\n", wantPort: "5433", hint: "systemctl start postgresql"}, + {name: "Arch", dir: "var/lib/postgres/data", conf: "#port = 5432\n", wantPort: "5432", hint: "systemctl start postgresql"}, + {name: "Homebrew", dir: "opt/homebrew/var/postgresql@16", conf: "port = 5440 # mine\n", wantPort: "5440", hint: "brew services start postgresql@16"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + withSystemClusterRoot(t, root) + dataDir := filepath.Join(root, filepath.FromSlash(tc.dir)) + if err := os.MkdirAll(dataDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dataDir, "PG_VERSION"), []byte("16\n"), 0o600); err != nil { + t.Fatal(err) + } + if tc.conf != "" { + if err := os.WriteFile(filepath.Join(dataDir, "postgresql.conf"), []byte(tc.conf), 0o600); err != nil { + t.Fatal(err) + } + } + c, ok := conflictingSystemCluster(tc.wantPort) + if !ok { + t.Fatalf("no cluster found at %s for port %s (found %+v)", dataDir, tc.wantPort, findSystemClusters()) + } + if c.dataDir != dataDir { + t.Errorf("data dir = %q, want %q", c.dataDir, dataDir) + } + if !strings.Contains(c.startCommand(), tc.hint) { + t.Errorf("start hint = %q, want it to mention %q", c.startCommand(), tc.hint) + } + if _, ok := conflictingSystemCluster("6543"); ok { + t.Errorf("a cluster on %s must not match another port", tc.wantPort) + } + }) + } +} + +// A directory without PG_VERSION is not a cluster (an empty package directory, +// a cluster that was dropped) — refusing there would block the fallback for +// nothing. +func TestFindSystemClusters_IgnoresUninitializedDirectories(t *testing.T) { + root := t.TempDir() + withSystemClusterRoot(t, root) + for _, d := range []string{"var/lib/postgresql/16/main", "var/lib/pgsql/data", "etc/postgresql/16/main"} { + if err := os.MkdirAll(filepath.Join(root, filepath.FromSlash(d)), 0o755); err != nil { + t.Fatal(err) + } + } + if got := findSystemClusters(); len(got) != 0 { + t.Fatalf("no initialized data directory exists, found %+v", got) + } +} + +// A non-root user cannot look inside /var/lib/postgresql/16/main (0700, +// owned by postgres), so stat(PG_VERSION) is refused rather than missing. That +// refusal is evidence of a cluster, not of its absence — the devcontainer user +// is exactly who #866 happened to. Injected, because this suite may run as root. +func TestFindSystemClusters_UnreadableDataDirectoryCounts(t *testing.T) { + root := t.TempDir() + withSystemClusterRoot(t, root) + dataDir := debianCluster(t, root, "16", "main", "5432") + old := clusterProbeStat + clusterProbeStat = func(name string) (fs.FileInfo, error) { + if strings.HasPrefix(name, dataDir+string(filepath.Separator)) { + return nil, &fs.PathError{Op: "stat", Path: name, Err: fs.ErrPermission} + } + return os.Stat(name) + } + t.Cleanup(func() { clusterProbeStat = old }) + + c, ok := conflictingSystemCluster("5432") + if !ok || c.dataDir != dataDir { + t.Fatalf("a permission-denied data directory must count as a cluster; got %+v, %v", c, ok) + } + // And a missing one still does not. + clusterProbeStat = func(name string) (fs.FileInfo, error) { + if strings.HasSuffix(name, "PG_VERSION") { + return nil, &fs.PathError{Op: "stat", Path: name, Err: fs.ErrNotExist} + } + return os.Stat(name) + } + if _, ok := conflictingSystemCluster("5432"); ok { + t.Fatal("a data directory without PG_VERSION is not a cluster") + } +} diff --git a/cmd/mxcli/docker/ensuredb_systemcluster_test.go b/cmd/mxcli/docker/ensuredb_systemcluster_test.go new file mode 100644 index 0000000000..94e9b993e7 --- /dev/null +++ b/cmd/mxcli/docker/ensuredb_systemcluster_test.go @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// ako/mxcli#866, verbatim: "In a container restarted with the system cluster +// down, the `mxcli init` SessionStart hook runs `--ensure-db`, provisions a +// fresh **empty** cluster, and both apps boot healthy against it. Every +// captured row (7,966 laps here) is absent, and no error says so. An empty +// database is indistinguishable from a new one." +// +// The #823 user-owned fallback is right when there is no PostgreSQL to start, +// and wrong when there is one that did not start: a new cluster on the same +// port answers every probe the app makes, so the only place the difference can +// be stated is here, before initdb runs. + +// withSystemClusterRoot points the system-cluster probe at root. +func withSystemClusterRoot(t *testing.T, root string) { + t.Helper() + old := systemClusterRoot + systemClusterRoot = root + t.Cleanup(func() { systemClusterRoot = old }) +} + +// debianCluster lays out what postgresql-common leaves for cluster ver/name +// under root: its configuration in /etc/postgresql/// (with the port +// and the data directory, as Debian writes them) and an initialized data +// directory in /var/lib/postgresql//. It returns the data directory. +func debianCluster(t *testing.T, root, ver, name, port string) string { + t.Helper() + dataDir := filepath.Join(root, "var", "lib", "postgresql", ver, name) + confDir := filepath.Join(root, "etc", "postgresql", ver, name) + for _, d := range []string{dataDir, confDir} { + if err := os.MkdirAll(d, 0o700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(dataDir, "PG_VERSION"), []byte(ver+"\n"), 0o600); err != nil { + t.Fatal(err) + } + conf := "# Debian postgresql-common\n" + + "data_directory = '/var/lib/postgresql/" + ver + "/" + name + "'\t\t# use data in another directory\n" + + "port = " + port + "\t\t\t\t# (change requires restart)\n" + if err := os.WriteFile(filepath.Join(confDir, "postgresql.conf"), []byte(conf), 0o644); err != nil { + t.Fatal(err) + } + return dataDir +} + +// stubSystemClusterThatWillNotStart is the reported machine: a service manager +// that runs and starts nothing, a server that never becomes ready, and the +// portable tools a user-owned fallback would use. +func stubSystemClusterThatWillNotStart(t *testing.T, dir, logPath string) { + t.Helper() + writeStub(t, dir, "service", `echo "service $*" >> "`+logPath+`"; echo "Starting PostgreSQL 16 database server: main failed!"; exit 1`) + writeStub(t, dir, "pg_isready", "exit 1") + writeStub(t, dir, "initdb", initdbStub(logPath)) + writeStub(t, dir, "pg_ctl", pgctlStub(logPath, 3, 0)) +} + +// The reported case: the system cluster exists, its port is the app's, and it +// is down. Falling back to `initdb` here is what produced the healthy-looking +// empty database. The run must stop and name the data directory instead. +func TestStartLocalPostgres_RefusesNewClusterWhileSystemClusterIsDown(t *testing.T) { + dir, logPath := newStubPATH(t) + withTimeouts(t, 20*time.Millisecond) + port := unusedTCPPort(t) + root := t.TempDir() + withSystemClusterRoot(t, root) + dataDir := debianCluster(t, root, "16", "main", port) + stubSystemClusterThatWillNotStart(t, dir, logPath) + + var out bytes.Buffer + _, err := startLocalPostgres("127.0.0.1", port, &out) + calls := readCalls(t, logPath) + if strings.Contains(calls, "initdb") || strings.Contains(calls, "pg_ctl_start") { + t.Fatalf("a NEW, EMPTY user-owned cluster was provisioned on port %s while the system "+ + "cluster at %s is down — the app now boots healthy against an empty database and "+ + "no error says so (ako/mxcli#866)\ncalls:\n%s\nerr: %v\noutput:\n%s", + port, dataDir, calls, err, out.String()) + } + if !strings.Contains(calls, "service postgresql start") { + t.Errorf("the system cluster must still be tried first, calls:\n%s", calls) + } + if err == nil { + t.Fatal("expected a refusal naming the stopped system cluster") + } + msg := err.Error() + first := strings.SplitN(msg, "\n", 2)[0] + for _, want := range []string{"refusing to create a new, empty PostgreSQL cluster", dataDir, "not running"} { + if !strings.Contains(first, want) { + t.Errorf("the refusal's FIRST line (the one a hook banner shows) lacks %q:\n%s", want, msg) + } + } + for _, want := range []string{ + "sudo pg_ctlcluster 16 main start", // how to start exactly that cluster + "sudo service postgresql start", + "MXCLI_ENSURE_DB_NEW_CLUSTER=1", // the deliberate way past it + "--db-type hsqldb", + "main failed!", // what the service manager said + } { + if !strings.Contains(msg, want) { + t.Errorf("refusal lacks %q:\n%s", want, msg) + } + } + // Only the first line may read as an error: the bootstrap hook names the + // failure by the first "Error:" line, and guidance must not compete. + rest := strings.SplitN(msg, "(a service manager ran first", 2)[0] + if strings.Contains(strings.ToLower(strings.SplitN(rest, "\n", 2)[1]), "error") { + t.Errorf("guidance lines must not contain 'error':\n%s", rest) + } +} + +// CONTROL: a system cluster configured for a different port is not this app's +// database — the fallback is the #823 behaviour and still runs, loudly. +func TestStartLocalPostgres_SystemClusterOnAnotherPortDoesNotBlock(t *testing.T) { + dir, logPath := newStubPATH(t) + withTimeouts(t, 20*time.Millisecond) + port := unusedTCPPort(t) + root := t.TempDir() + withSystemClusterRoot(t, root) + other := "1" + if port == other { + other = "2" + } + debianCluster(t, root, "16", "main", other) + stubSystemClusterThatWillNotStart(t, dir, logPath) + + var out bytes.Buffer + if _, err := startLocalPostgres("127.0.0.1", port, &out); err != nil { + t.Fatalf("a cluster on port %s must not block port %s: %v", other, port, err) + } + if calls := readCalls(t, logPath); !strings.Contains(calls, "initdb") { + t.Fatalf("expected the user-owned fallback to run:\n%s", calls) + } + if !strings.Contains(out.String(), "NEW, EMPTY") { + t.Errorf("a freshly initialized cluster must be announced loudly:\n%s", out.String()) + } +} + +// CONTROL: a user-owned cluster that already exists holds whatever the earlier +// runs put there — restarting it is not "provisioning an empty one", and must +// keep working. The stopped system cluster is still worth a line, for the +// users the old behaviour already moved onto an empty cluster. +func TestStartLocalPostgres_ExistingUserClusterIsRestartedNotRefused(t *testing.T) { + dir, logPath := newStubPATH(t) + withTimeouts(t, 20*time.Millisecond) + port := unusedTCPPort(t) + root := t.TempDir() + withSystemClusterRoot(t, root) + sysData := debianCluster(t, root, "16", "main", port) + stubSystemClusterThatWillNotStart(t, dir, logPath) + initClusterDir(t) + + var out bytes.Buffer + if _, err := startLocalPostgres("127.0.0.1", port, &out); err != nil { + t.Fatalf("restarting an existing user-owned cluster must not be refused: %v", err) + } + calls := readCalls(t, logPath) + if strings.Contains(calls, "initdb") || !strings.Contains(calls, "pg_ctl_start") { + t.Fatalf("expected a plain restart of the user-owned cluster:\n%s", calls) + } + if strings.Contains(out.String(), "NEW, EMPTY") { + t.Errorf("a restarted cluster is not new and must not be announced as such:\n%s", out.String()) + } + if !strings.Contains(out.String(), sysData) { + t.Errorf("the stopped system cluster should be named next to the cluster in use:\n%s", out.String()) + } +} + +// The deliberate way past the refusal, for a system cluster that is not this +// app's and cannot be started (no sudo): a new cluster, announced as such. +func TestStartLocalPostgres_NewClusterOptIn(t *testing.T) { + dir, logPath := newStubPATH(t) + withTimeouts(t, 20*time.Millisecond) + port := unusedTCPPort(t) + root := t.TempDir() + withSystemClusterRoot(t, root) + debianCluster(t, root, "16", "main", port) + stubSystemClusterThatWillNotStart(t, dir, logPath) + t.Setenv("MXCLI_ENSURE_DB_NEW_CLUSTER", "1") + + var out bytes.Buffer + if _, err := startLocalPostgres("127.0.0.1", port, &out); err != nil { + t.Fatal(err) + } + if calls := readCalls(t, logPath); !strings.Contains(calls, "initdb") { + t.Fatalf("the opt-in must allow the new cluster:\n%s", calls) + } + if !strings.Contains(out.String(), "NEW, EMPTY") { + t.Errorf("the new cluster must still be announced:\n%s", out.String()) + } +} + +// The warning belongs to the moment initdb runs, and only to it. +func TestStartUserCluster_AnnouncesOnlyANewCluster(t *testing.T) { + dir, logPath := newStubPATH(t) + writeStub(t, dir, "initdb", initdbStub(logPath)) + writeStub(t, dir, "pg_ctl", pgctlStub(logPath, 3, 0)) + + var first bytes.Buffer + if err := startUserCluster("127.0.0.1", "5544", &first); err != nil { + t.Fatal(err) + } + home, _ := os.UserHomeDir() + dataDir := filepath.Join(home, ".mxcli", "postgres", "data") + if !strings.Contains(first.String(), "WARNING") || !strings.Contains(first.String(), "NEW, EMPTY") || + !strings.Contains(first.String(), dataDir) { + t.Errorf("initializing a cluster must say loudly that the database is new and empty, and where:\n%s", first.String()) + } + + var second bytes.Buffer + if err := startUserCluster("127.0.0.1", "5544", &second); err != nil { + t.Fatal(err) + } + if strings.Contains(second.String(), "NEW, EMPTY") { + t.Errorf("restarting the same cluster must not claim it is new:\n%s", second.String()) + } +} diff --git a/cmd/mxcli/docker/ensuredb_test.go b/cmd/mxcli/docker/ensuredb_test.go index f8dcba2567..91f2e5d0bd 100644 --- a/cmd/mxcli/docker/ensuredb_test.go +++ b/cmd/mxcli/docker/ensuredb_test.go @@ -166,6 +166,10 @@ func newStubPATH(t *testing.T) (dir, logPath string) { // too: this host really does have /usr/lib/postgresql/*/bin, so a test that // asserts "the tools are missing" would otherwise run a real initdb. withServerBinGlobs(t) + // The same holds for the system-cluster probe: this host may well have a + // stopped /var/lib/postgresql/16/main, which would refuse every fallback + // test below (ako/mxcli#866). Each test opts in to a system cluster. + withSystemClusterRoot(t, t.TempDir()) // Pin the privilege level as well, so the same attempts are made whether the // suite runs as root in a container or as a user on a laptop. withEUID(t, 0) diff --git a/cmd/mxcli/docker/javadeps.go b/cmd/mxcli/docker/javadeps.go index 4038098d2a..95c5b1867c 100644 --- a/cmd/mxcli/docker/javadeps.go +++ b/cmd/mxcli/docker/javadeps.go @@ -10,6 +10,7 @@ import ( "os/exec" "path/filepath" "strings" + "time" ) // javadeps.go resolves a project's managed Java (JAR) dependencies. @@ -52,9 +53,7 @@ func SyncJavaDependencies(projectPath, mxPath, version string, w io.Writer) erro return fmt.Errorf("mx not found for Mendix %s (needed to resolve managed Java dependencies); run 'mxcli setup mxbuild --version %s'", version, version) } - cmd := exec.Command(mxPath, "sync-java-dependencies", projectPath) - cmd.Dir = filepath.Dir(projectPath) - PrepareMxCommand(cmd) // FreeType LD_PRELOAD workaround + cmd := syncJavaDepsCommand(mxPath, projectPath) out := &syncBuffer{} cmd.Stdout = out @@ -63,7 +62,14 @@ func SyncJavaDependencies(projectPath, mxPath, version string, w io.Writer) erro // (#1396). mx's own exit status is the result — ErrWaitDelay only says the // daemon outlived it. cmd.WaitDelay = orphanPipeWaitDelay - if err := cmd.Run(); err != nil && !errors.Is(err, exec.ErrWaitDelay) { + // Bounded, and not silent: the step used to wait for as long as Gradle did, + // which behind a stale daemon was ten minutes and more with no output at + // all (ako/mxcli#1340). The process tree is killed at the bound. + if err := runWithWatchdog(cmd, syncJavaDepsLimits(w), ""); err != nil && !errors.Is(err, exec.ErrWaitDelay) { + var werr *WatchdogError + if errors.As(err, &werr) { + return err // already names the step, the cause and the last output + } return fmt.Errorf("mx sync-java-dependencies failed: %w\n%s", err, lastLines(out.String(), 20)) } if w != nil { @@ -74,6 +80,51 @@ func SyncJavaDependencies(projectPath, mxPath, version string, w io.Writer) erro return nil } +// DefaultSyncJavaDepsTimeout bounds one `mx sync-java-dependencies`. A healthy +// sync takes seconds (6-12s measured in #1340, downloads included); the bound +// is generous because killing a slow download costs more than a late kill. +const DefaultSyncJavaDepsTimeout = 10 * time.Minute + +// syncJavaDepsTimeoutEnv raises (or, with 0, disables) the sync's overall bound. +const syncJavaDepsTimeoutEnv = "MXCLI_SYNC_JAVA_DEPS_TIMEOUT" + +// syncJavaDepsProgressEvery is how often a sync still running says so. A +// variable so a test need not wait it out. +var syncJavaDepsProgressEvery = 30 * time.Second + +// staleGradleDaemonHint is the likely cause of a sync that never finishes. +const staleGradleDaemonHint = "Likely cause: a Gradle daemon left running by an earlier mx or mxbuild run, which this\n" + + "sync attached to (ako/mxcli#1340). End the leftover Java processes whose command line\n" + + "contains GradleDaemon (Task Manager on Windows; `pkill -f GradleDaemon` elsewhere),\n" + + "then retry: mxcli sync-java-deps -p " + +// syncJavaDepsLimits are the sync's bounds: an overall timeout (no stall +// bound — Gradle can be quiet for a long download) and a progress line to w. +func syncJavaDepsLimits(w io.Writer) watchdogLimits { + lim := watchdogLimits{ + Timeout: durationEnv(syncJavaDepsTimeoutEnv, DefaultSyncJavaDepsTimeout), + Step: "mx sync-java-dependencies", + TimeoutEnv: syncJavaDepsTimeoutEnv, + Hint: staleGradleDaemonHint, + } + if w != nil { + lim.Progress = syncJavaDepsProgressEvery + lim.OnProgress = func(elapsed time.Duration) { + fmt.Fprintf(w, " still resolving Java dependencies... (%s)\n", elapsed.Round(time.Second)) + } + } + return lim +} + +// syncJavaDepsCommand builds the `mx sync-java-dependencies` invocation: a +// one-shot, so Gradle runs without a daemon (see prepareOneShotMxCommand). +func syncJavaDepsCommand(mxPath, projectPath string) *exec.Cmd { + cmd := exec.Command(mxPath, "sync-java-dependencies", projectPath) + cmd.Dir = filepath.Dir(projectPath) + prepareOneShotMxCommand(cmd) // FreeType LD_PRELOAD workaround + no Gradle daemon + return cmd +} + // UnvendoredJarDependencies returns the coordinates ("group:artifact:version") // the sync would vendor that have no matching jar in the project's vendorlib/, // so a caller can sync only when something is actually missing. diff --git a/cmd/mxcli/docker/javadeps_gradle_test.go b/cmd/mxcli/docker/javadeps_gradle_test.go new file mode 100644 index 0000000000..ce36a2d977 --- /dev/null +++ b/cmd/mxcli/docker/javadeps_gradle_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "os" + "os/exec" + "strings" + "testing" +) + +// ako/mxcli#1340 (mirror of mendixlabs/mxcli#1483), verbatim: "`mxcli run +// --local` sometimes stops for more than 10 minutes, without any output, at: +// `Resolving 4 managed Java dependency/dependencies (...)...` ... The same +// happened in the first build of `mxcli new` (step 6)." and: "Each `mx +// sync-java-dependencies` ... leaves a Gradle 8.5 daemon ... running for hours. +// When the hang occurred, an older daemon from a previous run was alive and +// idle ... With `GRADLE_OPTS=-Dorg.gradle.daemon=false`, the sync finishes in +// about 12 seconds and leaves no daemon behind; I have not seen the hang since." +// +// A one-shot mx/mxbuild run gains nothing from a daemon — it exits when its +// one job is done — and a daemon is what an earlier run left behind to hang +// the next one. Every one-shot that runs Gradle gets the reporter's setting. + +// cmdEnv is the environment cmd will run with, last value winning per key. +func cmdEnv(cmd *exec.Cmd) map[string]string { + env := cmd.Env + if env == nil { + env = os.Environ() + } + m := map[string]string{} + for _, kv := range env { + k, v, _ := strings.Cut(kv, "=") + m[k] = v + } + return m +} + +func TestOneShotMxCommands_RunGradleWithoutADaemon(t *testing.T) { + // The user's own GRADLE_OPTS (a proxy, here — needed to reach Maven at all + // behind one) must survive, and so must PrepareMxCommand's LD_PRELOAD. + t.Setenv("GRADLE_OPTS", "-Xmx2g -Dhttps.proxyHost=proxy.example") + t.Setenv("LD_PRELOAD", "/opt/custom/libpreload.so") + + for name, cmd := range map[string]*exec.Cmd{ + "mx sync-java-dependencies (run --local, sync-java-deps)": syncJavaDepsCommand("/opt/mx/mx", "/work/App.mpr"), + "mxbuild settle build (mxcli new step 6)": settleCommand("/opt/mx/mxbuild", "/jdk", "/work/App.mpr"), + } { + env := cmdEnv(cmd) + want := "-Xmx2g -Dhttps.proxyHost=proxy.example -Dorg.gradle.daemon=false" + if got := env["GRADLE_OPTS"]; got != want { + t.Errorf("%s: GRADLE_OPTS = %q, want %q — without it the run attaches to (or leaves) a "+ + "Gradle daemon, and a stale one hangs the step silently (ako/mxcli#1340)", name, got, want) + } + if !strings.Contains(env["LD_PRELOAD"], "/opt/custom/libpreload.so") { + t.Errorf("%s: LD_PRELOAD = %q, the caller's preload was dropped", name, env["LD_PRELOAD"]) + } + } +} + +func TestWithGradleNoDaemon(t *testing.T) { + cases := []struct { + name string + env []string + want string // GRADLE_OPTS the child sees + }{ + {"absent", []string{"PATH=/bin"}, "-Dorg.gradle.daemon=false"}, + {"empty", []string{"GRADLE_OPTS="}, "-Dorg.gradle.daemon=false"}, + {"appended", []string{"GRADLE_OPTS=-Xmx1g"}, "-Xmx1g -Dorg.gradle.daemon=false"}, + // The user asked for a daemon explicitly: their call, not ours. + {"explicit choice kept", []string{"GRADLE_OPTS=-Dorg.gradle.daemon=true"}, "-Dorg.gradle.daemon=true"}, + // exec keeps the last of duplicate keys, so that is the one to extend. + {"duplicate key", []string{"GRADLE_OPTS=-Xold", "GRADLE_OPTS=-Xnew"}, "-Xnew -Dorg.gradle.daemon=false"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + in := append([]string(nil), tc.env...) + got := cmdEnv(&exec.Cmd{Env: withGradleNoDaemon(in)})["GRADLE_OPTS"] + if got != tc.want { + t.Errorf("GRADLE_OPTS = %q, want %q", got, tc.want) + } + if strings.Join(in, "\x00") != strings.Join(tc.env, "\x00") { + t.Errorf("the input environment was mutated: %v", in) + } + }) + } +} diff --git a/cmd/mxcli/docker/javadeps_hang_test.go b/cmd/mxcli/docker/javadeps_hang_test.go new file mode 100644 index 0000000000..31ffb63f2f --- /dev/null +++ b/cmd/mxcli/docker/javadeps_hang_test.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows + +package docker + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// ako/mxcli#1340: the step "stops for more than 10 minutes, without any +// output". The reporter's suggestion: "show progress or a timeout instead of +// waiting silently". The fake mx prints what Gradle prints and then never +// finishes — the shape of a client attached to a wedged daemon. +func TestSyncJavaDependencies_HungSyncShowsProgressAndTimesOut(t *testing.T) { + dir := t.TempDir() + fake := filepath.Join(dir, "mx") + script := "#!/bin/sh\necho 'Starting a Gradle Daemon, 1 busy and 1 stopped Daemons could not be reused'\nsleep 30\n" + if err := os.WriteFile(fake, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv(syncJavaDepsTimeoutEnv, "700ms") + old := syncJavaDepsProgressEvery + syncJavaDepsProgressEvery = 150 * time.Millisecond + t.Cleanup(func() { syncJavaDepsProgressEvery = old }) + + var out syncBuffer + done := make(chan error, 1) + start := time.Now() + go func() { done <- SyncJavaDependencies(filepath.Join(dir, "App.mpr"), fake, "11.12.4", &out) }() + var err error + select { + case err = <-done: + case <-time.After(15 * time.Second): + t.Fatalf("SyncJavaDependencies still running after 15s, having printed only:\n%q\n"+ + "— the silent hang at 'Resolving managed Java dependencies' (ako/mxcli#1340)", out.String()) + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Errorf("a 700ms bound took %s: the process tree was not killed", elapsed) + } + if !strings.Contains(out.String(), "still resolving") { + t.Errorf("a long sync must say it is still working; output:\n%s", out.String()) + } + if err == nil { + t.Fatal("a sync killed by its bound must be an error") + } + var werr *WatchdogError + if !errors.As(err, &werr) { + t.Errorf("err = %T, want a *WatchdogError", err) + } + msg := err.Error() + t.Logf("output:\n%s\nerror:\n%s", out.String(), msg) + for _, want := range []string{ + "mx sync-java-dependencies", // the step + "timed out", + "Gradle daemon", // the likely cause + syncJavaDepsTimeoutEnv, // how to allow longer + "could not be reused", // the last thing mx said + } { + if !strings.Contains(msg, want) { + t.Errorf("error lacks %q:\n%s", want, msg) + } + } +} + +// CONTROL: a sync that finishes inside the progress interval is as quiet as +// before, and its success is its own. +func TestSyncJavaDependencies_FastSyncIsQuiet(t *testing.T) { + dir := t.TempDir() + fake := filepath.Join(dir, "mx") + if err := os.WriteFile(fake, []byte("#!/bin/sh\necho synced 1 jar\n"), 0o755); err != nil { + t.Fatal(err) + } + old := syncJavaDepsProgressEvery + syncJavaDepsProgressEvery = 5 * time.Second + t.Cleanup(func() { syncJavaDepsProgressEvery = old }) + + var out syncBuffer + if err := SyncJavaDependencies(filepath.Join(dir, "App.mpr"), fake, "11.12.4", &out); err != nil { + t.Fatal(err) + } + if strings.Contains(out.String(), "still resolving") { + t.Errorf("a fast sync printed progress:\n%s", out.String()) + } + if !strings.Contains(out.String(), "synced 1 jar") { + t.Errorf("mx output not forwarded:\n%s", out.String()) + } +} + +// The docker build's one-shot MxBuild shares the setting; asserted on the +// environment the child actually receives. +func TestMxBuildCmd_RunsGradleWithoutADaemon(t *testing.T) { + dir := t.TempDir() + fake := filepath.Join(dir, "mxbuild") + seen := filepath.Join(dir, "gradle_opts") + if err := os.WriteFile(fake, []byte("#!/bin/sh\nprintf '%s' \"$GRADLE_OPTS\" > '"+seen+"'\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("GRADLE_OPTS", "-Xmx1g") + var out syncBuffer + if err := mxbuildCmd(fake, "/usr", dir, filepath.Join(dir, "App.mpr"), &out, &out); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(seen) + if err != nil { + t.Fatal(err) + } + if string(got) != "-Xmx1g -Dorg.gradle.daemon=false" { + t.Errorf("mxbuild saw GRADLE_OPTS=%q, want the user's options plus -Dorg.gradle.daemon=false", got) + } +} diff --git a/cmd/mxcli/docker/loglevels.go b/cmd/mxcli/docker/loglevels.go index db097e96e8..7f0bd74657 100644 --- a/cmd/mxcli/docker/loglevels.go +++ b/cmd/mxcli/docker/loglevels.go @@ -68,7 +68,7 @@ type LogNodeLevel struct { func GetLogSettings(opts M2EEOptions) ([]LogNodeLevel, error) { resp, err := CallM2EE(opts, "get_log_settings", map[string]any{"sort": "node"}) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrAdminUnreachable, err) + return nil, fmt.Errorf("%w: %w", ErrAdminUnreachable, err) } if resp.Result != 0 { return nil, fmt.Errorf("reading log settings: %s", resp.M2EEError()) @@ -123,7 +123,7 @@ func SetLogLevels(opts M2EEOptions, levels []LogNodeLevel, force bool) error { resp, err := CallM2EE(opts, "set_log_level", params) if err != nil { - return fmt.Errorf("%w: %v", ErrAdminUnreachable, err) + return fmt.Errorf("%w: %w", ErrAdminUnreachable, err) } if resp.Result != 0 { // The HTTP response for an AdminException says only "See logging output diff --git a/cmd/mxcli/docker/m2ee.go b/cmd/mxcli/docker/m2ee.go index 2c8c24d2fa..29b40dbcd5 100644 --- a/cmd/mxcli/docker/m2ee.go +++ b/cmd/mxcli/docker/m2ee.go @@ -47,6 +47,31 @@ type M2EEOptions struct { Timeout time.Duration } +// ErrAdminAuth marks the admin API refusing the credentials (HTTP 401/403). +// Against a `mxcli run --local` that usually means the credentials were not +// that run's — its .mxcli/run-local.json was missing — rather than a wrong +// password, so a caller that knows the project can say so (ako/mxcli#1542). +var ErrAdminAuth = errors.New("admin API refused the credentials") + +// adminError keeps an admin-API failure's text while letting errors.Is see +// what kind of failure it was. +type adminError struct { + msg string + kind error +} + +func (e *adminError) Error() string { return e.msg } +func (e *adminError) Is(target error) bool { return target == e.kind } + +func adminAuthFailed(status int) error { + return &adminError{kind: ErrAdminAuth, + msg: fmt.Sprintf("authentication failed (HTTP %d) -- check the admin password (--token or M2EE_ADMIN_PASS)", status)} +} + +func adminUnreachable(msg string) error { + return &adminError{kind: ErrAdminUnreachable, msg: msg} +} + // M2EEResponse is the parsed envelope from the admin API. type M2EEResponse struct { Result int `json:"result"` @@ -117,12 +142,12 @@ func callM2EEDirect(opts M2EEOptions, action string, params map[string]any) (*M2 client := &http.Client{Timeout: timeout} resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("cannot connect to Mendix admin API at %s:%d -- is the app running? Start with 'mxcli docker up'", opts.Host, opts.Port) + return nil, adminUnreachable(fmt.Sprintf("cannot connect to Mendix admin API at %s:%d -- is the app running? Start with 'mxcli docker up'", opts.Host, opts.Port)) } defer resp.Body.Close() if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { - return nil, fmt.Errorf("authentication failed (HTTP %d) -- check the admin password (--token or M2EE_ADMIN_PASS)", resp.StatusCode) + return nil, adminAuthFailed(resp.StatusCode) } if resp.StatusCode != http.StatusOK { @@ -252,7 +277,7 @@ func previewOQLDevDirect(opts M2EEOptions, bodyBytes []byte) (json.RawMessage, e client := &http.Client{Timeout: timeout} resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("cannot connect to Mendix admin API at %s:%d -- is the app running? Start with 'mxcli docker up'", opts.Host, opts.Port) + return nil, adminUnreachable(fmt.Sprintf("cannot connect to Mendix admin API at %s:%d -- is the app running? Start with 'mxcli docker up'", opts.Host, opts.Port)) } defer resp.Body.Close() @@ -260,7 +285,7 @@ func previewOQLDevDirect(opts M2EEOptions, bodyBytes []byte) (json.RawMessage, e case resp.StatusCode == http.StatusNotFound: return nil, errDevEndpointNotFound case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden: - return nil, fmt.Errorf("authentication failed (HTTP %d) -- check the admin password (--token or M2EE_ADMIN_PASS)", resp.StatusCode) + return nil, adminAuthFailed(resp.StatusCode) case resp.StatusCode != http.StatusOK: return nil, fmt.Errorf("unexpected HTTP %d from admin API", resp.StatusCode) } @@ -300,13 +325,13 @@ func previewOQLDevDocker(opts M2EEOptions, dockerDir string, bodyBytes []byte) ( body, status, ok := splitCurlStatus(stdout.Bytes()) if !ok || status == 0 { - return nil, fmt.Errorf("cannot connect to Mendix admin API -- is the app running? Start with 'mxcli docker up'") + return nil, adminUnreachable("cannot connect to Mendix admin API -- is the app running? Start with 'mxcli docker up'") } switch status { case http.StatusNotFound: return nil, errDevEndpointNotFound case http.StatusUnauthorized, http.StatusForbidden: - return nil, fmt.Errorf("authentication failed (HTTP %d) -- check the admin password (--token or M2EE_ADMIN_PASS)", status) + return nil, adminAuthFailed(status) case http.StatusOK: return body, nil default: diff --git a/cmd/mxcli/docker/mxbuild_watchdog.go b/cmd/mxcli/docker/mxbuild_watchdog.go index a45abb1faa..e1f692ba39 100644 --- a/cmd/mxcli/docker/mxbuild_watchdog.go +++ b/cmd/mxcli/docker/mxbuild_watchdog.go @@ -35,6 +35,20 @@ const ( type watchdogLimits struct { Timeout time.Duration Stall time.Duration + + // Progress, when positive, calls OnProgress with the elapsed time every + // Progress while the command runs — so a slow step says it is still + // working instead of looking exactly like a hung one. + Progress time.Duration + OnProgress func(elapsed time.Duration) + + // Step names what is running in a WatchdogError ("mxbuild" when empty); + // TimeoutEnv and StallEnv are the variables that raise each bound (the + // MXCLI_MXBUILD_* ones when empty); Hint is the likely cause of a kill. + Step string + TimeoutEnv string + StallEnv string + Hint string } // mxbuildWatchdogLimits is what the one-shot MxBuild runs use; a variable so a @@ -67,32 +81,41 @@ func durationEnv(key string, def time.Duration) time.Duration { // WatchdogError reports a build the watchdog killed. type WatchdogError struct { + Step string // what was killed ("mxbuild" when empty) Reason string // "timed out" or "stalled" Limit time.Duration Elapsed time.Duration JavaHome string Tail string // the last lines of output EnvVar string // the variable that raises the limit + Hint string // the likely cause, when the step has a known one } func (e *WatchdogError) Error() string { + step := e.Step + if step == "" { + step = "mxbuild" + } var b strings.Builder if e.Reason == "stalled" { - fmt.Fprintf(&b, "mxbuild stalled: no output for %s (after %s), killed", e.Limit, e.Elapsed.Round(time.Second)) + fmt.Fprintf(&b, "%s stalled: no output for %s (after %s), killed", step, e.Limit, e.Elapsed.Round(time.Second)) } else { - fmt.Fprintf(&b, "mxbuild timed out after %s, killed", e.Limit) + fmt.Fprintf(&b, "%s timed out after %s, killed", step, e.Limit) } if e.JavaHome != "" { fmt.Fprintf(&b, "\n Java home used: %s", e.JavaHome) } - fmt.Fprintf(&b, "\n Raise the limit with %s= (e.g. 30m; 0 disables) if the build is merely slow.", e.EnvVar) + if e.Hint != "" { + b.WriteString("\n " + strings.ReplaceAll(e.Hint, "\n", "\n ")) + } + fmt.Fprintf(&b, "\n Raise the limit with %s= (e.g. 30m; 0 disables) if it is merely slow.", e.EnvVar) if e.Tail != "" { b.WriteString("\n Last output:\n") for _, l := range strings.Split(e.Tail, "\n") { b.WriteString(" " + l + "\n") } } else { - b.WriteString("\n (mxbuild printed nothing)") + fmt.Fprintf(&b, "\n (%s printed nothing)", step) } return strings.TrimRight(b.String(), "\n") } @@ -159,8 +182,18 @@ func runWithWatchdog(cmd *exec.Cmd, lim watchdogLimits, javaHome string) error { } setProcessGroup(cmd) // Once the tree is killed nothing should hold the pipes, but never let a - // stray descendant turn a kill into a second hang. - cmd.WaitDelay = 10 * time.Second + // stray descendant turn a kill into a second hang. A caller that chose its + // own delay (the orphaned-daemon backstop, #1396) keeps it. + if cmd.WaitDelay == 0 { + cmd.WaitDelay = 10 * time.Second + } + timeoutEnv, stallEnv := lim.TimeoutEnv, lim.StallEnv + if timeoutEnv == "" { + timeoutEnv = mxbuildTimeoutEnv + } + if stallEnv == "" { + stallEnv = mxbuildStallEnv + } start := time.Now() if err := cmd.Start(); err != nil { @@ -170,14 +203,14 @@ func runWithWatchdog(cmd *exec.Cmd, lim watchdogLimits, javaHome string) error { go func() { done <- cmd.Wait() }() tick := time.Second - if lim.Stall > 0 && lim.Stall/4 < tick { - tick = max(lim.Stall/4, 10*time.Millisecond) - } - if lim.Timeout > 0 && lim.Timeout/4 < tick { - tick = max(lim.Timeout/4, 10*time.Millisecond) + for _, bound := range []time.Duration{lim.Stall, lim.Timeout, lim.Progress} { + if bound > 0 && bound/4 < tick { + tick = max(bound/4, 10*time.Millisecond) + } } ticker := time.NewTicker(tick) defer ticker.Stop() + lastProgress := start for { select { case err := <-done: @@ -186,15 +219,21 @@ func runWithWatchdog(cmd *exec.Cmd, lim watchdogLimits, javaHome string) error { var werr *WatchdogError switch { case lim.Timeout > 0 && now.Sub(start) >= lim.Timeout: - werr = &WatchdogError{Reason: "timed out", Limit: lim.Timeout, EnvVar: mxbuildTimeoutEnv} + werr = &WatchdogError{Reason: "timed out", Limit: lim.Timeout, EnvVar: timeoutEnv} case lim.Stall > 0 && now.Sub(act.lastWrite()) >= lim.Stall: - werr = &WatchdogError{Reason: "stalled", Limit: lim.Stall, EnvVar: mxbuildStallEnv} + werr = &WatchdogError{Reason: "stalled", Limit: lim.Stall, EnvVar: stallEnv} } if werr == nil { + if lim.Progress > 0 && lim.OnProgress != nil && now.Sub(lastProgress) >= lim.Progress { + lastProgress = now + lim.OnProgress(now.Sub(start)) + } continue } _ = killProcessGroup(cmd.Process) <-done + werr.Step = lim.Step + werr.Hint = lim.Hint werr.Elapsed = time.Since(start) werr.JavaHome = javaHome werr.Tail = act.tailLines(20) diff --git a/cmd/mxcli/docker/mxenv.go b/cmd/mxcli/docker/mxenv.go index 1c496aea89..dd2ab0e00f 100644 --- a/cmd/mxcli/docker/mxenv.go +++ b/cmd/mxcli/docker/mxenv.go @@ -99,3 +99,63 @@ func injectLDPreload(env []string, lib string) (result []string, changed bool) { } return out, true } + +// gradleNoDaemonOpt makes Gradle build in the client process (or a single-use +// daemon it stops afterwards) instead of starting or reusing a long-lived one. +const gradleNoDaemonOpt = "-Dorg.gradle.daemon=false" + +// prepareOneShotMxCommand is PrepareMxCommand for an mx/mxbuild run that ends +// when its one job does — `mx sync-java-dependencies`, the settle build of +// `mxcli new`, a docker build — plus Gradle without a daemon. +// +// Each such run used to leave a Gradle daemon behind for hours, and the next +// run attached to it: with an idle daemon from an earlier run alive, `mx +// sync-java-dependencies` hung for over ten minutes printing nothing, where +// the same sync without a daemon took about twelve seconds and left nothing +// running (ako/mxcli#1340). A one-shot build gains nothing from a daemon, which +// only pays off for the NEXT build. The warm loop's `mxbuild --serve` is a +// different case — it builds repeatedly — and keeps its daemon. +func prepareOneShotMxCommand(cmd *exec.Cmd) { + PrepareMxCommand(cmd) + env := cmd.Env + if env == nil { + env = os.Environ() + } + cmd.Env = withGradleNoDaemon(env) +} + +// withGradleNoDaemon returns env with gradleNoDaemonOpt appended to GRADLE_OPTS +// (which is created when absent), keeping whatever the user put there — a +// proxy, a heap size. A GRADLE_OPTS that already sets org.gradle.daemon is the +// user's explicit choice and is left alone. env is not mutated. +func withGradleNoDaemon(env []string) []string { + out := append([]string(nil), env...) + // From the end: exec keeps the last of duplicate keys, so that is the one + // the child would see. + for i := len(out) - 1; i >= 0; i-- { + key, value, ok := strings.Cut(out[i], "=") + if !ok || !envKeyIs(key, "GRADLE_OPTS") { + continue + } + if strings.Contains(value, "-Dorg.gradle.daemon=") { + return out + } + if strings.TrimSpace(value) == "" { + out[i] = key + "=" + gradleNoDaemonOpt + } else { + out[i] = key + "=" + value + " " + gradleNoDaemonOpt + } + return out + } + return append(out, "GRADLE_OPTS="+gradleNoDaemonOpt) +} + +// envKeyIs compares an environment variable name the way the OS does: +// case-insensitively on Windows, where appending a second GRADLE_OPTS beside a +// "Gradle_Opts" would replace the user's value rather than extend it. +func envKeyIs(key, name string) bool { + if runtime.GOOS == "windows" { + return strings.EqualFold(key, name) + } + return key == name +} diff --git a/cmd/mxcli/docker/pagecheck.go b/cmd/mxcli/docker/pagecheck.go index 628027f6d9..6dfef0ca55 100644 --- a/cmd/mxcli/docker/pagecheck.go +++ b/cmd/mxcli/docker/pagecheck.go @@ -5,6 +5,7 @@ package docker import ( "encoding/json" "fmt" + "net/url" "os" "os/exec" "path/filepath" @@ -41,6 +42,9 @@ type PageSignals struct { Rows int `json:"rows"` // grid / list rows rendered ConsoleErrors []string `json:"consoleErrors"` // what a screenshot cannot show + // RequestedURL is the URL the page was loaded from, set by ProbePages so a + // verdict can tell when FinalURL is somewhere else. + RequestedURL string `json:"requestedUrl"` // FinalURL is where the browser ended up — a redirect to the login page // shows here, not in the title. FinalURL string `json:"finalUrl"` @@ -52,6 +56,10 @@ type PageSignals struct { // LoginForm reports that the Mendix sign-in form is on screen: the page // asked for was not rendered, whatever the title says. LoginForm bool `json:"loginForm"` + // SignedInAs is who the page was loaded as — "demo_user (user role User)" — + // set by `playwright check` when it signed in; empty for an anonymous load. + // It is not measured, only used to word a page the user was refused. + SignedInAs string `json:"-"` // TextFound and Counts answer the probe's text / count queries, index-aligned // with ProbeOptions.Texts and ProbeOptions.CountSelectors. TextFound []bool `json:"textFound"` @@ -151,6 +159,10 @@ const req = JSON.parse(require('fs').readFileSync(process.argv[2], 'utf8')); // sources load after the page shell, so a fixed sleep is either too short // or wasted — and the text alone holds still while a list view's data is // out, which is how a verdict used to be taken before the rows arrived. + // The sign-in form is the exception to the request rule: the client keeps + // a POST /xas/ open behind it (measured on 11.13), so waiting for quiet + // served the whole --wait and then reported the form as still loading. + // The form is a verdict in itself once it holds still. await p.waitForSelector('.mx-page, #usernameInput', { timeout: 20000 }).catch(() => {}); const mounted = Date.now(); const deadline = mounted + (req.waitMs || 15000); @@ -159,8 +171,9 @@ const req = JSON.parse(require('fs').readFileSync(process.argv[2], 'utf8')); const st = await p.evaluate(() => ({ len: (document.body && document.body.innerText || '').length, busy: document.querySelectorAll('.mx-progress, .mx-underlay-loading, [aria-busy="true"]').length, - })).catch(() => ({ len: -2, busy: 0 })); - const quiet = inflight.size === 0 && Date.now() - lastNet >= 500; + login: !!document.querySelector('#usernameInput'), + })).catch(() => ({ len: -2, busy: 0, login: false })); + const quiet = st.login || (inflight.size === 0 && Date.now() - lastNet >= 500); if (st.len === last && st.busy === 0 && quiet) { if (++stable >= 3) { settled = true; break; } } else { stable = 0; } @@ -177,7 +190,10 @@ const req = JSON.parse(require('fs').readFileSync(process.argv[2], 'utf8')); // heading, which is not the page's. out.headings = await p.$$eval('h1,h2', ns => ns.filter(n => !n.closest('.mx-demouserswitcher')) .map(n => (n.innerText||'').trim()).filter(Boolean).slice(0, 3)); - out.alerts = await p.$$eval('.alert-danger,.alert-warning,.mx-validation-message', + // An information dialog is how the client explains a redirect it made + // itself — "The page you requested was not found" for an unknown /p/ URL + // (ako/mxcli#998) — so it is shown, though it fails nothing. + out.alerts = await p.$$eval('.alert-danger,.alert-warning,.mx-validation-message,.mx-dialog-info .mx-dialog-body', ns => ns.map(n => (n.innerText||'').trim()).filter(Boolean).slice(0, 3)); out.errorAlerts = await p.$$eval('.alert-danger,.mx-dialog-error .mx-dialog-body', ns => ns.map(n => (n.innerText||'').trim()).filter(Boolean).slice(0, 3)); @@ -328,6 +344,9 @@ func ProbePages(opts ProbeOptions) ([]PageSignals, error) { if err := json.Unmarshal([]byte(lines[len(lines)-1]), &sigs); err != nil || len(sigs) != len(opts.Pages) { return nil, fmt.Errorf("page check produced no verdict: %v\n%s", err, clip(out.String(), 600)) } + for i := range sigs { + sigs[i].RequestedURL = opts.Pages[i].URL + } return sigs, nil } @@ -403,10 +422,21 @@ func globalNodeModules(node string) string { func formatPageVerdict(label string, s PageSignals) string { var b strings.Builder fmt.Fprintf(&b, "page %s", label) + // Where the browser ended, when that is not where it was sent: the title + // and heading describe the page at this URL, which for an unknown /p/ URL + // is the home page (ako/mxcli#998). + if where, moved := s.landedAt(); moved { + fmt.Fprintf(&b, " url=%s", clip(where, 100)) + } if s.Status >= 400 { fmt.Fprintf(&b, " http=%d", s.Status) } - if s.LoginForm { + // A page the signed-in user may not open lands on the sign-in form too, + // with messageCode=403 in the URL: saying LOGIN PAGE sent readers to sign in. + switch { + case s.AccessDenied(): + b.WriteString(" ACCESS DENIED") + case s.LoginForm: b.WriteString(" LOGIN PAGE") } if s.Title != "" { @@ -447,6 +477,80 @@ func formatPageVerdict(label string, s PageSignals) string { return b.String() } +// landedAt reports where the browser ended up when that differs from the URL +// the page was loaded from: the path, query and fragment on the same origin, +// the whole URL on another. A trailing slash on an empty path is no move. +func (s PageSignals) landedAt() (string, bool) { + req, fin, ok := s.locations() + if !ok { + return "", false + } + if strings.EqualFold(req.Scheme+"://"+req.Host, fin.Scheme+"://"+fin.Host) { + if pathOrRoot(req.Path) == pathOrRoot(fin.Path) && req.RawQuery == fin.RawQuery && req.Fragment == fin.Fragment { + return "", false + } + return relativeURL(fin), true + } + return s.FinalURL, true +} + +// relativeURL is u's path, query and fragment: how a URL on the app's own +// origin is shown. +func relativeURL(u *url.URL) string { + where := pathOrRoot(u.EscapedPath()) + if u.RawQuery != "" { + where += "?" + u.RawQuery + } + if u.Fragment != "" { + where += "#" + u.EscapedFragment() + } + return where +} + +// wrongPage reports a /p/ page that the browser did not stay on, and where it +// went instead. Measured on 11.13 and 11.15: the runtime answers every +// /p/ with a 301 to /#/, and the client replaces the location with +// /p/ for a page it has and with / for one it does not — rendering the home +// page, with HTTP 200 throughout (ako/mxcli#998). Matching is exact: /p/ITEMS +// and /p/items/ landed on / too. Only the path is compared; the query string +// survives the redirect, and so may a change of scheme in front of the app +// (http to https), which leaves the page the same. A page that went to the +// sign-in form is that form's failure, not this one, and so is a page the +// user was refused (/login.html?messageCode=403). +func (s PageSignals) wrongPage() (requested, landed string, home bool) { + req, fin, ok := s.locations() + if !ok || s.NeedsLogin() || s.AccessDenied() || !strings.Contains(req.Path, "/p/") || req.Path == fin.Path { + return "", "", false + } + sameOrigin := strings.EqualFold(req.Scheme+"://"+req.Host, fin.Scheme+"://"+fin.Host) + where, _ := s.landedAt() + if where == "" { + where = s.FinalURL + } + return req.Path, where, sameOrigin && pathOrRoot(fin.Path) == "/" +} + +// locations parses the requested and final URLs; ok is false when either is +// missing (a probe that never got the page). +func (s PageSignals) locations() (req, fin *url.URL, ok bool) { + if s.RequestedURL == "" || s.FinalURL == "" { + return nil, nil, false + } + req, err1 := url.Parse(s.RequestedURL) + fin, err2 := url.Parse(s.FinalURL) + if err1 != nil || err2 != nil { + return nil, nil, false + } + return req, fin, true +} + +func pathOrRoot(p string) string { + if p == "" { + return "/" + } + return p +} + // unsettledReason says why a page counted as still loading. func (s PageSignals) unsettledReason() string { waited := fmt.Sprintf("%.1fs", float64(s.WaitedMs)/1000) diff --git a/cmd/mxcli/docker/pagecheck_assert.go b/cmd/mxcli/docker/pagecheck_assert.go index 373ff967a9..8b021ee129 100644 --- a/cmd/mxcli/docker/pagecheck_assert.go +++ b/cmd/mxcli/docker/pagecheck_assert.go @@ -4,6 +4,7 @@ package docker import ( "fmt" + "net/url" "strconv" "strings" ) @@ -109,8 +110,10 @@ func (s CheckSpec) CountSelectors() []string { // EvaluatePage returns the reasons a page fails, empty when it passes. // // A page fails when it was still loading when the wait ran out, when the -// browser was refused it (HTTP >= 400), when it shows -// the sign-in form instead of itself, when it shows an error banner or error +// browser was refused it (HTTP >= 400), when the runtime refused it to the +// signed-in user (403: the user's roles may not open it), when it shows +// the sign-in form instead of itself, when a /p/ URL ended somewhere else (an +// unknown page URL lands on the home page), when it shows an error banner or error // dialog, when the console reports an error, when a same-origin request it // made failed, or when an assertion does not hold. Warnings and validation // messages are reported in the verdict but do not fail it. @@ -118,8 +121,10 @@ func EvaluatePage(sig PageSignals, spec CheckSpec) []string { var fails []string // A page measured before it finished loading may be missing exactly what // the check is for — the rows of a list whose data had not arrived - // (ako/mxcli#845) — so it cannot pass, whatever else it shows. - if sig.Unsettled { + // (ako/mxcli#845) — so it cannot pass, whatever else it shows. The sign-in + // form is not that page, and no longer wait would turn it into the page: + // its failure below is the one that says what to do. + if sig.Unsettled && !sig.LoginForm { fails = append(fails, "still loading when measured, so the verdict may describe a partly "+ "rendered page (raise --wait if it is only slow)") } @@ -127,19 +132,37 @@ func EvaluatePage(sig PageSignals, spec CheckSpec) []string { fails = append(fails, fmt.Sprintf("HTTP %d", sig.Status)) } switch { + case sig.AccessDenied(): + fails = append(fails, sig.accessDeniedReason()) case sig.LoginForm: fails = append(fails, "the sign-in form is showing, not the page (pass --user/--password or --role)") case sig.NeedsLogin(): fails = append(fails, "not signed in: the runtime answered 401 (pass --user/--password or --role)") } + // Everything else describes the page the browser ended on. For an unknown + // /p/ URL that is the home page, which renders cleanly (ako/mxcli#998). + if requested, landed, home := sig.wrongPage(); requested != "" { + f := fmt.Sprintf("the browser ended on %s, not on %s", landed, requested) + if home { + f += ": the client sends a /p/ URL it has no page for to the home page" + } + fails = append(fails, f) + } if n := len(sig.ErrorAlerts); n > 0 { fails = append(fails, fmt.Sprintf("%d error banner(s)", n)) } // On the sign-in form the client's own 401s are the consequence of not // being signed in, not separate failures; counting them would bury the one // reason that matters. + // + // A refused page never started at all: the 403 and the client's "[Startup] + // An error occurred while creating a new session" are the refusal itself, + // so none of them count. They stay on the verdict's ERR and HTTP lines. consoleErrs, httpErrs := sig.ConsoleErrors, sig.HTTPErrors - if sig.NeedsLogin() { + switch { + case sig.AccessDenied(): + consoleErrs, httpErrs = nil, nil + case sig.NeedsLogin(): consoleErrs, httpErrs = without401(consoleErrs), without401(httpErrs) } if n := len(consoleErrs); n > 0 && !spec.AllowConsoleErrors { @@ -173,7 +196,15 @@ func EvaluatePage(sig PageSignals, spec CheckSpec) []string { // signed in: the sign-in form is up, or the runtime refused the client's // session with a 401. A session the runtime has forgotten (it restarted) shows // the second: the client gets a 401 and renders nothing, with no form at all. +// +// A page the user may not open also ends on the sign-in form, but the browser +// is signed in: the runtime knew the session and refused the page. Signing in +// again as the same user gets the same answer, so that is not this — and +// reading it as this replaced a live session with a new one on every check. func (s PageSignals) NeedsLogin() bool { + if s.AccessDenied() { + return false + } if s.LoginForm || s.Status == 401 { return true } @@ -185,6 +216,40 @@ func (s PageSignals) NeedsLogin() bool { return false } +// AccessDenied reports that the runtime refused the page to the browser's +// signed-in user: the user's roles may not open it. Measured on 11.13 and +// 11.15: the runtime answers the client's session request for such a page +// (POST /xas/ get_session_data, referrer the page) with 403, and the client +// goes to /login.html?messageCode=403#/. That is the sign-in form, with +// no message on it — login.js misreads the code, so its "not enough +// permissions" text never shows, and it would be translated if it did. +// Anonymous, or with a session the runtime has forgotten, the same request is +// answered 401 and the client goes to /login.html#/: the same form +// without the code, guest access on or off. So the code in the URL is the one +// signal that tells "sign in" from "this user may not". +func (s PageSignals) AccessDenied() bool { + if s.FinalURL == "" { + return false + } + u, err := url.Parse(s.FinalURL) + if err != nil { + return false + } + return u.Query().Get("messageCode") == "403" +} + +// accessDeniedReason is the failure for a page the signed-in user was refused. +func (s PageSignals) accessDeniedReason() string { + page := "the page" + if req, _, ok := s.locations(); ok { + page = relativeURL(req) + } + if s.SignedInAs == "" { + return fmt.Sprintf("access denied: the runtime refused %s to the signed-in user (403)", page) + } + return fmt.Sprintf("access denied: signed in as %s, the runtime refused %s (403)", s.SignedInAs, page) +} + func without401(ss []string) []string { var out []string for _, s := range ss { diff --git a/cmd/mxcli/docker/pagecheck_check.go b/cmd/mxcli/docker/pagecheck_check.go index 98f96331c5..4ccf3021b9 100644 --- a/cmd/mxcli/docker/pagecheck_check.go +++ b/cmd/mxcli/docker/pagecheck_check.go @@ -3,6 +3,7 @@ package docker import ( + "errors" "fmt" "io" "net/url" @@ -27,6 +28,7 @@ import ( var ( probePagesFn = ProbePages loginFn = LoginAndSaveStorage + signOutFn = SignOut ) // PageCheckOptions configures RunPageCheck. @@ -39,11 +41,19 @@ type PageCheckOptions struct { Password string Role string // log in as the project's demo user holding this user role - // FreshLogin ignores a saved session and signs in again. + // FreshLogin ignores a saved session and signs in again. The session it + // replaces is signed out first, so it does not hold a slot until it expires. FreshLogin bool + // SignOut ends the run signed out: the session it used is signed out and + // not kept for the next check. + SignOut bool // StorageDir holds the saved sessions (default /.mxcli/playwright-check, // or ~/.mxcli/playwright-check without a project). StorageDir string + // RuntimeLogPath is read when a sign-in is refused, to name the cause the + // sign-in page does not (the unlicensed runtime's session cap). Default + // /.mxcli/runtime.log, where `run --local` writes it. + RuntimeLogPath string Spec CheckSpec Screenshot string // PNG path; per-page suffix when there are several pages @@ -62,7 +72,21 @@ type PageCheckResult struct { // credentials is who to sign in as, and why. type credentials struct { user, password string - source string // "", "--role Administrator", "demo user", ... + source string // "", "--role Administrator", "demo user", ... + roles []string // the demo user's user roles; unknown (nil) for --user +} + +// signedInAs names the user for a verdict: "demo_user (user role User)", or +// just the name when the roles are not known. +func (c credentials) signedInAs() string { + switch len(c.roles) { + case 0: + return c.user + case 1: + return c.user + " (user role " + c.roles[0] + ")" + default: + return c.user + " (user roles " + strings.Join(c.roles, ", ") + ")" + } } // RunPageCheck checks every target and prints one verdict per page. The error @@ -74,9 +98,15 @@ func RunPageCheck(opts PageCheckOptions) (PageCheckResult, error) { if w == nil { w = os.Stdout } + if opts.Stderr == nil { + opts.Stderr = os.Stderr + } if opts.BaseURL == "" { opts.BaseURL = "http://localhost:8080" } + if opts.RuntimeLogPath == "" && opts.ProjectPath != "" { + opts.RuntimeLogPath = filepath.Join(filepath.Dir(opts.ProjectPath), ".mxcli", "runtime.log") + } targets := opts.Targets if len(targets) == 0 { targets = []string{""} @@ -86,13 +116,6 @@ func RunPageCheck(opts PageCheckOptions) (PageCheckResult, error) { if err != nil { return res, err } - storage := "" - if creds.user != "" { - storage = checkStoragePath(opts, creds.user) - if opts.FreshLogin { - _ = os.Remove(storage) - } - } if opts.Screenshot != "" { if abs, err := filepath.Abs(opts.Screenshot); err == nil { @@ -112,45 +135,169 @@ func RunPageCheck(opts PageCheckOptions) (PageCheckResult, error) { } } } + // A session is a set of cookies, and a cookie belongs to the host it was + // set on: one signed in on localhost is not sent to 127.0.0.1 or to + // frontend.local, though all three reach the same app. Signing in on the + // base URL and then loading a page given as a full URL on another host + // showed the sign-in form on every run, and signed in again on each + // (ako/mxcli#1216). So the pages of each host get a session signed in on + // that host. + mxbuild := anyMxBuildPath() + sigs := make([]PageSignals, len(pages)) + for _, g := range groupByOrigin(opts.BaseURL, pages) { + o := opts + o.BaseURL = g.appURL + got, err := checkOrigin(o, creds, g.pages(pages), mxbuild, g.appURL != opts.BaseURL, w) + if err != nil { + return res, err + } + for j, i := range g.idx { + sigs[i] = got[j] + } + } + + for i, s := range sigs { + fails := EvaluatePage(s, opts.Spec) + if s.NeedsLogin() && creds.user == "" { + fails = replaceLoginHint(fails, opts.ProjectPath) + } + fmt.Fprint(w, formatCheckResult(pageLabel(targets[i]), s, fails)) + res.Pages++ + if len(fails) > 0 { + res.Failed++ + } + } + if res.Failed > 0 { + fmt.Fprintf(w, "FAIL %d of %d page(s)\n", res.Failed, res.Pages) + } else { + fmt.Fprintf(w, "OK %d page(s)\n", res.Pages) + } + return res, nil +} + +// originGroup is the pages of one origin, as indexes into the request. +type originGroup struct { + appURL string // where to sign in: the base URL, or the pages' own origin + idx []int +} + +func (g originGroup) pages(all []PageTarget) []PageTarget { + out := make([]PageTarget, len(g.idx)) + for j, i := range g.idx { + out[j] = all[i] + } + return out +} + +// groupByOrigin splits the pages by scheme, host and port, in the order each +// origin first appears. Pages on the base URL's origin sign in on the base URL +// itself; the others on their own origin. +func groupByOrigin(baseURL string, pages []PageTarget) []originGroup { + baseOrigin := originOf(baseURL) + var groups []originGroup + at := map[string]int{} + for i, p := range pages { + o := originOf(p.URL) + k, ok := at[o] + if !ok { + app := o + if o == baseOrigin || o == "" { + app = baseURL + } + k = len(groups) + at[o] = k + groups = append(groups, originGroup{appURL: app}) + } + groups[k].idx = append(groups[k].idx, i) + } + return groups +} + +// originOf is scheme://host[:port], lower-cased; "" when raw is not an +// absolute URL. +func originOf(raw string) string { + u, err := url.Parse(raw) + if err != nil || u.Scheme == "" || u.Host == "" { + return "" + } + return strings.ToLower(u.Scheme + "://" + u.Host) +} + +// checkOrigin probes pages that share one origin, opts.BaseURL, signing in on +// that origin when the check has credentials. nameHost adds the host to the +// login/logout lines, for an origin other than the one the user gave. +func checkOrigin(opts PageCheckOptions, creds credentials, pages []PageTarget, mxbuild string, nameHost bool, w io.Writer) ([]PageSignals, error) { probe := ProbeOptions{ Pages: pages, WaitMs: opts.WaitMs, Texts: opts.Spec.Texts, CountSelectors: opts.Spec.CountSelectors(), - MxBuildPath: anyMxBuildPath(), + MxBuildPath: mxbuild, + } + if creds.user == "" { + return probePagesFn(probe) } + storage := checkStoragePath(opts, creds.user) + where := "" + if nameHost { + if u, err := url.Parse(opts.BaseURL); err == nil { + where = " on " + u.Host + } + } + signOut := func() { + // Best effort, and never part of the verdict: a session that could not + // be signed out holds its slot only until the runtime expires it. + err := signOutFn(SignOutOptions{AppURL: opts.BaseURL, StoragePath: storage, MxBuildPath: mxbuild}) + if errors.Is(err, ErrNoSession) { + return // already gone: nothing held a slot + } + if err != nil { + fmt.Fprintf(opts.Stderr, "sign-out of %s%s did not complete (the session expires on its own): %v\n", creds.user, where, err) + return + } + fmt.Fprintf(w, "logout %s%s\n", creds.user, where) + } + if opts.FreshLogin { + if _, err := os.Stat(storage); err == nil { + signOut() + } + _ = os.Remove(storage) + } login := func() error { - fmt.Fprintf(w, "login %s%s\n", creds.user, sourceSuffix(creds.source)) + fmt.Fprintf(w, "login %s%s%s\n", creds.user, where, sourceSuffix(creds.source)) return loginFn(LoginOptions{ AppURL: opts.BaseURL, Username: creds.user, Password: creds.password, - StoragePath: storage, MxBuildPath: probe.MxBuildPath, + StoragePath: storage, MxBuildPath: mxbuild, RuntimeLogPath: opts.RuntimeLogPath, }) } // A saved session is reused as-is: signing in again costs a browser round // trip and, on the unlicensed runtime, one of a handful of session slots. + // Reused, it stays one session however many checks run. loggedIn := false - if storage != "" { - if _, err := os.Stat(storage); err != nil { - if err := login(); err != nil { - return res, err - } - loggedIn = true + if _, err := os.Stat(storage); err != nil { + if err := login(); err != nil { + return nil, err } - probe.Storage = storage + loggedIn = true } + probe.Storage = storage sigs, err := probePagesFn(probe) if err != nil { - return res, err + return nil, err } // A saved session the runtime no longer knows (it restarted, or the session // timed out) lands every page on the sign-in form. Sign in once and redo - // just those pages. - if storage != "" && !loggedIn && anyNeedsLogin(sigs) { + // just those pages. A page the user's roles may not open lands there too, + // but it is refused with 403, not 401, and does not count: signing in again + // would get the same refusal and replace a live session with a new one on + // every check, one more session per run. + if !loggedIn && anyNeedsLogin(sigs) { if err := login(); err != nil { - return res, err + return nil, err } + loggedIn = true var redo []int retry := probe retry.Pages = nil @@ -162,30 +309,27 @@ func RunPageCheck(opts PageCheckOptions) (PageCheckResult, error) { } again, err := probePagesFn(retry) if err != nil { - return res, err + return nil, err } for j, i := range redo { sigs[i] = again[j] } } - for i, s := range sigs { - fails := EvaluatePage(s, opts.Spec) - if s.NeedsLogin() && creds.user == "" { - fails = replaceLoginHint(fails, opts.ProjectPath) - } - fmt.Fprint(w, formatCheckResult(pageLabel(targets[i]), s, fails)) - res.Pages++ - if len(fails) > 0 { - res.Failed++ - } - } - if res.Failed > 0 { - fmt.Fprintf(w, "FAIL %d of %d page(s)\n", res.Failed, res.Pages) - } else { - fmt.Fprintf(w, "OK %d page(s)\n", res.Pages) - } - return res, nil + // A session this run signed in that got no page past the sign-in form is + // worth nothing to the next check, which would only sign in again beside + // it — the per-run leak of #1216. Sign it out instead of keeping it; and + // sign out whatever session was used when the run is to end signed out. + // A session that was only refused pages (403) is not that: the runtime + // knows it, and the next check can use it. + if opts.SignOut || (loggedIn && allNeedLogin(sigs)) { + signOut() + _ = os.Remove(storage) + } + for i := range sigs { + sigs[i].SignedInAs = creds.signedInAs() + } + return sigs, nil } func anyNeedsLogin(sigs []PageSignals) bool { @@ -197,6 +341,15 @@ func anyNeedsLogin(sigs []PageSignals) bool { return false } +func allNeedLogin(sigs []PageSignals) bool { + for _, s := range sigs { + if !s.NeedsLogin() { + return false + } + } + return len(sigs) > 0 +} + func sourceSuffix(src string) string { if src == "" { return "" @@ -267,7 +420,7 @@ func pickDemoUser(ps *security.ProjectSecurity, role string) (credentials, error for _, du := range ps.DemoUsers { for _, r := range du.UserRoles { if strings.EqualFold(shortRole(r), role) || strings.EqualFold(r, role) { - return credentials{user: du.UserName, password: du.Password, source: "--role " + role}, nil + return credentials{user: du.UserName, password: du.Password, source: "--role " + role, roles: du.UserRoles}, nil } } names = append(names, du.UserName) @@ -285,12 +438,12 @@ func pickDemoUser(ps *security.ProjectSecurity, role string) (credentials, error for _, du := range ps.DemoUsers { for _, r := range du.UserRoles { if admin.MatchString(r) { - return credentials{user: du.UserName, password: du.Password, source: "demo user"}, nil + return credentials{user: du.UserName, password: du.Password, source: "demo user", roles: du.UserRoles}, nil } } } du := ps.DemoUsers[0] - return credentials{user: du.UserName, password: du.Password, source: "demo user"}, nil + return credentials{user: du.UserName, password: du.Password, source: "demo user", roles: du.UserRoles}, nil } // shortRole strips a module or project qualifier from a role name. diff --git a/cmd/mxcli/docker/pagecheck_check_test.go b/cmd/mxcli/docker/pagecheck_check_test.go index 00e433e46f..70400bd54f 100644 --- a/cmd/mxcli/docker/pagecheck_check_test.go +++ b/cmd/mxcli/docker/pagecheck_check_test.go @@ -185,16 +185,21 @@ func TestFormatCheckResult_PrintsPathNotImage(t *testing.T) { // fakeBrowser stands in for the node probe and the login script. type fakeBrowser struct { - probes [][]PageTarget - storage []string - logins int - answer func(call int, p PageTarget) PageSignals + probes [][]PageTarget + storage []string + logins int + signOuts int + answer func(call int, p PageTarget) PageSignals } func (f *fakeBrowser) install(t *testing.T) { t.Helper() - oldProbe, oldLogin := probePagesFn, loginFn - t.Cleanup(func() { probePagesFn, loginFn = oldProbe, oldLogin }) + oldProbe, oldLogin, oldSignOut := probePagesFn, loginFn, signOutFn + t.Cleanup(func() { probePagesFn, loginFn, signOutFn = oldProbe, oldLogin, oldSignOut }) + signOutFn = func(SignOutOptions) error { + f.signOuts++ + return nil + } probePagesFn = func(o ProbeOptions) ([]PageSignals, error) { call := len(f.probes) f.probes = append(f.probes, o.Pages) diff --git a/cmd/mxcli/docker/pagecheck_denied_test.go b/cmd/mxcli/docker/pagecheck_denied_test.go new file mode 100644 index 0000000000..000dd3fc18 --- /dev/null +++ b/cmd/mxcli/docker/pagecheck_denied_test.go @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/sdk/security" +) + +// `mxcli playwright check /p/secret /p/items -p app.mpr --role User`, where the +// User role may not open /p/secret, signed in and then reported +// +// page /p/secret url=/login.html?messageCode=403#/secret LOGIN PAGE title="Login" h="Sign in" text=34 console-errors=1 +// ERR [Startup] An error occurred while creating a new session. i_: Internal server error … +// HTTP 403 POST /xas/ +// FAIL the sign-in form is showing, not the page (pass --user/--password or --role) +// +// — telling a signed-in user to sign in. Every later run then signed in again +// beside the live session (it read as one the runtime had forgotten), one +// more session per run; checked alone, the page had its working session +// signed out after every run. +// +// Measured on 11.13.0 and 11.15.0, the same on both. The runtime answers the +// client's POST /xas/ get_session_data (referrer /secret) with 403 and the +// client goes to /login.html?messageCode=403#/secret: the sign-in form, whose +// message element (#loginMessage) stays empty and hidden, because login.js +// reads the code as `"403".split("=")[1]`. Anonymous, and with a session the +// runtime has forgotten, the same request is answered 401 and the client goes +// to /login.html#/secret — the same form, no messageCode — and that holds with +// guest access enabled too (11.15). So the query parameter is the signal; the +// page itself has nothing in it to tell the two apart, and its text is +// translated anyway. + +// What the probe returned for /p/secret, signed in as a user whose role may not +// open it (11.13.0, from the live run above). +func deniedSignals(requested string) PageSignals { + u, _ := url.Parse(requested) + page := strings.TrimPrefix(u.Path, "/p/") + return PageSignals{ + RequestedURL: requested, FinalURL: u.Scheme + "://" + u.Host + "/login.html?messageCode=403#/" + page, + Title: "Login", Headings: []string{"Sign in"}, TextLen: 34, Status: 200, LoginForm: true, + ConsoleErrors: []string{"[Startup] An error occurred while creating a new session. i_: Internal server error\n at MO (http://localhost:8180/dist/chunks/y6JB9J-k.js:32:33456)"}, + HTTPErrors: []string{"403 POST /xas/"}, + } +} + +// What the probe returned for the same page with no session (11.13.0). +func anonymousSignInSignals(requested string) PageSignals { + u, _ := url.Parse(requested) + page := strings.TrimPrefix(u.Path, "/p/") + return PageSignals{ + RequestedURL: requested, FinalURL: u.Scheme + "://" + u.Host + "/login.html#/" + page, + Title: "Login", Headings: []string{"Sign in"}, TextLen: 34, Status: 200, LoginForm: true, + ConsoleErrors: []string{"[Startup] Could not create a session because the server responded with 401 (Unauthorized)."}, + HTTPErrors: []string{"401 POST /xas/"}, + } +} + +func TestEvaluatePage_RoleDeniedPageIsAccessDenied(t *testing.T) { + s := deniedSignals("http://localhost:8180/p/secret") + s.SignedInAs = "demo_user (user role User)" + fails := EvaluatePage(s, CheckSpec{}) + if len(fails) != 1 { + t.Fatalf("want one reason, got %d: %q", len(fails), fails) + } + f := fails[0] + for _, want := range []string{"access denied", "demo_user (user role User)", "/p/secret", "403"} { + if !strings.Contains(f, want) { + t.Errorf("the failure does not say %q: %q", want, f) + } + } + // The user is signed in: advice to sign in, or to pass credentials, is wrong. + for _, wrong := range []string{"sign-in form", "not signed in", "--user", "--password", "--role"} { + if strings.Contains(f, wrong) { + t.Errorf("the failure for a signed-in user's refused page says %q: %q", wrong, f) + } + } +} + +// Without a known identity the failure still names the refusal, and still +// does not send the user to sign in. +func TestEvaluatePage_RoleDeniedWithoutIdentity(t *testing.T) { + fails := EvaluatePage(deniedSignals("http://localhost:8180/p/secret"), CheckSpec{}) + if len(fails) != 1 || !strings.Contains(fails[0], "access denied") || !strings.Contains(fails[0], "/p/secret") || + strings.Contains(fails[0], "sign-in form") { + t.Errorf("got %q", fails) + } +} + +// Control: the anonymous sign-in form is still the sign-in failure. +func TestEvaluatePage_AnonymousSignInFormIsUnchanged(t *testing.T) { + fails := EvaluatePage(anonymousSignInSignals("http://localhost:8180/p/secret"), CheckSpec{}) + if len(fails) != 1 || !strings.Contains(fails[0], "the sign-in form is showing") { + t.Errorf("got %q, want only the sign-in failure", fails) + } +} + +// The retry and sign-out decisions hang on NeedsLogin. A refused page is not a +// session problem: the runtime knew the session and said no to the page, and +// signing in again as the same user gets the same answer. +func TestNeedsLogin_RoleDeniedIsNotASessionProblem(t *testing.T) { + denied := deniedSignals("http://localhost:8180/p/secret") + if !denied.AccessDenied() { + t.Error("the 403 landing is not recognised as access denied") + } + if denied.NeedsLogin() { + t.Error("a page refused to a signed-in user reads as needing a sign-in") + } + anon := anonymousSignInSignals("http://localhost:8180/p/secret") + if anon.AccessDenied() || !anon.NeedsLogin() { + t.Errorf("control: the anonymous sign-in form: AccessDenied=%v NeedsLogin=%v", anon.AccessDenied(), anon.NeedsLogin()) + } +} + +// The verdict line, which `run --page-check` prints too, says what happened +// rather than LOGIN PAGE. +func TestVerdictSaysAccessDenied(t *testing.T) { + got := formatPageVerdict("/p/secret", deniedSignals("http://localhost:8180/p/secret")) + if !strings.Contains(got, "ACCESS DENIED") || strings.Contains(got, "LOGIN PAGE") { + t.Errorf("verdict:\n%s", got) + } + if !strings.Contains(got, "url=/login.html?messageCode=403#/secret") { + t.Errorf("the landing URL is not reported:\n%s", got) + } + if got := formatPageVerdict("/p/secret", anonymousSignInSignals("http://localhost:8180/p/secret")); !strings.Contains(got, "LOGIN PAGE") { + t.Errorf("control: the anonymous sign-in form is no longer LOGIN PAGE:\n%s", got) + } +} + +// The reported pair, three runs on one saved session: the refused page fails +// as access denied, the other passes, and no run signs in again — before the +// fix runs two and three each signed in beside the live session. +func TestRunPageCheck_RoleDeniedPageKeepsTheSession(t *testing.T) { + fb := &cookieJarBrowser{denied: "/p/secret"} + fb.install(t) + var out bytes.Buffer + opts := PageCheckOptions{BaseURL: "http://localhost:8180", Targets: []string{"/p/secret", "/p/items"}, + User: "demo_user", Password: "x", StorageDir: t.TempDir(), Stdout: &out} + for i := 0; i < 3; i++ { + out.Reset() + res, err := RunPageCheck(opts) + if err != nil { + t.Fatal(err) + } + if res.Failed != 1 || !strings.Contains(out.String(), "access denied: signed in as demo_user") || + strings.Contains(out.String(), "sign-in form") { + t.Errorf("run %d:\n%s", i+1, out.String()) + } + } + if len(fb.logins) != 1 || len(fb.live) != 1 || len(fb.signOuts) != 0 { + t.Errorf("after three checks: %d sign-in(s), %d live session(s), %d sign-out(s); want 1, 1 and 0", + len(fb.logins), len(fb.live), len(fb.signOuts)) + } +} + +// Checked alone, a refused page leaves a valid session: it is kept for the +// next check, not signed out as one that "got no page past the sign-in form". +func TestRunPageCheck_RoleDeniedPageAloneKeepsTheSession(t *testing.T) { + fb := &cookieJarBrowser{denied: "/p/secret"} + fb.install(t) + var out bytes.Buffer + opts := PageCheckOptions{BaseURL: "http://localhost:8180", Targets: []string{"/p/secret"}, + User: "demo_user", Password: "x", StorageDir: t.TempDir(), Stdout: &out} + for i := 0; i < 3; i++ { + res, err := RunPageCheck(opts) + if err != nil { + t.Fatal(err) + } + if res.Failed != 1 { + t.Fatalf("run %d: the refused page passed:\n%s", i+1, out.String()) + } + } + if len(fb.logins) != 1 || len(fb.signOuts) != 0 || len(fb.live) != 1 { + t.Errorf("after three checks: %d sign-in(s), %d sign-out(s), %d live; want 1, 0 and 1:\n%s", + len(fb.logins), len(fb.signOuts), len(fb.live), out.String()) + } +} + +// --role names the user role in the failure; the role comes from the demo +// user, and --user names no role it does not know. +func TestCredentialsNameTheUserRoles(t *testing.T) { + ps := &security.ProjectSecurity{SecurityLevel: "CheckEverything", EnableDemoUsers: true, + DemoUsers: []*security.DemoUser{{UserName: "demo_user", Password: "u", UserRoles: []string{"User"}}}} + c, err := pickDemoUser(ps, "User") + if err != nil { + t.Fatal(err) + } + if got := c.signedInAs(); got != "demo_user (user role User)" { + t.Errorf("--role User: %q", got) + } + if got := (credentials{user: "someone"}).signedInAs(); got != "someone" { + t.Errorf("--user: %q", got) + } +} + +// fakeRefusingApp serves the measured startup: /p/ redirects to /#/, the +// client asks POST /xas/ for its session, and on a 403 goes to +// /login.html?messageCode=403#/, on a 401 to /login.html#/. Both land on +// the same sign-in form. +func fakeRefusingApp(sessionStatus int) *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/p/", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/#/"+strings.TrimPrefix(r.URL.Path, "/p/"), http.StatusMovedPermanently) + }) + mux.HandleFunc("/xas/", func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(sessionStatus) + }) + mux.HandleFunc("/login.html", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Login +

Sign in

+
+`) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Mendix`) + }) + return httptest.NewServer(mux) +} + +// The real probe against the measured startup: a refused page must come back +// as access denied, and the anonymous one as the sign-in form. +func TestProbeTellsARefusedPageFromTheSignInForm(t *testing.T) { + denied := fakeRefusingApp(http.StatusForbidden) + defer denied.Close() + sig := probeOrSkip(t, ProbeOptions{Pages: []PageTarget{{URL: denied.URL + "/p/secret"}}, WaitMs: 3000}) + if !sig.LoginForm || !strings.Contains(sig.FinalURL, "messageCode=403") { + t.Fatalf("the probe did not record the 403 landing: loginForm=%v finalUrl=%q", sig.LoginForm, sig.FinalURL) + } + sig.SignedInAs = "demo_user (user role User)" + if f := EvaluatePage(sig, CheckSpec{}); len(f) != 1 || !strings.Contains(f[0], "access denied") { + t.Errorf("refused page: %q\n%s", f, formatPageVerdict("/p/secret", sig)) + } + + anon := fakeRefusingApp(http.StatusUnauthorized) + defer anon.Close() + sig = probeOrSkip(t, ProbeOptions{Pages: []PageTarget{{URL: anon.URL + "/p/secret"}}, WaitMs: 3000}) + if f := EvaluatePage(sig, CheckSpec{}); len(f) != 1 || !strings.Contains(f[0], "the sign-in form is showing") { + t.Errorf("control, anonymous: %q\n%s", f, formatPageVerdict("/p/secret", sig)) + } +} diff --git a/cmd/mxcli/docker/pagecheck_landed_test.go b/cmd/mxcli/docker/pagecheck_landed_test.go new file mode 100644 index 0000000000..b2f2029786 --- /dev/null +++ b/cmd/mxcli/docker/pagecheck_landed_test.go @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// ako/mxcli#998: `mxcli playwright check / /p/doesnotexist` reported the home +// page's verdict for a page that does not exist, and exited 0: +// +// page / title="Mendix - Homepage" h="Welcome to your homepage" text=1680 console-errors=0 +// page /p/doesnotexist title="Mendix - Homepage" h="Welcome to your homepage" text=1774 console-errors=0 +// OK 2 page(s) +// +// Measured on 11.13.0 and 11.15.0, both the same: the runtime answers any +// /p/ with `301 Location: /#/`, whether or not a page has that URL. +// The client then resolves the fragment. A known page gets its location +// replaced with /p/. An unknown one gets / plus an information dialog, +// "The page you requested was not found. You are redirected to the home +// page.", and the home page renders with HTTP 200 throughout. So the location +// changes, and the final path is the signal: it holds for the client's +// redirect and for any in-place render alike, because both leave a location +// that is not the /p/ path asked for. The dialog's text would also tell, but it +// is translated, so it is reported and not matched. +// URL matching is exact: /p/ITEMS and /p/items/ also went to / on 11.13, and +// the query string survives (/p/items?x=1 stays). + +// What the probe returned for the reported pair, with the location it ended on. +func unknownPageSignals() PageSignals { + return PageSignals{ + RequestedURL: "http://localhost:8180/p/doesnotexist", FinalURL: "http://localhost:8180/", + Title: "Mendix - Homepage", Headings: []string{"Welcome to your homepage"}, TextLen: 1774, Status: 200, + Alerts: []string{"The page you requested was not found. You are redirected to the home page."}, + } +} + +func TestEvaluatePage_UnknownPageURLFails(t *testing.T) { + fails := EvaluatePage(unknownPageSignals(), CheckSpec{}) + if len(fails) != 1 || !strings.Contains(fails[0], "ended on /") || !strings.Contains(fails[0], "/p/doesnotexist") { + t.Errorf("a /p/ URL that landed on the home page passed, or failed for something else: %v", fails) + } +} + +// Controls: a page that stayed where it was asked for passes — the known page, +// the root, and the root asked for without its slash. +func TestEvaluatePage_PageThatStayedPasses(t *testing.T) { + for _, c := range []struct{ req, fin string }{ + {"http://localhost:8180/p/items", "http://localhost:8180/p/items"}, + {"http://localhost:8180/p/items?x=1", "http://localhost:8180/p/items?x=1"}, + {"http://localhost:8180/", "http://localhost:8180/"}, + {"http://localhost:8180", "http://localhost:8180/"}, + // Not a /p/ URL: where the client takes a fragment route is its business. + {"http://localhost:8180/#/items", "http://localhost:8180/p/items"}, + // A proxy upgrading the scheme leaves the page the same. + {"http://app.example.com/p/items", "https://app.example.com/p/items"}, + } { + s := healthy() + s.RequestedURL, s.FinalURL = c.req, c.fin + if f := EvaluatePage(s, CheckSpec{}); len(f) != 0 { + t.Errorf("%s -> %s failed: %v", c.req, c.fin, f) + } + } +} + +// Anonymous, a /p/ URL lands on /login.html#/. That is the sign-in form's +// failure, said once, not also a wrong-page one. +func TestEvaluatePage_SignInRedirectIsOneReason(t *testing.T) { + s := PageSignals{RequestedURL: "http://localhost:8180/p/items", FinalURL: "http://localhost:8180/login.html#/items", + Title: "Login", TextLen: 34, LoginForm: true} + f := EvaluatePage(s, CheckSpec{}) + if len(f) != 1 || !strings.Contains(f[0], "sign-in form") { + t.Errorf("got %v, want only the sign-in failure", f) + } +} + +// The verdict says where the browser ended when it is not where it was sent, +// and stays as it was when it is. +func TestVerdictReportsTheURLTheBrowserEndedOn(t *testing.T) { + got := formatPageVerdict("/p/doesnotexist", unknownPageSignals()) + if !strings.Contains(got, "url=/ ") { + t.Errorf("the final URL is not reported:\n%s", got) + } + if !strings.Contains(got, "was not found") { + t.Errorf("the client's own not-found message is not reported:\n%s", got) + } + s := healthy() + s.RequestedURL, s.FinalURL = "http://localhost:8180", "http://localhost:8180/" + if got := formatPageVerdict("home", s); strings.Contains(got, "url=") { + t.Errorf("a page that stayed put reports a url=:\n%s", got) + } +} + +// The reported command end to end, with the probe answering as the measured +// client does. +func TestRunPageCheck_UnknownPageURLFailsTheCheck(t *testing.T) { + fb := &fakeBrowser{answer: func(_ int, p PageTarget) PageSignals { + s := PageSignals{Title: "Mendix - Homepage", Headings: []string{"Welcome to your homepage"}, + TextLen: 1680, Status: 200, RequestedURL: p.URL, FinalURL: "http://localhost:8180/"} + if strings.HasSuffix(p.URL, "/p/doesnotexist") { + s.TextLen = 1774 + } + return s + }} + fb.install(t) + var out bytes.Buffer + res, err := RunPageCheck(PageCheckOptions{BaseURL: "http://localhost:8180", + Targets: []string{"/", "/p/doesnotexist"}, Stdout: &out}) + if err != nil { + t.Fatal(err) + } + if res.Failed != 1 || !strings.Contains(out.String(), "FAIL 1 of 2 page(s)") { + t.Errorf("want the unknown page to fail and the home page to pass:\n%s", out.String()) + } +} + +// fakeRoutingApp serves what the runtime and the client measurably do with a +// /p/ URL: a 301 to /#/, then a client that replaces the location with +// /p/ for a page it knows and with / (plus the not-found dialog) for one it +// does not. +func fakeRoutingApp() *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/p/", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/#/"+strings.TrimPrefix(r.URL.Path, "/p/"), http.StatusMovedPermanently) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Mendix`) + }) + return httptest.NewServer(mux) +} + +// The real probe against the measured routing: it must report where the +// browser ended, and the check must fail the unknown page and pass the known. +func TestProbeReportsWhereAPageURLLanded(t *testing.T) { + srv := fakeRoutingApp() + defer srv.Close() + opts := ProbeOptions{Pages: []PageTarget{{URL: srv.URL + "/p/doesnotexist"}, {URL: srv.URL + "/p/items"}}, WaitMs: 3000} + opts.MxBuildPath = anyMxBuildPath() + sigs, err := ProbePages(opts) + if err != nil { + if strings.Contains(err.Error(), "node not found") || strings.Contains(err.Error(), "Cannot find module") || + strings.Contains(err.Error(), "Executable doesn't exist") { + t.Skipf("no browser tooling here (set NODE_PATH to a Playwright install): %v", err) + } + t.Fatal(err) + } + unknown, known := sigs[0], sigs[1] + if unknown.RequestedURL != srv.URL+"/p/doesnotexist" || unknown.FinalURL != srv.URL+"/" { + t.Errorf("unknown page: requested %q, ended %q", unknown.RequestedURL, unknown.FinalURL) + } + if f := EvaluatePage(unknown, CheckSpec{}); len(f) == 0 { + t.Errorf("the unknown /p/ URL passed:\n%s", formatPageVerdict("/p/doesnotexist", unknown)) + } + if f := EvaluatePage(known, CheckSpec{}); len(f) != 0 { + t.Errorf("the known page failed: %v\n%s", f, formatPageVerdict("/p/items", known)) + } +} diff --git a/cmd/mxcli/docker/pagecheck_origin_test.go b/cmd/mxcli/docker/pagecheck_origin_test.go new file mode 100644 index 0000000000..2980f286ff --- /dev/null +++ b/cmd/mxcli/docker/pagecheck_origin_test.go @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "bytes" + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "testing" +) + +// ako/mxcli#1216: `playwright check http://frontend.local:8180/ -p app.mpr +// --role R` failed every time with "the sign-in form is showing", while the +// same command without the URL passed. With -p the base URL comes from the +// running loop's handshake, http://localhost:, and the sign-in ran against +// that — so the session cookies belonged to localhost, and the browser sent none +// of them to frontend.local. Cookies are scoped to a host, not to an app. +// +// Reproduced on a real 11.13.0 app with the host spelled differently +// (`playwright check http://127.0.0.1:8180/` against a loop the handshake lists +// as localhost:8180): FAIL, sign-in form; `http://localhost:8180/` and no URL +// both OK. Every failing run also signed in again — one more session on the +// unlicensed runtime per run, counted with the M2EE runtime_statistics action. + +// cookieJarBrowser stands in for the login script, the probe and the sign-out, +// keeping the two properties this bug is about: a session saved by signing in +// on one host is not sent to another, and every sign-in is one more live +// session on the runtime until something signs it out. +type cookieJarBrowser struct { + logins []LoginOptions + probes []ProbeOptions + signOuts []SignOutOptions + live map[string]bool // the runtime's sessions, by cookie value + // formOn pages render the sign-in form even to a signed-in browser: a + // session that does not take, such as one whose cookie the browser will + // not send back. + formOn string + // denied pages are refused to a signed-in session, as the runtime does for + // a page the user's role may not open: the session request is answered + // 403 and the client lands on /login.html?messageCode=403#/, which + // is the sign-in form (measured on 11.13 and 11.15). The session stays + // live. + denied string + signOutErr error +} + +// fakeStorage is the shape of a Playwright storage state, reduced to cookies. +type fakeStorage struct { + Cookies []struct { + Value string `json:"value"` + Domain string `json:"domain"` + } `json:"cookies"` +} + +func hostOf(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "" + } + return u.Hostname() +} + +func readFakeStorage(path string) fakeStorage { + var st fakeStorage + if b, err := os.ReadFile(path); err == nil { + _ = json.Unmarshal(b, &st) + } + return st +} + +func (f *cookieJarBrowser) install(t *testing.T) { + t.Helper() + f.live = map[string]bool{} + oldProbe, oldLogin, oldSignOut := probePagesFn, loginFn, signOutFn + t.Cleanup(func() { probePagesFn, loginFn, signOutFn = oldProbe, oldLogin, oldSignOut }) + loginFn = func(o LoginOptions) error { + f.logins = append(f.logins, o) + // A new session on the runtime, its cookie set for the host the + // sign-in ran on. + id := fmt.Sprintf("s%d", len(f.logins)) + f.live[id] = true + body := `{"cookies":[{"name":"XASSESSIONID","value":"` + id + `","domain":"` + hostOf(o.AppURL) + `"}]}` + if err := os.MkdirAll(filepath.Dir(o.StoragePath), 0o755); err != nil { + return err + } + return os.WriteFile(o.StoragePath, []byte(body), 0o600) + } + probePagesFn = func(o ProbeOptions) ([]PageSignals, error) { + f.probes = append(f.probes, o) + st := readFakeStorage(o.Storage) + out := make([]PageSignals, len(o.Pages)) + for i, p := range o.Pages { + signedIn := false + for _, c := range st.Cookies { + if c.Domain == hostOf(p.URL) && f.live[c.Value] { + signedIn = true + } + } + switch { + case signedIn && f.denied != "" && strings.HasSuffix(p.URL, f.denied): + out[i] = deniedSignals(p.URL) + continue + case signedIn && (f.formOn == "" || !strings.HasSuffix(p.URL, f.formOn)): + out[i] = healthy() + default: + // What the reporter saw: the client's anonymous session attempt + // is refused, and the sign-in form renders. + out[i] = PageSignals{Title: "Login", Headings: []string{"Sign in"}, TextLen: 34, LoginForm: true, + ConsoleErrors: []string{"[Startup] Could not create a session because the server responded with 401 (Unauthorized)."}, + HTTPErrors: []string{"401 POST /xas/"}} + } + out[i].FinalURL = p.URL + } + return out, nil + } + signOutFn = func(o SignOutOptions) error { + f.signOuts = append(f.signOuts, o) + if f.signOutErr != nil { + return f.signOutErr + } + for _, c := range readFakeStorage(o.StoragePath).Cookies { + if c.Domain == hostOf(o.AppURL) && f.live[c.Value] { + delete(f.live, c.Value) + return nil + } + } + return ErrNoSession + } +} + +// The reported command: the page's host is not the base URL's host. +func TestRunPageCheck_SignsInOnTheHostOfThePage(t *testing.T) { + fb := &cookieJarBrowser{} + fb.install(t) + var out bytes.Buffer + res, err := RunPageCheck(PageCheckOptions{ + BaseURL: "http://localhost:8180", Targets: []string{"http://frontend.local:8180/"}, + User: "demo_enthusiast", Password: "x", StorageDir: t.TempDir(), Stdout: &out, + }) + if err != nil { + t.Fatal(err) + } + if res.Failed != 0 { + t.Fatalf("a page the check had just signed in for failed:\n%s", out.String()) + } + if len(fb.logins) != 1 || hostOf(fb.logins[0].AppURL) != "frontend.local" { + t.Errorf("signed in on %v, want once on frontend.local", fb.logins) + } +} + +// Control: the same check with the page on the base URL's host passes before +// and after the fix, so the test above fails for the host and nothing else. +func TestRunPageCheck_SameHostControl(t *testing.T) { + fb := &cookieJarBrowser{} + fb.install(t) + var out bytes.Buffer + res, err := RunPageCheck(PageCheckOptions{ + BaseURL: "http://localhost:8180", Targets: []string{"http://localhost:8180/"}, + User: "demo_enthusiast", Password: "x", StorageDir: t.TempDir(), Stdout: &out, + }) + if err != nil { + t.Fatal(err) + } + if res.Failed != 0 || len(fb.logins) != 1 { + t.Errorf("control failed (logins=%d):\n%s", len(fb.logins), out.String()) + } +} + +// Pages on two hosts get a session each, every page passes, and each session +// is saved under its own host so a later check reuses the right one. +func TestRunPageCheck_PagesOnTwoHosts(t *testing.T) { + fb := &cookieJarBrowser{} + fb.install(t) + dir := t.TempDir() + var out bytes.Buffer + opts := PageCheckOptions{ + BaseURL: "http://localhost:8180", Targets: []string{"/p/items", "http://127.0.0.1:8180/p/items", "/"}, + User: "demo_user", Password: "x", StorageDir: dir, Stdout: &out, + } + res, err := RunPageCheck(opts) + if err != nil { + t.Fatal(err) + } + if res.Pages != 3 || res.Failed != 0 { + t.Fatalf("got %+v:\n%s", res, out.String()) + } + if len(fb.logins) != 2 { + t.Errorf("logins = %d, want one per host", len(fb.logins)) + } + // The verdicts keep the order the pages were asked for in. + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + var pageLines []string + for _, l := range lines { + if strings.HasPrefix(l, "page ") { + pageLines = append(pageLines, strings.Fields(l)[1]) + } + } + if strings.Join(pageLines, " ") != "/p/items http://127.0.0.1:8180/p/items /" { + t.Errorf("verdict order = %v", pageLines) + } + // A second check signs in nowhere: both sessions are reused. + if _, err := RunPageCheck(opts); err != nil { + t.Fatal(err) + } + if len(fb.logins) != 2 { + t.Errorf("second check signed in again (logins=%d); the sessions were not saved per host", len(fb.logins)) + } +} + +// The login failure explanation reads the runtime log; `playwright check` +// never told the login where that is, so the session-cap explanation could not +// fire from the command most likely to exhaust the cap. +func TestRunPageCheck_GivesTheLoginTheRuntimeLog(t *testing.T) { + fb := &cookieJarBrowser{} + fb.install(t) + proj := t.TempDir() + logPath := filepath.Join(proj, ".mxcli", "runtime.log") + if err := os.MkdirAll(filepath.Dir(logPath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(logPath, []byte("ERROR - Core: Maximum number of sessions exceeded! (You are currently using a trial license)\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := RunPageCheck(PageCheckOptions{ + BaseURL: "http://localhost:8180", ProjectPath: filepath.Join(proj, "app.mpr"), + User: "demo_user", Password: "x", Stdout: &bytes.Buffer{}, + }); err != nil { + t.Fatal(err) + } + if len(fb.logins) != 1 { + t.Fatalf("logins = %d", len(fb.logins)) + } + if got := fb.logins[0].RuntimeLogPath; got != logPath { + t.Errorf("login RuntimeLogPath = %q, want %q", got, logPath) + } + if loginFailureHint(fb.logins[0].RuntimeLogPath) == "" { + t.Error("with that path the session-cap hint still does not fire") + } +} + +// The reported runs, counted on the runtime's side: six checks of the page on +// the other host. Each one used to sign in again beside the last, which is how +// the command came to exhaust the unlicensed runtime's session cap. Measured on +// 11.13 with the M2EE runtime_statistics action: three failing runs, three +// sessions for demo_user. Reused, it is one session however many checks run. +func TestRunPageCheck_RepeatedChecksHoldOneSession(t *testing.T) { + fb := &cookieJarBrowser{} + fb.install(t) + opts := PageCheckOptions{ + BaseURL: "http://localhost:8180", Targets: []string{"http://frontend.local:8180/"}, + User: "demo_enthusiast", Password: "x", StorageDir: t.TempDir(), Stdout: &bytes.Buffer{}, + } + failed := 0 + for i := 0; i < 6; i++ { + res, err := RunPageCheck(opts) + if err != nil { + t.Fatal(err) + } + failed += res.Failed + } + if failed != 0 || len(fb.live) != 1 || len(fb.logins) != 1 { + t.Errorf("after six checks: %d failed, %d live session(s), %d sign-in(s); want 0, 1 and 1", + failed, len(fb.live), len(fb.logins)) + } +} + +// --fresh-login replaces the saved session; the one it replaces is signed out +// rather than left to hold a slot until it expires. +func TestRunPageCheck_FreshLoginSignsOutTheSessionItReplaces(t *testing.T) { + fb := &cookieJarBrowser{} + fb.install(t) + opts := PageCheckOptions{BaseURL: "http://localhost:8180", User: "demo_user", Password: "x", + StorageDir: t.TempDir(), Stdout: &bytes.Buffer{}} + if _, err := RunPageCheck(opts); err != nil { + t.Fatal(err) + } + opts.FreshLogin = true + if _, err := RunPageCheck(opts); err != nil { + t.Fatal(err) + } + if len(fb.logins) != 2 || len(fb.live) != 1 { + t.Errorf("logins=%d live=%d, want 2 sign-ins and 1 live session", len(fb.logins), len(fb.live)) + } +} + +// After a runtime restart the saved session is already gone: --fresh-login +// finds nothing to sign out and says nothing about it. +func TestRunPageCheck_FreshLoginAfterARestart(t *testing.T) { + fb := &cookieJarBrowser{} + fb.install(t) + var out, errOut bytes.Buffer + opts := PageCheckOptions{BaseURL: "http://localhost:8180", User: "demo_user", Password: "x", + StorageDir: t.TempDir(), Stdout: &out, Stderr: &errOut} + if _, err := RunPageCheck(opts); err != nil { + t.Fatal(err) + } + fb.live = map[string]bool{} // the runtime restarted + out.Reset() + opts.FreshLogin = true + res, err := RunPageCheck(opts) + if err != nil || res.Failed != 0 { + t.Fatalf("%+v %v\n%s", res, err, out.String()) + } + if strings.Contains(out.String(), "logout") || errOut.Len() != 0 { + t.Errorf("a session that was already gone was reported signed out:\n%s%s", out.String(), errOut.String()) + } +} + +// --sign-out ends the run signed out: no live session and nothing saved, and +// the verdict is the same as without it. +func TestRunPageCheck_SignOutEndsTheRunSignedOut(t *testing.T) { + fb := &cookieJarBrowser{} + fb.install(t) + var out bytes.Buffer + opts := PageCheckOptions{BaseURL: "http://localhost:8180", Targets: []string{"/p/items"}, + User: "demo_user", Password: "x", StorageDir: t.TempDir(), SignOut: true, Stdout: &out} + res, err := RunPageCheck(opts) + if err != nil { + t.Fatal(err) + } + if res.Failed != 0 { + t.Errorf("signing out changed the verdict:\n%s", out.String()) + } + if len(fb.live) != 0 { + t.Errorf("%d session(s) still live after --sign-out", len(fb.live)) + } + if _, err := os.Stat(checkStoragePath(opts, "demo_user")); err == nil { + t.Error("a signed-out session was kept for the next check") + } + if !strings.Contains(out.String(), "logout demo_user") { + t.Errorf("the sign-out is not reported:\n%s", out.String()) + } +} + +// A sign-out that fails is reported and changes nothing else: the check +// itself succeeded. +func TestRunPageCheck_FailedSignOutKeepsTheVerdict(t *testing.T) { + fb := &cookieJarBrowser{signOutErr: fmt.Errorf("no signed-in Mendix client")} + fb.install(t) + var out, errOut bytes.Buffer + res, err := RunPageCheck(PageCheckOptions{BaseURL: "http://localhost:8180", User: "demo_user", Password: "x", + StorageDir: t.TempDir(), SignOut: true, Stdout: &out, Stderr: &errOut}) + if err != nil || res.Failed != 0 { + t.Errorf("a failed sign-out changed the result: %+v %v\n%s", res, err, out.String()) + } + if !strings.Contains(errOut.String(), "no signed-in Mendix client") { + t.Errorf("the failed sign-out is not reported: %q", errOut.String()) + } +} + +// A session this run signed in that got no page past the sign-in form — one +// that did not take — is signed out and not kept. Kept, every later check +// re-signed-in beside it, one more session per run. (A page the role may not +// open is not this case: see pagecheck_denied_test.go.) +func TestRunPageCheck_SignsOutASessionThatGotNoPage(t *testing.T) { + fb := &cookieJarBrowser{formOn: "/p/secret"} + fb.install(t) + opts := PageCheckOptions{BaseURL: "http://localhost:8180", Targets: []string{"/p/secret"}, + User: "demo_user", Password: "x", StorageDir: t.TempDir(), Stdout: &bytes.Buffer{}} + for i := 0; i < 3; i++ { + res, err := RunPageCheck(opts) + if err != nil { + t.Fatal(err) + } + if res.Failed != 1 { + t.Fatalf("run %d: the page that showed the sign-in form passed", i+1) + } + } + if len(fb.live) != 0 { + t.Errorf("%d session(s) left live by three checks that never got past the sign-in form", len(fb.live)) + } + // Control: a page the session does get keeps its session for reuse. + fb.formOn = "" + opts.Targets = []string{"/p/items"} + for i := 0; i < 2; i++ { + if _, err := RunPageCheck(opts); err != nil { + t.Fatal(err) + } + } + if len(fb.live) != 1 || len(fb.signOuts) != 3 { + t.Errorf("control: live=%d signOuts=%d, want the session kept (1) and no further sign-out (3)", len(fb.live), len(fb.signOuts)) + } +} diff --git a/cmd/mxcli/docker/pagecheck_signin_wait_test.go b/cmd/mxcli/docker/pagecheck_signin_wait_test.go new file mode 100644 index 0000000000..eed6961faf --- /dev/null +++ b/cmd/mxcli/docker/pagecheck_signin_wait_test.go @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// `playwright check` on a page that shows the sign-in form waited out the whole +// --wait before answering, then reported the page as still loading on top of +// the sign-in form. Measured on 11.13.0 against a secured page with no session: +// +// WAIT not settled after 30.2s: still waiting for POST /xas/ +// FAIL still loading when measured, … (raise --wait if it is only slow) +// FAIL the sign-in form is showing, not the page (…) +// +// at --wait 30s (33s wall time; 8s at --wait 5s). The client keeps a POST /xas/ +// open behind the sign-in form, so the probe's "no request open" condition never +// held. The sign-in form is a verdict in itself, and raising --wait is the +// wrong advice for it. + +// fakeSignInPage serves the sign-in form the client shows for a page that needs +// a session, with a POST /xas/ that does not answer while the form is up. +func fakeSignInPage() *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/p/secret", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `Mendix +

Sign in

+
+ +`) + }) + mux.HandleFunc("/xas/", func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + select { + case <-time.After(60 * time.Second): + case <-r.Context().Done(): + } + }) + return httptest.NewServer(mux) +} + +func TestProbeDoesNotWaitOutTheWaitOnTheSignInForm(t *testing.T) { + srv := fakeSignInPage() + defer srv.Close() + const waitMs = 20000 + start := time.Now() + sig := probeOrSkip(t, ProbeOptions{Pages: []PageTarget{{URL: srv.URL + "/p/secret"}}, WaitMs: waitMs}) + took := time.Since(start) + + if !sig.LoginForm { + t.Fatalf("the sign-in form was not recognised: %+v", sig) + } + if sig.Unsettled || sig.WaitedMs >= waitMs { + t.Errorf("the probe waited out --wait on the sign-in form (waited %dms of %dms, unsettled=%v)", + sig.WaitedMs, waitMs, sig.Unsettled) + } + // Browser start-up is most of the remaining time; a wait served in full + // would put this above 20s. + if took > 12*time.Second { + t.Errorf("checking a sign-in form took %s with --wait %dms", took.Round(time.Second), waitMs) + } +} + +// The verdict for the sign-in form names the sign-in form, not loading: even a +// form measured while unsettled (a client still busy behind it) is not cured by +// a longer wait. +func TestEvaluatePage_SignInFormIsNotStillLoading(t *testing.T) { + sig := healthy() + sig.LoginForm, sig.Unsettled, sig.PendingRequests = true, true, []string{"POST /xas/"} + fails := EvaluatePage(sig, CheckSpec{}) + if len(fails) != 1 || !strings.Contains(fails[0], "sign-in form") { + t.Errorf("want the sign-in form as the one failure, got %q", fails) + } + // Control: an unsettled page that is not the sign-in form still fails as + // still loading. + sig.LoginForm = false + if fails := EvaluatePage(sig, CheckSpec{}); len(fails) == 0 || !strings.Contains(fails[0], "still loading") { + t.Errorf("control: an unsettled page no longer fails as still loading: %q", fails) + } +} diff --git a/cmd/mxcli/docker/runlocal.go b/cmd/mxcli/docker/runlocal.go index 6738483eb3..dcec942119 100644 --- a/cmd/mxcli/docker/runlocal.go +++ b/cmd/mxcli/docker/runlocal.go @@ -1075,18 +1075,31 @@ func RunLocal(opts LocalRunOptions) error { // still answers" the only evidence available. A non-zero exit is what any // supervisor — a shell loop, systemd, a container restart policy — already knows // how to act on. +// +// The reason is read from the runtime log FILE as well as from the JVM's own +// output: the application log — where the licence's "Maximum run time +// exceeded" is written — reaches only the file subscriber, never stdout +// (ako/mxcli#1542). It travels as RuntimeStoppedError.Reason so the run state +// can record it for `run status`. func runtimeStoppedError(rt *LocalRuntime) error { + fileLog := currentRuntimeLog(rt.opts.RuntimeLogPath) + reason := stopReason(fileLog, filepath.Base(rt.opts.RuntimeLogPath)) + if reason == "" { + reason = stopReason(rt.Log(), "runtime output") + } detail := "" - if reason := runtimeExitReason(rt.Log()); reason != "" { + if long := runtimeExitReason(rt.Log() + "\n" + fileLog); long != "" { + detail = "\n " + long + } else if reason != "" { detail = "\n " + reason } if err := rt.ExitErr(); err != nil { detail += "\n the runtime process exited with: " + err.Error() } - return fmt.Errorf("the app stopped: the Mendix runtime is no longer running%s\n"+ + return &RuntimeStoppedError{Reason: reason, msg: fmt.Sprintf("the app stopped: the Mendix runtime is no longer running%s\n"+ " mxcli is exiting rather than staying up over a dead app — a tunnel or reverse proxy\n"+ " keeps answering after the runtime is gone, so \"the URL responds\" is not evidence\n"+ - " the app is alive. Re-run to restart it.", detail) + " the app is alive. Restart it: mxcli run restart -p , or re-run.", detail)} } // maybeScreenshot captures the app (best-effort) when --screenshot is set. A diff --git a/cmd/mxcli/docker/runstate.go b/cmd/mxcli/docker/runstate.go index ac439cc2b0..e19d058329 100644 --- a/cmd/mxcli/docker/runstate.go +++ b/cmd/mxcli/docker/runstate.go @@ -4,6 +4,7 @@ package docker import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -102,6 +103,12 @@ type RunState struct { Ended time.Time `json:"ended,omitempty"` // Error is why the run failed or stopped on its own. Error string `json:"error,omitempty"` + // Reason is, for an app whose runtime stopped on its own, why — one line, + // from the runtime's log when it says ("trial licence time limit reached at + // 18:43:16 (runtime.log)"). It is written in the same update that sets + // phase stopped and ended, just before the run withdraws run-local.json, so + // a reader of either file sees one state (ako/mxcli#1542). + Reason string `json:"reason,omitempty"` // Gen is the newest build generation started; Building says whether it is // still in progress, and BuildingSource is the source mtime it is building. Gen int `json:"gen"` @@ -259,7 +266,7 @@ func (r *RunStateRecorder) Ready(url string, boot BuildOutcome) { // Exit records how the run ended: nil is a requested stop; an error before // readiness is a failed boot; an error after it is an app that stopped on its -// own. +// own — and when that is because the runtime stopped, why. func (r *RunStateRecorder) Exit(err error) { r.update(func(s *RunState) { s.Ended = r.now() @@ -267,6 +274,13 @@ func (r *RunStateRecorder) Exit(err error) { if err != nil { s.Error = err.Error() } + var stopped *RuntimeStoppedError + if errors.As(err, &stopped) { + s.Reason = stopped.Reason + if s.Reason == "" { + s.Reason = "the Mendix runtime stopped on its own (its log does not say why)" + } + } if s.Phase == PhaseStarting && err != nil { s.Phase = PhaseFailed return diff --git a/cmd/mxcli/docker/runtime_stop_reason.go b/cmd/mxcli/docker/runtime_stop_reason.go new file mode 100644 index 0000000000..cd85b91aa4 --- /dev/null +++ b/cmd/mxcli/docker/runtime_stop_reason.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "io" + "os" + "path/filepath" + "regexp" + "strings" +) + +// runtime_stop_reason.go says, in one line, why a local runtime stopped — for +// `run status` and run-state.json (ako/mxcli#1542). +// +// The reason is in the runtime's application log, which only exists because +// mxcli attaches a file log subscriber writing .mxcli/runtime.log: a standalone +// runtime has no subscriber of its own, so "Maximum run time exceeded" never +// reaches the JVM's stdout, which is all LocalRuntime.Log() holds. Reading the +// in-memory output alone found the licence in a unit test and never in a run. + +// RuntimeStoppedError ends a run whose Mendix runtime stopped without being +// asked to: the trial licence's time limit, a crash, a kill. +type RuntimeStoppedError struct { + // Reason is one line for `run status` and run-state.json, such as "trial + // licence time limit reached at 18:43:16 (runtime.log)"; "" when neither + // log says why. + Reason string + msg string +} + +func (e *RuntimeStoppedError) Error() string { return e.msg } + +// runtimeStartMarker begins each boot's section of the runtime log (see +// openRuntimeLog). +const runtimeStartMarker = "=== runtime start " + +// runtimeLogTailBytes bounds how much of the runtime log is read: a reason is +// in the last lines, and the file grows for as long as the project is run. +const runtimeLogTailBytes = 256 << 10 + +// currentRuntimeLog returns the runtime log written since the newest boot +// began, or "" when it cannot be read. The file is appended across restarts, +// and an earlier boot's licence stop must not be blamed for this one. +func currentRuntimeLog(path string) string { + if path == "" { + return "" + } + f, err := os.Open(path) + if err != nil { + return "" + } + defer f.Close() + if info, err := f.Stat(); err == nil && info.Size() > runtimeLogTailBytes { + if _, err := f.Seek(info.Size()-runtimeLogTailBytes, io.SeekStart); err != nil { + return "" + } + } + body, err := io.ReadAll(f) + if err != nil { + return "" + } + text := string(body) + if i := strings.LastIndex(text, runtimeStartMarker); i >= 0 { + text = text[i:] + } + return text +} + +// RuntimeLogStopReason is stopReason over the current boot of the runtime log +// at path — for a run that could not record its own end (it was killed), whose +// log may still say why the app went. +func RuntimeLogStopReason(path string) string { + return stopReason(currentRuntimeLog(path), filepath.Base(path)) +} + +var ( + // logClockRe finds the time of day on a runtime log line. + logClockRe = regexp.MustCompile(`(?:^|[^0-9T:])(\d{2}:\d{2}:\d{2})(?:[.,]\d+)?\b`) + // logErrorRe is a Mendix log line at ERROR or CRITICAL: "LEVEL - Node: message". + logErrorRe = regexp.MustCompile(`\b(?:ERROR|CRITICAL)\s+-\s+(.+)$`) +) + +// licenceStopMarkers are what the runtime logs when the trial licence ends the +// run. The boot warning ("...terminated when the maximum time is exceeded!") +// is deliberately not one: it is printed at every start. +var licenceStopMarkers = []string{"Maximum run time exceeded", "checkRuntimeDuration"} + +// stopReason picks the cause of a runtime stop out of log text, naming source +// as where it was read; "" when nothing in it says. The known causes come +// first; otherwise the last ERROR line is reported as exactly that, not as a +// cause — inventing one is worse than naming none. +func stopReason(text, source string) string { + if strings.TrimSpace(text) == "" { + return "" + } + lines := strings.Split(strings.ReplaceAll(text, "\r\n", "\n"), "\n") + for i := len(lines) - 1; i >= 0; i-- { + for _, m := range licenceStopMarkers { + if strings.Contains(lines[i], m) { + return "trial licence time limit reached" + clockNear(lines, i) + " (" + source + ")" + } + } + } + for i := len(lines) - 1; i >= 0; i-- { + if strings.Contains(lines[i], "java.lang.OutOfMemoryError") { + return "the runtime JVM ran out of memory" + clockNear(lines, i) + " (" + source + ")" + } + } + for i := len(lines) - 1; i >= 0; i-- { + if m := logErrorRe.FindStringSubmatch(lines[i]); m != nil { + msg := strings.TrimSpace(m[1]) + if len(msg) > 160 { + msg = msg[:160] + "…" + } + return "last runtime error" + clockNear(lines, i) + ": " + msg + " (" + source + ")" + } + } + return "" +} + +// clockNear is " at HH:MM:SS" from line i, or from the few lines above it — a +// stack frame carries no time, the line that logged it does — or "". +func clockNear(lines []string, i int) string { + for j := i; j >= 0 && j > i-6; j-- { + if strings.HasPrefix(lines[j], runtimeStartMarker) { + break + } + if m := logClockRe.FindStringSubmatch(lines[j]); m != nil { + return " at " + m[1] + } + } + return "" +} diff --git a/cmd/mxcli/docker/runtime_stop_reason_test.go b/cmd/mxcli/docker/runtime_stop_reason_test.go new file mode 100644 index 0000000000..6aaff457c0 --- /dev/null +++ b/cmd/mxcli/docker/runtime_stop_reason_test.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// ako/mxcli#1542: "One of the two times, the runtime had stopped itself on the +// trial licence's time limit (`LicenseUtil.checkRuntimeDuration`)" — and the +// callers got no reason. +// +// The licence message is an APPLICATION log line. A standalone runtime has no +// log subscriber until mxcli attaches the file subscriber that writes +// .mxcli/runtime.log, so the line lands in that file and never on the JVM's +// stdout — which is all LocalRuntime.Log() holds. Reading the reason from +// Log() alone (§60) worked in its unit test, on a string, and never in a run. + +// licenceStopLog is a runtime.log across two boots: the first stopped on the +// licence, the second (the run in question) too, hours later. +const licenceStopLog = "\n=== runtime start 2026-10-26T10:00:00Z ===\n" + + "Picked up JAVA_TOOL_OPTIONS\n" + + "2026-10-26 10:00:20.101 WARNING - LicenseService: The runtime has been started using a trial licence,\n" + + " the framework will be terminated when the maximum time is exceeded!\n" + + "\n=== runtime start 2026-10-26T14:51:02Z ===\n" + + "Picked up JAVA_TOOL_OPTIONS\n" + + "2026-10-26 14:51:20.101 WARNING - LicenseService: The runtime has been started using a trial licence,\n" + + " the framework will be terminated when the maximum time is exceeded!\n" + + "2026-10-26 16:02:11.007 INFO - Core: an ordinary line\n" + + "2026-10-26 18:43:16.412 CRITICAL - LicenseService: Maximum run time exceeded, framework is now terminating\n" + + "\tat com.mendix.basis.license.LicenseUtil.checkRuntimeDuration(LicenseUtil.java:212)\n" + + "2026-10-26 18:43:16.530 INFO - Core: Mendix Runtime is shutting down\n" + +func writeRuntimeLog(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), ".mxcli", "runtime.log") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestRuntimeStoppedError_ReadsTheReasonFromTheRuntimeLog(t *testing.T) { + logPath := writeRuntimeLog(t, licenceStopLog) + // What the JVM itself printed: the banner, and nothing the app logged. + mem := &syncBuffer{} + _, _ = mem.Write([]byte("Picked up JAVA_TOOL_OPTIONS\nMendix runtime launcher started\n")) + rt := &LocalRuntime{log: mem, opts: LocalRuntimeOptions{RuntimeLogPath: logPath}} + + err := runtimeStoppedError(rt) + msg := err.Error() + if !strings.Contains(strings.ToLower(msg), "licence") { + t.Fatalf("the runtime log says the trial licence ended the run, but the error does not:\n%s", msg) + } + stopped, ok := err.(*RuntimeStoppedError) + if !ok { + t.Fatalf("err = %T, want *RuntimeStoppedError", err) + } + if want := "trial licence time limit reached at 18:43:16 (runtime.log)"; stopped.Reason != want { + t.Errorf("Reason = %q, want %q", stopped.Reason, want) + } +} + +func TestRuntimeLogStopReason(t *testing.T) { + cases := []struct{ name, log, want string }{ + {"licence, timed from the line above the stack frame", licenceStopLog, + "trial licence time limit reached at 18:43:16 (runtime.log)"}, + // Only the newest boot counts: the file is appended across restarts. + {"an earlier boot's licence stop is not this one's", + licenceStopLog + "\n=== runtime start 2026-10-26T19:00:00Z ===\n2026-10-26 19:00:05.000 INFO - Core: started\n", ""}, + // The boot warning is printed at every start; it is not a stop. + {"boot warning is not a stop", + "\n=== runtime start 2026-10-26T10:00:00Z ===\nWARNING - LicenseService: the framework will be terminated when the maximum time is exceeded!\n", ""}, + {"out of memory", + "\n=== runtime start x ===\n2026-10-26 12:00:01.000 ERROR - Core: boom\nException in thread \"main\" java.lang.OutOfMemoryError: Java heap space\n", + "the runtime JVM ran out of memory at 12:00:01 (runtime.log)"}, + {"last error, reported as such", + "\n=== runtime start x ===\n2026-10-26 12:00:01.000 ERROR - Connector: first\n2026-10-26 12:30:02.500 CRITICAL - Core: Could not connect to the database\n2026-10-26 12:30:03.000 INFO - Core: bye\n", + "last runtime error at 12:30:02: Core: Could not connect to the database (runtime.log)"}, + {"nothing to say", "\n=== runtime start x ===\n2026-10-26 12:00:01.000 INFO - Core: fine\n", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := RuntimeLogStopReason(writeRuntimeLog(t, tc.log)); got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + }) + } + if got := RuntimeLogStopReason(filepath.Join(t.TempDir(), "missing.log")); got != "" { + t.Errorf("a missing log must give no reason, got %q", got) + } +} + +// Expected behaviour 2, verbatim: "`run-state.json` records it (`phase: +// stopped`, `ended` set)". The reason rides on the error the run returns. +func TestRunStateRecorder_RuntimeStopRecordsTheReason(t *testing.T) { + r, published := newTestRecorder(t) + r.Ready("http://127.0.0.1:8080/", BuildOutcome{Gen: 1, OK: true, Action: "boot"}) + r.Exit(&RuntimeStoppedError{Reason: "trial licence time limit reached at 18:43:16 (runtime.log)", msg: "the app stopped"}) + last := (*published)[len(*published)-1] + if last.Phase != PhaseStopped || last.Ended.IsZero() { + t.Fatalf("published %+v, want phase stopped with ended set", last) + } + if last.Reason != "trial licence time limit reached at 18:43:16 (runtime.log)" { + t.Errorf("reason = %q", last.Reason) + } + + // A stop whose logs say nothing still says the runtime stopped by itself. + r2, published2 := newTestRecorder(t) + r2.Ready("http://127.0.0.1:8080/", BuildOutcome{Gen: 1, OK: true, Action: "boot"}) + r2.Exit(&RuntimeStoppedError{msg: "the app stopped"}) + if got := (*published2)[len(*published2)-1].Reason; !strings.Contains(got, "stopped on its own") { + t.Errorf("reason = %q", got) + } + + // CONTROL: a requested stop has no reason to give. + r3, published3 := newTestRecorder(t) + r3.Ready("http://127.0.0.1:8080/", BuildOutcome{Gen: 1, OK: true, Action: "boot"}) + r3.Exit(nil) + if got := (*published3)[len(*published3)-1].Reason; got != "" { + t.Errorf("a requested stop recorded reason %q", got) + } +} diff --git a/cmd/mxcli/docker/screenshot_login.go b/cmd/mxcli/docker/screenshot_login.go index 2b503d5b39..745368f997 100644 --- a/cmd/mxcli/docker/screenshot_login.go +++ b/cmd/mxcli/docker/screenshot_login.go @@ -3,6 +3,7 @@ package docker import ( + "errors" "fmt" "io" "os" @@ -249,3 +250,93 @@ func LoginAndSaveStorage(opts LoginOptions) error { } return nil } + +// signOutScript signs a saved session out of the app the way the client's own +// sign-out does: mx.logout() posts {"action":"logout"} to /xas/ with the +// client's CSRF token. Measured on 11.13: the runtime's session count +// (M2EE runtime_statistics, user_sessions) drops by one. Args: appURL, +// storagePath. Exit 3 when there is no session left to sign out: a session +// the runtime has forgotten is answered 401 and sent to /login.html, which +// has no mx (measured on 11.15 after a restart, ~1.6s). +const signOutScript = ` +const [appURL, storagePath] = process.argv.slice(2); +const pw = (() => { try { return require("playwright"); } catch (e) { return require("playwright-core"); } })(); +const { chromium } = pw; +(async () => { + const exe = process.env.MXCLI_CHROMIUM_EXE; + const b = await chromium.launch(exe ? { executablePath: exe } : {}); + const ctx = await b.newContext({ storageState: storagePath }); + const p = await ctx.newPage(); + await p.goto(appURL, { waitUntil: "domcontentloaded", timeout: 30000 }); + await p.waitForSelector(".mx-page, #usernameInput", { timeout: 20000 }).catch(() => {}); + const client = await p.evaluate(() => !!(window.mx && typeof mx.logout === "function")).catch(() => false); + if (!client) { await b.close(); process.exit(3); } + const done = p.waitForResponse(r => r.request().method() === "POST" && + (r.request().postData() || "").includes('"logout"'), { timeout: 15000 }); + await p.evaluate(() => { mx.logout(); }); + const r = await done; + await b.close(); + if (!r.ok()) { process.stderr.write("logout answered HTTP " + r.status() + "\n"); process.exit(2); } +})().catch((e) => { process.stderr.write(String(e).split("\n")[0] + "\n"); process.exit(1); }); +` + +// SignOutOptions configures SignOut. +type SignOutOptions struct { + AppURL string + StoragePath string // the Playwright storage state holding the session + MxBuildPath string // fallback node source + Timeout time.Duration +} + +// ErrNoSession is SignOut's answer when the saved session is already gone — +// the runtime restarted or expired it — so there was nothing to sign out. +var ErrNoSession = errors.New("no live session to sign out") + +// SignOut ends the session saved in opts.StoragePath, so it stops holding one +// of the unlicensed runtime's few session slots. It does not remove the file. +func SignOut(opts SignOutOptions) error { + node := resolveNodeForScript(opts.MxBuildPath) + if node == "" { + return fmt.Errorf("node not found to run the sign-out script") + } + dir, err := os.MkdirTemp("", "mxcli-signout-") + if err != nil { + return err + } + defer os.RemoveAll(dir) + scriptPath := filepath.Join(dir, "signout.cjs") + if err := os.WriteFile(scriptPath, []byte(signOutScript), 0o600); err != nil { + return err + } + timeout := opts.Timeout + if timeout == 0 { + timeout = 60 * time.Second + } + cmd := exec.Command(node, scriptPath, opts.AppURL, opts.StoragePath) + cmd.Env = append(os.Environ(), "NODE_PATH="+playwrightNodePath(node)) + if exe := headlessShellPath(); exe != "" { + cmd.Env = append(cmd.Env, "MXCLI_CHROMIUM_EXE="+exe) + } + out := &syncBuffer{} + cmd.Stdout, cmd.Stderr = out, out + done := make(chan error, 1) + if err := cmd.Start(); err != nil { + return fmt.Errorf("launching the sign-out script: %w", err) + } + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 3 { + return ErrNoSession + } + if err != nil { + return fmt.Errorf("%w: %s", err, clip(out.String(), 200)) + } + case <-time.After(timeout): + _ = cmd.Process.Kill() + <-done + return fmt.Errorf("timed out after %s", timeout) + } + return nil +} diff --git a/cmd/mxcli/docker/settle.go b/cmd/mxcli/docker/settle.go index 82498c055e..b9838a5137 100644 --- a/cmd/mxcli/docker/settle.go +++ b/cmd/mxcli/docker/settle.go @@ -41,14 +41,7 @@ func SettleGeneratedSources(projectPath, mxPath, version string, w io.Writer) er return fmt.Errorf("no JDK %d available: %w", javaMajor, err) } - cmd := exec.Command(mxbuildPath, - "--target=deploy", - fmt.Sprintf("--java-home=%s", javaHome), - fmt.Sprintf("--java-exe-path=%s", JavaExePath(javaHome)), - projectPath, - ) - cmd.Dir = filepath.Dir(projectPath) - PrepareMxCommand(cmd) // FreeType LD_PRELOAD workaround + cmd := settleCommand(mxbuildPath, javaHome, projectPath) // MxBuild is very chatty and this build is incidental to what the user asked // for, so its output is only worth showing when it fails. @@ -66,6 +59,21 @@ func SettleGeneratedSources(projectPath, mxPath, version string, w io.Writer) er return nil } +// settleCommand builds the one-shot deploy build SettleGeneratedSources runs +// (`mxcli new` step 6) — without a Gradle daemon, which hung this step the same +// way it hung the dependency sync (ako/mxcli#1340). +func settleCommand(mxbuildPath, javaHome, projectPath string) *exec.Cmd { + cmd := exec.Command(mxbuildPath, + "--target=deploy", + fmt.Sprintf("--java-home=%s", javaHome), + fmt.Sprintf("--java-exe-path=%s", JavaExePath(javaHome)), + projectPath, + ) + cmd.Dir = filepath.Dir(projectPath) + prepareOneShotMxCommand(cmd) // FreeType LD_PRELOAD workaround + no Gradle daemon + return cmd +} + // ModelErrorsError is a build mxbuild refused because the MODEL has errors — // as opposed to one that could not run (no JDK, a killed JVM). The difference // matters to `mxcli new`: the first means the project it just created does not diff --git a/cmd/mxcli/docker_oql.go b/cmd/mxcli/docker_oql.go index 799b410247..230c55181d 100644 --- a/cmd/mxcli/docker_oql.go +++ b/cmd/mxcli/docker_oql.go @@ -47,9 +47,9 @@ Examples: // A `mxcli run --local` serving this project records its admin port and // password; use them unless the flags say otherwise (ako/mxcli#982). + set := adminFlagsSet{host: host != "", port: port != 0, token: token != ""} admin := devLoopAdminOptions(projectPath, - docker.M2EEOptions{Host: host, Port: port, Token: token, ProjectPath: projectPath, Direct: direct}, - adminFlagsSet{host: host != "", port: port != 0, token: token != ""}) + docker.M2EEOptions{Host: host, Port: port, Token: token, ProjectPath: projectPath, Direct: direct}, set) opts := docker.OQLOptions{ Host: admin.Host, @@ -63,7 +63,8 @@ Examples: result, err := docker.ExecuteOQL(opts, query) if err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) + // A 403 with no run-local.json is a stopped run, not a bad password. + fmt.Fprintf(os.Stderr, "Error: %v\n%s", err, devLoopAdminHint(projectPath, set, err)) os.Exit(1) } diff --git a/cmd/mxcli/init_hook.go b/cmd/mxcli/init_hook.go index d11dfbe1de..ad9ac03a17 100644 --- a/cmd/mxcli/init_hook.go +++ b/cmd/mxcli/init_hook.go @@ -213,7 +213,11 @@ rm -f "$st" status=0 [ -f "$st" ] && status=$(cat "$st") if [ "$status" -ne 0 ]; then - reason=$(grep -i 'error' "$log" | tail -n 1) + # mxcli's own failure is its first "Error:" line. Output it quotes after that + # line (a service manager's, pg_ctlcluster's) can carry "Error:" lines of its + # own, and the last of those is never the reason (ako/mxcli#866). + reason=$(grep '^Error:' "$log" | head -n 1) + [ -n "$reason" ] || reason=$(grep -i 'error' "$log" | tail -n 1) [ -n "$reason" ] || reason=$(grep -v '^[[:space:]]*$' "$log" | tail -n 1) reason=$(echo "$reason" | sed 's/^[Ee][Rr][Rr][Oo][Rr]:[[:space:]]*//') { diff --git a/cmd/mxcli/init_hook_failure_test.go b/cmd/mxcli/init_hook_failure_test.go index dff8a0769d..dddc844826 100644 --- a/cmd/mxcli/init_hook_failure_test.go +++ b/cmd/mxcli/init_hook_failure_test.go @@ -74,3 +74,48 @@ func TestBootstrapScript_ProvisioningSuccessIsQuiet(t *testing.T) { t.Errorf("run was not invoked:\n%s", log) } } + +// ako/mxcli#866: the refusal to provision an empty cluster over a stopped +// system one is only useful if the session sees WHICH cluster to start. The +// banner shows one line; it must be mxcli's own "Error:" line, not the last +// line that happens to contain "error" — a failing service manager quotes +// pg_ctlcluster, whose output carries "Error:" lines of its own, after it. +func TestBootstrapScript_BannerNamesTheRefusalNotTheServiceOutput(t *testing.T) { + requireSh(t) + e := newBootEnv(t, "") + stderrText := `Error: ensuring database: starting local PostgreSQL: refusing to create a new, empty PostgreSQL cluster: the cluster in /var/lib/postgresql/16/main (port 5432) exists but is not running + Start it, then rerun: + sudo pg_ctlcluster 16 main start (or: sudo service postgresql start) +(a service manager ran first but Postgres did not become ready: +sudo service: Starting PostgreSQL 16 database server: main +Error: /usr/lib/postgresql/16/bin/pg_ctl /usr/lib/postgresql/16/bin/pg_ctl start -D /var/lib/postgresql/16/main exited with status 1: +service: mkdir: cannot create directory '/var/run/postgresql': Permission denied)` + body := "#!/bin/sh\n" + + "if [ \"$1\" = \"--version\" ]; then echo \"mxcli version v0.25.0 (2026-10-01T00:00:00Z)\"; exit 0; fi\n" + + "if [ \"$1\" = \"run\" ]; then echo \"Ensuring database...\"; cat >&2 <<'MSG'\n" + stderrText + "\nMSG\nexit 1; fi\n" + + "exit 0\n" + if err := os.WriteFile(filepath.Join(e.project, "mxcli"), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + + cmd := exec.Command("sh", bootstrapScriptName) + cmd.Dir = e.project + cmd.Env = append(os.Environ(), "PATH="+e.bin+":/usr/bin:/bin", "BOOT_LOG="+e.log, "MXCLI_TAG=") + var stdout, stderr strings.Builder + cmd.Stdout, cmd.Stderr = &stdout, &stderr + _ = cmd.Run() + + var banner string + for _, l := range strings.Split(stderr.String(), "\n") { + if strings.HasPrefix(l, "ERROR: the app database was not provisioned") { + banner = l + } + } + if banner == "" { + t.Fatalf("no banner:\n%s", stderr.String()) + } + if !strings.Contains(banner, "refusing to create a new, empty PostgreSQL cluster") || + !strings.Contains(banner, "/var/lib/postgresql/16/main") { + t.Errorf("the banner must name the refusal and the data directory, got:\n%s", banner) + } +} diff --git a/cmd/mxcli/lint_rules_shipped.txt b/cmd/mxcli/lint_rules_shipped.txt index ba1c27c80b..fdc6c714cb 100644 --- a/cmd/mxcli/lint_rules_shipped.txt +++ b/cmd/mxcli/lint_rules_shipped.txt @@ -68,6 +68,7 @@ bbd935d1ee0fe6c635572f19c030f511e1010f13b38f49622f28060c3b3e8305 missing_docume a5fa3f0f37f7fde0206df2880fdeb676d0db656957ce89d7b09033af8a7a4476 orphaned_elements.star c27185493f9b8f0b26eff68634c522b61ddf41a59b614d14e046c28bbc60a7d3 orphaned_elements.star e05678135cbc750a3cfef07ddf348b7f207196f0a7762907e416e1a3c973de79 orphaned_elements.star +d5d37b24359dd2d8110860ced7107781016a730ac883e2353f3ae797ede58055 orphaned_elements.star 4500272ad6d0ae8876081de0825c633117b471913079be08fb78311e10470071 sec_guest_access.star 922394f50a7d8b8fab406dcd623536f98dc0d01b9970053776cd609900b2dc39 sec_missing_member_restriction.star 46d8bf64c7a30453685c4c470d6864e9b8f7cfd69d33985a50e24a358c309324 sec_pii_attributes.star diff --git a/cmd/mxcli/show_version_format_test.go b/cmd/mxcli/show_version_format_test.go new file mode 100644 index 0000000000..2b77f6df6a --- /dev/null +++ b/cmd/mxcli/show_version_format_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// SHOW VERSION printed `MPR Format: v2` for an MPR v1 project — a single .mpr +// with no mprcontents/ folder (ako/mxcli#1553, upstream mendixlabs/mxcli#1621). +// STATUS and the session log's connect record print the same value, so all +// three are asserted here, through the real CLI, on one project of each format. +// The v2 project is the control: it reported v2 before the fix as well. +func TestShowVersionReportsTheStorageFormatOnDisk(t *testing.T) { + if testing.Short() { + t.Skip("spawns mxcli against a project") + } + + v1 := filepath.Join(t.TempDir(), "App.mpr") + data, err := os.ReadFile(filepath.Join("..", "..", "modelsdk", "mpr", "testdata", "v1-project", "App.mpr")) + if err != nil { + t.Fatalf("read v1 fixture: %v", err) + } + if err := os.WriteFile(v1, data, 0o644); err != nil { + t.Fatal(err) + } + + v2dir := t.TempDir() + if err := os.CopyFS(v2dir, os.DirFS(filepath.Join("..", "..", "testdata", "testapp-views"))); err != nil { + t.Fatalf("copy v2 fixture: %v", err) + } + v2 := filepath.Join(v2dir, "TestApp.mpr") + + for _, tc := range []struct { + name string + mpr string + want string + }{ + {"v1", v1, "v1"}, + {"v2 control", v2, "v2"}, + } { + t.Run(tc.name, func(t *testing.T) { + // Precondition: the fixture is laid out as the format it stands for. + _, statErr := os.Stat(filepath.Join(filepath.Dir(tc.mpr), "mprcontents")) + if hasContents := statErr == nil; hasContents != (tc.want == "v2") { + t.Fatalf("fixture %s: mprcontents/ present = %v, want it present only for v2", tc.mpr, hasContents) + } + + res := runMainStreams(t, filepath.Dir(tc.mpr), "-p", tc.mpr, "-c", "show version") + if res.exitCode != 0 { + t.Fatalf("show version exited %d\nstdout:\n%s\nstderr:\n%s", res.exitCode, res.stdout, res.stderr) + } + if want := "MPR Format: " + tc.want + "\n"; !strings.Contains(res.stdout, want) { + t.Errorf("show version: want %q in\n%s", want, res.stdout) + } + + res = runMainStreams(t, filepath.Dir(tc.mpr), "-p", tc.mpr, "-c", "status") + if want := "MPR Format: " + tc.want + "\n"; !strings.Contains(res.stdout, want) { + t.Errorf("status: want %q in\n%s", want, res.stdout) + } + + // The session log is what `mxcli diag --bundle` ships with a bug report. + recs := runMainRecords(t, "connect", "-p", tc.mpr, "-c", "show version") + if len(recs) == 0 { + t.Fatal("no connect record in the session log") + } + for _, r := range recs { + if got, want := r["mpr_format"], float64(tc.want[1]-'0'); got != want { + t.Errorf("session log connect record: mpr_format = %v, want %v", got, want) + } + } + }) + } +} diff --git a/cmd/mxcli/syntax/features_page.go b/cmd/mxcli/syntax/features_page.go index e761c6ab09..10fd68afc7 100644 --- a/cmd/mxcli/syntax/features_page.go +++ b/cmd/mxcli/syntax/features_page.go @@ -293,7 +293,7 @@ LIST IMPACT OF htmlelement; "confirmation", "confirm", "are you sure", "progress bar", "progress message", "disabled during execution", "disabled during action", "asynchronous", "form validations", }, - Syntax: "Action: NOTHING -- deliberately no action (Forms$NoAction)\nAction: SAVE CHANGES\nAction: SAVE CHANGES CLOSE PAGE -- save, then close the pop-up\nAction: CANCEL CHANGES\nAction: CANCEL CHANGES CLOSE PAGE\nAction: CLOSE PAGE\nAction: DELETE\nAction: DELETE CLOSE PAGE\nAction: CALL NANOFLOW Module.NF\nAction: CALL NANOFLOW Module.NF(Param = $val)\nAction: OPEN LINK 'https://example.com'\nAction: OPEN LINK $currentObject/URL -- address read from an attribute\nAction: SIGN OUT\nAction: COMPLETE TASK 'OutcomeName'\nAction: SHOW PAGE Module.Page\nAction: SHOW PAGE Module.Page(Param = $currentObject)\nAction: CALL MICROFLOW Module.MF\nAction: CALL MICROFLOW Module.MF(Param = $val)\nAction: CREATE OBJECT Module.Entity THEN SHOW PAGE Module.Page\n\nThe list above is exhaustive. Anything else in an action slot is an\nERROR: `Action: OPEN LINK` without a URL or `Action: SHOW PAGE` without a\npage is a syntax error, and a value that is not an action at all\n(or a snake-case keyword short its argument) is MDL-WIDGET28.\nSuch a widget used to be written with NO action at all and rendered as a\ndead control, with check, exec and mxbuild all clean, because a\nno-action widget is legal Mendix (mendixlabs/mxcli#1062). Write NOTHING\nwhen a control really is meant to be inert.\n\nThe actions are the words a microflow uses (R8). The snake-case spellings\n(SAVE_CHANGES, SHOW_PAGE, CLOSE_PAGE, DELETE_OBJECT, OPEN_LINK, SIGN_OUT, ...)\nand a flow call without CALL still parse, but are deprecated (MDL-DEPR020);\n`mxcli fmt --upgrade` rewrites them.\n\nThe same forms serve `OnClick:` (an alias of `Action:`) and `OnChange:`.\n\nSETTINGS. Any action but NOTHING takes a `with ( … )` list of the\nsettings Studio Pro shows under the event (ako/mxcli#721):\n\n DisabledDuringExecution: true|false -- every action; default true\n ProgressBar: None|NonBlocking|Blocking -- flow calls; default None\n ProgressMessage: 'text' -- flow calls\n Confirmation: 'question' -- flow calls: ask before running\n ProceedCaption: 'text', CancelCaption: 'text' -- its buttons; default Proceed / Cancel\n Asynchronous: true|false -- call microflow only; default false\n FormValidations: All|Widget|None -- call microflow only; default All\n\nAction: CALL MICROFLOW M.Delete(Order = $currentObject) with (\n ProgressBar: Blocking, ProgressMessage: 'Deleting…',\n Confirmation: 'Delete this order?', ProceedCaption: 'Delete', CancelCaption: 'Keep')\n\nA setting left out keeps its default. DESCRIBE prints only the settings\nthat differ from it. An unknown key, or one the action does not have, is an\nerror. For CREATE OBJECT … THEN SHOW PAGE …, the list goes at the end and\nbelongs to the create.\n\nRETURN VALUE MAPPINGS. A flow call can map what the flow returns back onto\nthe page -- Studio Pro's \"Return value mappings\" (Mendix 11.5). The list\ngoes after the arguments and before `with ( … )`, target = source:\n\nAction: CALL MICROFLOW M.ACT_Vehicle_Get returns (\n Brand = $ActionReturnValue/Brand, -- an attribute of the returned object\n $stringResult = $ActionReturnValue, -- the return value itself\n Brand = 'Demo brand') -- any expression\n\nA bare target is an attribute of the enclosing data container's object;\n`$name` is a page variable. $ActionReturnValue/Attr needs a flow that\nreturns an object. A mapping into an attribute needs Mendix 11.15: mx\n11.13 and 11.14 report CE7006 on it. Leave the list out and the call maps\nnothing; DESCRIBE prints the stored mappings, and re-running a statement\nwithout them removes them (ako/mxcli#977).\n\nA microflow or nanoflow action is a CALL: it needs an argument for every\nparameter the flow declares, or Mendix rejects the page with CE1571. The\nargument list is the same on every widget that takes an action -- a\nCONTAINER (which is clickable) as much as an ACTIONBUTTON. An enclosing\ndata container of the right type supplies it without an argument; a data\ngrid's CONTROL BAR does not, because it is not row-scoped -- pass the\ngrid's selection there (`$dgOrders`).\n\nA SHOW PAGE argument must be the enclosing widget's context object --\neither $currentObject or the name of the variable the enclosing data\nwidget is bound to. Mendix infers it from that widget, so naming any\nother variable is refused (MDL-PAGEARG01); call a microflow instead.\nThe exception is a parameter of the page or snippet itself: it is bound\nby name (Studio Pro stores an explicit mapping), so `(Item = $Item)` works\nanywhere, outside every data widget too (mendixlabs/mxcli#1403).\nOutside any data widget there is no context object to infer, so such a\nbutton takes no OTHER argument -- not $currentObject, not a literal.\nmxcli used to drop it in silence and mxbuild then reported CE1571 per\nparameter of the target page (mendixlabs/mxcli#1029). Route that\nnavigation through a microflow.\n\nOPEN LINK takes a web address, stored as a Forms$StaticOrDynamicString:\neither a literal, or $currentObject/Attr to read it from an attribute of\nthe enclosing data container's object at runtime (Studio Pro's \"Address:\nattribute\"). An address over an association path is not supported yet.\n\nButton styles: Default, Primary, Success, Info, Warning, Danger\n\nMendix has THREE icon elements and the keyword picks which one:\n\nIcon: 'Atlas_Core.Atlas_Filled.pencil' -- an icon collection\nIcon: image MyModule.Images.logo -- an IMAGE collection\nIcon: glyph 57377 -- a font code point\n\nThe bare form is the icon-collection icon and any collection in the\nproject works, third-party ones included. The image form points into a\ndifferent document, and is spelled the same way apart from the keyword\n-- write it without `image` and mxcli stores a custom-icon reference,\nwhich fails the build with CE1613 (mendixlabs/mxcli#1059).\n`mxcli check -p … --references` resolves each kind against its own\ncollection and names the remedy when the kind is wrong.\n\nA glyph carries a code and no name. Codes are sparse, and an undefined\none fails only at `mxbuild --target=deploy`, naming the PAGE rather\nthan the icon -- so MDL078 refuses it (an error) against the font's own\ntable, naming the defined glyphs either side of the code.\nList them with `list glyphs`.\n\nA name may be quoted or bare; a hyphenated segment is double-quoted on\nits own: Atlas_Core.Atlas.\"align-center\".\n\nUse `linkbutton` instead of `actionbutton` for link render mode (same properties).", + Syntax: "Action: NOTHING -- deliberately no action (Forms$NoAction)\nAction: SAVE CHANGES\nAction: SAVE CHANGES CLOSE PAGE -- save, then close the pop-up\nAction: CANCEL CHANGES\nAction: CANCEL CHANGES CLOSE PAGE\nAction: CLOSE PAGE\nAction: DELETE\nAction: DELETE CLOSE PAGE\nAction: CALL NANOFLOW Module.NF\nAction: CALL NANOFLOW Module.NF(Param = $val)\nAction: OPEN LINK 'https://example.com'\nAction: OPEN LINK $currentObject/URL -- address read from an attribute\nAction: SIGN OUT\nAction: COMPLETE TASK 'OutcomeName'\nAction: SHOW PAGE Module.Page\nAction: SHOW PAGE Module.Page(Param = $currentObject)\nAction: CALL MICROFLOW Module.MF\nAction: CALL MICROFLOW Module.MF(Param = $val)\nAction: CREATE OBJECT Module.Entity THEN SHOW PAGE Module.Page\n\nThe list above is exhaustive. Anything else in an action slot is an\nERROR: `Action: OPEN LINK` without a URL or `Action: SHOW PAGE` without a\npage is a syntax error, and a value that is not an action at all\n(or a snake-case keyword short its argument) is MDL-WIDGET28.\nSuch a widget used to be written with NO action at all and rendered as a\ndead control, with check, exec and mxbuild all clean, because a\nno-action widget is legal Mendix (mendixlabs/mxcli#1062). Write NOTHING\nwhen a control really is meant to be inert.\n\nThe actions are the words a microflow uses (R8). The snake-case spellings\n(SAVE_CHANGES, SHOW_PAGE, CLOSE_PAGE, DELETE_OBJECT, OPEN_LINK, SIGN_OUT, ...)\nand a flow call without CALL still parse, but are deprecated (MDL-DEPR020);\n`mxcli fmt --upgrade` rewrites them.\n\nThe same forms serve `OnClick:` (an alias of `Action:`) and `OnChange:`.\n\nA call to a flow or page with no parameters may end in an empty `()`,\nas a flow body writes it. It stores the same action as no list at all,\nand DESCRIBE leaves it out (ako/mxcli#1516).\n\nSETTINGS. Any action but NOTHING takes a `with ( … )` list of the\nsettings Studio Pro shows under the event (ako/mxcli#721):\n\n DisabledDuringExecution: true|false -- every action; default true\n ProgressBar: None|NonBlocking|Blocking -- flow calls; default None\n ProgressMessage: 'text' -- flow calls\n Confirmation: 'question' -- flow calls: ask before running\n ProceedCaption: 'text', CancelCaption: 'text' -- its buttons; default Proceed / Cancel\n Asynchronous: true|false -- call microflow only; default false\n FormValidations: All|Widget|None -- call microflow only; default All\n\nAction: CALL MICROFLOW M.Delete(Order = $currentObject) with (\n ProgressBar: Blocking, ProgressMessage: 'Deleting…',\n Confirmation: 'Delete this order?', ProceedCaption: 'Delete', CancelCaption: 'Keep')\n\nA setting left out keeps its default. DESCRIBE prints only the settings\nthat differ from it. An unknown key, or one the action does not have, is an\nerror. For CREATE OBJECT … THEN SHOW PAGE …, the list goes at the end and\nbelongs to the create.\n\nRETURN VALUE MAPPINGS. A flow call can map what the flow returns back onto\nthe page -- Studio Pro's \"Return value mappings\" (Mendix 11.5). The list\ngoes after the arguments and before `with ( … )`, target = source:\n\nAction: CALL MICROFLOW M.ACT_Vehicle_Get returns (\n Brand = $ActionReturnValue/Brand, -- an attribute of the returned object\n $stringResult = $ActionReturnValue, -- the return value itself\n Brand = 'Demo brand') -- any expression\n\nA bare target is an attribute of the enclosing data container's object;\n`$name` is a page variable. $ActionReturnValue/Attr needs a flow that\nreturns an object. A mapping into an attribute needs Mendix 11.15: mx\n11.13 and 11.14 report CE7006 on it. Leave the list out and the call maps\nnothing; DESCRIBE prints the stored mappings, and re-running a statement\nwithout them removes them (ako/mxcli#977).\n\nA microflow or nanoflow action is a CALL: it needs an argument for every\nparameter the flow declares, or Mendix rejects the page with CE1571. The\nargument list is the same on every widget that takes an action -- a\nCONTAINER (which is clickable) as much as an ACTIONBUTTON. An enclosing\ndata container of the right type supplies it without an argument; a data\ngrid's CONTROL BAR does not, because it is not row-scoped -- pass the\ngrid's selection there (`$dgOrders`).\n\nA SHOW PAGE argument must be the enclosing widget's context object --\neither $currentObject or the name of the variable the enclosing data\nwidget is bound to. Mendix infers it from that widget, so naming any\nother variable is refused (MDL-PAGEARG01); call a microflow instead.\nThe exception is a parameter of the page or snippet itself: it is bound\nby name (Studio Pro stores an explicit mapping), so `(Item = $Item)` works\nanywhere, outside every data widget too (mendixlabs/mxcli#1403).\nOutside any data widget there is no context object to infer, so such a\nbutton takes no OTHER argument -- not $currentObject, not a literal.\nmxcli used to drop it in silence and mxbuild then reported CE1571 per\nparameter of the target page (mendixlabs/mxcli#1029). Route that\nnavigation through a microflow.\n\nOPEN LINK takes a web address, stored as a Forms$StaticOrDynamicString:\neither a literal, or $currentObject/Attr to read it from an attribute of\nthe enclosing data container's object at runtime (Studio Pro's \"Address:\nattribute\"). An address over an association path is not supported yet.\n\nButton styles: Default, Primary, Success, Info, Warning, Danger\n\nMendix has THREE icon elements and the keyword picks which one:\n\nIcon: 'Atlas_Core.Atlas_Filled.pencil' -- an icon collection\nIcon: image MyModule.Images.logo -- an IMAGE collection\nIcon: glyph 57377 -- a font code point\n\nThe bare form is the icon-collection icon and any collection in the\nproject works, third-party ones included. The image form points into a\ndifferent document, and is spelled the same way apart from the keyword\n-- write it without `image` and mxcli stores a custom-icon reference,\nwhich fails the build with CE1613 (mendixlabs/mxcli#1059).\n`mxcli check -p … --references` resolves each kind against its own\ncollection and names the remedy when the kind is wrong.\n\nA glyph carries a code and no name. Codes are sparse, and an undefined\none fails only at `mxbuild --target=deploy`, naming the PAGE rather\nthan the icon -- so MDL078 refuses it (an error) against the font's own\ntable, naming the defined glyphs either side of the code.\nList them with `list glyphs`.\n\nA name may be quoted or bare; a hyphenated segment is double-quoted on\nits own: Atlas_Core.Atlas.\"align-center\".\n\nUse `linkbutton` instead of `actionbutton` for link render mode (same properties).", Example: "ACTIONBUTTON btnSave (Caption: 'Save', Action: SAVE CHANGES, ButtonStyle: Primary)\nACTIONBUTTON btnEdit (Caption: 'Edit',\n Action: SHOW PAGE Module.EditPage(Item = $currentObject))\nLINKBUTTON btnDelete (Caption: 'Delete', Action: DELETE,\n Icon: 'Atlas_Core.Atlas_Filled.pencil')\n\n-- A clickable CONTAINER in a data grid's control bar, calling a nanoflow\n-- with the grid's selection as its argument.\nDATAGRID dgOrders (DataSource: DATABASE FROM Sales.Order, Selection: Single) {\n COLUMN (Attribute: Number, Caption: 'Order #')\n CONTROLBAR {\n CONTAINER cShip (Class: 'command',\n Action: CALL NANOFLOW Sales.ACT_Ship(Order = $dgOrders)) {\n ACTIONBUTTON btnShip (Caption: 'Ship')\n }\n }\n}", SeeAlso: []string{"page.widgets"}, }) @@ -496,9 +496,14 @@ LIST IMPACT OF htmlelement; " [, style: '...']\n" + " [, Layout: Module.Outer] -- built on another layout: see layout.nested\n" + ") {\n" + - " SCROLLCONTAINER name {\n" + + " SCROLLCONTAINER name [( LayoutMode: 'Headline' | 'Sidebar' )] {\n" + + " -- Headline (the default): the top region runs the full width and the\n" + + " -- left region starts under it. Sidebar: the left region runs the full\n" + + " -- height and the top bar sits beside it (Atlas_SideBar, Atlas_TopBar).\n" + " REGION top | right | bottom | left | center\n" + - " [( Size: 60, SizeMode: 'Fixed' | 'Pixels' | 'Auto', Class: '…' )] {\n" + + " [( Size: 60, SizeMode: 'Fixed' | 'Pixels' | 'Auto', Class: '…',\n" + + " ToggleMode: 'None' | 'PushContentAside' | 'SlideOverContent'\n" + + " | 'ShrinkContentInitiallyOpen' | 'ShrinkContentInitiallyClosed' )] {\n" + " -- widgets, plus:\n" + " NAVIGATIONTREE name (Profile: 'Responsive') -- vertical, for a sidebar\n" + " MENUBAR name (Profile: 'Responsive') -- horizontal, for a topbar\n" + diff --git a/cmd/mxcli/syntax/features_security.go b/cmd/mxcli/syntax/features_security.go index 7bf4215f40..83372b3499 100644 --- a/cmd/mxcli/syntax/features_security.go +++ b/cmd/mxcli/syntax/features_security.go @@ -142,10 +142,11 @@ func init() { Register(SyntaxFeature{ Path: "security.project-security", - Summary: "Set project security level, strict mode, demo user and guest access toggles, admin user name", + Summary: "Set project security level, strict mode, demo user and guest access toggles, admin user name and password", Keywords: []string{ "project security", "app security", "alter app security", "security level", "prototype", "production", "off", "strict mode", "SEC005", "admin user", "AdminUserName", "MxAdmin", + "admin password", "AdminPassword", "CE0129", }, Syntax: "ALTER APP SECURITY (\n" + " [SecurityLevel: OFF|PROTOTYPE|PRODUCTION,]\n" + @@ -153,17 +154,22 @@ func init() { " [EnableGuestAccess: TRUE|FALSE,]\n" + " [GuestUserRole: ,]\n" + " [StrictMode: TRUE|FALSE,] -- clears lint rule SEC005\n" + - " [AdminUserName: ''] -- the built-in administrator (default MxAdmin)\n" + + " [AdminUserName: '',] -- the built-in administrator (default MxAdmin)\n" + + " [AdminPassword: ''] -- its password; empty is CE0129\n" + ");\n\n" + "-- Set any subset of the properties, in create's ( Key: value ) list. The\n" + "-- clause forms (LEVEL …, DEMO USERS ON|OFF, GUEST ACCESS ON [ROLE r]|OFF,\n" + - "-- STRICT MODE ON|OFF) still run and warn MDL-DEPR133. The administrator's\n" + - "-- password is not settable from MDL.", + "-- STRICT MODE ON|OFF) still run and warn MDL-DEPR133.\n" + + "-- AdminPassword is stored in the model as written, as Studio Pro stores it;\n" + + "-- '' is refused (CE0129 \"Administrator password has not been set\" at\n" + + "-- prototype and production level). mxbuild does not check it against the\n" + + "-- password policy. `describe app security` says whether one is set.", Example: "mdl 1;\n" + "ALTER APP SECURITY ( SecurityLevel: PRODUCTION );\n" + "ALTER APP SECURITY ( EnableDemoUsers: FALSE );\n" + "ALTER APP SECURITY ( StrictMode: TRUE );\n" + - "ALTER APP SECURITY ( AdminUserName: 'appadmin' );", + "ALTER APP SECURITY ( AdminUserName: 'appadmin' );\n" + + "ALTER APP SECURITY ( AdminPassword: 'N3w-Admin-Passw0rd' ); -- resolves CE0129", SeeAlso: []string{"security.demo-user", "security.guest-access"}, }) diff --git a/cmd/mxcli/syntax/retired_spellings_test.go b/cmd/mxcli/syntax/retired_spellings_test.go index 5172f2684a..8009d503ed 100644 --- a/cmd/mxcli/syntax/retired_spellings_test.go +++ b/cmd/mxcli/syntax/retired_spellings_test.go @@ -25,8 +25,10 @@ func TestSyntaxDocs_NoRetiredSpellings(t *testing.T) { "the parser rejects it: \"'Binds:' is no longer supported, use 'Attribute:' instead\"", }, { + // Not retired since ako/mxcli#1516 — `()` parses and stores nothing — + // but the reference shows the form DESCRIBE writes. "MICROFLOW Module.MF()", - "a zero-argument microflow DATASOURCE takes no parentheses (unlike RETRIEVE/CALL, where they are normal)", + "the reference shows a zero-argument call or data source the way DESCRIBE writes it, without parentheses; `()` is accepted and means the same (ako/mxcli#1516)", }, } diff --git a/cmd/mxcli/testrunner/build_attribution.go b/cmd/mxcli/testrunner/build_attribution.go index f49a66b335..e17b44e667 100644 --- a/cmd/mxcli/testrunner/build_attribution.go +++ b/cmd/mxcli/testrunner/build_attribution.go @@ -175,7 +175,7 @@ func resultsFromFailedBuild(problems []docker.BuildProblem, suite *TestSuite) [] // buildFailureHint is appended to the error when a build failure could not be // attributed to any test, which means it is in the project rather than in the -// suite. +// suite — or in one of the documents the run generates besides its tests. // // The errors themselves are already rendered by BuildFailedError.Error(), so // this says what the reader cannot work out from them: that none of them belongs @@ -183,13 +183,34 @@ func resultsFromFailedBuild(problems []docker.BuildProblem, suite *TestSuite) [] // that it is a leftover from an earlier run, with the command to remove it. That // last case is the one that fails every subsequent run of every test file until // someone finds it by hand (mendixlabs/mxcli#1104). +// +// An error in the endpoint documents is mxcli's own, and says so. CE0114 in +// MxTest.RegisterEndpoint used to be reported as "in the project", which sent +// the reader through their own model for a defect in a microflow they never +// wrote (ako/mxcli#1654). func buildFailureHint(other []docker.BuildProblem) string { if len(other) == 0 { return "" } + generated, project := splitRunnerDocuments(other) var b strings.Builder - b.WriteString("\n These errors are in the project, not in this run's tests.") - for _, name := range leftoverFlowsIn(other) { + for _, name := range generated { + b.WriteString(fmt.Sprintf("\n %s is not part of your project: mxcli generated it for this run, %s."+ + " An error in it is a defect in mxcli, not in your project or your tests — please report it"+ + " (`mxcli diag --bundle` collects what is needed).", name, runnerDocuments[name])) + if name == endpointStartupFlow { + b.WriteString("\n Until it is fixed, --skip-app-startup runs the tests without the call to your after-startup microflow.") + } + } + if len(project) == 0 { + return b.String() + } + if len(generated) > 0 { + b.WriteString("\n The other errors are in the project, not in this run's tests.") + } else { + b.WriteString("\n These errors are in the project, not in this run's tests.") + } + for _, name := range leftoverFlowsIn(project) { b.WriteString(fmt.Sprintf( "\n %s is a microflow an earlier `mxcli test` run left behind. Remove it with:"+ "\n mxcli -p -c \"DROP MICROFLOW %s\"", name, name)) @@ -197,6 +218,48 @@ func buildFailureHint(other []docker.BuildProblem) string { return b.String() } +// runnerDocuments are the documents an endpoint run generates besides its test +// microflows, with what each is for. Their names are fixed, so an error located +// in one is the run's own whatever else is in the MxTest module. +var runnerDocuments = map[string]string{ + endpointStartupFlow: "to register the test endpoint at startup and then run your after-startup microflow", + endpointRegisterAction: "to register the test endpoint", +} + +// runnerDocumentPattern matches the `document` MxBuild reports — the document's +// kind and its bare name, `Microflow 'RegisterEndpoint'` or +// `Java action 'RegisterTestEndpoint'`; the module is carried separately. +var runnerDocumentPattern = regexp.MustCompile(`^[A-Za-z ]+ '([^']*)'$`) + +// splitRunnerDocuments separates the problems located in a runner document from +// the rest, naming each runner document once, in the order MxBuild reported it. +func splitRunnerDocuments(problems []docker.BuildProblem) (generated []string, rest []docker.BuildProblem) { + seen := map[string]bool{} + for _, p := range problems { + name := "" + for _, loc := range p.Locations { + if !strings.EqualFold(loc.Module, mxTestModule) { + continue + } + if m := runnerDocumentPattern.FindStringSubmatch(loc.Document); m != nil { + if q := mxTestModule + "." + m[1]; runnerDocuments[q] != "" { + name = q + break + } + } + } + if name == "" { + rest = append(rest, p) + continue + } + if !seen[name] { + seen[name] = true + generated = append(generated, name) + } + } + return generated, rest +} + // leftoverFlowsIn names the generated test microflows among a set of build // problems, deduplicated and in the order MxBuild reported them. // diff --git a/cmd/mxcli/testrunner/endpoint.go b/cmd/mxcli/testrunner/endpoint.go index 3b2061aa68..cf1acea01c 100644 --- a/cmd/mxcli/testrunner/endpoint.go +++ b/cmd/mxcli/testrunner/endpoint.go @@ -67,12 +67,21 @@ func testFlowName(tc TestCase) string { return testFlowPrefix + tc.ID } // // chainAfterStartup, when non-empty, is a microflow the generated startup flow // calls after registering the endpoint — the project's own after-startup -// microflow, which this one displaces. +// microflow, which this one displaces. The test runner passes it so tests see +// the app as it really boots, and "" under --skip-app-startup; a dev loop +// hosting the endpoint (`run --local --test-endpoint`) always passes it. // -// The test runner passes "" : a test run wants a known starting state, and the -// suite is the only thing that should execute. A dev loop hosting the endpoint -// (`run --local --test-endpoint`) passes the real one, because the developer's -// app must still seed its data and do whatever else it does at boot. +// The startup flow applies entity access, whatever the chained microflow does. +// MxBuild refuses a microflow that does not apply entity access calling one +// that does (CE0114), and the flow used to be written without the setting, so +// no project whose after-startup microflow applies entity access could run a +// single test (ako/mxcli#1654). The rule has one direction — measured with mx +// check on 11.13.0, an applying microflow may call an applying or a +// non-applying one at 0 errors, under security Off and Production alike (and +// MDL-SEC22 records the same table for 11.14.0) — so a constant serves every +// project, with nothing to read from the chained microflow first. The setting +// governs only this flow's own two activities: the Java action ignores it, and +// the chained microflow runs with its own. func GenerateEndpointMDL(chainAfterStartup string) string { var b strings.Builder @@ -91,6 +100,7 @@ func GenerateEndpointMDL(chainAfterStartup string) string { b.WriteString("\n$$;\n\n") b.WriteString("/** Registers the mxcli test endpoint at boot. Runs no tests. */\n") + b.WriteString("@applyentityaccess\n") b.WriteString(createFlow(langver.V1) + " " + endpointStartupFlow + " ()\n") b.WriteString("RETURNS Boolean AS $Registered\n") b.WriteString("BEGIN\n") diff --git a/cmd/mxcli/testrunner/generator.go b/cmd/mxcli/testrunner/generator.go index c28cb42c2e..674b81b3dc 100644 --- a/cmd/mxcli/testrunner/generator.go +++ b/cmd/mxcli/testrunner/generator.go @@ -161,9 +161,17 @@ func writeTestBlock(b *strings.Builder, tc TestCase, index int) { varNames := extractVariableNames(tc.MDL) renamedMDL := renameVariables(tc.MDL, varNames, suffix) + // Every assertion counts itself when evaluated; see assertionsEvaluatedVar. + // On this runner a skipped assertion logs no line at all, so without the + // guard the test would end with a RUN line and no verdict. + evaluated := assertionsEvaluatedVar + suffix + if len(tc.Expects) > 0 { + b.WriteString(fmt.Sprintf(" DECLARE %s Integer = 0;\n", evaluated)) + } + // Rewrite the MDL body: indent it and wrap CALL MICROFLOW with error handling lines := strings.Split(renamedMDL, "\n") - rewritten := rewriteWithErrorHandling(lines, tc.ID) + rewritten := rewriteWithErrorHandling(lines, tc.ID, tc.Version) writeBodyLines(b, rewritten, tc.Version) // Generate assertion checks for @expect (with renamed variables) @@ -175,8 +183,13 @@ func writeTestBlock(b *strings.Builder, tc TestCase, index int) { } writeExpectAggregates(b, " ", renamed) for _, exp := range renamed { - writeExpectAssertion(b, tc.ID, exp) + writeExpectAssertion(b, tc.ID, exp, evaluated) } + b.WriteString(fmt.Sprintf(" IF %s < %d THEN\n", evaluated, len(tc.Expects))) + b.WriteString(fmt.Sprintf(" LOG ERROR NODE 'MXTEST' '%s';\n", + escapeMDLString("MXTEST:FAIL:"+tc.ID+":"+assertionsSkippedReason))) + b.WriteString(" SET $AllPassed = false;\n") + b.WriteString(" END IF;\n") } else if !tc.expectsThrow() { // No expectations — just check it didn't throw b.WriteString(" IF $TestFailed = false THEN\n") @@ -214,7 +227,7 @@ func writeThrowsTestBlock(b *strings.Builder, tc TestCase, suffix string) { renamedMDL := renameVariables(tc.MDL, varNames, suffix) lines := strings.Split(renamedMDL, "\n") - rewritten := rewriteForThrowsTest(lines, didThrowVar, tc.Throws) + rewritten := rewriteForThrowsTest(lines, didThrowVar, tc.Throws, tc.Version) writeBodyLines(b, rewritten, tc.Version) // Check that it did throw @@ -238,13 +251,15 @@ func writeThrowsTestBlock(b *strings.Builder, tc TestCase, suffix string) { // Unlike the endpoint generator this path reports failures through the log // protocol, whose message is a single log line, so the observed value is // concatenated into that line rather than into a returned verdict. -func writeExpectAssertion(b *strings.Builder, testID string, exp Expect) { +func writeExpectAssertion(b *strings.Builder, testID string, exp Expect, evaluated string) { passCondition := fmt.Sprintf("$TestFailed = false and (%s)", exp.Condition) failMsg := "MXTEST:FAIL:" + testID + ":Expected " + exp.Raw b.WriteString(fmt.Sprintf(" IF %s THEN\n", passCondition)) + b.WriteString(fmt.Sprintf(" SET %s = %s + 1;\n", evaluated, evaluated)) b.WriteString(fmt.Sprintf(" LOG INFO NODE 'MXTEST' 'MXTEST:PASS:%s';\n", escapeMDLString(testID))) b.WriteString(" ELSE\n") + b.WriteString(fmt.Sprintf(" SET %s = %s + 1;\n", evaluated, evaluated)) if exp.Actual == "" { b.WriteString(fmt.Sprintf(" LOG ERROR NODE 'MXTEST' '%s';\n", escapeMDLString(failMsg))) } else { @@ -339,52 +354,16 @@ func renameExpect(exp Expect, names map[string]bool, suffix string) Expect { // rewriteWithErrorHandling adds ON ERROR clauses to the calls in the body. // The error handler ends with RETURN $AllPassed to properly terminate the flow path -// with a return value (required when the microflow returns a type). -func rewriteWithErrorHandling(lines []string, testID string) []string { - var result []string - - for i := 0; i < len(lines); i++ { - line := lines[i] - trimmed := strings.TrimSpace(line) - - if startsErrorRaisingCall(trimmed) && strings.HasSuffix(trimmed, ";") { - withoutSemicolon := strings.TrimSuffix(trimmed, ";") - result = append(result, withoutSemicolon+" ON ERROR BEGIN") - result = append(result, fmt.Sprintf( - " LOG ERROR NODE 'MXTEST' 'MXTEST:FAIL:%s:Exception during execution';", - escapeMDLString(testID))) - result = append(result, " SET $TestFailed = true;") - result = append(result, " SET $AllPassed = false;") - result = append(result, " RETURN $AllPassed;") - result = append(result, "END ERROR;") - } else if startsErrorRaisingCall(trimmed) && !strings.HasSuffix(trimmed, ";") { - var callLines []string - callLines = append(callLines, line) - for i+1 < len(lines) { - i++ - nextLine := lines[i] - callLines = append(callLines, nextLine) - if strings.HasSuffix(strings.TrimSpace(nextLine), ";") { - break - } - } - joined := strings.Join(callLines, "\n") - joined = strings.TrimSpace(joined) - joined = strings.TrimSuffix(joined, ";") - result = append(result, joined+" ON ERROR BEGIN") - result = append(result, fmt.Sprintf( - " LOG ERROR NODE 'MXTEST' 'MXTEST:FAIL:%s:Exception during execution';", - escapeMDLString(testID))) - result = append(result, " SET $TestFailed = true;") - result = append(result, " SET $AllPassed = false;") - result = append(result, " RETURN $AllPassed;") - result = append(result, "END ERROR;") - } else { - result = append(result, line) - } - } - - return result +// with a return value (required when the microflow returns a type). Which calls +// get one is attachOnError's decision, shared with the endpoint runner. +func rewriteWithErrorHandling(lines []string, testID string, v langver.Version) []string { + return attachOnError(lines, v, []string{ + fmt.Sprintf(" LOG ERROR NODE 'MXTEST' 'MXTEST:FAIL:%s:Exception during execution';", + escapeMDLString(testID)), + " SET $TestFailed = true;", + " SET $AllPassed = false;", + " RETURN $AllPassed;", + }) } // rewriteForThrowsTest wraps each call in the body with ON ERROR that sets the throw flag. @@ -393,41 +372,8 @@ func rewriteWithErrorHandling(lines []string, testID string) []string { // runner reports a wrong expectation the same way the endpoint runner does — as // "expected an exception but none was thrown", which is the only outcome its log // protocol has for a throws test that did not get what it asked for. -func rewriteForThrowsTest(lines []string, didThrowVar string, want string) []string { - var result []string - - for i := 0; i < len(lines); i++ { - line := lines[i] - trimmed := strings.TrimSpace(line) - - if startsErrorRaisingCall(trimmed) && strings.HasSuffix(trimmed, ";") { - withoutSemicolon := strings.TrimSuffix(trimmed, ";") - result = append(result, withoutSemicolon+" ON ERROR BEGIN") - result = append(result, throwsFlagHandler(didThrowVar, want)...) - result = append(result, "END ERROR;") - } else if startsErrorRaisingCall(trimmed) && !strings.HasSuffix(trimmed, ";") { - var callLines []string - callLines = append(callLines, line) - for i+1 < len(lines) { - i++ - nextLine := lines[i] - callLines = append(callLines, nextLine) - if strings.HasSuffix(strings.TrimSpace(nextLine), ";") { - break - } - } - joined := strings.Join(callLines, "\n") - joined = strings.TrimSpace(joined) - joined = strings.TrimSuffix(joined, ";") - result = append(result, joined+" ON ERROR BEGIN") - result = append(result, throwsFlagHandler(didThrowVar, want)...) - result = append(result, "END ERROR;") - } else { - result = append(result, line) - } - } - - return result +func rewriteForThrowsTest(lines []string, didThrowVar string, want string, v langver.Version) []string { + return attachOnError(lines, v, throwsFlagHandler(didThrowVar, want)) } // throwsFlagHandler sets the did-throw flag, gated on the expected message when @@ -444,22 +390,28 @@ func throwsFlagHandler(didThrowVar, want string) []string { } } -// startsErrorRaisingCall reports whether a line starts a call-shaped activity -// the runner must catch an error from: one that takes an ON ERROR clause in a -// microflow (onErrorClause on its rule in MDLMicroflow.g4). +// errorRaisingCallRules are the statements the runner must catch an error from: +// the call-shaped activities, named by their rule in MDLMicroflow.g4, each of +// which takes an onErrorClause there. TestErrorRaisingCallRulesAreGrammarRules +// fails if one is renamed, which would otherwise silently stop wrapping it. // // It used to match CALL MICROFLOW and CALL NANOFLOW only, so an error raised by // CALL JAVA ACTION escaped the test microflow — @throws could not catch it and // a test without one was reported as ERROR instead of FAIL (ako/mxcli#1164). -// Anchored at the statement start so a string that mentions a call is not one. -func startsErrorRaisingCall(s string) bool { - return errorRaisingCall.MatchString(s) +// It is a set of parsed statements rather than a pattern over a line, so a +// string or a comment that mentions a call is not one. +var errorRaisingCallRules = map[string]bool{ + "callMicroflowStatement": true, + "callNanoflowStatement": true, + "callJavaActionStatement": true, + "callExternalActionStatement": true, + "callWebServiceStatement": true, + "callWorkflowStatement": true, + "restCallStatement": true, + "sendRestRequestStatement": true, + "executeDatabaseQueryStatement": true, } -var errorRaisingCall = regexp.MustCompile(`(?i)^\s*(\$\w+\s*=\s*)?(` + - `call\s+(microflow|nanoflow|java\s+action|external\s+action|web\s+service|rest\s+service|workflow)` + - `|rest\s+call|send\s+rest\s+request|execute\s+database\s+query)\b`) - // escapeMDLString escapes single quotes for MDL string literals. func escapeMDLString(s string) string { return strings.ReplaceAll(s, "'", "''") diff --git a/cmd/mxcli/testrunner/generator_endpoint.go b/cmd/mxcli/testrunner/generator_endpoint.go index 2d0cfb53a2..c9ff710e28 100644 --- a/cmd/mxcli/testrunner/generator_endpoint.go +++ b/cmd/mxcli/testrunner/generator_endpoint.go @@ -6,7 +6,9 @@ import ( "fmt" "strings" + "github.com/mendixlabs/mxcli/mdl/ast" "github.com/mendixlabs/mxcli/mdl/langver" + "github.com/mendixlabs/mxcli/mdl/visitor" ) // Verdict protocol. A test microflow returns one string: either verdictPass, or @@ -95,12 +97,53 @@ func writeSetupCalls(b *strings.Builder, tc TestCase) { // writeExpectFlowBody writes the body of a normal test: run the MDL, then check // each @expect. An error during the body short-circuits to a FAIL verdict. +// +// Every assertion counts itself when it is evaluated, and the test fails if the +// count falls short (writeAssertionsEvaluatedGuard). See assertionsEvaluatedVar +// for the path that skips assertions without failing anything. func writeExpectFlowBody(b *strings.Builder, tc TestCase) { + if len(tc.Expects) > 0 { + fmt.Fprintf(b, " DECLARE %s Integer = 0;\n", assertionsEvaluatedVar) + } writeBodyLines(b, rewriteBodyForVerdict(strings.Split(tc.MDL, "\n"), tc), tc.Version) writeExpectAggregates(b, " ", tc.Expects) for _, exp := range tc.Expects { writeExpectCheck(b, exp) } + writeAssertionsEvaluatedGuard(b, len(tc.Expects)) +} + +// assertionsEvaluatedVar counts the assertions a test microflow actually +// evaluated. +// +// The count exists because a test can reach its RETURN without passing through +// its assertions, and nothing else would notice. A call the author gave a +// custom handler that ends normally (`on error begin log …; end error;`) +// rejoins the flow — and the microflow builder routes that rejoin past every +// statement reading the failed call's output, since the variable does not +// exist on the error path. The assertions read exactly that output. Measured on +// 11.13.0: a body whose call threw into such a handler reported PASS on +// `@expect $r = 'X'`, with $r never assigned (ako/mxcli#1448). +// +// The increments sit in both branches of each assertion's decision, so running +// one proves that the condition was evaluated; the guard after the last one +// cannot be skipped, because it reads nothing the body produced. +const assertionsEvaluatedVar = "$AssertionsEvaluated" + +// assertionsSkippedReason is the failure a short count reports. +const assertionsSkippedReason = "not every assertion was evaluated: an error in the body was caught by " + + "a call's own ON ERROR handler, and the flow resumed past the assertions that read that call's result" + +// writeAssertionsEvaluatedGuard fails a test that is still passing but did not +// evaluate all n of its assertions. A test that already failed keeps its own, +// more specific verdict. +func writeAssertionsEvaluatedGuard(b *strings.Builder, n int) { + if n == 0 { + return + } + fmt.Fprintf(b, " IF $Verdict = '%s' and %s < %d THEN\n", verdictPass, assertionsEvaluatedVar, n) + fmt.Fprintf(b, " SET $Verdict = '%s';\n", escapeMDLString(verdictFailPrefix+assertionsSkippedReason)) + b.WriteString(" END IF;\n") } // writeExpectAggregates emits the Aggregate list activities the assertions need, @@ -131,7 +174,7 @@ func writeExpectAggregates(b *strings.Builder, indent string, expects []Expect) func writeThrowsFlowBody(b *strings.Builder, tc TestCase) { fmt.Fprintf(b, " SET $Verdict = '%s';\n", escapeMDLString(verdictFailPrefix+"expected an exception but none was thrown")) - writeBodyLines(b, rewriteBodyForThrows(strings.Split(tc.MDL, "\n"), tc.Throws), tc.Version) + writeBodyLines(b, rewriteBodyForThrows(strings.Split(tc.MDL, "\n"), tc.Throws, tc.Version), tc.Version) } // writeExpectCheck writes one @expect assertion. @@ -145,7 +188,9 @@ func writeExpectCheck(b *strings.Builder, exp Expect) { // existing failure with a later assertion's result. fmt.Fprintf(b, " IF $Verdict = '%s' THEN\n", verdictPass) fmt.Fprintf(b, " IF %s THEN\n", exp.Condition) + fmt.Fprintf(b, " SET %s = %s + 1;\n", assertionsEvaluatedVar, assertionsEvaluatedVar) b.WriteString(" ELSE\n") + fmt.Fprintf(b, " SET %s = %s + 1;\n", assertionsEvaluatedVar, assertionsEvaluatedVar) fmt.Fprintf(b, " SET $Verdict = %s;\n", failVerdictExpr(exp)) b.WriteString(" END IF;\n") b.WriteString(" END IF;\n") @@ -179,7 +224,7 @@ func rewriteBodyForVerdict(lines []string, tc TestCase) []string { escapeMDLString(verdictFailPrefix+"exception during execution: "), latestErrorMessage), " RETURN $Verdict;", } - return attachOnError(lines, handler) + return attachOnError(lines, tc.Version, handler) } // rewriteBodyForThrows attaches an ON ERROR handler that clears the pre-set @@ -188,8 +233,8 @@ func rewriteBodyForVerdict(lines []string, tc TestCase) []string { // With a want, the handler clears it only for a matching message and otherwise // replaces it with one naming both sides, so a failing @throws says what was // raised instead of only what was expected. -func rewriteBodyForThrows(lines []string, want string) []string { - return attachOnError(lines, throwsHandler(want)) +func rewriteBodyForThrows(lines []string, want string, v langver.Version) []string { + return attachOnError(lines, v, throwsHandler(want)) } // throwsHandler builds the ON ERROR body of an @throws test. @@ -228,29 +273,100 @@ func quoteForMessage(s string) string { return "'" + s + "'" } -// attachOnError appends `ON ERROR BEGIN ... END ERROR` to each CALL statement in the body, -// joining a statement that spans several lines first. -func attachOnError(lines, handler []string) []string { - var out []string - for i := 0; i < len(lines); i++ { - trimmed := strings.TrimSpace(lines[i]) - if !startsErrorRaisingCall(trimmed) { - out = append(out, lines[i]) +// attachOnError gives each call in the body that has no error handling of its +// own the runner's `ON ERROR BEGIN … END ERROR`, so an error it raises becomes a +// verdict instead of escaping the test microflow. Every rewriting generator — +// both runners, with and without @throws — goes through here, so they cannot +// disagree about which calls are wrapped. +// +// A call the author already gave a clause keeps it, and gets no second one: two +// clauses on one activity is not MDL, and the generated script stopped the run +// at injection with a parse error on a line the author never wrote +// (ako/mxcli#1448). What the author wrote is what runs: +// +// - `on error continue` swallows the error, so the test goes on and its +// assertions decide — exactly what the production statement does. +// - A custom handler (`on error [without rollback] begin … end error`) runs +// as written. One that ends normally rejoins the flow; one that re-raises +// lets the error leave the test microflow, which the endpoint reports as an +// ERROR — never a pass. +// - `on error rollback` is the exception, because it is not a handler: it is +// Mendix's default spelled out, the ErrorHandlingType a call without a +// clause stores too. The two spellings are one model and get one verdict, +// so the clause is replaced by the runner's handler, which also rolls back. +// Keeping it would turn the same exception into an ERROR instead of a FAIL, +// fail an @throws test that should pass, and under the legacy runner abort +// the whole suite's after-startup flow. +// +// Statements are found by parsing the body, not by searching its text: the +// clause may be on a later line, a custom handler holds statements (and +// semicolons) of its own, and `on error` may be inside a string or a comment. +// A body that does not parse is returned unchanged — no handler can make it +// parse, and the error the author then sees is about their own text. +func attachOnError(lines []string, v langver.Version, handler []string) []string { + body := strings.Join(lines, "\n") + + // Parsed inside the same wrapper the generators put it in, under the same + // language version, so it reads exactly as the generated script will. + var wrap strings.Builder + writeScriptHeader(&wrap, v) + wrap.WriteString(createFlow(v) + " " + mxTestModule + ".Body ()\nBEGIN\n") + prefix := wrap.String() + stmts, ok := visitor.FlowStatements(prefix + body + "\nEND;\n") + if !ok { + return lines + } + + runes := []rune(body) + // The handler is laid out at the indentation of the line its call starts + // on, so a call nested in an IF or a loop reads as nested in --verbose. + block := func(callStart int) string { + lineStart := callStart + for lineStart > 0 && runes[lineStart-1] != '\n' { + lineStart-- + } + indent := lineStart + for indent < callStart && (runes[indent] == ' ' || runes[indent] == '\t') { + indent++ + } + pad := string(runes[lineStart:indent]) + var b strings.Builder + b.WriteString("ON ERROR BEGIN\n") + for _, l := range handler { + b.WriteString(pad + l + "\n") + } + b.WriteString(pad + "END ERROR") + return b.String() + } + type edit struct { + start, stop int + text string + } + var edits []edit + offset := len([]rune(prefix)) + for _, s := range stmts { + if !errorRaisingCallRules[s.Rule] { continue } - - stmt := lines[i] - for !strings.HasSuffix(strings.TrimSpace(stmt), ";") && i+1 < len(lines) { - i++ - stmt += "\n" + lines[i] + switch { + case s.OnError == nil: + edits = append(edits, edit{s.Stop - offset, s.Stop - offset, " " + block(s.Start-offset)}) + case s.OnError.Type == ast.ErrorHandlingRollback: + edits = append(edits, edit{s.OnError.Start - offset, s.OnError.Stop - offset, block(s.Start - offset)}) } - stmt = strings.TrimSuffix(strings.TrimSpace(stmt), ";") + } + if len(edits) == 0 { + return lines + } - out = append(out, stmt+" ON ERROR BEGIN") - out = append(out, handler...) - out = append(out, "END ERROR;") + // Back to front, so an edit never moves the offsets of one still to come. + // They cannot overlap: the only statements inside another are in a custom + // handler's body, and a call with a custom handler is never edited itself. + for i := len(edits) - 1; i >= 0; i-- { + e := edits[i] + runes = append(runes[:e.start:e.start], append([]rune(e.text), runes[e.stop:]...)...) } - return out + return strings.Split(string(runes), "\n") } // escapeMDLComment keeps a test name from closing the javadoc block it sits in. diff --git a/cmd/mxcli/testrunner/generator_test.go b/cmd/mxcli/testrunner/generator_test.go index 2b0a434ec9..cfc5ba11bc 100644 --- a/cmd/mxcli/testrunner/generator_test.go +++ b/cmd/mxcli/testrunner/generator_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/mendixlabs/mxcli/mdl/langver" "github.com/mendixlabs/mxcli/mdl/visitor" ) @@ -17,7 +18,7 @@ func TestRewriteWithErrorHandling_OnlyWrapsCalls(t *testing.T) { "DELETE $p1;", } - got := rewriteWithErrorHandling(lines, "test_1") + got := rewriteWithErrorHandling(lines, "test_1", langver.V0) joined := strings.Join(got, "\n") if strings.Contains(joined, "CHANGE $p1 (IsActive = false) ON ERROR CONTINUE;") { diff --git a/cmd/mxcli/testrunner/own_error_handler_test.go b/cmd/mxcli/testrunner/own_error_handler_test.go new file mode 100644 index 0000000000..f25e6bb98a --- /dev/null +++ b/cmd/mxcli/testrunner/own_error_handler_test.go @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testrunner + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/grammar/parser" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// The runner attaches its own `ON ERROR BEGIN … END ERROR` to every call in a +// test body, so an exception becomes a verdict. A call the author already gave +// an error-handling clause used to get a SECOND one appended, and the generated +// MDL did not parse: the run stopped at injection with "mismatched input 'ON' +// expecting ';'" on a line of a script the author never saw +// (ako/mxcli#1448, mendixlabs/mxcli#1550). +// +// The runner always writes its handler in upper case and these bodies write +// theirs in lower case, so counting "ON ERROR BEGIN" counts the runner's +// handlers only. +const runnerHandler = "ON ERROR BEGIN" + +// generatedBy renders one test body through each of the four generators that +// rewrite a body: the endpoint runner and the legacy after-startup runner, each +// for a test with @expect and one with @throws. +func generatedBy(t *testing.T, body string) map[string]string { + t.Helper() + expect := parseOne(t, "/**\n * @test it runs\n * @expect $r != empty\n */\n"+body+"\n/\n") + throws := parseOne(t, "/**\n * @test it raises\n * @throws 'boom'\n */\n"+body+"\n/\n") + one := func(tc TestCase) *TestSuite { return &TestSuite{Name: "s", Tests: []TestCase{tc}} } + return map[string]string{ + "endpoint @expect": GenerateTestFlows(one(expect)), + "endpoint @throws": GenerateTestFlows(one(throws)), + "legacy @expect": GenerateTestRunner(one(expect)), + "legacy @throws": GenerateTestRunner(one(throws)), + } +} + +func mustParse(t *testing.T, runner, mdl string) { + t.Helper() + if _, errs := visitor.Build(mdl); len(errs) > 0 { + t.Errorf("%s: generated MDL does not parse: %v\n%s", runner, errs[0], mdl) + } +} + +// TestCallWithItsOwnHandlerKeepsIt pins the semantics: a call that continues +// on error, or has a custom handler, keeps exactly what the author wrote — the +// test exercises what the production statement does — and gets no second +// handler from the runner. +func TestCallWithItsOwnHandlerKeepsIt(t *testing.T) { + cases := []struct{ name, body, clause string }{ + {"on error continue", + "$r = call java action Shop.JA_Upper(Text = 'abc') on error continue;", + "on error continue"}, + {"custom handler", + "$r = call java action Shop.JA_Upper(Text = 'abc') on error begin\n log error node 'T' 'handled';\nend error;", + "on error begin"}, + {"custom handler without rollback", + "$r = call java action Shop.JA_Upper(Text = 'abc') on error without rollback begin\n log error node 'T' 'handled';\nend error;", + "on error without rollback begin"}, + {"custom handler that re-raises", + "$r = call java action Shop.JA_Upper(Text = 'abc') on error begin\n log error node 'T' 'handled';\n raise error;\nend error;", + "raise error;"}, + {"deprecated brace form", + "$r = call java action Shop.JA_Upper(Text = 'abc') on error { log error node 'T' 'handled'; };", + "on error {"}, + {"clause on the line after the call", + "$r = call microflow Shop.Order_Label(Text = '1')\n on error continue;", + "on error continue"}, + {"clause after a multi-line argument list", + "$r = call microflow Shop.Order_Label(\n Text = '1'\n) on error continue;", + "on error continue"}, + {"nanoflow call", "$r = call nanoflow Shop.NF(Text = '1') on error continue;", "on error continue"}, + {"rest call", + "$r = call rest service get 'https://example.com' returns String on error continue;", + "on error continue"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + for runner, mdl := range generatedBy(t, c.body) { + mustParse(t, runner, mdl) + if n := strings.Count(mdl, runnerHandler); n != 0 { + t.Errorf("%s: the runner added %d handler(s) to a call that has its own:\n%s", runner, n, mdl) + } + if !strings.Contains(mdl, c.clause) { + t.Errorf("%s: the author's clause %q is gone:\n%s", runner, c.clause, mdl) + } + } + }) + } +} + +// TestOnErrorRollbackIsTheDefaultSpelledOut pins the one clause that is NOT +// kept. `on error rollback` is Mendix's default, the same ErrorHandlingType an +// un-annotated call stores, so `call X()` and `call X() on error rollback` are +// one model and must get one verdict. Keeping the clause would let the error +// escape the test microflow: an ERROR instead of a FAIL, a failed @throws that +// should pass, and under the legacy runner an exception that aborts the whole +// suite's after-startup flow. +func TestOnErrorRollbackIsTheDefaultSpelledOut(t *testing.T) { + // The statement from the report, verbatim, and its microflow twin. + for _, body := range []string{ + "$r = call java action Shop.JA_Upper(Text = 'abc') on error rollback;", + "$r = call microflow Shop.Order_Label(Text = '1') on error rollback;", + "$r = call microflow Shop.Order_Label(Text = '1')\n ON ERROR ROLLBACK;", + } { + for runner, mdl := range generatedBy(t, body) { + mustParse(t, runner, mdl) + if n := strings.Count(mdl, runnerHandler); n != 1 { + t.Errorf("%s: want the runner's handler once in place of `on error rollback`, got %d:\n%s", runner, n, mdl) + } + if strings.Contains(strings.ToLower(mdl), "on error rollback") { + t.Errorf("%s: `on error rollback` was kept beside the runner's handler:\n%s", runner, mdl) + } + } + } +} + +// TestUnhandledCallStillFails is the control for both tests above: a call with +// no clause still gets the handler that turns its exception into a FAIL, and an +// @throws test still gets the one that clears its pre-set failure. +func TestUnhandledCallStillFails(t *testing.T) { + got := generatedBy(t, "$r = call java action Shop.JA_Upper(Text = 'abc');") + for runner, mdl := range got { + mustParse(t, runner, mdl) + if n := strings.Count(mdl, runnerHandler); n != 1 { + t.Errorf("%s: want one runner handler on an unhandled call, got %d:\n%s", runner, n, mdl) + } + } + if !strings.Contains(got["endpoint @expect"], verdictFailPrefix+"exception during execution: ") { + t.Errorf("the endpoint handler does not FAIL the test:\n%s", got["endpoint @expect"]) + } + if !strings.Contains(got["legacy @expect"], "MXTEST:FAIL:") { + t.Errorf("the legacy handler does not FAIL the test:\n%s", got["legacy @expect"]) + } +} + +// TestOnErrorTextThatIsNotAClause pins that the clause is found by parsing, not +// by searching text: `on error` in a string literal or a comment is not a clause, +// so the call is unhandled and gets the runner's handler. +func TestOnErrorTextThatIsNotAClause(t *testing.T) { + for name, body := range map[string]string{ + "in a string literal": "$r = call microflow Shop.Order_Label(Text = 'on error continue');", + "in a line comment": "$r = call microflow Shop.Order_Label(Text = '1'); -- on error continue", + "in a block comment": "$r = call microflow Shop.Order_Label(Text = '1') /* on error continue */;", + "in a log after the call": "$r = call microflow Shop.Order_Label(Text = '1');\n" + + "log info node 'T' 'call microflow X() on error rollback';", + } { + t.Run(name, func(t *testing.T) { + for runner, mdl := range generatedBy(t, body) { + mustParse(t, runner, mdl) + if n := strings.Count(mdl, runnerHandler); n != 1 { + t.Errorf("%s: want exactly one runner handler, got %d:\n%s", runner, n, mdl) + } + } + }) + } +} + +// TestHandlerDecisionIsPerCall pins that the decision is made for each call on +// its own: a handled call beside an unhandled one, on one line or nested in a +// block, leaves the unhandled one wrapped. A call inside the author's own +// handler body has no clause of its own, so it is wrapped like any other. +func TestHandlerDecisionIsPerCall(t *testing.T) { + for name, body := range map[string]string{ + "two calls on one line": "$a = call microflow Shop.Order_Label(Text = '1'); " + + "$r = call microflow Shop.Order_Label(Text = '2') on error continue;", + "handled call inside an if": "if true then\n" + + " $r = call microflow Shop.Order_Label(Text = '1') on error continue;\n" + + " $a = call microflow Shop.Order_Label(Text = '2');\n" + + "end if;", + "call inside the author's handler": "$r = call microflow Shop.Order_Label(Text = '1') on error begin\n" + + " $a = call microflow Shop.Order_Label(Text = '2');\n" + + "end error;", + } { + t.Run(name, func(t *testing.T) { + for runner, mdl := range generatedBy(t, body) { + mustParse(t, runner, mdl) + if n := strings.Count(mdl, runnerHandler); n != 1 { + t.Errorf("%s: want the runner's handler on the one unhandled call, got %d:\n%s", runner, n, mdl) + } + } + }) + } +} + +// TestErrorRaisingCallRulesAreGrammarRules pins errorRaisingCallRules to the +// grammar. The set is matched against rule names from the parse, so a rule +// renamed in MDLMicroflow.g4 would silently stop being wrapped — every error it +// raised escaping the test microflow again (ako/mxcli#1164). +func TestErrorRaisingCallRulesAreGrammarRules(t *testing.T) { + parser.MDLParserInit() // the rule names are filled in lazily + known := map[string]bool{} + for _, name := range parser.MDLParserParserStaticData.RuleNames { + known[name] = true + } + if !known["callMicroflowStatement"] || len(errorRaisingCallRules) == 0 { + t.Fatalf("no rule names to check against; the guard would pass vacuously") + } + for rule := range errorRaisingCallRules { + if !known[rule] { + t.Errorf("errorRaisingCallRules names %q, which is not a rule of the MDL grammar", rule) + } + } +} + +// TestEveryAssertionIsCounted pins the guard against the false green that +// keeping the author's handler opens up. A custom handler that ends normally +// rejoins the flow past every statement reading the failed call's output — the +// assertions — so without a count the test returned PASS having evaluated +// nothing (measured on 11.13.0, ako/mxcli#1448). Each assertion counts itself in +// both branches of its decision, and a guard after the last one fails the test +// when the count is short. +func TestEveryAssertionIsCounted(t *testing.T) { + src := "/**\n * @test handled\n * @expect $r = 'X'\n * @expect $r != empty\n */\n" + + "$r = call java action Shop.JA_Fail(Text = 'a') on error begin\n log error node 'T' 'handled';\nend error;\n/\n" + tc := parseOne(t, src) + suite := &TestSuite{Name: "s", Tests: []TestCase{tc}} + + for runner, c := range map[string]struct { + mdl, counter, guard string + }{ + "endpoint": {GenerateTestFlows(suite), assertionsEvaluatedVar, + "IF $Verdict = '" + verdictPass + "' and " + assertionsEvaluatedVar + " < 2 THEN"}, + "legacy": {GenerateTestRunner(suite), assertionsEvaluatedVar + "_1", + "IF " + assertionsEvaluatedVar + "_1 < 2 THEN"}, + } { + mustParse(t, runner, c.mdl) + inc := "SET " + c.counter + " = " + c.counter + " + 1;" + if n := strings.Count(c.mdl, inc); n != 4 { + t.Errorf("%s: want the counter raised in both branches of both assertions (4), got %d:\n%s", runner, n, c.mdl) + } + g := strings.Index(c.mdl, c.guard) + if g < 0 { + t.Fatalf("%s: no guard %q:\n%s", runner, c.guard, c.mdl) + } + if g < strings.LastIndex(c.mdl, inc) { + t.Errorf("%s: the guard comes before the last assertion:\n%s", runner, c.mdl) + } + if !strings.Contains(c.mdl[g:], escapeMDLString(assertionsSkippedReason)) { + t.Errorf("%s: the guard does not fail the test with the reason:\n%s", runner, c.mdl[g:]) + } + } + + // Control: a test that asserts nothing has nothing to count. + bare := parseOne(t, "/**\n * @test bare\n */\n$r = call java action Shop.JA_Upper(Text = 'a');\n/\n") + for runner, mdl := range map[string]string{ + "endpoint": GenerateTestFlows(&TestSuite{Name: "s", Tests: []TestCase{bare}}), + "legacy": GenerateTestRunner(&TestSuite{Name: "s", Tests: []TestCase{bare}}), + } { + if strings.Contains(mdl, assertionsEvaluatedVar) { + t.Errorf("%s: a test without @expect got an assertion counter:\n%s", runner, mdl) + } + } +} + +// TestLegacyRunWithoutVerdictIsNotAPass pins the log runner's other false green. +// A test whose error leaves the monolithic runner — a call whose own handler +// re-raises, kept as written since ako/mxcli#1448 — aborts the flow after its +// RUN line and before any verdict. The parser created the result on RUN with +// the zero status, which is PASS: measured on 11.13.0 under --legacy-runner, the +// re-raising test reported PASS and the next one "not executed". +func TestLegacyRunWithoutVerdictIsNotAPass(t *testing.T) { + suite := &TestSuite{Name: "s", Tests: []TestCase{ + {ID: "test_1", Name: "passes", Expects: []Expect{expectOf("$a = 1")}}, + {ID: "test_2", Name: "re-raises", Expects: []Expect{expectOf("$b = 1")}}, + {ID: "test_3", Name: "never reached", Expects: []Expect{expectOf("$c = 1")}}, + }} + const log = "MXTEST: MXTEST:START:\n" + + "MXTEST: MXTEST:RUN:test_1:passes\n" + + "MXTEST: MXTEST:PASS:test_1\n" + + "MXTEST: MXTEST:RUN:test_2:re-raises\n" + got := ParseLogResults(strings.NewReader(log), suite, false) + if s := got.Tests[0].Status; s != StatusPass { + t.Errorf("control: test_1 logged PASS, status = %v", s) + } + if s := got.Tests[1].Status; s == StatusPass { + t.Errorf("test_2 logged RUN and no verdict, and reads as PASS") + } else if !strings.Contains(got.Tests[1].Message, "no verdict") { + t.Errorf("test_2: message %q does not say it reported no verdict", got.Tests[1].Message) + } + if s := got.Tests[2].Status; s != StatusError { + t.Errorf("control: test_3 never ran, status = %v", s) + } + if got.AllPassed() { + t.Error("a suite whose runner aborted mid-test passed") + } +} diff --git a/cmd/mxcli/testrunner/results.go b/cmd/mxcli/testrunner/results.go index 228f636f51..905b232d91 100644 --- a/cmd/mxcli/testrunner/results.go +++ b/cmd/mxcli/testrunner/results.go @@ -229,14 +229,21 @@ func ParseLogResults(logReader io.Reader, suite *TestSuite, requireAssertions bo if len(parts) >= 4 { name = parts[3] } + // Not the zero status, which is PASS. A test that logs RUN and no + // verdict did not finish: an error left the runner flow, which + // stops it here — a call whose own handler re-raises does exactly + // that (ako/mxcli#1448). Its PASS or FAIL line replaces this. resultMap[id] = &TestResult{ - ID: id, - Name: name, + ID: id, + Name: name, + Status: StatusError, + Message: "started and reported no verdict: an error left the test runner here, which stopped it (the runtime log has the error)", } case "PASS": if r, ok := resultMap[id]; ok { r.Status = StatusPass + r.Message = "" if t, ok := runTimes[id]; ok { r.Duration = time.Since(t) } diff --git a/cmd/mxcli/testrunner/startup_entity_access_test.go b/cmd/mxcli/testrunner/startup_entity_access_test.go new file mode 100644 index 0000000000..d1c7931a62 --- /dev/null +++ b/cmd/mxcli/testrunner/startup_entity_access_test.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testrunner + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// TestRegisterEndpointAppliesEntityAccess pins the generated startup flow's +// "apply entity access" setting (ako/mxcli#1654, mendixlabs/mxcli#1645). +// +// MxTest.RegisterEndpoint calls the project's own after-startup microflow. +// Written without the annotation it did not apply entity access, and mxbuild +// refuses a non-applying microflow that calls an applying one (CE0114): any +// project whose after-startup microflow applies entity access could not run a +// single test. Measured with mx check on 11.13.0, the rule has one direction — +// non-applying → applying is CE0114, applying → non-applying and applying → +// applying are 0 errors, under security Off and Production alike — so a +// RegisterEndpoint that applies entity access may call either kind. The setting +// governs only its own two activities: the Java action ignores it, and each +// chained microflow runs with its own. +func TestRegisterEndpointAppliesEntityAccess(t *testing.T) { + for _, chain := range []string{"", "MyModule.ASU_Startup"} { + prog, errs := visitor.Build(GenerateEndpointMDL(chain)) + if len(errs) > 0 { + t.Fatalf("chain %q: generated MDL does not parse: %v", chain, errs[0]) + } + var found *ast.CreateMicroflowStmt + for _, s := range prog.Statements { + if mf, ok := s.(*ast.CreateMicroflowStmt); ok && mf.Name.String() == endpointStartupFlow { + found = mf + } + } + if found == nil { + t.Fatalf("chain %q: no %s in the generated MDL", chain, endpointStartupFlow) + } + if found.ApplyEntityAccess == nil || !*found.ApplyEntityAccess { + t.Errorf("chain %q: %s does not apply entity access, so it cannot call an after-startup microflow that does (CE0114)", + chain, endpointStartupFlow) + } + } +} + +// TestBuildErrorInGeneratedStartupFlowIsMxclis pins the attribution half of +// ako/mxcli#1654. The CE0114 above was located in MxTest.RegisterEndpoint — a +// document mxcli generated for the run — and the error said "These errors are in +// the project, not in this run's tests", sending the reader through their own +// model for a defect in mxcli's. +func TestBuildErrorInGeneratedStartupFlowIsMxclis(t *testing.T) { + suite := suiteOf("test_1") + ce0114 := problem("CE0114", + "A microflow that does not apply entity access can only call microflows that also do not apply entity access.", + "MxTest", "Microflow 'RegisterEndpoint'", "Call microflow activity 'ASU_Startup'") + inProject := problem("CE0109", "Undefined variable 'x'.", "Sudoku", "Microflow 'SUB_Deal'", "End event") + + t.Run("only the generated flow", func(t *testing.T) { + _, err := resultsForBuildFailure(buildFailure(failedBuild(ce0114)), suite) + if err == nil { + t.Fatal("no error for a failed build") + } + msg := err.Error() + if strings.Contains(msg, "in the project") { + t.Errorf("an error in a document mxcli generated is blamed on the project:\n%s", msg) + } + for _, want := range []string{endpointStartupFlow, "generated", "mxcli", "--skip-app-startup"} { + if !strings.Contains(msg, want) { + t.Errorf("message does not say %q:\n%s", want, msg) + } + } + }) + + t.Run("the generated flow and the project", func(t *testing.T) { + _, err := resultsForBuildFailure(buildFailure(failedBuild(ce0114, inProject)), suite) + msg := err.Error() + if !strings.Contains(msg, endpointStartupFlow) || !strings.Contains(msg, "in the project") { + t.Errorf("want both attributions:\n%s", msg) + } + }) + + t.Run("control: only the project", func(t *testing.T) { + _, err := resultsForBuildFailure(buildFailure(failedBuild(inProject)), suite) + msg := err.Error() + if !strings.Contains(msg, "These errors are in the project, not in this run's tests.") { + t.Errorf("a project error is no longer attributed to the project:\n%s", msg) + } + if strings.Contains(msg, "generated") { + t.Errorf("a project error is attributed to mxcli:\n%s", msg) + } + }) +} diff --git a/docs-site/src/appendixes/quick-reference.md b/docs-site/src/appendixes/quick-reference.md index 84a271a91c..9469092469 100644 --- a/docs-site/src/appendixes/quick-reference.md +++ b/docs-site/src/appendixes/quick-reference.md @@ -229,7 +229,8 @@ Nested folders use `/` separator: `'Parent/Child/Grandchild'`. Missing folders a | Set security level | `ALTER APP SECURITY ( SecurityLevel: OFF\|PROTOTYPE\|PRODUCTION );` | | | Toggle demo users | `ALTER APP SECURITY ( EnableDemoUsers: TRUE\|FALSE );` | | | Toggle guest access | `ALTER APP SECURITY ( EnableGuestAccess: TRUE, GuestUserRole: UserRole );` / `( EnableGuestAccess: FALSE )` | Anonymous users; role required (CE0133) | -| Rename admin user | `ALTER APP SECURITY ( AdminUserName: 'appadmin' );` | Default `MxAdmin`; the password is not settable from MDL | +| Rename admin user | `ALTER APP SECURITY ( AdminUserName: 'appadmin' );` | Default `MxAdmin` | +| Set admin password | `ALTER APP SECURITY ( AdminPassword: 'N3w-Admin-Passw0rd' );` | Resolves CE0129 "Administrator password has not been set"; `''` is refused | | Create demo user | `CREATE DEMO USER 'name' ( Password: 'pass', [Entity: Module.Entity,] UserRoles: (UserRole, ...) );` | The clause form `PASSWORD … (…)` warns MDL-DEPR137 | | Drop demo user | `DROP DEMO USER [IF EXISTS] 'name';` | `IF EXISTS` makes a cleanup script re-runnable | diff --git a/docs-site/src/internals/catalog-schema.md b/docs-site/src/internals/catalog-schema.md index 54d708dff6..f167b9077a 100644 --- a/docs-site/src/internals/catalog-schema.md +++ b/docs-site/src/internals/catalog-schema.md @@ -289,7 +289,7 @@ rather than trusting a list here: | RefKind | Edge | |---------|------| -| `call` | flow calls a microflow / nanoflow / rule / Java action / REST operation | +| `call` | flow calls a microflow / nanoflow / rule / Java action / REST operation; a workflow calls a handler microflow; a consumed OData service calls its configuration / headers / error-handling microflow | | `create` / `change` / `delete` / `retrieve` | flow acts on an entity object | | `commit` | flow commits an entity object: a commit action, or a create / change that commits (`Yes` or `YesWithoutEvents`), beside its `create` / `change` edge | | `return` | flow returns an entity type | @@ -300,11 +300,11 @@ rather than trusting a list here: | `datasource` | page or widget reads an entity | | `action` | widget calls a microflow / nanoflow | | `show_page` | flow or widget action opens a page | -| `home_page` / `login_page` / `menu_item` | navigation profile references a page | +| `home_page` / `login_page` / `menu_item` | navigation profile references a page, or a microflow (home microflow, menu item that calls one) | | `calculate` | calculated attribute uses a microflow | | `schedule` | scheduled event runs a microflow | -| `publish` | published REST operation runs a microflow | -| `event` | entity event handler runs a microflow | +| `publish` | published service runs a microflow: a REST operation, an OData resource's read / insert / update / delete microflow, an OData action, a REST or OData service's custom authentication microflow | +| `event` | event handler runs a microflow: an entity event, or a business event a service subscribes to | | `settings` | a project setting (after-startup / before-shutdown / health check) names a microflow | | `sync` | offline navigation profile synchronizes an entity | | `validate` | attribute validation rule uses a regular expression | @@ -322,6 +322,11 @@ over), so it would mostly double existing edges. outside the call graph runs the microflow, so nothing in the model calls it. They are what stops `GRAPH_DEAD_ASSETS`, `LIST CALLERS OF` and lint rule QUAL004 from reporting a live scheduled job, API handler or event handler as unused. +`GRAPH_DEAD_ASSETS` and QUAL004 count an edge of any kind; `LIST CALLERS OF` +counts the kinds that invoke. The source of a service's edge is the service +itself (`PUBLISHED_ODATA_SERVICE`, `PUBLISHED_REST_SERVICE`, +`BUSINESS_EVENT_SERVICE`, `CONSUMED_ODATA_SERVICE`) with its catalog id in +`SourceId`; a published REST operation's edge comes from the operation. `sync` is not one — it names an entity a profile downloads, which is a use of a type, so `SHOW CALLERS` excludes it for the same reason it excludes `datasource`. diff --git a/docs-site/src/language/versions.md b/docs-site/src/language/versions.md index d0da629ee7..bc87d7ef0f 100644 --- a/docs-site/src/language/versions.md +++ b/docs-site/src/language/versions.md @@ -231,7 +231,7 @@ refuse the spelling; until then it only warns. ### Changes of meaning (`MDL-V1-*`) -23 constructs mean something different under the `mdl 1;` header. Without the header each keeps the meaning in the second column and warns with its code. +24 constructs mean something different under the `mdl 1;` header. Without the header each keeps the meaning in the second column and warns with its code. | Code | Without the header (mdl 0) | Under `mdl 1;` | Decided at | Rewritten by `fmt --upgrade` | |---|---|---|---|---| @@ -258,6 +258,7 @@ refuse the spelling; until then it only warns. | `MDL-V1-TEMPLATEATTR` | a text-template parameter bound to a non-String attribute (`{1} = $Order.Total`) was stored as `toString($Order/Total)` by earlier mxcli releases | an attribute reference, rendered with the attribute's formatting, as Studio Pro stores it | exec | no: decided by exec against the project; `check -p` reports it | | `MDL-V1-VIEWAUTONUMBER` | a view entity attribute declared `autonumber` over an AutoNumber column passes check, and mxbuild reports CE6770 "View Entity is out of sync with the OQL Query" | a check error that names the attribute and suggests `long`, the type the view gives the column | exec | no: decided by exec against the project; `check -p` reports it | | `MDL-V1-WHILE` | `while ` accepts a body without `begin` and an `end` without `while` | an error: a while loop is `while begin … end while;`, like `loop … begin … end loop;` | parse | yes: `fmt --upgrade --header` | +| `MDL-V1-WIDGETPROP` | A property a built-in page element does not take (`Caption:` on a textbox, `Label:` on a button, a misspelt key) is accepted and dropped on write | a refusal (MDL-WIDGET52), with nothing written | exec | no: decided by exec against the project; `check -p` reports it | ### Deprecated spellings (`MDL-DEPR*`) diff --git a/docs-site/src/language/widget-types.md b/docs-site/src/language/widget-types.md index 6a1c9bddc2..70f3937dfe 100644 --- a/docs-site/src/language/widget-types.md +++ b/docs-site/src/language/widget-types.md @@ -6,6 +6,8 @@ A widget is declared with a type keyword, a unique name, properties in parenthes WIDGET_TYPE widgetName (Property: value, ...) [{ children }] ``` +**Each built-in widget takes its own properties.** A property the widget does not take — `Caption:` on a `TEXTBOX` (an input's label is `Label:`), `Label:` on an `ACTIONBUTTON`, `Visible:` on a `SNIPPETCALL` (Mendix stores no visibility for one), a misspelt key — is refused under `mdl 1;` by `check` and by `exec`, with nothing written (MDL-WIDGET52). Without the header it is dropped as before and warned about (MDL-V1-WIDGETPROP). The message lists the properties that widget takes. + **The list below is not the boundary.** The built-in Mendix widgets are documented here because they have fixed, hand-written property sets. Every **pluggable or custom widget** installed in the project is also written by its own name, with a body derived from the widget's definition — see [Any installed widget](#any-installed-widget) below. If a widget is in `widgets/`, MDL can name it. ## Widget Categories @@ -106,7 +108,7 @@ CREATE PAGE MyModule.Task_Tabs ( | `BadgeParams` | The badge template's parameters, resolved against the enclosing data container | `BadgeParams: ({1} = OpenCount)` | | `Visible` | Visibility condition | `Visible: $Panel/ShowDone` | -A tab page has no `Class`, `Style` or `CaptionParams`: Mendix stores none for it. A `BadgeParams` without a `Badge`, or a `Badge` that is not a quoted template, is refused (MDL-WIDGET51); `Badge` on any other widget is not stored, and `check` says so (MDL-WIDGET07). +A tab page has no `Class`, `Style` or `CaptionParams`: Mendix stores none for it. A `BadgeParams` without a `Badge`, or a `Badge` that is not a quoted template, is refused (MDL-WIDGET51); `Badge` on any other widget is not stored, and is refused (MDL-WIDGET52). ## Data Widgets @@ -322,7 +324,7 @@ All input widgets share common properties: | Property | Description | Example | |----------|-------------|---------| -| `Label` | Field label text | `Label: 'Customer Name'` | +| `Label` | Field label text — not `Caption:`, which is a button's (MDL-WIDGET52) | `Label: 'Customer Name'` | | `Attribute` | Entity attribute to bind to | `Attribute: Name` | | `Editable` | Editability: `Always`, `Never`, `true`/`false`, or a condition expression (bare or quoted) | `Editable: Never` | | `Visible` | Visibility expression | `Visible: '$showField'` | diff --git a/docs-site/src/reference/page/create-layout.md b/docs-site/src/reference/page/create-layout.md index e08e74e8d1..a4c5812ee8 100644 --- a/docs-site/src/reference/page/create-layout.md +++ b/docs-site/src/reference/page/create-layout.md @@ -9,7 +9,7 @@ CREATE [OR REPLACE] LAYOUT module.Name class: 'layout-atlas layout-atlas-responsive-topbar' ) { - SCROLLCONTAINER name { + SCROLLCONTAINER name [( layoutmode: 'Headline' | 'Sidebar' )] { REGION top | right | bottom | left | center ( region_properties ) { widgets } @@ -171,13 +171,20 @@ CREATE OR MODIFY LAYOUT MyModule.Layout_Framed ( - Only web layouts can be built on another layout. - `DESCRIBE LAYOUT` prints this form, with one block per stored argument. +### Scroll container properties + +| Property | Values | Notes | +|----------|--------|-------| +| `layoutmode` | `Headline`, `Sidebar` | The frame. `Headline` runs the top region across the full width and starts the left one under it; `Sidebar` runs the left region the full height and puts the top bar beside it (Atlas's `Atlas_SideBar` and `Atlas_TopBar`). Unset is `Headline`, and `DESCRIBE LAYOUT` prints it only when it is `Sidebar`. Change it on a stored layout with `ALTER LAYOUT … { SET (layoutmode: 'Sidebar') ON layoutContainer; }` | + ### Region properties | Property | Values | Notes | |----------|--------|-------| | `size` | integer | Unset is Studio Pro's `200` | | `sizemode` | `Fixed`, `Pixels`, `Auto` | Unset is `Auto` | -| `class` | CSS class | e.g. `region-topbar`, `region-content` | +| `togglemode` | `None`, `PushContentAside`, `SlideOverContent`, `ShrinkContentInitiallyOpen`, `ShrinkContentInitiallyClosed` | How a `SIDEBARTOGGLE` opens and closes the region. Unset is `None` | +| `class` | CSS class | e.g. `region-topbar`, `region-content`. The only appearance property a region stores: `style` on one is refused (MDL-WIDGET52) | ## The four elements only a layout has @@ -217,16 +224,17 @@ CREATE OR MODIFY LAYOUT MyModule.App_Default }; ``` -A sidebar layout, with a navigation tree in the left region: +A sidebar layout, with a navigation tree in the left region running the full +height (`layoutmode: 'Sidebar'`; leave it out and the top region spans the page): ```sql CREATE LAYOUT MyModule.App_Sidebar ( layouttype: 'Responsive', - class: 'layout-atlas layout-atlas-responsive-default' + class: 'layout-atlas layout-atlas-responsive-sidebar' ) { - SCROLLCONTAINER layoutContainer { + SCROLLCONTAINER layoutContainer (layoutmode: 'Sidebar') { REGION left (size: 232, sizemode: 'Pixels', class: 'region-sidebar') { NAVIGATIONTREE navMenu (profile: 'Responsive') } diff --git a/docs-site/src/reference/page/create-page.md b/docs-site/src/reference/page/create-page.md index 906c7467a0..c5a39bdac7 100644 --- a/docs-site/src/reference/page/create-page.md +++ b/docs-site/src/reference/page/create-page.md @@ -141,6 +141,12 @@ mistake. The same values serve `OnClick:` (an alias of `Action:`) and `OnChange:`. +A flow or page that takes no parameters may be called with an empty argument +list, as a flow body writes it: `Action: CALL MICROFLOW Module.Name()` stores +exactly what `Action: CALL MICROFLOW Module.Name` stores, and `DESCRIBE` writes +it without the brackets. The same holds for a microflow or nanoflow data source +and for `USE FRAGMENT Name()`. + #### Action settings Any action but `NOTHING` takes a `with ( … )` list of the settings Studio Pro diff --git a/docs-site/src/tools/diff.md b/docs-site/src/tools/diff.md index ceced6462a..b2af742c38 100644 --- a/docs-site/src/tools/diff.md +++ b/docs-site/src/tools/diff.md @@ -120,6 +120,31 @@ mxcli diff-local -p app.mpr --ref main..feature-branch mxcli diff-local -p app.mpr --ref main...feature-branch ``` +With three dots the old side is the merge base of the two revisions, labelled +`(merge base of main and feature-branch)`, so the diff shows only what the +branch changed. With two dots it is `main`'s tip, so anything `main` changed +since the branch split also shows up, reversed. An empty side means `HEAD`, as +it does to git (`--ref main..` is `main..HEAD`). + +### What diff-local shows + +Each changed `.mxunit` is rendered as MDL at both revisions and the two +renderings are diffed. Domain models, microflows, nanoflows, pages, snippets, +layouts, enumerations, modules and the project settings render as MDL — a +domain model's entity access rules as the `grant` statements `describe entity` +prints, the settings as `describe settings` prints them, except that configuration +constant values are masked, since they often hold secrets. A changed value +still shows, as `value ` against `value `. + +A unit that changed only in properties its rendering leaves out — navigation, +project or module security, a page's allowed roles, an entity index — is +printed as one line naming the properties that changed, never their values, +rather than counted in the summary and left blank: + +``` +Modified: ModuleSecurity MyFirstModule: changed in properties the MDL rendering does not show: ModuleRoles (1 -> 2 items) +``` + ### MPR v2 Requirement `diff-local` only works with MPR v2 format (Mendix >= 10.18), where documents are stored as individual files. MPR v1 projects store everything in a single SQLite database, making file-level git diff impractical. diff --git a/docs-site/src/tools/playwright.md b/docs-site/src/tools/playwright.md index f51270550b..ef7cd331e4 100644 --- a/docs-site/src/tools/playwright.md +++ b/docs-site/src/tools/playwright.md @@ -80,13 +80,51 @@ page /p/Order_Overview title="Orders" STILL LOADING rows=0 text=37 console- FAIL still loading when measured, so the verdict may describe a partly rendered page (raise --wait if it is only slow) ``` +The sign-in form is not waited for: the client keeps a request open behind it, so +it would otherwise run out the whole `--wait` and then read as still loading. Once +the form holds still it is measured, and it fails as the sign-in form. + A page with a list view, data grid or gallery always reports `rows=`, `rows=0` included, so an empty list does not read the same as a page without one. +When the browser ends up somewhere other than the URL it was sent to, the verdict +says where, as `url=`, right after the page — the title and heading that follow +describe the page at *that* URL. A `/p/` URL that ends anywhere else fails: the +runtime answers every `/p/` with a redirect to `/#/`, and the client sends one +it has no page for to the home page, with HTTP 200 throughout (ako/mxcli#998). The +match is exact, so `/p/Items` and `/p/items/` fail where `/p/items` passes. + +``` +page /p/doesnotexist url=/ title="Mendix - Homepage" h="Welcome to your homepage" text=1925 console-errors=0 + ALERT The page you requested was not found. You are redirected to the home page. + FAIL the browser ended on /, not on /p/doesnotexist: the client sends a /p/ URL it has no page for to the home page +``` + +A page that ends on the sign-in form (`url=/login.html#/items`) fails for that +reason only. + +A page the signed-in user's roles may not open ends on the sign-in form too, but +with `messageCode=403` in the URL: the runtime refuses the client's session request +for that page with 403, and the client sends the browser to +`/login.html?messageCode=403#/`. The form shows no message, so the URL is the +only thing that tells it from "not signed in". It fails as access denied, naming who +the check signed in as; signing in again would get the same answer, so the check +does not: + +``` +page /p/secret url=/login.html?messageCode=403#/secret ACCESS DENIED title="Login" h="Sign in" text=34 console-errors=1 + ERR [Startup] An error occurred while creating a new session. i_: Internal server error … + HTTP 403 POST /xas/ + FAIL access denied: signed in as demo_user (user role User), the runtime refused /p/secret (403) +``` + +Check the page's allowed module roles (`mxcli -p app.mpr -c "show access on page +Module.Page"`) against the user role's (`describe user role R`). + | Exit status | Meaning | |---|---| | 0 | every page passed | -| 1 | a page failed: still loading when measured, HTTP error, sign-in form instead of the page (or a 401 from the runtime), error banner or error dialog, console error, failed same-origin request, or a failed `--assert-*` | +| 1 | a page failed: still loading when measured, HTTP error, access denied to the signed-in user (403), sign-in form instead of the page (or a 401 from the runtime), a `/p/` URL that ended elsewhere (no page has that URL), error banner or error dialog, console error, failed same-origin request, or a failed `--assert-*` | | 2 | the check could not run (no Node/Playwright, sign-in rejected, bad flag) | A warning banner or validation message is shown but does not fail the page. @@ -100,12 +138,35 @@ A warning banner or validation message is shown but does not fail the page. | `-p app.mpr` only | a demo user of the project (an administrator first), when demo users are enabled | | none | nobody; a secured page fails with "the sign-in form is showing" | +The sign-in happens on the **host of the pages checked**, not on `--base-url`: a +session's cookies belong to the host they were set on, so one signed in on +`localhost` is not sent to `127.0.0.1` or to another name for the same app. A page +given as a full URL on another host (`mxcli playwright check http://frontend.local:8180/ +-p app.mpr`) gets a session of its own; before ako/mxcli#1216 it was checked with +the base URL's session, showed the sign-in form on every run, and signed in again +on each. + The session is saved as a Playwright storage state under `/.mxcli/playwright-check/-.json` (or `~/.mxcli/playwright-check/` without `-p`) and reused by the next check, so repeated checks do not sign in again — -on the unlicensed runtime every sign-in takes one of a handful of session slots. When -the runtime no longer knows the session (it restarted), the check signs in once more -and reloads only the affected pages. `--fresh-login` ignores the saved session. +on the unlicensed runtime every sign-in takes one of a handful of session slots, and +a reused session stays one session however many checks run. When the runtime no +longer knows the session (it restarted), the check signs in once more and reloads +only the affected pages. + +- `--fresh-login` signs the saved session out, then signs in again. +- `--sign-out` ends the run signed out and keeps nothing for the next check — for a + last check in a script that should leave no session behind. +- A session the run signed in that got no page past the sign-in form is signed out + rather than kept: kept, every later check would sign in again beside it. A page + the user's roles may not open does not count — the runtime knew the session and + refused the page — so that session is kept and reused, and is not renewed either. + +Signing out is best effort and never changes the verdict; a sign-out that does not +complete is reported on stderr, and that session expires on its own. When the +runtime refuses a sign-in, the check reads `/.mxcli/runtime.log` for the +cause — the unlicensed runtime's session cap shows on the sign-in page only as +"Sign in failed". The sign-in types into the Mendix login form, so the "setting `.value` leaves the field empty" problem of hand-written `eval` logins does not arise. When the app root @@ -138,8 +199,9 @@ mxcli playwright check /p/Customer_Overview -p app.mpr \ | `--assert-text` | | Text the page must contain (repeatable) | | `--assert-count` | | `SELECTOR OP N` that must hold (repeatable) | | `--screenshot` | | Also write a PNG here; the path is printed | -| `--wait` | `4s` | Longest a page may keep changing after it mounts before it is measured | -| `--fresh-login` | `false` | Ignore the saved session | +| `--wait` | `15s` | Longest a page may keep loading after it mounts (open requests, changing text) before it is measured; the sign-in form is measured as soon as it holds still | +| `--fresh-login` | `false` | Sign the saved session out and sign in again | +| `--sign-out` | `false` | Sign out when the check is done instead of keeping the session | | `--allow-console-errors` | `false` | Report console errors without failing on them | The check needs Node and the Playwright package (`require('playwright')` or the diff --git a/docs-site/src/tools/run-local.md b/docs-site/src/tools/run-local.md index afe42c28b4..11291d140c 100644 --- a/docs-site/src/tools/run-local.md +++ b/docs-site/src/tools/run-local.md @@ -34,7 +34,7 @@ stopped: pid 30133 in 300ms | Command | Does | Exit | |---|---|---| | `run --local … --detach` | starts the loop in its own session (it survives the calling shell), writes `.mxcli/run-state.json` and `.mxcli/run.log`, returns once the app serves. A failed boot prints the reason and the last log lines. A second run for the same project is refused. | 0 / 1 | -| `run status` | one line: URL, pid, uptime, last build and how it was applied, a build in progress, a change still pending. A state file whose process is gone is reported as stopped, with any leftover processes. | 0 running, 3 stopped, 4 starting | +| `run status` | one line: URL, pid, uptime, last build and how it was applied, a build in progress, a change still pending. A state file whose process is gone is reported as stopped, with any leftover processes. A run whose runtime stopped on its own says why when its log does — `stopped: trial licence time limit reached at 18:43:16 (runtime.log); restart with mxcli run restart -p …`. | 0 running, 3 stopped, 4 starting | | `run wait [--timeout 5m]` | blocks until the watch loop has applied the model as it is now, or failed to; prints `applied: build #N via reload\|restart in …` or `failed:` with the CE codes / SCSS error lines. `--ready` waits for the boot instead; `--since N` for any build newer than N. | 0 applied, 1 failed, 2 timeout, 3 not running | | `run stop` | asks the run to shut down, waits until it is gone, then stops anything left in its process session — so no orphaned `runtimelauncher` or mxbuild JVM, even from a run killed with `-9`. Nothing running is a no-op. | 0 | | `run restart` | `run stop`, then `--detach` again with the arguments recorded in the state file. | as `--detach` | @@ -53,6 +53,24 @@ a change, and `wait` says so (exit 1). terminal — every run publishes its state. On Windows, `run stop` ends the process tree without a graceful shutdown. +### `.mxcli/run-local.json` + +Besides `run-state.json`, a run publishes **`.mxcli/run-local.json`** (mode 0600): the +app and admin ports, the admin password and the boot configuration — how `mxcli oql`, +`mxcli log`, `constant set --apply`, and any tool that calls the admin API, reach the +app. + +- **Written** once the app serves, carrying the run's pid. +- **Removed** when that run ends — a requested stop, or the runtime stopping on its own + (the trial licence's time limit, a crash) — immediately after `run-state.json` + records `"phase": "stopped"`, `"ended"` and a one-line `"reason"`. `run stop` + removes it for a run that was killed outright. No other process removes it: a + `run --local --setup` (the SessionStart hook) or a refused second run leaves it alone. +- **Missing?** Check `mxcli run status -p app.mpr`, not the credentials. It reports a + stopped run with its reason, or a live run whose handshake is gone (`mxcli run + restart` republishes it). A 401/403 from the admin API without the handshake means + the connection fell back to default settings; `oql` and `log` say so. + ## Why The Docker path (`mxcli docker run`) rebuilds a full deployment package and restarts @@ -101,6 +119,16 @@ again` when it happens, so a browser test knows to sign in before its next step. `~/.mxcli/postgres/server.log`. Stop it with `pg_ctl -D "$HOME/.mxcli/postgres/data" stop`. To remove it, stop it first and then delete `~/.mxcli/postgres` (this permanently deletes its databases). + The fallback never replaces a **stopped system cluster** for the same port + (`/var/lib/postgresql//`, `/var/lib/pgsql/data`, + `/var/lib/postgres/data`, Homebrew's): a new cluster there would hold none of the + app's data while looking healthy, so `--ensure-db` stops, names the data + directory and prints the command that starts it (e.g. + `sudo pg_ctlcluster 16 main start`). If that cluster is not this app's and cannot + be started, `MXCLI_ENSURE_DB_NEW_CLUSTER=1` creates the user-owned cluster anyway. + Whenever a new cluster is initialized, the output says + `WARNING: created a NEW, EMPTY PostgreSQL cluster`; an existing user-owned cluster + is restarted without it. A server whose `postgres` superuser has a password — **every Windows install**, where the installer always sets one — needs that login: `--ensure-db --db-admin-user postgres --db-admin-password `, or set diff --git a/docs-site/src/tools/running-tests.md b/docs-site/src/tools/running-tests.md index 0a11a18815..b457152ab7 100644 --- a/docs-site/src/tools/running-tests.md +++ b/docs-site/src/tools/running-tests.md @@ -356,6 +356,14 @@ This keeps a suite behaving the same under `--local` and `--attach`. Note that what the startup microflow writes is not covered by `@cleanup rollback` — it runs at boot, outside any test's transaction. +The generated startup flow, `MxTest.RegisterEndpoint`, **applies entity +access**, so it can call your after-startup microflow whichever way that one is +set: Mendix refuses a microflow that does not apply entity access calling one +that does (CE0114), and allows the other direction. Your microflow still runs +with its own setting. A build error located in `MxTest.RegisterEndpoint` or +`MxTest.RegisterTestEndpoint` is reported as mxcli's, not your project's — those +documents exist only for the run. + ### `@cleanup`: what happens to a test's data `rollback` is the **default**, so a test's database writes do not survive it. diff --git a/docs-site/src/tools/starlark-rules.md b/docs-site/src/tools/starlark-rules.md index ed0dd18617..59dc74bf9d 100644 --- a/docs-site/src/tools/starlark-rules.md +++ b/docs-site/src/tools/starlark-rules.md @@ -159,11 +159,15 @@ list holding a number) is not silently ignored — the rule runs with its defaul and the run reports a warning naming the option, listed with the rules that could not run and kept out of the score. -QUAL004 counts a microflow as called only when something **in the model** runs -it (a call, a schedule, a page or widget, a published service, a project -setting). A call from an MDL test script (`tests/*.test.mdl`) is not in the +QUAL004 counts a microflow or page as used when anything in the catalog's +reference graph points at it — exactly what `mxcli refs` lists: a call, a page +or widget, the navigation (a home page or home microflow, a menu item), a +schedule, a published service, an event handler, a project setting. Every edge +kind counts, including ones the catalog adds later, so the rule cannot fall +behind it. A call from an MDL test script (`tests/*.test.mdl`) is not in the model, so a helper used only by tests is reported; give such helpers a prefix -and list it in `entry_point_prefixes`. +and list it in `entry_point_prefixes`. If `refs` finds nothing for a document +you know is live, the gap is in the catalog, not the rule. ### CONV018 options {#conv018-options} diff --git a/docs-site/src/tools/test-annotations.md b/docs-site/src/tools/test-annotations.md index 2d1c3e9b83..4cd744b272 100644 --- a/docs-site/src/tools/test-annotations.md +++ b/docs-site/src/tools/test-annotations.md @@ -153,6 +153,35 @@ a `change`, a `create` — is not caught; move it into a microflow and call that The same handler turns an unexpected error in a test without `@throws` into a FAIL reading `exception during execution: `. +### A call with its own error handling + +A call that already has an `on error` clause keeps it and gets no second +handler, so a test runs the statement exactly as production code does. What the +test then reports depends on the clause (measured on Mendix 11.13 with a Java +action that throws): + +| Clause on the call | Without `@throws` | With `@throws` | +|---|---|---| +| none | FAIL `exception during execution: ` | PASS when the message matches | +| `on error rollback` | same as none — it is Mendix's default spelled out, so the runner's handler takes its place | same as none | +| `on error continue` | the error is swallowed; the assertions decide (`$r` is empty) | FAIL `expected an exception but none was thrown` | +| custom handler that ends normally | FAIL `not every assertion was evaluated` when an assertion reads the call's result; an assertion on what the handler did is evaluated | FAIL `expected an exception but none was thrown` | +| custom handler that re-raises (`raise error`) | ERROR with the message — the error leaves the test microflow | ERROR with the message | + +A custom handler that ends normally rejoins the flow after every statement that +reads the failed call's output, because that variable does not exist on the +error path. The assertions read exactly that, so the runner counts the +assertions it evaluated and fails the test when the count is short, instead of +reporting a pass for assertions that never ran. To test the handler's path, +assert on something the handler sets: `declare $Failed Boolean = false;` before +the call, `set $Failed = true;` inside its handler, and `@expect $Failed = true`. +That assertion does not read the call's result, so it is evaluated, and passes +only when the handler ran. + +Under the Docker / `--legacy-runner` path an error that leaves the test — the +re-raising row above — ends the whole runner: that test is reported as an ERROR +that started and gave no verdict, and every later test as not executed. + ## `@setup` Names a **microflow** to call before the test's own statements — a fixture in a diff --git a/docs-wiki/SYNC_LOG.md b/docs-wiki/SYNC_LOG.md index baa9e0f23a..833a20ecd2 100644 --- a/docs-wiki/SYNC_LOG.md +++ b/docs-wiki/SYNC_LOG.md @@ -88,3 +88,5 @@ not capture, because sources are upstream of the commit. | 2026-10-10 | bug-patterns/expression-translation-drift.md | .claude/skills/fix-issue/findings/ (same 32 records), mdl/visitor/visitor_xpath_quotes.go, plus the sources already listed | **Re-sync** (three beats added, rest unchanged). The page already carried the string-escape and source-interval material from the fixes themselves. New: `GetText()` has the opposite defect from a source-interval slice and a worse one — it concatenates visible tokens with nothing between them, so `true and false` stored as `trueandfalse`; the grep is `Expression().*GetText()`, and the reason it recurs is that the sites are grouped by document type rather than by what they do. Precedence is part of the translation (`/` shares a level with `*`/`div`/`mod`, so a member path after one parses as division), and the fix belongs where the AST is built. And position decides whether a quote is a name or a value: Mendix XPath never compares two members, so a quoted token after a comparison operator is always a mis-quoted string — which is why refusing it applies under every language version | | 2026-10-10 | bug-patterns/misleading-diagnostics.md | .claude/skills/fix-issue/findings/ (same 32 records), mdl/visitor/visitor.go (enhanceErrorMessage), plus the sources already listed | **Re-sync** (three beats added, rest unchanged). A bare parse error reads as a missing feature and gets filed as one — `Height:` on an annotation was reported as mxcli failing to expose a property Mendix does not store, and settling that needed four sources, the generated metamodel being insufficient because it is one version's snapshot (`mx dump-mpr` and decisively `mx convert` are the cheap oracles). When the grammar is right and the mistake is predictable from the source line, the fix is a hint keyed on the line rather than a grammar change, because accepting the input would need its own resolution path. And a hint must stand down on a line that already failed, or error recovery turns into a second phantom mistake | | 2026-10-23 | bug-patterns/package-operations-damage.md | .claude/skills/fix-issue/findings/cmd-mxcli/ (21 package-operation records, 2026-07-31 → 2026-10-22: update-widgets / module-import / rename-design-properties collapse, mx writes on check and build, bundled files ×4, widget rollback, CE0066 after transplant, cross-module association cascade, relative -p guard, diff inconclusive output, refcache action sources, 6.0.0 unpublished, newer-Studio-Pro mpk, foreign .mpr adjacency, install runs fix widgets, work-dir path too long), cmd/mxcli/marketplace/update.go, cmd/mxcli/marketplace/workdir.go, cmd/mxcli/marketplace/scratch.go, docs-site/src/guides/marketplace.md | **Re-sync** after mendixlabs/mxcli#1486/#1487. New beats: the package's `package.xml` `` list is the payload contract, and the bundled-file step guessed wrong three times before reading it (mx module-import with a before/after listing is the oracle); replacing a module is a delete whose cascade removes other modules' associations, and a transplant arrives unreconciled (reconcile only where stale); order steps so a failure is not half an install; inconclusive describe output, cache placeholders and miss-then-hit tests under "never unchanged"; mx's 260-char path limit is mx's own, so the scratch directory is mxcli's choice — shortest root the user owns, never a world-writable drive root. "Path too long" moved out of "environment failures". Dropped the quoted finding count; `covers:` adds cmd/mxcli/marketplace | +| 2026-10-27 | bug-patterns/diffs-narrower-than-the-change.md | .claude/skills/fix-issue/findings/mdl-executor/ (the diff-local records, 2026-09-10 → 2026-10-27: #1080 ×2, #1393 ×2, #1512/#1602, #1562/#1630, the git directory, the three-dot range and rename detection, and #997/#913 for `diff`), mdl/executor/cmd_diff_local.go, cmd_diff.go, cmd_diff_mdl.go | **New page.** Five shapes of one defect, "N modified" with no hunk: doctype stubs, an identical parse failure on both sides, a storage-name read, a doctype with no case, an unprinted part of a rendered doctype. Beats: a placeholder (or a mask) that does not vary with its input erases the change; close the class with a note naming the changed paths when the renderings agree; decode through the backend, not off the map; the `$Type` dispatch is containment-ordered; a test must change what the renderer cannot see; where git runs, the range's old side and rename detection are part of the contract | +| 2026-10-27 | bug-patterns/local-loop-silence.md | .claude/skills/fix-issue/findings/cmd-mxcli/ (the six 2026-10-27 records: ensure-db empty cluster #866, Gradle daemon #1340, run-local.json owner #1542, playwright check host #1216, unknown /p/ URL #998, sign-in form wait), cmd/mxcli/devloop_handshake.go, cmd/mxcli/docker/{runtime_stop_reason,ensuredb,javadeps,mxenv,pagecheck,pagecheck_check}.go | **Re-sync** (five beats added, rest unchanged), after the triage board's local-loop-silence cluster was fixed. New: a fallback whose success is the defect (decide before initdb; EACCES on PG_VERSION is evidence of a cluster; print the uid in a privilege-dropping harness); a shared file needs an owner (only the pid it names may remove it; the stop reason is in runtime.log, newest boot only); a one-shot Gradle run must neither leave a daemon nor wait on one without a bound (append to GRADLE_OPTS, never replace it); a signed-in check is a (host, session) pair; a settle rule for one page holds every page to it, and the final URL is the verdict's subject | diff --git a/docs-wiki/bug-patterns/diffs-narrower-than-the-change.md b/docs-wiki/bug-patterns/diffs-narrower-than-the-change.md new file mode 100644 index 0000000000..43b0123fb4 --- /dev/null +++ b/docs-wiki/bug-patterns/diffs-narrower-than-the-change.md @@ -0,0 +1,101 @@ +--- +title: A Diff Is Only as Wide as Its Renderer +category: bug-pattern +last-synced: cf9d5f0c +covers: + - mdl/executor +sources: + - .claude/skills/fix-issue/findings/mdl-executor/ + - mdl/executor/cmd_diff_local.go + - mdl/executor/cmd_diff.go + - mdl/executor/cmd_diff_mdl.go +--- + +> **Do not duplicate**: what `diff` and `diff-local` print, and which unit types +> have a renderer, is in `.claude/commands/mendix/diff-local.md` and +> `docs-site/src/tools/diff.md`. The individual fixes are in the findings. This +> page is about why "N modified" with no hunk keeps coming back. + +## What this is + +`diff-local` does not compare documents. It renders both revisions of a unit as +MDL and compares the **text**. So it can only show a change that its renderer +prints. Whatever the renderer leaves out comes out the same on both sides, and +`Current == Proposed` swallows it. Git, meanwhile, has already counted the file +as modified. The result is the signature of the class: + +``` +Summary: 0 new, 1 modified, 0 deleted +``` + +and no hunk. `-f struct` says `(no changes)` under the same count. + +This has shipped in five shapes: a renderer stub for a whole doctype (page, +nanoflow and layout bodies), a parse that failed identically on both sides (the +microflow stub when the engine lacked `ParseMicroflowFromRaw`), a property read +under its SDK name instead of its storage name (attribute `Type` vs `NewType`), +a doctype with no case at all (project settings, constants, module security), +and a part of a rendered doctype that was never printed (entity access rules). + +## How it fits + +**A placeholder that does not vary with its input erases the change it stands +for.** Each stub (`-- (page widgets)`, `-- parse failed --`, the generic +`-- : ` header) was written to degrade visibly. Both sides degrade +the same way, so the comparison sees nothing. The same trap applies to masking +a secret: a fixed `'***'` hides a changed value as completely as no renderer at +all. A mask for a diff must differ where the values differ and still give +nothing away. The settings renderer renders each side knowing the other's value, +so a changed constant prints as `` / `` +and an unchanged one as ``. + +**Close the class, not the case.** Adding renderers one doctype at a time (#1080, +#1393, #1602, #1630) never ends, because the next unprinted property is silent +again. Git has already said the bytes differ, so when the two renderings agree +the documents are compared instead, and the unit prints one line naming the +property paths that changed (`ChangedPaths` in `cmd_diff.go`, shared with +`mxcli diff`). It never prints the values, because the doctypes without a +renderer include the ones that hold secrets. The renderers still matter, because +a path list is no review. The note makes sure that a missing renderer is +visible, not silent. + +**Decode through the backend, never off the map.** BSON keys are storage names: +an access rule's roles are `AllowedModuleRoles`, an attribute's type is +`NewType`. A hand-written map reader is one wrong key away from printing nothing +on both sides. The `Parse…FromRaw` backend methods decode a `git show` revision +through the codec, and the `describe` renderer then prints it. One renderer then +serves both commands, which is also the remedy for +[duplicate-resolver drift](duplicate-resolver-drift.md). + +**The `$Type` dispatch is ordered by containment.** `Microflows$Nanoflow` +contains "Microflow", `Security$ModuleSecurity` contains "Module". A +`strings.Contains` case catches every type whose name includes its needle. Match +an exact type with `==`, and before adding a needle, list which other types +contain it. + +**A test for this class must make the documents differ where the renderer is +blind.** A test that changes something the renderer already prints passes +against the bug. A mock parser that accepts any `$Type` hides dispatch faults. +Make mocks refuse foreign documents the way the backend does. The cheapest real +repro is a copy of `testdata/pedapp`, `git init`, exec and commit, then +`diff-local --ref X~1..X`. The pre-fix binary prints the bare summary. + +**Where git runs, and what it is asked, are part of the contract.** Git's paths +are relative to the repository root, and git used to run in the process's +working directory. The two only line up when the working directory is the root +and the project sits at it. That is where every early repro ran, so `-p` from +anywhere else failed or read the wrong files. The same goes for the range: the +old side of `a...b` is the merge base, not `a`, and only a repository where both +branches changed the same unit after the split tells the two apart. Rename +detection is a third: a unit's file name is its ID, so a "renamed" file is one +unit deleted and another added. + +## See also + +- [describe-round-trip-gaps](describe-round-trip-gaps.md): the renderer + `diff-local` borrows has its own gaps, and every one of them is also a + diff-local blind spot. +- [duplicate-resolver-drift](duplicate-resolver-drift.md): `diff` once rendered + its two sides with two different renderers. +- [fix-issue findings](../../.claude/skills/fix-issue/findings/mdl-executor/): + `grep -l diff-local` for the instances. diff --git a/docs-wiki/bug-patterns/duplicate-resolver-drift.md b/docs-wiki/bug-patterns/duplicate-resolver-drift.md index b78ddc922f..b651590b81 100644 --- a/docs-wiki/bug-patterns/duplicate-resolver-drift.md +++ b/docs-wiki/bug-patterns/duplicate-resolver-drift.md @@ -84,6 +84,19 @@ next rehearsal found phantoms on 66 scripts (#907). The remedy was the general one taken literally: diff now runs exec itself on a scratch copy and compares units, so there is no second answer left to drift. +`diff-local` is the same shape with no exec to run instead: the older revision +is not a project anything can DESCRIBE, so it renders each unit's BSON with its +own per-doctype renderers, and every doctype they render lossily was counted +as modified and printed nothing — a stub body (mendixlabs/mxcli#1080, #1393), +a type with no case (project settings, ako/mxcli#1512), a member never printed +(entity access rules, ako/mxcli#1562). A probe of one statement per kind found +nine more silent unit kinds. The class is bounded at +the comparison, not per renderer: git has already said the bytes differ, so +identical renderings can only mean a lossy renderer, and the unit is reported +with the property paths that changed — the walker `mxcli diff` uses, now +shared. A renderer is extended by running the describer's own writer on the +parsed old document (`Parse…FromRaw`), never by writing a second one. + *A hand-kept list of statement kinds.* The most prolific variant found this cycle, because a `switch` over AST types is how everything in the executor is wired. `check`'s project tier resolves the module of every module-scoped diff --git a/docs-wiki/bug-patterns/local-loop-silence.md b/docs-wiki/bug-patterns/local-loop-silence.md index 89e4534824..12ca1fdba7 100644 --- a/docs-wiki/bug-patterns/local-loop-silence.md +++ b/docs-wiki/bug-patterns/local-loop-silence.md @@ -1,7 +1,7 @@ --- title: The Warm Loop Fails Quietly category: bug-pattern -last-synced: 5c24d899 +last-synced: b68b6e26 covers: - cmd/mxcli sources: @@ -13,6 +13,13 @@ sources: - cmd/mxcli/docker/webclient_pages.go - cmd/mxcli/docker/serve_recovery.go - cmd/mxcli/devloop_admin.go + - cmd/mxcli/devloop_handshake.go + - cmd/mxcli/docker/runtime_stop_reason.go + - cmd/mxcli/docker/ensuredb.go + - cmd/mxcli/docker/javadeps.go + - cmd/mxcli/docker/mxenv.go + - cmd/mxcli/docker/pagecheck.go + - cmd/mxcli/docker/pagecheck_check.go --- > **Do not duplicate**: the flags and their behaviour live in @@ -141,6 +148,51 @@ boot path, grep for the other. `mxcli exec` is mid-script can serve a partial model, and re-running the script fixes nothing because the write is idempotent and there is nothing left to write. +**A fallback that succeeds can be the defect.** `--ensure-db` fell back to a new +user-owned cluster when the system one would not start. Once that fallback worked +on Debian, it produced a database that passes every later check (role, database, +boot, health) and is empty. The only point where "new" and "already there" can be +told apart is the moment before `initdb`. So decide there, and announce every +`initdb` loudly. Detect an existing cluster by its data directory, not its +service. For a non-root user the directory is `0700 postgres`, so `EACCES` on +`PG_VERSION` is evidence that a cluster exists. A probe that only counts a +successful stat misses the devcontainer user. Any harness that drops privileges +should print its uid first: a control binary that silently ran as root started +the real cluster and provisioned into it. + +**A file other processes read needs an owner.** `.mxcli/run-local.json` was +removed by a deferred cleanup in *every* `run --local` process. That included +`--setup`, which the SessionStart hook runs on every resume and compaction, so a +live detached run lost its handshake. Callers then got a 403 and blamed the +password. The 403 was the clue: it needs a runtime that is up. Grep every remover +of a shared path and ask which processes reach it. Only the process whose pid the +file names may remove it. Reasons follow the same rule as logs (above). The +trial-licence stop is an application log line, so it is only in `runtime.log`, +and only the newest boot's section of that file may be blamed. + +**A one-shot that runs Gradle must neither leave a daemon nor wait on one without +a bound.** This is three findings with one cause (#1396, #1340 and an earlier +unbounded mxbuild). Append `-Dorg.gradle.daemon=false` to `GRADLE_OPTS`, never +replace the variable: replacing it drops a proxy setting and becomes mx's +misleading "Failed to find some hosts". Then run the step under the process-tree +watchdog with a progress line. The tell in the report was that Mendix's own `mx` +was fast on the same project: same tool, different state. `mxbuild --serve` +keeps its daemon on purpose. + +**A signed-in check is a (host, session) pair, not an app.** `playwright check` +signed in on the handshake's `localhost` and then probed the URL as typed. Cookies +belong to a host, so `127.0.0.1` (which the boot output prints) or a vhost got the +sign-in form on every run, and each retry added another session. Count sessions +(`runtime_statistics`) rather than trusting the verdict. + +**A settle rule written for one page holds every page to it.** The probe waits +for no open request because a list view's data source shows nothing else (#845). +The client keeps a `POST /xas/` open behind the sign-in form, so the same rule +spent the whole `--wait` there and then advised raising it. Exempt a state that +is a verdict in itself. And verify *where* the browser ended: an unknown `/p/` +URL is redirected to the home page with HTTP 200, so the final path is the only +signal that does not depend on the app's language. + ## See also - [fix-issue findings](../../.claude/skills/fix-issue/findings/) — the individual diff --git a/docs-wiki/bug-patterns/silent-property-drop.md b/docs-wiki/bug-patterns/silent-property-drop.md index 705bb3bf40..9210a75232 100644 --- a/docs-wiki/bug-patterns/silent-property-drop.md +++ b/docs-wiki/bug-patterns/silent-property-drop.md @@ -7,6 +7,7 @@ covers: sources: - .claude/skills/fix-issue/findings/mdl-executor/ - mdl/executor/validate_widgets.go + - mdl/executor/widget_native_props.go - modelsdk/widgets/definitions/ --- @@ -84,6 +85,22 @@ wrote would be dropped (mendixlabs/mxcli#1346). A `go/parser` scan of the builders' `Get*Prop` / `lookupPropCI` / `Properties["…"]` reads plus describe's `"Key: "` strings found four more keys in the same state on its first run. +**The union outlived its reason once the scan could be per builder.** A union +cannot tell a key from the widget it belongs to, so `Caption:` on a text box — +a button's property, the label of an input is `Label:` — passed for as long as +the union existed (ako/mxcli#1698), and so did every key in the class before it. +The same `go/parser` scan, run as a *taint walk* from each builder's widget +argument through the helpers it is handed to, yields each widget's own set; a +test then requires the per-kind table to equal it both ways, so a missing key is +a false refusal the test reports and a stale one is a silent drop it reports. +Two things the walk must know: a helper that reads a key only to *name* the +widget in an error is not consumption (the message helper read `Caption`, which +made every input appear to take it), and the common tail's writes are bounded by +the stored type, not the builder — `Visible:` reaches the Go struct of every +widget and is encoded only where the gen type has a visibility setter, so a +snippet call, scroll container or menu bar dropped it too. With the list derived, +the warning could become a refusal. + **Allow-list membership is not evidence about *this* widget.** The union is deliberately cross-widget, so a key being on it proves only that *some* builder consumes it — `Tooltip:` was parsed, listed, present as a struct field and @@ -116,8 +133,13 @@ re-declaring it, and when a property is missing, bisect by layer with the cheapest probe at each — a visitor probe says whether the AST carried it, a `bson dump` says whether disk did, and the gap between them names the hop. -**Warn; do not reject.** Neither the pluggable nor the built-in vocabulary can be -proven complete, so an error would trade silent drops for false refusals. The +**Warn on a vocabulary that is guessed; refuse on one that is derived.** A +hand-kept list cannot be proven complete, so an error on it trades silent drops +for false refusals. Where the list is *read off the code that writes* — the +built-in widgets' per-builder sets above, the pluggable engine's mapping below — +membership is the definition and a refusal cannot be false. Measure the blast +radius before switching anyway: the example corpus, every docs block, and +describe → check of real projects all came back at zero for #1698. The same reasoning appears wherever mxcli names a *specific* alternative — those lists are deliberately explicit rather than inferred, because inferring them would mean reimplementing the engine's dispatch and getting it wrong in the other diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index 387e049f69..ca481edf43 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -719,7 +719,8 @@ Nested folders use `/` separator: `'Parent/Child/Grandchild'`. Missing folders a | Toggle demo users | `alter app security ( EnableDemoUsers: true\|false );` | Several keys may go in one list | | Enable guest access | `alter app security ( EnableGuestAccess: true, GuestUserRole: UserRole );` | Anonymous users. The role is what visitors get — its entity access is the public surface. Mendix fails the build without one (CE0133), so `on` is refused unless a role is given or already stored. mxcli validates the role exists; Mendix does not | | Disable guest access | `alter app security ( EnableGuestAccess: false );` | Keeps the stored role, so re-enabling needs no `GuestUserRole` | -| Rename admin user | `alter app security ( AdminUserName: 'appadmin' );` | The built-in administrator (default `MxAdmin`), read by lint as `project_security().admin_user_name`. Its password is not settable from MDL | +| Rename admin user | `alter app security ( AdminUserName: 'appadmin' );` | The built-in administrator (default `MxAdmin`), read by lint as `project_security().admin_user_name` | +| Set admin password | `alter app security ( AdminPassword: 'N3w-Admin-Passw0rd' );` | Resolves **CE0129** "Administrator password has not been set", which mxbuild raises for an empty one at prototype and production level. `''` is refused; mxbuild does not check it against the password policy. Stored in the model as written, as Studio Pro stores it. `describe app security` says whether one is set | | Create demo user | `create demo user 'name' ( Password: 'pass', [Entity: Module.Entity,] UserRoles: (UserRole, ...) );` | The clause form `password … (…)` warns MDL-DEPR137 | | Drop demo user | `drop demo user [if exists] 'name';` | `if exists` makes a cleanup script re-runnable | | Update security | `update security [[in] Module];` | Re-syncs access rules with their domain model — Studio Pro's **Update security** button, headless. Repairs **CE0066** "Entity access is out of date", which a model authored elsewhere can carry (a module imported or updated outside Studio Pro). Not needed after mxcli's own writes: every write path reconciles as it writes. Writes nothing when the rules already match, and skips `System` | @@ -1516,7 +1517,7 @@ MDL uses explicit property declarations for pages: | Attribute binding | `attribute: AttrName` | `textbox txt (label: 'Name', attribute: Name)` | | Attribute over an association | `attribute: Assoc/Attr` (bare association name, multi-hop OK) | `textbox txt (label: 'Rule', attribute: RuleAction_BusinessRule/Name)` — works on textbox, textarea, datepicker, dropdown, checkbox and radiobuttons, the same as on a data grid column | | Password field | `Password: true` on a textbox | `textbox tbPw (attribute: Secret, Password: true)` — omitted when false. Without it a describe → exec round trip turns a password field into a plaintext one | -| Submit while typing | `SubmitBehaviour: OnEndEditing \| WhileEditing`, `SubmitOnInputDelay: ` on a textbox or textarea | `textbox tbSearch (attribute: Query, SubmitBehaviour: WhileEditing, SubmitOnInputDelay: 500)` — defaults `OnEndEditing` / `300`, omitted by describe when default. An invalid value is MDL-WIDGET18; on any other widget the keys draw MDL-WIDGET07 | +| Submit while typing | `SubmitBehaviour: OnEndEditing \| WhileEditing`, `SubmitOnInputDelay: ` on a textbox or textarea | `textbox tbSearch (attribute: Query, SubmitBehaviour: WhileEditing, SubmitOnInputDelay: 500)` — defaults `OnEndEditing` / `300`, omitted by describe when default. An invalid value is MDL-WIDGET18; on any other widget the keys are refused (MDL-WIDGET52) | | Layout grid width | `Width: FullWidth \| FixedWidth` on a layoutgrid | `layoutgrid lg (Width: FixedWidth) { … }` — default `FullWidth`, omitted by describe when default | | Widget validation | `Validation: ''`, `ValidationMessage: ''` | `Validation: 'length(toString($value)) > 0'` — a Mendix expression over `$value`, QUOTED not bracketed (`[...]` is the XPath spelling and parses as an array) | | Variable binding | `datasource: $Var` | `dataview dv (datasource: $Product) { ... }` | diff --git a/mdl-examples/bug-tests/624-alter-app-security-admin-password-empty.fail.mdl b/mdl-examples/bug-tests/624-alter-app-security-admin-password-empty.fail.mdl new file mode 100644 index 0000000000..4748751776 --- /dev/null +++ b/mdl-examples/bug-tests/624-alter-app-security-admin-password-empty.fail.mdl @@ -0,0 +1,5 @@ +mdl 1; +-- mendixlabs/mxcli#624: an empty administrator password is what CE0129 +-- reports, so writing one can never be the fix. Refused at parse time: +-- AdminPassword cannot be empty — an empty administrator password is what CE0129 reports +alter app security ( AdminPassword: '' ); diff --git a/mdl-examples/bug-tests/624-alter-app-security-admin-password.mdl b/mdl-examples/bug-tests/624-alter-app-security-admin-password.mdl new file mode 100644 index 0000000000..7ea9cf3de2 --- /dev/null +++ b/mdl-examples/bug-tests/624-alter-app-security-admin-password.mdl @@ -0,0 +1,16 @@ +mdl 1; +-- mendixlabs/mxcli#624 (ako/mxcli#1259) — CE0129 "Administrator password has +-- not been set" could only be resolved in Studio Pro: `alter app security` had +-- no AdminPassword property. +-- +-- Measured on Mendix 11.13.0 (testdata/pedapp with AdminPassword blanked): +-- security level off, 0 extra errors; production, one extra error +-- [CE0129] "Administrator password has not been set." at Security +-- A password that violates the project's password policy ('1' against a +-- 12-character minimum) adds no error at any level. +-- +-- After this script `mx check` reports the same errors as the untouched app, +-- and `describe app security` prints "Admin Password: set" (never the value). +-- Re-running it reports "Unchanged admin password". + +alter app security ( SecurityLevel: production, AdminPassword: 'N3w-Admin-Passw0rd' ); diff --git a/mdl-examples/bug-tests/catalog-1450-prefix-in-string-literals.mdl b/mdl-examples/bug-tests/catalog-1450-prefix-in-string-literals.mdl new file mode 100644 index 0000000000..e694a38745 --- /dev/null +++ b/mdl-examples/bug-tests/catalog-1450-prefix-in-string-literals.mdl @@ -0,0 +1,46 @@ +mdl 1; +-- ============================================================================ +-- ako/mxcli#1450 (mendixlabs/mxcli#1553): CATALOG. stripped inside literals +-- ============================================================================ +-- +-- Symptom (before fix): +-- A catalog query removed the text `Catalog.` everywhere in the query, not +-- only from the CATALOG. prefix, so a module named Catalog could not +-- be filtered on by qualified name: +-- select QualifiedName, 'Catalog.Product' as lit ... -> lit = 'Product' +-- ... where QualifiedName = 'Catalog.Product' -> 0 rows +-- The same rewrite also touched quoted identifiers and comments, and +-- `CATALOG . ENTITIES` (whitespace around the dot, which the grammar +-- accepts) failed with "no such table: CATALOG.ENTITIES". Every entry point +-- was affected: -c, scripts, the REPL, and the LSP (which runs -c). +-- +-- After fix: +-- Only the table prefix is rewritten; string literals, quoted identifiers +-- and comments reach SQLite verbatim. The first select returns +-- lit = 'Catalog.Product', the count is 1, and the last two each return the +-- one row Catalog.Product. +-- +-- Usage: +-- mxcli exec mdl-examples/bug-tests/catalog-1450-prefix-in-string-literals.mdl -p app.mpr +-- ============================================================================ + +create module Catalog; + +create persistent entity Catalog.Product ( + Name: String(100) +); + +-- A module whose name merely contains the word: must not be confused with it. +create module ProductCatalog; + +create persistent entity ProductCatalog.Item ( + Code: String(50) +); + +-- The issue's two queries. +select QualifiedName, 'Catalog.Product' as lit from CATALOG.ENTITIES where ModuleName = 'Catalog'; +select count(*) as n from CATALOG.ENTITIES where QualifiedName = 'Catalog.Product'; + +-- Lower-case prefix, a join of two catalog tables, and whitespace at the dot. +select e.QualifiedName from catalog.entities e join CATALOG.MODULES m on m.Name = e.ModuleName where e.QualifiedName = 'Catalog.Product'; +select QualifiedName as "Catalog.Name" from CATALOG . ENTITIES where QualifiedName = 'Catalog.Product'; diff --git a/mdl-examples/bug-tests/catalog-1639-service-microflow-refs.mdl b/mdl-examples/bug-tests/catalog-1639-service-microflow-refs.mdl new file mode 100644 index 0000000000..413006b8c6 --- /dev/null +++ b/mdl-examples/bug-tests/catalog-1639-service-microflow-refs.mdl @@ -0,0 +1,140 @@ +mdl 1; +-- ============================================================================ +-- Issue ako/mxcli#1639 — the catalog has no edge for the microflows a +-- published OData service runs, so `refs`, `impact` and QUAL004 call them dead +-- ============================================================================ +-- +-- Named with a topic prefix rather than a bare `1639-`: the historical files +-- here are named after mendixlabs/mxcli PR numbers, and the fork's issue +-- numbers collide with them. +-- +-- A published OData resource's `ReadMode: microflow X` and the service's +-- `Authentication: (microflow X)` are entry points: a client's request reaches +-- them, nothing in the model calls them. The catalog recorded neither as an +-- edge, so on the reporter's backend: +-- +-- mxcli refs Formula1Backend.Read_Drivers +-- (no references found: nothing in the catalog's reference graph points at …) +-- mxcli lint +-- ℹ Microflow 'Read_Drivers' is not called from anywhere. [QUAL004] +-- → Remove if unused, … +-- +-- for 34 Read_ microflows and the authentication microflow of six services — +-- 35 of the backend's 43 QUAL004 findings. Same gap, same mechanism, on the +-- siblings below: an OData action, a published REST service's authentication +-- microflow, a business event handler. (A consumed OData service's +-- configuration microflows got the same fix; they need a reachable $metadata +-- to author, so they are covered by unit tests, not here.) +-- +-- ---------------------------------------------------------------------------- +-- Running this example +-- +-- mxcli exec catalog-1639-service-microflow-refs.mdl -p app.mpr +-- mxcli -p app.mpr -c "refresh catalog full force" +-- mxcli refs Svc.Read_Rows -p app.mpr +-- # PUBLISHED_ODATA_SERVICE | Svc.LiveAPI | publish +-- mxcli refs Svc.AuthenticateApiClient -p app.mpr # same +-- mxcli refs Svc.OD_Recalculate -p app.mpr # same +-- mxcli refs Svc.AuthenticateRestClient -p app.mpr +-- # PUBLISHED_REST_SERVICE | Svc.PingAPI | publish +-- mxcli refs Svc.OnCustomerChanged -p app.mpr +-- # BUSINESS_EVENT_SERVICE | Svc.CustomerEvents | event +-- mxcli refs Svc.NoCaller -p app.mpr +-- # control: (no references found …) +-- mxcli lint -p app.mpr +-- # QUAL004 in Svc: Svc.NoCaller only +-- +-- Custom authentication also needs app security on (CE6600) before mx check +-- is clean; the edges do not depend on it. +-- ============================================================================ + +create module Svc; + +create non-persistent entity Svc.LiveRow ( RowKey: string(60), Label: string(200) ); + +create microflow Svc.Read_Rows () +returns List of Svc.LiveRow as $Rows +begin + $Rows = create list of Svc.LiveRow; + return $Rows; +end; + +create microflow Svc.AuthenticateApiClient ( + $Headers: List of System.HttpHeader +) +returns System.User as $User +begin + return empty; +end; + +create microflow Svc.OD_Recalculate ( $Note: string ) +returns Boolean as $Ok +begin + return true; +end; + +create published odata service Svc.LiveAPI ( + path: 'odata/live/', + version: '1.0.0', + ODataVersion: OData4, + namespace: 'Svc.LiveAPI', + ServiceName: 'LiveAPI', + Authentication: (microflow Svc.AuthenticateApiClient) +) +{ + publish entity Svc.LiveRow as 'Rows' ( + ReadMode: microflow Svc.Read_Rows, + InsertMode: not_supported, UpdateMode: not_supported, DeleteMode: not_supported, + Countable: false, TopSupported: false, SkipSupported: false + ) + expose ( RowKey as 'rowKey' (KEY), Label as 'label' ) + publish microflow Svc.OD_Recalculate as 'Recalculate' + expose ( Note as 'note' ); +}; + +create microflow Svc.PRS_Ping ( $httpRequest: System.HttpRequest ) +returns String as $R +begin + return 'pong'; +end; + +create microflow Svc.AuthenticateRestClient ( + $httpRequest: System.HttpRequest +) +returns System.User as $User +begin + return empty; +end; + +create published rest service Svc.PingAPI ( + Path: 'rest/ping/v1', + Version: '1.0.0', + ServiceName: 'Ping API', + Authentication: (microflow Svc.AuthenticateRestClient) +) +{ + resource 'ping' { + get '' microflow Svc.PRS_Ping; + } +}; + +create persistent entity Svc.BE_CustomerChanged ( CustomerId: long ); + +create microflow Svc.OnCustomerChanged ( $Event: Svc.BE_CustomerChanged ) begin + log info 'changed'; +end; + +create business event service Svc.CustomerEvents +( + ServiceName: 'CustomerEvents', + EventNamePrefix: '' +) +{ + message CustomerChanged (CustomerId: long) subscribe + entity Svc.BE_CustomerChanged microflow Svc.OnCustomerChanged; +}; + +-- control: nothing runs this one +create microflow Svc.NoCaller () begin + log info 'never'; +end; diff --git a/mdl-examples/bug-tests/difflocal-1512-settings-silent-modified.mdl b/mdl-examples/bug-tests/difflocal-1512-settings-silent-modified.mdl new file mode 100644 index 0000000000..27ed835f3b --- /dev/null +++ b/mdl-examples/bug-tests/difflocal-1512-settings-silent-modified.mdl @@ -0,0 +1,34 @@ +mdl 1; +-- ako/mxcli#1512 (mendixlabs/mxcli#1602) — after a settings change, +-- `mxcli diff-local` printed only +-- Summary: 0 new, 1 modified, 0 deleted +-- and `--format struct` said "(no changes)". The project settings unit had no +-- renderer: it fell to the generic case, two comment lines naming the unit +-- type, which read the same at every revision. +-- +-- The settings unit now renders with DESCRIBE SETTINGS' own writer, with +-- configuration constant values masked. And any unit whose rendering is the +-- same at both revisions (navigation, project or module security, a page's +-- access, an entity index, …) is printed as one line naming the properties +-- that changed, instead of nothing. +-- +-- Repro on an MPR v2 app in git: +-- 1. mxcli exec this script, commit. +-- 2. Run the commented statements below (uncommented), then from the app's +-- folder: mxcli diff-local -p App.mpr (and with --format struct) +-- Expected after the fix: +-- -alter settings constant 'MyFirstModule.DiffLocalApiKey' value +-- +alter settings constant 'MyFirstModule.DiffLocalApiKey' value +-- - BcryptCost: 12, (or whatever the app had) +-- + BcryptCost: 11, +-- Modified: ModuleSecurity MyFirstModule: changed in properties the MDL +-- rendering does not show: ModuleRoles (1 -> 2 items) +-- and neither constant value appears anywhere in the output. + +create or modify constant MyFirstModule.DiffLocalApiKey (Type: String, DefaultValue: 'x'); +alter settings constant @MyFirstModule.DiffLocalApiKey value 'first-secret' in configuration 'Default'; + +-- Step 2: +-- alter settings constant @MyFirstModule.DiffLocalApiKey value 'secret123' in configuration 'Default'; +-- alter settings runtime (BcryptCost: 11); +-- create module role MyFirstModule.DiffLocalReviewer; diff --git a/mdl-examples/bug-tests/difflocal-1562-entity-grant-silent-modified.mdl b/mdl-examples/bug-tests/difflocal-1562-entity-grant-silent-modified.mdl new file mode 100644 index 0000000000..cde73934cb --- /dev/null +++ b/mdl-examples/bug-tests/difflocal-1562-entity-grant-silent-modified.mdl @@ -0,0 +1,29 @@ +mdl 1; +-- ako/mxcli#1562 (mendixlabs/mxcli#1630) — after a `grant` on an entity, +-- `mxcli diff-local` counted the domain model as modified and printed +-- Summary: 0 new, 1 modified, 0 deleted +-- and nothing else. Its domain-model rendering printed each entity's +-- attributes and the associations, but not the entity access rules, so a +-- script that only changed access had nothing to review. +-- +-- The rules now render after each entity with DESCRIBE ENTITY's own grant +-- writer, read from a parse of each revision of the unit. +-- +-- Repro on an MPR v2 app in git: +-- 1. mxcli exec this script, commit. +-- 2. Run the commented statement below (uncommented), then from the app's +-- folder: mxcli diff-local -p App.mpr +-- Expected after the fix: +-- create persistent entity MyFirstModule.DiffLocalItem ( +-- Name: String(100) +-- ); +-- +-- -grant read * on entity MyFirstModule.DiffLocalItem to MyFirstModule.DiffLocalUser; +-- +grant read *, write * on entity MyFirstModule.DiffLocalItem to MyFirstModule.DiffLocalUser; + +create module role MyFirstModule.DiffLocalUser; +create or modify persistent entity MyFirstModule.DiffLocalItem (Name: String(100)); +grant read * on entity MyFirstModule.DiffLocalItem to MyFirstModule.DiffLocalUser; + +-- Step 2: +-- grant read *, write * on entity MyFirstModule.DiffLocalItem to MyFirstModule.DiffLocalUser; diff --git a/mdl-examples/bug-tests/layout-1703-scrollcontainer-layoutmode.mdl b/mdl-examples/bug-tests/layout-1703-scrollcontainer-layoutmode.mdl new file mode 100644 index 0000000000..06fe8ef8f9 --- /dev/null +++ b/mdl-examples/bug-tests/layout-1703-scrollcontainer-layoutmode.mdl @@ -0,0 +1,50 @@ +mdl 1; +-- ============================================================================ +-- ako/mxcli#1703 (mendixlabs/mxcli#1659): a scroll container's LayoutMode +-- ============================================================================ +-- +-- Symptom (before fix): +-- `describe layout Atlas_Core.Atlas_SideBar` printed `scrollcontainer +-- layoutContainer {` with no LayoutMode, and `create layout` could not write +-- one, so the documented describe → rename → exec copy of Atlas_SideBar +-- (stored LayoutMode "Sidebar") came out "Headline": a different frame, with +-- Atlas's sidebar CSS positioned for the other one. `(LayoutMode: 'Sidebar')` +-- on the scroll container passed check and exec and was dropped. mx check 0 +-- errors either way. +-- +-- Fix: +-- Forms$ScrollContainer.LayoutMode is read by the builder (Headline | +-- Sidebar, the metamodel's PagesLayoutMode, matched case-insensitively and +-- stored as spelled there), written by the modelsdk writer, settable with +-- `alter layout … set (LayoutMode: …) on `, and printed by +-- describe when it is Sidebar (Headline is Studio Pro's default for an unset +-- one). Before this, the per-widget property check of ako/mxcli#1698 refused +-- the key (MDL-WIDGET52) rather than dropping it. +-- +-- Expected: check and exec pass; `mxcli bson dump --type layout --object +-- BugLayout.App_SideBar` shows LayoutMode: "Sidebar"; describe prints +-- `scrollcontainer layoutContainer (LayoutMode: 'Sidebar') {`; mx check 0 errors. +-- +-- Usage: +-- mxcli exec mdl-examples/bug-tests/layout-1703-scrollcontainer-layoutmode.mdl -p app.mpr +-- ============================================================================ + +create module BugLayout; + +create or modify layout BugLayout.App_SideBar ( + layouttype: 'Responsive', + class: 'layout-atlas layout-atlas-responsive-sidebar' +) { + scrollcontainer layoutContainer (LayoutMode: 'Sidebar') { + region top (Size: 48, Class: 'border-bottom') { + dynamictext brand (Content: 'App') + } + region left (Size: 232, SizeMode: 'Pixels', ToggleMode: 'ShrinkContentInitiallyOpen', Class: 'region-sidebar') { + sidebartoggle toggle (Tooltip: 'Toggle Menu', ButtonStyle: Primary) + navigationtree nav (Profile: 'Responsive') + } + region center (Class: 'region-content') { + placeholder Main + } + } +}; diff --git a/mdl-examples/bug-tests/pagecheck-1216-explicit-url-other-host.mdl b/mdl-examples/bug-tests/pagecheck-1216-explicit-url-other-host.mdl new file mode 100644 index 0000000000..04720ae6fd --- /dev/null +++ b/mdl-examples/bug-tests/pagecheck-1216-explicit-url-other-host.mdl @@ -0,0 +1,30 @@ +-- ako/mxcli#1216: `mxcli playwright check -p app.mpr --role R` failed a +-- page it had just signed into ("the sign-in form is showing, not the page"), +-- while the same command without the URL passed. +-- +-- With -p the base URL comes from the run handshake, http://localhost:, +-- and the sign-in ran there; a page URL naming another host for the same app +-- received none of localhost's cookies, so the client started anonymous and +-- showed the sign-in form. Each failing run also signed in again, one more +-- session on the unlicensed runtime per run, until its cap refused the sign-in +-- — and then said nothing, because the check never told the login where the +-- runtime log is. +-- +-- No /etc/hosts entry is needed: `run --local` prints http://127.0.0.1:/ +-- while the handshake says localhost, so the boot URL itself reproduces it. +-- Against a blank 11.13.0 app: +-- +-- mxcli exec mdl-examples/bug-tests/pagecheck-1216-explicit-url-other-host.mdl -p app.mpr +-- mxcli run --local --detach -p app.mpr --db-type hsqldb --app-port 8180 +-- mxcli playwright check http://127.0.0.1:8180/ -p app.mpr --role User +-- mxcli playwright check -p app.mpr --role User # control: OK before and after +-- +-- Before the fix the first check printed `login demo_user`, then +-- `FAIL the sign-in form is showing, not the page (pass --user/--password or --role)` +-- and exited 1, and every run added a session (M2EE runtime_statistics, +-- user_sessions). After it the check signs in on 127.0.0.1:8180, passes, and +-- later runs reuse that session. +mdl 1; + +alter app security ( SecurityLevel: prototype ); +grant view on page MyFirstModule.Home_Web to MyFirstModule.User; diff --git a/mdl-examples/bug-tests/pagecheck-403-role-denied.mdl b/mdl-examples/bug-tests/pagecheck-403-role-denied.mdl new file mode 100644 index 0000000000..f84bdb2dd3 --- /dev/null +++ b/mdl-examples/bug-tests/pagecheck-403-role-denied.mdl @@ -0,0 +1,74 @@ +-- `mxcli playwright check` signed in, then checked a page the role may not +-- open, and told the user to sign in. +-- +-- The runtime refuses the signed-in client's session request for such a page +-- (POST /xas/ get_session_data, referrer /secret, HTTP 403), and the client +-- sends the browser to /login.html?messageCode=403#/secret: the sign-in form, +-- with no message on it (login.js parses the code wrongly, so the "not enough +-- permissions" text it has never shows). An anonymous browser gets a 401 and +-- /login.html#/secret, the same form with no messageCode; so does a session +-- the runtime has forgotten, with guest access on or off. Measured on 11.13.0 +-- and 11.15.0; messageCode=403 in the final URL is the only signal. +-- +-- Against a blank app, with this script applied: +-- +-- mxcli exec mdl-examples/bug-tests/pagecheck-403-role-denied.mdl -p app.mpr +-- mxcli run --local --detach -p app.mpr --db-type hsqldb --app-port 8180 +-- mxcli playwright check /p/secret /p/items -p app.mpr --role User +-- +-- Before the fix: +-- +-- login demo_user (--role User) +-- page /p/secret url=/login.html?messageCode=403#/secret LOGIN PAGE title="Login" h="Sign in" text=34 console-errors=1 +-- ERR [Startup] An error occurred while creating a new session. i_: Internal server error … +-- HTTP 403 POST /xas/ +-- FAIL the sign-in form is showing, not the page (pass --user/--password or --role) +-- FAIL 1 console error(s) +-- FAIL 1 failed request(s) +-- page /p/items title="Mendix - Items" h="Item overview" text=191 console-errors=0 +-- FAIL 1 of 2 page(s) +-- +-- and every later run printed `login demo_user` again: the 403 read as a +-- session the runtime had forgotten, so the check signed in beside the live +-- session it had just used (M2EE runtime_statistics: 1, 2, 3 sessions over +-- three runs). Checked alone, /p/secret had its session signed out after +-- every run, as one that "got no page past the sign-in form". +-- +-- After it (first run): +-- +-- login demo_user (--role User) +-- page /p/secret url=/login.html?messageCode=403#/secret ACCESS DENIED title="Login" h="Sign in" text=34 console-errors=1 +-- ERR [Startup] An error occurred while creating a new session. i_: Internal server error … +-- HTTP 403 POST /xas/ +-- FAIL access denied: signed in as demo_user (user role User), the runtime refused /p/secret (403) +-- page /p/items title="Mendix - Items" h="Item overview" text=191 console-errors=0 +-- FAIL 1 of 2 page(s) +-- +-- exit 1 still; later runs print no `login` line and the runtime holds one +-- session throughout, /p/secret checked alone included. +mdl 1; + +alter app security ( SecurityLevel: prototype ); + +create or modify module role MyFirstModule.Admin; + +create or modify page MyFirstModule.Items_Web ( + Title: 'Items', + Layout: MyFirstModule.App_Default, + Url: 'items' +) { + container c1 { dynamictext t1 (Content: 'Item overview', RenderMode: H1) } +}; + +create or modify page MyFirstModule.Secret_Web ( + Title: 'Secret', + Layout: MyFirstModule.App_Default, + Url: 'secret' +) { + container c1 { dynamictext t1 (Content: 'Secret overview', RenderMode: H1) } +}; + +grant view on page MyFirstModule.Home_Web to MyFirstModule.User; +grant view on page MyFirstModule.Items_Web to MyFirstModule.User; +-- No user role holds MyFirstModule.Admin, so every demo user is refused this page. +grant view on page MyFirstModule.Secret_Web to MyFirstModule.Admin; diff --git a/mdl-examples/bug-tests/pagecheck-998-unknown-page-url.mdl b/mdl-examples/bug-tests/pagecheck-998-unknown-page-url.mdl new file mode 100644 index 0000000000..bd0371976a --- /dev/null +++ b/mdl-examples/bug-tests/pagecheck-998-unknown-page-url.mdl @@ -0,0 +1,29 @@ +-- ako/mxcli#998: `mxcli playwright check / /p/doesnotexist` reported the home +-- page's verdict for a page that does not exist, and exited 0. +-- +-- Measured on 11.13.0 and 11.15.0: the runtime answers any /p/ with +-- `301 Location: /#/`, known page or not. The client then replaces the +-- location with /p/ when a page has that URL, and with / when none does — +-- rendering the home page and an information dialog ("The page you requested +-- was not found. You are redirected to the home page."), with HTTP 200 all +-- the way. The check now reports the location the browser ended on (`url=/`) +-- and fails a /p/ URL that did not stay put. +-- +-- Against a blank app: +-- +-- mxcli exec mdl-examples/bug-tests/pagecheck-998-unknown-page-url.mdl -p app.mpr +-- mxcli run --local --detach -p app.mpr --db-type hsqldb +-- mxcli playwright check / /p/doesnotexist /p/items /p/ITEMS -p app.mpr +-- +-- Before the fix: `OK 4 page(s)`, /p/doesnotexist and /p/ITEMS showing the home +-- page's title and heading. After it: both report `url=/` and fail +-- ("the browser ended on /, not on /p/doesnotexist"), /p/items passes, exit 1. +mdl 1; + +create or modify page MyFirstModule.Items_Web ( + Title: 'Items', + Layout: MyFirstModule.App_Default, + Url: 'items' +) { + container c1 { dynamictext t1 (Content: 'Item overview', RenderMode: H1) } +}; diff --git a/mdl-examples/bug-tests/pagecheck-signin-form-waits-out-wait.mdl b/mdl-examples/bug-tests/pagecheck-signin-form-waits-out-wait.mdl new file mode 100644 index 0000000000..d2acffd18a --- /dev/null +++ b/mdl-examples/bug-tests/pagecheck-signin-form-waits-out-wait.mdl @@ -0,0 +1,24 @@ +mdl 1; +-- `mxcli playwright check` on a page showing the sign-in form waited out the +-- whole --wait, then reported the form as still loading on top of the sign-in +-- failure. Measured on 11.13.0 and 11.15.0 with no session: +-- WAIT not settled after 30.2s: still waiting for POST /xas/ +-- FAIL still loading when measured, … (raise --wait if it is only slow) +-- FAIL the sign-in form is showing, not the page (…) +-- 33s at --wait 30s. The client keeps a POST /xas/ open behind the form, so +-- the probe's "no request open" condition never held. +-- +-- Repro: exec this on a blank app, `mxcli run --local --detach`, then +-- mxcli playwright check http://localhost:8080/p/secret --wait 30s +-- (no -p, so no sign-in). After the fix it answers in about 3s, exit 1, with +-- the sign-in form as its only failure. + +create or modify page MyFirstModule.Secret_Web ( + Title: 'Secret', + Layout: Atlas_Core.Atlas_Default, + Url: 'secret' +) { + container c1 { dynamictext t1 (Content: 'Secret overview', RenderMode: H1) } +}; + +alter app security ( SecurityLevel: prototype ); diff --git a/mdl-examples/bug-tests/qual004-1484-navigation-microflows.mdl b/mdl-examples/bug-tests/qual004-1484-navigation-microflows.mdl new file mode 100644 index 0000000000..881983abea --- /dev/null +++ b/mdl-examples/bug-tests/qual004-1484-navigation-microflows.mdl @@ -0,0 +1,69 @@ +mdl 1; +-- ============================================================================ +-- Issue ako/mxcli#1484 (upstream mendixlabs/mxcli#1579) — QUAL004 reports a +-- microflow started only from the navigation as "not called from anywhere" +-- ============================================================================ +-- +-- Named with a topic prefix rather than a bare `1484-`: the historical files +-- here are named after mendixlabs/mxcli PR numbers, and the fork's issue +-- numbers collide with them. +-- +-- A profile's home microflow and a menu item's `call microflow` are entry +-- points: the client starts them, nothing in the model calls them. The catalog +-- records both (`home_page` / `menu_item` edges, which `mxcli refs` lists), but +-- QUAL004 decided "has a caller" from its own list of reference kinds, +-- MICROFLOW_ENTRY_KINDS, which had the page kinds for the navigation and not the +-- microflow ones. So it said: +-- +-- ℹ Microflow 'NAV_Home' is not called from anywhere. [QUAL004] +-- at Shop.Shop.NAV_Home +-- → Remove if unused, ... +-- +-- The rule now counts every edge into a document — the same graph `refs` and +-- CATALOG.GRAPH_DEAD_ASSETS read — so a kind the catalog adds later cannot fall +-- behind it. (The doubled module in `at Shop.Shop.NAV_Home` was a separate +-- defect in the lint location, fixed alongside.) +-- +-- ---------------------------------------------------------------------------- +-- Running this example (QUAL004 is a Starlark rule: the project needs +-- .claude/lint-rules/orphaned_elements.star, which `mxcli init` writes) +-- +-- mxcli exec qual004-1484-navigation-microflows.mdl -p app.mpr +-- mxcli -p app.mpr -c "refresh catalog full force" +-- mxcli refs Shop.NAV_Orders -p app.mpr +-- # NAVIGATION | Navigation.Responsive | menu_item +-- mxcli lint -p app.mpr +-- # expected in Shop: QUAL004 for Shop.NAV_Unused only, `at Shop.NAV_Unused` +-- # before the fix: also Shop.NAV_Home and Shop.NAV_Orders +-- ============================================================================ + +create module Shop; + +-- started only from the navigation +create microflow Shop.NAV_Home () +begin + log info node 'Shop' 'home'; +end; + +create microflow Shop.NAV_Orders () +begin + log info node 'Shop' 'orders'; +end; + +-- control: no caller at all — still reported, also with a NAV_ name +create microflow Shop.NAV_Unused () +begin + log info node 'Shop' 'never'; +end; + +-- control: no caller, but an entry-point prefix exempts it by name +create microflow Shop.ACT_Entry () +begin + log info node 'Shop' 'act'; +end; + +create or modify navigation Responsive + home microflow Shop.NAV_Home +{ + menu item 'Orders' ( OnClick: call microflow Shop.NAV_Orders ) +}; diff --git a/mdl-examples/bug-tests/rest-grant-1486-service-in-new-folder.mdl b/mdl-examples/bug-tests/rest-grant-1486-service-in-new-folder.mdl new file mode 100644 index 0000000000..6ab5549a98 --- /dev/null +++ b/mdl-examples/bug-tests/rest-grant-1486-service-in-new-folder.mdl @@ -0,0 +1,112 @@ +mdl 1; +-- ============================================================================ +-- GRANT on a published REST service created in a new folder: "not found" +-- ako/mxcli#1486 (mirror of mendixlabs/mxcli#1581) +-- ============================================================================ +-- +-- `grant access on published rest service Rest1486.PingApi to Rest1486.User` +-- failed with "published rest service not found: Rest1486.PingApi" when the +-- service had been created with a `folder '…'` clause earlier in the same +-- script, and exec rolled the whole script back. `check` passed. Without the +-- folder clause, or with a folder that already existed, or with the grant in a +-- second exec, the same grant resolved. +-- +-- Cause: exec caches the container hierarchy (module <- folder <- folder) and +-- resolves every document's module through it. The CREATE built that cache in +-- its own existence check, then created the folder, and nothing recorded the +-- folder in the cache: walking up from the service ended at an unknown folder, +-- the module name came back "", and the lookup missed. Creating a folder was +-- the one container-changing write that did not keep the cache in step; it now +-- records the folder (createFolder, mdl/executor/helpers.go). +-- +-- Same mechanism, same fix: a scheduled event, task queue, regular expression +-- or consumed REST service created in a new folder could not be described, +-- dropped or re-run later in the same script — and a re-run CREATE OR MODIFY +-- missed its own document and silently created a second one of the same name. +-- +-- Expected after exec: every statement succeeds; the service is in folder +-- 'Api/Services' with Rest1486.User allowed; each `create or modify` re-run +-- reports "Unchanged"; SHOW PUBLISHED REST SERVICES IN Rest1486 lists one +-- service; mx check reports 0 errors (the service has an allowed role, so no +-- CE0338). Before the fix: "Error: published rest service not found" and +-- nothing written. +-- Go test: mdl/executor/hierarchy_created_folder_test.go. +-- ============================================================================ + +create module Rest1486; + +create module role Rest1486.User; + +create microflow Rest1486.PRS_Ping ($httpRequest: System.HttpRequest) +returns String as $Result +begin + return 'pong'; +end; + +create microflow Rest1486.SE_Run () +returns Boolean as $Result +begin + return true; +end; + +-- The reported case: the CREATE makes the folder, the GRANT looks it up. +create or modify published rest service Rest1486.PingApi folder 'Api/Services' ( + Path: 'rest/ping/v1', + Version: '1.0.0', + ServiceName: 'PingApi' +) +{ + resource 'ping' { + get '' microflow Rest1486.PRS_Ping; + } +}; + +grant access on published rest service Rest1486.PingApi to Rest1486.User; + +-- Re-running the CREATE OR MODIFY must find the service, not add a second one. +create or modify published rest service Rest1486.PingApi folder 'Api/Services' ( + Path: 'rest/ping/v1', + Version: '1.0.0', + ServiceName: 'PingApi' +) +{ + resource 'ping' { + get '' microflow Rest1486.PRS_Ping; + } +}; + +revoke access on published rest service Rest1486.PingApi from Rest1486.User; +grant access on published rest service Rest1486.PingApi to Rest1486.User; + +-- Siblings: each creates its own new folder, then is looked up by name. +create or modify scheduled event Rest1486.NightlyRun folder 'Jobs/Nightly' ( + Microflow: Rest1486.SE_Run, + Repeat: Daily, + HourOfDay: 4, + MinuteOfHour: 0, + TimeZone: Server, + Enabled: false, + StartDateTime: '2026-01-01T04:00:00Z' +); +describe scheduled event Rest1486.NightlyRun; + +create or modify task queue Rest1486.Work folder 'Jobs/Queues' (Parallelism: 2, ClusterWide: true); +describe task queue Rest1486.Work; + +create or modify regular expression Rest1486.Digits folder 'Validation/Patterns' ( + Expression: '[0-9]+' +); +describe regular expression Rest1486.Digits; + +create or modify consumed rest service Rest1486.EchoApi folder 'Api/Clients' ( + BaseUrl: 'https://example.org', + authentication: none +) +{ + operation GetStatus ( + method: get, + path: '/echo', + response: none + ) +}; +describe consumed rest service Rest1486.EchoApi; diff --git a/mdl-examples/bug-tests/test-runner-1448-own-error-handler.test.mdl b/mdl-examples/bug-tests/test-runner-1448-own-error-handler.test.mdl new file mode 100644 index 0000000000..7bf7545c7e --- /dev/null +++ b/mdl-examples/bug-tests/test-runner-1448-own-error-handler.test.mdl @@ -0,0 +1,166 @@ +-- ako/mxcli#1448 (upstream mendixlabs/mxcli#1550): a call in a test body that +-- had its own `on error` clause got a second `ON ERROR BEGIN … END ERROR` +-- appended by the generator, and the generated MDL did not parse ("mismatched +-- input 'ON' expecting ';'") — the run stopped at injection, before the app +-- started. A call now keeps the author's clause and gets no second one; +-- `on error rollback`, Mendix's default spelled out, is treated like no clause. +-- +-- Setup: test-runner-1448-setup.mdl. Then: +-- +-- mxcli test mdl-examples/bug-tests/test-runner-1448-own-error-handler.test.mdl -p app.mpr --local +-- +-- Measured on 11.13.0 (--local --db-type hsqldb). Before the fix: the parse +-- error above and no test runs. After: +-- +-- 01 FAIL exception during execution: boom: a +-- 02 FAIL exception during execution: boom: a (rollback = no clause) +-- 03 FAIL expected $r = 'X', actual: (continue swallowed it) +-- 04 FAIL not every assertion was evaluated: … (handler skipped the @expect) +-- 05 ERROR boom: a (the handler re-raised) +-- 06 ERROR boom: a +-- 07 PASS 08 PASS +-- 09 FAIL expected an exception but none was thrown +-- 10 FAIL expected an exception but none was thrown +-- 11 ERROR boom: a +-- 12-16 PASS +-- +-- 04 is the false green the fix had to close: a custom handler that ends +-- normally rejoins the flow past every statement reading the failed call's +-- output, so the assertion was never evaluated and the test reported PASS. 16 is +-- its control — an assertion on what the handler did is still evaluated. +-- +-- Under --legacy-runner a test whose error leaves the runner (05, 06, 11) ends +-- the whole run; 05 was reported PASS there (a RUN line with no verdict read as +-- the zero status) and is now an ERROR. That runner also needs a distinct +-- variable name per test. +mdl 1; +/** + * @test 01 plain call that throws + * @expect $r = 'X' + */ +$r = call java action Shop.JA_Fail(Text = 'a'); +/ + +/** + * @test 02 on error rollback, call throws + * @expect $r = 'X' + */ +$r = call java action Shop.JA_Fail(Text = 'a') on error rollback; +/ + +/** + * @test 03 on error continue, call throws + * @expect $r = 'X' + */ +$r = call java action Shop.JA_Fail(Text = 'a') on error continue; +/ + +/** + * @test 04 custom handler that logs, call throws + * @expect $r = 'X' + */ +$r = call java action Shop.JA_Fail(Text = 'a') on error begin + log error node 'T' 'handled'; +end error; +/ + +/** + * @test 05 custom handler that re-raises, call throws + * @expect $r = 'X' + */ +$r = call java action Shop.JA_Fail(Text = 'a') on error begin + log error node 'T' 'handled'; + raise error; +end error; +/ + +/** + * @test 06 custom handler without rollback that re-raises, call throws + * @expect $r = 'X' + */ +$r = call java action Shop.JA_Fail(Text = 'a') on error without rollback begin + log error node 'T' 'handled'; + raise error; +end error; +/ + +/** + * @test 07 throws: plain call + * @throws 'boom' + */ +$r = call java action Shop.JA_Fail(Text = 'a'); +/ + +/** + * @test 08 throws: on error rollback + * @throws 'boom' + */ +$r = call java action Shop.JA_Fail(Text = 'a') on error rollback; +/ + +/** + * @test 09 throws: on error continue + * @throws 'boom' + */ +$r = call java action Shop.JA_Fail(Text = 'a') on error continue; +/ + +/** + * @test 10 throws: custom handler that logs + * @throws 'boom' + */ +$r = call java action Shop.JA_Fail(Text = 'a') on error begin + log error node 'T' 'handled'; +end error; +/ + +/** + * @test 11 throws: custom handler that re-raises + * @throws 'boom' + */ +$r = call java action Shop.JA_Fail(Text = 'a') on error begin + log error node 'T' 'handled'; + raise error; +end error; +/ + +/** + * @test 12 the report: java action with on error rollback, call succeeds + * @expect $Result = 'ABC' + */ +$Result = call java action Shop.JA_Upper(Text = 'abc') on error rollback; +/ + +/** + * @test 13 the report: microflow with on error rollback, call succeeds + * @expect $Result = 'Order 1' + */ +$Result = call microflow Shop.Order_Label(Text = '1') on error rollback; +/ + +/** + * @test 14 on error continue, call succeeds + * @expect $r = 'ABC' + */ +$r = call java action Shop.JA_Upper(Text = 'abc') on error continue; +/ + +/** + * @test 15 custom handler, call succeeds + * @expect $r = 'ABC' + */ +$r = call java action Shop.JA_Upper(Text = 'abc') on error begin + log error node 'T' 'handled'; + raise error; +end error; +/ + +/** + * @test 16 custom handler path, asserted on what the handler did + * @expect $Handled = true + */ +declare $Handled Boolean = false; +$r = call java action Shop.JA_Fail(Text = 'a') on error begin + set $Handled = true; +end error; +/ diff --git a/mdl-examples/bug-tests/test-runner-1448-setup.mdl b/mdl-examples/bug-tests/test-runner-1448-setup.mdl new file mode 100644 index 0000000000..05973be5aa --- /dev/null +++ b/mdl-examples/bug-tests/test-runner-1448-setup.mdl @@ -0,0 +1,26 @@ +-- ako/mxcli#1448 (upstream mendixlabs/mxcli#1550): the setup script for +-- test-runner-1448-own-error-handler.test.mdl. Apply it before running the tests: +-- +-- mxcli exec mdl-examples/bug-tests/test-runner-1448-setup.mdl -p app.mpr +-- +-- JA_Fail throws on every call, so each clause's verdict on an exception can be +-- read off one run. +mdl 1; +create module Shop; + +create java action Shop.JA_Upper (Text: String not null) returns String +as $$ + return Text.toUpperCase(); +$$; + +create java action Shop.JA_Fail (Text: String not null) returns String +as $$ + throw new Exception("boom: " + Text); +$$; + +create microflow Shop.Order_Label ($Text: String) + returns String as $Label +begin + declare $Label String = 'Order ' + $Text; + return $Label; +end; diff --git a/mdl-examples/bug-tests/test-runner-1654-setup.mdl b/mdl-examples/bug-tests/test-runner-1654-setup.mdl new file mode 100644 index 0000000000..0ff9d829e5 --- /dev/null +++ b/mdl-examples/bug-tests/test-runner-1654-setup.mdl @@ -0,0 +1,31 @@ +-- ako/mxcli#1654 (upstream mendixlabs/mxcli#1645): the setup script for +-- test-runner-1654-startup-applies-entity-access.test.mdl. Apply it before +-- running the tests: +-- +-- mxcli exec mdl-examples/bug-tests/test-runner-1654-setup.mdl -p app.mpr +-- +-- The project's after-startup microflow applies entity access, which is what +-- the generated MxTest.RegisterEndpoint (it calls this one) could not call. It +-- leaves a marker, so a test can show that it ran at boot. +mdl 1; +create module MyModule; + +create persistent entity MyModule.StartupMarker ( + Note: String(100) +); + +/** Startup flow that applies entity access. */ +@applyentityaccess(true) +create microflow MyModule.ASU_Startup () returns Boolean +begin + $Marker = create MyModule.StartupMarker (Note = 'startup ran') commit; + return true; +end; + +/** Adds two integers. */ +create microflow MyModule.SUB_Add ($A: Integer, $B: Integer) returns Integer +begin + return $A + $B; +end; + +alter settings runtime (AfterStartupMicroflow: 'MyModule.ASU_Startup'); diff --git a/mdl-examples/bug-tests/test-runner-1654-startup-applies-entity-access.test.mdl b/mdl-examples/bug-tests/test-runner-1654-startup-applies-entity-access.test.mdl new file mode 100644 index 0000000000..cc1fc3bf2d --- /dev/null +++ b/mdl-examples/bug-tests/test-runner-1654-startup-applies-entity-access.test.mdl @@ -0,0 +1,41 @@ +-- ako/mxcli#1654 (upstream mendixlabs/mxcli#1645): `mxcli test --local` +-- generates the startup microflow MxTest.RegisterEndpoint, which registers the +-- test endpoint and then calls the project's own after-startup microflow. It was +-- written without "apply entity access", and MxBuild refuses a microflow that +-- does not apply entity access calling one that does (CE0114) — so in a project +-- whose after-startup microflow applies entity access no test could run, and the +-- error said it was "in the project, not in this run's tests". +-- +-- Setup: test-runner-1654-setup.mdl. Then: +-- +-- mxcli test mdl-examples/bug-tests/test-runner-1654-startup-applies-entity-access.test.mdl -p app.mpr --local +-- +-- Measured on 11.13.0 (--local --db-type hsqldb). Before the fix: "build failed +-- … [CE0114] … at MxTest / Microflow 'RegisterEndpoint' / Call microflow +-- activity 'ASU_Startup'" and no test runs. After: both tests PASS — the second +-- one shows the project's startup microflow ran at boot, through the generated +-- flow. It consumes the marker it counts (`@cleanup none`), so it cannot pass on +-- an earlier boot's: its control, the same run with --skip-app-startup, FAILs +-- it ("expected $Count > 0, actual: 0"). +-- +-- CE0114 has one direction (mx check, 11.13.0): non-applying → applying is the +-- error; applying → non-applying and applying → applying are 0 errors, at +-- security Off and Production. So RegisterEndpoint now always applies entity +-- access, and calls a startup microflow of either kind. +mdl 1; +/** + * @test Addition returns the sum + * @expect $result = 4 + */ +$result = call microflow MyModule.SUB_Add(A = 2, B = 2); +/ + +/** + * @test the app's own after-startup microflow ran at boot + * @cleanup none + * @expect $Count > 0 + */ +retrieve $Markers from MyModule.StartupMarker; +$Count = count $Markers; +delete $Markers; +/ diff --git a/mdl-examples/bug-tests/widget-action-1516-empty-call-args.mdl b/mdl-examples/bug-tests/widget-action-1516-empty-call-args.mdl new file mode 100644 index 0000000000..43220cae28 --- /dev/null +++ b/mdl-examples/bug-tests/widget-action-1516-empty-call-args.mdl @@ -0,0 +1,105 @@ +mdl 1; +-- ============================================================================ +-- ako/mxcli#1516 (mendixlabs/mxcli#1606) — an empty argument list on a page +-- action or data source +-- ============================================================================ +-- +-- Symptom: a flow with no parameters, called from a widget with `()`, +-- actionbutton btnPing (Caption: 'Ping', Action: call microflow M.Ping()) +-- was a syntax error at the `)`: +-- mismatched input ')' expecting the start of a statement (create, alter, …) +-- while the same call parses in a flow body, and the widget action parsed +-- without the brackets. Every site sharing the page argument rule failed the +-- same way: call microflow / call nanoflow / show page (and their deprecated +-- spellings), a microflow or nanoflow data source, OnClick, OnChange, a +-- pluggable widget's action property, alter page set, and the OnClick of a +-- navigation or menu document item. +-- +-- `use fragment Hdr()` was worse: it was not an error at all without a +-- project. It parsed as two widgets, of types `use` and `Hdr`, and only +-- `check -p` reported them ("`use` is not a widget in this project"). +-- +-- `M.F()` now means exactly what `M.F` means. Measured on 11.13.0: the two +-- pages below store the same BSON once element IDs and the page name are set +-- aside, describe identically (without brackets), re-running a description +-- reports "Unchanged page", and mx check reports 0 errors. +-- +-- mxcli check mdl-examples/bug-tests/widget-action-1516-empty-call-args.mdl +-- mxcli exec mdl-examples/bug-tests/widget-action-1516-empty-call-args.mdl -p app.mpr +-- mxcli -p app.mpr -c "describe page Bug1516.P_Empty" -- no brackets +-- ============================================================================ + +create module Bug1516; +create persistent entity Bug1516.Item (Name: string(50)); + +create microflow Bug1516.Ping () +begin + log info 'ping'; +end; + +create nanoflow Bug1516.NPing () +begin + log info 'nping'; +end; + +create microflow Bug1516.DS () returns list of Bug1516.Item +begin + retrieve $Items from Bug1516.Item; + return $Items; +end; + +create page Bug1516.Target (Title: 'Target', Layout: Atlas_Core.Atlas_Default) { + layoutgrid g { row { column (DesktopWidth: autofill) { dynamictext t (Content: 'target') } } } +}; + +create page Bug1516.EditItem (Title: 'Edit', Layout: Atlas_Core.Atlas_Default, Params: ($Item: Bug1516.Item)) { + layoutgrid g { row { column (DesktopWidth: autofill) { + dataview dv (DataSource: $Item) { textbox tb (Attribute: Name) } + } } } +}; + +create fragment Hdr as { + container hdrWrap { dynamictext hdrText (Content: 'header') } +}; + +-- Every call written with an empty argument list. +create page Bug1516.P_Empty (Title: 'Calls', Layout: Atlas_Core.Atlas_Default) { + layoutgrid g { + row { + column (DesktopWidth: autofill) { + actionbutton btnMf (Caption: 'Ping', Action: call microflow Bug1516.Ping()) + actionbutton btnMfWith (Caption: 'Ping', Action: call microflow Bug1516.Ping() with (ProgressBar: Blocking)) + actionbutton btnNf (Caption: 'NPing', Action: call nanoflow Bug1516.NPing()) + actionbutton btnShow (Caption: 'Show', Action: show page Bug1516.Target()) + actionbutton btnCreate (Caption: 'New', Action: create object Bug1516.Item then show page Bug1516.EditItem()) + container cClick (OnClick: call nanoflow Bug1516.NPing()) { dynamictext ct (Content: 'click') } + listview lvMf (DataSource: microflow Bug1516.DS()) { dynamictext lt (Content: '{1}', ContentParams: ({1} = Name)) } + use fragment Hdr() as h_ + } + } + } +}; + +-- Control: the same page without brackets. It must store the same document. +create page Bug1516.P_Bare (Title: 'Calls', Layout: Atlas_Core.Atlas_Default) { + layoutgrid g { + row { + column (DesktopWidth: autofill) { + actionbutton btnMf (Caption: 'Ping', Action: call microflow Bug1516.Ping) + actionbutton btnMfWith (Caption: 'Ping', Action: call microflow Bug1516.Ping with (ProgressBar: Blocking)) + actionbutton btnNf (Caption: 'NPing', Action: call nanoflow Bug1516.NPing) + actionbutton btnShow (Caption: 'Show', Action: show page Bug1516.Target) + actionbutton btnCreate (Caption: 'New', Action: create object Bug1516.Item then show page Bug1516.EditItem) + container cClick (OnClick: call nanoflow Bug1516.NPing) { dynamictext ct (Content: 'click') } + listview lvMf (DataSource: microflow Bug1516.DS) { dynamictext lt (Content: '{1}', ContentParams: ({1} = Name)) } + use fragment Hdr as h_ + } + } + } +}; + +-- A menu document item takes the same action expression. +create or modify menu Bug1516.Menu_Empty { + menu item 'Ping' (OnClick: call microflow Bug1516.Ping(), Icon: Atlas_Core.Atlas.home) + menu item 'Show' (OnClick: show page Bug1516.Target(), Icon: Atlas_Core.Atlas.home) +}; diff --git a/mdl-examples/bug-tests/widget-props-1698-caption-on-textbox.fail.mdl b/mdl-examples/bug-tests/widget-props-1698-caption-on-textbox.fail.mdl new file mode 100644 index 0000000000..5bfb581d52 --- /dev/null +++ b/mdl-examples/bug-tests/widget-props-1698-caption-on-textbox.fail.mdl @@ -0,0 +1,52 @@ +mdl 1; +-- ============================================================================ +-- ako/mxcli#1698 (mendixlabs/mxcli#1651): a property the widget does not take +-- ============================================================================ +-- +-- Symptom (before fix): +-- `textbox tbName (Attribute: Name, Caption: 'Item name')` passed `mxcli check` +-- and `exec`, and the text box was written with no label: `describe page` +-- read it back as `textbox tbName (Attribute: Name)`, `mx check` reported 0 +-- errors, and the page rendered an input with no label. An input widget's +-- label is `Label:`; `Caption:` is a button's and a data grid column's. +-- MDL-WIDGET07 judged keys against the UNION of every built-in widget's +-- vocabulary, so a key valid on some widget passed on all of them. +-- +-- Fix: +-- Each built-in page element has the set of properties its builder writes +-- (widget_native_props.go), derived from and tested against the builders' +-- source. A key outside it is refused under `mdl 1;` by check and by exec +-- (MDL-WIDGET52), with a hint naming what was meant; without the header it +-- is dropped as before and warned about (MDL-V1-WIDGETPROP). +-- +-- Expected: `mxcli check` fails with MDL-WIDGET52 on tbCaption only: +-- "textbox `tbCaption`: `Caption` is not a property mxcli writes on a textbox +-- (Forms$TextBox), so it would be dropped — an input widget's label is +-- `Label:` (`Caption:` is a button's and a column's)" +-- The controls (Label on a text box, Caption on a button) are written. +-- +-- Usage: +-- mxcli check mdl-examples/bug-tests/widget-props-1698-caption-on-textbox.fail.mdl +-- ============================================================================ + +create module ModC; +create persistent entity ModC.Item (Name: String(50)); +create page ModC.ItemEdit +( + Title: 'Item', + Layout: Atlas_Core.PopupLayout, + Params: ( $Item: ModC.Item ) +) +{ + layoutgrid lg { + row { + column (DesktopWidth: 12) { + dataview dvItem (DataSource: $Item) { + textbox tbCaption (Attribute: Name, Caption: 'Item name') + textbox tbLabel (Attribute: Name, Label: 'Item name (Label)') + actionbutton btnSave (Caption: 'Save', Action: save changes) + } + } + } + } +}; diff --git a/mdl-examples/bug-tests/widget-unknown-property.mdl b/mdl-examples/bug-tests/widget-unknown-property.fail.mdl similarity index 63% rename from mdl-examples/bug-tests/widget-unknown-property.mdl rename to mdl-examples/bug-tests/widget-unknown-property.fail.mdl index 7e95dd8586..d593f1f880 100644 --- a/mdl-examples/bug-tests/widget-unknown-property.mdl +++ b/mdl-examples/bug-tests/widget-unknown-property.fail.mdl @@ -1,6 +1,6 @@ mdl 1; -- ============================================================================ --- Unknown properties on a built-in widget are silently dropped (now warned) +-- Unknown properties on a built-in widget are silently dropped (now refused) -- ============================================================================ -- -- Symptom (before fix): @@ -10,16 +10,18 @@ mdl 1; -- rejected unknown keys (MDL-WIDGET01); built-in widgets had no such check. -- -- Fix: --- MDL-WIDGET07 — a WARNING (not an error) on any unrecognized property of a --- built-in widget, with a "did you mean" hint. It is a warning because the --- core-widget property vocabulary can't be proven complete per widget type --- (describe even emits keys like WidthUnit the native builder doesn't consume), --- so a hard reject would false-positive on valid MDL and break the --- describe→create roundtrip. The allow-list is the union of grammar keyword --- properties, builder-consumed keys, and the full describe-emit vocabulary. +-- MDL-WIDGET07 — a WARNING on any property outside the union of every +-- built-in widget's vocabulary, with a "did you mean" hint. -- --- Usage (observe the two MDL-WIDGET07 warnings — check still passes): --- mxcli check mdl-examples/bug-tests/widget-unknown-property.mdl +-- Superseded by ako/mxcli#1698: each built-in widget is now judged by the +-- properties ITS builder writes (widget_native_props.go), which is what made +-- a refusal safe. Under `mdl 1;` (this script) both keys below are refused +-- (MDL-WIDGET52), so the script is a negative test; without the header they +-- are dropped and warned about (MDL-V1-WIDGETPROP). Renamed from +-- widget-unknown-property.mdl. +-- +-- Usage (expect two MDL-WIDGET52 errors — check fails): +-- mxcli check mdl-examples/bug-tests/widget-unknown-property.fail.mdl -- ============================================================================ create or modify page MyFirstModule.P_UnknownProp @@ -27,9 +29,9 @@ create or modify page MyFirstModule.P_UnknownProp { -- all recognized → no warning dynamictext ok ( Content: 'hi', Class: 'c', DynamicClasses: if true then 'a' else '', RenderMode: H1 ) - -- typo of a real property → MDL-WIDGET07 "did you mean `Content`?" + -- typo of a real property → MDL-WIDGET52 "did you mean `Content`?" dynamictext typo ( Content: 'hi', Contnet: 'oops' ) - -- genuinely unknown property → MDL-WIDGET07 (no suggestion) + -- genuinely unknown property → MDL-WIDGET52 (no suggestion) dynamictext unknown ( Content: 'hi', TotallyMadeUp: 'x' ) -- describe-vocabulary props on a native image → no warning image img ( ImageType: imageUrl, ImageUrl: 'https://example.com/x.png', diff --git a/mdl-examples/doctype-tests/layouts.mdl b/mdl-examples/doctype-tests/layouts.mdl index 637bc0bec9..fcdcef3e99 100644 --- a/mdl-examples/doctype-tests/layouts.mdl +++ b/mdl-examples/doctype-tests/layouts.mdl @@ -62,6 +62,31 @@ create or modify layout LayoutTest.App_Default ( } }; +-- ---------------------------------------------------------------------------- +-- The frame: a scroll container's layoutmode. 'Headline' (the default, as +-- above) runs the top region across the full width and starts the left one +-- under it; 'Sidebar' runs the left region the full height and puts the top +-- bar beside it, as Atlas_SideBar and Atlas_TopBar do (ako/mxcli#1703). +-- `describe layout` prints it only when it is Sidebar. +-- ---------------------------------------------------------------------------- + +create or modify layout LayoutTest.App_Sidebar ( + layouttype: 'Responsive', + class: 'layout-atlas layout-atlas-responsive-sidebar' +) { + scrollcontainer layoutContainer (layoutmode: 'Sidebar') { + region top (size: 48, class: 'border-bottom') { + dynamictext sidebarBrand (content: 'My Application') + } + region left (size: 232, sizemode: 'Pixels', class: 'region-sidebar') { + navigationtree sidebarNav (profile: 'Responsive') + } + region center (class: 'region-content') { + placeholder Main + } + } +}; + -- ---------------------------------------------------------------------------- -- More than one placeholder. A page binds to each by name — the name is API, -- so renaming one unbinds every page that used it. diff --git a/mdl/ast/ast_security.go b/mdl/ast/ast_security.go index 0a110cb449..18c3d38fc4 100644 --- a/mdl/ast/ast_security.go +++ b/mdl/ast/ast_security.go @@ -211,9 +211,13 @@ type AlterProjectSecurityStmt struct { // from "the statement asked for off". StrictModeEnabled *bool // AdminUserName renames the built-in administrator account (MxAdmin by - // default). Empty means "keep what is stored". Its password is deliberately - // not settable from MDL (mendixlabs/mxcli#624). + // default). Empty means "keep what is stored". AdminUserName string + // AdminPassword sets that account's password, which Mendix stores in the + // model; empty means "keep what is stored". An empty stored password is + // CE0129 at prototype and production level (mendixlabs/mxcli#624), so the + // visitor refuses '' rather than letting a script write the error. + AdminPassword string } func (s *AlterProjectSecurityStmt) isStatement() {} diff --git a/mdl/backend/domainmodel.go b/mdl/backend/domainmodel.go index 3b1d28eaaa..490a787d53 100644 --- a/mdl/backend/domainmodel.go +++ b/mdl/backend/domainmodel.go @@ -16,6 +16,11 @@ type DomainModelBackend interface { GetDomainModel(moduleID model.ID) (*domainmodel.DomainModel, error) GetDomainModelByID(id model.ID) (*domainmodel.DomainModel, error) UpdateDomainModel(dm *domainmodel.DomainModel) error + // ParseDomainModelFromRaw converts a DomainModels$DomainModel document + // that is not in storage — diff-local's historical revision of the unit, + // read back with `git show` — the way GetDomainModel converts the stored + // one. Read-only. + ParseDomainModelFromRaw(raw map[string]any, unitID, containerID model.ID) (*domainmodel.DomainModel, error) // SetDomainModelAnnotations replaces the canvas notes on a domain model. // diff --git a/mdl/backend/infrastructure.go b/mdl/backend/infrastructure.go index 4e3aba77ba..55027df2b3 100644 --- a/mdl/backend/infrastructure.go +++ b/mdl/backend/infrastructure.go @@ -112,6 +112,11 @@ type AgentEditorBackend interface { type SettingsBackend interface { GetProjectSettings() (*model.ProjectSettings, error) UpdateProjectSettings(ps *model.ProjectSettings) error + // ParseProjectSettingsFromRaw converts a Settings$ProjectSettings document + // that is not in storage — diff-local's historical revision of the unit, + // read back with `git show` — the way GetProjectSettings converts the + // stored one. Read-only. + ParseProjectSettingsFromRaw(raw map[string]any) (*model.ProjectSettings, error) } // ImageBackend provides image collection operations. diff --git a/mdl/backend/mcp/unsupported_gen.go b/mdl/backend/mcp/unsupported_gen.go index cb76a41b55..a88608fbd7 100644 --- a/mdl/backend/mcp/unsupported_gen.go +++ b/mdl/backend/mcp/unsupported_gen.go @@ -1042,6 +1042,11 @@ func (unsupportedBackend) PageLayoutName(_ model.ID) (r0 string, err1 error) { return } +func (unsupportedBackend) ParseDomainModelFromRaw(_ map[string]any, _ model.ID, _ model.ID) (r0 *domainmodel.DomainModel, err1 error) { + err1 = errUnsupported("ParseDomainModelFromRaw") + return +} + func (unsupportedBackend) ParseMicroflowBSON(_ []uint8, _ model.ID, _ model.ID) (r0 *microflows.Microflow, err1 error) { err1 = errUnsupported("ParseMicroflowBSON") return @@ -1055,6 +1060,11 @@ func (unsupportedBackend) ParseNanoflowFromRaw(_ map[string]any, _ model.ID, _ m return } +func (unsupportedBackend) ParseProjectSettingsFromRaw(_ map[string]any) (r0 *model.ProjectSettings, err1 error) { + err1 = errUnsupported("ParseProjectSettingsFromRaw") + return +} + func (unsupportedBackend) Path() (r0 string) { return } @@ -1176,6 +1186,11 @@ func (unsupportedBackend) SetDomainModelDocumentation(_ model.ID, _ string) (err return } +func (unsupportedBackend) SetProjectAdminPassword(_ model.ID, _ string) (err0 error) { + err0 = errUnsupported("SetProjectAdminPassword") + return +} + func (unsupportedBackend) SetProjectAdminUserName(_ model.ID, _ string) (err0 error) { err0 = errUnsupported("SetProjectAdminUserName") return diff --git a/mdl/backend/mock/backend.go b/mdl/backend/mock/backend.go index 05f6a71098..b8b360aac4 100644 --- a/mdl/backend/mock/backend.go +++ b/mdl/backend/mock/backend.go @@ -63,6 +63,7 @@ type MockBackend struct { ListDomainModelsFunc func() ([]*domainmodel.DomainModel, error) GetDomainModelFunc func(moduleID model.ID) (*domainmodel.DomainModel, error) GetDomainModelByIDFunc func(id model.ID) (*domainmodel.DomainModel, error) + ParseDomainModelFromRawFunc func(raw map[string]any, unitID, containerID model.ID) (*domainmodel.DomainModel, error) UpdateDomainModelFunc func(dm *domainmodel.DomainModel) error SetDomainModelAnnotationsFunc func(domainModelID model.ID, annotations []*domainmodel.Annotation) error SetDomainModelDocumentationFunc func(domainModelID model.ID, documentation string) error @@ -158,6 +159,7 @@ type MockBackend struct { SetProjectDemoUsersEnabledFunc func(unitID model.ID, enabled bool) error SetProjectStrictModeFunc func(unitID model.ID, enabled bool) error SetProjectAdminUserNameFunc func(unitID model.ID, name string) error + SetProjectAdminPasswordFunc func(unitID model.ID, password string) error SetProjectGuestAccessFunc func(unitID model.ID, enabled bool, guestUserRole string) error AddUserRoleFunc func(unitID model.ID, name string, moduleRoles []string, manageAllRoles bool) error AlterUserRoleModuleRolesFunc func(unitID model.ID, userRoleName string, add bool, moduleRoles []string) error @@ -279,9 +281,10 @@ type MockBackend struct { DeleteWorkflowFunc func(id model.ID) error // SettingsBackend - GetProjectSettingsFunc func() (*model.ProjectSettings, error) - AddNavigationProfileFunc func(navDocID model.ID, name string) error - UpdateProjectSettingsFunc func(ps *model.ProjectSettings) error + GetProjectSettingsFunc func() (*model.ProjectSettings, error) + AddNavigationProfileFunc func(navDocID model.ID, name string) error + UpdateProjectSettingsFunc func(ps *model.ProjectSettings) error + ParseProjectSettingsFromRawFunc func(raw map[string]any) (*model.ProjectSettings, error) // ImageBackend ListImageCollectionsFunc func() ([]*types.ImageCollection, error) diff --git a/mdl/backend/mock/mock_domainmodel.go b/mdl/backend/mock/mock_domainmodel.go index 2dc8a4030b..b136d6c312 100644 --- a/mdl/backend/mock/mock_domainmodel.go +++ b/mdl/backend/mock/mock_domainmodel.go @@ -31,6 +31,13 @@ func (m *MockBackend) GetDomainModelByID(id model.ID) (*domainmodel.DomainModel, return nil, nil } +func (m *MockBackend) ParseDomainModelFromRaw(raw map[string]any, unitID, containerID model.ID) (*domainmodel.DomainModel, error) { + if m.ParseDomainModelFromRawFunc != nil { + return m.ParseDomainModelFromRawFunc(raw, unitID, containerID) + } + return nil, fmt.Errorf("MockBackend.ParseDomainModelFromRaw not configured") +} + func (m *MockBackend) UpdateDomainModel(dm *domainmodel.DomainModel) error { if m.UpdateDomainModelFunc != nil { return m.UpdateDomainModelFunc(dm) diff --git a/mdl/backend/mock/mock_security.go b/mdl/backend/mock/mock_security.go index a8ecea0fbd..12d9c286ca 100644 --- a/mdl/backend/mock/mock_security.go +++ b/mdl/backend/mock/mock_security.go @@ -39,6 +39,13 @@ func (m *MockBackend) SetProjectAdminUserName(unitID model.ID, name string) erro return fmt.Errorf("MockBackend.SetProjectAdminUserName not configured") } +func (m *MockBackend) SetProjectAdminPassword(unitID model.ID, password string) error { + if m.SetProjectAdminPasswordFunc != nil { + return m.SetProjectAdminPasswordFunc(unitID, password) + } + return fmt.Errorf("MockBackend.SetProjectAdminPassword not configured") +} + func (m *MockBackend) SetProjectStrictMode(unitID model.ID, enabled bool) error { if m.SetProjectStrictModeFunc != nil { return m.SetProjectStrictModeFunc(unitID, enabled) diff --git a/mdl/backend/mock/mock_workflow.go b/mdl/backend/mock/mock_workflow.go index 4d792482fd..2e1a988eb5 100644 --- a/mdl/backend/mock/mock_workflow.go +++ b/mdl/backend/mock/mock_workflow.go @@ -67,6 +67,13 @@ func (m *MockBackend) UpdateProjectSettings(ps *model.ProjectSettings) error { return nil } +func (m *MockBackend) ParseProjectSettingsFromRaw(raw map[string]any) (*model.ProjectSettings, error) { + if m.ParseProjectSettingsFromRawFunc != nil { + return m.ParseProjectSettingsFromRawFunc(raw) + } + return nil, fmt.Errorf("MockBackend.ParseProjectSettingsFromRaw not configured") +} + // --------------------------------------------------------------------------- // ImageBackend // --------------------------------------------------------------------------- diff --git a/mdl/backend/modelsdk/domainmodel_parse_raw_test.go b/mdl/backend/modelsdk/domainmodel_parse_raw_test.go new file mode 100644 index 0000000000..c148a6c457 --- /dev/null +++ b/mdl/backend/modelsdk/domainmodel_parse_raw_test.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "reflect" + "testing" + + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/modelsdk/bsoncompat" +) + +// TestParseDomainModelFromRawMatchesStoredRead feeds the method every domain +// model in the fixture as diff-local does — raw BSON unmarshalled to a map, +// with no unit to look up — and asserts each entity's access rules come out as +// GetDomainModel reads them. diff-local renders the rules from this parse with +// DESCRIBE ENTITY's writer, so a lossy parse would put the rules on the page +// and still hide a GRANT's change (ako/mxcli#1562). +func TestParseDomainModelFromRawMatchesStoredRead(t *testing.T) { + b := New() + if err := b.Connect(fixture); err != nil { + t.Fatalf("Connect(%s): %v", fixture, err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + refs, err := b.reader.ListUnitsByType("DomainModels$DomainModel") + if err != nil || len(refs) == 0 { + t.Fatalf("no domain model in the fixture: %v", err) + } + rules := 0 + for _, ref := range refs { + var raw map[string]any + if err := bsoncompat.UnmarshalDocMap(ref.Contents, &raw); err != nil { + t.Fatal(err) + } + got, err := b.ParseDomainModelFromRaw(raw, model.ID(ref.ID), model.ID(ref.ContainerID)) + if err != nil { + t.Fatalf("ParseDomainModelFromRaw(%s): %v", ref.ID, err) + } + want, err := b.GetDomainModel(model.ID(ref.ContainerID)) + if err != nil { + t.Fatal(err) + } + if len(got.Entities) != len(want.Entities) { + t.Fatalf("%s: %d entities, want %d", ref.ID, len(got.Entities), len(want.Entities)) + } + for i, e := range got.Entities { + if e.Name != want.Entities[i].Name { + t.Errorf("entity %d = %q, want %q", i, e.Name, want.Entities[i].Name) + } + if !reflect.DeepEqual(e.AccessRules, want.Entities[i].AccessRules) { + t.Errorf("%s access rules = %+v, want %+v", e.Name, e.AccessRules, want.Entities[i].AccessRules) + } + rules += len(e.AccessRules) + } + } + if rules == 0 { + t.Fatal("no entity in the fixture has an access rule — the comparison above checked nothing") + } +} + +func TestParseDomainModelFromRawRefusesOtherDocuments(t *testing.T) { + b := New() + if err := b.Connect(fixture); err != nil { + t.Fatalf("Connect(%s): %v", fixture, err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + refs, err := b.reader.ListUnitsByType("Settings$ProjectSettings") + if err != nil || len(refs) == 0 { + t.Fatalf("no settings unit in the fixture: %v", err) + } + var raw map[string]any + if err := bsoncompat.UnmarshalDocMap(refs[0].Contents, &raw); err != nil { + t.Fatal(err) + } + if dm, err := b.ParseDomainModelFromRaw(raw, "", ""); err == nil { + t.Errorf("a settings document parsed as a domain model: %+v", dm) + } +} diff --git a/mdl/backend/modelsdk/issue624_admin_password_test.go b/mdl/backend/modelsdk/issue624_admin_password_test.go new file mode 100644 index 0000000000..39851c1b60 --- /dev/null +++ b/mdl/backend/modelsdk/issue624_admin_password_test.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import "testing" + +// mendixlabs/mxcli#624 (ako/mxcli#1259): CE0129 "Administrator password has +// not been set" could only be resolved in Studio Pro. AdminPassword is written +// on Security$ProjectSecurity and nothing else about the administrator changes. +func TestSetProjectAdminPassword(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + ps, err := b.GetProjectSecurity() + if err != nil { + t.Fatalf("GetProjectSecurity: %v", err) + } + const pw = "N3w-Admin-Passw0rd" + // Control: the fixture does not already hold the value written below. + if before := storedString(t, b, ps.ID, "AdminPassword"); before == pw { + t.Fatalf("fixture AdminPassword is already %q; the control does not hold", before) + } + nameBefore := storedString(t, b, ps.ID, "AdminUserName") + + if err := b.SetProjectAdminPassword(ps.ID, pw); err != nil { + t.Fatalf("SetProjectAdminPassword: %v", err) + } + _ = b.Disconnect() + + b = New() + if err := b.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + if got := storedString(t, b, ps.ID, "AdminPassword"); got != pw { + t.Errorf("stored AdminPassword = %q, want %q", got, pw) + } + if got := storedString(t, b, ps.ID, "AdminUserName"); got != nameBefore { + t.Errorf("AdminUserName changed: %q -> %q", nameBefore, got) + } + got, err := b.GetProjectSecurity() + if err != nil || got.AdminPassword != pw { + t.Errorf("read back AdminPassword %q (%v), want %q", got.AdminPassword, err, pw) + } +} diff --git a/mdl/backend/modelsdk/parse_raw.go b/mdl/backend/modelsdk/parse_raw.go new file mode 100644 index 0000000000..57ba0e5db4 --- /dev/null +++ b/mdl/backend/modelsdk/parse_raw.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "fmt" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/modelsdk/codec" + "github.com/mendixlabs/mxcli/modelsdk/element" + genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" + genSet "github.com/mendixlabs/mxcli/modelsdk/gen/settings" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// decodeRaw decodes a unit document that did NOT come from this project's +// storage: diff-local hands over a historical unit read back with `git show`, +// so there is no unit to look up. The codec decodes bytes, not maps, so the map +// is re-marshalled first — safe for the same reason as ParseMicroflowFromRaw: +// the result is only rendered, never written back. +func decodeRaw(raw map[string]any) (element.Element, []byte, error) { + data, err := bson.Marshal(raw) + if err != nil { + return nil, nil, err + } + elem, err := codec.NewDecoder(codec.DefaultRegistry).Decode(bson.Raw(data)) + if err != nil { + return nil, nil, err + } + return elem, data, nil +} + +// ParseProjectSettingsFromRaw converts a Settings$ProjectSettings document to +// the semantic settings, the way GetProjectSettings does for the stored one — +// RawParts included, since DESCRIBE SETTINGS prints only the properties the +// document stores. diff-local renders both revisions of the settings unit with +// it; without it the unit had no renderer, and every change to it counted as +// modified and printed nothing (mendixlabs/mxcli#1602, ako/mxcli#1512). +func (b *Backend) ParseProjectSettingsFromRaw(raw map[string]any) (*model.ProjectSettings, error) { + elem, data, err := decodeRaw(raw) + if err != nil { + return nil, fmt.Errorf("decode project settings: %w", err) + } + g, ok := elem.(*genSet.ProjectSettings) + if !ok { + return nil, fmt.Errorf("not a project settings document: %T", elem) + } + ps := projectSettingsFromGen(g) + ps.RawParts = settingsRawParts(data) + return ps, nil +} + +// ParseDomainModelFromRaw converts a DomainModels$DomainModel document to the +// semantic domain model, under the same read-only reasoning. diff-local reads +// each entity's access rules from it, so a GRANT renders the way DESCRIBE +// ENTITY prints it (mendixlabs/mxcli#1630, ako/mxcli#1562). +func (b *Backend) ParseDomainModelFromRaw(raw map[string]any, unitID, containerID model.ID) (*domainmodel.DomainModel, error) { + elem, _, err := decodeRaw(raw) + if err != nil { + return nil, fmt.Errorf("decode domain model: %w", err) + } + g, ok := elem.(*genDm.DomainModel) + if !ok { + return nil, fmt.Errorf("not a domain model document: %T", elem) + } + dm := domainModelFromGen(g, containerID) + if unitID != "" { + dm.ID = unitID + } + return dm, nil +} diff --git a/mdl/backend/modelsdk/security_write.go b/mdl/backend/modelsdk/security_write.go index b2668720e9..a603d31812 100644 --- a/mdl/backend/modelsdk/security_write.go +++ b/mdl/backend/modelsdk/security_write.go @@ -219,7 +219,7 @@ func (b *Backend) SetProjectStrictMode(unitID model.ID, enabled bool) error { // SetProjectAdminUserName renames the built-in administrator account // (Security$ProjectSecurity.AdminUserName, MxAdmin by default). The password -// is left as stored: MDL does not set it (mendixlabs/mxcli#624). +// is left as stored. func (b *Backend) SetProjectAdminUserName(unitID model.ID, name string) error { ps, err := b.loadProjectSecurityGen(unitID) if err != nil { @@ -229,6 +229,18 @@ func (b *Backend) SetProjectAdminUserName(unitID model.ID, name string) error { return b.persistUnit(unitID, ps) } +// SetProjectAdminPassword sets the built-in administrator's password +// (Security$ProjectSecurity.AdminPassword), which Studio Pro stores in the +// model as entered. The user name is left as stored (mendixlabs/mxcli#624). +func (b *Backend) SetProjectAdminPassword(unitID model.ID, password string) error { + ps, err := b.loadProjectSecurityGen(unitID) + if err != nil { + return err + } + ps.SetAdminPassword(password) + return b.persistUnit(unitID, ps) +} + // SetProjectGuestAccess toggles anonymous (guest) access. An empty // guestUserRole leaves the stored role alone, so turning access off and back on // does not lose it. diff --git a/mdl/backend/modelsdk/settings_parse_raw_test.go b/mdl/backend/modelsdk/settings_parse_raw_test.go new file mode 100644 index 0000000000..86393257a5 --- /dev/null +++ b/mdl/backend/modelsdk/settings_parse_raw_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "reflect" + "testing" + + "github.com/mendixlabs/mxcli/modelsdk/bsoncompat" +) + +// TestParseProjectSettingsFromRawMatchesStoredRead feeds the method what +// diff-local feeds it — the settings unit's raw BSON, unmarshalled to a map, +// as if read back with `git show` — and asserts it converts the document the +// way GetProjectSettings converts the stored one. RawParts matter as much as +// the typed fields: DESCRIBE SETTINGS prints only the properties the document +// stores, so a parse without them renders a different property list than the +// describe it stands in for (ako/mxcli#1512). +func TestParseProjectSettingsFromRawMatchesStoredRead(t *testing.T) { + b := New() + if err := b.Connect(fixture); err != nil { + t.Fatalf("Connect(%s): %v", fixture, err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + want, err := b.GetProjectSettings() + if err != nil { + t.Fatalf("GetProjectSettings: %v", err) + } + refs, err := b.reader.ListUnitsByType("Settings$ProjectSettings") + if err != nil || len(refs) != 1 { + t.Fatalf("ListUnitsByType(Settings$ProjectSettings) = %d units, %v; want 1", len(refs), err) + } + var raw map[string]any + if err := bsoncompat.UnmarshalDocMap(refs[0].Contents, &raw); err != nil { + t.Fatal(err) + } + + got, err := b.ParseProjectSettingsFromRaw(raw) + if err != nil { + t.Fatalf("ParseProjectSettingsFromRaw: %v", err) + } + if got.Model == nil || got.Configuration == nil || len(got.Configuration.Configurations) == 0 { + t.Fatalf("parsed settings lack the runtime or configuration parts: %+v", got) + } + if !reflect.DeepEqual(got.Model, want.Model) { + t.Errorf("Model = %+v, want %+v", got.Model, want.Model) + } + if !reflect.DeepEqual(got.Configuration, want.Configuration) { + t.Errorf("Configuration = %+v, want %+v", got.Configuration, want.Configuration) + } + if len(got.RawParts) == 0 || len(got.RawParts) != len(want.RawParts) { + t.Errorf("RawParts: %d parts, want %d (the stored read's)", len(got.RawParts), len(want.RawParts)) + } +} + +// TestParseProjectSettingsFromRawRefusesOtherDocuments: diff-local renders +// whatever unit it is handed, so a document of another type must be an error, +// not an empty settings value that renders as "no settings". +func TestParseProjectSettingsFromRawRefusesOtherDocuments(t *testing.T) { + b := New() + if err := b.Connect(fixture); err != nil { + t.Fatalf("Connect(%s): %v", fixture, err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + refs, err := b.reader.ListUnitsByType("Microflows$Microflow") + if err != nil || len(refs) == 0 { + t.Fatalf("no microflow in the fixture: %v", err) + } + var raw map[string]any + if err := bsoncompat.UnmarshalDocMap(refs[0].Contents, &raw); err != nil { + t.Fatal(err) + } + if ps, err := b.ParseProjectSettingsFromRaw(raw); err == nil { + t.Errorf("a microflow parsed as project settings: %+v", ps) + } +} diff --git a/mdl/backend/modelsdk/settings_read.go b/mdl/backend/modelsdk/settings_read.go index 4bd4f80903..f200d3aede 100644 --- a/mdl/backend/modelsdk/settings_read.go +++ b/mdl/backend/modelsdk/settings_read.go @@ -55,6 +55,12 @@ func (b *Backend) readSettingsRawParts(unitID string) []map[string]any { if err != nil { return nil } + return settingsRawParts(raw) +} + +// settingsRawParts returns each element of a Settings$ProjectSettings +// document's Settings array as a map, skipping the versioned-array marker. +func settingsRawParts(raw []byte) []map[string]any { var doc bson.M if err := bsoncompat.UnmarshalDocM(raw, &doc); err != nil { return nil diff --git a/mdl/backend/modelsdk/unimplemented_gen.go b/mdl/backend/modelsdk/unimplemented_gen.go index 01241eb571..9eab3b463c 100644 --- a/mdl/backend/modelsdk/unimplemented_gen.go +++ b/mdl/backend/modelsdk/unimplemented_gen.go @@ -938,6 +938,11 @@ func (unimplemented) PageLayoutName(_ model.ID) (string, error) { return r0, errUnimplemented("PageLayoutName") } +func (unimplemented) ParseDomainModelFromRaw(_ map[string]interface{}, _ model.ID, _ model.ID) (*domainmodel.DomainModel, error) { + var r0 *domainmodel.DomainModel + return r0, errUnimplemented("ParseDomainModelFromRaw") +} + func (unimplemented) ParseMicroflowBSON(_ []uint8, _ model.ID, _ model.ID) (*microflows.Microflow, error) { var r0 *microflows.Microflow return r0, errUnimplemented("ParseMicroflowBSON") @@ -953,6 +958,11 @@ func (unimplemented) ParseNanoflowFromRaw(_ map[string]interface{}, _ model.ID, return r0 } +func (unimplemented) ParseProjectSettingsFromRaw(_ map[string]interface{}) (*model.ProjectSettings, error) { + var r0 *model.ProjectSettings + return r0, errUnimplemented("ParseProjectSettingsFromRaw") +} + func (unimplemented) Path() string { var r0 string return r0 @@ -1069,6 +1079,10 @@ func (unimplemented) SetDomainModelDocumentation(_ model.ID, _ string) error { return errUnimplemented("SetDomainModelDocumentation") } +func (unimplemented) SetProjectAdminPassword(_ model.ID, _ string) error { + return errUnimplemented("SetProjectAdminPassword") +} + func (unimplemented) SetProjectAdminUserName(_ model.ID, _ string) error { return errUnimplemented("SetProjectAdminUserName") } diff --git a/mdl/backend/modelsdk/widget_layout_write_test.go b/mdl/backend/modelsdk/widget_layout_write_test.go index 3d798f64ce..0c9f056245 100644 --- a/mdl/backend/modelsdk/widget_layout_write_test.go +++ b/mdl/backend/modelsdk/widget_layout_write_test.go @@ -241,3 +241,26 @@ func TestMenuWidgetToGen_MenuSourceKind(t *testing.T) { t.Error("a menu widget naming both a menu document and a profile was accepted") } } + +// A scroll container's LayoutMode, under the key Atlas_SideBar stores it +// (ako/mxcli#1703). Unsaid, the key is not written and Studio Pro's default +// (Headline) applies — so a layout that never said it encodes as before. +func TestScrollContainerToGen_LayoutMode(t *testing.T) { + for _, tc := range []struct{ mode, want string }{{"Sidebar", "Sidebar"}, {"Headline", "Headline"}, {"", ""}} { + g, err := widgetToGen(&pages.ScrollContainer{BaseWidget: pages.BaseWidget{Name: "layoutContainer"}, LayoutMode: tc.mode}) + if err != nil { + t.Fatal(err) + } + m := encodeToMap(t, g) + v, present := m["LayoutMode"] + if tc.want == "" { + if present { + t.Errorf("an unset LayoutMode encodes as %v; leave the key to Studio Pro's default", v) + } + continue + } + if got, _ := v.(string); got != tc.want { + t.Errorf("LayoutMode %q encodes as %v, want %q", tc.mode, v, tc.want) + } + } +} diff --git a/mdl/backend/modelsdk/widget_write.go b/mdl/backend/modelsdk/widget_write.go index 77169eec26..d2306faa7c 100644 --- a/mdl/backend/modelsdk/widget_write.go +++ b/mdl/backend/modelsdk/widget_write.go @@ -777,6 +777,12 @@ func widgetToGen(w pages.Widget) (element.Element, error) { case *pages.ScrollContainer: g := genPg.NewScrollContainer() applyWidgetBase(g, &x.BaseWidget) + // Only when stated: unset, the key is left to Studio Pro's default + // (Headline, canon's measured property defaults), so a layout that + // never said it is written as before (ako/mxcli#1703). + if x.LayoutMode != "" { + g.SetLayoutMode(x.LayoutMode) + } for _, r := range x.Regions { rg, err := scrollRegionToGen(r) if err != nil { diff --git a/mdl/backend/pagemutator/mutator.go b/mdl/backend/pagemutator/mutator.go index 5ee4c9096a..955c0992f0 100644 --- a/mdl/backend/pagemutator/mutator.go +++ b/mdl/backend/pagemutator/mutator.go @@ -3327,6 +3327,13 @@ func setRawWidgetPropertyMut(widget bson.D, propName string, value any) error { return setTextAreaPropertyMut(widget, propName, value) } return setPluggableWidgetPropertyMut(widget, propName, value) + case "layoutmode": + // A scroll container's Headline / Sidebar (ako/mxcli#1703). Anything + // else keeps the pluggable path. + if typ := bsonnav.DGetString(widget, "$Type"); typ == "Forms$ScrollContainer" || typ == "Pages$ScrollContainer" { + return setScrollContainerLayoutModeMut(widget, value) + } + return setPluggableWidgetPropertyMut(widget, propName, value) case "readonlystyle": // Stored on the built-in input widgets and the data view; a pluggable // widget with its own "readOnlyStyle" key keeps that path (ako/mxcli#977). @@ -3340,6 +3347,26 @@ func setRawWidgetPropertyMut(widget bson.D, propName string, value any) error { } } +// setScrollContainerLayoutModeMut writes a stored scroll container's +// LayoutMode, spelled as the metamodel spells it. It writes only a key the +// document already carries (CLAUDE.md, overlay writes): one without it is from +// a Mendix version mxcli has not measured the property on, and inventing the +// key there could leave a document Studio Pro cannot open. +func setScrollContainerLayoutModeMut(widget bson.D, value any) error { + s, ok := value.(string) + mode, valid := pages.CanonicalScrollContainerLayoutMode(s) + if !ok || !valid { + return fmt.Errorf("scroll container LayoutMode must be %s, got %v", + strings.Join(pages.ScrollContainerLayoutModes, " or "), value) + } + if bsonnav.DGet(widget, "LayoutMode") == nil { + return fmt.Errorf("this scroll container stores no LayoutMode, so it cannot be set here; " + + "rewrite the layout with `create or modify layout`") + } + bsonnav.DSet(widget, "LayoutMode", mode) + return nil +} + // dynamicTextRenderModes are the values a Forms$DynamicText's RenderMode takes — // the same set CREATE PAGE writes from `dynamictext x (RenderMode: …)`. var dynamicTextRenderModes = []pages.TextRenderMode{ diff --git a/mdl/backend/pagemutator/scrollcontainer_layoutmode_test.go b/mdl/backend/pagemutator/scrollcontainer_layoutmode_test.go new file mode 100644 index 0000000000..09a59546cf --- /dev/null +++ b/mdl/backend/pagemutator/scrollcontainer_layoutmode_test.go @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pagemutator + +import ( + "strings" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/backend/bsonnav" +) + +// `alter layout … { set (LayoutMode: 'Sidebar') on layoutContainer; }` — a +// scroll container's Headline / Sidebar, which CREATE LAYOUT now writes +// (ako/mxcli#1703). It fell to the pluggable path and was refused there. +func scrollContainerPage(withMode bool) bson.D { + w := bson.D{ + {Key: "$Type", Value: "Forms$ScrollContainer"}, + {Key: "Name", Value: "layoutContainer"}, + } + if withMode { + w = append(w, bson.E{Key: "LayoutMode", Value: "Headline"}) + } + return makeRawPage(w) +} + +func TestSetWidgetProperty_ScrollContainerLayoutMode(t *testing.T) { + for _, tc := range []struct{ in, want string }{{"Sidebar", "Sidebar"}, {"sidebar", "Sidebar"}, {"Headline", "Headline"}} { + rawData := scrollContainerPage(true) + m := &Mutator{rawData: rawData, widgetFinder: findBsonWidget} + if err := m.SetWidgetProperty("layoutContainer", "LayoutMode", tc.in); err != nil { + t.Fatalf("SetWidgetProperty(LayoutMode=%s): %v", tc.in, err) + } + if got := bsonnav.DGetString(findBsonWidget(rawData, "layoutContainer").widget, "LayoutMode"); got != tc.want { + t.Errorf("LayoutMode = %q, want %q", got, tc.want) + } + } +} + +func TestSetWidgetProperty_ScrollContainerLayoutModeRefusals(t *testing.T) { + // A value outside the enumeration, and the stored value is left alone. + rawData := scrollContainerPage(true) + m := &Mutator{rawData: rawData, widgetFinder: findBsonWidget} + if err := m.SetWidgetProperty("layoutContainer", "LayoutMode", "Topbar"); err == nil || + !strings.Contains(err.Error(), "Headline or Sidebar") { + t.Errorf("LayoutMode Topbar: err = %v, want a refusal naming Headline or Sidebar", err) + } + if got := bsonnav.DGetString(findBsonWidget(rawData, "layoutContainer").widget, "LayoutMode"); got != "Headline" { + t.Errorf("a refused value changed the stored LayoutMode to %q", got) + } + + // A document without the key: never invent one (overlay writes). + rawData = scrollContainerPage(false) + m = &Mutator{rawData: rawData, widgetFinder: findBsonWidget} + if err := m.SetWidgetProperty("layoutContainer", "LayoutMode", "Sidebar"); err == nil { + t.Error("a scroll container that stores no LayoutMode must not have one invented") + } + if bsonnav.DGet(findBsonWidget(rawData, "layoutContainer").widget, "LayoutMode") != nil { + t.Error("the refused write added a LayoutMode key") + } +} diff --git a/mdl/backend/security.go b/mdl/backend/security.go index 0112ab37fb..1cea4c42b6 100644 --- a/mdl/backend/security.go +++ b/mdl/backend/security.go @@ -34,6 +34,10 @@ type ProjectSecurityBackend interface { // (Security$ProjectSecurity.AdminUserName), which lint reads as // project_security().admin_user_name (mendixlabs/mxcli#1314). SetProjectAdminUserName(unitID model.ID, name string) error + // SetProjectAdminPassword sets that account's password + // (Security$ProjectSecurity.AdminPassword); an empty one is CE0129 at + // prototype and production level (mendixlabs/mxcli#624). + SetProjectAdminPassword(unitID model.ID, password string) error // SetProjectGuestAccess toggles anonymous access. An empty guestUserRole // leaves the stored role untouched — the caller is responsible for having // established that a role exists, because Mendix raises CE0133 on guest diff --git a/mdl/catalog/builder.go b/mdl/catalog/builder.go index 1a0c47f674..422984f454 100644 --- a/mdl/catalog/builder.go +++ b/mdl/catalog/builder.go @@ -121,6 +121,12 @@ type Builder struct { // cataloguing the handlers and emitted by buildReferences. eventHandlerRefs []eventHandlerRef + // Integration service → microflow edges (published OData resources, + // actions and authentication; published REST authentication; business + // event handlers; consumed OData configuration microflows), collected by + // the service builders and emitted by buildReferences (ako/mxcli#1639). + serviceMicroflowRefs []serviceMicroflowRef + // Page/snippet → document edges a widget makes beyond its single // widgets_data columns (a data grid 2's datasource AND on-click // microflow), collected by buildPages/buildSnippets and emitted by diff --git a/mdl/catalog/builder_graph.go b/mdl/catalog/builder_graph.go index 12e2a9de01..f5009eed09 100644 --- a/mdl/catalog/builder_graph.go +++ b/mdl/catalog/builder_graph.go @@ -30,10 +30,12 @@ var graphRefKinds = []string{ // of the analysis graph, so communities, layers, cycles and centrality all // see the API and scheduling surface as unreachable roots. // schedule a scheduled event runs it - // publish a published REST operation runs it + // publish a published service runs it: a REST operation, an OData + // resource or action, a service's authentication microflow // event an entity event handler runs it on every commit/delete — the // entity rather than the platform, but reached the same way, and - // structural rather than UI coupling + // structural rather than UI coupling — or a business event + // service runs it when a subscribed event arrives "schedule", "publish", "event", diff --git a/mdl/catalog/builder_modules.go b/mdl/catalog/builder_modules.go index a33a3f1ae7..d301fd484d 100644 --- a/mdl/catalog/builder_modules.go +++ b/mdl/catalog/builder_modules.go @@ -512,6 +512,11 @@ func (b *Builder) buildODataClients() error { qualifiedName := moduleName + "." + svc.Name + // The runtime calls these on every request the service makes, so the + // service is their caller. + b.addServiceMicroflowRefs(RefObjectConsumedODataService, svc.ID, qualifiedName, moduleName, + RefKindCall, svc.ConfigurationMicroflow, svc.HeadersMicroflow, svc.ErrorHandlingMicroflow) + validated := 0 if svc.Validated { validated = 1 @@ -563,6 +568,13 @@ func (b *Builder) buildODataServices() error { qualifiedName := moduleName + "." + svc.Name authTypes := strings.Join(svc.AuthenticationTypes, ", ") + // Every microflow the service runs is an entry point: a client's + // request reaches it, nothing in the model calls it. Without the edge a + // resource's read microflow and the service's authentication microflow + // read as dead to `refs`, `impact` and QUAL004 (ako/mxcli#1639). + b.addServiceMicroflowRefs(RefObjectPublishedODataService, svc.ID, qualifiedName, moduleName, + RefKindPublish, publishedODataMicroflows(svc)...) + _, err := stmt.Exec( string(svc.ID), svc.Name, @@ -616,6 +628,7 @@ func (b *Builder) buildBusinessEventServices() error { msgCount += len(ch.Messages) } } + handlers := make([]string, 0, len(svc.OperationImplementations)) for _, op := range svc.OperationImplementations { switch op.Operation { case "publish": @@ -623,7 +636,13 @@ func (b *Builder) buildBusinessEventServices() error { case "subscribe": subscribeCount++ } + handlers = append(handlers, op.Microflow) } + // An operation's microflow is the handler the platform runs when the + // event arrives — an event handler, like an entity's, and reached the + // same way: nothing in the model calls it. + b.addServiceMicroflowRefs(RefObjectBusinessEventService, svc.ID, qualifiedName, moduleName, + RefKindEvent, handlers...) serviceName := "" eventNamePrefix := "" diff --git a/mdl/catalog/builder_references.go b/mdl/catalog/builder_references.go index adade8b909..b0e66da663 100644 --- a/mdl/catalog/builder_references.go +++ b/mdl/catalog/builder_references.go @@ -14,7 +14,7 @@ import ( // Reference kinds for the refs table const ( - RefKindCall = "call" // Microflow calls microflow + RefKindCall = "call" // Microflow calls microflow (also: a workflow's handler, a consumed OData service's configuration microflow) RefKindCreate = "create" // Microflow creates entity RefKindRetrieve = "retrieve" // Microflow retrieves entity RefKindShowPage = "show_page" // Microflow shows page @@ -37,8 +37,8 @@ const ( RefKindWidget = "widget" // Page/snippet uses a pluggable or custom widget RefKindSettings = "settings" // A project setting names a microflow RefKindSync = "sync" // An offline navigation profile synchronizes an entity - RefKindPublish = "publish" // A published REST operation runs a microflow - RefKindEvent = "event" // An entity event handler runs a microflow + RefKindPublish = "publish" // A published service runs a microflow (REST operation, OData resource or action, authentication) + RefKindEvent = "event" // An event handler runs a microflow (entity event, or a business event a service subscribes to) RefKindMember = "member" // A document binds, reads or writes an attribute, or navigates an association RefKindXPath = "xpath" // An XPath constraint names an attribute, association or enumeration value RefKindType = "type" // An attribute, parameter or variable is typed as an enumeration @@ -46,6 +46,23 @@ const ( RefKindMapping = "mapping" // An import or export mapping maps an entity ) +// RefKinds is every value that reaches refs.RefKind. It is the vocabulary a +// consumer of the reference graph iterates when it must decide something per +// kind: TestQUAL004CountsEveryRefKind drives each one at the orphan rule, so an +// edge kind added here is covered there without anyone remembering the rule. +// TestEveryObjectTypeConstantIsInAPublishedList holds it complete against the +// RefKind constants above. +var RefKinds = []string{ + RefKindCall, RefKindCreate, RefKindRetrieve, RefKindShowPage, + RefKindGeneralize, RefKindAssociate, RefKindLayout, RefKindDatasource, + RefKindParameter, RefKindAction, RefKindHomePage, RefKindLoginPage, + RefKindMenuItem, RefKindChange, RefKindDelete, RefKindCommit, + RefKindCalculate, RefKindReturn, RefKindSchedule, RefKindValidate, + RefKindWidget, RefKindSettings, RefKindSync, RefKindPublish, + RefKindEvent, RefKindMember, RefKindXPath, RefKindType, + RefKindValue, RefKindMapping, +} + // Object types recorded in refs.SourceType and refs.TargetType — the catalog's // name for "what kind of document is at this end of the edge". They are the SDK // names, upper-cased, never Mendix's BSON storage names. @@ -71,6 +88,10 @@ const ( RefObjectJavaScriptAction = "JAVASCRIPT_ACTION" RefObjectRestOperation = "REST_OPERATION" RefObjectPublishedRestOperation = "PUBLISHED_REST_OPERATION" + RefObjectPublishedRestService = "PUBLISHED_REST_SERVICE" + RefObjectPublishedODataService = "PUBLISHED_ODATA_SERVICE" + RefObjectConsumedODataService = "CONSUMED_ODATA_SERVICE" + RefObjectBusinessEventService = "BUSINESS_EVENT_SERVICE" RefObjectRegularExpression = "REGULAR_EXPRESSION" RefObjectScheduledEvent = "SCHEDULED_EVENT" RefObjectProjectSettings = "PROJECT_SETTINGS" @@ -99,6 +120,10 @@ var ( RefObjectNavigation, RefObjectScheduledEvent, RefObjectPublishedRestOperation, + RefObjectPublishedRestService, + RefObjectPublishedODataService, + RefObjectConsumedODataService, + RefObjectBusinessEventService, RefObjectProjectSettings, RefObjectImportMapping, RefObjectExportMapping, @@ -799,6 +824,14 @@ func (b *Builder) buildReferences() error { // by GRAPH_DEAD_ASSETS, whose advice is to delete them (#1126). refCount += b.extractPublishedRestRefs(stmt, projectID, snapshotID) + // The other microflows an integration service names: a published OData + // resource's read/change microflow and actions, a published service's + // authentication microflow, a business event handler, a consumed OData + // service's configuration microflows. Same class again — the platform runs + // them — and found the same way: 34 Read_ microflows and the authentication + // microflow of six published OData services read as dead (ako/mxcli#1639). + refCount += b.extractServiceMicroflowRefs(stmt, projectID, snapshotID) + // Members, enumerations and mapped entities. Without these the graph ended // at documents, so `impact` on an attribute, an enumeration or a value said // "not referenced" however much it was used — the answer an agent reads as diff --git a/mdl/catalog/builder_rest.go b/mdl/catalog/builder_rest.go index 5b6f33a47c..0de19f65ed 100644 --- a/mdl/catalog/builder_rest.go +++ b/mdl/catalog/builder_rest.go @@ -161,6 +161,11 @@ func (b *Builder) buildPublishedRestServices() error { moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + svc.Name + // The custom authentication microflow runs on every authenticated + // request; the operation edges below do not cover it (ako/mxcli#1639). + b.addServiceMicroflowRefs(RefObjectPublishedRestService, svc.ID, qualifiedName, moduleName, + RefKindPublish, svc.AuthenticationMicroflow) + // Count total operations across all resources opCount := 0 for _, res := range svc.Resources { diff --git a/mdl/catalog/builder_service_refs.go b/mdl/catalog/builder_service_refs.go new file mode 100644 index 0000000000..6bc6568e7c --- /dev/null +++ b/mdl/catalog/builder_service_refs.go @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "database/sql" + "strings" + + "github.com/mendixlabs/mxcli/model" +) + +// serviceMicroflowRef is one edge from an integration service document to a +// microflow the platform runs on the service's behalf. The source is the +// service itself (its catalog id and qualified name), so the edge joins back to +// the service's own table — odata_services_data, published_rest_services_data, +// business_event_services_data or odata_clients_data — on SourceId. +// +// Collected while the service builders catalogue the services, and emitted by +// buildReferences, like scheduledEventRefs and publishedRestRefs. +type serviceMicroflowRef struct { + sourceType, sourceID, sourceName, moduleName, microflow, kind string +} + +// addServiceMicroflowRefs records one edge per distinct microflow the service +// names. An empty name is a real shape (a mode or a method chosen before its +// microflow is) and is skipped: an edge to "" would collide with every other +// unnamed target. A microflow named twice by one service — the same read +// microflow behind two resources — is one edge, as a page's widgets are. +func (b *Builder) addServiceMicroflowRefs(sourceType string, sourceID model.ID, sourceName, moduleName, kind string, microflows ...string) { + seen := map[string]bool{} + for _, mf := range microflows { + mf = strings.TrimSpace(mf) + if mf == "" || seen[mf] { + continue + } + seen[mf] = true + b.serviceMicroflowRefs = append(b.serviceMicroflowRefs, serviceMicroflowRef{ + sourceType: sourceType, + sourceID: string(sourceID), + sourceName: sourceName, + moduleName: moduleName, + microflow: mf, + kind: kind, + }) + } +} + +// publishedODataMicroflows returns every microflow a published OData service +// runs: the custom authentication microflow, each resource's read and +// insert/update/delete microflows, and each microflow published as an action. +func publishedODataMicroflows(svc *model.PublishedODataService) []string { + out := []string{svc.AuthMicroflow} + for _, es := range svc.EntitySets { + for _, mode := range []string{es.ReadMode, es.InsertMode, es.UpdateMode, es.DeleteMode} { + out = append(out, odataModeMicroflow(mode)) + } + } + for _, pm := range svc.Microflows { + out = append(out, pm.Microflow) + } + return out +} + +// odataModeMicroflow returns the microflow of a published resource's read or +// change mode, or "" when the mode names none. The reader spells a +// microflow-backed mode "CallMicroflow:Module.Name" (parseODataModeRaw), and a +// database-backed or unsupported one by a bare word. +func odataModeMicroflow(mode string) string { + if mf, ok := strings.CutPrefix(mode, "CallMicroflow:"); ok { + return mf + } + return "" +} + +// extractServiceMicroflowRefs emits the edges the service builders collected. +func (b *Builder) extractServiceMicroflowRefs(stmt *sql.Stmt, projectID, snapshotID string) int { + count := 0 + for _, r := range b.serviceMicroflowRefs { + if _, err := stmt.Exec( + r.sourceType, r.sourceID, r.sourceName, + RefObjectMicroflow, "", r.microflow, + r.kind, r.moduleName, projectID, snapshotID, + ); err == nil { + count++ + } + } + return count +} diff --git a/mdl/catalog/builder_service_refs_test.go b/mdl/catalog/builder_service_refs_test.go new file mode 100644 index 0000000000..a28a53c374 --- /dev/null +++ b/mdl/catalog/builder_service_refs_test.go @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "reflect" + "sort" + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" +) + +// A published OData service names microflows the platform runs for it — a +// resource's read microflow (`ReadMode: microflow X`), its insert/update/delete +// microflows, a microflow published as an OData action, and the custom +// authentication microflow (`Authentication: (microflow X)`). The catalog +// recorded none of them as an edge, so `mxcli refs` said "no references found", +// `impact` agreed, and QUAL004 advised "Remove if unused" on the read +// implementations behind six published services and the one microflow all six +// authenticate through — 35 of one backend's 43 findings (ako/mxcli#1639). +// +// The same gap, same mechanism — the builder holds the name and drops it — was +// open on three siblings: a published REST service's authentication microflow, +// a business event service's handler microflow, and a consumed OData service's +// configuration / headers / error-handling microflows. + +// serviceRefsFixture runs the real collecting halves — the service builders — +// and the real emitting half over the given services, and returns the catalog. +func serviceRefsFixture(t *testing.T, r *mock.MockBackend) *Catalog { + t.Helper() + cat, err := New() + if err != nil { + t.Fatalf("new catalog: %v", err) + } + t.Cleanup(func() { cat.Close() }) + + tx, err := cat.db.Begin() + if err != nil { + t.Fatalf("begin: %v", err) + } + // Every service list the builders read; the ones a test does not set are + // empty rather than the mock's "not configured" error. + if r.ListPublishedODataServicesFunc == nil { + r.ListPublishedODataServicesFunc = func() ([]*model.PublishedODataService, error) { return nil, nil } + } + if r.ListPublishedRestServicesFunc == nil { + r.ListPublishedRestServicesFunc = func() ([]*model.PublishedRestService, error) { return nil, nil } + } + if r.ListBusinessEventServicesFunc == nil { + r.ListBusinessEventServicesFunc = func() ([]*model.BusinessEventService, error) { return nil, nil } + } + if r.ListConsumedODataServicesFunc == nil { + r.ListConsumedODataServicesFunc = func() ([]*model.ConsumedODataService, error) { return nil, nil } + } + + b := &Builder{ + catalog: cat, + reader: r, + snapshot: &Snapshot{ID: "snap-1"}, + hierarchy: &hierarchy{ + moduleIDs: map[model.ID]bool{restRefsModuleID: true}, + moduleNames: map[model.ID]string{restRefsModuleID: "Sales"}, + containerParent: map[model.ID]model.ID{}, + folderNames: map[model.ID]string{}, + }, + tx: tx, + } + for _, step := range []struct { + name string + build func() error + }{ + {"buildODataClients", b.buildODataClients}, + {"buildODataServices", b.buildODataServices}, + {"buildBusinessEventServices", b.buildBusinessEventServices}, + {"buildPublishedRestServices", b.buildPublishedRestServices}, + } { + if err := step.build(); err != nil { + t.Fatalf("%s: %v", step.name, err) + } + } + + stmt, err := tx.Prepare(` + INSERT INTO refs (SourceType, SourceId, SourceName, TargetType, TargetId, TargetName, RefKind, ModuleName, ProjectId, SnapshotId) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + if err != nil { + t.Fatalf("prepare refs: %v", err) + } + projectID, snapshotID := b.snapshotMeta() + b.extractPublishedRestRefs(stmt, projectID, snapshotID) + b.extractServiceMicroflowRefs(stmt, projectID, snapshotID) + stmt.Close() + if err := tx.Commit(); err != nil { + t.Fatalf("commit: %v", err) + } + return cat +} + +// edgeRows returns "SourceType SourceName -> TargetName kind" for every refs +// row with the given source type, sorted. +func edgeRows(t *testing.T, cat *Catalog, sourceType string) []string { + t.Helper() + var out []string + for _, row := range queryRows(t, cat, `SELECT SourceName, TargetType, TargetName, RefKind, ModuleName FROM refs WHERE SourceType = '`+sourceType+`'`) { + out = append(out, row[0].(string)+" -> "+row[1].(string)+" "+row[2].(string)+" "+row[3].(string)+" ["+row[4].(string)+"]") + } + sort.Strings(out) + return out +} + +func publishedOData() *model.PublishedODataService { + svc := &model.PublishedODataService{ + ContainerID: restRefsModuleID, + Name: "LiveAPI", + AuthenticationTypes: []string{"Microflow"}, + AuthMicroflow: "Sales.AuthenticateApiClient", + EntitySets: []*model.PublishedEntitySet{ + { + ExposedName: "Rows", + ReadMode: "CallMicroflow:Sales.Read_Rows", + InsertMode: "CallMicroflow:Sales.Insert_Row", + UpdateMode: "ChangeFromDatabase", + DeleteMode: "NotSupported", + }, + // The same read microflow behind a second resource is one edge, + // and a microflow mode with no microflow named is none. + {ExposedName: "AllRows", ReadMode: "CallMicroflow:Sales.Read_Rows", UpdateMode: "CallMicroflow"}, + // A database-backed resource names no microflow. + {ExposedName: "Plain", ReadMode: "ReadFromDatabase", InsertMode: "ChangeFromDatabase"}, + }, + Microflows: []*model.PublishedMicroflow{{Microflow: "Sales.OD_DoThing", ExposedName: "DoThing"}}, + } + svc.ID = "odata-1" + return svc +} + +// TestPublishedODataServiceEmitsPublishEdges is the reported half of +// ako/mxcli#1639: ReadMode and Authentication, plus the change modes and the +// published actions, which are the same property family on the same document. +func TestPublishedODataServiceEmitsPublishEdges(t *testing.T) { + cat := serviceRefsFixture(t, &mock.MockBackend{ + ListPublishedODataServicesFunc: func() ([]*model.PublishedODataService, error) { + return []*model.PublishedODataService{publishedOData()}, nil + }, + }) + + got := edgeRows(t, cat, RefObjectPublishedODataService) + want := []string{ + "Sales.LiveAPI -> MICROFLOW Sales.AuthenticateApiClient publish [Sales]", + "Sales.LiveAPI -> MICROFLOW Sales.Insert_Row publish [Sales]", + "Sales.LiveAPI -> MICROFLOW Sales.OD_DoThing publish [Sales]", + "Sales.LiveAPI -> MICROFLOW Sales.Read_Rows publish [Sales]", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("published OData edges:\n got %v\nwant %v", got, want) + } + + // The edge joins back to the service row, so "who runs this microflow" is + // one join from the service's path and version. + joined := queryRows(t, cat, ` + SELECT COUNT(*) FROM refs r + JOIN odata_services_data s ON s.Id = r.SourceId + WHERE r.SourceType = 'PUBLISHED_ODATA_SERVICE'`) + if joined[0][0].(int64) != int64(len(want)) { + t.Errorf("the edges do not join to odata_services_data on SourceId; got %v of %d", joined[0][0], len(want)) + } +} + +// TestPublishedRestServiceAuthenticationMicroflowEmitsPublishEdge: the +// operation edges already existed; the service-level authentication microflow, +// which every authenticated request runs, had none. +func TestPublishedRestServiceAuthenticationMicroflowEmitsPublishEdge(t *testing.T) { + svc := oneService(&model.PublishedRestOperation{HTTPMethod: "GET", Path: "{id}", Microflow: "Sales.GetOrder"}) + svc.AuthenticationTypes = []string{"Microflow"} + svc.AuthenticationMicroflow = "Sales.AuthenticateRestClient" + cat := serviceRefsFixture(t, &mock.MockBackend{ + ListPublishedRestServicesFunc: func() ([]*model.PublishedRestService, error) { + return []*model.PublishedRestService{svc}, nil + }, + }) + + got := edgeRows(t, cat, RefObjectPublishedRestService) + want := []string{"Sales.OrdersApi -> MICROFLOW Sales.AuthenticateRestClient publish [Sales]"} + if !reflect.DeepEqual(got, want) { + t.Errorf("published REST service edges:\n got %v\nwant %v", got, want) + } + // CONTROL: the operation edge is still there, under its own source type. + if ops := edgeRows(t, cat, RefObjectPublishedRestOperation); len(ops) != 1 { + t.Errorf("control: want the one operation edge beside it, got %v", ops) + } +} + +// TestBusinessEventServiceHandlerEmitsEventEdge: a subscribe operation's +// microflow runs whenever the event arrives. business_events_data recorded it +// as SubscribeMicroflow and no edge followed. +func TestBusinessEventServiceHandlerEmitsEventEdge(t *testing.T) { + svc := &model.BusinessEventService{ + ContainerID: restRefsModuleID, + Name: "CustomerEvents", + OperationImplementations: []*model.ServiceOperation{ + {MessageName: "CustomerChanged", Operation: "subscribe", Entity: "Sales.CustomerChanged", Microflow: "Sales.OnCustomerChanged"}, + // A publish operation binds an entity and no microflow. + {MessageName: "OrderPlaced", Operation: "publish", Entity: "Sales.OrderPlaced"}, + }, + } + svc.ID = "bes-1" + cat := serviceRefsFixture(t, &mock.MockBackend{ + ListBusinessEventServicesFunc: func() ([]*model.BusinessEventService, error) { + return []*model.BusinessEventService{svc}, nil + }, + }) + + got := edgeRows(t, cat, RefObjectBusinessEventService) + want := []string{"Sales.CustomerEvents -> MICROFLOW Sales.OnCustomerChanged event [Sales]"} + if !reflect.DeepEqual(got, want) { + t.Errorf("business event service edges:\n got %v\nwant %v", got, want) + } +} + +// TestConsumedODataServiceMicroflowsEmitCallEdges: the runtime calls a consumed +// OData service's configuration, headers and error-handling microflows on every +// request it makes through the service. +func TestConsumedODataServiceMicroflowsEmitCallEdges(t *testing.T) { + svc := &model.ConsumedODataService{ + ContainerID: restRefsModuleID, + Name: "TripPin", + ConfigurationMicroflow: "Sales.ConfigureTripPin", + HeadersMicroflow: "Sales.TripPinHeaders", + ErrorHandlingMicroflow: "Sales.TripPinErrors", + } + svc.ID = "codata-1" + cat := serviceRefsFixture(t, &mock.MockBackend{ + ListConsumedODataServicesFunc: func() ([]*model.ConsumedODataService, error) { + return []*model.ConsumedODataService{svc}, nil + }, + }) + + got := edgeRows(t, cat, RefObjectConsumedODataService) + want := []string{ + "Sales.TripPin -> MICROFLOW Sales.ConfigureTripPin call [Sales]", + "Sales.TripPin -> MICROFLOW Sales.TripPinErrors call [Sales]", + "Sales.TripPin -> MICROFLOW Sales.TripPinHeaders call [Sales]", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("consumed OData service edges:\n got %v\nwant %v", got, want) + } +} + +// TestServiceMicroflowsAreNotDead is the consumer-facing symptom. +// GRAPH_DEAD_ASSETS is kind-agnostic, as are `refs` and (since ako/mxcli#1484) +// QUAL004, so the edge alone clears all three. The unreferenced microflow is the +// control: without it, a fix that emptied the view would pass. +func TestServiceMicroflowsAreNotDead(t *testing.T) { + cat := serviceRefsFixture(t, &mock.MockBackend{ + ListPublishedODataServicesFunc: func() ([]*model.PublishedODataService, error) { + return []*model.PublishedODataService{publishedOData()}, nil + }, + }) + for _, mf := range []string{"Read_Rows", "AuthenticateApiClient", "Orphan"} { + if _, err := cat.db.Exec( + `INSERT INTO microflows_data (Id, Name, QualifiedName, ModuleName, MicroflowType) + VALUES (?, ?, ?, 'Sales', 'MICROFLOW')`, + "mf-"+mf, mf, "Sales."+mf); err != nil { + t.Fatalf("insert microflow: %v", err) + } + } + + dead := map[string]bool{} + for _, row := range queryRows(t, cat, `SELECT QualifiedName FROM graph_dead_assets`) { + dead[row[0].(string)] = true + } + if !dead["Sales.Orphan"] { + t.Fatal("control failed: a microflow nothing references is not reported dead, " + + "so this test cannot detect the bug") + } + for _, live := range []string{"Sales.Read_Rows", "Sales.AuthenticateApiClient"} { + if dead[live] { + t.Errorf("%s runs behind a published OData service and is reported dead (ako/mxcli#1639)", live) + } + } +} diff --git a/mdl/catalog/lint_rule_doc_vocabulary_test.go b/mdl/catalog/lint_rule_doc_vocabulary_test.go index a1fdecc8d4..52bdc9b58d 100644 --- a/mdl/catalog/lint_rule_doc_vocabulary_test.go +++ b/mdl/catalog/lint_rule_doc_vocabulary_test.go @@ -285,6 +285,9 @@ func TestEveryObjectTypeConstantIsInAPublishedList(t *testing.T) { published []string }{ {"builder_references.go", "RefObject", append(append([]string{}, RefSourceObjectTypes...), RefTargetObjectTypes...)}, + // RefKinds is what TestQUAL004CountsEveryRefKind (mdl/linter) drives at + // the orphan rule, so a kind missing from it is a kind nothing checks. + {"builder_references.go", "RefKind", RefKinds}, {"builder_permissions.go", "PermissionElement", PermissionElementTypes}, {"builder_permissions.go", "AccessType", PermissionAccessTypes}, } { diff --git a/mdl/catalog/lint_rule_vocabulary_test.go b/mdl/catalog/lint_rule_vocabulary_test.go index f95d6df118..8c8cb5f6a2 100644 --- a/mdl/catalog/lint_rule_vocabulary_test.go +++ b/mdl/catalog/lint_rule_vocabulary_test.go @@ -28,7 +28,7 @@ import ( // QUAL004 had the sibling bug in the other vocabulary: it counted only the "call" // and "schedule" reference kinds, so a microflow reached through a page data // source, a widget action or a calculated attribute was reported as -// "not called from anywhere". +// "not called from anywhere". It now counts every kind (see below). func lintRulesDir(t *testing.T) string { t.Helper() @@ -95,63 +95,13 @@ func starListItems(src, name string) []string { return out } -// TestQUAL004EntryKindsAreRealRefKinds keeps the orphan rule's two kind lists -// spelled the way the reference builder emits them. A typo or an invented kind -// silently narrows the rule into false positives. -func TestQUAL004EntryKindsAreRealRefKinds(t *testing.T) { - known := map[string]bool{} - for _, k := range []string{ - RefKindCall, RefKindCreate, RefKindRetrieve, RefKindShowPage, - RefKindGeneralize, RefKindAssociate, RefKindLayout, RefKindDatasource, - RefKindParameter, RefKindAction, RefKindHomePage, RefKindLoginPage, - RefKindMenuItem, RefKindChange, RefKindDelete, RefKindCalculate, - RefKindReturn, RefKindSchedule, RefKindValidate, RefKindSettings, - RefKindWidget, RefKindSync, RefKindPublish, RefKindEvent, RefKindCommit, - } { - known[k] = true - } - - src := readRule(t, "orphaned_elements.star") - for _, listName := range []string{"MICROFLOW_ENTRY_KINDS", "PAGE_ENTRY_KINDS"} { - items := starListItems(src, listName) - if len(items) == 0 { - t.Fatalf("%s not found in orphaned_elements.star, or is empty", listName) - } - for _, kind := range items { - if !known[kind] { - t.Errorf("%s lists %q, which the reference builder never emits", listName, kind) - } - } - } -} - -// The kinds that mean "this runs" / "this opens" must actually be listed. Losing -// one is how the rule regresses into reporting live documents as dead. -func TestQUAL004CountsEveryEntryPointKind(t *testing.T) { - src := readRule(t, "orphaned_elements.star") - - for _, want := range []string{ - RefKindCall, RefKindSchedule, RefKindDatasource, RefKindAction, RefKindCalculate, - RefKindSettings, RefKindPublish, - // An entity event handler runs on every commit/delete of its entity. - // Of every kind in this list it is the one whose absence flags the - // hottest code (mendixlabs/mxcli#1127). - RefKindEvent, - } { - if !contains(starListItems(src, "MICROFLOW_ENTRY_KINDS"), want) { - t.Errorf("MICROFLOW_ENTRY_KINDS is missing %q — a microflow reached only that way "+ - "is reported as 'not called from anywhere'", want) - } - } - for _, want := range []string{ - RefKindShowPage, RefKindHomePage, RefKindLoginPage, RefKindMenuItem, - } { - if !contains(starListItems(src, "PAGE_ENTRY_KINDS"), want) { - t.Errorf("PAGE_ENTRY_KINDS is missing %q — a page reached only that way "+ - "is reported as orphaned", want) - } - } -} +// QUAL004 no longer keeps a list of the reference kinds that count as a caller: +// the two lists that were checked here (MICROFLOW_ENTRY_KINDS, PAGE_ENTRY_KINDS) +// drifted behind the catalog — the microflow one lacked home_page and menu_item, +// so a microflow started only from the navigation was reported as unused +// (ako/mxcli#1484). Every edge into a document now counts, and +// TestQUAL004CountsEveryRefKind in mdl/linter drives each of RefKinds at the +// rule instead of pinning a copy of the list here. func contains(hay []string, needle string) bool { for _, h := range hay { diff --git a/mdl/catalog/tables.go b/mdl/catalog/tables.go index da0b941349..28113f3ecc 100644 --- a/mdl/catalog/tables.go +++ b/mdl/catalog/tables.go @@ -7,6 +7,14 @@ package catalog // // History: // +// 29 (service microflow edges): refs gains the edges from an integration +// service to the microflows it runs — a published OData resource's +// read/change microflows, its actions and authentication microflow, a +// published REST service's authentication microflow (`publish`), a +// business event handler (`event`), a consumed OData service's +// configuration microflows (`call`) (ako/mxcli#1639). No column changes, +// but a cached catalog would keep answering "no references" for every one +// of them until something else rebuilt it. // 28 (message definitions): message_definition_collections and // message_definitions, unioned into objects as // MESSAGE_DEFINITION_COLLECTION / MESSAGE_DEFINITION, so documents() @@ -148,7 +156,7 @@ package catalog // SnapshotSource / SourceId / SourceBranch / SourceRevision columns // from every row (issue #576). // 1 — initial flat schema with denormalized snapshot columns on every row. -const CatalogSchemaVersion = "28" +const CatalogSchemaVersion = "29" // MetaSchemaVersion is the catalog_meta key that records the schema version // the cache was built against. diff --git a/mdl/executor/catalog_query_rewrite.go b/mdl/executor/catalog_query_rewrite.go new file mode 100644 index 0000000000..3161b0d13a --- /dev/null +++ b/mdl/executor/catalog_query_rewrite.go @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import "strings" + +// rewriteCatalogTableRefs turns each `CATALOG.
` table reference in a +// catalog SELECT into the bare
the SQLite catalog holds, and returns +// the tables it rewrote (lower case, first appearance first, no duplicates). +// +// It reads the query the way SQLite will, token by token, because the prefix +// is only a table reference where SQLite would read one. A regex over the raw +// text also matched `'Catalog.Product'` inside a string literal, so a filter on +// a module named Catalog compared against 'Product' and never matched +// (ako/mxcli#1450). Skipped verbatim: +// +// - string literals in single quotes, where a doubled quote is an escaped +// one, not the end; +// - quoted identifiers in double quotes, backticks or brackets (a doubled +// quote or backtick likewise escapes itself) — a name the user quoted is +// spelled exactly, even when it reads Catalog.x; +// - -- line comments and /* */ block comments. +// +// Of what is left, `catalog` followed by `.` and a name is the prefix, in any +// case and with whitespace or comments around the dot (the grammar reads +// CATALOG DOT name as three tokens, so `CATALOG . ENTITIES` parses; SQLite +// would read it as an unknown schema). `ProductCatalog.Item` is one identifier +// followed by a dot, so a name that merely contains the word is never touched. +// A `catalog` that itself follows a dot (`x.catalog.y`) is a column path, not +// the prefix. +// +// An unknown table name is still rewritten, so a new catalog view needs no +// allowlist entry: it yields SQLite's "no such table" rather than a silent +// passthrough. +func rewriteCatalogTableRefs(query string) (string, []string) { + var out strings.Builder + out.Grow(len(query)) + var tables []string + seen := map[string]bool{} + + copied := 0 // query[:copied] has been written to out + prevDot := false // the previous significant token was '.' + for i := 0; i < len(query); { + end, kind := sqlTokenEnd(query, i) + switch kind { + case sqlTokSpace, sqlTokComment: + i = end + continue + case sqlTokIdent: + if !prevDot && strings.EqualFold(query[i:end], "catalog") { + dot := skipSQLTrivia(query, end) + if dot < len(query) && query[dot] == '.' { + name := skipSQLTrivia(query, dot+1) + if name < len(query) { + if nameEnd, nameKind := sqlTokenEnd(query, name); nameKind == sqlTokIdent || nameKind == sqlTokQuotedIdent { + out.WriteString(query[copied:i]) + copied = name + table := strings.ToLower(unquoteSQLIdent(query[name:nameEnd])) + if !seen[table] { + seen[table] = true + tables = append(tables, table) + } + // Resume at the table name, which is an ordinary + // identifier from here on. + prevDot = false + i = name + continue + } + } + } + } + } + prevDot = kind == sqlTokOther && query[i:end] == "." + i = end + } + out.WriteString(query[copied:]) + return out.String(), tables +} + +type sqlTokKind int + +const ( + sqlTokSpace sqlTokKind = iota + sqlTokComment + sqlTokString + sqlTokQuotedIdent + sqlTokIdent + sqlTokOther +) + +// sqlTokenEnd returns the end of the token starting at i and its kind, +// following SQLite's tokenizer for the cases that matter here. An unterminated +// literal, quoted identifier or block comment runs to the end of the input, as +// it does for SQLite (which then reports it). +func sqlTokenEnd(s string, i int) (int, sqlTokKind) { + c := s[i] + switch { + case c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f': + j := i + 1 + for j < len(s) && (s[j] == ' ' || s[j] == '\t' || s[j] == '\n' || s[j] == '\r' || s[j] == '\f') { + j++ + } + return j, sqlTokSpace + case c == '-' && i+1 < len(s) && s[i+1] == '-': + if nl := strings.IndexByte(s[i:], '\n'); nl >= 0 { + return i + nl + 1, sqlTokComment + } + return len(s), sqlTokComment + case c == '/' && i+1 < len(s) && s[i+1] == '*': + if close := strings.Index(s[i+2:], "*/"); close >= 0 { + return i + 2 + close + 2, sqlTokComment + } + return len(s), sqlTokComment + case c == '\'': + return quotedEnd(s, i, '\''), sqlTokString + case c == '"' || c == '`': + return quotedEnd(s, i, c), sqlTokQuotedIdent + case c == '[': + if close := strings.IndexByte(s[i+1:], ']'); close >= 0 { + return i + 1 + close + 1, sqlTokQuotedIdent + } + return len(s), sqlTokQuotedIdent + case isSQLIdentByte(c) && !(c >= '0' && c <= '9'): + j := i + 1 + for j < len(s) && (isSQLIdentByte(s[j]) || s[j] == '$') { + j++ + } + return j, sqlTokIdent + case c >= '0' && c <= '9': + // A number, including a decimal point and exponent, so `1.5` is never + // read as an identifier followed by a dot. + j := i + 1 + for j < len(s) && (isSQLIdentByte(s[j]) || s[j] == '.') { + j++ + } + return j, sqlTokOther + case c == '$' || c == ':' || c == '@' || c == '#' || c == '?': + // A bound parameter: `$catalog` is one token, not the prefix. + j := i + 1 + for j < len(s) && (isSQLIdentByte(s[j]) || s[j] == '$') { + j++ + } + return j, sqlTokOther + } + return i + 1, sqlTokOther +} + +// quotedEnd returns the end of a token opened by q at s[i], where a doubled q +// stands for one q inside it. +func quotedEnd(s string, i int, q byte) int { + for j := i + 1; j < len(s); j++ { + if s[j] == q { + if j+1 < len(s) && s[j+1] == q { + j++ + continue + } + return j + 1 + } + } + return len(s) +} + +// isSQLIdentByte reports whether c can appear in an unquoted SQLite +// identifier. Bytes >= 0x80 are identifier characters in SQLite, which keeps a +// UTF-8 sequence inside the identifier it belongs to. +func isSQLIdentByte(c byte) bool { + return c == '_' || c >= 0x80 || + (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') +} + +// skipSQLTrivia returns the index of the first byte at or after i that is not +// whitespace or a comment. +func skipSQLTrivia(s string, i int) int { + for i < len(s) { + end, kind := sqlTokenEnd(s, i) + if kind != sqlTokSpace && kind != sqlTokComment { + return i + } + i = end + } + return i +} + +// unquoteSQLIdent strips the quotes from a quoted identifier token and leaves +// a bare one alone. +func unquoteSQLIdent(tok string) string { + if len(tok) < 2 { + return tok + } + switch tok[0] { + case '"', '`': + q := string(tok[0]) + return strings.ReplaceAll(strings.TrimSuffix(tok[1:], q), q+q, q) + case '[': + return strings.TrimSuffix(tok[1:], "]") + } + return tok +} diff --git a/mdl/executor/cmd_catalog.go b/mdl/executor/cmd_catalog.go index 8c765ca04e..6362c9e763 100644 --- a/mdl/executor/cmd_catalog.go +++ b/mdl/executor/cmd_catalog.go @@ -11,7 +11,6 @@ import ( "log" "os" "path/filepath" - "regexp" "strings" "sync" "time" @@ -117,27 +116,13 @@ var communitiesOnlyTables = map[string]bool{ "graph_integration_surface": true, } -// extractTableFromQuery extracts the table name after FROM in a converted catalog query. -func extractTableFromQuery(query string) string { - lower := strings.ToLower(query) - idx := strings.Index(lower, "from ") - if idx == -1 { - return "" - } - rest := strings.TrimSpace(lower[idx+5:]) - // Table name is the next word - end := strings.IndexAny(rest, " \t\n;,)") - if end == -1 { - return rest - } - return rest[:end] -} - -// warnIfCatalogModeInsufficient checks if the query targets a table that requires -// a higher catalog build mode than what's currently cached, and prints a warning. -func warnIfCatalogModeInsufficient(ctx *ExecContext, query string) { - table := extractTableFromQuery(query) - if table == "" { +// warnIfCatalogModeInsufficient warns for each queried table that requires a +// higher catalog build mode than what's currently cached. tables are the ones +// rewriteCatalogTableRefs found, so every table of a join or subquery is +// checked, and a literal containing "from" cannot stand in for one — the old +// guess took the word after the first "from " in the text (ako/mxcli#1450). +func warnIfCatalogModeInsufficient(ctx *ExecContext, tables []string) { + if len(tables) == 0 { return } @@ -156,14 +141,17 @@ func warnIfCatalogModeInsufficient(ctx *ExecContext, query string) { modeRank := map[string]int{"fast": 1, "full": 2, "source": 3} currentRank := modeRank[buildMode] - if communitiesOnlyTables[table] && !graphAnalysis { - fmt.Fprintf(ctx.progress(), "Warning: CATALOG.%s requires refresh catalog communities (not run for this catalog)\n", strings.ToUpper(table)) - return - } - if sourceOnlyTables[table] && currentRank < modeRank["source"] { - fmt.Fprintf(ctx.progress(), "Warning: CATALOG.%s requires refresh catalog full source (current mode: %s)\n", strings.ToUpper(table), buildMode) - } else if fullOnlyTables[table] && currentRank < modeRank["full"] { - fmt.Fprintf(ctx.progress(), "Warning: CATALOG.%s requires refresh catalog full (current mode: %s)\n", strings.ToUpper(table), buildMode) + for _, table := range tables { + switch { + case communitiesOnlyTables[table]: + if !graphAnalysis { + fmt.Fprintf(ctx.progress(), "Warning: CATALOG.%s requires refresh catalog communities (not run for this catalog)\n", strings.ToUpper(table)) + } + case sourceOnlyTables[table] && currentRank < modeRank["source"]: + fmt.Fprintf(ctx.progress(), "Warning: CATALOG.%s requires refresh catalog full source (current mode: %s)\n", strings.ToUpper(table), buildMode) + case fullOnlyTables[table] && currentRank < modeRank["full"]: + fmt.Fprintf(ctx.progress(), "Warning: CATALOG.%s requires refresh catalog full (current mode: %s)\n", strings.ToUpper(table), buildMode) + } } } @@ -177,10 +165,10 @@ func execCatalogQuery(ctx *ExecContext, query string) error { } // Convert CATALOG.xxx table names to actual table names - query = convertCatalogTableNames(query) + query, tables := rewriteCatalogTableRefs(query) // Warn if querying a table that requires a higher build mode - warnIfCatalogModeInsufficient(ctx, query) + warnIfCatalogModeInsufficient(ctx, tables) // Execute query result, err := ctx.Catalog.Query(query) @@ -881,16 +869,12 @@ func writeCatalogStatusJSON(ctx *ExecContext) error { return enc.Encode(status) } -// catalogTableRefRe matches a `CATALOG.` table reference (case-insensitive). -var catalogTableRefRe = regexp.MustCompile(`(?i)\bcatalog\.([a-z_][a-z0-9_]*)`) - // convertCatalogTableNames strips the CATALOG. prefix from table references so the -// query runs against the underlying SQLite views/tables. Generic by design: any -// CATALOG. is rewritten, so new catalog views (e.g. the graph_* analysis -// views) work without maintaining a per-name allowlist. An unknown name simply -// yields SQLite's "no such table" rather than a silent passthrough. +// query runs against the underlying SQLite views/tables; string literals, quoted +// identifiers and comments are left alone. See rewriteCatalogTableRefs. func convertCatalogTableNames(query string) string { - return catalogTableRefRe.ReplaceAllString(query, "$1") + converted, _ := rewriteCatalogTableRefs(query) + return converted } // outputCatalogResults outputs query results in table or JSON format. diff --git a/mdl/executor/cmd_catalog_prefix_rewrite_test.go b/mdl/executor/cmd_catalog_prefix_rewrite_test.go new file mode 100644 index 0000000000..fc28c710cc --- /dev/null +++ b/mdl/executor/cmd_catalog_prefix_rewrite_test.go @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/catalog" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// ako/mxcli#1450 (mendixlabs/mxcli#1553): the CATALOG. table prefix was +// stripped by a regex over the whole query, so it was also stripped inside a +// string literal. A module named Catalog could then never be filtered on: +// `WHERE QualifiedName = 'Catalog.Product'` compared against 'Product'. +// +// The rewrite is only correct if it touches what SQLite would read as a +// schema-qualified table name and nothing else, so the table below covers each +// lexical context SQLite has: string literals (with the doubled-quote escape), +// the three quoted-identifier spellings, both comment forms, and the controls +// that must keep working — upper/lower/mixed-case prefixes, joins, subqueries, +// a query with no prefix at all, and names that merely contain "catalog". +func TestConvertCatalogTableNames(t *testing.T) { + cases := []struct { + name, in, want string + }{ + // Controls: the prefix itself, in every case, is still removed. + {"upper", `SELECT * FROM CATALOG.ENTITIES`, `SELECT * FROM ENTITIES`}, + {"lower", `select * from catalog.entities`, `select * from entities`}, + {"mixed", `SELECT * FROM Catalog.Entities`, `SELECT * FROM Entities`}, + {"join of two catalog tables", + `SELECT e.Name, a.Name FROM CATALOG.ENTITIES e JOIN catalog.attributes a ON a.EntityQualifiedName = e.QualifiedName`, + `SELECT e.Name, a.Name FROM ENTITIES e JOIN attributes a ON a.EntityQualifiedName = e.QualifiedName`}, + {"subquery", + `SELECT Name FROM CATALOG.MICROFLOWS WHERE QualifiedName NOT IN (SELECT TargetName FROM CATALOG.REFS)`, + `SELECT Name FROM MICROFLOWS WHERE QualifiedName NOT IN (SELECT TargetName FROM REFS)`}, + {"no prefix", `SELECT * FROM entities WHERE ModuleName = 'Sales'`, `SELECT * FROM entities WHERE ModuleName = 'Sales'`}, + {"table-qualified column", `SELECT CATALOG.ENTITIES.Name FROM CATALOG.ENTITIES`, `SELECT ENTITIES.Name FROM ENTITIES`}, + {"catalog as a plain alias", `SELECT 1 AS catalog FROM CATALOG.MODULES`, `SELECT 1 AS catalog FROM MODULES`}, + + // The issue: the prefix inside a string literal is data, not a table. + {"literal in select list (the issue)", + `SELECT QualifiedName, 'Catalog.Product' AS lit FROM CATALOG.ENTITIES WHERE ModuleName = 'Catalog'`, + `SELECT QualifiedName, 'Catalog.Product' AS lit FROM ENTITIES WHERE ModuleName = 'Catalog'`}, + {"literal in where (the issue)", + `SELECT COUNT(*) AS n FROM CATALOG.ENTITIES WHERE QualifiedName = 'Catalog.Product'`, + `SELECT COUNT(*) AS n FROM ENTITIES WHERE QualifiedName = 'Catalog.Product'`}, + {"lower-case prefix in literal", + `SELECT * FROM catalog.entities WHERE Documentation LIKE '%see catalog.entities%'`, + `SELECT * FROM entities WHERE Documentation LIKE '%see catalog.entities%'`}, + {"doubled quote does not end the literal", + `SELECT * FROM CATALOG.ENTITIES WHERE Documentation = 'it''s Catalog.Product'`, + `SELECT * FROM ENTITIES WHERE Documentation = 'it''s Catalog.Product'`}, + {"literal ending in a doubled quote, then a prefix", + `SELECT 'x''' AS a, 'Catalog.Product' AS b FROM CATALOG.MODULES`, + `SELECT 'x''' AS a, 'Catalog.Product' AS b FROM MODULES`}, + + // Quoted identifiers are names the user spelled exactly. + {"double-quoted identifier", `SELECT Name AS "Catalog.Name" FROM CATALOG.ENTITIES`, `SELECT Name AS "Catalog.Name" FROM ENTITIES`}, + {"double-quoted with doubled quote", `SELECT Name AS "a""Catalog.b" FROM CATALOG.ENTITIES`, `SELECT Name AS "a""Catalog.b" FROM ENTITIES`}, + {"backtick identifier", "SELECT Name AS `Catalog.Name` FROM CATALOG.ENTITIES", "SELECT Name AS `Catalog.Name` FROM ENTITIES"}, + {"bracket identifier", `SELECT Name AS [Catalog.Name] FROM CATALOG.ENTITIES`, `SELECT Name AS [Catalog.Name] FROM ENTITIES`}, + + // Comments are kept verbatim. + {"line comment", "SELECT * FROM CATALOG.ENTITIES -- not Catalog.Product\nWHERE 1 = 1", "SELECT * FROM ENTITIES -- not Catalog.Product\nWHERE 1 = 1"}, + {"block comment", `SELECT /* Catalog.Product */ * FROM CATALOG.ENTITIES`, `SELECT /* Catalog.Product */ * FROM ENTITIES`}, + + // Names that merely contain "catalog" are not the prefix. + {"module name containing catalog, in literals", + `SELECT * FROM CATALOG.ENTITIES WHERE QualifiedName IN ('ProductCatalog.Item', 'MyCatalog.Thing', 'my_catalog.x')`, + `SELECT * FROM ENTITIES WHERE QualifiedName IN ('ProductCatalog.Item', 'MyCatalog.Thing', 'my_catalog.x')`}, + {"alias containing catalog", `SELECT ProductCatalog.Name FROM CATALOG.ENTITIES AS ProductCatalog`, `SELECT ProductCatalog.Name FROM ENTITIES AS ProductCatalog`}, + {"underscore alias containing catalog", `SELECT my_catalog.Name FROM CATALOG.ENTITIES my_catalog`, `SELECT my_catalog.Name FROM ENTITIES my_catalog`}, + + // The grammar accepts whitespace around the dot (CATALOG DOT name are + // three tokens); SQLite reads `CATALOG . ENTITIES` as a schema, so it + // has to be rewritten the same way. + {"whitespace around the dot", "SELECT * FROM CATALOG . ENTITIES e JOIN CATALOG\n.ATTRIBUTES a ON 1 = 1", "SELECT * FROM ENTITIES e JOIN ATTRIBUTES a ON 1 = 1"}, + + // An unterminated literal swallows the rest, as it does for SQLite, + // which then reports it; the rewrite must not invent a table there. + {"unterminated literal", `SELECT 'Catalog.x FROM CATALOG.ENTITIES`, `SELECT 'Catalog.x FROM CATALOG.ENTITIES`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := convertCatalogTableNames(tc.in); got != tc.want { + t.Errorf("convertCatalogTableNames(%q)\n got: %q\nwant: %q", tc.in, got, tc.want) + } + }) + } +} + +// catalogModuleFixture is an in-memory catalog holding a module literally +// named Catalog beside modules whose names only contain the word, which is the +// shape the issue needs and the controls need. +func catalogModuleFixture(t *testing.T) *catalog.Catalog { + t.Helper() + cat, err := catalog.New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + db := cat.CatalogDB() + for _, m := range []string{"Catalog", "ProductCatalog", "MyCatalog"} { + if _, err := db.Exec(`INSERT INTO modules_data (Id, Name, QualifiedName, ModuleName) VALUES (?, ?, ?, ?)`, "m-"+m, m, m, m); err != nil { + t.Fatalf("seed module: %v", err) + } + } + for _, e := range [][2]string{{"Catalog", "Product"}, {"ProductCatalog", "Item"}, {"MyCatalog", "Thing"}} { + qn := e[0] + "." + e[1] + if _, err := db.Exec(`INSERT INTO entities_data (Id, Name, QualifiedName, ModuleName, EntityType) VALUES (?, ?, ?, ?, 'PERSISTENT')`, "e-"+qn, e[1], qn, e[0]); err != nil { + t.Fatalf("seed entity: %v", err) + } + } + cat.SetBuilt(true) + return cat +} + +// runCatalogSelect parses src as MDL, exactly as -c, a script and the REPL +// do, and runs the SELECT it yields. +func runCatalogSelect(t *testing.T, ctx *ExecContext, src string) { + t.Helper() + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parse %q: %v", src, errs) + } + if len(prog.Statements) != 1 { + t.Fatalf("parse %q: got %d statements, want 1", src, len(prog.Statements)) + } + sel, ok := prog.Statements[0].(*ast.SelectStmt) + if !ok { + t.Fatalf("parse %q: got %T, want *ast.SelectStmt", src, prog.Statements[0]) + } + if err := execCatalogQuery(ctx, sel.Query); err != nil { + t.Fatalf("execCatalogQuery(%q): %v", sel.Query, err) + } +} + +// TestCatalogQueryOnModuleNamedCatalog runs the issue's two queries end to end +// (MDL parse → execCatalogQuery → SQLite → JSON) plus the controls. +func TestCatalogQueryOnModuleNamedCatalog(t *testing.T) { + cat := catalogModuleFixture(t) + cases := []struct { + name, src string + want []map[string]string + }{ + {"issue: literal in select list", + `SELECT QualifiedName, 'Catalog.Product' AS lit FROM CATALOG.ENTITIES WHERE ModuleName = 'Catalog'`, + []map[string]string{{"QualifiedName": "Catalog.Product", "lit": "Catalog.Product"}}}, + {"issue: filter on qualified name", + `SELECT COUNT(*) AS n FROM CATALOG.ENTITIES WHERE QualifiedName = 'Catalog.Product'`, + []map[string]string{{"n": "1"}}}, + {"lower-case prefix, filter on qualified name", + `select count(*) as n from catalog.entities where QualifiedName = 'Catalog.Product'`, + []map[string]string{{"n": "1"}}}, + {"module names containing catalog", + `SELECT QualifiedName FROM CATALOG.ENTITIES WHERE QualifiedName IN ('ProductCatalog.Item', 'MyCatalog.Thing') ORDER BY QualifiedName`, + []map[string]string{{"QualifiedName": "MyCatalog.Thing"}, {"QualifiedName": "ProductCatalog.Item"}}}, + {"join of two catalog tables", + `SELECT e.QualifiedName AS q FROM CATALOG.ENTITIES e JOIN CATALOG.MODULES m ON m.Name = e.ModuleName WHERE m.Name = 'Catalog'`, + []map[string]string{{"q": "Catalog.Product"}}}, + {"whitespace around the dot", + `SELECT COUNT(*) AS n FROM CATALOG . ENTITIES WHERE ModuleName = 'Catalog'`, + []map[string]string{{"n": "1"}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var diag bytes.Buffer + ctx, out := newMockCtx(t, withFormat(FormatJSON)) + ctx.Catalog = cat + ctx.Diagnostics = &diag + runCatalogSelect(t, ctx, tc.src) + var got []map[string]string + if err := json.Unmarshal(out.Bytes(), &got); err != nil { + t.Fatalf("output is not a JSON array of rows: %v\n%s", err, out.String()) + } + gotJSON, _ := json.Marshal(got) + wantJSON, _ := json.Marshal(tc.want) + if string(gotJSON) != string(wantJSON) { + t.Errorf("%s\n got: %s\nwant: %s", tc.src, gotJSON, wantJSON) + } + }) + } +} + +// TestCatalogQueryModeWarningReadsTheTables: the mode warning found its table +// by taking the word after the first "from " in the query text, so a literal +// containing "from" hid the real table, and only the first table of a join was +// ever checked. CATALOG.REFS is filled only by `refresh catalog full`; on this +// fast catalog a query against it must say so however it is spelled. +func TestCatalogQueryModeWarningReadsTheTables(t *testing.T) { + cat := catalogModuleFixture(t) + const warning = "Warning: CATALOG.REFS requires refresh catalog full" + cases := []struct { + name, src string + warn bool + }{ + {"control: plain query", `SELECT COUNT(*) AS n FROM CATALOG.REFS`, true}, + {"control: fast table does not warn", `SELECT COUNT(*) AS n FROM CATALOG.ENTITIES`, false}, + {"literal containing from", `SELECT 'moved from Sales' AS note FROM CATALOG.REFS`, true}, + {"gated table second in a join", `SELECT COUNT(*) AS n FROM CATALOG.ENTITIES e JOIN CATALOG.REFS r ON r.TargetName = e.QualifiedName`, true}, + {"literal naming the table does not warn", `SELECT 'from refs' AS note FROM CATALOG.ENTITIES`, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx, out := newMockCtx(t) + ctx.Catalog = cat + runCatalogSelect(t, ctx, tc.src) + if got := strings.Contains(out.String(), warning); got != tc.warn { + t.Errorf("%s\nwarned = %v, want %v; output:\n%s", tc.src, got, tc.warn, out.String()) + } + }) + } +} diff --git a/mdl/executor/cmd_diff.go b/mdl/executor/cmd_diff.go index ae0cf36394..491fea9631 100644 --- a/mdl/executor/cmd_diff.go +++ b/mdl/executor/cmd_diff.go @@ -6,10 +6,13 @@ package executor import ( + "bytes" "fmt" "io" + "strings" "github.com/pmezard/go-difflib/difflib" + "go.mongodb.org/mongo-driver/v2/bson" "github.com/mendixlabs/mxcli/mdl/ast" ) @@ -159,3 +162,107 @@ func LineChanges(current, proposed string) []StructuralChange { } return out } + +// ChangedPaths lists up to max property paths whose values differ between two +// unit documents, element $IDs aside. A list element is named by its Name +// where it has one. It names properties, never values, so a secret that +// changed is reported by where it is stored and not by what it holds. +// +// Both diffs use it for a unit that is written although its MDL rendering +// does not change: `mxcli diff` for what exec would write, diff-local for what +// changed between two revisions (ako/mxcli#1512). +func ChangedPaths(a, b []byte, max int) []string { + var out []string + var walk func(path string, x, y bson.RawValue) + walk = func(path string, x, y bson.RawValue) { + if len(out) >= max { + return + } + if x.Type != y.Type { + out = append(out, path) + return + } + switch x.Type { + case bson.TypeEmbeddedDocument: + dx, dy := x.Document(), y.Document() + ex, _ := dx.Elements() + keys := map[string]bool{} + for _, e := range ex { + k := e.Key() + keys[k] = true + if k == "$ID" { + continue + } + vy, err := dy.LookupErr(k) + if err != nil { + out = append(out, diffPathJoin(path, k)) + } else { + walk(diffPathJoin(path, k), e.Value(), vy) + } + if len(out) >= max { + return + } + } + ey, _ := dy.Elements() + for _, e := range ey { + if !keys[e.Key()] && len(out) < max { + out = append(out, diffPathJoin(path, e.Key())) + } + } + case bson.TypeArray: + vx, _ := x.Array().Values() + vy, _ := y.Array().Values() + vx, vy = withoutListMarker(vx), withoutListMarker(vy) + if len(vx) != len(vy) { + out = append(out, fmt.Sprintf("%s (%d -> %d items)", path, len(vx), len(vy))) + return + } + for i := range vx { + walk(fmt.Sprintf("%s[%s]", path, diffPathItemName(vx[i], i)), vx[i], vy[i]) + if len(out) >= max { + return + } + } + case bson.TypeBinary: + // A pointer holds an element $ID, so a pointer to an element exec + // re-minted differs without anything having changed; canon has + // already said whether the unit did. A GUID is reported. + if !bytes.Equal(x.Value, y.Value) && !strings.HasSuffix(path, "Pointer") { + out = append(out, path) + } + default: + if !bytes.Equal(x.Value, y.Value) { + out = append(out, path) + } + } + } + walk("", bson.RawValue{Type: bson.TypeEmbeddedDocument, Value: a}, bson.RawValue{Type: bson.TypeEmbeddedDocument, Value: b}) + return out +} + +func diffPathJoin(path, key string) string { + if path == "" { + return key + } + return path + "." + key +} + +func diffPathItemName(v bson.RawValue, i int) string { + if d, ok := v.DocumentOK(); ok { + if n, ok := d.Lookup("Name").StringValueOK(); ok && n != "" { + return n + } + } + return fmt.Sprint(i) +} + +// withoutListMarker drops the int32 Mendix stores ahead of a list's items (the +// typed-array marker), so a count or an index is the list's own: an empty +// list holds one value, and counting it reported a first grant as "1 -> 2 +// items". +func withoutListMarker(vals []bson.RawValue) []bson.RawValue { + if len(vals) > 0 && vals[0].Type == bson.TypeInt32 { + return vals[1:] + } + return vals +} diff --git a/mdl/executor/cmd_diff_local.go b/mdl/executor/cmd_diff_local.go index 825fc9394d..04642b0c8b 100644 --- a/mdl/executor/cmd_diff_local.go +++ b/mdl/executor/cmd_diff_local.go @@ -10,11 +10,13 @@ import ( "github.com/mendixlabs/mxcli/modelsdk/bsoncompat" "os" "os/exec" + "path/filepath" "strings" "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" "go.mongodb.org/mongo-driver/v2/bson" ) @@ -51,8 +53,18 @@ func diffLocal(ctx *ExecContext, ref string, opts DiffOptions) error { opts.Width = 120 } + // Every git command runs in the project's directory, with paths relative to + // it: running them in the process's working directory failed outright from + // outside the repository ("failed to git diff: exit status 1"), and from a + // subdirectory of it read the working-tree files at repo-root-relative + // paths that did not exist (mendixlabs/mxcli#1602). + projectDir, err := filepath.Abs(filepath.Dir(contentsDir)) + if err != nil { + return mdlerrors.NewBackend("resolve project directory", err) + } + // Find changed mxunit files using git - changedFiles, err := findChangedMxunitFiles(ctx, contentsDir, ref) + changedFiles, err := findChangedMxunitFiles(ctx, projectDir, filepath.Base(contentsDir), ref) if err != nil { return mdlerrors.NewBackend("find changed files", err) } @@ -62,11 +74,16 @@ func diffLocal(ctx *ExecContext, ref string, opts DiffOptions) error { return nil } + spec, err := resolveRefSpec(projectDir, ref) + if err != nil { + return err + } + var results []DiffResult var newCount, modifiedCount, deletedCount int for _, change := range changedFiles { - result, err := diffMxunitFile(ctx, change, contentsDir, ref) + result, err := diffMxunitFile(ctx, change, projectDir, spec) if err != nil { // Log error but continue with other files fmt.Fprintf(ctx.Output, "Warning: %v\n", err) @@ -84,17 +101,9 @@ func diffLocal(ctx *ExecContext, ref string, opts DiffOptions) error { } } - // Output results - for _, result := range results { - switch opts.Format { - case DiffFormatUnified: - outputUnifiedDiff(ctx, result, opts.UseColor) - case DiffFormatSideBySide: - outputSideBySideDiff(ctx, result, opts.Width, opts.UseColor) - case DiffFormatStructural: - outputStructuralDiff(ctx, result, opts.UseColor) - } - } + // Output results. A unit whose rendering is the same at both revisions is + // printed as one line saying what changed (Writes), in every format. + WriteDiffResults(ctx.Output, results, opts) // Output summary fmt.Fprintf(ctx.Output, "\nSummary: %d new, %d modified, %d deleted\n", @@ -111,13 +120,32 @@ func (e *Executor) DiffLocal(ref string, opts DiffOptions) error { // gitChange represents a file change from git type gitChange struct { Status string // "A" (added), "M" (modified), "D" (deleted) - FilePath string // relative path from repo root + FilePath string // path relative to the project directory +} + +// gitInProject runs git in the project directory. +func gitInProject(projectDir string, args ...string) *exec.Cmd { + cmd := execCommand("git", args...) + cmd.Dir = projectDir + return cmd } -// findChangedMxunitFiles uses git to find changed mxunit files -func findChangedMxunitFiles(_ *ExecContext, contentsDir, ref string) ([]gitChange, error) { - // Run git diff to find changed files in mprcontents - cmd := execCommand("git", "diff", "--name-status", ref, "--", contentsDir) +// gitShowArg names a project-relative path at a revision. "./" makes git read +// the path relative to the working directory rather than to the repo root, so +// a project below the root resolves the same as one at it. +func gitShowArg(rev, path string) string { + return rev + ":./" + path +} + +// findChangedMxunitFiles uses git to find changed mxunit files under +// contentsName (the mprcontents folder) of projectDir. +func findChangedMxunitFiles(_ *ExecContext, projectDir, contentsName, ref string) ([]gitChange, error) { + // --relative: paths relative to projectDir, the directory git runs in. + // --no-renames: a unit's file name is its ID, so a file that moved is a + // different unit — one deleted, one added. Rename detection reported it as + // "R100 old new", read here as a change to the old path, which the new + // revision does not have. + cmd := gitInProject(projectDir, "diff", "--name-status", "--relative", "--no-renames", ref, "--", contentsName) output, err := cmd.Output() if err != nil { return nil, mdlerrors.NewBackend("git diff", err) @@ -153,27 +181,54 @@ func findChangedMxunitFiles(_ *ExecContext, contentsDir, ref string) ([]gitChang return changes, nil } +// refSpec is a --ref resolved to the revisions diff-local reads its two sides +// at, and the labels it names them by. +type refSpec struct { + twoRevision bool // both sides from git; otherwise the working tree against a revision + base, target string // revisions to read; target alone in single-ref mode + fromLabel, toLabel string +} + +// resolveRefSpec resolves ref once, before any unit is read. The old side of a +// three-dot range is the merge base, which is what `git diff a...b` lists the +// changes against; reading it at a would put the other branch's changes into +// every hunk. +func resolveRefSpec(projectDir, ref string) (refSpec, error) { + base, target, isRange, mergeBase := parseRefRange(ref) + if !isRange { + return refSpec{target: target, fromLabel: target, toLabel: "working tree"}, nil + } + spec := refSpec{twoRevision: true, base: base, target: target, fromLabel: base, toLabel: target} + if mergeBase { + out, err := gitInProject(projectDir, "merge-base", base, target).Output() + if err != nil { + return refSpec{}, mdlerrors.NewBackend(fmt.Sprintf("find the merge base of %s and %s", base, target), err) + } + spec.base = strings.TrimSpace(string(out)) + spec.fromLabel = fmt.Sprintf("merge base of %s and %s", base, target) + } + return spec, nil +} + // diffMxunitFile generates a diff for a single mxunit file. -// For two-revision diffs (ref contains ".."), both sides are read from git. +// For two-revision diffs (a range), both sides are read from git. // For single-ref diffs, the "current" side is read from the working tree. -func diffMxunitFile(ctx *ExecContext, change gitChange, contentsDir, ref string) (*DiffResult, error) { +func diffMxunitFile(ctx *ExecContext, change gitChange, projectDir string, spec refSpec) (*DiffResult, error) { var currentContent, gitContent []byte var err error - // Determine if this is a two-revision diff - baseRef, targetRef, isTwoRevision := parseRefRange(ref) - - if isTwoRevision { + baseRef, targetRef := spec.base, spec.target + if spec.twoRevision { // Two-revision mode: both sides from git if change.Status != "D" { - cmd := execCommand("git", "show", targetRef+":"+change.FilePath) + cmd := gitInProject(projectDir, "show", gitShowArg(targetRef, change.FilePath)) currentContent, err = cmd.Output() if err != nil { return nil, mdlerrors.NewBackend(fmt.Sprintf("read %s version of %s", targetRef, change.FilePath), err) } } if change.Status != "A" { - cmd := execCommand("git", "show", baseRef+":"+change.FilePath) + cmd := gitInProject(projectDir, "show", gitShowArg(baseRef, change.FilePath)) gitContent, err = cmd.Output() if err != nil { return nil, mdlerrors.NewBackend(fmt.Sprintf("read %s version of %s", baseRef, change.FilePath), err) @@ -182,13 +237,13 @@ func diffMxunitFile(ctx *ExecContext, change gitChange, contentsDir, ref string) } else { // Single-ref mode: current from working tree, old from git if change.Status != "D" { - currentContent, err = readFile(change.FilePath) + currentContent, err = readFile(filepath.Join(projectDir, change.FilePath)) if err != nil { return nil, mdlerrors.NewBackend(fmt.Sprintf("read current file %s", change.FilePath), err) } } if change.Status != "A" { - cmd := execCommand("git", "show", ref+":"+change.FilePath) + cmd := gitInProject(projectDir, "show", gitShowArg(targetRef, change.FilePath)) gitContent, err = cmd.Output() if err != nil { return nil, mdlerrors.NewBackend(fmt.Sprintf("read git version of %s", change.FilePath), err) @@ -214,11 +269,7 @@ func diffMxunitFile(ctx *ExecContext, change gitChange, contentsDir, ref string) // Name the sides after what they are: both are git revisions in range // mode, and the working tree against a ref otherwise. Neither is a script. - if isTwoRevision { - result.FromLabel, result.ToLabel = baseRef, targetRef - } else { - result.FromLabel, result.ToLabel = ref, "working tree" - } + result.FromLabel, result.ToLabel = spec.fromLabel, spec.toLabel // Extract UUID from file path; it is the unit ID. unitID := extractUUIDFromPath(change.FilePath) @@ -231,12 +282,19 @@ func diffMxunitFile(ctx *ExecContext, change gitChange, contentsDir, ref string) qualifiedName := unitQualifiedName(ctx, nameSource, unitID) result.ObjectName = splitQualifiedName(qualifiedName) + // Values the rendering must not print are masked so that a change still + // shows: each side needs to know the other's value. + var oldMask, newMask constantValueFunc + if unitType == projectSettingsType { + oldMask, newMask = constantValueMasks(ctx, oldRaw, newRaw) + } + // Generate MDL for both versions based on type if len(currentContent) > 0 { - result.Proposed = bsonToMDL(ctx, unitType, unitID, currentContent, unitQualifiedName(ctx, newRaw, unitID)) + result.Proposed = bsonToMDL(ctx, unitType, unitID, currentContent, unitQualifiedName(ctx, newRaw, unitID), newMask) } if len(gitContent) > 0 { - result.Current = bsonToMDL(ctx, unitType, unitID, gitContent, unitQualifiedName(ctx, oldRaw, unitID)) + result.Current = bsonToMDL(ctx, unitType, unitID, gitContent, unitQualifiedName(ctx, oldRaw, unitID), oldMask) } // Generate structural changes @@ -244,6 +302,13 @@ func diffMxunitFile(ctx *ExecContext, change gitChange, contentsDir, ref string) result.Changes = compareGeneric(ctx, result.Current, result.Proposed) } + // git says the unit changed, so identical renderings mean the change is in + // what the rendering leaves out. Say so, rather than print nothing under a + // summary that counts it (ako/mxcli#1512, ako/mxcli#1562). + if !result.IsNew && !result.IsDeleted && result.Current == result.Proposed { + result.Writes = unrenderedChangeNote(gitContent, currentContent) + } + return &result, nil } @@ -341,8 +406,30 @@ func unitQualifiedName(ctx *ExecContext, raw map[string]any, unitID string) stri return modName + "." + name } -// bsonToMDL converts BSON content to MDL representation based on type -func bsonToMDL(ctx *ExecContext, unitType, unitID string, content []byte, qualifiedName string) string { +// unrenderedChangeNote names what changed in a unit whose MDL rendering is the +// same at both revisions: the property paths that differ, never their values. +// Without it the unit was counted as modified and printed nothing — every +// format, every unit type the rendering is lossy for (settings, access rules, +// navigation, security, …). +func unrenderedChangeNote(before, after []byte) string { + const shown = 4 + paths := ChangedPaths(before, after, shown+1) + if len(paths) == 0 { + return "rewritten with no property change (element $IDs, pointers or property order only)" + } + if len(paths) > shown { + paths = append(paths[:shown], "…") + } + return "changed in properties the MDL rendering does not show: " + strings.Join(paths, ", ") +} + +// projectSettingsType is the storage type of the project settings unit. +const projectSettingsType = "Settings$ProjectSettings" + +// bsonToMDL converts BSON content to MDL representation based on type. +// constantValue renders a configuration constant's value in the settings unit +// (see constantValueMasks); other unit types ignore it. +func bsonToMDL(ctx *ExecContext, unitType, unitID string, content []byte, qualifiedName string, constantValue constantValueFunc) string { var raw map[string]any if err := bsoncompat.UnmarshalDocMap(content, &raw); err != nil { return fmt.Sprintf("-- Error parsing BSON: %v", err) @@ -352,6 +439,10 @@ func bsonToMDL(ctx *ExecContext, unitType, unitID string, content []byte, qualif } switch { + // An exact match, first: the dispatch below is by containment, and + // "Settings" is in "Projects$ModuleSettings" too. + case unitType == projectSettingsType: + return projectSettingsBsonToMDL(ctx, raw, constantValue) case strings.Contains(unitType, "DomainModel"): return domainModelBsonToMDL(ctx, raw, qualifiedName) case strings.Contains(unitType, "Entity"): @@ -371,7 +462,9 @@ func bsonToMDL(ctx *ExecContext, unitType, unitID string, content []byte, qualif return snippetBsonToMDL(ctx, raw, qualifiedName) case strings.Contains(unitType, "Layout"): return layoutBsonToMDL(ctx, raw, qualifiedName) - case strings.Contains(unitType, "Module"): + // Exact, not Contains: Security$ModuleSecurity and Projects$ModuleSettings + // contain "Module" too, and rendered as `create module ;` on both sides. + case unitType == "Projects$ModuleImpl" || unitType == "Projects$Module": return moduleBsonToMDL(ctx, raw) default: // Generic representation @@ -379,13 +472,91 @@ func bsonToMDL(ctx *ExecContext, unitType, unitID string, content []byte, qualif } } +// projectSettingsBsonToMDL renders the project settings unit with DESCRIBE +// SETTINGS' own writer. The unit had no case here, so it fell to the generic +// two-comment rendering, which is the same at every revision: a changed +// constant value or runtime setting counted as modified and printed nothing +// (mendixlabs/mxcli#1602, ako/mxcli#1512). Configuration constant values are +// masked — they often hold secrets, and diff output lands in reviews and CI +// logs — so constantValue is never left to print them. +func projectSettingsBsonToMDL(ctx *ExecContext, raw map[string]any, constantValue constantValueFunc) string { + ps, err := ctx.Backend.ParseProjectSettingsFromRaw(raw) + if err != nil { + return "-- project settings could not be read: " + err.Error() + } + if constantValue == nil { + constantValue = func(_, _, _ string) string { return maskedValue } + } + return strings.TrimRight(renderToString(ctx, func() { + writeProjectSettingsMDL(ctx, ps, constantValue) + }), "\n") +} + +// maskedValue stands in for a configuration constant's value. It is not a +// string literal on purpose: pasted into a script, it fails to parse instead +// of setting the constant to the text "masked". +const maskedValue = "" + +// constantValueMasks returns the value renderers for the old and the new +// revision of the settings unit. A value is printed as — except where +// the same constant in the same configuration holds a different value on the +// other side, where it is "" / "", so +// the change shows as a -/+ pair without either value appearing. +func constantValueMasks(ctx *ExecContext, oldRaw, newRaw map[string]any) (oldMask, newMask constantValueFunc) { + oldVals, newVals := constantValues(ctx, oldRaw), constantValues(ctx, newRaw) + mask := func(other map[string]string, label string) constantValueFunc { + return func(configuration, constant, value string) string { + if o, ok := other[configuration+"\x00"+constant]; ok && o != value { + return "" + } + return maskedValue + } + } + return mask(newVals, "old value"), mask(oldVals, "new value") +} + +// constantValues maps configuration + "\x00" + constant to the shared value +// the settings document holds for it; private values have none. +func constantValues(ctx *ExecContext, raw map[string]any) map[string]string { + vals := map[string]string{} + if raw == nil { + return vals + } + ps, err := ctx.Backend.ParseProjectSettingsFromRaw(raw) + if err != nil || ps.Configuration == nil { + return vals + } + for _, cfg := range ps.Configuration.Configurations { + for _, cv := range cfg.ConstantValues { + if !cv.IsPrivate { + vals[cfg.Name+"\x00"+cv.ConstantId] = cv.Value + } + } + } + return vals +} + // domainModelBsonToMDL converts a domain model BSON to MDL. -// Includes full entity definitions (attributes) so diffs show schema changes. +// Includes full entity definitions (attributes) so diffs show schema changes, +// each followed by its access rules as DESCRIBE ENTITY prints them. func domainModelBsonToMDL(ctx *ExecContext, raw map[string]any, name string) string { var lines []string lines = append(lines, fmt.Sprintf("-- Domain Model: %s", name)) lines = append(lines, "") + // The access rules are rendered by DESCRIBE ENTITY's own grant writer, so + // they need the semantic entity, not the raw map the rest of this renderer + // reads. Without them a GRANT or REVOKE changed the unit but nothing this + // printed, so it diffed to nothing (mendixlabs/mxcli#1630, ako/mxcli#1562). + accessByEntity := map[string]*domainmodel.Entity{} + if dm, err := ctx.Backend.ParseDomainModelFromRaw(raw, "", ""); err != nil { + lines = append(lines, "-- access rules not shown: "+err.Error(), "") + } else { + for _, e := range dm.Entities { + accessByEntity[e.Name] = e + } + } + // Render full entity definitions so attribute changes show up in diffs entities := extractBsonArray(raw["Entities"]) for _, ent := range entities { @@ -396,6 +567,14 @@ func domainModelBsonToMDL(ctx *ExecContext, raw map[string]any, name string) str } qn := name + "." + entName lines = append(lines, entityBsonToMDL(ctx, entMap, qn)) + if e := accessByEntity[entName]; e != nil { + grants := renderToString(ctx, func() { + outputEntityAccessGrants(ctx, e, name, entName) + }) + if grants = strings.TrimRight(grants, "\n"); grants != "" { + lines = append(lines, grants) + } + } lines = append(lines, "") } } @@ -883,13 +1062,30 @@ func compareGeneric(_ *ExecContext, current, proposed string) []StructuralChange // Helper Functions for Git and BSON Parsing // ============================================================================ -// parseRefRange splits a ref like "base..target" into its parts. -// Returns (base, target, true) for ranges, or ("", ref, false) for single refs. -func parseRefRange(ref string) (base, target string, isRange bool) { - if idx := strings.Index(ref, ".."); idx >= 0 { - return ref[:idx], ref[idx+2:], true +// parseRefRange splits a ref like "base..target" or "base...target" into its +// parts; mergeBase reports the three-dot form, whose old side is the merge base +// of the two. An empty side is HEAD, as it is to git. Returns ("", ref, false, +// false) for a single ref. +// +// "..." is matched before "..": splitting "a...b" on the first ".." read the +// target as ".b", and every unit failed to load. +func parseRefRange(ref string) (base, target string, isRange, mergeBase bool) { + sep := ".." + if strings.Contains(ref, "...") { + sep, mergeBase = "...", true + } + idx := strings.Index(ref, sep) + if idx < 0 { + return "", ref, false, false + } + base, target = ref[:idx], ref[idx+len(sep):] + if base == "" { + base = "HEAD" + } + if target == "" { + target = "HEAD" } - return "", ref, false + return base, target, true, mergeBase } // execCommand creates an exec.Cmd for running git commands. diff --git a/mdl/executor/cmd_diff_local_access_rules_test.go b/mdl/executor/cmd_diff_local_access_rules_test.go new file mode 100644 index 0000000000..8033ceff4a --- /dev/null +++ b/mdl/executor/cmd_diff_local_access_rules_test.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 + +// ako/mxcli#1562 (mendixlabs/mxcli#1630): diff-local counted a domain model as +// modified after a `grant` on one of its entities and printed no difference. +// The domain model renderer printed each entity's attributes and the +// associations, but not the entity access rules, so a script that only changed +// access had nothing to review. The rules now render with DESCRIBE ENTITY's own +// grant writer, read from a parse of each revision of the unit. +package executor + +import ( + "bytes" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend" + modelsdkbackend "github.com/mendixlabs/mxcli/mdl/backend/modelsdk" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +const diffAccessSetup = `create module M; +create module role M.User; +create persistent entity M.Item (Name: String(100)); +grant read * on entity M.Item to M.User;` + +func TestDiffLocal_EntityGrantChangeRenders(t *testing.T) { + got := diffLocalAfter(t, diffAccessSetup, "grant read *, write * on entity M.Item to M.User;", DiffOptions{}) + + for _, want := range []string{ + "\n-grant read * on entity M.Item to M.User;\n", + "\n+grant read *, write * on entity M.Item to M.User;\n", + "Summary: 0 new, 1 modified, 0 deleted", + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + if strings.Contains(got, "Modified: DomainModel") { + t.Errorf("the grant fell through to the not-rendered note:\n%s", got) + } +} + +func TestDiffLocal_EntityGrantChangeInStructFormat(t *testing.T) { + got := diffLocalAfter(t, diffAccessSetup, "grant read *, write * on entity M.Item to M.User;", + DiffOptions{Format: DiffFormatStructural}) + if !strings.Contains(got, "DomainModel: M\n ~ Line") { + t.Errorf("struct format shows no changed line:\n%s", got) + } + if strings.Contains(got, "(no changes)") || strings.Contains(got, "Modified: DomainModel") { + t.Errorf("struct format did not render the grant:\n%s", got) + } +} + +// A grant on an entity with no rule yet is a + line, and a revoke of the last +// rule a - line. +func TestDiffLocal_EntityGrantAddedAndRevoked(t *testing.T) { + setup := "create module M;\ncreate module role M.User;\ncreate persistent entity M.Item (Name: String(100));" + got := diffLocalAfter(t, setup, "grant read * on entity M.Item to M.User;", DiffOptions{}) + if !strings.Contains(got, "\n+grant read * on entity M.Item to M.User;\n") { + t.Errorf("a new rule is not a + line:\n%s", got) + } + got = diffLocalAfter(t, diffAccessSetup, "revoke all on entity M.Item from M.User;", DiffOptions{}) + if !strings.Contains(got, "\n-grant read * on entity M.Item to M.User;\n") { + t.Errorf("a revoked rule is not a - line:\n%s", got) + } +} + +// The grant lines are DESCRIBE ENTITY's, not a second renderer's: every grant +// describe prints for the entity appears verbatim on diff-local's new side. +func TestDiffLocal_EntityGrantLinesAreDescribeEntitys(t *testing.T) { + const change = "grant create, delete, read (Name), write (Name) on entity M.Item to M.User where [Name != empty];" + got := diffLocalAfter(t, diffAccessSetup, change, DiffOptions{}) + + var out bytes.Buffer + x := New(&out) + x.SetQuiet(true) + x.SetBackendFactory(func() backend.FullBackend { return modelsdkbackend.New() }) + t.Cleanup(func() { x.Close() }) + run(t, x, "CONNECT LOCAL '"+visitor.QuoteString(projectFixture(t))+"'") + run(t, x, diffAccessSetup) + run(t, x, change) + out.Reset() + run(t, x, "describe entity M.Item;") + + n := 0 + for line := range strings.SplitSeq(out.String(), "\n") { + if strings.HasPrefix(line, "grant ") { + n++ + if !strings.Contains(got, "\n+"+line+"\n") && !strings.Contains(got, "\n "+line+"\n") { + t.Errorf("describe entity prints %q, diff-local does not:\n%s", line, got) + } + } + } + if n == 0 { + t.Fatalf("describe entity printed no grant:\n%s", out.String()) + } +} diff --git a/mdl/executor/cmd_diff_local_dispatch_test.go b/mdl/executor/cmd_diff_local_dispatch_test.go new file mode 100644 index 0000000000..0d40318bac --- /dev/null +++ b/mdl/executor/cmd_diff_local_dispatch_test.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// Security$ModuleSecurity and Projects$ModuleSettings contain "Module", so the +// containment dispatch sent them to the module renderer: a new module's +// security unit printed as `+create module ;`, naming the wrong thing and no +// module at all. +func TestDiffLocal_ModuleSecurityIsNotRenderedAsAModule(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + proj := t.TempDir() + writeEnumUnit(t, proj, "Base") + gitIn(t, proj, "init", "-q") + gitIn(t, proj, "add", "-A") + gitIn(t, proj, "commit", "-q", "-m", "base") + unit := filepath.Join(proj, "mprcontents", "7c", "7c", "7c7c7c7c-1111-2222-3333-444444444444.mxunit") + if err := os.MkdirAll(filepath.Dir(unit), 0o755); err != nil { + t.Fatal(err) + } + data, err := bson.Marshal(map[string]any{"$Type": "Security$ModuleSecurity", + "ModuleRoles": []any{int32(3), map[string]any{"$Type": "Security$ModuleRole", "Name": "User"}}}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(unit, data, 0o644); err != nil { + t.Fatal(err) + } + gitIn(t, proj, "add", "-A") + gitIn(t, proj, "commit", "-q", "-m", "module security") + + ctx, out := gitDiffContext(filepath.Join(proj, "mprcontents")) + if err := diffLocal(ctx, "HEAD~1..HEAD", DiffOptions{}); err != nil { + t.Fatal(err) + } + if got := out.String(); strings.Contains(got, "create module") || !strings.Contains(got, "ModuleSecurity") { + t.Errorf("a new module security unit is rendered as a module:\n%s", got) + } +} diff --git a/mdl/executor/cmd_diff_local_gitdir_test.go b/mdl/executor/cmd_diff_local_gitdir_test.go new file mode 100644 index 0000000000..9cd4113a67 --- /dev/null +++ b/mdl/executor/cmd_diff_local_gitdir_test.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +// mendixlabs/mxcli#1602 (ako/mxcli#1512), side note: "`diff-local` runs `git +// diff` in the process working directory, not in the project's repo. Run from +// outside the repo with `-p /App.mpr`, it fails with `Error: failed to +// find changed files: failed to git diff: exit status 1`". +package executor + +import ( + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "go.mongodb.org/mongo-driver/v2/bson" +) + +const gitUnit = "5a5a5a5a-1111-2222-3333-444444444444" + +// gitDiffContext is a connected v2 project whose mprcontents is contentsDir. +// An enumeration renders its values, so a test can see which revision each +// side was read at. +func gitDiffContext(contentsDir string) (*ExecContext, *bytes.Buffer) { + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + VersionFunc: func() types.MPRVersion { return 2 }, + ContentsDirFunc: func() string { return contentsDir }, + } + out := &bytes.Buffer{} + return &ExecContext{Context: context.Background(), Backend: mb, Output: out, Cache: &executorCache{}}, out +} + +// writeEnumUnit writes an enumeration unit with one value to proj's mprcontents. +func writeEnumUnit(t *testing.T, proj, value string) { + t.Helper() + unit := filepath.Join(proj, "mprcontents", "5a", "5a", gitUnit+".mxunit") + if err := os.MkdirAll(filepath.Dir(unit), 0o755); err != nil { + t.Fatal(err) + } + data, err := bson.Marshal(map[string]any{ + "$Type": "Enumerations$Enumeration", + "Name": "Status", + "Values": []any{int32(3), map[string]any{"$Type": "Enumerations$EnumerationValue", "Name": value}}, + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(unit, data, 0o644); err != nil { + t.Fatal(err) + } +} + +// gitIn runs git in dir with a throwaway identity, failing the test on error. +func gitIn(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-c", "user.name=t", "-c", "user.email=t@example.com", + "-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main"}, args...)...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } +} + +// gitProjectRepo commits an enumeration under /App/mprcontents, then +// changes its value in the working tree. It returns the repo root and the +// project directory. +func gitProjectRepo(t *testing.T) (root, proj string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + root = t.TempDir() + proj = filepath.Join(root, "App") + writeEnumUnit(t, proj, "Before") + gitIn(t, root, "init", "-q") + gitIn(t, root, "add", "-A") + gitIn(t, root, "commit", "-q", "-m", "base") + writeEnumUnit(t, proj, "After") + return root, proj +} + +func TestDiffLocal_RunsGitInTheProjectRepo(t *testing.T) { + for _, tc := range []struct { + name string + cwd func(root, proj string) string + dir func(root, proj string) string // what ContentsDir reports + }{ + {"cwd outside any repo", func(_, _ string) string { return t.TempDir() }, + func(_, proj string) string { return filepath.Join(proj, "mprcontents") }}, + {"cwd in the project, below the repo root", func(_, proj string) string { return proj }, + func(_, _ string) string { return "mprcontents" }}, + } { + t.Run(tc.name, func(t *testing.T) { + root, proj := gitProjectRepo(t) + t.Chdir(tc.cwd(root, proj)) + ctx, out := gitDiffContext(tc.dir(root, proj)) + + if err := diffLocal(ctx, "HEAD", DiffOptions{}); err != nil { + t.Fatalf("diff-local from %s: %v (#1602)", tc.name, err) + } + got := out.String() + if !strings.Contains(got, "- Before 'Before'") || !strings.Contains(got, "+ After 'After'") { + t.Errorf("diff-local from %s did not diff the changed unit:\n%s", tc.name, got) + } + }) + } +} diff --git a/mdl/executor/cmd_diff_local_ref_range_test.go b/mdl/executor/cmd_diff_local_ref_range_test.go index b574805acf..04041b3477 100644 --- a/mdl/executor/cmd_diff_local_ref_range_test.go +++ b/mdl/executor/cmd_diff_local_ref_range_test.go @@ -62,7 +62,7 @@ func stubGitRevisions(t *testing.T, unitPath string, revs map[string]map[string] return exec.Command("printf", "M\t%s\n", unitPath) } if len(args) == 2 && args[0] == "show" { - if f, ok := files[args[1]]; ok { + if f, ok := files[strings.Replace(args[1], ":./", ":", 1)]; ok { return exec.Command("cat", f) } } diff --git a/mdl/executor/cmd_diff_local_refrange_test.go b/mdl/executor/cmd_diff_local_refrange_test.go new file mode 100644 index 0000000000..20e277ce27 --- /dev/null +++ b/mdl/executor/cmd_diff_local_refrange_test.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 + +// `mxcli diff-local --ref main...feature-branch` is documented ("changes since +// common ancestor") and printed, on a git copy of testdata/pedapp: +// +// Warning: failed to read .grant version of mprcontents/84/77/….mxunit: exit status 128 +// +// Summary: 0 new, 0 modified, 0 deleted +// +// parseRefRange split on the first "..", so "a...b" became base "a" and target +// ".b". For `git diff a...b` the old side is the merge base of a and b, not a. +package executor + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestParseRefRange(t *testing.T) { + for _, tc := range []struct { + ref string + base, target string + isRange, mergeBase bool + }{ + {"HEAD", "", "HEAD", false, false}, + {"main", "", "main", false, false}, + {"HEAD~1..HEAD", "HEAD~1", "HEAD", true, false}, + {"main...feature", "main", "feature", true, true}, + // An empty side is HEAD, as it is to git. + {"main..", "main", "HEAD", true, false}, + {"..feature", "HEAD", "feature", true, false}, + {"main...", "main", "HEAD", true, true}, + } { + base, target, isRange, mergeBase := parseRefRange(tc.ref) + if base != tc.base || target != tc.target || isRange != tc.isRange || mergeBase != tc.mergeBase { + t.Errorf("parseRefRange(%q) = (%q, %q, %v, %v), want (%q, %q, %v, %v)", tc.ref, + base, target, isRange, mergeBase, tc.base, tc.target, tc.isRange, tc.mergeBase) + } + } +} + +// divergedRepo commits an enumeration, branches `feature` off that commit and +// changes it there, then changes it differently on main. Only the merge base +// holds the value the feature branch started from. +func divergedRepo(t *testing.T) (proj string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + proj = t.TempDir() + writeEnumUnit(t, proj, "Base") + gitIn(t, proj, "init", "-q") + gitIn(t, proj, "add", "-A") + gitIn(t, proj, "commit", "-q", "-m", "base") + gitIn(t, proj, "checkout", "-q", "-b", "feature") + writeEnumUnit(t, proj, "Feat") + gitIn(t, proj, "commit", "-q", "-am", "feature") + gitIn(t, proj, "checkout", "-q", "main") + writeEnumUnit(t, proj, "Mainval") + gitIn(t, proj, "commit", "-q", "-am", "main moves on") + return proj +} + +func TestDiffLocal_ThreeDotRangeDiffsFromTheMergeBase(t *testing.T) { + proj := divergedRepo(t) + ctx, out := gitDiffContext(filepath.Join(proj, "mprcontents")) + + if err := diffLocal(ctx, "main...feature", DiffOptions{}); err != nil { + t.Fatal(err) + } + got := out.String() + if strings.Contains(got, "Warning") || !strings.Contains(got, "1 modified") { + t.Fatalf("diff-local --ref main...feature did not diff the feature branch's change:\n%s", got) + } + // The merge base holds "Base"; reading the old side at main would give + // "Mainval". + if !strings.Contains(got, "- Base 'Base'") || !strings.Contains(got, "+ Feat 'Feat'") || + strings.Contains(got, "Mainval") || !strings.Contains(got, "(merge base of main and feature)") { + t.Errorf("the old side is not the merge base of main and feature:\n%s", got) + } + + // Control: two dots compare the tips, main's value against feature's. + ctx, out = gitDiffContext(filepath.Join(proj, "mprcontents")) + if err := diffLocal(ctx, "main..feature", DiffOptions{}); err != nil { + t.Fatal(err) + } + if got := out.String(); !strings.Contains(got, "- Mainval 'Mainval'") || !strings.Contains(got, "+ Feat 'Feat'") { + t.Errorf("control: main..feature does not compare main's tip with feature's:\n%s", got) + } +} + +// A unit whose file moves to another name is a different unit to Mendix — the +// file name is the unit ID — but git's rename detection reported it as +// "R100\told\tnew", which diff-local read as a modification of the OLD path: +// the new revision has no such file, so the unit was skipped with a warning and +// counted nowhere. +func TestDiffLocal_RenamedUnitFileIsADeleteAndAnAdd(t *testing.T) { + proj := divergedRepo(t) + const moved = "6b6b6b6b-1111-2222-3333-444444444444" + if err := os.MkdirAll(filepath.Join(proj, "mprcontents", "6b", "6b"), 0o755); err != nil { + t.Fatal(err) + } + gitIn(t, proj, "mv", filepath.Join("mprcontents", "5a", "5a", gitUnit+".mxunit"), + filepath.Join("mprcontents", "6b", "6b", moved+".mxunit")) + gitIn(t, proj, "commit", "-q", "-m", "unit gets a new ID") + + ctx, out := gitDiffContext(filepath.Join(proj, "mprcontents")) + if err := diffLocal(ctx, "HEAD~1..HEAD", DiffOptions{}); err != nil { + t.Fatal(err) + } + if got := out.String(); strings.Contains(got, "Warning") || !strings.Contains(got, "Summary: 1 new, 0 modified, 1 deleted") { + t.Errorf("a unit file moved to a new ID is not reported as one new and one deleted unit:\n%s", got) + } +} diff --git a/mdl/executor/cmd_diff_local_unrendered_test.go b/mdl/executor/cmd_diff_local_unrendered_test.go new file mode 100644 index 0000000000..6fbbaef4f3 --- /dev/null +++ b/mdl/executor/cmd_diff_local_unrendered_test.go @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: Apache-2.0 + +// ako/mxcli#1512 (mendixlabs/mxcli#1602): after `alter settings constant … +// in configuration …` or `alter settings runtime (BcryptCost: 11)`, diff-local +// printed only "Summary: 0 new, 1 modified, 0 deleted". The settings unit had +// no renderer: it fell to the generic case, two comment lines naming the type, +// which are the same at every revision, so the change counted and showed +// nothing — in every format (`--format struct` said "(no changes)"). +// +// The class behind it: a unit whose change lies outside what diff-local's +// renderer prints is counted as modified and silent. Navigation, project and +// module security, a page's or a published REST service's access, an entity +// index — all were. Such a unit is now printed as one line naming the +// properties that changed. +package executor + +import ( + "bytes" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend" + modelsdkbackend "github.com/mendixlabs/mxcli/mdl/backend/modelsdk" + "github.com/mendixlabs/mxcli/mdl/visitor" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// diffLocalAfter runs diff-local against a real MPR v2 project, the way a user +// does: setup is "committed", change is applied to the working tree, and +// diff-local compares the two. git is answered from a snapshot of the units +// taken between the two scripts, so the bytes on both sides are the real +// engine's. +func diffLocalAfter(t *testing.T, setup, change string, opts DiffOptions) string { + t.Helper() + var out bytes.Buffer + x := New(&out) + x.SetQuiet(true) + x.SetBackendFactory(func() backend.FullBackend { return modelsdkbackend.New() }) + t.Cleanup(func() { x.Close() }) + run(t, x, "CONNECT LOCAL '"+visitor.QuoteString(projectFixture(t))+"'") + if setup != "" { + run(t, x, setup) + } + contents := x.backend.ContentsDir() + head := readUnits(t, contents) + if change != "" { + run(t, x, change) + } + work := readUnits(t, contents) + + // `git diff --name-status --relative HEAD` and `git show HEAD:./`, + // from the snapshot. Like git run in the project directory, the paths are + // relative to it. + var status strings.Builder + snap := t.TempDir() + files := map[string]string{} + projectDir := filepath.Dir(contents) + rel := func(path string) string { + r, err := filepath.Rel(projectDir, path) + if err != nil { + t.Fatal(err) + } + return filepath.ToSlash(r) + } + for path, data := range head { + f := filepath.Join(snap, filepath.Base(path)) + if err := os.WriteFile(f, data, 0o644); err != nil { + t.Fatal(err) + } + files["HEAD:./"+rel(path)] = f + if w, ok := work[path]; !ok { + status.WriteString("D\t" + rel(path) + "\n") + } else if !bytes.Equal(w, data) { + status.WriteString("M\t" + rel(path) + "\n") + } + } + for path := range work { + if _, ok := head[path]; !ok { + status.WriteString("A\t" + rel(path) + "\n") + } + } + listing := filepath.Join(snap, "name-status") + if err := os.WriteFile(listing, []byte(status.String()), 0o644); err != nil { + t.Fatal(err) + } + orig := execCommand + t.Cleanup(func() { execCommand = orig }) + execCommand = func(name string, args ...string) *exec.Cmd { + if len(args) > 0 && args[0] == "diff" { + return exec.Command("cat", listing) + } + if len(args) == 2 && args[0] == "show" { + if f, ok := files[args[1]]; ok { + return exec.Command("cat", f) + } + } + return exec.Command("sh", "-c", "exit 128") + } + + out.Reset() + if err := x.DiffLocal("HEAD", opts); err != nil { + t.Fatalf("diff-local: %v", err) + } + return out.String() +} + +// readUnits reads every .mxunit under dir, by path. +func readUnits(t *testing.T, dir string) map[string][]byte { + t.Helper() + units := map[string][]byte{} + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".mxunit") { + return err + } + data, err := os.ReadFile(path) + units[path] = data + return err + }) + if err != nil { + t.Fatal(err) + } + return units +} + +const diffSettingsSetup = `create module M; +create constant M.ApiKey type String default 'x'; +alter settings constant @M.ApiKey value 'first-secret' in configuration 'Default';` + +func TestDiffLocal_SettingsConstantValueChangeShowsMasked(t *testing.T) { + got := diffLocalAfter(t, diffSettingsSetup, + `alter settings constant @M.ApiKey value 'secret123' in configuration 'Default';`, DiffOptions{}) + + for _, want := range []string{ + "-alter settings constant 'M.ApiKey' value ", + "+alter settings constant 'M.ApiKey' value ", + "Summary: 0 new, 1 modified, 0 deleted", + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + for _, secret := range []string{"secret123", "first-secret"} { + if strings.Contains(got, secret) { + t.Errorf("diff-local printed the constant value %q:\n%s", secret, got) + } + } +} + +// A value set where there was none is a new line, masked like any other. +func TestDiffLocal_SettingsConstantValueAddedShowsMasked(t *testing.T) { + got := diffLocalAfter(t, "create module M;\ncreate constant M.ApiKey type String default 'x';", + `alter settings constant @M.ApiKey value 'secret123' in configuration 'Default';`, DiffOptions{}) + + if !strings.Contains(got, "+alter settings constant 'M.ApiKey' value \n+ in configuration 'Default';") { + t.Errorf("the new constant value is not a masked + line:\n%s", got) + } + if strings.Contains(got, "secret123") { + t.Errorf("diff-local printed the constant value:\n%s", got) + } +} + +func TestDiffLocal_SettingsRuntimeChangeRenders(t *testing.T) { + for _, tc := range []struct { + format DiffFormat + want []string + }{ + {DiffFormatUnified, []string{"- BcryptCost: ", "+ BcryptCost: 11,"}}, + {DiffFormatSideBySide, []string{"BcryptCost: 11, ~"}}, + {DiffFormatStructural, []string{"ProjectSettings: ", "~ Line"}}, + } { + t.Run(string(tc.format), func(t *testing.T) { + got := diffLocalAfter(t, "", "alter settings runtime (BcryptCost: 11);", DiffOptions{Format: tc.format}) + for _, want := range tc.want { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + for _, bad := range []string{"(no changes)", "-- Type: Settings$ProjectSettings"} { + if strings.Contains(got, bad) { + t.Errorf("still the silent rendering (%q):\n%s", bad, got) + } + } + }) + } +} + +// The class: a unit diff-local's renderer is lossy for (module security +// renders as its module's name only) is named, with what changed, in every +// format — not counted and left blank. +func TestDiffLocal_UnrenderedChangeIsNamedInEveryFormat(t *testing.T) { + const want = "Modified: ModuleSecurity M: changed in properties the MDL rendering does not show: ModuleRoles (0 -> 1 items)\n" + for _, format := range []DiffFormat{DiffFormatUnified, DiffFormatSideBySide, DiffFormatStructural} { + t.Run(string(format), func(t *testing.T) { + got := diffLocalAfter(t, "create module M;", "create module role M.Admin;", DiffOptions{Format: format}) + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + if strings.Contains(got, "(no changes)") { + t.Errorf("a changed unit reported as \"(no changes)\":\n%s", got) + } + if !strings.Contains(got, "Summary: 0 new, 1 modified, 0 deleted") { + t.Errorf("summary changed:\n%s", got) + } + }) + } +} + +// Control: a statement that writes nothing (re-applying the stored value is +// elided) leaves nothing to report — the note is about units git says +// changed, not something every run prints. +func TestDiffLocal_UnchangedUnitStaysUnreported(t *testing.T) { + got := diffLocalAfter(t, diffSettingsSetup, + `alter settings constant @M.ApiKey value 'first-secret' in configuration 'Default';`, DiffOptions{}) + if strings.TrimSpace(got) != "No local changes found in mxunit files." { + t.Errorf("an unchanged project produced:\n%s", got) + } +} + +// Control: a change the rendering always showed renders byte for byte as it +// did before the note existed (output captured from 97e1cceac). +func TestDiffLocal_RenderedChangeIsUnchanged(t *testing.T) { + got := diffLocalAfter(t, "create module M;\ncreate persistent entity M.Item (Name: String(100));", + "alter entity M.Item modify attribute Name: String(50);", DiffOptions{}) + want := strings.Join([]string{ + "--- DomainModel.M (HEAD)", + "+++ DomainModel.M (working tree)", + "@@ -1,6 +1,6 @@", + " -- Domain Model: M", + " ", + " create persistent entity M.Item (", + "- Name: String(100)", + "+ Name: String(50)", + " );", + " ", + "", + "", + "Summary: 0 new, 1 modified, 0 deleted", + "", + }, "\n") + if got != want { + t.Errorf("rendered change differs from before:\n%s\nwant:\n%s", got, want) + } +} + +func TestChangedPaths_NamesPropertiesNotValues(t *testing.T) { + doc := func(roles bson.A, value string) []byte { + b, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "Forms$Page"}, + {Key: "AllowedModuleRoles", Value: roles}, + {Key: "Secret", Value: value}, + }) + if err != nil { + t.Fatal(err) + } + return b + } + got := ChangedPaths(doc(bson.A{int32(1)}, "old-secret"), doc(bson.A{int32(1), "M.User"}, "new-secret"), 4) + // The leading int32 is Mendix's typed-array marker, not an item. + want := []string{"AllowedModuleRoles (0 -> 1 items)", "Secret"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("ChangedPaths = %q, want %q", got, want) + } +} diff --git a/mdl/executor/cmd_misc.go b/mdl/executor/cmd_misc.go index 374fc44b0d..fe8ff6c18b 100644 --- a/mdl/executor/cmd_misc.go +++ b/mdl/executor/cmd_misc.go @@ -248,7 +248,7 @@ Security - Project Settings: alter app security ( SecurityLevel: off|prototype|production ); alter app security ( EnableDemoUsers: true|false, StrictMode: true|false ); alter app security ( EnableGuestAccess: true|false [, GuestUserRole: ] ); - alter app security ( AdminUserName: '' ); + alter app security ( AdminUserName: '', AdminPassword: '' ); create demo user 'name' ( Password: 'pass', UserRoles: (UserRole [, ...]) ); drop demo user 'name'; diff --git a/mdl/executor/cmd_module_documentation_test.go b/mdl/executor/cmd_module_documentation_test.go index 759542f3af..80f6d0d2b9 100644 --- a/mdl/executor/cmd_module_documentation_test.go +++ b/mdl/executor/cmd_module_documentation_test.go @@ -156,3 +156,60 @@ func TestAlterProjectSecurity_AdminUserName(t *testing.T) { t.Error("a StrictMode statement wrote the admin user name") } } + +// mendixlabs/mxcli#624 (ako/mxcli#1259): AdminPassword is written through the +// backend, and the write report never echoes the value. +func TestAlterProjectSecurity_AdminPassword(t *testing.T) { + const pw = "N3w-Admin-Passw0rd" + var gotID model.ID + var gotPW string + called := false + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + GetProjectSecurityFunc: func() (*security.ProjectSecurity, error) { + return &security.ProjectSecurity{BaseElement: model.BaseElement{ID: "ps-1"}}, nil + }, + SetProjectAdminPasswordFunc: func(id model.ID, password string) error { + called, gotID, gotPW = true, id, password + return nil + }, + } + ctx, buf := newMockCtx(t, withBackend(mb)) + assertNoError(t, execAlterProjectSecurity(ctx, &ast.AlterProjectSecurityStmt{AdminPassword: pw})) + if !called || gotID != "ps-1" || gotPW != pw { + t.Fatalf("SetProjectAdminPassword(%q, %q) called=%v, want (ps-1, %s)", gotID, gotPW, called, pw) + } + if strings.Contains(buf.String(), pw) { + t.Errorf("the write report printed the password:\n%s", buf.String()) + } + + // Control: a statement about something else leaves the password alone. + called = false + on := true + mb.SetProjectStrictModeFunc = func(model.ID, bool) error { return nil } + assertNoError(t, execAlterProjectSecurity(ctx, &ast.AlterProjectSecurityStmt{StrictModeEnabled: &on})) + if called { + t.Error("a StrictMode statement wrote the admin password") + } +} + +// describe app security says whether the administrator has a password — the cause +// of CE0129 — and never what it is. +func TestShowProjectSecurity_AdminPasswordSetNotShown(t *testing.T) { + for pw, want := range map[string]string{"": "Admin Password: not set", "N3w-Admin-Passw0rd": "Admin Password: set"} { + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + GetProjectSecurityFunc: func() (*security.ProjectSecurity, error) { + return &security.ProjectSecurity{AdminUserName: "MxAdmin", AdminPassword: pw}, nil + }, + } + ctx, buf := newMockCtx(t, withBackend(mb)) + assertNoError(t, listProjectSecurity(ctx)) + if !strings.Contains(buf.String(), want) { + t.Errorf("password %q: output lacks %q:\n%s", pw, want, buf.String()) + } + if pw != "" && strings.Contains(buf.String(), pw) { + t.Errorf("describe app security printed the password:\n%s", buf.String()) + } + } +} diff --git a/mdl/executor/cmd_pages_builder.go b/mdl/executor/cmd_pages_builder.go index 76f0422d57..89bc98a5c1 100644 --- a/mdl/executor/cmd_pages_builder.go +++ b/mdl/executor/cmd_pages_builder.go @@ -415,22 +415,10 @@ func (pb *pageBuilder) resolveFolder(folderPath string) (model.ID, error) { return currentContainerID, nil } -// createFolder creates a new folder in the project. +// createFolder creates a new folder in the project through the executor's one +// folder creator, which also keeps the cached hierarchy in step (ako/mxcli#1486). func (pb *pageBuilder) createFolder(name string, containerID model.ID) (model.ID, error) { - folder := &model.Folder{ - BaseElement: model.BaseElement{ - ID: model.ID(types.GenerateID()), - TypeName: "Projects$Folder", - }, - ContainerID: containerID, - Name: name, - } - - if err := pb.backend.CreateFolder(folder); err != nil { - return "", err - } - - return folder.ID, nil + return createFolder(pb.ctx, name, containerID) } // execDropPage handles DROP PAGE statement. diff --git a/mdl/executor/cmd_pages_builder_v3.go b/mdl/executor/cmd_pages_builder_v3.go index ba051213d8..44f364f1fa 100644 --- a/mdl/executor/cmd_pages_builder_v3.go +++ b/mdl/executor/cmd_pages_builder_v3.go @@ -405,6 +405,12 @@ func (pb *pageBuilder) buildWidgetV3(w *ast.WidgetV3) (pages.Widget, error) { if err := checkWidgetHasName(w); err != nil { return nil, err } + // A property this widget does not take — `Caption:` on a textbox — was + // accepted and dropped (ako/mxcli#1698). Elements a parent builds itself + // (a region, a layout grid row) are judged where that parent reads them. + if err := refuseNativeWidgetProps(pb.ctx, nativeWidgetKindOf(w, nil), w); err != nil { + return nil, err + } switch strings.ToLower(w.Type) { case "dataview": @@ -3314,6 +3320,18 @@ func (pb *pageBuilder) buildScrollContainerV3(w *ast.WidgetV3) (pages.Widget, er Name: w.Name, }, } + // Headline or Sidebar: which regions run the full width (ako/mxcli#1703). + // It was neither read nor written, so a describe → exec copy of + // Atlas_SideBar came out a Headline layout. + if v := w.GetStringProp("LayoutMode"); v != "" { + mode, ok := pages.CanonicalScrollContainerLayoutMode(v) + if !ok { + return nil, mdlerrors.NewValidation(fmt.Sprintf( + "scrollcontainer %q: unknown LayoutMode %q (want %s)", + w.Name, v, strings.Join(pages.ScrollContainerLayoutModes, " or "))) + } + sc.LayoutMode = mode + } seen := map[pages.ScrollContainerSlot]bool{} for _, child := range w.Children { @@ -3337,32 +3355,46 @@ func (pb *pageBuilder) buildScrollContainerV3(w *ast.WidgetV3) (pages.Widget, er } seen[slot] = true - toggleMode, ok := canonicalRegionToggleMode(child.GetStringProp("ToggleMode")) - if !ok { - return nil, mdlerrors.NewValidation(fmt.Sprintf( - "scrollcontainer %q: region %q: unknown ToggleMode %q (want %s)", - w.Name, slot, child.GetStringProp("ToggleMode"), strings.Join(regionToggleModes, ", "))) - } - region := &pages.ScrollContainerRegion{ - BaseElement: model.BaseElement{ID: model.ID(types.GenerateID()), TypeName: "Forms$ScrollContainerRegion"}, - Slot: slot, - Class: child.GetStringProp("Class"), - SizeMode: child.GetStringProp("SizeMode"), - Size: child.GetIntProp("Size"), - ToggleMode: toggleMode, - } - for _, gc := range child.Children { - cw, err := pb.buildWidgetV3(gc) - if err != nil { - return nil, err - } - region.Widgets = append(region.Widgets, cw) + region, err := pb.buildScrollContainerRegionV3(w.Name, slot, child) + if err != nil { + return nil, err } sc.Regions = append(sc.Regions, region) } return sc, nil } +// buildScrollContainerRegionV3 builds one region of the scroll container +// named container. A region is not a widget: buildWidgetV3 never sees it, so +// what it takes is judged here (ako/mxcli#1698). +func (pb *pageBuilder) buildScrollContainerRegionV3(container string, slot pages.ScrollContainerSlot, w *ast.WidgetV3) (*pages.ScrollContainerRegion, error) { + if err := refuseNativeWidgetProps(pb.ctx, "scrollcontainer region", w); err != nil { + return nil, err + } + toggleMode, ok := canonicalRegionToggleMode(w.GetStringProp("ToggleMode")) + if !ok { + return nil, mdlerrors.NewValidation(fmt.Sprintf( + "scrollcontainer %q: region %q: unknown ToggleMode %q (want %s)", + container, slot, w.GetStringProp("ToggleMode"), strings.Join(regionToggleModes, ", "))) + } + region := &pages.ScrollContainerRegion{ + BaseElement: model.BaseElement{ID: model.ID(types.GenerateID()), TypeName: "Forms$ScrollContainerRegion"}, + Slot: slot, + Class: w.GetStringProp("Class"), + SizeMode: w.GetStringProp("SizeMode"), + Size: w.GetIntProp("Size"), + ToggleMode: toggleMode, + } + for _, gc := range w.Children { + cw, err := pb.buildWidgetV3(gc) + if err != nil { + return nil, err + } + region.Widgets = append(region.Widgets, cw) + } + return region, nil +} + // buildLayoutHeaderV3 builds a phone or tablet layout's header bar // (Forms$Header). Its children are its two sides, spelled like a scroll // container's regions: `region left { … }` and `region right { … }`. @@ -3387,6 +3419,10 @@ func (pb *pageBuilder) buildLayoutHeaderV3(w *ast.WidgetV3) (pages.Widget, error return nil, mdlerrors.NewValidation(fmt.Sprintf("layoutheader %q: region %q given twice", w.Name, side)) } seen[side] = true + // A header side stores nothing but its widgets (ako/mxcli#1698). + if err := refuseNativeWidgetProps(pb.ctx, "layoutheader region", child); err != nil { + return nil, err + } for _, gc := range child.Children { cw, err := pb.buildWidgetV3(gc) if err != nil { diff --git a/mdl/executor/cmd_pages_builder_v3_layout.go b/mdl/executor/cmd_pages_builder_v3_layout.go index 6ecf4db494..9d869fba0d 100644 --- a/mdl/executor/cmd_pages_builder_v3_layout.go +++ b/mdl/executor/cmd_pages_builder_v3_layout.go @@ -33,6 +33,10 @@ func (pb *pageBuilder) buildLayoutGridV3(w *ast.WidgetV3) (*pages.LayoutGrid, er // Build rows from children for _, child := range w.Children { if strings.ToLower(child.Type) == "row" { + // Built here, not by buildWidgetV3 (ako/mxcli#1698). + if err := refuseNativeWidgetProps(pb.ctx, "layoutgrid row", child); err != nil { + return nil, err + } row, err := pb.buildLayoutGridRowV3(child) if err != nil { return nil, err @@ -82,6 +86,10 @@ func (pb *pageBuilder) buildLayoutGridRowV3(w *ast.WidgetV3) (*pages.LayoutGridR // Build columns from children for _, child := range w.Children { if strings.ToLower(child.Type) == "column" { + // Built here, not by buildWidgetV3 (ako/mxcli#1698). + if err := refuseNativeWidgetProps(pb.ctx, "layoutgrid column", child); err != nil { + return nil, err + } col, err := pb.buildLayoutGridColumnV3(child) if err != nil { return nil, err diff --git a/mdl/executor/cmd_pages_builder_v3_widgets.go b/mdl/executor/cmd_pages_builder_v3_widgets.go index 9c03d6b5d6..9e6f121d18 100644 --- a/mdl/executor/cmd_pages_builder_v3_widgets.go +++ b/mdl/executor/cmd_pages_builder_v3_widgets.go @@ -117,6 +117,10 @@ func (pb *pageBuilder) buildDataViewV3(w *ast.WidgetV3) (*pages.DataView, error) for _, child := range w.Children { // Check if this is a FOOTER widget - its children go to FooterWidgets if child.Type == "footer" { + // The block is a slot; nothing of its own is stored (ako/mxcli#1698). + if err := refuseNativeWidgetProps(pb.ctx, "dataview footer", child); err != nil { + return nil, err + } if !showFooterSet { dv.ShowFooter = true } @@ -399,6 +403,10 @@ func (pb *pageBuilder) buildListViewV3(w *ast.WidgetV3) (*pages.ListView, error) lv.Widgets = append(lv.Widgets, widget) continue } + // A specialization template stores its widgets only (ako/mxcli#1698). + if err := refuseNativeWidgetProps(pb.ctx, "listview template", child); err != nil { + return nil, err + } tpl, err := pb.buildListViewTemplateV3(child, w.Name, listEntity, seen) if err != nil { return nil, err @@ -1406,6 +1414,10 @@ func (pb *pageBuilder) buildNavigationListV3(w *ast.WidgetV3) (*pages.Navigation // Build items from children (ITEM widgets) for _, child := range w.Children { if strings.ToLower(child.Type) == "item" { + // Built here, not by buildWidgetV3 (ako/mxcli#1698). + if err := refuseNativeWidgetProps(pb.ctx, "navigationlist item", child); err != nil { + return nil, err + } item, err := pb.buildNavigationListItemV3(child) if err != nil { return nil, err diff --git a/mdl/executor/cmd_pages_describe.go b/mdl/executor/cmd_pages_describe.go index ce1e52aa0d..6fa26c1fbf 100644 --- a/mdl/executor/cmd_pages_describe.go +++ b/mdl/executor/cmd_pages_describe.go @@ -951,6 +951,9 @@ type rawWidget struct { // RegionToggleMode is a scroll container region's ToggleMode — how a // sidebar toggle opens it. None (the default) is not printed. RegionToggleMode string + // ScrollLayoutMode is a scroll container's LayoutMode, Headline (the + // default, not printed) or Sidebar (ako/mxcli#1703). + ScrollLayoutMode string // NavigationProfile is a Forms$NavigationTree's profile, which the document // keeps one level down in MenuSource rather than on the tree. NavigationProfile string diff --git a/mdl/executor/cmd_pages_describe_output.go b/mdl/executor/cmd_pages_describe_output.go index 346b276e0b..67056b0139 100644 --- a/mdl/executor/cmd_pages_describe_output.go +++ b/mdl/executor/cmd_pages_describe_output.go @@ -13,6 +13,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/pages" ) // mdl0Quote writes s as an mdl 0 string literal: a backslash, a line break, a @@ -323,7 +324,13 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { switch w.Type { case "Forms$ScrollContainer", "Pages$ScrollContainer": header := fmt.Sprintf("scrollcontainer %s", mdlIdent(w.Name)) - props := appendAppearanceProps(ctx, nil, w) + var props []string + // Only Sidebar: Headline is what an unset scroll container is, so a + // layout that never set it describes as before (ako/mxcli#1703). + if w.ScrollLayoutMode != "" && w.ScrollLayoutMode != pages.DefaultScrollContainerLayoutMode { + props = append(props, fmt.Sprintf("LayoutMode: %s", mdlQuote(ctx, w.ScrollLayoutMode))) + } + props = appendAppearanceProps(ctx, props, w) if len(w.Children) > 0 { formatWidgetProps(ctx.Output, prefix, header, props, " {\n") for _, child := range w.Children { @@ -370,7 +377,12 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { if w.RegionToggleMode != "" && w.RegionToggleMode != "None" { props = append(props, fmt.Sprintf("ToggleMode: %s", mdlQuote(ctx, w.RegionToggleMode))) } - props = appendAppearanceProps(ctx, props, w) + // Class is the one appearance key a region is read with and written + // with (buildScrollContainerRegionV3); printing the rest would be + // printing what re-execution refuses (ako/mxcli#1698). + if w.Class != "" { + props = append(props, fmt.Sprintf("Class: %s", mdlQuote(ctx, w.Class))) + } if len(w.Children) > 0 { formatWidgetProps(ctx.Output, prefix, header, props, " {\n") for _, child := range w.Children { diff --git a/mdl/executor/cmd_pages_describe_parse.go b/mdl/executor/cmd_pages_describe_parse.go index 73462d9f52..c89dd5d41c 100644 --- a/mdl/executor/cmd_pages_describe_parse.go +++ b/mdl/executor/cmd_pages_describe_parse.go @@ -119,6 +119,8 @@ func parseRawWidget(ctx *ExecContext, w map[string]any, parentEntityContext ...s Type: typeName, Name: name, } + // Which regions run the full width (ako/mxcli#1703). + widget.ScrollLayoutMode, _ = w["LayoutMode"].(string) if appearance, ok := w["Appearance"].(map[string]any); ok { if class, ok := appearance["Class"].(string); ok && class != "" { widget.Class = class diff --git a/mdl/executor/cmd_search.go b/mdl/executor/cmd_search.go index 74614a20db..a908a46591 100644 --- a/mdl/executor/cmd_search.go +++ b/mdl/executor/cmd_search.go @@ -25,14 +25,17 @@ import ( // The recurring shape is an ENTRY POINT: something outside the call graph // invokes it, so nothing in the model calls it. 'schedule' was the first (a // microflow run only by a scheduled event reported "(no callers found)" on a job -// that runs nightly); 'publish' is a published REST operation (#1126); 'event' -// is an entity event handler, run by the entity on commit or delete; 'settings' +// that runs nightly); 'publish' is a published REST operation (#1126), or a +// published OData resource, action or authentication microflow +// (ako/mxcli#1639); 'event' is an entity event handler, run by the entity on +// commit or delete, or a business event handler; 'settings' // is a microflow wired as after-startup, before-shutdown or health check, whose // edge shipped in v0.22.0 and was added to QUAL004 but not here, so // `show callers` stayed blind to it. Every new way for something to run a -// microflow belongs in this list, in graphRefKinds, and in QUAL004's -// MICROFLOW_ENTRY_KINDS — three consumers, none of which shares the others' -// list. +// microflow belongs in this list and in graphRefKinds — two consumers, neither +// of which shares the other's list. (QUAL004 had a third, MICROFLOW_ENTRY_KINDS; +// it fell behind on the navigation kinds and was removed in favour of counting +// every edge, ako/mxcli#1484.) // // Deliberately excluded: 'datasource', 'parameter', 'return', 'retrieve', // 'create', 'change', 'delete', 'commit', 'associate', 'generalize', 'layout' diff --git a/mdl/executor/cmd_security.go b/mdl/executor/cmd_security.go index 8bf1025579..1041681369 100644 --- a/mdl/executor/cmd_security.go +++ b/mdl/executor/cmd_security.go @@ -36,6 +36,7 @@ func listProjectSecurity(ctx *ExecContext) error { if ps.AdminUserName != "" { result.Rows = append(result.Rows, []any{"AdminUser", ps.AdminUserName}) } + result.Rows = append(result.Rows, []any{"AdminPasswordSet", fmt.Sprintf("%v", ps.AdminPassword != "")}) if ps.GuestUserRole != "" { result.Rows = append(result.Rows, []any{"GuestUserRole", ps.GuestUserRole}) } @@ -59,6 +60,13 @@ func listProjectSecurity(ctx *ExecContext) error { if ps.AdminUserName != "" { fmt.Fprintf(ctx.Output, "Admin User: %s\n", ps.AdminUserName) } + // Whether, never what: an empty one is CE0129 at prototype and production + // level, which `alter app security ( AdminPassword: '…' )` resolves. + if ps.AdminPassword != "" { + fmt.Fprintln(ctx.Output, "Admin Password: set") + } else { + fmt.Fprintln(ctx.Output, "Admin Password: not set (CE0129 unless the security level is off)") + } if ps.GuestUserRole != "" { fmt.Fprintf(ctx.Output, "Guest User Role: %s\n", ps.GuestUserRole) } diff --git a/mdl/executor/cmd_security_write.go b/mdl/executor/cmd_security_write.go index 0226dcba12..808f4354a4 100644 --- a/mdl/executor/cmd_security_write.go +++ b/mdl/executor/cmd_security_write.go @@ -1227,6 +1227,14 @@ func execAlterProjectSecurity(ctx *ExecContext, s *ast.AlterProjectSecurityStmt) ctx.reportWrite("admin user name: "+s.AdminUserName, "Admin user name set to %s", s.AdminUserName) } + if s.AdminPassword != "" { + if err := ctx.Backend.SetProjectAdminPassword(ps.ID, s.AdminPassword); err != nil { + return mdlerrors.NewBackend("set admin password", err) + } + // Never echo the value: exec output ends up in logs and transcripts. + ctx.reportWrite("admin password", "Admin password set") + } + if s.StrictModeEnabled != nil { if err := ctx.Backend.SetProjectStrictMode(ps.ID, *s.StrictModeEnabled); err != nil { return mdlerrors.NewBackend("set strict mode", err) diff --git a/mdl/executor/cmd_settings.go b/mdl/executor/cmd_settings.go index e11f5f9932..068e240598 100644 --- a/mdl/executor/cmd_settings.go +++ b/mdl/executor/cmd_settings.go @@ -143,6 +143,19 @@ func describeSettings(ctx *ExecContext, configName string) error { fmt.Fprintf(ctx.Output, "-- Project ID: %s (read-only; Business Events uses it as the app id, and a copy of the model shares it)\n\n", id) } + writeProjectSettingsMDL(ctx, ps, nil) + return nil +} + +// constantValueFunc renders the value of a configuration's constant override +// in a settings description; nil prints it as a quoted literal. +type constantValueFunc func(configuration, constant, value string) string + +// writeProjectSettingsMDL prints the settings document as DESCRIBE SETTINGS +// does, below the project id (which is not part of the document). diff-local +// renders the settings unit with it, passing a constantValue that masks the +// values, so the two cannot drift apart (ako/mxcli#1512). +func writeProjectSettingsMDL(ctx *ExecContext, ps *model.ProjectSettings, constantValue constantValueFunc) { // Model settings if ps.Model != nil { ms := ps.Model @@ -200,7 +213,7 @@ func describeSettings(ctx *ExecContext, configName string) error { // Configuration settings if ps.Configuration != nil { for _, cfg := range ps.Configuration.Configurations { - writeSettingsConfiguration(ctx, cfg) + writeSettingsConfiguration(ctx, cfg, constantValue) } } @@ -277,8 +290,6 @@ func describeSettings(ctx *ExecContext, configName string) error { fmt.Fprintf(ctx.Output, "alter settings webui (\n OptimizedClient: %s\n);\n\n", mdlQuoted(ps.WebUI.UseOptimizedClient)) } } - - return nil } // modelSettingKeys names every property the ALTER SETTINGS MODEL switch below @@ -1165,7 +1176,7 @@ func settingsValueToString(val any) string { } // writeSettingsConfiguration emits one configuration as re-executable MDL. -func writeSettingsConfiguration(ctx *ExecContext, cfg *model.ServerConfiguration) { +func writeSettingsConfiguration(ctx *ExecContext, cfg *model.ServerConfiguration, constantValue constantValueFunc) { var parts []string parts = append(parts, " DatabaseType: "+mdlQuoted(cfg.DatabaseType)) parts = append(parts, " DatabaseUrl: "+mdlQuoted(cfg.DatabaseUrl)) @@ -1211,8 +1222,12 @@ func writeSettingsConfiguration(ctx *ExecContext, cfg *model.ServerConfiguration cv.ConstantId, cfg.Name) continue } + value := mdlQuoted(cv.Value) + if constantValue != nil { + value = constantValue(cfg.Name, cv.ConstantId, cv.Value) + } fmt.Fprintf(ctx.Output, "alter settings constant %s value %s\n in configuration %s;\n\n", - mdlQuoted(cv.ConstantId), mdlQuoted(cv.Value), mdlQuoted(cfg.Name)) + mdlQuoted(cv.ConstantId), value, mdlQuoted(cfg.Name)) } } @@ -1223,7 +1238,7 @@ func describeSettingsConfiguration(ctx *ExecContext, ps *model.ProjectSettings, if ps.Configuration != nil { for _, cfg := range ps.Configuration.Configurations { if strings.EqualFold(cfg.Name, name) { - writeSettingsConfiguration(ctx, cfg) + writeSettingsConfiguration(ctx, cfg, nil) return nil } available = append(available, "'"+cfg.Name+"'") diff --git a/mdl/executor/helpers.go b/mdl/executor/helpers.go index f0b50e438c..62d078b3cd 100644 --- a/mdl/executor/helpers.go +++ b/mdl/executor/helpers.go @@ -151,7 +151,19 @@ func resolveFolder(ctx *ExecContext, moduleID model.ID, folderPath string) (mode return currentContainerID, nil } -// createFolder creates a new folder in the project. +// createFolder creates a new folder in the project, and is the only place the +// executor does (TestFolderCreationHasOneCallSite). +// +// It also records the folder in the cached container hierarchy. Every document +// lookup by qualified name walks up from the document's container through that +// cache to find its module, and the cache is commonly built by the very CREATE +// that is about to make the folder — its own existence check runs first. A +// folder the cache has not seen ends the walk at the folder, the module name +// comes back "", and the document is "not found": a GRANT on a published REST +// service created in a new folder earlier in the same script was rolled back +// that way, and a re-run CREATE OR MODIFY fell through to creating a duplicate +// (ako/mxcli#1486). Every other write that changes containers already kept the +// cache in step; this one did not. func createFolder(ctx *ExecContext, name string, containerID model.ID) (model.ID, error) { folder := &model.Folder{ BaseElement: model.BaseElement{ @@ -165,6 +177,9 @@ func createFolder(ctx *ExecContext, name string, containerID model.ID) (model.ID if err := ctx.Backend.CreateFolder(folder); err != nil { return "", err } + if ctx.Cache != nil && ctx.Cache.hierarchy != nil { + ctx.Cache.hierarchy.addFolder(folder.ID, containerID, name) + } return folder.ID, nil } diff --git a/mdl/executor/hierarchy.go b/mdl/executor/hierarchy.go index 837843c171..46029bf48f 100644 --- a/mdl/executor/hierarchy.go +++ b/mdl/executor/hierarchy.go @@ -68,6 +68,13 @@ func newContainerHierarchyImpl(src hierarchySource) (*ContainerHierarchy, error) return h, nil } +// addFolder records a folder created after the hierarchy was built, so a +// cached hierarchy can resolve documents placed in it (ako/mxcli#1486). +func (h *ContainerHierarchy) addFolder(id, containerID model.ID, name string) { + h.folderNames[id] = name + h.containerParent[id] = containerID +} + // FindModuleID finds the module ID for any container by traversing the hierarchy. func (h *ContainerHierarchy) FindModuleID(containerID model.ID) model.ID { current := containerID diff --git a/mdl/executor/hierarchy_created_folder_test.go b/mdl/executor/hierarchy_created_folder_test.go new file mode 100644 index 0000000000..f22f9b7c6a --- /dev/null +++ b/mdl/executor/hierarchy_created_folder_test.go @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" + "github.com/mendixlabs/mxcli/sdk/security" +) + +// ako/mxcli#1486 (mendixlabs/mxcli#1581): `grant access on published rest +// service M.PingApi to M.User` reported "published rest service not found" +// when the service was created with a `folder '…'` clause earlier in the same +// script, and exec rolled the whole script back. +// +// The executor caches the container hierarchy (module ← folder ← folder) once +// per exec and resolves every document's module through it. CREATE PUBLISHED +// REST SERVICE built that cache in its own existence check, then created the +// folder — and nothing told the cache. The grant found the service, walked up +// from its container, hit a folder the cache had never seen, and so compared +// "" against "M". Every other write that changes containers (create module, +// move document, move/drop folder, apply folder) already kept the cache in +// step; creating a folder was the one that did not. + +// folderStore is a mock project that behaves like a store: what a handler +// creates, the next statement lists. Its CreateFolder deliberately does NOT +// touch the hierarchy — the fixture in document_placement_test.go registers +// each new folder into the cached hierarchy itself, which is exactly the +// bookkeeping the executor was missing, and it hid this bug from that suite. +type folderStore struct { + mod *model.Module + folders []*types.FolderInfo + services []*model.PublishedRestService +} + +func newFolderStoreCtx(t *testing.T) (*folderStore, *ExecContext, *strings.Builder) { + t.Helper() + s := &folderStore{mod: mkModule("M")} + ping := µflows.Microflow{ + BaseElement: model.BaseElement{ID: nextID("mf")}, + ContainerID: s.mod.ID, + Name: "PRS_Ping", + Parameters: []*microflows.MicroflowParameter{{ + Name: "httpRequest", + Type: µflows.ObjectType{EntityQualifiedName: "System.HttpRequest"}, + }}, + ReturnType: µflows.StringType{}, + } + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{s.mod}, nil }, + ListFoldersFunc: func() ([]*types.FolderInfo, error) { return s.folders, nil }, + CreateFolderFunc: func(f *model.Folder) error { + s.folders = append(s.folders, &types.FolderInfo{ID: f.ID, ContainerID: f.ContainerID, Name: f.Name}) + return nil + }, + ListMicroflowsFunc: func() ([]*microflows.Microflow, error) { + return []*microflows.Microflow{ping}, nil + }, + ListPublishedRestServicesFunc: func() ([]*model.PublishedRestService, error) { return s.services, nil }, + CreatePublishedRestServiceFunc: func(svc *model.PublishedRestService) error { + if svc.ID == "" { + svc.ID = nextID("prs") + } + s.services = append(s.services, svc) + return nil + }, + UpdatePublishedRestServiceFunc: func(svc *model.PublishedRestService) error { return nil }, + MoveDocumentFunc: func(unitID, containerID model.ID) error { return nil }, + GetModuleSecurityFunc: func(model.ID) (*security.ModuleSecurity, error) { + return &security.ModuleSecurity{ModuleRoles: []*security.ModuleRole{{Name: "User"}}}, nil + }, + UpdatePublishedRestServiceRolesFunc: func(id model.ID, roles []string) error { + for _, svc := range s.services { + if svc.ID == id { + svc.AllowedRoles = roles + } + } + return nil + }, + } + // No hierarchy is seeded: the first statement that needs it builds it from + // the store, as exec does. + ctx, _ := newMockCtx(t, withBackend(mb)) + out := &strings.Builder{} + ctx.Output = out + return s, ctx, out +} + +// execFolderScript runs a script statement by statement against one context, +// the way exec does, and stops at the first error. +func execFolderScript(t *testing.T, ctx *ExecContext, src string) error { + t.Helper() + for _, stmt := range parseMDL(t, src).Statements { + var err error + switch s := stmt.(type) { + case *ast.CreatePublishedRestServiceStmt: + err = execCreatePublishedRestService(ctx, s) + case *ast.GrantPublishedRestServiceAccessStmt: + err = execGrantPublishedRestServiceAccess(ctx, s) + case *ast.RevokePublishedRestServiceAccessStmt: + err = execRevokePublishedRestServiceAccess(ctx, s) + default: + t.Fatalf("unexpected statement %T", stmt) + } + if err != nil { + return err + } + } + return nil +} + +func pingService(folderClause string) string { + return `create or modify published rest service M.PingApi ` + folderClause + ` ( + Path: 'rest/ping/v1', + Version: '1.0.0', + ServiceName: 'PingApi' +) +{ + resource 'ping' { + get '' microflow M.PRS_Ping; + } +}; +` +} + +func TestGrantPublishedRestService_CreatedInFolderSameScript(t *testing.T) { + cases := map[string]string{ + // The reported case: the create makes the folder. + "new nested folder": `folder 'Api/Services'`, + // Control: the module root. Passed before the fix; it proves the + // failure is the folder, not the grant. + "module root": ``, + } + for name, folder := range cases { + t.Run(name, func(t *testing.T) { + s, ctx, out := newFolderStoreCtx(t) + err := execFolderScript(t, ctx, pingService(folder)+` +grant access on published rest service M.PingApi to M.User;`) + if err != nil { + t.Fatalf("grant after create failed: %v\noutput:\n%s", err, out) + } + if len(s.services) != 1 { + t.Fatalf("want 1 service, got %d", len(s.services)) + } + if got := strings.Join(s.services[0].AllowedRoles, ","); got != "M.User" { + t.Errorf("stored roles = %q, want M.User", got) + } + + if err := execFolderScript(t, ctx, `revoke access on published rest service M.PingApi from M.User;`); err != nil { + t.Fatalf("revoke after create failed: %v", err) + } + if got := s.services[0].AllowedRoles; len(got) != 0 { + t.Errorf("roles after revoke = %v, want none", got) + } + }) + } +} + +// A re-run of the same CREATE OR MODIFY looks the service up by the same +// route. With a stale cache that lookup missed too, and the statement fell +// through to CREATE: a second service with the same name, silently. +func TestCreateOrModifyPublishedRestService_InNewFolderTwiceIsOneService(t *testing.T) { + s, ctx, out := newFolderStoreCtx(t) + if err := execFolderScript(t, ctx, pingService(`folder 'Api/Services'`)+pingService(`folder 'Api/Services'`)); err != nil { + t.Fatalf("exec: %v\noutput:\n%s", err, out) + } + if len(s.services) != 1 { + t.Errorf("the second statement created a duplicate: %d services named M.PingApi\noutput:\n%s", len(s.services), out) + } +} + +// Control: the not-found refusal itself is intact. +func TestGrantPublishedRestService_MissingServiceStillRefused(t *testing.T) { + _, ctx, _ := newFolderStoreCtx(t) + err := execFolderScript(t, ctx, pingService(`folder 'Api/Services'`)+` +grant access on published rest service M.Missing to M.User;`) + if err == nil || !strings.Contains(err.Error(), "published rest service not found: M.Missing") { + t.Fatalf("want the not-found refusal for M.Missing, got %v", err) + } +} + +// The mechanism, below every doctype: a folder created while the hierarchy is +// cached must be resolvable through that cache. Every handler with a FOLDER +// clause creates folders through resolveFolder, so this is what the audit's +// other failures (scheduled event, task queue, regular expression, consumed +// REST service) had in common with the reported one. +func TestCreatedFolderIsVisibleThroughTheCachedHierarchy(t *testing.T) { + _, ctx, _ := newFolderStoreCtx(t) + mod, err := findModule(ctx, "M") + assertNoError(t, err) + + _, err = getHierarchy(ctx) // cached before the folder exists + assertNoError(t, err) + + leaf, err := resolveFolder(ctx, mod.ID, "Api/Services") + assertNoError(t, err) + + h, err := getHierarchy(ctx) + assertNoError(t, err) + if got := h.GetModuleName(h.FindModuleID(leaf)); got != "M" { + t.Errorf("module of the new folder = %q, want M (the cached hierarchy never learned of the folder)", got) + } + if got := h.BuildFolderPath(leaf); got != "Api/Services" { + t.Errorf("folder path = %q, want Api/Services", got) + } + + // The page builder resolves FOLDER clauses on pages and snippets with its + // own walk; it must create through the same helper. + pb := &pageBuilder{ctx: ctx, backend: ctx.Backend, moduleID: mod.ID, moduleName: "M", execCache: ctx.Cache} + pageLeaf, err := pb.resolveFolder("Pages/Admin") + assertNoError(t, err) + h, _ = getHierarchy(ctx) + if got := h.BuildFolderPath(pageLeaf); got != "Pages/Admin" { + t.Errorf("page folder path = %q, want Pages/Admin", got) + } +} + +// Every folder the executor creates goes through createFolder, which keeps the +// cached hierarchy in step. A second call site would be a second chance to +// forget that, which is how this bug happened: the hierarchy's other writers +// all remembered, the folder creator did not. +func TestFolderCreationHasOneCallSite(t *testing.T) { + files, err := filepath.Glob("*.go") + assertNoError(t, err) + call := regexp.MustCompile(`\.CreateFolder\(`) + var sites []string + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + src, err := os.ReadFile(f) + assertNoError(t, err) + for i, line := range strings.Split(string(src), "\n") { + if call.MatchString(line) { + sites = append(sites, f+":"+strconv.Itoa(i+1)) + } + } + } + if len(sites) != 1 || !strings.HasPrefix(sites[0], "helpers.go:") { + t.Errorf("Backend.CreateFolder is called from %v; route new call sites through createFolder (helpers.go), "+ + "which records the folder in the cached hierarchy (ako/mxcli#1486)", sites) + } +} diff --git a/mdl/executor/input_formatting_test.go b/mdl/executor/input_formatting_test.go index b95f1c3942..133745003f 100644 --- a/mdl/executor/input_formatting_test.go +++ b/mdl/executor/input_formatting_test.go @@ -31,10 +31,11 @@ func TestValidateInputFormatting(t *testing.T) { {"custom without pattern", `datepicker d (Attribute: Moment, DateFormat: Custom)`, "MDL-WIDGET18", "needs a non-empty CustomDateFormat"}, // -1 is Studio Pro's "no fixed precision" and is accepted (mendixlabs/mxcli#1407). {"precision below -1", `textbox t (Attribute: Amount, DecimalPrecision: -2)`, "MDL-WIDGET18", "non-negative integer"}, - // A widget with no FormattingInfo still drops the key, and says so. - {"DateFormat on a text area", `textarea a (Attribute: Notes, DateFormat: Time)`, "MDL-WIDGET07", "DateFormat"}, + // A widget with no FormattingInfo does not take the key, and says so + // (MDL-V1-WIDGETPROP: a warning here, without the header; ako/mxcli#1698). + {"DateFormat on a text area", `textarea a (Attribute: Notes, DateFormat: Time)`, "MDL-V1-WIDGETPROP", "DateFormat"}, // A text box binds no dates (CE2421), so DateFormat means nothing there. - {"DateFormat on a text box", `textbox t (Attribute: Amount, DateFormat: Time)`, "MDL-WIDGET07", "DateFormat"}, + {"DateFormat on a text box", `textbox t (Attribute: Amount, DateFormat: Time)`, "MDL-V1-WIDGETPROP", "DateFormat"}, } { t.Run(c.name, func(t *testing.T) { prog, errs := visitor.Build(page(c.widget)) @@ -44,7 +45,8 @@ func TestValidateInputFormatting(t *testing.T) { vs := ValidateWidgetProperties(prog, "") var hit []string for _, v := range vs { - if strings.HasPrefix(v.RuleID, "MDL-WIDGET18") || strings.HasPrefix(v.RuleID, "MDL-WIDGET07") { + if strings.HasPrefix(v.RuleID, "MDL-WIDGET18") || strings.HasPrefix(v.RuleID, "MDL-WIDGET07") || + v.RuleID == nativeWidgetPropDropped.Code || v.RuleID == "MDL-WIDGET52" { hit = append(hit, v.RuleID+": "+v.Message) } } diff --git a/mdl/executor/issue1653_tabpage_badge_test.go b/mdl/executor/issue1653_tabpage_badge_test.go index 7e45dba952..ba75bcf089 100644 --- a/mdl/executor/issue1653_tabpage_badge_test.go +++ b/mdl/executor/issue1653_tabpage_badge_test.go @@ -180,7 +180,7 @@ func TestIssue1653_CheckAcceptsTheBadgeOnATabPageOnly(t *testing.T) { got := issue1653Violations(t, `create page M.P (Title: 'x', Layout: Atlas_Core.Atlas_Default) { dynamictext txt (Content: 'a', Badge: 'b') };`) - if len(got) == 0 || !strings.Contains(strings.Join(got, "\n"), "MDL-WIDGET07") { + if len(got) == 0 || !strings.Contains(strings.Join(got, "\n"), nativeWidgetPropDropped.Code) { t.Errorf("Badge on a dynamic text is not reported: %v", got) } } diff --git a/mdl/executor/language_changes.go b/mdl/executor/language_changes.go index 638aaa2a4a..56b42704d6 100644 --- a/mdl/executor/language_changes.go +++ b/mdl/executor/language_changes.go @@ -13,5 +13,6 @@ func LanguageChanges() []langver.Change { return []langver.Change{ actionSlotRefused, galleryClickRefused, flowRebuildRefused, boundaryDropAmbiguous, remoteTypeChangeRefused, templateAttrBinding, viewAutoNumberRefused, tabPagePropDropped, + nativeWidgetPropDropped, } } diff --git a/mdl/executor/layout_mode_1703_test.go b/mdl/executor/layout_mode_1703_test.go new file mode 100644 index 0000000000..e29a77daa9 --- /dev/null +++ b/mdl/executor/layout_mode_1703_test.go @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "context" + "strings" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/langver" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/sdk/pages" +) + +// ako/mxcli#1703 (mendixlabs/mxcli#1659): a scroll container's LayoutMode — +// Headline or Sidebar, which decides whether the top bar or the left region +// runs the full width — was neither described nor written, so the documented +// describe → rename → exec copy of Atlas_Core.Atlas_SideBar was a Headline +// layout, and `(LayoutMode: 'Sidebar')` was dropped in silence. + +func scrollContainerOf(t *testing.T, props string) *ast.WidgetV3 { + t.Helper() + prog, errs := visitor.Build("create layout M.L (layouttype: 'Responsive') {\n scrollcontainer sc " + props + + " {\n region center { placeholder Main }\n }\n};\n") + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + return prog.Statements[0].(*ast.CreateLayoutStmt).Widgets[0] +} + +func TestScrollContainerLayoutMode_IsWritten(t *testing.T) { + for _, c := range []struct{ props, want string }{ + {"(LayoutMode: 'Sidebar')", "Sidebar"}, + {"(layoutmode: sidebar)", "Sidebar"}, // any case, bare or quoted; stored as the metamodel spells it + {"(LayoutMode: 'Headline')", "Headline"}, + {"", ""}, // control: unsaid stays unwritten, which Studio Pro reads as Headline + } { + pb := visibleWhenPB("") + pb.ctx = &ExecContext{Output: &bytes.Buffer{}, LanguageVersion: langver.V1} + built, err := pb.buildWidgetV3(scrollContainerOf(t, c.props)) + if err != nil { + t.Fatalf("%q: %v", c.props, err) + } + if got := built.(*pages.ScrollContainer).LayoutMode; got != c.want { + t.Errorf("%q: LayoutMode = %q, want %q", c.props, got, c.want) + } + } +} + +func TestScrollContainerLayoutMode_UnknownValueIsRefused(t *testing.T) { + pb := visibleWhenPB("") + pb.ctx = &ExecContext{Output: &bytes.Buffer{}, LanguageVersion: langver.V1} + _, err := pb.buildWidgetV3(scrollContainerOf(t, "(LayoutMode: 'Topbar')")) + if err == nil || !strings.Contains(err.Error(), "Headline or Sidebar") { + t.Errorf("err = %v, want a refusal naming Headline or Sidebar", err) + } +} + +// The property is checked, not just built: `check` accepts it where, before +// the support, the per-widget verdict refused it (MDL-WIDGET52). +func TestScrollContainerLayoutMode_CheckAccepts(t *testing.T) { + vs := nativePropViolations(t, "mdl 1;\ncreate layout M.L (layouttype: 'Responsive') {\n"+ + " scrollcontainer sc (LayoutMode: 'Sidebar') { region center { placeholder Main } }\n};\n") + if len(vs) != 0 { + t.Errorf("check refuses a scroll container's LayoutMode:\n%s", violationText(vs)) + } +} + +// describe prints Sidebar, and leaves Headline — the default — unsaid, so a +// layout that never set it describes as before. +func TestScrollContainerLayoutMode_Describe(t *testing.T) { + for _, c := range []struct{ stored, want string }{ + {"Sidebar", "scrollcontainer layoutContainer (LayoutMode: 'Sidebar') {"}, + {"Headline", "scrollcontainer layoutContainer {"}, + {"", "scrollcontainer layoutContainer {"}, + } { + stored := map[string]any{ + "$Type": "Forms$ScrollContainer", + "Name": "layoutContainer", + "CenterRegion": map[string]any{ + "$Type": "Forms$ScrollContainerRegion", + "Widgets": []any{int32(2), map[string]any{"$Type": "Forms$Placeholder", "Name": "Main"}}, + }, + } + if c.stored != "" { + stored["LayoutMode"] = c.stored + } + var buf bytes.Buffer + ctx := (&Executor{}).newExecContext(context.Background()) + ctx.Output = &buf + outputWidgetMDLV3(ctx, parseRawWidget(ctx, stored)[0], 1) + if !strings.Contains(buf.String(), c.want) { + t.Errorf("stored %q: describe lacks %q:\n%s", c.stored, c.want, buf.String()) + } + } +} + +// The reported workflow end to end on the Studio Pro-authored PedApp: describe +// Atlas_SideBar, run it under a new name, and the copy stores Sidebar. The +// control is Atlas_Default, a Headline layout, whose copy stays Headline and +// whose describe says nothing about it. +func TestCopyAtlasSideBarKeepsItsLayoutMode(t *testing.T) { + exec, out, dir := openPedAppCopy(t) + for _, c := range []struct{ name, want string }{ + {"Atlas_SideBar", "Sidebar"}, + {"Atlas_Default", "Headline"}, + } { + if got := storedScrollLayoutModes(t, exec, "Atlas_Core."+c.name); len(got) != 1 || got[0] != c.want { + t.Fatalf("fixture: %s stores LayoutMode %v, want [%s]; the test proves nothing", c.name, got, c.want) + } + out.Reset() + if err := exec.Execute(&ast.DescribeStmt{ObjectType: ast.DescribeLayout, Name: ast.QualifiedName{Module: "Atlas_Core", Name: c.name}}); err != nil { + t.Fatalf("describe layout %s: %v", c.name, err) + } + described := languageHeader.ReplaceAllString(out.String(), "") + if says := strings.Contains(described, "LayoutMode:"); says != (c.want == "Sidebar") { + t.Errorf("%s: describe prints LayoutMode = %v, want %v:\n%s", c.name, says, c.want == "Sidebar", described) + } + copied := strings.Replace(described, "create or modify layout Atlas_Core."+c.name+" ", "create layout MyFirstModule.Copy_"+c.name+" ", 1) + if got := agreeCheck(t, exec, dir, "mdl 1;\n"+copied); len(got) != 0 { + t.Fatalf("%s: check refuses the copy:\n%s", c.name, strings.Join(got, "\n")) + } + if err := agreeExec(t, exec, "mdl 1;\n"+copied); err != nil { + t.Fatalf("%s: exec of the copy: %v", c.name, err) + } + if got := storedScrollLayoutModes(t, exec, "MyFirstModule.Copy_"+c.name); len(got) != 1 || got[0] != c.want { + t.Errorf("%s: the copy stores LayoutMode %v, Studio Pro stores [%s]", c.name, got, c.want) + } + } +} + +// storedScrollLayoutModes is the LayoutMode of every scroll container in a +// layout, in document order ("" for one that stores none). +func storedScrollLayoutModes(t *testing.T, exec *Executor, qn string) []string { + t.Helper() + ctx := exec.newExecContext(context.Background()) + u, err := ctx.Backend.GetRawUnitByName("layout", qn) + if err != nil || u == nil { + t.Fatalf("raw layout %s: %v", qn, err) + } + var modes []string + var walk func(v bson.RawValue) + walk = func(v bson.RawValue) { + switch v.Type { + case bson.TypeEmbeddedDocument: + doc := v.Document() + if doc.Lookup("$Type").StringValue() == "Forms$ScrollContainer" { + mode, _ := doc.Lookup("LayoutMode").StringValueOK() + modes = append(modes, mode) + } + elems, _ := doc.Elements() + for _, e := range elems { + walk(e.Value()) + } + case bson.TypeArray: + vals, _ := v.Array().Values() + for _, x := range vals { + walk(x) + } + } + } + walk(bson.RawValue{Type: bson.TypeEmbeddedDocument, Value: u.Contents}) + return modes +} diff --git a/mdl/executor/sidebartoggle_copy_layout_pedapp_test.go b/mdl/executor/sidebartoggle_copy_layout_pedapp_test.go index 7a02d71340..c6eb2631c3 100644 --- a/mdl/executor/sidebartoggle_copy_layout_pedapp_test.go +++ b/mdl/executor/sidebartoggle_copy_layout_pedapp_test.go @@ -187,7 +187,7 @@ func TestCopyEveryAtlasLayoutKeepsEveryWidget(t *testing.T) { if len(names) < 20 { t.Fatalf("found only %d Atlas_Core layouts in the fixture: %v", len(names), names) } - headers := 0 + headers, sidebars := 0, 0 for _, name := range names { out.Reset() if err := exec.Execute(&ast.DescribeStmt{ObjectType: ast.DescribeLayout, Name: ast.QualifiedName{Module: "Atlas_Core", Name: name}}); err != nil { @@ -232,6 +232,20 @@ func TestCopyEveryAtlasLayoutKeepsEveryWidget(t *testing.T) { t.Errorf("%s: region %s ToggleMode = %q in the copy, Studio Pro stores %q", name, slot, cm[slot], want) } } + // Headline or Sidebar (ako/mxcli#1703): a copy of Atlas_SideBar was a + // Headline layout, with every key and every widget still matching. + ol, cl := storedScrollLayoutModes(t, exec, "Atlas_Core."+name), storedScrollLayoutModes(t, exec, "MyFirstModule.Copy_"+name) + if strings.Join(ol, ",") != strings.Join(cl, ",") { + t.Errorf("%s: scroll container LayoutMode %v in the copy, Studio Pro stores %v", name, cl, ol) + } + for _, m := range ol { + if m == "Sidebar" { + sidebars++ + } + } + } + if sidebars == 0 { + t.Error("no Atlas_Core layout in the fixture is a Sidebar layout; the LayoutMode half of this test proves nothing") } if headers == 0 { t.Error("no Atlas_Core layout in the fixture holds a Forms$Header; the header half of this test proves nothing") diff --git a/mdl/executor/validate_widget_action_slot.go b/mdl/executor/validate_widget_action_slot.go index e355df433c..e0ef5f37a9 100644 --- a/mdl/executor/validate_widget_action_slot.go +++ b/mdl/executor/validate_widget_action_slot.go @@ -32,6 +32,7 @@ package executor import ( "fmt" + "sort" "strings" "github.com/mendixlabs/mxcli/mdl/ast" @@ -82,27 +83,51 @@ func validateWidgetActionSlot(w *ast.WidgetV3, locationPrefix string) []linter.V // Iterated over a fixed slice rather than over w.Properties: map order is // not stable, and a widget with two faulty slots must report them the same // way every run (CLAUDE.md, determinism). - for _, key := range actionSlotKeys { - raw, present := w.Properties[key] - if !present { - continue + for _, slot := range actionSlotKeys { + for _, key := range actionSlotSpellings(w, slot) { + raw := w.Properties[key] + if _, ok := raw.(*ast.ActionV3); ok { + continue + } + out = append(out, linter.Violation{ + RuleID: "MDL-WIDGET28", + Severity: linter.SeverityError, + Message: fmt.Sprintf( + "%s: widget `%s` has `%s: %s`, which is not an action — the widget is written with "+ + "no action at all and renders as a dead control (Mendix accepts it, so the build stays clean)", + locationPrefix, w.Name, key, renderActionSlotValue(raw)), + Suggestion: actionSlotSuggestion(raw), + }) } - if _, ok := raw.(*ast.ActionV3); ok { - continue - } - out = append(out, linter.Violation{ - RuleID: "MDL-WIDGET28", - Severity: linter.SeverityError, - Message: fmt.Sprintf( - "%s: widget `%s` has `%s: %s`, which is not an action — the widget is written with "+ - "no action at all and renders as a dead control (Mendix accepts it, so the build stays clean)", - locationPrefix, w.Name, key, renderActionSlotValue(raw)), - Suggestion: actionSlotSuggestion(raw), - }) } return out } +// actionSlotSpellings are the property keys on w that spell slot. A value that +// does not parse keeps its key AS WRITTEN, so `onClick: TOTALLY_MADE_UP` — the +// spelling the skills teach — is stored under "onClick", and an exact lookup of +// "OnClick" missed it: the widget was written with no action and nothing said +// so (found with ako/mxcli#1698, whose per-widget rule now leaves these to this +// one). On a built-in widget every case of the slot's name is that slot. On a +// pluggable widget only the exact spelling is: its keys are its package's own, +// and one of them may be an `action` that is not an action slot. +func actionSlotSpellings(w *ast.WidgetV3, slot string) []string { + if _, native := nativeWidgetKinds[strings.ToLower(w.Type)]; !native { + if _, ok := w.Properties[slot]; ok { + return []string{slot} + } + return nil + } + var keys []string + for k := range w.Properties { + if strings.EqualFold(k, slot) { + keys = append(keys, k) + } + } + sort.Strings(keys) + return keys +} + // renderActionSlotValue prints the offending value the way the author wrote it, // so the message can be matched against the source line. func renderActionSlotValue(raw any) string { diff --git a/mdl/executor/validate_widget_embedded_schema_test.go b/mdl/executor/validate_widget_embedded_schema_test.go index 5cee163494..feef105358 100644 --- a/mdl/executor/validate_widget_embedded_schema_test.go +++ b/mdl/executor/validate_widget_embedded_schema_test.go @@ -29,7 +29,7 @@ func noProjectFindings(t *testing.T, script string) []linter.Violation { t.Helper() var out []linter.Violation for _, v := range ValidateProgram(parseMDL(t, script), "") { - if v.RuleID == "MDL-WIDGET07" || v.RuleID == "MDL-WIDGET42" { + if v.RuleID == "MDL-WIDGET07" || v.RuleID == "MDL-WIDGET42" || v.RuleID == nativeWidgetPropDropped.Code { out = append(out, v) } } @@ -150,14 +150,15 @@ func TestCheckWithoutProject_UndeclaredKeyIsNotClaimedToBeDropped(t *testing.T) } } -// A built-in widget has no embedded template; its message is the one it always -// had, so the widgets the vocabulary was written for are judged as before. +// A built-in widget has no embedded template: it is judged by the properties +// its own builder writes (ako/mxcli#1698), not by a data grid's schema. func TestCheckWithoutProject_BuiltInWidgetUnknownKeyIsUnchanged(t *testing.T) { got := noProjectFindings(t, `create page Shop.P ( Title: 'P', Layout: Atlas_Core.Atlas_Default ) { textbox tb (Attribute: Code, columnsHidable: false) };`) - if len(got) != 1 || !strings.Contains(got[0].Message, "is not recognized and will be silently dropped on write") { - t.Errorf("want the unchanged MDL-WIDGET07 message for a text box, got:\n %s", describeFindings(got)) + if len(got) != 1 || got[0].RuleID != nativeWidgetPropDropped.Code || + !strings.Contains(got[0].Message, "`columnsHidable` is not a property mxcli writes on a textbox") { + t.Errorf("want the text box's own verdict (%s), got:\n %s", nativeWidgetPropDropped.Code, describeFindings(got)) } } diff --git a/mdl/executor/validate_widget_language.go b/mdl/executor/validate_widget_language.go index 5ee260212c..8bce3dcdf4 100644 --- a/mdl/executor/validate_widget_language.go +++ b/mdl/executor/validate_widget_language.go @@ -48,9 +48,10 @@ var widgetLanguageChanges = map[string]struct { change langver.Change errorID string }{ - actionSlotRefused.Code: {actionSlotRefused, "MDL-WIDGET37"}, - galleryClickRefused.Code: {galleryClickRefused, "MDL-WIDGET36"}, - tabPagePropDropped.Code: {tabPagePropDropped, "MDL-WIDGET44"}, + actionSlotRefused.Code: {actionSlotRefused, "MDL-WIDGET37"}, + galleryClickRefused.Code: {galleryClickRefused, "MDL-WIDGET36"}, + tabPagePropDropped.Code: {tabPagePropDropped, "MDL-WIDGET44"}, + nativeWidgetPropDropped.Code: {nativeWidgetPropDropped, "MDL-WIDGET52"}, } // GateWidgetViolations applies the script's language version to the widget diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index 0352e74917..29f294e3e4 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -164,6 +164,11 @@ func validateWidgetPropertiesOf(stmt ast.Statement, registry *WidgetRegistry) [] return append(out, validatePageVariableBindings(widgets, documentVariables(stmt), label)...) } switch s := stmt.(type) { + case *ast.CreateLayoutStmt: + // A layout's widgets were checked by no widget rule at all, so a + // property its scroll container does not take passed check and was + // refused — or, before ako/mxcli#1698, dropped — only at exec. + return validateNativeWidgetPropsTree(s.AllWidgets(), nil, nil, nil, registry, "layout "+s.Name.String()) case *ast.AlterPageStmt: var out []linter.Violation for _, op := range s.Operations { @@ -355,7 +360,14 @@ func validateWidgetTreeIn(widgets []*ast.WidgetV3, registry *WidgetRegistry, loc // no definition for the project's own widget — the built-in allow-list // would report every property it has. if def == nil && !isObjectListItem && !w.TypeIsGeneric && explicitWidgetID(w) == "" { - out = append(out, validateStaticWidgetUnknownPropsIn(w, locationPrefix, embedded, registryProjectPath(registry) != "")...) + // A built-in element is judged by the properties ITS builder writes + // (ako/mxcli#1698); the cross-widget vocabulary of MDL-WIDGET07 is + // kept for the keywords without such a set. + if kind := nativeWidgetKindInPosition(w, parent, parentItem, registry, argCtx.known); kind != "" && embedded == nil { + out = append(out, validateNativeWidgetProps(kind, w, locationPrefix)...) + } else { + out = append(out, validateStaticWidgetUnknownPropsIn(w, locationPrefix, embedded, registryProjectPath(registry) != "")...) + } // #1465: the vocabulary above is a cross-widget union, so a key a // tab page cannot store (CaptionParams, Class, Style) passes it. out = append(out, validateTabPageProps(w, locationPrefix)...) @@ -1192,6 +1204,8 @@ var staticWidgetKnownProps = func() map[string]bool { // a scroll-container region's toggle mode, which a sidebartoggle opens // (mendixlabs/mxcli#1220). "ToggleMode", + // a scroll container's Headline / Sidebar frame (ako/mxcli#1703). + "LayoutMode", // a dynamic text's style on a native page (ako/mxcli#1677). "NativeTextStyle", // a layout-grid row's and column's alignment, read by diff --git a/mdl/executor/validate_widgets_explicit_id_test.go b/mdl/executor/validate_widgets_explicit_id_test.go index ca347853bc..a6e8c66bf0 100644 --- a/mdl/executor/validate_widgets_explicit_id_test.go +++ b/mdl/executor/validate_widgets_explicit_id_test.go @@ -18,7 +18,9 @@ func widget07Hits(t *testing.T, body string) int { } n := 0 for _, v := range ValidateWidgetPropertiesForStatement(prog.Statements[0], LoadWidgetRegistry("")) { - if v.RuleID == "MDL-WIDGET07" { + // A built-in widget's own verdict replaced the union allow-list for + // the widgets that have one (ako/mxcli#1698); either is the judge. + if v.RuleID == "MDL-WIDGET07" || v.RuleID == nativeWidgetPropDropped.Code { n++ t.Logf("%s", v.Message) } diff --git a/mdl/executor/widget_native_props.go b/mdl/executor/widget_native_props.go new file mode 100644 index 0000000000..b4a96169db --- /dev/null +++ b/mdl/executor/widget_native_props.go @@ -0,0 +1,563 @@ +// SPDX-License-Identifier: Apache-2.0 + +// The properties each built-in page element takes (ako/mxcli#1698). +// +// `textbox tb (Attribute: Name, Caption: 'Item name')` passed check and exec and +// wrote a text box with no label: a text box takes `Label:`, and `Caption:` is +// a button's. MDL-WIDGET07 judged the key against a vocabulary that is the +// UNION over every built-in widget, so a key valid on some widget passed on all +// of them. That union is how #1307 (an action button's Tooltip), #1465 (a tab +// page's CaptionParams), #1582 (a container's RenderMode) and #1643 each +// reached a user before anyone saw them. +// +// This file replaces the union with one set per element kind: what that kind's +// builder reads off the element, plus what buildWidgetV3's common tail writes +// for it (appearance for every widget, visibility only where the stored $Type +// has ConditionalVisibilitySettings). nativeWidgetPropRefusals is the single +// verdict; check reports it (validateNativeWidgetProps) and exec refuses on it +// (refuseNativeWidgetProps), so the two cannot disagree. +// +// The sets are not a promise kept by hand. TestNativeWidgetProps_MatchBuilderReads +// derives, from the builders' source, every key each one reads off its element +// and fails unless that is exactly the kind's `reads`; TestNativeWidgetProps_TailMatchesGen +// ties `visible` to whether the gen type the writer encodes has a +// ConditionalVisibilitySettings setter; TestNativeWidgetProps_DescribeEmitsOnlyAccepted +// holds describe to the same sets, so describe → exec never refuses its own output. +// +// It is a new rejection, so per ADR-0011 it applies under `mdl 1;`; a script +// without the header keeps the old outcome — the key is dropped — and is told +// so with MDL-V1-WIDGETPROP. Same shape as MDL-V1-TABPAGEPROP (#1465). +package executor + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/mdl/langver" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +var nativeWidgetPropDropped = langver.Change{ + Code: "MDL-V1-WIDGETPROP", + Since: langver.V1, + Old: "A property a built-in page element does not take (`Caption:` on a textbox, `Label:` on a button, " + + "a misspelt key) is accepted and dropped on write", + New: "a refusal (MDL-WIDGET52), with nothing written", +} + +// nativeWidgetKind is one built-in page element: a widget keyword, or a child +// element its parent's builder reads directly (a scroll container's region, a +// layout grid's row). +type nativeWidgetKind struct { + // stored is the $Type the element is written as. + stored string + // reads are the keys its builder reads off the element — exactly, see + // TestNativeWidgetProps_MatchBuilderReads. + reads []string + // tail: built by buildWidgetV3, whose common tail writes Class, Style, + // DynamicClasses and DesignProperties (every widget has an Appearance). + tail bool + // visible: the tail also writes Visible — the stored type has + // ConditionalVisibilitySettings. A scroll container, placeholder, menu + // widget or snippet call does not, so `Visible:` on one was dropped. + visible bool +} + +var ( + appearanceKeys = []string{"Class", "Style", "DynamicClasses", "DesignProperties"} + // The visitor lowers `Visible:` by shape: an expression to VisibleIf, a + // plain value to Visible, `Attr in (…)` to VisibleWhen, `roles (…)` to + // VisibleRoles. One property, four keys. + visibilityKeys = []string{"Visible", "VisibleIf", "VisibleWhen", "VisibleRoles"} +) + +// inputReads are what every attribute-bound input's builder reads. +var inputReads = []string{"Attribute", "Label", "ShowLabel", "OnChange"} + +func withReads(base []string, more ...string) []string { + return append(append([]string(nil), base...), more...) +} + +// nativeWidgetKinds is keyed by kind: the lower-case keyword, or " +// " for an element its parent builds (nativeWidgetKindOf). +var nativeWidgetKinds = map[string]nativeWidgetKind{ + // Data and list widgets. `Footer` is the legacy property form of a data + // view's footer block, still read by buildDataViewV3. + "dataview": {stored: "Forms$DataView", tail: true, visible: true, + reads: []string{"DataSource", "FormOrientation", "LabelWidth", "ReadOnlyStyle", "ShowFooter", "Footer"}}, + "listview": {stored: "Forms$ListView", tail: true, visible: true, + reads: []string{"DataSource", "Action", "Editable", "PageSize"}}, + + // Inputs. + "textbox": {stored: "Forms$TextBox", tail: true, visible: true, + reads: withReads(inputReads, "Placeholder", "ReadOnlyStyle", "Password", "Validation", "ValidationMessage", + "SubmitBehaviour", "SubmitOnInputDelay")}, + "textarea": {stored: "Forms$TextArea", tail: true, visible: true, + reads: withReads(inputReads, "Placeholder", "ReadOnlyStyle", "NumberOfLines", "Autocomplete", + "SubmitBehaviour", "SubmitOnInputDelay")}, + "datepicker": {stored: "Forms$DatePicker", tail: true, visible: true, + reads: withReads(inputReads, "ReadOnlyStyle")}, + "dropdown": {stored: "Forms$DropDown", tail: true, visible: true, reads: withReads(inputReads)}, + "checkbox": {stored: "Forms$CheckBox", tail: true, visible: true, reads: withReads(inputReads, "ReadOnlyStyle")}, + "radiobuttons": {stored: "Forms$RadioButtonGroup", tail: true, visible: true, reads: withReads(inputReads, "ReadOnlyStyle")}, + "filemanager": {stored: "Forms$FileManager", tail: true, visible: true, + reads: []string{"Label", "ShowLabel", "Type", "MaxFileSize", "AllowedExtensions", "ShowFileInBrowser"}}, + + // Text. + "dynamictext": {stored: "Forms$DynamicText", tail: true, visible: true, + reads: []string{"Content", "ContentParams", "Attribute", "RenderMode", "NativeTextStyle"}}, + "title": {stored: "Forms$Title", tail: true, visible: true, reads: []string{"Content"}}, + "label": {stored: "Forms$Label", tail: true, visible: true, reads: []string{"Content"}}, + + // Buttons. `ContentParams:` is what describe printed for a button's caption + // parameters before #632, so it is still read. + "actionbutton": {stored: "Forms$ActionButton", tail: true, visible: true, reads: buttonReads}, + "button": {stored: "Forms$ActionButton", tail: true, visible: true, reads: buttonReads}, + "linkbutton": {stored: "Forms$ActionButton", tail: true, visible: true, reads: buttonReads}, + "sidebartoggle": {stored: "Forms$SidebarToggleButton", tail: true, visible: true, + reads: []string{"Caption", "CaptionParams", "ContentParams", "Tooltip", "ButtonStyle", "Icon", "RenderMode"}}, + + // Structure. + "container": {stored: "Forms$DivContainer", tail: true, visible: true, reads: []string{"Action", "RenderMode"}}, + "customcontainer": {stored: "Forms$DivContainer", tail: true, visible: true, reads: []string{"Action", "RenderMode"}}, + "groupbox": {stored: "Forms$GroupBox", tail: true, visible: true, + reads: []string{"Caption", "Collapsible", "HeaderMode"}}, + "tabcontainer": {stored: "Forms$TabControl", tail: true, visible: true}, + "layoutgrid": {stored: "Forms$LayoutGrid", tail: true, visible: true, reads: []string{"Width"}}, + // A top-level `row` / `column` is a container holding a one-row layout + // grid: the tail writes the container, the row's or column's own builder + // reads the rest. + "row": {stored: "Forms$DivContainer", tail: true, visible: true, reads: layoutGridRowReads}, + "column": {stored: "Forms$DivContainer", tail: true, visible: true, + reads: withReads(appearanceKeys, "VerticalAlignment", "DesktopWidth", "TabletWidth", "PhoneWidth")}, + "navigationlist": {stored: "Forms$NavigationList", tail: true, visible: true}, + "snippetcall": {stored: "Forms$SnippetCallWidget", tail: true, reads: []string{"Snippet", "Params"}}, + // Containers a list or data view's body is split into. + "footer": {stored: "Forms$DivContainer", tail: true, visible: true}, + "header": {stored: "Forms$DivContainer", tail: true, visible: true}, + "controlbar": {stored: "Forms$DivContainer", tail: true, visible: true}, + "template": {stored: "Forms$DivContainer", tail: true, visible: true}, + "filter": {stored: "Forms$DivContainer", tail: true, visible: true}, + + // Images. + "staticimage": {stored: "Forms$StaticImageViewer", tail: true, visible: true, + reads: []string{"Image", "Action", "Width", "Height", "WidthUnit", "HeightUnit", "Responsive"}}, + "dynamicimage": {stored: "Forms$ImageViewer", tail: true, visible: true, + reads: []string{"DataSource", "DefaultImage", "DisplayAs", "OnClickType", "Action", + "Width", "Height", "WidthUnit", "HeightUnit", "Responsive"}}, + + // Layout elements. + "scrollcontainer": {stored: "Forms$ScrollContainer", tail: true, reads: []string{"LayoutMode"}}, + "layoutheader": {stored: "Forms$Header", tail: true}, + "placeholder": {stored: "Forms$Placeholder", tail: true}, + "navigationtree": {stored: "Forms$NavigationTree", tail: true, reads: []string{"Menu", "Profile"}}, + "menubar": {stored: "Forms$MenuBar", tail: true, reads: []string{"Menu", "Profile"}}, + "simplemenubar": {stored: "Forms$SimpleMenuBar", tail: true, reads: []string{"Menu", "Profile", "Orientation"}}, + + // Elements a parent builds itself, so buildWidgetV3's tail never runs. + "scrollcontainer region": {stored: "Forms$ScrollContainerRegion", + reads: []string{"Class", "Size", "SizeMode", "ToggleMode"}}, + "layoutheader region": {stored: "Forms$Header"}, + "layoutgrid row": {stored: "Forms$LayoutGridRow", reads: layoutGridRowReads}, + "layoutgrid column": {stored: "Forms$LayoutGridColumn", + reads: withReads(appearanceKeys, "VerticalAlignment", "DesktopWidth", "TabletWidth", "PhoneWidth")}, + "navigationlist item": {stored: "Forms$NavigationListItem", + reads: []string{"Caption", "Action", "Class", "Style", "DynamicClasses", "Visible", "VisibleIf", "VisibleWhen", "VisibleRoles"}}, + "dataview footer": {stored: "Forms$DataView"}, + "listview template": {stored: "Forms$ListViewTemplate"}, +} + +var buttonReads = []string{"Caption", "CaptionParams", "ContentParams", "Tooltip", "ButtonStyle", "Icon", "Action"} + +var layoutGridRowReads = withReads(appearanceKeys, visibilityKeys[0], visibilityKeys[1], visibilityKeys[2], visibilityKeys[3], + "VerticalAlignment", "HorizontalAlignment", "SpacingBetweenColumns") + +// nativeChildKinds are the elements a parent's builder reads itself, keyed by +// " ". +var nativeChildKinds = map[string]string{ + "scrollcontainer region": "scrollcontainer region", + "layoutheader region": "layoutheader region", + "layoutgrid row": "layoutgrid row", + "row column": "layoutgrid column", + "navigationlist item": "navigationlist item", + "dataview footer": "dataview footer", +} + +// nativeWidgetKindOf is the kind w is built as when it sits under parent (nil +// at the document root), or "" when it is not a built-in element mxcli builds +// — a pluggable widget, a fragment sentinel, a keyword with no builder. +func nativeWidgetKindOf(w, parent *ast.WidgetV3) string { + if w == nil { + return "" + } + t := strings.ToLower(w.Type) + if parent != nil { + p := strings.ToLower(parent.Type) + if k, ok := nativeChildKinds[p+" "+t]; ok { + return k + } + // `template for Module.Entity { … }` is a list view's specialization. + if p == "listview" && w.Specialization != "" { + return "listview template" + } + } + if _, ok := nativeWidgetKinds[t]; ok { + return t + } + return "" +} + +// nativeElementKeywords are the keywords that name a built-in element in some +// position: every kind's keyword and every child a native parent builds. +var nativeElementKeywords = func() map[string]bool { + m := map[string]bool{"tabpage": true, "region": true, "item": true} + for k := range nativeWidgetKinds { + if !strings.Contains(k, " ") { + m[k] = true + } + } + return m +}() + +// nativePositionalKeywords are keywords whose meaning depends on the parent: a +// `column` is a layout-grid column under a row, a data grid's column under a +// data grid, and a container shorthand elsewhere; a `template` is a list +// view's specialization, a gallery's item slot or a container. +var nativePositionalKeywords = map[string]bool{ + "column": true, "row": true, "item": true, "region": true, + "template": true, "filter": true, "header": true, "footer": true, "controlbar": true, +} + +// nativeWidgetKindInPosition is nativeWidgetKindOf for the check-time walk. +// A positional keyword is judged only where its parent is known and is a +// built-in element: under a pluggable widget or an object-list item it is that +// widget's column, item or slot, which the engine reads and this file does not +// judge; at the root of an ALTER … INSERT (enclosureKnown false) the parent is +// out of sight. +func nativeWidgetKindInPosition(w, parent *ast.WidgetV3, parentItem *ObjectListMapping, registry *WidgetRegistry, enclosureKnown bool) string { + if w == nil { + return "" + } + if nativePositionalKeywords[strings.ToLower(w.Type)] { + if parent == nil && !enclosureKnown { + return "" + } + if parent != nil && !isNativeParent(parent, parentItem, registry) { + return "" + } + } + return nativeWidgetKindOf(w, parent) +} + +// isNativeParent reports whether parent is a built-in element, whose children +// buildWidgetV3 or the parent's own builder reads — not a pluggable widget, an +// object-list item, or a keyword with no builder. +func isNativeParent(parent *ast.WidgetV3, parentItem *ObjectListMapping, registry *WidgetRegistry) bool { + if parentItem != nil || parent.TypeIsGeneric || explicitWidgetID(parent) != "" || + isUniversalObjectListKeyword(parent.Type) || lookupWidgetDef(parent, registry) != nil { + return false + } + return nativeElementKeywords[strings.ToLower(parent.Type)] +} + +// nativeWidgetAccepts reports whether a kind takes key, case-insensitively. +func nativeWidgetAccepts(kind, keyword, key string) bool { + k, ok := nativeWidgetKinds[kind] + if !ok { + return true + } + for _, set := range nativeWidgetKeySets(k) { + for _, a := range set { + if strings.EqualFold(a, key) { + return true + } + } + } + // The table-driven reads: a date picker's or text box's FormattingInfo, + // a text box's or text area's submit settings. Their builders read the + // keys from those tables, not by literal. + if kind == keyword { + if _, ok := inputFormatProp(keyword, key); ok { + return true + } + } + // A spelling of a key this kind takes, kept as written because its value + // did not parse (nativeWidgetKeyAliases). + if target, ok := nativeWidgetKeyAliases[strings.ToLower(key)]; ok { + return nativeWidgetAccepts(kind, keyword, target) + } + return false +} + +// nativeWidgetKeyAliases are spellings the visitor stores under another key — +// but only when the value is well formed. `onClick: close page` is stored as +// "Action" (#603, actionKeywordSource); `onClick: TOTALLY_MADE_UP` does not +// parse as an action, falls through to the generic `key: value` branch and +// keeps its key as written. That is not a property the widget lacks: it is +// the widget's click action holding something that is not an action, which +// MDL-WIDGET28 reports. Refusing it here as well named the wrong problem, and — +// this rule being gated on the header and that one not — made `mdl 1;` change +// what the script writes (ako/mxcli#1698, the upgrade property test on +// action-slot-not-an-action-1062.fail.mdl). Keyed lower-case: the alias, and +// the key it is stored under when it parses. An alias is not a builder read; +// TestNativeWidgetProps_AliasesFollowTheVisitor ties it to actionKeywordSource. +// +// The visitor renames no other key this way: every other dedicated branch +// stores under the key's own name (case aside, which nativeWidgetAccepts +// ignores), and `Visible:` / `Editable:` lower to keys judged as one group. +var nativeWidgetKeyAliases = map[string]string{"onclick": "Action"} + +func nativeWidgetKeySets(k nativeWidgetKind) [][]string { + sets := [][]string{k.reads} + if k.tail { + sets = append(sets, appearanceKeys) + if k.visible { + sets = append(sets, visibilityKeys) + } + } + return sets +} + +// nativeWidgetAcceptedNames lists a kind's keys for messages, sorted and +// without the internal spellings of Visible. +func nativeWidgetAcceptedNames(kind, keyword string) []string { + k := nativeWidgetKinds[kind] + seen := map[string]bool{} + var out []string + add := func(n string) { + if l := strings.ToLower(n); !seen[l] { + seen[l] = true + out = append(out, n) + } + } + for _, set := range nativeWidgetKeySets(k) { + for _, n := range set { + switch n { + case "VisibleIf", "VisibleWhen", "VisibleRoles", "Footer": + continue // spellings of Visible, and a legacy form + } + add(n) + } + } + if kind == keyword { + for _, n := range inputFormatProps[keyword] { + add(n) + } + // Judged by MDL-WIDGET20 rather than by the set, but taken all the same. + // (plainEditableWidgetTypes' `column` is a data grid's, not this kind.) + if editableWidgetTypes[keyword] || keyword == "listview" { + add("Editable") + } + } + sort.Strings(out) + return out +} + +// nativeWidgetKeyOwnedElsewhere reports keys a narrower rule already judges on +// this kind, so they are not reported twice: editability (MDL-WIDGET20), a +// click action on a widget Mendix gives none (MDL-WIDGET23), a snippet call's +// appearance (#1511), a layout-grid column's Visible (#1461) and a dynamic +// text's per-parameter format keys (MDL-WIDGET18). +func nativeWidgetKeyOwnedElsewhere(kind, keyword, key string) bool { + l := strings.ToLower(key) + switch l { + case "editable", "editableif": + return true + case "action", "onclick": + if clickDroppedNoSlot[keyword] || clickCapableInMendix[keyword] { + return true + } + } + switch kind { + case "snippetcall": + for _, a := range appearanceKeys { + if strings.EqualFold(a, key) { + return true + } + } + case "layoutgrid column": + for _, v := range visibilityKeys { + if strings.EqualFold(v, key) { + return true + } + } + case "dynamictext": + return paramFormatKeys[l] + } + return false +} + +// nativeWidgetPropRefusal is one key a built-in element does not take. +type nativeWidgetPropRefusal struct { + Key string + Hint string // a cross-widget mix-up or a "did you mean", "" when neither +} + +// nativeWidgetPropRefusals is THE verdict: the keys on w, built as kind, that +// nothing writes. Check and exec both call it. +func nativeWidgetPropRefusals(kind string, w *ast.WidgetV3) []nativeWidgetPropRefusal { + if w == nil || kind == "" { + return nil + } + if _, ok := nativeWidgetKinds[kind]; !ok { + return nil + } + keyword := strings.ToLower(w.Type) + var out []nativeWidgetPropRefusal + for _, key := range sortedPropertyKeys(w) { + if nativeWidgetAccepts(kind, keyword, key) || nativeWidgetKeyOwnedElsewhere(kind, keyword, key) { + continue + } + out = append(out, nativeWidgetPropRefusal{Key: key, Hint: nativeWidgetPropHint(kind, keyword, key)}) + } + return out +} + +// Kinds by what they are, for the cross-widget hints. +var ( + nativeInputKinds = map[string]bool{"textbox": true, "textarea": true, "datepicker": true, "dropdown": true, "checkbox": true, "radiobuttons": true, "filemanager": true} + nativeButtonKinds = map[string]bool{"actionbutton": true, "button": true, "linkbutton": true, "sidebartoggle": true} + nativeTextKinds = map[string]bool{"dynamictext": true, "title": true, "label": true} +) + +// nativeWidgetPropHint names what the author most likely meant: the property +// this kind uses for the thing the key means on another widget, or the +// nearest key it takes. +func nativeWidgetPropHint(kind, keyword, key string) string { + switch strings.ToLower(key) { + case "caption", "captionparams": + switch { + case nativeInputKinds[kind]: + return "an input widget's label is `Label:` (`Caption:` is a button's and a column's)" + case nativeTextKinds[kind]: + return "its text is `Content:` (with `ContentParams:`)" + } + case "label": + switch { + case nativeButtonKinds[kind]: + return "a button's text is `Caption:` (`Label:` is an input widget's)" + case kind == "groupbox" || kind == "navigationlist item": + return "its text is `Caption:`" + case nativeTextKinds[kind]: + return "its text is `Content:`" + } + case "content", "contentparams": + switch { + case nativeInputKinds[kind]: + return "an input widget's label is `Label:`" + case kind == "groupbox" || kind == "navigationlist item": + return "its text is `Caption:`" + } + case "tooltip": + return "only a button stores a tooltip" + case "action", "onclick": + return fmt.Sprintf("%s has no click action — put it on a `container` inside it, or use an `actionbutton`", + nativeWidgetKinds[kind].stored) + case "onchange": + return "only an input widget has an on-change action" + case "datasource": + return "put the widget inside a `dataview` with that data source" + case "visible", "visibleif", "visiblewhen", "visibleroles": + k := nativeWidgetKinds[kind] + return fmt.Sprintf("%s has no conditional visibility — wrap it in a `container` with Visible:", k.stored) + case "class", "style", "dynamicclasses", "designproperties": + if nativeWidgetAccepts(kind, keyword, "Class") { + return "of its appearance only `Class:` is written" + } + return "none of its appearance is written — style a widget inside it" + } + if near := nearestKey(key, nativeWidgetAcceptedNames(kind, keyword)); near != "" { + return fmt.Sprintf("did you mean `%s`?", near) + } + return "" +} + +// nativeWidgetPropMessage is the shared wording of a refusal. +func nativeWidgetPropMessage(kind string, w *ast.WidgetV3, r nativeWidgetPropRefusal) string { + what := strings.ToLower(w.Type) + if strings.Contains(kind, " ") { + what = kind + } + name := "" + if w.Name != "" { + name = " `" + w.Name + "`" + } + msg := fmt.Sprintf("%s%s: `%s` is not a property mxcli writes on a %s (%s), so it would be dropped", + what, name, r.Key, what, nativeWidgetKinds[kind].stored) + if r.Hint != "" { + msg += " — " + r.Hint + } + return msg +} + +func nativeWidgetPropSuggestion(kind string, w *ast.WidgetV3) string { + names := nativeWidgetAcceptedNames(kind, strings.ToLower(w.Type)) + if len(names) == 0 { + return "It takes no properties: remove them." + } + return "Properties it takes: " + strings.Join(names, ", ") + "." +} + +// validateNativeWidgetProps is the check-time half (MDL-V1-WIDGETPROP, gated by +// GateWidgetViolations to MDL-WIDGET52 under mdl 1). +func validateNativeWidgetProps(kind string, w *ast.WidgetV3, locationPrefix string) []linter.Violation { + var out []linter.Violation + for _, r := range nativeWidgetPropRefusals(kind, w) { + out = append(out, linter.Violation{ + RuleID: nativeWidgetPropDropped.Code, + Severity: linter.SeverityError, + Message: fmt.Sprintf("%s: %s", locationPrefix, nativeWidgetPropMessage(kind, w, r)), + Suggestion: nativeWidgetPropSuggestion(kind, w), + }) + } + return out +} + +// validateNativeWidgetPropsTree applies validateNativeWidgetProps to a widget +// tree that no other widget rule walks — a layout's. Pluggable widgets and +// their object-list items are skipped, as validateWidgetTreeIn skips them. +func validateNativeWidgetPropsTree(widgets []*ast.WidgetV3, parent *ast.WidgetV3, parentItem *ObjectListMapping, + parentLists map[string]*ObjectListMapping, registry *WidgetRegistry, locationPrefix string) []linter.Violation { + var out []linter.Violation + for _, w := range widgets { + if w == nil { + continue + } + mapping := parentLists[strings.ToUpper(w.Type)] + def := lookupWidgetDef(w, registry) + if mapping == nil && def == nil && !w.TypeIsGeneric && explicitWidgetID(w) == "" && !isUniversalObjectListKeyword(w.Type) { + kind := nativeWidgetKindInPosition(w, parent, parentItem, registry, true) + out = append(out, validateNativeWidgetProps(kind, w, locationPrefix)...) + } + out = append(out, validateNativeWidgetPropsTree(w.Children, w, mapping, objectListMappingSet(def), registry, locationPrefix)...) + } + return out +} + +// refuseNativeWidgetProps is the exec-time half, for `exec --no-check` and the +// paths no check pass sees: an error under mdl 1, a warning under mdl 0 (the +// keys are then dropped, as they always were). +func refuseNativeWidgetProps(ctx *ExecContext, kind string, w *ast.WidgetV3) error { + refusals := nativeWidgetPropRefusals(kind, w) + if len(refusals) == 0 { + return nil + } + version := langver.V0 + if ctx != nil { + version = ctx.LanguageVersion + } + if nativeWidgetPropDropped.Applies(version) { + return mdlerrors.NewValidationf("%s, so nothing was written [MDL-WIDGET52]. %s", + nativeWidgetPropMessage(kind, w, refusals[0]), nativeWidgetPropSuggestion(kind, w)) + } + if ctx != nil { + for _, r := range refusals { + fmt.Fprintf(ctx.progress(), "Warning [%s]: %s; not written. %s\n", + nativeWidgetPropDropped.Code, nativeWidgetPropMessage(kind, w, r), nativeWidgetPropDropped.Warning(version)) + } + } + return nil +} diff --git a/mdl/executor/widget_native_props_source_test.go b/mdl/executor/widget_native_props_source_test.go new file mode 100644 index 0000000000..7a32151665 --- /dev/null +++ b/mdl/executor/widget_native_props_source_test.go @@ -0,0 +1,694 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + goast "go/ast" + "go/parser" + "go/token" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/modelsdk/element" + genPg "github.com/mendixlabs/mxcli/modelsdk/gen/pages" +) + +// The per-kind property sets in widget_native_props.go are only as good as +// their agreement with the builders. A set that is missing a key the builder +// reads refuses a property that IS written; a set holding a key nothing reads +// accepts one that is dropped — the bug class itself. So the sets are checked +// against the source, not against a second hand-typed list (the lesson of +// mendixlabs/mxcli#1346, where the guard drifted with the list it guarded). + +// builderSource is the executor package's non-test source, indexed for a +// taint walk: which keys does a function read off the *ast.WidgetV3 it is +// given, directly or through the functions it hands that widget to? +type builderSource struct { + fset *token.FileSet + funcs map[string]*builderFunc // "pageBuilder.buildTextBoxV3", "inputLabel" + names map[string][]string // bare name -> keys of funcs +} + +type builderFunc struct { + decl *goast.FuncDecl + widgets map[int]string // index of each *ast.WidgetV3 parameter -> its name + keyArgs map[int]bool // string parameters the function uses as a property key +} + +// widgetGetterKeys are ast.WidgetV3's typed getters and the key each reads. +var widgetGetterKeys = map[string]string{ + "GetDataSource": "DataSource", "GetAction": "Action", "GetOnChange": "OnChange", "GetPlaceholder": "Placeholder", + "GetAttribute": "Attribute", "GetBinds": "Binds", "GetLabel": "Label", "GetCaption": "Caption", "GetContent": "Content", + "GetRenderMode": "RenderMode", "GetButtonStyle": "ButtonStyle", "GetClass": "Class", "GetStyle": "Style", + "GetDynamicClasses": "DynamicClasses", "GetDesktopWidth": "DesktopWidth", "GetContentParams": "ContentParams", + "GetCaptionParams": "CaptionParams", "GetAttributes": "Attributes", "GetFilterType": "FilterType", "GetAttr": "Attr", + "GetSnippet": "Snippet", "GetSnippetParams": "Params", "GetSelection": "Selection", "GetDesignProperties": "DesignProperties", +} + +// messageOnlyReaders read a key to NAME the widget in an error, not to write +// it, so their reads are not consumption. widgetRefForMessage falls back to +// the Caption when a widget has no name — which, counted, made every input +// "read" Caption, the very key #1698 is about. +// refuseNativeWidgetProps is the verdict itself: it reads every key to judge +// it, not to write it. +var messageOnlyReaders = map[string]bool{"widgetRefForMessage": true, "refuseNativeWidgetProps": true} + +// tableDrivenReaders read keys from a per-type table rather than by literal; +// nativeWidgetAccepts consults the same table (inputFormatProp). +var tableDrivenReaders = map[string]bool{"inputFormattingInfo": true, "inputFormattingProblems": true} + +func loadBuilderSource(t *testing.T) *builderSource { + t.Helper() + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + src := &builderSource{fset: token.NewFileSet(), funcs: map[string]*builderFunc{}, names: map[string][]string{}} + for _, path := range files { + if strings.HasSuffix(path, "_test.go") { + continue + } + f, err := parser.ParseFile(src.fset, path, nil, 0) + if err != nil { + t.Fatal(err) + } + for _, d := range f.Decls { + fd, ok := d.(*goast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + bf := &builderFunc{decl: fd, widgets: map[int]string{}, keyArgs: map[int]bool{}} + strParams := map[string]int{} + i := 0 + for _, field := range fd.Type.Params.List { + for _, n := range field.Names { + if isWidgetV3Type(field.Type) { + bf.widgets[i] = n.Name + } + if id, ok := field.Type.(*goast.Ident); ok && id.Name == "string" { + strParams[n.Name] = i + } + i++ + } + if len(field.Names) == 0 { + i++ + } + } + if len(bf.widgets) == 0 { + continue + } + goast.Inspect(fd.Body, func(n goast.Node) bool { + switch x := n.(type) { + case *goast.IndexExpr: + if sel, ok := x.X.(*goast.SelectorExpr); ok && sel.Sel.Name == "Properties" { + if id, ok := x.Index.(*goast.Ident); ok { + if j, ok := strParams[id.Name]; ok { + bf.keyArgs[j] = true + } + } + } + case *goast.CallExpr: + name := calleeName(x) + if !isKeyAccessor(name) { + break + } + for _, a := range x.Args { + if id, ok := a.(*goast.Ident); ok { + if j, ok := strParams[id.Name]; ok { + bf.keyArgs[j] = true + } + } + } + } + return true + }) + key := fd.Name.Name + if r := receiverTypeName(fd); r != "" { + key = r + "." + key + } + src.funcs[key] = bf + src.names[fd.Name.Name] = append(src.names[fd.Name.Name], key) + } + } + return src +} + +func isWidgetV3Type(e goast.Expr) bool { + st, ok := e.(*goast.StarExpr) + if !ok { + return false + } + sel, ok := st.X.(*goast.SelectorExpr) + return ok && sel.Sel.Name == "WidgetV3" +} + +func receiverTypeName(fd *goast.FuncDecl) string { + if fd.Recv == nil || len(fd.Recv.List) == 0 { + return "" + } + t := fd.Recv.List[0].Type + if st, ok := t.(*goast.StarExpr); ok { + t = st.X + } + if id, ok := t.(*goast.Ident); ok { + return id.Name + } + return "" +} + +func calleeName(c *goast.CallExpr) string { + switch f := c.Fun.(type) { + case *goast.Ident: + return f.Name + case *goast.SelectorExpr: + return f.Sel.Name + } + return "" +} + +func isKeyAccessor(name string) bool { + return (strings.HasPrefix(name, "Get") && strings.HasSuffix(name, "Prop")) || name == "lookupPropCI" || name == "ExpressionSource" +} + +func stringLit(e goast.Expr) (string, bool) { + bl, ok := e.(*goast.BasicLit) + if !ok || bl.Kind != token.STRING { + return "", false + } + s, err := strconv.Unquote(bl.Value) + return s, err == nil +} + +// builderReads is what one taint walk found. +type builderReads struct { + keys map[string]bool + dynamic map[string]bool // functions reading a key that is not a literal + unknown map[string]bool // calls passing the widget to a function not found +} + +// reads walks fn from its widget parameter(s). +func (s *builderSource) reads(fn string) builderReads { + r := builderReads{keys: map[string]bool{}, dynamic: map[string]bool{}, unknown: map[string]bool{}} + visited := map[string]bool{} + bf := s.funcs[fn] + if bf == nil { + r.unknown[fn] = true + return r + } + for idx := range bf.widgets { + s.walk(fn, idx, &r, visited) + } + return r +} + +func (s *builderSource) walk(fn string, idx int, r *builderReads, visited map[string]bool) { + vk := fn + "#" + strconv.Itoa(idx) + if visited[vk] { + return + } + visited[vk] = true + bf := s.funcs[fn] + if bf == nil || messageOnlyReaders[bf.decl.Name.Name] { + return + } + if tableDrivenReaders[bf.decl.Name.Name] { + r.dynamic[bf.decl.Name.Name] = true + return + } + w := map[string]bool{bf.widgets[idx]: true} + recv := "" + if bf.decl.Recv != nil && len(bf.decl.Recv.List) > 0 && len(bf.decl.Recv.List[0].Names) > 0 { + recv = bf.decl.Recv.List[0].Names[0].Name + } + // `for _, key := range []string{"Icon", "icon"}` — literal keys by variable. + litVars := map[string][]string{} + goast.Inspect(bf.decl.Body, func(n goast.Node) bool { + if as, ok := n.(*goast.AssignStmt); ok && len(as.Lhs) == 1 && len(as.Rhs) == 1 { + if id, ok := as.Rhs[0].(*goast.Ident); ok && w[id.Name] { + if l, ok := as.Lhs[0].(*goast.Ident); ok { + w[l.Name] = true + } + } + } + if rs, ok := n.(*goast.RangeStmt); ok { + if cl, ok := rs.X.(*goast.CompositeLit); ok { + if v, ok := rs.Value.(*goast.Ident); ok { + for _, e := range cl.Elts { + if lit, ok := stringLit(e); ok { + litVars[v.Name] = append(litVars[v.Name], lit) + } + } + } + } + } + return true + }) + isW := func(e goast.Expr) bool { + id, ok := e.(*goast.Ident) + return ok && w[id.Name] + } + keyOf := func(e goast.Expr) ([]string, bool) { + if lit, ok := stringLit(e); ok { + return []string{lit}, true + } + if id, ok := e.(*goast.Ident); ok && len(litVars[id.Name]) > 0 { + return litVars[id.Name], true + } + return nil, false + } + goast.Inspect(bf.decl.Body, func(n goast.Node) bool { + switch x := n.(type) { + case *goast.RangeStmt: + if sel, ok := x.X.(*goast.SelectorExpr); ok && sel.Sel.Name == "Properties" && isW(sel.X) { + r.dynamic[fn] = true + } + case *goast.IndexExpr: + if sel, ok := x.X.(*goast.SelectorExpr); ok && sel.Sel.Name == "Properties" && isW(sel.X) { + if ks, ok := keyOf(x.Index); ok { + for _, k := range ks { + r.keys[k] = true + } + } else if len(bf.keyArgs) == 0 { + r.dynamic[fn] = true + } + } + case *goast.CallExpr: + if sel, ok := x.Fun.(*goast.SelectorExpr); ok && isW(sel.X) { + if k, ok := widgetGetterKeys[sel.Sel.Name]; ok { + r.keys[k] = true + } + if isKeyAccessor(sel.Sel.Name) && len(x.Args) >= 1 { + if ks, ok := keyOf(x.Args[0]); ok { + for _, k := range ks { + r.keys[k] = true + } + } else if len(bf.keyArgs) == 0 { + r.dynamic[fn] = true + } + } + } + for i, a := range x.Args { + if !isW(a) { + continue + } + found := false + for _, target := range s.resolve(x, recv) { + tf := s.funcs[target] + if tf == nil { + continue + } + if _, ok := tf.widgets[i]; !ok { + continue + } + found = true + if len(tf.keyArgs) > 0 { + // A key accessor: the read is the key this call passes. + for j, b := range x.Args { + if !tf.keyArgs[j] { + continue + } + if ks, ok := keyOf(b); ok { + for _, k := range ks { + r.keys[k] = true + } + } else { + r.dynamic[fn] = true + } + } + continue + } + s.walk(target, i, r, visited) + } + if !found { + r.unknown[calleeName(x)] = true + } + } + } + return true + }) +} + +func (s *builderSource) resolve(c *goast.CallExpr, recv string) []string { + switch f := c.Fun.(type) { + case *goast.Ident: + if _, ok := s.funcs[f.Name]; ok { + return []string{f.Name} + } + case *goast.SelectorExpr: + if id, ok := f.X.(*goast.Ident); ok && recv != "" && id.Name == recv { + for _, k := range s.names[f.Sel.Name] { + if strings.Contains(k, ".") { + return []string{k} + } + } + } + return s.names[f.Sel.Name] + } + return nil +} + +// dispatch reads buildWidgetV3's switch: keyword -> the builder it calls. +func (s *builderSource) dispatch(t *testing.T) map[string]string { + t.Helper() + bw := s.funcs["pageBuilder.buildWidgetV3"] + if bw == nil { + t.Fatal("pageBuilder.buildWidgetV3 not found") + } + out := map[string]string{} + goast.Inspect(bw.decl.Body, func(n goast.Node) bool { + cc, ok := n.(*goast.CaseClause) + if !ok { + return true + } + callee := "" + for _, stmt := range cc.Body { + goast.Inspect(stmt, func(m goast.Node) bool { + if c, ok := m.(*goast.CallExpr); ok && callee == "" { + if sel, ok := c.Fun.(*goast.SelectorExpr); ok && strings.HasPrefix(sel.Sel.Name, "build") { + callee = "pageBuilder." + sel.Sel.Name + } + } + return true + }) + } + for _, e := range cc.List { + if lit, ok := stringLit(e); ok { + out[lit] = callee + } + } + return true + }) + if len(out) < 30 { + t.Fatalf("found only %d keywords in buildWidgetV3's switch — the scan no longer matches its shape", len(out)) + } + return out +} + +// nativeChildBuilders are the builders of the elements a parent reads itself. +// "layoutheader region" and "dataview footer" are read inline by their parent +// (only their children are used), so they have none and take nothing. +var nativeChildBuilders = map[string]string{ + "scrollcontainer region": "pageBuilder.buildScrollContainerRegionV3", + "layoutgrid row": "pageBuilder.buildLayoutGridRowV3", + "layoutgrid column": "pageBuilder.buildLayoutGridColumnV3", + "navigationlist item": "pageBuilder.buildNavigationListItemV3", + "listview template": "pageBuilder.buildListViewTemplateV3", +} + +// dispatchedWithoutKind are buildWidgetV3 keywords with no property set, and why. +var dispatchedWithoutKind = map[string]string{ + "text": "refused outright: Forms$Text is not a Mendix type", + "statictext": "refused outright: Forms$Text is not a Mendix type", + "image": "the pluggable image widget; the static fallback is used only with no engine", + "legacydatagrid": "refused outright: no writer", + "tabpage": "refused outside a tab container; judged there by MDL-WIDGET44 (#1465)", + "region": "refused outside a scroll container or layout header; judged there", + "item": "refused outside a navigation list; judged there", + "slot": "refused outside a fragment body", +} + +func lowerSet(keys []string) map[string]bool { + m := map[string]bool{} + for _, k := range keys { + m[strings.ToLower(k)] = true + } + return m +} + +func sortedNameSet(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// Every key a kind's builder reads is in its `reads`, and every key in its +// `reads` is read by the builder — exactly, both ways. +func TestNativeWidgetProps_MatchBuilderReads(t *testing.T) { + src := loadBuilderSource(t) + dispatch := src.dispatch(t) + + for keyword, fn := range dispatch { + if _, ok := nativeWidgetKinds[keyword]; !ok && dispatchedWithoutKind[keyword] == "" { + t.Errorf("buildWidgetV3 builds `%s` (%s) but nativeWidgetKinds has no entry for it — add its "+ + "property set, or list it in dispatchedWithoutKind with the reason", keyword, fn) + } + } + + for kind, k := range nativeWidgetKinds { + fn := nativeChildBuilders[kind] + if !strings.Contains(kind, " ") { + fn = dispatch[kind] + } + want := lowerSet(k.reads) + if fn == "" { + if len(want) > 0 { + t.Errorf("%s: no builder found, but reads lists %v", kind, k.reads) + } + continue + } + r := src.reads(fn) + if len(r.unknown) > 0 { + t.Errorf("%s (%s): passes the widget to functions the scan cannot resolve: %v", kind, fn, sortedNameSet(r.unknown)) + } + for d := range r.dynamic { + if !tableDrivenReaders[d] { + t.Errorf("%s (%s): %s reads a key that is not a literal — make it one, or add it to "+ + "tableDrivenReaders with the predicate nativeWidgetAccepts uses for it", kind, fn, d) + } + } + got := map[string]bool{} + for key := range r.keys { + got[strings.ToLower(key)] = true + } + for key := range got { + if !want[key] { + t.Errorf("%s: %s reads `%s`, which nativeWidgetKinds[%q].reads lacks — check and exec would "+ + "refuse a property that is written", kind, fn, key, kind) + } + } + for key := range want { + if !got[key] { + t.Errorf("%s: nativeWidgetKinds[%q].reads has `%s`, which %s never reads — it would be "+ + "accepted and dropped", kind, kind, key, fn) + } + } + } + + // The common tail: exactly appearance, visibility and editability. + tail := map[string]bool{} + for _, fn := range []string{"applyWidgetAppearance", "applyConditionalSettings", "pageBuilder.applyVisibleWhen", "applyVisibleRoles"} { + for k := range src.reads(fn).keys { + tail[strings.ToLower(k)] = true + } + } + want := lowerSet(append(append(append([]string(nil), appearanceKeys...), visibilityKeys...), "Editable", "EditableIf")) + if strings.Join(sortedNameSet(tail), ",") != strings.Join(sortedNameSet(want), ",") { + t.Errorf("buildWidgetV3's tail reads %v, want %v (appearanceKeys + visibilityKeys + editability)", + sortedNameSet(tail), sortedNameSet(want)) + } +} + +// genWidgetConstructors build the gen element the writer encodes for each +// stored $Type. Its setters are what the writer can store: applyWidgetBase +// writes visibility only through SetConditionalVisibilitySettings, and drops +// it silently on a type without one. +var genWidgetConstructors = map[string]func() element.Element{ + "Forms$DataView": func() element.Element { return genPg.NewDataView() }, + "Forms$ListView": func() element.Element { return genPg.NewListView() }, + "Forms$TextBox": func() element.Element { return genPg.NewTextBox() }, + "Forms$TextArea": func() element.Element { return genPg.NewTextArea() }, + "Forms$DatePicker": func() element.Element { return genPg.NewDatePicker() }, + "Forms$DropDown": func() element.Element { return genPg.NewDropDown() }, + "Forms$CheckBox": func() element.Element { return genPg.NewCheckBox() }, + "Forms$RadioButtonGroup": func() element.Element { return genPg.NewRadioButtonGroup() }, + "Forms$FileManager": func() element.Element { return genPg.NewFileManager() }, + "Forms$DynamicText": func() element.Element { return genPg.NewDynamicText() }, + "Forms$Title": func() element.Element { return genPg.NewTitle() }, + "Forms$Label": func() element.Element { return genPg.NewLabel() }, + "Forms$ActionButton": func() element.Element { return genPg.NewActionButton() }, + "Forms$SidebarToggleButton": func() element.Element { return genPg.NewSidebarToggleButton() }, + "Forms$DivContainer": func() element.Element { return genPg.NewDivContainer() }, + "Forms$GroupBox": func() element.Element { return genPg.NewGroupBox() }, + "Forms$TabControl": func() element.Element { return genPg.NewTabContainer() }, + "Forms$LayoutGrid": func() element.Element { return genPg.NewLayoutGrid() }, + "Forms$NavigationList": func() element.Element { return genPg.NewNavigationList() }, + "Forms$SnippetCallWidget": func() element.Element { return genPg.NewSnippetCallWidget() }, + "Forms$StaticImageViewer": func() element.Element { return genPg.NewStaticImageViewer() }, + "Forms$ImageViewer": func() element.Element { return genPg.NewDynamicImageViewer() }, + "Forms$ScrollContainer": func() element.Element { return genPg.NewScrollContainer() }, + "Forms$Header": func() element.Element { return genPg.NewHeader() }, + "Forms$Placeholder": func() element.Element { return genPg.NewPlaceholder() }, + "Forms$NavigationTree": func() element.Element { return genPg.NewNavigationTree() }, + "Forms$MenuBar": func() element.Element { return genPg.NewMenuBar() }, + "Forms$SimpleMenuBar": func() element.Element { return genPg.NewSimpleMenuBar() }, +} + +// A tail kind accepts Visible exactly when its stored type can hold it. +func TestNativeWidgetProps_TailMatchesGen(t *testing.T) { + for kind, k := range nativeWidgetKinds { + if !k.tail { + continue + } + ctor, ok := genWidgetConstructors[k.stored] + if !ok { + t.Errorf("%s: no gen constructor for %s in genWidgetConstructors", kind, k.stored) + continue + } + g := ctor() + if g.TypeName() != k.stored { + t.Errorf("%s: genWidgetConstructors[%s] builds %s", kind, k.stored, g.TypeName()) + } + _, hasVisibility := g.(interface { + SetConditionalVisibilitySettings(element.Element) + }) + if hasVisibility != k.visible { + t.Errorf("%s (%s): visible = %v, but the gen type the writer encodes has a "+ + "ConditionalVisibilitySettings setter: %v", kind, k.stored, k.visible, hasVisibility) + } + if _, ok := g.(interface{ SetAppearance(element.Element) }); !ok { + t.Errorf("%s (%s): the tail writes appearance, but the gen type has no SetAppearance", kind, k.stored) + } + } +} + +// describeCaseKinds maps each $Type case of outputWidgetMDLV3 to the kinds its +// body prints. A case printing several elements (a layout grid with its rows +// and columns) is held to the union of their sets. +var describeCaseKinds = map[string][]string{ + "Forms$ScrollContainer": {"scrollcontainer"}, + "Forms$Header": {"layoutheader", "layoutheader region"}, + "Forms$ScrollContainerRegion": {"scrollcontainer region"}, + "Forms$Placeholder": {"placeholder"}, + "Forms$NavigationTree": {"navigationtree", "menubar", "simplemenubar"}, + "Forms$TabControl": {"tabcontainer"}, + "Forms$DivContainer": {"container"}, + "Forms$GroupBox": {"groupbox"}, + "Forms$LayoutGrid": {"layoutgrid", "layoutgrid row", "layoutgrid column"}, + "Forms$DynamicText": {"dynamictext"}, + "Forms$ActionButton": {"actionbutton", "linkbutton", "sidebartoggle"}, + "Forms$Title": {"title"}, + "Forms$DataView": {"dataview"}, + "Forms$TextBox": {"textbox"}, + "Forms$TextArea": {"textarea"}, + "Forms$FileManager": {"filemanager"}, + "Forms$DatePicker": {"datepicker"}, + "Forms$RadioButtons": {"radiobuttons"}, + "Forms$CheckBox": {"checkbox"}, + "Forms$NavigationList": {"navigationlist", "navigationlist item"}, + "Forms$Label": {"label"}, + "Forms$StaticImageViewer": {"staticimage"}, + "Forms$ImageViewer": {"dynamicimage"}, + "Forms$SnippetCallWidget": {"snippetcall"}, + "Forms$ListViewTemplate": {"listview template"}, + "Forms$ListView": {"listview"}, +} + +// describePrintsKeyRe matches the `Key: ` a describe format string starts a +// property with. +var describePrintsKeyRe = regexp.MustCompile(`^([A-Z][A-Za-z]+): `) + +// describe → exec must not refuse its own output: every key a describe case +// prints is one its kind takes. Visibility and editability are printed only +// when the stored document has them, which the metamodel decides per type +// (TestNativeWidgetProps_TailMatchesGen), so they are not judged here. +func TestNativeWidgetProps_DescribeEmitsOnlyAccepted(t *testing.T) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "cmd_pages_describe_output.go", nil, 0) + if err != nil { + t.Fatal(err) + } + funcs := map[string]*goast.FuncDecl{} + for _, d := range f.Decls { + if fd, ok := d.(*goast.FuncDecl); ok && fd.Body != nil && fd.Recv == nil { + funcs[fd.Name.Name] = fd + } + } + out := funcs["outputWidgetMDLV3"] + if out == nil { + t.Fatal("outputWidgetMDLV3 not found") + } + // printed collects the `Key: ` literals of a body and of the functions it + // calls in this file (not outputWidgetMDLV3 itself: that prints children). + var printed func(n goast.Node, keys map[string]bool, seen map[string]bool) + printed = func(n goast.Node, keys map[string]bool, seen map[string]bool) { + goast.Inspect(n, func(m goast.Node) bool { + switch x := m.(type) { + case *goast.BasicLit: + if lit, ok := stringLit(x); ok { + if sm := describePrintsKeyRe.FindStringSubmatch(lit); sm != nil { + keys[sm[1]] = true + } + } + case *goast.CallExpr: + name := calleeName(x) + // widgetConditionMDL("Visible", …) and actionProp("Action", …) + // take the key as their first argument. + if (name == "widgetConditionMDL" || name == "actionProp") && len(x.Args) > 0 { + if lit, ok := stringLit(x.Args[0]); ok { + keys[lit] = true + } + } + if fd := funcs[name]; fd != nil && name != "outputWidgetMDLV3" && !seen[name] { + seen[name] = true + printed(fd.Body, keys, seen) + } + } + return true + }) + } + judgedLater := lowerSet(append(append([]string(nil), visibilityKeys...), "Editable", "EditableIf")) + covered := map[string]bool{} + goast.Inspect(out.Body, func(n goast.Node) bool { + cc, ok := n.(*goast.CaseClause) + if !ok { + return true + } + var kinds []string + for _, e := range cc.List { + if lit, ok := stringLit(e); ok { + if ks, ok := describeCaseKinds[lit]; ok { + kinds = ks + covered[lit] = true + } + } + } + if kinds == nil { + return true + } + keys := map[string]bool{} + for _, stmt := range cc.Body { + printed(stmt, keys, map[string]bool{}) + } + for key := range keys { + if judgedLater[strings.ToLower(key)] { + continue + } + accepted := false + for _, kind := range kinds { + keyword := kind + if i := strings.LastIndex(kind, " "); i >= 0 { + keyword = kind[i+1:] + } + if nativeWidgetAccepts(kind, keyword, key) || nativeWidgetKeyOwnedElsewhere(kind, keyword, key) { + accepted = true + } + } + if !accepted { + t.Errorf("describe prints `%s:` for %v, which none of them takes — describe → exec would refuse its own output", key, kinds) + } + } + return false + }) + for typ := range describeCaseKinds { + if !covered[typ] { + t.Errorf("describeCaseKinds names %s, but outputWidgetMDLV3 has no such case", typ) + } + } +} diff --git a/mdl/executor/widget_native_props_test.go b/mdl/executor/widget_native_props_test.go new file mode 100644 index 0000000000..bd5d77884f --- /dev/null +++ b/mdl/executor/widget_native_props_test.go @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/langver" + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/sdk/pages" +) + +// ako/mxcli#1698 (mendixlabs/mxcli#1651): `textbox tb (Attribute: Name, +// Caption: 'Item name')` passed check and exec, and the text box was written +// with no label — inputs take `Label:`. MDL-WIDGET07 judged keys against the +// union of every built-in widget's vocabulary, where Caption is a button's. + +const itemEditPage = `create page M.ItemEdit (Title: 'Item', Layout: Atlas_Core.PopupLayout, Params: ($Job: M.Job)) { + dataview dv (DataSource: $Job) { + textbox tbCaption (Attribute: Title, Caption: 'Item name') + textbox tbLabel (Attribute: Title, Label: 'Item name (Label)') + actionbutton btnSave (Caption: 'Save', Action: save changes) + } +}; +` + +// nativePropViolations runs the page-widget checks over the first statement +// and returns the ones this rule reports, gated by the script's header. +func nativePropViolations(t *testing.T, src string) []linter.Violation { + t.Helper() + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parse: %v\n%s", errs, src) + } + reg, _ := NewWidgetRegistry() + var out []linter.Violation + for _, stmt := range prog.Statements { + for _, v := range ValidateWidgetPropertiesForStatement(stmt, reg) { + if v.RuleID == nativeWidgetPropDropped.Code || v.RuleID == "MDL-WIDGET52" { + out = append(out, v) + } + } + } + return GateWidgetViolations(out, prog.LanguageVersion) +} + +func violationText(vs []linter.Violation) string { + var b strings.Builder + for _, v := range vs { + b.WriteString(v.RuleID + " " + v.Message + "\n") + } + return b.String() +} + +func TestNativeWidgetProps_CaptionOnTextboxIsRefused(t *testing.T) { + vs := nativePropViolations(t, "mdl 1;\n"+itemEditPage) + if len(vs) != 1 { + t.Fatalf("got %d violations, want exactly one — for tbCaption's Caption. Label on a text box and Caption "+ + "on a button are what those widgets take (controls):\n%s", len(vs), violationText(vs)) + } + v := vs[0] + if v.RuleID != "MDL-WIDGET52" || v.Severity != linter.SeverityError { + t.Errorf("mdl 1: %s/%v, want an MDL-WIDGET52 error", v.RuleID, v.Severity) + } + for _, want := range []string{"tbCaption", "`Caption`", "Forms$TextBox", "`Label:`"} { + if !strings.Contains(v.Message, want) { + t.Errorf("message lacks %q:\n%s", want, v.Message) + } + } + if !strings.Contains(v.Suggestion, "Label") || strings.Contains(v.Suggestion, "Caption") { + t.Errorf("suggestion should list what a text box takes (Label, not Caption): %s", v.Suggestion) + } +} + +// A script without the header keeps the old outcome and is told (ADR-0011). +func TestNativeWidgetProps_HeaderlessWarns(t *testing.T) { + vs := nativePropViolations(t, itemEditPage) + if len(vs) != 1 { + t.Fatalf("got %d violations, want one:\n%s", len(vs), violationText(vs)) + } + if vs[0].RuleID != nativeWidgetPropDropped.Code || vs[0].Severity != linter.SeverityWarning { + t.Errorf("mdl 0: %s/%v, want a %s warning", vs[0].RuleID, vs[0].Severity, nativeWidgetPropDropped.Code) + } +} + +// The other direction of the mix-up, and a typo: each is a key the widget does +// not take, named with what was most likely meant. +func TestNativeWidgetProps_HintsNameTheIntendedProperty(t *testing.T) { + vs := nativePropViolations(t, `mdl 1; +create page M.P (Title: 'x', Layout: A.L, Params: ($Job: M.Job)) { + dataview dv (DataSource: $Job) { + actionbutton btn (Label: 'Save', Action: save changes) + dynamictext dt (Contnet: 'hi') + groupbox gb (Label: 'Group') { dynamictext inner (Content: 'x') } + } +}; +`) + text := violationText(vs) + for _, want := range []string{ + "btn`: `Label`", "a button's text is `Caption:`", + "dt`: `Contnet`", "did you mean `Content`?", + "gb`: `Label`", "its text is `Caption:`", + } { + if !strings.Contains(text, want) { + t.Errorf("no violation says %q:\n%s", want, text) + } + } +} + +// Visible on an element whose stored type has no ConditionalVisibilitySettings +// was applied to the Go struct by buildWidgetV3's tail and then not encoded. +// Control: the same key on a container, which stores it. +func TestNativeWidgetProps_VisibleWhereNotStored(t *testing.T) { + vs := nativePropViolations(t, `mdl 1; +create page M.P (Title: 'x', Layout: A.L) { + snippetcall sc (Snippet: M.S, Visible: false) + container c (Visible: false) { dynamictext t (Content: 'x') } +}; +`) + text := violationText(vs) + if len(vs) != 1 || !strings.Contains(text, "sc`: `Visible`") || !strings.Contains(text, "Forms$SnippetCallWidget has no conditional visibility") { + t.Errorf("want one refusal, for the snippet call's Visible:\n%s", text) + } +} + +// A layout's elements: a region takes Class, Size, SizeMode and ToggleMode; its +// Style was read by nothing. Control: those four. +func TestNativeWidgetProps_LayoutRegions(t *testing.T) { + vs := nativePropViolations(t, `mdl 1; +create layout M.L (layouttype: 'Responsive') { + scrollcontainer sc { + region top (Size: 48, SizeMode: 'Pixels', ToggleMode: 'None', Class: 'top') { placeholder Top } + region center (Style: 'color: red') { placeholder Main } + } +}; +`) + text := violationText(vs) + if len(vs) != 1 || !strings.Contains(text, "scrollcontainer region `center`: `Style`") || + !strings.Contains(text, "only `Class:` is written") { + t.Errorf("want one refusal, for the centre region's Style:\n%s", text) + } +} + +// A positional keyword is judged only where its parent is a built-in element. +// A data grid's `column` (an object-list item of a pluggable widget) and the +// root of an ALTER … INSERT (whose parent is out of sight) are not this rule's; +// a layout-grid column is. +func TestNativeWidgetProps_PositionalKeywords(t *testing.T) { + vs := nativePropViolations(t, `mdl 1; +create page M.P (Title: 'x', Layout: A.L) { + pluggablewidget 'com.mendix.widget.web.datagrid.Datagrid' dg (DataSource: database M.Job) { + column colTitle (Attribute: Title, Caption: 'Title') + } + layoutgrid lg { + row { + column (DesktopWidth: 6, Caption: 'not a column property') { dynamictext t (Content: 'x') } + } + } +}; +alter page M.Q { + insert after dg.colA { + column colB (Attribute: Title, Caption: 'B') + } +}; +`) + text := violationText(vs) + if len(vs) != 1 || !strings.Contains(text, "layoutgrid column: `Caption`") { + t.Errorf("want one refusal, for the layout-grid column's Caption — a data grid column's Caption and "+ + "Attribute are its own:\n%s", text) + } +} + +// exec applies the same verdict, so `exec --no-check` refuses too; without +// the header it warns and writes what it wrote before. Controls: the text box +// with Label: and the button with Caption: keep them. +func TestNativeWidgetProps_ExecRefuses(t *testing.T) { + prog, errs := visitor.Build(itemEditPage) + if len(errs) > 0 { + t.Fatal(errs) + } + // The data view's children, built in its context (M.Job). + children := prog.Statements[0].(*ast.CreatePageStmtV3).Widgets[0].Children + build := func(v langver.Version, w *ast.WidgetV3) (pages.Widget, string, error) { + var out bytes.Buffer + pb := visibleWhenPB("M.Job") + pb.ctx = &ExecContext{Output: &out, LanguageVersion: v} + built, err := pb.buildWidgetV3(w) + return built, out.String(), err + } + + if _, _, err := build(langver.V1, children[0]); err == nil || !strings.Contains(err.Error(), "MDL-WIDGET52") || + !strings.Contains(err.Error(), "Caption") { + t.Errorf("mdl 1: err = %v, want an MDL-WIDGET52 refusal naming Caption", err) + } + built, out, err := build(langver.V0, children[0]) + if err != nil { + t.Fatalf("mdl 0: refused (%v) — a new refusal applies only under the header", err) + } + if !strings.Contains(out, nativeWidgetPropDropped.Code) || !strings.Contains(out, "tbCaption") { + t.Errorf("mdl 0: the drop was silent, want the %s warning; output:\n%s", nativeWidgetPropDropped.Code, out) + } + if tb := built.(*pages.TextBox); tb.Label != "" { + t.Errorf("tbCaption's Label = %q: Caption is not mapped to the label, it is refused", tb.Label) + } + + // Controls, under mdl 1: what each widget does take is written. + built, _, err = build(langver.V1, children[1]) + if err != nil { + t.Fatalf("control: tbLabel refused: %v", err) + } + if tb := built.(*pages.TextBox); tb.Label != "Item name (Label)" { + t.Errorf("control: tbLabel's Label = %q, want it written", tb.Label) + } + built, _, err = build(langver.V1, children[2]) + if err != nil { + t.Fatalf("control: btnSave refused: %v", err) + } + if btn := built.(*pages.ActionButton); btn.CaptionTemplate == nil { + t.Errorf("control: btnSave's Caption was not written") + } +} + +// Every kind's own keys are accepted by the verdict — no kind refuses what its +// builder reads (a sanity net under the source-derived test). +func TestNativeWidgetProps_EveryReadIsAccepted(t *testing.T) { + for kind, k := range nativeWidgetKinds { + keyword := kind + if i := strings.LastIndex(kind, " "); i >= 0 { + keyword = kind[i+1:] + } + for _, key := range k.reads { + if !nativeWidgetAccepts(kind, keyword, key) { + t.Errorf("%s: reads %s but does not accept it", kind, key) + } + } + } +} + +// An `OnClick:` whose value is not an action keeps its key as written — the +// visitor renames it to Action only when it parses — so the verdict saw a +// container carrying an unknown `OnClick`. It is the container's click action +// holding something that is not one: MDL-WIDGET28's, not this rule's. Refusing +// it here too named the wrong problem and, this rule being gated and that one +// not, made `mdl 1;` change what action-slot-not-an-action-1062.fail.mdl writes +// (the upgrade property test). Every case of the spelling, since a malformed +// value keeps the author's case. +func TestNativeWidgetProps_MalformedActionIsTheActionSlotRules(t *testing.T) { + prog, errs := visitor.Build(`mdl 1; +create page M.P (Title: 'x', Layout: A.L, Params: ($Job: M.Job)) { + dataview dv (DataSource: $Job) { + container c1 (OnClick: TOTALLY_MADE_UP) { dynamictext t1 (Content: 'a') } + container c2 (onClick: TOTALLY_MADE_UP) { dynamictext t2 (Content: 'a') } + container c3 (action: OPEN_LINK) { dynamictext t3 (Content: 'a') } + textbox tb (Attribute: Title, onchange: TOTALLY_MADE_UP) + dynamictext t4 (Content: 'a', onClick: TOTALLY_MADE_UP) + } +}; +`) + if len(errs) > 0 { + t.Fatal(errs) + } + reg, _ := NewWidgetRegistry() + var native, slot []string + for _, v := range GateWidgetViolations(ValidateWidgetPropertiesForStatement(prog.Statements[0], reg), prog.LanguageVersion) { + switch v.RuleID { + case "MDL-WIDGET52", nativeWidgetPropDropped.Code: + native = append(native, v.Message) + case "MDL-WIDGET28": + slot = append(slot, v.Message) + } + } + if len(native) != 0 { + t.Errorf("a malformed action is reported as a property the widget lacks:\n%s", strings.Join(native, "\n")) + } + all := strings.Join(slot, "\n") + for _, want := range []string{"`c1` has `OnClick:", "`c2` has `onClick:", "`c3` has `action:", "`tb` has `onchange:", "`t4` has `onClick:"} { + if !strings.Contains(all, want) { + t.Errorf("MDL-WIDGET28 does not report %q:\n%s", want, all) + } + } + + // exec: the same verdict, so `mdl 1;` builds what mdl 0 builds. + pb := visibleWhenPB("M.Job") + pb.ctx = &ExecContext{Output: &bytes.Buffer{}, LanguageVersion: langver.V1} + if _, err := pb.buildWidgetV3(prog.Statements[0].(*ast.CreatePageStmtV3).Widgets[0].Children[0]); err != nil { + t.Errorf("mdl 1: exec refused the container — %v", err) + } +} + +// Controls for the alias: a well-formed OnClick is the container's action and +// is written; a key no container takes is still refused, and so is a click +// action on a widget Mendix gives none. +func TestNativeWidgetProps_OnClickAliasControls(t *testing.T) { + prog, errs := visitor.Build("mdl 1;\ncreate page M.P (Title: 'x', Layout: A.L) {\n" + + " container c1 (OnClick: close page) { dynamictext t1 (Content: 'a') }\n};\n") + if len(errs) > 0 { + t.Fatal(errs) + } + pb := visibleWhenPB("") + pb.ctx = &ExecContext{Output: &bytes.Buffer{}, LanguageVersion: langver.V1} + built, err := pb.buildWidgetV3(prog.Statements[0].(*ast.CreatePageStmtV3).Widgets[0]) + if err != nil { + t.Fatalf("container with a well-formed OnClick: %v", err) + } + if c := built.(*pages.Container); c.OnClickAction == nil { + t.Error("a well-formed OnClick on a container was not written") + } + + vs := nativePropViolations(t, `mdl 1; +create page M.P (Title: 'x', Layout: A.L) { + container c2 (Bogus: 1) { dynamictext t2 (Content: 'a') } + title ttl (Content: 'x', OnClick: TOTALLY_MADE_UP) + label lbl (Content: 'x', OnClick: close page) +}; +`) + text := violationText(vs) + for _, want := range []string{"c2`: `Bogus`", "lbl`: `Action`", "Forms$Label has no click action"} { + if !strings.Contains(text, want) { + t.Errorf("no refusal says %q:\n%s", want, text) + } + } + // A title is MDL-WIDGET23's (a click action Mendix does not model there). + if strings.Contains(text, "ttl") { + t.Errorf("the title's OnClick belongs to the click-action rules, not this one:\n%s", text) + } +} + +// The alias is the visitor's rename, not a builder read: an alias and its +// target must be the same action source. +func TestNativeWidgetProps_AliasesFollowTheVisitor(t *testing.T) { + for alias, target := range nativeWidgetKeyAliases { + a, okA := actionKeywordSource(alias) + b, okB := actionKeywordSource(target) + if !okA || !okB || a != b { + t.Errorf("alias %q -> %q: the visitor stores them as %q / %q", alias, target, a, b) + } + } +} diff --git a/mdl/grammar/domains/MDLPage.g4 b/mdl/grammar/domains/MDLPage.g4 index bbf73585f5..0485c3b531 100644 --- a/mdl/grammar/domains/MDLPage.g4 +++ b/mdl/grammar/domains/MDLPage.g4 @@ -349,8 +349,14 @@ useFragmentPayload ; // Fragment argument list: ($p: $Data, $q: microflow Module.Flow) +// +// May be empty, meaning no arguments, as microflowArgsV3 may (ako/mxcli#1516). +// Unlike the flow calls this was not a syntax error: `use fragment Frag()` +// failed this rule and the page body re-read it as two generic widgets, one of +// type `use` named `fragment` and one of type `Frag` — and `mxcli check` +// without a project passed it. fragmentArgs - : LPAREN fragmentArg (COMMA fragmentArg)* RPAREN + : LPAREN (fragmentArg (COMMA fragmentArg)*)? RPAREN ; fragmentArg @@ -786,8 +792,19 @@ openLinkV3 // V3 Microflow arguments: (Param = value, ...) — R4, the argument form of every // call site. `Param: value` and `$Param = value` are deprecated spellings. +// +// The list may be EMPTY, and `M.F()` means exactly what `M.F` means — no +// mappings (ako/mxcli#1516). A flow body writes `call microflow M.F()`, so the +// same call on a button was written that way by analogy and was a syntax error +// at the `)`, reported as "expecting the start of a statement". This one rule +// serves every page action (call microflow/nanoflow, show page and their +// deprecated spellings), both flow data sources, and so every site that takes +// an action: Action/OnClick/OnChange, a pluggable widget's action property, +// alter page set, a navigation or menu document item, a fragment argument and +// a building-block override. describe keeps writing a parameterless call +// without the brackets. microflowArgsV3 - : LPAREN microflowArgV3 (COMMA microflowArgV3)* RPAREN + : LPAREN (microflowArgV3 (COMMA microflowArgV3)*)? RPAREN ; microflowArgV3 diff --git a/mdl/linter/linter.go b/mdl/linter/linter.go index 0141a8a96f..70f611fd7e 100644 --- a/mdl/linter/linter.go +++ b/mdl/linter/linter.go @@ -5,6 +5,7 @@ package linter import ( "context" + "strings" ) // Severity indicates how serious a violation is. @@ -72,10 +73,28 @@ type Location struct { } // QualifiedName returns the full qualified name of the location. +// +// DocumentName is meant to be the document's own name, but most bundled +// Starlark rules pass the qualified one (`document_name=mf.qualified_name`), and +// so does missing_translations. Prefixing the module again printed every such +// finding as `at M.M.NAV_Home` (ako/mxcli#1484). This is the one place a +// location is turned into a name — text, Markdown, SARIF and `lint -d` all come +// through here — so both spellings are accepted here rather than in each rule, +// which would leave a project's stale rule copies and every user-written rule +// still repeating the module. A dot cannot occur in a Mendix document name, so +// "." at the front is always the module and never part of the name. func (l Location) QualifiedName() string { if l.Module == "" { return l.DocumentName } + if strings.HasPrefix(l.DocumentName, l.Module+".") { + return l.DocumentName + } + // A finding about the module itself (CONV018) names the module as its + // document; "Sales.Sales" would read as a document called Sales. + if strings.EqualFold(l.DocumentType, "module") && l.DocumentName == l.Module { + return l.Module + } return l.Module + "." + l.DocumentName } diff --git a/mdl/linter/location_test.go b/mdl/linter/location_test.go new file mode 100644 index 0000000000..5cd32cc3dd --- /dev/null +++ b/mdl/linter/location_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package linter_test + +import ( + "bytes" + "path/filepath" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// Location.DocumentName is documented as the document's own name ("Customer"), +// and QualifiedName put the module in front of it. Most of the bundled Starlark +// rules pass `document_name=.qualified_name` instead — QUAL004, CONV001-010, +// the SEC rules, ~30 in all — so every one of their findings printed as +// `at M.M.NAV_Home`, the module twice (ako/mxcli#1484's repro). The Go rules +// mostly strip the module first; missing_translations does not. +// +// Fixing the rules one by one would leave a project's stale copies of them +// (rules are written into the project by `mxcli init` and kept until it is +// re-run) and every user-written rule still printing the module twice. So the +// one place the qualified name is assembled tolerates both spellings. +func TestLocationQualifiedNameDoesNotRepeatTheModule(t *testing.T) { + for _, tc := range []struct { + name string + loc linter.Location + want string + }{ + {"short document name", linter.Location{Module: "M", DocumentType: "Microflow", DocumentName: "NAV_Home"}, "M.NAV_Home"}, + {"qualified document name", linter.Location{Module: "M", DocumentType: "Microflow", DocumentName: "M.NAV_Home"}, "M.NAV_Home"}, + // missing_translations names an element below a document. + {"qualified name of a member", linter.Location{Module: "Sales", DocumentType: "attribute", DocumentName: "Sales.Order.Status"}, "Sales.Order.Status"}, + // CONV018 reports a module about itself. + {"module-level finding", linter.Location{Module: "Sales", DocumentType: "Module", DocumentName: "Sales"}, "Sales"}, + {"project-level finding", linter.Location{DocumentType: "security", DocumentName: "ProjectSecurity"}, "ProjectSecurity"}, + + // CONTROLS: a document may share its module's name, and a module name + // may be a prefix of another module's — neither is a repeat. + {"document named like its module", linter.Location{Module: "Sales", DocumentType: "Microflow", DocumentName: "Sales"}, "Sales.Sales"}, + {"module name is a prefix of the document's", linter.Location{Module: "M", DocumentType: "Microflow", DocumentName: "MX_Home"}, "M.MX_Home"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := tc.loc.QualifiedName(); got != tc.want { + t.Errorf("QualifiedName() = %q, want %q", got, tc.want) + } + }) + } +} + +// TestShippedRuleLocationPrintsTheModuleOnce is the reported line, end to end: +// the shipped QUAL004 rule over a catalog, through the text formatter. +func TestShippedRuleLocationPrintsTheModuleOnce(t *testing.T) { + r, err := linter.LoadStarlarkRule(filepath.Join(flowRulesDir, "orphaned_elements.star")) + if err != nil { + t.Fatalf("LoadStarlarkRule: %v", err) + } + violations := r.Check(linter.NewLintContextFromDB(flowKindsFixtureDB(t))) + + var hit *linter.Violation + for i := range violations { + if strings.HasSuffix(violations[i].Location.DocumentName, "Big_MF") { + hit = &violations[i] + } + } + // CONTROL: the fixture's Big_MF is unreferenced, so QUAL004 reports it. + if hit == nil { + t.Fatalf("QUAL004 did not report Sales.Big_MF; got %d findings", len(violations)) + } + + var out bytes.Buffer + if err := (&linter.TextFormatter{NoSummary: true}).Format([]linter.Violation{*hit}, &out); err != nil { + t.Fatal(err) + } + if strings.Contains(out.String(), "Sales.Sales.") { + t.Errorf("the location repeats the module:\n%s", out.String()) + } + if !strings.Contains(out.String(), "at Sales.Big_MF") { + t.Errorf("want the location `at Sales.Big_MF`, got:\n%s", out.String()) + } +} diff --git a/mdl/linter/starlark_qual004_refs_test.go b/mdl/linter/starlark_qual004_refs_test.go new file mode 100644 index 0000000000..55f9d3672f --- /dev/null +++ b/mdl/linter/starlark_qual004_refs_test.go @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 + +package linter_test + +import ( + "database/sql" + "path/filepath" + "slices" + "sort" + "testing" + + "github.com/mendixlabs/mxcli/mdl/catalog" + "github.com/mendixlabs/mxcli/mdl/linter" + _ "modernc.org/sqlite" +) + +// QUAL004 decided "has a caller" from a hand-kept list of reference kinds, +// MICROFLOW_ENTRY_KINDS, and the list drifted behind the catalog every time an +// edge kind was added. A microflow started only from the navigation — a +// profile's home microflow, or a menu item's `call microflow` — has a +// `home_page` / `menu_item` edge that `mxcli refs` lists, and QUAL004 still said +// "is not called from anywhere. Remove if unused" (ako/mxcli#1484, upstream +// mendixlabs/mxcli#1579; four such NAV_ microflows in ako/mxcli#1639's +// Formula1Frontend). PAGE_ENTRY_KINDS had both kinds; the microflow list did not. + +// qual004Ref is one refs row pointing at a document. +type qual004Ref struct{ sourceType, sourceName, targetType, targetName, kind string } + +// qual004FixtureDB is a catalog with microflows and pages in module M, and the +// given refs rows. Nothing references a document unless a row says so. +func qual004FixtureDB(t *testing.T, refs []qual004Ref) catalog.CatalogDB { + t.Helper() + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + stmts := []string{ + `CREATE TABLE modules (Id TEXT, Name TEXT, Source TEXT)`, + `INSERT INTO modules VALUES ('m1', 'M', '')`, + `CREATE TABLE microflows ( + Id TEXT, Name TEXT, QualifiedName TEXT, ModuleName TEXT, Folder TEXT, + MicroflowType TEXT, Description TEXT, ReturnType TEXT, + ParameterCount INTEGER, ActivityCount INTEGER, TotalActivityCount INTEGER DEFAULT 0, Complexity INTEGER)`, + `INSERT INTO microflows VALUES + ('f1','NAV_Home','M.NAV_Home','M','','MICROFLOW','','',0,1,1,1), + ('f2','NAV_Orders','M.NAV_Orders','M','','MICROFLOW','','',0,1,1,1), + ('f3','Unused','M.Unused','M','','MICROFLOW','','',0,1,1,1), + ('f4','ACT_Entry','M.ACT_Entry','M','','MICROFLOW','','',0,1,1,1), + ('f5','Target','M.Target','M','','MICROFLOW','','',0,1,1,1), + ('f6','TargetNF','M.TargetNF','M','','NANOFLOW','','',0,1,1,1)`, + `CREATE TABLE pages (Id TEXT, Name TEXT, QualifiedName TEXT, ModuleName TEXT, Folder TEXT, + Title TEXT, URL TEXT, Description TEXT, WidgetCount INTEGER)`, + `INSERT INTO pages VALUES + ('p1','Orders_Overview','M.Orders_Overview','M','','','','',0), + ('p2','Unused_Page','M.Unused_Page','M','','','','',0), + ('p3','TargetPage','M.TargetPage','M','','','','',0)`, + `CREATE TABLE refs ( + SourceType TEXT, SourceId TEXT, SourceName TEXT, TargetType TEXT, + TargetId TEXT, TargetName TEXT, RefKind TEXT, ModuleName TEXT)`, + // One unrelated row, so the refs table reads as populated. + `INSERT INTO refs VALUES ('PAGE','','M.X','ENTITY','','M.E','parameter','M')`, + `CREATE TABLE entities ( + Id TEXT, Name TEXT, QualifiedName TEXT, ModuleName TEXT, Folder TEXT, + EntityType TEXT, Description TEXT, Generalization TEXT, + AttributeCount INTEGER, AccessRuleCount INTEGER, ValidationRuleCount INTEGER, + HasEventHandlers INTEGER, IsExternal INTEGER, + HasCreatedDate INTEGER, HasChangedDate INTEGER, + HasOwner INTEGER, HasChangedBy INTEGER)`, + } + for _, s := range stmts { + if _, err := db.Exec(s); err != nil { + t.Fatalf("fixture %q: %v", s, err) + } + } + for _, r := range refs { + if _, err := db.Exec(`INSERT INTO refs VALUES (?, '', ?, ?, '', ?, ?, 'M')`, + r.sourceType, r.sourceName, r.targetType, r.targetName, r.kind); err != nil { + t.Fatalf("insert ref %+v: %v", r, err) + } + } + return catalog.WrapSqlDB(db) +} + +// qual004Flagged runs the shipped QUAL004 rule and returns the qualified names +// it reports. +func qual004Flagged(t *testing.T, db catalog.CatalogDB) []string { + t.Helper() + r, err := linter.LoadStarlarkRule(filepath.Join(flowRulesDir, "orphaned_elements.star")) + if err != nil { + t.Fatalf("LoadStarlarkRule: %v", err) + } + var out []string + for _, v := range r.Check(linter.NewLintContextFromDB(db)) { + if v.RuleFailure { + t.Fatalf("QUAL004 failed to run: %s", v.Message) + } + out = append(out, v.Location.QualifiedName()) + } + sort.Strings(out) + return out +} + +// TestQUAL004CountsNavigationAsACaller is the reported case: a home microflow +// and a menu item's microflow, with the controls the report itself used. +func TestQUAL004CountsNavigationAsACaller(t *testing.T) { + flagged := qual004Flagged(t, qual004FixtureDB(t, []qual004Ref{ + {"NAVIGATION", "Navigation.Responsive", "MICROFLOW", "M.NAV_Home", catalog.RefKindHomePage}, + {"NAVIGATION", "Navigation.Responsive", "MICROFLOW", "M.NAV_Orders", catalog.RefKindMenuItem}, + {"NAVIGATION", "Navigation.Responsive", "PAGE", "M.Orders_Overview", catalog.RefKindMenuItem}, + })) + + for _, live := range []string{"M.NAV_Home", "M.NAV_Orders", "M.Orders_Overview"} { + if slices.Contains(flagged, live) { + t.Errorf("QUAL004 reports %s, which the navigation starts — `mxcli refs` lists the edge, "+ + "and the rule advises deleting a live entry point", live) + } + } + + // CONTROLS: an unreferenced microflow and page are still reported — a rule + // that reported nothing would pass the assertions above — and an + // entry-point prefix still exempts by name. + for _, dead := range []string{"M.Unused", "M.Unused_Page"} { + if !slices.Contains(flagged, dead) { + t.Errorf("control: %s has no reference and is not reported; got %v", dead, flagged) + } + } + if slices.Contains(flagged, "M.ACT_Entry") { + t.Errorf("control: M.ACT_Entry carries an entry-point prefix and is reported; got %v", flagged) + } +} + +// TestQUAL004CountsEveryRefKind is the drift guard. The rule does not keep its +// own list of the kinds that mean "something starts this" — every edge into a +// document counts, which is the graph `mxcli refs` and GRAPH_DEAD_ASSETS read — +// so a kind the catalog adds later counts without anyone remembering this rule. +// +// Every kind the catalog declares is driven at a microflow, a nanoflow and a +// page, one at a time. catalog.RefKinds is held complete against the RefKind +// constants by TestEveryObjectTypeConstantIsInAPublishedList, so a new kind is +// in this loop the moment it is declared; re-introducing an allowlist that +// omits it fails here, by name. +func TestQUAL004CountsEveryRefKind(t *testing.T) { + // CONTROL: a scan of an empty vocabulary would pass whatever the rule did. + if len(catalog.RefKinds) < 20 { + t.Fatalf("catalog.RefKinds holds %d kinds — the vocabulary is broken, not the rule", len(catalog.RefKinds)) + } + for _, kind := range catalog.RefKinds { + t.Run(kind, func(t *testing.T) { + flagged := qual004Flagged(t, qual004FixtureDB(t, []qual004Ref{ + {"MICROFLOW", "M.Caller", "MICROFLOW", "M.Target", kind}, + {"MICROFLOW", "M.Caller", "NANOFLOW", "M.TargetNF", kind}, + {"MICROFLOW", "M.Caller", "PAGE", "M.TargetPage", kind}, + })) + for _, target := range []string{"M.Target", "M.TargetNF", "M.TargetPage"} { + if slices.Contains(flagged, target) { + t.Errorf("a %q edge into %s does not count: QUAL004 reports it as not called from "+ + "anywhere while `mxcli refs` lists the reference", kind, target) + } + } + // CONTROL, per kind: the edge must not silence the unrelated document. + if !slices.Contains(flagged, "M.Unused") { + t.Errorf("control: M.Unused stopped being reported; got %v", flagged) + } + }) + } +} diff --git a/mdl/scriptdiff/render.go b/mdl/scriptdiff/render.go index 31613eb4bd..f17ed74056 100644 --- a/mdl/scriptdiff/render.go +++ b/mdl/scriptdiff/render.go @@ -312,7 +312,7 @@ func writesNote(u UnitChange, renderedSame, described bool, errs []string) strin case u.IDChurnOnly: what = append(what, "rewritten with only element $IDs changed (canonically equal)") default: - if paths := changedPaths(u.before, u.after, 4); len(paths) > 0 { + if paths := executor.ChangedPaths(u.before, u.after, 4); len(paths) > 0 { what = append(what, "changed: "+strings.Join(paths, ", ")) } else { what = append(what, "rewritten") @@ -365,93 +365,6 @@ func memberSet(typ string, raw []byte) map[member]bool { return m } -// changedPaths lists up to max property paths whose values differ between two -// units, element $IDs aside. A list element is named by its Name where it has -// one. -func changedPaths(a, b []byte, max int) []string { - var out []string - var walk func(path string, x, y bson.RawValue) - walk = func(path string, x, y bson.RawValue) { - if len(out) >= max { - return - } - if x.Type != y.Type { - out = append(out, path) - return - } - switch x.Type { - case bson.TypeEmbeddedDocument: - dx, dy := x.Document(), y.Document() - ex, _ := dx.Elements() - keys := map[string]bool{} - for _, e := range ex { - k := e.Key() - keys[k] = true - if k == "$ID" { - continue - } - vy, err := dy.LookupErr(k) - if err != nil { - out = append(out, join(path, k)) - } else { - walk(join(path, k), e.Value(), vy) - } - if len(out) >= max { - return - } - } - ey, _ := dy.Elements() - for _, e := range ey { - if !keys[e.Key()] && len(out) < max { - out = append(out, join(path, e.Key())) - } - } - case bson.TypeArray: - vx, _ := x.Array().Values() - vy, _ := y.Array().Values() - if len(vx) != len(vy) { - out = append(out, fmt.Sprintf("%s (%d -> %d items)", path, len(vx), len(vy))) - return - } - for i := range vx { - walk(fmt.Sprintf("%s[%s]", path, itemName(vx[i], i)), vx[i], vy[i]) - if len(out) >= max { - return - } - } - case bson.TypeBinary: - // A pointer holds an element $ID, so a pointer to an element exec - // re-minted differs without anything having changed; canon has - // already said whether the unit did. A GUID is reported. - if !bytes.Equal(x.Value, y.Value) && !strings.HasSuffix(path, "Pointer") { - out = append(out, path) - } - default: - if !bytes.Equal(x.Value, y.Value) { - out = append(out, path) - } - } - } - walk("", bson.RawValue{Type: bson.TypeEmbeddedDocument, Value: a}, bson.RawValue{Type: bson.TypeEmbeddedDocument, Value: b}) - return out -} - -func join(path, key string) string { - if path == "" { - return key - } - return path + "." + key -} - -func itemName(v bson.RawValue, i int) string { - if d, ok := v.DocumentOK(); ok { - if n, ok := d.Lookup("Name").StringValueOK(); ok && n != "" { - return n - } - } - return fmt.Sprint(i) -} - // labelOf turns a describe kind into a label: "Java Action". func labelOf(k ast.DescribeObjectType) string { words := strings.Fields(strings.ToLower(k.String())) diff --git a/mdl/types/infrastructure.go b/mdl/types/infrastructure.go index ff32ae78ac..b92f3f88e6 100644 --- a/mdl/types/infrastructure.go +++ b/mdl/types/infrastructure.go @@ -13,6 +13,10 @@ const ( ) // ProjectVersion holds the parsed Mendix project version. +// +// FormatVersion is the MPR storage format (MPRVersionV1/V2) the reader detected +// on disk when it opened the project — not read from _MetaData, which cannot say +// (ako/mxcli#1553). It is zero when no reader filled it in. type ProjectVersion struct { ProductVersion string BuildVersion string diff --git a/mdl/visitor/flow_statements.go b/mdl/visitor/flow_statements.go new file mode 100644 index 0000000000..3290e1629e --- /dev/null +++ b/mdl/visitor/flow_statements.go @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "github.com/antlr4-go/antlr/v4" + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/grammar/parser" + "github.com/mendixlabs/mxcli/mdl/langver" +) + +// FlowStatement is one statement of a microflow body, located in the text it +// was parsed from. It exists for code that rewrites a body as text and has to +// know where a statement ends and whether it already handles its own errors — +// the `mxcli test` generators, which attach an error handler to every call in +// a test body (ako/mxcli#1448). Searching the text for `on error` cannot answer +// that: the clause may sit on a later line, a custom handler holds statements +// of its own, and the words may be inside a string or a comment. +type FlowStatement struct { + // Rule is the grammar rule the statement matched — the alternative of + // microflowStatement, e.g. "callMicroflowStatement". + Rule string + // Start and Stop are the rune offsets [Start, Stop) of the statement, + // without its annotations and without the `;` that terminates it. + Start, Stop int + // OnError is the statement's own ON ERROR clause, nil when it has none. + // A clause on a statement inside a nested body (an IF branch, a loop, a + // handler) belongs to that statement, which is listed separately. + OnError *FlowOnError +} + +// FlowOnError is an ON ERROR clause, located. +type FlowOnError struct { + Type ast.ErrorHandlingType + // Start and Stop are the rune offsets [Start, Stop) of the whole clause, + // from `on` to `rollback`, `continue`, `end error` or the closing brace. + Start, Stop int +} + +// FlowStatements parses src as a script and returns every microflow-body +// statement in it, nested ones included, in source order. ok is false when src +// does not parse, and the caller must then leave the text alone. +// +// Offsets are in runes, as ANTLR counts them: index []rune(src), not src. +func FlowStatements(src string) (stmts []FlowStatement, ok bool) { + errs := newErrorListener() + lexer := parser.NewMDLLexer(newScriptStream(src, langver.V0)) + lexer.RemoveErrorListeners() + lexer.AddErrorListener(errs) + stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel) + p := parser.NewMDLParser(stream) + p.RemoveErrorListeners() + p.AddErrorListener(errs) + tree := p.Program() + if len(errs.errors) > 0 { + return nil, false + } + ruleNames := p.GetRuleNames() + + var walk func(t antlr.Tree) + walk = func(t antlr.Tree) { + if ms, isStmt := t.(*parser.MicroflowStatementContext); isStmt { + if fs, found := flowStatement(ms, ruleNames); found { + stmts = append(stmts, fs) + } + } + for i := 0; i < t.GetChildCount(); i++ { + walk(t.GetChild(i)) + } + } + walk(tree) + return stmts, true +} + +// flowStatement locates one microflowStatement: the statement rule inside it, +// past any annotations, and that rule's own ON ERROR clause. +func flowStatement(ms *parser.MicroflowStatementContext, ruleNames []string) (FlowStatement, bool) { + for i := 0; i < ms.GetChildCount(); i++ { + inner, isRule := ms.GetChild(i).(antlr.ParserRuleContext) + if !isRule { + continue + } + if _, isAnnotation := inner.(*parser.AnnotationContext); isAnnotation { + continue + } + start, stop := inner.GetStart(), inner.GetStop() + if start == nil || stop == nil { + return FlowStatement{}, false + } + fs := FlowStatement{ + Rule: ruleNames[inner.GetRuleIndex()], + Start: start.GetStart(), + Stop: stop.GetStop() + 1, + } + if oe := ownOnErrorClause(inner); oe != nil { + clause := buildOnErrorClause(oe) + if clause != nil { + fs.OnError = &FlowOnError{ + Type: clause.Type, + Start: oe.GetStart().GetStart(), + Stop: oe.GetStop().GetStop() + 1, + } + } + } + return fs, true + } + return FlowStatement{}, false +} + +// ownOnErrorClause finds the ON ERROR clause that belongs to stmt itself. It +// does not descend into a nested body, whose statements own their clauses. +func ownOnErrorClause(t antlr.Tree) *parser.OnErrorClauseContext { + for i := 0; i < t.GetChildCount(); i++ { + switch c := t.GetChild(i).(type) { + case *parser.OnErrorClauseContext: + return c + case *parser.MicroflowBodyContext: + continue + case antlr.ParserRuleContext: + if found := ownOnErrorClause(c); found != nil { + return found + } + } + } + return nil +} diff --git a/mdl/visitor/flow_statements_test.go b/mdl/visitor/flow_statements_test.go new file mode 100644 index 0000000000..38e86e14b5 --- /dev/null +++ b/mdl/visitor/flow_statements_test.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +func TestFlowStatementsLocatesEachStatementAndItsOwnClause(t *testing.T) { + src := "mdl 1;\ncreate or modify microflow M.F ()\nbegin\n" + + " $a = call microflow M.A(Msg = 'on error continue'); -- on error rollback\n" + + " $b = call microflow M.B()\n on error rollback;\n" + + " $c = call java action M.C() on error begin\n" + + " $d = call microflow M.D() on error continue;\n" + + " end error;\n" + + " log info node 'T' 'é';\n" + + "end;\n" + stmts, ok := FlowStatements(src) + if !ok { + t.Fatal("the script did not parse") + } + runes := []rune(src) + text := func(start, stop int) string { return string(runes[start:stop]) } + + type want struct { + rule, text string + onError ast.ErrorHandlingType // "" = no clause of its own + clause string + } + wants := []want{ + {"callMicroflowStatement", "$a = call microflow M.A(Msg = 'on error continue')", "", ""}, + {"callMicroflowStatement", "$b = call microflow M.B()\n on error rollback", ast.ErrorHandlingRollback, "on error rollback"}, + {"callJavaActionStatement", "", ast.ErrorHandlingCustom, ""}, + {"callMicroflowStatement", "$d = call microflow M.D() on error continue", ast.ErrorHandlingContinue, "on error continue"}, + {"logStatement", "log info node 'T' 'é'", "", ""}, + } + if len(stmts) != len(wants) { + t.Fatalf("got %d statements, want %d: %+v", len(stmts), len(wants), stmts) + } + for i, w := range wants { + s := stmts[i] + if s.Rule != w.rule { + t.Errorf("statement %d: rule %q, want %q", i, s.Rule, w.rule) + } + if w.text != "" && text(s.Start, s.Stop) != w.text { + t.Errorf("statement %d: text %q, want %q", i, text(s.Start, s.Stop), w.text) + } + switch { + case w.onError == "" && s.OnError != nil: + t.Errorf("statement %d: found a clause %q that is not its own", i, text(s.OnError.Start, s.OnError.Stop)) + case w.onError != "" && s.OnError == nil: + t.Errorf("statement %d: its %s clause was not found", i, w.onError) + case w.onError != "" && s.OnError.Type != w.onError: + t.Errorf("statement %d: clause type %s, want %s", i, s.OnError.Type, w.onError) + case w.clause != "" && text(s.OnError.Start, s.OnError.Stop) != w.clause: + t.Errorf("statement %d: clause text %q, want %q", i, text(s.OnError.Start, s.OnError.Stop), w.clause) + } + } + // The custom handler's clause spans its whole body, nested call included. + if c := stmts[2].OnError; c != nil { + if got := text(c.Start, c.Stop); got[:14] != "on error begin" || got[len(got)-9:] != "end error" { + t.Errorf("custom clause text %q", got) + } + } +} + +func TestFlowStatementsRefusesAScriptThatDoesNotParse(t *testing.T) { + if _, ok := FlowStatements("create microflow M.F () begin $a = call microflow M.A() on error rollback on error continue; end;"); ok { + t.Error("a statement with two clauses parsed") + } +} diff --git a/mdl/visitor/visitor_app_security_admin_password_test.go b/mdl/visitor/visitor_app_security_admin_password_test.go new file mode 100644 index 0000000000..040bc7d769 --- /dev/null +++ b/mdl/visitor/visitor_app_security_admin_password_test.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "reflect" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// mendixlabs/mxcli#624 (ako/mxcli#1259): CE0129 "Administrator password has +// not been set" had no MDL remedy — `alter app security` took no AdminPassword. +func TestAlterAppSecurityAdminPassword(t *testing.T) { + prog := mustBuild(t, "alter app security (AdminPassword: 'N3w-Admin-Passw0rd');") + want := &ast.AlterProjectSecurityStmt{AdminPassword: "N3w-Admin-Passw0rd"} + if len(prog.Statements) != 1 || !reflect.DeepEqual(prog.Statements[0], want) { + t.Fatalf("got %#v, want %#v", prog.Statements, want) + } + for src, msg := range map[string]string{ + // An empty password is what CE0129 reports; writing one cannot be the fix. + "alter app security ( AdminPassword: '' );": "CE0129", + "alter app security ( AdminPassword: true );": "string", + "alter app security ( AdminPassword: 12 );": "string", + } { + _, errs := Build(src) + if len(errs) == 0 { + t.Errorf("%s: accepted, want an error mentioning %q", src, msg) + continue + } + if !strings.Contains(errs[0].Error(), msg) { + t.Errorf("%s: error %q does not mention %q", src, errs[0], msg) + } + } +} diff --git a/mdl/visitor/visitor_empty_call_args_test.go b/mdl/visitor/visitor_empty_call_args_test.go new file mode 100644 index 0000000000..56c8690d78 --- /dev/null +++ b/mdl/visitor/visitor_empty_call_args_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "reflect" + "strings" + "testing" +) + +// ako/mxcli#1516 (mendixlabs/mxcli#1606): a page action or data source that +// calls a flow with no parameters may be written with an empty argument list, +// `call microflow M.Ping()`, as a flow body writes it. It was a syntax error +// ("mismatched input ')' expecting the start of a statement") everywhere the +// shared argument rule is used, while the bracket-less form and a non-empty +// list both parsed. +// +// The empty list must mean exactly what no list means: the same statement, the +// same deprecations. An empty-but-present mapping list in the AST would be a +// change in what exec writes, which is why this compares whole statements. +// +// The `args` form is the control: it must build a DIFFERENT statement from the +// bare one, or the equality below could not detect a stray mapping. +func TestEmptyCallArgumentListMeansNoArguments(t *testing.T) { + const page = "create page M.P (Title: 'P', Layout: Atlas_Core.Atlas_Default, Params: ($O: M.E)) { dataview dv (DataSource: $O) { %s } };" + pg := func(widget string) string { return strings.Replace(page, "%s", widget, 1) } + cases := []struct { + site string + // src holds %s where the argument list goes. + src string + // args is a non-empty list for the control; "" where the site has none + // of its own (the fragment and building-block forms carry an action). + args string + }{ + {"button call microflow", pg("actionbutton b (Caption: 'Go', Action: call microflow M.F%s)"), "(Order = $currentObject)"}, + {"button call nanoflow", pg("actionbutton b (Caption: 'Go', Action: call nanoflow M.N%s)"), "(Order = $currentObject)"}, + {"button show page", pg("actionbutton b (Caption: 'Go', Action: show page M.Q%s)"), "(Order = $currentObject)"}, + {"button microflow (MDL-DEPR020)", pg("actionbutton b (Caption: 'Go', Action: microflow M.F%s)"), "(Order = $currentObject)"}, + {"button nanoflow (MDL-DEPR020)", pg("actionbutton b (Caption: 'Go', Action: nanoflow M.N%s)"), "(Order = $currentObject)"}, + {"button show_page (MDL-DEPR020)", pg("actionbutton b (Caption: 'Go', Action: show_page M.Q%s)"), "(Order = $currentObject)"}, + {"call microflow then settings", pg("actionbutton b (Caption: 'Go', Action: call microflow M.F%s with (ProgressBar: Blocking))"), "(Order = $currentObject)"}, + {"call microflow then returns", pg("actionbutton b (Caption: 'Go', Action: call microflow M.F%s returns ($result = $ActionReturnValue))"), "(Order = $currentObject)"}, + {"create object then show page", pg("actionbutton b (Caption: 'Go', Action: create object M.E then show page M.Q%s)"), "(Order = $currentObject)"}, + {"container OnClick", pg("container c (OnClick: call microflow M.F%s) { }"), "(Order = $currentObject)"}, + {"textbox OnChange", pg("textbox t (Attribute: Name, OnChange: call nanoflow M.N%s)"), "(Order = $currentObject)"}, + {"pluggable action property", pg("gallery g (DataSource: database M.E, onSelectionChange: call microflow M.F%s) { }"), "(Order = $currentObject)"}, + {"data source microflow", pg("listview lv (DataSource: microflow M.DS%s) { }"), "(Order = $O)"}, + {"data source nanoflow", pg("listview lv (DataSource: nanoflow M.DS%s) { }"), "(Order = $O)"}, + {"data source microflow then settings", pg("listview lv (DataSource: microflow M.DS%s with (FormValidations: None)) { }"), "(Order = $O)"}, + {"alter page set Action", "alter page M.P { set Action = call microflow M.F%s on b };", "(Order = $currentObject)"}, + {"alter page set DataSource", "alter page M.P { set DataSource = microflow M.DS%s on lv };", "(Order = $O)"}, + {"navigation menu item", "create or replace navigation Responsive home page M.Home { menu item 'Go' (OnClick: call microflow M.F%s) };", "(Order = $currentObject)"}, + {"navigation menu item show page", "create or replace navigation Responsive home page M.Home { menu item 'Go' (OnClick: show page M.Q%s) };", "(Order = $currentObject)"}, + {"menu document item", "create or modify menu M.Menu { menu item 'Go' (OnClick: call nanoflow M.N%s) };", "(Order = $currentObject)"}, + {"fragment argument action", pg("use fragment Frag($act: call microflow M.F%s)"), "(Order = $currentObject)"}, + {"building block action override", pg("use building block M.BB(Action: call microflow M.F%s)"), "(Order = $currentObject)"}, + // A fragment use has an argument list of its own. + {"use fragment", pg("use fragment Frag%s"), "($p: $O)"}, + } + for _, c := range cases { + t.Run(c.site, func(t *testing.T) { + bareSrc := strings.Replace(c.src, "%s", "", 1) + emptySrc := strings.Replace(c.src, "%s", "()", 1) + bare := mustBuild(t, bareSrc) + empty := mustBuild(t, emptySrc) + if !reflect.DeepEqual(empty.Statements, bare.Statements) { + t.Errorf("X() builds a different statement from X:\n X(): %#v\n X: %#v", empty.Statements, bare.Statements) + } + if got, want := deprecationCodes(empty), deprecationCodes(bare); !reflect.DeepEqual(got, want) { + t.Errorf("X() recorded %v, X recorded %v", got, want) + } + // Control: a real argument list must change the statement, or the + // comparison above proves nothing. + args := mustBuild(t, strings.Replace(c.src, "%s", c.args, 1)) + if reflect.DeepEqual(args.Statements, bare.Statements) { + t.Errorf("control: X%s builds the same statement as X, so the comparison cannot see an argument", c.args) + } + }) + } +} + +// The flow body already took `()`; it must keep requiring the brackets (they +// are not optional there) and keep accepting the empty list. +func TestFlowBodyCallStillRequiresItsBrackets(t *testing.T) { + mustBuild(t, "create microflow M.G () begin call microflow M.F(); end;") + if _, errs := Build("create microflow M.G () begin call microflow M.F; end;"); len(errs) == 0 { + t.Error("a flow-body call without () parsed; it was required before and must stay required") + } +} diff --git a/mdl/visitor/visitor_module_documentation_test.go b/mdl/visitor/visitor_module_documentation_test.go index 9eb6113b0f..b4c150e8fc 100644 --- a/mdl/visitor/visitor_module_documentation_test.go +++ b/mdl/visitor/visitor_module_documentation_test.go @@ -55,8 +55,8 @@ func TestCreateModuleDocCommentIsStored(t *testing.T) { // mendixlabs/mxcli#1314: "alter app security has no property "AdminUserName"". // Security$ProjectSecurity.AdminUserName is the name of the built-in -// administrator account (MxAdmin by default); the password stays out of MDL -// (mendixlabs/mxcli#624). +// administrator account (MxAdmin by default); its password is +// TestAlterAppSecurityAdminPassword (mendixlabs/mxcli#624). func TestAlterAppSecurityAdminUserName(t *testing.T) { prog := mustBuild(t, "alter app security (AdminUserName: 'appadmin');") want := &ast.AlterProjectSecurityStmt{AdminUserName: "appadmin"} diff --git a/mdl/visitor/visitor_security.go b/mdl/visitor/visitor_security.go index e4387a9e13..0defad6da3 100644 --- a/mdl/visitor/visitor_security.go +++ b/mdl/visitor/visitor_security.go @@ -562,9 +562,8 @@ func nextToken(stop antlr.Token, ctx antlr.ParserRuleContext) antlr.Token { } // appSecurityKeys are the properties `alter app security ( … )` takes: the -// Security$ProjectSecurity property names. AdminPassword is deliberately absent -// (mendixlabs/mxcli#624). -var appSecurityKeys = []string{"SecurityLevel", "EnableDemoUsers", "EnableGuestAccess", "GuestUserRole", "StrictMode", "AdminUserName"} +// Security$ProjectSecurity property names. +var appSecurityKeys = []string{"SecurityLevel", "EnableDemoUsers", "EnableGuestAccess", "GuestUserRole", "StrictMode", "AdminUserName", "AdminPassword"} // appSecurityProperties reads the property list into stmt. A key or value the // list does not take is an error, never an ignored property. @@ -644,6 +643,20 @@ func (b *Builder) appSecurityProperties(stmt *ast.AlterProjectSecurityStmt, opts continue } stmt.AdminUserName = name + case "AdminPassword": + if sv.STRING_LITERAL() == nil { + b.addError(fmt.Errorf("line %d: AdminPassword takes the administrator's password as a string, not %s", line, sv.GetText())) + continue + } + pw := settingsValueText(sv) + if pw == "" { + // Measured on 11.13.0: an empty AdminPassword at prototype or + // production level is CE0129 "Administrator password has not been + // set"; at level off it passes. Either way '' is never the fix. + b.addError(fmt.Errorf("line %d: AdminPassword cannot be empty — an empty administrator password is what CE0129 reports", line)) + continue + } + stmt.AdminPassword = pw } } } diff --git a/modelsdk/mpr/reader.go b/modelsdk/mpr/reader.go index 6d287fecd3..6456f7faab 100644 --- a/modelsdk/mpr/reader.go +++ b/modelsdk/mpr/reader.go @@ -186,6 +186,8 @@ func OpenWithOptions(path string, opts OpenOptions) (*Reader, error) { } } + r.reportDetectedFormat() + // Verify it's a valid MPR file if err := r.verify(); err != nil { r.Close() @@ -195,6 +197,18 @@ func OpenWithOptions(path string, opts OpenOptions) (*Reader, error) { return r, nil } +// reportDetectedFormat makes ProjectVersion().FormatVersion the storage format +// this reader decided on — the one it reads units with — so SHOW VERSION, +// STATUS and the session log cannot disagree with how the project was opened. +// Call it once the format is final: after OpenWithOptions' Unit.Contents +// reconciliation, which can still turn a folder-less .mpr into v2. +// version.DetectFromDB deliberately leaves the field unset (ako/mxcli#1553). +func (r *Reader) reportDetectedFormat() { + if r.projectVersion != nil { + r.projectVersion.FormatVersion = int(r.version) + } +} + // ListUnitHashes returns a map from unit UUID string to its ContentsHash for // every unit that has a non-empty ContentsHash in the Unit table. // The key format matches what gen-typed elements return from .ID(). @@ -250,6 +264,7 @@ func OpenWithDB(db *sql.DB, path, contentsDir string) (*Reader, error) { return nil, fmt.Errorf("detect project version: %w", err) } r.projectVersion = pv + r.reportDetectedFormat() if err := r.verify(); err != nil { return nil, err } diff --git a/modelsdk/mpr/reader_format_version_test.go b/modelsdk/mpr/reader_format_version_test.go new file mode 100644 index 0000000000..9b5f9afb2d --- /dev/null +++ b/modelsdk/mpr/reader_format_version_test.go @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/mendixlabs/mxcli/modelsdk/mpr/version" +) + +// ProjectVersion().FormatVersion is what SHOW VERSION, STATUS and the session +// log's `mpr_format` print. It must be the storage format the reader detected to +// open the project — Version() — not a second answer worked out from _MetaData. +// +// The second answer was wrong for every v1 project: _MetaData._FormatVersion +// exists only in the v2 schema (measured: absent in a 9.24.30 and an 11.13.0 v1 +// project, present and 2 in 11.6.6/11.13.0/11.14.0 v2 ones), and the fallback +// for a missing column defaulted to 2, so a v1 project reported `MPR Format: v2` +// (ako/mxcli#1553, upstream mendixlabs/mxcli#1621). +func TestProjectVersionReportsTheDetectedStorageFormat(t *testing.T) { + // A v2-schema .mpr copied away from its mprcontents/ folder: the directory + // test says v1 and the Unit.Contents reconciliation corrects it to v2. The + // format must be taken after that correction, not before it. + detached := filepath.Join(t.TempDir(), "minimal.mpr") + data, err := os.ReadFile(v2Fixture) + if err != nil { + t.Fatalf("read v2 fixture: %v", err) + } + if err := os.WriteFile(detached, data, 0o644); err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + name string + path string + want MPRVersion + }{ + {"v1 project (no _FormatVersion column)", v1Fixture, MPRVersionV1}, + {"v2 project (control: _FormatVersion = 2)", v2Fixture, MPRVersionV2}, + {"v2 schema without its mprcontents folder", detached, MPRVersionV2}, + } { + t.Run(tc.name, func(t *testing.T) { + r, err := Open(tc.path) + if err != nil { + t.Fatalf("open: %v", err) + } + defer r.Close() + + // Precondition: the reader's own detection is right. Without this a + // wrong fixture would make the assertion below vacuous. + if r.Version() != tc.want { + t.Fatalf("Version() = %d, want %d — fixture is not the format this case is about", r.Version(), tc.want) + } + if got := r.ProjectVersion().FormatVersion; got != int(tc.want) { + t.Errorf("ProjectVersion().FormatVersion = %d, want %d (Version() = %d) — "+ + "SHOW VERSION would print `MPR Format: v%d`", got, tc.want, r.Version(), got) + } + }) + } +} + +// OpenWithDB (the writer's shared-connection path) is told the format by its +// caller rather than probing for it. It must report that, too. +func TestOpenWithDBReportsTheFormatItWasGiven(t *testing.T) { + db, err := sql.Open("sqlite", fmt.Sprintf("file:%s?mode=ro", v1Fixture)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + r, err := OpenWithDB(db, v1Fixture, "") + if err != nil { + t.Fatalf("OpenWithDB: %v", err) + } + defer r.Close() // does not close db: the caller owns it + if r.Version() != MPRVersionV1 { + t.Fatalf("Version() = %d, want 1", r.Version()) + } + if got := r.ProjectVersion().FormatVersion; got != int(MPRVersionV1) { + t.Errorf("ProjectVersion().FormatVersion = %d, want 1", got) + } +} + +// DetectFromDB reads _MetaData, which cannot say which storage format the +// project is in — that is decided by what is on disk. It must leave the field +// for the reader to fill in rather than guess, because a guess is what shipped. +func TestDetectFromDBDoesNotGuessTheStorageFormat(t *testing.T) { + db, err := sql.Open("sqlite", fmt.Sprintf("file:%s?mode=ro", v1Fixture)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + pv, err := version.DetectFromDB(db) + if err != nil { + t.Fatalf("DetectFromDB: %v", err) + } + if pv.ProductVersion == "" { + t.Fatal("DetectFromDB read no product version from the v1 fixture") + } + if pv.FormatVersion != 0 { + t.Errorf("DetectFromDB set FormatVersion = %d on a database with no _FormatVersion column; "+ + "the storage format is the reader's to report", pv.FormatVersion) + } +} diff --git a/modelsdk/mpr/version/version.go b/modelsdk/mpr/version/version.go index 2b5a737075..103e2ffcc8 100644 --- a/modelsdk/mpr/version/version.go +++ b/modelsdk/mpr/version/version.go @@ -27,11 +27,12 @@ import ( type ProjectVersion = types.ProjectVersion // DefaultVersion returns the default version (11.6.0) used when detection fails. +// +// FormatVersion is left zero, as DetectFromDB leaves it: see there. func DefaultVersion() *ProjectVersion { return &ProjectVersion{ ProductVersion: "11.6.0", BuildVersion: "11.6.0", - FormatVersion: 2, MajorVersion: 11, MinorVersion: 6, PatchVersion: 0, @@ -39,35 +40,35 @@ func DefaultVersion() *ProjectVersion { } // DetectFromDB reads version information from the MPR database. +// +// It does NOT set FormatVersion. Whether a project is MPR v1 or v2 is a fact +// about what is on disk — an mprcontents/ folder beside the .mpr, and a Unit +// table with or without a Contents column — and the reader decides it from +// exactly that before it can read a single unit (mpr.OpenWithOptions). The +// reader stamps its answer onto the ProjectVersion it returns, so there is one +// answer to the question, not two. +// +// _MetaData cannot stand in for it. Its _FormatVersion column exists only in +// the v2 schema — measured absent in a 9.24.30 and an 11.13.0 v1 project, +// present and 2 in 11.6.6, 11.13.0 and 11.14.0 v2 ones — and this function used +// to read a missing column as "a newer schema, so 2". Every v1 project therefore +// reported `MPR Format: v2` (ako/mxcli#1553, upstream mendixlabs/mxcli#1621). +// The three columns read here are present in both schemas. func DetectFromDB(db *sql.DB) (*ProjectVersion, error) { - var formatVersion int var productVersion, buildVersion, schemaHash string - // Try the old schema first (with _FormatVersion) - row := db.QueryRow("SELECT _FormatVersion, _ProductVersion, _BuildVersion, _SchemaHash FROM _MetaData LIMIT 1") - err := row.Scan(&formatVersion, &productVersion, &buildVersion, &schemaHash) - if err != nil { + row := db.QueryRow("SELECT _ProductVersion, _BuildVersion, _SchemaHash FROM _MetaData LIMIT 1") + if err := row.Scan(&productVersion, &buildVersion, &schemaHash); err != nil { if err == sql.ErrNoRows { // Return default if no metadata found return DefaultVersion(), nil } - // Try new schema without _FormatVersion (Mendix 11.6.2+) - row = db.QueryRow("SELECT _ProductVersion, _BuildVersion, _SchemaHash FROM _MetaData LIMIT 1") - err = row.Scan(&productVersion, &buildVersion, &schemaHash) - if err != nil { - if err == sql.ErrNoRows { - return DefaultVersion(), nil - } - return nil, fmt.Errorf("failed to read version metadata: %w", err) - } - // Default format version to 2 for newer schemas - formatVersion = 2 + return nil, fmt.Errorf("failed to read version metadata: %w", err) } pv := &ProjectVersion{ ProductVersion: productVersion, BuildVersion: buildVersion, - FormatVersion: formatVersion, SchemaHash: schemaHash, } diff --git a/sdk/pages/pages_widgets_container.go b/sdk/pages/pages_widgets_container.go index 265d960d72..251e28fe99 100644 --- a/sdk/pages/pages_widgets_container.go +++ b/sdk/pages/pages_widgets_container.go @@ -137,9 +137,31 @@ func (tp *TabPage) GetName() string { // is the flat form some older documents use and is kept for reading them. type ScrollContainer struct { BaseWidget - ScrollBehavior ScrollBehavior `json:"scrollBehavior"` - Regions []*ScrollContainerRegion `json:"regions,omitempty"` - Widgets []Widget `json:"widgets,omitempty"` + ScrollBehavior ScrollBehavior `json:"scrollBehavior"` + // LayoutMode is how the regions share the frame (ako/mxcli#1703): in + // Headline the top region runs the full width and the left region starts + // under it; in Sidebar the left region runs the full height and the top + // bar sits beside it. Empty leaves it unwritten, which Studio Pro reads as + // DefaultScrollContainerLayoutMode. + LayoutMode string `json:"layoutMode,omitempty"` + Regions []*ScrollContainerRegion `json:"regions,omitempty"` + Widgets []Widget `json:"widgets,omitempty"` +} + +// ScrollContainerLayoutModes are the members of Forms$ScrollContainer.LayoutMode +// (generated/metamodel PagesLayoutMode), spelled as stored. +var ScrollContainerLayoutModes = []string{"Headline", "Sidebar"} + +// DefaultScrollContainerLayoutMode is the mode Studio Pro stores on a scroll +// container that was never set, and fills in when the key is absent +// (modelsdk/canon's measured property defaults). describe leaves it unsaid. +const DefaultScrollContainerLayoutMode = "Headline" + +// CanonicalScrollContainerLayoutMode returns the stored spelling of a scroll +// container's LayoutMode, matched case-insensitively. Shared by the CREATE +// LAYOUT builder and the ALTER mutator, so the two accept the same values. +func CanonicalScrollContainerLayoutMode(s string) (string, bool) { + return canonicalMember(s, ScrollContainerLayoutModes...) } // ScrollContainerRegion is one of a ScrollContainer's five slots. diff --git a/sdk/pages/scrollcontainer_layoutmode_test.go b/sdk/pages/scrollcontainer_layoutmode_test.go new file mode 100644 index 0000000000..699e8420eb --- /dev/null +++ b/sdk/pages/scrollcontainer_layoutmode_test.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import ( + "testing" + + "github.com/mendixlabs/mxcli/generated/metamodel" +) + +// The modes a scroll container's LayoutMode takes are the metamodel's, spelled +// as stored (ako/mxcli#1703): a value Studio Pro does not know leaves a +// document it cannot open, and a member missing here refuses a valid one. +func TestScrollContainerLayoutModes_MatchTheMetamodel(t *testing.T) { + want := []string{string(metamodel.PagesLayoutModeHeadline), string(metamodel.PagesLayoutModeSidebar)} + if len(ScrollContainerLayoutModes) != len(want) { + t.Fatalf("ScrollContainerLayoutModes = %v, the metamodel's PagesLayoutMode is %v", ScrollContainerLayoutModes, want) + } + for i, m := range want { + if ScrollContainerLayoutModes[i] != m { + t.Errorf("ScrollContainerLayoutModes[%d] = %q, metamodel %q", i, ScrollContainerLayoutModes[i], m) + } + if got, ok := CanonicalScrollContainerLayoutMode(m); !ok || got != m { + t.Errorf("CanonicalScrollContainerLayoutMode(%q) = %q, %v", m, got, ok) + } + } + if DefaultScrollContainerLayoutMode != string(metamodel.PagesLayoutModeHeadline) { + t.Errorf("default %q is not a member", DefaultScrollContainerLayoutMode) + } + if got, ok := CanonicalScrollContainerLayoutMode("sidebar"); !ok || got != "Sidebar" { + t.Errorf("CanonicalScrollContainerLayoutMode(sidebar) = %q, %v — want the stored spelling", got, ok) + } + if _, ok := CanonicalScrollContainerLayoutMode("Topbar"); ok { + t.Error("Topbar is not a LayoutMode") + } +}