From de1615264aa7ec2ef997d8e73fede1f0abc71965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 1/6] feat(index): add worker resource watchdogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery limits bound what indexing accepts, not what it then costs. A repository well inside those bounds can still exhaust the host through parser memory, or simply never finish, and a supervised worker that hangs leaves the parent waiting with nothing to report. Add index_max_rss_mb and index_max_duration_seconds, enforced by the parent against the worker process tree rather than the worker process alone, so a runaway child cannot hide behind a small parent. Resident memory is sampled through the platform interface on macOS, Linux and Windows. Crossing a limit terminates the tree and yields one trusted, structured terminal result that attributes the failure to the resource that caused it. Both limits default to off. A measurement that cannot be taken fails the attempt instead of passing it: a watchdog that quietly stops watching is worse than no watchdog at all. The shell fixture that stands in for the supervisor names the two new keys. The worker accepts only a policy that spells out every key it knows, which is what keeps a stale supervisor from starting a worker it cannot bound. Signed-off-by: 刘冲 --- README.md | 4 +- docs/CONFIGURATION.md | 10 +- docs/INDEX_RESOURCE_LIMITS.md | 48 +++++- scripts/test-runtime.sh | 3 +- src/cli/cli.c | 10 +- src/daemon/application.c | 20 ++- src/foundation/index_policy.c | 126 +++++++++----- src/foundation/index_policy.h | 11 ++ src/foundation/subprocess.c | 285 ++++++++++++++++++++++++++++++++ src/foundation/subprocess.h | 18 ++ src/mcp/index_supervisor.c | 267 ++++++++++++++++++++++++++++-- src/mcp/index_supervisor.h | 38 ++++- src/mcp/mcp.c | 114 ++++++++++--- src/mcp/mcp.h | 1 + src/mcp/mcp_internal.h | 4 + src/pipeline/pipeline.c | 5 +- tests/test_daemon_application.c | 133 +++++++++++++-- tests/test_index_policy.c | 72 +++++++- tests/test_index_supervisor.c | 269 ++++++++++++++++++++++++++++++ tests/test_mcp.c | 98 +++++++++++ tests/test_subprocess.c | 47 ++++++ 21 files changed, 1472 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 590c9eef96..4466dae50b 100644 --- a/README.md +++ b/README.md @@ -928,10 +928,12 @@ codebase-memory-mcp config set watcher_enabled false # stop the watcher thre codebase-memory-mcp config set watch_non_git true # also poll non-git project roots (default: false) codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit +codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree current RSS limit +codebase-memory-mcp config set index_max_duration_seconds 3600 # optional total worker duration codebase-memory-mcp config reset auto_index # reset to default ``` -The two `index_max_*` settings default to `off`. Exceeding one fails the complete +The four `index_max_*` settings default to `off`. Exceeding one fails the complete index attempt rather than publishing a partial graph; an existing serving index is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3e9956d601..3d3e9a9716 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -93,6 +93,8 @@ Current keys: | `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | +| `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). | +| `index_max_duration_seconds` | `off` | Optional maximum total worker duration in seconds (`1..86400`). | For a watched project, `index_status` names the polling method in `watch.strategy`: `git` (HEAD and dirty-state polling), `tree` (a non-git root polled by the @@ -126,11 +128,13 @@ has not run yet). > `auto_index` still runs, and `index_repository` stays available for manual > reindexing. -The two `index_max_*` settings are independent and disabled by default. They +The four `index_max_*` settings are independent and disabled by default. They apply to explicit indexing, automatic indexing, and watcher re-indexing, but not to `cross-repo-intelligence`, which does not scan repository source files. -Equality is allowed; exceeding either setting fails the complete index request -and preserves any previously serving database. See +Equality is allowed; exceeding any setting fails the complete index request and +preserves any previously serving database. Worker RSS covers descendants and is +not the same as the internal `CBM_MEM_BUDGET_MB` allocation budget. Total +duration is independent of the existing 15-minute no-log-progress timeout. See [Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and error-response details. diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index a3d57e0155..66be4ecf46 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -1,8 +1,8 @@ # Index resource limits Index resource limits are optional operator controls for repositories whose -discovery breadth is not known in advance. They are disabled by default so -existing large-repository workloads retain their current behavior. +discovery breadth or worker runtime is not known in advance. They are disabled +by default so existing large-repository workloads retain their current behavior. ## Discovery settings @@ -23,6 +23,39 @@ Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero, negative values, suffixes, trailing characters, and values outside the stated ranges are rejected without changing the stored value. +## Worker settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Current RSS of the complete worker process tree | +| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Total worker wall-clock duration | + +```bash +codebase-memory-mcp config set index_max_rss_mb 8192 +codebase-memory-mcp config set index_max_duration_seconds 3600 +``` + +RSS is the current resident memory of the contained worker and every descendant, +not the worker's allocation budget and not peak memory. This hard watchdog is +separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor +samples RSS at most once every 250 milliseconds so the watchdog does not turn +full process-table enumeration into a busy loop. + +Duration uses a monotonic clock from successful spawn. It is independent of the +existing 15-minute quiet timeout: continuous log progress does not reset total +duration, while the quiet timeout continues to identify a worker that stops +making progress. + +Equality is allowed. The first RSS or elapsed-duration observation above its +limit starts the existing graceful-to-force process-tree shutdown. CBM reports +terminal only after the tree is quiescent or a bounded containment failure is +explicitly surfaced. Resource termination is not retried and does not +quarantine a source file. + +If RSS is enabled and three consecutive probes cannot obtain any trustworthy +tree measurement while the root worker is still running, CBM fails closed with +`code=resource_probe_failed`. + ## Counting and failure semantics `index_max_files` counts a file only after it passes directory pruning, ignore @@ -55,6 +88,12 @@ The previous database remains available because publication occurs only after a complete discovery and successful staged build. If no previous database exists, `serving_index_preserved` is false. +Worker limit failures use the same shape with `stage=worker`, +`resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and +`unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed` +and omit `observed`, `limit`, and `unit` because no trustworthy observation was +available. + ## Trust and compatibility Limits are read from the CLI-managed `_config.db`; they are not MCP request @@ -64,5 +103,6 @@ parent policy. These settings do not replace or increase `auto_index_limit`, change the 512 MiB single-file cap, alter workspace-root authorization, or affect -`cross-repo-intelligence`. With both settings `off`, discovery follows the -existing unbounded path. +`cross-repo-intelligence`. With all settings `off`, discovery follows the +existing path and the supervisor performs no periodic RSS probe or total-duration +termination. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index 5d2c0c3aee..e6e60b15a7 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -112,7 +112,8 @@ _cbm_test_runtime_daemon() { # cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and # nothing else. cbm_test_index_worker_policy_json() { - printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' + printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"' + printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"}' } cbm_test_runtime_cleanup() { diff --git a/src/cli/cli.c b/src/cli/cli.c index 1ebe1687e8..00b2db3410 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7603,6 +7603,8 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max worker duration in seconds, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing @@ -7629,8 +7631,12 @@ static bool config_key_is_ui(const char *key) { } static bool config_key_is_index_policy(const char *key) { - return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 || - strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, cbm_index_policy_key_at(index)) == 0) { + return true; + } + } + return false; } static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { diff --git a/src/daemon/application.c b/src/daemon/application.c index 22b6f0d1a7..a7b2f4f5b0 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -363,6 +363,14 @@ static int application_worker_start_default(void *context, const char *args_json const char *quarantine_file, cbm_daemon_application_worker_t *worker_out) { (void)context; + cbm_index_resource_policy_t resource_policy; + char error[CBM_SZ_256] = {0}; + if (!cbm_mcp_index_policy_from_internal_args(args_json, &resource_policy, error, + sizeof(error))) { + cbm_log_error("daemon.index.policy", "error", error); + *worker_out = NULL; + return -1; + } application_default_worker_t *worker = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*worker)); if (!worker) { return -1; @@ -372,9 +380,9 @@ static int application_worker_start_default(void *context, const char *args_json const char *root = yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(args_doc), "repo_path")); worker->project = application_index_project_key(root, args_json); yyjson_doc_free(args_doc); - int result = cbm_index_worker_start_with_log(args_json, memory_budget_bytes, false, marker_file, - quarantine_file, application_worker_diagnostic, - worker, &worker->handle); + int result = cbm_index_worker_start_with_policy_log( + args_json, memory_budget_bytes, &resource_policy, false, marker_file, quarantine_file, + application_worker_diagnostic, worker, &worker->handle); if (result != 0) { application_worker_destroy_default(NULL, worker); *worker_out = NULL; @@ -1567,6 +1575,12 @@ static application_attempt_decision_t application_consume_attempt( application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_SUCCESS; } + if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + execution->response = + cbm_mcp_index_worker_resource_response(job->args_json, &attempt->result); + application_attempt_free(attempt); + return APPLICATION_ATTEMPT_DECISION_STOP; + } if (disposition == CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL) { execution->unsafe_terminal = true; execution->supervision_failed = diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c index 0d2b986bf2..80aa2f8a65 100644 --- a/src/foundation/index_policy.c +++ b/src/foundation/index_policy.c @@ -1,21 +1,38 @@ #include "foundation/index_policy.h" +#include #include #include -static const char *const INDEX_POLICY_KEYS[] = { - CBM_INDEX_CONFIG_MAX_FILES, - CBM_INDEX_CONFIG_MAX_SOURCE_MB, +typedef struct { + const char *key; + size_t field_offset; + uint64_t minimum; + uint64_t maximum; + uint64_t multiplier; +} index_policy_metadata_t; + +static const index_policy_metadata_t INDEX_POLICY_METADATA[] = { + {CBM_INDEX_CONFIG_MAX_FILES, offsetof(cbm_index_resource_policy_t, max_files), 1, + CBM_INDEX_MAX_FILES_VALUE, 1}, + {CBM_INDEX_CONFIG_MAX_SOURCE_MB, offsetof(cbm_index_resource_policy_t, max_source_bytes), 1, + CBM_INDEX_MAX_SOURCE_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, offsetof(cbm_index_resource_policy_t, max_rss_bytes), + CBM_INDEX_MIN_RSS_MB_VALUE, CBM_INDEX_MAX_RSS_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, offsetof(cbm_index_resource_policy_t, max_duration_ms), + 1, CBM_INDEX_MAX_DURATION_SECONDS_VALUE, UINT64_C(1000)}, }; -static void set_error(char *error, size_t error_size, const char *key, uint64_t maximum) { +static void set_error(char *error, size_t error_size, const char *key, uint64_t minimum, + uint64_t maximum) { if (error && error_size > 0) { - (void)snprintf(error, error_size, "%s must be off or an integer from 1 to %llu", key, - (unsigned long long)maximum); + (void)snprintf(error, error_size, "%s must be off or an integer from %llu to %llu", key, + (unsigned long long)minimum, (unsigned long long)maximum); } } -static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t *parsed) { +static bool parse_bounded_uint64(const char *value, uint64_t minimum, uint64_t maximum, + uint64_t *parsed) { if (!value || !value[0] || !parsed) { return false; } @@ -30,7 +47,7 @@ static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t * } result = result * 10U + digit; } - if (result == 0) { + if (result < minimum) { return false; } *parsed = result; @@ -44,20 +61,39 @@ void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { } bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { + if (!policy) { + return false; + } + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + + INDEX_POLICY_METADATA[index].field_offset); + if (limit->enabled) { + return true; + } + } + return false; +} + +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy) { return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled); } +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_rss_bytes.enabled || policy->max_duration_ms.enabled); +} + size_t cbm_index_policy_key_count(void) { - return sizeof(INDEX_POLICY_KEYS) / sizeof(INDEX_POLICY_KEYS[0]); + return sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0]); } const char *cbm_index_policy_key_at(size_t index) { - return index < cbm_index_policy_key_count() ? INDEX_POLICY_KEYS[index] : NULL; + return index < cbm_index_policy_key_count() ? INDEX_POLICY_METADATA[index].key : NULL; } const char *cbm_index_policy_default_value(const char *key) { for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { - if (key && strcmp(key, INDEX_POLICY_KEYS[index]) == 0) { + if (key && strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { return "off"; } } @@ -70,34 +106,33 @@ bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, error[0] = '\0'; } if (!policy || !key || !value) { - set_error(error, error_size, key ? key : "index resource limit", 0); + set_error(error, error_size, key ? key : "index resource limit", 0, 0); return false; } - cbm_index_limit_u64_t *target = NULL; - uint64_t maximum = 0; - uint64_t multiplier = 1; - if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { - target = &policy->max_files; - maximum = CBM_INDEX_MAX_FILES_VALUE; - } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { - target = &policy->max_source_bytes; - maximum = CBM_INDEX_MAX_SOURCE_MB_VALUE; - multiplier = CBM_INDEX_MIB_BYTES; - } else { - set_error(error, error_size, key, 0); + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { + set_error(error, error_size, key, 0, 0); return false; } + cbm_index_limit_u64_t *target = + (cbm_index_limit_u64_t *)((unsigned char *)policy + metadata->field_offset); cbm_index_limit_u64_t candidate = {0}; if (strcmp(value, "off") != 0) { uint64_t parsed = 0; - if (!parse_bounded_uint64(value, maximum, &parsed)) { - set_error(error, error_size, key, maximum); + if (!parse_bounded_uint64(value, metadata->minimum, metadata->maximum, &parsed)) { + set_error(error, error_size, key, metadata->minimum, metadata->maximum); return false; } candidate.enabled = true; - candidate.value = parsed * multiplier; + candidate.value = parsed * metadata->multiplier; } *target = candidate; return true; @@ -108,18 +143,21 @@ bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const ch if (!policy || !key || !out || out_size == 0) { return false; } - const cbm_index_limit_u64_t *limit = NULL; - uint64_t divisor = 1; - if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { - limit = &policy->max_files; - } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { - limit = &policy->max_source_bytes; - divisor = CBM_INDEX_MIB_BYTES; - } else { + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { return false; } + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + metadata->field_offset); int length = limit->enabled - ? snprintf(out, out_size, "%llu", (unsigned long long)(limit->value / divisor)) + ? snprintf(out, out_size, "%llu", + (unsigned long long)(limit->value / metadata->multiplier)) : snprintf(out, out_size, "off"); return length >= 0 && (size_t)length < out_size; } @@ -130,6 +168,10 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { return "files"; case CBM_INDEX_RESOURCE_SOURCE_BYTES: return "source_bytes"; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return "rss_bytes"; + case CBM_INDEX_RESOURCE_DURATION_MS: + return "duration_ms"; case CBM_INDEX_RESOURCE_NONE: default: return "unknown"; @@ -137,7 +179,13 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { } const char *cbm_index_resource_unit(cbm_index_resource_t resource) { - return resource == CBM_INDEX_RESOURCE_FILES ? "files" : "bytes"; + if (resource == CBM_INDEX_RESOURCE_FILES) { + return "files"; + } + if (resource == CBM_INDEX_RESOURCE_DURATION_MS) { + return "milliseconds"; + } + return "bytes"; } const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { @@ -146,6 +194,10 @@ const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { return CBM_INDEX_CONFIG_MAX_FILES; case CBM_INDEX_RESOURCE_SOURCE_BYTES: return CBM_INDEX_CONFIG_MAX_SOURCE_MB; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return CBM_INDEX_CONFIG_MAX_RSS_MB; + case CBM_INDEX_RESOURCE_DURATION_MS: + return CBM_INDEX_CONFIG_MAX_DURATION_SECONDS; case CBM_INDEX_RESOURCE_NONE: default: return "index_resource_limit"; diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h index 6a6c499fc6..617a6ab7de 100644 --- a/src/foundation/index_policy.h +++ b/src/foundation/index_policy.h @@ -7,9 +7,14 @@ #define CBM_INDEX_CONFIG_MAX_FILES "index_max_files" #define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" +#define CBM_INDEX_CONFIG_MAX_RSS_MB "index_max_rss_mb" +#define CBM_INDEX_CONFIG_MAX_DURATION_SECONDS "index_max_duration_seconds" #define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) #define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MIN_RSS_MB_VALUE UINT64_C(64) +#define CBM_INDEX_MAX_RSS_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MAX_DURATION_SECONDS_VALUE UINT64_C(86400) #define CBM_INDEX_MIB_BYTES UINT64_C(1048576) typedef struct { @@ -20,12 +25,16 @@ typedef struct { typedef struct { cbm_index_limit_u64_t max_files; cbm_index_limit_u64_t max_source_bytes; + cbm_index_limit_u64_t max_rss_bytes; + cbm_index_limit_u64_t max_duration_ms; } cbm_index_resource_policy_t; typedef enum { CBM_INDEX_RESOURCE_NONE = 0, CBM_INDEX_RESOURCE_FILES, CBM_INDEX_RESOURCE_SOURCE_BYTES, + CBM_INDEX_RESOURCE_RSS_BYTES, + CBM_INDEX_RESOURCE_DURATION_MS, } cbm_index_resource_t; typedef struct { @@ -36,6 +45,8 @@ typedef struct { void cbm_index_policy_init(cbm_index_resource_policy_t *policy); bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy); size_t cbm_index_policy_key_count(void); const char *cbm_index_policy_key_at(size_t index); diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 5e3a549ed2..474ada1d2b 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -14,6 +14,7 @@ #include "platform.h" /* cbm_now_ms */ #include "sanitized.h" /* CBM_SANITIZED — spawn-retry budget */ +#include #include #include #include @@ -21,15 +22,18 @@ #ifdef _WIN32 #include +#include #include "win_utf8.h" /* cbm_utf8_to_wide — spawn the worker with a wide command line so a * non-ASCII repo path survives CreateProcess (#423/#20) */ #include /* free */ #else +#include #include #include #include #ifdef __APPLE__ #include +#include #include #endif extern char **environ; @@ -456,6 +460,287 @@ struct cbm_subprocess { #endif }; +static void cbm_rss_add_saturated(uint64_t *total, uint64_t value) { + if (value > UINT64_MAX - *total) { + *total = UINT64_MAX; + } else { + *total += value; + } +} + +#ifdef CBM_ENABLE_TEST_SEAMS +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count) { + uint64_t total = 0; + for (size_t index = 0; values && index < count; index++) { + cbm_rss_add_saturated(&total, values[index]); + } + return total; +} +#endif + +#ifdef _WIN32 +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DWORD capacity = 32; + JOBOBJECT_BASIC_PROCESS_ID_LIST *processes = NULL; + for (;;) { + size_t bytes = sizeof(*processes) + (size_t)(capacity - 1) * sizeof(ULONG_PTR); + if (bytes > UINT32_MAX) { + cbm_free(CBM_MEM_CLASS_OTHER, processes); + return CBM_PROC_TREE_RSS_ERROR; + } + JOBOBJECT_BASIC_PROCESS_ID_LIST *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, processes, bytes); + if (!grown) { + cbm_free(CBM_MEM_CLASS_OTHER, processes); + return CBM_PROC_TREE_RSS_ERROR; + } + processes = grown; + ZeroMemory(processes, bytes); + processes->NumberOfAssignedProcesses = capacity; + if (QueryInformationJobObject(process->job, JobObjectBasicProcessIdList, processes, + (DWORD)bytes, NULL)) { + break; + } + if (GetLastError() != ERROR_MORE_DATA || capacity > (UINT32_MAX / 2U)) { + cbm_free(CBM_MEM_CLASS_OTHER, processes); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2U; + } + + uint64_t total = 0; + DWORD measured = 0; + bool root_failed = false; + for (DWORD index = 0; index < processes->NumberOfProcessIdsInList; index++) { + DWORD pid = (DWORD)processes->ProcessIdList[index]; + HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); + if (!member) { + root_failed = root_failed || pid == process->process_id; + continue; + } + PROCESS_MEMORY_COUNTERS memory; + ZeroMemory(&memory, sizeof(memory)); + if (GetProcessMemoryInfo(member, &memory, sizeof(memory))) { + cbm_rss_add_saturated(&total, (uint64_t)memory.WorkingSetSize); + measured++; + } else if (pid == process->process_id) { + root_failed = true; + } + CloseHandle(member); + } + DWORD listed = processes->NumberOfProcessIdsInList; + cbm_free(CBM_MEM_CLASS_OTHER, processes); + if (root_failed || (measured == 0 && listed > 0)) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#elif defined(__APPLE__) +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + int capacity = proc_listallpids(NULL, 0); + if (capacity <= 0 || capacity > INT_MAX / (int)sizeof(pid_t) - 64) { + return CBM_PROC_TREE_RSS_ERROR; + } + capacity += 64; + pid_t *pids = NULL; + int count = 0; + for (;;) { + pid_t *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, pids, (size_t)capacity * sizeof(*pids)); + if (!grown) { + cbm_free(CBM_MEM_CLASS_OTHER, pids); + return CBM_PROC_TREE_RSS_ERROR; + } + pids = grown; + count = proc_listallpids(pids, capacity * (int)sizeof(*pids)); + if (count <= 0) { + cbm_free(CBM_MEM_CLASS_OTHER, pids); + return CBM_PROC_TREE_RSS_ERROR; + } + if (count < capacity) { + break; + } + if (capacity > INT_MAX / (int)sizeof(*pids) / 2) { + cbm_free(CBM_MEM_CLASS_OTHER, pids); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2; + } + + uint64_t total = 0; + int measured = 0; + bool root_failed = false; + for (int index = 0; index < count; index++) { + struct proc_bsdinfo info; + if (pids[index] <= 0) { + continue; + } + if (proc_pidinfo(pids[index], PROC_PIDTBSDINFO, 0, &info, sizeof(info)) != + (int)sizeof(info)) { + root_failed = root_failed || pids[index] == process->pid; + continue; + } + if ((pid_t)info.pbi_pgid != process->pgid) { + continue; + } + struct rusage_info_v2 usage; + if (proc_pid_rusage(pids[index], RUSAGE_INFO_V2, (rusage_info_t *)&usage) != 0) { + root_failed = root_failed || pids[index] == process->pid; + continue; + } + cbm_rss_add_saturated(&total, usage.ri_resident_size); + measured++; + } + cbm_free(CBM_MEM_CLASS_OTHER, pids); + if (root_failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#else +static bool cbm_proc_pid_name(const char *name) { + if (!name || !name[0]) { + return false; + } + for (const unsigned char *cursor = (const unsigned char *)name; *cursor; cursor++) { + if (*cursor < '0' || *cursor > '9') { + return false; + } + } + return true; +} + +static bool cbm_linux_proc_stat(const char *pid_name, pid_t *group, int64_t *rss_pages) { + char path[64]; + int length = snprintf(path, sizeof(path), "/proc/%s/stat", pid_name); + if (length <= 0 || length >= (int)sizeof(path)) { + return false; + } + FILE *stat_file = cbm_fopen(path, "rb"); + if (!stat_file) { + return false; + } + char stat_line[4096]; + bool read = fgets(stat_line, sizeof(stat_line), stat_file) != NULL; + (void)fclose(stat_file); + char *command_end = read ? strrchr(stat_line, ')') : NULL; + if (!command_end || command_end[1] != ' ') { + return false; + } + + char *save = NULL; + char *token = strtok_r(command_end + 2, " ", &save); + int field = 3; + bool have_group = false; + bool have_rss = false; + while (token) { + if (field == 5 || field == 24) { + char *end = NULL; + long long value = strtoll(token, &end, 10); + if (!end || *end != '\0') { + return false; + } + if (field == 5) { + *group = (pid_t)value; + have_group = true; + } else { + *rss_pages = (int64_t)value; + have_rss = true; + break; + } + } + token = strtok_r(NULL, " ", &save); + field++; + } + return have_group && have_rss; +} + +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DIR *proc = opendir("/proc"); + if (!proc) { + return CBM_PROC_TREE_RSS_ERROR; + } + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) { + (void)closedir(proc); + return CBM_PROC_TREE_RSS_ERROR; + } + + uint64_t total = 0; + int measured = 0; + bool root_failed = false; + struct dirent *entry; + while ((entry = readdir(proc)) != NULL) { + if (!cbm_proc_pid_name(entry->d_name)) { + continue; + } + pid_t group = 0; + int64_t pages = 0; + bool root_entry = strtol(entry->d_name, NULL, 10) == (long)process->pid; + if (!cbm_linux_proc_stat(entry->d_name, &group, &pages)) { + root_failed = root_failed || root_entry; + continue; + } + if (group != process->pgid || pages < 0) { + continue; + } + uint64_t page_count = (uint64_t)pages; + uint64_t bytes = page_count > UINT64_MAX / (uint64_t)page_size + ? UINT64_MAX + : page_count * (uint64_t)page_size; + cbm_rss_add_saturated(&total, bytes); + measured++; + } + (void)closedir(proc); + if (root_failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#endif + +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + if (!process || !rss_bytes) { + return CBM_PROC_TREE_RSS_ERROR; + } + *rss_bytes = 0; + return cbm_subprocess_tree_rss_platform(process, rss_bytes); +} + +bool cbm_subprocess_root_running(const cbm_subprocess_t *process) { + if (!process || process->root_reaped) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process) { + return process && process->termination_started; +} + +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process) { + if (!process) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + static void cbm_subprocess_result_init(cbm_proc_result_t *result) { result->outcome = CBM_PROC_SPAWN_FAILED; result->exit_code = -1; diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 4dc8f18705..bfcb2311b7 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -22,6 +22,7 @@ #include #include /* size_t (cbm_build_win_cmdline) */ +#include /* How a supervised child ended. */ typedef enum { @@ -118,6 +119,12 @@ typedef enum { CBM_PROC_POLL_TERMINAL = 1 } cbm_proc_poll_t; +typedef enum { + CBM_PROC_TREE_RSS_ERROR = -1, + CBM_PROC_TREE_RSS_EMPTY = 0, + CBM_PROC_TREE_RSS_OK = 1, +} cbm_proc_tree_rss_status_t; + /* Spawn opts->bin and return immediately with a supervisor handle. On success, * *out owns the process until a terminal poll followed by destroy. Spawn copies * the option strings/argv it needs after return; log_ud remains caller-owned until @@ -148,6 +155,16 @@ int cbm_subprocess_spawn(const cbm_proc_opts_t *opts, cbm_subprocess_t **out); * tree when cancel_grace_ms elapses. Callers must keep polling to make progress. */ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t *out); +/* Read the current resident-set size of the complete contained process tree. + * OK returns an overflow-safe byte total, EMPTY means the owned tree currently + * has no observable members, and ERROR means no trustworthy measurement could + * be obtained. Individual processes that exit during enumeration are ignored. */ +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes); +bool cbm_subprocess_root_running(const cbm_subprocess_t *process); +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process); +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process); + /* Record an explicit cancellation request without waiting. Safe to repeat and * safe to call from a cancellation thread while one owner thread polls. The * owner must stop cancellation producers before destroying the handle. true @@ -223,6 +240,7 @@ void cbm_subprocess_hold_parent_until_child_exits_for_testing(int spawns); * shows for a child that is EXITING: setpgid and getpgid answer ESRCH while * waitid does not report it yet. Test builds only. */ void cbm_subprocess_observe_exiting_child_for_testing(int spawns); +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count); #endif /* How the POSIX fork+exec child closed its inherited descriptors (#1484). */ diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 223945d3c3..4ffeecbb89 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -481,8 +481,56 @@ enum { INDEX_WORKER_SYNC_POLL_NS = 10000000, INDEX_WORKER_RELAY_LINES_PER_POLL = 64, INDEX_WORKER_RELAY_BYTES_PER_POLL = 64 * 1024, + INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT = 3, + INDEX_WORKER_RSS_PROBE_INTERVAL_MS = 250, }; +typedef enum { + INDEX_WORKER_TERMINATION_NONE = 0, + INDEX_WORKER_TERMINATION_CANCEL_PENDING, + INDEX_WORKER_TERMINATION_CANCEL, + INDEX_WORKER_TERMINATION_RESOURCE, +} index_worker_termination_reason_t; + +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_index_supervisor_clock_fn g_resource_clock; +static cbm_index_supervisor_rss_fn g_resource_rss; +static void *g_resource_hook_context; + +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context) { + g_resource_clock = clock_fn; + g_resource_rss = rss_fn; + g_resource_hook_context = context; +} + +void cbm_index_supervisor_reset_resource_hooks_for_testing(void) { + g_resource_clock = NULL; + g_resource_rss = NULL; + g_resource_hook_context = NULL; +} +#endif + +static uint64_t worker_resource_now_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_clock) { + return g_resource_clock(g_resource_hook_context); + } +#endif + return cbm_now_ms(); +} + +static cbm_proc_tree_rss_status_t worker_resource_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_rss) { + return g_resource_rss(process, rss_bytes, g_resource_hook_context); + } +#endif + return cbm_subprocess_tree_rss_bytes(process, rss_bytes); +} + struct cbm_index_worker_handle { cbm_subprocess_t *process; char response_path[INDEX_WORKER_PATH_CAP]; @@ -492,6 +540,12 @@ struct cbm_index_worker_handle { long relay_tail_pos; bool process_terminal; cbm_proc_result_t process_result; + cbm_index_resource_policy_t resource_policy; + uint64_t started_ms; + uint64_t last_rss_probe_ms; + unsigned int rss_probe_failures; + bool rss_probe_started; + atomic_int termination_reason; atomic_bool terminal; cbm_index_worker_result_t result; }; @@ -670,7 +724,8 @@ static bool worker_unique_file(char *out, size_t out_size, const char *kind) { static bool worker_result_succeeded(const cbm_index_worker_result_t *result) { return result && result->outcome == CBM_PROC_CLEAN && !result->cancellation_requested && - result->tree_quiesced && !result->supervision_failed; + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced && !result->supervision_failed; } static void worker_terminal_log(cbm_index_worker_handle_t *handle) { @@ -701,6 +756,13 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { } else if (handle->result.supervision_failed || !handle->result.tree_quiesced) { cbm_log_error("index.supervisor.containment_failed", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); + } else if (handle->result.resource_probe_failed) { + cbm_log_error("index.supervisor.resource_probe_failed", "resource", "rss_bytes", "log", + handle->log_path); + } else if (handle->result.resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + cbm_log_warn("index.supervisor.resource_limit", "resource", + cbm_index_resource_name(handle->result.resource_violation.resource), "log", + handle->log_path); } else if (handle->result.cancellation_requested) { cbm_log_warn("index.supervisor.worker_cancelled", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); @@ -739,10 +801,87 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes) { return memory_budget_bytes > SIZE_MAX - headroom ? SIZE_MAX : memory_budget_bytes + headroom; } -int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, - bool single_thread, const char *marker_file, - const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, cbm_index_worker_handle_t **handle_out) { +static void worker_request_resource_termination(cbm_index_worker_handle_t *handle, + cbm_index_resource_t resource, uint64_t observed, + uint64_t limit, bool probe_failed) { + int expected = INDEX_WORKER_TERMINATION_NONE; + if (!atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_RESOURCE, + memory_order_acq_rel, memory_order_acquire)) { + return; + } + handle->result.resource_violation = (cbm_index_resource_violation_t){ + .resource = resource, .observed = observed, .limit = limit}; + handle->result.resource_probe_failed = probe_failed; + if (!cbm_subprocess_request_cancel(handle->process)) { + handle->result.resource_violation = (cbm_index_resource_violation_t){0}; + handle->result.resource_probe_failed = false; + expected = INDEX_WORKER_TERMINATION_RESOURCE; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + } +} + +static bool worker_check_resource_limits(cbm_index_worker_handle_t *handle) { + if (atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != + INDEX_WORKER_TERMINATION_NONE || + cbm_subprocess_termination_pending(handle->process) || + !cbm_subprocess_supervision_active(handle->process)) { + return false; + } + uint64_t now = 0; + bool have_now = false; + bool rss_probe_failed = false; + if (handle->resource_policy.max_rss_bytes.enabled) { + now = worker_resource_now_ms(); + have_now = true; + bool probe_due = !handle->rss_probe_started || now < handle->last_rss_probe_ms || + now - handle->last_rss_probe_ms >= INDEX_WORKER_RSS_PROBE_INTERVAL_MS; + if (!probe_due) { + goto duration_check; + } + handle->rss_probe_started = true; + handle->last_rss_probe_ms = now; + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t status = worker_resource_rss(handle->process, &rss_bytes); + if (status == CBM_PROC_TREE_RSS_OK) { + handle->rss_probe_failures = 0; + if (rss_bytes > handle->resource_policy.max_rss_bytes.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, rss_bytes, + handle->resource_policy.max_rss_bytes.value, + false); + return false; + } + } else if (status == CBM_PROC_TREE_RSS_ERROR) { + handle->rss_probe_failures++; + if (handle->rss_probe_failures >= INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT) { + rss_probe_failed = true; + } + } else { + handle->rss_probe_failures = 0; + } + } +duration_check: + if (handle->resource_policy.max_duration_ms.enabled) { + if (!have_now) { + now = worker_resource_now_ms(); + } + uint64_t elapsed = now >= handle->started_ms ? now - handle->started_ms : 0; + if (elapsed > handle->resource_policy.max_duration_ms.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_DURATION_MS, elapsed, + handle->resource_policy.max_duration_ms.value, + false); + } + } + return rss_probe_failed; +} + +static int worker_start_internal(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; } @@ -774,6 +913,11 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ if (!handle) { return -1; } + cbm_index_policy_init(&handle->resource_policy); + if (resource_policy) { + handle->resource_policy = *resource_policy; + } + atomic_init(&handle->termination_reason, INDEX_WORKER_TERMINATION_NONE); atomic_init(&handle->terminal, false); handle->log_callback = log_callback; handle->log_context = log_context; @@ -840,15 +984,46 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ cbm_log_error("index.supervisor.spawn_failed", "action", "fail_closed"); return -1; } + if (handle->resource_policy.max_duration_ms.enabled) { + handle->started_ms = worker_resource_now_ms(); + } *handle_out = handle; return 0; } +int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, + quarantine_file, log_callback, log_context, handle_out); +} + int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { - return cbm_index_worker_start_with_log(args_json, memory_budget_bytes, single_thread, - marker_file, quarantine_file, NULL, NULL, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, + quarantine_file, NULL, NULL, handle_out); +} + +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, + marker_file, quarantine_file, NULL, NULL, handle_out); +} + +int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, + marker_file, quarantine_file, log_callback, log_context, + handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -865,10 +1040,16 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool relay_caught_up = true; if (!handle->process_terminal) { + bool rss_probe_failed = worker_check_resource_limits(handle); cbm_proc_result_t process_result; cbm_proc_poll_t state = cbm_subprocess_poll(handle->process, &process_result); relay_caught_up = worker_relay_log(handle); if (state == CBM_PROC_POLL_RUNNING) { + if (rss_probe_failed && cbm_subprocess_root_running(handle->process)) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, 0, + handle->resource_policy.max_rss_bytes.value, + true); + } return CBM_INDEX_WORKER_POLL_RUNNING; } if (state != CBM_PROC_POLL_TERMINAL) { @@ -889,7 +1070,12 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->result.outcome = process_result->outcome; handle->result.exit_code = process_result->exit_code; handle->result.term_signal = process_result->term_signal; - handle->result.cancellation_requested = process_result->cancellation_requested; + int termination_reason = + atomic_load_explicit(&handle->termination_reason, memory_order_acquire); + handle->result.cancellation_requested = + termination_reason == INDEX_WORKER_TERMINATION_CANCEL || + (termination_reason == INDEX_WORKER_TERMINATION_CANCEL_PENDING && + process_result->cancellation_requested); handle->result.forced = process_result->forced; handle->result.tree_quiesced = process_result->tree_quiesced; handle->result.supervision_failed = process_result->supervision_failed; @@ -922,8 +1108,36 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool cbm_index_worker_request_cancel(cbm_index_worker_handle_t *handle) { - return handle && !atomic_load_explicit(&handle->terminal, memory_order_acquire) && - cbm_subprocess_request_cancel(handle->process); + if (!handle || atomic_load_explicit(&handle->terminal, memory_order_acquire)) { + return false; + } + int expected = INDEX_WORKER_TERMINATION_NONE; + bool claimed = atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL_PENDING, + memory_order_acq_rel, memory_order_acquire); + if (!claimed) { + if (expected == INDEX_WORKER_TERMINATION_CANCEL) { + return true; + } + if (expected != INDEX_WORKER_TERMINATION_CANCEL_PENDING) { + return false; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + if (accepted) { + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL, + memory_order_acq_rel, memory_order_acquire); + } + return accepted; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + accepted ? INDEX_WORKER_TERMINATION_CANCEL + : INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + return accepted; } const char *cbm_index_worker_response_path(const cbm_index_worker_handle_t *handle) { @@ -948,18 +1162,19 @@ void cbm_index_worker_destroy(cbm_index_worker_handle_t *handle) { free(handle); } -int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, - const char *marker_file, const char *quarantine_file, - cbm_proc_log_cb log_callback, void *log_context, - const atomic_int *cancel_requested, - cbm_index_worker_result_t *result) { +static int worker_spawn_internal(const char *args_json, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; - if (cbm_index_worker_start_with_log(args_json, 0, single_thread, marker_file, quarantine_file, - log_callback, log_context, &handle) != 0) { + if (worker_start_internal(args_json, 0, resource_policy, single_thread, marker_file, + quarantine_file, log_callback, log_context, &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -985,6 +1200,24 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th return 0; } +int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, NULL, single_thread, marker_file, quarantine_file, + log_callback, log_context, cancel_requested, result); +} + +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, resource_policy, single_thread, marker_file, + quarantine_file, log_callback, log_context, cancel_requested, + result); +} + int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index c231a563e3..a86761d773 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -22,8 +22,10 @@ #include #include +#include #include +#include "foundation/index_policy.h" #include "foundation/subprocess.h" /* cbm_proc_outcome_t */ /* Worker-role state, set once from the CLI arg parser (main.c) when this process @@ -156,8 +158,10 @@ typedef struct { char last_phase[128]; char worker_log[4096]; bool response_rejected; /* clean worker exceeded the bounded response protocol */ - char *response; /* worker result only after a contained, uncancelled CLEAN exit; - * borrowed for async polls, caller-owned from the sync wrapper */ + cbm_index_resource_violation_t resource_violation; + bool resource_probe_failed; + char *response; /* worker result only after a contained, uncancelled CLEAN exit; + * borrowed for async polls, caller-owned from the sync wrapper */ } cbm_index_worker_result_t; /* Daemon-owned, nonblocking supervisor for one contained worker process tree. */ @@ -179,6 +183,21 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes); int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out); +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_index_worker_handle_t **handle_out); + +/* Policy plus the request-scoped log callback. The daemon default start path + * needs both: a missing policy fails closed, and worker diagnostics still + * reach the daemon journal. */ +int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + cbm_index_worker_handle_t **handle_out); /* Request-scoped variant used by interactive local CLI calls. The callback is * invoked by the owner thread while it polls the contained worker; log_context @@ -242,7 +261,22 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); +#ifdef CBM_ENABLE_TEST_SEAMS +typedef uint64_t (*cbm_index_supervisor_clock_fn)(void *context); +typedef cbm_proc_tree_rss_status_t (*cbm_index_supervisor_rss_fn)(cbm_subprocess_t *process, + uint64_t *rss_bytes, + void *context); +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context); +void cbm_index_supervisor_reset_resource_hooks_for_testing(void); +#endif + #endif /* CBM_INDEX_SUPERVISOR_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 1d1489e7fc..1016f781ab 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11205,6 +11205,22 @@ static bool build_index_success_response(cbm_mcp_server_t *srv, yyjson_mut_doc * return degraded; } +static bool project_db_is_servable(const char *project, const char *db_path); + +/* Serialize a worker JSON document and release the heap strings it borrowed. + * Extra arguments may be NULL; free(NULL) is a no-op. */ +static char *index_worker_json_result(yyjson_mut_doc *doc, char *repo_path, char *name_override, + char *project_name) { + char *json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + free(repo_path); + free(name_override); + free(project_name); + char *result = cbm_mcp_text_result(json, true); + free(json); + return result; +} + /* Build the response for a worker that crashed/hung/failed without producing a * result. The crash is already contained (this process survived); we report it * rather than dying. Precise skip-and-continue (quarantine the culprit, index the @@ -11237,12 +11253,7 @@ static char *build_worker_failure_response(const char *args, cbm_proc_outcome_t if (repo_path) { yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); } - char *json = yy_doc_to_str(doc); - yyjson_mut_doc_free(doc); - free(repo_path); - char *result = cbm_mcp_text_result(json, true); - free(json); - return result; + return index_worker_json_result(doc, repo_path, NULL, NULL); } static char *build_worker_unsafe_terminal_response(const char *args, cbm_proc_outcome_t outcome, @@ -11263,12 +11274,50 @@ static char *build_worker_unsafe_terminal_response(const char *args, cbm_proc_ou if (repo_path) { yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); } - char *json = yy_doc_to_str(doc); - yyjson_mut_doc_free(doc); - free(repo_path); - char *response = cbm_mcp_text_result(json, true); - free(json); - return response; + return index_worker_json_result(doc, repo_path, NULL, NULL); +} + +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result) { + char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); + char *name_override = cbm_mcp_get_string_arg(args, "name"); + char *project_name = + cbm_project_name_from_path(name_override && name_override[0] ? name_override : repo_path); + char db_path[CBM_SZ_1K] = {0}; + if (project_name) { + project_db_path(project_name, db_path, sizeof(db_path)); + } + + const cbm_index_resource_violation_t *violation = &worker_result->resource_violation; + const char *config_key = cbm_index_resource_config_key(violation->resource); + char message[CBM_SZ_256]; + (void)snprintf(message, sizeof(message), + worker_result->resource_probe_failed + ? "Worker resource measurement failed for %s" + : "Index worker exceeded %s", + config_key); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "code", + worker_result->resource_probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", "worker"); + yyjson_mut_obj_add_str(doc, root, "resource", cbm_index_resource_name(violation->resource)); + if (!worker_result->resource_probe_failed) { + yyjson_mut_obj_add_uint(doc, root, "observed", violation->observed); + yyjson_mut_obj_add_uint(doc, root, "limit", violation->limit); + yyjson_mut_obj_add_str(doc, root, "unit", cbm_index_resource_unit(violation->resource)); + } + yyjson_mut_obj_add_bool(doc, root, "retryable", true); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", + project_name && project_db_is_servable(project_name, db_path)); + yyjson_mut_obj_add_strcpy(doc, root, "message", message); + if (repo_path) { + yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); + } + return index_worker_json_result(doc, repo_path, name_override, project_name); } /* Drop the cached store so the next query reopens whatever the worker wrote (each @@ -11414,6 +11463,9 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( worker_result->supervision_failed) { return CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL; } + if (worker_result->resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + return CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE; + } if (worker_result->outcome == CBM_PROC_CLEAN) { return worker_result->response ? CBM_MCP_SUPERVISED_RESULT_SUCCESS : CBM_MCP_SUPERVISED_RESULT_FALLBACK; @@ -11421,8 +11473,8 @@ cbm_mcp_supervised_result_disposition_t cbm_mcp_supervised_result_disposition( return CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE; } -static bool index_policy_from_worker_args(const char *args, cbm_index_resource_policy_t *policy, - char *error, size_t error_size) { +bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; yyjson_val *encoded = @@ -11452,7 +11504,7 @@ static bool index_policy_from_worker_args(const char *args, cbm_index_resource_p static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, cbm_index_resource_policy_t *policy, char *error, size_t error_size) { if (cbm_index_worker_active()) { - return index_policy_from_worker_args(args, policy, error, error_size); + return cbm_mcp_index_policy_from_internal_args(args, policy, error, error_size); } cbm_config_t *owned_config = NULL; cbm_config_t *config = srv ? srv->config : NULL; @@ -11562,13 +11614,14 @@ char *cbm_mcp_index_worker_no_response_failure(const char *args, return cbm_mcp_text_result(written ? json : hint, true); } -static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { +static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, + const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; - int rc = cbm_index_spawn_worker_with_log_cancel( - args, false, NULL, NULL, srv ? srv->index_log_callback : NULL, + int rc = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, false, NULL, NULL, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); @@ -11593,6 +11646,12 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { invalidate_cached_store(srv); return failure; } + if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + char *failure = cbm_mcp_index_worker_resource_response(args, &wr); + cbm_index_worker_result_free(&wr); + invalidate_cached_store(srv); + return failure; + } if (disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { /* Clean exit → transfer the worker's response (the common path). */ char *resp = wr.response; /* transfer ownership to caller (may be NULL) */ @@ -11642,8 +11701,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { bool terminal_cancelled = false; for (int i = 0; i < cap; i++) { cbm_index_worker_result_t wr2; - int rc2 = cbm_index_spawn_worker_with_log_cancel( - args, /*single_thread=*/false, marker_path, quarantine_path, + int rc2 = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, /*single_thread=*/false, marker_path, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = @@ -11670,6 +11729,11 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { cbm_index_worker_result_free(&wr2); break; } + if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + resp = cbm_mcp_index_worker_resource_response(args, &wr2); + cbm_index_worker_result_free(&wr2); + break; + } if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { resp = wr2.response; /* transfer ownership to caller */ wr2.response = NULL; @@ -11764,8 +11828,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { * so it cannot itself hang. Rare given monotonic progress. */ if (!resp && !unsafe_terminal && quarantined > 0) { cbm_index_worker_result_t wrp; - int rcp = cbm_index_spawn_worker_with_log_cancel( - args, /*single_thread=*/false, NULL, quarantine_path, + int rcp = cbm_index_spawn_worker_with_policy_log_cancel( + args, resource_policy, /*single_thread=*/false, NULL, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = @@ -11781,6 +11845,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { last_outcome = wrp.outcome; unsafe_terminal = true; terminal_cancelled = wrp.cancellation_requested; + } else if (partial_disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { + resp = cbm_mcp_index_worker_resource_response(args, &wrp); } cbm_index_worker_result_free(&wrp); } @@ -11827,7 +11893,7 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p if (!args) { return NULL; } - char *resp = index_run_supervised(srv, args); + char *resp = index_run_supervised(srv, args, &policy); free(args); return resp; } @@ -12366,7 +12432,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result("failed to prepare supervised index request", true); } - char *supervised = index_run_supervised(srv, worker_args); + char *supervised = index_run_supervised(srv, worker_args, &resource_policy); free(worker_args); if (supervised) { free(mutation_project); diff --git a/src/mcp/mcp.h b/src/mcp/mcp.h index 3a5fce5ff8..a7284672b2 100644 --- a/src/mcp/mcp.h +++ b/src/mcp/mcp.h @@ -275,6 +275,7 @@ char *cbm_mcp_index_worker_no_response_failure(const char *args, typedef enum { CBM_MCP_SUPERVISED_RESULT_FALLBACK = 0, CBM_MCP_SUPERVISED_RESULT_SUCCESS, + CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE, CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE, CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL, } cbm_mcp_supervised_result_disposition_t; diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index 7035d80baf..3ca7c6ee9f 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -60,6 +60,10 @@ bool cbm_mcp_jsonrpc_response_prepend_notice(char **response_io, const char *not * must remove any untrusted field with the same name before invoking this. */ bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_index_resource_policy_t *policy); +bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, + char *error, size_t error_size); +char *cbm_mcp_index_worker_resource_response(const char *args, + const cbm_index_worker_result_t *worker_result); /* Encode a doc_candidates setting the user set to a boolean on an internal * worker request (the worker reads no config). Unset or unparseable adds no diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 92e23de64f..e14bfb7f76 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1195,7 +1195,8 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { } const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) { - return p && cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL; + return p && cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy + : NULL; } cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) { @@ -3941,7 +3942,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { .ignore_file = NULL, .max_file_size = 0, .resource_policy = - cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL, + cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy : NULL, .resource_violation = &p->resource_violation, }; cbm_file_info_t *files = NULL; diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index a923043ca4..5fe4e618f5 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1650,6 +1650,8 @@ typedef struct { cbm_project_lock_manager_t *project_locks; bool project_lock_busy_is_error; cbm_proc_outcome_t outcomes[APP_FAKE_MAX_ATTEMPTS]; + cbm_index_resource_violation_t resource_violations[APP_FAKE_MAX_ATTEMPTS]; + bool resource_probe_failures[APP_FAKE_MAX_ATTEMPTS]; const char *responses[APP_FAKE_MAX_ATTEMPTS]; const char *marker_payloads[APP_FAKE_MAX_ATTEMPTS]; char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; @@ -1754,7 +1756,9 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, - const char *max_files, const char *max_source_mb) { + const char *max_files, const char *max_source_mb, + const char *max_rss_mb, + const char *max_duration_seconds) { if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { return false; } @@ -1771,9 +1775,18 @@ static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, in yyjson_val *bytes = policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB) : NULL; - bool equal = files && bytes && yyjson_is_str(files) && yyjson_is_str(bytes) && + yyjson_val *rss = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_RSS_MB) + : NULL; + yyjson_val *duration = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) + : NULL; + bool equal = files && bytes && rss && duration && yyjson_is_str(files) && + yyjson_is_str(bytes) && yyjson_is_str(rss) && yyjson_is_str(duration) && strcmp(yyjson_get_str(files), max_files) == 0 && - strcmp(yyjson_get_str(bytes), max_source_mb) == 0; + strcmp(yyjson_get_str(bytes), max_source_mb) == 0 && + strcmp(yyjson_get_str(rss), max_rss_mb) == 0 && + strcmp(yyjson_get_str(duration), max_duration_seconds) == 0; yyjson_doc_free(document); return equal; } @@ -1815,6 +1828,12 @@ static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, worker->result.outcome = outcome; worker->result.exit_code = outcome == CBM_PROC_CLEAN ? 0 : -1; worker->result.tree_quiesced = true; + if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + worker->result.resource_violation = + worker->context->resource_violations[worker->attempt]; + worker->result.resource_probe_failed = + worker->context->resource_probe_failures[worker->attempt]; + } const char *response = worker->attempt < APP_FAKE_MAX_ATTEMPTS && worker->context->responses[worker->attempt] ? worker->context->responses[worker->attempt] @@ -2385,11 +2404,13 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool dirs_ok = cbm_mkdtemp(root) != NULL && cbm_mkdtemp(cache) != NULL; bool cache_set = dirs_ok && cache_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL; - bool config_ready = stored_config && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0; + bool config_ready = + stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -2434,7 +2455,8 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool first_owned = first_initialized && project && app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); - bool auto_policy_propagated = first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4"); + bool auto_policy_propagated = + first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7"); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2777,9 +2799,11 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { ASSERT_NOT_NULL(root); ASSERT_NOT_NULL(cache); cbm_config_t *stored_config = cbm_config_open(cache); - bool configured = stored_config && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0; + bool configured = + stored_config && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2799,7 +2823,8 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL; int index_rc = application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; - bool policy_propagated = index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6"); + bool policy_propagated = + index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6", "64", "7"); bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); cbm_daemon_application_free(application); @@ -2884,6 +2909,87 @@ TEST(daemon_application_programmatic_index_forwards_set_doc_candidates) { ASSERT_EQ(garbage, -1); PASS(); } + +TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { + char *cache = th_mktempdir("cbm_app_worker_resource_cache"); + cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.scripted, true); + fake.outcomes[0] = CBM_PROC_KILLED; + fake.resource_violations[0] = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_DURATION_MS, .observed = 7001, .limit = 7000}; + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = + stored_config ? cbm_daemon_application_new(&config) : NULL; + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 4188); + char *root = th_mktempdir("cbm_app_worker_resource"); + uint8_t *context = NULL; + uint32_t context_length = 0; + uint8_t *tool = NULL; + uint32_t tool_length = 0; + char args[APP_TEST_PATH_CAP + 96]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"DaemonWorkerResourceFixture\"}", + root ? root : ""); + bool setup = cache && stored_config && application && session && root && + app_test_context_request(root, root, &context, &context_length) && + app_test_tool_request("index_repository", args, &tool, &tool_length); + uint8_t *response = NULL; + uint32_t response_length = 0; + if (setup) { + uint8_t *context_response = NULL; + uint32_t context_response_length = 0; + setup = app_test_request(&callbacks, session, context, context_length, &context_response, + &context_response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(context_response); + } + cbm_daemon_runtime_application_status_t status = + setup + ? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length) + : CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR; + bool structured = response && strstr((char *)response, "resource_limit_exceeded") && + strstr((char *)response, "\\\"stage\\\":\\\"worker\\\"") && + strstr((char *)response, "\\\"resource\\\":\\\"duration_ms\\\"") && + strstr((char *)response, "\\\"observed\\\":7001") && + strstr((char *)response, "\\\"limit\\\":7000") && + strstr((char *)response, "\\\"unit\\\":\\\"milliseconds\\\""); + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + int starts = atomic_load(&fake.starts); + int destroys = atomic_load(&fake.destroys); + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + free(response); + free(context); + free(tool); + th_cleanup(root); + th_cleanup(cache); + + ASSERT_TRUE(setup); + ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); + ASSERT_TRUE(structured); + ASSERT_EQ(starts, 1); + ASSERT_EQ(destroys, 1); + ASSERT_TRUE(stopped); + PASS(); +} + TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -6953,6 +7059,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_unnamed_async_and_status_refuse_ambiguous_root_issue2134); RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); RUN_TEST(daemon_application_programmatic_index_forwards_set_doc_candidates); + RUN_TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index d866a06313..b33d64cdb0 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -21,9 +21,15 @@ TEST(index_policy_defaults_are_disabled) { ASSERT_FALSE(policy.max_files.enabled); ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(policy.max_rss_bytes.enabled); + ASSERT_FALSE(policy.max_duration_ms.enabled); ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_worker_enabled(&policy)); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_RSS_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_DURATION_SECONDS), "off"); PASS(); } @@ -60,6 +66,38 @@ TEST(index_policy_source_limit_converts_mib_without_overflow) { PASS(); } +TEST(index_policy_worker_limits_validate_and_convert_units) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE(cbm_index_policy_enabled(&policy)); + ASSERT_TRUE(cbm_index_policy_worker_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "1048576", error, + sizeof(error))); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(1048576) * CBM_INDEX_MIB_BYTES); + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "63", error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, 1000); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86400", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(86400000)); + before = policy; + ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86401", + error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + PASS(); +} + TEST(index_policy_invalid_value_is_rejected_atomically) { static const char *const invalid[] = {"", "0", "-1", "1MB", "1 ", "+1", "10000001", "18446744073709551616"}; @@ -95,6 +133,16 @@ TEST(index_policy_format_round_trips_public_values) { ASSERT_TRUE( cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value))); ASSERT_STR_EQ(value, "7"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "64"); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "9", error, + sizeof(error))); + ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, value, + sizeof(value))); + ASSERT_STR_EQ(value, "9"); PASS(); } @@ -103,10 +151,18 @@ TEST(index_policy_violation_metadata_is_stable) { ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_RSS_BYTES), "rss_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_DURATION_MS), "duration_ms"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_RSS_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_DURATION_MS), "milliseconds"); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES), CBM_INDEX_CONFIG_MAX_FILES); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES), CBM_INDEX_CONFIG_MAX_SOURCE_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_RSS_BYTES), + CBM_INDEX_CONFIG_MAX_RSS_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_DURATION_MS), + CBM_INDEX_CONFIG_MAX_DURATION_SECONDS); PASS(); } @@ -122,9 +178,13 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_FALSE(cbm_index_policy_enabled(&policy)); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7"), 0); ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_EQ(policy.max_files.value, 9); ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7000)); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); @@ -135,16 +195,23 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { PASS(); } -TEST(index_policy_cli_lists_both_operator_keys) { +TEST(index_policy_cli_lists_all_operator_keys) { bool files_found = false; bool bytes_found = false; + bool rss_found = false; + bool duration_found = false; for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { const char *key = cbm_cli_config_key_at_for_testing(index); files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + rss_found = rss_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_RSS_MB) == 0); + duration_found = + duration_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) == 0); } ASSERT_TRUE(files_found); ASSERT_TRUE(bytes_found); + ASSERT_TRUE(rss_found); + ASSERT_TRUE(duration_found); PASS(); } @@ -391,11 +458,12 @@ SUITE(index_policy) { RUN_TEST(index_policy_defaults_are_disabled); RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); + RUN_TEST(index_policy_worker_limits_validate_and_convert_units); RUN_TEST(index_policy_invalid_value_is_rejected_atomically); RUN_TEST(index_policy_format_round_trips_public_values); RUN_TEST(index_policy_violation_metadata_is_stable); RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption); - RUN_TEST(index_policy_cli_lists_both_operator_keys); + RUN_TEST(index_policy_cli_lists_all_operator_keys); RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); RUN_TEST(index_policy_cli_set_reports_a_failed_write); RUN_TEST(index_policy_worker_rejects_missing_parent_policy); diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index 1861c8076c..88d58c0d70 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -1019,6 +1019,269 @@ TEST(index_supervisor_job_memory_limit_has_floor_headroom_and_no_overflow) { PASS(); } +typedef struct { + uint64_t now_ms; + uint64_t rss_values[4]; + int rss_value_count; + int rss_calls; + int clock_calls; + cbm_proc_tree_rss_status_t rss_status; +} index_supervisor_resource_fake_t; + +static uint64_t index_supervisor_fake_clock(void *context) { + index_supervisor_resource_fake_t *fake = context; + fake->clock_calls++; + return fake->now_ms; +} + +static cbm_proc_tree_rss_status_t index_supervisor_fake_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes, void *context) { + (void)process; + index_supervisor_resource_fake_t *fake = context; + int value_index = + fake->rss_calls < fake->rss_value_count ? fake->rss_calls : fake->rss_value_count - 1; + fake->rss_calls++; + if (fake->rss_status == CBM_PROC_TREE_RSS_OK && rss_bytes && value_index >= 0) { + *rss_bytes = fake->rss_values[value_index]; + } + return fake->rss_status; +} + +static cbm_index_resource_policy_t index_supervisor_test_worker_policy(uint64_t rss_bytes, + uint64_t duration_ms) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_rss_bytes = (cbm_index_limit_u64_t){.enabled = rss_bytes > 0, .value = rss_bytes}; + policy.max_duration_ms = + (cbm_index_limit_u64_t){.enabled = duration_ms > 0, .value = duration_ms}; + return policy; +} + +TEST(index_supervisor_disabled_limits_do_not_probe) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {UINT64_MAX}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"clean\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + cbm_index_resource_t resource = + result ? result->resource_violation.resource : CBM_INDEX_RESOURCE_RSS_BYTES; + bool probe_failed = result && result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_EQ(fake.rss_calls, 0); + ASSERT_EQ(fake.clock_calls, 0); + ASSERT_EQ(resource, CBM_INDEX_RESOURCE_NONE); + ASSERT_FALSE(probe_failed); + PASS(); +} + +TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit, limit + 1}, + .rss_value_count = 2, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + const cbm_index_worker_result_t *cached = NULL; + bool cached_terminal = + terminal && cbm_index_worker_poll(handle, &cached) == CBM_INDEX_WORKER_POLL_TERMINAL && + cached == result; + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + result->resource_violation.observed == limit + 1 && + result->resource_violation.limit == limit && !result->cancellation_requested && + result->tree_quiesced && !result->supervision_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + ASSERT_TRUE(cached_terminal); + PASS(); +} + +TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + fake.now_ms = 1100; + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms = 1101; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + result->resource_violation.observed == 1001 && + result->resource_violation.limit == 1000 && result->outcome != CBM_PROC_HANG && + result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + PASS(); +} + +TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { + const char *saved_timeout = getenv("CBM_INDEX_WORKER_TIMEOUT_S"); + char *saved_timeout_copy = saved_timeout ? cbm_strdup(saved_timeout) : NULL; + (void)cbm_setenv("CBM_INDEX_WORKER_TIMEOUT_S", "1", 1); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 60000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool remained_hang = terminal && result && result->outcome == CBM_PROC_HANG && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + index_supervisor_test_restore_env("CBM_INDEX_WORKER_TIMEOUT_S", saved_timeout_copy); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_TRUE(remained_hang); + PASS(); +} + +TEST(index_supervisor_cancel_precedes_resource_probe) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit + 1}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + fake.now_ms = 2000; + bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool cancelled = terminal && result && result->cancellation_requested && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(cancelled); + ASSERT_EQ(fake.rss_calls, 0); + PASS(); +} + +TEST(index_supervisor_three_failed_rss_probes_fail_closed) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_ERROR, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t first = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + cbm_index_worker_poll_t throttled = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + int calls_before_interval = fake.rss_calls; + fake.now_ms += 250; + cbm_index_worker_poll_t second = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + cbm_index_worker_poll_t third = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool failed_closed = terminal && result && result->resource_probe_failed && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + !result->cancellation_requested && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(first, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(throttled, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(calls_before_interval, 1); + ASSERT_EQ(second, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(third, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(failed_closed); + ASSERT_EQ(fake.rss_calls, 3); + PASS(); +} + SUITE(index_supervisor) { RUN_TEST(index_supervisor_job_memory_limit_has_floor_headroom_and_no_overflow); RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar); @@ -1030,4 +1293,10 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_oversized_response_is_contained_and_log_is_retained); RUN_TEST(index_supervisor_clean_exit_without_response_is_named_failure_issue1300); RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run); + RUN_TEST(index_supervisor_disabled_limits_do_not_probe); + RUN_TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree); + RUN_TEST(index_supervisor_duration_is_total_time_not_quiet_timeout); + RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); + RUN_TEST(index_supervisor_cancel_precedes_resource_probe); + RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); } diff --git a/tests/test_mcp.c b/tests/test_mcp.c index ebf24a827f..3bfcb018ff 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -21581,6 +21581,103 @@ TEST(index_supervisor_clean_exit_without_response_response_issue1300) { PASS(); } +#ifndef _WIN32 +typedef struct { + int calls; +} index_worker_resource_clock_t; + +static uint64_t index_worker_resource_clock(void *context) { + index_worker_resource_clock_t *clock = context; + clock->calls++; + return clock->calls == 1 ? 100 : 1101; +} + +enum { + IDXRESOURCE_OK = 0, + IDXRESOURCE_SETUP = 71, + IDXRESOURCE_NO_RESPONSE = 72, + IDXRESOURCE_BAD_CONTRACT = 73, + IDXRESOURCE_RETRIED = 74, +}; + +static int index_worker_resource_response_check(const char *repo, const char *cache) { + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_unsetenv("CBM_INDEX_SUPERVISOR"); + cbm_index_supervisor_mark_host(); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (!config || !server || + cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1") != 0) { + cbm_mcp_server_free(server); + cbm_config_close(config); + return IDXRESOURCE_SETUP; + } + cbm_mcp_server_set_config(server, config); + index_worker_resource_clock_t clock = {0}; + cbm_index_supervisor_set_resource_hooks_for_testing(index_worker_resource_clock, NULL, &clock); + int before = cbm_index_supervisor_spawn_count(); + char args[CBM_SZ_2K]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"mode\":\"fast\"," + "\"_cbm_index_policy\":{\"index_max_files\":\"off\"," + "\"index_max_source_mb\":\"off\",\"index_max_rss_mb\":\"off\"," + "\"index_max_duration_seconds\":\"off\"}}", + repo); + char *response = cbm_mcp_handle_tool(server, "index_repository", args); + int after = cbm_index_supervisor_spawn_count(); + cbm_index_supervisor_reset_resource_hooks_for_testing(); + int result = IDXRESOURCE_OK; + if (!response) { + result = IDXRESOURCE_NO_RESPONSE; + } else if (!strstr(response, "resource_limit_exceeded") || + !strstr(response, "\\\"stage\\\":\\\"worker\\\"") || + !strstr(response, "\\\"resource\\\":\\\"duration_ms\\\"") || + !strstr(response, "\\\"observed\\\":1001") || + !strstr(response, "\\\"limit\\\":1000") || + !strstr(response, "\\\"unit\\\":\\\"milliseconds\\\"")) { + result = IDXRESOURCE_BAD_CONTRACT; + } else if (after - before != 1) { + result = IDXRESOURCE_RETRIED; + } + free(response); + cbm_mcp_server_free(server); + cbm_config_close(config); + return result; +} +#endif + +TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried) { +#ifdef _WIN32 + SKIP_PLATFORM("fork-isolated MCP host harness; supervisor state machine is cross-platform"); +#else + char *repo = th_mktempdir("cbm_worker_resource_repo"); + char *cache = th_mktempdir("cbm_worker_resource_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + fflush(NULL); + pid_t pid = fork(); + if (pid == 0) { + alarm(60); + _exit(index_worker_resource_response_check(repo, cache)); + } + int status = 0; + bool waited = waitpid(pid, &status, 0) == pid; + int exit_code = waited && WIFEXITED(status) ? WEXITSTATUS(status) : -1; + int signal_code = waited && WIFSIGNALED(status) ? WTERMSIG(status) : 0; + th_cleanup(repo); + th_cleanup(cache); + if (exit_code != IDXRESOURCE_OK) { + printf(" worker resource child exit=%d signal=%d\n", exit_code, signal_code); + } + ASSERT_TRUE(waited); + ASSERT_EQ(signal_code, 0); + ASSERT_EQ(exit_code, IDXRESOURCE_OK); + PASS(); +#endif +} + + /* Child-side check: index a tiny fixture and verify it ran IN-PROCESS. * Distinct exit codes so the parent can report the exact failure mode. */ enum { @@ -25140,6 +25237,7 @@ SUITE(mcp) { RUN_TEST(index_supervisor_gate_requires_marked_host_issue845); RUN_TEST(index_supervisor_start_failure_is_fail_closed_in_real_host); RUN_TEST(index_supervisor_clean_exit_without_response_response_issue1300); + RUN_TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried); RUN_TEST(index_bg_paths_route_through_supervisor_issue832); RUN_TEST(sequential_service_edge_props_are_valid_json_issue898); RUN_TEST(index_second_inprocess_run_survives_issue773); diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index c7f78b3d87..385293e085 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -101,6 +101,12 @@ TEST(subprocess_outcome_str) { PASS(); } +TEST(subprocess_tree_rss_sum_saturates_on_overflow) { + const uint64_t values[] = {UINT64_MAX - 4, 4, 1}; + ASSERT_EQ(cbm_subprocess_rss_sum_for_testing(values, 3), UINT64_MAX); + PASS(); +} + /* ── Layer 2: real spawn/reap (POSIX) ─────────────────────────────────────── */ extern const char *cbm_test_program_path; @@ -801,6 +807,45 @@ TEST(subprocess_spawn_returns_while_child_is_running) { #endif } +TEST(subprocess_tree_rss_measures_contained_descendants) { +#ifdef _WIN32 + SKIP_PLATFORM("native Job Object process-tree RSS runs on Windows CI"); +#else + char pid_path[64]; + ASSERT_TRUE(make_tree_pid_path(pid_path)); + cbm_subprocess_t *process = NULL; + ASSERT_EQ(spawn_ignoring_tree(pid_path, 0, 100, &process), 0); + pid_t parent_pid = 0; + pid_t grandchild_pid = 0; + bool ready = wait_for_tree_pids(pid_path, process, &parent_pid, &grandchild_pid, 3000); + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t rss_status = + ready ? cbm_subprocess_tree_rss_bytes(process, &rss_bytes) : CBM_PROC_TREE_RSS_ERROR; + bool cancel_accepted = cbm_subprocess_request_cancel(process); + cbm_proc_result_t result; + bool terminal = poll_until_terminal(process, 5000, &result); + uint64_t final_rss = UINT64_MAX; + cbm_proc_tree_rss_status_t final_rss_status = + terminal ? cbm_subprocess_tree_rss_bytes(process, &final_rss) : CBM_PROC_TREE_RSS_ERROR; + if (terminal) { + cbm_subprocess_destroy(process); + } else { + force_probe_cleanup(parent_pid, grandchild_pid); + } + (void)unlink(pid_path); + + ASSERT_TRUE(ready); + ASSERT_EQ(rss_status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(rss_bytes > 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(terminal); + ASSERT_TRUE(result.tree_quiesced); + ASSERT_EQ(final_rss_status, CBM_PROC_TREE_RSS_EMPTY); + ASSERT_EQ(final_rss, 0); + PASS(); +#endif +} + TEST(subprocess_natural_completion_is_cached_across_polls) { #ifdef _WIN32 SKIP_PLATFORM("POSIX /bin/sh completion-cache probe; native Job Object coverage pending"); @@ -2635,6 +2680,7 @@ SUITE(subprocess) { #ifndef _WIN32 RUN_TEST(subprocess_environment_path_preserves_caller_lookup); #endif + RUN_TEST(subprocess_tree_rss_sum_saturates_on_overflow); RUN_TEST(subprocess_run_clean); RUN_TEST(subprocess_run_exit_nonzero); RUN_TEST(subprocess_run_child_exited_before_group_check); @@ -2648,6 +2694,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_run_spawn_failure); RUN_TEST(subprocess_run_null_bin_rejected); RUN_TEST(subprocess_spawn_returns_while_child_is_running); + RUN_TEST(subprocess_tree_rss_measures_contained_descendants); RUN_TEST(subprocess_natural_completion_is_cached_across_polls); RUN_TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree); RUN_TEST(subprocess_quiet_timeout_kills_ignoring_tree); From a45858ffb5a53a3c4d7707d7a237f7f37d749aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Tue, 22 Sep 2026 01:28:50 +0800 Subject: [PATCH 2/6] fix(test): send the complete worker policy from the session-scope fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR 2257 still named only the two discovery keys in the worker-scope script. The worker now fail-closes unless every known policy key is present, so the fixture must stand in for the supervisor with cbm_test_index_worker_policy_json. Signed-off-by: 刘冲 --- tests/test_worker_session_scope.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_worker_session_scope.sh b/tests/test_worker_session_scope.sh index ab018ba765..1f32c46827 100644 --- a/tests/test_worker_session_scope.sh +++ b/tests/test_worker_session_scope.sh @@ -73,9 +73,9 @@ run_worker() { # A worker never reads the resource policy from config or environment: the # supervisor resolves it and sends it inside the request, and a request # without one is refused ("missing or incomplete trusted worker policy"). -# This script plays the supervisor, so it sends what the supervisor sends -- -# both limits off, the default. -policy='"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' +# This script plays the supervisor, so it sends the same complete object +# cbm_test_index_worker_policy_json produces -- every known key, all off. +policy="$(cbm_test_index_worker_policy_json)" response="${tmpdir}/scoped.response" if ! run_worker "{\"repo_path\":\"${repo}\",\"mode\":\"fast\",${policy}}" "${response}" \ From 1e4751d3ca4dab07fef38416729ff7b445d8cedf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Tue, 22 Sep 2026 03:05:13 +0800 Subject: [PATCH 3/6] fix(index): charge macOS phys_footprint and pin duration across recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Watchdogs now sample the OS-charged footprint instead of resident size on Darwin, and index_max_duration_seconds covers the whole request so a crash/hang respawn cannot reset the clock. Signed-off-by: 刘冲 --- README.md | 4 +- docs/CONFIGURATION.md | 4 +- docs/INDEX_RESOURCE_LIMITS.md | 28 ++++++---- src/cli/cli.c | 5 +- src/daemon/application.c | 13 +++-- src/daemon/application.h | 2 +- src/foundation/subprocess.c | 12 ++++- src/foundation/subprocess.h | 10 ++-- src/mcp/index_supervisor.c | 36 ++++++++----- src/mcp/index_supervisor.h | 12 +++-- src/mcp/mcp.c | 8 +-- tests/test_daemon_application.c | 2 + tests/test_index_supervisor.c | 94 ++++++++++++++++++++++++++------- 13 files changed, 166 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 4466dae50b..c34d20b982 100644 --- a/README.md +++ b/README.md @@ -928,8 +928,8 @@ codebase-memory-mcp config set watcher_enabled false # stop the watcher thre codebase-memory-mcp config set watch_non_git true # also poll non-git project roots (default: false) codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit -codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree current RSS limit -codebase-memory-mcp config set index_max_duration_seconds 3600 # optional total worker duration +codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree charged-memory limit +codebase-memory-mcp config set index_max_duration_seconds 3600 # optional per-request duration codebase-memory-mcp config reset auto_index # reset to default ``` diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3d3e9a9716..a02459b4ab 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -93,8 +93,8 @@ Current keys: | `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | -| `index_max_rss_mb` | `off` | Optional maximum current RSS in MiB for the complete contained index-worker process tree (`64..1048576`). | -| `index_max_duration_seconds` | `off` | Optional maximum total worker duration in seconds (`1..86400`). | +| `index_max_rss_mb` | `off` | Optional maximum charged memory in MiB for the complete contained index-worker process tree (`64..1048576`). On macOS this is phys_footprint, not resident size. | +| `index_max_duration_seconds` | `off` | Optional maximum wall-clock duration in seconds for the whole index request, including crash/hang recovery (`1..86400`). | For a watched project, `index_status` names the polling method in `watch.strategy`: `git` (HEAD and dirty-state polling), `tree` (a non-git root polled by the diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 66be4ecf46..1525993606 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -27,24 +27,30 @@ ranges are rejected without changing the stored value. | Key | Default | Accepted value | Protects | |---|---:|---:|---| -| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Current RSS of the complete worker process tree | -| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Total worker wall-clock duration | +| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Charged memory of the complete worker process tree | +| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Wall-clock duration of the whole index request | ```bash codebase-memory-mcp config set index_max_rss_mb 8192 codebase-memory-mcp config set index_max_duration_seconds 3600 ``` -RSS is the current resident memory of the contained worker and every descendant, -not the worker's allocation budget and not peak memory. This hard watchdog is +RSS is the charged memory of the contained worker and every descendant, not +the worker's allocation budget and not peak memory. On macOS that quantity is +`phys_footprint` (the same number `cbm_mem_charged()` enforces), not +`resident_size`, which still counts pages the allocator has already handed +back. Linux and Windows use RSS / working set. This hard watchdog is separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor -samples RSS at most once every 250 milliseconds so the watchdog does not turn -full process-table enumeration into a busy loop. - -Duration uses a monotonic clock from successful spawn. It is independent of the -existing 15-minute quiet timeout: continuous log progress does not reset total -duration, while the quiet timeout continues to identify a worker that stops -making progress. +samples the tree at most once every 250 milliseconds so the watchdog does not +turn full process-table enumeration into a busy loop. + +Duration is per request: the clock starts at the first worker spawn and is +not reset when crash/hang recovery starts a later attempt. Continuous log +progress does not reset it. It is independent of the existing 15-minute quiet +timeout, which still identifies a worker that stops making progress. A +duration limit shorter than that quiet timeout kills a hung worker before hang +quarantine can name the in-flight file, so the next attempt may hang on the +same file. Equality is allowed. The first RSS or elapsed-duration observation above its limit starts the existing graceful-to-force process-tree shutdown. CBM reports diff --git a/src/cli/cli.c b/src/cli/cli.c index 00b2db3410..fd51c48e9f 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7603,8 +7603,9 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, - {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree RSS MiB, or off"}, - {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max worker duration in seconds, or off"}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree charged memory MiB, or off"}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", + "Max wall-clock seconds for the whole index request, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing diff --git a/src/daemon/application.c b/src/daemon/application.c index a7b2f4f5b0..0f696d59c4 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -159,6 +159,7 @@ struct cbm_daemon_application_job { * job outlives the request and session that started it. */ bool async_owned; bool worker_started; + uint64_t request_started_ms; cbm_daemon_application_job_t *next; }; @@ -361,6 +362,7 @@ static void application_worker_diagnostic(const char *line, void *context) { static int application_worker_start_default(void *context, const char *args_json, size_t memory_budget_bytes, const char *marker_file, const char *quarantine_file, + uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out) { (void)context; cbm_index_resource_policy_t resource_policy; @@ -382,7 +384,7 @@ static int application_worker_start_default(void *context, const char *args_json yyjson_doc_free(args_doc); int result = cbm_index_worker_start_with_policy_log( args_json, memory_budget_bytes, &resource_policy, false, marker_file, quarantine_file, - application_worker_diagnostic, worker, &worker->handle); + application_worker_diagnostic, worker, duration_origin_ms, &worker->handle); if (result != 0) { application_worker_destroy_default(NULL, worker); *worker_out = NULL; @@ -1377,10 +1379,13 @@ static application_attempt_status_t application_job_run_attempt(cbm_daemon_appli } cbm_daemon_application_worker_t worker = NULL; + if (job->request_started_ms == 0) { + job->request_started_ms = cbm_index_worker_now_ms(); + } application_tmp_lock(); - int start_result = - application->worker_ops.start(application->worker_ops.context, job->args_json, - memory_budget_bytes, marker_path, quarantine_path, &worker); + int start_result = application->worker_ops.start( + application->worker_ops.context, job->args_json, memory_budget_bytes, marker_path, + quarantine_path, job->request_started_ms, &worker); application_tmp_unlock(); if (start_result != 0 || !worker) { return application_job_cancel_requested(job) ? APPLICATION_ATTEMPT_CANCELLED diff --git a/src/daemon/application.h b/src/daemon/application.h index 933186aa73..426bef3db9 100644 --- a/src/daemon/application.h +++ b/src/daemon/application.h @@ -31,7 +31,7 @@ typedef void *cbm_daemon_application_update_worker_t; typedef struct { void *context; int (*start)(void *context, const char *args_json, size_t memory_budget_bytes, - const char *marker_file, const char *quarantine_file, + const char *marker_file, const char *quarantine_file, uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out); cbm_index_worker_poll_t (*poll)(void *context, cbm_daemon_application_worker_t worker, const cbm_index_worker_result_t **result_out); diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 474ada1d2b..519a554e23 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -513,6 +513,12 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces bool root_failed = false; for (DWORD index = 0; index < processes->NumberOfProcessIdsInList; index++) { DWORD pid = (DWORD)processes->ProcessIdList[index]; + /* GetProcessMemoryInfo documents PROCESS_QUERY_INFORMATION | + * PROCESS_VM_READ. PROCESS_QUERY_LIMITED_INFORMATION is a lesser + * alternative on newer Windows; the Job Object membership list + * already closes most of the pid-reuse window between listing and + * opening. A member we cannot open is a probe failure when it is + * the root. */ HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); if (!member) { root_failed = root_failed || pid == process->process_id; @@ -592,7 +598,11 @@ static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subproces root_failed = root_failed || pids[index] == process->pid; continue; } - cbm_rss_add_saturated(&total, usage.ri_resident_size); + /* phys_footprint is what the OS charges, matching cbm_mem_charged(). + * ri_resident_size still counts pages mimalloc already marked + * MADV_FREE_REUSABLE, so a Mac worker can look like 17 GB RSS while + * the kernel holds it to 5.5 GB. */ + cbm_rss_add_saturated(&total, usage.ri_phys_footprint); measured++; } cbm_free(CBM_MEM_CLASS_OTHER, pids); diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index bfcb2311b7..94a8a2ed01 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -155,10 +155,12 @@ int cbm_subprocess_spawn(const cbm_proc_opts_t *opts, cbm_subprocess_t **out); * tree when cancel_grace_ms elapses. Callers must keep polling to make progress. */ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t *out); -/* Read the current resident-set size of the complete contained process tree. - * OK returns an overflow-safe byte total, EMPTY means the owned tree currently - * has no observable members, and ERROR means no trustworthy measurement could - * be obtained. Individual processes that exit during enumeration are ignored. */ +/* Read the current charged memory of the complete contained process tree. + * On macOS this is phys_footprint (what the OS holds against the process); + * elsewhere it is RSS. OK returns an overflow-safe byte total, EMPTY means + * the owned tree currently has no observable members, and ERROR means no + * trustworthy measurement could be obtained. Individual processes that exit + * during enumeration are ignored. */ cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, uint64_t *rss_bytes); bool cbm_subprocess_root_running(const cbm_subprocess_t *process); diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 4ffeecbb89..e84dff7557 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -881,7 +881,8 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, cbm_index_worker_handle_t **handle_out) { + void *log_context, uint64_t duration_origin_ms, + cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; } @@ -985,34 +986,42 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt return -1; } if (handle->resource_policy.max_duration_ms.enabled) { - handle->started_ms = worker_resource_now_ms(); + /* Duration is per request, not per spawn: a crash/hang recovery must + * not reset the clock. Callers pass the request origin; 0 means now. */ + uint64_t origin = duration_origin_ms != 0 ? duration_origin_ms : worker_resource_now_ms(); + handle->started_ms = origin; } *handle_out = handle; return 0; } +uint64_t cbm_index_worker_now_ms(void) { + return worker_resource_now_ms(); +} + int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, - quarantine_file, log_callback, log_context, handle_out); + quarantine_file, log_callback, log_context, 0, handle_out); } int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, - quarantine_file, NULL, NULL, handle_out); + quarantine_file, NULL, NULL, 0, handle_out); } int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, - const char *quarantine_file, + const char *quarantine_file, uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, - marker_file, quarantine_file, NULL, NULL, handle_out); + marker_file, quarantine_file, NULL, NULL, duration_origin_ms, + handle_out); } int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, @@ -1020,10 +1029,11 @@ int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_ bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out) { return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, marker_file, quarantine_file, log_callback, log_context, - handle_out); + duration_origin_ms, handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -1167,14 +1177,15 @@ static int worker_spawn_internal(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, - cbm_index_worker_result_t *result) { + uint64_t duration_origin_ms, cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; if (worker_start_internal(args_json, 0, resource_policy, single_thread, marker_file, - quarantine_file, log_callback, log_context, &handle) != 0) { + quarantine_file, log_callback, log_context, duration_origin_ms, + &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -1206,16 +1217,17 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { return worker_spawn_internal(args_json, NULL, single_thread, marker_file, quarantine_file, - log_callback, log_context, cancel_requested, result); + log_callback, log_context, cancel_requested, 0, result); } int cbm_index_spawn_worker_with_policy_log_cancel( const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { + void *log_context, const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result) { return worker_spawn_internal(args_json, resource_policy, single_thread, marker_file, quarantine_file, log_callback, log_context, cancel_requested, - result); + duration_origin_ms, result); } int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index a86761d773..2cb244a7ee 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -186,19 +186,24 @@ int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bo int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, - const char *quarantine_file, + const char *quarantine_file, uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out); /* Policy plus the request-scoped log callback. The daemon default start path * needs both: a missing policy fails closed, and worker diagnostics still - * reach the daemon journal. */ + * reach the daemon journal. duration_origin_ms is the request clock; 0 means now. */ int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out); +/* Supervisor clock used to stamp a request-scoped duration origin. Tests may + * replace it; production is cbm_now_ms(). */ +uint64_t cbm_index_worker_now_ms(void); + /* Request-scoped variant used by interactive local CLI calls. The callback is * invoked by the owner thread while it polls the contained worker; log_context * remains caller-owned until terminal. No process-global sink is installed. @@ -264,7 +269,8 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th int cbm_index_spawn_worker_with_policy_log_cancel( const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); + void *log_context, const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 1016f781ab..f23a00e5cd 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11617,12 +11617,14 @@ char *cbm_mcp_index_worker_no_response_failure(const char *args, static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); + uint64_t request_started_ms = cbm_index_worker_now_ms(); /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; int rc = cbm_index_spawn_worker_with_policy_log_cancel( args, resource_policy, false, NULL, NULL, srv ? srv->index_log_callback : NULL, - srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, &wr); + srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, + request_started_ms, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); @@ -11704,7 +11706,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, int rc2 = cbm_index_spawn_worker_with_policy_log_cancel( args, resource_policy, /*single_thread=*/false, marker_path, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, - srv ? &srv->pipeline_cancel_requested : NULL, &wr2); + srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = cbm_mcp_supervised_result_disposition(rc2, &wr2); if (wr2.response_missing) { @@ -11831,7 +11833,7 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, int rcp = cbm_index_spawn_worker_with_policy_log_cancel( args, resource_policy, /*single_thread=*/false, NULL, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, - srv ? &srv->pipeline_cancel_requested : NULL, &wrp); + srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = cbm_mcp_supervised_result_disposition(rcp, &wrp); if (partial_disposition == CBM_MCP_SUPERVISED_RESULT_SUCCESS) { diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 5fe4e618f5..5641ecbac5 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1703,7 +1703,9 @@ static void app_fake_worker_read_file(const char *path, char *out, size_t out_si static int app_fake_worker_start(void *opaque, const char *args_json, size_t memory_budget_bytes, const char *marker_file, const char *quarantine_file, + uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out) { + (void)duration_origin_ms; app_fake_worker_context_t *context = opaque; app_fake_worker_t *worker = calloc(1, sizeof(*worker)); if (!worker) { diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index 88d58c0d70..df55407da5 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -689,8 +689,8 @@ TEST(index_supervisor_worker_keeps_default_info_liveness_heartbeat) { (void)cbm_unsetenv("CBM_LOG_LEVEL"); cbm_index_worker_handle_t *handle = NULL; - int start_rc = cbm_index_worker_start("{\"__cbm_test_worker\":\"heartbeat\"}", 0, false, - NULL, NULL, &handle); + int start_rc = cbm_index_worker_start("{\"__cbm_test_worker\":\"heartbeat\"}", 0, false, NULL, + NULL, &handle); char log_path[INDEX_SUPERVISOR_TEST_PATH_CAP] = {0}; if (handle) { (void)snprintf(log_path, sizeof(log_path), "%s", cbm_index_worker_log_path(handle)); @@ -698,8 +698,8 @@ TEST(index_supervisor_worker_keeps_default_info_liveness_heartbeat) { bool ready = log_path[0] && index_supervisor_test_wait_file_text( log_path, "async worker heartbeat probe ready", INDEX_SUPERVISOR_TEST_READY_MS); - bool heartbeat = ready && index_supervisor_test_wait_file_text( - log_path, "msg=pipeline.discover", 1000); + bool heartbeat = + ready && index_supervisor_test_wait_file_text(log_path, "msg=pipeline.discover", 1000); const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); @@ -1034,8 +1034,9 @@ static uint64_t index_supervisor_fake_clock(void *context) { return fake->now_ms; } -static cbm_proc_tree_rss_status_t index_supervisor_fake_rss(cbm_subprocess_t *process, - uint64_t *rss_bytes, void *context) { +static cbm_proc_tree_rss_status_t index_supervisor_fake_charged_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes, + void *context) { (void)process; index_supervisor_resource_fake_t *fake = context; int value_index = @@ -1065,11 +1066,11 @@ TEST(index_supervisor_disabled_limits_do_not_probe) { .rss_status = CBM_PROC_TREE_RSS_OK, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 0); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"clean\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); @@ -1092,7 +1093,7 @@ TEST(index_supervisor_disabled_limits_do_not_probe) { PASS(); } -TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { +TEST(index_supervisor_charged_rss_equality_runs_then_excess_terminates_tree) { const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; index_supervisor_resource_fake_t fake = { .now_ms = 100, @@ -1101,11 +1102,11 @@ TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree) { .rss_status = CBM_PROC_TREE_RSS_OK, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; cbm_index_worker_poll_t equal_state = handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; @@ -1141,11 +1142,11 @@ TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { .rss_status = CBM_PROC_TREE_RSS_EMPTY, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); fake.now_ms = 1100; const cbm_index_worker_result_t *result = NULL; cbm_index_worker_poll_t equal_state = @@ -1171,6 +1172,60 @@ TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { PASS(); } +TEST(index_supervisor_duration_spans_recovery_attempts) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *first = NULL; + int first_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 100, &first); + fake.now_ms = 500; + const cbm_index_worker_result_t *first_result = NULL; + cbm_index_worker_poll_t mid = + first ? cbm_index_worker_poll(first, &first_result) : CBM_INDEX_WORKER_POLL_ERROR; + bool cancel_accepted = first && cbm_index_worker_request_cancel(first); + bool first_terminal = first && index_supervisor_test_poll_terminal( + first, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &first_result); + if (first_terminal) { + cbm_index_worker_destroy(first); + first = NULL; + } else { + index_supervisor_test_cleanup_handle(first); + first = NULL; + } + + fake.now_ms = 1101; + cbm_index_worker_handle_t *second = NULL; + int second_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 100, &second); + const cbm_index_worker_result_t *second_result = NULL; + bool second_terminal = second && index_supervisor_test_poll_terminal( + second, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &second_result); + bool limited = second_terminal && second_result && + second_result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + second_result->resource_violation.observed == 1001 && + second_result->resource_violation.limit == 1000 && + second_result->outcome != CBM_PROC_HANG && second_result->tree_quiesced; + if (second_terminal) { + cbm_index_worker_destroy(second); + } else { + index_supervisor_test_cleanup_handle(second); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(first_rc, 0); + ASSERT_EQ(mid, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(first_terminal); + ASSERT_EQ(second_rc, 0); + ASSERT_TRUE(limited); + PASS(); +} + TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { const char *saved_timeout = getenv("CBM_INDEX_WORKER_TIMEOUT_S"); char *saved_timeout_copy = saved_timeout ? cbm_strdup(saved_timeout) : NULL; @@ -1179,7 +1234,7 @@ TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 60000); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; bool terminal = handle && index_supervisor_test_poll_terminal( handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); @@ -1208,11 +1263,11 @@ TEST(index_supervisor_cancel_precedes_resource_probe) { .rss_status = CBM_PROC_TREE_RSS_OK, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 1000); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); fake.now_ms = 2000; bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); const cbm_index_worker_result_t *result = NULL; @@ -1242,11 +1297,11 @@ TEST(index_supervisor_three_failed_rss_probes_fail_closed) { .rss_status = CBM_PROC_TREE_RSS_ERROR, }; cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, - index_supervisor_fake_rss, &fake); + index_supervisor_fake_charged_rss, &fake); cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); cbm_index_worker_handle_t *handle = NULL; int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, - &policy, false, NULL, NULL, &handle); + &policy, false, NULL, NULL, 0, &handle); const cbm_index_worker_result_t *result = NULL; cbm_index_worker_poll_t first = handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; @@ -1294,8 +1349,9 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_clean_exit_without_response_is_named_failure_issue1300); RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run); RUN_TEST(index_supervisor_disabled_limits_do_not_probe); - RUN_TEST(index_supervisor_rss_equality_runs_then_excess_terminates_tree); + RUN_TEST(index_supervisor_charged_rss_equality_runs_then_excess_terminates_tree); RUN_TEST(index_supervisor_duration_is_total_time_not_quiet_timeout); + RUN_TEST(index_supervisor_duration_spans_recovery_attempts); RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); RUN_TEST(index_supervisor_cancel_precedes_resource_probe); RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); From 225a8b1583df24e7221a20b37c6b923e36a68492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 4/6] feat(index): enforce opt-in storage resource limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An index that fits in memory and finishes in time can still fill the disk. Publication needs room for the staging artifacts and the final database at the same time, and running out of space during publication is the one failure that can cost a working index. Add index_cache_max_mb and index_min_free_disk_mb, measured before staging and again before publication, together with internal ceilings on the final database, the staging artifacts and the task temporary directory. Those three have no public keys because they are only meaningful as part of one composed decision. Crossing any of them fails the attempt before the old database is touched, so atomic publication is unchanged and the previous index keeps serving. Staging cleanup is scoped by a private per-task token, so a worker removes only the artifacts it created and never a concurrent run's. An old database is treated as replaceable only after an integrity verdict distinguishes real corruption from a transient busy error. A probe that cannot complete fails closed. Both public keys default to off. The shell fixture that stands in for the supervisor names them too, so the worker still recognises the policy it is handed. Signed-off-by: 刘冲 --- README.md | 10 +- docs/CONFIGURATION.md | 17 +- docs/INDEX_RESOURCE_LIMITS.md | 29 ++- scripts/test-runtime.sh | 3 +- src/cli/cli.c | 5 +- src/daemon/application.c | 14 +- src/daemon/bootstrap.c | 21 ++ src/foundation/compat_fs.c | 311 ++++++++++++++++++++++++ src/foundation/compat_fs.h | 19 ++ src/foundation/index_policy.c | 92 ++++++- src/foundation/index_policy.h | 32 +++ src/main.c | 6 +- src/mcp/index_supervisor.c | 185 ++++++++++++-- src/mcp/index_supervisor.h | 23 +- src/mcp/mcp.c | 83 +++++-- src/mcp/mcp_internal.h | 1 + src/pipeline/pipeline.c | 400 ++++++++++++++++++++++++++++--- src/pipeline/pipeline.h | 15 ++ src/pipeline/pipeline_internal.h | 3 + tests/test_daemon_application.c | 62 ++++- tests/test_index_policy.c | 172 +++++++++++++ tests/test_index_supervisor.c | 77 +++++- tests/test_main.c | 2 +- tests/test_mcp.c | 91 ++++++- tests/test_pipeline.c | 209 ++++++++++++++++ tests/test_platform.c | 93 +++++++ 26 files changed, 1845 insertions(+), 130 deletions(-) diff --git a/README.md b/README.md index c34d20b982..17825dfff4 100644 --- a/README.md +++ b/README.md @@ -930,12 +930,16 @@ codebase-memory-mcp config set index_max_files 250000 # optional per-index so codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree charged-memory limit codebase-memory-mcp config set index_max_duration_seconds 3600 # optional per-request duration +codebase-memory-mcp config set index_cache_max_mb 32768 # optional projected cache-size limit +codebase-memory-mcp config set index_min_free_disk_mb 4096 # optional free-space reserve codebase-memory-mcp config reset auto_index # reset to default ``` -The four `index_max_*` settings default to `off`. Exceeding one fails the complete -index attempt rather than publishing a partial graph; an existing serving index -is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). +The six index resource settings default to `off`. Exceeding one fails the +complete index attempt rather than publishing a partial graph; an existing +serving index is preserved. Storage probes also fail closed when an enabled +measurement cannot be completed. See +[Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). ### Environment Variables diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index a02459b4ab..7d45a85b21 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -95,6 +95,8 @@ Current keys: | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | | `index_max_rss_mb` | `off` | Optional maximum charged memory in MiB for the complete contained index-worker process tree (`64..1048576`). On macOS this is phys_footprint, not resident size. | | `index_max_duration_seconds` | `off` | Optional maximum wall-clock duration in seconds for the whole index request, including crash/hang recovery (`1..86400`). | +| `index_cache_max_mb` | `off` | Optional maximum projected cache size in MiB after publication. | +| `index_min_free_disk_mb` | `off` | Optional minimum free MiB reserved on the cache filesystem while indexing. | For a watched project, `index_status` names the polling method in `watch.strategy`: `git` (HEAD and dirty-state polling), `tree` (a non-git root polled by the @@ -128,13 +130,14 @@ has not run yet). > `auto_index` still runs, and `index_repository` stays available for manual > reindexing. -The four `index_max_*` settings are independent and disabled by default. They -apply to explicit indexing, automatic indexing, and watcher re-indexing, but not -to `cross-repo-intelligence`, which does not scan repository source files. -Equality is allowed; exceeding any setting fails the complete index request and -preserves any previously serving database. Worker RSS covers descendants and is -not the same as the internal `CBM_MEM_BUDGET_MB` allocation budget. Total -duration is independent of the existing 15-minute no-log-progress timeout. See +The six index resource settings are independent and disabled by default. +They apply to explicit indexing, automatic indexing, and watcher re-indexing, +but not to `cross-repo-intelligence`, which does not scan or publish repository +source indexes. Equality is allowed; exceeding a setting fails the complete +index request and preserves any previously serving database. Worker RSS covers +descendants and is not the same as the internal `CBM_MEM_BUDGET_MB` allocation +budget. Total duration is independent of the existing 15-minute no-log-progress +timeout. See [Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and error-response details. diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 1525993606..74cce0bd95 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -100,6 +100,31 @@ Worker limit failures use the same shape with `stage=worker`, and omit `observed`, `limit`, and `unit` because no trustworthy observation was available. +## Storage settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_cache_max_mb` | `off` | `off` or `1..1048576` | Projected cache bytes after replacement | +| `index_min_free_disk_mb` | `off` | `off` or `1..1048576` | Free bytes reserved on the cache filesystem | + +Set or reset these keys through the same `config set` and `config reset` +commands. With both keys `off`, indexing does not scan the cache tree or probe +filesystem capacity. + +Projected cache usage is the current cache size, minus the old project +database and SQLite sidecars only when that generation is confirmed valid and +replaceable, plus the current operation's staging artifacts. Other projects +and unrelated files always count toward the limit and are never evicted. + +Free space is checked before staging, after the staged build completes, and +immediately before atomic publication. Equality is allowed. An enabled +measurement that cannot be completed fails closed with +`code: "resource_probe_failed"` and +`stage: "storage"`. A limit breach uses `code: "resource_limit_exceeded"`. +Both cases preserve the old serving database. Staging files created by a +terminated supervised worker are tagged with a private task token and removed +after its process tree is quiescent; cleanup cannot match another attempt. + ## Trust and compatibility Limits are read from the CLI-managed `_config.db`; they are not MCP request @@ -110,5 +135,5 @@ parent policy. These settings do not replace or increase `auto_index_limit`, change the 512 MiB single-file cap, alter workspace-root authorization, or affect `cross-repo-intelligence`. With all settings `off`, discovery follows the -existing path and the supervisor performs no periodic RSS probe or total-duration -termination. +existing path, the supervisor performs no periodic RSS probe or total-duration +termination, and the pipeline performs no cache-tree or free-space probe. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index e6e60b15a7..b7e3b885a3 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -113,7 +113,8 @@ _cbm_test_runtime_daemon() { # nothing else. cbm_test_index_worker_policy_json() { printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"' - printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"}' + printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"' + printf '%s' ',"index_cache_max_mb":"off","index_min_free_disk_mb":"off"}' } cbm_test_runtime_cleanup() { diff --git a/src/cli/cli.c b/src/cli/cli.c index fd51c48e9f..759512e626 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -67,7 +67,7 @@ enum { MIN_ARGC_CMD = 3, /* minimum argc for subcommand with arg */ /* sqlite3_bind parameter index 2 */ /* 10 MB cap factor */ - CLI_MB_FACTOR = CLI_BUF_1K * CLI_BUF_1K, + CLI_MB_FACTOR = CLI_BUF_1K * CLI_BUF_1K, NUM_RETRIES = 5, NUM_DIRS = 4, DECOMP_FACTOR = 10, @@ -7606,6 +7606,9 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree charged memory MiB, or off"}, {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", "Max wall-clock seconds for the whole index request, or off"}, + {CBM_INDEX_CONFIG_CACHE_MAX_MB, "off", "Max projected cache MiB after publish, or off"}, + {CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "off", + "Minimum free cache-filesystem MiB during indexing, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing diff --git a/src/daemon/application.c b/src/daemon/application.c index 0f696d59c4..e4d801e29c 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -367,9 +367,12 @@ static int application_worker_start_default(void *context, const char *args_json (void)context; cbm_index_resource_policy_t resource_policy; char error[CBM_SZ_256] = {0}; + char task_db_path[CBM_SZ_1K]; if (!cbm_mcp_index_policy_from_internal_args(args_json, &resource_policy, error, - sizeof(error))) { - cbm_log_error("daemon.index.policy", "error", error); + sizeof(error)) || + !cbm_mcp_index_task_db_path(args_json, task_db_path, sizeof(task_db_path))) { + cbm_log_error("daemon.index.worker_policy", "error", + error[0] ? error : "could not resolve worker index path"); *worker_out = NULL; return -1; } @@ -382,9 +385,10 @@ static int application_worker_start_default(void *context, const char *args_json const char *root = yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(args_doc), "repo_path")); worker->project = application_index_project_key(root, args_json); yyjson_doc_free(args_doc); - int result = cbm_index_worker_start_with_policy_log( - args_json, memory_budget_bytes, &resource_policy, false, marker_file, quarantine_file, - application_worker_diagnostic, worker, duration_origin_ms, &worker->handle); + int result = cbm_index_worker_start_with_storage_policy( + args_json, memory_budget_bytes, &resource_policy, task_db_path, false, marker_file, + quarantine_file, application_worker_diagnostic, worker, duration_origin_ms, + &worker->handle); if (result != 0) { application_worker_destroy_default(NULL, worker); *worker_out = NULL; diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index cb384cd20c..db8422ecae 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -107,6 +107,21 @@ static bool bootstrap_worker_budget_valid(const char *text) { return value > 0; } +static bool bootstrap_worker_stage_token_valid(const char *token) { + size_t length = token ? strlen(token) : 0; + if (length < 6U || length >= 64U) { + return false; + } + for (size_t index = 0; index < length; index++) { + unsigned char ch = (unsigned char)token[index]; + if (!((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || + ch == '-' || ch == '_')) { + return false; + } + } + return true; +} + /* Keep the bootstrap role boundary exact and fail closed before any client or * worker state is initialized. index_supervisor owns the matching builder and * performs the captured-build comparison after this syntax-only classification. */ @@ -141,6 +156,12 @@ static bool bootstrap_worker_argv_exact(int argc, char *const argv[]) { } next += 2; } + if (next < argc && bootstrap_arg_is(argv[next], "--index-worker-stage-token")) { + if (next + 1 >= argc || !bootstrap_worker_stage_token_valid(argv[next + 1])) { + return false; + } + next += 2; + } return next == argc; } diff --git a/src/foundation/compat_fs.c b/src/foundation/compat_fs.c index 604faa50ff..f1e0b03ab8 100644 --- a/src/foundation/compat_fs.c +++ b/src/foundation/compat_fs.c @@ -163,6 +163,24 @@ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { return CBM_PATH_INFO_OK; } +bool cbm_filesystem_free_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + wchar_t *wpath = cbm_path_to_wide(path); + if (!wpath) { + return false; + } + ULARGE_INTEGER available; + BOOL ok = GetDiskFreeSpaceExW(wpath, &available, NULL, NULL); + free(wpath); + if (!ok) { + return false; + } + *bytes_out = available.QuadPart; + return true; +} + void cbm_closedir(cbm_dir_t *d) { if (d) { if (d->find_handle != INVALID_HANDLE_VALUE) { @@ -786,6 +804,7 @@ int cbm_exec_no_shell(const char *const *argv) { #include #include #include +#include #include #include @@ -875,6 +894,23 @@ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out) { return CBM_PATH_INFO_OK; } +bool cbm_filesystem_free_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + struct statvfs status; + if (statvfs(path, &status) != 0) { + return false; + } + uint64_t blocks = (uint64_t)status.f_bavail; + uint64_t fragment = status.f_frsize != 0 ? (uint64_t)status.f_frsize : (uint64_t)status.f_bsize; + if (fragment == 0) { + return false; + } + *bytes_out = blocks > UINT64_MAX / fragment ? UINT64_MAX : blocks * fragment; + return true; +} + void cbm_closedir(cbm_dir_t *d) { if (d) { if (d->dir) { @@ -1258,6 +1294,281 @@ int cbm_exec_no_shell(const char *const *argv) { #endif /* _WIN32 */ +static uint64_t fs_add_saturated(uint64_t left, uint64_t right) { + return left > UINT64_MAX - right ? UINT64_MAX : left + right; +} + +static bool directory_size_recursive(const char *path, unsigned int depth, uint64_t *total) { + enum { CBM_DIRECTORY_SCAN_MAX_DEPTH = 1024 }; + if (!path || !total || depth > CBM_DIRECTORY_SCAN_MAX_DEPTH) { + return false; + } + cbm_dir_t *directory = cbm_opendir(path); + if (!directory) { + return false; + } + bool ok = true; + cbm_dirent_t *entry; + while (ok && (entry = cbm_readdir(directory)) != NULL) { + size_t path_len = strlen(path); + size_t name_len = strlen(entry->name); + bool separator_needed = + path_len > 0 && path[path_len - 1] != '/' && path[path_len - 1] != '\\'; + if (path_len > SIZE_MAX - name_len - (separator_needed ? 2U : 1U)) { + ok = false; + break; + } + size_t child_size = path_len + name_len + (separator_needed ? 2U : 1U); + char *child = (char *)malloc(child_size); + if (!child) { + ok = false; + break; + } + int written = + snprintf(child, child_size, separator_needed ? "%s/%s" : "%s%s", path, entry->name); + cbm_path_info_t info; + int status = written > 0 && (size_t)written < child_size ? cbm_path_info_utf8(child, &info) + : CBM_PATH_INFO_UNAVAILABLE; + if (status == CBM_PATH_INFO_UNAVAILABLE) { + ok = false; + } else if (status == CBM_PATH_INFO_OK && !info.is_symlink && info.is_regular) { + if (info.size < 0) { + ok = false; + } else { + *total = fs_add_saturated(*total, (uint64_t)info.size); + } + } else if (status == CBM_PATH_INFO_OK && !info.is_symlink && info.is_directory) { + ok = directory_size_recursive(child, depth + 1U, total); + } + free(child); + } + cbm_closedir(directory); + return ok; +} + +bool cbm_directory_size_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + *bytes_out = 0; + return directory_size_recursive(path, 0, bytes_out); +} + +bool cbm_db_artifact_bytes(const char *db_path, uint64_t *bytes_out) { + if (!db_path || !bytes_out) { + return false; + } + *bytes_out = 0; + static const char *const suffixes[] = {"", "-wal", "-shm", "-journal"}; + size_t base_len = strlen(db_path); + for (size_t index = 0; index < sizeof(suffixes) / sizeof(suffixes[0]); index++) { + size_t suffix_len = strlen(suffixes[index]); + if (base_len > SIZE_MAX - suffix_len - 1U) { + return false; + } + size_t path_size = base_len + suffix_len + 1U; + char *path = (char *)malloc(path_size); + if (!path) { + return false; + } + (void)snprintf(path, path_size, "%s%s", db_path, suffixes[index]); + cbm_path_info_t info; + int status = cbm_path_info_utf8(path, &info); + free(path); + if (status == CBM_PATH_INFO_ABSENT) { + continue; + } + if (status != CBM_PATH_INFO_OK || !info.is_regular || info.is_symlink || info.size < 0) { + return false; + } + *bytes_out = fs_add_saturated(*bytes_out, (uint64_t)info.size); + } + return true; +} + +static bool fs_optional_regular_bytes(const char *path, uint64_t *total) { + if (!path || !path[0]) { + return true; + } + cbm_path_info_t info; + int status = cbm_path_info_utf8(path, &info); + if (status == CBM_PATH_INFO_ABSENT) { + return true; + } + if (status != CBM_PATH_INFO_OK || !info.is_regular || info.is_symlink || info.size < 0) { + return false; + } + *total = fs_add_saturated(*total, (uint64_t)info.size); + return true; +} + +bool cbm_index_task_temp_bytes(const char *final_db_path, const char *stage_token, + const char *log_path, const char *response_path, + uint64_t *bytes_out) { + if (!final_db_path || !final_db_path[0] || !stage_token || !stage_token[0] || !bytes_out) { + return false; + } + *bytes_out = 0; + if (!fs_optional_regular_bytes(log_path, bytes_out) || + !fs_optional_regular_bytes(response_path, bytes_out)) { + return false; + } + const char *basename = strrchr(final_db_path, '/'); +#ifdef _WIN32 + const char *backslash = strrchr(final_db_path, '\\'); + if (backslash && (!basename || backslash > basename)) { + basename = backslash; + } +#endif + basename = basename ? basename + 1 : final_db_path; + size_t directory_len = (size_t)(basename - final_db_path); + char *directory = NULL; + if (directory_len == 0) { + directory = strdup("."); + directory_len = 1; + } else { + directory = (char *)malloc(directory_len + 1U); + if (directory) { + memcpy(directory, final_db_path, directory_len); + while (directory_len > 1U && (directory[directory_len - 1U] == '/' || + directory[directory_len - 1U] == '\\')) { + directory_len--; + } + directory[directory_len] = '\0'; + } + } + size_t basename_len = strlen(basename); + size_t token_len = strlen(stage_token); + static const char stage_marker[] = ".stage."; + if (!directory || token_len > SIZE_MAX - sizeof(stage_marker) - 1U || + basename_len > SIZE_MAX - token_len - sizeof(stage_marker) - 1U) { + free(directory); + return false; + } + size_t prefix_size = basename_len + (sizeof(stage_marker) - 1U) + token_len + 2U; + char *prefix = (char *)malloc(prefix_size); + if (!prefix) { + free(directory); + return false; + } + (void)snprintf(prefix, prefix_size, "%s%s%s.", basename, stage_marker, stage_token); + cbm_dir_t *dir = cbm_opendir(directory); + if (!dir) { + free(prefix); + free(directory); + return false; + } + bool ok = true; + cbm_dirent_t *entry; + while (ok && (entry = cbm_readdir(dir)) != NULL) { + if (strncmp(entry->name, prefix, prefix_size - 1U) != 0) { + continue; + } + size_t name_len = strlen(entry->name); + if (directory_len > SIZE_MAX - name_len - 2U) { + ok = false; + break; + } + size_t path_size = directory_len + name_len + 2U; + char *path = (char *)malloc(path_size); + if (!path) { + ok = false; + break; + } + (void)snprintf(path, path_size, "%s/%s", directory, entry->name); + ok = fs_optional_regular_bytes(path, bytes_out); + free(path); + } + cbm_closedir(dir); + free(prefix); + free(directory); + return ok; +} + +bool cbm_index_staging_cleanup(const char *final_db_path, const char *stage_token) { + if (!final_db_path || !final_db_path[0] || !stage_token || !stage_token[0]) { + return false; + } + const char *basename = strrchr(final_db_path, '/'); +#ifdef _WIN32 + const char *backslash = strrchr(final_db_path, '\\'); + if (backslash && (!basename || backslash > basename)) { + basename = backslash; + } +#endif + basename = basename ? basename + 1 : final_db_path; + size_t directory_len = (size_t)(basename - final_db_path); + char *directory = NULL; + if (directory_len == 0) { + directory = strdup("."); + directory_len = 1; + } else { + directory = (char *)malloc(directory_len + 1U); + if (directory) { + memcpy(directory, final_db_path, directory_len); + while (directory_len > 1U && (directory[directory_len - 1U] == '/' || + directory[directory_len - 1U] == '\\')) { + directory_len--; + } + directory[directory_len] = '\0'; + } + } + static const char stage_marker[] = ".stage."; + size_t basename_len = strlen(basename); + size_t token_len = strlen(stage_token); + if (!directory || token_len > SIZE_MAX - sizeof(stage_marker) - 1U || + basename_len > SIZE_MAX - token_len - sizeof(stage_marker) - 1U) { + free(directory); + return false; + } + size_t prefix_size = basename_len + (sizeof(stage_marker) - 1U) + token_len + 2U; + char *prefix = (char *)malloc(prefix_size); + if (!prefix) { + free(directory); + return false; + } + (void)snprintf(prefix, prefix_size, "%s%s%s.", basename, stage_marker, stage_token); + cbm_dir_t *dir = cbm_opendir(directory); + if (!dir) { + free(prefix); + free(directory); + return false; + } + bool ok = true; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + if (strncmp(entry->name, prefix, prefix_size - 1U) != 0) { + continue; + } + size_t name_len = strlen(entry->name); + if (directory_len > SIZE_MAX - name_len - 2U) { + ok = false; + continue; + } + size_t path_size = directory_len + name_len + 2U; + char *path = (char *)malloc(path_size); + if (!path) { + ok = false; + continue; + } + (void)snprintf(path, path_size, "%s/%s", directory, entry->name); + cbm_path_info_t info; + int status = cbm_path_info_utf8(path, &info); + if (status == CBM_PATH_INFO_OK && !info.is_symlink && info.is_regular) { + if (cbm_unlink(path) != 0 && errno != ENOENT) { + ok = false; + } + } else if (status != CBM_PATH_INFO_ABSENT) { + ok = false; + } + free(path); + } + cbm_closedir(dir); + free(prefix); + free(directory); + return ok; +} + /* Canonicalize an EXISTING path (collapse `..`, resolve links/junctions): * realpath on POSIX; a final path queried from an opened handle on Windows. * The previous Windows callers used the ANSI CRT (_access/_fullpath) on UTF-8 diff --git a/src/foundation/compat_fs.h b/src/foundation/compat_fs.h index 1a572a4785..8264c795c8 100644 --- a/src/foundation/compat_fs.h +++ b/src/foundation/compat_fs.h @@ -47,6 +47,25 @@ enum { * their behavior. */ int cbm_path_info_utf8(const char *path, cbm_path_info_t *out); +/* Measure logical regular-file bytes beneath path without following symlinks + * or Windows reparse points. Saturates at UINT64_MAX. */ +bool cbm_directory_size_bytes(const char *path, uint64_t *bytes_out); + +/* Return bytes available to the current process on the filesystem containing + * an existing path (statvfs / GetDiskFreeSpaceExW). */ +bool cbm_filesystem_free_bytes(const char *path, uint64_t *bytes_out); + +/* Sum a SQLite main file and its -wal/-shm/-journal sidecars when present. + * Missing artifacts contribute zero; inspection errors fail the measurement. */ +bool cbm_db_artifact_bytes(const char *db_path, uint64_t *bytes_out); + +/* Sum this indexing attempt's explicit worker log/response plus every regular + * sibling whose name starts with ".stage.". */ +bool cbm_index_task_temp_bytes(const char *final_db_path, const char *stage_token, + const char *log_path, const char *response_path, + uint64_t *bytes_out); +bool cbm_index_staging_cleanup(const char *final_db_path, const char *stage_token); + /* Open a directory for iteration. Returns NULL on error. */ cbm_dir_t *cbm_opendir(const char *path); diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c index 80aa2f8a65..9c0f22e9c0 100644 --- a/src/foundation/index_policy.c +++ b/src/foundation/index_policy.c @@ -21,6 +21,10 @@ static const index_policy_metadata_t INDEX_POLICY_METADATA[] = { CBM_INDEX_MIN_RSS_MB_VALUE, CBM_INDEX_MAX_RSS_MB_VALUE, CBM_INDEX_MIB_BYTES}, {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, offsetof(cbm_index_resource_policy_t, max_duration_ms), 1, CBM_INDEX_MAX_DURATION_SECONDS_VALUE, UINT64_C(1000)}, + {CBM_INDEX_CONFIG_CACHE_MAX_MB, offsetof(cbm_index_resource_policy_t, max_cache_bytes), 1, + CBM_INDEX_MAX_STORAGE_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, offsetof(cbm_index_resource_policy_t, min_free_disk_bytes), + 1, CBM_INDEX_MAX_STORAGE_MB_VALUE, CBM_INDEX_MIB_BYTES}, }; static void set_error(char *error, size_t error_size, const char *key, uint64_t minimum, @@ -61,18 +65,8 @@ void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { } bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { - if (!policy) { - return false; - } - for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { - const cbm_index_limit_u64_t *limit = - (const cbm_index_limit_u64_t *)((const unsigned char *)policy + - INDEX_POLICY_METADATA[index].field_offset); - if (limit->enabled) { - return true; - } - } - return false; + return cbm_index_policy_discovery_enabled(policy) || cbm_index_policy_worker_enabled(policy) || + cbm_index_policy_storage_enabled(policy); } bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy) { @@ -83,6 +77,62 @@ bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy) return policy && (policy->max_rss_bytes.enabled || policy->max_duration_ms.enabled); } +bool cbm_index_policy_storage_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_cache_bytes.enabled || policy->min_free_disk_bytes.enabled || + policy->max_final_db_bytes.enabled || policy->max_staging_bytes.enabled || + policy->max_task_temp_bytes.enabled); +} + +static uint64_t add_saturated(uint64_t left, uint64_t right) { + return left > UINT64_MAX - right ? UINT64_MAX : left + right; +} + +static bool storage_within_max(cbm_index_resource_t resource, uint64_t observed, + const cbm_index_limit_u64_t *limit, + cbm_index_resource_violation_t *violation) { + if (!limit->enabled || observed <= limit->value) { + return true; + } + *violation = (cbm_index_resource_violation_t){ + .resource = resource, + .observed = observed, + .limit = limit->value, + }; + return false; +} + +bool cbm_index_policy_check_storage(const cbm_index_resource_policy_t *policy, + const cbm_index_storage_sample_t *sample, + cbm_index_resource_violation_t *violation) { + if (!policy || !sample || !violation) { + return false; + } + *violation = (cbm_index_resource_violation_t){0}; + uint64_t remaining_cache = sample->current_cache_bytes > sample->replaceable_old_bytes + ? sample->current_cache_bytes - sample->replaceable_old_bytes + : 0; + uint64_t projected_cache = add_saturated(remaining_cache, sample->operation_bytes); + if (!storage_within_max(CBM_INDEX_RESOURCE_CACHE_BYTES, projected_cache, + &policy->max_cache_bytes, violation)) { + return false; + } + if (policy->min_free_disk_bytes.enabled && + sample->free_disk_bytes < policy->min_free_disk_bytes.value) { + *violation = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES, + .observed = sample->free_disk_bytes, + .limit = policy->min_free_disk_bytes.value, + }; + return false; + } + return storage_within_max(CBM_INDEX_RESOURCE_FINAL_DB_BYTES, sample->final_db_bytes, + &policy->max_final_db_bytes, violation) && + storage_within_max(CBM_INDEX_RESOURCE_STAGING_BYTES, sample->staging_bytes, + &policy->max_staging_bytes, violation) && + storage_within_max(CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, sample->task_temp_bytes, + &policy->max_task_temp_bytes, violation); +} + size_t cbm_index_policy_key_count(void) { return sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0]); } @@ -172,6 +222,16 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { return "rss_bytes"; case CBM_INDEX_RESOURCE_DURATION_MS: return "duration_ms"; + case CBM_INDEX_RESOURCE_CACHE_BYTES: + return "cache_bytes"; + case CBM_INDEX_RESOURCE_FREE_DISK_BYTES: + return "free_disk_bytes"; + case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: + return "final_db_bytes"; + case CBM_INDEX_RESOURCE_STAGING_BYTES: + return "staging_bytes"; + case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: + return "task_temp_bytes"; case CBM_INDEX_RESOURCE_NONE: default: return "unknown"; @@ -198,6 +258,14 @@ const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { return CBM_INDEX_CONFIG_MAX_RSS_MB; case CBM_INDEX_RESOURCE_DURATION_MS: return CBM_INDEX_CONFIG_MAX_DURATION_SECONDS; + case CBM_INDEX_RESOURCE_CACHE_BYTES: + return CBM_INDEX_CONFIG_CACHE_MAX_MB; + case CBM_INDEX_RESOURCE_FREE_DISK_BYTES: + return CBM_INDEX_CONFIG_MIN_FREE_DISK_MB; + case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: + case CBM_INDEX_RESOURCE_STAGING_BYTES: + case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: + return "index_resource_profile"; case CBM_INDEX_RESOURCE_NONE: default: return "index_resource_limit"; diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h index 617a6ab7de..bac059f2e4 100644 --- a/src/foundation/index_policy.h +++ b/src/foundation/index_policy.h @@ -9,12 +9,15 @@ #define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" #define CBM_INDEX_CONFIG_MAX_RSS_MB "index_max_rss_mb" #define CBM_INDEX_CONFIG_MAX_DURATION_SECONDS "index_max_duration_seconds" +#define CBM_INDEX_CONFIG_CACHE_MAX_MB "index_cache_max_mb" +#define CBM_INDEX_CONFIG_MIN_FREE_DISK_MB "index_min_free_disk_mb" #define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) #define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) #define CBM_INDEX_MIN_RSS_MB_VALUE UINT64_C(64) #define CBM_INDEX_MAX_RSS_MB_VALUE UINT64_C(1048576) #define CBM_INDEX_MAX_DURATION_SECONDS_VALUE UINT64_C(86400) +#define CBM_INDEX_MAX_STORAGE_MB_VALUE UINT64_C(1048576) #define CBM_INDEX_MIB_BYTES UINT64_C(1048576) typedef struct { @@ -27,6 +30,15 @@ typedef struct { cbm_index_limit_u64_t max_source_bytes; cbm_index_limit_u64_t max_rss_bytes; cbm_index_limit_u64_t max_duration_ms; + cbm_index_limit_u64_t max_cache_bytes; + cbm_index_limit_u64_t min_free_disk_bytes; + /* Internal storage dimensions with no public config key. Nothing enables + * them on its own; they exist so that a single composed decision can bound + * the database, the staging artifacts and the task temporary directory + * together, which no individual key can express. */ + cbm_index_limit_u64_t max_final_db_bytes; + cbm_index_limit_u64_t max_staging_bytes; + cbm_index_limit_u64_t max_task_temp_bytes; } cbm_index_resource_policy_t; typedef enum { @@ -35,18 +47,38 @@ typedef enum { CBM_INDEX_RESOURCE_SOURCE_BYTES, CBM_INDEX_RESOURCE_RSS_BYTES, CBM_INDEX_RESOURCE_DURATION_MS, + CBM_INDEX_RESOURCE_CACHE_BYTES, + CBM_INDEX_RESOURCE_FREE_DISK_BYTES, + CBM_INDEX_RESOURCE_FINAL_DB_BYTES, + CBM_INDEX_RESOURCE_STAGING_BYTES, + CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, } cbm_index_resource_t; typedef struct { cbm_index_resource_t resource; uint64_t observed; uint64_t limit; + bool probe_failed; } cbm_index_resource_violation_t; +typedef struct { + uint64_t current_cache_bytes; + uint64_t replaceable_old_bytes; + uint64_t operation_bytes; + uint64_t free_disk_bytes; + uint64_t final_db_bytes; + uint64_t staging_bytes; + uint64_t task_temp_bytes; +} cbm_index_storage_sample_t; + void cbm_index_policy_init(cbm_index_resource_policy_t *policy); bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy); bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_storage_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_check_storage(const cbm_index_resource_policy_t *policy, + const cbm_index_storage_sample_t *sample, + cbm_index_resource_violation_t *violation); size_t cbm_index_policy_key_count(void); const char *cbm_index_policy_key_at(size_t index); diff --git a/src/main.c b/src/main.c index 54bad875eb..019fe59e0b 100644 --- a/src/main.c +++ b/src/main.c @@ -897,6 +897,8 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo const char *worker_marker = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_MARKER_ARG); const char *worker_quarantine = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_QUARANTINE_ARG); + const char *worker_stage_token = + cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_STAGE_TOKEN_ARG); if (index_worker) { /* The graph lives on this process's heaps: SQLite gets heaps of its * own here, and only here (cbm_sqlite_dedicated_heap). */ @@ -905,7 +907,7 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo } cbm_index_set_worker_role_options(index_worker, response_out, worker_single_thread, worker_marker, worker_quarantine, - cbm_index_worker_memory_budget_bytes()); + cbm_index_worker_memory_budget_bytes(), worker_stage_token); if (argc < MAIN_MIN_ARGC) { (void)fprintf(stderr, CLI_USAGE); @@ -3187,7 +3189,7 @@ int main(int argc, char **argv) { } cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread, invocation.marker_file, invocation.quarantine_file, - invocation.memory_budget_bytes); + invocation.memory_budget_bytes, invocation.stage_token); #ifndef _WIN32 /* Split into three ordered steps rather than one condition, because the * ORDER is load-bearing and the middle step only exists in test builds: diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index e84dff7557..a612f7780d 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -69,7 +69,7 @@ static void worker_set_local_env(const char *name, const char *value) { void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, bool single_thread, const char *marker_file, const char *quarantine_file, - size_t memory_budget_bytes) { + size_t memory_budget_bytes, const char *stage_token) { cbm_index_set_worker_role(is_worker, response_out); g_worker_memory_budget_bytes = is_worker ? memory_budget_bytes : 0; if (!is_worker) { @@ -78,6 +78,7 @@ void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, worker_set_local_env("CBM_INDEX_SINGLE_THREAD", single_thread ? "1" : NULL); worker_set_local_env("CBM_INDEX_MARKER_FILE", marker_file); worker_set_local_env("CBM_INDEX_QUARANTINE_FILE", quarantine_file); + worker_set_local_env("CBM_INDEX_STAGE_TOKEN", stage_token); } bool cbm_index_worker_active(void) { @@ -173,6 +174,21 @@ static bool worker_fingerprint_valid(const char *fingerprint) { return true; } +static bool worker_stage_token_valid(const char *token) { + size_t length = token ? strlen(token) : 0; + if (length < 6U || length >= CBM_SZ_64) { + return false; + } + for (size_t index = 0; index < length; index++) { + unsigned char ch = (unsigned char)token[index]; + if (!((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || + ch == '-' || ch == '_')) { + return false; + } + } + return true; +} + static bool worker_parse_positive_size(const char *text, size_t *value_out) { if (!text || !text[0] || !value_out) { return false; @@ -250,6 +266,13 @@ cbm_index_worker_argv_status_t cbm_index_worker_parse_process_argv( parsed.quarantine_file = argv[next + 1]; next += 2; } + if (next < argc && argv[next] && strcmp(argv[next], CBM_INDEX_WORKER_STAGE_TOKEN_ARG) == 0) { + if (next + 1 >= argc || !worker_stage_token_valid(argv[next + 1])) { + return CBM_INDEX_WORKER_ARGV_INVALID; + } + parsed.stage_token = argv[next + 1]; + next += 2; + } if (next != argc) { return CBM_INDEX_WORKER_ARGV_INVALID; } @@ -477,12 +500,13 @@ static void worker_log_last_phase(const char *log_path, char *out, size_t out_si enum { INDEX_WORKER_PATH_CAP = CBM_SZ_4K, - INDEX_WORKER_ARGV_CAP = 17, + INDEX_WORKER_ARGV_CAP = 19, INDEX_WORKER_SYNC_POLL_NS = 10000000, INDEX_WORKER_RELAY_LINES_PER_POLL = 64, INDEX_WORKER_RELAY_BYTES_PER_POLL = 64 * 1024, INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT = 3, INDEX_WORKER_RSS_PROBE_INTERVAL_MS = 250, + INDEX_WORKER_TASK_TEMP_PROBE_INTERVAL_MS = 250, }; typedef enum { @@ -496,6 +520,8 @@ typedef enum { static cbm_index_supervisor_clock_fn g_resource_clock; static cbm_index_supervisor_rss_fn g_resource_rss; static void *g_resource_hook_context; +static cbm_index_supervisor_task_temp_fn g_task_temp_hook; +static void *g_task_temp_hook_context; void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, cbm_index_supervisor_rss_fn rss_fn, @@ -510,6 +536,17 @@ void cbm_index_supervisor_reset_resource_hooks_for_testing(void) { g_resource_rss = NULL; g_resource_hook_context = NULL; } + +void cbm_index_supervisor_set_task_temp_hook_for_testing( + cbm_index_supervisor_task_temp_fn task_temp_fn, void *context) { + g_task_temp_hook = task_temp_fn; + g_task_temp_hook_context = context; +} + +void cbm_index_supervisor_reset_task_temp_hook_for_testing(void) { + g_task_temp_hook = NULL; + g_task_temp_hook_context = NULL; +} #endif static uint64_t worker_resource_now_ms(void) { @@ -541,10 +578,14 @@ struct cbm_index_worker_handle { bool process_terminal; cbm_proc_result_t process_result; cbm_index_resource_policy_t resource_policy; + char task_db_path[INDEX_WORKER_PATH_CAP]; + char stage_token[CBM_SZ_64]; uint64_t started_ms; uint64_t last_rss_probe_ms; + uint64_t last_task_temp_probe_ms; unsigned int rss_probe_failures; bool rss_probe_started; + bool task_temp_probe_started; atomic_int termination_reason; atomic_bool terminal; cbm_index_worker_result_t result; @@ -811,7 +852,7 @@ static void worker_request_resource_termination(cbm_index_worker_handle_t *handl return; } handle->result.resource_violation = (cbm_index_resource_violation_t){ - .resource = resource, .observed = observed, .limit = limit}; + .resource = resource, .observed = observed, .limit = limit, .probe_failed = probe_failed}; handle->result.resource_probe_failed = probe_failed; if (!cbm_subprocess_request_cancel(handle->process)) { handle->result.resource_violation = (cbm_index_resource_violation_t){0}; @@ -877,11 +918,49 @@ static bool worker_check_resource_limits(cbm_index_worker_handle_t *handle) { return rss_probe_failed; } +static bool worker_task_temp_bytes(cbm_index_worker_handle_t *handle, uint64_t *bytes_out) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_task_temp_hook) { + return g_task_temp_hook(handle->task_db_path, handle->log_path, handle->response_path, + bytes_out, g_task_temp_hook_context); + } +#endif + return cbm_index_task_temp_bytes(handle->task_db_path, handle->stage_token, handle->log_path, + handle->response_path, bytes_out); +} + +static void worker_check_task_temp_limit(cbm_index_worker_handle_t *handle) { + const cbm_index_limit_u64_t *limit = &handle->resource_policy.max_task_temp_bytes; + if (!limit->enabled || + atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != + INDEX_WORKER_TERMINATION_NONE || + cbm_subprocess_termination_pending(handle->process) || + !cbm_subprocess_supervision_active(handle->process)) { + return; + } + uint64_t now = worker_resource_now_ms(); + bool probe_due = + !handle->task_temp_probe_started || now < handle->last_task_temp_probe_ms || + now - handle->last_task_temp_probe_ms >= INDEX_WORKER_TASK_TEMP_PROBE_INTERVAL_MS; + if (!probe_due) { + return; + } + handle->task_temp_probe_started = true; + handle->last_task_temp_probe_ms = now; + uint64_t observed = 0; + bool measured = worker_task_temp_bytes(handle, &observed); + if (!measured || observed > limit->value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, observed, + limit->value, !measured); + } +} + static int worker_start_internal(const char *args_json, size_t memory_budget_bytes, const cbm_index_resource_policy_t *resource_policy, - bool single_thread, const char *marker_file, - const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, uint64_t duration_origin_ms, + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; @@ -918,6 +997,15 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt if (resource_policy) { handle->resource_policy = *resource_policy; } + if (handle->resource_policy.max_task_temp_bytes.enabled) { + int written = task_db_path ? snprintf(handle->task_db_path, sizeof(handle->task_db_path), + "%s", task_db_path) + : -1; + if (written <= 0 || written >= (int)sizeof(handle->task_db_path)) { + free(handle); + return -1; + } + } atomic_init(&handle->termination_reason, INDEX_WORKER_TERMINATION_NONE); atomic_init(&handle->terminal, false); handle->log_callback = log_callback; @@ -935,6 +1023,17 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt free(handle); return -1; } + const char *random_suffix = strrchr(handle->response_path, '-'); + int token_written = random_suffix ? snprintf(handle->stage_token, sizeof(handle->stage_token), + "%ld%s", (long)worker_getpid(), random_suffix) + : -1; + if (token_written <= 0 || token_written >= (int)sizeof(handle->stage_token) || + !worker_stage_token_valid(handle->stage_token)) { + (void)cbm_unlink(handle->response_path); + (void)cbm_unlink(handle->log_path); + free(handle); + return -1; + } const char *argv[INDEX_WORKER_ARGV_CAP]; size_t argc = 0; @@ -964,6 +1063,8 @@ static int worker_start_internal(const char *args_json, size_t memory_budget_byt argv[argc++] = CBM_INDEX_WORKER_QUARANTINE_ARG; argv[argc++] = quarantine_file; } + argv[argc++] = CBM_INDEX_WORKER_STAGE_TOKEN_ARG; + argv[argc++] = handle->stage_token; argv[argc] = NULL; cbm_proc_opts_t options = {0}; @@ -1003,15 +1104,16 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, cbm_index_worker_handle_t **handle_out) { - return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, - quarantine_file, log_callback, log_context, 0, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, NULL, NULL, single_thread, + marker_file, quarantine_file, log_callback, log_context, 0, + handle_out); } int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { - return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, - quarantine_file, NULL, NULL, 0, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, NULL, NULL, single_thread, + marker_file, quarantine_file, NULL, NULL, 0, handle_out); } int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, @@ -1019,9 +1121,9 @@ int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budg bool single_thread, const char *marker_file, const char *quarantine_file, uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out) { - return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, - marker_file, quarantine_file, NULL, NULL, duration_origin_ms, - handle_out); + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, NULL, + single_thread, marker_file, quarantine_file, NULL, NULL, + duration_origin_ms, handle_out); } int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, @@ -1031,9 +1133,21 @@ int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_ cbm_proc_log_cb log_callback, void *log_context, uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out) { - return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, - marker_file, quarantine_file, log_callback, log_context, - duration_origin_ms, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, NULL, + single_thread, marker_file, quarantine_file, log_callback, + log_context, duration_origin_ms, handle_out); +} + +int cbm_index_worker_start_with_storage_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, task_db_path, + single_thread, marker_file, quarantine_file, log_callback, + log_context, duration_origin_ms, handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -1051,6 +1165,7 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, bool relay_caught_up = true; if (!handle->process_terminal) { bool rss_probe_failed = worker_check_resource_limits(handle); + worker_check_task_temp_limit(handle); cbm_proc_result_t process_result; cbm_proc_poll_t state = cbm_subprocess_poll(handle->process, &process_result); relay_caught_up = worker_relay_log(handle); @@ -1110,6 +1225,14 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->log_path); } } + if (termination_reason == INDEX_WORKER_TERMINATION_RESOURCE && + handle->result.resource_violation.resource == CBM_INDEX_RESOURCE_TASK_TEMP_BYTES) { + if (!handle->result.tree_quiesced || handle->result.supervision_failed) { + cbm_log_warn("index.supervisor.staging_cleanup_deferred", "path", handle->task_db_path); + } else if (!cbm_index_staging_cleanup(handle->task_db_path, handle->stage_token)) { + cbm_log_warn("index.supervisor.staging_cleanup_failed", "path", handle->task_db_path); + } + } (void)cbm_unlink(handle->response_path); worker_terminal_log(handle); atomic_store_explicit(&handle->terminal, true, memory_order_release); @@ -1174,18 +1297,19 @@ void cbm_index_worker_destroy(cbm_index_worker_handle_t *handle) { static int worker_spawn_internal(const char *args_json, const cbm_index_resource_policy_t *resource_policy, - bool single_thread, const char *marker_file, - const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, const atomic_int *cancel_requested, - uint64_t duration_origin_ms, cbm_index_worker_result_t *result) { + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; - if (worker_start_internal(args_json, 0, resource_policy, single_thread, marker_file, - quarantine_file, log_callback, log_context, duration_origin_ms, - &handle) != 0) { + if (worker_start_internal(args_json, 0, resource_policy, task_db_path, single_thread, + marker_file, quarantine_file, log_callback, log_context, + duration_origin_ms, &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -1216,7 +1340,7 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result) { - return worker_spawn_internal(args_json, NULL, single_thread, marker_file, quarantine_file, + return worker_spawn_internal(args_json, NULL, NULL, single_thread, marker_file, quarantine_file, log_callback, log_context, cancel_requested, 0, result); } @@ -1225,11 +1349,22 @@ int cbm_index_spawn_worker_with_policy_log_cancel( const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, uint64_t duration_origin_ms, cbm_index_worker_result_t *result) { - return worker_spawn_internal(args_json, resource_policy, single_thread, marker_file, + return worker_spawn_internal(args_json, resource_policy, NULL, single_thread, marker_file, quarantine_file, log_callback, log_context, cancel_requested, duration_origin_ms, result); } +int cbm_index_spawn_worker_with_storage_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, resource_policy, task_db_path, single_thread, + marker_file, quarantine_file, log_callback, log_context, + cancel_requested, duration_origin_ms, result); +} + int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index 2cb244a7ee..7de4ac056e 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -42,11 +42,12 @@ void cbm_index_set_worker_role(bool is_worker, const char *response_out); #define CBM_INDEX_WORKER_QUARANTINE_ARG "--index-worker-quarantine" #define CBM_INDEX_WORKER_MEMORY_BUDGET_ARG "--index-worker-memory-budget-bytes" #define CBM_INDEX_WORKER_BUILD_ARG "--index-worker-build" +#define CBM_INDEX_WORKER_STAGE_TOKEN_ARG "--index-worker-stage-token" #define CBM_INDEX_WORKER_BUILD_FINGERPRINT_LENGTH 64U #define CBM_INDEX_WORKER_BUILD_FINGERPRINT_SIZE 65U void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, bool single_thread, const char *marker_file, const char *quarantine_file, - size_t memory_budget_bytes); + size_t memory_budget_bytes, const char *stage_token); bool cbm_index_worker_active(void); const char *cbm_index_worker_response_out(void); size_t cbm_index_worker_memory_budget_bytes(void); @@ -96,6 +97,7 @@ typedef struct { const char *marker_file; const char *quarantine_file; size_t memory_budget_bytes; + const char *stage_token; } cbm_index_worker_invocation_t; typedef enum { @@ -188,6 +190,13 @@ int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budg bool single_thread, const char *marker_file, const char *quarantine_file, uint64_t duration_origin_ms, cbm_index_worker_handle_t **handle_out); +int cbm_index_worker_start_with_storage_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, + cbm_index_worker_handle_t **handle_out); /* Policy plus the request-scoped log callback. The daemon default start path * needs both: a missing policy fails closed, and worker diagnostics still @@ -271,6 +280,12 @@ int cbm_index_spawn_worker_with_policy_log_cancel( const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, uint64_t duration_origin_ms, cbm_index_worker_result_t *result); +int cbm_index_spawn_worker_with_storage_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, + const char *task_db_path, bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); @@ -283,6 +298,12 @@ void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_cl cbm_index_supervisor_rss_fn rss_fn, void *context); void cbm_index_supervisor_reset_resource_hooks_for_testing(void); +typedef bool (*cbm_index_supervisor_task_temp_fn)(const char *final_db_path, const char *log_path, + const char *response_path, uint64_t *bytes_out, + void *context); +void cbm_index_supervisor_set_task_temp_hook_for_testing( + cbm_index_supervisor_task_temp_fn task_temp_fn, void *context); +void cbm_index_supervisor_reset_task_temp_hook_for_testing(void); #endif #endif /* CBM_INDEX_SUPERVISOR_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index f23a00e5cd..c8e7d4d444 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11225,6 +11225,8 @@ static char *index_worker_json_result(yyjson_mut_doc *doc, char *repo_path, char * result. The crash is already contained (this process survived); we report it * rather than dying. Precise skip-and-continue (quarantine the culprit, index the * rest) is layered on in the probe stage. */ +static bool project_db_is_servable(const char *project, const char *db_path); + static char *build_worker_failure_response(const char *args, cbm_proc_outcome_t outcome) { char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); @@ -11279,6 +11281,9 @@ static char *build_worker_unsafe_terminal_response(const char *args, cbm_proc_ou char *cbm_mcp_index_worker_resource_response(const char *args, const cbm_index_worker_result_t *worker_result) { + if (!worker_result || worker_result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE) { + return NULL; + } char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); char *name_override = cbm_mcp_get_string_arg(args, "name"); char *project_name = @@ -11289,23 +11294,23 @@ char *cbm_mcp_index_worker_resource_response(const char *args, } const cbm_index_resource_violation_t *violation = &worker_result->resource_violation; + bool probe_failed = worker_result->resource_probe_failed || violation->probe_failed; + bool worker_stage = violation->resource == CBM_INDEX_RESOURCE_RSS_BYTES || + violation->resource == CBM_INDEX_RESOURCE_DURATION_MS; const char *config_key = cbm_index_resource_config_key(violation->resource); char message[CBM_SZ_256]; (void)snprintf(message, sizeof(message), - worker_result->resource_probe_failed - ? "Worker resource measurement failed for %s" - : "Index worker exceeded %s", + probe_failed ? "Index resource measurement failed for %s" : "Index exceeded %s", config_key); yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); yyjson_mut_obj_add_str(doc, root, "status", "error"); yyjson_mut_obj_add_str(doc, root, "code", - worker_result->resource_probe_failed ? "resource_probe_failed" - : "resource_limit_exceeded"); - yyjson_mut_obj_add_str(doc, root, "stage", "worker"); + probe_failed ? "resource_probe_failed" : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", worker_stage ? "worker" : "storage"); yyjson_mut_obj_add_str(doc, root, "resource", cbm_index_resource_name(violation->resource)); - if (!worker_result->resource_probe_failed) { + if (!probe_failed) { yyjson_mut_obj_add_uint(doc, root, "observed", violation->observed); yyjson_mut_obj_add_uint(doc, root, "limit", violation->limit); yyjson_mut_obj_add_str(doc, root, "unit", cbm_index_resource_unit(violation->resource)); @@ -11501,6 +11506,24 @@ bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resourc return valid; } +bool cbm_mcp_index_task_db_path(const char *args, char *path_out, size_t path_size) { + if (!path_out || path_size == 0) { + return false; + } + path_out[0] = '\0'; + char *repo_path = cbm_mcp_get_string_arg(args, "repo_path"); + char *name_override = cbm_mcp_get_string_arg(args, "name"); + char *project = + cbm_project_name_from_path(name_override && name_override[0] ? name_override : repo_path); + char cache_path[CBM_SZ_1K]; + cache_dir(cache_path, sizeof(cache_path)); + int written = project ? snprintf(path_out, path_size, "%s/%s.db", cache_path, project) : -1; + free(project); + free(name_override); + free(repo_path); + return written > 0 && (size_t)written < path_size; +} + static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, cbm_index_resource_policy_t *policy, char *error, size_t error_size) { if (cbm_index_worker_active()) { @@ -11618,13 +11641,17 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, const cbm_index_resource_policy_t *resource_policy) { invalidate_cached_store(srv); uint64_t request_started_ms = cbm_index_worker_now_ms(); + char task_db_path[CBM_SZ_1K]; + if (!resource_policy || !cbm_mcp_index_task_db_path(args, task_db_path, sizeof(task_db_path))) { + return cbm_mcp_text_result("could not resolve worker index path", true); + } /* First attempt: normal parallel run. */ cbm_index_worker_result_t wr; - int rc = cbm_index_spawn_worker_with_policy_log_cancel( - args, resource_policy, false, NULL, NULL, srv ? srv->index_log_callback : NULL, - srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, - request_started_ms, &wr); + int rc = cbm_index_spawn_worker_with_storage_policy_log_cancel( + args, resource_policy, task_db_path, false, NULL, NULL, + srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, + srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wr); cbm_mcp_supervised_result_disposition_t disposition = cbm_mcp_supervised_result_disposition(rc, &wr); @@ -11662,7 +11689,6 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, invalidate_cached_store(srv); return resp; } - /* Crash / hang / nonzero exit → skip-and-continue recovery. Re-run the * worker PARALLEL (there are no sequential production runs) with the * per-file marker JOURNAL armed; after each failed run the journal's @@ -11703,10 +11729,11 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, bool terminal_cancelled = false; for (int i = 0; i < cap; i++) { cbm_index_worker_result_t wr2; - int rc2 = cbm_index_spawn_worker_with_policy_log_cancel( - args, resource_policy, /*single_thread=*/false, marker_path, quarantine_path, - srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, - srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wr2); + int rc2 = cbm_index_spawn_worker_with_storage_policy_log_cancel( + args, resource_policy, task_db_path, /*single_thread=*/false, marker_path, + quarantine_path, srv ? srv->index_log_callback : NULL, + srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, + request_started_ms, &wr2); cbm_mcp_supervised_result_disposition_t recovery_disposition = cbm_mcp_supervised_result_disposition(rc2, &wr2); if (wr2.response_missing) { @@ -11830,8 +11857,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, * so it cannot itself hang. Rare given monotonic progress. */ if (!resp && !unsafe_terminal && quarantined > 0) { cbm_index_worker_result_t wrp; - int rcp = cbm_index_spawn_worker_with_policy_log_cancel( - args, resource_policy, /*single_thread=*/false, NULL, quarantine_path, + int rcp = cbm_index_spawn_worker_with_storage_policy_log_cancel( + args, resource_policy, task_db_path, /*single_thread=*/false, NULL, quarantine_path, srv ? srv->index_log_callback : NULL, srv ? srv->index_log_context : NULL, srv ? &srv->pipeline_cancel_requested : NULL, request_started_ms, &wrp); cbm_mcp_supervised_result_disposition_t partial_disposition = @@ -12634,11 +12661,18 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { } else if (rc == CBM_PIPELINE_RESOURCE_LIMIT && resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { const char *config_key = cbm_index_resource_config_key(resource_violation.resource); + bool discovery_resource = resource_violation.resource == CBM_INDEX_RESOURCE_FILES || + resource_violation.resource == CBM_INDEX_RESOURCE_SOURCE_BYTES; char message[CBM_SZ_256]; - (void)snprintf(message, sizeof(message), "Index discovery exceeded %s", config_key); + (void)snprintf(message, sizeof(message), + resource_violation.probe_failed ? "Index resource probe failed for %s" + : "Index exceeded %s", + config_key); yyjson_mut_obj_add_str(doc, root, "status", "error"); - yyjson_mut_obj_add_str(doc, root, "code", "resource_limit_exceeded"); - yyjson_mut_obj_add_str(doc, root, "stage", "discovery"); + yyjson_mut_obj_add_str(doc, root, "code", + resource_violation.probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", discovery_resource ? "discovery" : "storage"); yyjson_mut_obj_add_str(doc, root, "resource", cbm_index_resource_name(resource_violation.resource)); yyjson_mut_obj_add_uint(doc, root, "observed", resource_violation.observed); @@ -20794,11 +20828,18 @@ static void *autoindex_thread(void *arg) { return NULL; } + cbm_index_resource_policy_t resource_policy; + char policy_error[CBM_SZ_256] = {0}; + if (!load_index_policy(srv, NULL, &resource_policy, policy_error, sizeof(policy_error))) { + cbm_log_warn("autoindex.err", "msg", "resource_policy_load_failed", "error", policy_error); + return NULL; + } cbm_pipeline_t *p = cbm_pipeline_new(srv->session_root, NULL, CBM_MODE_FULL); if (!p) { cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed"); return NULL; } + cbm_pipeline_set_resource_policy(p, &resource_policy); index_doc_candidates_apply(p, index_doc_candidates_choice(srv, NULL)); /* Block until any concurrent pipeline finishes */ diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index 3ca7c6ee9f..354ae36a15 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -62,6 +62,7 @@ bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_index_resource_policy_t *policy); bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resource_policy_t *policy, char *error, size_t error_size); +bool cbm_mcp_index_task_db_path(const char *args, char *path_out, size_t path_size); char *cbm_mcp_index_worker_resource_response(const char *args, const cbm_index_worker_result_t *worker_result); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index e14bfb7f76..2d08d75cc5 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -203,6 +203,10 @@ struct cbm_pipeline { bool persistence; /* write .codebase-memory/graph.db.zst after indexing */ cbm_index_resource_policy_t resource_policy; cbm_index_resource_violation_t resource_violation; +#ifdef CBM_ENABLE_TEST_SEAMS + cbm_pipeline_storage_probe_fn storage_probe; + void *storage_probe_context; +#endif /* Snapshot of the artifact export failure of THIS run (set only by * export_after_publish failure, zeroed at run start, cleared on success). @@ -935,6 +939,17 @@ const char *cbm_pipeline_export_error(const cbm_pipeline_t *p) { return p ? p->export_error : ""; } +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_pipeline_set_storage_probe_for_testing(cbm_pipeline_t *p, + cbm_pipeline_storage_probe_fn probe, + void *context) { + if (p) { + p->storage_probe = probe; + p->storage_probe_context = context; + } +} +#endif + bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) { if (!p || !name || !name[0]) { return false; @@ -3292,7 +3307,6 @@ static int publish_generation_impl(const cbm_pipeline_generation_t *generation, if (generation->cancelled && atomic_load(generation->cancelled)) { return CBM_PIPELINE_ABORT_PRESERVE_DB; } - /* The staging name must be unpredictable and created exclusively. It used * to be ".stage..", which any other process can compute * in advance; this path is then unlinked and written, so in a @@ -4123,6 +4137,216 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { return rc; } +static const cbm_index_limit_u64_t *storage_limit_for_resource( + const cbm_index_resource_policy_t *policy, cbm_index_resource_t resource) { + if (!policy) { + return NULL; + } + switch (resource) { + case CBM_INDEX_RESOURCE_CACHE_BYTES: + return &policy->max_cache_bytes; + case CBM_INDEX_RESOURCE_FREE_DISK_BYTES: + return &policy->min_free_disk_bytes; + case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: + return &policy->max_final_db_bytes; + case CBM_INDEX_RESOURCE_STAGING_BYTES: + return &policy->max_staging_bytes; + case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: + return &policy->max_task_temp_bytes; + default: + return NULL; + } +} + +static cbm_index_resource_t storage_enabled_probe_resource( + const cbm_index_resource_policy_t *policy) { + if (policy->max_cache_bytes.enabled) { + return CBM_INDEX_RESOURCE_CACHE_BYTES; + } + if (policy->min_free_disk_bytes.enabled) { + return CBM_INDEX_RESOURCE_FREE_DISK_BYTES; + } + if (policy->max_staging_bytes.enabled) { + return CBM_INDEX_RESOURCE_STAGING_BYTES; + } + if (policy->max_task_temp_bytes.enabled) { + return CBM_INDEX_RESOURCE_TASK_TEMP_BYTES; + } + return policy->max_final_db_bytes.enabled ? CBM_INDEX_RESOURCE_FINAL_DB_BYTES + : CBM_INDEX_RESOURCE_NONE; +} + +static char *storage_root_from_final_path(const char *final_path) { + if (!final_path) { + return NULL; + } + char *root = strdup(final_path); + if (!root) { + return NULL; + } + char *slash = strrchr(root, '/'); +#ifdef _WIN32 + char *backslash = strrchr(root, '\\'); + if (backslash && (!slash || backslash > slash)) { + slash = backslash; + } +#endif + if (!slash) { + free(root); + return strdup("."); + } + if (slash == root) { + slash[1] = '\0'; + } else { + *slash = '\0'; + } + return root; +} + +static bool storage_regular_file_bytes(const char *path, uint64_t *bytes_out) { + if (!path || !bytes_out) { + return false; + } + cbm_path_info_t info; + int status = cbm_path_info_utf8(path, &info); + if (status == CBM_PATH_INFO_ABSENT) { + *bytes_out = 0; + return true; + } + if (status != CBM_PATH_INFO_OK || !info.is_regular || info.is_symlink || info.size < 0) { + return false; + } + *bytes_out = (uint64_t)info.size; + return true; +} + +static bool storage_replaceable_old_bytes(const char *final_path, uint64_t *bytes_out) { + if (!final_path || !bytes_out) { + return false; + } + *bytes_out = 0; + cbm_path_info_t info; + int status = cbm_path_info_utf8(final_path, &info); + if (status == CBM_PATH_INFO_ABSENT) { + return true; + } + if (status != CBM_PATH_INFO_OK || !info.is_regular || info.is_symlink) { + return false; + } + cbm_store_t *store = cbm_store_open_path_query(final_path); + if (!store) { + /* A file that cannot be opened is not proven replaceable. Keep its + * bytes in the projection rather than blocking repair of a confirmed + * corrupt/non-database destination. */ + return true; + } + cbm_integrity_verdict_t verdict = cbm_store_check_integrity_verdict(store); + cbm_store_close(store); + if (verdict == CBM_INTEGRITY_TRANSIENT) { + return false; + } + /* A generation that cannot be proven healthy remains in current_cache but is + * not deducted: publication may quarantine it instead of replacing it. */ + return verdict != CBM_INTEGRITY_OK || cbm_db_artifact_bytes(final_path, bytes_out); +} + +static bool storage_probe_default(const cbm_index_resource_policy_t *policy, + cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_path, const char *staging_path, + cbm_index_storage_sample_t *sample, + cbm_index_resource_t *failed_resource) { + *sample = (cbm_index_storage_sample_t){0}; + *failed_resource = CBM_INDEX_RESOURCE_NONE; + char *storage_root = NULL; + if (policy->max_cache_bytes.enabled || policy->min_free_disk_bytes.enabled) { + storage_root = storage_root_from_final_path(final_path); + if (!storage_root) { + *failed_resource = storage_enabled_probe_resource(policy); + return false; + } + } + uint64_t operation_bytes = 0; + bool need_operation = + staging_path && + (policy->max_cache_bytes.enabled || policy->max_staging_bytes.enabled || + policy->max_task_temp_bytes.enabled || + (checkpoint == CBM_PIPELINE_STORAGE_PREPUBLISH && policy->max_final_db_bytes.enabled)); + if (need_operation && !cbm_db_artifact_bytes(staging_path, &operation_bytes)) { + *failed_resource = + policy->max_staging_bytes.enabled + ? CBM_INDEX_RESOURCE_STAGING_BYTES + : (policy->max_task_temp_bytes.enabled + ? CBM_INDEX_RESOURCE_TASK_TEMP_BYTES + : (policy->max_final_db_bytes.enabled ? CBM_INDEX_RESOURCE_FINAL_DB_BYTES + : CBM_INDEX_RESOURCE_CACHE_BYTES)); + free(storage_root); + return false; + } + if (policy->max_cache_bytes.enabled) { + uint64_t cache_with_operation = 0; + if (!cbm_directory_size_bytes(storage_root, &cache_with_operation) || + !storage_replaceable_old_bytes(final_path, &sample->replaceable_old_bytes)) { + *failed_resource = CBM_INDEX_RESOURCE_CACHE_BYTES; + free(storage_root); + return false; + } + sample->current_cache_bytes = + cache_with_operation > operation_bytes ? cache_with_operation - operation_bytes : 0; + sample->operation_bytes = operation_bytes; + } + if (policy->min_free_disk_bytes.enabled && + !cbm_filesystem_free_bytes(storage_root, &sample->free_disk_bytes)) { + *failed_resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES; + free(storage_root); + return false; + } + free(storage_root); + if (checkpoint != CBM_PIPELINE_STORAGE_PREFLIGHT) { + sample->staging_bytes = operation_bytes; + sample->task_temp_bytes = operation_bytes; + } + if (checkpoint == CBM_PIPELINE_STORAGE_PREPUBLISH && policy->max_final_db_bytes.enabled && + !storage_regular_file_bytes(staging_path, &sample->final_db_bytes)) { + *failed_resource = CBM_INDEX_RESOURCE_FINAL_DB_BYTES; + return false; + } + return true; +} + +int cbm_pipeline_storage_admit(cbm_pipeline_t *p, cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_path, const char *staging_path) { + if (!p || !cbm_index_policy_storage_enabled(&p->resource_policy)) { + return 0; + } + cbm_index_storage_sample_t sample; + cbm_index_resource_t failed_resource = CBM_INDEX_RESOURCE_NONE; + bool measured; +#ifdef CBM_ENABLE_TEST_SEAMS + if (p->storage_probe) { + measured = p->storage_probe(checkpoint, final_path, staging_path, &sample, &failed_resource, + p->storage_probe_context); + } else +#endif + { + measured = storage_probe_default(&p->resource_policy, checkpoint, final_path, staging_path, + &sample, &failed_resource); + } + if (!measured) { + const cbm_index_limit_u64_t *limit = + storage_limit_for_resource(&p->resource_policy, failed_resource); + p->resource_violation = (cbm_index_resource_violation_t){ + .resource = failed_resource, + .limit = limit && limit->enabled ? limit->value : 0, + .probe_failed = true, + }; + return CBM_PIPELINE_RESOURCE_LIMIT; + } + if (!cbm_index_policy_check_storage(&p->resource_policy, &sample, &p->resource_violation)) { + return CBM_PIPELINE_RESOURCE_LIMIT; + } + return 0; +} + static void cleanup_staging_db(const char *path) { if (!path) { return; @@ -4157,66 +4381,122 @@ static bool ensure_db_parent(const char *path) { } /* Length of the path a stage was minted for: the input itself unless its - * basename has exactly the minted shape ".stage.<6 alphanumerics>", in - * which case the root is . The outer run rewrites the pipeline's db_path - * to its stage, so the inner publication (dump and delta clone) used to mint - * ITS stage from that stage: .stage.A.stage.B, with -wal/-shm beside it - * (#1839). Minting from the root keeps every generation's stage a sibling of - * the live database. Only the exact minted shape is recognised: a database - * named "x.stage.y.db" is not a stage and keeps its full name. */ + * basename has exactly the minted shape ".stage.<6 alphanumerics>" or + * ".stage..<6 alphanumerics>", in which case the root is . + * The outer run rewrites the pipeline's db_path to its stage, so the inner + * publication (dump and delta clone) used to mint ITS stage from that stage: + * .stage.A.stage.B, with -wal/-shm beside it (#1839). Minting from the + * root keeps every generation's stage a sibling of the live database. Only + * the exact minted shape is recognised: a database named "x.stage.y.db" is + * not a stage and keeps its full name. A worker stage token, when present, + * sits between the marker and the random suffix so the supervisor can find + * this attempt's temporary files. */ enum { CBM_STAGE_SUFFIX_RANDOM_CHARS = 6 }; static const char cbm_stage_marker[] = ".stage."; -static bool stage_suffix_at(const char *tail) { - if (strncmp(tail, cbm_stage_marker, sizeof(cbm_stage_marker) - 1) != 0) { +static bool stage_token_char(unsigned char ch) { + return (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || + ch == '-' || ch == '_'; +} + +static bool stage_random_suffix(const char *text) { + for (int i = 0; i < CBM_STAGE_SUFFIX_RANDOM_CHARS; i++) { + if (!isalnum((unsigned char)text[i])) { + return false; + } + } + return text[CBM_STAGE_SUFFIX_RANDOM_CHARS] == '\0'; +} + +/* after ".stage.": 6 alphanumerics, or .<6 alphanumerics>, through end. */ +static bool stage_body_at(const char *after_marker) { + if (stage_random_suffix(after_marker)) { + return true; + } + const char *dot = strchr(after_marker, '.'); + if (!dot || dot == after_marker) { return false; } - const char *random = tail + sizeof(cbm_stage_marker) - 1; - for (int i = 0; i < CBM_STAGE_SUFFIX_RANDOM_CHARS; i++) { - if (!isalnum((unsigned char)random[i])) { + size_t token_len = (size_t)(dot - after_marker); + if (token_len < 6U || token_len >= CBM_SZ_64) { + return false; + } + for (size_t i = 0; i < token_len; i++) { + if (!stage_token_char((unsigned char)after_marker[i])) { return false; } } - return random[CBM_STAGE_SUFFIX_RANDOM_CHARS] == '\0'; + return stage_random_suffix(dot + 1); } static size_t stage_root_length(const char *path) { size_t len = strlen(path); - const size_t suffix_len = sizeof(cbm_stage_marker) - 1 + CBM_STAGE_SUFFIX_RANDOM_CHARS; - if (len <= suffix_len) { + const char *marker = NULL; + for (const char *next = strstr(path, cbm_stage_marker); next; + next = strstr(next + 1, cbm_stage_marker)) { + marker = next; + } + if (!marker) { return len; } - size_t root_len = len - suffix_len; - /* The marker must sit inside the basename, never span a separator. */ - for (size_t i = root_len; i < len; i++) { - if (path[i] == '/' + for (const char *scan = marker; *scan; scan++) { + if (*scan == '/' #ifdef _WIN32 - || path[i] == '\\' + || *scan == '\\' #endif ) { return len; } } - return stage_suffix_at(path + root_len) ? root_len : len; + if (!stage_body_at(marker + sizeof(cbm_stage_marker) - 1)) { + return len; + } + return (size_t)(marker - path); +} + +static bool staging_token_valid(const char *token) { + size_t length = token ? strlen(token) : 0; + if (length < 6U || length >= CBM_SZ_64) { + return false; + } + for (size_t index = 0; index < length; index++) { + if (!stage_token_char((unsigned char)token[index])) { + return false; + } + } + return true; } static char *stage_allocate_template(const char *final_path) { - if (!final_path) + if (!final_path) { return NULL; - static const char suffix[] = ".stage.XXXXXX"; + } + char suffix[CBM_SZ_128]; + const char *stage_token = getenv("CBM_INDEX_STAGE_TOKEN"); + if (!staging_token_valid(stage_token)) { + stage_token = NULL; + } + int suffix_length = stage_token && stage_token[0] + ? snprintf(suffix, sizeof(suffix), ".stage.%s.XXXXXX", stage_token) + : snprintf(suffix, sizeof(suffix), ".stage.XXXXXX"); + if (suffix_length <= 0 || (size_t)suffix_length >= sizeof(suffix)) { + return NULL; + } size_t final_len = stage_root_length(final_path); - if (final_len > SIZE_MAX - sizeof(suffix)) + if (final_len > SIZE_MAX - (size_t)suffix_length - 1U) { return NULL; - size_t path_size = final_len + sizeof(suffix); + } + size_t path_size = final_len + (size_t)suffix_length + 1U; #ifdef _WIN32 - if (path_size > CBM_SZ_4K) + if (path_size > CBM_SZ_4K) { return NULL; + } path_size = CBM_SZ_4K; #endif char *path = malloc(path_size); if (path) { memcpy(path, final_path, final_len); - memcpy(path + final_len, suffix, sizeof(suffix)); + memcpy(path + final_len, suffix, (size_t)suffix_length + 1U); } return path; } @@ -4270,7 +4550,7 @@ static char *create_staging_path_impl(const char *final_path, cbm_pipeline_t *fr char *path = stage_allocate_template(final_path); if (!path) return NULL; - char *random_at = path + stage_root_length(final_path) + sizeof(cbm_stage_marker) - 1; + char *random_at = path + strlen(path) - CBM_STAGE_SUFFIX_RANDOM_CHARS; for (int attempt = 0; attempt < 128; attempt++) { int rc = stage_create_attempt(path, random_at, frozen); if (rc > 0) @@ -4436,6 +4716,34 @@ static size_t stage_entry_stage_length_any_base(const char *name) { return stage_entry_stage_length(name, name, (size_t)(marker - name)); } +static const char *stage_name_after_marker(const char *after_marker) { + for (int i = 0; i < CBM_STAGE_SUFFIX_RANDOM_CHARS; i++) { + if (!isalnum((unsigned char)after_marker[i])) { + const char *dot = strchr(after_marker, '.'); + if (!dot || dot == after_marker) { + return NULL; + } + size_t token_len = (size_t)(dot - after_marker); + if (token_len < 6U || token_len >= CBM_SZ_64) { + return NULL; + } + for (size_t n = 0; n < token_len; n++) { + if (!stage_token_char((unsigned char)after_marker[n])) { + return NULL; + } + } + after_marker = dot + 1; + for (int j = 0; j < CBM_STAGE_SUFFIX_RANDOM_CHARS; j++) { + if (!isalnum((unsigned char)after_marker[j])) { + return NULL; + } + } + return after_marker + CBM_STAGE_SUFFIX_RANDOM_CHARS; + } + } + return after_marker + CBM_STAGE_SUFFIX_RANDOM_CHARS; +} + static size_t stage_entry_stage_length(const char *name, const char *base, size_t base_len) { if (strncmp(name, base, base_len) != 0) { return 0; @@ -4444,14 +4752,10 @@ static size_t stage_entry_stage_length(const char *name, const char *base, size_ if (strncmp(at, cbm_stage_marker, sizeof(cbm_stage_marker) - 1) != 0) { return 0; } - at += sizeof(cbm_stage_marker) - 1; - for (int i = 0; i < CBM_STAGE_SUFFIX_RANDOM_CHARS; i++) { - /* NUL is not alphanumeric, so a short name fails here too. */ - if (!isalnum((unsigned char)at[i])) { - return 0; - } + at = stage_name_after_marker(at + sizeof(cbm_stage_marker) - 1); + if (!at) { + return 0; } - at += CBM_STAGE_SUFFIX_RANDOM_CHARS; for (size_t i = 0; i < sizeof(cbm_stage_sidecar_tails) / sizeof(cbm_stage_sidecar_tails[0]); i++) { if (strcmp(at, cbm_stage_sidecar_tails[i]) == 0) { @@ -4653,11 +4957,18 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { cbm_log_debug("pipeline.setup", "step", "stage"); pipeline_release_test_config(p); p->export_error[0] = '\0'; + p->resource_violation = (cbm_index_resource_violation_t){0}; char *final_path = resolve_db_path(p); if (!final_path || !ensure_db_parent(final_path)) { free(final_path); return CBM_NOT_FOUND; } + int storage_rc = + cbm_pipeline_storage_admit(p, CBM_PIPELINE_STORAGE_PREFLIGHT, final_path, NULL); + if (storage_rc != 0) { + free(final_path); + return storage_rc; + } struct stat final_st; bool final_existed = stat(final_path, &final_st) == 0; sweep_orphan_stages(final_path); @@ -4708,6 +5019,15 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { pipeline_release_test_config(p); return rc; } + storage_rc = + cbm_pipeline_storage_admit(p, CBM_PIPELINE_STORAGE_GROWTH, final_path, staging_path); + if (storage_rc != 0) { + cleanup_staging_db(staging_path); + free(staging_path); + free(final_path); + pipeline_release_test_config(p); + return storage_rc; + } /* A cancelled run and a live no-op both end here without publishing: * the stage placeholder is discarded, and only the no-op goes on to * finish on the live generation. */ @@ -4748,6 +5068,14 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { pipeline_release_test_config(p); return CBM_PIPELINE_PERSIST_FAILED; } + storage_rc = + cbm_pipeline_storage_admit(p, CBM_PIPELINE_STORAGE_PREPUBLISH, final_path, staging_path); + if (storage_rc != 0) { + cleanup_staging_db(staging_path); + free(staging_path); + free(final_path); + return storage_rc; + } /* Check after the before-publish hook and the final seal, while the * original bytes are still owned. ERROR and absent/present changes abort. */ diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 072674d8d4..ab4f9b9080 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -78,6 +78,21 @@ void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p, * cbm_pipeline_free(). Returns "" for NULL p. */ const char *cbm_pipeline_export_error(const cbm_pipeline_t *p); +typedef enum { + CBM_PIPELINE_STORAGE_PREFLIGHT = 0, + CBM_PIPELINE_STORAGE_GROWTH, + CBM_PIPELINE_STORAGE_PREPUBLISH, +} cbm_pipeline_storage_checkpoint_t; +#ifdef CBM_ENABLE_TEST_SEAMS +typedef bool (*cbm_pipeline_storage_probe_fn)(cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_db_path, + const char *staging_db_path, + cbm_index_storage_sample_t *sample, + cbm_index_resource_t *failed_resource, void *context); +void cbm_pipeline_set_storage_probe_for_testing(cbm_pipeline_t *p, + cbm_pipeline_storage_probe_fn probe, void *context); +#endif + /* Free a pipeline and all its internal state. NULL-safe. */ void cbm_pipeline_free(cbm_pipeline_t *p); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 06722aca73..8a51d46806 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1655,6 +1655,9 @@ typedef struct { bool doc_candidates_in_place; } cbm_pipeline_generation_t; +int cbm_pipeline_storage_admit(cbm_pipeline_t *p, cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_path, const char *staging_path); + /* Serialize and fully populate a sibling staging database, then atomically * replace final_db_path. The old generation is untouched on every failure or * cancellation observed before the rename commit point. */ diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 5641ecbac5..04a5f7a302 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1728,6 +1728,7 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem atomic_init(&worker->cancelled, false); worker->result.exit_code = -1; if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + worker->result.resource_violation = context->resource_violations[worker->attempt]; (void)snprintf(context->args_json[worker->attempt], APP_FAKE_ARGS_CAP, "%s", args_json ? args_json : ""); atomic_store(&context->args_captured[worker->attempt], true); @@ -1759,8 +1760,8 @@ static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, const char *max_files, const char *max_source_mb, - const char *max_rss_mb, - const char *max_duration_seconds) { + const char *max_rss_mb, const char *max_duration_seconds, + const char *cache_max_mb, const char *min_free_disk_mb) { if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { return false; } @@ -1783,12 +1784,21 @@ static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, in yyjson_val *duration = policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) : NULL; - bool equal = files && bytes && rss && duration && yyjson_is_str(files) && + yyjson_val *cache = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_CACHE_MAX_MB) + : NULL; + yyjson_val *free_disk = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB) + : NULL; + bool equal = files && bytes && rss && duration && cache && free_disk && yyjson_is_str(files) && yyjson_is_str(bytes) && yyjson_is_str(rss) && yyjson_is_str(duration) && + yyjson_is_str(cache) && yyjson_is_str(free_disk) && strcmp(yyjson_get_str(files), max_files) == 0 && strcmp(yyjson_get_str(bytes), max_source_mb) == 0 && strcmp(yyjson_get_str(rss), max_rss_mb) == 0 && - strcmp(yyjson_get_str(duration), max_duration_seconds) == 0; + strcmp(yyjson_get_str(duration), max_duration_seconds) == 0 && + strcmp(yyjson_get_str(cache), cache_max_mb) == 0 && + strcmp(yyjson_get_str(free_disk), min_free_disk_mb) == 0; yyjson_doc_free(document); return equal; } @@ -2458,7 +2468,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); bool auto_policy_propagated = - first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7"); + first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7", "off", "off"); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2805,7 +2815,9 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { stored_config && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0 && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_CACHE_MAX_MB, "8") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "9") == 0; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2826,7 +2838,7 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { int index_rc = application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; bool policy_propagated = - index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6", "64", "7"); + index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6", "64", "7", "8", "9"); bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); cbm_daemon_application_free(application); @@ -2992,6 +3004,41 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { PASS(); } +TEST(daemon_application_worker_storage_resource_failure_is_structured_and_not_retried) { + app_watch_race_fixture_t fixture; + bool ready = app_watch_race_fixture_init(&fixture, 77); + atomic_store(&fixture.fake.scripted, true); + fixture.fake.outcomes[0] = CBM_PROC_KILLED; + fixture.fake.resource_violations[0] = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, .observed = 2048, .limit = 1024}; + + char args[APP_TEST_PATH_CAP + 32]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"}", fixture.root); + uint8_t *request = NULL; + uint32_t request_length = 0; + uint8_t *response = NULL; + uint32_t response_length = 0; + bool encoded = + ready && app_test_tool_request("index_repository", args, &request, &request_length); + bool completed = encoded && app_test_request(&fixture.callbacks, fixture.session, request, + request_length, &response, &response_length) == + CBM_DAEMON_RUNTIME_APPLICATION_OK; + bool structured = + completed && response && strstr((char *)response, "resource_limit_exceeded") && + strstr((char *)response, "task_temp_bytes") && strstr((char *)response, "storage"); + int starts = atomic_load(&fixture.fake.starts); + free(request); + free(response); + bool cleaned = app_watch_race_fixture_finish(&fixture); + + ASSERT_TRUE(ready); + ASSERT_TRUE(completed); + ASSERT_TRUE(structured); + ASSERT_EQ(starts, 1); + ASSERT_TRUE(cleaned); + PASS(); +} + TEST(daemon_application_auto_index_honors_tracked_file_limit) { app_env_backup_t cache_environment; bool cache_saved = app_env_backup_capture(&cache_environment, "CBM_CACHE_DIR"); @@ -7062,6 +7109,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); RUN_TEST(daemon_application_programmatic_index_forwards_set_doc_candidates); RUN_TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried); + RUN_TEST(daemon_application_worker_storage_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index b33d64cdb0..dc362f6b5b 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -3,6 +3,7 @@ #include "cli/cli.h" #include "foundation/compat.h" +#include "foundation/constants.h" #include "foundation/index_policy.h" #include "mcp/index_supervisor.h" #include "mcp/mcp.h" @@ -23,13 +24,21 @@ TEST(index_policy_defaults_are_disabled) { ASSERT_FALSE(policy.max_source_bytes.enabled); ASSERT_FALSE(policy.max_rss_bytes.enabled); ASSERT_FALSE(policy.max_duration_ms.enabled); + ASSERT_FALSE(policy.max_cache_bytes.enabled); + ASSERT_FALSE(policy.min_free_disk_bytes.enabled); + ASSERT_FALSE(policy.max_final_db_bytes.enabled); + ASSERT_FALSE(policy.max_staging_bytes.enabled); + ASSERT_FALSE(policy.max_task_temp_bytes.enabled); ASSERT_FALSE(cbm_index_policy_enabled(&policy)); ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); ASSERT_FALSE(cbm_index_policy_worker_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_storage_enabled(&policy)); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_RSS_MB), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_DURATION_SECONDS), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_CACHE_MAX_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MIN_FREE_DISK_MB), "off"); PASS(); } @@ -98,6 +107,70 @@ TEST(index_policy_worker_limits_validate_and_convert_units) { PASS(); } +TEST(index_policy_storage_limits_validate_and_convert_units) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + char value[64]; + + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_CACHE_MAX_MB, "131072", error, + sizeof(error))); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "4096", error, + sizeof(error))); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(131072) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(cbm_index_policy_storage_enabled(&policy)); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_CACHE_MAX_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "131072"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_CACHE_MAX_MB, "off", error, sizeof(error))); + ASSERT_FALSE(policy.max_cache_bytes.enabled); + PASS(); +} + +TEST(index_policy_storage_projection_boundaries_are_overflow_safe) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_cache_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 100}; + policy.min_free_disk_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 40}; + policy.max_final_db_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 60}; + policy.max_staging_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 70}; + policy.max_task_temp_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 80}; + cbm_index_storage_sample_t sample = { + .current_cache_bytes = 130, + .replaceable_old_bytes = 50, + .operation_bytes = 20, + .free_disk_bytes = 40, + .final_db_bytes = 60, + .staging_bytes = 70, + .task_temp_bytes = 80, + }; + cbm_index_resource_violation_t violation = {0}; + + ASSERT_TRUE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + sample.current_cache_bytes++; + ASSERT_FALSE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_CACHE_BYTES); + ASSERT_EQ(violation.observed, 101); + ASSERT_EQ(violation.limit, 100); + sample.current_cache_bytes--; + sample.free_disk_bytes--; + ASSERT_FALSE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FREE_DISK_BYTES); + ASSERT_EQ(violation.observed, 39); + sample = (cbm_index_storage_sample_t){ + .current_cache_bytes = UINT64_MAX, + .replaceable_old_bytes = 0, + .operation_bytes = UINT64_MAX, + .free_disk_bytes = UINT64_MAX, + }; + ASSERT_FALSE(cbm_index_policy_check_storage(&policy, &sample, &violation)); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_CACHE_BYTES); + ASSERT_EQ(violation.observed, UINT64_MAX); + PASS(); +} + TEST(index_policy_invalid_value_is_rejected_atomically) { static const char *const invalid[] = {"", "0", "-1", "1MB", "1 ", "+1", "10000001", "18446744073709551616"}; @@ -149,6 +222,9 @@ TEST(index_policy_format_round_trips_public_values) { TEST(index_policy_violation_metadata_is_stable) { ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FILES), "files"); ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_CACHE_BYTES), "cache_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_FREE_DISK_BYTES), "free_disk_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_STAGING_BYTES), "staging_bytes"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_RSS_BYTES), "rss_bytes"); @@ -163,6 +239,10 @@ TEST(index_policy_violation_metadata_is_stable) { CBM_INDEX_CONFIG_MAX_RSS_MB); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_DURATION_MS), CBM_INDEX_CONFIG_MAX_DURATION_SECONDS); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_CACHE_BYTES), + CBM_INDEX_CONFIG_CACHE_MAX_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FREE_DISK_BYTES), + CBM_INDEX_CONFIG_MIN_FREE_DISK_MB); PASS(); } @@ -180,11 +260,15 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_CACHE_MAX_MB, "17"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "5"), 0); ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_EQ(policy.max_files.value, 9); ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7000)); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(17) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(5) * CBM_INDEX_MIB_BYTES); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); @@ -200,6 +284,8 @@ TEST(index_policy_cli_lists_all_operator_keys) { bool bytes_found = false; bool rss_found = false; bool duration_found = false; + bool cache_found = false; + bool free_found = false; for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { const char *key = cbm_cli_config_key_at_for_testing(index); files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); @@ -207,11 +293,15 @@ TEST(index_policy_cli_lists_all_operator_keys) { rss_found = rss_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_RSS_MB) == 0); duration_found = duration_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) == 0); + cache_found = cache_found || (key && strcmp(key, CBM_INDEX_CONFIG_CACHE_MAX_MB) == 0); + free_found = free_found || (key && strcmp(key, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB) == 0); } ASSERT_TRUE(files_found); ASSERT_TRUE(bytes_found); ASSERT_TRUE(rss_found); ASSERT_TRUE(duration_found); + ASSERT_TRUE(cache_found); + ASSERT_TRUE(free_found); PASS(); } @@ -454,11 +544,92 @@ TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { PASS(); } +TEST(index_policy_storage_limit_preserves_old_index_and_unrelated_cache_files) { + char *repo = th_mktempdir("cbm_index_storage_repo"); + char *cache = th_mktempdir("cbm_index_storage_cache"); + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int original(void) { return 1; }\n"), 0); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + cbm_config_t *config = cbm_config_open(cache); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + if (server && config) { + cbm_mcp_server_set_config(server, config); + } + char args[CBM_SZ_4K]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"StoragePolicyFixture\",\"mode\":\"fast\"}", + repo); + char *first = server ? cbm_mcp_handle_tool(server, "index_repository", args) : NULL; + bool first_ok = first && strstr(first, "\\\"status\\\":\\\"indexed\\\""); + free(first); + + char unrelated[CBM_SZ_4K]; + (void)snprintf(unrelated, sizeof(unrelated), "%s/unrelated.bin", cache); + FILE *large = cbm_fopen(unrelated, "wb"); + char block[4096] = {0}; + bool wrote_large = large != NULL; + for (int index = 0; wrote_large && index < 512; index++) { + wrote_large = fwrite(block, 1, sizeof(block), large) == sizeof(block); + } + if (large) { + wrote_large = fclose(large) == 0 && wrote_large; + } + bool configured = + config && cbm_config_set(config, CBM_INDEX_CONFIG_CACHE_MAX_MB, "1") == 0 && + th_write_file(TH_PATH(repo, "main.c"), "int changed(void) { return 2; }\n") == 0; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"StoragePolicyFixture\",\"mode\":\"fast\"," + "\"_cbm_index_policy\":{\"index_max_files\":\"off\"," + "\"index_max_source_mb\":\"off\",\"index_cache_max_mb\":\"off\"," + "\"index_min_free_disk_mb\":\"off\"}}", + repo); + char *limited = configured && wrote_large && server + ? cbm_mcp_handle_tool(server, "index_repository", args) + : NULL; + bool contract_ok = limited && strstr(limited, "resource_limit_exceeded") && + strstr(limited, "\\\"stage\\\":\\\"storage\\\"") && + strstr(limited, "\\\"resource\\\":\\\"cache_bytes\\\"") && + strstr(limited, "\\\"limit\\\":1048576") && + strstr(limited, "\\\"unit\\\":\\\"bytes\\\"") && + strstr(limited, "\\\"serving_index_preserved\\\":true"); + free(limited); + int64_t unrelated_size = cbm_file_size(unrelated); + char db_path[CBM_SZ_4K]; + (void)snprintf(db_path, sizeof(db_path), "%s/StoragePolicyFixture.db", cache); + cbm_store_t *store = cbm_store_open_path_query(db_path); + bool old_queryable = store && cbm_store_count_nodes(store, "StoragePolicyFixture") > 0; + cbm_store_close(store); + + cbm_mcp_server_free(server); + cbm_config_close(config); + th_cleanup(repo); + th_cleanup(cache); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + + ASSERT_TRUE(first_ok); + ASSERT_TRUE(wrote_large); + ASSERT_TRUE(configured); + ASSERT_TRUE(contract_ok); + ASSERT_EQ(unrelated_size, (int64_t)sizeof(block) * 512); + ASSERT_TRUE(old_queryable); + PASS(); +} + SUITE(index_policy) { RUN_TEST(index_policy_defaults_are_disabled); RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); RUN_TEST(index_policy_worker_limits_validate_and_convert_units); + RUN_TEST(index_policy_storage_limits_validate_and_convert_units); + RUN_TEST(index_policy_storage_projection_boundaries_are_overflow_safe); RUN_TEST(index_policy_invalid_value_is_rejected_atomically); RUN_TEST(index_policy_format_round_trips_public_values); RUN_TEST(index_policy_violation_metadata_is_stable); @@ -468,4 +639,5 @@ SUITE(index_policy) { RUN_TEST(index_policy_cli_set_reports_a_failed_write); RUN_TEST(index_policy_worker_rejects_missing_parent_policy); RUN_TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index); + RUN_TEST(index_policy_storage_limit_preserves_old_index_and_unrelated_cache_files); } diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index df55407da5..58e4346c13 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -124,6 +124,24 @@ static bool index_supervisor_test_append_log(const char *path, const char *line) return fclose(file) == 0 && written; } +typedef struct { + uint64_t bytes; + bool succeeds; + int calls; +} index_supervisor_task_temp_fake_t; + +static bool index_supervisor_task_temp_fake(const char *final_db_path, const char *log_path, + const char *response_path, uint64_t *bytes_out, + void *context) { + index_supervisor_task_temp_fake_t *fake = context; + fake->calls++; + if (!final_db_path || !log_path || !response_path || !fake->succeeds) { + return false; + } + *bytes_out = fake->bytes; + return true; +} + static bool index_supervisor_test_append_terminal_backlog(const char *path) { FILE *file = cbm_fopen(path, "ab"); if (!file) { @@ -239,18 +257,21 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { "/tmp/m", CBM_INDEX_WORKER_QUARANTINE_ARG, "/tmp/q", + CBM_INDEX_WORKER_STAGE_TOKEN_ARG, + "task123", NULL, }; cbm_index_worker_invocation_t invocation; - ASSERT_EQ(cbm_index_worker_parse_process_argv(16, valid, &invocation), + ASSERT_EQ(cbm_index_worker_parse_process_argv(18, valid, &invocation), CBM_INDEX_WORKER_ARGV_VALID); - ASSERT_EQ(cbm_daemon_process_role(16, valid), CBM_DAEMON_PROCESS_WORKER); + ASSERT_EQ(cbm_daemon_process_role(18, valid), CBM_DAEMON_PROCESS_WORKER); ASSERT_STR_EQ(invocation.args_json, valid[6]); ASSERT_STR_EQ(invocation.response_out, valid[8]); ASSERT_EQ(invocation.memory_budget_bytes, 1024); ASSERT_TRUE(invocation.single_thread); ASSERT_STR_EQ(invocation.marker_file, valid[13]); ASSERT_STR_EQ(invocation.quarantine_file, valid[15]); + ASSERT_STR_EQ(invocation.stage_token, valid[17]); char *missing_build[] = {"test-runner", "cli", "--index-worker", "index_repository", "{}", "--response-out", @@ -314,6 +335,18 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { CBM_INDEX_WORKER_MEMORY_BUDGET_ARG, "184467440737095516160", NULL}; + char *invalid_stage_token[] = {"test-runner", + "cli", + "--index-worker", + CBM_INDEX_WORKER_BUILD_ARG, + (char *)captured, + "index_repository", + "{}", + "--response-out", + "/tmp/r", + CBM_INDEX_WORKER_STAGE_TOKEN_ARG, + "../escape", + NULL}; char *user_value[] = {"test-runner", "cli", "search_code", "--query", "--index-worker", NULL}; ASSERT_EQ(cbm_index_worker_parse_process_argv(7, missing_build, &invocation), CBM_INDEX_WORKER_ARGV_INVALID); @@ -330,6 +363,9 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { ASSERT_EQ(cbm_index_worker_parse_process_argv(11, overflow_budget, &invocation), CBM_INDEX_WORKER_ARGV_INVALID); ASSERT_EQ(cbm_daemon_process_role(11, overflow_budget), CBM_DAEMON_PROCESS_INVALID); + ASSERT_EQ(cbm_index_worker_parse_process_argv(11, invalid_stage_token, &invocation), + CBM_INDEX_WORKER_ARGV_INVALID); + ASSERT_EQ(cbm_daemon_process_role(11, invalid_stage_token), CBM_DAEMON_PROCESS_INVALID); ASSERT_EQ(cbm_index_worker_parse_process_argv(5, user_value, &invocation), CBM_INDEX_WORKER_ARGV_INVALID); ASSERT_EQ(cbm_daemon_process_role(5, user_value), CBM_DAEMON_PROCESS_INVALID); @@ -1337,6 +1373,42 @@ TEST(index_supervisor_three_failed_rss_probes_fail_closed) { PASS(); } +TEST(index_supervisor_task_temp_limit_terminates_as_resource_failure) { + char *root = th_mktempdir("cbm_supervisor_task_temp"); + ASSERT_NOT_NULL(root); + char db_path[INDEX_SUPERVISOR_TEST_PATH_CAP]; + (void)snprintf(db_path, sizeof(db_path), "%s/index.db", root); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_task_temp_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + index_supervisor_task_temp_fake_t fake = {.bytes = 2, .succeeds = true}; + cbm_index_supervisor_set_task_temp_hook_for_testing(index_supervisor_task_temp_fake, &fake); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = + cbm_index_worker_start_with_storage_policy("{\"__cbm_test_worker\":\"clean\"}", 0, &policy, + db_path, false, NULL, NULL, NULL, NULL, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool cancelled = result ? result->cancellation_requested : true; + cbm_index_resource_violation_t violation = + result ? result->resource_violation : (cbm_index_resource_violation_t){0}; + cbm_index_supervisor_reset_task_temp_hook_for_testing(); + if (handle) { + cbm_index_worker_destroy(handle); + } + th_cleanup(root); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_FALSE(cancelled); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_TASK_TEMP_BYTES); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + ASSERT_TRUE(fake.calls > 0); + PASS(); +} + SUITE(index_supervisor) { RUN_TEST(index_supervisor_job_memory_limit_has_floor_headroom_and_no_overflow); RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar); @@ -1355,4 +1427,5 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); RUN_TEST(index_supervisor_cancel_precedes_resource_probe); RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); + RUN_TEST(index_supervisor_task_temp_limit_terminates_as_resource_failure); } diff --git a/tests/test_main.c b/tests/test_main.c index e0526cf858..a8909ca381 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -507,7 +507,7 @@ static int tf_maybe_run_index_worker(int argc, char **argv) { free(worker_repo_path); cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread, invocation.marker_file, invocation.quarantine_file, - invocation.memory_budget_bytes); + invocation.memory_budget_bytes, invocation.stage_token); cbm_mem_init_with_cap(0.5, invocation.memory_budget_bytes); cbm_log_init_for_process(false, true); tf_index_worker_probe(invocation.args_json, invocation.response_out); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 3bfcb018ff..560f15d26c 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -15361,8 +15361,8 @@ static int doccand_rows(const char *cache) { sqlite3_finalize(st); st = NULL; n = 0; - if (has_table && sqlite3_prepare_v2(db, "SELECT count(*) FROM doc_link_candidates", -1, - &st, NULL) == SQLITE_OK) { + if (has_table && sqlite3_prepare_v2(db, "SELECT count(*) FROM doc_link_candidates", -1, &st, + NULL) == SQLITE_OK) { n = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1; sqlite3_finalize(st); } @@ -21533,6 +21533,17 @@ TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery) { CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL); result.supervision_failed = false; + result.tree_quiesced = true; + result.outcome = CBM_PROC_KILLED; + result.resource_violation = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, .observed = 2048, .limit = 1024}; + ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), + CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE); + result.tree_quiesced = false; + ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), + CBM_MCP_SUPERVISED_RESULT_UNSAFE_TERMINAL); + result.resource_violation = (cbm_index_resource_violation_t){0}; + result.tree_quiesced = true; result.outcome = CBM_PROC_CRASH; result.response = NULL; ASSERT_EQ(cbm_mcp_supervised_result_disposition(0, &result), @@ -21677,6 +21688,25 @@ TEST(index_worker_resource_limit_is_trusted_structured_and_not_retried) { #endif } +TEST(index_supervisor_resource_probe_failure_response_is_structured) { + cbm_index_worker_result_t worker_result = { + .resource_violation = + { + .resource = CBM_INDEX_RESOURCE_CACHE_BYTES, + .limit = 1024, + .probe_failed = true, + }, + }; + char *response = + cbm_mcp_index_worker_resource_response("{\"repo_path\":\"/tmp/missing\"}", &worker_result); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"code\":\"resource_probe_failed\"")); + ASSERT_NOT_NULL(strstr(response, "\"stage\":\"storage\"")); + ASSERT_NOT_NULL(strstr(response, "\"resource\":\"cache_bytes\"")); + ASSERT_NOT_NULL(strstr(response, "\"retryable\":true")); + free(response); + PASS(); +} /* Child-side check: index a tiny fixture and verify it ran IN-PROCESS. * Distinct exit codes so the parent can report the exact failure mode. */ @@ -22847,6 +22877,57 @@ TEST(mcp_auto_watch_false_skips_watcher_on_connect) { PASS(); } +TEST(mcp_inprocess_autoindex_inherits_resource_policy) { + char *root = th_mktempdir("cbm-autoindex-policy-root"); + char *cache = th_mktempdir("cbm-autoindex-policy-cache"); + ASSERT_NOT_NULL(root); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(root, "first.c"), "int first(void) { return 1; }\n"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "second.c"), "int second(void) { return 2; }\n"), 0); + + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + char saved_cwd[CBM_SZ_4K]; + bool cwd_saved = cbm_getcwd(saved_cwd, sizeof(saved_cwd)) != NULL; + bool environment_ready = + cwd_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0 && cbm_chdir(root) == 0; + cbm_config_t *config = environment_ready ? cbm_config_open(cache) : NULL; + bool configured = config && cbm_config_set(config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "1") == 0; + char *project = configured ? cbm_project_name_from_path(root) : NULL; + char db_path[CBM_SZ_4K] = {0}; + if (project) { + (void)snprintf(db_path, sizeof(db_path), "%s/%s.db", cache, project); + } + + cbm_mcp_server_t *server = configured && project ? cbm_mcp_server_new(NULL) : NULL; + if (server) { + cbm_mcp_server_set_config(server, config); + char *response = cbm_mcp_server_handle( + server, "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{}}"); + free(response); + cbm_mcp_server_free(server); /* joins the in-process auto-index thread */ + } + bool limit_preserved_no_index = project && !cbm_file_exists(db_path); + + cbm_config_close(config); + if (cwd_saved) { + (void)cbm_chdir(saved_cwd); + } + restore_cache_dir(saved_cache_copy); + free(saved_cache_copy); + free(project); + th_cleanup(root); + th_cleanup(cache); + + ASSERT_TRUE(environment_ready); + ASSERT_TRUE(configured); + ASSERT_NOT_NULL(server); + ASSERT_TRUE(limit_preserved_no_index); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * #1466 / #713 — the auto_index_limit guard * @@ -24732,8 +24813,8 @@ static bool scan_fixture_list(const scan_fixture_t *fx, const char *extra) { /* Run the scan; returns the records it wrote (caller frees), CRs kept. */ static char *scan_fixture_run(const scan_fixture_t *fx, const char *pattern, bool use_regex, size_t output_limit, int *cause) { - *cause = cbm_mcp_search_scan_for_test(NULL, fx->list, pattern, use_regex, fx->out, - output_limit, UINT64_MAX); + *cause = cbm_mcp_search_scan_for_test(NULL, fx->list, pattern, use_regex, fx->out, output_limit, + UINT64_MAX); FILE *fp = cbm_fopen(fx->out, "rb"); if (!fp) { return NULL; @@ -25234,6 +25315,7 @@ SUITE(mcp) { RUN_TEST(index_repository_cli_name_override_issue823); RUN_TEST(index_repository_over_budget_reports_named_reason); RUN_TEST(index_supervisor_unsafe_clean_is_never_fallback_or_recovery); + RUN_TEST(index_supervisor_resource_probe_failure_response_is_structured); RUN_TEST(index_supervisor_gate_requires_marked_host_issue845); RUN_TEST(index_supervisor_start_failure_is_fail_closed_in_real_host); RUN_TEST(index_supervisor_clean_exit_without_response_response_issue1300); @@ -25336,6 +25418,7 @@ SUITE(mcp) { /* auto_watch gate (distilled from PR #625) */ RUN_TEST(mcp_auto_watch_default_registers_watcher_on_connect); RUN_TEST(mcp_auto_watch_false_skips_watcher_on_connect); + RUN_TEST(mcp_inprocess_autoindex_inherits_resource_policy); RUN_TEST(mcp_auto_watch_false_skips_supervised_autoindex_issue853); RUN_TEST(autoindex_skip_reports_numeric_limit_issue1466); RUN_TEST(autoindex_limit_guards_non_git_root_issue713); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 2b80f51bdf..3624af7f32 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -184,6 +184,212 @@ TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing) { PASS(); } +typedef struct { + const char *target_db; + cbm_pipeline_storage_checkpoint_t fail_at; + cbm_index_resource_t failed_resource; + cbm_index_storage_sample_t samples[3]; + bool sample_set[3]; + int calls[3]; + int unexpected_paths; +} pipeline_storage_probe_fake_t; + +static bool pipeline_storage_probe_fake(cbm_pipeline_storage_checkpoint_t checkpoint, + const char *final_db_path, const char *staging_db_path, + cbm_index_storage_sample_t *sample, + cbm_index_resource_t *failed_resource, void *context) { + (void)staging_db_path; + pipeline_storage_probe_fake_t *fake = (pipeline_storage_probe_fake_t *)context; + fake->calls[checkpoint]++; + if (fake->target_db && (!final_db_path || strcmp(fake->target_db, final_db_path) != 0)) { + fake->unexpected_paths++; + } + *sample = (cbm_index_storage_sample_t){ + .current_cache_bytes = 0, + .free_disk_bytes = UINT64_MAX, + }; + if (fake->sample_set[checkpoint]) { + *sample = fake->samples[checkpoint]; + } + if (fake->failed_resource != CBM_INDEX_RESOURCE_NONE && checkpoint == fake->fail_at && + fake->target_db && final_db_path && strcmp(fake->target_db, final_db_path) == 0) { + *failed_resource = fake->failed_resource; + return false; + } + return true; +} + +static int pipeline_stage_artifact_count(const char *directory) { + cbm_dir_t *dir = cbm_opendir(directory); + if (!dir) { + return -1; + } + int count = 0; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + if (strstr(entry->name, ".stage.")) { + count++; + } + } + cbm_closedir(dir); + return count; +} + +TEST(pipeline_disabled_storage_policy_performs_no_probes) { + char *repo = th_mktempdir("cbm_pipeline_storage_off"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + char db_path[CBM_SZ_4K]; + (void)snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + cbm_pipeline_t *pipeline = cbm_pipeline_new(repo, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + pipeline_storage_probe_fake_t fake = {.target_db = db_path, + .fail_at = CBM_PIPELINE_STORAGE_PREFLIGHT, + .failed_resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES}; + cbm_pipeline_set_storage_probe_for_testing(pipeline, pipeline_storage_probe_fake, &fake); + + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + ASSERT_EQ(fake.calls[CBM_PIPELINE_STORAGE_PREFLIGHT], 0); + ASSERT_EQ(fake.calls[CBM_PIPELINE_STORAGE_GROWTH], 0); + ASSERT_EQ(fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH], 0); + cbm_pipeline_free(pipeline); + th_cleanup(repo); + PASS(); +} + +TEST(pipeline_storage_probe_failure_preserves_old_db_and_cleans_stage) { + char *repo = th_mktempdir("cbm_pipeline_storage_probe"); + ASSERT_NOT_NULL(repo); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int original(void) { return 1; }\n"), 0); + char db_path[CBM_SZ_4K]; + (void)snprintf(db_path, sizeof(db_path), "%s/index.db", repo); + cbm_pipeline_t *first = cbm_pipeline_new(repo, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(first); + ASSERT_EQ(cbm_pipeline_run(first), 0); + const char *project = cbm_pipeline_project_name(first); + char *project_copy = project ? strdup(project) : NULL; + cbm_pipeline_free(first); + ASSERT_NOT_NULL(project_copy); + + cbm_store_t *before = cbm_store_open_path_query(db_path); + cbm_project_t before_project = {0}; + ASSERT_NOT_NULL(before); + ASSERT_EQ(cbm_store_get_project(before, project_copy, &before_project), CBM_STORE_OK); + char *indexed_at = before_project.indexed_at ? strdup(before_project.indexed_at) : NULL; + cbm_project_free_fields(&before_project); + cbm_store_close(before); + ASSERT_NOT_NULL(indexed_at); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int changed(void) { return 2; }\n"), 0); + + cbm_pipeline_t *second = cbm_pipeline_new(repo, db_path, CBM_MODE_FAST); + ASSERT_NOT_NULL(second); + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.min_free_disk_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_pipeline_set_resource_policy(second, &policy); + pipeline_storage_probe_fake_t fake = {.target_db = db_path, + .fail_at = CBM_PIPELINE_STORAGE_PREPUBLISH, + .failed_resource = CBM_INDEX_RESOURCE_FREE_DISK_BYTES}; + cbm_pipeline_set_storage_probe_for_testing(second, pipeline_storage_probe_fake, &fake); + int rc = cbm_pipeline_run(second); + cbm_index_resource_violation_t violation = {0}; + cbm_pipeline_get_resource_violation(second, &violation); + cbm_pipeline_free(second); + + cbm_store_t *after = cbm_store_open_path_query(db_path); + cbm_project_t after_project = {0}; + bool preserved = after && + cbm_store_get_project(after, project_copy, &after_project) == CBM_STORE_OK && + after_project.indexed_at && strcmp(indexed_at, after_project.indexed_at) == 0; + cbm_project_free_fields(&after_project); + cbm_store_close(after); + int stage_count = pipeline_stage_artifact_count(repo); + free(indexed_at); + free(project_copy); + th_cleanup(repo); + + ASSERT_EQ(rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_FREE_DISK_BYTES); + ASSERT_TRUE(violation.probe_failed); + ASSERT_TRUE(fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH] > 0); + ASSERT_TRUE(preserved); + ASSERT_EQ(stage_count, 0); + PASS(); +} + +TEST(pipeline_full_and_incremental_storage_limits_report_identical_failure) { + char *full_repo = th_mktempdir("cbm_pipeline_storage_full"); + char *incremental_repo = th_mktempdir("cbm_pipeline_storage_incremental"); + ASSERT_NOT_NULL(full_repo); + ASSERT_NOT_NULL(incremental_repo); + ASSERT_EQ(th_write_file(TH_PATH(full_repo, "main.c"), "int full(void) { return 1; }\n"), 0); + ASSERT_EQ( + th_write_file(TH_PATH(incremental_repo, "main.c"), "int incremental(void) { return 1; }\n"), + 0); + char full_db[CBM_SZ_4K]; + char incremental_db[CBM_SZ_4K]; + (void)snprintf(full_db, sizeof(full_db), "%s/index.db", full_repo); + (void)snprintf(incremental_db, sizeof(incremental_db), "%s/index.db", incremental_repo); + + cbm_pipeline_t *seed = cbm_pipeline_new(incremental_repo, incremental_db, CBM_MODE_FAST); + ASSERT_NOT_NULL(seed); + ASSERT_EQ(cbm_pipeline_run(seed), 0); + cbm_pipeline_free(seed); + ASSERT_EQ( + th_write_file(TH_PATH(incremental_repo, "main.c"), "int incremental(void) { return 2; }\n"), + 0); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_staging_bytes = (cbm_index_limit_u64_t){.enabled = true, .value = 100}; + pipeline_storage_probe_fake_t full_fake = {.target_db = full_db}; + full_fake.sample_set[CBM_PIPELINE_STORAGE_GROWTH] = true; + full_fake.samples[CBM_PIPELINE_STORAGE_GROWTH] = + (cbm_index_storage_sample_t){.free_disk_bytes = UINT64_MAX, .staging_bytes = 101}; + pipeline_storage_probe_fake_t incremental_fake = full_fake; + incremental_fake.target_db = incremental_db; + + cbm_pipeline_t *full = cbm_pipeline_new(full_repo, full_db, CBM_MODE_FAST); + cbm_pipeline_t *incremental = cbm_pipeline_new(incremental_repo, incremental_db, CBM_MODE_FAST); + ASSERT_NOT_NULL(full); + ASSERT_NOT_NULL(incremental); + cbm_pipeline_set_resource_policy(full, &policy); + cbm_pipeline_set_resource_policy(incremental, &policy); + cbm_pipeline_set_storage_probe_for_testing(full, pipeline_storage_probe_fake, &full_fake); + cbm_pipeline_set_storage_probe_for_testing(incremental, pipeline_storage_probe_fake, + &incremental_fake); + int full_rc = cbm_pipeline_run(full); + int incremental_rc = cbm_pipeline_run(incremental); + cbm_index_resource_violation_t full_violation = {0}; + cbm_index_resource_violation_t incremental_violation = {0}; + cbm_pipeline_get_resource_violation(full, &full_violation); + cbm_pipeline_get_resource_violation(incremental, &incremental_violation); + cbm_pipeline_free(full); + cbm_pipeline_free(incremental); + + ASSERT_EQ(full_rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(incremental_rc, CBM_PIPELINE_RESOURCE_LIMIT); + ASSERT_EQ(full_violation.resource, CBM_INDEX_RESOURCE_STAGING_BYTES); + ASSERT_EQ(incremental_violation.resource, full_violation.resource); + ASSERT_EQ(incremental_violation.observed, full_violation.observed); + ASSERT_EQ(incremental_violation.limit, full_violation.limit); + ASSERT_STR_EQ(cbm_index_resource_unit(incremental_violation.resource), + cbm_index_resource_unit(full_violation.resource)); + ASSERT_EQ(full_fake.unexpected_paths, 0); + ASSERT_EQ(incremental_fake.unexpected_paths, 0); + ASSERT_EQ(full_fake.calls[CBM_PIPELINE_STORAGE_PREFLIGHT], 1); + ASSERT_EQ(full_fake.calls[CBM_PIPELINE_STORAGE_GROWTH], 1); + ASSERT_EQ(full_fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH], 0); + ASSERT_EQ(incremental_fake.calls[CBM_PIPELINE_STORAGE_PREFLIGHT], 1); + ASSERT_EQ(incremental_fake.calls[CBM_PIPELINE_STORAGE_GROWTH], 1); + ASSERT_EQ(incremental_fake.calls[CBM_PIPELINE_STORAGE_PREPUBLISH], 0); + ASSERT_EQ(pipeline_stage_artifact_count(full_repo), 0); + ASSERT_EQ(pipeline_stage_artifact_count(incremental_repo), 0); + th_cleanup(full_repo); + th_cleanup(incremental_repo); + PASS(); +} + /* ── Focused: file-backed store persistence ─────────────────────── */ TEST(store_file_persistence) { @@ -26110,6 +26316,9 @@ SUITE(pipeline) { RUN_TEST(pipeline_cancel_null); RUN_TEST(pipeline_run_null); RUN_TEST(pipeline_discovery_limit_returns_exact_violation_without_publishing); + RUN_TEST(pipeline_disabled_storage_policy_performs_no_probes); + RUN_TEST(pipeline_storage_probe_failure_preserves_old_db_and_cleans_stage); + RUN_TEST(pipeline_full_and_incremental_storage_limits_report_identical_failure); RUN_TEST(pipeline_late_source_limit_preserves_previous_generation); /* Extraction back-pressure */ RUN_TEST(pipeline_backpressure_futile_nap_disengages); diff --git a/tests/test_platform.c b/tests/test_platform.c index d1434c2ee3..8de61f7042 100644 --- a/tests/test_platform.c +++ b/tests/test_platform.c @@ -2,6 +2,7 @@ * test_platform.c — RED phase tests for foundation/platform. */ #include "test_framework.h" +#include "test_helpers.h" #include "../src/foundation/compat.h" /* cbm_setenv / cbm_unsetenv (Windows-portable) */ #include "../src/foundation/compat_fs.h" #include "../src/foundation/compat_regex.h" @@ -686,6 +687,93 @@ TEST(platform_file_size) { PASS(); } +TEST(platform_storage_measurement_uses_logical_bytes_without_following_links) { + char *root = th_mktempdir("cbm_storage_measure"); + ASSERT_NOT_NULL(root); + ASSERT_TRUE(cbm_mkdir_p(TH_PATH(root, "nested"), 0700)); + ASSERT_EQ(th_write_file(TH_PATH(root, "first.bin"), "12345"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "nested/second.bin"), "1234567"), 0); +#ifndef _WIN32 + ASSERT_EQ(symlink("/dev/null", TH_PATH(root, "external-link")), 0); +#endif + uint64_t bytes = 0; + uint64_t free_bytes = 0; + ASSERT_TRUE(cbm_directory_size_bytes(root, &bytes)); + ASSERT_EQ(bytes, 12); + ASSERT_TRUE(cbm_filesystem_free_bytes(root, &free_bytes)); + ASSERT_GT(free_bytes, 0); + th_cleanup(root); + PASS(); +} + +TEST(platform_db_artifact_measurement_saturates_and_counts_sidecars) { + char *root = th_mktempdir("cbm_storage_db_artifacts"); + ASSERT_NOT_NULL(root); + char db[CBM_SZ_4K]; + (void)snprintf(db, sizeof(db), "%s/project.db", root); + ASSERT_EQ(th_write_file(db, "12345"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db-wal"), "1234567"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db-shm"), "123"), 0); + uint64_t bytes = 0; + ASSERT_TRUE(cbm_db_artifact_bytes(db, &bytes)); + ASSERT_EQ(bytes, 15); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db.stage.task123.ABCDEF"), "12345678901"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db.stage.task123.ABCDEF-wal"), "1234567890123"), + 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "project.db.stage.other456.ABCDEF"), "unrelated"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "worker.log"), "12"), 0); + ASSERT_EQ(th_write_file(TH_PATH(root, "worker.response"), "123"), 0); + ASSERT_TRUE(cbm_index_task_temp_bytes(db, "task123", TH_PATH(root, "worker.log"), + TH_PATH(root, "worker.response"), &bytes)); + ASSERT_EQ(bytes, 29); + ASSERT_TRUE(cbm_index_staging_cleanup(db, "task123")); + ASSERT_FALSE(cbm_file_exists(TH_PATH(root, "project.db.stage.task123.ABCDEF"))); + ASSERT_FALSE(cbm_file_exists(TH_PATH(root, "project.db.stage.task123.ABCDEF-wal"))); + ASSERT_TRUE(cbm_file_exists(TH_PATH(root, "project.db.stage.other456.ABCDEF"))); + ASSERT_TRUE(cbm_file_exists(db)); + ASSERT_TRUE(cbm_file_exists(TH_PATH(root, "worker.log"))); + th_cleanup(root); + PASS(); +} + +#ifndef _WIN32 +TEST(platform_task_temp_measurement_supports_relative_database_paths) { + char *root = th_mktempdir("cbm_task_temp_relative"); + ASSERT_NOT_NULL(root); + char saved_cwd[CBM_SZ_4K]; + ASSERT_NOT_NULL(getcwd(saved_cwd, sizeof(saved_cwd))); + ASSERT_EQ(chdir(root), 0); + + int write_rc = th_write_file("project.db.stage.task123.ABCDEF", "12345"); + int wal_rc = th_write_file("project.db.stage.task123.ABCDEF-wal", "123456"); + int other_rc = th_write_file("project.db.stage.other456.ABCDEF", "unrelated"); + int log_rc = th_write_file("worker.log", "123"); + int response_rc = th_write_file("worker.response", "1234"); + uint64_t bytes = 0; + bool measured = + cbm_index_task_temp_bytes("project.db", "task123", "worker.log", "worker.response", &bytes); + bool cleaned = cbm_index_staging_cleanup("project.db", "task123"); + bool own_stage_removed = !cbm_file_exists("project.db.stage.task123.ABCDEF") && + !cbm_file_exists("project.db.stage.task123.ABCDEF-wal"); + bool other_stage_preserved = cbm_file_exists("project.db.stage.other456.ABCDEF"); + int restore_rc = chdir(saved_cwd); + th_cleanup(root); + + ASSERT_EQ(write_rc, 0); + ASSERT_EQ(wal_rc, 0); + ASSERT_EQ(other_rc, 0); + ASSERT_EQ(log_rc, 0); + ASSERT_EQ(response_rc, 0); + ASSERT_TRUE(measured); + ASSERT_EQ(bytes, 18); + ASSERT_TRUE(cleaned); + ASSERT_TRUE(own_stage_removed); + ASSERT_TRUE(other_stage_preserved); + ASSERT_EQ(restore_rc, 0); + PASS(); +} +#endif + TEST(platform_mmap) { /* mmap this test file and verify first bytes */ size_t sz = 0; @@ -1407,6 +1495,11 @@ SUITE(platform) { RUN_TEST(platform_file_exists); RUN_TEST(platform_is_dir); RUN_TEST(platform_file_size); + RUN_TEST(platform_storage_measurement_uses_logical_bytes_without_following_links); + RUN_TEST(platform_db_artifact_measurement_saturates_and_counts_sidecars); +#ifndef _WIN32 + RUN_TEST(platform_task_temp_measurement_supports_relative_database_paths); +#endif RUN_TEST(platform_mmap); RUN_TEST(platform_mmap_nonexistent); RUN_TEST(platform_path_helpers_use_per_thread_storage); From 7c4d1a63673a7faf1b080295d119fd05832ee46c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 5/6] feat(index): add opt-in index resource profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six independent keys are an honest interface and a poor default. An operator who only wants indexing to stay within reason has to learn all six, choose a value for each, and keep them consistent as the machine changes. Add index_resource_profile, accepting off, balanced or strict. Balanced guards the host: it lets an index use what the machine can spare and stops a runaway worker before the host is exhausted. Strict holds indexing to the daemon's own budget, so a repository that needs more fails fast and attributed instead of finishing at the host's expense. Any individual key, including an explicit off, replaces that dimension of the selected profile, so a profile is a baseline rather than a lock. The balanced worker ceiling is derived from detected host memory instead of being tabled. Large-repository indexing peaks in the tens of gigabytes, so any round number low enough to feel safe would reject repositories that index successfully today. When host memory cannot be read, balanced falls back to a fixed floor. Profiles also bound directory count, entry count, traversal depth and discovery time. These have no individual keys for the same reason as the storage internals: they are only meaningful inside a composed decision. The resolved profile travels through the same trusted worker contract as the individual limits, so every entry point enforces the same decision. That contract now also carries the resolved profile and the override mask, so the shell fixture standing in for the supervisor sends both. The coverage that proves a forged policy cannot fork a second job asserts that an explicit request joins the auto-index job already running for the same root, and that assertion found the join broken on Windows. A project root reaches the daemon through the platform canonicalizer, which answers in backslash form there, while every tool handler normalizes the separators of the path it canonicalizes. One directory therefore had two spellings, and they met in comparisons that are exact: whether an index request may join the running job, and whether a watch is still live for that root. Both were wrong on Windows, in opposite directions, so aligning either one alone moved the failure rather than removing it. The session context now stores the normalized spelling, and the daemon canonicalizes project roots through a single function that normalizes on the way out. Only the storage side is directly assertable off Windows, since separators are folded on every platform; the watch-liveness side is held by the existing watcher-ownership tests, which is where the second half of this surfaced. Signed-off-by: 刘冲 --- README.md | 12 +- docs/CONFIGURATION.md | 9 +- docs/INDEX_RESOURCE_LIMITS.md | 51 ++++++++ scripts/test-runtime.sh | 9 +- src/cli/cli.c | 24 ++-- src/daemon/application.c | 34 +++++- src/discover/discover.c | 80 ++++++++++++- src/discover/discover.h | 5 + src/foundation/index_policy.c | 204 ++++++++++++++++++++++++++++++- src/foundation/index_policy.h | 29 ++++- src/mcp/mcp.c | 31 ++++- tests/test_daemon_application.c | 44 ++++++- tests/test_discover.c | 182 ++++++++++++++++++++++++++++ tests/test_index_policy.c | 205 ++++++++++++++++++++++++++++++++ 14 files changed, 882 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 17825dfff4..144edee95c 100644 --- a/README.md +++ b/README.md @@ -926,6 +926,7 @@ codebase-memory-mcp config set auto_watch false # don't register backgr codebase-memory-mcp config set doc_candidates false # skip doc<->code candidate scoring on huge repos (default: auto = on for full, off for moderate/fast) codebase-memory-mcp config set watcher_enabled false # stop the watcher thread entirely (default: true) codebase-memory-mcp config set watch_non_git true # also poll non-git project roots (default: false) +codebase-memory-mcp config set index_resource_profile balanced # opt-in composed resource limits codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree charged-memory limit @@ -935,10 +936,13 @@ codebase-memory-mcp config set index_min_free_disk_mb 4096 # optional free-space codebase-memory-mcp config reset auto_index # reset to default ``` -The six index resource settings default to `off`. Exceeding one fails the -complete index attempt rather than publishing a partial graph; an existing -serving index is preserved. Storage probes also fail closed when an enabled -measurement cannot be completed. See +`index_resource_profile` defaults to `off` and accepts `off`, `balanced`, or +`strict`. The six individual resource settings also default to `off`; an +explicit value, including `off`, replaces that dimension of a selected +profile. Exceeding an effective limit fails the complete index attempt rather +than publishing a partial graph, and an existing serving index is preserved. +Storage probes also fail closed when an enabled measurement cannot be +completed. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). ### Environment Variables diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 7d45a85b21..933890d98d 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -91,6 +91,7 @@ Current keys: | `doc_candidates` | `auto` | Score documentation sections against code to store doc↔code candidates (the possible targets of a section, and the sections that may describe a function or file, each with a judged probability). Unset (`auto`), the index mode decides: on for `full` indexes (the default mode), off for `moderate` and `fast`. Set `true` or `false` to choose for every mode; `config reset doc_candidates` returns to `auto`. The scoring grows with sections × functions: under a second up to ~100k functions, ~15 s at ~400k, ~3 minutes on the Linux kernel. Off stores no candidates and leaves the rest of the graph unchanged; on in `fast` mode stores candidates without adding `fast`'s skipped `SIMILAR_TO` / `SEMANTICALLY_RELATED` edges. `index_repository` also takes `doc_candidates` for a single index, which wins over this setting. Applies from the next index. | | `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. Only an open MCP session's own project is ever watched; `index_status` reports the current state in its `watch` object. | | `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. | +| `index_resource_profile` | `off` | Composed index resource baseline: `off`, `balanced`, or `strict`. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | | `index_max_rss_mb` | `off` | Optional maximum charged memory in MiB for the complete contained index-worker process tree (`64..1048576`). On macOS this is phys_footprint, not resident size. | @@ -130,8 +131,12 @@ has not run yet). > `auto_index` still runs, and `index_repository` stays available for manual > reindexing. -The six index resource settings are independent and disabled by default. -They apply to explicit indexing, automatic indexing, and watcher re-indexing, +The profile and all six individual resource settings are disabled by default. +An individual stored value replaces its profile dimension; storing `off` +disables only that dimension. Profile-only limits also bound traversed +directories, directory entries, depth, discovery time, final database size, +staging size, and task temporary size. Effective limits apply to explicit +indexing, automatic indexing, and watcher re-indexing, but not to `cross-repo-intelligence`, which does not scan or publish repository source indexes. Equality is allowed; exceeding a setting fails the complete index request and preserves any previously serving database. Worker RSS covers diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 74cce0bd95..736c7d05f5 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -4,6 +4,51 @@ Index resource limits are optional operator controls for repositories whose discovery breadth or worker runtime is not known in advance. They are disabled by default so existing large-repository workloads retain their current behavior. +## Resource profiles + +`index_resource_profile` accepts `off`, `balanced`, or `strict` and defaults to +`off`. A profile supplies a baseline; each explicitly stored individual +resource key then replaces its dimension. An explicit `off` disables only that +dimension. + +| Dimension | `balanced` | `strict` | +|---|---:|---:| +| Accepted files | 500,000 | 100,000 | +| Traversed directories | 250,000 | 20,000 | +| Directory entries | 2,000,000 | 500,000 | +| Directory depth | 128 | 64 | +| Accepted source | 65,536 MiB | 4,096 MiB | +| Discovery duration | 1,800 seconds | 30 seconds | +| Worker duration | 7,200 seconds | 3,600 seconds | +| Final database | 65,536 MiB | 16,384 MiB | +| Staging artifacts | 81,920 MiB | 20,480 MiB | +| Task temporary artifacts | 98,304 MiB | 24,576 MiB | +| Projected cache | 131,072 MiB | 32,768 MiB | +| Free-disk reserve | 4,096 MiB | 4,096 MiB | + +The two profiles answer different questions about worker memory, so their RSS +ceilings are derived rather than tabled. + +Balanced protects the host. Its ceiling is 80% of detected host memory, which +leaves an index free to use everything this machine can spare and still stops a +runaway worker before the host is exhausted. A fixed number is deliberately not +used here: large-repository indexing peaks in the tens of gigabytes, so any +round ceiling low enough to feel safe would reject repositories that index +successfully with the profile off. When host memory cannot be read, balanced +falls back to the higher of twice the worker soft budget or 8,192 MiB. + +Strict protects the budget. Its ceiling is the lower of the worker soft budget +and 8,192 MiB, which is the point of the profile: a repository that needs more +memory than the daemon budgets for itself fails fast with an attributed +`rss_bytes` violation instead of completing at the host's expense. On a large +repository this is expected to fail, and the failure names the limit and the +observed value so the operator can choose `balanced`, raise +`index_max_rss_mb`, or leave the profile `off`. + +Both derived values are held inside the same 64 to 1,048,576 MiB range that +`index_max_rss_mb` accepts, and detected host memory can lower but never raise +an explicitly configured `index_max_rss_mb`. + ## Discovery settings | Key | Default | Accepted value | Protects | @@ -69,6 +114,12 @@ rules, filename and suffix filters, language detection, and the existing per-file size rule. `index_max_source_mb` sums the filesystem sizes of that same accepted set. +Profile directory counting includes the request root and each non-skipped +directory admitted for traversal. Entry counting happens before ignore, +language, and file-size filtering. Root depth is zero. The monotonic discovery +deadline is checked before opening a directory and before processing each +entry. Equality is allowed for every dimension. + Equality is allowed. The first file or byte that makes an observed value greater than its limit stops discovery. CBM discards the partial file list and does not publish a partial graph as a complete index. diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index b7e3b885a3..8978680a60 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -109,10 +109,13 @@ _cbm_test_runtime_daemon() { # worker in argv; a worker that finds no complete policy refuses to start rather # than index unbounded. A shell test that spawns `cli --index-worker` itself # stands in for the supervisor and owes the worker the same object. Mirrors -# cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and -# nothing else. +# cbm_mcp_index_policy_add_to_args: the resolved profile, the override mask +# that says which keys the operator set by hand, every key in +# cbm_index_policy_key_at, and nothing else. cbm_test_index_worker_policy_json() { - printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"' + printf '%s' '"_cbm_index_policy":{"index_resource_profile":"off"' + printf '%s' ',"_cbm_index_override_mask":0' + printf '%s' ',"index_max_files":"off","index_max_source_mb":"off"' printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"' printf '%s' ',"index_cache_max_mb":"off","index_min_free_disk_mb":"off"}' } diff --git a/src/cli/cli.c b/src/cli/cli.c index 759512e626..360d64524e 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -23,6 +23,7 @@ #include "foundation/platform.h" #include "foundation/constants.h" #include "foundation/log.h" +#include "foundation/mem.h" #include "foundation/sha256.h" #include "foundation/str_util.h" #include "cli/client_adapter.h" @@ -7562,13 +7563,19 @@ bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t return false; } cbm_index_policy_init(policy); + const char *profile = cbm_config_get(cfg, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "off"); + if (!cbm_index_policy_set_profile(policy, profile, error, error_size)) { + return false; + } for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { const char *key = cbm_index_policy_key_at(index); - const char *value = cbm_config_get(cfg, key, cbm_index_policy_default_value(key)); - if (!cbm_index_policy_set(policy, key, value, error, error_size)) { + const char *value = cbm_config_get(cfg, key, NULL); + if (value && !cbm_index_policy_set(policy, key, value, error, error_size)) { return false; } } + cbm_system_info_t system = cbm_system_info(); + cbm_index_policy_finalize(policy, (uint64_t)system.total_ram, (uint64_t)cbm_mem_budget()); return true; } @@ -7601,6 +7608,7 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"}, {CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"}, {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, + {CBM_INDEX_CONFIG_RESOURCE_PROFILE, "off", "Index resource profile: off, balanced, or strict"}, {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree charged memory MiB, or off"}, @@ -7635,19 +7643,17 @@ static bool config_key_is_ui(const char *key) { } static bool config_key_is_index_policy(const char *key) { - for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) { - if (strcmp(key, cbm_index_policy_key_at(index)) == 0) { - return true; - } - } - return false; + return cbm_index_policy_is_config_key(key); } static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { cbm_index_resource_policy_t candidate; cbm_index_policy_init(&candidate); char error[CLI_BUF_256]; - if (!cbm_index_policy_set(&candidate, key, value, error, sizeof(error))) { + bool valid = strcmp(key, CBM_INDEX_CONFIG_RESOURCE_PROFILE) == 0 + ? cbm_index_policy_set_profile(&candidate, value, error, sizeof(error)) + : cbm_index_policy_set(&candidate, key, value, error, sizeof(error)); + if (!valid) { (void)fprintf(stderr, "error: %s\n", error); return CLI_ERR; } diff --git a/src/daemon/application.c b/src/daemon/application.c index e4d801e29c..f2e2e38085 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -2367,9 +2367,32 @@ static bool application_index_args_add_policy(cbm_daemon_application_t *applicat cbm_log_error("daemon.index.policy", "error", error); return false; } + cbm_system_info_t system = cbm_system_info(); + cbm_index_policy_finalize(&policy, (uint64_t)system.total_ram, + application ? (uint64_t)application->worker_memory_budget_bytes : 0); return doc_candidates_encoded && cbm_mcp_index_policy_add_to_args(document, root, &policy); } +static char *application_index_args_replace_policy(cbm_daemon_application_t *application, + const char *args_json) { + yyjson_doc *source = args_json ? yyjson_read(args_json, strlen(args_json), 0) : NULL; + yyjson_mut_doc *document = source ? yyjson_doc_mut_copy(source, NULL) : NULL; + yyjson_doc_free(source); + yyjson_mut_val *root = document ? yyjson_mut_doc_get_root(document) : NULL; + if (!root || !yyjson_mut_is_obj(root)) { + yyjson_mut_doc_free(document); + return NULL; + } + while (yyjson_mut_obj_get(root, "_cbm_index_policy")) { + (void)yyjson_mut_obj_remove_key(root, "_cbm_index_policy"); + } + char *rewritten = application_index_args_add_policy(application, document, root) + ? yyjson_mut_write(document, 0, NULL) + : NULL; + yyjson_mut_doc_free(document); + return rewritten; +} + static char *application_auto_index_args(cbm_daemon_application_t *application, const char *root_path) { yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); @@ -2663,14 +2686,19 @@ static const char *application_index_admission_error(application_job_subscribe_s * physical job limit); a cancelled request drops only its own reference. */ static char *application_index_execute_sync(cbm_daemon_application_session_t *session, const char *root_path, const char *args_json) { - char *project_key = application_index_project_key(root_path, args_json); + char *trusted_args = application_index_args_replace_policy(session->application, args_json); + if (!trusted_args) { + return cbm_mcp_text_result("failed to resolve daemon index resource policy", true); + } + char *project_key = application_index_project_key(root_path, trusted_args); if (!project_key) { + free(trusted_args); return cbm_mcp_text_result("failed to derive index project identity", true); } application_job_subscribe_status_t subscribe_status = APPLICATION_JOB_SUBSCRIBE_UNAVAILABLE; cbm_daemon_application_job_t *job = NULL; for (;;) { - job = application_job_subscribe(session->application, project_key, root_path, args_json, + job = application_job_subscribe(session->application, project_key, root_path, trusted_args, &subscribe_status); if (job || (subscribe_status != APPLICATION_JOB_SUBSCRIBE_BUSY && subscribe_status != APPLICATION_JOB_SUBSCRIBE_CANCELLING)) { @@ -2692,12 +2720,14 @@ static char *application_index_execute_sync(cbm_daemon_application_session_t *se cbm_mutex_unlock(&session->application->mutex); if (queued_cancelled) { free(project_key); + free(trusted_args); return cbm_mcp_text_result( "index operation cancelled for this session. " CBM_MCP_INDEX_ASYNC_HINT, true); } cbm_usleep(APPLICATION_JOB_POLL_US); } free(project_key); + free(trusted_args); if (!job) { return cbm_mcp_text_result(application_index_admission_error(subscribe_status), true); } diff --git a/src/discover/discover.c b/src/discover/discover.c index 134dd18c88..6c10b09c3e 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -39,6 +39,25 @@ int cbm_gitignore_match_result(const cbm_gitignore_t *gi, const char *rel_path, bool is_dir); +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_discover_now_ms_fn g_discover_now_ms; +static void *g_discover_now_ms_context; + +void cbm_discover_set_now_ms_for_testing(cbm_discover_now_ms_fn now_ms, void *context) { + g_discover_now_ms = now_ms; + g_discover_now_ms_context = context; +} +#endif + +static uint64_t discover_now_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_discover_now_ms) { + return g_discover_now_ms(g_discover_now_ms_context); + } +#endif + return cbm_now_ms(); +} + /* ── Hardcoded always-skip directories ──────────────────────────── */ static const char *ALWAYS_SKIP_DIRS[] = { @@ -471,6 +490,9 @@ typedef struct { uint64_t deadline_ms; const cbm_index_resource_policy_t *resource_policy; cbm_index_resource_violation_t *resource_violation; + uint64_t resource_started_ms; + uint64_t directories; + uint64_t entries; uint64_t source_files; uint64_t source_bytes; bool count_only; @@ -514,9 +536,21 @@ static bool file_list_should_stop(file_list_t *fl) { if (!fl) { return true; } - if (!fl->failed && fl->deadline_ms != 0 && cbm_now_ms() >= fl->deadline_ms) { + bool needs_now = fl->deadline_ms != 0 || + (fl->resource_policy && fl->resource_policy->discovery_deadline_ms.enabled); + uint64_t now = needs_now ? discover_now_ms() : 0; + if (!fl->failed && fl->deadline_ms != 0 && now >= fl->deadline_ms) { fl->failed = true; } + if (!fl->failed && fl->resource_policy && fl->resource_policy->discovery_deadline_ms.enabled) { + uint64_t elapsed = + now >= fl->resource_started_ms ? now - fl->resource_started_ms : UINT64_C(0); + uint64_t limit = fl->resource_policy->discovery_deadline_ms.value; + if (elapsed > limit) { + file_list_resource_violation(fl, CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS, elapsed, + limit); + } + } return fl->failed || fl->limit_exceeded; } @@ -1112,6 +1146,7 @@ typedef struct { const gitignore_link_t *ignore_chain; /* deepest .gitignore governing this dir */ sphinx_suffixes_t sphinx; /* the documentation set's extra reST suffixes */ bool git_ignored; /* ignored by git's rules, entered only for its tracked content */ + uint64_t depth; } walk_frame_t; /* Initial capacity only — the stack grows on demand. A single directory can * hold more pending sibling frames than any fixed cap (dotnet/runtime has 855 @@ -1171,6 +1206,7 @@ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char slot->ignore_chain = parent->ignore_chain; slot->sphinx = parent->sphinx; slot->git_ignored = git_ignored; + slot->depth = parent->depth == UINT64_MAX ? UINT64_MAX : parent->depth + 1U; ws->top++; } @@ -1179,6 +1215,18 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram const cbm_gitignore_t *global_gi, const cbm_gitignore_t *cbmignore, walk_stack_t *ws, file_list_t *out) { + if (out->resource_policy && out->resource_policy->max_entries.enabled) { + uint64_t limit = out->resource_policy->max_entries.value; + if (out->entries >= limit) { + uint64_t observed = out->entries == UINT64_MAX ? UINT64_MAX : out->entries + 1U; + file_list_resource_violation(out, CBM_INDEX_RESOURCE_ENTRIES, observed, limit); + return; + } + } + if (out->entries < UINT64_MAX) { + out->entries++; + } + char abs_path[CBM_SZ_4K]; char rel_path[CBM_SZ_4K]; int absolute_length = snprintf(abs_path, sizeof(abs_path), "%s/%s", frame->dir, entry->name); @@ -1218,6 +1266,24 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram } if (verdict == DIR_KEEP || (verdict == DIR_GIT_IGNORED && tracked_has_dir(out->tracked, rel_path))) { + uint64_t child_depth = frame->depth == UINT64_MAX ? UINT64_MAX : frame->depth + 1U; + if (out->resource_policy && out->resource_policy->max_depth.enabled && + child_depth > out->resource_policy->max_depth.value) { + file_list_resource_violation(out, CBM_INDEX_RESOURCE_DEPTH, child_depth, + out->resource_policy->max_depth.value); + return; + } + if (out->resource_policy && out->resource_policy->max_directories.enabled && + out->directories >= out->resource_policy->max_directories.value) { + uint64_t observed = + out->directories == UINT64_MAX ? UINT64_MAX : out->directories + 1U; + file_list_resource_violation(out, CBM_INDEX_RESOURCE_DIRECTORIES, observed, + out->resource_policy->max_directories.value); + return; + } + if (out->directories < UINT64_MAX) { + out->directories++; + } walk_push_subdir(ws, abs_path, rel_path, frame, verdict == DIR_GIT_IGNORED, out); } else { /* Record the excluded subtree root so callers can report it (#411). */ @@ -1297,6 +1363,7 @@ static void walk_dir(const char *dir_path, const char *rel_prefix, const cbm_dis return; } } + ws.frames[0].depth = 0; ws.top++; while (ws.top > 0 && !file_list_should_stop(out)) { @@ -1844,6 +1911,11 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc .deadline_ms = count_only ? deadline_ms : 0, .resource_policy = opts ? opts->resource_policy : NULL, .resource_violation = opts ? opts->resource_violation : NULL, + .resource_started_ms = + opts && opts->resource_policy && opts->resource_policy->discovery_deadline_ms.enabled + ? discover_now_ms() + : 0, + .directories = 1, .count_only = count_only, .collect_excluded = !count_only && excluded_out != NULL, .collect_ignored = !count_only && ignored_out != NULL, @@ -1853,8 +1925,14 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc tracked_load(&tracked, repo_path); fl.tracked = &tracked; } + if (fl.resource_policy && fl.resource_policy->max_directories.enabled && + fl.resource_policy->max_directories.value < 1U) { + file_list_resource_violation(&fl, CBM_INDEX_RESOURCE_DIRECTORIES, 1, + fl.resource_policy->max_directories.value); + } walk_cache_dir_snapshot(); walk_dir(repo_path, "", opts, gitignore, ancestors.chain, global_gi, cbmignore, &fl); + (void)file_list_should_stop(&fl); /* Cleanup */ tracked_free(&tracked); diff --git a/src/discover/discover.h b/src/discover/discover.h index 85bc052945..9b31905f71 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -191,6 +191,11 @@ typedef enum { CBM_DISCOVER_LIMIT_EXCEEDED = 1, } cbm_discover_status_t; +#ifdef CBM_ENABLE_TEST_SEAMS +typedef uint64_t (*cbm_discover_now_ms_fn)(void *context); +void cbm_discover_set_now_ms_for_testing(cbm_discover_now_ms_fn now_ms, void *context); +#endif + /* Walk a repository directory tree and discover all source files. * Applies hardcoded filters, gitignore patterns, and language detection. * Returns 0 on success, -1 on error. diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c index 9c0f22e9c0..6b9cab1caa 100644 --- a/src/foundation/index_policy.c +++ b/src/foundation/index_policy.c @@ -27,6 +27,61 @@ static const index_policy_metadata_t INDEX_POLICY_METADATA[] = { 1, CBM_INDEX_MAX_STORAGE_MB_VALUE, CBM_INDEX_MIB_BYTES}, }; +static cbm_index_limit_u64_t limit_value(uint64_t value) { + return (cbm_index_limit_u64_t){.enabled = true, .value = value}; +} + +static uint64_t mib(uint64_t value) { + return value * CBM_INDEX_MIB_BYTES; +} + +static void apply_profile_baseline(cbm_index_resource_policy_t *policy, + cbm_index_resource_profile_t profile) { + policy->max_files = (cbm_index_limit_u64_t){0}; + policy->max_directories = (cbm_index_limit_u64_t){0}; + policy->max_entries = (cbm_index_limit_u64_t){0}; + policy->max_depth = (cbm_index_limit_u64_t){0}; + policy->max_source_bytes = (cbm_index_limit_u64_t){0}; + policy->discovery_deadline_ms = (cbm_index_limit_u64_t){0}; + policy->max_rss_bytes = (cbm_index_limit_u64_t){0}; + policy->max_duration_ms = (cbm_index_limit_u64_t){0}; + policy->max_cache_bytes = (cbm_index_limit_u64_t){0}; + policy->min_free_disk_bytes = (cbm_index_limit_u64_t){0}; + policy->max_final_db_bytes = (cbm_index_limit_u64_t){0}; + policy->max_staging_bytes = (cbm_index_limit_u64_t){0}; + policy->max_task_temp_bytes = (cbm_index_limit_u64_t){0}; + policy->profile = profile; + if (profile == CBM_INDEX_PROFILE_BALANCED) { + policy->max_files = limit_value(UINT64_C(500000)); + policy->max_directories = limit_value(UINT64_C(250000)); + policy->max_entries = limit_value(UINT64_C(2000000)); + policy->max_depth = limit_value(UINT64_C(128)); + policy->max_source_bytes = limit_value(mib(UINT64_C(65536))); + policy->discovery_deadline_ms = limit_value(UINT64_C(1800000)); + policy->max_rss_bytes = limit_value(mib(UINT64_C(8192))); + policy->max_duration_ms = limit_value(UINT64_C(7200000)); + policy->max_final_db_bytes = limit_value(mib(UINT64_C(65536))); + policy->max_staging_bytes = limit_value(mib(UINT64_C(81920))); + policy->max_task_temp_bytes = limit_value(mib(UINT64_C(98304))); + policy->max_cache_bytes = limit_value(mib(UINT64_C(131072))); + policy->min_free_disk_bytes = limit_value(mib(UINT64_C(4096))); + } else if (profile == CBM_INDEX_PROFILE_STRICT) { + policy->max_files = limit_value(UINT64_C(100000)); + policy->max_directories = limit_value(UINT64_C(20000)); + policy->max_entries = limit_value(UINT64_C(500000)); + policy->max_depth = limit_value(UINT64_C(64)); + policy->max_source_bytes = limit_value(mib(UINT64_C(4096))); + policy->discovery_deadline_ms = limit_value(UINT64_C(30000)); + policy->max_rss_bytes = limit_value(mib(UINT64_C(8192))); + policy->max_duration_ms = limit_value(UINT64_C(3600000)); + policy->max_final_db_bytes = limit_value(mib(UINT64_C(16384))); + policy->max_staging_bytes = limit_value(mib(UINT64_C(20480))); + policy->max_task_temp_bytes = limit_value(mib(UINT64_C(24576))); + policy->max_cache_bytes = limit_value(mib(UINT64_C(32768))); + policy->min_free_disk_bytes = limit_value(mib(UINT64_C(4096))); + } +} + static void set_error(char *error, size_t error_size, const char *key, uint64_t minimum, uint64_t maximum) { if (error && error_size > 0) { @@ -64,13 +119,138 @@ void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { } } +bool cbm_index_policy_set_profile(cbm_index_resource_policy_t *policy, const char *value, + char *error, size_t error_size) { + if (error && error_size > 0) { + error[0] = '\0'; + } + cbm_index_resource_profile_t profile; + if (value && strcmp(value, "off") == 0) { + profile = CBM_INDEX_PROFILE_OFF; + } else if (value && strcmp(value, "balanced") == 0) { + profile = CBM_INDEX_PROFILE_BALANCED; + } else if (value && strcmp(value, "strict") == 0) { + profile = CBM_INDEX_PROFILE_STRICT; + } else { + if (error && error_size > 0) { + (void)snprintf(error, error_size, "%s must be off, balanced, or strict", + CBM_INDEX_CONFIG_RESOURCE_PROFILE); + } + return false; + } + if (!policy) { + return false; + } + + cbm_index_resource_policy_t candidate = *policy; + cbm_index_limit_u64_t + overrides[sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0])]; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const cbm_index_limit_u64_t *source = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + + INDEX_POLICY_METADATA[index].field_offset); + overrides[index] = *source; + } + apply_profile_baseline(&candidate, profile); + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if ((candidate.override_mask & (UINT64_C(1) << index)) == 0) { + continue; + } + cbm_index_limit_u64_t *target = + (cbm_index_limit_u64_t *)((unsigned char *)&candidate + + INDEX_POLICY_METADATA[index].field_offset); + *target = overrides[index]; + } + *policy = candidate; + return true; +} + +void cbm_index_policy_finalize(cbm_index_resource_policy_t *policy, uint64_t host_memory_bytes, + uint64_t soft_budget_bytes) { + if (!policy) { + return; + } + uint64_t host_cap = host_memory_bytes > 0 + ? (host_memory_bytes / 5U) * 4U + ((host_memory_bytes % 5U) * 4U) / 5U + : 0; + const uint64_t rss_override_bit = UINT64_C(1) << 2U; + if ((policy->override_mask & rss_override_bit) == 0) { + if (policy->profile == CBM_INDEX_PROFILE_BALANCED) { + /* Balanced guards the host, not the workload. Any fixed ceiling + * below what this machine can actually finish would reject indexes + * that succeed today — the recorded large-repository runs peak far + * above any round number worth hard-coding. The host share is the + * only honest balanced answer; the fixed floor is a fallback for + * hosts whose memory cannot be read. */ + uint64_t doubled = + soft_budget_bytes > UINT64_MAX / 2U ? UINT64_MAX : soft_budget_bytes * 2U; + uint64_t fallback = doubled > mib(UINT64_C(8192)) ? doubled : mib(UINT64_C(8192)); + policy->max_rss_bytes = limit_value(host_cap > 0 ? host_cap : fallback); + } else if (policy->profile == CBM_INDEX_PROFILE_STRICT) { + /* Strict holds the worker to the daemon's own soft budget, so a + * repository larger than that budget fails fast and attributed + * instead of being finished at the host's expense. */ + uint64_t baseline = soft_budget_bytes > 0 ? soft_budget_bytes : mib(UINT64_C(8192)); + if (baseline > mib(UINT64_C(8192))) { + baseline = mib(UINT64_C(8192)); + } + policy->max_rss_bytes = limit_value(baseline); + } + } + if (policy->max_rss_bytes.enabled && host_cap > 0 && policy->max_rss_bytes.value > host_cap) { + policy->max_rss_bytes.value = host_cap; + } + if (policy->max_rss_bytes.enabled) { + uint64_t minimum = mib(CBM_INDEX_MIN_RSS_MB_VALUE); + uint64_t maximum = mib(CBM_INDEX_MAX_RSS_MB_VALUE); + if (policy->max_rss_bytes.value < minimum) { + policy->max_rss_bytes.value = minimum; + } else if (policy->max_rss_bytes.value > maximum) { + policy->max_rss_bytes.value = maximum; + } + } +} + +const char *cbm_index_policy_profile_name(const cbm_index_resource_policy_t *policy) { + if (!policy || policy->profile == CBM_INDEX_PROFILE_OFF) { + return "off"; + } + return policy->profile == CBM_INDEX_PROFILE_BALANCED ? "balanced" : "strict"; +} + +const char *cbm_index_policy_source_name(const cbm_index_resource_policy_t *policy) { + bool profile = policy && policy->profile != CBM_INDEX_PROFILE_OFF; + bool override = policy && policy->override_mask != 0; + if (profile && override) { + return "profile+override"; + } + if (profile) { + return "profile"; + } + return override ? "override" : "off"; +} + +bool cbm_index_policy_is_config_key(const char *key) { + if (key && strcmp(key, CBM_INDEX_CONFIG_RESOURCE_PROFILE) == 0) { + return true; + } + for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + return true; + } + } + return false; +} + bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { return cbm_index_policy_discovery_enabled(policy) || cbm_index_policy_worker_enabled(policy) || cbm_index_policy_storage_enabled(policy); } bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy) { - return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled); + return policy && (policy->max_files.enabled || policy->max_directories.enabled || + policy->max_entries.enabled || policy->max_depth.enabled || + policy->max_source_bytes.enabled || policy->discovery_deadline_ms.enabled); } bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy) { @@ -185,6 +365,7 @@ bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, candidate.value = parsed * metadata->multiplier; } *target = candidate; + policy->override_mask |= UINT64_C(1) << (size_t)(metadata - INDEX_POLICY_METADATA); return true; } @@ -232,6 +413,14 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { return "staging_bytes"; case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: return "task_temp_bytes"; + case CBM_INDEX_RESOURCE_DIRECTORIES: + return "directories"; + case CBM_INDEX_RESOURCE_ENTRIES: + return "entries"; + case CBM_INDEX_RESOURCE_DEPTH: + return "depth"; + case CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS: + return "discovery_duration_ms"; case CBM_INDEX_RESOURCE_NONE: default: return "unknown"; @@ -245,6 +434,13 @@ const char *cbm_index_resource_unit(cbm_index_resource_t resource) { if (resource == CBM_INDEX_RESOURCE_DURATION_MS) { return "milliseconds"; } + if (resource == CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS) { + return "milliseconds"; + } + if (resource == CBM_INDEX_RESOURCE_DIRECTORIES || resource == CBM_INDEX_RESOURCE_ENTRIES || + resource == CBM_INDEX_RESOURCE_DEPTH) { + return "count"; + } return "bytes"; } @@ -265,7 +461,11 @@ const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { case CBM_INDEX_RESOURCE_FINAL_DB_BYTES: case CBM_INDEX_RESOURCE_STAGING_BYTES: case CBM_INDEX_RESOURCE_TASK_TEMP_BYTES: - return "index_resource_profile"; + case CBM_INDEX_RESOURCE_DIRECTORIES: + case CBM_INDEX_RESOURCE_ENTRIES: + case CBM_INDEX_RESOURCE_DEPTH: + case CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS: + return CBM_INDEX_CONFIG_RESOURCE_PROFILE; case CBM_INDEX_RESOURCE_NONE: default: return "index_resource_limit"; diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h index bac059f2e4..85d1234b06 100644 --- a/src/foundation/index_policy.h +++ b/src/foundation/index_policy.h @@ -11,6 +11,7 @@ #define CBM_INDEX_CONFIG_MAX_DURATION_SECONDS "index_max_duration_seconds" #define CBM_INDEX_CONFIG_CACHE_MAX_MB "index_cache_max_mb" #define CBM_INDEX_CONFIG_MIN_FREE_DISK_MB "index_min_free_disk_mb" +#define CBM_INDEX_CONFIG_RESOURCE_PROFILE "index_resource_profile" #define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) #define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) @@ -25,17 +26,26 @@ typedef struct { uint64_t value; } cbm_index_limit_u64_t; +typedef enum { + CBM_INDEX_PROFILE_OFF = 0, + CBM_INDEX_PROFILE_BALANCED, + CBM_INDEX_PROFILE_STRICT, +} cbm_index_resource_profile_t; + typedef struct { + cbm_index_resource_profile_t profile; + uint64_t override_mask; cbm_index_limit_u64_t max_files; + cbm_index_limit_u64_t max_directories; + cbm_index_limit_u64_t max_entries; + cbm_index_limit_u64_t max_depth; cbm_index_limit_u64_t max_source_bytes; + cbm_index_limit_u64_t discovery_deadline_ms; cbm_index_limit_u64_t max_rss_bytes; cbm_index_limit_u64_t max_duration_ms; cbm_index_limit_u64_t max_cache_bytes; cbm_index_limit_u64_t min_free_disk_bytes; - /* Internal storage dimensions with no public config key. Nothing enables - * them on its own; they exist so that a single composed decision can bound - * the database, the staging artifacts and the task temporary directory - * together, which no individual key can express. */ + /* Profile-only dimensions. They are deliberately not public config keys. */ cbm_index_limit_u64_t max_final_db_bytes; cbm_index_limit_u64_t max_staging_bytes; cbm_index_limit_u64_t max_task_temp_bytes; @@ -52,6 +62,10 @@ typedef enum { CBM_INDEX_RESOURCE_FINAL_DB_BYTES, CBM_INDEX_RESOURCE_STAGING_BYTES, CBM_INDEX_RESOURCE_TASK_TEMP_BYTES, + CBM_INDEX_RESOURCE_DIRECTORIES, + CBM_INDEX_RESOURCE_ENTRIES, + CBM_INDEX_RESOURCE_DEPTH, + CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS, } cbm_index_resource_t; typedef struct { @@ -72,6 +86,13 @@ typedef struct { } cbm_index_storage_sample_t; void cbm_index_policy_init(cbm_index_resource_policy_t *policy); +bool cbm_index_policy_set_profile(cbm_index_resource_policy_t *policy, const char *value, + char *error, size_t error_size); +void cbm_index_policy_finalize(cbm_index_resource_policy_t *policy, uint64_t host_memory_bytes, + uint64_t soft_budget_bytes); +const char *cbm_index_policy_profile_name(const cbm_index_resource_policy_t *policy); +const char *cbm_index_policy_source_name(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_is_config_key(const char *key); bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy); bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index c8e7d4d444..1feb8b89a3 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -11485,20 +11485,32 @@ bool cbm_mcp_index_policy_from_internal_args(const char *args, cbm_index_resourc yyjson_val *encoded = root && yyjson_is_obj(root) ? yyjson_obj_get(root, "_cbm_index_policy") : NULL; if (!encoded || !yyjson_is_obj(encoded) || - yyjson_obj_size(encoded) != cbm_index_policy_key_count()) { + yyjson_obj_size(encoded) != cbm_index_policy_key_count() + 2U) { yyjson_doc_free(doc); (void)snprintf(error, error_size, "missing or incomplete trusted worker policy"); return false; } cbm_index_policy_init(policy); - bool valid = true; + yyjson_val *profile = yyjson_obj_get(encoded, CBM_INDEX_CONFIG_RESOURCE_PROFILE); + yyjson_val *override_mask = yyjson_obj_get(encoded, "_cbm_index_override_mask"); + bool valid = profile && yyjson_is_str(profile) && override_mask && + yyjson_is_uint(override_mask) && + cbm_index_policy_set_profile(policy, yyjson_get_str(profile), error, error_size); for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { const char *key = cbm_index_policy_key_at(index); yyjson_val *value = yyjson_obj_get(encoded, key); valid = value && yyjson_is_str(value) && cbm_index_policy_set(policy, key, yyjson_get_str(value), error, error_size); } + if (valid) { + uint64_t known_mask = cbm_index_policy_key_count() >= 64U + ? UINT64_MAX + : (UINT64_C(1) << cbm_index_policy_key_count()) - 1U; + uint64_t decoded_mask = yyjson_get_uint(override_mask); + valid = (decoded_mask & ~known_mask) == 0; + policy->override_mask = decoded_mask; + } yyjson_doc_free(doc); if (!valid && error && error_size > 0 && error[0] == '\0') { (void)snprintf(error, error_size, "invalid trusted worker policy"); @@ -11543,7 +11555,11 @@ static bool load_index_policy(cbm_mcp_server_t *srv, const char *args, bool cbm_mcp_index_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_index_resource_policy_t *policy) { yyjson_mut_val *encoded = yyjson_mut_obj(doc); - bool valid = encoded != NULL; + bool valid = encoded != NULL && + yyjson_mut_obj_add_strcpy(doc, encoded, CBM_INDEX_CONFIG_RESOURCE_PROFILE, + cbm_index_policy_profile_name(policy)) && + yyjson_mut_obj_add_uint(doc, encoded, "_cbm_index_override_mask", + policy ? policy->override_mask : 0); for (size_t index = 0; valid && index < cbm_index_policy_key_count(); index++) { const char *key = cbm_index_policy_key_at(index); char value[CBM_SZ_64]; @@ -12661,8 +12677,13 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { } else if (rc == CBM_PIPELINE_RESOURCE_LIMIT && resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { const char *config_key = cbm_index_resource_config_key(resource_violation.resource); - bool discovery_resource = resource_violation.resource == CBM_INDEX_RESOURCE_FILES || - resource_violation.resource == CBM_INDEX_RESOURCE_SOURCE_BYTES; + bool discovery_resource = + resource_violation.resource == CBM_INDEX_RESOURCE_FILES || + resource_violation.resource == CBM_INDEX_RESOURCE_SOURCE_BYTES || + resource_violation.resource == CBM_INDEX_RESOURCE_DIRECTORIES || + resource_violation.resource == CBM_INDEX_RESOURCE_ENTRIES || + resource_violation.resource == CBM_INDEX_RESOURCE_DEPTH || + resource_violation.resource == CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS; char message[CBM_SZ_256]; (void)snprintf(message, sizeof(message), resource_violation.probe_failed ? "Index resource probe failed for %s" diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 04a5f7a302..1035756aca 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -2419,10 +2419,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool config_ready = stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; + cbm_config_set(stored_config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "strict") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -2441,6 +2438,7 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { .config = stored_config, .worker_ops = &worker_ops, .update_ops = &update_ops, + .aggregate_memory_budget_bytes = UINT64_C(128) * CBM_INDEX_MIB_BYTES, }; cbm_daemon_application_t *application = config_ready ? cbm_daemon_application_new(&config) : NULL; @@ -2468,7 +2466,41 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); bool auto_policy_propagated = - first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7", "off", "off"); + first_owned && + app_fake_worker_policy_equals(&fake, 0, "100000", "4096", "64", "3600", "32768", "4096"); + char explicit_args[APP_TEST_PATH_CAP + 64]; + (void)snprintf(explicit_args, sizeof(explicit_args), "{\"repo_path\":\"%s\"}", root); + uint8_t *explicit_tool = NULL; + uint32_t explicit_tool_length = 0; + bool explicit_encoded = app_test_tool_request("index_repository", explicit_args, &explicit_tool, + &explicit_tool_length); + app_request_thread_t explicit_request = { + .callbacks = callbacks, + .session = sessions[3], + .request = explicit_tool, + .request_length = explicit_tool_length, + }; + cbm_thread_t explicit_thread; + bool explicit_started = + explicit_encoded && + cbm_thread_create(&explicit_thread, 0, app_request_thread, &explicit_request) == 0; + bool explicit_coalesced = explicit_started && project && + app_wait_for_subscribers(application, project, 2) && + atomic_load(&fake.starts) == 1; + if (sessions[3]) { + callbacks.session_cancel(callbacks.context, sessions[3]); + } + if (explicit_started) { + (void)cbm_thread_join(&explicit_thread); + } + free(explicit_request.response); + free(explicit_tool); + if (sessions[3]) { + callbacks.session_close(callbacks.context, sessions[3]); + sessions[3] = NULL; + } + bool explicit_detached = + explicit_coalesced && project && app_wait_for_subscribers(application, project, 1); bool second_initialized = app_test_initialize_profile(&callbacks, sessions[1], root, CBM_MCP_TOOL_PROFILE_ALL, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2531,6 +2563,8 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { ASSERT_TRUE(first_initialized); ASSERT_TRUE(first_owned); ASSERT_TRUE(auto_policy_propagated); + ASSERT_TRUE(explicit_coalesced); + ASSERT_TRUE(explicit_detached); ASSERT_TRUE(second_initialized); ASSERT_TRUE(coalesced); ASSERT_TRUE(restricted_disconnect_kept_job); diff --git a/tests/test_discover.c b/tests/test_discover.c index 3b1c5a213f..38fe357640 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -13,6 +13,21 @@ typedef struct { char *xdg_config_home; } git_env_snapshot_t; +#ifdef CBM_ENABLE_TEST_SEAMS +typedef struct { + uint64_t first; + uint64_t second; + uint64_t later; + int calls; +} discover_clock_fake_t; + +static uint64_t discover_clock_fake(void *context) { + discover_clock_fake_t *fake = context; + int call = fake->calls++; + return call == 0 ? fake->first : call == 1 ? fake->second : fake->later; +} +#endif + static git_env_snapshot_t save_git_env(void) { git_env_snapshot_t snapshot = {0}; const char *home = getenv("HOME"); @@ -585,6 +600,169 @@ TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte) { PASS(); } +TEST(discover_profile_directory_limit_counts_root_and_admitted_directories) { + char *base = th_mktempdir("cbm_disc_policy_dirs"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "first/source.c"), "int first;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_directories = (cbm_index_limit_u64_t){.enabled = true, .value = 2}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + th_write_file(TH_PATH(base, "second/source.py"), "second = 2\n"); + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DIRECTORIES); + ASSERT_EQ(violation.observed, 3); + ASSERT_EQ(violation.limit, 2); + th_cleanup(base); + PASS(); +} + +TEST(discover_profile_entry_limit_counts_before_ignore_and_language_filters) { + char *base = th_mktempdir("cbm_disc_policy_entries"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, ".gitignore"), "ignored.c\n"); + th_write_file(TH_PATH(base, "ignored.c"), "int ignored;\n"); + th_write_file(TH_PATH(base, "asset.png"), "not source\n"); + th_write_file(TH_PATH(base, "accepted.c"), "int accepted;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_entries = (cbm_index_limit_u64_t){.enabled = true, .value = 4}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + policy.max_entries.value = 3; + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_ENTRIES); + ASSERT_EQ(violation.observed, 4); + ASSERT_EQ(violation.limit, 3); + th_cleanup(base); + PASS(); +} + +TEST(discover_profile_depth_limit_allows_equality_and_rejects_child) { + char *base = th_mktempdir("cbm_disc_policy_depth"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "one/two/source.c"), "int source;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_depth = (cbm_index_limit_u64_t){.enabled = true, .value = 2}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + cbm_file_info_t *files = NULL; + int count = 0; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_OK); + ASSERT_EQ(count, 1); + cbm_discover_free(files, count); + + policy.max_depth.value = 1; + files = NULL; + count = 99; + ASSERT_EQ(cbm_discover(base, &opts, &files, &count), CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DEPTH); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + th_cleanup(base); + PASS(); +} + +TEST(discover_profile_deadline_uses_monotonic_test_clock) { +#ifndef CBM_ENABLE_TEST_SEAMS + SKIP_PLATFORM("discovery clock seam unavailable"); +#else + char *base = th_mktempdir("cbm_disc_policy_deadline"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "source.c"), "int source;\n"); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.discovery_deadline_ms = (cbm_index_limit_u64_t){.enabled = true, .value = 1}; + cbm_index_resource_violation_t violation = {0}; + cbm_discover_opts_t opts = { + .mode = CBM_MODE_FULL, + .resource_policy = &policy, + .resource_violation = &violation, + }; + discover_clock_fake_t clock = {.first = 1000, .second = 1001, .later = 1002}; + cbm_discover_set_now_ms_for_testing(discover_clock_fake, &clock); + cbm_file_info_t *files = NULL; + int count = 99; + cbm_discover_status_t status = cbm_discover(base, &opts, &files, &count); + cbm_discover_set_now_ms_for_testing(NULL, NULL); + + ASSERT_EQ(status, CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + char empty[4096]; + (void)snprintf(empty, sizeof(empty), "%s/empty", base); + ASSERT_EQ(th_mkdir_p(empty), 0); + violation = (cbm_index_resource_violation_t){0}; + clock = (discover_clock_fake_t){.first = 2000, .second = 2001, .later = 2002}; + cbm_discover_set_now_ms_for_testing(discover_clock_fake, &clock); + status = cbm_discover(empty, &opts, &files, &count); + cbm_discover_set_now_ms_for_testing(NULL, NULL); + ASSERT_EQ(status, CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT(files == NULL); + ASSERT_EQ(count, 0); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS); + ASSERT_EQ(violation.observed, 2); + ASSERT_EQ(violation.limit, 1); + + violation = (cbm_index_resource_violation_t){0}; + clock = (discover_clock_fake_t){.first = 3000, .second = 3001, .later = 3001}; + cbm_discover_set_now_ms_for_testing(discover_clock_fake, &clock); + status = cbm_discover(empty, &opts, &files, &count); + cbm_discover_set_now_ms_for_testing(NULL, NULL); + ASSERT_EQ(status, CBM_DISCOVER_OK); + ASSERT_EQ(count, 0); + ASSERT(files == NULL); + ASSERT_EQ(violation.resource, CBM_INDEX_RESOURCE_NONE); + th_cleanup(base); + PASS(); +#endif +} + TEST(discover_resource_file_budget_excludes_existing_oversized_skip) { char *base = th_mktempdir("cbm_disc_policy_oversized"); ASSERT(base != NULL); @@ -2443,6 +2621,10 @@ SUITE(discover) { RUN_TEST(discover_resource_policy_off_matches_legacy_discovery); RUN_TEST(discover_resource_file_limit_is_exact_and_counts_only_accepted_sources); RUN_TEST(discover_resource_source_bytes_allows_equality_and_rejects_one_more_byte); + RUN_TEST(discover_profile_directory_limit_counts_root_and_admitted_directories); + RUN_TEST(discover_profile_entry_limit_counts_before_ignore_and_language_filters); + RUN_TEST(discover_profile_depth_limit_allows_equality_and_rejects_child); + RUN_TEST(discover_profile_deadline_uses_monotonic_test_clock); RUN_TEST(discover_resource_file_budget_excludes_existing_oversized_skip); RUN_TEST(discover_skips_git_dir); RUN_TEST(discover_with_gitignore); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index dc362f6b5b..71f8240c76 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -7,7 +7,9 @@ #include "foundation/index_policy.h" #include "mcp/index_supervisor.h" #include "mcp/mcp.h" +#include "mcp/mcp_internal.h" #include "store/store.h" +#include #include #include @@ -20,8 +22,14 @@ TEST(index_policy_defaults_are_disabled) { cbm_index_resource_policy_t policy; cbm_index_policy_init(&policy); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "off"); + ASSERT_STR_EQ(cbm_index_policy_source_name(&policy), "off"); ASSERT_FALSE(policy.max_files.enabled); + ASSERT_FALSE(policy.max_directories.enabled); + ASSERT_FALSE(policy.max_entries.enabled); + ASSERT_FALSE(policy.max_depth.enabled); ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(policy.discovery_deadline_ms.enabled); ASSERT_FALSE(policy.max_rss_bytes.enabled); ASSERT_FALSE(policy.max_duration_ms.enabled); ASSERT_FALSE(policy.max_cache_bytes.enabled); @@ -42,6 +50,140 @@ TEST(index_policy_defaults_are_disabled) { PASS(); } +TEST(index_policy_balanced_profile_expands_exact_baseline) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(20) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(4) * 1024 * CBM_INDEX_MIB_BYTES); + + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "balanced"); + ASSERT_STR_EQ(cbm_index_policy_source_name(&policy), "profile"); + ASSERT_EQ(policy.max_files.value, UINT64_C(500000)); + ASSERT_EQ(policy.max_directories.value, UINT64_C(250000)); + ASSERT_EQ(policy.max_entries.value, UINT64_C(2000000)); + ASSERT_EQ(policy.max_depth.value, UINT64_C(128)); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(65536) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.discovery_deadline_ms.value, UINT64_C(1800000)); + /* Four fifths of a 20 GiB host. */ + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(16) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7200000)); + ASSERT_EQ(policy.max_final_db_bytes.value, UINT64_C(65536) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_staging_bytes.value, UINT64_C(81920) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_task_temp_bytes.value, UINT64_C(98304) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(131072) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + PASS(); +} + +/* A balanced ceiling must never sit below what the host can actually finish. + * The large-repository runs this project records as reference workloads peak + * in the tens of gigabytes, so a fixed balanced ceiling would reject indexes + * that complete today with the profile off. */ +TEST(index_policy_balanced_rss_follows_the_host_share) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(40) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(4) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(policy.max_rss_bytes.enabled); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(32) * 1024 * CBM_INDEX_MIB_BYTES); + + /* A small host still receives a real ceiling below its own memory. */ + cbm_index_policy_init(&policy); + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(10) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(2) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(8) * 1024 * CBM_INDEX_MIB_BYTES); + + /* Host memory that cannot be read falls back to the fixed floor. */ + cbm_index_policy_init(&policy); + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + cbm_index_policy_finalize(&policy, 0, UINT64_C(6) * 1024 * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(12) * 1024 * CBM_INDEX_MIB_BYTES); + PASS(); +} + +TEST(index_policy_strict_profile_expands_exact_baseline) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "strict", error, sizeof(error))); + cbm_index_policy_finalize(&policy, 0, UINT64_C(3) * 1024 * CBM_INDEX_MIB_BYTES); + + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "strict"); + ASSERT_EQ(policy.max_files.value, UINT64_C(100000)); + ASSERT_EQ(policy.max_directories.value, UINT64_C(20000)); + ASSERT_EQ(policy.max_entries.value, UINT64_C(500000)); + ASSERT_EQ(policy.max_depth.value, UINT64_C(64)); + ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.discovery_deadline_ms.value, UINT64_C(30000)); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(3072) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(3600000)); + ASSERT_EQ(policy.max_final_db_bytes.value, UINT64_C(16384) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_staging_bytes.value, UINT64_C(20480) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_task_temp_bytes.value, UINT64_C(24576) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_cache_bytes.value, UINT64_C(32768) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.min_free_disk_bytes.value, UINT64_C(4096) * CBM_INDEX_MIB_BYTES); + PASS(); +} + +TEST(index_policy_explicit_override_and_off_replace_profile_dimensions) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "balanced", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_FILES, "42", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "off", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(1024) * CBM_INDEX_MIB_BYTES, + UINT64_C(512) * CBM_INDEX_MIB_BYTES); + + ASSERT_EQ(policy.max_files.value, UINT64_C(42)); + ASSERT_FALSE(policy.max_rss_bytes.enabled); + ASSERT_TRUE(policy.max_entries.enabled); + ASSERT_STR_EQ(cbm_index_policy_source_name(&policy), "profile+override"); + PASS(); +} + +TEST(index_policy_host_clamp_only_tightens_explicit_rss) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "4096", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(2048) * CBM_INDEX_MIB_BYTES, 0); + ASSERT_EQ(policy.max_rss_bytes.value, + (UINT64_C(2048) * CBM_INDEX_MIB_BYTES * UINT64_C(4)) / UINT64_C(5)); + + cbm_index_policy_init(&policy); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "512", error, sizeof(error))); + cbm_index_policy_finalize(&policy, UINT64_C(2048) * CBM_INDEX_MIB_BYTES, 0); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(512) * CBM_INDEX_MIB_BYTES); + PASS(); +} + +TEST(index_policy_profile_validation_is_atomic) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "strict", error, sizeof(error))); + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE(cbm_index_policy_set_profile(&policy, "STRICT", error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + ASSERT_TRUE(strstr(error, CBM_INDEX_CONFIG_RESOURCE_PROFILE) != NULL); + PASS(); +} + TEST(index_policy_file_limit_accepts_off_and_exact_range) { cbm_index_resource_policy_t policy; cbm_index_policy_init(&policy); @@ -256,6 +398,8 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "off"); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "strict"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64"), 0); @@ -264,6 +408,8 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB, "5"), 0); ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_EQ(policy.max_files.value, 9); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&policy), "strict"); + ASSERT_EQ(policy.max_directories.value, UINT64_C(20000)); ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7000)); @@ -280,6 +426,7 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { } TEST(index_policy_cli_lists_all_operator_keys) { + bool profile_found = false; bool files_found = false; bool bytes_found = false; bool rss_found = false; @@ -288,6 +435,8 @@ TEST(index_policy_cli_lists_all_operator_keys) { bool free_found = false; for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { const char *key = cbm_cli_config_key_at_for_testing(index); + profile_found = + profile_found || (key && strcmp(key, CBM_INDEX_CONFIG_RESOURCE_PROFILE) == 0); files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); rss_found = rss_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_RSS_MB) == 0); @@ -296,6 +445,7 @@ TEST(index_policy_cli_lists_all_operator_keys) { cache_found = cache_found || (key && strcmp(key, CBM_INDEX_CONFIG_CACHE_MAX_MB) == 0); free_found = free_found || (key && strcmp(key, CBM_INDEX_CONFIG_MIN_FREE_DISK_MB) == 0); } + ASSERT_TRUE(profile_found); ASSERT_TRUE(files_found); ASSERT_TRUE(bytes_found); ASSERT_TRUE(rss_found); @@ -314,6 +464,8 @@ TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) { char *set_valid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "7"}; char *set_invalid[] = {"set", CBM_INDEX_CONFIG_MAX_FILES, "0"}; + char *set_profile[] = {"set", CBM_INDEX_CONFIG_RESOURCE_PROFILE, "balanced"}; + char *set_invalid_profile[] = {"set", CBM_INDEX_CONFIG_RESOURCE_PROFILE, "BALANCED"}; char *reset[] = {"reset", CBM_INDEX_CONFIG_MAX_FILES}; int valid_rc = cbm_cmd_config(3, set_valid); cbm_config_t *config = cbm_config_open(cache); @@ -323,6 +475,14 @@ TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) { int invalid_rc = cbm_cmd_config(3, set_invalid); stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "missing") : "missing"; bool invalid_preserved = strcmp(stored, "7") == 0; + int profile_rc = cbm_cmd_config(3, set_profile); + stored = + config ? cbm_config_get(config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "missing") : "missing"; + bool profile_stored = strcmp(stored, "balanced") == 0; + int invalid_profile_rc = cbm_cmd_config(3, set_invalid_profile); + stored = + config ? cbm_config_get(config, CBM_INDEX_CONFIG_RESOURCE_PROFILE, "missing") : "missing"; + bool invalid_profile_preserved = strcmp(stored, "balanced") == 0; int reset_rc = cbm_cmd_config(2, reset); stored = config ? cbm_config_get(config, CBM_INDEX_CONFIG_MAX_FILES, "off") : "missing"; bool reset_to_default = strcmp(stored, "off") == 0; @@ -340,6 +500,10 @@ TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite) { ASSERT_TRUE(valid_stored); ASSERT_TRUE(invalid_rc != 0); ASSERT_TRUE(invalid_preserved); + ASSERT_EQ(profile_rc, 0); + ASSERT_TRUE(profile_stored); + ASSERT_TRUE(invalid_profile_rc != 0); + ASSERT_TRUE(invalid_profile_preserved); ASSERT_EQ(reset_rc, 0); ASSERT_TRUE(reset_to_default); PASS(); @@ -443,6 +607,40 @@ TEST(index_policy_worker_rejects_missing_parent_policy) { PASS(); } +TEST(index_policy_worker_contract_preserves_profile_only_dimensions) { + cbm_index_resource_policy_t parent; + cbm_index_policy_init(&parent); + char error[256]; + ASSERT_TRUE(cbm_index_policy_set_profile(&parent, "strict", error, sizeof(error))); + cbm_index_policy_finalize(&parent, UINT64_C(16) * 1024 * CBM_INDEX_MIB_BYTES, + UINT64_C(32) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(parent.max_rss_bytes.value, CBM_INDEX_MIN_RSS_MB_VALUE * CBM_INDEX_MIB_BYTES); + + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_TRUE(cbm_mcp_index_policy_add_to_args(document, root, &parent)); + char *args = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(args); + + cbm_index_resource_policy_t worker; + ASSERT_TRUE(cbm_mcp_index_policy_from_internal_args(args, &worker, error, sizeof(error))); + ASSERT_STR_EQ(cbm_index_policy_profile_name(&worker), "strict"); + ASSERT_STR_EQ(cbm_index_policy_source_name(&worker), "profile"); + ASSERT_EQ(worker.max_directories.value, parent.max_directories.value); + ASSERT_EQ(worker.max_entries.value, parent.max_entries.value); + ASSERT_EQ(worker.max_depth.value, parent.max_depth.value); + ASSERT_EQ(worker.discovery_deadline_ms.value, parent.discovery_deadline_ms.value); + ASSERT_EQ(worker.max_final_db_bytes.value, parent.max_final_db_bytes.value); + ASSERT_EQ(worker.max_staging_bytes.value, parent.max_staging_bytes.value); + ASSERT_EQ(worker.max_task_temp_bytes.value, parent.max_task_temp_bytes.value); + ASSERT_EQ(worker.max_rss_bytes.value, parent.max_rss_bytes.value); + + free(args); + yyjson_mut_doc_free(document); + PASS(); +} + TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { char *repo = th_mktempdir("cbm_index_policy_mcp_repo"); char *cache = th_mktempdir("cbm_index_policy_mcp_cache"); @@ -625,6 +823,12 @@ TEST(index_policy_storage_limit_preserves_old_index_and_unrelated_cache_files) { SUITE(index_policy) { RUN_TEST(index_policy_defaults_are_disabled); + RUN_TEST(index_policy_balanced_profile_expands_exact_baseline); + RUN_TEST(index_policy_balanced_rss_follows_the_host_share); + RUN_TEST(index_policy_strict_profile_expands_exact_baseline); + RUN_TEST(index_policy_explicit_override_and_off_replace_profile_dimensions); + RUN_TEST(index_policy_host_clamp_only_tightens_explicit_rss); + RUN_TEST(index_policy_profile_validation_is_atomic); RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); RUN_TEST(index_policy_worker_limits_validate_and_convert_units); @@ -638,6 +842,7 @@ SUITE(index_policy) { RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); RUN_TEST(index_policy_cli_set_reports_a_failed_write); RUN_TEST(index_policy_worker_rejects_missing_parent_policy); + RUN_TEST(index_policy_worker_contract_preserves_profile_only_dimensions); RUN_TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index); RUN_TEST(index_policy_storage_limit_preserves_old_index_and_unrelated_cache_files); } From 9e753e17e24ea7ebc967b66c5864c20b843c25cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=86=B2?= Date: Wed, 19 Aug 2026 14:39:38 +0800 Subject: [PATCH 6/6] feat(index): record and report the latest index attempt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed index is currently a log line. A watcher-triggered rebuild that dies on a resource limit leaves the previously published graph in service and nothing durable behind it, so index_status reports a healthy generation and the operator has no way to learn that it is no longer the tree. Record one attempt per project under the cache directory, updated under a file lock across queued, running and terminal states, carrying the origin, timestamps, the effective profile source and the resource failure when there is one. index_status returns it as last_index_attempt, and delete_project removes a matching record when no database was published. Freshness is reported alongside it and is deliberately conservative. A generation is comparable only when matching clean Git snapshots were observed before and after indexing, and it is fresh only while the worktree is still clean at that HEAD. A different clean HEAD is stale. Non-Git roots, dirty trees, a tree that changed during indexing, a failed probe and a corrupt record are all unknown, because a dirty tree is evidence of neither freshness nor staleness. Daemon startup fails attempts whose worker no longer exists. Liveness is judged by process identity rather than the pid alone, so a recycled pid cannot keep a dead attempt alive, and an unavailable start time falls back to plain liveness rather than declaring the worker gone. Projects with no record keep the previous response shape. Signed-off-by: 刘冲 --- README.md | 4 +- docs/CONFIGURATION.md | 9 + docs/INDEX_RESOURCE_LIMITS.md | 19 + src/daemon/application.c | 153 ++++++ src/git/git_context.c | 59 ++ src/git/git_context.h | 8 + src/mcp/mcp.c | 925 +++++++++++++++++++++++++++++++- src/mcp/mcp_internal.h | 22 + tests/test_daemon_application.c | 123 ++++- tests/test_git_context.c | 69 ++- tests/test_index_policy.c | 14 +- tests/test_mcp.c | 346 ++++++++++++ 12 files changed, 1728 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 144edee95c..53e9ee1cec 100644 --- a/README.md +++ b/README.md @@ -941,8 +941,8 @@ codebase-memory-mcp config reset auto_index # reset to default explicit value, including `off`, replaces that dimension of a selected profile. Exceeding an effective limit fails the complete index attempt rather than publishing a partial graph, and an existing serving index is preserved. -Storage probes also fail closed when an enabled measurement cannot be -completed. See +`index_status` reports the latest physical indexing attempt and Git-based +freshness after the first recorded attempt. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md). ### Environment Variables diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 933890d98d..4ea8d1f5ea 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -10,6 +10,7 @@ This page documents the configuration files that `codebase-memory-mcp` reads or | Per-project custom extension mapping | `{repo_root}/.codebase-memory.json` | JSON | Overrides conflicting global `extra_extensions` entries. | | CLI-managed runtime settings | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/_config.db` | SQLite | Written by `codebase-memory-mcp config set/reset`. | | UI settings | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/config.json` | JSON | Stores `ui_enabled` and `ui_port`. | +| Latest indexing attempt | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/status/.json` | JSON | Owner-private, atomically replaced status used by `index_status`. | | Daemon operation log | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/logs/cbm-daemon.log` | Structured log | Durable daemon lifecycle, watcher/indexing, UI, resource, and error events. | | Admission conflict log | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/logs/daemon-conflicts.ndjson` | NDJSON | Exact-build, ABI, and canonical-cache conflicts. | | Activation log | `${CBM_CACHE_DIR:-~/.cache/codebase-memory-mcp}/logs/activation-events.ndjson` | NDJSON | Install/update/uninstall activation progress and outcomes. | @@ -146,6 +147,14 @@ timeout. See [Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and error-response details. +After the first physical indexing attempt for a project, `index_status` adds +`last_index_attempt` and `freshness`. Freshness is `fresh` only when a clean +Git snapshot observed both before and after indexing still matches the current +clean `HEAD`. A different clean `HEAD` is `stale`, while non-Git, dirty, +changed-during-index, missing, or unreadable snapshots are `unknown`. +`delete_project` also removes a matching attempt record when no database was +published. + ## 3. UI Settings The optional built-in graph UI stores its settings in: diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 736c7d05f5..f8ff2508bf 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -176,6 +176,25 @@ Both cases preserve the old serving database. Staging files created by a terminated supervised worker are tagged with a private task token and removed after its process tree is quiescent; cleanup cannot match another attempt. +## Attempt visibility and freshness + +The latest physical attempt for each project is atomically stored in an +owner-private file below `${CBM_CACHE_DIR}/status/`. It records the +`explicit`, `auto`, or `watcher` origin; `queued`, `running`, `completed`, +`failed`, or `cancelled` state; effective profile source; timestamps; and +stable resource failure details when applicable. Daemon startup changes +abandoned `queued` or `running` records to `failed` with +`failure_code=worker_lost`. + +After a record exists, `index_status` returns `last_index_attempt` and +`freshness`. A generation is comparable only when matching clean Git snapshots +were observed before and after indexing, and it is `fresh` only while the +current worktree remains clean at that `HEAD`. A different clean `HEAD`, or a +failed watcher rebuild after an observed change, is `stale`. Non-Git roots, +dirty or changed-during-index snapshots, failed Git probes, and corrupt records +are `unknown`. Dirty state is never proof of freshness or staleness. Projects +with no attempt record retain the previous response shape. + ## Trust and compatibility Limits are read from the CLI-managed `_config.db`; they are not MCP request diff --git a/src/daemon/application.c b/src/daemon/application.c index f2e2e38085..c0cf5fd00e 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -160,6 +160,8 @@ struct cbm_daemon_application_job { bool async_owned; bool worker_started; uint64_t request_started_ms; + bool attempt_recorded; + char attempt_id[33]; cbm_daemon_application_job_t *next; }; @@ -1520,6 +1522,63 @@ static void application_record_cancelled(cbm_index_worker_result_t *last_result, last_log[0] = '\0'; } +static cbm_index_resource_t application_resource_from_name(const char *name) { + for (int resource = CBM_INDEX_RESOURCE_FILES; + resource <= CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS; resource++) { + if (name && strcmp(name, cbm_index_resource_name((cbm_index_resource_t)resource)) == 0) { + return (cbm_index_resource_t)resource; + } + } + return CBM_INDEX_RESOURCE_NONE; +} + +static bool application_response_error_metadata(const char *response, char *code, size_t code_size, + cbm_index_resource_violation_t *violation) { + if (code && code_size > 0) { + code[0] = '\0'; + } + if (violation) { + *violation = (cbm_index_resource_violation_t){0}; + } + yyjson_doc *outer = response ? yyjson_read(response, strlen(response), 0) : NULL; + yyjson_val *outer_root = outer ? yyjson_doc_get_root(outer) : NULL; + yyjson_val *is_error = + outer_root && yyjson_is_obj(outer_root) ? yyjson_obj_get(outer_root, "isError") : NULL; + yyjson_val *content = + outer_root && yyjson_is_obj(outer_root) ? yyjson_obj_get(outer_root, "content") : NULL; + yyjson_val *first = content && yyjson_is_arr(content) ? yyjson_arr_get_first(content) : NULL; + yyjson_val *text = first && yyjson_is_obj(first) ? yyjson_obj_get(first, "text") : NULL; + yyjson_doc *inner = text && yyjson_is_str(text) + ? yyjson_read(yyjson_get_str(text), yyjson_get_len(text), 0) + : NULL; + yyjson_val *inner_root = inner ? yyjson_doc_get_root(inner) : NULL; + yyjson_val *inner_status = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "status") : NULL; + bool failed = (is_error && yyjson_is_bool(is_error) && yyjson_get_bool(is_error)) || + (inner_status && yyjson_is_str(inner_status) && + strcmp(yyjson_get_str(inner_status), "error") == 0); + if (failed && inner_root && yyjson_is_obj(inner_root)) { + yyjson_val *failure_code = yyjson_obj_get(inner_root, "code"); + if (code && code_size > 0 && failure_code && yyjson_is_str(failure_code)) { + (void)snprintf(code, code_size, "%s", yyjson_get_str(failure_code)); + } + yyjson_val *resource = yyjson_obj_get(inner_root, "resource"); + yyjson_val *observed = yyjson_obj_get(inner_root, "observed"); + yyjson_val *limit = yyjson_obj_get(inner_root, "limit"); + if (violation && resource && yyjson_is_str(resource)) { + violation->resource = application_resource_from_name(yyjson_get_str(resource)); + violation->probe_failed = code && strcmp(code, "resource_probe_failed") == 0; + if (observed && yyjson_is_uint(observed) && limit && yyjson_is_uint(limit)) { + violation->observed = yyjson_get_uint(observed); + violation->limit = yyjson_get_uint(limit); + } + } + } + yyjson_doc_free(inner); + yyjson_doc_free(outer); + return failed; +} + static bool application_result_is_attributable_failure( const application_attempt_t *attempt, cbm_mcp_supervised_result_disposition_t disposition) { return disposition == CBM_MCP_SUPERVISED_RESULT_CONTAINED_FAILURE && attempt->has_result && @@ -1535,6 +1594,7 @@ typedef enum { typedef struct { char *response; bool successful; + bool response_is_error; bool unsafe_terminal; bool supervision_failed; cbm_index_worker_result_t last_result; @@ -1581,12 +1641,17 @@ static application_attempt_decision_t application_consume_attempt( execution->response = attempt->result.response; attempt->result.response = NULL; execution->successful = application_index_response_succeeded(execution->response); + execution->response_is_error = + execution->response && + application_response_error_metadata(execution->response, NULL, 0, NULL); application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_SUCCESS; } if (disposition == CBM_MCP_SUPERVISED_RESULT_RESOURCE_FAILURE) { execution->response = cbm_mcp_index_worker_resource_response(job->args_json, &attempt->result); + execution->successful = execution->response != NULL; + execution->response_is_error = execution->response != NULL; application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_STOP; } @@ -1800,6 +1865,46 @@ static void application_job_publish(cbm_daemon_application_job_t *job, : cancelled ? "cancelled" : "failed", NULL); + if (job->attempt_recorded) { + bool cancelled = + execution->have_last_result && execution->last_result.cancellation_requested; + bool attempt_completed = execution->successful && !execution->response_is_error; + const char *state = attempt_completed ? "completed" : cancelled ? "cancelled" : "failed"; + const char *failure_code = NULL; + const cbm_index_resource_violation_t *violation = NULL; + char response_failure_code[CBM_SZ_128] = {0}; + cbm_index_resource_violation_t response_violation = {0}; + if (!attempt_completed && !cancelled) { + if (application_response_error_metadata(execution->response, response_failure_code, + sizeof(response_failure_code), + &response_violation)) { + failure_code = response_failure_code[0] ? response_failure_code : "index_failed"; + if (response_violation.resource != CBM_INDEX_RESOURCE_NONE) { + violation = &response_violation; + } + } else if (execution->have_last_result && + execution->last_result.resource_violation.resource != + CBM_INDEX_RESOURCE_NONE) { + violation = &execution->last_result.resource_violation; + failure_code = + violation->probe_failed ? "resource_probe_failed" : "resource_limit_exceeded"; + } else if (execution->supervision_failed || execution->unsafe_terminal) { + failure_code = "supervision_failed"; + } else if (!execution->have_last_result || + execution->last_result.outcome == CBM_PROC_SPAWN_FAILED) { + failure_code = "worker_start_failed"; + } else { + failure_code = "index_failed"; + } + } else if (cancelled) { + failure_code = "cancelled"; + } + if (!cbm_mcp_index_attempt_transition(job->project_key, job->root_path, job->attempt_id, + state, failure_code, violation, attempt_completed)) { + cbm_log_warn("daemon.index.attempt_record", "project", job->project_key, "state", + "terminal_write_failed"); + } + } cbm_daemon_application_t *application = job->application; cbm_mutex_lock(&application->mutex); job->response = execution->response; @@ -1854,6 +1959,12 @@ static void *application_job_thread(void *opaque) { if (!application_job_wait_for_mutations(job)) { application_job_execution_cancel(&execution); } else { + if (job->attempt_recorded && + !cbm_mcp_index_attempt_transition(job->project_key, job->root_path, job->attempt_id, + "running", NULL, NULL, false)) { + cbm_log_warn("daemon.index.attempt_record", "project", job->project_key, "state", + "running_write_failed"); + } application_attempt_t attempt; application_attempt_status_t status = application_job_run_attempt(job, NULL, NULL, &attempt); @@ -1904,6 +2015,18 @@ static char *application_index_project_key(const char *root_path, const char *ar return key; } +static const char *application_index_origin(const char *args_json, char **owned_origin) { + *owned_origin = cbm_mcp_get_string_arg(args_json, "_cbm_index_origin"); + if (*owned_origin && + (strcmp(*owned_origin, "auto") == 0 || strcmp(*owned_origin, "watcher") == 0 || + strcmp(*owned_origin, "explicit") == 0)) { + return *owned_origin; + } + free(*owned_origin); + *owned_origin = NULL; + return "explicit"; +} + static size_t application_active_job_count_locked(cbm_daemon_application_t *application) { size_t count = 0; for (cbm_daemon_application_job_t *job = application->jobs; job; job = job->next) { @@ -1966,6 +2089,7 @@ static bool application_index_args_normalize_defaults(yyjson_mut_val *root) { if (name && (!yyjson_mut_is_str(name) || yyjson_mut_get_len(name) == 0)) { (void)yyjson_mut_obj_remove_key(root, "name"); } + (void)yyjson_mut_obj_remove_key(root, "_cbm_index_origin"); return true; } @@ -2031,6 +2155,9 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( if (application->stopping) { return NULL; } + char *requested_origin = cbm_mcp_get_string_arg(args_json, "_cbm_index_origin"); + bool watcher_request = requested_origin && strcmp(requested_origin, "watcher") == 0; + free(requested_origin); cbm_daemon_application_job_t *job = application_find_active_job_locked(application, project_key); if (job) { @@ -2043,6 +2170,9 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( return NULL; } job->subscribers++; + if (watcher_request && job->attempt_recorded) { + (void)cbm_mcp_index_attempt_mark_watcher_change(job->project_key, job->attempt_id); + } *status_out = APPLICATION_JOB_SUBSCRIBE_OK; return job; } @@ -2068,6 +2198,20 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( } job->application = application; job->subscribers = 1; + cbm_index_resource_policy_t policy; + char policy_error[CBM_SZ_256] = {0}; + char *owned_origin = NULL; + const char *origin = application_index_origin(args_json, &owned_origin); + if (cbm_mcp_index_policy_from_internal_args(args_json, &policy, policy_error, + sizeof(policy_error))) { + job->attempt_recorded = + cbm_mcp_index_attempt_begin(project_key, root_path, origin, &policy, + strcmp(origin, "watcher") == 0, job->attempt_id); + } + free(owned_origin); + if (!job->attempt_recorded) { + cbm_log_warn("daemon.index.attempt_record", "project", project_key, "state", "unavailable"); + } job->next = application->jobs; application->jobs = job; if (application_job_thread_create(&job->thread, job) == 0) { @@ -2080,6 +2224,11 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( * reservation back synchronously and let background callers retry. */ application->jobs = job->next; job->next = NULL; + if (job->attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(job->project_key, job->root_path, + job->attempt_id, "failed", "worker_start_failed", + NULL, false); + } application_job_free(job); *status_out = APPLICATION_JOB_SUBSCRIBE_UNAVAILABLE; cbm_log_warn("daemon.index.thread_start_failed", "action", "retry"); @@ -2403,6 +2552,7 @@ static char *application_auto_index_args(cbm_daemon_application_t *application, } yyjson_mut_doc_set_root(document, root); char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) && + yyjson_mut_obj_add_str(document, root, "_cbm_index_origin", "auto") && application_index_args_add_policy(application, document, root) ? yyjson_mut_write(document, 0, NULL) : NULL; @@ -3679,6 +3829,7 @@ cbm_daemon_application_t *cbm_daemon_application_new( application->watcher, application_watcher_mutation_begin, application_watcher_mutation_end, application_watcher_project_pruned, application); } + cbm_mcp_index_attempt_recover_abandoned(); return application; } @@ -4105,6 +4256,8 @@ static int application_background_index(cbm_daemon_application_t *application, } yyjson_mut_doc_set_root(document, root); bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) && + yyjson_mut_obj_add_str(document, root, "_cbm_index_origin", + require_live_watch ? "watcher" : "explicit") && application_index_args_add_policy(application, document, root); char *default_project = cbm_project_name_from_path(canonical_root); bool custom_project = diff --git a/src/git/git_context.c b/src/git/git_context.c index 66431333b4..0462e0d954 100644 --- a/src/git/git_context.c +++ b/src/git/git_context.c @@ -111,6 +111,35 @@ static int git_capture(const char *repo_path, const char *git_args, char **out) return *out ? 0 : CBM_NOT_FOUND; } +static int git_dirty_status(const char *repo_path, bool *dirty_out) { + if (!dirty_out || !repo_path || !git_validate_repo_path(repo_path)) { + return CBM_NOT_FOUND; + } + *dirty_out = false; +#ifdef _WIN32 + const char *null_dev = "NUL"; +#else + const char *null_dev = "/dev/null"; +#endif + char cmd[GIT_CMD_MAX]; + int n = snprintf( + cmd, sizeof(cmd), + "git --no-optional-locks -C \"%s\" status --porcelain --untracked-files=normal 2>%s", + repo_path, null_dev); + if (n < 0 || n >= (int)sizeof(cmd)) { + return CBM_NOT_FOUND; + } + FILE *fp = cbm_popen(cmd, "r"); + if (!fp) { + return CBM_NOT_FOUND; + } + char buffer[CBM_SZ_4K]; + while (fread(buffer, 1, sizeof(buffer), fp) > 0) { + *dirty_out = true; + } + return cbm_pclose(fp) == 0 ? 0 : CBM_NOT_FOUND; +} + static bool path_is_absolute(const char *path) { if (!path || !path[0]) { return false; @@ -264,6 +293,36 @@ void cbm_git_context_free(cbm_git_context_t *ctx) { memset(ctx, 0, sizeof(*ctx)); } +int cbm_git_snapshot_read(const char *path, cbm_git_snapshot_t *out) { + if (!out) { + return CBM_NOT_FOUND; + } + memset(out, 0, sizeof(*out)); + char *head_sha = NULL; + int status = git_capture(path, "rev-parse --verify HEAD", &head_sha); + if (status != 0 || !head_sha || !head_sha[0]) { + free(head_sha); + return status == 0 ? CBM_NOT_FOUND : status; + } + bool dirty = false; + if (git_dirty_status(path, &dirty) != 0) { + free(head_sha); + return CBM_NOT_FOUND; + } + out->head_sha = head_sha; + out->available = true; + out->dirty = dirty; + return 0; +} + +void cbm_git_snapshot_free(cbm_git_snapshot_t *snapshot) { + if (!snapshot) { + return; + } + free(snapshot->head_sha); + memset(snapshot, 0, sizeof(*snapshot)); +} + int cbm_git_context_resolve(const char *path, cbm_git_context_t *out) { if (!out) { return CBM_NOT_FOUND; diff --git a/src/git/git_context.h b/src/git/git_context.h index 2ac4195228..b1e3ba3cdd 100644 --- a/src/git/git_context.h +++ b/src/git/git_context.h @@ -20,6 +20,12 @@ typedef struct { char *base_sha; } cbm_git_context_t; +typedef struct { + bool available; + bool dirty; + char *head_sha; +} cbm_git_snapshot_t; + int cbm_git_context_resolve(const char *path, cbm_git_context_t *out); /* Only the is_git and branch fields of cbm_git_context_resolve, from the same * two git commands it uses for them (2 git processes instead of 7). branch @@ -30,5 +36,7 @@ int cbm_git_branch_resolve(const char *path, bool *is_git, char *branch, size_t void cbm_git_context_free(cbm_git_context_t *ctx); char *cbm_git_context_branch_qn(const char *project_name, const cbm_git_context_t *ctx); int cbm_git_context_props_json(const cbm_git_context_t *ctx, char *buf, int buf_size); +int cbm_git_snapshot_read(const char *path, cbm_git_snapshot_t *out); +void cbm_git_snapshot_free(cbm_git_snapshot_t *snapshot); #endif diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 1feb8b89a3..d7bdbb4eb3 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -86,6 +86,7 @@ enum { #include "foundation/hash_table.h" #include "foundation/subprocess.h" #include "foundation/sha256.h" +#include "foundation/secure_random.h" #include "mcp/index_supervisor.h" #include "mcp/compact_out.h" #include "foundation/str_util.h" @@ -105,14 +106,22 @@ enum { * path — see search_scratch_open. Mirrors config_toml_edit.c's toml_fdopen. */ #define mcp_fdopen _fdopen #define mcp_close _close +#define mcp_fsync _commit #else #include #include #include #include #include +#include +#include +#ifdef __APPLE__ +#include +#include +#endif #define mcp_fdopen fdopen #define mcp_close close +#define mcp_fsync fsync #endif #include #include @@ -7377,10 +7386,807 @@ static void add_watch_status_json(cbm_mcp_server_t *srv, yyjson_mut_doc *doc, yy yyjson_mut_obj_add_val(doc, root, "watch", watch); } +static bool index_attempt_paths(const char *project, bool create_directory, char *directory, + size_t directory_size, char *record, size_t record_size) { + if (!cbm_validate_project_name(project) || !directory || directory_size == 0 || !record || + record_size == 0) { + return false; + } + int directory_length = + snprintf(directory, directory_size, "%s/status", cbm_resolve_cache_dir()); + if (directory_length <= 0 || (size_t)directory_length >= directory_size || + (create_directory && !cbm_mkdir_p(directory, 0700))) { + return false; + } + int record_length = snprintf(record, record_size, "%s/%s.json", directory, project); + return record_length > 0 && (size_t)record_length < record_size; +} + +typedef struct { +#ifdef _WIN32 + HANDLE handle; + OVERLAPPED overlapped; +#else + int descriptor; +#endif +} index_attempt_lock_t; + +static bool index_attempt_lock_acquire(index_attempt_lock_t *lock, const char *project) { + if (!lock || !project) { + return false; + } + memset(lock, 0, sizeof(*lock)); + char directory[CBM_SZ_4K]; + char record[CBM_SZ_4K]; + if (!index_attempt_paths(project, true, directory, sizeof(directory), record, sizeof(record))) { + return false; + } + char path[CBM_SZ_4K]; + int path_length = snprintf(path, sizeof(path), "%s", record); + if (path_length <= 5 || (size_t)path_length >= sizeof(path)) { + return false; + } + memcpy(path + path_length - 5, ".lock", 6); +#ifdef _WIN32 + wchar_t *wide_path = cbm_utf8_to_wide(path); + if (!wide_path) { + return false; + } + lock->handle = + CreateFileW(wide_path, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_HIDDEN | FILE_FLAG_OPEN_REPARSE_POINT, NULL); + free(wide_path); + if (lock->handle == INVALID_HANDLE_VALUE) { + lock->handle = NULL; + return false; + } + if (!LockFileEx(lock->handle, LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &lock->overlapped)) { + (void)CloseHandle(lock->handle); + lock->handle = NULL; + return false; + } + return true; +#else + int flags = O_CREAT | O_RDWR; +#ifdef O_CLOEXEC + flags |= O_CLOEXEC; +#endif +#ifdef O_NOFOLLOW + flags |= O_NOFOLLOW; +#endif + lock->descriptor = open(path, flags, 0600); + if (lock->descriptor < 0) { + return false; + } + (void)fchmod(lock->descriptor, 0600); + if (flock(lock->descriptor, LOCK_EX) != 0) { + (void)close(lock->descriptor); + lock->descriptor = -1; + return false; + } + return true; +#endif +} + +static void index_attempt_lock_release(index_attempt_lock_t *lock) { + if (!lock) { + return; + } +#ifdef _WIN32 + if (lock->handle) { + (void)UnlockFileEx(lock->handle, 0, 1, 0, &lock->overlapped); + (void)CloseHandle(lock->handle); + lock->handle = NULL; + } +#else + if (lock->descriptor >= 0) { + (void)flock(lock->descriptor, LOCK_UN); + (void)close(lock->descriptor); + lock->descriptor = -1; + } +#endif +} + +static yyjson_doc *index_attempt_read(const char *project, bool *exists_out) { + if (exists_out) { + *exists_out = false; + } + char directory[CBM_SZ_4K]; + char record[CBM_SZ_4K]; + if (!index_attempt_paths(project, false, directory, sizeof(directory), record, + sizeof(record))) { + return NULL; + } + if (access(record, F_OK) != 0) { + return NULL; + } + if (exists_out) { + *exists_out = true; + } + yyjson_doc *document = yyjson_read_file(record, 0, NULL, NULL); + yyjson_val *root = document ? yyjson_doc_get_root(document) : NULL; + yyjson_val *schema = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "schema_version") : NULL; + yyjson_val *stored_project = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "project") : NULL; + yyjson_val *attempt_id = + root && yyjson_is_obj(root) ? yyjson_obj_get(root, "attempt_id") : NULL; + yyjson_val *state = root && yyjson_is_obj(root) ? yyjson_obj_get(root, "state") : NULL; + if (!schema || !yyjson_is_uint(schema) || yyjson_get_uint(schema) != 1U || !stored_project || + !yyjson_is_str(stored_project) || strcmp(yyjson_get_str(stored_project), project) != 0 || + !attempt_id || !yyjson_is_str(attempt_id) || yyjson_get_len(attempt_id) != 32U || !state || + !yyjson_is_str(state)) { + yyjson_doc_free(document); + return NULL; + } + return document; +} + +static bool index_attempt_put_string(yyjson_mut_doc *document, yyjson_mut_val *object, + const char *key, const char *value) { + yyjson_mut_val *json_key = yyjson_mut_strcpy(document, key); + yyjson_mut_val *json_value = yyjson_mut_strcpy(document, value ? value : ""); + return json_key && json_value && yyjson_mut_obj_put(object, json_key, json_value); +} + +static bool index_attempt_put_bool(yyjson_mut_doc *document, yyjson_mut_val *object, + const char *key, bool value) { + yyjson_mut_val *json_key = yyjson_mut_strcpy(document, key); + yyjson_mut_val *json_value = yyjson_mut_bool(document, value); + return json_key && json_value && yyjson_mut_obj_put(object, json_key, json_value); +} + +static bool index_attempt_put_uint(yyjson_mut_doc *document, yyjson_mut_val *object, + const char *key, uint64_t value) { + yyjson_mut_val *json_key = yyjson_mut_strcpy(document, key); + yyjson_mut_val *json_value = yyjson_mut_uint(document, value); + return json_key && json_value && yyjson_mut_obj_put(object, json_key, json_value); +} + +static bool index_attempt_put_value(yyjson_mut_doc *document, yyjson_mut_val *object, + const char *key, yyjson_mut_val *value) { + yyjson_mut_val *json_key = yyjson_mut_strcpy(document, key); + return json_key && value && yyjson_mut_obj_put(object, json_key, value); +} + +static bool index_attempt_now(char out[32]) { + time_t now = time(NULL); + struct tm utc; + return cbm_gmtime_r(&now, &utc) && strftime(out, 32, "%Y-%m-%dT%H:%M:%SZ", &utc) > 0; +} + +/* A false `available` means only "no trustworthy start time", never "the + * process is gone"; callers must fall back to plain liveness before deciding + * an attempt was abandoned. */ +typedef struct { + bool available; + uint64_t token; +} index_attempt_process_start_t; + +static void index_attempt_process_start(uint64_t process_id, index_attempt_process_start_t *start) { + if (!start) { + return; + } + start->available = false; + start->token = 0; + if (process_id == 0) { + return; + } +#ifdef _WIN32 + if (process_id > UINT32_MAX) { + return; + } + HANDLE process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, (DWORD)process_id); + if (!process) { + return; + } + FILETIME created; + FILETIME exited; + FILETIME kernel; + FILETIME user; + DWORD exit_code = 0; + if (GetProcessTimes(process, &created, &exited, &kernel, &user) != 0 && + GetExitCodeProcess(process, &exit_code) != 0 && exit_code == STILL_ACTIVE) { + start->available = true; + start->token = ((uint64_t)created.dwHighDateTime << 32U) | created.dwLowDateTime; + } + (void)CloseHandle(process); +#elif defined(__APPLE__) + if (process_id > INT_MAX) { + return; + } + struct proc_bsdinfo info; + memset(&info, 0, sizeof(info)); + if (proc_pidinfo((int)process_id, PROC_PIDTBSDINFO, 0, &info, sizeof(info)) != + (int)sizeof(info) || + info.pbi_pid != (uint32_t)process_id || info.pbi_status == SZOMB) { + return; + } + start->available = true; + start->token = info.pbi_start_tvsec > UINT64_MAX / UINT64_C(1000000) + ? UINT64_MAX + : info.pbi_start_tvsec * UINT64_C(1000000) + info.pbi_start_tvusec; +#elif defined(__linux__) + if (process_id > INT_MAX) { + return; + } + char path[64]; + int length = snprintf(path, sizeof(path), "/proc/%llu/stat", (unsigned long long)process_id); + if (length <= 0 || (size_t)length >= sizeof(path)) { + return; + } + FILE *file = fopen(path, "rb"); + char buffer[CBM_SZ_4K]; + bool read = file && fgets(buffer, sizeof(buffer), file) != NULL; + if (file) { + (void)fclose(file); + } + char *fields = read ? strrchr(buffer, ')') : NULL; + if (!fields || fields[1] != ' ') { + return; + } + fields += 2; + char *save = NULL; + char *field = strtok_r(fields, " ", &save); + if (!field || field[0] == 'Z') { + return; + } + for (int index = 0; field && index < 19; index++) { + field = strtok_r(NULL, " ", &save); + } + if (!field) { + return; + } + char *end = NULL; + unsigned long long parsed = strtoull(field, &end, 10); + if (end == field || (*end != '\0' && *end != '\n')) { + return; + } + start->available = true; + start->token = (uint64_t)parsed; +#else + (void)process_id; +#endif +} + +static bool index_attempt_generate_id(char out[33]) { + unsigned char bytes[16]; + if (!cbm_secure_random(bytes, sizeof(bytes))) { + return false; + } + static const char HEX[] = "0123456789abcdef"; + for (size_t index = 0; index < sizeof(bytes); index++) { + out[index * 2U] = HEX[bytes[index] >> 4U]; + out[index * 2U + 1U] = HEX[bytes[index] & 0x0FU]; + } + out[32] = '\0'; + return true; +} + +static bool index_attempt_write(const char *project, yyjson_mut_doc *document) { + char directory[CBM_SZ_4K]; + char record[CBM_SZ_4K]; + if (!index_attempt_paths(project, true, directory, sizeof(directory), record, sizeof(record))) { + return false; + } + char temporary[CBM_SZ_4K]; + int temporary_length = + snprintf(temporary, sizeof(temporary), "%s/.%s.XXXXXX", directory, project); + if (temporary_length <= 0 || (size_t)temporary_length >= sizeof(temporary)) { + return false; + } + int descriptor = cbm_mkstemp(temporary); + if (descriptor < 0) { + return false; + } + char *json = yyjson_mut_write(document, 0, NULL); + FILE *file = json ? mcp_fdopen(descriptor, "wb") : NULL; + bool written = false; + if (file) { + size_t length = strlen(json); + bool content_written = fwrite(json, 1, length, file) == length; + bool flushed = fflush(file) == 0; + bool synced = flushed && mcp_fsync(cbm_fileno(file)) == 0; + bool closed = fclose(file) == 0; + written = content_written && flushed && synced && closed; + } else { + (void)mcp_close(descriptor); + } + free(json); + if (!written || cbm_rename_replace(temporary, record) != 0) { + (void)cbm_unlink(temporary); + return false; + } + return true; +} + +static void index_attempt_copy_generation(yyjson_mut_doc *document, yyjson_mut_val *target, + yyjson_val *previous) { + static const char *KEYS[] = { + "generation_snapshot_available", "generation_commit", "generation_dirty", + "serviceable_generation", "watcher_observed_change", + }; + for (size_t index = 0; previous && index < sizeof(KEYS) / sizeof(KEYS[0]); index++) { + yyjson_val *value = yyjson_obj_get(previous, KEYS[index]); + yyjson_mut_val *copy = value ? yyjson_val_mut_copy(document, value) : NULL; + if (copy) { + yyjson_mut_obj_add_val(document, target, KEYS[index], copy); + } + } +} + +bool cbm_mcp_index_attempt_begin(const char *project, const char *repo_path, const char *origin, + const cbm_index_resource_policy_t *policy, + bool watcher_observed_change, char attempt_id[33]) { + if (!project || !repo_path || !repo_path[0] || !policy || !attempt_id || + (!origin || (strcmp(origin, "explicit") != 0 && strcmp(origin, "auto") != 0 && + strcmp(origin, "watcher") != 0)) || + !index_attempt_generate_id(attempt_id)) { + return false; + } + index_attempt_lock_t lock; + if (!index_attempt_lock_acquire(&lock, project)) { + return false; + } + yyjson_doc *previous_document = index_attempt_read(project, NULL); + yyjson_val *previous = previous_document ? yyjson_doc_get_root(previous_document) : NULL; + yyjson_val *previous_watcher = previous && yyjson_is_obj(previous) + ? yyjson_obj_get(previous, "watcher_observed_change") + : NULL; + yyjson_val *previous_state = + previous && yyjson_is_obj(previous) ? yyjson_obj_get(previous, "state") : NULL; + bool unresolved_watcher_change = previous_watcher && yyjson_is_bool(previous_watcher) && + yyjson_get_bool(previous_watcher) && previous_state && + yyjson_is_str(previous_state) && + strcmp(yyjson_get_str(previous_state), "completed") != 0; + + char queued_at[32]; + uint64_t owner_pid = (uint64_t)getpid(); + index_attempt_process_start_t owner_start; + index_attempt_process_start(owner_pid, &owner_start); + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = document ? yyjson_mut_obj(document) : NULL; + bool valid = document && root && index_attempt_now(queued_at); + if (valid) { + yyjson_mut_doc_set_root(document, root); + valid = index_attempt_put_uint(document, root, "schema_version", 1U) && + index_attempt_put_string(document, root, "project", project) && + index_attempt_put_string(document, root, "repo_path", repo_path) && + index_attempt_put_string(document, root, "attempt_id", attempt_id) && + index_attempt_put_uint(document, root, "owner_pid", owner_pid) && + index_attempt_put_string(document, root, "origin", origin) && + index_attempt_put_string(document, root, "state", "queued") && + index_attempt_put_string(document, root, "profile", + cbm_index_policy_profile_name(policy)) && + index_attempt_put_string(document, root, "policy_source", + cbm_index_policy_source_name(policy)) && + index_attempt_put_string(document, root, "queued_at", queued_at); + if (valid && owner_start.available) { + valid = index_attempt_put_uint(document, root, "owner_start_token", owner_start.token); + } + } + if (valid) { + index_attempt_copy_generation(document, root, previous); + valid = index_attempt_put_bool(document, root, "watcher_observed_change", + watcher_observed_change || unresolved_watcher_change) && + index_attempt_write(project, document); + } + yyjson_mut_doc_free(document); + yyjson_doc_free(previous_document); + index_attempt_lock_release(&lock); + return valid; +} + +static bool index_attempt_transition_allowed(const char *from, const char *to) { + if (!from || !to) { + return false; + } + if (strcmp(from, "queued") == 0) { + return strcmp(to, "running") == 0 || strcmp(to, "completed") == 0 || + strcmp(to, "failed") == 0 || strcmp(to, "cancelled") == 0; + } + return strcmp(from, "running") == 0 && + (strcmp(to, "completed") == 0 || strcmp(to, "failed") == 0 || + strcmp(to, "cancelled") == 0); +} + +static bool index_attempt_capture_start_snapshot(yyjson_mut_doc *document, yyjson_mut_val *root, + const char *repo_path) { + cbm_git_snapshot_t snapshot = {0}; + bool snapshot_available = + cbm_git_snapshot_read(repo_path, &snapshot) == 0 && snapshot.available; + bool valid = + index_attempt_put_bool(document, root, "attempt_snapshot_available", snapshot_available); + if (valid && snapshot_available) { + valid = index_attempt_put_string(document, root, "attempt_commit", snapshot.head_sha) && + index_attempt_put_bool(document, root, "attempt_dirty", snapshot.dirty); + } + cbm_git_snapshot_free(&snapshot); + return valid; +} + +static bool index_attempt_capture_generation(yyjson_mut_doc *document, yyjson_mut_val *root, + const char *project, const char *repo_path) { + yyjson_mut_val *attempt_available = yyjson_mut_obj_get(root, "attempt_snapshot_available"); + yyjson_mut_val *attempt_commit = yyjson_mut_obj_get(root, "attempt_commit"); + yyjson_mut_val *attempt_dirty = yyjson_mut_obj_get(root, "attempt_dirty"); + cbm_git_snapshot_t completed = {0}; + bool completed_available = + cbm_git_snapshot_read(repo_path, &completed) == 0 && completed.available; + char database_path[CBM_SZ_1K]; + project_db_path(project, database_path, sizeof(database_path)); + cbm_store_t *store = database_path[0] ? cbm_store_open_path_query(database_path) : NULL; + cbm_project_t stored = {0}; + bool serviceable = store && cbm_store_get_project(store, project, &stored) == CBM_STORE_OK && + stored.indexed_at; + bool available = attempt_available && yyjson_mut_is_bool(attempt_available) && + yyjson_mut_get_bool(attempt_available) && attempt_commit && + yyjson_mut_is_str(attempt_commit) && attempt_dirty && + yyjson_mut_is_bool(attempt_dirty) && !yyjson_mut_get_bool(attempt_dirty) && + completed_available && !completed.dirty && serviceable && + strcmp(yyjson_mut_get_str(attempt_commit), completed.head_sha) == 0; + bool valid = index_attempt_put_bool(document, root, "generation_snapshot_available", available); + if (available) { + valid = valid && + index_attempt_put_string(document, root, "generation_commit", completed.head_sha) && + index_attempt_put_bool(document, root, "generation_dirty", false); + } else { + (void)yyjson_mut_obj_remove_key(root, "generation_commit"); + (void)yyjson_mut_obj_remove_key(root, "generation_dirty"); + } + cbm_git_snapshot_free(&completed); + + if (serviceable) { + valid = valid && index_attempt_put_string(document, root, "serviceable_generation", + stored.indexed_at); + } + cbm_project_free_fields(&stored); + if (store) { + cbm_store_close(store); + } + return valid && index_attempt_put_bool(document, root, "watcher_observed_change", false); +} + +bool cbm_mcp_index_attempt_transition(const char *project, const char *repo_path, + const char *attempt_id, const char *state, + const char *failure_code, + const cbm_index_resource_violation_t *violation, + bool generation_completed) { + index_attempt_lock_t lock; + if (!index_attempt_lock_acquire(&lock, project)) { + return false; + } + yyjson_doc *source = index_attempt_read(project, NULL); + yyjson_val *source_root = source ? yyjson_doc_get_root(source) : NULL; + yyjson_val *stored_id = source_root && yyjson_is_obj(source_root) + ? yyjson_obj_get(source_root, "attempt_id") + : NULL; + yyjson_val *stored_state = + source_root && yyjson_is_obj(source_root) ? yyjson_obj_get(source_root, "state") : NULL; + if (!source_root || !stored_id || !yyjson_is_str(stored_id) || + strcmp(yyjson_get_str(stored_id), attempt_id ? attempt_id : "") != 0 || !stored_state || + !yyjson_is_str(stored_state) || + !index_attempt_transition_allowed(yyjson_get_str(stored_state), state)) { + yyjson_doc_free(source); + index_attempt_lock_release(&lock); + return false; + } + + yyjson_mut_doc *document = yyjson_doc_mut_copy(source, NULL); + yyjson_mut_val *root = document ? yyjson_mut_doc_get_root(document) : NULL; + char timestamp[32]; + bool terminal = strcmp(state, "completed") == 0 || strcmp(state, "failed") == 0 || + strcmp(state, "cancelled") == 0; + bool valid = root && index_attempt_now(timestamp) && + index_attempt_put_string(document, root, "state", state); + if (valid && strcmp(state, "running") == 0) { + valid = index_attempt_put_string(document, root, "started_at", timestamp) && + index_attempt_capture_start_snapshot(document, root, repo_path); + } else if (valid && terminal) { + valid = index_attempt_put_string(document, root, "finished_at", timestamp); + } + if (valid && failure_code && failure_code[0]) { + valid = index_attempt_put_string(document, root, "failure_code", failure_code); + } else if (valid && terminal) { + (void)yyjson_mut_obj_remove_key(root, "failure_code"); + } + if (valid && violation && violation->resource != CBM_INDEX_RESOURCE_NONE) { + yyjson_mut_val *resource = yyjson_mut_obj(document); + valid = resource && + index_attempt_put_string(document, resource, "resource", + cbm_index_resource_name(violation->resource)) && + index_attempt_put_bool(document, resource, "probe_failed", violation->probe_failed); + if (valid && !violation->probe_failed) { + valid = index_attempt_put_uint(document, resource, "observed", violation->observed) && + index_attempt_put_uint(document, resource, "limit", violation->limit) && + index_attempt_put_string(document, resource, "unit", + cbm_index_resource_unit(violation->resource)); + } + if (valid) { + valid = index_attempt_put_value(document, root, "resource_failure", resource); + } + } + if (valid && generation_completed) { + valid = index_attempt_capture_generation(document, root, project, repo_path); + } + if (valid) { + valid = index_attempt_write(project, document); + } + yyjson_mut_doc_free(document); + yyjson_doc_free(source); + index_attempt_lock_release(&lock); + return valid; +} + +bool cbm_mcp_index_attempt_mark_watcher_change(const char *project, const char *attempt_id) { + index_attempt_lock_t lock; + if (!index_attempt_lock_acquire(&lock, project)) { + return false; + } + yyjson_doc *source = index_attempt_read(project, NULL); + yyjson_val *source_root = source ? yyjson_doc_get_root(source) : NULL; + yyjson_val *stored_id = source_root && yyjson_is_obj(source_root) + ? yyjson_obj_get(source_root, "attempt_id") + : NULL; + if (!stored_id || !yyjson_is_str(stored_id) || + strcmp(yyjson_get_str(stored_id), attempt_id ? attempt_id : "") != 0) { + yyjson_doc_free(source); + index_attempt_lock_release(&lock); + return false; + } + yyjson_mut_doc *document = yyjson_doc_mut_copy(source, NULL); + yyjson_mut_val *root = document ? yyjson_mut_doc_get_root(document) : NULL; + bool valid = root && index_attempt_put_bool(document, root, "watcher_observed_change", true) && + index_attempt_write(project, document); + yyjson_mut_doc_free(document); + yyjson_doc_free(source); + index_attempt_lock_release(&lock); + return valid; +} + +static cbm_index_resource_t index_attempt_resource_from_name(const char *name) { + for (int resource = CBM_INDEX_RESOURCE_FILES; + resource <= CBM_INDEX_RESOURCE_DISCOVERY_DURATION_MS; resource++) { + if (name && strcmp(name, cbm_index_resource_name((cbm_index_resource_t)resource)) == 0) { + return (cbm_index_resource_t)resource; + } + } + return CBM_INDEX_RESOURCE_NONE; +} + +static void index_attempt_finish_response(const char *project, const char *repo_path, + const char *attempt_id, const char *response) { + yyjson_doc *outer = response ? yyjson_read(response, strlen(response), 0) : NULL; + yyjson_val *outer_root = outer ? yyjson_doc_get_root(outer) : NULL; + yyjson_val *is_error = + outer_root && yyjson_is_obj(outer_root) ? yyjson_obj_get(outer_root, "isError") : NULL; + yyjson_val *content = + outer_root && yyjson_is_obj(outer_root) ? yyjson_obj_get(outer_root, "content") : NULL; + yyjson_val *first = content && yyjson_is_arr(content) ? yyjson_arr_get_first(content) : NULL; + yyjson_val *text = first && yyjson_is_obj(first) ? yyjson_obj_get(first, "text") : NULL; + yyjson_doc *inner = text && yyjson_is_str(text) + ? yyjson_read(yyjson_get_str(text), yyjson_get_len(text), 0) + : NULL; + yyjson_val *inner_root = inner ? yyjson_doc_get_root(inner) : NULL; + yyjson_val *status = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "status") : NULL; + bool successful = (!is_error || !yyjson_is_bool(is_error) || !yyjson_get_bool(is_error)) && + status && yyjson_is_str(status) && + (strcmp(yyjson_get_str(status), "indexed") == 0 || + strcmp(yyjson_get_str(status), "degraded") == 0); + const char *failure_code = successful ? NULL : "index_failed"; + yyjson_val *code = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "code") : NULL; + if (!successful && code && yyjson_is_str(code) && yyjson_get_len(code) > 0) { + failure_code = yyjson_get_str(code); + } + cbm_index_resource_violation_t violation = {0}; + yyjson_val *resource = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "resource") : NULL; + yyjson_val *observed = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "observed") : NULL; + yyjson_val *limit = + inner_root && yyjson_is_obj(inner_root) ? yyjson_obj_get(inner_root, "limit") : NULL; + if (resource && yyjson_is_str(resource)) { + violation.resource = index_attempt_resource_from_name(yyjson_get_str(resource)); + violation.probe_failed = failure_code && strcmp(failure_code, "resource_probe_failed") == 0; + if (observed && yyjson_is_uint(observed) && limit && yyjson_is_uint(limit)) { + violation.observed = yyjson_get_uint(observed); + violation.limit = yyjson_get_uint(limit); + } + } + (void)cbm_mcp_index_attempt_transition( + project, repo_path, attempt_id, successful ? "completed" : "failed", failure_code, + violation.resource != CBM_INDEX_RESOURCE_NONE ? &violation : NULL, successful); + yyjson_doc_free(inner); + yyjson_doc_free(outer); +} + +cbm_index_attempt_read_status_t cbm_mcp_index_attempt_add_status(yyjson_mut_doc *document, + yyjson_mut_val *root, + const char *project, + const char *current_root_path) { + bool exists = false; + yyjson_doc *attempt = index_attempt_read(project, &exists); + if (!exists) { + return CBM_INDEX_ATTEMPT_NONE; + } + if (!attempt) { + yyjson_mut_obj_add_null(document, root, "last_index_attempt"); + yyjson_mut_obj_add_str(document, root, "freshness", "unknown"); + return CBM_INDEX_ATTEMPT_CORRUPT; + } + yyjson_val *attempt_root = yyjson_doc_get_root(attempt); + yyjson_mut_val *copy = yyjson_val_mut_copy(document, attempt_root); + if (!copy || !yyjson_mut_obj_add_val(document, root, "last_index_attempt", copy)) { + yyjson_doc_free(attempt); + return CBM_INDEX_ATTEMPT_CORRUPT; + } + + const char *freshness = "unknown"; + yyjson_val *watcher_change = yyjson_obj_get(attempt_root, "watcher_observed_change"); + yyjson_val *state = yyjson_obj_get(attempt_root, "state"); + bool watcher_stale = watcher_change && yyjson_is_bool(watcher_change) && + yyjson_get_bool(watcher_change) && state && yyjson_is_str(state) && + strcmp(yyjson_get_str(state), "completed") != 0; + yyjson_val *available = yyjson_obj_get(attempt_root, "generation_snapshot_available"); + yyjson_val *generation_commit = yyjson_obj_get(attempt_root, "generation_commit"); + yyjson_val *generation_dirty = yyjson_obj_get(attempt_root, "generation_dirty"); + if (watcher_stale) { + freshness = "stale"; + } else if (available && yyjson_is_bool(available) && yyjson_get_bool(available) && + generation_commit && yyjson_is_str(generation_commit) && generation_dirty && + yyjson_is_bool(generation_dirty) && !yyjson_get_bool(generation_dirty) && + current_root_path && current_root_path[0]) { + cbm_git_snapshot_t current = {0}; + if (cbm_git_snapshot_read(current_root_path, ¤t) == 0 && current.available) { + if (!current.dirty) { + freshness = strcmp(current.head_sha, yyjson_get_str(generation_commit)) == 0 + ? "fresh" + : "stale"; + } + } + cbm_git_snapshot_free(¤t); + } + yyjson_mut_obj_add_str(document, root, "freshness", freshness); + yyjson_doc_free(attempt); + return CBM_INDEX_ATTEMPT_AVAILABLE; +} + +bool cbm_mcp_index_attempt_remove(const char *project) { + index_attempt_lock_t lock; + if (!index_attempt_lock_acquire(&lock, project)) { + return false; + } + char directory[CBM_SZ_4K]; + char record[CBM_SZ_4K]; + if (!index_attempt_paths(project, false, directory, sizeof(directory), record, + sizeof(record))) { + index_attempt_lock_release(&lock); + return false; + } + bool removed = access(record, F_OK) != 0 || cbm_unlink(record) == 0; + index_attempt_lock_release(&lock); + return removed; +} + +void cbm_mcp_index_attempt_recover_abandoned(void) { + char directory[CBM_SZ_4K]; + char unused[CBM_SZ_4K]; + if (!index_attempt_paths("recovery-probe", false, directory, sizeof(directory), unused, + sizeof(unused))) { + return; + } + cbm_dir_t *status_directory = cbm_opendir(directory); + if (!status_directory) { + return; + } + cbm_dirent_t *entry; + while ((entry = cbm_readdir(status_directory)) != NULL) { + size_t length = strlen(entry->name); + if (length <= 5U || strcmp(entry->name + length - 5U, ".json") != 0) { + continue; + } + char project[CBM_SZ_1K]; + if (length - 5U >= sizeof(project)) { + continue; + } + memcpy(project, entry->name, length - 5U); + project[length - 5U] = '\0'; + if (!cbm_validate_project_name(project)) { + continue; + } + bool exists = false; + yyjson_doc *document = index_attempt_read(project, &exists); + yyjson_val *root = document ? yyjson_doc_get_root(document) : NULL; + yyjson_val *state = root ? yyjson_obj_get(root, "state") : NULL; + yyjson_val *attempt_id = root ? yyjson_obj_get(root, "attempt_id") : NULL; + yyjson_val *repo_path = root ? yyjson_obj_get(root, "repo_path") : NULL; + yyjson_val *owner_pid = root ? yyjson_obj_get(root, "owner_pid") : NULL; + yyjson_val *owner_start_token = root ? yyjson_obj_get(root, "owner_start_token") : NULL; + bool abandoned = state && yyjson_is_str(state) && + (strcmp(yyjson_get_str(state), "queued") == 0 || + strcmp(yyjson_get_str(state), "running") == 0); + if (abandoned && owner_pid && yyjson_is_uint(owner_pid)) { + uint64_t process_id = yyjson_get_uint(owner_pid); + bool same_process = false; + if (owner_start_token && yyjson_is_uint(owner_start_token)) { + index_attempt_process_start_t current; + index_attempt_process_start(process_id, ¤t); + same_process = + current.available && current.token == yyjson_get_uint(owner_start_token); +#ifdef _WIN32 + if (!current.available && GetLastError() == ERROR_ACCESS_DENIED) { + same_process = true; + } +#else + if (!current.available && process_id > 0 && process_id <= (uint64_t)INT_MAX && + kill((pid_t)process_id, 0) != 0 && errno == EPERM) { + same_process = true; + } +#endif + } else { +#ifdef _WIN32 + HANDLE process = process_id > 0 && process_id <= UINT32_MAX + ? OpenProcess(SYNCHRONIZE, FALSE, (DWORD)process_id) + : NULL; + if (process) { + same_process = WaitForSingleObject(process, 0) == WAIT_TIMEOUT; + CloseHandle(process); + } else if (GetLastError() == ERROR_ACCESS_DENIED) { + same_process = true; + } +#else + same_process = process_id > 0 && process_id <= (uint64_t)INT_MAX && + (kill((pid_t)process_id, 0) == 0 || errno == EPERM); +#endif + } + if (same_process) { + abandoned = false; + } + } + if (abandoned && attempt_id && yyjson_is_str(attempt_id) && repo_path && + yyjson_is_str(repo_path)) { + char id[33]; + (void)snprintf(id, sizeof(id), "%s", yyjson_get_str(attempt_id)); + char path[CBM_SZ_4K]; + (void)snprintf(path, sizeof(path), "%s", yyjson_get_str(repo_path)); + yyjson_doc_free(document); + document = NULL; + (void)cbm_mcp_index_attempt_transition(project, path, id, "failed", "worker_lost", NULL, + false); + } + yyjson_doc_free(document); + } + cbm_closedir(status_directory); +} + static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { char *project = get_project_arg(args); cbm_store_t *store = resolve_store(srv, project); - REQUIRE_STORE(store, project); + if (!store) { + yyjson_mut_doc *attempt_document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *attempt_root = attempt_document ? yyjson_mut_obj(attempt_document) : NULL; + cbm_index_attempt_read_status_t attempt_status = + attempt_root && project + ? cbm_mcp_index_attempt_add_status(attempt_document, attempt_root, project, NULL) + : CBM_INDEX_ATTEMPT_NONE; + if (attempt_status != CBM_INDEX_ATTEMPT_NONE) { + yyjson_mut_doc_set_root(attempt_document, attempt_root); + yyjson_mut_obj_add_strcpy(attempt_document, attempt_root, "project", project); + yyjson_mut_obj_add_int(attempt_document, attempt_root, "nodes", 0); + yyjson_mut_obj_add_int(attempt_document, attempt_root, "edges", 0); + yyjson_mut_obj_add_str(attempt_document, attempt_root, "status", "unavailable"); + char *json = yy_doc_to_str(attempt_document); + yyjson_mut_doc_free(attempt_document); + free(project); + char *result = cbm_mcp_text_result(json, false); + free(json); + return result; + } + yyjson_mut_doc_free(attempt_document); + REQUIRE_STORE(store, project); + } /* The git context block (worktree/shadow path variants) only matters when * debugging index-location issues — gate it so the common status call * stays lean. */ @@ -7429,6 +8235,7 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) { if (verbose) { add_git_context_json(doc, root, proj_info.root_path); } + (void)cbm_mcp_index_attempt_add_status(doc, root, project, proj_info.root_path); } add_coverage_report(doc, root, store, project, have_proj_info ? proj_info.indexed_at : NULL, coverage_samples); @@ -7518,13 +8325,29 @@ static char *handle_delete_project(cbm_mcp_server_t *srv, const char *args) { (void)cbm_unlink(shm); if (rc == 0) { status = "deleted"; + if (!cbm_mcp_index_attempt_remove(name)) { + status = "delete_failed"; + error_detail = "project deleted but attempt status cleanup failed"; + is_error = true; + } } else { status = "delete_failed"; error_detail = strerror(errno); is_error = true; } } else { - is_error = true; + bool attempt_exists = false; + yyjson_doc *attempt = index_attempt_read(name, &attempt_exists); + yyjson_doc_free(attempt); + if (attempt_exists && cbm_mcp_index_attempt_remove(name)) { + status = "deleted"; + } else { + is_error = true; + if (attempt_exists) { + status = "delete_failed"; + error_detail = "attempt status cleanup failed"; + } + } } cbm_pipeline_unlock(); @@ -11912,7 +12735,8 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args, * run it through index_run_supervised. Shared by the session auto-index (srv * present → its cached store is invalidated) and the watcher re-index (srv NULL). * Returns the worker's response string (caller frees) or NULL to degrade. */ -static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_path) { +static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_path, + const char *origin) { if (!root_path || !root_path[0]) { return NULL; } @@ -11928,6 +12752,7 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p /* A doc_candidates setting the user set travels with the request (the * worker reads no config); unset, the worker's index mode decides. */ if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) || + !yyjson_mut_obj_add_strcpy(doc, root, "_cbm_index_origin", origin) || !cbm_mcp_index_policy_add_to_args(doc, root, &policy) || !cbm_mcp_index_doc_candidates_add_to_args(doc, root, srv ? srv->config : NULL)) { yyjson_mut_doc_free(doc); @@ -11938,7 +12763,20 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p if (!args) { return NULL; } + char *project = cbm_project_name_from_path(root_path); + char attempt_id[33]; + bool attempt_recorded = + project && cbm_mcp_index_attempt_begin(project, root_path, origin, &policy, + strcmp(origin, "watcher") == 0, attempt_id); + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(project, root_path, attempt_id, "running", NULL, + NULL, false); + } char *resp = index_run_supervised(srv, args, &policy); + if (attempt_recorded) { + index_attempt_finish_response(project, root_path, attempt_id, resp); + } + free(project); free(args); return resp; } @@ -11946,7 +12784,7 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p /* Public entry (see mcp.h): the watcher re-index in main.c has no MCP server, so * it reaches the supervised runner through this srv-less wrapper. */ char *cbm_mcp_index_run_supervised_path(const char *root_path) { - return index_run_supervised_path(NULL, root_path); + return index_run_supervised_path(NULL, root_path, "watcher"); } bool cbm_path_within_root(const char *root_path, const char *abs_path); /* defined below */ @@ -12048,6 +12886,9 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r while (yyjson_mut_obj_get(copy_root, "_cbm_index_policy")) { (void)yyjson_mut_obj_remove_key(copy_root, "_cbm_index_policy"); } + while (yyjson_mut_obj_get(copy_root, "_cbm_index_origin")) { + (void)yyjson_mut_obj_remove_key(copy_root, "_cbm_index_origin"); + } /* #2144: async/status choose how the caller waits, not what is indexed. * They never reach the worker, and an async request coalesces with an * identical synchronous one instead of being refused as an options @@ -12067,6 +12908,7 @@ static char *index_args_with_repo_path(const char *args, const char *canonical_r } if ((has_name && !yyjson_mut_obj_add_strcpy(copy, copy_root, "name", project_name)) || !yyjson_mut_obj_add_strcpy(copy, copy_root, "repo_path", canonical_repo_path) || + !yyjson_mut_obj_add_str(copy, copy_root, "_cbm_index_origin", "explicit") || !cbm_mcp_index_policy_add_to_args(copy, copy_root, policy) || (doc_candidates != INDEX_DOC_CANDIDATES_BY_MODE && !yyjson_mut_obj_add_bool(copy, copy_root, "doc_candidates", @@ -12477,7 +13319,24 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { index_args_free(repo_path, mode_str, name_override); return cbm_mcp_text_result("failed to prepare supervised index request", true); } + char *owned_origin = cbm_mcp_get_string_arg(worker_args, "_cbm_index_origin"); + const char *origin = owned_origin && (strcmp(owned_origin, "auto") == 0 || + strcmp(owned_origin, "watcher") == 0) + ? owned_origin + : "explicit"; + char attempt_id[33]; + bool attempt_recorded = + cbm_mcp_index_attempt_begin(mutation_project, repo_path, origin, &resource_policy, + strcmp(origin, "watcher") == 0, attempt_id); + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(mutation_project, repo_path, attempt_id, + "running", NULL, NULL, false); + } char *supervised = index_run_supervised(srv, worker_args, &resource_policy); + if (attempt_recorded) { + index_attempt_finish_response(mutation_project, repo_path, attempt_id, supervised); + } + free(owned_origin); free(worker_args); if (supervised) { free(mutation_project); @@ -12515,9 +13374,17 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(mode_str); bool persistence = cbm_mcp_get_bool_arg(args, "persistence"); + char attempt_id[33]; + bool attempt_recorded = !cbm_index_worker_active() && + cbm_mcp_index_attempt_begin(mutation_project, repo_path, "explicit", + &resource_policy, false, attempt_id); cbm_pipeline_t *p = cbm_pipeline_new(repo_path, NULL, mode); if (!p) { + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(mutation_project, repo_path, attempt_id, + "failed", "index_failed", NULL, false); + } mcp_project_mutation_end(srv, mutation_project); free(mutation_project); free(name_override); @@ -12525,6 +13392,10 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { return cbm_mcp_text_result("failed to create pipeline", true); } if (name_override && name_override[0] && !cbm_pipeline_set_project_name(p, name_override)) { + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(mutation_project, repo_path, attempt_id, + "failed", "index_failed", NULL, false); + } cbm_pipeline_free(p); mcp_project_mutation_end(srv, mutation_project); free(mutation_project); @@ -12553,6 +13424,11 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { free(srv->current_project); srv->current_project = NULL; + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(project_name, repo_path, attempt_id, "running", NULL, + NULL, false); + } + /* Serialize pipeline runs to prevent concurrent writes. * Track active pipeline so signal handler and notifications/cancelled * can cancel it mid-run. */ @@ -12577,6 +13453,18 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { cbm_pipeline_get_file_errors(p, &file_errors, &file_error_count); cbm_index_resource_violation_t resource_violation = {0}; cbm_pipeline_get_resource_violation(p, &resource_violation); + if (attempt_recorded) { + const char *failure_code = + rc == 0 ? NULL + : resource_violation.resource != CBM_INDEX_RESOURCE_NONE + ? (resource_violation.probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded") + : "index_failed"; + (void)cbm_mcp_index_attempt_transition( + project_name, repo_path, attempt_id, rc == 0 ? "completed" : "failed", failure_code, + resource_violation.resource != CBM_INDEX_RESOURCE_NONE ? &resource_violation : NULL, + rc == 0); + } cbm_mem_collect(); /* return mimalloc pages to OS after large indexing */ @@ -20833,7 +21721,7 @@ static void *autoindex_thread(void *arg) { * 100% of that memory back to the OS every cycle. In a marked host this is a * safety boundary: preparation/start failure stops the operation. */ if (cbm_index_supervisor_should_wrap()) { - char *resp = index_run_supervised_path(srv, srv->session_root); + char *resp = index_run_supervised_path(srv, srv->session_root, "auto"); if (resp) { free(resp); cbm_log_info("autoindex.done", "project", srv->session_project, "mode", "supervised"); @@ -20855,19 +21743,46 @@ static void *autoindex_thread(void *arg) { cbm_log_warn("autoindex.err", "msg", "resource_policy_load_failed", "error", policy_error); return NULL; } + char attempt_id[33]; + bool attempt_recorded = cbm_mcp_index_attempt_begin( + srv->session_project, srv->session_root, "auto", &resource_policy, false, attempt_id); cbm_pipeline_t *p = cbm_pipeline_new(srv->session_root, NULL, CBM_MODE_FULL); if (!p) { + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(srv->session_project, srv->session_root, + attempt_id, "failed", "index_failed", NULL, + false); + } cbm_log_warn("autoindex.err", "msg", "pipeline_create_failed"); return NULL; } cbm_pipeline_set_resource_policy(p, &resource_policy); index_doc_candidates_apply(p, index_doc_candidates_choice(srv, NULL)); + if (attempt_recorded) { + (void)cbm_mcp_index_attempt_transition(srv->session_project, srv->session_root, attempt_id, + "running", NULL, NULL, false); + } /* Block until any concurrent pipeline finishes */ cbm_pipeline_lock(); int rc = cbm_pipeline_run(p); cbm_pipeline_unlock(); + cbm_index_resource_violation_t resource_violation = {0}; + cbm_pipeline_get_resource_violation(p, &resource_violation); + if (attempt_recorded) { + const char *failure_code = + rc == 0 ? NULL + : resource_violation.resource != CBM_INDEX_RESOURCE_NONE + ? (resource_violation.probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded") + : "index_failed"; + (void)cbm_mcp_index_attempt_transition( + srv->session_project, srv->session_root, attempt_id, rc == 0 ? "completed" : "failed", + failure_code, + resource_violation.resource != CBM_INDEX_RESOURCE_NONE ? &resource_violation : NULL, + rc == 0); + } cbm_pipeline_free(p); cbm_kind_in_set_free_cache(); /* free THIS thread's extraction bitset cache (see above) */ cbm_mem_collect(); /* return mimalloc pages to OS after indexing (in-process only) */ diff --git a/src/mcp/mcp_internal.h b/src/mcp/mcp_internal.h index 354ae36a15..337ce5c358 100644 --- a/src/mcp/mcp_internal.h +++ b/src/mcp/mcp_internal.h @@ -72,6 +72,28 @@ char *cbm_mcp_index_worker_resource_response(const char *args, bool cbm_mcp_index_doc_candidates_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, struct cbm_config *config); +typedef enum { + CBM_INDEX_ATTEMPT_NONE = 0, + CBM_INDEX_ATTEMPT_AVAILABLE, + CBM_INDEX_ATTEMPT_CORRUPT, +} cbm_index_attempt_read_status_t; + +bool cbm_mcp_index_attempt_begin(const char *project, const char *repo_path, const char *origin, + const cbm_index_resource_policy_t *policy, + bool watcher_observed_change, char attempt_id[33]); +bool cbm_mcp_index_attempt_transition(const char *project, const char *repo_path, + const char *attempt_id, const char *state, + const char *failure_code, + const cbm_index_resource_violation_t *violation, + bool generation_completed); +bool cbm_mcp_index_attempt_mark_watcher_change(const char *project, const char *attempt_id); +cbm_index_attempt_read_status_t cbm_mcp_index_attempt_add_status(yyjson_mut_doc *document, + yyjson_mut_val *root, + const char *project, + const char *current_root_path); +bool cbm_mcp_index_attempt_remove(const char *project); +void cbm_mcp_index_attempt_recover_abandoned(void); + enum { CBM_MCP_DEFAULT_AUTO_INDEX_LIMIT = 50000 }; /* Count indexable files with the pipeline's native full-mode discovery policy, diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 1035756aca..5cb0a0146d 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -2958,8 +2958,81 @@ TEST(daemon_application_programmatic_index_forwards_set_doc_candidates) { PASS(); } +TEST(daemon_application_controlled_tool_error_completes_background_observation) { + char *root_created = th_mktempdir("cbm_app_controlled_error_root"); + char *root = root_created ? strdup(root_created) : NULL; + char *cache_created = th_mktempdir("cbm_app_controlled_error_cache"); + char *cache = cache_created ? strdup(cache_created) : NULL; + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + bool cache_env = cache && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; + cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.scripted, true); + fake.outcomes[0] = CBM_PROC_CLEAN; + fake.responses[0] = "{\"content\":[{\"type\":\"text\",\"text\":\"{" + "\\\"status\\\":\\\"error\\\"," + "\\\"code\\\":\\\"resource_limit_exceeded\\\"," + "\\\"stage\\\":\\\"discovery\\\"," + "\\\"resource\\\":\\\"entries\\\"," + "\\\"observed\\\":11," + "\\\"limit\\\":10," + "\\\"unit\\\":\\\"entries\\\"}\"}],\"isError\":true}"; + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = + cache_env && stored_config ? cbm_daemon_application_new(&config) : NULL; + int index_rc = application && root + ? cbm_daemon_application_index(application, "ControlledFailureFixture", root) + : -1; + char attempt_path[APP_TEST_PATH_CAP]; + char attempt_record[APP_TEST_PATH_CAP] = {0}; + (void)snprintf(attempt_path, sizeof(attempt_path), "%s/status/ControlledFailureFixture.json", + cache ? cache : ""); + app_fake_worker_read_file(attempt_path, attempt_record, sizeof(attempt_record)); + bool attempt_failed = strstr(attempt_record, "\"state\":\"failed\"") && + strstr(attempt_record, "\"failure_code\":\"resource_limit_exceeded\"") && + strstr(attempt_record, "\"resource\":\"entries\""); + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + int starts = atomic_load(&fake.starts); + + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); + th_cleanup(root); + th_cleanup(cache); + free(root); + free(cache); + + ASSERT_EQ(index_rc, 0); + ASSERT_TRUE(attempt_failed); + ASSERT_EQ(starts, 1); + ASSERT_TRUE(stopped); + PASS(); +} + TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { - char *cache = th_mktempdir("cbm_app_worker_resource_cache"); + char *cache_created = th_mktempdir("cbm_app_worker_resource_cache"); + char *cache = cache_created ? strdup(cache_created) : NULL; + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + bool cache_env = cache && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2993,7 +3066,7 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"name\":\"DaemonWorkerResourceFixture\"}", root ? root : ""); - bool setup = cache && stored_config && application && session && root && + bool setup = cache_env && stored_config && application && session && root && app_test_context_request(root, root, &context, &context_length) && app_test_tool_request("index_repository", args, &tool, &tool_length); uint8_t *response = NULL; @@ -3015,6 +3088,17 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { strstr((char *)response, "\\\"observed\\\":7001") && strstr((char *)response, "\\\"limit\\\":7000") && strstr((char *)response, "\\\"unit\\\":\\\"milliseconds\\\""); + char attempt_path[APP_TEST_PATH_CAP]; + char attempt_record[APP_TEST_PATH_CAP] = {0}; + (void)snprintf(attempt_path, sizeof(attempt_path), "%s/status/DaemonWorkerResourceFixture.json", + cache ? cache : ""); + app_fake_worker_read_file(attempt_path, attempt_record, sizeof(attempt_record)); + bool attempt_visible = strstr(attempt_record, "\"origin\":\"explicit\"") && + strstr(attempt_record, "\"state\":\"failed\"") && + strstr(attempt_record, "\"failure_code\":\"resource_limit_exceeded\"") && + strstr(attempt_record, "\"resource\":\"duration_ms\"") && + strstr(attempt_record, "\"observed\":7001") && + strstr(attempt_record, "\"limit\":7000"); if (session) { callbacks.session_close(callbacks.context, session); } @@ -3026,12 +3110,20 @@ TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { free(response); free(context); free(tool); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); th_cleanup(root); th_cleanup(cache); + free(cache); ASSERT_TRUE(setup); ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); ASSERT_TRUE(structured); + ASSERT_TRUE(attempt_visible); ASSERT_EQ(starts, 1); ASSERT_EQ(destroys, 1); ASSERT_TRUE(stopped); @@ -5746,6 +5838,11 @@ TEST(daemon_application_cancellation_between_recovery_attempts_stops_retry) { } TEST(daemon_application_thread_start_failure_rolls_back_job_reservation) { + char *cache_created = th_mktempdir("cbm_app_thread_start_cache"); + char *cache = cache_created ? strdup(cache_created) : NULL; + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache_copy = saved_cache ? strdup(saved_cache) : NULL; + bool cache_ready = cache && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); atomic_store(&fake.allow_completion, true); @@ -5758,7 +5855,8 @@ TEST(daemon_application_thread_start_failure_rolls_back_job_reservation) { .destroy = app_fake_worker_destroy, }; cbm_daemon_application_config_t config = {.worker_ops = &worker_ops}; - cbm_daemon_application_t *application = cbm_daemon_application_new(&config); + cbm_daemon_application_t *application = + cache_ready ? cbm_daemon_application_new(&config) : NULL; char root[APP_TEST_PATH_CAP]; (void)snprintf(root, sizeof(root), "%s/cbm-app-thread-start-XXXXXX", cbm_tmpdir()); bool root_ok = cbm_mkdtemp(root) != NULL; @@ -5771,17 +5869,35 @@ TEST(daemon_application_thread_start_failure_rolls_back_job_reservation) { size_t active_after_failure = application ? cbm_daemon_application_active_jobs(application) : 1; size_t subscribers_after_failure = application ? cbm_daemon_application_job_subscribers(application, "thread-start") : 1; + char attempt_path[APP_TEST_PATH_CAP]; + char attempt_record[APP_TEST_PATH_CAP] = {0}; + (void)snprintf(attempt_path, sizeof(attempt_path), "%s/status/thread-start.json", + cache ? cache : ""); + app_fake_worker_read_file(attempt_path, attempt_record, sizeof(attempt_record)); + bool failure_recorded = strstr(attempt_record, "\"state\":\"failed\"") && + strstr(attempt_record, "\"failure_code\":\"worker_start_failed\"") && + strstr(attempt_record, "\"origin\":\"explicit\""); int retried = application && root_ok ? cbm_daemon_application_index(application, "thread-start", root) : -1; bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); cbm_daemon_application_free(application); + if (saved_cache_copy) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache_copy, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache_copy); (void)cbm_rmdir(root); + th_cleanup(cache); + free(cache); + ASSERT_TRUE(cache_ready); ASSERT_TRUE(root_ok); ASSERT_LT(failed, 0); ASSERT_EQ(active_after_failure, 0); ASSERT_EQ(subscribers_after_failure, 0); + ASSERT_TRUE(failure_recorded); ASSERT_EQ(retried, 0); ASSERT_EQ(atomic_load(&fake.starts), 1); ASSERT_EQ(atomic_load(&fake.destroys), 1); @@ -7142,6 +7258,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_unnamed_async_and_status_refuse_ambiguous_root_issue2134); RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); RUN_TEST(daemon_application_programmatic_index_forwards_set_doc_candidates); + RUN_TEST(daemon_application_controlled_tool_error_completes_background_observation); RUN_TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_worker_storage_resource_failure_is_structured_and_not_retried); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); diff --git a/tests/test_git_context.c b/tests/test_git_context.c index c508e25f34..33399b2098 100644 --- a/tests/test_git_context.c +++ b/tests/test_git_context.c @@ -73,16 +73,22 @@ static int git_run(const char *dir, const char *args) { /* Create a minimal git repo at dir (init + empty commit so HEAD exists). */ static int make_git_repo(const char *dir) { - if (th_mkdir_p(dir) != 0) return -1; - if (git_run(dir, "init -q") != 0) return -1; - if (git_run(dir, "config user.email test@example.com") != 0) return -1; - if (git_run(dir, "config user.name Test") != 0) return -1; + if (th_mkdir_p(dir) != 0) + return -1; + if (git_run(dir, "init -q") != 0) + return -1; + if (git_run(dir, "config user.email test@example.com") != 0) + return -1; + if (git_run(dir, "config user.name Test") != 0) + return -1; /* Create a file so HEAD points to a real commit. */ char path[1024]; snprintf(path, sizeof(path), "%s/.keep", dir); th_write_file(path, ""); - if (git_run(dir, "add .keep") != 0) return -1; - if (git_run(dir, "commit -q -m init") != 0) return -1; + if (git_run(dir, "add .keep") != 0) + return -1; + if (git_run(dir, "commit -q -m init") != 0) + return -1; return 0; } #endif /* _WIN32 */ @@ -182,7 +188,8 @@ TEST(canonical_root_repo_root) { SKIP_PLATFORM("git-based canonical_root test not supported on Windows CI"); #else char *tmp = th_mktempdir("cbm_gitctx"); - if (!tmp) FAIL("th_mktempdir returned NULL"); + if (!tmp) + FAIL("th_mktempdir returned NULL"); if (make_git_repo(tmp) != 0) { th_rmtree(tmp); @@ -223,7 +230,8 @@ TEST(canonical_root_subdir) { SKIP_PLATFORM("git-based canonical_root test not supported on Windows CI"); #else char *tmp = th_mktempdir("cbm_gitctx"); - if (!tmp) FAIL("th_mktempdir returned NULL"); + if (!tmp) + FAIL("th_mktempdir returned NULL"); if (make_git_repo(tmp) != 0) { th_rmtree(tmp); @@ -282,13 +290,15 @@ TEST(canonical_root_linked_worktree) { /* th_mktempdir() returns a static buffer — copy before the second call. */ char main_tmp[256]; char *raw = th_mktempdir("cbm_main"); - if (!raw) FAIL("th_mktempdir returned NULL"); + if (!raw) + FAIL("th_mktempdir returned NULL"); strncpy(main_tmp, raw, sizeof(main_tmp) - 1); main_tmp[sizeof(main_tmp) - 1] = '\0'; char wt_tmp[256]; raw = th_mktempdir("cbm_worktree"); - if (!raw) FAIL("th_mktempdir returned NULL"); + if (!raw) + FAIL("th_mktempdir returned NULL"); strncpy(wt_tmp, raw, sizeof(wt_tmp) - 1); wt_tmp[sizeof(wt_tmp) - 1] = '\0'; @@ -466,10 +476,12 @@ TEST(git_branch_resolve_matches_full_context) { #else char plain[512]; char *tmp = th_mktempdir("cbm_gitbranch_plain"); - if (!tmp) FAIL("th_mktempdir returned NULL"); + if (!tmp) + FAIL("th_mktempdir returned NULL"); snprintf(plain, sizeof(plain), "%s", tmp); tmp = th_mktempdir("cbm_gitbranch"); - if (!tmp) FAIL("th_mktempdir returned NULL"); + if (!tmp) + FAIL("th_mktempdir returned NULL"); if (make_git_repo(tmp) != 0) { th_rmtree(tmp); th_rmtree(plain); @@ -495,6 +507,38 @@ TEST(git_branch_resolve_matches_full_context) { #endif /* _WIN32 */ } +TEST(git_snapshot_reports_head_and_dirty_state_read_only) { +#ifdef _WIN32 + SKIP_PLATFORM("git snapshot test not supported on Windows CI"); +#else + char *tmp = th_mktempdir("cbm_git_snapshot"); + ASSERT_NOT_NULL(tmp); + if (make_git_repo(tmp) != 0) { + th_rmtree(tmp); + SKIP_PLATFORM("git not available to init a repo"); + } + + cbm_git_snapshot_t clean = {0}; + ASSERT_EQ(cbm_git_snapshot_read(tmp, &clean), 0); + ASSERT_TRUE(clean.available); + ASSERT_NOT_NULL(clean.head_sha); + ASSERT_EQ(strlen(clean.head_sha), 40); + ASSERT_FALSE(clean.dirty); + + ASSERT_EQ(th_write_file(TH_PATH(tmp, "untracked.c"), "int untracked;\n"), 0); + cbm_git_snapshot_t dirty = {0}; + ASSERT_EQ(cbm_git_snapshot_read(tmp, &dirty), 0); + ASSERT_TRUE(dirty.available); + ASSERT_STR_EQ(dirty.head_sha, clean.head_sha); + ASSERT_TRUE(dirty.dirty); + + cbm_git_snapshot_free(&dirty); + cbm_git_snapshot_free(&clean); + th_rmtree(tmp); + PASS(); +#endif +} + /* ── Suite ──────────────────────────────────────────────────────── */ SUITE(git_context) { @@ -505,4 +549,5 @@ SUITE(git_context) { RUN_TEST(canonical_root_linked_worktree); RUN_TEST(git_context_ignores_inherited_repo_env); RUN_TEST(githistory_ignores_inherited_repo_env); + RUN_TEST(git_snapshot_reports_head_and_dirty_state_read_only); } diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index 71f8240c76..2cc6b51307 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -683,7 +683,8 @@ TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { th_write_file(TH_PATH(repo, "second.py"), "def second():\n return 2\n") == 0; (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"name\":\"ResourcePolicyFixture\"," - "\"mode\":\"fast\",\"_cbm_index_policy\":{" + "\"mode\":\"fast\",\"_cbm_index_origin\":\"watcher\"," + "\"_cbm_index_policy\":{" "\"index_max_files\":\"off\",\"index_max_source_mb\":\"off\"}}", repo); char *limited_response = @@ -697,6 +698,16 @@ TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { strstr(limited_response, "\\\"retryable\\\":true") && strstr(limited_response, "\\\"serving_index_preserved\\\":true"); free(limited_response); + char attempt_path[2048]; + (void)snprintf(attempt_path, sizeof(attempt_path), "%s/status/ResourcePolicyFixture.json", + cache); + yyjson_doc *attempt_record = yyjson_read_file(attempt_path, 0, NULL, NULL); + yyjson_val *attempt_root = attempt_record ? yyjson_doc_get_root(attempt_record) : NULL; + yyjson_val *attempt_origin = + attempt_root && yyjson_is_obj(attempt_root) ? yyjson_obj_get(attempt_root, "origin") : NULL; + bool origin_not_forged = attempt_origin && yyjson_is_str(attempt_origin) && + strcmp(yyjson_get_str(attempt_origin), "explicit") == 0; + yyjson_doc_free(attempt_record); cbm_store_t *after_store = cbm_store_open_path_query(db_path); cbm_project_t after_project = {0}; @@ -737,6 +748,7 @@ TEST(index_policy_mcp_rejects_forged_override_and_preserves_serving_index) { ASSERT_TRUE(first_indexed); ASSERT_TRUE(configured); ASSERT_TRUE(contract_ok); + ASSERT_TRUE(origin_not_forged); ASSERT_TRUE(generation_preserved); ASSERT_TRUE(cross_repo_unaffected); PASS(); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index 560f15d26c..8f24839a76 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -1052,6 +1052,41 @@ TEST(tree_table_uses_prefix_dictionary_only_when_it_materially_wins) { PASS(); } +TEST(index_attempt_only_project_can_be_deleted) { + char *cache = th_mktempdir("cbm_attempt_only_delete"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved_cache ? strdup(saved_cache) : NULL; + ASSERT_EQ(cbm_setenv("CBM_CACHE_DIR", cache, 1), 0); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptOnlyFixture", cache, "explicit", &policy, false, + attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptOnlyFixture", cache, attempt_id, "failed", + "worker_start_failed", NULL, false)); + cbm_mcp_server_t *server = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(server); + char *response = + cbm_mcp_handle_tool(server, "delete_project", "{\"project\":\"AttemptOnlyFixture\"}"); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\\\"status\\\":\\\"deleted\\\"")); + free(response); + cbm_mcp_server_free(server); + + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "AttemptOnlyFixture", cache), + CBM_INDEX_ATTEMPT_NONE); + yyjson_mut_doc_free(document); + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * JSON-RPC PARSING * ══════════════════════════════════════════════════════════════════ */ @@ -6888,6 +6923,276 @@ TEST(tool_index_status_includes_git_metadata) { PASS(); } +TEST(index_attempt_record_rejects_stale_transition_and_recovers_abandoned_job) { + char *cache = th_mktempdir("cbm_attempt_record"); + ASSERT_NOT_NULL(cache); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved_cache ? strdup(saved_cache) : NULL; + ASSERT_EQ(cbm_setenv("CBM_CACHE_DIR", cache, 1), 0); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[128]; + ASSERT_TRUE(cbm_index_policy_set_profile(&policy, "strict", error, sizeof(error))); + char attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptFixture", cache, "explicit", &policy, false, + attempt_id)); + ASSERT_EQ(strlen(attempt_id), 32); + ASSERT_FALSE(cbm_mcp_index_attempt_transition( + "AttemptFixture", cache, "00000000000000000000000000000000", "running", NULL, NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, attempt_id, "running", + NULL, NULL, false)); + char replacement_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptFixture", cache, "auto", &policy, false, + replacement_attempt_id)); + ASSERT_FALSE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, attempt_id, "failed", + "index_failed", NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, replacement_attempt_id, + "running", NULL, NULL, false)); + + char attempt_path[512]; + (void)snprintf(attempt_path, sizeof(attempt_path), "%s/status/AttemptFixture.json", cache); + yyjson_read_err read_error; + yyjson_doc *stored = yyjson_read_file(attempt_path, 0, NULL, &read_error); + ASSERT_NOT_NULL(stored); + yyjson_mut_doc *abandoned = yyjson_doc_mut_copy(stored, NULL); + yyjson_doc_free(stored); + ASSERT_NOT_NULL(abandoned); + yyjson_mut_val *abandoned_root = yyjson_mut_doc_get_root(abandoned); + ASSERT_TRUE(yyjson_mut_obj_put(abandoned_root, + yyjson_mut_strcpy(abandoned, "owner_start_token"), + yyjson_mut_uint(abandoned, 0))); + char *abandoned_json = yyjson_mut_write(abandoned, 0, NULL); + ASSERT_NOT_NULL(abandoned_json); + ASSERT_EQ(th_write_file(attempt_path, abandoned_json), 0); + free(abandoned_json); + yyjson_mut_doc_free(abandoned); + + cbm_mcp_index_attempt_recover_abandoned(); + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "AttemptFixture", cache), + CBM_INDEX_ATTEMPT_AVAILABLE); + char *json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"state\":\"failed\"")); + ASSERT_NOT_NULL(strstr(json, "\"failure_code\":\"worker_lost\"")); + ASSERT_NOT_NULL(strstr(json, "\"profile\":\"strict\"")); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"unknown\"")); + free(json); + yyjson_mut_doc_free(document); + + char probe_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptFixture", cache, "auto", &policy, false, + probe_attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, probe_attempt_id, + "running", NULL, NULL, false)); + cbm_index_resource_violation_t probe_failure = { + .resource = CBM_INDEX_RESOURCE_RSS_BYTES, + .probe_failed = true, + }; + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, probe_attempt_id, + "failed", "resource_probe_failed", &probe_failure, + false)); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "AttemptFixture", cache), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"failure_code\":\"resource_probe_failed\"")); + ASSERT_NOT_NULL(strstr(json, "\"resource\":\"rss_bytes\"")); + ASSERT_NOT_NULL(strstr(json, "\"probe_failed\":true")); + ASSERT_NULL(strstr(json, "\"observed\"")); + free(json); + yyjson_mut_doc_free(document); + + char watcher_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptFixture", cache, "watcher", &policy, true, + watcher_attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, watcher_attempt_id, + "running", NULL, NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, watcher_attempt_id, + "failed", "index_failed", NULL, false)); + char retry_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("AttemptFixture", cache, "explicit", &policy, false, + retry_attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, retry_attempt_id, + "running", NULL, NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("AttemptFixture", cache, retry_attempt_id, + "failed", "index_failed", NULL, false)); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "AttemptFixture", cache), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"origin\":\"explicit\"")); + ASSERT_NOT_NULL(strstr(json, "\"watcher_observed_change\":true")); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"stale\"")); + free(json); + yyjson_mut_doc_free(document); + + ASSERT_TRUE(cbm_mcp_index_attempt_remove("AttemptFixture")); + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + PASS(); +} + +TEST(index_attempt_freshness_requires_same_clean_git_snapshot) { + char *cache_created = th_mktempdir("cbm_attempt_fresh_cache"); + char *cache = cache_created ? strdup(cache_created) : NULL; + char *repo_created = th_mktempdir("cbm_attempt_fresh_repo"); + char *repo = repo_created ? strdup(repo_created) : NULL; + ASSERT_NOT_NULL(cache); + ASSERT_NOT_NULL(repo); + const char *saved_cache = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved_cache ? strdup(saved_cache) : NULL; + ASSERT_EQ(cbm_setenv("CBM_CACHE_DIR", cache, 1), 0); + + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + const char *init[] = {"-c", "init.defaultBranch=main", "init", "-q", NULL}; + const char *add[] = {"add", "main.c", NULL}; + const char *commit[] = {"-c", "user.name=Test", + "-c", "user.email=test@example.com", + "-c", "commit.gpgsign=false", + "commit", "-q", + "-m", "init", + NULL}; + if (mcp_test_git(repo, init) != 0 || mcp_test_git(repo, add) != 0 || + mcp_test_git(repo, commit) != 0) { + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + th_cleanup(repo); + free(cache); + free(repo); + SKIP_PLATFORM("git fixture unavailable"); + } + ASSERT_EQ(cbm_unlink(TH_PATH(repo, ".cbm-empty-gitconfig")), 0); + + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char db_path[512]; + (void)snprintf(db_path, sizeof(db_path), "%s/FreshFixture.db", cache); + ASSERT_TRUE(mcp_make_valid_project_store_at(db_path, "FreshFixture", repo)); + char attempt_id[33]; + ASSERT_TRUE( + cbm_mcp_index_attempt_begin("FreshFixture", repo, "explicit", &policy, false, attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, attempt_id, "running", NULL, + NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, attempt_id, "completed", + NULL, NULL, true)); + + yyjson_mut_doc *document = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "FreshFixture", repo), + CBM_INDEX_ATTEMPT_AVAILABLE); + char *json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"fresh\"")); + free(json); + yyjson_mut_doc_free(document); + + char failed_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("FreshFixture", repo, "explicit", &policy, false, + failed_attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, failed_attempt_id, "running", + NULL, NULL, false)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, failed_attempt_id, "failed", + "index_failed", NULL, false)); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "FreshFixture", repo), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"state\":\"failed\"")); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"fresh\"")); + free(json); + yyjson_mut_doc_free(document); + + ASSERT_EQ(th_write_file(TH_PATH(repo, "untracked.c"), "int dirty;\n"), 0); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "FreshFixture", repo), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"unknown\"")); + free(json); + yyjson_mut_doc_free(document); + + ASSERT_EQ(cbm_unlink(TH_PATH(repo, "untracked.c")), 0); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 1; }\n"), 0); + if (mcp_test_git(repo, add) != 0 || mcp_test_git(repo, commit) != 0) { + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + th_cleanup(repo); + free(cache); + free(repo); + SKIP_PLATFORM("git fixture update unavailable"); + } + ASSERT_EQ(cbm_unlink(TH_PATH(repo, ".cbm-empty-gitconfig")), 0); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "FreshFixture", repo), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"stale\"")); + free(json); + yyjson_mut_doc_free(document); + + char changed_attempt_id[33]; + ASSERT_TRUE(cbm_mcp_index_attempt_begin("FreshFixture", repo, "explicit", &policy, false, + changed_attempt_id)); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, changed_attempt_id, + "running", NULL, NULL, false)); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 2; }\n"), 0); + if (mcp_test_git(repo, add) != 0 || mcp_test_git(repo, commit) != 0) { + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + th_cleanup(repo); + free(cache); + free(repo); + SKIP_PLATFORM("git fixture update unavailable"); + } + ASSERT_EQ(cbm_unlink(TH_PATH(repo, ".cbm-empty-gitconfig")), 0); + ASSERT_TRUE(cbm_mcp_index_attempt_transition("FreshFixture", repo, changed_attempt_id, + "completed", NULL, NULL, true)); + document = yyjson_mut_doc_new(NULL); + root = yyjson_mut_obj(document); + yyjson_mut_doc_set_root(document, root); + ASSERT_EQ(cbm_mcp_index_attempt_add_status(document, root, "FreshFixture", repo), + CBM_INDEX_ATTEMPT_AVAILABLE); + json = yyjson_mut_write(document, 0, NULL); + ASSERT_NOT_NULL(json); + ASSERT_NOT_NULL(strstr(json, "\"generation_snapshot_available\":false")); + ASSERT_NOT_NULL(strstr(json, "\"freshness\":\"unknown\"")); + free(json); + yyjson_mut_doc_free(document); + + ASSERT_TRUE(cbm_mcp_index_attempt_remove("FreshFixture")); + restore_cache_dir(saved_copy); + free(saved_copy); + th_cleanup(cache); + th_cleanup(repo); + free(cache); + free(repo); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * TOOL HANDLERS WITH DATA * ══════════════════════════════════════════════════════════════════ */ @@ -21992,6 +22297,7 @@ enum { IDX832_NULL_RESP = 52, /* supervised entry degraded to NULL */ IDX832_NOT_INDEXED = 53, /* response/store lacks the indexed Function node */ IDX832_SERVER_FAIL = 54, + IDX832_ATTEMPT_MISSING = 55, }; #ifndef _WIN32 /* helper used only by the POSIX fork harness below */ @@ -22026,6 +22332,24 @@ static int idx832_supervised_route_check(const char *repo_dir) { /* Store-level proof the worker child did real work: the Function node it wrote * must be queryable from a fresh server reading the DB the child produced. */ char *project = cbm_project_name_from_path(repo_dir); + const char *cache = cbm_resolve_cache_dir(); + char attempt_path[1024]; + int attempt_length = project && cache ? snprintf(attempt_path, sizeof(attempt_path), + "%s/status/%s.json", cache, project) + : -1; + yyjson_doc *attempt = attempt_length > 0 && (size_t)attempt_length < sizeof(attempt_path) + ? yyjson_read_file(attempt_path, 0, NULL, NULL) + : NULL; + yyjson_val *attempt_root = attempt ? yyjson_doc_get_root(attempt) : NULL; + yyjson_val *attempt_origin = + attempt_root && yyjson_is_obj(attempt_root) ? yyjson_obj_get(attempt_root, "origin") : NULL; + bool watcher_attempt = attempt_origin && yyjson_is_str(attempt_origin) && + strcmp(yyjson_get_str(attempt_origin), "watcher") == 0; + yyjson_doc_free(attempt); + if (!watcher_attempt) { + free(project); + return IDX832_ATTEMPT_MISSING; + } cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); if (!srv) { free(project); @@ -23160,6 +23484,7 @@ enum { IDX853_NO_SPAWN = 62, /* spawn_count unchanged → supervised path not exercised */ IDX853_SETUP_FAIL = 63, /* config/watcher/server/cwd setup failed */ IDX853_BAD_COUNT = 64, /* unexpected watch_count (<0 or >1) */ + IDX853_ATTEMPT_MISSING = 65, /* supervised auto path omitted its attempt record */ }; #ifndef _WIN32 /* helper used only by the POSIX fork harness below */ @@ -23218,9 +23543,27 @@ static int idx853_supervised_autowatch_check(const char *repo_dir, const char *c int spawns_after = cbm_index_supervisor_spawn_count(); int watch_count = cbm_watcher_watch_count(watcher); + char *project = cbm_project_name_from_path(repo_dir); + char attempt_path[1024]; + int attempt_length = project ? snprintf(attempt_path, sizeof(attempt_path), + "%s/status/%s.json", cache_dir, project) + : -1; + yyjson_doc *attempt = attempt_length > 0 && (size_t)attempt_length < sizeof(attempt_path) + ? yyjson_read_file(attempt_path, 0, NULL, NULL) + : NULL; + yyjson_val *attempt_root = attempt ? yyjson_doc_get_root(attempt) : NULL; + yyjson_val *attempt_origin = attempt_root && yyjson_is_obj(attempt_root) + ? yyjson_obj_get(attempt_root, "origin") + : NULL; + bool attempt_recorded = attempt_origin && yyjson_is_str(attempt_origin) && + strcmp(yyjson_get_str(attempt_origin), "auto") == 0; + yyjson_doc_free(attempt); + free(project); if (spawns_after == spawns_before) { code = IDX853_NO_SPAWN; /* supervised branch never ran — not a valid probe */ + } else if (!attempt_recorded) { + code = IDX853_ATTEMPT_MISSING; } else if (watch_count == 1) { code = IDX853_WATCHER_REGISTERED; /* the discriminating RED assertion */ } else if (watch_count == 0) { @@ -25181,6 +25524,9 @@ SUITE(mcp) { RUN_TEST(tool_check_index_coverage_requires_source_when_file_metadata_changed); RUN_TEST(tool_check_index_coverage_surfaces_lookup_errors); RUN_TEST(tool_index_status_includes_git_metadata); + RUN_TEST(index_attempt_record_rejects_stale_transition_and_recovers_abandoned_job); + RUN_TEST(index_attempt_freshness_requires_same_clean_git_snapshot); + RUN_TEST(index_attempt_only_project_can_be_deleted); /* Tool handlers with validation */ RUN_TEST(tool_trace_call_path_not_found);