From 50ee4a1972102fe32f4dbc1412899f3d62be74f8 Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Tue, 14 Jul 2026 18:35:47 +0300 Subject: [PATCH 1/2] fix: rebuild index when mode adds capabilities Signed-off-by: Anton Standrik --- src/mcp/mcp.c | 3 +- src/pipeline/pipeline.c | 157 +++++++--- src/pipeline/pipeline_incremental.c | 35 +-- src/pipeline/pipeline_internal.h | 11 +- src/store/store.h | 2 + tests/test_pipeline.c | 436 ++++++++++++++++++++++++++++ 6 files changed, 588 insertions(+), 56 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index fb5a750ec..e3f0d5100 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -375,7 +375,8 @@ static const tool_def_t TOOLS[] = { "are normalized.\"}," "\"persistence\":{\"type\":\"boolean\",\"default\":false,\"description\":" "\"Write compressed artifact to .codebase-memory/graph.db.zst for team sharing. " - "Teammates can bootstrap from the artifact instead of full re-indexing.\"}" + "Teammates can bootstrap from the artifact instead of full re-indexing. " + "An existing artifact is refreshed after any reindex even when persistence is false.\"}" "},\"required\":[\"repo_path\"]}"}, {"search_graph", "Search graph", diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 0c3041959..e0ea30d0f 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -35,6 +35,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include "foundation/compat_thread.h" #include "foundation/profile.h" #include "foundation/mem.h" +#include "yyjson/yyjson.h" #include #include @@ -162,6 +163,82 @@ static const char *itoa_buf(int val) { return bufs[i]; } +const char *cbm_pipeline_mode_name(cbm_index_mode_t mode) { + switch (mode) { + case CBM_MODE_FULL: + return "full"; + case CBM_MODE_MODERATE: + return "moderate"; + case CBM_MODE_FAST: + return "fast"; + default: + return "unknown"; + } +} + +typedef enum { + INDEX_MODE_METADATA_OK = 0, + INDEX_MODE_METADATA_MISSING_OR_INVALID, + INDEX_MODE_METADATA_ERROR, +} index_mode_metadata_status_t; + +static index_mode_metadata_status_t parse_index_mode(const char *properties_json, + cbm_index_mode_t *out_mode) { + if (!out_mode) { + return INDEX_MODE_METADATA_ERROR; + } + if (!properties_json || !properties_json[0]) { + return INDEX_MODE_METADATA_MISSING_OR_INVALID; + } + + yyjson_read_err read_error = {0}; + yyjson_doc *doc = + yyjson_read_opts((char *)properties_json, strlen(properties_json), 0, NULL, &read_error); + if (!doc) { + return read_error.code == YYJSON_READ_ERROR_MEMORY_ALLOCATION + ? INDEX_MODE_METADATA_ERROR + : INDEX_MODE_METADATA_MISSING_OR_INVALID; + } + + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *mode_value = yyjson_is_obj(root) ? yyjson_obj_get(root, "index_mode") : NULL; + index_mode_metadata_status_t status = INDEX_MODE_METADATA_OK; + if (yyjson_equals_str(mode_value, "full")) { + *out_mode = CBM_MODE_FULL; + } else if (yyjson_equals_str(mode_value, "moderate")) { + *out_mode = CBM_MODE_MODERATE; + } else if (yyjson_equals_str(mode_value, "fast")) { + *out_mode = CBM_MODE_FAST; + } else { + status = INDEX_MODE_METADATA_MISSING_OR_INVALID; + } + + yyjson_doc_free(doc); + return status; +} + +static bool index_mode_covers(cbm_index_mode_t stored_mode, cbm_index_mode_t requested_mode) { + return stored_mode >= CBM_MODE_FULL && stored_mode <= CBM_MODE_FAST && + requested_mode >= CBM_MODE_FULL && requested_mode <= CBM_MODE_FAST && + stored_mode <= requested_mode; +} + +static index_mode_metadata_status_t read_stored_index_mode(cbm_store_t *store, const char *project, + cbm_index_mode_t *out_mode) { + cbm_node_t node = {0}; + int rc = cbm_store_find_node_by_qn(store, project, project, &node); + if (rc == CBM_STORE_NOT_FOUND) { + return INDEX_MODE_METADATA_MISSING_OR_INVALID; + } + if (rc != CBM_STORE_OK) { + return INDEX_MODE_METADATA_ERROR; + } + + index_mode_metadata_status_t status = parse_index_mode(node.properties_json, out_mode); + cbm_node_free_fields(&node); + return status; +} + /* Log current + peak RSS at a pipeline phase boundary (memory profiling). */ static void log_phase_mem(const char *phase) { enum { PL_BYTES_PER_MB = 1024 * 1024 }; @@ -503,7 +580,11 @@ static int pass_structure(cbm_pipeline_t *p, const cbm_file_info_t *files, int f cbm_log_info("pass.start", "pass", "structure", "files", itoa_buf(file_count)); /* Project node */ - cbm_gbuf_upsert_node(p->gbuf, "Project", p->project_name, p->project_name, NULL, 0, 0, "{}"); + char project_props[CBM_SZ_64]; + snprintf(project_props, sizeof(project_props), "{\"index_mode\":\"%s\"}", + cbm_pipeline_mode_name(p->mode)); + cbm_gbuf_upsert_node(p->gbuf, "Project", p->project_name, p->project_name, NULL, 0, 0, + project_props); const char *branch_qn = p->branch_qn ? p->branch_qn : p->project_name; const char *branch_name = p->git_ctx.branch ? p->git_ctx.branch : "working-tree"; char branch_props[CBM_SZ_2K]; @@ -1156,7 +1237,8 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, } /* Try incremental pipeline or delete old DB for reindex. - * Returns >= 0 if incremental was used (the return code), or -1 to proceed with full. */ + * Returns >= 0 if incremental was used (the return code), -1 to proceed with full, + * or -2 to abort the run and preserve the existing DB. */ static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *files, int file_count) { char *db_path = resolve_db_path(p); if (!db_path) { @@ -1171,17 +1253,36 @@ static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *file if (check_store && cbm_store_check_integrity(check_store)) { cbm_file_hash_t *hashes = NULL; int hash_count = 0; + cbm_index_mode_t stored_mode = CBM_MODE_FAST; + index_mode_metadata_status_t stored_mode_status = + read_stored_index_mode(check_store, p->project_name, &stored_mode); + if (stored_mode_status == INDEX_MODE_METADATA_ERROR) { + cbm_log_error("pipeline.route", "path", "metadata_read_error", "stored_mode", "error", + "requested_mode", cbm_pipeline_mode_name(p->mode), "action", + "preserve_db"); + cbm_store_close(check_store); + free(db_path); + return CBM_PIPELINE_ABORT_PRESERVE_DB; + } + bool stored_mode_known = stored_mode_status == INDEX_MODE_METADATA_OK; cbm_store_get_file_hashes(check_store, p->project_name, &hashes, &hash_count); cbm_store_free_file_hashes(hashes, hash_count); cbm_store_close(check_store); - if (hash_count > 0 && file_count <= hash_count + (hash_count / PAIR_LEN)) { + bool mode_covered = stored_mode_known && index_mode_covers(stored_mode, p->mode); + if (hash_count > 0 && mode_covered && file_count <= hash_count + (hash_count / PAIR_LEN)) { cbm_log_info("pipeline.route", "path", "incremental", "stored_hashes", - itoa_buf(hash_count)); - int rc = cbm_pipeline_run_incremental(p, db_path, files, file_count); + itoa_buf(hash_count), "stored_mode", cbm_pipeline_mode_name(stored_mode), + "requested_mode", cbm_pipeline_mode_name(p->mode), "effective_mode", + cbm_pipeline_mode_name(stored_mode)); + int rc = cbm_pipeline_run_incremental(p, db_path, files, file_count, stored_mode); free(db_path); return rc; } - if (hash_count > 0) { + if (hash_count > 0 && !mode_covered) { + cbm_log_info("pipeline.route", "path", "mode_upgrade_reindex", "stored_mode", + stored_mode_known ? cbm_pipeline_mode_name(stored_mode) : "unknown", + "requested_mode", cbm_pipeline_mode_name(p->mode)); + } else if (hash_count > 0) { cbm_log_info("pipeline.route", "path", "mode_change_reindex", "stored_hashes", itoa_buf(hash_count), "discovered", itoa_buf(file_count)); } @@ -1223,19 +1324,6 @@ static int64_t stat_mtime_ns(const struct stat *fst) { #endif } -static const char *pipeline_mode_name(cbm_index_mode_t mode) { - switch (mode) { - case CBM_MODE_FULL: - return "full"; - case CBM_MODE_MODERATE: - return "moderate"; - case CBM_MODE_FAST: - return "fast"; - default: - return "unknown"; - } -} - /* Dump graph to SQLite and persist file hashes for incremental indexing. */ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *files, int file_count, struct timespec *t) { @@ -1394,7 +1482,7 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil : (p->ignored_total > p->ignored_count ? "truncated" : "complete"); cbm_coverage_meta_t coverage_meta = { .generation = have_project_info ? project_info.indexed_at : NULL, - .index_mode = pipeline_mode_name(p->mode), + .index_mode = cbm_pipeline_mode_name(p->mode), .recording_status = recording_status, .ignored_files_stored = p->ignored_count, .ignored_files_total = p->ignored_total, @@ -1809,21 +1897,22 @@ static int seal_staging_db(const char *staging_path) { return rc; } -static int export_after_publish(cbm_pipeline_t *p, const char *final_path, bool was_incremental) { - if (p->persistence) { - CBM_PROF_START(t_art); - int rc = cbm_artifact_export(final_path, p->repo_path, p->project_name, CBM_ARTIFACT_BEST); - CBM_PROF_END("persist", "6_artifact_export", t_art); - if (rc != 0) { - const char *err = cbm_artifact_export_last_error(); - cbm_log_error("pipeline.err", "phase", "artifact_export", "err", err ? err : "unknown"); - } - return rc; +static int export_after_publish(cbm_pipeline_t *p, const char *final_path) { + bool persistence_required = p->persistence; + bool refresh_existing = p->repo_path && cbm_artifact_exists(p->repo_path); + if (!persistence_required && !refresh_existing) { + return 0; } - if (was_incremental && p->repo_path && cbm_artifact_exists(p->repo_path)) { - (void)cbm_artifact_export(final_path, p->repo_path, p->project_name, CBM_ARTIFACT_FAST); + + int quality = persistence_required ? CBM_ARTIFACT_BEST : CBM_ARTIFACT_FAST; + CBM_PROF_START(t_art); + int rc = cbm_artifact_export(final_path, p->repo_path, p->project_name, quality); + CBM_PROF_END("persist", "6_artifact_export", t_art); + if (rc != 0) { + const char *err = cbm_artifact_export_last_error(); + cbm_log_error("pipeline.err", "phase", "artifact_export", "err", err ? err : "unknown"); } - return 0; + return persistence_required ? rc : 0; } int cbm_pipeline_run(cbm_pipeline_t *p) { @@ -1903,7 +1992,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { return CBM_NOT_FOUND; } - rc = export_after_publish(p, final_path, was_incremental); + rc = export_after_publish(p, final_path); free(staging_path); free(final_path); return rc; diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 21740ba64..955d50741 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -69,19 +69,6 @@ static int64_t stat_mtime_ns(const struct stat *st) { #endif } -static const char *incr_mode_name(int mode) { - switch (mode) { - case CBM_MODE_FULL: - return "full"; - case CBM_MODE_MODERATE: - return "moderate"; - case CBM_MODE_FAST: - return "fast"; - default: - return "unknown"; - } -} - /* ── File classification ─────────────────────────────────────────── */ /* Classify discovered files against stored hashes using mtime+size. @@ -633,6 +620,7 @@ static void run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_fil itoa_buf((int)elapsed_ms(t))); } } + /* Delete old DB and dump merged graph + hashes to disk. * Mode-skipped hash rows are preserved across the rebuild so subsequent * reindexes can correctly distinguish "never indexed" from "indexed but @@ -707,11 +695,12 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p /* ── Incremental pipeline entry point ────────────────────────────── */ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files, - int file_count) { + int file_count, cbm_index_mode_t effective_mode) { struct timespec t0; cbm_clock_gettime(CLOCK_MONOTONIC, &t0); const char *project = cbm_pipeline_project_name(p); + cbm_index_mode_t requested_mode = (cbm_index_mode_t)cbm_pipeline_get_mode(p); /* Open existing disk DB */ cbm_store_t *store = cbm_store_open_path(db_path); @@ -873,7 +862,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil .registry = registry, .cancelled = cbm_pipeline_cancelled_ptr(p), .pipeline = p, /* so passes can record per-file skips (Track B) */ - .mode = cbm_pipeline_get_mode(p), + .mode = effective_mode, .path_aliases = path_aliases, .excluded_dirs = excluded_dirs, .excluded_count = excluded_count, @@ -897,7 +886,12 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil } } + /* Discovery already used requested_mode. Re-extraction must preserve the + * stronger capabilities recorded in Project.index_mode, including full-only + * C/C++ Macro nodes. Restore the process-wide gate immediately afterwards. */ + cbm_set_macro_extraction(effective_mode == CBM_MODE_FULL); run_extract_resolve(&ctx, changed_files, ci); + cbm_set_macro_extraction(requested_mode == CBM_MODE_FULL); cbm_pipeline_pass_k8s(&ctx, changed_files, ci); run_postpasses(&ctx, changed_files, ci, project); @@ -923,6 +917,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * Rows for deleted files are pruned against file_hashes inside the * replace. Borrowed strings: old_cov and the pipeline own them past the * dump_and_persist call below. */ + int cov_n = 0; cbm_file_error_t *run_errs = NULL; int run_err_count = 0; cbm_pipeline_get_file_errors(p, &run_errs, &run_err_count); @@ -934,7 +929,6 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil int run_ignored_total = 0; cbm_pipeline_get_ignored(p, &run_ignored, &run_ignored_count, &run_ignored_total); cbm_coverage_row_t *cov = NULL; - int cov_n = 0; int cov_cap = old_cov_count + run_err_count + run_excluded_count + run_ignored_count; if (cov_cap > 0) { cov = (cbm_coverage_row_t *)malloc((size_t)cov_cap * sizeof(*cov)); @@ -995,9 +989,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * covers incremental reindexes, not just full ones. */ cbm_pipeline_set_committed_counts(p, cbm_gbuf_node_count(existing), cbm_gbuf_edge_count(existing)); - int index_mode = cbm_pipeline_get_mode(p); cbm_coverage_meta_t coverage_meta = { - .index_mode = incr_mode_name(index_mode), + .index_mode = cbm_pipeline_mode_name((cbm_index_mode_t)cbm_pipeline_get_mode(p)), .recording_status = !coverage_rows_available ? "unavailable" @@ -1013,6 +1006,10 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil free_mode_skipped(mode_skipped, mode_skipped_count); cbm_gbuf_free(existing); + if (persist_rc != 0) { + return persist_rc; + } + cbm_log_info("incremental.done", "elapsed_ms", itoa_buf((int)elapsed_ms(t0))); - return persist_rc; + return 0; } diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 9886d5718..29ce9381f 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -31,6 +31,10 @@ * "no incremental route; continue with a full index" sentinel. */ #define CBM_PIPELINE_ABORT_PRESERVE_DB (-2) +/* Stable internal serialization used by Project metadata, coverage metadata, + * and route diagnostics. */ +const char *cbm_pipeline_mode_name(cbm_index_mode_t mode); + /* Canonicalize route-path parameter placeholders (":id", "{id}", "", * "${...}") to a single "{}" token so that client call sites and server * handlers rendezvous on the same Route QN regardless of framework syntax. @@ -634,9 +638,12 @@ int cbm_scan_project_env_urls_excluded(const char *root_path, cbm_env_binding_t /* Run incremental re-index on an existing disk DB. * Classifies files by mtime+size, deletes changed nodes, re-parses changed - * files, merges into disk DB. Returns 0 on success. */ + * files with effective_mode capabilities, then merges into disk DB. The + * pipeline's requested mode still owns discovery/exclusions. Returns 0 on + * success, or CBM_PIPELINE_ABORT_PRESERVE_DB when the existing coverage + * table is unreadable and the on-disk DB must be left untouched. */ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files, - int file_count); + int file_count, cbm_index_mode_t effective_mode); /* Pipeline accessors for incremental use */ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); diff --git a/src/store/store.h b/src/store/store.h index 55be4a48b..5eb144ab5 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -463,6 +463,8 @@ typedef struct { typedef struct { const char *project; const char *generation; + /* Requested discovery mode of the last run; the effective capability + * mode lives in the Project node's properties.index_mode. */ const char *index_mode; const char *recorded_at; const char *recording_status; diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 6cb2a6150..ec97e0760 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -9,6 +9,7 @@ #include "test_framework.h" #include "test_helpers.h" #include "foundation/mem.h" // cbm_mem_init/budget (back-pressure futile-nap test) +#include "pipeline/artifact.h" #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" #include "store/store.h" @@ -6221,7 +6222,435 @@ TEST(full_reindex_preserves_exact_long_db_path) { PASS(); } #endif +static bool project_has_index_mode(cbm_store_t *store, const char *project, + const char *expected_mode) { + cbm_node_t node = {0}; + if (cbm_store_find_node_by_qn(store, project, project, &node) != CBM_STORE_OK) { + return false; + } + + bool matches = false; + if (node.properties_json) { + yyjson_doc *doc = yyjson_read_opts((char *)node.properties_json, + strlen(node.properties_json), 0, NULL, NULL); + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *mode = root ? yyjson_obj_get(root, "index_mode") : NULL; + matches = yyjson_equals_str(mode, expected_mode); + if (doc) { + yyjson_doc_free(doc); + } + } + + cbm_node_free_fields(&node); + return matches; +} + +static int project_count_nodes_by_label(cbm_store_t *store, const char *project, + const char *label) { + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_label(store, project, label, &nodes, &count) != CBM_STORE_OK) { + return -1; + } + cbm_store_free_nodes(nodes, count); + return count; +} + +static int setup_mode_upgrade_repo(char *tmpdir, size_t tmpdir_size, char *dbpath, + size_t dbpath_size) { + snprintf(tmpdir, tmpdir_size, "/tmp/cbm_mode_upgrade_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) { + return -1; + } + snprintf(dbpath, dbpath_size, "%s/test.db", tmpdir); + + if (th_write_file( + TH_PATH(tmpdir, "user_validator.go"), + "package validation\n" + "import \"errors\"\n" + "import \"strings\"\n" + "func ValidateUser(u User) error {\n" + " if u.Name == \"\" { return errors.New(\"name required\") }\n" + " if len(u.Name) > 100 { return errors.New(\"name too long\") }\n" + " if u.Age < 0 { return errors.New(\"invalid age\") }\n" + " if u.Age > 200 { return errors.New(\"age too high\") }\n" + " if u.Email == \"\" { return errors.New(\"email required\") }\n" + " if !strings.Contains(u.Email, \"@\") { return errors.New(\"invalid email\") }\n" + " if u.Phone == \"\" { return errors.New(\"phone required\") }\n" + " if len(u.Phone) < 7 { return errors.New(\"phone too short\") }\n" + " if u.Country == \"\" { return errors.New(\"country required\") }\n" + " for _, tag := range u.Tags {\n" + " if tag == \"\" { return errors.New(\"empty tag\") }\n" + " }\n" + " return nil\n" + "}\n") != 0) { + return -1; + } + + return th_write_file( + TH_PATH(tmpdir, "order_validator.go"), + "package validation\n" + "import \"errors\"\n" + "import \"strings\"\n" + "func ValidateOrder(o Order) error {\n" + " if o.Title == \"\" { return errors.New(\"title required\") }\n" + " if len(o.Title) > 100 { return errors.New(\"title too long\") }\n" + " if o.Amount < 0 { return errors.New(\"invalid amount\") }\n" + " if o.Amount > 200 { return errors.New(\"amount too high\") }\n" + " if o.Status == \"\" { return errors.New(\"status required\") }\n" + " if !strings.Contains(o.Status, \"@\") { return errors.New(\"invalid status\") }\n" + " if o.Region == \"\" { return errors.New(\"region required\") }\n" + " if len(o.Region) < 7 { return errors.New(\"region too short\") }\n" + " if o.Vendor == \"\" { return errors.New(\"vendor required\") }\n" + " for _, item := range o.Items {\n" + " if item == \"\" { return errors.New(\"empty item\") }\n" + " }\n" + " return nil\n" + "}\n"); +} + +TEST(incremental_mode_upgrade_reindexes_capabilities) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + cbm_pipeline_set_persistence(pipeline, true); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"), 0); + ASSERT_TRUE(project_has_index_mode(store, project, "fast")); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_MODERATE); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + int similarity_edges = cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"); + ASSERT_GT(similarity_edges, 0); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + char artifact_dbpath[512]; + snprintf(artifact_dbpath, sizeof(artifact_dbpath), "%s/artifact-upgrade.db", tmpdir); + ASSERT_EQ(cbm_artifact_import(tmpdir, artifact_dbpath), 0); + cbm_store_t *artifact_store = cbm_store_open_path(artifact_dbpath); + ASSERT_NOT_NULL(artifact_store); + ASSERT_EQ(cbm_store_count_edges_by_type(artifact_store, project, "SIMILAR_TO"), + similarity_edges); + ASSERT_TRUE(project_has_index_mode(artifact_store, project, "moderate")); + cbm_store_close(artifact_store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"), similarity_edges); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + PASS(); +} + +TEST(incremental_mode_downgrade_preserves_similarity_for_changed_file) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_MODERATE); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + int similarity_edges = cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"); + ASSERT_GT(similarity_edges, 0); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + FILE *changed = fopen(TH_PATH(tmpdir, "user_validator.go"), "a"); + ASSERT_NOT_NULL(changed); + ASSERT_GT(fprintf(changed, "\n// changed before fast reindex\n"), 0); + fclose(changed); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"), similarity_edges); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_MODERATE); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"), similarity_edges); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + PASS(); +} + +TEST(incremental_mode_downgrade_preserves_full_extraction_for_changed_file) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + ASSERT_EQ(th_write_file(TH_PATH(tmpdir, "macros.c"), + "#define REVIEW_LIMIT 42\n" + "int review_value(void) { return REVIEW_LIMIT; }\n"), + 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_MODERATE); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(project_count_nodes_by_label(store, project, "Macro"), 0); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + int macro_nodes = project_count_nodes_by_label(store, project, "Macro"); + ASSERT_GT(macro_nodes, 0); + ASSERT_TRUE(project_has_index_mode(store, project, "full")); + cbm_store_close(store); + + FILE *changed = fopen(TH_PATH(tmpdir, "macros.c"), "a"); + ASSERT_NOT_NULL(changed); + ASSERT_GT(fprintf(changed, "\n/* changed before moderate reindex */\n"), 0); + fclose(changed); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_MODERATE); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(project_count_nodes_by_label(store, project, "Macro"), macro_nodes); + ASSERT_TRUE(project_has_index_mode(store, project, "full")); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(project_count_nodes_by_label(store, project, "Macro"), macro_nodes); + ASSERT_TRUE(project_has_index_mode(store, project, "full")); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + PASS(); +} +TEST(incremental_noop_downgrade_honors_explicit_persistence) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + ASSERT_FALSE(cbm_artifact_exists(tmpdir)); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + cbm_pipeline_set_persistence(pipeline, true); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + ASSERT_TRUE(cbm_artifact_exists(tmpdir)); + + th_rmtree(tmpdir); + PASS(); +} + +TEST(incremental_changed_file_propagates_explicit_persistence_failure) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + FILE *changed = fopen(TH_PATH(tmpdir, "user_validator.go"), "a"); + ASSERT_NOT_NULL(changed); + ASSERT_GT(fprintf(changed, "\n// force incremental artifact export\n"), 0); + fclose(changed); + + char artifact_dir[512]; + snprintf(artifact_dir, sizeof(artifact_dir), "%s/.codebase-memory", tmpdir); + cbm_mkdir_p(artifact_dir, 0755); + char artifact_path[512]; + snprintf(artifact_path, sizeof(artifact_path), "%s/graph.db.zst", artifact_dir); + cbm_mkdir_p(artifact_path, 0755); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + cbm_pipeline_set_persistence(pipeline, true); + int rc = cbm_pipeline_run(pipeline); + cbm_pipeline_free(pipeline); + + ASSERT_NEQ(rc, 0); + + th_rmtree(tmpdir); + PASS(); +} + +TEST(incremental_missing_mode_metadata_forces_reindex) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_mode_legacy_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmpdir)); + + char dbpath[512]; + snprintf(dbpath, sizeof(dbpath), "%s/test.db", tmpdir); + ASSERT_EQ(th_write_file(TH_PATH(tmpdir, "main.go"), + "package main\n\nfunc main() { println(\"hello\") }\n"), + 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + cbm_node_t project_node = {0}; + ASSERT_EQ(cbm_store_find_node_by_qn(store, project, project, &project_node), CBM_STORE_OK); + cbm_node_t legacy_project_node = project_node; + legacy_project_node.properties_json = "{}"; + ASSERT_GT(cbm_store_upsert_node(store, &legacy_project_node), 0); + cbm_node_free_fields(&project_node); + + char sentinel_qn[512]; + snprintf(sentinel_qn, sizeof(sentinel_qn), "%s.legacy_sentinel", project); + cbm_node_t sentinel = {.project = project, + .label = "Function", + .name = "legacy_sentinel", + .qualified_name = sentinel_qn, + .file_path = "legacy.go", + .properties_json = "{}"}; + ASSERT_GT(cbm_store_upsert_node(store, &sentinel), 0); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + cbm_node_t sentinel_after = {0}; + ASSERT_EQ(cbm_store_find_node_by_qn(store, project, sentinel_qn, &sentinel_after), + CBM_STORE_NOT_FOUND); + ASSERT_TRUE(project_has_index_mode(store, project, "fast")); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + PASS(); +} + +TEST(incremental_escaped_nul_mode_metadata_forces_reindex) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_mode_nul_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmpdir)); + + char dbpath[512]; + snprintf(dbpath, sizeof(dbpath), "%s/test.db", tmpdir); + ASSERT_EQ(th_write_file(TH_PATH(tmpdir, "main.go"), + "package main\n\nfunc main() { println(\"hello\") }\n"), + 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + + /* Valid JSON whose mode string carries an escaped NUL: yyjson unescapes it + * to "full\0garbage" (12 chars), which a prefix strcmp would accept but an + * exact comparison must reject as unknown mode metadata. */ + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + cbm_node_t corrupt_project = {.project = project, + .label = "Project", + .name = project, + .qualified_name = project, + .file_path = "", + .properties_json = "{\"index_mode\":\"full\\u0000garbage\"}"}; + ASSERT_GT(cbm_store_upsert_node(store, &corrupt_project), 0); + ASSERT_FALSE(project_has_index_mode(store, project, "full")); + + char sentinel_qn[512]; + snprintf(sentinel_qn, sizeof(sentinel_qn), "%s.nul_mode_sentinel", project); + cbm_node_t sentinel = {.project = project, + .label = "Function", + .name = "nul_mode_sentinel", + .qualified_name = sentinel_qn, + .file_path = "nul-mode.go", + .properties_json = "{}"}; + ASSERT_GT(cbm_store_upsert_node(store, &sentinel), 0); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + cbm_node_t sentinel_after = {0}; + ASSERT_EQ(cbm_store_find_node_by_qn(store, project, sentinel_qn, &sentinel_after), + CBM_STORE_NOT_FOUND); + ASSERT_TRUE(project_has_index_mode(store, project, "fast")); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + PASS(); +} TEST(incremental_fast_preserves_mode_skipped_tools_dir) { /* Regression: 2026-04-13. A fast-mode reindex after a full-mode index * was silently destroying every file under FAST_SKIP_DIRS directories @@ -7517,6 +7946,13 @@ SUITE(pipeline) { #ifdef __linux__ RUN_TEST(full_reindex_preserves_exact_long_db_path); #endif + RUN_TEST(incremental_mode_upgrade_reindexes_capabilities); + RUN_TEST(incremental_mode_downgrade_preserves_similarity_for_changed_file); + RUN_TEST(incremental_mode_downgrade_preserves_full_extraction_for_changed_file); + RUN_TEST(incremental_noop_downgrade_honors_explicit_persistence); + RUN_TEST(incremental_changed_file_propagates_explicit_persistence_failure); + RUN_TEST(incremental_missing_mode_metadata_forces_reindex); + RUN_TEST(incremental_escaped_nul_mode_metadata_forces_reindex); RUN_TEST(incremental_fast_preserves_mode_skipped_tools_dir); RUN_TEST(incremental_k8s_manifest_indexed); RUN_TEST(incremental_kustomize_module_indexed); From 4d856e1f3a878835d5ca1a953326930d9739d561 Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Sat, 25 Jul 2026 11:12:43 +0300 Subject: [PATCH 2/2] fix: address persistence review feedback Signed-off-by: Anton Standrik --- src/pipeline/pipeline.c | 7 ++----- tests/test_pipeline.c | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index e0ea30d0f..44b7ce0e7 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1641,11 +1641,10 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, return rc; } -static int cbm_pipeline_run_staged(cbm_pipeline_t *p, bool *was_incremental) { +static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { if (!p) { return CBM_NOT_FOUND; } - *was_incremental = false; CBM_PROF_START(t_pipeline_total); struct timespec t0; @@ -1703,7 +1702,6 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p, bool *was_incremental) { goto cleanup; } if (rc >= 0) { - *was_incremental = true; goto cleanup; } cbm_log_info("pipeline.route", "path", "full"); @@ -1951,8 +1949,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { free(final_path); return CBM_NOT_FOUND; } - bool was_incremental = false; - int rc = cbm_pipeline_run_staged(p, &was_incremental); + int rc = cbm_pipeline_run_staged(p); free(p->db_path); p->db_path = configured_db_path; diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index ec97e0760..d1e08fd95 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -6520,10 +6520,10 @@ TEST(incremental_changed_file_propagates_explicit_persistence_failure) { char artifact_dir[512]; snprintf(artifact_dir, sizeof(artifact_dir), "%s/.codebase-memory", tmpdir); - cbm_mkdir_p(artifact_dir, 0755); + ASSERT_TRUE(cbm_mkdir_p(artifact_dir, 0755)); char artifact_path[512]; snprintf(artifact_path, sizeof(artifact_path), "%s/graph.db.zst", artifact_dir); - cbm_mkdir_p(artifact_path, 0755); + ASSERT_TRUE(cbm_mkdir_p(artifact_path, 0755)); pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); ASSERT_NOT_NULL(pipeline);