From 5af45c3e2a19212faf254f8fbab00ae367044e84 Mon Sep 17 00:00:00 2001 From: Joe Esquibel Date: Fri, 28 Aug 2026 09:28:53 -0400 Subject: [PATCH 1/2] fix(#367): read has_credentials from durable hit/risk vectors, not equations #381 fixed the always-zero has_credentials SQLite column by reading file_data["equations"]["sec_hardcoded_secrets"], verified only against a hand-built equations dict. Population-scale validation (#1144: 718 repos, 1.2M files) showed the column still 100% zero -- file_data["equations"] is not a durable carrier, it's rebuilt/emptied/pruned across several galaxyscope.py phases before record_keeper.py reads it. Read the durable carriers instead: hit_vector["sec_hardcoded_secrets"] (raw SecurityLens count) and risk_vector["secrets_risk"] (thresholded score, also spiked to 100 for critical leaks via a path that zeroes the hit_vector -- so both slots are checked). These are the same vectors this recorder already writes as the threat_private_info / risk_secrets_risk columns, which #1144 confirms carry real nonzero data for 437 files in the same dataset. equations kept only as a last-resort fallback. Tests: positive assertion now exercises the durable path (mock's equations dict deliberately has no sec_hardcoded_secrets key); new test_has_credentials_zero_without_signal guards against an always-on read. Closes #367. Co-Authored-By: Claude Sonnet 5 --- gitgalaxy/recorders/record_keeper.py | 34 ++++++++++++++--- tests/tools_recorders/test_record_keeper.py | 41 +++++++++++++++++---- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/gitgalaxy/recorders/record_keeper.py b/gitgalaxy/recorders/record_keeper.py index a04689b5e..8f058447e 100644 --- a/gitgalaxy/recorders/record_keeper.py +++ b/gitgalaxy/recorders/record_keeper.py @@ -492,11 +492,35 @@ def record_mission( # #366: security_auditor.py's real output key is "is_ml_threat", not # "is_malware" -- a near-miss rename that left this column always 0. is_malware = 1 if file_data.get("is_ml_threat", False) else 0 - # #367: no producer ever set file_data["has_credentials"]; the engine's - # real hardcoded-secrets signal is equations["sec_hardcoded_secrets"] - # (security_lens.py, correlated in galaxyscope.py's Active Hemorrhage - # step, #348). - has_creds = 1 if file_data.get("equations", {}).get("sec_hardcoded_secrets", 0) > 0 else 0 + # #367: no producer ever set file_data["has_credentials"]. #381 then + # read equations["sec_hardcoded_secrets"], but file_data["equations"] + # is not a durable carrier -- it's rebuilt/emptied/pruned across + # several galaxyscope.py phases (= {} at ~L1039, per-key del at + # ~L1090, assembled on a separate logic_data object in the worker), + # so that read was still 100% zero at population scale + # (squid-protocol/gitgalaxy#1144). The durable carriers are the same + # hit_vector / risk_vector slots this recorder already writes as + # threat_private_info / risk_secrets_risk: + # - hit_vector["sec_hardcoded_secrets"]: raw SecurityLens count + # - risk_vector["secrets_risk"]: thresholded score, also spiked to + # 100 for critical leaks via a path that zeroes hit_vector + # (signal_processor.py:360), so both slots are checked. + # equations is kept only as a last-resort fallback. + hs_idx = ( + self.SIGNAL_SCHEMA.index("sec_hardcoded_secrets") + if "sec_hardcoded_secrets" in self.SIGNAL_SCHEMA + else -1 + ) + sr_idx = self.RISK_SCHEMA.index("secrets_risk") if "secrets_risk" in self.RISK_SCHEMA else -1 + has_creds = ( + 1 + if ( + (hs_idx >= 0 and len(hv) > hs_idx and hv[hs_idx] > 0) + or (sr_idx >= 0 and len(rv) > sr_idx and rv[sr_idx] > 0) + or file_data.get("equations", {}).get("sec_hardcoded_secrets", 0) > 0 + ) + else 0 + ) # #368: no producer ever set file_data["binary_anomaly"]. The Binary # Analysis Sensor (galaxyscope.py) does exist and does run # SecurityLens.scan_binary() on suspicious binaries, but its findings diff --git a/tests/tools_recorders/test_record_keeper.py b/tests/tools_recorders/test_record_keeper.py index 951ac108e..749831ff5 100644 --- a/tests/tools_recorders/test_record_keeper.py +++ b/tests/tools_recorders/test_record_keeper.py @@ -10,8 +10,8 @@ def keeper(): """Initializes the RecordKeeper with a controlled schema for deterministic testing.""" mock_schemas = { - "RISK_SCHEMA": ["tech_debt", "cognitive_load"], - "SIGNAL_SCHEMA": ["high_risk_execution", "io", "sec_tainted_injection"], + "RISK_SCHEMA": ["tech_debt", "cognitive_load", "secrets_risk"], + "SIGNAL_SCHEMA": ["high_risk_execution", "io", "sec_tainted_injection", "sec_hardcoded_secrets"], } with patch("gitgalaxy.recorders.record_keeper.RECORDING_SCHEMAS", mock_schemas): return RecordKeeper() @@ -58,12 +58,11 @@ def mock_pipeline_state(): }, "is_ml_threat": True, "equations": { - "sec_hardcoded_secrets": 1, # Maps to has_credentials, #367 "sec_extension_mismatch": 1, # Maps to binary_anomaly, #368 "sec_self_propagation": 1, # Maps to obfuscation_flag, #1150 }, - "risk_vector": [80.0, 60.0], # debt, cog_load - "hit_vector": [2, 5, 1], # danger, io, tainted_injection + "risk_vector": [80.0, 60.0, 42.0], # debt, cog_load, secrets_risk (#367) + "hit_vector": [2, 5, 1, 3], # danger, io, tainted_injection, sec_hardcoded_secrets (#367) "classes": [{"name": "APIRouter", "inheritance": ["BaseRouter"], "method_count": 5}], "functions": [ { @@ -181,8 +180,10 @@ def test_record_keeper_data_insertion(keeper, mock_pipeline_state, tmp_path): # #366: is_malware reads file_data["is_ml_threat"] (security_auditor.py's # real output key), not the never-produced "is_malware". assert file_row["is_malware"] == 1 - # #367: has_credentials reads equations["sec_hardcoded_secrets"], not the - # never-produced "has_credentials". + # #367: has_credentials reads the durable hit_vector["sec_hardcoded_secrets"] / + # risk_vector["secrets_risk"] slots (the never-produced "has_credentials" key and + # #381's non-durable equations["sec_hardcoded_secrets"] read were both always 0). + # Note the mock's equations dict deliberately has NO sec_hardcoded_secrets key. assert file_row["has_credentials"] == 1 # #368: binary_anomaly reads equations["sec_extension_mismatch"], not the # never-produced "binary_anomaly". @@ -221,6 +222,32 @@ def test_record_keeper_data_insertion(keeper, mock_pipeline_state, tmp_path): conn.close() +# ============================================================================== +# TEST 2b: #367 -- has_credentials stays 0 when no real secret signal survives +# ============================================================================== +def test_has_credentials_zero_without_signal(keeper, mock_pipeline_state, tmp_path): + """ + #367 guard: with both durable carriers empty (hit_vector["sec_hardcoded_secrets"] + == 0, risk_vector["secrets_risk"] == 0.0) and nothing in the equations dict, the + column must be 0 -- proves the new read isn't just always-on. + """ + db_path = tmp_path / "test_no_creds.sqlite" + parsed, unparsable, summary, session = mock_pipeline_state + + parsed[0]["hit_vector"] = [2, 5, 1, 0] # sec_hardcoded_secrets slot -> 0 + parsed[0]["risk_vector"] = [80.0, 60.0, 0.0] # secrets_risk slot -> 0.0 + parsed[0]["equations"] = {} + + keeper.record_mission(parsed, unparsable, summary, session, str(db_path)) + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute("SELECT * FROM file_data WHERE file_name='router.py'") + assert cursor.fetchone()["has_credentials"] == 0 + conn.close() + + # ============================================================================== # TEST 3: IDEMPOTENCY (THE CASCADE DELETE) # ============================================================================== From 3ca01e44a2372f885ec9e44f2debf4dd1b061d8f Mon Sep 17 00:00:00 2001 From: Joe Esquibel Date: Fri, 28 Aug 2026 09:29:26 -0400 Subject: [PATCH 2/2] chore: regen ruff baseline for record_keeper.py line-shift The #367 fix inserts ~24 lines above the pre-existing W291/RUF005 baseline findings in record_keeper.py; audit_check.py --regenerate confirms all 10 are pure line-shifts (same file/code/message), no new findings. Co-Authored-By: Claude Sonnet 5 --- tests/ruff_audit_baseline.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/ruff_audit_baseline.json b/tests/ruff_audit_baseline.json index 64fe1f941..188ccb062 100644 --- a/tests/ruff_audit_baseline.json +++ b/tests/ruff_audit_baseline.json @@ -33,16 +33,16 @@ "gitgalaxy/recorders/record_keeper.py:291: W291": "Trailing whitespace", "gitgalaxy/recorders/record_keeper.py:292: W291": "Trailing whitespace", "gitgalaxy/recorders/record_keeper.py:293: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:652: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:653: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:656: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:658: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:659: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:704: RUF005": "Consider iterable unpacking instead of concatenation", - "gitgalaxy/recorders/record_keeper.py:773: RUF005": "Consider iterable unpacking instead of concatenation", - "gitgalaxy/recorders/record_keeper.py:810: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:811: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:842: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:676: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:677: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:680: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:682: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:683: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:728: RUF005": "Consider iterable unpacking instead of concatenation", + "gitgalaxy/recorders/record_keeper.py:797: RUF005": "Consider iterable unpacking instead of concatenation", + "gitgalaxy/recorders/record_keeper.py:834: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:835: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:866: W291": "Trailing whitespace", "gitgalaxy/recorders/sbom_recorder.py:221: PERF401": "Use `list.extend` to create a transformed list", "gitgalaxy/security/security_auditor.py:359: RUF046": "Value being cast to `int` is already an integer", "gitgalaxy/security/security_auditor.py:424: PERF203": "`try`-`except` within a loop incurs performance overhead",