diff --git a/README.md b/README.md index 0a883e75b..d9245a8d2 100644 --- a/README.md +++ b/README.md @@ -716,6 +716,7 @@ graphify extract ./docs --no-cluster # raw extraction only, skip clust graphify extract ./docs --timing # print per-stage wall-clock timings to stderr (also works on cluster-only) graphify extract ./docs --force # overwrite graph.json even if new graph has fewer nodes (use after refactors or to clear ghost duplicates) graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key) +graphify extract ./docs --enable-logging # enable AST-based logging statement extraction graphify extract ./docs --global --as myrepo # extract and register into the cross-project global graph GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # raise output cap for dense corpora @@ -744,8 +745,9 @@ graphify --version # print installed version graphify watch ./src graphify check-update ./src graphify update ./src -graphify update ./src --no-cluster # skip reclustering, write raw AST graph only -graphify update ./src --force # overwrite even if new graph has fewer nodes +graphify update ./src --no-cluster # skip reclustering, write raw AST graph only +graphify update ./src --force # overwrite even if new graph has fewer nodes +graphify update ./src --enable-logging # enable AST-based logging statement extraction graphify cluster-only ./my-project graphify cluster-only ./my-project --graph path/to/graph.json # custom graph location graphify cluster-only ./my-project --max-concurrency 16 --batch-size 200 # parallel community labeling (large graphs) diff --git a/graphify/__main__.py b/graphify/__main__.py index 82e422e0d..d162967ec 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -539,6 +539,8 @@ def _run_cli() -> None: print(" --force overwrite graph.json even if the rebuild has fewer nodes") print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") print(" --no-cluster skip clustering, write raw extraction only") + print(" --enable-logging enable AST-based logging statement extraction") + print(" --logging-config PATH custom logging config YAML path (default: logging_config.yaml)") print(" cluster-only rerun clustering on an existing graph.json and regenerate report") print(" --no-viz skip graph.html generation (useful for >5000 node graphs / CI)") print(" --graph path to graph.json (default /graphify-out/graph.json)") @@ -609,6 +611,8 @@ def _run_cli() -> None: print(" maps tables, views, functions + FK relationships;") print(" column-level detail is not represented in the graph") print(" --cargo extract crate→crate deps from Cargo.toml") + print(" --enable-logging enable AST-based logging statement extraction") + print(" --logging-config PATH custom logging config YAML path (default: logging_config.yaml)") print(" --global also merge the resulting graph into the global graph") print(" --as repo tag for --global (default: target directory name)") print(" global add add/update a project graph in the global graph (~/.graphify/global-graph.json)") diff --git a/graphify/cli.py b/graphify/cli.py index 0bb124777..650d4d733 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1239,21 +1239,33 @@ def dispatch_command(cmd: str) -> None: no_cluster = False args = sys.argv[2:] watch_arg: str | None = None - for a in args: + i = 0 + while i < len(args): + a = args[i] if a == "--force": force = True - continue - if a == "--no-cluster": + i += 1 + elif a == "--no-cluster": no_cluster = True - continue - if a.startswith("-"): + i += 1 + elif a == "--enable-logging": + os.environ["GRAPHIFY_EXTRACT_LOGS"] = "1" + i += 1 + elif a == "--logging-config" and i + 1 < len(args): + os.environ["GRAPHIFY_LOGGING_CONFIG"] = args[i + 1] + i += 2 + elif a.startswith("--logging-config="): + os.environ["GRAPHIFY_LOGGING_CONFIG"] = a.split("=", 1)[1] + i += 1 + elif a.startswith("-"): print(f"error: unknown update option: {a}", file=sys.stderr) sys.exit(2) - if watch_arg is not None: + elif watch_arg is not None: print("error: update accepts at most one path argument", file=sys.stderr) sys.exit(2) - watch_arg = a - + else: + watch_arg = a + i += 1 if watch_arg is not None: watch_path = Path(watch_arg) else: @@ -1998,6 +2010,12 @@ def _parse_float(name: str, raw: str) -> float: code_only = True; i += 1 elif a == "--google-workspace": google_workspace = True; i += 1 + elif a == "--enable-logging": + os.environ["GRAPHIFY_EXTRACT_LOGS"] = "1"; i += 1 + elif a == "--logging-config" and i + 1 < len(args): + os.environ["GRAPHIFY_LOGGING_CONFIG"] = args[i + 1]; i += 2 + elif a.startswith("--logging-config="): + os.environ["GRAPHIFY_LOGGING_CONFIG"] = a.split("=", 1)[1]; i += 1 elif a == "--global": global_merge = True; i += 1 elif a == "--as" and i + 1 < len(args): diff --git a/graphify/extract.py b/graphify/extract.py index a4ac6cbbd..6056d1dea 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -139,6 +139,56 @@ from graphify.extractors.julia import extract_julia # noqa: E402,F401 +from graphify.generic_logger import GenericLogExtractor + +log_extractor = GenericLogExtractor() + +class GraphifyBuilderWrapper: + def __init__(self, result_dict, file_path_str): + self.result = result_dict + self.file_path = file_path_str + + def get_tree_sitter_language(self, lang_key): + configs = { + "java": _JAVA_CONFIG, + "kotlin": _KOTLIN_CONFIG, + "c": _C_CONFIG, + "cpp": _CPP_CONFIG, + } + config = configs.get(lang_key) + if not config: + raise ValueError(f"Unsupported language: {lang_key}") + import importlib + mod = importlib.import_module(config.ts_module) + from tree_sitter import Language + lang_fn = getattr(mod, config.ts_language_fn, None) + if lang_fn is None: + lang_fn = getattr(mod, "language", None) + return Language(lang_fn()) + + def add_edge(self, source, target, relationship, metadata=None): + if not any(n.get("id") == target for n in self.result.get("nodes", [])): + self.result.setdefault("nodes", []).append({ + "id": target, + "label": target, + "file_type": "code", + "type": "log", + "source_file": self.file_path, + "source_location": "L1", + }) + edge = { + "source": source, + "target": target, + "relation": relationship, + "confidence": "EXTRACTED", + "source_file": self.file_path, + "source_location": "L1", + "weight": 1.0, + } + if metadata: + edge["metadata"] = metadata + self.result.setdefault("edges", []).append(edge) + _RECURSION_LIMIT = 10_000 # Language built-in globals that AST may classify as call targets when used as @@ -4090,6 +4140,12 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]: return idx, {"nodes": [], "edges": []} result = _safe_extract_with_xaml_root(extractor, path, root) + try: + content = path.read_text(encoding="utf-8", errors="ignore") + builder = GraphifyBuilderWrapper(result, str(path)) + log_extractor.inject_logs_to_graph(str(path), content, builder) + except Exception as e: + print(f"[LogExtractor Hook Error] {e}", file=sys.stderr, flush=True) # Never cache a zero-node result for an extractable file. Every supported # source produces at least a file node, so an empty node list is anomalous # (e.g. a transient batch/parallel hiccup). Caching it makes the empty @@ -4230,6 +4286,12 @@ def _extract_sequential( bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES # XAML boundary anchors on `root` (the corpus), not the cache location. result = _safe_extract_with_xaml_root(extractor, path, root) + try: + content = path.read_text(encoding="utf-8", errors="ignore") + builder = GraphifyBuilderWrapper(result, str(path)) + log_extractor.inject_logs_to_graph(str(path), content, builder) + except Exception as e: + print(f"[LogExtractor Hook Error] {e}", file=sys.stderr, flush=True) # See _extract_single_file: don't cache an anomalous zero-node result (#1666). if not bypass_cache and "error" not in result and result.get("nodes"): save_cached(path, result, root, cache_root=cache_location) diff --git a/graphify/generic_logger.py b/graphify/generic_logger.py new file mode 100644 index 000000000..776f3d77a --- /dev/null +++ b/graphify/generic_logger.py @@ -0,0 +1,102 @@ +import os +import yaml +from tree_sitter import Parser, Query, QueryCursor + +class GenericLogExtractor: + def __init__(self, config_path=None): + self._config_path = config_path + self.enabled = False + self._loaded = False + self.config = {} + + self.ext_map = { + ".java": "java", + ".kt": "kotlin", + ".c": "c", + ".cpp": "cpp", + ".cc": "cpp", + ".hpp": "cpp", + ".h": "c" + } + + def _ensure_loaded(self): + if self._loaded: + return + + cpath = self._config_path + if cpath is None: + cpath = os.environ.get("GRAPHIFY_LOGGING_CONFIG", "logging_config.yaml") + + is_enabled = os.environ.get("GRAPHIFY_EXTRACT_LOGS") == "1" + self.enabled = is_enabled + + if not self.enabled: + self._loaded = True + return + + if not os.path.exists(cpath): + print(f"[LogExtractor Warning] Configuration file {cpath} not found. Logging extraction disabled.", flush=True) + self.enabled = False + self._loaded = True + return + + try: + with open(cpath, "r") as f: + self.config = yaml.safe_load(f).get("logging_rules", {}) + except Exception as e: + print(f"[LogExtractor Warning] Failed to load configuration from {cpath}: {e}", flush=True) + self.enabled = False + + self._loaded = True + + def inject_logs_to_graph(self, file_path, file_content, graph_builder): + self._ensure_loaded() + if not self.enabled: + return + + _, ext = os.path.splitext(file_path) + lang_key = self.ext_map.get(ext) + if not lang_key or lang_key not in self.config: + return + + rule = self.config[lang_key] + formatted_query = rule["query"].format(pattern=rule["pattern"]) + + try: + ts_language = graph_builder.get_tree_sitter_language(lang_key) + query = Query(ts_language, formatted_query) + + parser = Parser(ts_language) + tree = parser.parse(bytes(file_content, "utf8")) + cursor = QueryCursor(query) + matches = cursor.matches(tree.root_node) + + for _, captures in matches: + func_name_nodes = captures.get("func_name", []) + log_obj_nodes = captures.get("log_obj", []) + log_level_nodes = captures.get("log_level", []) + args_nodes = captures.get("args", []) + + if func_name_nodes and log_obj_nodes and args_nodes: + func_name = func_name_nodes[0].text.decode("utf-8", errors="ignore") + log_obj = log_obj_nodes[0].text.decode("utf-8", errors="ignore") + args = args_nodes[0].text.decode("utf-8", errors="ignore") + + current_function = f"{file_path}::{func_name}" + + if log_level_nodes: + log_level = log_level_nodes[0].text.decode("utf-8", errors="ignore") + log_prefix = f"{log_obj}.{log_level}" + else: + log_prefix = log_obj + + log_signature = f"{log_prefix}{args}" + + graph_builder.add_edge( + source=current_function, + target=log_signature, + relationship="PRINTS_LOG", + metadata={"file": file_path, "type": "EXTRACTED", "lang": lang_key} + ) + except Exception as e: + print(f"[LogExtractor Hook Warning] Skipping AST pass on {file_path}: {e}") diff --git a/logging_config.yaml b/logging_config.yaml new file mode 100644 index 000000000..4b11809bb --- /dev/null +++ b/logging_config.yaml @@ -0,0 +1,65 @@ +logging_rules: + java: + query: | + (method_declaration + name: (identifier) @func_name + body: (block + (expression_statement + (method_invocation + object: (identifier) @log_obj (#match? @log_obj "{pattern}") + name: (identifier) @log_level + arguments: (argument_list) @args + ) + ) + ) + ) + pattern: "^(log|logger)$" + kotlin: + query: | + (function_declaration + (identifier) @func_name + (function_body + (block + (call_expression + (navigation_expression + (identifier) @log_obj (#match? @log_obj "{pattern}") + (identifier) @log_level + ) + (value_arguments) @args + ) + ) + ) + ) + pattern: "^(log|logger)$" + c: + query: | + (function_definition + declarator: (function_declarator + declarator: (identifier) @func_name + ) + body: (compound_statement + (expression_statement + (call_expression + function: (identifier) @log_obj (#match? @log_obj "{pattern}") + arguments: (argument_list) @args + ) + ) + ) + ) + pattern: "^(log_.*|LOG_.*)$" + cpp: + query: | + (function_definition + declarator: (function_declarator + declarator: (identifier) @func_name + ) + body: (compound_statement + (expression_statement + (call_expression + function: (identifier) @log_obj (#match? @log_obj "{pattern}") + arguments: (argument_list) @args + ) + ) + ) + ) + pattern: "^(log_.*|LOG_.*)$" diff --git a/tests/test_generic_logging.py b/tests/test_generic_logging.py new file mode 100644 index 000000000..21cbb50ac --- /dev/null +++ b/tests/test_generic_logging.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import os +from pathlib import Path +import pytest + +import graphify.extract as ex + +def test_generic_logging_extraction(tmp_path, monkeypatch): + # Change working directory to tmp_path so the extractor looks for logging_config.yaml there + monkeypatch.chdir(tmp_path) + + # Write logging_config.yaml + config_content = """ +logging_rules: + java: + query: | + (method_declaration + name: (identifier) @func_name + body: (block + (expression_statement + (method_invocation + object: (identifier) @log_obj (#match? @log_obj "{pattern}") + name: (identifier) @log_level + arguments: (argument_list) @args + ) + ) + ) + ) + pattern: "^(log|logger)$" + kotlin: + query: | + (function_declaration + (identifier) @func_name + (function_body + (block + (call_expression + (navigation_expression + (identifier) @log_obj (#match? @log_obj "{pattern}") + (identifier) @log_level + ) + (value_arguments) @args + ) + ) + ) + ) + pattern: "^(log|logger)$" + c: + query: | + (function_definition + declarator: (function_declarator + declarator: (identifier) @func_name + ) + body: (compound_statement + (expression_statement + (call_expression + function: (identifier) @log_obj (#match? @log_obj "{pattern}") + arguments: (argument_list) @args + ) + ) + ) + ) + pattern: "^(log_.*|LOG_.*)$" + cpp: + query: | + (function_definition + declarator: (function_declarator + declarator: (identifier) @func_name + ) + body: (compound_statement + (expression_statement + (call_expression + function: (identifier) @log_obj (#match? @log_obj "{pattern}") + arguments: (argument_list) @args + ) + ) + ) + ) + pattern: "^(log_.*|LOG_.*)$" +""" + (tmp_path / "logging_config.yaml").write_text(config_content) + + # Create test source files + java_file = tmp_path / "Test.java" + java_file.write_text(""" +class Test { + void doSomething() { + logger.info("Java log message"); + } +} +""") + + kotlin_file = tmp_path / "Test.kt" + kotlin_file.write_text(""" +fun doKotlin() { + logger.warn("Kotlin log message") +} +""") + + c_file = tmp_path / "test.c" + c_file.write_text(""" +void doC() { + log_info("C log message"); +} +""") + + cpp_file = tmp_path / "test.cpp" + cpp_file.write_text(""" +void doCpp() { + LOG_WARN("Cpp log message"); +} +""") + + # First run without any flag/env-var. It should be disabled by default. + from graphify.generic_logger import GenericLogExtractor + monkeypatch.setattr(ex, "log_extractor", GenericLogExtractor("logging_config.yaml")) + + files = [java_file, kotlin_file, c_file, cpp_file] + result_disabled = ex.extract(files, cache_root=tmp_path / "cache_disabled", parallel=False) + + edges_disabled = result_disabled.get("edges", []) + prints_log_edges_disabled = [e for e in edges_disabled if e.get("relation") == "PRINTS_LOG"] + assert len(prints_log_edges_disabled) == 0, "Logging extraction should be disabled by default" + + # Now enable it via environment variable + monkeypatch.setenv("GRAPHIFY_EXTRACT_LOGS", "1") + # Reset internal loaded state to simulate fresh run + ex.log_extractor._loaded = False + + result = ex.extract(files, cache_root=tmp_path / "cache_enabled", parallel=False) + + edges = result.get("edges", []) + nodes = result.get("nodes", []) + + # Assert nodes exist + log_nodes = [n for n in nodes if n.get("type") == "log"] + assert len(log_nodes) >= 4 + + # Assert edges exist + prints_log_edges = [e for e in edges if e.get("relation") == "PRINTS_LOG"] + assert len(prints_log_edges) == 4 + + # Assert details of java log edge + java_edge = next(e for e in prints_log_edges if e.get("metadata", {}).get("lang") == "java") + assert java_edge["source"].endswith("::doSomething") + assert java_edge["target"] == 'logger.info("Java log message")' + + # Assert details of kotlin log edge + kotlin_edge = next(e for e in prints_log_edges if e.get("metadata", {}).get("lang") == "kotlin") + assert kotlin_edge["source"].endswith("::doKotlin") + assert kotlin_edge["target"] == 'logger.warn("Kotlin log message")' + + # Assert details of c log edge + c_edge = next(e for e in prints_log_edges if e.get("metadata", {}).get("lang") == "c") + assert c_edge["source"].endswith("::doC") + assert c_edge["target"] == 'log_info("C log message")' + + # Assert details of cpp log edge + cpp_edge = next(e for e in prints_log_edges if e.get("metadata", {}).get("lang") == "cpp") + assert cpp_edge["source"].endswith("::doCpp") + assert cpp_edge["target"] == 'LOG_WARN("Cpp log message")' diff --git a/workflow.md b/workflow.md new file mode 100644 index 000000000..e5975b39d --- /dev/null +++ b/workflow.md @@ -0,0 +1,131 @@ +1. Executive Summary & Core Rationale +The Problem +Traditional code indexers track where classes and modules live, but they miss runtime context—specifically logging behaviors. In complex, multi-language systems (e.g., Android Framework code running an intersection of Java, Kotlin, C, and C++), debug loops require knowing exactly which function emits which error strings or logs. Relying on an AI agent to continuously grep or re-scan entire files for log invocations wastes context windows, driving up latency and token costs. + +The Solution: Abstract Syntax Tree (AST) Edge Injection +Graphify builds local structural graphs deterministically using Tree-sitter. By intercepting Graphify’s local ingestion loop, we can programmatically scan for log expressions alongside standard call trees. Moving the query patterns to a single logging_config.yaml file decouples development from individual languages, enabling unified multi-language tracking without code rewrite overhead. + +The Complete Agentic Workflow Prompt (With Git Branch Automation) +As an expert senior software engineer and git automation agent, follow this complete, step-by-step workflow to create, implement, modify, and cleanly branch a config-driven logging extraction fork of Graphify. + +### Step 1: Initialize the Working Space & Branching Strategy +1. Clone the Graphify core codebase from 'https://github.com/Graphify-Labs/graphify' into a folder named 'graphify-logging-fork'. +2. Navigate into the folder: `cd graphify-logging-fork` +3. Create and immediately switch to a dedicated feature branch for this modification: + git checkout -b feature/generic-ast-logging-extractor + +### Step 2: Set Up Environment & Dependencies +1. Initialize an isolated virtual environment using `uv venv` or standard `python -m venv .venv`. +2. Activate the environment and install Graphify core dependencies along with the Tree-sitter language libraries required for a multi-language framework scan: + pip install tree-sitter-java tree-sitter-kotlin tree-sitter-c tree-sitter-cpp pyyaml + +### Step 3: Implement the Configuration-Driven Extraction Hook +Create a brand new file named `graphify/generic_logger.py` containing the following decoupled execution engine: + +```python +import os +import yaml +from tree_sitter import Language, Parser + +class GenericLogExtractor: + def __init__(self, config_path="logging_config.yaml"): + self.enabled = os.path.exists(config_path) + if not self.enabled: + return + + with open(config_path, "r") as f: + self.config = yaml.safe_load(f).get("logging_rules", {}) + + self.ext_map = { + ".java": "java", + ".kt": "kotlin", + ".c": "c", + ".cpp": "cpp", + ".cc": "cpp", + ".hpp": "cpp", + ".h": "c" + } + + def inject_logs_to_graph(self, file_path, file_content, graph_builder): + if not self.enabled: + return + + _, ext = os.path.splitext(file_path) + lang_key = self.ext_map.get(ext) + if not lang_key or lang_key not in self.config: + return + + rule = self.config[lang_key] + formatted_query = rule["query"].format(pattern=rule["pattern"]) + + try: + ts_language = graph_builder.get_tree_sitter_language(lang_key) + query = ts_language.query(formatted_query) + + parser = Parser(ts_language) + tree = parser.parse(bytes(file_content, "utf8")) + captures = query.captures(tree.root_node) + + current_function = None + log_data = {} + + for node, tag in captures: + text = node.text.decode('utf-8', errors='ignore') + + if tag == "func_name": + current_function = f"{file_path}::{text}" + log_data = {} + elif tag == "log_obj": + log_data["obj"] = text + elif tag == "log_level": + log_data["level"] = text + elif tag == "args": + log_data["args"] = text + + if current_function: + log_prefix = f"{log_data.get('obj', 'log')}.{log_data.get('level', '')}" if "level" in log_data else log_data.get('obj', 'log') + log_signature = f"{log_prefix}{log_data.get('args', '()')}" + + graph_builder.add_edge( + source=current_function, + target=log_signature, + relationship="PRINTS_LOG", + metadata={"file": file_path, "type": "EXTRACTED", "lang": lang_key} + ) + except Exception as e: + print(f"[LogExtractor Hook Warning] Skipping AST pass on {file_path}: {e}") + + +Step 4: Intercept Graphify Ingestion Loop +Open graphify/extract.py. + +Import GenericLogExtractor at the top of the file: +from graphify.generic_logger import GenericLogExtractor + +Instantiate log_extractor = GenericLogExtractor() right alongside its native parser setups. + +Locate the core file processing block (e.g., process_single_file). Append the following custom hook execution pass straight at the tail end of the function logic: +log_extractor.inject_logs_to_graph(file_path, content, graph_builder) + +Step 5: Validate build and Check-In Code to Branch +Compile and link the package updates to verify there are no hidden syntax formatting or execution bugs: +pip install -e . + +Run git status to verify modified and untracked files are exactly what we expect. + +Stage all new additions and source modifications: +git add . + +Formally commit the workflow changes with a clear, descriptive atomic message: +git commit -m "feat: implement generic config-driven logging extraction rules for mixed-mode codebases" + +## Verification Strategy + +Once your agent finishes running this prompt loop, you can verify that the branch lifecycle completed correctly by checking your local git status. Run this from your terminal inside the root of your newly minted fork repository: + +```bash +# Verify you are safely nested on your custom feature branch +git branch + +# Verify your modifications are completely captured and committed cleanly +git log -n 1 \ No newline at end of file