Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -716,6 +716,7 @@ graphify extract ./docs --no-cluster # raw extraction only, skip clust
graphify extract ./docs --timing # print per-stage wall-clock timings to stderr (also works on cluster-only)
graphify extract ./docs --force # overwrite graph.json even if new graph has fewer nodes (use after refactors or to clear ghost duplicates)
graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key)
graphify extract ./docs --enable-logging # enable AST-based logging statement extraction
graphify extract ./docs --global --as myrepo # extract and register into the cross-project global graph
GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # raise output cap for dense corpora

Expand Down Expand Up @@ -744,8 +745,9 @@ graphify --version # print installed version
graphify watch ./src
graphify check-update ./src
graphify update ./src
graphify update ./src --no-cluster # skip reclustering, write raw AST graph only
graphify update ./src --force # overwrite even if new graph has fewer nodes
graphify update ./src --no-cluster # skip reclustering, write raw AST graph only
graphify update ./src --force # overwrite even if new graph has fewer nodes
graphify update ./src --enable-logging # enable AST-based logging statement extraction
graphify cluster-only ./my-project
graphify cluster-only ./my-project --graph path/to/graph.json # custom graph location
graphify cluster-only ./my-project --max-concurrency 16 --batch-size 200 # parallel community labeling (large graphs)
Expand Down
4 changes: 4 additions & 0 deletions graphify/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -539,6 +539,8 @@ def _run_cli() -> None:
print(" --force overwrite graph.json even if the rebuild has fewer nodes")
print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)")
print(" --no-cluster skip clustering, write raw extraction only")
print(" --enable-logging enable AST-based logging statement extraction")
print(" --logging-config PATH custom logging config YAML path (default: logging_config.yaml)")
print(" cluster-only <path> rerun clustering on an existing graph.json and regenerate report")
print(" --no-viz skip graph.html generation (useful for >5000 node graphs / CI)")
print(" --graph <path> path to graph.json (default <path>/graphify-out/graph.json)")
Expand Down Expand Up @@ -609,6 +611,8 @@ def _run_cli() -> None:
print(" maps tables, views, functions + FK relationships;")
print(" column-level detail is not represented in the graph")
print(" --cargo extract crate→crate deps from Cargo.toml")
print(" --enable-logging enable AST-based logging statement extraction")
print(" --logging-config PATH custom logging config YAML path (default: logging_config.yaml)")
print(" --global also merge the resulting graph into the global graph")
print(" --as <tag> repo tag for --global (default: target directory name)")
print(" global add <graph.json> add/update a project graph in the global graph (~/.graphify/global-graph.json)")
Expand Down
34 changes: 26 additions & 8 deletions graphify/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -1239,21 +1239,33 @@ def dispatch_command(cmd: str) -> None:
no_cluster = False
args = sys.argv[2:]
watch_arg: str | None = None
for a in args:
i = 0
while i < len(args):
a = args[i]
if a == "--force":
force = True
continue
if a == "--no-cluster":
i += 1
elif a == "--no-cluster":
no_cluster = True
continue
if a.startswith("-"):
i += 1
elif a == "--enable-logging":
os.environ["GRAPHIFY_EXTRACT_LOGS"] = "1"
i += 1
elif a == "--logging-config" and i + 1 < len(args):
os.environ["GRAPHIFY_LOGGING_CONFIG"] = args[i + 1]
i += 2
elif a.startswith("--logging-config="):
os.environ["GRAPHIFY_LOGGING_CONFIG"] = a.split("=", 1)[1]
i += 1
elif a.startswith("-"):
print(f"error: unknown update option: {a}", file=sys.stderr)
sys.exit(2)
if watch_arg is not None:
elif watch_arg is not None:
print("error: update accepts at most one path argument", file=sys.stderr)
sys.exit(2)
watch_arg = a

else:
watch_arg = a
i += 1
if watch_arg is not None:
watch_path = Path(watch_arg)
else:
Expand Down Expand Up @@ -1998,6 +2010,12 @@ def _parse_float(name: str, raw: str) -> float:
code_only = True; i += 1
elif a == "--google-workspace":
google_workspace = True; i += 1
elif a == "--enable-logging":
os.environ["GRAPHIFY_EXTRACT_LOGS"] = "1"; i += 1
elif a == "--logging-config" and i + 1 < len(args):
os.environ["GRAPHIFY_LOGGING_CONFIG"] = args[i + 1]; i += 2
elif a.startswith("--logging-config="):
os.environ["GRAPHIFY_LOGGING_CONFIG"] = a.split("=", 1)[1]; i += 1
elif a == "--global":
global_merge = True; i += 1
elif a == "--as" and i + 1 < len(args):
Expand Down
62 changes: 62 additions & 0 deletions graphify/extract.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,56 @@

from graphify.extractors.julia import extract_julia # noqa: E402,F401

from graphify.generic_logger import GenericLogExtractor

log_extractor = GenericLogExtractor()

class GraphifyBuilderWrapper:
def __init__(self, result_dict, file_path_str):
self.result = result_dict
self.file_path = file_path_str

def get_tree_sitter_language(self, lang_key):
configs = {
"java": _JAVA_CONFIG,
"kotlin": _KOTLIN_CONFIG,
"c": _C_CONFIG,
"cpp": _CPP_CONFIG,
}
config = configs.get(lang_key)
if not config:
raise ValueError(f"Unsupported language: {lang_key}")
import importlib
mod = importlib.import_module(config.ts_module)
from tree_sitter import Language
lang_fn = getattr(mod, config.ts_language_fn, None)
if lang_fn is None:
lang_fn = getattr(mod, "language", None)
return Language(lang_fn())

def add_edge(self, source, target, relationship, metadata=None):
if not any(n.get("id") == target for n in self.result.get("nodes", [])):
self.result.setdefault("nodes", []).append({
"id": target,
"label": target,
"file_type": "code",
"type": "log",
"source_file": self.file_path,
"source_location": "L1",
})
edge = {
"source": source,
"target": target,
"relation": relationship,
"confidence": "EXTRACTED",
"source_file": self.file_path,
"source_location": "L1",
"weight": 1.0,
}
if metadata:
edge["metadata"] = metadata
self.result.setdefault("edges", []).append(edge)

_RECURSION_LIMIT = 10_000

# Language built-in globals that AST may classify as call targets when used as
Expand Down Expand Up @@ -4090,6 +4140,12 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]:
return idx, {"nodes": [], "edges": []}

result = _safe_extract_with_xaml_root(extractor, path, root)
try:
content = path.read_text(encoding="utf-8", errors="ignore")
builder = GraphifyBuilderWrapper(result, str(path))
log_extractor.inject_logs_to_graph(str(path), content, builder)
except Exception as e:
print(f"[LogExtractor Hook Error] {e}", file=sys.stderr, flush=True)
# Never cache a zero-node result for an extractable file. Every supported
# source produces at least a file node, so an empty node list is anomalous
# (e.g. a transient batch/parallel hiccup). Caching it makes the empty
Expand Down Expand Up @@ -4230,6 +4286,12 @@ def _extract_sequential(
bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES
# XAML boundary anchors on `root` (the corpus), not the cache location.
result = _safe_extract_with_xaml_root(extractor, path, root)
try:
content = path.read_text(encoding="utf-8", errors="ignore")
builder = GraphifyBuilderWrapper(result, str(path))
log_extractor.inject_logs_to_graph(str(path), content, builder)
except Exception as e:
print(f"[LogExtractor Hook Error] {e}", file=sys.stderr, flush=True)
# See _extract_single_file: don't cache an anomalous zero-node result (#1666).
if not bypass_cache and "error" not in result and result.get("nodes"):
save_cached(path, result, root, cache_root=cache_location)
Expand Down
102 changes: 102 additions & 0 deletions graphify/generic_logger.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
import os
import yaml
from tree_sitter import Parser, Query, QueryCursor

class GenericLogExtractor:
def __init__(self, config_path=None):
self._config_path = config_path
self.enabled = False
self._loaded = False
self.config = {}

self.ext_map = {
".java": "java",
".kt": "kotlin",
".c": "c",
".cpp": "cpp",
".cc": "cpp",
".hpp": "cpp",
".h": "c"
}

def _ensure_loaded(self):
if self._loaded:
return

cpath = self._config_path
if cpath is None:
cpath = os.environ.get("GRAPHIFY_LOGGING_CONFIG", "logging_config.yaml")

is_enabled = os.environ.get("GRAPHIFY_EXTRACT_LOGS") == "1"
self.enabled = is_enabled

if not self.enabled:
self._loaded = True
return

if not os.path.exists(cpath):
print(f"[LogExtractor Warning] Configuration file {cpath} not found. Logging extraction disabled.", flush=True)
self.enabled = False
self._loaded = True
return

try:
with open(cpath, "r") as f:
self.config = yaml.safe_load(f).get("logging_rules", {})
except Exception as e:
print(f"[LogExtractor Warning] Failed to load configuration from {cpath}: {e}", flush=True)
self.enabled = False

self._loaded = True

def inject_logs_to_graph(self, file_path, file_content, graph_builder):
self._ensure_loaded()
if not self.enabled:
return

_, ext = os.path.splitext(file_path)
lang_key = self.ext_map.get(ext)
if not lang_key or lang_key not in self.config:
return

rule = self.config[lang_key]
formatted_query = rule["query"].format(pattern=rule["pattern"])

try:
ts_language = graph_builder.get_tree_sitter_language(lang_key)
query = Query(ts_language, formatted_query)

parser = Parser(ts_language)
tree = parser.parse(bytes(file_content, "utf8"))
cursor = QueryCursor(query)
matches = cursor.matches(tree.root_node)

for _, captures in matches:
func_name_nodes = captures.get("func_name", [])
log_obj_nodes = captures.get("log_obj", [])
log_level_nodes = captures.get("log_level", [])
args_nodes = captures.get("args", [])

if func_name_nodes and log_obj_nodes and args_nodes:
func_name = func_name_nodes[0].text.decode("utf-8", errors="ignore")
log_obj = log_obj_nodes[0].text.decode("utf-8", errors="ignore")
args = args_nodes[0].text.decode("utf-8", errors="ignore")

current_function = f"{file_path}::{func_name}"

if log_level_nodes:
log_level = log_level_nodes[0].text.decode("utf-8", errors="ignore")
log_prefix = f"{log_obj}.{log_level}"
else:
log_prefix = log_obj

log_signature = f"{log_prefix}{args}"

graph_builder.add_edge(
source=current_function,
target=log_signature,
relationship="PRINTS_LOG",
metadata={"file": file_path, "type": "EXTRACTED", "lang": lang_key}
)
except Exception as e:
print(f"[LogExtractor Hook Warning] Skipping AST pass on {file_path}: {e}")
65 changes: 65 additions & 0 deletions logging_config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
logging_rules:
java:
query: |
(method_declaration
name: (identifier) @func_name
body: (block
(expression_statement
(method_invocation
object: (identifier) @log_obj (#match? @log_obj "{pattern}")
name: (identifier) @log_level
arguments: (argument_list) @args
)
)
)
)
pattern: "^(log|logger)$"
kotlin:
query: |
(function_declaration
(identifier) @func_name
(function_body
(block
(call_expression
(navigation_expression
(identifier) @log_obj (#match? @log_obj "{pattern}")
(identifier) @log_level
)
(value_arguments) @args
)
)
)
)
pattern: "^(log|logger)$"
c:
query: |
(function_definition
declarator: (function_declarator
declarator: (identifier) @func_name
)
body: (compound_statement
(expression_statement
(call_expression
function: (identifier) @log_obj (#match? @log_obj "{pattern}")
arguments: (argument_list) @args
)
)
)
)
pattern: "^(log_.*|LOG_.*)$"
cpp:
query: |
(function_definition
declarator: (function_declarator
declarator: (identifier) @func_name
)
body: (compound_statement
(expression_statement
(call_expression
function: (identifier) @log_obj (#match? @log_obj "{pattern}")
arguments: (argument_list) @args
)
)
)
)
pattern: "^(log_.*|LOG_.*)$"
Loading