diff --git a/README.md b/README.md index ea3c862e3..5520b1cca 100644 --- a/README.md +++ b/README.md @@ -726,6 +726,7 @@ GRAPHIFY_TRIAGE_BACKEND=kimi graphify prs --triage # use a specific backend fo graphify clone https://github.com/karpathy/nanoGPT graphify merge-graphs a.json b.json --out merged.json +graphify merge-graphs a.json b.json --repo-tag api --repo-tag web --out merged.json graphify --version # print installed version graphify watch ./src graphify check-update ./src diff --git a/graphify/__main__.py b/graphify/__main__.py index e620d97a5..62e917404 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -2282,6 +2282,7 @@ def main() -> None: print(" merge-driver git merge driver: union-merge two graph.json files (set up via hook install)") print(" merge-graphs merge two or more graph.json files into one cross-repo graph") print(" --out output path (default: graphify-out/merged-graph.json)") + print(" --repo-tag repository tag for each input, in input order (repeatable)") print(" --branch checkout a specific branch (default: repo default)") print(" --out clone to a custom directory (default: ~/.graphify/repos//)") print(" add fetch a URL and save it to ./raw, then update the graph") @@ -3890,24 +3891,37 @@ def _load_graph(p: str): # graphify merge-graphs graph1.json graph2.json ... --out merged.json args = sys.argv[2:] graph_paths: list[Path] = [] + repo_tags: list[str] = [] out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" i = 0 while i < len(args): if args[i] == "--out" and i + 1 < len(args): out_path = Path(args[i + 1]) i += 2 + elif args[i] == "--repo-tag" and i + 1 < len(args): + repo_tags.append(args[i + 1]) + i += 2 else: graph_paths.append(Path(args[i])) i += 1 if len(graph_paths) < 2: print( - "Usage: graphify merge-graphs [...] [--out merged.json]", + "Usage: graphify merge-graphs [...] " + "[--repo-tag TAG ...] [--out merged.json]", + file=sys.stderr, + ) + sys.exit(1) + if repo_tags and len(repo_tags) != len(graph_paths): + print( + "error: --repo-tag must be repeated once for every input graph", file=sys.stderr, ) sys.exit(1) - import networkx as _nx from networkx.readwrite import json_graph as _jg - from graphify.build import prefix_graph_for_global as _prefix + from graphify.build import ( + compose_repository_graphs as _compose_repository_graphs, + derive_repository_tag as _derive_repository_tag, + ) graphs = [] for gp in graph_paths: if not gp.exists(): @@ -3923,34 +3937,29 @@ def _load_graph(p: str): G = _jg.node_link_graph(data, edges="links") except TypeError: G = _jg.node_link_graph(data) + G.graph["hyperedges"] = data.get( + "hyperedges", G.graph.get("hyperedges", []) + ) graphs.append(G) - # nx.compose requires all graphs to be the same type. When input graphs - # come from different sources (e.g. an AST-only run vs a full LLM run) one - # may be a MultiGraph and another a Graph. Normalise everything to Graph - # (the graphify default) by converting MultiGraphs with nx.Graph(). - def _to_simple(g: "_nx.Graph") -> "_nx.Graph": - # nx.compose requires every graph to be the same type. Inputs may - # disagree on BOTH axes — directed vs undirected, and multi vs simple - # — because per-repo graph.json files are written by different extract - # paths at different times. Normalise everything to a plain undirected - # Graph (the merged cross-repo view is undirected anyway), which covers - # DiGraph / MultiGraph / MultiDiGraph. Without this a directed input - # crashed compose with "All graphs must be directed or undirected" (#1606). - if type(g) is not _nx.Graph: - return _nx.Graph(g) - return g - merged = _nx.Graph() - for G, gp in zip(graphs, graph_paths): - repo_tag = gp.parent.parent.name # graphify-out/../ → repo dir name - prefixed = _to_simple(_prefix(G, repo_tag)) - merged = _nx.compose(merged, prefixed) + if not repo_tags: + repo_tags = [_derive_repository_tag(path) for path in graph_paths] + try: + merged = _compose_repository_graphs(list(zip(repo_tags, graphs))) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) try: out_data = _jg.node_link_data(merged, edges="links") except TypeError: out_data = _jg.node_link_data(merged) + out_data["hyperedges"] = merged.graph.get("hyperedges", []) out_path.parent.mkdir(parents=True, exist_ok=True) out_path.write_text(json.dumps(out_data, indent=2), encoding="utf-8") - print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") + print( + f"Merged {len(graphs)} graphs [{', '.join(repo_tags)}] -> " + f"{merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges, " + f"{len(merged.graph.get('hyperedges', []))} hyperedges" + ) print(f"Written to: {out_path}") elif cmd == "clone": diff --git a/graphify/build.py b/graphify/build.py index 279eccbf4..72dbe1689 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -21,6 +21,7 @@ # before any graph construction happens. # from __future__ import annotations +import copy import json import os import re @@ -925,6 +926,85 @@ def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph: return H +def derive_repository_tag(graph_path: Path) -> str: + """Derive a readable repository tag from a graph path. + + Standard ``/graphify-out/graph.json`` inputs use the repository + directory. Flat JSON inputs use the filename stem instead of the empty tag + produced by ``path.parent.parent.name``. + """ + if graph_path.name == "graph.json" and graph_path.parent.name == "graphify-out": + return graph_path.parent.parent.resolve().name + return graph_path.stem or graph_path.parent.name + + +def _community_identity(value: object) -> str: + """Return a stable, sortable identity for a repository-local community id.""" + try: + return json.dumps(value, ensure_ascii=False, sort_keys=True) + except TypeError: + return repr(value) + + +def compose_repository_graphs(tagged_graphs: list[tuple[str, nx.Graph]]) -> nx.Graph: + """Compose repository graphs without conflating repository-local identity. + + Node and hyperedge ids are namespaced by repository tag. Community ids are + remapped deterministically into one non-overlapping integer range while + preserving each repository's existing partition. + """ + tags = [tag for tag, _graph in tagged_graphs] + invalid_tags = [tag for tag in tags if not tag or "::" in tag] + if invalid_tags: + raise ValueError("repository tags must be non-empty and cannot contain '::'") + duplicate_tags = sorted({tag for tag in tags if tags.count(tag) > 1}) + if duplicate_tags: + raise ValueError( + "duplicate repository tag(s): " + ", ".join(repr(tag) for tag in duplicate_tags) + ) + + community_keys = { + (tag, _community_identity(data["community"])) + for tag, graph in tagged_graphs + for _node, data in graph.nodes(data=True) + if data.get("community") is not None + } + community_ids = { + key: community_id for community_id, key in enumerate(sorted(community_keys)) + } + + merged = nx.Graph() + merged_hyperedges: list = [] + for tag, source_graph in tagged_graphs: + graph = source_graph if type(source_graph) is nx.Graph else nx.Graph(source_graph) + prefixed = prefix_graph_for_global(graph, tag) + + for _node, data in prefixed.nodes(data=True): + local_community = data.get("community") + if local_community is not None: + merged_community = community_ids[(tag, _community_identity(local_community))] + data["community"] = merged_community + if data.get("community_name") == f"Community {local_community}": + data["community_name"] = f"Community {merged_community}" + + for raw_hyperedge in graph.graph.get("hyperedges", []) or []: + hyperedge = copy.deepcopy(raw_hyperedge) + if not isinstance(hyperedge, dict): + merged_hyperedges.append(hyperedge) + continue + _normalize_hyperedge_members(hyperedge) + if hyperedge.get("id") is not None: + hyperedge["id"] = f"{tag}::{hyperedge['id']}" + if isinstance(hyperedge.get("nodes"), list): + hyperedge["nodes"] = [f"{tag}::{node}" for node in hyperedge["nodes"]] + merged_hyperedges.append(hyperedge) + + merged = nx.compose(merged, prefixed) + + merged.graph["hyperedges"] = merged_hyperedges + return merged + + def prune_repo_from_graph(G: nx.Graph, repo_tag: str) -> int: """Remove all nodes tagged with repo_tag from G in-place. Returns count removed.""" to_remove = [n for n, d in G.nodes(data=True) if d.get("repo") == repo_tag] diff --git a/tests/test_merge_graphs_cli.py b/tests/test_merge_graphs_cli.py index f3e0e05d2..96c4af453 100644 --- a/tests/test_merge_graphs_cli.py +++ b/tests/test_merge_graphs_cli.py @@ -28,6 +28,35 @@ def _write(p: Path, directed: bool, multigraph: bool, node_id: str): })) +def _write_repository_graph(p: Path, label: str): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps({ + "directed": True, + "multigraph": False, + "graph": {}, + "nodes": [ + { + "id": "entry", + "label": f"{label} entry", + "community": 0, + "community_name": "Community 0", + }, + { + "id": "worker", + "label": f"{label} worker", + "community": 1, + "community_name": f"{label} workers", + }, + ], + "links": [ + {"source": "entry", "target": "worker", "relation": "calls"}, + ], + "hyperedges": [ + {"id": "request_flow", "nodes": ["entry", "worker"], "relation": "flow"}, + ], + })) + + def test_merge_graphs_mixed_directed_and_multigraph(tmp_path): a = tmp_path / "r1" / "graphify-out" / "graph.json" b = tmp_path / "r2" / "graphify-out" / "graph.json" @@ -43,6 +72,126 @@ def test_merge_graphs_mixed_directed_and_multigraph(tmp_path): data = json.loads(out.read_text()) ids = {n["id"] for n in data["nodes"]} # every input's node survives, normalized into one undirected simple graph - assert {"r1::x", "r2::y", "r3::z"} <= ids or len(ids) == 3 + assert ids == {"r1::x", "r2::y", "r3::z"} assert data.get("directed") is False assert data.get("multigraph") is False + + +def test_merge_graphs_preserves_repository_local_identity(tmp_path): + a = tmp_path / "a.json" + b = tmp_path / "b.json" + _write_repository_graph(a, "A") + _write_repository_graph(b, "B") + out = tmp_path / "merged.json" + + r = _run(["merge-graphs", str(a), str(b), "--out", str(out)], tmp_path) + + assert r.returncode == 0, r.stderr + data = json.loads(out.read_text()) + nodes = {node["id"]: node for node in data["nodes"]} + assert set(nodes) == {"a::entry", "a::worker", "b::entry", "b::worker"} + assert {nodes["a::entry"]["community"], nodes["a::worker"]["community"]}.isdisjoint( + {nodes["b::entry"]["community"], nodes["b::worker"]["community"]} + ) + assert nodes["b::entry"]["community_name"] == ( + f"Community {nodes['b::entry']['community']}" + ) + assert nodes["b::worker"]["community_name"] == "B workers" + assert { + (edge["source"], edge["target"], edge["relation"]) + for edge in data["links"] + } == { + ("a::entry", "a::worker", "calls"), + ("b::entry", "b::worker", "calls"), + } + assert { + hyperedge["id"]: hyperedge["nodes"] for hyperedge in data["hyperedges"] + } == { + "a::request_flow": ["a::entry", "a::worker"], + "b::request_flow": ["b::entry", "b::worker"], + } + assert "4 nodes, 2 edges, 2 hyperedges" in r.stdout + + +def test_merge_graphs_derives_tag_from_relative_canonical_path(tmp_path): + a = tmp_path / "repo-a" / "graphify-out" / "graph.json" + b = tmp_path / "repo-b" / "graphify-out" / "graph.json" + _write_repository_graph(a, "A") + _write_repository_graph(b, "B") + + r = _run([ + "merge-graphs", "graphify-out/graph.json", + "../repo-b/graphify-out/graph.json", "--out", "merged.json", + ], tmp_path / "repo-a") + + assert r.returncode == 0, r.stderr + ids = {node["id"] for node in json.loads((tmp_path / "repo-a/merged.json").read_text())["nodes"]} + assert {"repo-a::entry", "repo-b::entry"} <= ids + + +def test_merge_graphs_community_remap_is_input_order_independent(tmp_path): + a = tmp_path / "a.json" + b = tmp_path / "b.json" + _write_repository_graph(a, "A") + _write_repository_graph(b, "B") + out_ab = tmp_path / "merged-ab.json" + out_ba = tmp_path / "merged-ba.json" + + first = _run(["merge-graphs", str(a), str(b), "--out", str(out_ab)], tmp_path) + second = _run(["merge-graphs", str(b), str(a), "--out", str(out_ba)], tmp_path) + + assert first.returncode == second.returncode == 0 + communities_ab = { + node["id"]: node.get("community") + for node in json.loads(out_ab.read_text())["nodes"] + } + communities_ba = { + node["id"]: node.get("community") + for node in json.loads(out_ba.read_text())["nodes"] + } + assert communities_ab == communities_ba + + +def test_merge_graphs_rejects_duplicate_derived_tags(tmp_path): + a = tmp_path / "owner-a" / "service" / "graphify-out" / "graph.json" + b = tmp_path / "owner-b" / "service" / "graphify-out" / "graph.json" + _write_repository_graph(a, "A") + _write_repository_graph(b, "B") + out = tmp_path / "merged.json" + + r = _run(["merge-graphs", str(a), str(b), "--out", str(out)], tmp_path) + + assert r.returncode != 0 + assert "duplicate repository tag(s): 'service'" in r.stderr + assert not out.exists() + + +def test_merge_graphs_accepts_explicit_repository_tags(tmp_path): + a = tmp_path / "owner-a" / "service" / "graphify-out" / "graph.json" + b = tmp_path / "owner-b" / "service" / "graphify-out" / "graph.json" + _write_repository_graph(a, "A") + _write_repository_graph(b, "B") + out = tmp_path / "merged.json" + + r = _run([ + "merge-graphs", str(a), str(b), + "--repo-tag", "owner-a-service", "--repo-tag", "owner-b-service", + "--out", str(out), + ], tmp_path) + + assert r.returncode == 0, r.stderr + ids = {node["id"] for node in json.loads(out.read_text())["nodes"]} + assert "owner-a-service::entry" in ids + assert "owner-b-service::entry" in ids + + +def test_merge_graphs_requires_one_explicit_tag_per_input(tmp_path): + a = tmp_path / "a.json" + b = tmp_path / "b.json" + _write_repository_graph(a, "A") + _write_repository_graph(b, "B") + + r = _run(["merge-graphs", str(a), str(b), "--repo-tag", "a"], tmp_path) + + assert r.returncode != 0 + assert "--repo-tag must be repeated once for every input graph" in r.stderr