diff --git a/app.py b/app.py index 7d12d84..f8b78bf 100644 --- a/app.py +++ b/app.py @@ -83,8 +83,8 @@ def analyze_diff_and_report(payload: dict): head_code = head_files[file_path] if base_code != head_code: # Ingest code and extract function boundary definitions - base_funcs = extract_functions(base_code, "python") - head_funcs = extract_functions(head_code, "python") + base_funcs = extract_functions(base_code, file_path=file_path) + head_funcs = extract_functions(head_code, file_path=file_path) # Calculate exact line changes between base and head base_changed, head_changed = get_changed_line_numbers(base_code, head_code) @@ -146,7 +146,7 @@ def analyze_diff_and_report(payload: dict): if code_bytes: for c_func_name, c_type in changed_entities: if c_type in ("modified", "deleted"): - found_funcs = find_functions_using_symbol(code_bytes, c_func_name) + found_funcs = find_functions_using_symbol(code_bytes, c_func_name, file_path=imp_file) if found_funcs: if imp_file not in at_risk_functions: at_risk_functions[imp_file] = [] @@ -158,7 +158,7 @@ def analyze_diff_and_report(payload: dict): all_api_routes = {} for f_path, h_code in head_files.items(): - routes = extract_api_routes(h_code) + routes = extract_api_routes(h_code, file_path=f_path) for f_name, r_info in routes.items(): all_api_routes[(f_path, f_name)] = r_info @@ -219,7 +219,7 @@ def format_markdown_report( md += "### 🔍 Modified Semantic Entities\n" if not modified_files: - md += "_No Python code modifications detected._\n" + md += "_No supported code modifications detected._\n" else: md += "| File | Entity | Change Type |\n" md += "| --- | --- | --- |\n" @@ -337,8 +337,8 @@ async def api_analyze(req: AnalyzeRequest): else: head_code = head_files[file_path] if base_code != head_code: - base_funcs = extract_functions(base_code, "python") - head_funcs = extract_functions(head_code, "python") + base_funcs = extract_functions(base_code, file_path=file_path) + head_funcs = extract_functions(head_code, file_path=file_path) base_changed, head_changed = get_changed_line_numbers(base_code, head_code) changed_funcs = [] @@ -386,7 +386,7 @@ async def api_analyze(req: AnalyzeRequest): if code_bytes: for c_func_name, c_type in changed_entities: if c_type in ("modified", "deleted"): - found_funcs = find_functions_using_symbol(code_bytes, c_func_name) + found_funcs = find_functions_using_symbol(code_bytes, c_func_name, file_path=imp_file) if found_funcs: if imp_file not in at_risk_functions: at_risk_functions[imp_file] = [] @@ -397,7 +397,7 @@ async def api_analyze(req: AnalyzeRequest): all_api_routes = {} for f_path, h_code in head_files.items(): - routes = extract_api_routes(h_code) + routes = extract_api_routes(h_code, file_path=f_path) for f_name, r_info in routes.items(): all_api_routes[(f_path, f_name)] = r_info diff --git a/cli.py b/cli.py index e842916..b00c43d 100644 --- a/cli.py +++ b/cli.py @@ -8,6 +8,7 @@ from graph_engine import build_dependency_graph, get_impacted_files from parser import extract_functions, find_functions_using_symbol import networkx as nx +from languages import registry def git_list_files(repo_path: str, commit: str) -> list[str]: """ @@ -22,7 +23,7 @@ def git_list_files(repo_path: str, commit: str) -> list[str]: check=True, text=True ) - return [line.strip() for line in result.stdout.splitlines() if line.strip().endswith(".py")] + return [line.strip() for line in result.stdout.splitlines() if registry.is_supported(line.strip())] except subprocess.CalledProcessError as e: print(f"Error listing files for commit {commit}: {e.stderr.strip()}", file=sys.stderr) sys.exit(1) @@ -58,7 +59,7 @@ def git_diff_files(repo_path: str, base: str, head: str) -> list[str]: check=True, text=True ) - return [line.strip() for line in result.stdout.splitlines() if line.strip().endswith(".py")] + return [line.strip() for line in result.stdout.splitlines() if registry.is_supported(line.strip())] except subprocess.CalledProcessError as e: print(f"Error running git diff: {e.stderr.strip()}", file=sys.stderr) sys.exit(1) @@ -87,14 +88,14 @@ def run_analysis(repo_path: str, base: str, head: str) -> str: if not base_code: # File newly added in head - head_funcs = extract_functions(head_code, "python") + head_funcs = extract_functions(head_code, file_path=file_path) changed_funcs = [(f_name, "added") for f_name in head_funcs.keys()] if changed_funcs: modified_files.append(file_path) modified_functions_by_file[file_path] = changed_funcs elif not head_code: # File deleted in head - base_funcs = extract_functions(base_code, "python") + base_funcs = extract_functions(base_code, file_path=file_path) changed_funcs = [(f_name, "deleted") for f_name in base_funcs.keys()] if changed_funcs: modified_files.append(file_path) @@ -102,8 +103,8 @@ def run_analysis(repo_path: str, base: str, head: str) -> str: else: # File modified if base_code != head_code: - base_funcs = extract_functions(base_code, "python") - head_funcs = extract_functions(head_code, "python") + base_funcs = extract_functions(base_code, file_path=file_path) + head_funcs = extract_functions(head_code, file_path=file_path) base_changed, head_changed = get_changed_line_numbers(base_code, head_code) changed_funcs = [] @@ -156,7 +157,7 @@ def run_analysis(repo_path: str, base: str, head: str) -> str: if code_bytes: for c_func_name, c_type in changed_entities: if c_type in ("modified", "deleted"): - found_funcs = find_functions_using_symbol(code_bytes, c_func_name) + found_funcs = find_functions_using_symbol(code_bytes, c_func_name, file_path=imp_file) if found_funcs: if imp_file not in at_risk_functions: at_risk_functions[imp_file] = [] @@ -172,7 +173,7 @@ def run_analysis(repo_path: str, base: str, head: str) -> str: for imp_file, funcs in at_risk_functions.items(): code_bytes = git_show(repo_path, head, imp_file) if code_bytes: - routes = extract_api_routes(code_bytes) + routes = extract_api_routes(code_bytes, file_path=imp_file) for f in funcs: if f in routes: impacted_apis.append({ diff --git a/frontend/App.js b/frontend/App.js index 792d0aa..de5ce69 100644 --- a/frontend/App.js +++ b/frontend/App.js @@ -110,9 +110,9 @@ const Icons = { }; export default function App() { - const [repoUrl, setRepoUrl] = useState('https://github.com/pallets/flask'); - const [base, setBase] = useState('2.3.0'); - const [head, setHead] = useState('3.0.0'); + const [repoUrl, setRepoUrl] = useState(() => localStorage.getItem('diff_guard_repo_url') || 'https://github.com/pallets/flask'); + const [base, setBase] = useState(() => localStorage.getItem('diff_guard_base_ref') || '2.3.0'); + const [head, setHead] = useState(() => localStorage.getItem('diff_guard_head_ref') || '3.0.0'); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const [data, setData] = useState(null); @@ -128,6 +128,35 @@ export default function App() { const containerRef = useRef(null); const cyRef = useRef(null); + const getLanguageGroup = (filePath) => { + const ext = filePath.split('.').pop().toLowerCase(); + if (['py'].includes(ext)) return 'Python'; + if (['js', 'jsx', 'ts', 'tsx'].includes(ext)) return 'JavaScript/TypeScript'; + if (['go'].includes(ext)) return 'Go'; + return 'Other'; + }; + + let groupedModified = {}; + let groupedImpacted = {}; + if (data) { + if (data.modified_files) { + groupedModified = data.modified_files.reduce((acc, item) => { + const lang = getLanguageGroup(item.file); + if (!acc[lang]) acc[lang] = []; + acc[lang].push(item); + return acc; + }, {}); + } + if (data.all_impacted_files) { + groupedImpacted = data.all_impacted_files.reduce((acc, file) => { + const lang = getLanguageGroup(file); + if (!acc[lang]) acc[lang] = []; + acc[lang].push(file); + return acc; + }, {}); + } + } + // Load history from localStorage on mount useEffect(() => { const storedHistory = localStorage.getItem('diff_guard_history'); @@ -140,6 +169,13 @@ export default function App() { } }, []); + // Persist input values to localStorage on change + useEffect(() => { + localStorage.setItem('diff_guard_repo_url', repoUrl); + localStorage.setItem('diff_guard_base_ref', base); + localStorage.setItem('diff_guard_head_ref', head); + }, [repoUrl, base, head]); + const saveToHistory = (url, b, h, result) => { let repoName = url; try { @@ -575,22 +611,41 @@ export default function App() { ${modifiedExpanded && html`
${data.modified_files.length === 0 ? html` -
No Python modifications.
- ` : data.modified_files.map(item => html` -
setSelectedNode({ id: item.file, isModified: true, isImpacted: false })} - > -
- ${item.file} - Modified -
-
- ${item.changed_entities.map(e => html` - ${e.entity} - `)} +
No modifications detected.
+ ` : Object.entries(groupedModified).map(([lang, items]) => html` +
+
+ ${lang} Files + ${items.length}
+ ${items.map(item => html` +
setSelectedNode({ id: item.file, isModified: true, isImpacted: false })} + > +
+ ${item.file} + Modified +
+
+ ${item.changed_entities.map(e => html` + ${e.entity} + `)} +
+
+ `)}
`)}
@@ -772,28 +827,47 @@ export default function App() {
${data.all_impacted_files.length === 0 ? html`
🟢 No downstream modules impacted!
- ` : data.all_impacted_files.map(file => { - const atRiskFuncs = data.at_risk_functions[file] || []; - return html` -
setSelectedNode({ id: file, isModified: false, isImpacted: true })} - > -
- ${file} - Impacted -
- ${atRiskFuncs.length > 0 && html` -
- ${atRiskFuncs.map(f => html` - ${f}() - `)} -
- `} + ` : Object.entries(groupedImpacted).map(([lang, files]) => html` +
+
+ ${lang} Files + ${files.length}
- `; - })} + ${files.map(file => { + const atRiskFuncs = data.at_risk_functions[file] || []; + return html` +
setSelectedNode({ id: file, isModified: false, isImpacted: true })} + > +
+ ${file} + Impacted +
+ ${atRiskFuncs.length > 0 && html` +
+ ${atRiskFuncs.map(f => html` + ${f}() + `)} +
+ `} +
+ `; + })} +
+ `)}
`}
diff --git a/graph_engine.py b/graph_engine.py index a27ff68..8ae4e86 100644 --- a/graph_engine.py +++ b/graph_engine.py @@ -1,6 +1,5 @@ import networkx as nx -import tree_sitter_languages -from parser import get_ast_parser, parse_code +from languages import registry def resolve_relative_import(current_file_path: str, import_from_name: str) -> str: """ @@ -53,51 +52,29 @@ def build_dependency_graph(files_data: dict) -> nx.DiGraph: Edges point from the dependency target (supplier) to the file importing it (consumer). """ graph = nx.DiGraph() - module_to_file = {} - # 1. Register file paths as nodes and build module map + # 1. Register supported files as nodes for file_path in files_data.keys(): - graph.add_node(file_path) - - normalized = file_path.replace("\\", "/") - if normalized.endswith(".py"): - normalized = normalized[:-3] - if normalized.endswith("/__init__"): - normalized = normalized[:-9] - - module_name = normalized.replace("/", ".") - # Strip leading dots or directory indicators - if module_name.startswith("."): - module_name = module_name.lstrip(".") - - module_to_file[module_name] = file_path + if registry.is_supported(file_path): + graph.add_node(file_path) - # 2. Extract and resolve imports - parser = get_ast_parser("python") - import_query_str = """ - (import_statement name: (dotted_name) @import_name) - (import_from_statement module_name: [(dotted_name) (relative_import)] @import_from_name) - """ - lang = tree_sitter_languages.get_language("python") - query = lang.query(import_query_str) - + # 2. Extract and resolve imports using registered language providers for file_path, code_bytes in files_data.items(): - root = parse_code(code_bytes, parser) - captures = query.captures(root) - - for node, capture_name in captures: - raw_import = code_bytes[node.start_byte:node.end_byte].decode("utf-8") + if not registry.is_supported(file_path): + continue + + provider = registry.get_provider_for_file(file_path) + if not provider: + continue - if capture_name == "import_from_name" and raw_import.startswith("."): - resolved_module = resolve_relative_import(file_path, raw_import) - else: - resolved_module = raw_import - - dep_file = find_matching_file(resolved_module, module_to_file) - if dep_file and dep_file != file_path: - # Add edge from supplier -> consumer - graph.add_edge(dep_file, file_path) - + raw_imports = provider.extract_imports(code_bytes, file_path) + for imp in raw_imports: + dep_files = provider.resolve_import_to_file(imp, file_path, files_data) + for dep_file in dep_files: + if dep_file and dep_file != file_path: + # Add edge from supplier -> consumer + graph.add_edge(dep_file, file_path) + return graph def get_impacted_files(graph: nx.DiGraph, changed_file: str) -> set: diff --git a/languages/__init__.py b/languages/__init__.py new file mode 100644 index 0000000..a862af8 --- /dev/null +++ b/languages/__init__.py @@ -0,0 +1,29 @@ +import os +from languages.python import PythonLanguageProvider +from languages.javascript import JavaScriptLanguageProvider +from languages.go import GoLanguageProvider + +class LanguageRegistry: + def __init__(self): + self.providers = [ + PythonLanguageProvider(), + JavaScriptLanguageProvider(), + GoLanguageProvider(), + ] + self.extension_map = {} + for provider in self.providers: + for ext in provider.extensions: + self.extension_map[ext] = provider + + def get_provider_for_file(self, file_path: str): + """Finds the correct LanguageProvider based on the file extension.""" + _, ext = os.path.splitext(file_path.lower()) + return self.extension_map.get(ext) + + def is_supported(self, file_path: str) -> bool: + """Checks if Diff-Guard can parse this file type.""" + _, ext = os.path.splitext(file_path.lower()) + return ext in self.extension_map + +# Export a global instance +registry = LanguageRegistry() diff --git a/languages/base.py b/languages/base.py new file mode 100644 index 0000000..ab14dd1 --- /dev/null +++ b/languages/base.py @@ -0,0 +1,66 @@ +from abc import ABC, abstractmethod +from tree_sitter import Parser, Node +import tree_sitter_languages + +class LanguageProvider(ABC): + + @property + @abstractmethod + def extensions(self) -> list[str]: + """Returns the list of file extensions supported by this language (e.g. ['.py']).""" + pass + + @property + @abstractmethod + def language_name(self) -> str: + """Returns the tree-sitter language identifier (e.g. 'python', 'javascript').""" + pass + + def get_parser(self, file_path: str = None) -> Parser: + """Instantiates and returns the Tree-sitter parser for this language.""" + return tree_sitter_languages.get_parser(self.language_name) + + def parse_code(self, code_bytes: bytes, file_path: str = None) -> Node: + """Parses raw bytes of a file and returns the root node of the AST.""" + parser = self.get_parser(file_path) + tree = parser.parse(code_bytes) + return tree.root_node + + @abstractmethod + def extract_functions(self, code_bytes: bytes, file_path: str = None) -> dict[str, tuple[int, int]]: + """ + Parses file and returns a dict mapping: + function_name -> (start_line_1_indexed, end_line_1_indexed) + """ + pass + + @abstractmethod + def find_functions_using_symbol(self, code_bytes: bytes, symbol: str, file_path: str = None) -> list[str]: + """ + Scans a file's AST for function definitions that contain the specified symbol name + inside their body text. Returns a list of function names that are at risk. + """ + pass + + @abstractmethod + def extract_imports(self, code_bytes: bytes, file_path: str = None) -> list[str]: + """ + Extracts raw import statements / dependency targets from the file. + """ + pass + + @abstractmethod + def resolve_import_to_file(self, imported_ref: str, current_file: str, files_data: dict[str, bytes]) -> list[str]: + """ + Resolves a raw import string to actual file paths in the workspace. + Returns a list of matching file paths (since a package import in Go can map to multiple files). + """ + pass + + @abstractmethod + def extract_api_routes(self, code_bytes: bytes, file_path: str = None) -> dict[str, dict]: + """ + Parses code to find functions decorated or defined as HTTP route entrypoints. + Returns a dict mapping function name to {"method": "HTTP_METHOD", "path": "/url/path"}. + """ + pass diff --git a/languages/go.py b/languages/go.py new file mode 100644 index 0000000..020c8df --- /dev/null +++ b/languages/go.py @@ -0,0 +1,137 @@ +import os +import tree_sitter_languages +from languages.base import LanguageProvider + +class GoLanguageProvider(LanguageProvider): + + @property + def extensions(self) -> list[str]: + return [".go"] + + @property + def language_name(self) -> str: + return "go" + + def extract_functions(self, code_bytes: bytes, file_path: str = None) -> dict[str, tuple[int, int]]: + root = self.parse_code(code_bytes, file_path) + lang = tree_sitter_languages.get_language(self.language_name) + + query_str = """ + (function_declaration name: (identifier) @function_name) + (method_declaration name: (field_identifier) @function_name) + """ + query = lang.query(query_str) + captures = query.captures(root) + + functions = {} + for node, capture_name in captures: + if capture_name == "function_name": + func_name = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") + func_def_node = node.parent + if func_def_node and func_def_node.type in ("function_declaration", "method_declaration"): + start_line = func_def_node.start_point[0] + 1 + end_line = func_def_node.end_point[0] + 1 + functions[func_name] = (start_line, end_line) + + return functions + + def find_functions_using_symbol(self, code_bytes: bytes, symbol: str, file_path: str = None) -> list[str]: + root = self.parse_code(code_bytes, file_path) + lang = tree_sitter_languages.get_language(self.language_name) + + query_str = """ + (function_declaration name: (identifier) @func_name) + (method_declaration name: (field_identifier) @func_name) + """ + query = lang.query(query_str) + captures = query.captures(root) + + at_risk = [] + for node, capture_name in captures: + if capture_name == "func_name": + func_name = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") + func_def_node = node.parent + if func_def_node and func_def_node.type in ("function_declaration", "method_declaration"): + func_text = code_bytes[func_def_node.start_byte:func_def_node.end_byte].decode("utf-8", errors="ignore") + if symbol in func_text: + at_risk.append(func_name) + + return list(dict.fromkeys(at_risk)) + + def extract_imports(self, code_bytes: bytes, file_path: str = None) -> list[str]: + root = self.parse_code(code_bytes, file_path) + lang = tree_sitter_languages.get_language(self.language_name) + + query_str = "(import_spec path: _ @import_path)" + query = lang.query(query_str) + captures = query.captures(root) + + imports = [] + for node, _ in captures: + raw_import = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore").strip("\"'`") + imports.append(raw_import) + + return imports + + def resolve_import_to_file(self, imported_ref: str, current_file: str, files_data: dict[str, bytes]) -> list[str]: + # Try to find module path from go.mod + module_name = "" + go_mod_path = next((k for k in files_data.keys() if os.path.basename(k) == "go.mod"), None) + if go_mod_path: + content = files_data[go_mod_path].decode("utf-8", errors="ignore") + for line in content.splitlines(): + if line.strip().startswith("module "): + parts = line.strip().split() + if len(parts) > 1: + module_name = parts[1] + break + + suffix = "" + if module_name and imported_ref.startswith(module_name + "/"): + suffix = imported_ref[len(module_name) + 1:] + elif module_name and imported_ref == module_name: + suffix = "." # Root module level + else: + # Fallback if no module name matches, treat last segment as potential package dir name + suffix = imported_ref.split("/")[-1] + + matching_files = [] + for f in files_data.keys(): + if f.endswith(".go"): + dir_path = os.path.dirname(f.replace("\\", "/")) + # Match suffix exactly, or if suffix is package directory + if dir_path == suffix or dir_path.endswith("/" + suffix) or (suffix == "." and dir_path == ""): + matching_files.append(f) + + return matching_files + + def extract_api_routes(self, code_bytes: bytes, file_path: str = None) -> dict[str, dict]: + root = self.parse_code(code_bytes, file_path) + lang = tree_sitter_languages.get_language(self.language_name) + + query_str = """ + (call_expression + function: (selector_expression + field: (field_identifier) @method) + arguments: (argument_list [(interpreted_string_literal) (raw_string_literal)] @path (identifier) @func_name)) + """ + query = lang.query(query_str) + captures = query.captures(root) + + routes = {} + current_method = None + current_path = None + + for node, capture_name in captures: + text = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") + if capture_name == "method": + current_method = text.upper() + elif capture_name == "path": + current_path = text.strip("\"'`") + elif capture_name == "func_name": + if current_method in ("GET", "POST", "PUT", "DELETE", "PATCH") and current_path: + routes[text] = {"method": current_method, "path": current_path} + current_method = None + current_path = None + + return routes diff --git a/languages/javascript.py b/languages/javascript.py new file mode 100644 index 0000000..03b2d1e --- /dev/null +++ b/languages/javascript.py @@ -0,0 +1,191 @@ +import os +import tree_sitter_languages +from tree_sitter import Parser +from languages.base import LanguageProvider + +class JavaScriptLanguageProvider(LanguageProvider): + + @property + def extensions(self) -> list[str]: + return [".js", ".jsx", ".ts", ".tsx"] + + @property + def language_name(self) -> str: + return "javascript" + + def get_parser(self, file_path: str = None) -> Parser: + if file_path: + _, ext = os.path.splitext(file_path.lower()) + if ext == ".ts": + return tree_sitter_languages.get_parser("typescript") + elif ext == ".tsx": + return tree_sitter_languages.get_parser("tsx") + return tree_sitter_languages.get_parser("javascript") + + def get_tree_sitter_language(self, file_path: str = None): + if file_path: + _, ext = os.path.splitext(file_path.lower()) + if ext == ".ts": + return tree_sitter_languages.get_language("typescript") + elif ext == ".tsx": + return tree_sitter_languages.get_language("tsx") + return tree_sitter_languages.get_language("javascript") + + def extract_functions(self, code_bytes: bytes, file_path: str = None) -> dict[str, tuple[int, int]]: + root = self.parse_code(code_bytes, file_path) + lang = self.get_tree_sitter_language(file_path) + + query_str = """ + (function_declaration name: (identifier) @function_name) + (method_definition name: (property_identifier) @function_name) + (variable_declarator name: (identifier) @function_name value: [(arrow_function) (function)]) + """ + query = lang.query(query_str) + captures = query.captures(root) + + functions = {} + for node, capture_name in captures: + if capture_name == "function_name": + func_name = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") + + # Resolve the node range + func_def_node = node.parent + if func_def_node: + if func_def_node.parent and func_def_node.parent.type in ("variable_declaration", "lexical_declaration"): + func_def_node = func_def_node.parent + + start_line = func_def_node.start_point[0] + 1 + end_line = func_def_node.end_point[0] + 1 + functions[func_name] = (start_line, end_line) + + return functions + + def find_functions_using_symbol(self, code_bytes: bytes, symbol: str, file_path: str = None) -> list[str]: + root = self.parse_code(code_bytes, file_path) + lang = self.get_tree_sitter_language(file_path) + + query_str = """ + (function_declaration name: (identifier) @func_name) + (method_definition name: (property_identifier) @func_name) + (variable_declarator name: (identifier) @func_name value: [(arrow_function) (function)]) + """ + query = lang.query(query_str) + captures = query.captures(root) + + at_risk = [] + for node, capture_name in captures: + if capture_name == "func_name": + func_name = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") + + func_def_node = node.parent + if func_def_node: + if func_def_node.parent and func_def_node.parent.type in ("variable_declaration", "lexical_declaration"): + func_def_node = func_def_node.parent + + func_text = code_bytes[func_def_node.start_byte:func_def_node.end_byte].decode("utf-8", errors="ignore") + if symbol in func_text: + at_risk.append(func_name) + + return list(dict.fromkeys(at_risk)) + + def extract_imports(self, code_bytes: bytes, file_path: str = None) -> list[str]: + root = self.parse_code(code_bytes, file_path) + lang = self.get_tree_sitter_language(file_path) + + query_str = """ + (import_statement source: (string) @import_path) + (export_statement source: (string) @import_path) + (call_expression + function: [(identifier) (import)] @require_or_import + arguments: (arguments (string) @import_path)) + """ + query = lang.query(query_str) + captures = query.captures(root) + + imports = [] + for node, capture_name in captures: + if capture_name == "import_path": + # Validate if require_name/import check is needed + parent = node.parent + is_valid = False + while parent: + if parent.type in ("import_statement", "export_statement"): + is_valid = True + break + if parent.type == "call_expression": + func_node = parent.child_by_field_name("function") + if func_node: + if func_node.type == "import": + is_valid = True + elif func_node.type == "identifier": + func_name = code_bytes[func_node.start_byte:func_node.end_byte].decode("utf-8", errors="ignore") + if func_name == "require": + is_valid = True + break + parent = parent.parent + + if is_valid: + path_str = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore").strip("\"'") + imports.append(path_str) + + return imports + + def resolve_import_to_file(self, imported_ref: str, current_file: str, files_data: dict[str, bytes]) -> list[str]: + if not (imported_ref.startswith("./") or imported_ref.startswith("../")): + return [] # Ignore absolute / library imports + + current_dir = os.path.dirname(current_file.replace("\\", "/")) + raw_target = os.path.normpath(os.path.join(current_dir, imported_ref)).replace("\\", "/") + + # Candidate extensions and index structures + candidates = [ + raw_target, + raw_target + ".js", + raw_target + ".ts", + raw_target + ".jsx", + raw_target + ".tsx", + os.path.join(raw_target, "index.js").replace("\\", "/"), + os.path.join(raw_target, "index.ts").replace("\\", "/"), + os.path.join(raw_target, "index.jsx").replace("\\", "/"), + os.path.join(raw_target, "index.tsx").replace("\\", "/"), + ] + + # Build normalized files set for quick lookup + normalized_files = {f.replace("\\", "/"): f for f in files_data.keys()} + + for candidate in candidates: + if candidate in normalized_files: + return [normalized_files[candidate]] + + return [] + + def extract_api_routes(self, code_bytes: bytes, file_path: str = None) -> dict[str, dict]: + root = self.parse_code(code_bytes, file_path) + lang = self.get_tree_sitter_language(file_path) + + query_str = """ + (call_expression + function: (member_expression + property: (property_identifier) @method) + arguments: (arguments (string) @path (identifier) @func_name)) + """ + query = lang.query(query_str) + captures = query.captures(root) + + routes = {} + current_method = None + current_path = None + + for node, capture_name in captures: + text = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") + if capture_name == "method": + current_method = text.upper() + elif capture_name == "path": + current_path = text.strip("\"'") + elif capture_name == "func_name": + if current_method in ("GET", "POST", "PUT", "DELETE", "PATCH") and current_path: + routes[text] = {"method": current_method, "path": current_path} + current_method = None + current_path = None + + return routes diff --git a/languages/python.py b/languages/python.py new file mode 100644 index 0000000..cf36d9f --- /dev/null +++ b/languages/python.py @@ -0,0 +1,128 @@ +import os +import tree_sitter_languages +from languages.base import LanguageProvider + +class PythonLanguageProvider(LanguageProvider): + + @property + def extensions(self) -> list[str]: + return [".py"] + + @property + def language_name(self) -> str: + return "python" + + def extract_functions(self, code_bytes: bytes, file_path: str = None) -> dict[str, tuple[int, int]]: + root = self.parse_code(code_bytes, file_path) + lang = tree_sitter_languages.get_language(self.language_name) + + query_str = "(function_definition name: (identifier) @function_name)" + query = lang.query(query_str) + captures = query.captures(root) + + functions = {} + for node, capture_name in captures: + if capture_name == "function_name": + func_name = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") + func_def_node = node.parent + if func_def_node and func_def_node.type == "function_definition": + start_line = func_def_node.start_point[0] + 1 + end_line = func_def_node.end_point[0] + 1 + functions[func_name] = (start_line, end_line) + + return functions + + def find_functions_using_symbol(self, code_bytes: bytes, symbol: str, file_path: str = None) -> list[str]: + root = self.parse_code(code_bytes, file_path) + lang = tree_sitter_languages.get_language(self.language_name) + + query_str = "(function_definition name: (identifier) @func_name)" + query = lang.query(query_str) + captures = query.captures(root) + + at_risk = [] + for node, capture_name in captures: + if capture_name == "func_name": + func_name = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") + func_def_node = node.parent + if func_def_node and func_def_node.type == "function_definition": + func_text = code_bytes[func_def_node.start_byte:func_def_node.end_byte].decode("utf-8", errors="ignore") + if symbol in func_text: + at_risk.append(func_name) + + return list(dict.fromkeys(at_risk)) + + def extract_imports(self, code_bytes: bytes, file_path: str = None) -> list[str]: + root = self.parse_code(code_bytes, file_path) + lang = tree_sitter_languages.get_language(self.language_name) + + query_str = """ + (import_statement name: (dotted_name) @import_name) + (import_from_statement module_name: [(dotted_name) (relative_import)] @import_from_name) + """ + query = lang.query(query_str) + captures = query.captures(root) + + imports = [] + for node, _ in captures: + raw_import = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") + imports.append(raw_import) + + return imports + + def resolve_import_to_file(self, imported_ref: str, current_file: str, files_data: dict[str, bytes]) -> list[str]: + from graph_engine import resolve_relative_import, find_matching_file + + # Build Python module map + module_to_file = {} + for f in files_data.keys(): + if f.endswith(".py"): + normalized = f.replace("\\", "/")[:-3] + if normalized.endswith("/__init__"): + normalized = normalized[:-9] + mod = normalized.replace("/", ".") + if mod.startswith("."): + mod = mod.lstrip(".") + module_to_file[mod] = f + + resolved_module = resolve_relative_import(current_file, imported_ref) + dep_file = find_matching_file(resolved_module, module_to_file) + if dep_file: + return [dep_file] + return [] + + def extract_api_routes(self, code_bytes: bytes, file_path: str = None) -> dict[str, dict]: + root = self.parse_code(code_bytes, file_path) + lang = tree_sitter_languages.get_language(self.language_name) + + query_str = """ + (decorated_definition + (decorator + (call + function: (attribute + attribute: (identifier) @method) + arguments: (argument_list + (string) @path))) + (function_definition + name: (identifier) @func_name)) + """ + query = lang.query(query_str) + captures = query.captures(root) + + routes = {} + current_method = None + current_path = None + + for node, capture_name in captures: + text = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") + if capture_name == "method": + current_method = text.upper() + elif capture_name == "path": + current_path = text.strip("\"'") + elif capture_name == "func_name": + if current_method and current_path: + routes[text] = {"method": current_method, "path": current_path} + current_method = None + current_path = None + + return routes diff --git a/parser.py b/parser.py index 1138268..5127f91 100644 --- a/parser.py +++ b/parser.py @@ -1,5 +1,6 @@ import tree_sitter_languages from tree_sitter import Parser, Node +from languages import registry def get_ast_parser(language_name: str = "python") -> Parser: """ @@ -14,117 +15,42 @@ def parse_code(code_bytes: bytes, parser: Parser) -> Node: tree = parser.parse(code_bytes) return tree.root_node -def extract_functions(code_bytes: bytes, language_name: str = "python") -> dict: +def get_provider(language_name: str = None, file_path: str = None): + """ + Retrieves the corresponding LanguageProvider based on file path or language name. + Falls back to the first registered provider (Python) if none match. + """ + if file_path: + provider = registry.get_provider_for_file(file_path) + if provider: + return provider + if language_name: + for p in registry.providers: + if p.language_name == language_name or language_name in p.extensions: + return p + # Default fallback to Python provider + return registry.providers[0] + +def extract_functions(code_bytes: bytes, language_name: str = "python", file_path: str = None) -> dict: """ Parses code and extracts function names with their 1-indexed start and end line ranges. Returns a dict mapping function name -> (start_line, end_line). """ - parser = get_ast_parser(language_name) - root = parse_code(code_bytes, parser) - - lang = tree_sitter_languages.get_language(language_name) - - # Python-specific query to find function definitions - if language_name == "python": - query_str = "(function_definition name: (identifier) @function_name)" - else: - # Fallback / placeholder for other languages if needed - query_str = "(function_definition name: (identifier) @function_name)" - - query = lang.query(query_str) - captures = query.captures(root) - functions = {} - - for node, capture_name in captures: - if capture_name == "function_name": - # Extract function name from raw bytes - func_name = code_bytes[node.start_byte:node.end_byte].decode("utf-8") - - # The function definition block is the parent node of the identifier name - func_def_node = node.parent - if func_def_node and func_def_node.type == "function_definition": - start_line = func_def_node.start_point[0] + 1 - end_line = func_def_node.end_point[0] + 1 - functions[func_name] = (start_line, end_line) - - return functions + provider = get_provider(language_name, file_path) + return provider.extract_functions(code_bytes, file_path) -def find_functions_using_symbol(code_bytes: bytes, symbol: str, language_name: str = "python") -> list: +def find_functions_using_symbol(code_bytes: bytes, symbol: str, language_name: str = "python", file_path: str = None) -> list: """ Scans a file's AST for function definitions that contain the specified symbol name inside their body text. Returns a list of function names that are at risk. """ - parser = get_ast_parser(language_name) - root = parse_code(code_bytes, parser) - lang = tree_sitter_languages.get_language(language_name) - - if language_name == "python": - query_str = "(function_definition name: (identifier) @func_name)" - else: - query_str = "(function_definition name: (identifier) @func_name)" - - query = lang.query(query_str) - captures = query.captures(root) - at_risk = [] - - for node, capture_name in captures: - if capture_name == "func_name": - func_name = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") - # Get the full text of the function definition - func_def_node = node.parent - if func_def_node and func_def_node.type == "function_definition": - func_text = code_bytes[func_def_node.start_byte:func_def_node.end_byte].decode("utf-8", errors="ignore") - # Simple string matching inside the function body - if symbol in func_text: - at_risk.append(func_name) - - # Maintain insertion order but remove duplicates - return list(dict.fromkeys(at_risk)) + provider = get_provider(language_name, file_path) + return provider.find_functions_using_symbol(code_bytes, symbol, file_path) -def extract_api_routes(code_bytes: bytes, language_name: str = "python") -> dict: +def extract_api_routes(code_bytes: bytes, language_name: str = "python", file_path: str = None) -> dict: """ - Parses code to find functions decorated with HTTP route decorators. + Parses code to find functions decorated or defined as HTTP route entrypoints. Returns a dict mapping function name to {"method": "HTTP_METHOD", "path": "/url/path"}. """ - routes = {} - if language_name != "python": - return routes - - parser = get_ast_parser(language_name) - root = parse_code(code_bytes, parser) - lang = tree_sitter_languages.get_language(language_name) - - query_str = """ - (decorated_definition - (decorator - (call - function: (attribute - attribute: (identifier) @method) - arguments: (argument_list - (string) @path))) - (function_definition - name: (identifier) @func_name)) - """ - - query = lang.query(query_str) - captures = query.captures(root) - - # Process captures which are returned sequentially - # We expect sets of [method, path, func_name] per decorated function - current_method = None - current_path = None - - for node, capture_name in captures: - text = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore") - if capture_name == "method": - current_method = text.upper() - elif capture_name == "path": - # Remove quotes from string literal - current_path = text.strip("\"'") - elif capture_name == "func_name": - if current_method and current_path: - routes[text] = {"method": current_method, "path": current_path} - current_method = None - current_path = None - - return routes + provider = get_provider(language_name, file_path) + return provider.extract_api_routes(code_bytes, file_path) diff --git a/streaming_client.py b/streaming_client.py index 539a6de..2ce83e5 100644 --- a/streaming_client.py +++ b/streaming_client.py @@ -1,17 +1,19 @@ import io +import os import tarfile import httpx +from languages import registry def fetch_repository_archive( owner: str, repo: str, ref: str, github_token: str = None, - target_extension: str = ".py" + target_extension: str = None ) -> dict: """ Downloads the repository tarball from GitHub for a specific ref (commit SHA, branch, tag), - decompresses it in-memory, and extracts all files matching the target extension. + decompresses it in-memory, and extracts all files matching target extension or registered language extensions. Returns a dict: file_path (str) -> code_bytes (bytes). """ url = f"https://api.github.com/repos/{owner}/{repo}/tarball/{ref}" @@ -37,13 +39,23 @@ def fetch_repository_archive( # Decompress and stream read from the in-memory tarball with tarfile.open(fileobj=buffer, mode="r:gz") as tar: for member in tar.getmembers(): - if member.isfile() and member.name.endswith(target_extension): + if member.isfile(): # GitHub tarball root dir is dynamic (e.g. owner-repo-sha/), strip it parts = member.name.split("/", 1) repo_relative_path = parts[1] if len(parts) > 1 else member.name - f = tar.extractfile(member) - if f: - files_data[repo_relative_path] = f.read() - + should_extract = False + if target_extension is not None: + should_extract = member.name.endswith(target_extension) + else: + should_extract = ( + registry.is_supported(repo_relative_path) or + os.path.basename(repo_relative_path) == "go.mod" + ) + + if should_extract: + f = tar.extractfile(member) + if f: + files_data[repo_relative_path] = f.read() + return files_data diff --git a/tests/test_multi_lang.py b/tests/test_multi_lang.py new file mode 100644 index 0000000..a0e7d49 --- /dev/null +++ b/tests/test_multi_lang.py @@ -0,0 +1,123 @@ +import pytest +from parser import extract_functions, find_functions_using_symbol, extract_api_routes +from graph_engine import build_dependency_graph + +def test_javascript_extraction(): + js_code = b""" + function standardFunc() { + return 42; + } + + const arrowFunc = () => { + console.log("hello"); + }; + + class MyClass { + methodName() { + // inside method + } + } + """ + + funcs = extract_functions(js_code, file_path="app.js") + assert "standardFunc" in funcs + assert "arrowFunc" in funcs + assert "methodName" in funcs + + assert funcs["standardFunc"] == (2, 4) + assert funcs["arrowFunc"] == (6, 8) + assert funcs["methodName"] == (11, 13) + +def test_javascript_find_using_symbol(): + js_code = b""" + function caller() { + targetSymbol(); + } + function safe() {} + """ + at_risk = find_functions_using_symbol(js_code, "targetSymbol", file_path="app.js") + assert "caller" in at_risk + assert "safe" not in at_risk + +def test_javascript_resolve_imports(): + files_data = { + "src/app.js": b"import { x } from './utils'; import y from '../config'; const z = require('./components/Button');", + "src/utils.js": b"export const x = 1;", + "config.ts": b"export default {};", + "src/components/Button/index.tsx": b"export default function Button() {}", + } + + graph = build_dependency_graph(files_data) + + # Edges go supplier -> consumer + assert graph.has_edge("src/utils.js", "src/app.js") + assert graph.has_edge("config.ts", "src/app.js") + assert graph.has_edge("src/components/Button/index.tsx", "src/app.js") + +def test_javascript_api_routes(): + js_code = b""" + app.get('/api/users', getUsers); + router.post('/login', loginUser); + """ + routes = extract_api_routes(js_code, file_path="server.js") + assert "getUsers" in routes + assert routes["getUsers"] == {"method": "GET", "path": "/api/users"} + assert "loginUser" in routes + assert routes["loginUser"] == {"method": "POST", "path": "/login"} + +def test_go_extraction(): + go_code = b""" + package main + + func StandardFunc() int { + return 42 + } + + func (r *Receiver) MethodName() { + // do something + } + """ + funcs = extract_functions(go_code, file_path="main.go") + assert "StandardFunc" in funcs + assert "MethodName" in funcs + + assert funcs["StandardFunc"] == (4, 6) + assert funcs["MethodName"] == (8, 10) + +def test_go_find_using_symbol(): + go_code = b""" + package main + func caller() { + TargetSymbol() + } + """ + at_risk = find_functions_using_symbol(go_code, "TargetSymbol", file_path="main.go") + assert "caller" in at_risk + +def test_go_resolve_imports(): + files_data = { + "go.mod": b"module github.com/user/myproject\n\ngo 1.18", + "main.go": b'package main\nimport "github.com/user/myproject/pkg/utils"', + "pkg/utils/math.go": b"package utils", + "pkg/utils/string.go": b"package utils", + } + + graph = build_dependency_graph(files_data) + + # main.go should depend on all files in utils folder + assert graph.has_edge("pkg/utils/math.go", "main.go") + assert graph.has_edge("pkg/utils/string.go", "main.go") + +def test_go_api_routes(): + go_code = b""" + package main + func main() { + r.GET("/users", GetUsers) + router.POST("/v1/login", LoginHandler) + } + """ + routes = extract_api_routes(go_code, file_path="main.go") + assert "GetUsers" in routes + assert routes["GetUsers"] == {"method": "GET", "path": "/users"} + assert "LoginHandler" in routes + assert routes["LoginHandler"] == {"method": "POST", "path": "/v1/login"}