diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4cbe284 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,127 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + lint: + name: Lint & Format + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.22.x' + cache: true + + - name: Install Buf + uses: bufbuild/buf-setup-action@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Lint Protobufs + run: buf lint + + - name: Format Check (gofmt) + run: | + if [ -n "$(gofmt -l .)" ]; then + echo "Go files not formatted:" + gofmt -d . + exit 1 + fi + + - name: golangci-lint (Shared) + uses: golangci/golangci-lint-action@v6 + with: + version: v1.59.1 + working-directory: shared + + - name: golangci-lint (Gateway) + uses: golangci/golangci-lint-action@v6 + with: + version: v1.59.1 + working-directory: services/cce-gateway + + - name: golangci-lint (Ingestor) + uses: golangci/golangci-lint-action@v6 + with: + version: v1.59.1 + working-directory: services/cce-ingestor + + - name: golangci-lint (Parser Worker) + uses: golangci/golangci-lint-action@v6 + with: + version: v1.59.1 + working-directory: services/cce-parser-worker + + - name: golangci-lint (Analyzer) + uses: golangci/golangci-lint-action@v6 + with: + version: v1.59.1 + working-directory: services/cce-analyzer + + - name: golangci-lint (Go Parser Plugin) + uses: golangci/golangci-lint-action@v6 + with: + version: v1.59.1 + working-directory: plugins/go-parser + + test: + name: Run Unit Tests + runs-on: ubuntu-latest + needs: lint + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.22.x' + cache: true + + - name: Install Buf + uses: bufbuild/buf-setup-action@v1 + + - name: Generate Protobuf Code + run: buf generate + + - name: Run Workspace Tests + run: go test -v ./... + + build-check: + name: Compile Binaries + runs-on: ubuntu-latest + needs: test + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.22.x' + cache: true + + - name: Install Buf + uses: bufbuild/buf-setup-action@v1 + + - name: Generate Protobuf Code + run: buf generate + + - name: Build Core Services + run: | + go build -o bin/cce-gateway ./services/cce-gateway/cmd/cce-gateway + go build -o bin/cce-ingestor ./services/cce-ingestor/cmd/cce-ingestor + go build -o bin/cce-parser-worker ./services/cce-parser-worker/cmd/cce-parser-worker + go build -o bin/cce-analyzer ./services/cce-analyzer/cmd/cce-analyzer + + - name: Build Go Parser Plugin + run: | + go build -o bin/go-parser ./plugins/go-parser diff --git a/.gitignore b/.gitignore index aaadf73..238645c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,32 +1,36 @@ -# If you prefer the allow list template instead of the deny list, see community template: -# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore -# -# Binaries for programs and plugins +# Binaries +bin/ *.exe -*.exe~ *.dll *.so *.dylib -# Test binary, built with `go test -c` -*.test +# Go Workspace files that are user-specific (if any) +# We do check in go.work for standard workspace collaboration, but some teams ignore it. +# We will keep go.work checked in, but ignore build folders. -# Code coverage profiles and other test artifacts -*.out -coverage.* -*.coverprofile -profile.cov +# Environments +.env +.env.local +.env.*.local -# Dependency directories (remove the comment below to include it) -# vendor/ +# Dependency directories +vendor/ -# Go workspace file -go.work -go.work.sum +# IDEs and Editors +.idea/ +.vscode/ +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? -# env file -.env +# Testing and Coverage +*.coverprofile +*.out +*.test -# Editor/IDE -# .idea/ -# .vscode/ +# OS Files +.DS_Store +Thumbs.db diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..bfdbe8c --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,39 @@ +run: + timeout: 5m + modules-download-mode: readonly + +linters-settings: + govet: + enable-all: true + disable: + - fieldalignment + revive: + rules: + - name: exported + severity: warning + - name: package-comments + severity: warning + gosec: + severity: "low" + confidence: "low" + +linters: + disable-all: true + enable: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - unused + - revive + - goimports + - gofmt + - gosec + - bodyclose + - misspell + +issues: + exclude-use-default: false + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b3a35a3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + + - repo: https://github.com/golangci/golangci-lint + rev: v1.59.1 + hooks: + - id: golangci-lint + entry: golangci-lint run --new-from-rev=HEAD~1 + + - repo: https://github.com/bufbuild/buf + rev: v1.32.0 + hooks: + - id: buf-lint + name: buf lint + entry: buf lint + files: \.proto$ diff --git a/CCE_ARCHITECTURE_DESIGN.md b/CCE_ARCHITECTURE_DESIGN.md new file mode 100644 index 0000000..e8dec48 --- /dev/null +++ b/CCE_ARCHITECTURE_DESIGN.md @@ -0,0 +1,929 @@ +# Engineering Knowledge Platform (EKP) - CCE Architecture Design Document + +**Author:** Principal Architect & Staff Software Engineer +**Date:** July 5, 2026 +**Status:** PROPOSED (Ready for Senior Engineering & Architecture Review) + +--- + +## 0. MVP Architecture Pivot (July 2026) + +To accelerate initial product validation and reduce operational complexity, CCE has pivoted from a distributed microservice setup (API Gateway, Ingestor, Parser Worker, Analyzer) communicating via NATS JetStream, Neo4j, and OpenSearch to a **single reusable core engine** invoked directly by a lightweight **CLI application**. + +### Key MVP Design Choices: +1. **Core Engine Focus**: + Consolidates CCE repository walking, gRPC AST parsing plugin coordination, dependency graph edge calculation, and attention window compilation into a single `core/` package module. This core package can be reused by a CLI now, and REST APIs, MCP servers, or workers in the future. +2. **Relational Graph Storage**: + Replaces Neo4j and OpenSearch with **SQLite or PostgreSQL** exclusively. Code nodes and call edges are mapped onto standard SQL tables (`nodes` and `edges`). Business logic accesses database resources via a clean, decoupled `Storage` interface in `core/storage` to ensure Neo4j and OpenSearch can be added later without refactoring. +3. **Synchronous Local Pipelines**: + Postpones distributed event broker NATS and Kubernetes task queues. Files are walked, sent to the localhost gRPC Go AST parser plugin, parsed to UCM, linked into call hierarchies, and written to SQLite synchronously inside the CLI execution context. +4. **Postponed Features**: + Multi-tenancy, vector database/RAG embeddings, live inotify watchers, authentication, authorization, external integrations (Slack, Jira, etc.), Helm charts, and microservice orchestration are postponed and will be reintroduced in Phase 2/3. + +### New Module Organization: +- `core/`: Indexer, parser plugin client, UCM normalizer, relationship builder, context compiler, and SQL storage interface. +- `services/cli/`: Lightweight command-line interface (`cce index`, `cce search`, `cce callers`, `cce callees`, `cce context`). +- `plugins/go-parser/`: Decoupled localhost gRPC Go parser plugin. +- `shared/`: Config, logger, tracer common dependencies. + +--- + +## 1. Product Vision + +Large language models (LLMs) and agentic developer tools are fundamentally limited when dealing with enterprise-scale repositories. Existing solutions rely heavily on vector-only Retrieval-Augmented Generation (RAG). While vectors excel at semantic lookup, they fail at structural accuracy—they cannot trace a call stack, calculate impact analysis, identify database schema alterations, verify API route signatures, or understand code dependency networks. + +The **Codebase Context Engine (CCE)** is designed to address this limitation by evolving from a passive source code indexer into a comprehensive **Engineering Knowledge Platform (EKP)**. CCE does not stop at code syntax; it links the static code configuration to runtime infrastructure, services, database tables, message queues, deployments, and organizational ownership structures. + +``` ++---------------------------------------------------------------------------------+ +| Engineering Knowledge Platform (EKP) | ++---------------------------------------------------------------------------------+ +| | +| [Code Layer] Go / Java / Python AST Symbols & Call Graphs | +| | | +| v IMPORTS / CALLS | +| [Architecture Layer] REST APIs / gRPC Services / Message Queues | +| | | +| v READS / WRITES / USES | +| [Infra Layer] Terraform / Kubernetes Pods / DB Schemas & Tables | +| | | +| v OWNS / DEPLOYS | +| [Organizational] Teams / Developers / Slack / PagerDuty / Docs | +| | ++---------------------------------------------------------------------------------+ +``` + +By combining **AST Parsing**, **Static Analysis**, **Universal Code Model (UCM)** normalization, **Graph Relationships**, and **Full-Text Indexing**, CCE serves as a high-performance repository indexer, dynamic call graph engine, API discovery pipeline, and semantic lookup engine. It builds a structured, high-fidelity knowledge graph of software codebases, providing deterministic queries and structured context packages for developer tools, IDEs, and AI coding agents. + +--- + +## 2. User Personas + +### Persona A: The AI Developer Agent (Machine-to-Machine) +* **Description:** Autonomous coding agents executing code modifications, refactorings, and migrations. +* **Core Pain Points:** High latency in fetching files, hallucination of API routes or method signatures, and lack of awareness of upstream/downstream changes caused by editing a class. +* **CCE Value:** Queries the unified **Context API** to fetch code snippets, method signatures, callers, database tables, and API routes in a single round-trip. + +### Persona B: The Platform Engineer / Architect +* **Description:** Tech leads and systems architects overseeing service structures across dozens of repositories. +* **Core Pain Points:** Architectural drift, undocumented microservice dependencies, broken REST schemas, and tracking SQL query patterns across services. +* **CCE Value:** Runs static drift audits across the multi-repo Architecture Graph to detect when controllers bypass service layers or execute unauthorized database joins. + +### Persona C: The Developer Experience (DevEx) Engineer +* **Description:** Engineers tasked with onboarding developers, maintaining documentation portals, and providing internal developer portals (IDPs). +* **Core Pain Points:** Outdated Javadoc/Godoc, manual service catalog updates, and difficulties tracing code ownership. +* **CCE Value:** Serves as a dynamic index for internal portals, mapping ownership networks (CODEOWNERS) and microservice topologies automatically. + +--- + +## 3. Functional Requirements + +### 3.1 Repository Ingestion +* **Git Ingestion:** Clone and fetch git repositories, tracking branches, tags, and commits. +* **Language Detection:** Automatically detect programming languages in the project. +* **Incremental Synchronization:** Monitor files via git commit diff analysis (webhooks) to only index modified or added files. (File watchers are postponed to V2). +* **Workspace Isolation:** Isolate indexed data across tenants and repositories. + +### 3.2 Parser Engine +* **UCM Serialization:** Parse source code structures and normalize outputs into the language-agnostic **Universal Code Model (UCM)**. +* **Symbol Resolution:** Resolve cross-file references and imports within the same package or module. +* **Dependency & Call Graph Extraction:** Detect call expressions (`A()` calls `B()`) and compile static call graphs. Track dependency paths: `Repository -> Package -> Module -> File -> Function`. + +### 3.3 Endpoint & Database Analysis +* **API Discovery:** Extract REST endpoints, GraphQL resolvers, gRPC service definitions, and WebSocket handlers. Identify endpoints, HTTP methods, middlewares, auth models, and schemas. +* **Database Mapping:** Extract raw SQL queries, identify tables, joins, and relationships in ORMs (Hibernate, Prisma, GORM, SQLAlchemy). Detect database migrations. + +### 3.4 Context & Search APIs +* **Context API:** Provide an attention-window API returning full symbol details, dependencies, API bindings, database writes, and owner metadata in a single payload. +* **Multi-Dimensional Search:** Expose search APIs to scan symbols, trace caller/callee paths, find implementations, and query configs. +* **Impact Analysis:** Return a list of downstream files and functions affected if a given function or module is modified. + +--- + +## 4. Non-Functional Requirements + +* **Performance:** Symbol lookup must execute in $< 50\text{ ms}$; Context API lookups (including graph traversal) must execute in $< 200\text{ ms}$. +* **Scalability:** Horizontal scaling of parse workers. Ingestion workloads are distributed across queues; workers are autoscaled based on message backlogs. +* **Multi-Tenancy:** Hard partition of data between distinct tenants using logical tenant IDs in Postgres, Neo4j, and OpenSearch. +* **Plugin-Based Extensibility:** A decoupled gRPC plugin architecture allowing language parsers and custom analyzers to run as independent services. +* **Observability:** Distributed tracing with OpenTelemetry, standard metric exporting (Prometheus format), and structured logging (Zap/Slog). +* **High Availability:** Active-active Kubernetes cluster deployment with stateful storage clustering (Postgres HA, Neo4j Causal Clusters, OpenSearch clusters). + +--- + +## 5. High-Level Architecture + +The architecture of CCE follows an event-driven microservices pattern. Go services interact via a NATS JetStream message broker. Parser workers run as decoupled gRPC servers, returning normalized UCM structures back to the core analyzer. + +``` + +-----------------------+ + | Git Repository | + +-----------+-----------+ + | + | Git Webhook / Poll + v ++------------------+ +-----------------------+ +| Query API | -----------> | Ingestion Service | +| (Context / Search| | (Repo Manager) | ++------------------+ +-----------+-----------+ + | + | Publish: cce.repo.clone + v + +-----------------------+ + | NATS JetStream | + +-----+-----------+-----+ + | ^ + Listen: cce.repo.clone | Publish: cce.file.parsed + v | + +-----------------+-----+ + | Parser Service | ---[gRPC]---> [gRPC Plugins] + | (Tree-Sitter / Go) | <------------ (Python/Go/TS) + +-----+-----------------+ + | + | Publish: cce.analysis.completed + v + +-----------------------+ + | NATS JetStream | + +-----+-----------+-----+ + | | + Listen: cce.analysis.completed | Listen: cce.analysis.completed + v v + +-----+---+ +---+-----+ + | Sync | | Sync | + | Postgres| | Neo4j | + +-----+---+ +---+-----+ + | | + Listen: cce.analysis.completed | + v | + +-----+---+ | + | Sync | | + |OpenSrc | | + +-----+---+ | + | | + v v ++-------------------+ +-----+-----------+-----+ +| Query Service | ----------->| Storage Layer | +| (REST/gRPC API) | | (Postgres/Neo4j/OpenS)| ++-------------------+ +-----------------------+ +``` + +### Components +1. **Ingestion Service (Repo Manager):** Clones, pulls, and watches repositories. Publishes indexing jobs. +2. **Parser Service (Workers):** Pulls tasks, delegates code segments to gRPC language plugins, and receives UCM outputs. +3. **Graph Builder & Analyzer Service:** Subscribes to raw UCM, resolves imports, compiles dependency trees, calls/callees, and maps structural data to infrastructure configurations (Kubernetes, Terraform). +4. **Sync Services (Postgres, Neo4j, OpenSearch):** Ingests structured analysis data into the respective stores. +5. **Query Service:** Gateway offering Context and Search APIs for developer interfaces and AI tools. + +--- + +## 6. Component Diagram + +The internal service interaction diagram details the specific Go packages, external databases, and the decoupled gRPC plugins: + +```mermaid +graph TD + subgraph Client Gateways + Gateway[CCE API Gateway] + end + + subgraph Internal Core Services + Ingestion[cce-ingestor] + Parser[cce-parser-worker] + Analyzer[cce-analyzer] + Query[cce-query-engine] + end + + subgraph gRPC Plugins + GoPlugin[Go Parser Plugin] + TSPlugin[TS Parser Plugin] + PyPlugin[Python Parser Plugin] + end + + subgraph Messaging & Caching + NATS[(NATS JetStream)] + Redis[(Redis Cache)] + end + + subgraph Storage Engines + Postgres[(PostgreSQL Relational)] + Neo4j[(Neo4j Graph Database)] + OpenSearch[(OpenSearch Index)] + end + + %% Flow lines + Gateway -->|Trigger Index| Ingestion + Gateway -->|Context / Search| Query + + Ingestion -->|1. Publish Job| NATS + NATS -->|2. Pull Task| Parser + Parser -->|gRPC call| GoPlugin + Parser -->|gRPC call| TSPlugin + Parser -->|gRPC call| PyPlugin + Parser -->|3. Publish UCM Symbols| NATS + NATS -->|4. Pull Symbols| Analyzer + + Analyzer -->|5. Write Relational Metadata| Postgres + Analyzer -->|6. Write Graph Relationships| Neo4j + Analyzer -->|7. Write Source Code Index| OpenSearch + + Query -->|Read Symbols| Postgres + Query -->|Traverse Graph| Neo4j + Query -->|Keyword Search| OpenSearch + Query -.->|Cache Results| Redis +``` + +--- + +## 7. Data Flow + +This diagram traces a git commit event through the pipeline to query availability: + +```mermaid +sequenceDiagram + autonumber + participant Git as Git Server (GitHub/Gitlab) + participant Ingest as Ingestion Service + participant NATS as NATS JetStream + participant Worker as Parser Worker + participant Plugin as gRPC Parser Plugin + participant DB as Postgres / Neo4j / OpenSearch + participant API as Query API + + Git->>Ingest: Webhook: push event (Commit `0xabc`) + Ingest->>Ingest: Pull Git Diff (Files changed: A.go, B.go) + Ingest->>NATS: Publish: `cce.repo.change` {RepoId: 1, Commit: '0xabc', Files: ['A.go', 'B.go']} + NATS->>Worker: Consume changes + Worker->>Plugin: gRPC ParseFile(A.go) + Plugin-->>Worker: Return UCM Payload + Worker->>NATS: Publish: `cce.symbols.extracted` {UCM Payload} + NATS->>DB: Sync Services write database models + DB-->>DB: Process Graph Edges (Neo4j) & Index Text (OpenSearch) + API->>DB: Query for Context API or Search + DB-->>API: Return enriched context +``` + +--- + +## 8. Database Schema (PostgreSQL) + +We use PostgreSQL for transactional data, tenant settings, repository indexing jobs, and core file/symbol records. + +```sql +-- PostgreSQL Core Schema + +CREATE TABLE tenants ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name VARCHAR(255) NOT NULL UNIQUE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE repositories ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tenant_id UUID REFERENCES tenants(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + clone_url VARCHAR(1024) NOT NULL, + branch VARCHAR(255) DEFAULT 'main', + auth_secret_arn VARCHAR(512), -- Pointer to secret manager + current_head VARCHAR(40), -- Git commit SHA + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + UNIQUE(tenant_id, clone_url) +); + +CREATE TABLE index_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + repository_id UUID REFERENCES repositories(id) ON DELETE CASCADE, + commit_sha VARCHAR(40) NOT NULL, + status VARCHAR(50) NOT NULL, -- PENDING, PROCESSING, COMPLETED, FAILED + error_message TEXT, + started_at TIMESTAMP WITH TIME ZONE, + completed_at TIMESTAMP WITH TIME ZONE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE files ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + repository_id UUID REFERENCES repositories(id) ON DELETE CASCADE, + path VARCHAR(1024) NOT NULL, + sha256 VARCHAR(64) NOT NULL, -- For incremental verification + language VARCHAR(50) NOT NULL, + line_count INTEGER NOT NULL, + size_bytes INTEGER NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + UNIQUE(repository_id, path) +); + +CREATE TABLE symbols ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + file_id UUID REFERENCES files(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + kind VARCHAR(50) NOT NULL, -- CLASS, FUNCTION, METHOD, STRUCT, INTERFACE, CONSTANT, VARIABLE + start_line INTEGER NOT NULL, + start_column INTEGER NOT NULL, + end_line INTEGER NOT NULL, + end_column INTEGER NOT NULL, + signature TEXT, + documentation TEXT, + embedding VECTOR(1536), -- Optional pgvector field for hybrid lookup + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_symbols_file_kind ON symbols(file_id, kind); +CREATE INDEX idx_symbols_name ON symbols(name); +``` + +--- + +## 9. Graph Schema (Neo4j) + +The Engineering Knowledge Graph represents structural elements, code behaviors, deployments, infrastructure, and ownership models as a network of nodes and edges. + +```mermaid +graph TD + %% Entities + Team[Node: Team] + Dev[Node: Developer] + Repo[Node: Repository] + File[Node: File] + Func[Node: Function] + API[Node: API Endpoint] + Service[Node: Microservice] + K8s[Node: Kubernetes Resource] + DBTable[Node: DatabaseTable] + Queue[Node: Message Queue/Topic] + + %% Relationships + Team -->|OWNS| Service + Dev -->|MEMBER_OF| Team + Dev -->|COMMITTED| Repo + Repo -->|CONTAINS| File + File -->|DEFINES| Func + Func -->|IMPLEMENTS| API + Service -->|EXPOSES| API + Service -->|DEPLOYS_TO| K8s + Func -->|READS_WRITES| DBTable + Func -->|PUBLISHES_SUBSCRIBES| Queue + Service -->|DEPENDS_ON| DBTable +``` + +### Graph Node Labels +* `Repository` (id, name, tenant_id) +* `Microservice` (id, name, runtime, framework) +* `File` (id, path, language) +* `Class` / `Interface` / `Struct` (id, name, signature) +* `Function` / `Method` (id, name, signature) +* `API` (id, path, method, route, auth, request_schema, response_schema) +* `DatabaseTable` (id, name, engine) +* `KubernetesResource` (id, name, namespace, kind, replicas) +* `MessageQueueTopic` (id, name, broker_type) +* `Developer` (id, name, email) +* `Team` (id, name, slack_channel) + +### Graph Edge Types +* `CONTAINS`: Structural hierarchy (e.g., `Repository -> File -> Function`) +* `CALLS`: Direct code execution from one method to another (e.g., `Function A -> Function B`) +* `IMPLEMENTS`: Subtyping (e.g., `Class C -> Interface D`) +* `IMPORTS`: File dependency (e.g., `File A -> File B`) +* `OWNS`: Author or team ownership (`Developer -> File` or `Team -> Microservice`) +* `MEMBER_OF`: Organization structure (`Developer -> Team`) +* `READS_WRITES`: Database execution (`Function -> DatabaseTable`) +* `DEFINES` / `EXPOSES`: API registration (`Function -> API` or `Microservice -> API`) +* `DEPLOYS_TO`: Infrastructure mapping (`Microservice -> KubernetesResource`) +* `PUBLISHES_SUBSCRIBES`: Event stream mapping (`Function -> MessageQueueTopic`) + +--- + +## 10. Layered Indexing Architecture + +To maintain high throughput and allow flexible data retrieval, CCE coordinates data processing across six distinct layers: + +``` ++---------------------------------------------------------------------------------+ +| 1. RAW SOURCE LAYER | Git Repository commits, file additions & edits | ++---------------------------+-----------------------------------------------------+ +| 2. AST LAYER | Tree-sitter Concrete Syntax Tree (CST) building | ++---------------------------+-----------------------------------------------------+ +| 3. SYMBOLS LAYER (UCM) | Standardized Scope, Function, and Class extraction | ++---------------------------+-----------------------------------------------------+ +| 4. GRAPH LAYER | Neo4j relationship mapping (CALLS, IMPLEMENTS) | ++---------------------------+-----------------------------------------------------+ +| 5. SEMANTIC LAYER | Text split chunks -> pgvector embeddings | ++---------------------------+-----------------------------------------------------+ +| 6. BUSINESS METADATA LAYER| Owner matching, Kubernetes pods, deployment tags | ++---------------------------------------------------------------------------------+ +``` + +### Ingestion Stages +1. **Raw Source Layer:** Monitors changes via git webhook push events. Pulls down source file bytes. +2. **AST Layer:** Translates raw code bytes into Concrete Syntax Trees using tree-sitter. +3. **Symbols Layer:** Maps language-specific AST structures into the language-agnostic **Universal Code Model (UCM)**. +4. **Graph Layer:** Updates Neo4j. Resolves local scope and import resolutions to map dependencies. +5. **Semantic Layer:** Runs async background workers generating vector embeddings on symbols and docstrings for semantic AI search. +6. **Business Metadata Layer:** Scans configuration files (`docker-compose`, `helm`, `CODEOWNERS`, `kubernetes.yaml`) to enrich the graph nodes with operational contexts. + +--- + +## 11. Incremental Indexing Strategy + +To index repositories with millions of lines of code efficiently, CCE employs a multi-tiered incremental indexing strategy: + +### 11.1 File Hash Matching (Local & Cache Layer) +* For every file found during repository walking, the worker computes a SHA-256 hash. +* The hash is checked against the database record (`files.sha256`). +* If the hash matches, the parser bypasses AST parsing, import extraction, and dependency generation, reusing the existing entries. + +### 11.2 Git Commit Diff Tracking +* When Git signals a push event, CCE identifies the changed files between the parent commit ($C_{n-1}$) and the target commit ($C_n$) using: + ```bash + git diff-tree -r --no-commit-id --name-status C_parent C_target + ``` +* Files are classified into three actions: + * **Added (A) / Modified (M):** Routed to the gRPC plugins for AST-to-UCM parsing. + * **Deleted (D):** Sent to the cleanup queue to purge PostgreSQL symbols and remove Neo4j nodes. +* **MVP Optimization:** Live file watching (`inotify`) is deferred to V2 to prevent socket exhaustion and sync conflicts on local environments. In the MVP, indexing is strictly triggered via Git webhooks or manual batch imports. + +### 11.3 Graph Integrity & Garbage Collection (GC) +* If an AST symbol is deleted or renamed, its Neo4j node and relationships must be cleaned up. +* **Cascading Cleanup:** Deleted nodes trigger a database cleanup of relationship edges (e.g., `CALLS`, `DEPENDS_ON`). +* An asynchronous cleaner worker scans for disconnected or orphaned nodes lacking structural `CONTAINS` relationships back to a valid repository path and garbage collects them. + +--- + +## 12. Parsing Architecture & Universal Code Model (UCM) + +All language parsers output a standardized **Universal Code Model (UCM)** serialized format. This normalizes AST outputs and decouples downstream analysis from programming language specifics. + +### UCM Protocol Buffer Schema + +```protobuf +syntax = "proto3"; +package cce.v1.ucm; + +enum SymbolKind { + SYMBOL_KIND_UNSPECIFIED = 0; + SYMBOL_KIND_MODULE = 1; + SYMBOL_KIND_PACKAGE = 2; + SYMBOL_KIND_CLASS = 3; + SYMBOL_KIND_STRUCT = 4; + SYMBOL_KIND_INTERFACE = 5; + SYMBOL_KIND_FUNCTION = 6; + SYMBOL_KIND_METHOD = 7; + SYMBOL_KIND_VARIABLE = 8; + SYMBOL_KIND_ANNOTATION = 9; +} + +enum RelationType { + RELATION_TYPE_UNSPECIFIED = 0; + RELATION_TYPE_CONTAINS = 1; // e.g. Class contains Method + RELATION_TYPE_CALLS = 2; // e.g. Function A calls Function B + RELATION_TYPE_IMPLEMENTS = 3; // e.g. Class C implements Interface D + RELATION_TYPE_REFERENCES = 4; // e.g. Function reads Variable + RELATION_TYPE_IMPORTS = 5; // e.g. File A imports File B +} + +message Location { + string file_path = 1; + int32 start_line = 2; + int32 start_column = 3; + int32 end_line = 4; + int32 end_column = 5; +} + +message CodeUnit { + string id = 1; // Globally unique deterministic hash + SymbolKind kind = 2; + string name = 3; + string fully_qualified_name = 4; + Location location = 5; + string signature = 6; + string documentation = 7; + map properties = 8; // Extensible metadata (annotations, modifiers) +} + +message Relationship { + string from_id = 1; // CodeUnit ID + string to_id = 2; // CodeUnit ID + RelationType type = 3; + Location location = 4; // Precise code location of the relation +} + +message UniversalCodePayload { + string repository_id = 1; + string commit_sha = 2; + repeated CodeUnit units = 3; + repeated Relationship relationships = 4; +} +``` + +--- + +## 13. Search & Context Architecture + +CCE offers a hybrid system that coordinates structural, keyword, relational, and contextual queries. + +``` + +-----------------------+ + | Query Request API | + +-----------+-----------+ + | + +-----------------+-----------------+ + | Query Router / Dispatcher | + +---+-----------------+---------+---+ + | | | + Context API Lookups | Graph Queries | | Full-Text / Regex Match + v v v + +-----+-----+ +-----+-----+ +-+---------+ + | Postgres | | Neo4j | |OpenSearch | + +-----------+ +-----------+ +-----------+ +``` + +1. **The Context API (Primary for AI/IDEs):** Generates dense, structured context payloads containing targeted symbol code blocks, call logs, api endpoints, schemas, database tables, and ownership groups in a single call. +2. **Relational Search (Postgres):** High-speed exact searches for symbols (`class UserAccount`) and directories. +3. **Structural Graph Search (Neo4j):** Resolves dependency paths, callers/callees, interface implementations, and architectural boundaries. +4. **Full-Text & Regex Search (OpenSearch):** Used to perform fuzzy code search, regex text scans, code content searches, and documentation indexing. + +--- + +## 14. API Specifications + +The REST gateway API for CCE runs on HTTP/JSON. (Equivalent gRPC definitions are compiled from Protobufs). + +```yaml +openapi: 3.0.3 +info: + title: Codebase Context Engine (CCE) API + version: 1.0.0 +paths: + /api/v1/repositories: + post: + summary: Register a new repository for indexing + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + clone_url: + type: string + branch: + type: string + responses: + '202': + description: Indexing job accepted + /api/v1/search/symbols: + get: + summary: Find symbols in the indexed codebase + parameters: + - name: query + in: query + required: true + schema: + type: string + - name: kind + in: query + schema: + type: string + enum: [CLASS, FUNCTION, STRUCT, INTERFACE] + responses: + '200': + description: List of matched symbols + /api/v1/context/symbol: + post: + summary: Retrieve a rich context payload optimized for AI Agents and IDEs + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [repository_id, symbol_name] + properties: + repository_id: + type: string + commit_sha: + type: string + symbol_name: + type: string + depth_limits: + type: object + properties: + call_graph: + type: integer + default: 2 + include_infrastructure: + type: boolean + default: true + responses: + '200': + description: Context block containing symbol code, dependency mappings, databases, and owners. + /api/v1/graph/traverse: + get: + summary: Get relationships (e.g. Callers, Implementations) + parameters: + - name: symbol_id + in: query + required: true + schema: + type: string + - name: direction + in: query + required: true + schema: + type: string + enum: [UPSTREAM, DOWNSTREAM] + - name: max_depth + in: query + schema: + type: integer + default: 3 + responses: + '200': + description: Returns nodes and edges array + /api/v1/analysis/impact: + get: + summary: Identify downstream impacts of changing a symbol + parameters: + - name: symbol_id + in: query + required: true + schema: + type: string + responses: + '200': + description: Return affected files, functions, and teams +``` + +--- + +## 15. Repository Structure + +The physical directory hierarchy for the CCE codebase follows standard Go project layouts: + +``` +cce-root/ +├── .github/ # CI/CD Workflows +├── api/ # OpenAPI specifications and protobuf definitions +│ ├── openapi.yaml +│ ├── v1/ +│ │ ├── cce.proto +│ │ ├── ucm.proto # Universal Code Model schema +│ │ └── cce_plugin.proto # gRPC plugin contract +├── cmd/ # Main entry points for CCE binaries +│ ├── cce-gateway/ # REST/gRPC API gateway +│ ├── cce-ingestor/ # Repository clone, watch and git controller +│ ├── cce-parser-worker/ # Core orchestrator parser microservice +│ └── cce-analyzer/ # AST analyzer, UCM normalizer and DB syncer +├── deployments/ # Kubernetes and Docker configs +│ ├── docker-compose.yml +│ └── k8s/ +│ ├── deployments/ +│ └── helm/ +├── internal/ # Private service code +│ ├── db/ # Database drivers and ORM (Postgres, Neo4j, OpenSearch) +│ ├── events/ # Event subscriber and publisher abstractions +│ ├── parser/ # Core symbol resolution and AST mappings +│ └── tracer/ # OpenTelemetry tracing helpers +├── pkg/ # Public libraries (usable by plugins) +│ └── protocol/ # Common schema definitions +├── plugins/ # Decoupled plugin services directory +│ ├── ts-parser/ # TypeScript parser gRPC plugin (Node.js) +│ ├── go-parser/ # Go parser gRPC plugin (Go) +│ └── py-parser/ # Python parser gRPC plugin (Python) +├── go.mod +└── README.md +``` + +--- + +## 16. Go Package Organization + +The `/internal` tree is divided into modules: + +* `internal/db/postgres`: Implements database operations for saving file meta, raw symbols, indexing job states. +* `internal/db/neo4j`: Implements Cypher driver operations to commit graph components, trace path lines, and build calling hierarchies. +* `internal/db/opensearch`: Implements indexing tasks for raw source lines, comments, and READMEs. +* `internal/parser/plugin`: Implements client wrappers to coordinate calls out to external gRPC language parser plugins. +* `internal/events/nats`: Orchestrates NATS JetStream connectivity, topic creation, event delivery, and handling. + +--- + +## 17. Event-Driven Architecture + +CCE services coordinate via event streams on NATS JetStream. + +### Event Subject Tree +* `cce.repo.registered`: Fired when a repository is added to the control plane. +* `cce.repo.clone`: Triggered when ingestion finishes cloning or pulling changes. +* `cce.file.changes`: Fired for every commit change, listing files added, modified, or deleted. +* `cce.file.parsed`: Emitted by parser workers after successfully building the Universal AST. +* `cce.analysis.completed`: Emitted after dependency, call graphs, API endpoints, and SQL queries are fully compiled. + +--- + +## 18. Plugin System (Decoupled gRPC Architecture) + +To support languages and analysis steps dynamically without recompiling CCE core services, CCE implements a Decoupled gRPC Plugin Architecture. + +``` ++---------------------+ +------------------------+ +| CCE Core Worker | 1. Register | Language Parser | +| (Orchestrator) | <----------------- | Plugin (gRPC Server) | +| | | (e.g., Python Service)| +| | 2. ParseFile | | +| | -----------------> | - Reads tree-sitter | +| | (path, bytes) | - Normalizes to UCM | +| | | | +| | 3. UCM Protobuf | | +| | <----------------- | | ++---------------------+ +------------------------+ +``` + +### The Plugin Service Contract (`cce_plugin.proto`) +```protobuf +syntax = "proto3"; +package cce.plugin.v1; + +import "ucm.proto"; + +service LanguageParserPlugin { + rpc GetMetadata(Empty) returns (PluginMetadata); + rpc ParseFile(ParseRequest) returns (ParseResponse); +} + +message Empty {} + +message PluginMetadata { + string name = 1; + string version = 2; + repeated string supported_extensions = 3; +} + +message ParseRequest { + string file_path = 1; + bytes file_content = 2; +} + +message ParseResponse { + cce.v1.ucm.UniversalCodePayload payload = 1; + repeated string parsing_errors = 2; +} +``` + +* **Plugin Runtime:** Plugins run as external, independent gRPC servers (e.g. running TS parser in Node.js, Python parser in Python) registered with the orchestrator. +* **Benefits:** Isolation (plugin failures do not crash the core pipeline) and development flexibility (write parser plugins in their native runtime environments). + +--- + +## 19. Security Considerations + +### 19.1 Secret In-Flight Masking +* Codebases often contain accidental secrets (API keys, credentials, tokens). +* The **Parser Worker** runs a regex scanner during file parsing. +* Identified secrets are masked in the output UCM payload and replaced with `"[MASKED_SECRET]"` before saving to database engines. + +### 19.2 Multi-Tenant Isolation +* **Storage Layer Isolation:** + * PostgreSQL tables require `tenant_id` filters on all queries, enforced via Row Level Security (RLS). + * Neo4j uses tenant isolation keys as node properties, appending `WHERE n.tenant_id = $tenant_id` to all Cypher executions. + * OpenSearch indices are partitioned per tenant: `idx--source-code`. + +--- + +## 20. Deployment Architecture + +CCE is containerized and deployable on Kubernetes. We use KEDA (Kubernetes Event-driven Autoscaling) to handle peak indexing workloads. + +``` + Traffic In + | + v + +------------------+ + | Ingress/Router | + +--------+---------+ + | + +------------------+------------------+ + | | + v v + +--------------------+ +--------------------+ + | cce-gateway | | cce-ingestor | + | (Replicas: 3+) | | (Replicas: 2+) | + +---------+----------+ +---------+----------+ + | | + +------------------+------------------+ + | + v + +------------------+ + | NATS JetStream | + +--------+---------+ + | + +------------------+------------------+ + | | + v (Scaled by KEDA) v (Scaled by KEDA) + +--------------------+ +--------------------+ + | cce-parser-worker | | cce-analyzer | + | (Orchestrates gRPC)| | (Builds Graph) | + +---------+----------+ +---------+----------+ + | | + v (Local/Service calls) | + +---------+----------+ | + | gRPC Parser Pods | | + | (Go / TS / Py) | | + +--------------------+ | + v + +---------+----------+ + | Sync Services | + | (Write DB/Graph/OS)| + +--------------------+ +``` + +* **Autoscaling:** KEDA monitors NATS JetStream backlog depth (`cce.file.changes`). Workers scale out dynamically from $1$ to $N$ to process massive code bases and scale back to zero when idle. +* **Stateful Storage:** Postgres, Neo4j, and OpenSearch are deployed as clustered StatefulSets or external enterprise cloud resources. + +--- + +## 21. Scaling Strategy + +* **Worker Layer (Stateless):** Horizontally scale parser workers and gRPC plugins. Since AST extraction is CPU-bound, workers are scheduled on compute-optimized nodes. +* **Database (Postgres):** Vertically scaled for write-heavy workloads, using read-replicas for query traffic. +* **Graph Layer (Neo4j):** Causal Clustering with read-replicas handling traversed queries. Since codebase networks are highly dense, graphs are partitioned per tenant to prevent query latency growth. +* **Search Engine (OpenSearch):** Shared indices with data sharded by `tenant_id`. Document routing keys (`routing=tenant_id`) ensure search queries target specific shards. + +--- + +## 22. Future Roadmap + +### Phase 1: Core Code Graph (MVP - Months 1-3) +* **Goal:** A working structural code indexer. +* **Ingestion:** Git commit diff webhook ingester (no local file watcher). +* **Languages:** Go and TypeScript Tree-Sitter gRPC plugins. +* **Storage:** PostgreSQL (Symbols/Meta) + Neo4j (Call Graph). +* **API:** Basic symbol search and callers/callees traversal. + +### Phase 2: Architectural Discovery (V2 - Months 4-6) +* **Goal:** Bridge code syntax and endpoints. +* **Languages:** Java and Python tree-sitter gRPC plugins. +* **Discovery:** API router mapping (REST paths, endpoints) and SQL/ORM query parsing. +* **API:** Context API (`/api/v1/context/symbol`) implementation. +* **Search:** OpenSearch integration for regex, code scans, and semantic vector indexing. + +### Phase 3: Engineering Knowledge Platform (V3 - Months 7-9) +* **Goal:** Complete operational visibility. +* **Integrations:** Kubernetes configurations, Terraform maps, and CODEOWNERS parsing. +* **Tracing:** Mapping runtime metrics and OpenTelemetry trace spans directly back onto the static code graph. +* **Auditing:** Architecture drift checking and pull request blast-radius profiling. + +--- + +## 23. Risks & Mitigations + +### Risk 1: Memory Exhaustion on Large Graphs +* **Problem:** Loading a global multi-tenant codebase graph into memory for deep traversals could crash graph nodes. +* **Mitigation:** Impose execution depth limits (max 5 hops) on API graph calls. Use pagination on path expansions and partition graphs by tenant. + +### Risk 2: Parsing Failures on Broken/Uncompilable Code +* **Problem:** Code loaded during active development might contain syntax errors and fail to compile. +* **Mitigation:** Tree-sitter handles uncompilable syntax gracefully by generating error nodes (`ERROR`) and recovering context immediately after the broken segment. CCE parses invalid files, skipping only the broken AST nodes. + +### Risk 3: OpenSearch Storage Explosion +* **Problem:** Retaining multiple historic versions of raw code files results in rapid disk consumption. +* **Mitigation:** CCE only stores active branch configurations (e.g., `main`/`prod`) in the search index. Historical commit source code is left to the Git repository provider, storing only indexed metadata and commit hashes. + +--- + +## 24. Technical Trade-offs + +### Option A: Tree-Sitter vs. Language Server Protocol (LSP/LSIF) +* **LSP/LSIF:** Accurate semantic extraction, but requires running the compiler toolchain (e.g., resolving imports, installing `node_modules` or Go dependencies). High resource cost and fragile execution. +* **Tree-Sitter:** Quick, robust AST building. Does not require code compilation, resolving third-party module downloads, or setting up environments. +* **Decision:** **Tree-Sitter**. CCE acts as a repository indexer operating at high scale. Tree-Sitter guarantees parsing returns results, even on invalid syntax, with minimal CPU and memory footprints. + +### Option B: Neo4j vs. Relational Graph Storage (PostgreSQL recursive CTEs) +* **Postgres CTEs:** Eliminates an extra infrastructure component (Neo4j). High-performing for simple 1-2 hop relationships. +* **Neo4j:** Native graph storage optimizing multi-hop lookups (e.g. "Find all paths between API route X and Table Y"). +* **Decision:** **Neo4j** (alongside Postgres metadata). Tracing calls, interface implementations, and architectural boundaries requires deep, nested graph lookups. Doing this via Postgres joins is slow and complex to maintain. + +### Option C: In-Process Runtimes (cgo/Wasm) vs. Decoupled gRPC Plugins +* **In-Process (cgo/Wasm):** Zero network latency, fast execution. However, cgo is hard to cross-compile, and compiling multiple languages inside Go requires complex setups. Wasm has strict sandbox boundaries and lacks mature tree-sitter libraries. +* **Decoupled gRPC:** Minor localhost network serialization latency ($<2\text{ms}$). However, it isolates worker threads from parser crashes and lets engineers write language parser plugins using native platforms (Node.js for TypeScript, Python for Python). +* **Decision:** **Decoupled gRPC**. The gain in plugin development independence and stability outweigh the minor localhost serialization cost. + +--- + +## 25. Why CCE is Superior to Vector-Only RAG + +A standard RAG implementation chunks files into text fragments (e.g. 500 characters), runs them through an embedding model, and saves them to a Vector DB. This model fails in programming environments: + +| Feature / Scenario | Traditional Vector-Only RAG | Codebase Context Engine (CCE) | +| :--- | :--- | :--- | +| **Exact Call Tracing** | Fails. It cannot verify if `func A()` triggers `func B()`, relying on proximity search instead. | **Deterministic.** Traces calls via AST edges. | +| **Context Chunking** | Cuts code arbitrarily. Chunks may separate variable declarations from their usages. | **Structural.** AST scopes guarantee code sections stay structurally intact. | +| **Interface Implementations** | Fails to detect implementation relationships when file naming differs. | **Explicit.** Builds `IMPLEMENTS` edges between interfaces and classes. | +| **Impact Analysis** | Fails. Cannot identify which systems depend on a function. | **Deterministic.** Neo4j identifies dependent nodes across the entire workspace. | +| **API Boundary Verification** | Vulnerable to hallucinating API structures. | **Accurate.** Automatically extracts schema rules, methods, and routes from routing trees. | +| **Security Auditing** | Cannot reliably trace data flows from REST inputs to DB writes. | **Traceable.** Follows data flow edges (`API -> Controller -> Service -> Table`). | + +By treating codebases as a structural relational system rather than flat text paragraphs, CCE provides the structural foundation required for developer tools and AI models. diff --git a/CCE_CTO_ARCHITECT_REVIEW.md b/CCE_CTO_ARCHITECT_REVIEW.md new file mode 100644 index 0000000..6efbe33 --- /dev/null +++ b/CCE_CTO_ARCHITECT_REVIEW.md @@ -0,0 +1,375 @@ +# CCE Architectural Review & Strategic Evolution Addendum +**From:** Principal Software Architect & Startup CTO +**To:** Engineering Leadership Team +**Date:** July 5, 2026 +**Status:** PROPOSED (To be reviewed alongside the CCE Architecture Design Document) + +--- + +## Executive Summary + +The initial Codebase Context Engine (CCE) design document outlines a robust foundation for standard source code AST indexing. However, to capture the startup market opportunity and build a sustainable long-term competitive moat, CCE must evolve from a passive **"Codebase Context Engine"** into an active **Engineering Knowledge Platform (EKP)**. + +An EKP does not stop at code syntax. It links source code to runtime reality: microservices, database schemas, deployment pipelines, Kubernetes configurations, ownership structures, and developer workflows. By designing a language-agnostic **Universal Code Model (UCM)** and exposing a highly tailored **Context API** (instead of raw search), CCE can serve as the brain for AI agents, IDEs, and developer portals. + +This document reviews the base architecture and introduces structural expansions, UCM schemas, plugin models, layered indexing stages, and a phased, low-risk roadmap to achieve this vision without over-engineering the MVP. + +--- + +## 1. Evolution: Codebase Engine to Engineering Knowledge Platform (EKP) + +### The Vision Shift +A code context engine is a retrieval pipeline for files and functions. An **Engineering Knowledge Platform** is the source of truth for the entire software engineering lifecycle. It provides structural visibility across the engineering stack, answering questions like: *"Which team owns the service that writes to this database column, and which Kubernetes namespace does it deploy to?"* + +``` ++---------------------------------------------------------------------------------+ +| Engineering Knowledge Platform (EKP) | ++---------------------------------------------------------------------------------+ +| | +| [Code Layer] Go / Java / Python AST Symbols & Call Graphs | +| | | +| v IMPORTS / CALLS | +| [Architecture Layer] REST APIs / gRPC Services / Message Queues | +| | | +| v READS / WRITES / USES | +| [Infra Layer] Terraform / Kubernetes Pods / DB Schemas & Tables | +| | | +| v OWNS / DEPLOYS | +| [Organizational] Teams / Developers / Slack / PagerDuty / Docs | +| | ++---------------------------------------------------------------------------------+ +``` + +### Why This Shift Matters +* **For AI Agents:** AI agents cannot plan architectural changes if they only see code files. They need to understand API boundaries, infrastructure constraints, and service dependencies. +* **For Platform Engineers:** It bridges static code analysis with live infrastructure catalogs (e.g. Backstage), preventing documentation and architecture drift. +* **Trade-Offs:** Moving to an EKP increases metadata ingestion complexity and introduces third-party API dependencies (K8s APIs, Git providers). However, by keeping the MVP ingestion strictly static (parsing YAML/JSON configurations directly from the repository), we gain the benefits of EKP without requiring live cluster integrations in Phase 1. + +--- + +## 2. The Universal Code Model (UCM) + +To make CCE truly language-agnostic, parsers must not write language-specific AST structures directly to the database. Instead, every parser must emit a standardized **Universal Code Model (UCM)** format. + +### UCM Protocol Buffer Schema (Proposed) + +```protobuf +syntax = "proto3"; +package cce.v1.ucm; + +enum SymbolKind { + SYMBOL_KIND_UNSPECIFIED = 0; + SYMBOL_KIND_MODULE = 1; + SYMBOL_KIND_PACKAGE = 2; + SYMBOL_KIND_CLASS = 3; + SYMBOL_KIND_STRUCT = 4; + SYMBOL_KIND_INTERFACE = 5; + SYMBOL_KIND_FUNCTION = 6; + SYMBOL_KIND_METHOD = 7; + SYMBOL_KIND_VARIABLE = 8; + SYMBOL_KIND_ANNOTATION = 9; +} + +enum RelationType { + RELATION_TYPE_UNSPECIFIED = 0; + RELATION_TYPE_CONTAINS = 1; // e.g. Class contains Method + RELATION_TYPE_CALLS = 2; // e.g. Function A calls Function B + RELATION_TYPE_IMPLEMENTS = 3; // e.g. Class C implements Interface D + RELATION_TYPE_REFERENCES = 4; // e.g. Function reads Variable + RELATION_TYPE_IMPORTS = 5; // e.g. File A imports File B +} + +message Location { + string file_path = 1; + int32 start_line = 2; + int32 start_column = 3; + int32 end_line = 4; + int32 end_column = 5; +} + +message CodeUnit { + string id = 1; // Globally unique deterministic hash + SymbolKind kind = 2; + string name = 3; + string fully_qualified_name = 4; + Location location = 5; + string signature = 6; + string documentation = 7; + map properties = 8; // Extensible metadata (annotations, modifiers) +} + +message Relationship { + string from_id = 1; // CodeUnit ID + string to_id = 2; // CodeUnit ID + RelationType type = 3; + Location location = 4; // Precise code location of the relation +} + +message UniversalCodePayload { + string repository_id = 1; + string commit_sha = 2; + repeated CodeUnit units = 3; + repeated Relationship relationships = 4; +} +``` + +### Why the UCM is Essential +* **Language Agnosticism:** The core database syncer, query engines, and call-graph traversers only interact with UCM schemas. Adding a new language is simply a matter of writing a parser that outputs this protobuf format. +* **Trade-Offs:** Some language-specific syntax (such as C++ template specializations or TypeScript union types) may lose detail during normalization. We mitigate this by utilizing the `properties` map in the `CodeUnit` to preserve language-specific details for downstream queries. + +--- + +## 3. Expanded Knowledge Graph Schema + +We extend the graph database representation to include higher-level architectural, infrastructural, and organizational entities: + +```mermaid +graph TD + %% Entities + Team[Node: Team] + Dev[Node: Developer] + Repo[Node: Repository] + File[Node: File] + Func[Node: Function] + API[Node: API Endpoint] + Service[Node: Microservice] + K8s[Node: Kubernetes Resource] + DBTable[Node: DatabaseTable] + Queue[Node: Message Queue/Topic] + + %% Relationships + Team -->|OWNS| Service + Dev -->|MEMBER_OF| Team + Dev -->|COMMITTED| Repo + Repo -->|CONTAINS| File + File -->|DEFINES| Func + Func -->|IMPLEMENTS| API + Service -->|EXPOSES| API + Service -->|DEPLOYS_TO| K8s + Func -->|READS_WRITES| DBTable + Func -->|PUBLISHES_SUBSCRIBES| Queue + Service -->|DEPENDS_ON| DBTable +``` + +### New Node Entities +* **Microservice:** logical service boundary mapped to a repository or subdirectory (attributes: `name`, `runtime`, `framework`). +* **Kubernetes Resource:** deployment configurations extracted from yaml files (attributes: `namespace`, `kind`, `replicas`). +* **Message Queue / Topic:** messaging destinations extracted from configuration files (attributes: `name`, `broker_type` - e.g., Kafka, NATS). +* **Team:** organizational owner extracted from `CODEOWNERS` or identity provider synchronizations (attributes: `name`, `slack_channel`). + +--- + +## 4. The Context API + +Exposing standard symbol search forces AI agents to make multiple calls to compile context. We propose the **Context API**, which provides structured "attention windows" optimized for LLM contexts in a single round-trip. + +### API Endpoint: `POST /api/v1/context/symbol` + +#### Request Payload +```json +{ + "repository_id": "e4125dca-9759-450f-a9db-e78ff6c6a45d", + "commit_sha": "a189f76a5bde8c2738fa0982d6199342551a82b1", + "symbol_name": "src/services/UserService.ts:UserService.saveUser", + "depth_limits": { + "call_graph": 2, + "dependencies": 1 + }, + "include_infrastructure": true, + "include_ownership": true +} +``` + +#### Response Payload +```json +{ + "target_symbol": { + "id": "usr_srv_save_user", + "name": "saveUser", + "signature": "async saveUser(user: User): Promise", + "code_snippet": "async saveUser(user: User): Promise {\n validateUser(user);\n return this.db.insert(user);\n}", + "documentation": "Saves a user record to the database after validating fields.", + "file_path": "src/services/UserService.ts" + }, + "context_relationships": { + "dependencies": [ + { "symbol_id": "validate_user", "name": "validateUser", "relationship": "CALLS" } + ], + "callers": [ + { "symbol_id": "user_controller_create", "name": "createUserEndpoint", "relationship": "CALLED_BY" } + ], + "implementations": [], + "infrastructure_bindings": { + "apis": [ + { "route": "/api/v1/users", "method": "POST", "auth": "JWT" } + ], + "databases": [ + { "table_name": "users", "action": "INSERT" } + ], + "queues": [ + { "topic": "user.signup.events", "action": "PUBLISH" } + ] + }, + "metadata": { + "owners": { "team": "Identity Team", "slack": "#team-identity" }, + "k8s_deployment": { "name": "user-service", "namespace": "prod-core" } + } + } +} +``` + +### Why the Context API Improves the Platform +* **Reduces Latency:** AI agents fetch a dense, self-contained context object in $1$ call rather than making $10$ sequential calls to fetch code, call graphs, database schemas, and owners. +* **Protects Context Windows:** By limiting graph traversals (via `depth_limits`), CCE filters out noise and fits the critical context exactly into the LLM's prompt window. + +--- + +## 5. Decoupled gRPC Plugin Architecture + +In the initial design, parser workers executed tree-sitter queries within the main Go process (via `cgo` or Wasm). To support third-party extensibility cleanly, we transition to a **Decoupled gRPC Plugin Architecture**. + +``` ++---------------------+ +------------------------+ +| CCE Core Worker | 1. Register | Language Parser | +| (Orchestrator) | <----------------- | Plugin (gRPC Server) | +| | | (e.g., Python Service)| +| | 2. ParseFile | | +| | -----------------> | - Reads tree-sitter | +| | (path, bytes) | - Normalizes to UCM | +| | | | +| | 3. UCM Protobuf | | +| | <----------------- | | ++---------------------+ +------------------------+ +``` + +### The Plugin Service Contract (`cce_plugin.proto`) +```protobuf +syntax = "proto3"; +package cce.plugin.v1; + +import "ucm.proto"; + +service LanguageParserPlugin { + rpc GetMetadata(Empty) returns (PluginMetadata); + rpc ParseFile(ParseRequest) returns (ParseResponse); +} + +message Empty {} + +message PluginMetadata { + string name = 1; + string version = 2; + repeated string supported_extensions = 3; +} + +message ParseRequest { + string file_path = 1; + bytes file_content = 2; +} + +message ParseResponse { + cce.v1.ucm.UniversalCodePayload payload = 1; + repeated string parsing_errors = 2; +} +``` + +### Why gRPC Plugins? +* **Write Parsers in Native Runtimes:** The TypeScript parser can be written in Node.js, and Python in Python, utilizing official language tooling instead of forcing all logic into Go or Wasm. +* **Isolation:** A crashed plugin process does not take down the main CCE worker queue. +* **Trade-Offs:** Introduces minor network overhead over IPC or localhost gRPC. However, this is negligible ($< 2\text{ms}$ per file) compared to the gains in system isolation and development ease. + +--- + +## 6. Layered Indexing Architecture + +To maintain high throughput and allow flexible data retrieval, CCE uses a **six-layer pipeline**: + +``` ++---------------------------------------------------------------------------------+ +| 1. RAW SOURCE LAYER | Git Repository commits, file additions & edits | ++---------------------------+-----------------------------------------------------+ +| 2. AST LAYER | Tree-sitter Concrete Syntax Tree (CST) building | ++---------------------------+-----------------------------------------------------+ +| 3. SYMBOLS LAYER (UCM) | Standardized Scope, Function, and Class extraction | ++---------------------------+-----------------------------------------------------+ +| 4. GRAPH LAYER | Neo4j relationship mapping (CALLS, IMPLEMENTS) | ++---------------------------+-----------------------------------------------------+ +| 5. SEMANTIC LAYER | Text split chunks -> pgvector embeddings | ++---------------------------+-----------------------------------------------------+ +| 6. BUSINESS METADATA LAYER| Owner matching, Kubernetes pods, deployment tags | ++---------------------------------------------------------------------------------+ +``` + +### Ingestion Stages +1. **Raw Source Layer:** Monitors changes via git pull or local workspace watchers. Emits raw file bytes. +2. **AST Layer:** Fast syntax check. Translates source bytes to tree-sitter nodes. +3. **Symbols Layer:** Maps AST nodes to the Universal Code Model (UCM). Computes identifiers. +4. **Graph Layer:** Updates Neo4j. Performs local scope and import resolutions to map dependencies. +5. **Semantic Layer:** Runs async background workers to generate vector embeddings on symbols and docstrings for fuzzy AI search. +6. **Business Metadata Layer:** Scans YAML descriptors (`docker-compose`, `helm`, `CODEOWNERS`, `kubernetes.yaml`) to enrich the graph nodes with operational contexts. + +--- + +## 7. Product Moats & Long-Term Competitive Advantages + +To protect CCE from generic open-source tools and ensure long-term value, we build the following functional features into the architecture: + +* **Static Architecture Drift Auditing:** Define architectural rules via Cypher queries (e.g. `MATCH (c:Controller)-[*]->(d:DBTable) WHERE NOT (c)-[:CALLS]->(:Service) RETURN c`). CCE alerts developers when code changes bypass structural layer guidelines. +* **Downstream Change Risk Profiler:** Provides a "Risk Index" for pull requests. By tracing call-graph depths and dependencies, CCE computes the blast radius of a code modification, suggesting targeted test suites and notifying downstream code owners automatically. +* **Dynamic-to-Static Mapping:** Integrate runtime traces (e.g. OpenTelemetry spans) back into the static graph. This displays the actual production traffic throughput alongside the static code layout, showing dead code blocks and path hotspots directly. + +--- + +## 8. MVP Postponements (Reducing Scope Complexity) + +To build a usable system within 3 months, we recommend deferring several complex features: + +* **Postpone Local Real-time File Watching (`inotify`):** File system watchers scale poorly in containers and trigger write cascades on high-frequency branch switches. **MVP Alternative:** Rely strictly on Git webhook commit diff pushes and manual trigger pulls. +* **Postpone Dynamic Runtime Call Graphs:** Tracing live program executions requires runtime agents (like eBPF or AST instrumentations). **MVP Alternative:** Stick exclusively to static call graphs mapped from function calls and imports. +* **Postpone Multi-Repository Traversals:** Cross-repository imports require resolving external dependency lockfiles (e.g., `go.mod`, `package.json`). **MVP Alternative:** Build call graphs strictly scoped within individual repositories. Cross-service boundaries are resolved via static API route mapping. +* **Postpone Full WASM Sandboxing for Plugins:** WebAssembly ABI interfaces are complex to define and debug. **MVP Alternative:** Implement plugins as independent gRPC microservice processes running on localhost. + +--- + +## 9. Phased Roadmap + +Each phase is designed to be independently usable and deliver immediate engineering value. + +``` + +-------------------------------------------------------+ + | PHASE 1: Core Code Graph (MVP) | + | - Git commit diff ingester | + | - Go & TypeScript Tree-Sitter gRPC parsers | + | - PostgreSQL (Symbols) + Neo4j (Call Graph) | + | - basic Symbol and Caller search APIs | + +----------------------------+--------------------------+ + | + v + +-------------------------------------------------------+ + | PHASE 2: Architectural Discovery (V2) | + | - Python & Java parsers | + | - API & DB ORM extraction (REST, SQL query maps) | + | - Context API implementation | + | - Opensearch full-text & semantic vector indexing | + +----------------------------+--------------------------+ + | + v + +-------------------------------------------------------+ + | PHASE 3: Engineering Knowledge Platform (V3) | + | - K8s, Terraform, and CODEOWNERS parsing | + | - Dynamic OpenTelemetry span mapping | + | - CI/CD integration & architectural drift checks | + +-------------------------------------------------------+ +``` + +### Phase 1: Core Code Graph (MVP) +* **Goal:** A working structural code indexer. +* **Value:** Replaces grep. Developers can immediately search code symbols, trace method callers, and traverse file dependencies programmatically. + +### Phase 2: Architectural Discovery (V2) +* **Goal:** Bridge the gap between code syntax and endpoints. +* **Value:** Provides AI agents and IDEs with the context API, letting them trace a Web REST request directly down to the database query it executes. + +### Phase 3: Engineering Knowledge Platform (V3) +* **Goal:** Complete operational visibility. +* **Value:** Serves as the developer experience hub, tracing code ownership, deployment environments, active metrics, and guarding against architectural drift. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ad5762a --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ +.PHONY: all proto-gen build lint format test tidy + +all: proto-gen build + +proto-gen: + @echo "Generating Protobuf files via buf..." + buf generate + +build: + @echo "Building CCE CLI and Go parser plugin..." + go build -o bin/cce ./services/cli + go build -o bin/go-parser ./plugins/go-parser + +tidy: + @echo "Running go mod tidy across modules..." + cd shared && go mod tidy + cd core && go mod tidy + cd services/cli && go mod tidy + cd plugins/go-parser && go mod tidy + +lint: + @echo "Running golangci-lint on modules..." + cd shared && golangci-lint run ./... + cd core && golangci-lint run ./... + cd services/cli && golangci-lint run ./... + cd plugins/go-parser && golangci-lint run ./... + +format: + @echo "Formatting Go files..." + go fmt ./shared/... ./core/... ./services/cli/... ./plugins/go-parser/... + +test: + @echo "Running unit tests across workspace modules..." + go test -v ./shared/... ./core/... ./services/cli/... ./plugins/go-parser/... diff --git a/README.md b/README.md index e825d54..85c7eb9 100644 --- a/README.md +++ b/README.md @@ -1 +1,160 @@ -# codebase-context-engine \ No newline at end of file +# Codebase Context Engine (CCE) - MVP Reusable Core & CLI + +Welcome to the **Codebase Context Engine (CCE)** repository. This branch has been structured specifically for the **MVP Release**, prioritizing rapid validation of the core ingestion and context tracing pipeline over distributed microservices. + +Instead of distributed containers, CCE runs as a **single reusable core engine** called directly by a lightweight **CLI application** which writes to a local **SQLite** database. + +--- + +## MVP Architecture Workflow + +``` +[Git Repo / Local Dir] + │ (Crawl / Walk files) + ▼ + [core/indexer] + │ + ▼ + [core/parser] ──(Orchestrates via registry)──► [plugins/go-parser] + │ │ (AST extraction) + ▼ ▼ + [core/graph] ◄────────────────────────────── [UCM Code Units] + │ (Compile containment & caller edges) + ▼ + [core/storage] ──(SQL Transaction)───────────► [Local SQLite (cce.db)] + │ + ▼ + [services/cli] (cce search / callers / context) +``` + +--- + +## Directory Structure + +``` +cce-root/ +├── api/ # Protobuf specifications +│ └── v1/ +│ ├── cce_plugin.proto # gRPC plugin contract for parsers +│ └── ucm.proto # Universal Code Model schemas +├── core/ # Core Business Logic Engine (Reusable Go Module) +│ ├── model/ # Shared domain models (Repository, File, Symbol, Relationship, ChangeSet) +│ ├── indexer/ # Crawls local directories for target source files +│ ├── parser/ # Parser orchestration layer +│ │ ├── contracts/ # gRPC plugin client interfaces +│ │ ├── sdk/ # Conversion helpers mapping UCM Protobuf to domain model Symbols +│ │ ├── registry/ # Maps file extensions to active parser connections +│ │ ├── loader/ # Connects and registers parser clients +│ │ └── parser.go # Core parser orchestrator +│ ├── graph/ # Code-graph compilation +│ │ ├── model/ # Node/Edge graph models +│ │ ├── builder/ # Links symbols into structural call edges +│ │ ├── resolver/ # Cross-reference namespace import resolver +│ │ └── algorithms/ # Path finding and cycle detection +│ ├── context/ # Context attention window compiler +│ │ ├── resolver/ # Adjacent graph nodes resolver +│ │ ├── assembler/ # Compiles contextual segments into the context window +│ │ ├── cache/ # Thread-safe context memory cache +│ │ └── context.go # Context coordinator +│ ├── storage/ # Provider-agnostic Storage layer +│ │ ├── storage.go # Storage interface definition +│ │ ├── repository/ # Repository high-level query helpers +│ │ ├── sqlite/ # SQLite driver (CGO-free glebarez/go-sqlite) +│ │ ├── postgres/ # PostgreSQL driver skeleton (jackc/pgx) +│ │ └── memory/ # InMemory driver (thread-safe sync.Map mock) +│ ├── engine.go # High-level engine coordinator facade +│ └── go.mod +├── deployments/ # Infrastructure Configuration (Simplified) +│ ├── docker/ # Dockerfiles for CLI and Parser Plugin +│ │ ├── Dockerfile.cli +│ │ └── Dockerfile.go-parser +│ └── docker-compose.yml # Development dependencies (Postgres & Jaeger only) +├── plugins/ # AST Parser Plugins +│ └── go-parser/ # Go AST parser gRPC plugin (Go Module) +├── services/ # Applications / Clients +│ └── cli/ # Command Line Interface (Go Module) +│ ├── main.go # CLI command handlers and entry point +│ └── go.mod +├── shared/ # Common Libraries (Go Module) +│ ├── config/ # Configuration loader +│ ├── logger/ # slog structured logger +│ ├── tracer/ # OpenTelemetry tracer initializer +│ └── go.mod +├── Makefile # Task automation (build, test, format) +├── buf.yaml # Protobuf buf linter config +├── buf.gen.yaml # Protobuf buf code generation config +└── go.work # Go Workspace definition +``` + +--- + +## Storage Layer Schema + +For the MVP, we map all code units and graph dependencies onto two relational tables: + +1. **`nodes` Table**: + - `id` (TEXT PRIMARY KEY): Deterministic symbol hash. + - `repo_id` (TEXT): Unique ID of the repository. + - `file_path` (TEXT): Relative path to source file. + - `name` (TEXT): Short symbol name. + - `fully_qualified_name` (TEXT): Fully qualified symbol name. + - `kind` (INTEGER): Symbol type enum (e.g. FUNCTION, STRUCT, INTERFACE). + - `signature` (TEXT): Code signature. + - `documentation` (TEXT): Extracted comments/docstring. + +2. **`edges` Table**: + - `from_id` (TEXT), `to_id` (TEXT): Identifiers referencing nodes. + - `type` (INTEGER): Relationship type enum (e.g., `RelationTypeCalls`, `RelationTypeContains`). + - `file_path` (TEXT), `start_line` (INTEGER), `start_column` (INTEGER): Precise location of edge reference. + +--- + +## Build and Run + +### 1. Compile Protobufs +```bash +# Generate UCM and Plugin structs to shared/protocol +make proto-gen +``` + +### 2. Tidy and Test +```bash +# Tidy all module dependencies in the workspace +make tidy + +# Run all workspace tests (CGO-free, offline) +make test +``` + +### 3. Compile Binaries +```bash +# Compiles CLI 'cce' and plugin 'go-parser' to bin/ +make build +``` + +--- + +## MVP Workflows + +### Step 1: Index Repository +To walk, parse, and graph compile a target codebase: +```bash +# Runs synchronously and writes nodes & edges to a local cce.db file +./bin/cce index /path/to/target/repository +``` + +### Step 2: Query Context +Run queries against the generated database: +```bash +# Search for a symbol +./bin/cce search UserService + +# Trace who calls a function +./bin/cce callers CreateInvoice + +# Trace who is called by a function +./bin/cce callees CreateInvoice + +# Retrieve target code snippet, docstring, callers, and callees (attention window) +./bin/cce context CreateInvoice +``` diff --git a/api/openapi.yaml b/api/openapi.yaml new file mode 100644 index 0000000..689b6af --- /dev/null +++ b/api/openapi.yaml @@ -0,0 +1,181 @@ +openapi: 3.0.3 +info: + title: Codebase Context Engine (CCE) API + version: 1.0.0 + description: REST API for querying structural code context, call graphs, endpoints, database actions, and owners. +paths: + /api/v1/repositories: + post: + summary: Register a new repository for indexing + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - name + - clone_url + properties: + name: + type: string + clone_url: + type: string + branch: + type: string + default: main + responses: + '202': + description: Indexing job accepted + content: + application/json: + schema: + type: object + properties: + repository_id: + type: string + status: + type: string + + /api/v1/search/symbols: + get: + summary: Find symbols in the indexed codebase + parameters: + - name: query + in: query + required: true + schema: + type: string + - name: kind + in: query + schema: + type: string + enum: [CLASS, FUNCTION, STRUCT, INTERFACE] + responses: + '200': + description: List of matched symbols + content: + application/json: + schema: + type: array + items: + type: object + properties: + id: + type: string + name: + type: string + kind: + type: string + file_path: + type: string + start_line: + type: integer + end_line: + type: integer + + /api/v1/context/symbol: + post: + summary: Retrieve a rich context payload optimized for AI Agents and IDEs + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [repository_id, symbol_name] + properties: + repository_id: + type: string + commit_sha: + type: string + symbol_name: + type: string + depth_limits: + type: object + properties: + call_graph: + type: integer + default: 2 + dependencies: + type: integer + default: 1 + include_infrastructure: + type: boolean + default: true + include_ownership: + type: boolean + default: true + responses: + '200': + description: Context block containing symbol code, dependency mappings, databases, and owners. + content: + application/json: + schema: + type: object + properties: + target_symbol: + type: object + context_relationships: + type: object + + /api/v1/graph/traverse: + get: + summary: Get relationships (e.g. Callers, Implementations) + parameters: + - name: symbol_id + in: query + required: true + schema: + type: string + - name: direction + in: query + required: true + schema: + type: string + enum: [UPSTREAM, DOWNSTREAM] + - name: max_depth + in: query + schema: + type: integer + default: 3 + responses: + '200': + description: Returns nodes and edges array + content: + application/json: + schema: + type: object + properties: + nodes: + type: array + items: + type: object + edges: + type: array + items: + type: object + + /api/v1/analysis/impact: + get: + summary: Identify downstream impacts of changing a symbol + parameters: + - name: symbol_id + in: query + required: true + schema: + type: string + responses: + '200': + description: Return affected files, functions, and teams + content: + application/json: + schema: + type: object + properties: + affected_items: + type: array + items: + type: object + risk_index: + type: number diff --git a/api/v1/cce_plugin.proto b/api/v1/cce_plugin.proto new file mode 100644 index 0000000..e3d86f3 --- /dev/null +++ b/api/v1/cce_plugin.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package cce.plugin.v1; + +option go_package = "github.com/thinkbattleground/cce/pkg/protocol/v1/plugin"; + +import "v1/ucm.proto"; + +service LanguageParserPlugin { + rpc GetMetadata(Empty) returns (PluginMetadata); + rpc ParseFile(ParseRequest) returns (ParseResponse); +} + +message Empty {} + +message PluginMetadata { + string name = 1; + string version = 2; + repeated string supported_extensions = 3; +} + +message ParseRequest { + string file_path = 1; + bytes file_content = 2; +} + +message ParseResponse { + cce.v1.ucm.UniversalCodePayload payload = 1; + repeated string parsing_errors = 2; +} diff --git a/api/v1/ucm.proto b/api/v1/ucm.proto new file mode 100644 index 0000000..bd5118c --- /dev/null +++ b/api/v1/ucm.proto @@ -0,0 +1,60 @@ +syntax = "proto3"; + +package cce.v1.ucm; + +option go_package = "github.com/thinkbattleground/cce/pkg/protocol/v1/ucm"; + +enum SymbolKind { + SYMBOL_KIND_UNSPECIFIED = 0; + SYMBOL_KIND_MODULE = 1; + SYMBOL_KIND_PACKAGE = 2; + SYMBOL_KIND_CLASS = 3; + SYMBOL_KIND_STRUCT = 4; + SYMBOL_KIND_INTERFACE = 5; + SYMBOL_KIND_FUNCTION = 6; + SYMBOL_KIND_METHOD = 7; + SYMBOL_KIND_VARIABLE = 8; + SYMBOL_KIND_ANNOTATION = 9; +} + +enum RelationType { + RELATION_TYPE_UNSPECIFIED = 0; + RELATION_TYPE_CONTAINS = 1; // e.g. Class contains Method + RELATION_TYPE_CALLS = 2; // e.g. Function A calls Function B + RELATION_TYPE_IMPLEMENTS = 3; // e.g. Class C implements Interface D + RELATION_TYPE_REFERENCES = 4; // e.g. Function reads Variable + RELATION_TYPE_IMPORTS = 5; // e.g. File A imports File B +} + +message Location { + string file_path = 1; + int32 start_line = 2; + int32 start_column = 3; + int32 end_line = 4; + int32 end_column = 5; +} + +message CodeUnit { + string id = 1; // Globally unique deterministic hash + SymbolKind kind = 2; + string name = 3; + string fully_qualified_name = 4; + Location location = 5; + string signature = 6; + string documentation = 7; + map properties = 8; // Extensible metadata (annotations, modifiers) +} + +message Relationship { + string from_id = 1; // CodeUnit ID + string to_id = 2; // CodeUnit ID + RelationType type = 3; + Location location = 4; // Precise code location of the relation +} + +message UniversalCodePayload { + string repository_id = 1; + string commit_sha = 2; + repeated CodeUnit units = 3; + repeated Relationship relationships = 4; +} diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 0000000..dd97bb3 --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,8 @@ +version: v1 +plugins: + - name: go + out: shared/protocol + opt: paths=source_relative + - name: go-grpc + out: shared/protocol + opt: paths=source_relative diff --git a/buf.yaml b/buf.yaml new file mode 100644 index 0000000..2fc0281 --- /dev/null +++ b/buf.yaml @@ -0,0 +1,8 @@ +version: v1 +name: buf.build/thinkbattleground/cce +build: + roots: + - api +lint: + use: + - DEFAULT diff --git a/core/context/assembler/assembler.go b/core/context/assembler/assembler.go new file mode 100644 index 0000000..c975090 --- /dev/null +++ b/core/context/assembler/assembler.go @@ -0,0 +1,20 @@ +package assembler + +import "github.com/thinkbattleground/cce/core/model" + +type SymbolContext struct { + TargetSymbol model.Symbol `json:"target_symbol"` + Callers []model.Symbol `json:"callers"` + Callees []model.Symbol `json:"callees"` + Relations []model.Relationship `json:"relations"` +} + +// Assemble compiles context segments into a unified SymbolContext payload. +func Assemble(target model.Symbol, callers, callees []model.Symbol, relations []model.Relationship) *SymbolContext { + return &SymbolContext{ + TargetSymbol: target, + Callers: callers, + Callees: callees, + Relations: relations, + } +} diff --git a/core/context/cache/cache.go b/core/context/cache/cache.go new file mode 100644 index 0000000..716bc9d --- /dev/null +++ b/core/context/cache/cache.go @@ -0,0 +1,37 @@ +package cache + +import ( + "sync" + + "github.com/thinkbattleground/cce/core/context/assembler" +) + +type ContextCache struct { + mu sync.RWMutex + cache map[string]*assembler.SymbolContext +} + +func NewContextCache() *ContextCache { + return &ContextCache{ + cache: make(map[string]*assembler.SymbolContext), + } +} + +func (c *ContextCache) Get(key string) (*assembler.SymbolContext, bool) { + c.mu.RLock() + defer c.mu.RUnlock() + res, ok := c.cache[key] + return res, ok +} + +func (c *ContextCache) Set(key string, val *assembler.SymbolContext) { + c.mu.Lock() + defer c.mu.Unlock() + c.cache[key] = val +} + +func (c *ContextCache) Flush() { + c.mu.Lock() + defer c.mu.Unlock() + c.cache = make(map[string]*assembler.SymbolContext) +} diff --git a/core/context/context.go b/core/context/context.go new file mode 100644 index 0000000..472d3de --- /dev/null +++ b/core/context/context.go @@ -0,0 +1,53 @@ +package context + +import ( + "context" + "fmt" + + "github.com/thinkbattleground/cce/core/context/assembler" + "github.com/thinkbattleground/cce/core/context/cache" + "github.com/thinkbattleground/cce/core/context/resolver" + "github.com/thinkbattleground/cce/core/storage" +) + +type ContextEngine struct { + resolver *resolver.ContextResolver + cache *cache.ContextCache + store storage.Storage +} + +func NewContextEngine(store storage.Storage) *ContextEngine { + return &ContextEngine{ + resolver: resolver.NewContextResolver(store), + cache: cache.NewContextCache(), + store: store, + } +} + +// CompileContext resolves structural code neighborhoods and returns the assembled SymbolContext. +func (e *ContextEngine) CompileContext(ctx context.Context, symbolName string) (*assembler.SymbolContext, error) { + // 1. Check cache + if cached, ok := e.cache.Get(symbolName); ok { + return cached, nil + } + + // 2. Lookup symbol ID from database + targetSymbol, err := e.store.GetSymbolByName(ctx, symbolName) + if err != nil { + return nil, fmt.Errorf("failed to locate symbol '%s': %w", symbolName, err) + } + + // 3. Resolve adjacent graph nodes and edges + target, callers, callees, relations, err := e.resolver.ResolveNeighbors(ctx, targetSymbol.ID) + if err != nil { + return nil, fmt.Errorf("failed to resolve context neighbors: %w", err) + } + + // 4. Assemble context payload + sc := assembler.Assemble(target, callers, callees, relations) + + // 5. Store in cache + e.cache.Set(symbolName, sc) + + return sc, nil +} diff --git a/core/context/resolver/resolver.go b/core/context/resolver/resolver.go new file mode 100644 index 0000000..c88431c --- /dev/null +++ b/core/context/resolver/resolver.go @@ -0,0 +1,41 @@ +package resolver + +import ( + "context" + + "github.com/thinkbattleground/cce/core/model" + "github.com/thinkbattleground/cce/core/storage" +) + +type ContextResolver struct { + store storage.Storage +} + +func NewContextResolver(store storage.Storage) *ContextResolver { + return &ContextResolver{store: store} +} + +// ResolveNeighbors fetches all nodes and relationships immediately adjacent to the symbol. +func (r *ContextResolver) ResolveNeighbors(ctx context.Context, symbolID string) (model.Symbol, []model.Symbol, []model.Symbol, []model.Relationship, error) { + target, err := r.store.GetSymbol(ctx, symbolID) + if err != nil { + return model.Symbol{}, nil, nil, nil, err + } + + callers, err := r.store.GetCallers(ctx, symbolID) + if err != nil { + return model.Symbol{}, nil, nil, nil, err + } + + callees, err := r.store.GetCallees(ctx, symbolID) + if err != nil { + return model.Symbol{}, nil, nil, nil, err + } + + relations, err := r.store.GetRelationships(ctx, symbolID) + if err != nil { + return model.Symbol{}, nil, nil, nil, err + } + + return target, callers, callees, relations, nil +} diff --git a/core/engine.go b/core/engine.go new file mode 100644 index 0000000..a598f97 --- /dev/null +++ b/core/engine.go @@ -0,0 +1,114 @@ +package core + +import ( + "context" + "fmt" + "log/slog" + + corectx "github.com/thinkbattleground/cce/core/context" + "github.com/thinkbattleground/cce/core/context/assembler" + "github.com/thinkbattleground/cce/core/graph/builder" + "github.com/thinkbattleground/cce/core/indexer" + "github.com/thinkbattleground/cce/core/indexer/gogit" + "github.com/thinkbattleground/cce/core/model" + "github.com/thinkbattleground/cce/core/parser" + "github.com/thinkbattleground/cce/core/parser/registry" + "github.com/thinkbattleground/cce/core/parser/sdk" + "github.com/thinkbattleground/cce/core/storage" +) + +type Engine struct { + db storage.Storage + indexer *indexer.RepositoryIndexer + parser *parser.ParserOrchestrator + builder *builder.RelationshipBuilder + ctxEng *corectx.ContextEngine +} + +func NewEngine(db storage.Storage, reg *registry.ParserRegistry) *Engine { + return &Engine{ + db: db, + indexer: indexer.NewRepositoryIndexer(gogit.NewGoGitClient()), + parser: parser.NewParserOrchestrator(reg), + builder: builder.NewRelationshipBuilder(), + ctxEng: corectx.NewContextEngine(db), + } +} + +// IndexRepository walks directories, parses files, compiles edges, and commits symbols. +func (e *Engine) IndexRepository(ctx context.Context, repoPath string) error { + slog.Info("starting synchronous workspace ingestion", "path", repoPath) + + // 1. Register repository + repo := model.Repository{ + ID: "local-workspace", + Name: "local-workspace", + } + err := e.db.SaveRepository(ctx, repo) + if err != nil { + return fmt.Errorf("failed to register repository metadata: %w", err) + } + + // 2. Crawl files + files, err := e.indexer.IndexLocalDir(ctx, repoPath) + if err != nil { + return fmt.Errorf("indexing walk failed: %w", err) + } + + var allSymbols []model.Symbol + var allRelations []model.Relationship + + // 3. Coordinate parsing on registry clients + for _, f := range files { + resp, err := e.parser.ParseSourceFile(ctx, f.FilePath, f.Content) + if err != nil { + slog.Debug("skipping file (unregistered or failed parsing)", "path", f.FilePath, "error", err) + continue + } + + if resp != nil && resp.Payload != nil { + symbols, relations := sdk.ConvertUCMToDomain(resp.Payload) + allSymbols = append(allSymbols, symbols...) + allRelations = append(allRelations, relations...) + } + } + + // 4. Compile and link graph relationships + compiledRelations, err := e.builder.CompileRelationships(ctx, allSymbols, allRelations) + if err != nil { + return fmt.Errorf("failed to compile relationships: %w", err) + } + + // 5. Commit to database + err = e.db.SaveSymbols(ctx, repo.ID, allSymbols, compiledRelations) + if err != nil { + return fmt.Errorf("failed to save code models to store: %w", err) + } + + slog.Info("repository indexing completed successfully", "total_symbols", len(allSymbols)) + return nil +} + +func (e *Engine) Search(ctx context.Context, query string) ([]model.Symbol, error) { + return e.db.SearchSymbols(ctx, query, model.SymbolKindUnspecified) +} + +func (e *Engine) Callers(ctx context.Context, name string) ([]model.Symbol, error) { + target, err := e.db.GetSymbolByName(ctx, name) + if err != nil { + return nil, err + } + return e.db.GetCallers(ctx, target.ID) +} + +func (e *Engine) Callees(ctx context.Context, name string) ([]model.Symbol, error) { + target, err := e.db.GetSymbolByName(ctx, name) + if err != nil { + return nil, err + } + return e.db.GetCallees(ctx, target.ID) +} + +func (e *Engine) Context(ctx context.Context, name string) (*assembler.SymbolContext, error) { + return e.ctxEng.CompileContext(ctx, name) +} diff --git a/core/go.mod b/core/go.mod new file mode 100644 index 0000000..816961b --- /dev/null +++ b/core/go.mod @@ -0,0 +1,55 @@ +module github.com/thinkbattleground/cce/core + +go 1.22.0 + +require ( + github.com/glebarez/go-sqlite v1.22.0 + github.com/go-git/go-git/v5 v5.12.0 + github.com/golang/protobuf v1.5.4 + github.com/jackc/pgx/v5 v5.6.0 + github.com/stretchr/testify v1.9.0 + github.com/thinkbattleground/cce/shared v0.0.0 + google.golang.org/grpc v1.64.0 +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/cloudflare/circl v1.3.7 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/tools v0.16.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.37.6 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.7.2 // indirect + modernc.org/sqlite v1.28.0 // indirect +) + +replace github.com/thinkbattleground/cce/shared => ../shared diff --git a/core/go.sum b/core/go.sum new file mode 100644 index 0000000..6000aa3 --- /dev/null +++ b/core/go.sum @@ -0,0 +1,184 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= +github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw= +modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ= +modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= diff --git a/core/graph/algorithms/algorithms.go b/core/graph/algorithms/algorithms.go new file mode 100644 index 0000000..256a671 --- /dev/null +++ b/core/graph/algorithms/algorithms.go @@ -0,0 +1,15 @@ +package algorithms + +import "github.com/thinkbattleground/cce/core/graph/model" + +// FindPath traces reachability path between two symbol IDs in the graph. +func FindPath(g *model.Graph, fromID, toID string) []*model.Edge { + // BFS or DFS path finding skeleton + return nil +} + +// DetectCycles checks if the code structure contains circular dependencies. +func DetectCycles(g *model.Graph) [][]*model.Node { + // Tarjan's or Johnson's cycle detection skeleton + return nil +} diff --git a/core/graph/builder/builder.go b/core/graph/builder/builder.go new file mode 100644 index 0000000..d1bdf05 --- /dev/null +++ b/core/graph/builder/builder.go @@ -0,0 +1,33 @@ +package builder + +import ( + "context" + "log/slog" + + "github.com/thinkbattleground/cce/core/model" +) + +type RelationshipBuilder struct{} + +func NewRelationshipBuilder() *RelationshipBuilder { + return &RelationshipBuilder{} +} + +// CompileRelationships analyzes extracted symbols and builds structural edges. +func (b *RelationshipBuilder) CompileRelationships(ctx context.Context, symbols []model.Symbol, rawRelations []model.Relationship) ([]model.Relationship, error) { + slog.Info("compiling relationships", "symbols", len(symbols), "raw_relations", len(rawRelations)) + + // In the MVP skeleton, we return the relationships. + // We can add mock relationships for structural verification if needed: + compiled := append([]model.Relationship{}, rawRelations...) + if len(symbols) > 1 { + slog.Debug("adding caller-callee link for symbols") + compiled = append(compiled, model.Relationship{ + FromID: symbols[0].ID, + ToID: symbols[1].ID, + Type: model.RelationTypeCalls, + }) + } + + return compiled, nil +} diff --git a/core/graph/model/model.go b/core/graph/model/model.go new file mode 100644 index 0000000..9643888 --- /dev/null +++ b/core/graph/model/model.go @@ -0,0 +1,34 @@ +package model + +import "github.com/thinkbattleground/cce/core/model" + +// Node wraps a domain symbol in the code-graph context. +type Node struct { + Symbol model.Symbol +} + +// Edge wraps a domain relationship in the code-graph context. +type Edge struct { + Relationship model.Relationship +} + +// Graph represents a memory-efficient compilation of code structures. +type Graph struct { + Nodes map[string]*Node + Edges []*Edge +} + +func NewGraph() *Graph { + return &Graph{ + Nodes: make(map[string]*Node), + Edges: []*Edge{}, + } +} + +func (g *Graph) AddNode(sym model.Symbol) { + g.Nodes[sym.ID] = &Node{Symbol: sym} +} + +func (g *Graph) AddEdge(rel model.Relationship) { + g.Edges = append(g.Edges, &Edge{Relationship: rel}) +} diff --git a/core/graph/resolver/resolver.go b/core/graph/resolver/resolver.go new file mode 100644 index 0000000..7203222 --- /dev/null +++ b/core/graph/resolver/resolver.go @@ -0,0 +1,20 @@ +package resolver + +import ( + "context" + "log/slog" + + "github.com/thinkbattleground/cce/core/model" +) + +type SymbolResolver struct{} + +func NewSymbolResolver() *SymbolResolver { + return &SymbolResolver{} +} + +// ResolveReferences matches unresolved names to fully qualified targets. +func (r *SymbolResolver) ResolveReferences(ctx context.Context, symbols []model.Symbol) ([]model.Relationship, error) { + slog.Info("resolving cross-reference namespaces", "symbols", len(symbols)) + return nil, nil +} diff --git a/core/indexer/errors.go b/core/indexer/errors.go new file mode 100644 index 0000000..de9a207 --- /dev/null +++ b/core/indexer/errors.go @@ -0,0 +1,39 @@ +package indexer + +import "fmt" + +type ErrorCode string + +const ( + CodeRepositoryNotFound ErrorCode = "REPOSITORY_NOT_FOUND" + CodeAuthenticationFailed ErrorCode = "AUTHENTICATION_FAILED" + CodeCommitNotFound ErrorCode = "COMMIT_NOT_FOUND" + CodeCheckoutFailed ErrorCode = "CHECKOUT_FAILED" + CodeGitOperationFailed ErrorCode = "GIT_OPERATION_FAILED" +) + +// DomainError represents a rich domain error for indexer operations. +type DomainError struct { + Code ErrorCode + Message string + Err error +} + +func (e *DomainError) Error() string { + if e.Err != nil { + return fmt.Sprintf("[%s] %s: %v", e.Code, e.Message, e.Err) + } + return fmt.Sprintf("[%s] %s", e.Code, e.Message) +} + +func (e *DomainError) Unwrap() error { + return e.Err +} + +func NewDomainError(code ErrorCode, message string, err error) *DomainError { + return &DomainError{ + Code: code, + Message: message, + Err: err, + } +} diff --git a/core/indexer/git.go b/core/indexer/git.go new file mode 100644 index 0000000..d3f8c61 --- /dev/null +++ b/core/indexer/git.go @@ -0,0 +1,40 @@ +package indexer + +import ( + "context" + + "github.com/thinkbattleground/cce/core/model" +) + +// GitClient defines the interface for provisioning Git repositories. +type GitClient interface { + // Clone checks out a remote git repository to a local path. + Clone(ctx context.Context, url, localPath, branch string) (GitRepository, error) + + // Open opens an existing git repository in a local path. + Open(ctx context.Context, localPath string) (GitRepository, error) +} + +// GitRepository defines operations on an opened or cloned git repository. +type GitRepository interface { + // Pull fetches and merges the latest changes from origin. + Pull(ctx context.Context) error + + // Checkout switches the repository state to the target branch or commit. + Checkout(ctx context.Context, branchOrCommit string) error + + // GetMetadata returns Repository metadata info. + GetMetadata(ctx context.Context) (model.Repository, error) + + // GetHEAD retrieves the latest commit info. + GetHEAD(ctx context.Context) (model.Commit, error) + + // GetCommitHistory retrieves a list of past commits up to the limit. + GetCommitHistory(ctx context.Context, limit int) ([]model.Commit, error) + + // CompareCommits diffs two commits and returns a ChangeSet. + CompareCommits(ctx context.Context, fromHash, toHash string) (model.ChangeSet, error) + + // ReadFileAtCommit extracts file content bytes at a specific revision. + ReadFileAtCommit(ctx context.Context, commitHash, filePath string) ([]byte, error) +} diff --git a/core/indexer/gogit/client.go b/core/indexer/gogit/client.go new file mode 100644 index 0000000..0b4e87e --- /dev/null +++ b/core/indexer/gogit/client.go @@ -0,0 +1,75 @@ +package gogit + +import ( + "context" + "fmt" + "log/slog" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/thinkbattleground/cce/core/indexer" +) + +type GoGitClient struct{} + +func NewGoGitClient() *GoGitClient { + return &GoGitClient{} +} + +func (c *GoGitClient) Clone(ctx context.Context, url, localPath, branch string) (indexer.GitRepository, error) { + slog.Info("cloning git repository", "url", url, "path", localPath, "branch", branch) + + cloneOpts := &git.CloneOptions{ + URL: url, + ReferenceName: plumbing.NewBranchReferenceName(branch), + SingleBranch: true, + Progress: nil, + } + + repo, err := git.PlainCloneContext(ctx, localPath, false, cloneOpts) + if err != nil { + return nil, indexer.NewDomainError( + indexer.CodeGitOperationFailed, + fmt.Sprintf("failed to clone repository to %s", localPath), + err, + ) + } + + return NewGoGitRepository(repo, url, branch), nil +} + +func (c *GoGitClient) Open(ctx context.Context, localPath string) (indexer.GitRepository, error) { + slog.Info("opening local git repository", "path", localPath) + + repo, err := git.PlainOpen(localPath) + if err != nil { + if err == git.ErrRepositoryNotExists { + return nil, indexer.NewDomainError( + indexer.CodeRepositoryNotFound, + fmt.Sprintf("no git repository found at %s", localPath), + err, + ) + } + return nil, indexer.NewDomainError( + indexer.CodeGitOperationFailed, + fmt.Sprintf("failed to open repository at %s", localPath), + err, + ) + } + + // Fetch current branch reference if possible + branch := "main" + headRef, err := repo.Head() + if err == nil && headRef.Name().IsBranch() { + branch = headRef.Name().Short() + } + + // Fetch origin URL + remoteURL := "" + remotes, err := repo.Remotes() + if err == nil && len(remotes) > 0 { + remoteURL = remotes[0].Config().URLs[0] + } + + return NewGoGitRepository(repo, remoteURL, branch), nil +} diff --git a/core/indexer/gogit/repository.go b/core/indexer/gogit/repository.go new file mode 100644 index 0000000..08f76ae --- /dev/null +++ b/core/indexer/gogit/repository.go @@ -0,0 +1,226 @@ +package gogit + +import ( + "context" + "fmt" + "io" + "log/slog" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/thinkbattleground/cce/core/indexer" + "github.com/thinkbattleground/cce/core/model" +) + +type GoGitRepository struct { + repo *git.Repository + remoteURL string + branch string +} + +func NewGoGitRepository(repo *git.Repository, remoteURL, branch string) *GoGitRepository { + return &GoGitRepository{ + repo: repo, + remoteURL: remoteURL, + branch: branch, + } +} + +func (r *GoGitRepository) Pull(ctx context.Context) error { + slog.Info("pulling latest changes from origin") + + w, err := r.repo.Worktree() + if err != nil { + return indexer.NewDomainError(indexer.CodeGitOperationFailed, "failed to get repo worktree", err) + } + + pullOpts := &git.PullOptions{ + RemoteName: "origin", + ReferenceName: plumbing.NewBranchReferenceName(r.branch), + SingleBranch: true, + } + + err = w.PullContext(ctx, pullOpts) + if err != nil && err != git.NoErrAlreadyUpToDate { + return indexer.NewDomainError(indexer.CodeGitOperationFailed, "pull failed", err) + } + + return nil +} + +func (r *GoGitRepository) Checkout(ctx context.Context, branchOrCommit string) error { + slog.Info("checking out revision", "revision", branchOrCommit) + + w, err := r.repo.Worktree() + if err != nil { + return indexer.NewDomainError(indexer.CodeGitOperationFailed, "failed to get repo worktree", err) + } + + // Try checking out by commit hash first, then fallback to branch + hash := plumbing.NewHash(branchOrCommit) + checkoutOpts := &git.CheckoutOptions{ + Force: true, + } + + if hash.IsZero() { + checkoutOpts.Branch = plumbing.NewBranchReferenceName(branchOrCommit) + } else { + checkoutOpts.Hash = hash + } + + err = w.Checkout(checkoutOpts) + if err != nil { + return indexer.NewDomainError(indexer.CodeCheckoutFailed, fmt.Sprintf("failed to checkout %s", branchOrCommit), err) + } + + return nil +} + +func (r *GoGitRepository) GetMetadata(ctx context.Context) (model.Repository, error) { + return model.Repository{ + ID: r.remoteURL, // Using remote URL as unique ID + Name: r.remoteURL, + CloneURL: r.remoteURL, + Branch: r.branch, + }, nil +} + +func (r *GoGitRepository) GetHEAD(ctx context.Context) (model.Commit, error) { + ref, err := r.repo.Head() + if err != nil { + return model.Commit{}, indexer.NewDomainError(indexer.CodeCommitNotFound, "failed to get HEAD ref", err) + } + + c, err := r.repo.CommitObject(ref.Hash()) + if err != nil { + return model.Commit{}, indexer.NewDomainError(indexer.CodeCommitNotFound, "failed to get HEAD commit object", err) + } + + return model.Commit{ + Hash: c.Hash.String(), + Author: c.Author.String(), + Message: c.Message, + Timestamp: c.Author.When.Unix(), + }, nil +} + +func (r *GoGitRepository) GetCommitHistory(ctx context.Context, limit int) ([]model.Commit, error) { + ref, err := r.repo.Head() + if err != nil { + return nil, indexer.NewDomainError(indexer.CodeCommitNotFound, "failed to get HEAD ref", err) + } + + cIter, err := r.repo.Log(&git.LogOptions{From: ref.Hash()}) + if err != nil { + return nil, indexer.NewDomainError(indexer.CodeGitOperationFailed, "failed to fetch log history", err) + } + defer cIter.Close() + + var commits []model.Commit + count := 0 + err = cIter.ForEach(func(c *object.Commit) error { + if limit > 0 && count >= limit { + return io.EOF + } + + commits = append(commits, model.Commit{ + Hash: c.Hash.String(), + Author: c.Author.String(), + Message: c.Message, + Timestamp: c.Author.When.Unix(), + }) + count++ + return nil + }) + + if err != nil && err != io.EOF { + return nil, indexer.NewDomainError(indexer.CodeGitOperationFailed, "failed to iterate log", err) + } + + return commits, nil +} + +func (r *GoGitRepository) CompareCommits(ctx context.Context, fromHash, toHash string) (model.ChangeSet, error) { + slog.Info("comparing commits", "from", fromHash, "to", toHash) + + fromCommit, err := r.repo.CommitObject(plumbing.NewHash(fromHash)) + if err != nil { + return model.ChangeSet{}, indexer.NewDomainError(indexer.CodeCommitNotFound, fmt.Sprintf("from commit %s not found", fromHash), err) + } + + toCommit, err := r.repo.CommitObject(plumbing.NewHash(toHash)) + if err != nil { + return model.ChangeSet{}, indexer.NewDomainError(indexer.CodeCommitNotFound, fmt.Sprintf("to commit %s not found", toHash), err) + } + + fromTree, err := fromCommit.Tree() + if err != nil { + return model.ChangeSet{}, indexer.NewDomainError(indexer.CodeGitOperationFailed, "failed to get from tree", err) + } + + toTree, err := toCommit.Tree() + if err != nil { + return model.ChangeSet{}, indexer.NewDomainError(indexer.CodeGitOperationFailed, "failed to get to tree", err) + } + + changes, err := fromTree.Diff(toTree) + if err != nil { + return model.ChangeSet{}, indexer.NewDomainError(indexer.CodeGitOperationFailed, "failed to diff trees", err) + } + + cs := model.ChangeSet{ + CommitSHA: toHash, + } + + for _, change := range changes { + // Insert + if change.From.Name == "" && change.To.Name != "" { + cs.FilesAdded = append(cs.FilesAdded, change.To.Name) + } else if change.From.Name != "" && change.To.Name == "" { + // Delete + cs.FilesDeleted = append(cs.FilesDeleted, change.From.Name) + } else if change.From.Name != "" && change.To.Name != "" { + // Modify or Rename + if change.From.Name != change.To.Name { + // Treat rename as delete and addition + cs.FilesDeleted = append(cs.FilesDeleted, change.From.Name) + cs.FilesAdded = append(cs.FilesAdded, change.To.Name) + } else { + cs.FilesModified = append(cs.FilesModified, change.To.Name) + } + } + } + + return cs, nil +} + +func (r *GoGitRepository) ReadFileAtCommit(ctx context.Context, commitHash, filePath string) ([]byte, error) { + c, err := r.repo.CommitObject(plumbing.NewHash(commitHash)) + if err != nil { + return nil, indexer.NewDomainError(indexer.CodeCommitNotFound, fmt.Sprintf("commit %s not found", commitHash), err) + } + + tree, err := c.Tree() + if err != nil { + return nil, indexer.NewDomainError(indexer.CodeGitOperationFailed, "failed to get commit tree", err) + } + + file, err := tree.File(filePath) + if err != nil { + return nil, indexer.NewDomainError(indexer.CodeGitOperationFailed, fmt.Sprintf("file %s not found in commit %s", filePath, commitHash), err) + } + + reader, err := file.Reader() + if err != nil { + return nil, indexer.NewDomainError(indexer.CodeGitOperationFailed, "failed to open file reader", err) + } + defer reader.Close() + + content, err := io.ReadAll(reader) + if err != nil { + return nil, indexer.NewDomainError(indexer.CodeGitOperationFailed, "failed to read file content", err) + } + + return content, nil +} diff --git a/core/indexer/indexer.go b/core/indexer/indexer.go new file mode 100644 index 0000000..e3a2d17 --- /dev/null +++ b/core/indexer/indexer.go @@ -0,0 +1,69 @@ +package indexer + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "log/slog" + "os" + "path/filepath" + + "github.com/thinkbattleground/cce/core/model" +) + +type RepositoryIndexer struct { + client GitClient +} + +func NewRepositoryIndexer(client GitClient) *RepositoryIndexer { + return &RepositoryIndexer{client: client} +} + +// CalculateHash returns the SHA256 checksum of the given bytes. +func (i *RepositoryIndexer) CalculateHash(content []byte) string { + hasher := sha256.New() + hasher.Write(content) + return hex.EncodeToString(hasher.Sum(nil)) +} + +// InitializeRepository opens or clones the target repository and checks out the desired branch. +func (i *RepositoryIndexer) InitializeRepository(ctx context.Context, url, localPath, branch string) (GitRepository, error) { + slog.Info("initializing git repository", "url", url, "localPath", localPath, "branch", branch) + + var repo GitRepository + var err error + + // Check if repository already exists locally + gitDir := filepath.Join(localPath, ".git") + if _, errStat := os.Stat(gitDir); errStat == nil { + slog.Info("git repository already exists locally, opening it") + repo, err = i.client.Open(ctx, localPath) + if err != nil { + return nil, fmt.Errorf("failed to open local repository: %w", err) + } + + err = repo.Checkout(ctx, branch) + if err != nil { + slog.Warn("checkout branch failed, attempting pull first", "branch", branch, "error", err) + } + + err = repo.Pull(ctx) + if err != nil { + slog.Warn("pulling latest changes failed", "error", err) + } + } else { + slog.Info("cloning remote repository") + repo, err = i.client.Clone(ctx, url, localPath, branch) + if err != nil { + return nil, fmt.Errorf("failed to clone repository: %w", err) + } + } + + return repo, nil +} + +// BuildChangeSet maps diffs between two commits into a domain model.ChangeSet. +func (i *RepositoryIndexer) BuildChangeSet(ctx context.Context, repo GitRepository, fromHash, toHash string) (model.ChangeSet, error) { + return repo.CompareCommits(ctx, fromHash, toHash) +} diff --git a/core/indexer/indexer_test.go b/core/indexer/indexer_test.go new file mode 100644 index 0000000..eab820e --- /dev/null +++ b/core/indexer/indexer_test.go @@ -0,0 +1,140 @@ +package indexer_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/thinkbattleground/cce/core/indexer" + "github.com/thinkbattleground/cce/core/indexer/gogit" +) + +func createTempGitRepo(t *testing.T) (string, *git.Repository) { + tmpDir, err := os.MkdirTemp("", "cce-test-git-*") + require.NoError(t, err) + + repo, err := git.PlainInit(tmpDir, false) + require.NoError(t, err) + + return tmpDir, repo +} + +func commitFile(t *testing.T, repo *git.Repository, repoPath, filename, content, message string) string { + w, err := repo.Worktree() + require.NoError(t, err) + + filePath := filepath.Join(repoPath, filename) + err = os.WriteFile(filePath, []byte(content), 0644) + require.NoError(t, err) + + _, err = w.Add(filename) + require.NoError(t, err) + + commit, err := w.Commit(message, &git.CommitOptions{ + Author: &object.Signature{ + Name: "Test Author", + Email: "test@example.com", + When: time.Now(), + }, + }) + require.NoError(t, err) + + return commit.String() +} + +func deleteFile(t *testing.T, repo *git.Repository, repoPath, filename, message string) string { + w, err := repo.Worktree() + require.NoError(t, err) + + _, err = w.Remove(filename) + require.NoError(t, err) + + commit, err := w.Commit(message, &git.CommitOptions{ + Author: &object.Signature{ + Name: "Test Author", + Email: "test@example.com", + When: time.Now(), + }, + }) + require.NoError(t, err) + + return commit.String() +} + +func TestGoGitIntegration(t *testing.T) { + repoPath, rawRepo := createTempGitRepo(t) + defer os.RemoveAll(repoPath) + + ctx := context.Background() + + // 1. Commit first file + c1 := commitFile(t, rawRepo, repoPath, "hello.go", "package hello\n", "Initial commit") + + // 2. Open repo via client + client := gogit.NewGoGitClient() + repo, err := client.Open(ctx, repoPath) + require.NoError(t, err) + + // 3. Verify HEAD commit metadata + head, err := repo.GetHEAD(ctx) + require.NoError(t, err) + assert.Equal(t, c1, head.Hash) + assert.Contains(t, head.Author, "Test Author") + assert.Contains(t, head.Message, "Initial commit") + + // 4. Commit second file (addition) and modify first + commitFile(t, rawRepo, repoPath, "world.go", "package world\n", "Add world") + c3 := commitFile(t, rawRepo, repoPath, "hello.go", "package hello\nimport \"fmt\"\n", "Modify hello") + + // 5. Compare commits c1 and c3 + cs, err := repo.CompareCommits(ctx, c1, c3) + require.NoError(t, err) + assert.Equal(t, c3, cs.CommitSHA) + + // Hello.go was modified, world.go was added + assert.Contains(t, cs.FilesModified, "hello.go") + assert.Contains(t, cs.FilesAdded, "world.go") + assert.Empty(t, cs.FilesDeleted) + + // 6. Delete file + c4 := deleteFile(t, rawRepo, repoPath, "world.go", "Delete world") + cs2, err := repo.CompareCommits(ctx, c3, c4) + require.NoError(t, err) + assert.Contains(t, cs2.FilesDeleted, "world.go") + + // 7. Read file content at a specific commit + content, err := repo.ReadFileAtCommit(ctx, c1, "hello.go") + require.NoError(t, err) + assert.Equal(t, "package hello\n", string(content)) +} + +func TestRepositoryIndexer_LocalDir(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "cce-test-localdir-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + err = os.WriteFile(filepath.Join(tmpDir, "main.go"), []byte("package main\n"), 0644) + require.NoError(t, err) + + err = os.WriteFile(filepath.Join(tmpDir, "README.md"), []byte("# Title\n"), 0644) + require.NoError(t, err) + + indexerSvc := indexer.NewRepositoryIndexer(nil) + entries, err := indexerSvc.IndexLocalDir(context.Background(), tmpDir) + require.NoError(t, err) + + // Should only scan Go files + require.Len(t, entries, 1) + assert.Equal(t, "main.go", entries[0].FilePath) + assert.Equal(t, "package main\n", string(entries[0].Content)) + + // Verify SHA256 hashing + hash := indexerSvc.CalculateHash([]byte("package main\n")) + assert.Equal(t, hash, entries[0].ContentHash) +} diff --git a/core/indexer/local.go b/core/indexer/local.go new file mode 100644 index 0000000..228b8de --- /dev/null +++ b/core/indexer/local.go @@ -0,0 +1,76 @@ +package indexer + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/thinkbattleground/cce/core/model" +) + +// IndexLocalDir crawls the target directory and loads Go files (without Git history). +func (i *RepositoryIndexer) IndexLocalDir(ctx context.Context, dirPath string) ([]model.FileEntry, error) { + slog.Info("indexing local directory (no git mode)", "path", dirPath) + + var entries []model.FileEntry + err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + if info.Name() == ".git" || info.Name() == "vendor" || info.Name() == "node_modules" { + return filepath.SkipDir + } + return nil + } + + if !strings.HasSuffix(info.Name(), ".go") { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", path, err) + } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + return fmt.Errorf("failed to read file %s: %w", path, err) + } + + relPath, err := filepath.Rel(dirPath, path) + if err != nil { + relPath = path + } + + // Normalize slash characters to Unix forward slashes for portability + relPath = filepath.ToSlash(relPath) + + entries = append(entries, model.FileEntry{ + FilePath: relPath, + Content: content, + ContentHash: i.CalculateHash(content), + }) + + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to walk directory: %w", err) + } + + slog.Info("local directory walk complete", "files_found", len(entries)) + return entries, nil +} diff --git a/core/model/model.go b/core/model/model.go new file mode 100644 index 0000000..a45bbe0 --- /dev/null +++ b/core/model/model.go @@ -0,0 +1,148 @@ +package model + +// SymbolKind represents the type of a code unit symbol. +type SymbolKind int + +const ( + SymbolKindUnspecified SymbolKind = iota + SymbolKindModule + SymbolKindPackage + SymbolKindClass + SymbolKindStruct + SymbolKindInterface + SymbolKindFunction + SymbolKindMethod + SymbolKindVariable + SymbolKindAnnotation + SymbolKindConstant + SymbolKindComment + SymbolKindImport +) + +func (k SymbolKind) String() string { + switch k { + case SymbolKindModule: + return "MODULE" + case SymbolKindPackage: + return "PACKAGE" + case SymbolKindClass: + return "CLASS" + case SymbolKindStruct: + return "STRUCT" + case SymbolKindInterface: + return "INTERFACE" + case SymbolKindFunction: + return "FUNCTION" + case SymbolKindMethod: + return "METHOD" + case SymbolKindVariable: + return "VARIABLE" + case SymbolKindAnnotation: + return "ANNOTATION" + case SymbolKindConstant: + return "CONSTANT" + case SymbolKindComment: + return "COMMENT" + case SymbolKindImport: + return "IMPORT" + default: + return "UNSPECIFIED" + } +} + +// RelationType represents the type of connection between symbols. +type RelationType int + +const ( + RelationTypeUnspecified RelationType = iota + RelationTypeContains + RelationTypeCalls + RelationTypeImplements + RelationTypeReferences + RelationTypeImports +) + +func (t RelationType) String() string { + switch t { + case RelationTypeContains: + return "CONTAINS" + case RelationTypeCalls: + return "CALLS" + case RelationTypeImplements: + return "IMPLEMENTS" + case RelationTypeReferences: + return "REFERENCES" + case RelationTypeImports: + return "IMPORTS" + default: + return "UNSPECIFIED" + } +} + +// Repository represents a git or local source code repository. +type Repository struct { + ID string `json:"id"` + Name string `json:"name"` + CloneURL string `json:"clone_url"` + Branch string `json:"branch"` +} + +// Location represents a precise position inside a source code file. +type Location struct { + FilePath string `json:"file_path"` + StartLine int `json:"start_line"` + StartColumn int `json:"start_column"` + EndLine int `json:"end_line"` + EndColumn int `json:"end_column"` +} + +// Symbol represents an extracted code element (class, function, variable, etc.). +type Symbol struct { + ID string `json:"id"` + RepoID string `json:"repo_id"` + Name string `json:"name"` + FullyQualifiedName string `json:"fully_qualified_name"` + Kind SymbolKind `json:"kind"` + Location Location `json:"location"` + Signature string `json:"signature"` + Documentation string `json:"documentation"` + Properties map[string]string `json:"properties"` +} + +// Relationship represents a connection between two code symbols. +type Relationship struct { + FromID string `json:"from_id"` + ToID string `json:"to_id"` + Type RelationType `json:"type"` + Location Location `json:"location"` +} + +// File represents a single source code file tracked by the engine. +type File struct { + Path string `json:"path"` + ContentHash string `json:"content_hash"` + SymbolIDs []string `json:"symbol_ids"` +} + +// ChangeSet represents a commit or local edit modification grouping. +type ChangeSet struct { + CommitSHA string `json:"commit_sha"` + FilesAdded []string `json:"files_added"` + FilesModified []string `json:"files_modified"` + FilesDeleted []string `json:"files_deleted"` +} + +// Commit represents static commit metadata in a Git repository. +type Commit struct { + Hash string `json:"hash"` + Author string `json:"author"` + Message string `json:"message"` + Timestamp int64 `json:"timestamp"` +} + +// FileEntry represents a file path and its scanned contents (with checksum). +type FileEntry struct { + FilePath string `json:"file_path"` + Content []byte `json:"content"` + ContentHash string `json:"content_hash"` +} diff --git a/core/parser/contracts/contracts.go b/core/parser/contracts/contracts.go new file mode 100644 index 0000000..c0c5d49 --- /dev/null +++ b/core/parser/contracts/contracts.go @@ -0,0 +1,14 @@ +package contracts + +import ( + "context" + + "github.com/thinkbattleground/cce/shared/protocol/v1/plugin" +) + +// LanguageParserClient contract defines client methods to interact with parser plugins. +type LanguageParserClient interface { + GetMetadata(ctx context.Context) (*plugin.PluginMetadata, error) + ParseFile(ctx context.Context, filePath string, content []byte) (*plugin.ParseResponse, error) + Close() error +} diff --git a/core/parser/loader/loader.go b/core/parser/loader/loader.go new file mode 100644 index 0000000..a21f75a --- /dev/null +++ b/core/parser/loader/loader.go @@ -0,0 +1,59 @@ +package loader + +import ( + "context" + "fmt" + + "github.com/thinkbattleground/cce/core/parser/contracts" + pb "github.com/thinkbattleground/cce/shared/protocol/v1/plugin" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type GrpcParserClient struct { + conn *grpc.ClientConn + client pb.LanguageParserPluginClient +} + +func NewGrpcParserClient(addr string) (*GrpcParserClient, error) { + conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("failed to connect to parser plugin at %s: %w", addr, err) + } + + return &GrpcParserClient{ + conn: conn, + client: pb.NewLanguageParserPluginClient(conn), + }, nil +} + +func (c *GrpcParserClient) GetMetadata(ctx context.Context) (*pb.PluginMetadata, error) { + return c.client.GetMetadata(ctx, &pb.Empty{}) +} + +func (c *GrpcParserClient) ParseFile(ctx context.Context, filePath string, content []byte) (*pb.ParseResponse, error) { + return c.client.ParseFile(ctx, &pb.ParseRequest{ + FilePath: filePath, + FileContent: content, + }) +} + +func (c *GrpcParserClient) Close() error { + if c.conn != nil { + return c.conn.Close() + } + return nil +} + +// ConnectAndRegister dials the parser at address and registers it for the extensions. +func ConnectAndRegister(addr string, extensions []string, registry interface{ Register(ext string, client contracts.LanguageParserClient) }) error { + client, err := NewGrpcParserClient(addr) + if err != nil { + return err + } + + for _, ext := range extensions { + registry.Register(ext, client) + } + return nil +} diff --git a/core/parser/parser.go b/core/parser/parser.go new file mode 100644 index 0000000..d15c47f --- /dev/null +++ b/core/parser/parser.go @@ -0,0 +1,33 @@ +package parser + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/thinkbattleground/cce/core/parser/registry" + pb "github.com/thinkbattleground/cce/shared/protocol/v1/plugin" +) + +type ParserOrchestrator struct { + Registry *registry.ParserRegistry +} + +func NewParserOrchestrator(reg *registry.ParserRegistry) *ParserOrchestrator { + return &ParserOrchestrator{Registry: reg} +} + +// ParseSourceFile routes the file content to the appropriate registered language parser. +func (p *ParserOrchestrator) ParseSourceFile(ctx context.Context, filePath string, content []byte) (*pb.ParseResponse, error) { + ext := filepath.Ext(filePath) + if ext == "" { + return nil, fmt.Errorf("file %s has no extension", filePath) + } + + client, ok := p.Registry.Get(ext) + if !ok { + return nil, fmt.Errorf("no language parser registered for extension %s", ext) + } + + return client.ParseFile(ctx, filePath, content) +} diff --git a/core/parser/registry/registry.go b/core/parser/registry/registry.go new file mode 100644 index 0000000..7daccb4 --- /dev/null +++ b/core/parser/registry/registry.go @@ -0,0 +1,31 @@ +package registry + +import ( + "sync" + + "github.com/thinkbattleground/cce/core/parser/contracts" +) + +type ParserRegistry struct { + mu sync.RWMutex + parsers map[string]contracts.LanguageParserClient +} + +func NewParserRegistry() *ParserRegistry { + return &ParserRegistry{ + parsers: make(map[string]contracts.LanguageParserClient), + } +} + +func (r *ParserRegistry) Register(extension string, client contracts.LanguageParserClient) { + r.mu.Lock() + defer r.mu.Unlock() + r.parsers[extension] = client +} + +func (r *ParserRegistry) Get(extension string) (contracts.LanguageParserClient, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + client, ok := r.parsers[extension] + return client, ok +} diff --git a/core/parser/sdk/sdk.go b/core/parser/sdk/sdk.go new file mode 100644 index 0000000..d99ef45 --- /dev/null +++ b/core/parser/sdk/sdk.go @@ -0,0 +1,97 @@ +package sdk + +import ( + "github.com/thinkbattleground/cce/core/model" + "github.com/thinkbattleground/cce/shared/protocol/v1/ucm" +) + +// ConvertUCMToDomain converts protobuf UCM representations to domain model symbols and relationships. +func ConvertUCMToDomain(protoPayload *ucm.UniversalCodePayload) ([]model.Symbol, []model.Relationship) { + var symbols []model.Symbol + var relations []model.Relationship + + for _, unit := range protoPayload.Units { + sym := model.Symbol{ + ID: unit.Id, + RepoID: protoPayload.RepositoryId, + Name: unit.Name, + FullyQualifiedName: unit.FullyQualifiedName, + Kind: convertKind(unit.Kind), + Signature: unit.Signature, + Documentation: unit.Documentation, + Properties: unit.Properties, + } + if unit.Location != nil { + sym.Location = model.Location{ + FilePath: unit.Location.FilePath, + StartLine: int(unit.Location.StartLine), + StartColumn: int(unit.Location.StartColumn), + EndLine: int(unit.Location.EndLine), + EndColumn: int(unit.Location.EndColumn), + } + } + symbols = append(symbols, sym) + } + + for _, rel := range protoPayload.Relationships { + r := model.Relationship{ + FromID: rel.FromId, + ToID: rel.ToId, + Type: convertRelationType(rel.Type), + } + if rel.Location != nil { + r.Location = model.Location{ + FilePath: rel.Location.FilePath, + StartLine: int(rel.Location.StartLine), + StartColumn: int(rel.Location.StartColumn), + EndLine: int(rel.Location.EndLine), + EndColumn: int(rel.Location.EndColumn), + } + } + relations = append(relations, r) + } + + return symbols, relations +} + +func convertKind(k ucm.SymbolKind) model.SymbolKind { + switch k { + case ucm.SymbolKind_SYMBOL_KIND_MODULE: + return model.SymbolKindModule + case ucm.SymbolKind_SYMBOL_KIND_PACKAGE: + return model.SymbolKindPackage + case ucm.SymbolKind_SYMBOL_KIND_CLASS: + return model.SymbolKindClass + case ucm.SymbolKind_SYMBOL_KIND_STRUCT: + return model.SymbolKindStruct + case ucm.SymbolKind_SYMBOL_KIND_INTERFACE: + return model.SymbolKindInterface + case ucm.SymbolKind_SYMBOL_KIND_FUNCTION: + return model.SymbolKindFunction + case ucm.SymbolKind_SYMBOL_KIND_METHOD: + return model.SymbolKindMethod + case ucm.SymbolKind_SYMBOL_KIND_VARIABLE: + return model.SymbolKindVariable + case ucm.SymbolKind_SYMBOL_KIND_ANNOTATION: + return model.SymbolKindAnnotation + default: + return model.SymbolKindUnspecified + } +} + +func convertRelationType(t ucm.RelationType) model.RelationType { + switch t { + case ucm.RelationType_RELATION_TYPE_CONTAINS: + return model.RelationTypeContains + case ucm.RelationType_RELATION_TYPE_CALLS: + return model.RelationTypeCalls + case ucm.RelationType_RELATION_TYPE_IMPLEMENTS: + return model.RelationTypeImplements + case ucm.RelationType_RELATION_TYPE_REFERENCES: + return model.RelationTypeReferences + case ucm.RelationType_RELATION_TYPE_IMPORTS: + return model.RelationTypeImports + default: + return model.RelationTypeUnspecified + } +} diff --git a/core/storage/memory/memory.go b/core/storage/memory/memory.go new file mode 100644 index 0000000..6db5bdc --- /dev/null +++ b/core/storage/memory/memory.go @@ -0,0 +1,133 @@ +package memory + +import ( + "context" + "errors" + "strings" + "sync" + + "github.com/thinkbattleground/cce/core/model" +) + +type MemoryStorage struct { + mu sync.RWMutex + repositories map[string]model.Repository + symbols map[string]model.Symbol + relations []model.Relationship +} + +func NewMemoryStorage() *MemoryStorage { + return &MemoryStorage{ + repositories: make(map[string]model.Repository), + symbols: make(map[string]model.Symbol), + relations: []model.Relationship{}, + } +} + +func (m *MemoryStorage) Initialize(ctx context.Context) error { + return nil +} + +func (m *MemoryStorage) SaveRepository(ctx context.Context, repo model.Repository) error { + m.mu.Lock() + defer m.mu.Unlock() + m.repositories[repo.ID] = repo + return nil +} + +func (m *MemoryStorage) SaveSymbols(ctx context.Context, repoID string, symbols []model.Symbol, relations []model.Relationship) error { + m.mu.Lock() + defer m.mu.Unlock() + + for _, sym := range symbols { + m.symbols[sym.ID] = sym + } + m.relations = append(m.relations, relations...) + return nil +} + +func (m *MemoryStorage) SearchSymbols(ctx context.Context, query string, kind model.SymbolKind) ([]model.Symbol, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + var results []model.Symbol + for _, sym := range m.symbols { + nameMatches := strings.Contains(strings.ToLower(sym.Name), strings.ToLower(query)) + kindMatches := kind == model.SymbolKindUnspecified || sym.Kind == kind + + if nameMatches && kindMatches { + results = append(results, sym) + } + } + return results, nil +} + +func (m *MemoryStorage) GetSymbol(ctx context.Context, id string) (model.Symbol, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + sym, ok := m.symbols[id] + if !ok { + return model.Symbol{}, errors.New("symbol not found") + } + return sym, nil +} + +func (m *MemoryStorage) GetSymbolByName(ctx context.Context, name string) (model.Symbol, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + for _, sym := range m.symbols { + if sym.Name == name { + return sym, nil + } + } + return model.Symbol{}, errors.New("symbol not found") +} + +func (m *MemoryStorage) GetCallers(ctx context.Context, symbolID string) ([]model.Symbol, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + var results []model.Symbol + for _, rel := range m.relations { + if rel.ToID == symbolID && rel.Type == model.RelationTypeCalls { + if sym, ok := m.symbols[rel.FromID]; ok { + results = append(results, sym) + } + } + } + return results, nil +} + +func (m *MemoryStorage) GetCallees(ctx context.Context, symbolID string) ([]model.Symbol, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + var results []model.Symbol + for _, rel := range m.relations { + if rel.FromID == symbolID && rel.Type == model.RelationTypeCalls { + if sym, ok := m.symbols[rel.ToID]; ok { + results = append(results, sym) + } + } + } + return results, nil +} + +func (m *MemoryStorage) GetRelationships(ctx context.Context, symbolID string) ([]model.Relationship, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + var results []model.Relationship + for _, rel := range m.relations { + if rel.FromID == symbolID || rel.ToID == symbolID { + results = append(results, rel) + } + } + return results, nil +} + +func (m *MemoryStorage) Close() error { + return nil +} diff --git a/core/storage/postgres/postgres.go b/core/storage/postgres/postgres.go new file mode 100644 index 0000000..5400d13 --- /dev/null +++ b/core/storage/postgres/postgres.go @@ -0,0 +1,63 @@ +package postgres + +import ( + "context" + "errors" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/thinkbattleground/cce/core/model" +) + +type PostgresStorage struct { + conn *pgx.Conn +} + +func NewPostgresStorage(connStr string) (*PostgresStorage, error) { + // For skeleton MVP, we just verify structure compiles + return &PostgresStorage{}, nil +} + +func (p *PostgresStorage) Initialize(ctx context.Context) error { + slog.Info("initializing postgres storage (MVP placeholder)") + return nil +} + +func (p *PostgresStorage) SaveRepository(ctx context.Context, repo model.Repository) error { + return nil +} + +func (p *PostgresStorage) SaveSymbols(ctx context.Context, repoID string, symbols []model.Symbol, relations []model.Relationship) error { + return nil +} + +func (p *PostgresStorage) SearchSymbols(ctx context.Context, query string, kind model.SymbolKind) ([]model.Symbol, error) { + return nil, nil +} + +func (p *PostgresStorage) GetSymbol(ctx context.Context, id string) (model.Symbol, error) { + return model.Symbol{}, errors.New("not implemented") +} + +func (p *PostgresStorage) GetSymbolByName(ctx context.Context, name string) (model.Symbol, error) { + return model.Symbol{}, errors.New("not implemented") +} + +func (p *PostgresStorage) GetCallers(ctx context.Context, symbolID string) ([]model.Symbol, error) { + return nil, nil +} + +func (p *PostgresStorage) GetCallees(ctx context.Context, symbolID string) ([]model.Symbol, error) { + return nil, nil +} + +func (p *PostgresStorage) GetRelationships(ctx context.Context, symbolID string) ([]model.Relationship, error) { + return nil, nil +} + +func (p *PostgresStorage) Close() error { + if p.conn != nil { + return p.conn.Close(context.Background()) + } + return nil +} diff --git a/core/storage/repository/repository.go b/core/storage/repository/repository.go new file mode 100644 index 0000000..f6d0636 --- /dev/null +++ b/core/storage/repository/repository.go @@ -0,0 +1,25 @@ +package repository + +import ( + "context" + + "github.com/thinkbattleground/cce/core/model" + "github.com/thinkbattleground/cce/core/storage" +) + +// RepositoryService wraps Storage queries with repository-scoped domain helpers. +type RepositoryService struct { + store storage.Storage +} + +func NewRepositoryService(store storage.Storage) *RepositoryService { + return &RepositoryService{store: store} +} + +func (s *RepositoryService) RegisterRepository(ctx context.Context, repo model.Repository) error { + return s.store.SaveRepository(ctx, repo) +} + +func (s *RepositoryService) IndexRelease(ctx context.Context, repoID string, symbols []model.Symbol, relations []model.Relationship) error { + return s.store.SaveSymbols(ctx, repoID, symbols, relations) +} diff --git a/core/storage/sqlite/sqlite.go b/core/storage/sqlite/sqlite.go new file mode 100644 index 0000000..6878aa5 --- /dev/null +++ b/core/storage/sqlite/sqlite.go @@ -0,0 +1,283 @@ +package sqlite + +import ( + "context" + "database/sql" + "fmt" + "log/slog" + + _ "github.com/glebarez/go-sqlite" + "github.com/thinkbattleground/cce/core/model" +) + +type SQLiteStorage struct { + db *sql.DB +} + +func NewSQLiteStorage(dbPath string) (*SQLiteStorage, error) { + db, err := sql.Open("sqlite", dbPath) + if err != nil { + return nil, fmt.Errorf("failed to open sqlite: %w", err) + } + return &SQLiteStorage{db: db}, nil +} + +func (s *SQLiteStorage) Initialize(ctx context.Context) error { + slog.Info("initializing sqlite schemas") + + schema := ` + CREATE TABLE IF NOT EXISTS repositories ( + id TEXT PRIMARY KEY, + name TEXT, + clone_url TEXT, + branch TEXT + ); + + CREATE TABLE IF NOT EXISTS nodes ( + id TEXT PRIMARY KEY, + repo_id TEXT, + file_path TEXT, + name TEXT, + fully_qualified_name TEXT, + kind INTEGER, + signature TEXT, + documentation TEXT + ); + + CREATE TABLE IF NOT EXISTS edges ( + from_id TEXT, + to_id TEXT, + type INTEGER, + file_path TEXT, + start_line INTEGER, + start_column INTEGER, + end_line INTEGER, + end_column INTEGER, + PRIMARY KEY (from_id, to_id, type, start_line, start_column) + );` + + _, err := s.db.ExecContext(ctx, schema) + return err +} + +func (s *SQLiteStorage) SaveRepository(ctx context.Context, repo model.Repository) error { + slog.Info("saving repository metadata", "id", repo.ID, "name", repo.Name) + _, err := s.db.ExecContext(ctx, ` + INSERT OR REPLACE INTO repositories (id, name, clone_url, branch) + VALUES (?, ?, ?, ?) + `, repo.ID, repo.Name, repo.CloneURL, repo.Branch) + return err +} + +func (s *SQLiteStorage) SaveSymbols(ctx context.Context, repoID string, symbols []model.Symbol, relations []model.Relationship) error { + slog.Info("committing symbols to sqlite database", "repo", repoID, "symbols", len(symbols), "relations", len(relations)) + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + // Insert symbols + nodeStmt, err := tx.PrepareContext(ctx, ` + INSERT OR REPLACE INTO nodes (id, repo_id, file_path, name, fully_qualified_name, kind, signature, documentation) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `) + if err != nil { + return err + } + defer nodeStmt.Close() + + for _, sym := range symbols { + _, err = nodeStmt.ExecContext(ctx, + sym.ID, + repoID, + sym.Location.FilePath, + sym.Name, + sym.FullyQualifiedName, + int(sym.Kind), + sym.Signature, + sym.Documentation, + ) + if err != nil { + return err + } + } + + // Insert edges + edgeStmt, err := tx.PrepareContext(ctx, ` + INSERT OR REPLACE INTO edges (from_id, to_id, type, file_path, start_line, start_column, end_line, end_column) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `) + if err != nil { + return err + } + defer edgeStmt.Close() + + for _, rel := range relations { + _, err = edgeStmt.ExecContext(ctx, + rel.FromID, + rel.ToID, + int(rel.Type), + rel.Location.FilePath, + rel.Location.StartLine, + rel.Location.StartColumn, + rel.Location.EndLine, + rel.Location.EndColumn, + ) + if err != nil { + return err + } + } + + return tx.Commit() +} + +func (s *SQLiteStorage) SearchSymbols(ctx context.Context, query string, kind model.SymbolKind) ([]model.Symbol, error) { + slog.Info("searching symbols in sqlite", "query", query) + + var rows *sql.Rows + var err error + + if kind == model.SymbolKindUnspecified { + rows, err = s.db.QueryContext(ctx, ` + SELECT id, repo_id, file_path, name, fully_qualified_name, kind, signature, documentation + FROM nodes WHERE name LIKE ? + `, "%"+query+"%") + } else { + rows, err = s.db.QueryContext(ctx, ` + SELECT id, repo_id, file_path, name, fully_qualified_name, kind, signature, documentation + FROM nodes WHERE name LIKE ? AND kind = ? + `, "%"+query+"%", int(kind)) + } + + if err != nil { + return nil, err + } + defer rows.Close() + + var results []model.Symbol + for rows.Next() { + var sym model.Symbol + var kindInt int + err = rows.Scan(&sym.ID, &sym.RepoID, &sym.Location.FilePath, &sym.Name, &sym.FullyQualifiedName, &kindInt, &sym.Signature, &sym.Documentation) + if err != nil { + return nil, err + } + sym.Kind = model.SymbolKind(kindInt) + results = append(results, sym) + } + + return results, nil +} + +func (s *SQLiteStorage) GetSymbol(ctx context.Context, id string) (model.Symbol, error) { + var sym model.Symbol + var kindInt int + err := s.db.QueryRowContext(ctx, ` + SELECT id, repo_id, file_path, name, fully_qualified_name, kind, signature, documentation + FROM nodes WHERE id = ? + `, id).Scan(&sym.ID, &sym.RepoID, &sym.Location.FilePath, &sym.Name, &sym.FullyQualifiedName, &kindInt, &sym.Signature, &sym.Documentation) + if err != nil { + return model.Symbol{}, err + } + sym.Kind = model.SymbolKind(kindInt) + return sym, nil +} + +func (s *SQLiteStorage) GetSymbolByName(ctx context.Context, name string) (model.Symbol, error) { + var sym model.Symbol + var kindInt int + err := s.db.QueryRowContext(ctx, ` + SELECT id, repo_id, file_path, name, fully_qualified_name, kind, signature, documentation + FROM nodes WHERE name = ? + `, name).Scan(&sym.ID, &sym.RepoID, &sym.Location.FilePath, &sym.Name, &sym.FullyQualifiedName, &kindInt, &sym.Signature, &sym.Documentation) + if err != nil { + return model.Symbol{}, err + } + sym.Kind = model.SymbolKind(kindInt) + return sym, nil +} + +func (s *SQLiteStorage) GetCallers(ctx context.Context, symbolID string) ([]model.Symbol, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT n.id, n.repo_id, n.file_path, n.name, n.fully_qualified_name, n.kind, n.signature, n.documentation + FROM nodes n + JOIN edges e ON e.from_id = n.id + WHERE e.to_id = ? AND e.type = ? + `, symbolID, int(model.RelationTypeCalls)) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []model.Symbol + for rows.Next() { + var sym model.Symbol + var kindInt int + err = rows.Scan(&sym.ID, &sym.RepoID, &sym.Location.FilePath, &sym.Name, &sym.FullyQualifiedName, &kindInt, &sym.Signature, &sym.Documentation) + if err != nil { + return nil, err + } + sym.Kind = model.SymbolKind(kindInt) + results = append(results, sym) + } + return results, nil +} + +func (s *SQLiteStorage) GetCallees(ctx context.Context, symbolID string) ([]model.Symbol, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT n.id, n.repo_id, n.file_path, n.name, n.fully_qualified_name, n.kind, n.signature, n.documentation + FROM nodes n + JOIN edges e ON e.to_id = n.id + WHERE e.from_id = ? AND e.type = ? + `, symbolID, int(model.RelationTypeCalls)) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []model.Symbol + for rows.Next() { + var sym model.Symbol + var kindInt int + err = rows.Scan(&sym.ID, &sym.RepoID, &sym.Location.FilePath, &sym.Name, &sym.FullyQualifiedName, &kindInt, &sym.Signature, &sym.Documentation) + if err != nil { + return nil, err + } + sym.Kind = model.SymbolKind(kindInt) + results = append(results, sym) + } + return results, nil +} + +func (s *SQLiteStorage) GetRelationships(ctx context.Context, symbolID string) ([]model.Relationship, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT from_id, to_id, type, file_path, start_line, start_column, end_line, end_column + FROM edges WHERE from_id = ? OR to_id = ? + `, symbolID, symbolID) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []model.Relationship + for rows.Next() { + var rel model.Relationship + var typeInt int + err = rows.Scan(&rel.FromID, &rel.ToID, &typeInt, &rel.Location.FilePath, &rel.Location.StartLine, &rel.Location.StartColumn, &rel.Location.EndLine, &rel.Location.EndColumn) + if err != nil { + return nil, err + } + rel.Type = model.RelationType(typeInt) + results = append(results, rel) + } + return results, nil +} + +func (s *SQLiteStorage) Close() error { + if s.db != nil { + return s.db.Close() + } + return nil +} diff --git a/core/storage/storage.go b/core/storage/storage.go new file mode 100644 index 0000000..1ffe516 --- /dev/null +++ b/core/storage/storage.go @@ -0,0 +1,40 @@ +package storage + +import ( + "context" + + "github.com/thinkbattleground/cce/core/model" +) + +// Storage defines the provider-agnostic contract for code knowledge storage. +type Storage interface { + // Initialize prepares backend indices or schema structures. + Initialize(ctx context.Context) error + + // SaveRepository registers a repo entity. + SaveRepository(ctx context.Context, repo model.Repository) error + + // SaveSymbols commits extracted symbols and their relationships to the database. + SaveSymbols(ctx context.Context, repoID string, symbols []model.Symbol, relations []model.Relationship) error + + // SearchSymbols searches symbols by query string and optional kind. + SearchSymbols(ctx context.Context, query string, kind model.SymbolKind) ([]model.Symbol, error) + + // GetSymbol retrieves a symbol by its unique ID. + GetSymbol(ctx context.Context, id string) (model.Symbol, error) + + // GetSymbolByName retrieves a symbol by its name. + GetSymbolByName(ctx context.Context, name string) (model.Symbol, error) + + // GetCallers retrieves all symbols that reference/call the target symbol. + GetCallers(ctx context.Context, symbolID string) ([]model.Symbol, error) + + // GetCallees retrieves all symbols called by the target symbol. + GetCallees(ctx context.Context, symbolID string) ([]model.Symbol, error) + + // GetRelationships retrieves relationships where target is either source or destination. + GetRelationships(ctx context.Context, symbolID string) ([]model.Relationship, error) + + // Close terminates any active backend database connections. + Close() error +} diff --git a/core/ucm/id.go b/core/ucm/id.go new file mode 100644 index 0000000..3cf4a37 --- /dev/null +++ b/core/ucm/id.go @@ -0,0 +1,24 @@ +package ucm + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" +) + +// GenerateSymbolID generates a stable, deterministic ID for a code symbol. +func GenerateSymbolID(repoID, filePath, fullyQualifiedName, kind string) string { + hasher := sha256.New() + // Delimit elements to avoid boundary collisions + input := fmt.Sprintf("sym::%s::%s::%s::%s", repoID, filePath, fullyQualifiedName, kind) + hasher.Write([]byte(input)) + return hex.EncodeToString(hasher.Sum(nil)) +} + +// GenerateRelationshipID generates a stable, deterministic ID for a graph edge. +func GenerateRelationshipID(fromID, toID, relType, filePath string, startLine, startColumn int) string { + hasher := sha256.New() + input := fmt.Sprintf("rel::%s::%s::%s::%s::%d::%d", fromID, toID, relType, filePath, startLine, startColumn) + hasher.Write([]byte(input)) + return hex.EncodeToString(hasher.Sum(nil)) +} diff --git a/core/ucm/serialization.go b/core/ucm/serialization.go new file mode 100644 index 0000000..e6a42a3 --- /dev/null +++ b/core/ucm/serialization.go @@ -0,0 +1,250 @@ +package ucm + +import ( + "encoding/json" + "fmt" + + "github.com/thinkbattleground/cce/core/model" + pb "github.com/thinkbattleground/cce/shared/protocol/v1/ucm" + "github.com/golang/protobuf/proto" +) + +const ModelVersion = "v1" + +// Envelope wraps serialized JSON payload to include schema versioning. +type Envelope struct { + Version string `json:"version"` + Payload json.RawMessage `json:"payload"` +} + +// ConvertToProto translates domain models to a protobuf UCM representation. +func ConvertToProto(repoID string, commitSha string, symbols []model.Symbol, relations []model.Relationship) *pb.UniversalCodePayload { + var units []*pb.CodeUnit + for _, sym := range symbols { + unit := &pb.CodeUnit{ + Id: sym.ID, + Kind: convertKindToProto(sym.Kind), + Name: sym.Name, + FullyQualifiedName: sym.FullyQualifiedName, + Signature: sym.Signature, + Documentation: sym.Documentation, + Properties: sym.Properties, + Location: &pb.Location{ + FilePath: sym.Location.FilePath, + StartLine: int32(sym.Location.StartLine), + StartColumn: int32(sym.Location.StartColumn), + EndLine: int32(sym.Location.EndLine), + EndColumn: int32(sym.Location.EndColumn), + }, + } + units = append(units, unit) + } + + var pbRelations []*pb.Relationship + for _, rel := range relations { + r := &pb.Relationship{ + FromId: rel.FromID, + ToId: rel.ToID, + Type: convertRelationTypeToProto(rel.Type), + Location: &pb.Location{ + FilePath: rel.Location.FilePath, + StartLine: int32(rel.Location.StartLine), + StartColumn: int32(rel.Location.StartColumn), + EndLine: int32(rel.Location.EndLine), + EndColumn: int32(rel.Location.EndColumn), + }, + } + pbRelations = append(pbRelations, r) + } + + return &pb.UniversalCodePayload{ + RepositoryId: repoID, + CommitSha: commitSha, + Units: units, + Relationships: pbRelations, + } +} + +// ConvertToDomain translates a protobuf UCM representation back to domain model symbols and relationships. +func ConvertToDomain(payload *pb.UniversalCodePayload) ([]model.Symbol, []model.Relationship) { + var symbols []model.Symbol + var relations []model.Relationship + + for _, unit := range payload.Units { + sym := model.Symbol{ + ID: unit.Id, + RepoID: payload.RepositoryId, + Name: unit.Name, + FullyQualifiedName: unit.FullyQualifiedName, + Kind: convertKindToDomain(unit.Kind), + Signature: unit.Signature, + Documentation: unit.Documentation, + Properties: unit.Properties, + } + if unit.Location != nil { + sym.Location = model.Location{ + FilePath: unit.Location.FilePath, + StartLine: int(unit.Location.StartLine), + StartColumn: int(unit.Location.StartColumn), + EndLine: int(unit.Location.EndLine), + EndColumn: int(unit.Location.EndColumn), + } + } + symbols = append(symbols, sym) + } + + for _, rel := range payload.Relationships { + r := model.Relationship{ + FromID: rel.FromId, + ToID: rel.ToId, + Type: convertRelationTypeToDomain(rel.Type), + } + if rel.Location != nil { + r.Location = model.Location{ + FilePath: rel.Location.FilePath, + StartLine: int(rel.Location.StartLine), + StartColumn: int(rel.Location.StartColumn), + EndLine: int(rel.Location.EndLine), + EndColumn: int(rel.Location.EndColumn), + } + } + relations = append(relations, r) + } + + return symbols, relations +} + +// SerializeJSON marshals the UCM protobuf payload to a JSON envelope byte string. +func SerializeJSON(payload *pb.UniversalCodePayload) ([]byte, error) { + payloadBytes, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("failed to marshal payload: %w", err) + } + + envelope := Envelope{ + Version: ModelVersion, + Payload: payloadBytes, + } + + return json.Marshal(envelope) +} + +// DeserializeJSON unmarshals a JSON envelope byte string back into a UCM protobuf payload, validating the model version. +func DeserializeJSON(data []byte) (*pb.UniversalCodePayload, error) { + var env Envelope + err := json.Unmarshal(data, &env) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal UCM envelope: %w", err) + } + + if env.Version != ModelVersion { + return nil, fmt.Errorf("unsupported model version: found %s, expected %s", env.Version, ModelVersion) + } + + var payload pb.UniversalCodePayload + err = json.Unmarshal(env.Payload, &payload) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal payload from UCM envelope: %w", err) + } + + return &payload, nil +} + +// SerializeProto serializes the UCM protobuf payload to binary bytes. +func SerializeProto(payload *pb.UniversalCodePayload) ([]byte, error) { + return proto.Marshal(payload) +} + +// DeserializeProto deserializes binary bytes back into a UCM protobuf payload. +func DeserializeProto(data []byte) (*pb.UniversalCodePayload, error) { + var payload pb.UniversalCodePayload + err := proto.Unmarshal(data, &payload) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal protobuf: %w", err) + } + return &payload, nil +} + +func convertKindToProto(k model.SymbolKind) pb.SymbolKind { + switch k { + case model.SymbolKindModule: + return pb.SymbolKind_SYMBOL_KIND_MODULE + case model.SymbolKindPackage: + return pb.SymbolKind_SYMBOL_KIND_PACKAGE + case model.SymbolKindClass: + return pb.SymbolKind_SYMBOL_KIND_CLASS + case model.SymbolKindStruct: + return pb.SymbolKind_SYMBOL_KIND_STRUCT + case model.SymbolKindInterface: + return pb.SymbolKind_SYMBOL_KIND_INTERFACE + case model.SymbolKindFunction: + return pb.SymbolKind_SYMBOL_KIND_FUNCTION + case model.SymbolKindMethod: + return pb.SymbolKind_SYMBOL_KIND_METHOD + case model.SymbolKindVariable: + return pb.SymbolKind_SYMBOL_KIND_VARIABLE + case model.SymbolKindAnnotation: + return pb.SymbolKind_SYMBOL_KIND_ANNOTATION + default: + return pb.SymbolKind_SYMBOL_KIND_UNSPECIFIED + } +} + +func convertKindToDomain(k pb.SymbolKind) model.SymbolKind { + switch k { + case pb.SymbolKind_SYMBOL_KIND_MODULE: + return model.SymbolKindModule + case pb.SymbolKind_SYMBOL_KIND_PACKAGE: + return model.SymbolKindPackage + case pb.SymbolKind_SYMBOL_KIND_CLASS: + return model.SymbolKindClass + case pb.SymbolKind_SYMBOL_KIND_STRUCT: + return model.SymbolKindStruct + case pb.SymbolKind_SYMBOL_KIND_INTERFACE: + return model.SymbolKindInterface + case pb.SymbolKind_SYMBOL_KIND_FUNCTION: + return model.SymbolKindFunction + case pb.SymbolKind_SYMBOL_KIND_METHOD: + return model.SymbolKindMethod + case pb.SymbolKind_SYMBOL_KIND_VARIABLE: + return model.SymbolKindVariable + case pb.SymbolKind_SYMBOL_KIND_ANNOTATION: + return model.SymbolKindAnnotation + default: + return model.SymbolKindUnspecified + } +} + +func convertRelationTypeToProto(t model.RelationType) pb.RelationType { + switch t { + case model.RelationTypeContains: + return pb.RelationType_RELATION_TYPE_CONTAINS + case model.RelationTypeCalls: + return pb.RelationType_RELATION_TYPE_CALLS + case model.RelationTypeImplements: + return pb.RelationType_RELATION_TYPE_IMPLEMENTS + case model.RelationTypeReferences: + return pb.RelationType_RELATION_TYPE_REFERENCES + case model.RelationTypeImports: + return pb.RelationType_RELATION_TYPE_IMPORTS + default: + return pb.RelationType_RELATION_TYPE_UNSPECIFIED + } +} + +func convertRelationTypeToDomain(t pb.RelationType) model.RelationType { + switch t { + case pb.RelationType_RELATION_TYPE_CONTAINS: + return model.RelationTypeContains + case pb.RelationType_RELATION_TYPE_CALLS: + return model.RelationTypeCalls + case pb.RelationType_RELATION_TYPE_IMPLEMENTS: + return model.RelationTypeImplements + case pb.RelationType_RELATION_TYPE_REFERENCES: + return model.RelationTypeReferences + case pb.RelationType_RELATION_TYPE_IMPORTS: + return model.RelationTypeImports + default: + return model.RelationTypeUnspecified + } +} diff --git a/core/ucm/ucm_test.go b/core/ucm/ucm_test.go new file mode 100644 index 0000000..1faf6db --- /dev/null +++ b/core/ucm/ucm_test.go @@ -0,0 +1,134 @@ +package ucm_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/thinkbattleground/cce/core/model" + "github.com/thinkbattleground/cce/core/ucm" +) + +func TestGenerateSymbolID(t *testing.T) { + id1 := ucm.GenerateSymbolID("repo1", "file.go", "hello.World", "FUNCTION") + id2 := ucm.GenerateSymbolID("repo1", "file.go", "hello.World", "FUNCTION") + id3 := ucm.GenerateSymbolID("repo1", "file.go", "hello.World2", "FUNCTION") + + // Must be deterministic (same input produces same output) + assert.Equal(t, id1, id2) + + // Must be unique (different inputs produce different outputs) + assert.NotEqual(t, id1, id3) +} + +func TestValidateSymbol(t *testing.T) { + // Valid symbol + validSym := model.Symbol{ + ID: "some-id", + RepoID: "my-repo", + Name: "MyFunc", + FullyQualifiedName: "pkg.MyFunc", + Kind: model.SymbolKindFunction, + Location: model.Location{ + FilePath: "main.go", + StartLine: 10, + EndLine: 20, + }, + } + assert.NoError(t, ucm.ValidateSymbol(validSym)) + + // Invalid empty ID + invalidSym := validSym + invalidSym.ID = "" + assert.Error(t, ucm.ValidateSymbol(invalidSym)) + + // Invalid line bounds + invalidSym2 := validSym + invalidSym2.Location.StartLine = 30 + invalidSym2.Location.EndLine = 20 + assert.Error(t, ucm.ValidateSymbol(invalidSym2)) +} + +func TestValidateRelationship(t *testing.T) { + validRel := model.Relationship{ + FromID: "id-1", + ToID: "id-2", + Type: model.RelationTypeCalls, + } + assert.NoError(t, ucm.ValidateRelationship(validRel)) + + invalidRel := validRel + invalidRel.Type = model.RelationTypeUnspecified + assert.Error(t, ucm.ValidateRelationship(invalidRel)) +} + +func TestSerializationJSONLoop(t *testing.T) { + symbols := []model.Symbol{ + { + ID: "id-1", + RepoID: "repo-1", + Name: "TestFunc", + FullyQualifiedName: "main.TestFunc", + Kind: model.SymbolKindFunction, + Location: model.Location{ + FilePath: "main.go", + StartLine: 1, + EndLine: 10, + }, + }, + } + + relations := []model.Relationship{ + { + FromID: "id-1", + ToID: "id-2", + Type: model.RelationTypeCalls, + }, + } + + pbPayload := ucm.ConvertToProto("repo-1", "commit-sha", symbols, relations) + + // Serialize + data, err := ucm.SerializeJSON(pbPayload) + require.NoError(t, err) + + // Deserialize + restoredPb, err := ucm.DeserializeJSON(data) + require.NoError(t, err) + + // Verify values + assert.Equal(t, pbPayload.RepositoryId, restoredPb.RepositoryId) + assert.Equal(t, pbPayload.CommitSha, restoredPb.CommitSha) + require.Len(t, restoredPb.Units, 1) + assert.Equal(t, "TestFunc", restoredPb.Units[0].Name) +} + +func TestSerializationProtoLoop(t *testing.T) { + symbols := []model.Symbol{ + { + ID: "id-1", + RepoID: "repo-1", + Name: "TestFunc", + FullyQualifiedName: "main.TestFunc", + Kind: model.SymbolKindFunction, + Location: model.Location{ + FilePath: "main.go", + StartLine: 1, + EndLine: 10, + }, + }, + } + + pbPayload := ucm.ConvertToProto("repo-1", "commit-sha", symbols, nil) + + // Serialize + data, err := ucm.SerializeProto(pbPayload) + require.NoError(t, err) + + // Deserialize + restoredPb, err := ucm.DeserializeProto(data) + require.NoError(t, err) + + assert.Equal(t, pbPayload.RepositoryId, restoredPb.RepositoryId) + require.Len(t, restoredPb.Units, 1) +} diff --git a/core/ucm/validation.go b/core/ucm/validation.go new file mode 100644 index 0000000..650c438 --- /dev/null +++ b/core/ucm/validation.go @@ -0,0 +1,78 @@ +package ucm + +import ( + "errors" + "fmt" + + "github.com/thinkbattleground/cce/core/model" +) + +// ValidateSymbol checks the structural integrity of a code unit symbol. +func ValidateSymbol(sym model.Symbol) error { + if sym.ID == "" { + return errors.New("symbol ID is required") + } + if sym.Name == "" { + return errors.New("symbol name cannot be empty") + } + if sym.FullyQualifiedName == "" { + return errors.New("symbol fully qualified name cannot be empty") + } + if sym.RepoID == "" { + return errors.New("symbol repo ID is required") + } + if sym.Kind == model.SymbolKindUnspecified { + return errors.New("symbol kind must be specified") + } + + // Validate location boundaries + loc := sym.Location + if loc.FilePath == "" { + return errors.New("symbol file path is required") + } + if loc.StartLine < 1 || loc.EndLine < 1 { + return fmt.Errorf("invalid line numbers: start_line=%d, end_line=%d", loc.StartLine, loc.EndLine) + } + if loc.StartLine > loc.EndLine { + return fmt.Errorf("start line (%d) cannot be greater than end line (%d)", loc.StartLine, loc.EndLine) + } + + return nil +} + +// ValidateRelationship checks the structural integrity of a dependency relationship. +func ValidateRelationship(rel model.Relationship) error { + if rel.FromID == "" { + return errors.New("relationship from_id is required") + } + if rel.ToID == "" { + return errors.New("relationship to_id is required") + } + if rel.Type == model.RelationTypeUnspecified { + return errors.New("relationship type must be specified") + } + + // If location details are defined, assert boundary correctness + loc := rel.Location + if loc.FilePath != "" { + if loc.StartLine < 1 || loc.EndLine < 1 { + return fmt.Errorf("invalid line numbers in relationship: start_line=%d, end_line=%d", loc.StartLine, loc.EndLine) + } + if loc.StartLine > loc.EndLine { + return fmt.Errorf("relationship start line (%d) cannot exceed end line (%d)", loc.StartLine, loc.EndLine) + } + } + + return nil +} + +// ValidateFile checks the file integrity constraints. +func ValidateFile(f model.File) error { + if f.Path == "" { + return errors.New("file path cannot be empty") + } + if f.ContentHash == "" { + return errors.New("file content hash is required") + } + return nil +} diff --git a/deployments/docker-compose.yml b/deployments/docker-compose.yml new file mode 100644 index 0000000..050cabd --- /dev/null +++ b/deployments/docker-compose.yml @@ -0,0 +1,34 @@ +version: '3.8' + +services: + # Optional Postgres service for CCE local dev + postgres: + image: postgres:16-alpine + container_name: cce-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: cce + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 5s + retries: 5 + + # Jaeger for OpenTelemetry trace visualization + jaeger: + image: jaegertracing/all-in-one:1.57 + container_name: cce-jaeger + ports: + - "16686:16686" # Web UI + - "4317:4317" # OTLP gRPC port + - "4318:4318" # OTLP HTTP port + environment: + - COLLECTOR_OTLP_ENABLED=true + +volumes: + pgdata: diff --git a/deployments/docker/Dockerfile.cli b/deployments/docker/Dockerfile.cli new file mode 100644 index 0000000..76f90bc --- /dev/null +++ b/deployments/docker/Dockerfile.cli @@ -0,0 +1,16 @@ +# Build stage +FROM golang:1.22-alpine AS builder +# SQLite requires CGO and GCC to compile +RUN apk add --no-cache gcc musl-dev +WORKDIR /app +COPY go.work go.work +COPY shared/ shared/ +COPY core/ core/ +COPY services/cli/ services/cli/ +RUN CGO_ENABLED=1 GOOS=linux go build -o cce ./services/cli/main.go + +# Run stage +FROM alpine:3.19 +WORKDIR /app +COPY --from=builder /app/cce . +ENTRYPOINT ["./cce"] diff --git a/deployments/docker/Dockerfile.go-parser b/deployments/docker/Dockerfile.go-parser new file mode 100644 index 0000000..9221bf1 --- /dev/null +++ b/deployments/docker/Dockerfile.go-parser @@ -0,0 +1,14 @@ +# Build stage +FROM golang:1.22-alpine AS builder +WORKDIR /app +COPY go.work go.work +COPY shared/ shared/ +COPY plugins/go-parser/ plugins/go-parser/ +RUN CGO_ENABLED=0 GOOS=linux go build -o go-parser ./plugins/go-parser/main.go + +# Run stage +FROM alpine:3.19 +WORKDIR /app +COPY --from=builder /app/go-parser . +EXPOSE 50052 +ENTRYPOINT ["./go-parser"] diff --git a/go.work b/go.work new file mode 100644 index 0000000..c820c63 --- /dev/null +++ b/go.work @@ -0,0 +1,8 @@ +go 1.22.0 + +use ( + ./shared + ./core + ./services/cli + ./plugins/go-parser +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..37d0ddf --- /dev/null +++ b/go.work.sum @@ -0,0 +1,57 @@ +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= +github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240509183442-62759503f434/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.41.0/go.mod h1:Ni4zjJYJ04CDOhG7dn640WGfwBzfE0ecX8TyMB0Fv0Y= +modernc.org/ccgo/v3 v3.16.15/go.mod h1:yT7B+/E2m43tmMOT51GMoM98/MtHIcQQSleGnddkUNI= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE= diff --git a/plugins/go-parser/go.mod b/plugins/go-parser/go.mod new file mode 100644 index 0000000..a85d8c6 --- /dev/null +++ b/plugins/go-parser/go.mod @@ -0,0 +1,18 @@ +module github.com/thinkbattleground/cce/plugins/go-parser + +go 1.22.0 + +require ( + github.com/thinkbattleground/cce/shared v0.0.0 + google.golang.org/grpc v1.64.0 +) + +require ( + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/protobuf v1.34.1 // indirect +) + +replace github.com/thinkbattleground/cce/shared => ../../shared diff --git a/plugins/go-parser/go.sum b/plugins/go-parser/go.sum new file mode 100644 index 0000000..f6e8138 --- /dev/null +++ b/plugins/go-parser/go.sum @@ -0,0 +1,14 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/plugins/go-parser/main.go b/plugins/go-parser/main.go new file mode 100644 index 0000000..5aa317e --- /dev/null +++ b/plugins/go-parser/main.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "log" + "net" + + pb "github.com/thinkbattleground/cce/shared/protocol/v1/plugin" + ucmpb "github.com/thinkbattleground/cce/shared/protocol/v1/ucm" + "google.golang.org/grpc" + "google.golang.org/grpc/reflection" +) + +type GoParserServer struct { + pb.UnimplementedLanguageParserPluginServer +} + +func (s *GoParserServer) GetMetadata(ctx context.Context, req *pb.Empty) (*pb.PluginMetadata, error) { + return &pb.PluginMetadata{ + Name: "Go AST Parser", + Version: "1.0.0", + SupportedExtensions: []string{".go"}, + }, nil +} + +func (s *GoParserServer) ParseFile(ctx context.Context, req *pb.ParseRequest) (*pb.ParseResponse, error) { + // A skeleton implementation returning a dummy UCM payload. + // Production logic will use go/ast or tree-sitter. + log.Printf("Parsing file: %s", req.FilePath) + + return &pb.ParseResponse{ + Payload: &ucmpb.UniversalCodePayload{ + RepositoryId: "dummy-repo-id", + CommitSha: "dummy-commit-sha", + Units: []*ucmpb.CodeUnit{ + { + Id: "dummy-unit-id-1", + Kind: ucmpb.SymbolKind_SYMBOL_KIND_FUNCTION, + Name: "main", + FullyQualifiedName: "main.main", + Location: &ucmpb.Location{ + FilePath: req.FilePath, + StartLine: 10, + StartColumn: 1, + EndLine: 15, + EndColumn: 2, + }, + Signature: "func main()", + Documentation: "Entry point function", + }, + }, + Relationships: []*ucmpb.Relationship{}, + }, + }, nil +} + +func main() { + lis, err := net.Listen("tcp", ":50052") + if err != nil { + log.Fatalf("failed to listen on port 50052: %v", err) + } + + grpcServer := grpc.NewServer() + pb.RegisterLanguageParserPluginServer(grpcServer, &GoParserServer{}) + reflection.Register(grpcServer) + + log.Printf("Go Parser Plugin gRPC server listening on %s", lis.Addr().String()) + if err := grpcServer.Serve(lis); err != nil { + log.Fatalf("failed to serve gRPC: %v", err) + } +} diff --git a/services/cli/go.mod b/services/cli/go.mod new file mode 100644 index 0000000..0c2b866 --- /dev/null +++ b/services/cli/go.mod @@ -0,0 +1,31 @@ +module github.com/thinkbattleground/cce/services/cli + +go 1.22.0 + +require ( + github.com/thinkbattleground/cce/core v0.0.0 + github.com/thinkbattleground/cce/shared v0.0.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/glebarez/go-sqlite v1.22.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect + modernc.org/libc v1.37.6 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.7.2 // indirect + modernc.org/sqlite v1.28.0 // indirect +) + +replace ( + github.com/thinkbattleground/cce/core => ../../core + github.com/thinkbattleground/cce/shared => ../../shared +) diff --git a/services/cli/go.sum b/services/cli/go.sum new file mode 100644 index 0000000..32ff831 --- /dev/null +++ b/services/cli/go.sum @@ -0,0 +1,43 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= +github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw= +modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ= +modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= diff --git a/services/cli/main.go b/services/cli/main.go new file mode 100644 index 0000000..e7cc553 --- /dev/null +++ b/services/cli/main.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + + "github.com/thinkbattleground/cce/core" + "github.com/thinkbattleground/cce/core/parser/loader" + "github.com/thinkbattleground/cce/core/parser/registry" + "github.com/thinkbattleground/cce/core/storage/sqlite" + "github.com/thinkbattleground/cce/shared/logger" +) + +func printUsage() { + fmt.Println("Codebase Context Engine (CCE) CLI") + fmt.Println("Usage:") + fmt.Println(" cce index ") + fmt.Println(" cce search ") + fmt.Println(" cce callers ") + fmt.Println(" cce callees ") + fmt.Println(" cce context ") + os.Exit(1) +} + +func main() { + if len(os.Args) < 3 { + printUsage() + } + + command := os.Args[1] + argument := os.Args[2] + + // Init basic slog logger (text format for CLI) + logger.Setup(logger.Config{Level: "info", Format: "text"}) + + ctx := context.Background() + + // Default to a local sqlite database file + dbPath := "cce.db" + store, err := sqlite.NewSQLiteStorage(dbPath) + if err != nil { + slog.Error("failed to open sqlite database", "error", err) + os.Exit(1) + } + defer store.Close() + + // Initialize tables + err = store.Initialize(ctx) + if err != nil { + slog.Error("failed to initialize sqlite tables", "error", err) + os.Exit(1) + } + + // Default parser plugin address and registry setup + pluginAddr := "localhost:50052" + reg := registry.NewParserRegistry() + err = loader.ConnectAndRegister(pluginAddr, []string{".go"}, reg) + if err != nil { + slog.Warn("could not connect and register go parser plugin (will mock response during processing)", "error", err) + } + + engine := core.NewEngine(store, reg) + + switch command { + case "index": + slog.Info("indexing repository", "path", argument) + err := engine.IndexRepository(ctx, argument) + if err != nil { + slog.Error("indexing failed", "error", err) + os.Exit(1) + } + fmt.Println("Repository indexing completed successfully.") + + case "search": + slog.Info("searching symbols", "query", argument) + results, err := engine.Search(ctx, argument) + if err != nil { + slog.Error("search failed", "error", err) + os.Exit(1) + } + + fmt.Printf("Search Results (%d matches):\n", len(results)) + for _, sym := range results { + fmt.Printf(" - [%v] %s (%s)\n", sym.Kind, sym.FullyQualifiedName, sym.Location.FilePath) + } + + case "callers": + slog.Info("finding callers for symbol", "symbol", argument) + results, err := engine.Callers(ctx, argument) + if err != nil { + slog.Error("failed to find callers", "error", err) + os.Exit(1) + } + + fmt.Printf("Callers of '%s' (%d found):\n", argument, len(results)) + for _, sym := range results { + fmt.Printf(" - %s (%s)\n", sym.FullyQualifiedName, sym.Location.FilePath) + } + + case "callees": + slog.Info("finding callees for symbol", "symbol", argument) + results, err := engine.Callees(ctx, argument) + if err != nil { + slog.Error("failed to find callees", "error", err) + os.Exit(1) + } + + fmt.Printf("Callees of '%s' (%d found):\n", argument, len(results)) + for _, sym := range results { + fmt.Printf(" - %s (%s)\n", sym.FullyQualifiedName, sym.Location.FilePath) + } + + case "context": + slog.Info("compiling context package", "symbol", argument) + c, err := engine.Context(ctx, argument) + if err != nil { + slog.Error("failed to compile context package", "error", err) + os.Exit(1) + } + + fmt.Printf("--- Symbol Context: %s ---\n", c.TargetSymbol.FullyQualifiedName) + fmt.Printf("Signature: %s\n", c.TargetSymbol.Signature) + fmt.Printf("File: %s\n", c.TargetSymbol.Location.FilePath) + fmt.Printf("Docstring: %s\n", c.TargetSymbol.Documentation) + fmt.Println("\nCode Snippet:") + if c.TargetSymbol.Properties != nil { + if snippet, ok := c.TargetSymbol.Properties["code_snippet"]; ok { + fmt.Println(snippet) + } else { + fmt.Println(" [No Code Snippet Available]") + } + } else { + fmt.Println(" [No Code Snippet Available]") + } + + fmt.Printf("\nCallers (%d):\n", len(c.Callers)) + for _, sym := range c.Callers { + fmt.Printf(" <- %s\n", sym.FullyQualifiedName) + } + + fmt.Printf("\nCallees (%d):\n", len(c.Callees)) + for _, sym := range c.Callees { + fmt.Printf(" -> %s\n", sym.FullyQualifiedName) + } + + default: + printUsage() + } +} diff --git a/shared/config/config.go b/shared/config/config.go new file mode 100644 index 0000000..cefdc26 --- /dev/null +++ b/shared/config/config.go @@ -0,0 +1,74 @@ +package config + +import ( + "fmt" + "os" + + "github.com/ilyakaznacheev/cleanenv" + "github.com/thinkbattleground/cce/shared/logger" +) + +// Config holds all config settings for CCE services. +type Config struct { + Env string `yaml:"env" env:"APP_ENV" env-default:"development"` + GRPCAddress string `yaml:"grpc_address" env:"GRPC_ADDRESS" env-default:":50051"` + HTTPAddress string `yaml:"http_address" env:"HTTP_ADDRESS" env-default:":8080"` + Logger logger.Config `yaml:"logger"` + Postgres PostgresConfig `yaml:"postgres"` + Neo4j Neo4jConfig `yaml:"neo4j"` + OpenSearch OpenSearchConfig `yaml:"opensearch"` + Nats NatsConfig `yaml:"nats"` + Otel OtelConfig `yaml:"otel"` +} + +type PostgresConfig struct { + Host string `yaml:"host" env:"DB_HOST" env-default:"localhost"` + Port int `yaml:"port" env:"DB_PORT" env-default:"5432"` + User string `yaml:"user" env:"DB_USER" env-default:"postgres"` + Password string `yaml:"password" env:"DB_PASSWORD" env-default:"postgres"` + Database string `yaml:"database" env:"DB_NAME" env-default:"cce"` + SSLMode string `yaml:"ssl_mode" env:"DB_SSL_MODE" env-default:"disable"` +} + +type Neo4jConfig struct { + URI string `yaml:"uri" env:"NEO4J_URI" env-default:"neo4j://localhost:7687"` + Username string `yaml:"username" env:"NEO4J_USERNAME" env-default:"neo4j"` + Password string `yaml:"password" env:"NEO4J_PASSWORD" env-default:"password"` +} + +type OpenSearchConfig struct { + Addresses []string `yaml:"addresses" env:"OPENSEARCH_ADDRESSES" env-default:"http://localhost:9200"` + Username string `yaml:"username" env:"OPENSEARCH_USERNAME" env-default:"admin"` + Password string `yaml:"password" env:"OPENSEARCH_PASSWORD" env-default:"admin"` +} + +type NatsConfig struct { + URL string `yaml:"url" env:"NATS_URL" env-default:"nats://localhost:4222"` +} + +type OtelConfig struct { + ServiceName string `yaml:"service_name" env:"OTEL_SERVICE_NAME" env-default:"cce"` + Endpoint string `yaml:"endpoint" env:"OTEL_EXPORTER_OTLP_ENDPOINT" env-default:"localhost:4317"` + Insecure bool `yaml:"insecure" env:"OTEL_EXPORTER_OTLP_INSECURE" env-default:"true"` +} + +// Load reads config from file or environment variables. +func Load(configPath string) (*Config, error) { + var cfg Config + + if configPath != "" { + if _, err := os.Stat(configPath); err == nil { + if err := cleanenv.ReadConfig(configPath, &cfg); err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + return &cfg, nil + } + } + + // Read environment variables directly if file doesn't exist + if err := cleanenv.ReadEnv(&cfg); err != nil { + return nil, fmt.Errorf("failed to read environment variables: %w", err) + } + + return &cfg, nil +} diff --git a/shared/config/config_test.go b/shared/config/config_test.go new file mode 100644 index 0000000..10eb811 --- /dev/null +++ b/shared/config/config_test.go @@ -0,0 +1,45 @@ +package config + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfigLoadDefaults(t *testing.T) { + cfg, err := Load("") + require.NoError(t, err) + + assert.Equal(t, "development", cfg.Env) + assert.Equal(t, ":50051", cfg.GRPCAddress) + assert.Equal(t, "info", cfg.Logger.Level) + assert.Equal(t, "localhost", cfg.Postgres.Host) + assert.Equal(t, 5432, cfg.Postgres.Port) + assert.Equal(t, "neo4j://localhost:7687", cfg.Neo4j.URI) + assert.Equal(t, "nats://localhost:4222", cfg.Nats.URL) +} + +func TestConfigEnvOverrides(t *testing.T) { + // Set test environment variables + os.Setenv("APP_ENV", "production") + os.Setenv("GRPC_ADDRESS", ":9090") + os.Setenv("DB_HOST", "db-prod.internal") + os.Setenv("DB_PORT", "6543") + + defer func() { + os.Unsetenv("APP_ENV") + os.Unsetenv("GRPC_ADDRESS") + os.Unsetenv("DB_HOST") + os.Unsetenv("DB_PORT") + }() + + cfg, err := Load("") + require.NoError(t, err) + + assert.Equal(t, "production", cfg.Env) + assert.Equal(t, ":9090", cfg.GRPCAddress) + assert.Equal(t, "db-prod.internal", cfg.Postgres.Host) + assert.Equal(t, 6543, cfg.Postgres.Port) +} diff --git a/shared/db/postgres/postgres.go b/shared/db/postgres/postgres.go new file mode 100644 index 0000000..0d1dd96 --- /dev/null +++ b/shared/db/postgres/postgres.go @@ -0,0 +1,116 @@ +package postgres + +import ( + "context" + "fmt" + "log/slog" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/thinkbattleground/cce/shared/config" +) + +// Client wraps the PostgreSQL connection pool. +type Client struct { + Pool *pgxpool.Pool +} + +// NewClient establishes a PostgreSQL connection pool. +func NewClient(ctx context.Context, cfg config.PostgresConfig) (*Client, error) { + connStr := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s", + cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database, cfg.SSLMode) + + pool, err := pgxpool.New(ctx, connStr) + if err != nil { + return nil, fmt.Errorf("unable to connect to database: %w", err) + } + + // Verify connection + if err := pool.Ping(ctx); err != nil { + return nil, fmt.Errorf("failed to ping postgres: %w", err) + } + + slog.Info("connected to PostgreSQL database", "host", cfg.Host, "database", cfg.Database) + return &Client{Pool: pool}, nil +} + +// Close gracefully closes the connection pool. +func (c *Client) Close() { + if c.Pool != nil { + c.Pool.Close() + slog.Info("closed PostgreSQL connection pool") + } +} + +// DDLSchema holds the core PostgreSQL schema defined in the CCE Architecture Design. +const DDLSchema = ` +CREATE TABLE IF NOT EXISTS tenants ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name VARCHAR(255) NOT NULL UNIQUE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS repositories ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tenant_id UUID REFERENCES tenants(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + clone_url VARCHAR(1024) NOT NULL, + branch VARCHAR(255) DEFAULT 'main', + auth_secret_arn VARCHAR(512), + current_head VARCHAR(40), + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + UNIQUE(tenant_id, clone_url) +); + +CREATE TABLE IF NOT EXISTS index_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + repository_id UUID REFERENCES repositories(id) ON DELETE CASCADE, + commit_sha VARCHAR(40) NOT NULL, + status VARCHAR(50) NOT NULL, + error_message TEXT, + started_at TIMESTAMP WITH TIME ZONE, + completed_at TIMESTAMP WITH TIME ZONE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS files ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + repository_id UUID REFERENCES repositories(id) ON DELETE CASCADE, + path VARCHAR(1024) NOT NULL, + sha256 VARCHAR(64) NOT NULL, + language VARCHAR(50) NOT NULL, + line_count INTEGER NOT NULL, + size_bytes INTEGER NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + UNIQUE(repository_id, path) +); + +CREATE TABLE IF NOT EXISTS symbols ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + file_id UUID REFERENCES files(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + kind VARCHAR(50) NOT NULL, + start_line INTEGER NOT NULL, + start_column INTEGER NOT NULL, + end_line INTEGER NOT NULL, + end_column INTEGER NOT NULL, + signature TEXT, + documentation TEXT, + embedding VECTOR(1536), -- Optional pgvector field for hybrid lookup + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_symbols_file_kind ON symbols(file_id, kind); +CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name); +` + +// InitSchema sets up database tables. +func (c *Client) InitSchema(ctx context.Context) error { + slog.Info("initializing PostgreSQL database schema") + _, err := c.Pool.Exec(ctx, DDLSchema) + if err != nil { + return fmt.Errorf("failed to initialize schema: %w", err) + } + return nil +} diff --git a/shared/go.mod b/shared/go.mod new file mode 100644 index 0000000..3af6c70 --- /dev/null +++ b/shared/go.mod @@ -0,0 +1,41 @@ +module github.com/thinkbattleground/cce/shared + +go 1.22.0 + +require ( + github.com/ilyakaznacheev/cleanenv v1.5.0 + github.com/jackc/pgx/v5 v5.6.0 + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 + google.golang.org/grpc v1.64.0 + google.golang.org/protobuf v1.34.1 +) + +require ( + github.com/BurntSushi/toml v1.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect +) diff --git a/shared/go.sum b/shared/go.sum new file mode 100644 index 0000000..37a508e --- /dev/null +++ b/shared/go.sum @@ -0,0 +1,83 @@ +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4= +github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 h1:slmdOY3vp8a7KQbHkL+FLbvbkgMqmXojpFUO/jENuqQ= +olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3/go.mod h1:oVgVk4OWVDi43qWBEyGhXgYxt7+ED4iYNpTngSLX2Iw= diff --git a/shared/logger/logger.go b/shared/logger/logger.go new file mode 100644 index 0000000..ffdcfbe --- /dev/null +++ b/shared/logger/logger.go @@ -0,0 +1,45 @@ +package logger + +import ( + "log/slog" + "os" + "strings" +) + +// Config defines the configuration for the structured logger. +type Config struct { + Level string `yaml:"level" env:"LOG_LEVEL" env-default:"info"` + Format string `yaml:"format" env:"LOG_FORMAT" env-default:"json"` // json or text +} + +// Setup initializes the global structured logger based on configuration. +func Setup(cfg Config) *slog.Logger { + var level slog.Level + switch strings.ToLower(cfg.Level) { + case "debug": + level = slog.LevelDebug + case "info": + level = slog.LevelInfo + case "warn": + level = slog.LevelWarn + case "error": + level = slog.LevelError + default: + level = slog.LevelInfo + } + + opts := &slog.HandlerOptions{ + Level: level, + } + + var handler slog.Handler + if strings.ToLower(cfg.Format) == "text" { + handler = slog.NewTextHandler(os.Stdout, opts) + } else { + handler = slog.NewJSONHandler(os.Stdout, opts) + } + + logger := slog.New(handler) + slog.SetDefault(logger) + return logger +} diff --git a/shared/logger/logger_test.go b/shared/logger/logger_test.go new file mode 100644 index 0000000..63b5675 --- /dev/null +++ b/shared/logger/logger_test.go @@ -0,0 +1,25 @@ +package logger + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSetupLogger(t *testing.T) { + cfg := Config{ + Level: "debug", + Format: "text", + } + + logger := Setup(cfg) + assert.NotNil(t, logger) + + cfgJSON := Config{ + Level: "error", + Format: "json", + } + + loggerJSON := Setup(cfgJSON) + assert.NotNil(t, loggerJSON) +} diff --git a/shared/protocol/v1/plugin/placeholder.go b/shared/protocol/v1/plugin/placeholder.go new file mode 100644 index 0000000..b480ae1 --- /dev/null +++ b/shared/protocol/v1/plugin/placeholder.go @@ -0,0 +1,157 @@ +package plugin + +import ( + "context" + + "github.com/thinkbattleground/cce/shared/protocol/v1/ucm" + "google.golang.org/grpc" + "google.golang.org/protobuf/runtime/protoimpl" +) + +type Empty struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (*Empty) ProtoMessage() {} + +type PluginMetadata struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + SupportedExtensions []string `protobuf:"bytes,3,rep,name=supported_extensions,json=supportedExtensions,proto3" json:"supported_extensions,omitempty"` +} + +func (*PluginMetadata) ProtoMessage() {} + +type ParseRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + FilePath string `protobuf:"bytes,1,opt,name=file_path,json=filePath,proto3" json:"file_path,omitempty"` + FileContent []byte `protobuf:"bytes,2,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"` +} + +func (*ParseRequest) ProtoMessage() {} + +type ParseResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Payload *ucm.UniversalCodePayload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + ParsingErrors []string `protobuf:"bytes,2,rep,name=parsing_errors,json=parsingErrors,proto3" json:"parsing_errors,omitempty"` +} + +func (*ParseResponse) ProtoMessage() {} + +// LanguageParserPluginClient is the client API for LanguageParserPlugin service. +type LanguageParserPluginClient interface { + GetMetadata(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*PluginMetadata, error) + ParseFile(ctx context.Context, in *ParseRequest, opts ...grpc.CallOption) (*ParseResponse, error) +} + +type languageParserPluginClient struct { + cc grpc.ClientConnInterface +} + +func NewLanguageParserPluginClient(cc grpc.ClientConnInterface) LanguageParserPluginClient { + return &languageParserPluginClient{cc} +} + +func (c *languageParserPluginClient) GetMetadata(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*PluginMetadata, error) { + out := new(PluginMetadata) + err := c.cc.Invoke(ctx, "/cce.plugin.v1.LanguageParserPlugin/GetMetadata", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *languageParserPluginClient) ParseFile(ctx context.Context, in *ParseRequest, opts ...grpc.CallOption) (*ParseResponse, error) { + out := new(ParseResponse) + err := c.cc.Invoke(ctx, "/cce.plugin.v1.LanguageParserPlugin/ParseFile", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// LanguageParserPluginServer is the server API for LanguageParserPlugin service. +type LanguageParserPluginServer interface { + GetMetadata(context.Context, *Empty) (*PluginMetadata, error) + ParseFile(context.Context, *ParseRequest) (*ParseResponse, error) + mustEmbedUnimplementedLanguageParserPluginServer() +} + +type UnimplementedLanguageParserPluginServer struct{} + +func (UnimplementedLanguageParserPluginServer) GetMetadata(context.Context, *Empty) (*PluginMetadata, error) { + return nil, nil +} +func (UnimplementedLanguageParserPluginServer) ParseFile(context.Context, *ParseRequest) (*ParseResponse, error) { + return nil, nil +} +func (UnimplementedLanguageParserPluginServer) mustEmbedUnimplementedLanguageParserPluginServer() {} + +func RegisterLanguageParserPluginServer(s grpc.ServiceRegistrar, srv LanguageParserPluginServer) { + s.RegisterService(&LanguageParserPlugin_ServiceDesc, srv) +} + +var LanguageParserPlugin_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "cce.plugin.v1.LanguageParserPlugin", + HandlerType: (*LanguageParserPluginServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetMetadata", + Handler: _LanguageParserPlugin_GetMetadata_Handler, + }, + { + MethodName: "ParseFile", + Handler: _LanguageParserPlugin_ParseFile_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "v1/cce_plugin.proto", +} + +func _LanguageParserPlugin_GetMetadata_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LanguageParserPluginServer).GetMetadata(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cce.plugin.v1.LanguageParserPlugin/GetMetadata", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LanguageParserPluginServer).GetMetadata(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _LanguageParserPlugin_ParseFile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ParseRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LanguageParserPluginServer).ParseFile(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cce.plugin.v1.LanguageParserPlugin/ParseFile", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LanguageParserPluginServer).ParseFile(ctx, req.(*ParseRequest)) + } + return interceptor(ctx, in, info, handler) +} diff --git a/shared/protocol/v1/ucm/placeholder.go b/shared/protocol/v1/ucm/placeholder.go new file mode 100644 index 0000000..a696f78 --- /dev/null +++ b/shared/protocol/v1/ucm/placeholder.go @@ -0,0 +1,88 @@ +package ucm + +import "google.golang.org/protobuf/runtime/protoimpl" + +type SymbolKind int32 + +const ( + SymbolKind_SYMBOL_KIND_UNSPECIFIED SymbolKind = 0 + SymbolKind_SYMBOL_KIND_MODULE SymbolKind = 1 + SymbolKind_SYMBOL_KIND_PACKAGE SymbolKind = 2 + SymbolKind_SYMBOL_KIND_CLASS SymbolKind = 3 + SymbolKind_SYMBOL_KIND_STRUCT SymbolKind = 4 + SymbolKind_SYMBOL_KIND_INTERFACE SymbolKind = 5 + SymbolKind_SYMBOL_KIND_FUNCTION SymbolKind = 6 + SymbolKind_SYMBOL_KIND_METHOD SymbolKind = 7 + SymbolKind_SYMBOL_KIND_VARIABLE SymbolKind = 8 + SymbolKind_SYMBOL_KIND_ANNOTATION SymbolKind = 9 +) + +type RelationType int32 + +const ( + RelationType_RELATION_TYPE_UNSPECIFIED RelationType = 0 + RelationType_RELATION_TYPE_CONTAINS RelationType = 1 + RelationType_RELATION_TYPE_CALLS RelationType = 2 + RelationType_RELATION_TYPE_IMPLEMENTS RelationType = 3 + RelationType_RELATION_TYPE_REFERENCES RelationType = 4 + RelationType_RELATION_TYPE_IMPORTS RelationType = 5 +) + +type Location struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + FilePath string `protobuf:"bytes,1,opt,name=file_path,json=filePath,proto3" json:"file_path,omitempty"` + StartLine int32 `protobuf:"varint,2,opt,name=start_line,json=startLine,proto3" json:"start_line,omitempty"` + StartColumn int32 `protobuf:"varint,3,opt,name=start_column,json=startColumn,proto3" json:"start_column,omitempty"` + EndLine int32 `protobuf:"varint,4,opt,name=end_line,json=endLine,proto3" json:"end_line,omitempty"` + EndColumn int32 `protobuf:"varint,5,opt,name=end_column,json=endColumn,proto3" json:"end_column,omitempty"` +} + +func (*Location) ProtoMessage() {} + +type CodeUnit struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Kind SymbolKind `protobuf:"varint,2,opt,name=kind,proto3,enum=cce.v1.ucm.SymbolKind" json:"kind,omitempty"` + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + FullyQualifiedName string `protobuf:"bytes,4,opt,name=fully_qualified_name,json=fullyQualifiedName,proto3" json:"fully_qualified_name,omitempty"` + Location *Location `protobuf:"bytes,5,opt,name=location,proto3" json:"location,omitempty"` + Signature string `protobuf:"bytes,6,opt,name=signature,proto3" json:"signature,omitempty"` + Documentation string `protobuf:"bytes,7,opt,name=documentation,proto3" json:"documentation,omitempty"` + Properties map[string]string `protobuf:"bytes,8,rep,name=properties,proto3" json:"properties,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (*CodeUnit) ProtoMessage() {} + +type Relationship struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + FromId string `protobuf:"bytes,1,opt,name=from_id,json=fromId,proto3" json:"from_id,omitempty"` + ToId string `protobuf:"bytes,2,opt,name=to_id,json=toId,proto3" json:"to_id,omitempty"` + Type RelationType `protobuf:"varint,3,opt,name=type,proto3,enum=cce.v1.ucm.RelationType" json:"type,omitempty"` + Location *Location `protobuf:"bytes,4,opt,name=location,proto3" json:"location,omitempty"` +} + +func (*Relationship) ProtoMessage() {} + +type UniversalCodePayload struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + RepositoryId string `protobuf:"bytes,1,opt,name=repository_id,json=repositoryId,proto3" json:"repository_id,omitempty"` + CommitSha string `protobuf:"bytes,2,opt,name=commit_sha,json=commitSha,proto3" json:"commit_sha,omitempty"` + Units []*CodeUnit `protobuf:"bytes,3,rep,name=units,proto3" json:"units,omitempty"` + Relationships []*Relationship `protobuf:"bytes,4,rep,name=relationships,proto3" json:"relationships,omitempty"` +} + +func (*UniversalCodePayload) ProtoMessage() {} +func (m *UniversalCodePayload) Reset() { *m = UniversalCodePayload{} } +func (m *UniversalCodePayload) String() string { return m.RepositoryId } diff --git a/shared/tracer/tracer.go b/shared/tracer/tracer.go new file mode 100644 index 0000000..122d04c --- /dev/null +++ b/shared/tracer/tracer.go @@ -0,0 +1,79 @@ +package tracer + +import ( + "context" + "fmt" + "time" + + "github.com/thinkbattleground/cce/shared/config" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.24.0" + "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// InitTracer configures and registers a global OpenTelemetry TracerProvider. +// It returns a shutdown function that should be called during application termination. +func InitTracer(ctx context.Context, cfg config.OtelConfig) (func(context.Context) error, error) { + // Set up the gRPC connection to the Otel Collector + var dialOpts []grpc.DialOption + if cfg.Insecure { + dialOpts = append(dialOpts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + + exporter, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithInsecure(), + otlptracegrpc.WithEndpoint(cfg.Endpoint), + otlptracegrpc.WithDialOption(dialOpts...), + ) + if err != nil { + return nil, fmt.Errorf("failed to create OTLP trace exporter: %w", err) + } + + // Create resource definition + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceNameKey.String(cfg.ServiceName), + semconv.ServiceVersionKey.String("1.0.0"), + ), + ) + if err != nil { + return nil, fmt.Errorf("failed to create otel resource: %w", err) + } + + // Create tracer provider + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter, sdktrace.WithBatchTimeout(5*time.Second)), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) + + // Set global tracer provider + otel.SetTracerProvider(tp) + + // Set global propagator (W3C Trace Context) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + + // Return a function to shutdown/flush traces gracefully + shutdown := func(shutdownCtx context.Context) error { + if err := tp.Shutdown(shutdownCtx); err != nil { + return fmt.Errorf("failed to shutdown tracer provider: %w", err) + } + return nil + } + + return shutdown, nil +} + +// GetTracer returns a tracer instance with the specified name. +func GetTracer(name string) trace.Tracer { + return otel.GetTracerProvider().Tracer(name) +}