feat(docs): add TODO list for completed and future improvements

feat(package): update version to 0.1.5 in package.json
feat(storage): implement bulk file fetching and processing in repo client

Author: ErcinDedeoglu

README.md

@@ -114,6 +114,7 @@ See [Configuration Guide](docs/CONFIG.md) for all options.
 | [Development Setup](docs/DEV-SETUP.md) | Local development environment |
 | [Publishing](docs/PUBLISH.md) | npm release process |
 | [LLM Installation](docs/LLM-INSTALL.md) | Instructions for AI coding agents |
+| [TODO](docs/TODO.md) | Future improvements |
 
 ## License
 

docs/TODO.md

@@ -0,0 +1,15 @@
+# TODO
+
+## Completed
+
+- [x] Push optimization: inline content in tree (O(n) → O(1))
+- [x] Pull optimization: raw.githubusercontent.com (0 API calls)
+- [x] Rate limit detection and backoff
+- [x] Skip push when no changes
+
+## Future
+
+- [ ] ETag caching for manifest fetch (saves bandwidth, not API calls)
+- [ ] GitHub webhook for push notifications (eliminate polling)
+- [ ] Compression improvements for large messages
+- [ ] Parallel category processing during sync

package.json

@@ -1,6 +1,6 @@
 {
   "name": "oc-sync",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "Sync OpenCode data across machines using a private GitHub repository with vector clock conflict resolution",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

src/storage/interface.ts

@@ -32,6 +32,9 @@ export interface StorageBackend {
   /** Get file content by path */
   getFile(path: string): Promise<string | null>;
 
+  /** Get multiple files in bulk (optimized for fewer API calls) */
+  getFiles(paths: string[]): Promise<Record<string, string | null>>;
+
   /** Update multiple files atomically (null value = delete) */
   updateFiles(files: Record<string, string | null>): Promise<void>;
 

src/storage/repo/repo-client.ts

@@ -31,17 +31,21 @@ interface GitRef {
   object: { sha: string; type: string };
 }
 
-interface GitTree {
+interface GitTreeResponse {
   sha: string;
-  tree: { path: string; mode: string; type: string; sha: string }[];
 }
 
-interface GitBlob {
+interface GitCommit {
   sha: string;
 }
 
-interface GitCommit {
-  sha: string;
+/** Tree entry for GitHub API - can use either sha (existing blob) or content (inline) */
+interface TreeEntry {
+  path: string;
+  mode: string;
+  type: string;
+  sha?: string | null;
+  content?: string;
 }
 
 interface ContentFile {
@@ -132,17 +136,12 @@ export class RepoStorageBackend implements StorageBackend {
     // Get current HEAD commit SHA
     const headSha = await this.getHeadSha();
 
-    // Get current tree
-    const currentTree = await this.getTree(headSha);
-
-    // Build new tree entries (includes blob creation)
-    const blobStart = Date.now();
-    const treeEntries = await this.buildTreeEntries(files, currentTree);
-    const blobDuration = Date.now() - blobStart;
-    this.logProgress(`Created ${String(treeEntries.length)} blobs in ${String(blobDuration)}ms`);
+    // Build tree entries with inline content (no separate blob creation needed)
+    const treeEntries = this.buildTreeEntries(files);
+    this.logProgress(`Prepared ${String(treeEntries.length)} tree entries`);
 
-    // Create new tree
-    const newTreeSha = await this.createTree(treeEntries, currentTree.sha);
+    // Create new tree with base_tree for incremental update
+    const newTreeSha = await this.createTree(treeEntries, headSha);
 
     // Create commit
     const message = `Sync update: ${String(fileCount)} files`;
@@ -152,7 +151,9 @@ export class RepoStorageBackend implements StorageBackend {
     await this.updateRef(commitSha);
 
     const totalDuration = Date.now() - startTime;
-    this.logProgress(`Upload complete: ${String(fileCount)} files in ${String(totalDuration)}ms`);
+    this.logProgress(
+      `Upload complete: ${String(fileCount)} files in ${String(totalDuration)}ms (~5 API calls)`
+    );
   }
 
   /** Log progress for debugging */
@@ -185,6 +186,66 @@ export class RepoStorageBackend implements StorageBackend {
     }));
   }
 
+  /**
+   * Bulk fetch multiple files using raw.githubusercontent.com.
+   * No API rate limits - all files fetched in parallel.
+   */
+  public async getFiles(paths: string[]): Promise<Record<string, string | null>> {
+    if (paths.length === 0) return {};
+
+    const startTime = Date.now();
+    const fetched = await this.fetchFilesViaRaw(paths);
+
+    const result: Record<string, string | null> = {};
+    for (const { path, content } of fetched) {
+      result[path] = content;
+    }
+
+    const duration = Date.now() - startTime;
+    const found = fetched.filter((f) => f.content !== null).length;
+    this.logProgress(
+      `Bulk fetch: ${String(found)}/${String(paths.length)} files in ${String(duration)}ms (0 API calls)`
+    );
+
+    return result;
+  }
+
+  /**
+   * Fetch files via raw.githubusercontent.com (no API rate limits).
+   * All fetches run in parallel - raw content endpoint has no rate limiting.
+   */
+  private async fetchFilesViaRaw(
+    paths: string[]
+  ): Promise<{ path: string; content: string | null }[]> {
+    const branch = await this.getBranch();
+    const baseUrl = `https://raw.githubusercontent.com/${this.owner}/${this.repo}/${branch}`;
+
+    const results = await Promise.all(
+      paths.map(async (path) => {
+        const content = await this.fetchRawFile(`${baseUrl}/${SYNC_DIR}/${path}`);
+        return { path, content };
+      })
+    );
+
+    return results;
+  }
+
+  /** Fetch a single file from raw.githubusercontent.com */
+  private async fetchRawFile(url: string): Promise<string | null> {
+    try {
+      const res = await fetchWithRetry(
+        url,
+        { headers: { Authorization: `Bearer ${this.token}` } },
+        this.maxRetries,
+        this.retryDelayMs
+      );
+      if (!res.ok) return null;
+      return await res.text();
+    } catch {
+      return null;
+    }
+  }
+
   // --- Private helpers ---
 
   private async fetch(path: string, options?: RequestInit): Promise<Response> {
@@ -336,122 +397,51 @@ export class RepoStorageBackend implements StorageBackend {
     return data.object.sha;
   }
 
-  private async getTree(commitSha: string): Promise<GitTree> {
-    const res = await this.fetch(`/git/trees/${commitSha}?recursive=1`);
-    if (!res.ok) {
-      throw new RepoApiError('Failed to get tree', res.status);
-    }
-    return (await res.json()) as GitTree;
-  }
-
-  private async buildTreeEntries(
-    files: Record<string, string | null>,
-    currentTree: GitTree
-  ): Promise<{ path: string; mode: string; type: string; sha?: string | null }[]> {
-    // Start with existing tree entries (excluding ones we're updating/deleting)
-    const updatedPaths = new Set(Object.keys(files).map((p) => `${SYNC_DIR}/${p}`));
-
-    const entries = currentTree.tree
-      .filter((e) => !updatedPaths.has(e.path))
-      .map((e) => ({ path: e.path, mode: e.mode, type: e.type, sha: e.sha }));
+  /**
+   * Build tree entries with inline content.
+   * Uses GitHub's ability to accept `content` directly instead of blob SHA,
+   * reducing API calls from N+5 to just 5 (regardless of file count).
+   */
+  private buildTreeEntries(files: Record<string, string | null>): TreeEntry[] {
+    const entries: TreeEntry[] = [];
 
-    // Collect files that need blob creation
-    const filesToUpload: { path: string; content: string }[] = [];
     for (const [path, content] of Object.entries(files)) {
-      if (content !== null) {
-        filesToUpload.push({ path: `${SYNC_DIR}/${path}`, content });
+      const fullPath = `${SYNC_DIR}/${path}`;
+      if (content === null) {
+        // Delete file by setting sha to null
+        entries.push({ path: fullPath, mode: '100644', type: 'blob', sha: null });
+      } else {
+        // Add/update file with inline content (no blob creation needed!)
+        entries.push({ path: fullPath, mode: '100644', type: 'blob', content });
       }
     }
 
-    // Create blobs in small parallel batches with delays to avoid secondary rate limits
-    const BATCH_SIZE = 5;
-    const blobResults = await this.createBlobsInBatches(filesToUpload, BATCH_SIZE);
-
-    // Add new entries from batch results
-    for (const result of blobResults) {
-      entries.push({
-        path: result.path,
-        mode: '100644',
-        type: 'blob',
-        sha: result.sha,
-      });
-    }
-
     return entries;
   }
 
   /**
-   * Create blobs in parallel batches with rate limit protection.
-   * Uses small batches with delays to avoid GitHub's secondary rate limits.
+   * Create a new tree using base_tree for incremental updates.
+   * Entries can use either `sha` (for existing blobs) or `content` (inline).
    */
-  private async createBlobsInBatches(
-    files: { path: string; content: string }[],
-    batchSize: number
-  ): Promise<{ path: string; sha: string }[]> {
-    const results: { path: string; sha: string }[] = [];
-    const totalBatches = Math.ceil(files.length / batchSize);
-
-    for (let i = 0; i < files.length; i += batchSize) {
-      const batchNum = Math.floor(i / batchSize) + 1;
-      const batch = files.slice(i, i + batchSize);
-
-      if (files.length > batchSize) {
-        this.logProgress(
-          `Batch ${String(batchNum)}/${String(totalBatches)} (${String(batch.length)} files)`
-        );
-      }
-
-      const batchPromises = batch.map(async (file) => {
-        const sha = await this.createBlob(file.content);
-        return { path: file.path, sha };
-      });
-      const batchResults = await Promise.all(batchPromises);
-      results.push(...batchResults);
-
-      // Add delay between batches to avoid secondary rate limits
-      if (i + batchSize < files.length) {
-        await new Promise((r) => setTimeout(r, 100));
-      }
-    }
-
-    return results;
-  }
-
-  private async createBlob(content: string): Promise<string> {
-    const body = JSON.stringify({
-      content: Buffer.from(content).toString('base64'),
-      encoding: 'base64',
-    });
-
-    const res = await this.fetch('/git/blobs', { method: 'POST', body });
-    if (!res.ok) {
-      throw new RepoApiError('Failed to create blob', res.status);
-    }
-
-    const data = (await res.json()) as GitBlob;
-    return data.sha;
-  }
-
-  private async createTree(
-    entries: { path: string; mode: string; type: string; sha?: string | null }[],
-    baseSha?: string
-  ): Promise<string> {
+  private async createTree(entries: TreeEntry[], baseSha: string): Promise<string> {
     const body = JSON.stringify({
       base_tree: baseSha,
-      tree: entries.map((e) => ({
-        path: e.path,
-        mode: e.mode,
-        type: e.type,
-        sha: e.sha,
-      })),
+      tree: entries.map((e) => {
+        if (e.content !== undefined) {
+          // Inline content - GitHub will create the blob automatically
+          return { path: e.path, mode: e.mode, type: e.type, content: e.content };
+        }
+        // Delete (sha: null) or reference existing blob
+        return { path: e.path, mode: e.mode, type: e.type, sha: e.sha };
+      }),
     });
 
     const res = await this.fetch('/git/trees', { method: 'POST', body });
     if (!res.ok) {
       throw new RepoApiError('Failed to create tree', res.status);
     }
 
-    const data = (await res.json()) as GitTree;
+    const data = (await res.json()) as GitTreeResponse;
     return data.sha;
   }