chore(internal): cache fetch instruction calls in MCP server

Author: stainless-app[bot]

packages/mcp-server/src/instructions.ts

@@ -0,0 +1,74 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { readEnv } from './util';
+
+const INSTRUCTIONS_CACHE_TTL_MS = 15 * 60 * 1000; // 15 minutes
+
+interface InstructionsCacheEntry {
+  fetchedInstructions: string;
+  fetchedAt: number;
+}
+
+const instructionsCache = new Map<string, InstructionsCacheEntry>();
+
+// Periodically evict stale entries so the cache doesn't grow unboundedly.
+const _cacheCleanupInterval = setInterval(() => {
+  const now = Date.now();
+  for (const [key, entry] of instructionsCache) {
+    if (now - entry.fetchedAt > INSTRUCTIONS_CACHE_TTL_MS) {
+      instructionsCache.delete(key);
+    }
+  }
+}, INSTRUCTIONS_CACHE_TTL_MS);
+
+// Don't keep the process alive just for cleanup.
+_cacheCleanupInterval.unref();
+
+export async function getInstructions(stainlessApiKey: string | undefined): Promise<string> {
+  const cacheKey = stainlessApiKey ?? '';
+  const cached = instructionsCache.get(cacheKey);
+
+  if (cached && Date.now() - cached.fetchedAt <= INSTRUCTIONS_CACHE_TTL_MS) {
+    return cached.fetchedInstructions;
+  }
+
+  const fetchedInstructions = await fetchLatestInstructions(stainlessApiKey);
+  instructionsCache.set(cacheKey, { fetchedInstructions, fetchedAt: Date.now() });
+  return fetchedInstructions;
+}
+
+async function fetchLatestInstructions(stainlessApiKey: string | undefined): Promise<string> {
+  // Setting the stainless API key is optional, but may be required
+  // to authenticate requests to the Stainless API.
+  const response = await fetch(
+    readEnv('CODE_MODE_INSTRUCTIONS_URL') ?? 'https://api.stainless.com/api/ai/instructions/scan-documents',
+    {
+      method: 'GET',
+      headers: { ...(stainlessApiKey && { Authorization: stainlessApiKey }) },
+    },
+  );
+
+  let instructions: string | undefined;
+  if (!response.ok) {
+    console.warn(
+      'Warning: failed to retrieve MCP server instructions. Proceeding with default instructions...',
+    );
+
+    instructions = `
+      This is the scan-documents MCP server. You will use Code Mode to help the user perform
+      actions. You can use search_docs tool to learn about how to take action with this server. Then,
+      you will write TypeScript code using the execute tool take action. It is CRITICAL that you be
+      thoughtful and deliberate when executing code. Always try to entirely solve the problem in code
+      block: it can be as long as you need to get the job done!
+    `;
+  }
+
+  instructions ??= ((await response.json()) as { instructions: string }).instructions;
+  instructions = `
+    If needed, you can get the current time by executing Date.now().
+
+    ${instructions}
+  `;
+
+  return instructions;
+}

packages/mcp-server/src/server.ts

@@ -11,46 +11,10 @@ import { ClientOptions } from 'scan-documents';
 import ScanDocuments from 'scan-documents';
 import { codeTool } from './code-tool';
 import docsSearchTool from './docs-search-tool';
+import { getInstructions } from './instructions';
 import { McpOptions } from './options';
 import { blockedMethodsForCodeTool } from './methods';
 import { HandlerFunction, McpRequestContext, ToolCallResult, McpTool } from './types';
-import { readEnv } from './util';
-
-async function getInstructions(stainlessApiKey: string | undefined): Promise<string> {
-  // Setting the stainless API key is optional, but may be required
-  // to authenticate requests to the Stainless API.
-  const response = await fetch(
-    readEnv('CODE_MODE_INSTRUCTIONS_URL') ?? 'https://api.stainless.com/api/ai/instructions/scan-documents',
-    {
-      method: 'GET',
-      headers: { ...(stainlessApiKey && { Authorization: stainlessApiKey }) },
-    },
-  );
-
-  let instructions: string | undefined;
-  if (!response.ok) {
-    console.warn(
-      'Warning: failed to retrieve MCP server instructions. Proceeding with default instructions...',
-    );
-
-    instructions = `
-      This is the scan-documents MCP server. You will use Code Mode to help the user perform
-      actions. You can use search_docs tool to learn about how to take action with this server. Then,
-      you will write TypeScript code using the execute tool take action. It is CRITICAL that you be
-      thoughtful and deliberate when executing code. Always try to entirely solve the problem in code
-      block: it can be as long as you need to get the job done!
-    `;
-  }
-
-  instructions ??= ((await response.json()) as { instructions: string }).instructions;
-  instructions = `
-    The current time in Unix timestamps is ${Date.now()}.
-
-    ${instructions}
-  `;
-
-  return instructions;
-}
 
 export const newMcpServer = async (stainlessApiKey: string | undefined) =>
   new McpServer(