diff --git a/.gitignore b/.gitignore index c6434a449..b09ff0480 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,9 @@ reference_gen/.gen-cache*.json reference_gen/.node-incremental-cache.json # LLM documentation files (generated at build time) -static/llms.txt static/llms-full.txt +static/llms-summary.txt +static/llms.json # other generated content runtime/reference/std diff --git a/_config.ts b/_config.ts index 628e48f28..902995538 100644 --- a/_config.ts +++ b/_config.ts @@ -195,7 +195,6 @@ site.addEventListener("afterBuild", async () => { ); const { collectFiles, - generateLlmsTxt, generateLlmsSummaryTxt, generateLlmsFullTxt, generateLlmsJson, @@ -207,11 +206,6 @@ site.addEventListener("afterBuild", async () => { const files = await collectFiles(); log.info(`Collected ${files.length} documentation files for LLMs`); - // Generate llms.txt - const llmsTxt = generateLlmsTxt(files); - Deno.writeTextFileSync(site.dest("llms.txt"), llmsTxt); - log.info("Generated llms.txt in site root"); - // Generate llms-summary.txt const llmsSummaryTxt = generateLlmsSummaryTxt(files); Deno.writeTextFileSync(site.dest("llms-summary.txt"), llmsSummaryTxt); diff --git a/ai/index.md b/ai/index.md index 66973c904..490a17550 100644 --- a/ai/index.md +++ b/ai/index.md @@ -149,18 +149,22 @@ deno init --serve ## Key resources +- [llms.txt](/llms.txt): curated section index with key documentation links +- [llms-full-guide.txt](/llms-full-guide.txt): agent-oriented quick reference + with CLI commands, code examples, and usage patterns - [llms-summary.txt](/llms-summary.txt): compact, high-signal index -- [llms.txt](/llms.txt): full section index - [llms.json](/llms.json): structured index (Orama summary) - [llms-full.txt](/llms-full.txt): full content dump (large) - [Site search](/): use the on-site search UI for human browsing -- [Skills](https://github.com/denoland/skills): AI skills build for coding - assistants. +- [Skills](https://github.com/denoland/skills): AI skills for coding assistants ## Usage notes -- Prefer `llms-summary.txt` for quick orientation; fall back to `llms.txt` for - breadth. +- Start with `llms.txt` for a curated overview of all documentation sections. +- Use `llms-full-guide.txt` for a self-contained quick reference with CLI + commands, permissions, configuration, and code examples. +- Use `llms-summary.txt` for a compact, scored selection of the most important + pages. - Use `llms.json` when you need structured metadata (category, section, docType). - `llms-full.txt` is large; only fetch it when you need full-text extraction. diff --git a/generate_llms_files.ts b/generate_llms_files.ts index 77b3df5e4..e8fbc5e5a 100644 --- a/generate_llms_files.ts +++ b/generate_llms_files.ts @@ -1,11 +1,14 @@ #!/usr/bin/env -S deno run --allow-read --allow-write /** - * This script generates llms.txt and llms-full.txt files for the Deno documentation, - * following the llmstxt.org standard to create LLM-friendly documentation. + * This script generates llms-summary.txt, llms-full.txt, and llms.json for the + * Deno documentation, following the llmstxt.org standard. * - * By default, files are generated in the same directory as the generation script. When running - * during the build process, files are generated in the _site directory so they can be served - * at the site's root. + * The hand-written llms.txt and llms-full-guide.txt live in static/ and are + * copied to _site/ by Lume's site.copy("static", "."). + * + * By default, generated files are written to the static directory. When running + * during the build process, files are generated in the _site directory so they + * can be served at the site's root. */ import { walk } from "@std/fs"; @@ -20,6 +23,7 @@ const ROOT_DIR = fromFileUrl(new URL(".", import.meta.url)); const INCLUDE_DIRS = [ "runtime", "deploy", + "sandbox", "examples", "subhosting", "lint", @@ -243,112 +247,6 @@ async function collectFiles(): Promise { return files; } -function generateLlmsTxt(files: FileInfo[]): string { - // Extract information about the main sections - const sections = INCLUDE_DIRS.map((dir) => { - const sectionFiles = files.filter((file) => - file.relativePath.startsWith(dir) - ); - const sectionName = dir.charAt(0).toUpperCase() + dir.slice(1); - - return { - name: sectionName, - files: sectionFiles, - description: getSectionDescription(dir), - }; - }); - - // Build the llms.txt content - let content = "# Deno Documentation\n\n"; - content += - "> Deno is an open source JavaScript, TypeScript, and WebAssembly runtime with secure defaults and a great developer experience. It's built on V8, Rust, and Tokio.\n\n"; - content += - "Deno is a modern, secure-by-default runtime for JavaScript, TypeScript, and WebAssembly. This documentation covers the Deno runtime, Deno Deploy cloud service, and related tools and services.\n\n"; - - // Add sections - for (const section of sections) { - content += `## ${section.name} Documentation\n\n`; - if (section.description) { - content += `${section.description}\n\n`; - } - - // Get section index page - const indexPage = section.files.find((file) => - file.relativePath === `${section.name.toLowerCase()}/index.md` || - file.relativePath === `${section.name.toLowerCase()}/index.mdx` - ); - - if (indexPage) { - content += `- [${indexPage.title}](${indexPage.url})`; - if (indexPage.description ?? indexPage.summary) { - content += `: ${indexPage.description ?? indexPage.summary}`; - } - content += "\n"; - } - - // Group files by directories (if needed) - const filesByDirectory = new Map(); - for (const file of section.files) { - // Skip the index page as it's already included - if (file === indexPage) continue; - - const parts = file.relativePath.split("/"); - if (parts.length > 2) { - // This is a subdirectory - const subdir = parts.slice(0, 2).join("/"); - if (!filesByDirectory.has(subdir)) { - filesByDirectory.set(subdir, []); - } - filesByDirectory.get(subdir)?.push(file); - } else { - // This is a direct child of the section - if (!filesByDirectory.has(section.name.toLowerCase())) { - filesByDirectory.set(section.name.toLowerCase(), []); - } - filesByDirectory.get(section.name.toLowerCase())?.push(file); - } - } - - // Add links to important files - for (const [_, dirFiles] of filesByDirectory) { - // Sort files to ensure consistent output - dirFiles.sort((a, b) => a.relativePath.localeCompare(b.relativePath)); - - for (const file of dirFiles) { - // Skip files with no title - if (!file.title) continue; - - // Format for llms.txt - use the pattern: [Title](url): Description - const descriptionOrSummary = file.description ?? file.summary; - content += `- [${file.title}](${file.url})`; - if (descriptionOrSummary) { - content += `: ${descriptionOrSummary}`; - } - content += "\n"; - } - } - - content += "\n"; - } - - // Add Optional section with additional resources - content += "## Optional\n\n"; - content += - `- [AI Entrypoint](${BASE_URL}/ai/): Overview and key links for LLMs and AI agents\n`; - content += - `- [LLM Index (JSON)](${BASE_URL}/llms.json): Structured index built from the Orama summary\n`; - content += - `- [Contribution Guidelines](${BASE_URL}/runtime/contributing): How to contribute to Deno\n`; - content += - `- [Style Guide](${BASE_URL}/runtime/contributing/style_guide): Coding style guidelines for Deno\n`; - content += - `- [Release Schedule](${BASE_URL}/runtime/contributing/release_schedule): Deno's release cadence and versioning\n`; - content += - "- [Deno LLM Skills](https://github.com/denoland/skills): Skills and playbooks for LLMs and AI agents working with Deno\n"; - - return content; -} - function scoreSummaryCandidate(file: FileInfo): number { let score = 0; const depth = file.relativePath.split("/").length; @@ -523,6 +421,8 @@ function getSectionDescription(section: string): string { "Documentation for the Deno CLI and runtime environment, including installation, configuration, and core concepts.", deploy: "Documentation for Deno Deploy, a serverless platform for deploying JavaScript to a global edge network.", + sandbox: + "Documentation for Deno Sandbox, ephemeral Linux microVMs for running untrusted code safely.", examples: "Code examples and tutorials demonstrating how to build applications with Deno.", subhosting: @@ -555,11 +455,6 @@ async function main(outputDir?: string) { throw error; } - // Generate llms.txt - const llmsTxt = generateLlmsTxt(files); - await Deno.writeTextFile(join(outDir, "llms.txt"), llmsTxt); - console.log(`Generated llms.txt in ${outDir}`); - // Generate llms-summary.txt const llmsSummaryTxt = generateLlmsSummaryTxt(files); await Deno.writeTextFile(join(outDir, "llms-summary.txt"), llmsSummaryTxt); @@ -596,7 +491,6 @@ if (import.meta.main) { // Export functions for use in the site build process export default { collectFiles, - generateLlmsTxt, generateLlmsSummaryTxt, generateLlmsFullTxt, generateLlmsJson, diff --git a/middleware/llmsFiles.ts b/middleware/llmsFiles.ts index 3bf360a8d..d595bd9af 100644 --- a/middleware/llmsFiles.ts +++ b/middleware/llmsFiles.ts @@ -13,6 +13,7 @@ const LLMS_CONTENT_TYPES = new Map([ ["/llms.txt", "text/plain; charset=utf-8"], ["/llms-summary.txt", "text/plain; charset=utf-8"], ["/llms-full.txt", "text/plain; charset=utf-8"], + ["/llms-full-guide.txt", "text/plain; charset=utf-8"], ["/llms.json", "application/json; charset=utf-8"], ]); diff --git a/static/llms-full-guide.txt b/static/llms-full-guide.txt new file mode 100644 index 000000000..e2608aff7 --- /dev/null +++ b/static/llms-full-guide.txt @@ -0,0 +1,204 @@ +# Deno + +Deno is a secure JavaScript/TypeScript runtime built on V8 and Rust, distributed +as a single `deno` binary. TypeScript, formatting, linting, testing, and a +standard library work with zero configuration. Programs are sandboxed by +default; capabilities are granted explicitly via `--allow-*` flags. + +## CLI quick reference + +``` +deno run main.ts # run a script (sandboxed by default) +deno run -A main.ts # run with all permissions +deno test # run tests (*_test.ts, *.test.ts) +deno fmt # format code +deno lint # lint code +deno task # run a task defined in deno.json +deno add # add a dependency to deno.json +deno init my_project # scaffold a new project +deno init --lib # scaffold a library +deno init --serve # scaffold an HTTP server +deno compile main.ts # compile to standalone binary +deno install --global -A jsr:@std/http/file-server # install a CLI globally +deno deploy # deploy to Deno Deploy +``` + +## Permissions + +By default `deno run` blocks network, filesystem, and environment access. Grant +specific capabilities with flags: + +- `--allow-net` — all network access +- `--allow-net=example.com` — network access to specific hosts +- `--allow-read` — all filesystem reads +- `--allow-read=./data` — reads scoped to a path +- `--allow-write` — all filesystem writes +- `--allow-env` — environment variable access +- `--allow-run` — subprocess execution +- `-A` — all permissions (shorthand for all flags) + +Typical server invocation: + +``` +deno run --allow-net --allow-read server.ts +``` + +## Project configuration (deno.json) + +Deno auto-detects `deno.json` (or `deno.jsonc`) up the directory tree. Key +fields: + +```jsonc +{ + // Tasks (run via `deno task `) + "tasks": { + "dev": "deno run --watch --allow-net main.ts", + "test": "deno test" + }, + + // Import map — bare specifier aliases for dependencies + "imports": { + "@std/assert": "jsr:@std/assert@^1.0.0", + "chalk": "npm:chalk@5" + } +} +``` + +Dependencies are added manually or via `deno add`: + +``` +deno add jsr:@std/http # from JSR (Deno-native registry) +deno add npm:express # from npm +``` + +## Module resolution and imports + +Deno supports three specifier types: + +```ts +// JSR — Deno's native registry (preferred for Deno-first packages) +import { assertEquals } from "jsr:@std/assert@^1.0.0"; + +// npm — use any npm package without an install step +import chalk from "npm:chalk@5"; + +// node: — Node.js built-in modules +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +``` + +When an import map is configured in `deno.json`, use bare specifiers instead: + +```ts +import { assertEquals } from "@std/assert"; +import chalk from "chalk"; +``` + +## Node.js compatibility + +Deno implements Node.js built-in modules and supports `package.json`. Most +Node.js projects run without modification. Key differences: + +- Node built-ins require the `node:` prefix (`import fs from "node:fs"`) +- npm packages can be imported via `npm:` specifiers without `node_modules` +- Deno respects `package.json` if present, but prefers `deno.json` + +## Testing + +Deno has a built-in test runner. Tests are functions registered with +`Deno.test()` and discovered by file name convention (*_test.ts, *.test.ts). + +```ts +import { assertEquals } from "jsr:@std/assert"; + +Deno.test("addition works", () => { + assertEquals(1 + 1, 2); +}); + +Deno.test("async test", async () => { + const data = await Promise.resolve("hello"); + assertEquals(data, "hello"); +}); +``` + +Run tests: + +``` +deno test # run all tests +deno test src/ # run tests in a directory +deno test --filter "add" # filter by test name +deno test --coverage # collect coverage +``` + +## HTTP server + +Deno provides `Deno.serve()` for HTTP servers: + +```ts +Deno.serve({ port: 8000 }, (req: Request) => { + return new Response("Hello, World!"); +}); +``` + +## Deno Deploy + +Deno Deploy is a managed edge platform for JavaScript/TypeScript apps. + +Deploy from the CLI: + +``` +deno deploy +``` + +Or connect a GitHub repo via https://console.deno.com/ for automatic deploys on +push, with preview deploys per branch/PR. + +Framework apps (e.g., Fresh) typically use a build step: `deno task build`. + +Deno Deploy includes Deno KV, a built-in key-value database: + +```ts +const kv = await Deno.openKv(); +await kv.set(["users", "alice"], { name: "Alice", age: 30 }); +const result = await kv.get(["users", "alice"]); +``` + +## Deno Sandbox + +Deno Sandbox provides ephemeral Linux microVMs (sub-second startup) via API. +Designed for executing untrusted code safely — particularly useful for AI agent +workflows (generate code, execute, inspect output, iterate). + +SDKs: `@deno/sandbox` (JavaScript — works in Deno and Node), plus a Python SDK. + +Basic usage (JS): + +```ts +import { Sandbox } from "@deno/sandbox"; + +const sandbox = await Sandbox.create(); +const result = await sandbox.runCommand("echo hello"); +console.log(result.output); // "hello" +await sandbox.close(); +``` + +Capabilities: + +- Full Linux environment (filesystem, processes, package managers) +- Configurable region, memory, and lifetime +- HTTP port exposure and SSH access +- Persistent storage via volumes and snapshots +- Network allowlisting (`allowNet: ["example.com"]`) + +Security model: Secrets never enter the sandbox. The platform substitutes secret +values only on outbound requests to approved hosts, preventing exfiltration by +generated code. + +## Key documentation links + +- Runtime docs: https://docs.deno.com/runtime/ +- Deploy docs: https://docs.deno.com/deploy/ +- Sandbox docs: https://docs.deno.com/sandbox/ +- Standard library: https://jsr.io/@std +- Examples: https://docs.deno.com/examples/ +- AI skills: https://github.com/denoland/skills diff --git a/static/llms.txt b/static/llms.txt new file mode 100644 index 000000000..0e83b4c1e --- /dev/null +++ b/static/llms.txt @@ -0,0 +1,59 @@ +# Deno + +> Deno is a secure JavaScript/TypeScript runtime built on V8 and Rust, +> distributed as a single binary. TypeScript, formatting, linting, testing, and +> a standard library work with zero configuration. Programs are sandboxed by +> default; capabilities (network, filesystem, etc.) are granted explicitly via +> --allow-* flags. Deno natively supports npm packages and Node.js built-in +> modules. + +- [llms-full-guide.txt](https://docs.deno.com/llms-full-guide.txt): Complete agent-oriented guide with CLI reference, code examples, and usage patterns (publish alongside this file) +- [llms-summary.txt](https://docs.deno.com/llms-summary.txt): Compact index of all documentation sections +- [llms-full.txt](https://docs.deno.com/llms-full.txt): Full documentation content dump (large) + +## Runtime + +- [Getting Started](https://docs.deno.com/runtime/getting_started/first_project): Scaffold a project, run code, and execute tests +- [CLI Reference](https://docs.deno.com/runtime/reference/cli/): All deno subcommands and flags (run, test, fmt, lint, task, compile, install) +- [Configuration (deno.json)](https://docs.deno.com/runtime/fundamentals/configuration): Project config, tasks, import maps, TypeScript settings +- [Modules and Imports](https://docs.deno.com/runtime/fundamentals/modules): jsr:, npm:, and node: specifiers; import maps; dependency management +- [Security and Permissions](https://docs.deno.com/runtime/fundamentals/security): Sandbox model and --allow-* permission flags +- [Node.js Compatibility](https://docs.deno.com/runtime/fundamentals/node): Running Node projects, npm packages, and node: built-ins in Deno +- [Testing](https://docs.deno.com/runtime/fundamentals/testing): Built-in test runner, assertions, mocking, coverage +- [TypeScript Support](https://docs.deno.com/runtime/fundamentals/typescript): TypeScript configuration and type checking +- [HTTP Server](https://docs.deno.com/runtime/fundamentals/http_server): Deno.serve API, request handling, WebSockets +- [Standard Library (@std)](https://docs.deno.com/runtime/reference/std/): Deno's standard library modules on JSR +- [Linting and Formatting](https://docs.deno.com/runtime/fundamentals/linting_and_formatting): deno lint and deno fmt configuration and usage +- [Workspaces](https://docs.deno.com/runtime/fundamentals/workspaces): Monorepo and multi-package project configuration +- [Web Development](https://docs.deno.com/runtime/fundamentals/web_dev): Frameworks (Fresh, Next.js, Astro, SvelteKit) with Deno + +## Deploy + +- [Deno Deploy Overview](https://docs.deno.com/deploy/): Managed edge platform for JavaScript/TypeScript apps +- [Getting Started](https://docs.deno.com/deploy/getting_started): Create and configure your first Deploy application +- [Deno KV](https://docs.deno.com/deploy/kv/): Built-in key-value database available in CLI and on Deploy + +## Sandbox + +- [Deno Sandbox Overview](https://docs.deno.com/sandbox/): Ephemeral Linux microVMs for running untrusted code safely +- [Getting Started](https://docs.deno.com/sandbox/getting_started/): Enable sandboxes, create a microVM, run commands, manage secrets +- [Create a Sandbox](https://docs.deno.com/sandbox/create/): Sandbox.create() API reference and configuration options +- [Sandbox Timeouts](https://docs.deno.com/sandbox/timeouts/): Ephemeral vs. persistent sandbox lifecycle management +- [Deploy App Management](https://docs.deno.com/sandbox/apps/): Programmatically create and manage Deploy apps via the SDK + +## Examples + +- [Build a Fresh App](https://docs.deno.com/examples/fresh_tutorial/): Full-stack app with Fresh framework and islands architecture +- [Deploy with deno deploy CLI](https://docs.deno.com/examples/deploy_command_tutorial/): Deploy a local project to Deno Deploy +- [Chat App with WebSockets](https://docs.deno.com/examples/chat_app_tutorial/): Real-time WebSocket server with Oak +- [LLM Chat App](https://docs.deno.com/examples/llm_tutorial/): Integrate OpenAI/Anthropic APIs with Deno +- [Connecting to Databases](https://docs.deno.com/examples/connecting_to_databases_tutorial/): MySQL, PostgreSQL, MongoDB, SQLite, and ORMs +- [Sandbox Volumes](https://docs.deno.com/examples/volumes_tutorial/): Persistent storage for sandbox microVMs +- [Sandbox Snapshots](https://docs.deno.com/examples/snapshots_tutorial/): Reproducible sandbox environments with snapshots + +## Optional + +- [Contributing](https://docs.deno.com/runtime/contributing/): How to contribute to the Deno project +- [Style Guide](https://docs.deno.com/runtime/contributing/style_guide): Coding conventions for Deno internals +- [Subhosting](https://docs.deno.com/subhosting/): Platform for SaaS providers to run customer code securely +- [AI Skills for Coding Assistants](https://github.com/denoland/skills): Deno-specific skills and playbooks for LLMs