From 849759412f2c9b0599f72b43905bc96173247ae2 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:05:55 -0700 Subject: [PATCH 1/2] docs: add sitemap, canonical tags, and crawlable robots for SEO migration --- packages/docs/public/robots.txt | 5 --- .../docs/src/app/docs/[[...mdxPath]]/page.tsx | 12 +++++- packages/docs/src/app/page.tsx | 4 ++ packages/docs/src/app/robots.ts | 13 ++++++ packages/docs/src/app/sitemap.ts | 40 +++++++++++++++++++ 5 files changed, 68 insertions(+), 6 deletions(-) delete mode 100644 packages/docs/public/robots.txt create mode 100644 packages/docs/src/app/robots.ts create mode 100644 packages/docs/src/app/sitemap.ts diff --git a/packages/docs/public/robots.txt b/packages/docs/public/robots.txt deleted file mode 100644 index b9de6e0b..00000000 --- a/packages/docs/public/robots.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Robots.txt for https://new.docs.re.video -# Disallow everything while we're still migrating - -User-agent: * -Disallow: / diff --git a/packages/docs/src/app/docs/[[...mdxPath]]/page.tsx b/packages/docs/src/app/docs/[[...mdxPath]]/page.tsx index 76e5edc5..c81d328a 100644 --- a/packages/docs/src/app/docs/[[...mdxPath]]/page.tsx +++ b/packages/docs/src/app/docs/[[...mdxPath]]/page.tsx @@ -6,7 +6,17 @@ import {useMDXComponents} from "../../../../mdx-components"; export async function generateMetadata(props: {params: Promise<{mdxPath: string[]}>}) { const params = await props.params; const {metadata} = await importPage(params.mdxPath); - return metadata; + // Self-referencing canonical so Google settles on the new URL (important for + // the docs.re.video -> midrender.com/revideo/docs migration). Absolute URL: + // metadataBase doesn't include the "/revideo" basePath. + const path = params.mdxPath?.length ? `/${params.mdxPath.join("/")}` : ""; + return { + ...metadata, + alternates: { + ...metadata?.alternates, + canonical: `https://midrender.com/revideo/docs${path}`, + }, + }; } const Wrapper = useMDXComponents().wrapper; diff --git a/packages/docs/src/app/page.tsx b/packages/docs/src/app/page.tsx index 4fc1609d..66d9b94c 100644 --- a/packages/docs/src/app/page.tsx +++ b/packages/docs/src/app/page.tsx @@ -9,6 +9,10 @@ export const metadata: Metadata = { title: 'Revideo', description: 'Revideo, the open-source framework for programmatic video creation, is now part of Midrender.', + // Absolute canonical: metadataBase doesn't include the "/revideo" basePath. + alternates: { + canonical: 'https://midrender.com/revideo', + }, openGraph: { title: 'The next chapter of Revideo', description: diff --git a/packages/docs/src/app/robots.ts b/packages/docs/src/app/robots.ts new file mode 100644 index 00000000..6a7368a1 --- /dev/null +++ b/packages/docs/src/app/robots.ts @@ -0,0 +1,13 @@ +import type {MetadataRoute} from "next"; + +// NOTE: with basePath "/revideo" this is served at /revideo/robots.txt. Crawlers +// only read robots.txt from the domain root (midrender.com/robots.txt), which is +// owned by the main app, so the authoritative robots.txt must live there and +// reference this sitemap. This file keeps the docs zone self-describing and works +// if the docs are ever served on their own domain. +export default function robots(): MetadataRoute.Robots { + return { + rules: {userAgent: "*", allow: "/"}, + sitemap: "https://midrender.com/revideo/sitemap.xml", + }; +} diff --git a/packages/docs/src/app/sitemap.ts b/packages/docs/src/app/sitemap.ts new file mode 100644 index 00000000..7398e145 --- /dev/null +++ b/packages/docs/src/app/sitemap.ts @@ -0,0 +1,40 @@ +import type {MetadataRoute} from "next"; +import {readdirSync, statSync} from "node:fs"; +import {join, relative} from "node:path"; + +// Absolute origin (no basePath); the docs are served under midrender.com/revideo. +const ORIGIN = "https://midrender.com"; +const CONTENT_DIR = join(process.cwd(), "src", "content"); + +/** Collect every doc page (content-relative route + last-modified time). */ +function collect(dir: string, out: {route: string; mtime: Date}[]) { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const stat = statSync(full); + if (stat.isDirectory()) { + collect(full, out); + } else if (/\.mdx?$/.test(entry)) { + const route = relative(CONTENT_DIR, full) + .replace(/\\/g, "/") + .replace(/\.mdx?$/, "") + .replace(/(^|\/)index$/, ""); // index -> its folder + out.push({route, mtime: stat.mtime}); + } + } +} + +export default function sitemap(): MetadataRoute.Sitemap { + const pages: {route: string; mtime: Date}[] = []; + collect(CONTENT_DIR, pages); + + const docs = pages + .map(({route, mtime}) => ({ + url: `${ORIGIN}/revideo/docs${route ? `/${route}` : ""}`, + lastModified: mtime, + })) + .sort((a, b) => a.url.localeCompare(b.url)); + + // The marketing landing page lives at /revideo (app/page.tsx), outside the + // docs content tree, so add it explicitly. + return [{url: `${ORIGIN}/revideo`}, ...docs]; +} From 4956dd5e7bdda2a0fc06ae4d9ee41238c2fb44d4 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:53:43 -0700 Subject: [PATCH 2/2] docs: override Vercel proxy noindex with X-Robots-Tag index,follow --- packages/docs/next.config.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/docs/next.config.ts b/packages/docs/next.config.ts index 11699e03..ea82fac7 100644 --- a/packages/docs/next.config.ts +++ b/packages/docs/next.config.ts @@ -32,4 +32,18 @@ export default withNextra({ // would externalize it and fail to `require()` it at runtime. Transpiling the // package makes Next bundle it instead of externalizing. transpilePackages: ["@theguild/remark-mermaid"], + // Because midrender.com proxies /revideo* to this deployment via its + // *.vercel.app URL, Vercel automatically injects `X-Robots-Tag: noindex` on the + // proxied response, which would keep the docs out of Google. Setting the header + // here — at the source, before the response leaves this deployment — replaces + // that value so the proxy forwards a single, clean `index, follow`. basePath is + // applied to `source`, so `/:path*` covers every route under /revideo. + async headers() { + return [ + { + source: "/:path*", + headers: [{key: "X-Robots-Tag", value: "index, follow"}], + }, + ]; + }, });