fix: harden scrape inputs against SSRF and unsafe URL targets

apps/api/src/lib/urlSafety.test.ts

@@ -0,0 +1,89 @@
+import { validateUrl, resolveAndValidate, getRequestPolicy, safeErrorMessage } from './urlSafety';
+
+describe('validateUrl', () => {
+  it('allows public HTTPS URLs', () => {
+    const result = validateUrl('https://developers.stellar.org/docs');
+    expect(result.safe).toBe(true);
+  });
+
+  it('allows public HTTP URLs', () => {
+    const result = validateUrl('http://example.com/page');
+    expect(result.safe).toBe(true);
+  });
+
+  it('rejects FTP protocol', () => {
+    const result = validateUrl('ftp://files.example.com');
+    expect(result.safe).toBe(false);
+  });
+
+  it('rejects URLs with credentials', () => {
+    const result = validateUrl('https://user:pass@example.com');
+    expect(result.safe).toBe(false);
+  });
+
+  it('rejects localhost', () => {
+    const result = validateUrl('http://localhost:3000/admin');
+    expect(result.safe).toBe(false);
+  });
+
+  it('rejects 127.0.0.1', () => {
+    const result = validateUrl('http://127.0.0.1:8080');
+    expect(result.safe).toBe(false);
+  });
+
+  it('rejects 192.168.x.x', () => {
+    const result = validateUrl('http://192.168.1.100');
+    expect(result.safe).toBe(false);
+  });
+
+  it('rejects 10.x.x.x', () => {
+    const result = validateUrl('http://10.0.0.5');
+    expect(result.safe).toBe(false);
+  });
+
+  it('rejects 172.16.x.x', () => {
+    const result = validateUrl('http://172.16.0.1');
+    expect(result.safe).toBe(false);
+  });
+
+  it('rejects link-local 169.254.x.x', () => {
+    const result = validateUrl('http://169.254.1.1');
+    expect(result.safe).toBe(false);
+  });
+
+  it('rejects cloud metadata IP', () => {
+    const result = validateUrl('http://169.254.169.254/latest/meta-data');
+    expect(result.safe).toBe(false);
+  });
+
+  it('rejects IPv6 loopback', () => {
+    const result = validateUrl('http://[::1]:8080');
+    expect(result.safe).toBe(false);
+  });
+
+  it('rejects IPv6 link-local', () => {
+    const result = validateUrl('http://[fe80::1]:8080');
+    expect(result.safe).toBe(false);
+  });
+
+  it('sanitizes URL by removing fragment', () => {
+    const result = validateUrl('https://example.com/page?q=1');
+    expect(result.sanitizedUrl).toBe('https://example.com/page?q=1');
+  });
+
+  it('returns safe error message without internal details', () => {
+    const msg = safeErrorMessage(new Error('ECONNREFUSED'));
+    expect(msg).toContain('not reachable');
+    expect(msg).not.toContain('ECONNREFUSED');
+  });
+});
+
+describe('getRequestPolicy', () => {
+  it('returns timeout, size limit, and redirect limit', () => {
+    const policy = getRequestPolicy();
+    expect(policy.timeout).toBeGreaterThan(0);
+    expect(policy.maxResponseSize).toBeGreaterThan(0);
+    expect(policy.maxRedirects).toBeGreaterThan(0);
+    expect(policy.allowedContentTypes).toContain('text/html');
+  });
+});

apps/api/src/lib/urlSafety.ts

@@ -0,0 +1,104 @@
+import { URL } from 'url';
+import { promises as dns } from 'dns';
+
+const BLOCKED_HOSTS = new Set([
+  'localhost', '127.0.0.1', '0.0.0.0', '::1', '[::1]',
+]);
+
+const BLOCKED_PREFIXES = ['127.', '10.', '192.168.', '172.16.', '172.17.', '172.18.',
+  '172.19.', '172.20.', '172.21.', '172.22.', '172.23.', '172.24.', '172.25.',
+  '172.26.', '172.27.', '172.28.', '172.29.', '172.30.', '172.31.', '169.254.',
+  'fc00:', 'fd00:', 'fe80:',
+];
+
+const BLOCKED_CLOUD_META = ['169.254.169.254', '[fd00:ec2::254]'];
+
+const MAX_REDIRECTS = 5;
+const REQUEST_TIMEOUT_MS = 10_000;
+const MAX_RESPONSE_SIZE_BYTES = 5 * 1024 * 1024;
+const ALLOWED_CONTENT_TYPES = ['text/html', 'application/json', 'text/plain', 'application/xml', 'text/xml'];
+
+export interface UrlPolicyResult {
+  safe: boolean;
+  sanitizedUrl: string;
+  error?: string;
+}
+
+export function validateUrl(raw: string): UrlPolicyResult {
+  try {
+    let url = new URL(raw);
+
+    if (!['http:', 'https:'].includes(url.protocol)) {
+      return { safe: false, sanitizedUrl: '', error: 'Only HTTP/HTTPS protocols are allowed' };
+    }
+
+    if (url.username || url.password) {
+      return { safe: false, sanitizedUrl: '', error: 'Credentials in URLs are not allowed' };
+    }
+
+    const hostname = url.hostname.toLowerCase();
+
+    if (BLOCKED_HOSTS.has(hostname) || BLOCKED_CLOUD_META.includes(hostname)) {
+      return { safe: false, sanitizedUrl: '', error: 'URL targets a blocked host' };
+    }
+
+    for (const prefix of BLOCKED_PREFIXES) {
+      if (hostname.startsWith(prefix)) {
+        return { safe: false, sanitizedUrl: '', error: 'URL targets a private/restricted network' };
+      }
+    }
+
+    const sanitized = url.protocol + '//' + url.hostname + url.pathname + url.search;
+
+    return { safe: true, sanitizedUrl: sanitized };
+  } catch {
+    return { safe: false, sanitizedUrl: '', error: 'Invalid URL format' };
+  }
+}
+
+export async function resolveAndValidate(raw: string): Promise<UrlPolicyResult> {
+  const result = validateUrl(raw);
+  if (!result.safe) return result;
+
+  try {
+    const hostname = new URL(raw).hostname;
+    const addresses = await dns.resolve(hostname);
+
+    for (const addr of addresses) {
+      for (const prefix of BLOCKED_PREFIXES) {
+        if (addr.startsWith(prefix)) {
+          return { safe: false, sanitizedUrl: '', error: 'DNS resolved to a blocked address' };
+        }
+      }
+      if (BLOCKED_HOSTS.has(addr) || BLOCKED_CLOUD_META.includes(addr)) {
+        return { safe: false, sanitizedUrl: '', error: 'DNS resolved to a blocked address' };
+      }
+    }
+
+    return result;
+  } catch {
+    return { safe: false, sanitizedUrl: '', error: 'DNS resolution failed' };
+  }
+}
+
+export function getRequestPolicy() {
+  return {
+    timeout: REQUEST_TIMEOUT_MS,
+    maxResponseSize: MAX_RESPONSE_SIZE_BYTES,
+    maxRedirects: MAX_REDIRECTS,
+    allowedContentTypes: ALLOWED_CONTENT_TYPES,
+    validateRedirect: (url: string) => validateUrl(url),
+  };
+}
+
+export function safeErrorMessage(err: unknown): string {
+  if (err instanceof Error) {
+    if (err.message.includes('DNS') || err.message.includes('ENOTFOUND')) {
+      return 'Unable to resolve the requested URL';
+    }
+    if (err.message.includes('ECONNREFUSED') || err.message.includes('ETIMEDOUT')) {
+      return 'The target server is not reachable';
+    }
+  }
+  return 'The request could not be completed due to security restrictions';
+}