Fix Metaforecast platform fetchers: Manifold, Kalshi, Polymarket

apps/metaforecast/src/backend/platforms/kalshi.ts

@@ -1,142 +1,198 @@
-import api from "api";
+import { z } from "zod";
 
-import { Platform } from "../types";
+import { average } from "../../utils";
+import { FetchedQuestion, Platform } from "../types";
+import { fetchJson } from "../utils/fetchUtils";
 
-/**
- * https://kalshi.com
- *
- * Broken/disabled, FIXME.
- */
-
-const kalshi_api = api("@trading-api/v2.0#13mtbs10lc863irx");
-
-/* Definitions */
 const platformName = "kalshi";
-let jsonEndpoint = "https://trading-api.kalshi.com/v2";
-
-async function fetchAllMarkets() {
-  try {
-    let response = await kalshi_api.login({
-      email: process.env["KALSHI_EMAIL"]!,
-      password: process.env["KALSHI_PASSWORD"]!,
+const API_BASE = "https://api.elections.kalshi.com/trade-api/v2";
+
+// Zod schemas for Kalshi API response validation
+
+const kalshiMarketSchema = z.object({
+  ticker: z.string(),
+  event_ticker: z.string(),
+  series_ticker: z.string().optional().default(""),
+  title: z.string(),
+  yes_sub_title: z.string().optional().default(""),
+  no_sub_title: z.string().optional().default(""),
+  market_type: z.string(),
+  status: z.string(),
+  last_price_dollars: z.string().optional().default("0"),
+  volume_fp: z.string().optional().default("0"),
+  volume_24h_fp: z.string().optional().default("0"),
+  open_interest_fp: z.string().optional().default("0"),
+  open_time: z.string().optional(),
+  close_time: z.string().optional(),
+  rules_primary: z.string().optional().default(""),
+  result: z.string().optional().default(""),
+});
+
+type KalshiMarket = z.infer<typeof kalshiMarketSchema>;
+
+const kalshiMarketsResponseSchema = z.object({
+  markets: z.array(z.unknown()),
+  cursor: z.string().optional().default(""),
+});
+
+async function* fetchAllActiveMarkets(): AsyncGenerator<KalshiMarket> {
+  let cursor = "";
+  const limit = 1000;
+  const seen = new Set<string>();
+
+  while (true) {
+    const params = new URLSearchParams({
+      status: "active",
+      limit: String(limit),
     });
-    console.log(response.data);
-    let exchange_status = await kalshi_api.getExchangeStatus();
-    console.log(exchange_status.data);
-
-    // kalshi_api.auth(process.env.KALSHI_EMAIL, process.env.KALSHI_PASSWORD);
-    kalshi_api.auth(response.member_id, response.token);
-    /*
-     */
-    let market_params = {
-      limit: "100",
-      cursor: null,
-      event_ticker: null,
-      series_ticker: null,
-      max_close_ts: null,
-      min_close_ts: null,
-      status: null,
-      tickers: null,
-    };
-    // let markets = await kalshi_api.getMarkets(market_params).then(({data: any}) => console.log(data))
-    // console.log(markets)
-  } catch (error) {
-    console.log(error);
-  }
+    if (cursor) {
+      params.set("cursor", cursor);
+    }
 
-  return 1;
+    const url = `${API_BASE}/markets?${params.toString()}`;
+    console.log(`Fetching Kalshi markets: ${url}`);
+
+    const rawData = await fetchJson(url);
+    const response = kalshiMarketsResponseSchema.parse(rawData);
+
+    for (let i = 0; i < response.markets.length; i++) {
+      const parsed = kalshiMarketSchema.safeParse(response.markets[i]);
+      if (parsed.success) {
+        const market = parsed.data;
+        if (seen.has(market.ticker)) {
+          continue;
+        }
+        seen.add(market.ticker);
+        yield market;
+      } else {
+        console.error(
+          `Error parsing Kalshi market[${i}]: ${parsed.error.issues.length} issues. First issue:\n${JSON.stringify(parsed.error.issues[0], null, 2)}`
+        );
+      }
+    }
+
+    if (!response.cursor || response.markets.length < limit) {
+      break;
+    }
+    cursor = response.cursor;
+  }
 }
 
-/*
-async function fetchAllMarkets() {
-  let response = await axios
-    .get(jsonEndpoint)
-    .then((response) => response.data.markets);
+function marketToQuestion(market: KalshiMarket): FetchedQuestion | null {
+  if (market.market_type !== "binary") {
+    return null;
+  }
 
-  return response;
-}
+  if (market.result === "yes" || market.result === "no") {
+    return null;
+  }
+
+  const probability = parseFloat(market.last_price_dollars);
+  if (isNaN(probability) || probability < 0 || probability > 1) {
+    console.warn(
+      `Skipping market ${market.ticker}: invalid probability ${market.last_price_dollars}`
+    );
+    return null;
+  }
 
-async function processMarkets(markets: any[]) {
-  let dateNow = new Date().toISOString();
-  // console.log(markets)
-  markets = markets.filter((market) => market.close_date > dateNow);
-  let results = await markets.map((market) => {
-    const probability = market.last_price / 100;
-    const options: FetchedQuestion["options"] = [
-      {
-        name: "Yes",
-        probability: probability,
-        type: "PROBABILITY",
-      },
-      {
-        name: "No",
-        probability: 1 - probability,
-        type: "PROBABILITY",
-      },
-    ];
-    const id = `${platformName}-${market.id}`;
-    const result: FetchedQuestion = {
-      id,
-      title: market.title.replaceAll("*", ""),
-      url: `https://kalshi.com/markets/${market.ticker_name}`,
-      description: `${market.settle_details}. The resolution source is: ${market.ranged_group_name} (${market.settle_source_url})`,
-      options,
-      qualityindicators: {
-        yes_bid: market.yes_bid,
-        yes_ask: market.yes_ask,
-        spread: Math.abs(market.yes_bid - market.yes_ask),
-        shares_volume: market.volume, // Assuming that half of all buys are for yes and half for no, which is a big if.
-        // "open_interest": market.open_interest, also in shares
-      },
-      extra: {
-        open_interest: market.open_interest,
-      },
-    };
-    return result;
-  });
-
-  console.log([...new Set(results.map((result) => result.title))]);
-  console.log(
-    "Number of unique questions: ",
-    [...new Set(results.map((result) => result.title))].length
-  );
-
-  return results;
+  const volume = parseFloat(market.volume_fp) || 0;
+  const volume24h = parseFloat(market.volume_24h_fp) || 0;
+  const openInterest = parseFloat(market.open_interest_fp) || 0;
+
+  const seriesTicker = market.series_ticker || market.event_ticker;
+  const marketUrl = `https://kalshi.com/markets/${seriesTicker.toLowerCase()}`;
+
+  const title = market.title.replaceAll("*", "");
+
+  const description = [
+    market.rules_primary,
+    market.close_time ? `Closes: ${market.close_time}` : "",
+  ]
+    .filter(Boolean)
+    .join("\n\n");
+
+  const options: FetchedQuestion["options"] = [
+    {
+      name: market.yes_sub_title || "Yes",
+      probability,
+      type: "PROBABILITY",
+    },
+    {
+      name: market.no_sub_title || "No",
+      probability: 1 - probability,
+      type: "PROBABILITY",
+    },
+  ];
+
+  return {
+    id: `${platformName}-${market.ticker}`,
+    title,
+    url: marketUrl,
+    description,
+    options,
+    qualityindicators: {
+      trade_volume: volume,
+      volume24Hours: volume24h,
+      open_interest: openInterest,
+    },
+    extra: {
+      ticker: market.ticker,
+      event_ticker: market.event_ticker,
+      series_ticker: market.series_ticker,
+      open_interest: openInterest,
+    },
+  };
 }
-*/
 
 export const kalshi: Platform = {
   name: platformName,
   label: "Kalshi",
   color: "#615691",
 
-  fetcher: async function () {
-    // let markets = await fetchAllMarkets();
-    // console.log(markets)
-    return { questions: [] };
+  async fetcher() {
+    const questions: FetchedQuestion[] = [];
+
+    for await (const market of fetchAllActiveMarkets()) {
+      try {
+        const question = marketToQuestion(market);
+        if (question) {
+          questions.push(question);
+        }
+      } catch (error) {
+        console.error(
+          `Error processing Kalshi market ${market.ticker}:`,
+          error
+        );
+      }
+    }
+
+    console.log(`Fetched ${questions.length} Kalshi questions`);
+    return { questions };
   },
 
   calculateStars(data) {
+    const volume = Number(data.qualityindicators.trade_volume) || 0;
+    const extra = data.extra as Record<string, unknown> | undefined;
+    const openInterest = Number(extra?.["open_interest"]) || 0;
+
     const nuno = () =>
-      ((data.extra as any)?.open_interest || 0) > 500 &&
-      data.qualityindicators.shares_volume > 10000
+      openInterest > 500 && volume > 10000
         ? 4
-        : data.qualityindicators.shares_volume > 2000
+        : volume > 2000
           ? 3
           : 2;
-    let starsDecimal = nuno();
+    const starsDecimal = average([nuno()]);
 
-    // Substract 1 star if probability is above 90% or below 10%
+    // Subtract 1 star if probability is above 90% or below 10%
     if (
       data.options instanceof Array &&
       data.options[0] &&
       ((data.options[0].probability || 0) < 0.1 ||
         (data.options[0].probability || 0) > 0.9)
     ) {
-      starsDecimal = starsDecimal - 1;
+      return Math.round(starsDecimal - 1);
     }
 
-    const starsInteger = Math.round(starsDecimal);
-    return starsInteger;
+    return Math.round(starsDecimal);
   },
 };

apps/metaforecast/src/backend/platforms/manifold/api.ts

@@ -48,7 +48,9 @@ export async function fetchAllMarketsLite({
     let filteredMarkets = markets;
     if (upToUpdatedTime) {
       filteredMarkets = markets.filter(
-        (market) => market.lastUpdatedTime! >= upToUpdatedTime // keep only the markets that were updated after upToUpdatedTime
+        (market) =>
+          market.lastUpdatedTime !== undefined &&
+          market.lastUpdatedTime >= upToUpdatedTime
       );
     }
 
@@ -60,8 +62,8 @@ export async function fetchAllMarketsLite({
     {
       const total = allMarkets.length;
       const added = filteredMarkets.length;
-      const minDate = filteredMarkets[0].lastUpdatedTime!;
-      const maxDate = filteredMarkets.at(-1)?.lastUpdatedTime!;
+      const minDate = filteredMarkets[0].lastUpdatedTime ?? "unknown";
+      const maxDate = filteredMarkets.at(-1)?.lastUpdatedTime ?? "unknown";
       console.log(
         `Total: ${total}, added: ${added}, minDate: ${minDate}, maxDate: ${maxDate}`
       );

apps/metaforecast/src/backend/platforms/manifold/fetch.ts

@@ -34,11 +34,26 @@ export async function upgradeLiteMarketsToFull(
   liteMarkets: ManifoldApiLiteMarket[]
 ): Promise<ManifoldApiFullMarket[]> {
   const fullMarkets: ManifoldApiFullMarket[] = [];
+  let skippedCount = 0;
 
   for (const market of liteMarkets) {
-    console.log(`Fetching full market ${market.url}`);
-    const fullMarket = await fetchFullMarket(market.id);
-    fullMarkets.push(fullMarket);
+    try {
+      console.log(`Fetching full market ${market.url}`);
+      const fullMarket = await fetchFullMarket(market.id);
+      fullMarkets.push(fullMarket);
+    } catch (e) {
+      skippedCount++;
+      console.error(
+        `Failed to fetch full market ${market.id} (${market.url}):`,
+        e
+      );
+    }
+  }
+
+  if (skippedCount > 0) {
+    console.log(
+      `Skipped ${skippedCount} out of ${liteMarkets.length} markets due to errors`
+    );
   }
 
   return fullMarkets;

apps/metaforecast/src/backend/platforms/manifold/marketsToQuestions.ts

@@ -13,10 +13,12 @@ export function marketsToQuestions(
   markets: ManifoldMarket[]
 ): FetchedQuestion[] {
   const questions: FetchedQuestion[] = [];
+  let skippedCount = 0;
 
   for (const market of markets) {
     // Skip markets without probability (multiple choice questions)
     if (market.probability === undefined || market.probability === null) {
+      skippedCount++;
       continue;
     }
 
@@ -52,5 +54,11 @@ export function marketsToQuestions(
     questions.push(question);
   }
 
+  if (skippedCount > 0) {
+    console.log(
+      `Skipped ${skippedCount} out of ${markets.length} markets without probability`
+    );
+  }
+
   return questions;
 }