Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added CC BY-NC 4.0.docx
Binary file not shown.
122 changes: 80 additions & 42 deletions app/actions.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,27 +13,25 @@ import { Spinner } from '@/components/ui/spinner'
import { Section } from '@/components/section'
import { FollowupPanel } from '@/components/followup-panel'
import { inquire, researcher, taskManager, querySuggestor, resolutionSearch, type DrawnFeature } from '@/lib/agents'
// Removed import of useGeospatialToolMcp as it no longer exists and was incorrectly used here.
// The geospatialTool (if used by agents like researcher) now manages its own MCP client.
import { writer } from '@/lib/agents/writer'
import { saveChat, getSystemPrompt } from '@/lib/actions/chat' // Added getSystemPrompt
import { saveChat, getSystemPrompt } from '@/lib/actions/chat'
import { Chat, AIMessage } from '@/lib/types'
import { UserMessage } from '@/components/user-message'
import { BotMessage } from '@/components/message'
import { SearchSection } from '@/components/search-section'
import SearchRelated from '@/components/search-related'
import { GeoJsonLayer } from '@/components/map/geojson-layer'
import { ResolutionImage } from '@/components/resolution-image'
import { CopilotDisplay } from '@/components/copilot-display'
import RetrieveSection from '@/components/retrieve-section'
import { VideoSearchSection } from '@/components/video-search-section'
import { MapQueryHandler } from '@/components/map/map-query-handler' // Add this import
import { MapQueryHandler } from '@/components/map/map-query-handler'

// Define the type for related queries
type RelatedQueries = {
items: { query: string }[]
}

// Removed mcp parameter from submit, as geospatialTool now handles its client.
async function submit(formData?: FormData, skip?: boolean) {
'use server'

Expand All @@ -43,16 +41,17 @@ async function submit(formData?: FormData, skip?: boolean) {
const isCollapsed = createStreamableValue(false)

const action = formData?.get('action') as string;
const drawnFeaturesString = formData?.get('drawnFeatures') as string;
let drawnFeatures: DrawnFeature[] = [];
try {
drawnFeatures = drawnFeaturesString ? JSON.parse(drawnFeaturesString) : [];
} catch (e) {
console.error('Failed to parse drawnFeatures:', e);
}

if (action === 'resolution_search') {
const file = formData?.get('file') as File;
const timezone = (formData?.get('timezone') as string) || 'UTC';
const drawnFeaturesString = formData?.get('drawnFeatures') as string;
let drawnFeatures: DrawnFeature[] = [];
try {
drawnFeatures = drawnFeaturesString ? JSON.parse(drawnFeaturesString) : [];
} catch (e) {
console.error('Failed to parse drawnFeatures:', e);
}

if (!file) {
throw new Error('No file provided for resolution search.');
Expand All @@ -61,7 +60,6 @@ async function submit(formData?: FormData, skip?: boolean) {
const buffer = await file.arrayBuffer();
const dataUrl = `data:${file.type};base64,${Buffer.from(buffer).toString('base64')}`;

// Get the current messages, excluding tool-related ones.
const messages: CoreMessage[] = [...(aiState.get().messages as any[])].filter(
message =>
message.role !== 'tool' &&
Expand All @@ -71,16 +69,12 @@ async function submit(formData?: FormData, skip?: boolean) {
message.type !== 'resolution_search_result'
);

// The user's prompt for this action is static.
const userInput = 'Analyze this map view.';

// Construct the multimodal content for the user message.
const content: CoreMessage['content'] = [
{ type: 'text', text: userInput },
{ type: 'image', image: dataUrl, mimeType: file.type }
];

// Add the new user message to the AI state.
aiState.update({
...aiState.get(),
messages: [
Expand All @@ -90,12 +84,11 @@ async function submit(formData?: FormData, skip?: boolean) {
});
messages.push({ role: 'user', content });

// Create a streamable value for the summary.
const summaryStream = createStreamableValue<string>('');
const summaryStream = createStreamableValue<string>('Analyzing map view...');
const groupeId = nanoid();

async function processResolutionSearch() {
try {
// Call the simplified agent, which now returns a stream.
const streamResult = await resolutionSearch(messages, timezone, drawnFeatures);

let fullSummary = '';
Expand All @@ -107,22 +100,41 @@ async function submit(formData?: FormData, skip?: boolean) {
}

const analysisResult = await streamResult.object;

// Mark the summary stream as done with the result.
summaryStream.done(analysisResult.summary || 'Analysis complete.');

if (analysisResult.geoJson) {
uiStream.append(
<GeoJsonLayer
id={groupeId}
data={analysisResult.geoJson as FeatureCollection}
/>
);
}

messages.push({ role: 'assistant', content: analysisResult.summary || 'Analysis complete.' });

const sanitizedMessages: CoreMessage[] = messages.map(m => {
if (Array.isArray(m.content)) {
return {
...m,
content: m.content.filter(part => part.type !== 'image')
content: m.content.filter((part: any) => part.type !== 'image')
} as CoreMessage
}
return m
})
Comment on lines 116 to 124
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🧹 Nitpick | 🔵 Trivial

Consider consolidating image sanitization logic.

There are two similar sanitization patterns in this file:

  1. Lines 116-124: Filters out image parts entirely from messages
  2. Lines 126-137: Replaces image data with "IMAGE_PROCESSED" placeholder

These serve different purposes but the logic is similar. Consider extracting helper functions to clarify intent and reduce duplication.

Suggested helpers
// Remove image parts entirely (for AI context)
const stripImageParts = (messages: CoreMessage[]): CoreMessage[] =>
  messages.map(m => {
    if (Array.isArray(m.content)) {
      return { ...m, content: m.content.filter((part: any) => part.type !== 'image') } as CoreMessage;
    }
    return m;
  });

// Replace image data with placeholder (for storage/history)
const sanitizeImageData = (messages: AIMessage[]): AIMessage[] =>
  messages.map(m => {
    if (m.role === 'user' && Array.isArray(m.content)) {
      return {
        ...m,
        content: m.content.map((part: any) =>
          part.type === 'image' ? { ...part, image: 'IMAGE_PROCESSED' } : part
        )
      };
    }
    return m;
  });
🤖 Prompt for AI Agents
In `@app/actions.tsx` around lines 116 - 124, Extract the duplicated
image-sanitization logic into two small helper functions and replace the inline
code around sanitizedMessages and the subsequent image-placeholder block: create
stripImageParts(messages: CoreMessage[]) to remove parts with part.type ===
'image' and use it where sanitizedMessages is built, and create
sanitizeImageData(messages: AIMessage[]) to map user messages and replace image
fields with the "IMAGE_PROCESSED" placeholder, then call that helper in the
storage/history path; update callers to use these helpers (referencing
sanitizedMessages, stripImageParts, and sanitizeImageData) to keep intent clear
and remove duplication.


const currentMessages = aiState.get().messages;
const sanitizedHistory = currentMessages.map(m => {
if (m.role === "user" && Array.isArray(m.content)) {
return {
...m,
content: m.content.map((part: any) =>
part.type === "image" ? { ...part, image: "IMAGE_PROCESSED" } : part
)
}
}
return m
});
Comment on lines +126 to +137
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Bug: sanitizedHistory is created but not used.

The code creates sanitizedHistory (lines 126-137) to replace image data with "IMAGE_PROCESSED", but then aiState.done() uses ...aiState.get().messages (line 150) instead of sanitizedHistory. This means the sanitization is never applied to the final state.

🐛 Proposed fix
         aiState.done({
           ...aiState.get(),
           messages: [
-            ...aiState.get().messages,
+            ...sanitizedHistory,
             {
               id: groupeId,
               role: 'assistant',

Also applies to: 147-150

🤖 Prompt for AI Agents
In `@app/actions.tsx` around lines 126 - 137, The code builds sanitizedHistory but
then still passes aiState.get().messages into aiState.done, so the sanitized
image replacements are never applied; update calls that use
aiState.get().messages (notably the aiState.done(...) invocation and any nearby
usages around the end of the action) to pass sanitizedHistory instead (or
construct a finalMessages variable set to sanitizedHistory and use that) so the
sanitized content is what gets saved/emitted; ensure you only change the
argument passed to aiState.done (and any other subsequent consumers) to
reference sanitizedHistory rather than aiState.get().messages.

const relatedQueries = await querySuggestor(uiStream, sanitizedMessages);
uiStream.append(
<Section title="Follow-up">
Expand All @@ -132,8 +144,6 @@ async function submit(formData?: FormData, skip?: boolean) {

await new Promise(resolve => setTimeout(resolve, 500));

const groupeId = nanoid();

aiState.done({
...aiState.get(),
messages: [
Expand All @@ -147,7 +157,10 @@ async function submit(formData?: FormData, skip?: boolean) {
{
id: groupeId,
role: 'assistant',
content: JSON.stringify(analysisResult),
content: JSON.stringify({
...analysisResult,
image: dataUrl
}),
type: 'resolution_search_result'
},
{
Expand All @@ -173,12 +186,11 @@ async function submit(formData?: FormData, skip?: boolean) {
}
}

// Start the background process without awaiting it.
processResolutionSearch();

// Immediately update the UI stream with the BotMessage component.
uiStream.update(
<Section title="response">
<ResolutionImage src={dataUrl} />
<BotMessage content={summaryStream.value} />
</Section>
);
Expand All @@ -198,7 +210,17 @@ async function submit(formData?: FormData, skip?: boolean) {
message.type !== 'related' &&
message.type !== 'end' &&
message.type !== 'resolution_search_result'
)
).map(m => {
if (Array.isArray(m.content)) {
return {
...m,
content: m.content.filter((part: any) =>
part.type !== "image" || (typeof part.image === "string" && part.image.startsWith("data:"))
)
} as any
}
return m
})

const groupeId = nanoid()
const useSpecificAPI = process.env.USE_SPECIFIC_API_FOR_WRITER === 'true'
Expand Down Expand Up @@ -241,9 +263,8 @@ async function submit(formData?: FormData, skip?: boolean) {
</Section>
);

uiStream.append(answerSection);
uiStream.update(answerSection);

const groupeId = nanoid();
const relatedQueries = { items: [] };

aiState.done({
Expand Down Expand Up @@ -327,7 +348,6 @@ async function submit(formData?: FormData, skip?: boolean) {
}

const hasImage = messageParts.some(part => part.type === 'image')
// Properly type the content based on whether it contains images
const content: CoreMessage['content'] = hasImage
? messageParts as CoreMessage['content']
: messageParts.map(part => part.text).join('\n')
Expand Down Expand Up @@ -361,7 +381,6 @@ async function submit(formData?: FormData, skip?: boolean) {

const userId = 'anonymous'
const currentSystemPrompt = (await getSystemPrompt(userId)) || ''

const mapProvider = formData?.get('mapProvider') as 'mapbox' | 'google'

async function processEvents() {
Expand Down Expand Up @@ -410,7 +429,8 @@ async function submit(formData?: FormData, skip?: boolean) {
streamText,
messages,
mapProvider,
useSpecificAPI
useSpecificAPI,
drawnFeatures
)
answer = fullResponse
toolOutputs = toolResponses
Expand Down Expand Up @@ -643,12 +663,10 @@ export const getUIStateFromAIState = (aiState: AIState): UIState => {
case 'input_related':
let messageContent: string | any[]
try {
// For backward compatibility with old messages that stored a JSON string
const json = JSON.parse(content as string)
messageContent =
type === 'input' ? json.input : json.related_query
} catch (e) {
// New messages will store the content array or string directly
messageContent = content
}
return {
Expand All @@ -669,8 +687,8 @@ export const getUIStateFromAIState = (aiState: AIState): UIState => {
}
break
case 'assistant':
const answer = createStreamableValue()
answer.done(content)
const answer = createStreamableValue(content as string)
answer.done(content as string)
switch (type) {
case 'response':
return {
Expand All @@ -682,7 +700,9 @@ export const getUIStateFromAIState = (aiState: AIState): UIState => {
)
}
case 'related':
const relatedQueries = createStreamableValue<RelatedQueries>()
const relatedQueries = createStreamableValue<RelatedQueries>({
items: []
})
relatedQueries.done(JSON.parse(content as string))
return {
id,
Expand All @@ -704,11 +724,13 @@ export const getUIStateFromAIState = (aiState: AIState): UIState => {
case 'resolution_search_result': {
const analysisResult = JSON.parse(content as string);
const geoJson = analysisResult.geoJson as FeatureCollection;
const image = analysisResult.image as string;

return {
id,
component: (
<>
{image && <ResolutionImage src={image} />}
{geoJson && (
<GeoJsonLayer id={id} data={geoJson} />
)}
Expand All @@ -721,21 +743,37 @@ export const getUIStateFromAIState = (aiState: AIState): UIState => {
case 'tool':
try {
const toolOutput = JSON.parse(content as string)
const isCollapsed = createStreamableValue()
const isCollapsed = createStreamableValue(true)
isCollapsed.done(true)

if (
toolOutput.type === 'MAP_QUERY_TRIGGER' &&
name === 'geospatialQueryTool'
) {
const mapUrl = toolOutput.mcp_response?.mapUrl;
const placeName = toolOutput.mcp_response?.location?.place_name;

return {
id,
component: <MapQueryHandler toolOutput={toolOutput} />,
component: (
<>
{mapUrl && (
<ResolutionImage
src={mapUrl}
className="mb-0"
alt={placeName ? `Map of ${placeName}` : 'Map Preview'}
/>
)}
<MapQueryHandler toolOutput={toolOutput} />
</>
),
isCollapsed: false
}
}

const searchResults = createStreamableValue()
const searchResults = createStreamableValue(
JSON.stringify(toolOutput)
)
searchResults.done(JSON.stringify(toolOutput))
switch (name) {
case 'search':
Expand Down
52 changes: 30 additions & 22 deletions app/layout.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@ import { SpeedInsights } from "@vercel/speed-insights/next"
import { Toaster } from '@/components/ui/sonner'
import { MapToggleProvider } from '@/components/map-toggle-context'
import { ProfileToggleProvider } from '@/components/profile-toggle-context'
import { UsageToggleProvider } from '@/components/usage-toggle-context'
import { CalendarToggleProvider } from '@/components/calendar-toggle-context'
import { HistoryToggleProvider } from '@/components/history-toggle-context'
import { HistorySidebar } from '@/components/history-sidebar'
import { MapLoadingProvider } from '@/components/map-loading-context';
import ConditionalLottie from '@/components/conditional-lottie';
import { MapProvider as MapContextProvider } from '@/components/map/map-context'
Expand Down Expand Up @@ -70,28 +73,33 @@ export default function RootLayout({
)}
>
<CalendarToggleProvider>
<MapToggleProvider>
<ProfileToggleProvider>
<ThemeProvider
attribute="class"
defaultTheme="earth"
enableSystem
disableTransitionOnChange
themes={['light', 'dark', 'earth']}
>
<MapContextProvider>
<MapLoadingProvider>
<Header />
<ConditionalLottie />
{children}
<Sidebar />
<Footer />
<Toaster />
</MapLoadingProvider>
</MapContextProvider>
</ThemeProvider>
</ProfileToggleProvider>
</MapToggleProvider>
<HistoryToggleProvider>
<MapToggleProvider>
<ProfileToggleProvider>
<UsageToggleProvider>
<ThemeProvider
attribute="class"
defaultTheme="earth"
enableSystem
disableTransitionOnChange
themes={['light', 'dark', 'earth']}
>
<MapContextProvider>
<MapLoadingProvider>
<Header />
<ConditionalLottie />
{children}
<Sidebar />
<HistorySidebar />
<Footer />
<Toaster />
</MapLoadingProvider>
</MapContextProvider>
</ThemeProvider>
</UsageToggleProvider>
</ProfileToggleProvider>
</MapToggleProvider>
</HistoryToggleProvider>
</CalendarToggleProvider>
<Analytics />
<SpeedInsights />
Expand Down
Loading