diff --git a/infra/stacks/email-service/index.ts b/infra/stacks/email-service/index.ts index ddd8eed201..c4da76e15d 100644 --- a/infra/stacks/email-service/index.ts +++ b/infra/stacks/email-service/index.ts @@ -46,6 +46,19 @@ const ISSUER = config.require(`fusionauth_issuer`); const NOTIFICATIONS_ENABLED = config.require(`notifications_enabled`); const USE_APOLLO_CRM_ENRICHMENT = config.require(`use_apollo_crm_enrichment`); const APOLLO_API_KEY_SECRET_NAME = config.require(`apollo_api_key_secret_name`); +// Secret holding the base64-encoded AES-256 key that encrypts stored +// IMAP/SMTP passwords at rest. Optional: when unset the env var is empty and +// IMAP/SMTP account linking is disabled for the deployment. +const emailCredentialsEncryptionKeySecretName = config.get( + `email_credentials_encryption_key_secret_name` +); +const EMAIL_CREDENTIALS_ENCRYPTION_KEY = emailCredentialsEncryptionKeySecretName + ? aws.secretsmanager + .getSecretVersionOutput({ + secretId: emailCredentialsEncryptionKeySecretName, + }) + .apply((secret) => secret.secretString) + : pulumi.output(''); const SENT_UNDO_DELAY_SECS = config.require(`sent_undo_delay_secs`); const REDIS_RATE_LIMIT_REQS = config.require(`redis_rate_limit_reqs`); const REDIS_RATE_LIMIT_REQS_BACKFILL = config.require( @@ -498,6 +511,10 @@ const containerEnvVars = [ name: 'SENT_UNDO_DELAY_SECS', value: pulumi.interpolate`${SENT_UNDO_DELAY_SECS}`, }, + { + name: 'EMAIL_CREDENTIALS_ENCRYPTION_KEY', + value: pulumi.interpolate`${EMAIL_CREDENTIALS_ENCRYPTION_KEY}`, + }, { name: 'REDIS_RATE_LIMIT_REQS', value: pulumi.interpolate`${REDIS_RATE_LIMIT_REQS}`, diff --git a/js/app/packages/app/component/settings/Account.tsx b/js/app/packages/app/component/settings/Account.tsx index 32fd12aa15..96f7269944 100644 --- a/js/app/packages/app/component/settings/Account.tsx +++ b/js/app/packages/app/component/settings/Account.tsx @@ -34,8 +34,10 @@ import SignOutIcon from '@phosphor-icons/core/regular/sign-out.svg?component-sol import XIcon from '@phosphor-icons/core/regular/x.svg?component-solid'; import ArrowsClockwiseIcon from '@phosphor-icons/core/regular/arrows-clockwise.svg?component-solid'; import PlusIcon from '@phosphor-icons/core/regular/plus.svg?component-solid'; +import AtIcon from '@phosphor-icons/core/regular/at.svg?component-solid'; import { authServiceClient } from '@service-auth/client'; import type { + ConnectionSecurity, Link as EmailLink, SyncStatus, } from '@service-email/generated/schemas'; @@ -59,7 +61,10 @@ import PaywallTeamOwnerView from '../paywall/PaywallTeamOwnerView'; import { ROUTER_BASE_CONCAT } from '@app/constants/routerBase'; import { useEmailLinks, useEmailLinksStatus } from '@core/email-link'; import { useInitGmailLink } from '@queries/auth'; -import { useRemoveInboxMutation } from '@queries/email/link'; +import { + useCreateImapLinkMutation, + useRemoveInboxMutation, +} from '@queries/email/link'; import { type SupportedNotificationSettings, useNotificationSettings, @@ -295,6 +300,7 @@ export function Account() { email: string; isOwn: boolean; } | null>(null); + const [showImapDialog, setShowImapDialog] = createSignal(false); const [resyncingIds, setResyncingIds] = createSignal>( new Set() ); @@ -653,17 +659,30 @@ export function Account() { } > - - - +
+ + + + + + +
@@ -774,6 +793,11 @@ export function Account() { + + { @@ -1019,6 +1043,235 @@ function InboxRow(props: { ); } + +/** + * Form dialog for connecting an inbox on an arbitrary email server over + * IMAP (receive) + SMTP (send). The backend verifies both connections with + * the supplied credentials before anything is persisted, so errors shown + * here are specific (wrong password, unreachable host, ...). + */ +function ImapConnectDialog(props: { + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const createImapLink = useCreateImapLinkMutation(); + + const [email, setEmail] = createSignal(''); + const [username, setUsername] = createSignal(''); + const [password, setPassword] = createSignal(''); + const [imapHost, setImapHost] = createSignal(''); + const [imapPort, setImapPort] = createSignal('993'); + const [imapSecurity, setImapSecurity] = + createSignal('SSL_TLS'); + const [smtpHost, setSmtpHost] = createSignal(''); + const [smtpPort, setSmtpPort] = createSignal('465'); + const [smtpSecurity, setSmtpSecurity] = + createSignal('SSL_TLS'); + const [error, setError] = createSignal(null); + + const canSubmit = createMemo( + () => + email().trim() !== '' && + username().trim() !== '' && + password() !== '' && + imapHost().trim() !== '' && + smtpHost().trim() !== '' && + Number.parseInt(imapPort(), 10) > 0 && + Number.parseInt(smtpPort(), 10) > 0 && + !createImapLink.isPending + ); + + const handleConnect = async () => { + if (!canSubmit()) return; + setError(null); + try { + await createImapLink.mutateAsync({ + email_address: email().trim(), + imap: { + host: imapHost().trim(), + port: Number.parseInt(imapPort(), 10), + security: imapSecurity(), + username: username().trim(), + password: password(), + }, + smtp: { + host: smtpHost().trim(), + port: Number.parseInt(smtpPort(), 10), + security: smtpSecurity(), + username: username().trim(), + password: password(), + }, + }); + toast.success('Inbox connected — syncing recent mail'); + props.onOpenChange(false); + } catch (e) { + setError( + e instanceof Error && e.message + ? e.message + : 'Failed to connect. Check your server settings and credentials.' + ); + } + }; + + return ( + + + + + Connect IMAP/SMTP account + + + + + Connect any email account using its IMAP and SMTP servers. Many + providers require an app-specific password. + + + +
+ + +
+ +
+ + + +
+ +
+ + + +
+ + +
+ {error()} +
+ + +
+ + +
+ + + + ); +} + +function ImapField(props: { + label: string; + value: string; + onInput: (value: string) => void; + placeholder?: string; + type?: string; +}) { + return ( + + ); +} + +function ImapSecuritySelect(props: { + label: string; + value: ConnectionSecurity; + onChange: (value: ConnectionSecurity) => void; +}) { + return ( + + ); +} + function NameInput(props: { value?: string; placeholder?: string; diff --git a/js/app/packages/queries/email/link.ts b/js/app/packages/queries/email/link.ts index 7c8b0c911d..69114cb71d 100644 --- a/js/app/packages/queries/email/link.ts +++ b/js/app/packages/queries/email/link.ts @@ -3,7 +3,10 @@ import { throwOnErr } from '@core/util/result'; import { invalidateUserInfo } from '@queries/auth/user-info'; import { queryClient } from '@queries/client'; import { emailClient } from '@service-email/client'; -import type { ListLinksResponse } from '@service-email/generated/schemas'; +import type { + CreateImapLinkRequest, + ListLinksResponse, +} from '@service-email/generated/schemas'; import { useMutation, useQuery } from '@tanstack/solid-query'; import { createMemo } from 'solid-js'; import { type MutationCallbacks, withCallbacks } from '../utils'; @@ -67,6 +70,21 @@ export function invalidateEmailLinks() { }); } +/** + * Connects an inbox on an arbitrary IMAP/SMTP server. The backend validates + * both server connections before persisting, so errors carry a specific, + * user-presentable reason. Refreshes the links list on success. + */ +export function useCreateImapLinkMutation() { + return useMutation(() => ({ + mutationFn: async (request: CreateImapLinkRequest) => + throwOnErr(() => emailClient.createImapLink(request)), + onSuccess: () => { + invalidateEmailLinks(); + }, + })); +} + type RemoveInboxContext = { previousLinks: ListLinksResponse | undefined }; type RemoveInboxCallbacks = MutationCallbacks< void, diff --git a/js/app/packages/service-clients/service-email/client.ts b/js/app/packages/service-clients/service-email/client.ts index f9133856d9..32dd235af7 100644 --- a/js/app/packages/service-clients/service-email/client.ts +++ b/js/app/packages/service-clients/service-email/client.ts @@ -12,6 +12,8 @@ import type { ApiPaginatedThreadCursor, CreateDraftRequest, CreateDraftResponse, + CreateImapLinkRequest, + CreateImapLinkResponse, GetAttachmentDocumentIDResponse, GetAttachmentResponse, GetThreadResponse, @@ -241,6 +243,31 @@ export const emailClient = { ).map((result) => result); }, + /** + * Connects an email account on an arbitrary IMAP/SMTP server. The backend + * verifies both server connections before persisting anything, so failures + * surface as 400s whose body is a human-readable reason — propagate it so + * the form can tell the user which half is misconfigured. + */ + async createImapLink(args: CreateImapLinkRequest) { + return ( + await fetchWithToken( + `${emailHost}/email/links/imap`, + { + method: 'POST', + body: JSON.stringify(args), + errorResponseHandler: async (response) => { + const body = await response.text().catch(() => ''); + return { + code: 'IMAP_LINK_FAILED', + message: body || `HTTP error! status: ${response.status}`, + }; + }, + } + ) + ).map((result) => result); + }, + async resyncLink(args: { linkId: string }) { const { linkId } = args; return ( diff --git a/js/app/packages/service-clients/service-email/generated/client.ts b/js/app/packages/service-clients/service-email/generated/client.ts index 61eaea6f8d..a8605679a6 100644 --- a/js/app/packages/service-clients/service-email/generated/client.ts +++ b/js/app/packages/service-clients/service-email/generated/client.ts @@ -15,6 +15,8 @@ import type { CancelBackfillParams, CreateDraftRequest, CreateDraftResponse, + CreateImapLinkRequest, + CreateImapLinkResponse, CreateLabelRequest, CreateLabelResponse, EmptyResponse, @@ -1748,6 +1750,78 @@ export const listLinks = async ( } as listLinksResponse; }; +/** + * Verifies both server connections with the supplied credentials before +persisting anything; passwords are stored encrypted. On success an initial +sync of the inbox is scheduled. + * @summary Connects an email account on an arbitrary IMAP/SMTP server as an inbox. + */ +export type createImapLinkResponse200 = { + data: CreateImapLinkResponse; + status: 200; +}; + +export type createImapLinkResponse400 = { + data: ErrorResponse; + status: 400; +}; + +export type createImapLinkResponse401 = { + data: ErrorResponse; + status: 401; +}; + +export type createImapLinkResponse500 = { + data: ErrorResponse; + status: 500; +}; + +export type createImapLinkResponse501 = { + data: ErrorResponse; + status: 501; +}; + +export type createImapLinkResponseSuccess = createImapLinkResponse200 & { + headers: Headers; +}; +export type createImapLinkResponseError = ( + | createImapLinkResponse400 + | createImapLinkResponse401 + | createImapLinkResponse500 + | createImapLinkResponse501 +) & { + headers: Headers; +}; + +export type createImapLinkResponse = + | createImapLinkResponseSuccess + | createImapLinkResponseError; + +export const getCreateImapLinkUrl = () => { + return `/email/links/imap`; +}; + +export const createImapLink = async ( + createImapLinkRequest: CreateImapLinkRequest, + options?: RequestInit +): Promise => { + const res = await fetch(getCreateImapLinkUrl(), { + ...options, + method: 'POST', + headers: { 'Content-Type': 'application/json', ...options?.headers }, + body: JSON.stringify(createImapLinkRequest), + }); + + const body = [204, 205, 304].includes(res.status) ? null : await res.text(); + + const data: createImapLinkResponse['data'] = body ? JSON.parse(body) : {}; + return { + data, + status: res.status, + headers: res.headers, + } as createImapLinkResponse; +}; + /** * For an inbox the caller owns this enqueues a full cascade teardown (`LinkManagerMessage::DeleteLink`). For an inbox reached via delegation it diff --git a/js/app/packages/service-clients/service-email/generated/schemas/connectionSecurity.ts b/js/app/packages/service-clients/service-email/generated/schemas/connectionSecurity.ts new file mode 100644 index 0000000000..820b38e92d --- /dev/null +++ b/js/app/packages/service-clients/service-email/generated/schemas/connectionSecurity.ts @@ -0,0 +1,20 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * email_service + * OpenAPI spec version: 0.1.0 + */ + +/** + * How a connection to an IMAP/SMTP server is secured. + +Mirrors the `email_connection_security_enum` Postgres enum. + */ +export type ConnectionSecurity = + (typeof ConnectionSecurity)[keyof typeof ConnectionSecurity]; + +// eslint-disable-next-line @typescript-eslint/no-redeclare +export const ConnectionSecurity = { + SSL_TLS: 'SSL_TLS', + STARTTLS: 'STARTTLS', +} as const; diff --git a/js/app/packages/service-clients/service-email/generated/schemas/createImapLinkRequest.ts b/js/app/packages/service-clients/service-email/generated/schemas/createImapLinkRequest.ts new file mode 100644 index 0000000000..b75f26be46 --- /dev/null +++ b/js/app/packages/service-clients/service-email/generated/schemas/createImapLinkRequest.ts @@ -0,0 +1,16 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * email_service + * OpenAPI spec version: 0.1.0 + */ +import type { ServerSettingsInput } from './serverSettingsInput'; + +export interface CreateImapLinkRequest { + /** The mailbox address being connected. */ + email_address: string; + /** IMAP (receiving) server settings. */ + imap: ServerSettingsInput; + /** SMTP (sending) server settings. */ + smtp: ServerSettingsInput; +} diff --git a/js/app/packages/service-clients/service-email/generated/schemas/createImapLinkResponse.ts b/js/app/packages/service-clients/service-email/generated/schemas/createImapLinkResponse.ts new file mode 100644 index 0000000000..b75506f929 --- /dev/null +++ b/js/app/packages/service-clients/service-email/generated/schemas/createImapLinkResponse.ts @@ -0,0 +1,11 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * email_service + * OpenAPI spec version: 0.1.0 + */ + +export interface CreateImapLinkResponse { + /** The email_links row id for the newly connected inbox. */ + link_id: string; +} diff --git a/js/app/packages/service-clients/service-email/generated/schemas/index.ts b/js/app/packages/service-clients/service-email/generated/schemas/index.ts index 1765357cca..04e9a4864d 100644 --- a/js/app/packages/service-clients/service-email/generated/schemas/index.ts +++ b/js/app/packages/service-clients/service-email/generated/schemas/index.ts @@ -141,6 +141,7 @@ export * from './backfillJobStatus'; export * from './backfillJobThreadsRequestedLimit'; export * from './blockSenderRequest'; export * from './cancelBackfillParams'; +export * from './connectionSecurity'; export * from './contact'; export * from './contactEmailAddress'; export * from './contactInfo'; @@ -157,6 +158,8 @@ export * from './contactSfsPhotoUrl'; export * from './createDraftRequest'; export * from './createDraftRequestSendTime'; export * from './createDraftResponse'; +export * from './createImapLinkRequest'; +export * from './createImapLinkResponse'; export * from './createLabelRequest'; export * from './createLabelResponse'; export * from './emptyResponse'; @@ -224,8 +227,10 @@ export * from './previewsInboxCursorParams'; export * from './previewView'; export * from './previewViewStandardLabel'; export * from './resyncResponse'; +export * from './resyncResponseBackfillJobId'; export * from './sendMessageRequest'; export * from './sendMessageResponse'; +export * from './serverSettingsInput'; export * from './settings'; export * from './settingsSignatureOnRepliesForwards'; export * from './syncStatus'; diff --git a/js/app/packages/service-clients/service-email/generated/schemas/resyncResponse.ts b/js/app/packages/service-clients/service-email/generated/schemas/resyncResponse.ts index 3ec9044f97..b02a80813d 100644 --- a/js/app/packages/service-clients/service-email/generated/schemas/resyncResponse.ts +++ b/js/app/packages/service-clients/service-email/generated/schemas/resyncResponse.ts @@ -4,6 +4,7 @@ * email_service * OpenAPI spec version: 0.1.0 */ +import type { ResyncResponseBackfillJobId } from './resyncResponseBackfillJobId'; /** * The response returned from the resync endpoint. @@ -12,6 +13,7 @@ export interface ResyncResponse { /** True when a backfill was already running and this call was a no-op. */ already_in_progress: boolean; /** The backfill job driving the (re-)sync. Either the freshly enqueued job or -the one already in progress. */ - backfill_job_id: string; +the one already in progress. Absent for IMAP/SMTP links, whose resync is a +direct poll of the server rather than a tracked backfill job. */ + backfill_job_id?: ResyncResponseBackfillJobId; } diff --git a/js/app/packages/service-clients/service-email/generated/schemas/resyncResponseBackfillJobId.ts b/js/app/packages/service-clients/service-email/generated/schemas/resyncResponseBackfillJobId.ts new file mode 100644 index 0000000000..728f0bc166 --- /dev/null +++ b/js/app/packages/service-clients/service-email/generated/schemas/resyncResponseBackfillJobId.ts @@ -0,0 +1,13 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * email_service + * OpenAPI spec version: 0.1.0 + */ + +/** + * The backfill job driving the (re-)sync. Either the freshly enqueued job or +the one already in progress. Absent for IMAP/SMTP links, whose resync is a +direct poll of the server rather than a tracked backfill job. + */ +export type ResyncResponseBackfillJobId = string | null; diff --git a/js/app/packages/service-clients/service-email/generated/schemas/serverSettingsInput.ts b/js/app/packages/service-clients/service-email/generated/schemas/serverSettingsInput.ts new file mode 100644 index 0000000000..e70a400b07 --- /dev/null +++ b/js/app/packages/service-clients/service-email/generated/schemas/serverSettingsInput.ts @@ -0,0 +1,26 @@ +/** + * Generated by orval v7.21.0 🍺 + * Do not edit manually. + * email_service + * OpenAPI spec version: 0.1.0 + */ +import type { ConnectionSecurity } from './connectionSecurity'; + +/** + * Connection settings for one server (the IMAP or SMTP half). + */ +export interface ServerSettingsInput { + /** Server hostname, e.g. `imap.fastmail.com`. */ + host: string; + /** Login password. Many providers require an app-specific password. */ + password: string; + /** + * Server port, e.g. 993 for IMAP over TLS, 465/587 for SMTP. + * @minimum 0 + */ + port: number; + /** How the connection is secured. */ + security: ConnectionSecurity; + /** Login username (usually the email address). */ + username: string; +} diff --git a/js/app/packages/service-clients/service-email/generated/schemas/userProvider.ts b/js/app/packages/service-clients/service-email/generated/schemas/userProvider.ts index eca0217670..dac6a498d0 100644 --- a/js/app/packages/service-clients/service-email/generated/schemas/userProvider.ts +++ b/js/app/packages/service-clients/service-email/generated/schemas/userProvider.ts @@ -10,4 +10,5 @@ export type UserProvider = (typeof UserProvider)[keyof typeof UserProvider]; // eslint-disable-next-line @typescript-eslint/no-redeclare export const UserProvider = { GMAIL: 'GMAIL', + IMAP_SMTP: 'IMAP_SMTP', } as const; diff --git a/js/app/packages/service-clients/service-email/openapi.json b/js/app/packages/service-clients/service-email/openapi.json index 084fb47066..20405d0870 100644 --- a/js/app/packages/service-clients/service-email/openapi.json +++ b/js/app/packages/service-clients/service-email/openapi.json @@ -1576,6 +1576,76 @@ } } }, + "/email/links/imap": { + "post": { + "tags": ["Links"], + "summary": "Connects an email account on an arbitrary IMAP/SMTP server as an inbox.", + "description": "Verifies both server connections with the supplied credentials before\npersisting anything; passwords are stored encrypted. On success an initial\nsync of the inbox is scheduled.", + "operationId": "create_imap_link", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateImapLinkRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateImapLinkResponse" + } + } + } + }, + "400": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "401": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "500": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "501": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, "/email/links/{link_id}": { "delete": { "tags": ["Links"], @@ -3806,6 +3876,11 @@ } } }, + "ConnectionSecurity": { + "type": "string", + "description": "How a connection to an IMAP/SMTP server is secured.\n\nMirrors the `email_connection_security_enum` Postgres enum.", + "enum": ["SSL_TLS", "STARTTLS"] + }, "Contact": { "type": "object", "required": ["id", "link_id"], @@ -3909,6 +3984,35 @@ } } }, + "CreateImapLinkRequest": { + "type": "object", + "required": ["email_address", "imap", "smtp"], + "properties": { + "email_address": { + "type": "string", + "description": "The mailbox address being connected." + }, + "imap": { + "$ref": "#/components/schemas/ServerSettingsInput", + "description": "IMAP (receiving) server settings." + }, + "smtp": { + "$ref": "#/components/schemas/ServerSettingsInput", + "description": "SMTP (sending) server settings." + } + } + }, + "CreateImapLinkResponse": { + "type": "object", + "required": ["link_id"], + "properties": { + "link_id": { + "type": "string", + "format": "uuid", + "description": "The email_links row id for the newly connected inbox." + } + } + }, "CreateLabelRequest": { "type": "object", "required": ["label_name"], @@ -4557,16 +4661,16 @@ "ResyncResponse": { "type": "object", "description": "The response returned from the resync endpoint.", - "required": ["backfill_job_id", "already_in_progress"], + "required": ["already_in_progress"], "properties": { "already_in_progress": { "type": "boolean", "description": "True when a backfill was already running and this call was a no-op." }, "backfill_job_id": { - "type": "string", + "type": ["string", "null"], "format": "uuid", - "description": "The backfill job driving the (re-)sync. Either the freshly enqueued job or\nthe one already in progress." + "description": "The backfill job driving the (re-)sync. Either the freshly enqueued job or\nthe one already in progress. Absent for IMAP/SMTP links, whose resync is a\ndirect poll of the server rather than a tracked backfill job." } } }, @@ -4592,6 +4696,35 @@ } } }, + "ServerSettingsInput": { + "type": "object", + "description": "Connection settings for one server (the IMAP or SMTP half).", + "required": ["host", "port", "security", "username", "password"], + "properties": { + "host": { + "type": "string", + "description": "Server hostname, e.g. `imap.fastmail.com`." + }, + "password": { + "type": "string", + "description": "Login password. Many providers require an app-specific password." + }, + "port": { + "type": "integer", + "format": "int32", + "description": "Server port, e.g. 993 for IMAP over TLS, 465/587 for SMTP.", + "minimum": 0 + }, + "security": { + "$ref": "#/components/schemas/ConnectionSecurity", + "description": "How the connection is secured." + }, + "username": { + "type": "string", + "description": "Login username (usually the email address)." + } + } + }, "Settings": { "type": "object", "properties": { @@ -4939,7 +5072,7 @@ }, "UserProvider": { "type": "string", - "enum": ["GMAIL"] + "enum": ["GMAIL", "IMAP_SMTP"] }, "Value": {} } diff --git a/rust/cloud-storage/.sqlx/query-15b00cf9b57c2c077f7e55bd25c267a6cf83c9efdc10ba23981f7b85847881c3.json b/rust/cloud-storage/.sqlx/query-15b00cf9b57c2c077f7e55bd25c267a6cf83c9efdc10ba23981f7b85847881c3.json index ef0ebb8a40..f873678e62 100644 --- a/rust/cloud-storage/.sqlx/query-15b00cf9b57c2c077f7e55bd25c267a6cf83c9efdc10ba23981f7b85847881c3.json +++ b/rust/cloud-storage/.sqlx/query-15b00cf9b57c2c077f7e55bd25c267a6cf83c9efdc10ba23981f7b85847881c3.json @@ -31,7 +31,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-2ba42e1b8b6e432c5cbe966076ca05523a4b1f4be0798ae362a325c172b19a43.json b/rust/cloud-storage/.sqlx/query-2ba42e1b8b6e432c5cbe966076ca05523a4b1f4be0798ae362a325c172b19a43.json index 5ea23b9b81..71ef9ee006 100644 --- a/rust/cloud-storage/.sqlx/query-2ba42e1b8b6e432c5cbe966076ca05523a4b1f4be0798ae362a325c172b19a43.json +++ b/rust/cloud-storage/.sqlx/query-2ba42e1b8b6e432c5cbe966076ca05523a4b1f4be0798ae362a325c172b19a43.json @@ -17,7 +17,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-2c1df8ba2035cc1f63f9d566e8ac763f4d0b5e28370570971935d29237a0211a.json b/rust/cloud-storage/.sqlx/query-2c1df8ba2035cc1f63f9d566e8ac763f4d0b5e28370570971935d29237a0211a.json index 4e5c86734c..9a03d066d3 100644 --- a/rust/cloud-storage/.sqlx/query-2c1df8ba2035cc1f63f9d566e8ac763f4d0b5e28370570971935d29237a0211a.json +++ b/rust/cloud-storage/.sqlx/query-2c1df8ba2035cc1f63f9d566e8ac763f4d0b5e28370570971935d29237a0211a.json @@ -31,7 +31,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-45fced9bee3cf06b7f54821c6c96e3b483654160636aab0f6c5ab18f64addc99.json b/rust/cloud-storage/.sqlx/query-45fced9bee3cf06b7f54821c6c96e3b483654160636aab0f6c5ab18f64addc99.json new file mode 100644 index 0000000000..97b9ecedc0 --- /dev/null +++ b/rust/cloud-storage/.sqlx/query-45fced9bee3cf06b7f54821c6c96e3b483654160636aab0f6c5ab18f64addc99.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n id as \"link_id\"\n FROM email_links\n WHERE\n is_sync_active = TRUE\n AND provider = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "link_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + { + "Custom": { + "name": "email_user_provider_enum", + "kind": { + "Enum": [ + "GMAIL", + "IMAP_SMTP" + ] + } + } + } + ] + }, + "nullable": [ + false + ] + }, + "hash": "45fced9bee3cf06b7f54821c6c96e3b483654160636aab0f6c5ab18f64addc99" +} diff --git a/rust/cloud-storage/.sqlx/query-4857541516ce135939f34754fde00c114cdf9ed7f203553020cf032732892e80.json b/rust/cloud-storage/.sqlx/query-4857541516ce135939f34754fde00c114cdf9ed7f203553020cf032732892e80.json index cb80ad13ca..a90d57a357 100644 --- a/rust/cloud-storage/.sqlx/query-4857541516ce135939f34754fde00c114cdf9ed7f203553020cf032732892e80.json +++ b/rust/cloud-storage/.sqlx/query-4857541516ce135939f34754fde00c114cdf9ed7f203553020cf032732892e80.json @@ -14,7 +14,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-4a8b04d2fa922cd2f007241f83d6ad7e4dbcb5e7f6b0560316ca474878971e63.json b/rust/cloud-storage/.sqlx/query-4a8b04d2fa922cd2f007241f83d6ad7e4dbcb5e7f6b0560316ca474878971e63.json index c905db25bd..2e692f7543 100644 --- a/rust/cloud-storage/.sqlx/query-4a8b04d2fa922cd2f007241f83d6ad7e4dbcb5e7f6b0560316ca474878971e63.json +++ b/rust/cloud-storage/.sqlx/query-4a8b04d2fa922cd2f007241f83d6ad7e4dbcb5e7f6b0560316ca474878971e63.json @@ -31,7 +31,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } @@ -62,7 +63,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-50318d6d13fb931307f2115b33d277b5298c45ed798719e792e722bd30615a95.json b/rust/cloud-storage/.sqlx/query-50318d6d13fb931307f2115b33d277b5298c45ed798719e792e722bd30615a95.json index 7a4d0490be..422eac8bf8 100644 --- a/rust/cloud-storage/.sqlx/query-50318d6d13fb931307f2115b33d277b5298c45ed798719e792e722bd30615a95.json +++ b/rust/cloud-storage/.sqlx/query-50318d6d13fb931307f2115b33d277b5298c45ed798719e792e722bd30615a95.json @@ -31,7 +31,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-6b5a41c0c33c20537ef484ad312d80da7dfa7251e2333f8981e2a76f40e623c2.json b/rust/cloud-storage/.sqlx/query-6b5a41c0c33c20537ef484ad312d80da7dfa7251e2333f8981e2a76f40e623c2.json index 9cd4ca4c09..248bee83c6 100644 --- a/rust/cloud-storage/.sqlx/query-6b5a41c0c33c20537ef484ad312d80da7dfa7251e2333f8981e2a76f40e623c2.json +++ b/rust/cloud-storage/.sqlx/query-6b5a41c0c33c20537ef484ad312d80da7dfa7251e2333f8981e2a76f40e623c2.json @@ -16,7 +16,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-7630425280ebc21d6c451df7474e42b3ae2d0a78219ea44d4831cbbc6ecbd2ca.json b/rust/cloud-storage/.sqlx/query-7630425280ebc21d6c451df7474e42b3ae2d0a78219ea44d4831cbbc6ecbd2ca.json new file mode 100644 index 0000000000..88cf9ed0c2 --- /dev/null +++ b/rust/cloud-storage/.sqlx/query-7630425280ebc21d6c451df7474e42b3ae2d0a78219ea44d4831cbbc6ecbd2ca.json @@ -0,0 +1,102 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT link_id,\n imap_host, imap_port, imap_security as \"imap_security: _\",\n imap_username, imap_password_ciphertext,\n smtp_host, smtp_port, smtp_security as \"smtp_security: _\",\n smtp_username, smtp_password_ciphertext\n FROM email_imap_smtp_credentials\n WHERE link_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "link_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "imap_host", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "imap_port", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "imap_security: _", + "type_info": { + "Custom": { + "name": "email_connection_security_enum", + "kind": { + "Enum": [ + "SSL_TLS", + "STARTTLS" + ] + } + } + } + }, + { + "ordinal": 4, + "name": "imap_username", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "imap_password_ciphertext", + "type_info": "Bytea" + }, + { + "ordinal": 6, + "name": "smtp_host", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "smtp_port", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "smtp_security: _", + "type_info": { + "Custom": { + "name": "email_connection_security_enum", + "kind": { + "Enum": [ + "SSL_TLS", + "STARTTLS" + ] + } + } + } + }, + { + "ordinal": 9, + "name": "smtp_username", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "smtp_password_ciphertext", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "7630425280ebc21d6c451df7474e42b3ae2d0a78219ea44d4831cbbc6ecbd2ca" +} diff --git a/rust/cloud-storage/.sqlx/query-7bdcbf27b9edb2a2efc0e30c83230f6400385b295a50b48776985089d6ae3c84.json b/rust/cloud-storage/.sqlx/query-7bdcbf27b9edb2a2efc0e30c83230f6400385b295a50b48776985089d6ae3c84.json index ee23701453..1b20c5f3a5 100644 --- a/rust/cloud-storage/.sqlx/query-7bdcbf27b9edb2a2efc0e30c83230f6400385b295a50b48776985089d6ae3c84.json +++ b/rust/cloud-storage/.sqlx/query-7bdcbf27b9edb2a2efc0e30c83230f6400385b295a50b48776985089d6ae3c84.json @@ -31,7 +31,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-7f9dea3f16070c9e5c67107643cef03d9c7e1704353496d79faf02dc998bf6fa.json b/rust/cloud-storage/.sqlx/query-7f9dea3f16070c9e5c67107643cef03d9c7e1704353496d79faf02dc998bf6fa.json new file mode 100644 index 0000000000..b515632ad8 --- /dev/null +++ b/rust/cloud-storage/.sqlx/query-7f9dea3f16070c9e5c67107643cef03d9c7e1704353496d79faf02dc998bf6fa.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT thread_id\n FROM email_messages\n WHERE link_id = $1 AND global_id = ANY($2)\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "thread_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7f9dea3f16070c9e5c67107643cef03d9c7e1704353496d79faf02dc998bf6fa" +} diff --git a/rust/cloud-storage/.sqlx/query-8942100b50a073f4f95b37246d355dc8b17c5a105492901ad72da89438ec6601.json b/rust/cloud-storage/.sqlx/query-8942100b50a073f4f95b37246d355dc8b17c5a105492901ad72da89438ec6601.json index 8a954a7761..7ab7c97498 100644 --- a/rust/cloud-storage/.sqlx/query-8942100b50a073f4f95b37246d355dc8b17c5a105492901ad72da89438ec6601.json +++ b/rust/cloud-storage/.sqlx/query-8942100b50a073f4f95b37246d355dc8b17c5a105492901ad72da89438ec6601.json @@ -20,7 +20,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-8a1debc8a93f83a63f33dfd12a1eb8f707e99f0a207a9b34f27848ab362a7bff.json b/rust/cloud-storage/.sqlx/query-8a1debc8a93f83a63f33dfd12a1eb8f707e99f0a207a9b34f27848ab362a7bff.json index d389ea18d1..27f24e1b35 100644 --- a/rust/cloud-storage/.sqlx/query-8a1debc8a93f83a63f33dfd12a1eb8f707e99f0a207a9b34f27848ab362a7bff.json +++ b/rust/cloud-storage/.sqlx/query-8a1debc8a93f83a63f33dfd12a1eb8f707e99f0a207a9b34f27848ab362a7bff.json @@ -31,7 +31,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } @@ -61,7 +62,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-94058ba40f828251432cad4a2514e4ec0dcd8d9f6517694f3c00d7c9aa8ccc82.json b/rust/cloud-storage/.sqlx/query-94058ba40f828251432cad4a2514e4ec0dcd8d9f6517694f3c00d7c9aa8ccc82.json new file mode 100644 index 0000000000..92bf9be0a9 --- /dev/null +++ b/rust/cloud-storage/.sqlx/query-94058ba40f828251432cad4a2514e4ec0dcd8d9f6517694f3c00d7c9aa8ccc82.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT link_id, folder, uid_validity, last_seen_uid\n FROM email_imap_folder_states\n WHERE link_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "link_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "folder", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "uid_validity", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "last_seen_uid", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "94058ba40f828251432cad4a2514e4ec0dcd8d9f6517694f3c00d7c9aa8ccc82" +} diff --git a/rust/cloud-storage/.sqlx/query-95c5d4f79e729713642efcda97838eddd466c0ec2ec9f96ae2be185ef9ff1547.json b/rust/cloud-storage/.sqlx/query-95c5d4f79e729713642efcda97838eddd466c0ec2ec9f96ae2be185ef9ff1547.json index 94b5b88dd8..42dc8e92c7 100644 --- a/rust/cloud-storage/.sqlx/query-95c5d4f79e729713642efcda97838eddd466c0ec2ec9f96ae2be185ef9ff1547.json +++ b/rust/cloud-storage/.sqlx/query-95c5d4f79e729713642efcda97838eddd466c0ec2ec9f96ae2be185ef9ff1547.json @@ -31,7 +31,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-a81b373188c98bdfa52a878a659909f73796aef0302323f7930fe4d17ccfe6af.json b/rust/cloud-storage/.sqlx/query-a81b373188c98bdfa52a878a659909f73796aef0302323f7930fe4d17ccfe6af.json index c03715ba1e..15296a48cf 100644 --- a/rust/cloud-storage/.sqlx/query-a81b373188c98bdfa52a878a659909f73796aef0302323f7930fe4d17ccfe6af.json +++ b/rust/cloud-storage/.sqlx/query-a81b373188c98bdfa52a878a659909f73796aef0302323f7930fe4d17ccfe6af.json @@ -31,7 +31,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-c588fc5c8adc1e58d7127b11242eb9f9a29d35648304934dc750040aad9e8f1c.json b/rust/cloud-storage/.sqlx/query-c588fc5c8adc1e58d7127b11242eb9f9a29d35648304934dc750040aad9e8f1c.json new file mode 100644 index 0000000000..2621af9f80 --- /dev/null +++ b/rust/cloud-storage/.sqlx/query-c588fc5c8adc1e58d7127b11242eb9f9a29d35648304934dc750040aad9e8f1c.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO email_imap_smtp_credentials (\n link_id,\n imap_host, imap_port, imap_security, imap_username, imap_password_ciphertext,\n smtp_host, smtp_port, smtp_security, smtp_username, smtp_password_ciphertext,\n updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NOW())\n ON CONFLICT (link_id)\n DO UPDATE SET\n imap_host = EXCLUDED.imap_host,\n imap_port = EXCLUDED.imap_port,\n imap_security = EXCLUDED.imap_security,\n imap_username = EXCLUDED.imap_username,\n imap_password_ciphertext = EXCLUDED.imap_password_ciphertext,\n smtp_host = EXCLUDED.smtp_host,\n smtp_port = EXCLUDED.smtp_port,\n smtp_security = EXCLUDED.smtp_security,\n smtp_username = EXCLUDED.smtp_username,\n smtp_password_ciphertext = EXCLUDED.smtp_password_ciphertext,\n updated_at = NOW()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Int4", + { + "Custom": { + "name": "email_connection_security_enum", + "kind": { + "Enum": [ + "SSL_TLS", + "STARTTLS" + ] + } + } + }, + "Varchar", + "Bytea", + "Varchar", + "Int4", + { + "Custom": { + "name": "email_connection_security_enum", + "kind": { + "Enum": [ + "SSL_TLS", + "STARTTLS" + ] + } + } + }, + "Varchar", + "Bytea" + ] + }, + "nullable": [] + }, + "hash": "c588fc5c8adc1e58d7127b11242eb9f9a29d35648304934dc750040aad9e8f1c" +} diff --git a/rust/cloud-storage/.sqlx/query-c8569a737734eab039a3ca9759478b203596330442305a6f3925edf67a8b52a3.json b/rust/cloud-storage/.sqlx/query-c8569a737734eab039a3ca9759478b203596330442305a6f3925edf67a8b52a3.json index dec05aa8a8..ceb9ff2367 100644 --- a/rust/cloud-storage/.sqlx/query-c8569a737734eab039a3ca9759478b203596330442305a6f3925edf67a8b52a3.json +++ b/rust/cloud-storage/.sqlx/query-c8569a737734eab039a3ca9759478b203596330442305a6f3925edf67a8b52a3.json @@ -31,7 +31,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } @@ -62,7 +63,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-f4dd3bd30a64a7fd82d9a254a0fb0ea36e8607ebd4c272509afba882a3e0b572.json b/rust/cloud-storage/.sqlx/query-f4dd3bd30a64a7fd82d9a254a0fb0ea36e8607ebd4c272509afba882a3e0b572.json index 3cdb5b9433..5dc35953c7 100644 --- a/rust/cloud-storage/.sqlx/query-f4dd3bd30a64a7fd82d9a254a0fb0ea36e8607ebd4c272509afba882a3e0b572.json +++ b/rust/cloud-storage/.sqlx/query-f4dd3bd30a64a7fd82d9a254a0fb0ea36e8607ebd4c272509afba882a3e0b572.json @@ -31,7 +31,8 @@ "name": "email_user_provider_enum", "kind": { "Enum": [ - "GMAIL" + "GMAIL", + "IMAP_SMTP" ] } } diff --git a/rust/cloud-storage/.sqlx/query-f634430e82099c69b9aead94a2870bcf8a7052dce3ea4b07da7ee1586945726f.json b/rust/cloud-storage/.sqlx/query-f634430e82099c69b9aead94a2870bcf8a7052dce3ea4b07da7ee1586945726f.json new file mode 100644 index 0000000000..8ccd751b72 --- /dev/null +++ b/rust/cloud-storage/.sqlx/query-f634430e82099c69b9aead94a2870bcf8a7052dce3ea4b07da7ee1586945726f.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO email_imap_folder_states (link_id, folder, uid_validity, last_seen_uid, updated_at)\n VALUES ($1, $2, $3, $4, NOW())\n ON CONFLICT (link_id, folder)\n DO UPDATE SET\n uid_validity = EXCLUDED.uid_validity,\n last_seen_uid = EXCLUDED.last_seen_uid,\n updated_at = NOW()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "f634430e82099c69b9aead94a2870bcf8a7052dce3ea4b07da7ee1586945726f" +} diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lock index 4b26d299a3..52b78b9785 100644 --- a/rust/cloud-storage/Cargo.lock +++ b/rust/cloud-storage/Cargo.lock @@ -434,6 +434,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" version = "0.4.42" @@ -446,6 +458,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-imap" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78dceaba06f029d8f4d7df20addd4b7370a30206e3926267ecda2915b0f3f66" +dependencies = [ + "async-channel 2.5.0", + "async-compression", + "base64 0.22.1", + "bytes", + "chrono", + "futures", + "imap-proto", + "log", + "nom 7.1.3", + "pin-project", + "pin-utils", + "self_cell", + "stop-token", + "thiserror 1.0.69", + "tokio", +] + [[package]] name = "async-lock" version = "3.4.2" @@ -4048,6 +4083,7 @@ dependencies = [ "gmail_client", "html-escape", "http-body-util", + "imap_smtp_client", "infer 0.19.0", "item_filters", "last_online_tracker", @@ -4160,6 +4196,9 @@ dependencies = [ name = "email_utils" version = "0.1.0" dependencies = [ + "aes-gcm", + "anyhow", + "base64 0.22.1", "ego-tree", "html2text", "lazy_static", @@ -5299,7 +5338,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" dependencies = [ "anyhow", - "async-channel", + "async-channel 1.9.0", "base64 0.13.1", "futures-lite", "http 0.2.12", @@ -5680,6 +5719,33 @@ dependencies = [ "utoipa-swagger-ui", ] +[[package]] +name = "imap-proto" +version = "0.16.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f6af35c6a517aea5c72314abe90134980d2ae6a763809b50c208b3e429d71f" +dependencies = [ + "nom 7.1.3", +] + +[[package]] +name = "imap_smtp_client" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-imap", + "chrono", + "futures", + "macro_uuid", + "mail-builder", + "mail-send", + "models_email", + "tokio", + "tokio-rustls 0.26.4", + "tracing", + "webpki-roots 0.26.11", +] + [[package]] name = "imgref" version = "1.12.1" @@ -6707,6 +6773,24 @@ dependencies = [ "gethostname", ] +[[package]] +name = "mail-send" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d06f18f6ad15354b83645a05466570ced7bf770a17b138742e705d54407d83ba" +dependencies = [ + "base64 0.22.1", + "gethostname", + "md5", + "rand 0.9.4", + "rustls 0.23.40", + "rustls-pki-types", + "rustls-platform-verifier", + "smtp-proto", + "tokio", + "tokio-rustls 0.26.4", +] + [[package]] name = "mailparse" version = "0.16.1" @@ -6936,6 +7020,12 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "md5" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" + [[package]] name = "memchr" version = "2.8.0" @@ -9922,6 +10012,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", @@ -10462,6 +10553,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "self_cell" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + [[package]] name = "semver" version = "1.0.28" @@ -10927,6 +11024,12 @@ dependencies = [ "serde", ] +[[package]] +name = "smtp-proto" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97c9f5dd7ec5cc6d743f33fcb96de4eb91bb1cc51c5e0ba40cb285a9012043da" + [[package]] name = "sns_client" version = "0.1.0" @@ -11350,6 +11453,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "stop-token" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af91f480ee899ab2d9f8435bfdfc14d08a5754bd9d3fef1f1a1c23336aad6c8b" +dependencies = [ + "async-channel 1.9.0", + "cfg-if", + "futures-core", + "pin-project-lite", +] + [[package]] name = "stream" version = "0.1.0" diff --git a/rust/cloud-storage/email/src/domain/models/link.rs b/rust/cloud-storage/email/src/domain/models/link.rs index fc025bc3bd..f99c53978d 100644 --- a/rust/cloud-storage/email/src/domain/models/link.rs +++ b/rust/cloud-storage/email/src/domain/models/link.rs @@ -6,12 +6,15 @@ use uuid::Uuid; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum UserProvider { Gmail, + /// A generic email server reached over IMAP (receive) and SMTP (send). + ImapSmtp, } impl UserProvider { pub fn as_str(&self) -> &'static str { match self { UserProvider::Gmail => "GMAIL", + UserProvider::ImapSmtp => "IMAP_SMTP", } } } diff --git a/rust/cloud-storage/email/src/inbound/axum/thread_labels_router.rs b/rust/cloud-storage/email/src/inbound/axum/thread_labels_router.rs index 7d7cc29e70..1b5a13d4ca 100644 --- a/rust/cloud-storage/email/src/inbound/axum/thread_labels_router.rs +++ b/rust/cloud-storage/email/src/inbound/axum/thread_labels_router.rs @@ -126,7 +126,14 @@ pub async fn update_thread_labels_handler { + token_state.inner.fetch_gmail_access_token(&link).await? + } + crate::domain::models::UserProvider::ImapSmtp => String::new(), + }; let result = state .inner diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/db_types.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/db_types.rs index 055642f33a..6bd5f9ef1f 100644 --- a/rust/cloud-storage/email/src/outbound/email_pg_repo/db_types.rs +++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/db_types.rs @@ -150,6 +150,8 @@ impl ThreadPreviewCursorDbRow { #[dg(forward = crate::domain::models::UserProvider)] pub enum DbUserProvider { Gmail, + #[sqlx(rename = "IMAP_SMTP")] + ImapSmtp, } #[derive(Debug, Clone)] diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/link.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/link.rs index f96826c4f8..b17a70b5df 100644 --- a/rust/cloud-storage/email/src/outbound/email_pg_repo/link.rs +++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/link.rs @@ -14,6 +14,7 @@ pub(super) async fn link_by_fusionauth_and_macro_id( ) -> Result, sqlx::Error> { let provider: DbUserProvider = match provider { UserProvider::Gmail => DbUserProvider::Gmail, + UserProvider::ImapSmtp => DbUserProvider::ImapSmtp, }; let db_link = sqlx::query_as!( @@ -47,6 +48,7 @@ pub(super) async fn link_by_fusionauth_email_provider( ) -> Result, sqlx::Error> { let provider: DbUserProvider = match provider { UserProvider::Gmail => DbUserProvider::Gmail, + UserProvider::ImapSmtp => DbUserProvider::ImapSmtp, }; let db_link = sqlx::query_as!( diff --git a/rust/cloud-storage/email_db_client/src/imap/mod.rs b/rust/cloud-storage/email_db_client/src/imap/mod.rs new file mode 100644 index 0000000000..ca1029748e --- /dev/null +++ b/rust/cloud-storage/email_db_client/src/imap/mod.rs @@ -0,0 +1,195 @@ +//! Storage for IMAP/SMTP link connection settings and per-folder sync state. +//! +//! Passwords are stored as AES-256-GCM ciphertext (see +//! `email_utils::credential_crypto`); this module only ever sees the +//! ciphertext bytes. + +use models_email::service::imap::ConnectionSecurity; +use sqlx::PgPool; +use sqlx::types::Uuid; + +#[cfg(test)] +mod test; + +/// Mirrors the `email_connection_security_enum` Postgres enum. +#[derive(sqlx::Type, Debug, Clone, Copy, PartialEq, Eq)] +#[sqlx( + type_name = "email_connection_security_enum", + rename_all = "SCREAMING_SNAKE_CASE" +)] +pub enum DbConnectionSecurity { + SslTls, + Starttls, +} + +impl From for DbConnectionSecurity { + fn from(value: ConnectionSecurity) -> Self { + match value { + ConnectionSecurity::SslTls => DbConnectionSecurity::SslTls, + ConnectionSecurity::Starttls => DbConnectionSecurity::Starttls, + } + } +} + +impl From for ConnectionSecurity { + fn from(value: DbConnectionSecurity) -> Self { + match value { + DbConnectionSecurity::SslTls => ConnectionSecurity::SslTls, + DbConnectionSecurity::Starttls => ConnectionSecurity::Starttls, + } + } +} + +/// Row in `email_imap_smtp_credentials`. Passwords are ciphertext. +#[derive(Debug, Clone)] +pub struct DbImapSmtpCredentials { + pub link_id: Uuid, + pub imap_host: String, + pub imap_port: i32, + pub imap_security: DbConnectionSecurity, + pub imap_username: String, + pub imap_password_ciphertext: Vec, + pub smtp_host: String, + pub smtp_port: i32, + pub smtp_security: DbConnectionSecurity, + pub smtp_username: String, + pub smtp_password_ciphertext: Vec, +} + +/// Row in `email_imap_folder_states`. +#[derive(Debug, Clone)] +pub struct DbImapFolderState { + pub link_id: Uuid, + pub folder: String, + pub uid_validity: i64, + pub last_seen_uid: i64, +} + +/// Inserts or replaces the IMAP/SMTP connection settings for a link. +#[tracing::instrument(skip(executor, credentials), fields(link_id = %credentials.link_id), err)] +pub async fn upsert_credentials<'e, E>( + executor: E, + credentials: &DbImapSmtpCredentials, +) -> anyhow::Result<()> +where + E: sqlx::Executor<'e, Database = sqlx::Postgres>, +{ + sqlx::query!( + r#" + INSERT INTO email_imap_smtp_credentials ( + link_id, + imap_host, imap_port, imap_security, imap_username, imap_password_ciphertext, + smtp_host, smtp_port, smtp_security, smtp_username, smtp_password_ciphertext, + updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NOW()) + ON CONFLICT (link_id) + DO UPDATE SET + imap_host = EXCLUDED.imap_host, + imap_port = EXCLUDED.imap_port, + imap_security = EXCLUDED.imap_security, + imap_username = EXCLUDED.imap_username, + imap_password_ciphertext = EXCLUDED.imap_password_ciphertext, + smtp_host = EXCLUDED.smtp_host, + smtp_port = EXCLUDED.smtp_port, + smtp_security = EXCLUDED.smtp_security, + smtp_username = EXCLUDED.smtp_username, + smtp_password_ciphertext = EXCLUDED.smtp_password_ciphertext, + updated_at = NOW() + "#, + credentials.link_id, + credentials.imap_host, + credentials.imap_port, + credentials.imap_security as _, + credentials.imap_username, + credentials.imap_password_ciphertext, + credentials.smtp_host, + credentials.smtp_port, + credentials.smtp_security as _, + credentials.smtp_username, + credentials.smtp_password_ciphertext, + ) + .execute(executor) + .await?; + + Ok(()) +} + +/// Fetches the IMAP/SMTP connection settings for a link, if any. +#[tracing::instrument(skip(pool), err)] +pub async fn fetch_credentials_by_link_id( + pool: &PgPool, + link_id: Uuid, +) -> anyhow::Result> { + let row = sqlx::query_as!( + DbImapSmtpCredentials, + r#" + SELECT link_id, + imap_host, imap_port, imap_security as "imap_security: _", + imap_username, imap_password_ciphertext, + smtp_host, smtp_port, smtp_security as "smtp_security: _", + smtp_username, smtp_password_ciphertext + FROM email_imap_smtp_credentials + WHERE link_id = $1 + "#, + link_id + ) + .fetch_optional(pool) + .await?; + + Ok(row) +} + +/// Fetches all per-folder sync states for a link. +#[tracing::instrument(skip(pool), err)] +pub async fn fetch_folder_states( + pool: &PgPool, + link_id: Uuid, +) -> anyhow::Result> { + let rows = sqlx::query_as!( + DbImapFolderState, + r#" + SELECT link_id, folder, uid_validity, last_seen_uid + FROM email_imap_folder_states + WHERE link_id = $1 + "#, + link_id + ) + .fetch_all(pool) + .await?; + + Ok(rows) +} + +/// Inserts or updates the sync state for one folder of a link. +#[tracing::instrument(skip(executor), err)] +pub async fn upsert_folder_state<'e, E>( + executor: E, + link_id: Uuid, + folder: &str, + uid_validity: i64, + last_seen_uid: i64, +) -> anyhow::Result<()> +where + E: sqlx::Executor<'e, Database = sqlx::Postgres>, +{ + sqlx::query!( + r#" + INSERT INTO email_imap_folder_states (link_id, folder, uid_validity, last_seen_uid, updated_at) + VALUES ($1, $2, $3, $4, NOW()) + ON CONFLICT (link_id, folder) + DO UPDATE SET + uid_validity = EXCLUDED.uid_validity, + last_seen_uid = EXCLUDED.last_seen_uid, + updated_at = NOW() + "#, + link_id, + folder, + uid_validity, + last_seen_uid, + ) + .execute(executor) + .await?; + + Ok(()) +} diff --git a/rust/cloud-storage/email_db_client/src/imap/test.rs b/rust/cloud-storage/email_db_client/src/imap/test.rs new file mode 100644 index 0000000000..0af4923d89 --- /dev/null +++ b/rust/cloud-storage/email_db_client/src/imap/test.rs @@ -0,0 +1,124 @@ +use super::*; +use macro_db_migrator::MACRO_DB_MIGRATIONS; +use macro_user_id::email::EmailStr; +use macro_user_id::user_id::MacroUserIdStr; +use models_email::email::service::link::{Link, UserProvider}; +use sqlx::{Pool, Postgres}; + +async fn insert_imap_link(pool: &Pool, email: &str) -> anyhow::Result { + let link = Link { + id: macro_uuid::generate_uuid_v7(), + macro_id: MacroUserIdStr::try_from(format!("macro|{email}"))?, + fusionauth_user_id: "22222222-2222-2222-2222-222222222222".to_string(), + email_address: EmailStr::try_from(email.to_string())?, + provider: UserProvider::ImapSmtp, + is_sync_active: true, + created_at: Default::default(), + updated_at: Default::default(), + }; + + let mut tx = pool.begin().await?; + let inserted = crate::links::insert::upsert_link(&mut tx, link).await?; + tx.commit().await?; + Ok(inserted) +} + +fn test_credentials(link_id: sqlx::types::Uuid) -> DbImapSmtpCredentials { + DbImapSmtpCredentials { + link_id, + imap_host: "imap.example.com".to_string(), + imap_port: 993, + imap_security: DbConnectionSecurity::SslTls, + imap_username: "user@example.com".to_string(), + imap_password_ciphertext: vec![1, 2, 3], + smtp_host: "smtp.example.com".to_string(), + smtp_port: 587, + smtp_security: DbConnectionSecurity::Starttls, + smtp_username: "user@example.com".to_string(), + smtp_password_ciphertext: vec![4, 5, 6], + } +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn upsert_and_fetch_credentials_roundtrip(pool: Pool) -> anyhow::Result<()> { + let link = insert_imap_link(&pool, "user@example.com").await?; + + let credentials = test_credentials(link.id); + upsert_credentials(&pool, &credentials).await?; + + let fetched = fetch_credentials_by_link_id(&pool, link.id) + .await? + .expect("credentials should exist"); + assert_eq!(fetched.imap_host, "imap.example.com"); + assert_eq!(fetched.imap_port, 993); + assert_eq!(fetched.imap_security, DbConnectionSecurity::SslTls); + assert_eq!(fetched.imap_password_ciphertext, vec![1, 2, 3]); + assert_eq!(fetched.smtp_security, DbConnectionSecurity::Starttls); + assert_eq!(fetched.smtp_port, 587); + + // Upsert with new values replaces the row. + let mut updated = test_credentials(link.id); + updated.imap_host = "imap2.example.com".to_string(); + updated.imap_password_ciphertext = vec![9, 9]; + upsert_credentials(&pool, &updated).await?; + + let fetched = fetch_credentials_by_link_id(&pool, link.id) + .await? + .expect("credentials should exist"); + assert_eq!(fetched.imap_host, "imap2.example.com"); + assert_eq!(fetched.imap_password_ciphertext, vec![9, 9]); + + Ok(()) +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn fetch_credentials_missing_returns_none(pool: Pool) -> anyhow::Result<()> { + let link = insert_imap_link(&pool, "user@example.com").await?; + assert!( + fetch_credentials_by_link_id(&pool, link.id) + .await? + .is_none() + ); + Ok(()) +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn credentials_cascade_on_link_delete(pool: Pool) -> anyhow::Result<()> { + let link = insert_imap_link(&pool, "user@example.com").await?; + upsert_credentials(&pool, &test_credentials(link.id)).await?; + upsert_folder_state(&pool, link.id, "INBOX", 1, 10).await?; + + crate::links::delete::delete_link_by_id(&pool, link.id).await?; + + assert!( + fetch_credentials_by_link_id(&pool, link.id) + .await? + .is_none() + ); + assert!(fetch_folder_states(&pool, link.id).await?.is_empty()); + Ok(()) +} + +#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")] +async fn folder_state_upsert_and_fetch(pool: Pool) -> anyhow::Result<()> { + let link = insert_imap_link(&pool, "user@example.com").await?; + + upsert_folder_state(&pool, link.id, "INBOX", 100, 5).await?; + upsert_folder_state(&pool, link.id, "Sent", 200, 7).await?; + + let mut states = fetch_folder_states(&pool, link.id).await?; + states.sort_by(|a, b| a.folder.cmp(&b.folder)); + assert_eq!(states.len(), 2); + assert_eq!(states[0].folder, "INBOX"); + assert_eq!(states[0].uid_validity, 100); + assert_eq!(states[0].last_seen_uid, 5); + + // UIDVALIDITY change resets the stored state via upsert. + upsert_folder_state(&pool, link.id, "INBOX", 101, 0).await?; + let states = fetch_folder_states(&pool, link.id).await?; + let inbox = states.iter().find(|s| s.folder == "INBOX").unwrap(); + assert_eq!(inbox.uid_validity, 101); + assert_eq!(inbox.last_seen_uid, 0); + + Ok(()) +} diff --git a/rust/cloud-storage/email_db_client/src/lib.rs b/rust/cloud-storage/email_db_client/src/lib.rs index c1bdc26cd7..eae43249a2 100644 --- a/rust/cloud-storage/email_db_client/src/lib.rs +++ b/rust/cloud-storage/email_db_client/src/lib.rs @@ -2,6 +2,7 @@ pub mod attachments; pub mod backfill; pub mod contacts; pub mod histories; +pub mod imap; pub mod labels; pub mod links; pub mod links_history; diff --git a/rust/cloud-storage/email_db_client/src/links/types.rs b/rust/cloud-storage/email_db_client/src/links/types.rs index 8f65e69093..406de75c08 100644 --- a/rust/cloud-storage/email_db_client/src/links/types.rs +++ b/rust/cloud-storage/email_db_client/src/links/types.rs @@ -9,12 +9,15 @@ use sqlx::{Type, types::Uuid}; #[dg(backward = models_email::email::service::link::UserProvider)] pub enum DbUserProvider { Gmail, + #[sqlx(rename = "IMAP_SMTP")] + ImapSmtp, } impl DbUserProvider { pub fn as_str(&self) -> &'static str { match self { DbUserProvider::Gmail => "GMAIL", + DbUserProvider::ImapSmtp => "IMAP_SMTP", } } } @@ -45,6 +48,7 @@ impl From for DbLink { .to_string(), provider: match service_link.provider { models_email::service::link::UserProvider::Gmail => DbUserProvider::Gmail, + models_email::service::link::UserProvider::ImapSmtp => DbUserProvider::ImapSmtp, }, is_sync_active: service_link.is_sync_active, created_at: service_link.created_at, @@ -74,6 +78,7 @@ impl TryFrom for models_email::email::service::link::Link { email_address: EmailStr::try_from(email_address)?, provider: match provider { DbUserProvider::Gmail => UserProvider::Gmail, + DbUserProvider::ImapSmtp => UserProvider::ImapSmtp, }, is_sync_active, created_at, diff --git a/rust/cloud-storage/email_db_client/src/messages/get.rs b/rust/cloud-storage/email_db_client/src/messages/get.rs index bbe071e73c..6f7bac6285 100644 --- a/rust/cloud-storage/email_db_client/src/messages/get.rs +++ b/rust/cloud-storage/email_db_client/src/messages/get.rs @@ -318,6 +318,38 @@ where Ok(message_id) } +/// Finds the thread containing any message whose global_id (RFC 5322 +/// Message-ID) is in `global_ids`. Used to thread IMAP messages by their +/// References/In-Reply-To chain. +#[tracing::instrument(skip(executor, global_ids), err)] +pub async fn get_thread_id_by_global_ids<'e, E>( + executor: E, + link_id: Uuid, + global_ids: &[String], +) -> anyhow::Result> +where + E: sqlx::Executor<'e, Database = sqlx::Postgres>, +{ + if global_ids.is_empty() { + return Ok(None); + } + + let thread_id = sqlx::query_scalar!( + r#" + SELECT thread_id + FROM email_messages + WHERE link_id = $1 AND global_id = ANY($2) + LIMIT 1 + "#, + link_id, + global_ids + ) + .fetch_optional(executor) + .await?; + + Ok(thread_id) +} + /// fetch draft message and sender contact info from database for sending #[tracing::instrument(skip(pool), err)] pub async fn get_message_to_send( diff --git a/rust/cloud-storage/email_refresh_handler/src/handler.rs b/rust/cloud-storage/email_refresh_handler/src/handler.rs index 70c9a38850..4446ae5dac 100644 --- a/rust/cloud-storage/email_refresh_handler/src/handler.rs +++ b/rust/cloud-storage/email_refresh_handler/src/handler.rs @@ -14,6 +14,8 @@ use sqlx::{Pool, Postgres, Type}; #[sqlx(type_name = "email_user_provider_enum", rename_all = "UPPERCASE")] pub enum DbUserProvider { Gmail, + #[sqlx(rename = "IMAP_SMTP")] + ImapSmtp, } #[tracing::instrument(skip(ctx, _event))] @@ -79,9 +81,55 @@ async fn send_refresh_messages(ctx: &context::Context) -> Result<(), Error> { .ok(); } } + + send_imap_refresh_messages(ctx).await; + Ok(()) } +/// IMAP links have no push channel, so unlike Gmail's daily watch renewal +/// they are refreshed (polled) on every handler invocation. +async fn send_imap_refresh_messages(ctx: &context::Context) { + let provider_filter = DbUserProvider::ImapSmtp; + let link_ids = sqlx::query_scalar!( + r#" + SELECT + id as "link_id" + FROM email_links + WHERE + is_sync_active = TRUE + AND provider = $1 + "#, + provider_filter as _, + ) + .fetch_all(&ctx.db) + .await + .unwrap_or_else(|e| { + tracing::error!(error = ?e, "Error fetching IMAP links for refresh"); + Vec::new() + }); + + if link_ids.is_empty() { + return; + } + + tracing::info!( + "Sending refresh notifications for {} IMAP links", + link_ids.len() + ); + + for link_id in link_ids { + let notif = LinkManagerMessage::Refresh { link_id }; + ctx.sqs_client + .enqueue_link_manager_notification(notif) + .await + .inspect_err(|e| { + tracing::error!(error=?e, link_id=%link_id, "Error enqueueing refresh notification for IMAP link"); + }) + .ok(); + } +} + /// delete unused and inactive links from our database async fn send_delete_messages(ctx: &context::Context) -> Result<(), Error> { let unused_links = fetch_unused_link_ids(&ctx.db, ctx.config.delete_unused_after_days as i32) diff --git a/rust/cloud-storage/email_service/Cargo.toml b/rust/cloud-storage/email_service/Cargo.toml index 1e5bbc9727..dad7a02b15 100644 --- a/rust/cloud-storage/email_service/Cargo.toml +++ b/rust/cloud-storage/email_service/Cargo.toml @@ -91,6 +91,7 @@ filter_ast = { path = "../filter_ast" } frecency = { path = "../frecency", features = ["postgres"] } futures = { workspace = true } gmail_client = { path = "../gmail_client" } +imap_smtp_client = { path = "../imap_smtp_client" } html-escape = "0.2" http-body-util = { workspace = true } infer = "0.19.0" diff --git a/rust/cloud-storage/email_service/src/api/context.rs b/rust/cloud-storage/email_service/src/api/context.rs index 02a496ee6a..897ed03f12 100644 --- a/rust/cloud-storage/email_service/src/api/context.rs +++ b/rust/cloud-storage/email_service/src/api/context.rs @@ -49,4 +49,7 @@ pub(crate) struct ApiContext { pub entity_access_service: Arc, pub email_thread_state: EmailThreadRouterState, pub gmail_token_state: GmailTokenState, + /// Key for encrypting/decrypting stored IMAP/SMTP credentials; `None` + /// when IMAP/SMTP links aren't configured for this deployment. + pub credential_key: Option, } diff --git a/rust/cloud-storage/email_service/src/api/email/attachments/get.rs b/rust/cloud-storage/email_service/src/api/email/attachments/get.rs index 9df516144c..ee69f9086a 100644 --- a/rust/cloud-storage/email_service/src/api/email/attachments/get.rs +++ b/rust/cloud-storage/email_service/src/api/email/attachments/get.rs @@ -128,6 +128,19 @@ pub async fn handler( })?; presigned_request.to_string() } else { + // On-demand fetch is Gmail-only for now: IMAP attachments aren't + // re-downloadable yet (no per-part provider id is stored), so fail + // with a clear message instead of a token error. + if link.provider == models_email::service::link::UserProvider::ImapSmtp { + return Err(( + StatusCode::NOT_IMPLEMENTED, + Json(ErrorResponse { + message: "attachment downloads are not yet supported for IMAP inboxes".into(), + }), + ) + .into_response()); + } + // Object doesn't exist, need to fetch from Gmail and upload. // Use the owning inbox's own token, not the caller's primary inbox token. let gmail_token = email_service::util::gmail::auth::fetch_gmail_access_token_from_link( diff --git a/rust/cloud-storage/email_service/src/api/email/links/imap.rs b/rust/cloud-storage/email_service/src/api/email/links/imap.rs new file mode 100644 index 0000000000..7ea3d8c923 --- /dev/null +++ b/rust/cloud-storage/email_service/src/api/email/links/imap.rs @@ -0,0 +1,260 @@ +//! Connects an arbitrary email server as an inbox via IMAP (receive) and +//! SMTP (send), the "bring your own server" counterpart of the Gmail OAuth +//! init flow. + +use crate::api::ApiContext; +use anyhow::Context; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Json, Response}; +use email_service::util::imap::{encrypt_credentials, require_credential_key}; +use imap_smtp_client::ImapSession; +use macro_user_id::email::EmailStr; +use model::response::ErrorResponse; +use model::user::axum_extractor::MacroUserExtractor; +use models_email::gmail::inbox_sync::{ + ImapPollPayload, InboxSyncOperation, InboxSyncPubsubMessage, +}; +use models_email::service::imap::{ConnectionSecurity, ServerSettings}; +use models_email::service::link; +use std::time::Duration; +use strum_macros::AsRefStr; +use thiserror::Error; +use utoipa::ToSchema; +use uuid::Uuid; + +/// How long we give each of the IMAP/SMTP verification handshakes before +/// telling the user their server is unreachable. +const VERIFY_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, Error, AsRefStr)] +pub enum CreateImapLinkError { + #[error("An inbox for this email address is already connected")] + AlreadyInitialized, + + #[error("Invalid input: {0}")] + BadRequest(String), + + #[error("Could not connect to the IMAP server: {0}")] + ImapVerificationFailed(String), + + #[error("Could not connect to the SMTP server: {0}")] + SmtpVerificationFailed(String), + + #[error("IMAP/SMTP accounts are not enabled for this deployment")] + NotConfigured, + + #[error("Internal error")] + Internal(#[from] anyhow::Error), +} + +impl IntoResponse for CreateImapLinkError { + fn into_response(self) -> Response { + let status_code = match &self { + CreateImapLinkError::AlreadyInitialized + | CreateImapLinkError::BadRequest(_) + | CreateImapLinkError::ImapVerificationFailed(_) + | CreateImapLinkError::SmtpVerificationFailed(_) => StatusCode::BAD_REQUEST, + CreateImapLinkError::NotConfigured => StatusCode::NOT_IMPLEMENTED, + CreateImapLinkError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + }; + + (status_code, self.to_string()).into_response() + } +} + +/// Connection settings for one server (the IMAP or SMTP half). +#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)] +pub struct ServerSettingsInput { + /// Server hostname, e.g. `imap.fastmail.com`. + pub host: String, + /// Server port, e.g. 993 for IMAP over TLS, 465/587 for SMTP. + pub port: u16, + /// How the connection is secured. + pub security: ConnectionSecurity, + /// Login username (usually the email address). + pub username: String, + /// Login password. Many providers require an app-specific password. + pub password: String, +} + +impl ServerSettingsInput { + fn into_settings(self) -> Result { + let host = self.host.trim().to_string(); + if host.is_empty() { + return Err(CreateImapLinkError::BadRequest( + "server host must not be empty".to_string(), + )); + } + if self.port == 0 { + return Err(CreateImapLinkError::BadRequest( + "server port must not be 0".to_string(), + )); + } + if self.username.is_empty() || self.password.is_empty() { + return Err(CreateImapLinkError::BadRequest( + "username and password must not be empty".to_string(), + )); + } + Ok(ServerSettings { + host, + port: self.port, + security: self.security, + username: self.username, + password: self.password, + }) + } +} + +#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)] +pub struct CreateImapLinkRequest { + /// The mailbox address being connected. + pub email_address: String, + /// IMAP (receiving) server settings. + pub imap: ServerSettingsInput, + /// SMTP (sending) server settings. + pub smtp: ServerSettingsInput, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)] +pub struct CreateImapLinkResponse { + /// The email_links row id for the newly connected inbox. + pub link_id: Uuid, +} + +/// Connects an email account on an arbitrary IMAP/SMTP server as an inbox. +/// +/// Verifies both server connections with the supplied credentials before +/// persisting anything; passwords are stored encrypted. On success an initial +/// sync of the inbox is scheduled. +#[utoipa::path( + post, + tag = "Links", + path = "/email/links/imap", + operation_id = "create_imap_link", + request_body = CreateImapLinkRequest, + responses( + (status = 200, body=CreateImapLinkResponse), + (status = 400, body=ErrorResponse), + (status = 401, body=ErrorResponse), + (status = 500, body=ErrorResponse), + (status = 501, body=ErrorResponse), + ) +)] +#[tracing::instrument(skip(ctx, user_extractor, request), fields(user_id=user_extractor.user_context.user_id), err)] +pub async fn create_imap_link_handler( + State(ctx): State, + user_extractor: MacroUserExtractor, + Json(request): Json, +) -> Result { + let MacroUserExtractor { + macro_user_id, + user_context, + .. + } = user_extractor; + + let key = require_credential_key(&ctx.credential_key) + .map_err(|_| CreateImapLinkError::NotConfigured)?; + + let email_address = EmailStr::try_from(request.email_address.trim().to_lowercase()) + .map_err(|e| CreateImapLinkError::BadRequest(format!("invalid email address: {e}")))?; + + let imap = request.imap.into_settings()?; + let smtp = request.smtp.into_settings()?; + + let existing = email_db_client::links::get::fetch_link_by_email( + &ctx.db, + email_address.0.as_ref(), + link::UserProvider::ImapSmtp, + ) + .await + .context("failed to check for existing link")?; + if existing.is_some() { + return Err(CreateImapLinkError::AlreadyInitialized); + } + + // Validate both connections before persisting anything so the user gets + // immediate, specific feedback on which half is misconfigured. + tokio::time::timeout(VERIFY_TIMEOUT, ImapSession::verify(&imap)) + .await + .map_err(|_| { + CreateImapLinkError::ImapVerificationFailed(format!( + "timed out connecting to {}:{}", + imap.host, imap.port + )) + })? + .map_err(|e| CreateImapLinkError::ImapVerificationFailed(format!("{e:#}")))?; + + tokio::time::timeout(VERIFY_TIMEOUT, imap_smtp_client::smtp::verify(&smtp)) + .await + .map_err(|_| { + CreateImapLinkError::SmtpVerificationFailed(format!( + "timed out connecting to {}:{}", + smtp.host, smtp.port + )) + })? + .map_err(|e| CreateImapLinkError::SmtpVerificationFailed(format!("{e:#}")))?; + + let new_link = link::Link { + id: macro_uuid::generate_uuid_v7(), + macro_id: macro_user_id, + fusionauth_user_id: user_context.fusion_user_id.clone(), + email_address, + provider: link::UserProvider::ImapSmtp, + is_sync_active: true, + created_at: Default::default(), + updated_at: Default::default(), + }; + + let mut tx = ctx + .db + .begin() + .await + .context("failed to begin link transaction")?; + + let inserted = email_db_client::links::insert::upsert_link(&mut tx, new_link) + .await + .context("failed to upsert link")?; + + let credentials = encrypt_credentials(key, inserted.id, &imap, &smtp) + .context("failed to encrypt credentials")?; + email_db_client::imap::upsert_credentials(&mut *tx, &credentials) + .await + .context("failed to store credentials")?; + + tx.commit() + .await + .context("failed to commit link transaction")?; + + // Record link creation in history table for tracking (best-effort) + email_db_client::links_history::insert::insert_email_link_history( + &ctx.db, + inserted.id, + &inserted.fusionauth_user_id, + inserted.email_address.0.as_ref(), + inserted.provider, + ) + .await + .inspect_err(|e| { + tracing::error!(error=?e, link_id=?inserted.id, "Failed to insert email link history"); + }) + .ok(); + + // Seed the inbox with recent mail; subsequent polls are scheduled by the + // link refresh cycle. + ctx.sqs_client + .enqueue_gmail_inbox_sync_notification(InboxSyncPubsubMessage { + link_id: inserted.id, + operation: InboxSyncOperation::ImapPoll(ImapPollPayload { initial: true }), + }) + .await + .context("failed to enqueue initial IMAP sync")?; + + Ok(( + StatusCode::OK, + Json(CreateImapLinkResponse { + link_id: inserted.id, + }), + ) + .into_response()) +} diff --git a/rust/cloud-storage/email_service/src/api/email/links/list.rs b/rust/cloud-storage/email_service/src/api/email/links/list.rs index efe740000a..1947e69f30 100644 --- a/rust/cloud-storage/email_service/src/api/email/links/list.rs +++ b/rust/cloud-storage/email_service/src/api/email/links/list.rs @@ -109,7 +109,11 @@ pub async fn list_links_handler( .map_err(ListLinksError::DatabaseError)? .and_then(|contact| contact.photo_url); - let signature = if query_params.include_signature { + // Signatures come from the Gmail settings API; IMAP servers have + // no equivalent to fetch from. + let signature = if query_params.include_signature + && link.provider == models_email::service::link::UserProvider::Gmail + { let access_token = fetch_gmail_access_token_from_link( &link, &ctx.redis_client, diff --git a/rust/cloud-storage/email_service/src/api/email/links/mod.rs b/rust/cloud-storage/email_service/src/api/email/links/mod.rs index 6eb4469850..bca4699a31 100644 --- a/rust/cloud-storage/email_service/src/api/email/links/mod.rs +++ b/rust/cloud-storage/email_service/src/api/email/links/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod access; pub(crate) mod delete; +pub(crate) mod imap; pub(crate) mod list; pub(crate) mod resync; @@ -10,6 +11,7 @@ use axum::routing::{delete, get, post}; pub fn router() -> Router { Router::new() .route("/", get(list::list_links_handler)) + .route("/imap", post(imap::create_imap_link_handler)) .route("/{link_id}", delete(delete::delete_link_handler)) .route("/{link_id}/resync", post(resync::resync_link_handler)) } diff --git a/rust/cloud-storage/email_service/src/api/email/links/resync.rs b/rust/cloud-storage/email_service/src/api/email/links/resync.rs index 26250ce059..f3b02c2fb6 100644 --- a/rust/cloud-storage/email_service/src/api/email/links/resync.rs +++ b/rust/cloud-storage/email_service/src/api/email/links/resync.rs @@ -9,6 +9,9 @@ use model::user::UserContext; use models_email::email::service::backfill::{ BackfillJobStatus, BackfillOperation, BackfillPubsubMessage, InitPayload, JobScopedPayload, }; +use models_email::gmail::inbox_sync::{ + ImapPollPayload, InboxSyncOperation, InboxSyncPubsubMessage, +}; use utoipa::ToSchema; use uuid::Uuid; @@ -16,8 +19,10 @@ use uuid::Uuid; #[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)] pub struct ResyncResponse { /// The backfill job driving the (re-)sync. Either the freshly enqueued job or - /// the one already in progress. - pub backfill_job_id: Uuid, + /// the one already in progress. Absent for IMAP/SMTP links, whose resync is a + /// direct poll of the server rather than a tracked backfill job. + #[serde(skip_serializing_if = "Option::is_none")] + pub backfill_job_id: Option, /// True when a backfill was already running and this call was a no-op. pub already_in_progress: bool, } @@ -50,13 +55,31 @@ pub async fn resync_link_handler( ) -> Result { let (link, _access) = authorize_inbox_access(&ctx, &user_context.user_id, link_id).await?; + // IMAP links have no backfill pipeline; a resync is an immediate poll of + // the server, so there's no job id to report. + if link.provider == models_email::service::link::UserProvider::ImapSmtp { + ctx.sqs_client + .enqueue_gmail_inbox_sync_notification(InboxSyncPubsubMessage { + link_id: link.id, + operation: InboxSyncOperation::ImapPoll(ImapPollPayload { initial: false }), + }) + .await + .context("failed to enqueue IMAP poll for resync")?; + + return Ok(Json(ResyncResponse { + backfill_job_id: None, + already_in_progress: false, + }) + .into_response()); + } + if let Some(active) = email_db_client::backfill::job::get::get_active_backfill_job(&ctx.db, link.id) .await .context("failed to check active backfill job")? { return Ok(Json(ResyncResponse { - backfill_job_id: active.id, + backfill_job_id: Some(active.id), already_in_progress: true, }) .into_response()); @@ -98,7 +121,7 @@ pub async fn resync_link_handler( } Ok(Json(ResyncResponse { - backfill_job_id: backfill_job.id, + backfill_job_id: Some(backfill_job.id), already_in_progress: false, }) .into_response()) diff --git a/rust/cloud-storage/email_service/src/api/middleware/link.rs b/rust/cloud-storage/email_service/src/api/middleware/link.rs index d075bfa681..9a5f598ceb 100644 --- a/rust/cloud-storage/email_service/src/api/middleware/link.rs +++ b/rust/cloud-storage/email_service/src/api/middleware/link.rs @@ -13,6 +13,7 @@ pub(in crate::api) async fn attach_link_context( ) -> Result { let provider = match link.provider.as_str() { "GMAIL" => models_email::email::service::link::UserProvider::Gmail, + "IMAP_SMTP" => models_email::email::service::link::UserProvider::ImapSmtp, other => { tracing::error!(provider = other, "unknown provider in link"); return Err(( diff --git a/rust/cloud-storage/email_service/src/api/swagger.rs b/rust/cloud-storage/email_service/src/api/swagger.rs index b9c6edd241..471e17e96c 100644 --- a/rust/cloud-storage/email_service/src/api/swagger.rs +++ b/rust/cloud-storage/email_service/src/api/swagger.rs @@ -15,6 +15,9 @@ use crate::api::email::drafts::add_forwarded_attachment::{ use crate::api::email::init::InitResponse; use crate::api::email::labels::create::CreateLabelRequest; use crate::api::email::labels::create::CreateLabelResponse; +use crate::api::email::links::imap::{ + CreateImapLinkRequest, CreateImapLinkResponse, ServerSettingsInput, +}; use crate::api::email::links::list::ListLinksResponse; use crate::api::email::links::resync::ResyncResponse; use crate::api::email::messages::labels::{UpdateLabelBatchRequest, UpdateLabelBatchResponse}; @@ -88,6 +91,7 @@ use utoipa::OpenApi; email::links::list::list_links_handler, email::links::delete::delete_link_handler, email::links::resync::resync_link_handler, + email::links::imap::create_imap_link_handler, email::labels::create::handler, email::labels::delete::handler, inbound::axum::list_labels_router::list_labels_handler, @@ -154,6 +158,9 @@ use utoipa::OpenApi; Link, SyncStatus, ResyncResponse, + CreateImapLinkRequest, + CreateImapLinkResponse, + ServerSettingsInput, Settings, // Contact types ListContactsResponse, diff --git a/rust/cloud-storage/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/rust/cloud-storage/email_service/src/bin/pubsub_workers/pubsub_workers.rs index 3dc73ebe11..8f19b590d3 100644 --- a/rust/cloud-storage/email_service/src/bin/pubsub_workers/pubsub_workers.rs +++ b/rust/cloud-storage/email_service/src/bin/pubsub_workers/pubsub_workers.rs @@ -36,6 +36,10 @@ async fn main() -> anyhow::Result<()> { let config = Config::from_env(cloudfront_signer_private_key) .context("expected to be able to generate config")?; + let credential_key = + email_service::util::imap::parse_credential_key(&config.email_credentials_encryption_key) + .context("invalid EMAIL_CREDENTIALS_ENCRYPTION_KEY")?; + let auth_service_secret_key = match config.environment { Environment::Local => config.auth_service_secret_key.clone(), _ => secretsmanager_client @@ -308,6 +312,7 @@ async fn main() -> anyhow::Result<()> { let dss_client_inbox_sync = dss_client.clone(); let system_properties_service_inbox_sync = system_properties_service.clone(); let crm_service_inbox_sync = crm_service.clone(); + let credential_key_inbox_sync = credential_key.clone(); tokio::spawn(async move { email_service::pubsub::inbox_sync::worker::run_worker( db_inbox_sync, @@ -325,6 +330,7 @@ async fn main() -> anyhow::Result<()> { crm_service_inbox_sync, config.notifications_enabled, false, + credential_key_inbox_sync, ) .await; }); @@ -348,6 +354,7 @@ async fn main() -> anyhow::Result<()> { let dss_client_inbox_sync = dss_client.clone(); let system_properties_service_inbox_sync = system_properties_service.clone(); let crm_service_inbox_sync = crm_service.clone(); + let credential_key_inbox_sync = credential_key.clone(); tokio::spawn(async move { email_service::pubsub::inbox_sync::worker::run_worker( db_inbox_sync, @@ -365,6 +372,7 @@ async fn main() -> anyhow::Result<()> { crm_service_inbox_sync, config.notifications_enabled, true, + credential_key_inbox_sync, ) .await; }); @@ -381,6 +389,7 @@ async fn main() -> anyhow::Result<()> { let gmail_client_gmail_ops = gmail_client.clone(); let auth_service_client_gmail_ops = auth_service_client.clone(); let redis_client_gmail_ops = redis_client.clone(); + let credential_key_gmail_ops = credential_key.clone(); tokio::spawn(async move { email_service::pubsub::gmail_ops::worker::run_worker( db_gmail_ops, @@ -390,6 +399,7 @@ async fn main() -> anyhow::Result<()> { auth_service_client_gmail_ops, redis_client_gmail_ops, false, + credential_key_gmail_ops, ) .await; }); @@ -406,6 +416,7 @@ async fn main() -> anyhow::Result<()> { let gmail_client_gmail_ops = gmail_client.clone(); let auth_service_client_gmail_ops = auth_service_client.clone(); let redis_client_gmail_ops = redis_client.clone(); + let credential_key_gmail_ops = credential_key.clone(); tokio::spawn(async move { email_service::pubsub::gmail_ops::worker::run_worker( db_gmail_ops, @@ -415,6 +426,7 @@ async fn main() -> anyhow::Result<()> { auth_service_client_gmail_ops, redis_client_gmail_ops, true, + credential_key_gmail_ops, ) .await; }); @@ -489,6 +501,7 @@ async fn main() -> anyhow::Result<()> { let redis_client_scheduled = redis_client.clone(); let s3_client_scheduled = s3_client.clone(); let attachment_bucket_scheduled = config.attachment_bucket.clone(); + let credential_key_scheduled = credential_key.clone(); // send scheduled emails tokio::spawn(async move { email_service::pubsub::scheduled::worker::run_worker( @@ -499,6 +512,7 @@ async fn main() -> anyhow::Result<()> { redis_client_scheduled, s3_client_scheduled, attachment_bucket_scheduled, + credential_key_scheduled, ) .await; }); diff --git a/rust/cloud-storage/email_service/src/config.rs b/rust/cloud-storage/email_service/src/config.rs index 28becc2b09..78d958294e 100644 --- a/rust/cloud-storage/email_service/src/config.rs +++ b/rust/cloud-storage/email_service/src/config.rs @@ -154,6 +154,11 @@ pub struct Config { // How long presigned urls should be valid for attachments pub email_service_presigned_url_ttl_secs: u64, + + /// Base64-encoded 32-byte AES-256 key used to encrypt stored IMAP/SMTP + /// passwords at rest. Empty when IMAP/SMTP links are not enabled for this + /// deployment; attempts to create or use such links then fail cleanly. + pub email_credentials_encryption_key: String, } env_var! { pub struct EmailServiceCloudfrontSignerPrivateKey; } @@ -343,6 +348,10 @@ impl Config { .parse::() .unwrap(); + let email_credentials_encryption_key = + std::env::var(email_utils::credential_crypto::ENCRYPTION_KEY_ENV_VAR) + .unwrap_or_default(); + Ok(Config { macro_db_url: database_url, port, @@ -391,6 +400,7 @@ impl Config { email_service_cloudfront_signer_public_key_id, email_service_cloudfront_signer_private_key, email_service_presigned_url_ttl_secs, + email_credentials_encryption_key, }) } } diff --git a/rust/cloud-storage/email_service/src/convert/imap.rs b/rust/cloud-storage/email_service/src/convert/imap.rs new file mode 100644 index 0000000000..994ae2efda --- /dev/null +++ b/rust/cloud-storage/email_service/src/convert/imap.rs @@ -0,0 +1,289 @@ +//! Maps raw RFC 5322 messages fetched over IMAP to service layer models. +//! +//! The Gmail path gets a structured `MessageResource` from the API; IMAP only +//! hands us the raw message bytes plus folder/flag context, so this module +//! parses the MIME structure with `mailparse` and synthesizes the +//! Gmail-compatible system labels (`INBOX`, `SENT`, `UNREAD`, ...) the rest of +//! the email stack keys off. + +use crate::convert::message::parse_address_header; +use crate::convert::sanitizer::sanitize_email_html; +use anyhow::{Context, Result}; +use chrono::{TimeZone, Utc}; +use imap_smtp_client::FetchedMessage; +use macro_uuid::generate_uuid_v7; +use mailparse::{MailHeaderMap, ParsedMail}; +use models_email::email::service; +use models_email::gmail::Header; +use models_email::gmail::labels::SystemLabelID; +use uuid::Uuid; + +#[cfg(test)] +mod test; + +const SNIPPET_MAX_CHARS: usize = 200; + +/// A message mapped from IMAP, along with the threading hints needed to +/// attach it to an existing conversation. +#[derive(Debug)] +pub struct MappedImapMessage { + /// The mapped service message. `provider_id` is the message's RFC 5322 + /// `Message-ID` (bracketed), which doubles as its `global_id`; + /// `provider_thread_id` is left unset for the caller to fill in after + /// thread resolution. + pub message: service::message::Message, + /// Message-IDs (bracketed) of ancestors, from `References` and + /// `In-Reply-To`, used to find the thread this message belongs to. + pub ancestor_global_ids: Vec, +} + +/// Maps a raw message fetched from an IMAP folder to the service model. +/// +/// `is_sent_folder` marks messages from the sent mailbox so they get the +/// `SENT` label instead of `INBOX`. +#[tracing::instrument(skip(fetched), err)] +pub fn map_imap_message_to_service( + fetched: &FetchedMessage, + link_id: Uuid, + folder: &str, + uid_validity: u32, + is_sent_folder: bool, +) -> Result { + let parsed = mailparse::parse_mail(&fetched.body).context("failed to parse RFC 5322 body")?; + + let all_headers: Vec
= parsed + .headers + .iter() + .map(|h| Header { + name: h.get_key(), + value: h.get_value(), + }) + .collect(); + + let find_header = |name: &str| -> Option<&str> { + all_headers + .iter() + .find(|h| h.name.eq_ignore_ascii_case(name)) + .map(|h| h.value.as_str()) + }; + + // Some messages (rarely) lack a Message-ID; synthesize a stable one from + // the folder coordinates so dedupe still works across polls. + let global_id = find_header("Message-ID") + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| format!("", fetched.uid)); + + let ancestor_global_ids = parse_ancestor_ids( + find_header("References"), + find_header("In-Reply-To"), + &global_id, + ); + + let subject = find_header("Subject").map(str::to_string); + + let sent_at = find_header("Date") + .and_then(|d| mailparse::dateparse(d).ok()) + .and_then(|ts| Utc.timestamp_opt(ts, 0).single()); + let internal_date_ts = fetched.internal_date.or(sent_at); + + let from = find_header("From") + .and_then(|v| parse_address_header(v).into_iter().next()) + .map(|(name, email)| service::address::ContactInfo { + email, + name, + photo_url: None, + }); + + let parse_contacts = |name: &str| -> Vec { + find_header(name) + .map(|v| { + parse_address_header(v) + .into_iter() + .map(|(name, email)| service::address::ContactInfo { + email, + name, + photo_url: None, + }) + .collect() + }) + .unwrap_or_default() + }; + + let bodies = extract_bodies(&parsed); + + let mut provider_label_ids = vec![if is_sent_folder { + SystemLabelID::Sent.as_str().to_string() + } else { + SystemLabelID::Inbox.as_str().to_string() + }]; + if !fetched.seen { + provider_label_ids.push(SystemLabelID::Unread.as_str().to_string()); + } + if fetched.flagged { + provider_label_ids.push(SystemLabelID::Starred.as_str().to_string()); + } + if fetched.draft { + provider_label_ids.push(SystemLabelID::Draft.as_str().to_string()); + } + + let labels = provider_label_ids + .into_iter() + .map(|id| service::label::Label { + id: None, + link_id, + provider_label_id: id, + name: None, + created_at: Default::default(), + message_list_visibility: None, + label_list_visibility: None, + type_: None, + }) + .collect(); + + let snippet = bodies.text.as_deref().map(make_snippet).unwrap_or_default(); + + let message = service::message::Message { + db_id: generate_uuid_v7(), + provider_id: Some(global_id.clone()), + thread_db_id: generate_uuid_v7(), + // Filled in by the caller once the thread is resolved. + provider_thread_id: None, + replying_to_id: None, // gets generated later, once message has been inserted + global_id: Some(global_id), + link_id, + subject, + snippet: Some(snippet), + provider_history_id: None, + internal_date_ts, + sent_at: sent_at.or(internal_date_ts), + size_estimate: Some(fetched.body.len() as i64), + is_read: fetched.seen, + is_starred: fetched.flagged, + is_sent: is_sent_folder, + is_draft: fetched.draft, + scheduled_send_time: None, + has_attachments: !bodies.attachments.is_empty(), + from, + to: parse_contacts("To"), + cc: parse_contacts("Cc"), + bcc: parse_contacts("Bcc"), + labels, + body_text: bodies.text, + body_html_sanitized: bodies.html_sanitized, + body_macro: None, + attachments: bodies.attachments, + attachments_draft: Vec::new(), + attachments_forwarded: Vec::new(), + headers_json: Some(serde_json::to_value(all_headers)?), + created_at: Utc::now(), + updated_at: Utc::now(), + }; + + Ok(MappedImapMessage { + message, + ancestor_global_ids, + }) +} + +/// Collects bracketed ancestor Message-IDs from `References` and +/// `In-Reply-To`, oldest first, excluding the message's own id. +fn parse_ancestor_ids( + references: Option<&str>, + in_reply_to: Option<&str>, + own_global_id: &str, +) -> Vec { + let mut ids: Vec = Vec::new(); + for source in [references, in_reply_to].into_iter().flatten() { + for token in source.split_whitespace() { + let token = token.trim(); + if token.starts_with('<') + && token.ends_with('>') + && token != own_global_id + && !ids.iter().any(|existing| existing == token) + { + ids.push(token.to_string()); + } + } + } + ids +} + +#[derive(Default)] +struct ExtractedBodies { + text: Option, + html_sanitized: Option, + attachments: Vec, +} + +/// Walks the MIME tree collecting the first text/plain and text/html bodies +/// plus attachment metadata, mirroring `parse_gmail_payload`'s traversal. +fn extract_bodies(root: &ParsedMail) -> ExtractedBodies { + let mut out = ExtractedBodies::default(); + let mut stack: Vec<&ParsedMail> = vec![root]; + + while let Some(part) = stack.pop() { + let mime_type = part.ctype.mimetype.to_lowercase(); + let is_multipart = mime_type.starts_with("multipart/"); + + let disposition = part.get_content_disposition(); + let is_attachment_disposition = + disposition.disposition == mailparse::DispositionType::Attachment; + let filename = disposition.params.get("filename").cloned(); + + let is_inline_non_text = disposition.disposition == mailparse::DispositionType::Inline + && !mime_type.starts_with("text/") + && !is_multipart; + let is_regular_attachment = is_attachment_disposition + || (filename.is_some() && !is_multipart && !mime_type.starts_with("text/")); + + if !is_multipart { + if is_inline_non_text || is_regular_attachment { + let size_bytes = part.get_body_raw().map(|b| b.len() as i64).ok(); + let content_id = part.headers.get_first_value("Content-ID"); + + out.attachments.push(service::attachment::Attachment { + db_id: generate_uuid_v7(), + // IMAP has no per-attachment provider id; downloads + // re-fetch the message by Message-ID instead. + provider_id: None, + data_url: None, + filename: filename.map(lowercase_extension), + mime_type: Some(part.ctype.mimetype.clone()), + size_bytes, + content_id, + sfs_id: None, + }); + } else if mime_type == "text/plain" && out.text.is_none() { + if let Ok(body) = part.get_body() { + out.text = Some(body); + } + } else if mime_type == "text/html" + && out.html_sanitized.is_none() + && let Ok(body) = part.get_body() + { + out.html_sanitized = Some(sanitize_email_html(&body)); + } + } + + for sub_part in part.subparts.iter().rev() { + stack.push(sub_part); + } + } + + out +} + +fn lowercase_extension(filename: String) -> String { + if let Some((base, ext)) = filename.rsplit_once('.') + && !ext.is_empty() + { + return format!("{}.{}", base, ext.to_lowercase()); + } + filename +} + +fn make_snippet(text: &str) -> String { + let collapsed = text.split_whitespace().collect::>().join(" "); + collapsed.chars().take(SNIPPET_MAX_CHARS).collect() +} diff --git a/rust/cloud-storage/email_service/src/convert/imap/test.rs b/rust/cloud-storage/email_service/src/convert/imap/test.rs new file mode 100644 index 0000000000..0ad88ff783 --- /dev/null +++ b/rust/cloud-storage/email_service/src/convert/imap/test.rs @@ -0,0 +1,175 @@ +use super::*; + +fn fetched(body: &str, seen: bool) -> FetchedMessage { + FetchedMessage { + uid: 7, + seen, + flagged: false, + draft: false, + internal_date: Some(Utc.with_ymd_and_hms(2026, 6, 1, 12, 0, 0).unwrap()), + body: body.replace('\n', "\r\n").into_bytes(), + } +} + +const SIMPLE_MESSAGE: &str = "\ +Message-ID: +Date: Mon, 1 Jun 2026 08:00:00 -0400 +From: Alice Sender +To: Bob Recipient +Cc: carol@example.org +Subject: Hello world +References: +In-Reply-To: +Content-Type: text/plain; charset=utf-8 + +This is the body of the message. +"; + +#[test] +fn maps_simple_plaintext_message() { + let link_id = uuid::Uuid::nil(); + let mapped = + map_imap_message_to_service(&fetched(SIMPLE_MESSAGE, false), link_id, "INBOX", 1, false) + .unwrap(); + + let m = &mapped.message; + assert_eq!(m.global_id.as_deref(), Some("")); + assert_eq!(m.provider_id.as_deref(), Some("")); + assert_eq!(m.subject.as_deref(), Some("Hello world")); + assert_eq!(m.from.as_ref().unwrap().email, "alice@example.com"); + assert_eq!( + m.from.as_ref().unwrap().name.as_deref(), + Some("Alice Sender") + ); + assert_eq!(m.to.len(), 1); + assert_eq!(m.to[0].email, "bob@example.net"); + assert_eq!(m.cc.len(), 1); + assert!(!m.is_read); + assert!(!m.is_sent); + assert!(!m.is_draft); + assert!( + m.body_text + .as_deref() + .unwrap() + .contains("body of the message") + ); + assert_eq!( + m.snippet.as_deref(), + Some("This is the body of the message.") + ); + assert!(m.sent_at.is_some()); + + // Ancestors come from References + In-Reply-To, deduped, oldest first. + assert_eq!( + mapped.ancestor_global_ids, + vec!["", ""] + ); + + // Unseen inbox message gets INBOX + UNREAD labels. + let label_ids: Vec<&str> = m + .labels + .iter() + .map(|l| l.provider_label_id.as_str()) + .collect(); + assert_eq!(label_ids, vec!["INBOX", "UNREAD"]); +} + +#[test] +fn sent_folder_message_gets_sent_label_and_is_read() { + let mapped = map_imap_message_to_service( + &fetched(SIMPLE_MESSAGE, true), + uuid::Uuid::nil(), + "Sent", + 1, + true, + ) + .unwrap(); + + let m = &mapped.message; + assert!(m.is_sent); + assert!(m.is_read); + let label_ids: Vec<&str> = m + .labels + .iter() + .map(|l| l.provider_label_id.as_str()) + .collect(); + assert_eq!(label_ids, vec!["SENT"]); +} + +const MULTIPART_MESSAGE: &str = "\ +Message-ID: +From: alice@example.com +To: bob@example.net +Subject: With attachment +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary=\"outer\" + +--outer +Content-Type: multipart/alternative; boundary=\"inner\" + +--inner +Content-Type: text/plain; charset=utf-8 + +plain body +--inner +Content-Type: text/html; charset=utf-8 + +

html body

+--inner-- +--outer +Content-Type: application/pdf; name=\"Report.PDF\" +Content-Disposition: attachment; filename=\"Report.PDF\" +Content-Transfer-Encoding: base64 + +aGVsbG8= +--outer-- +"; + +#[test] +fn maps_multipart_with_attachment() { + let mapped = map_imap_message_to_service( + &fetched(MULTIPART_MESSAGE, true), + uuid::Uuid::nil(), + "INBOX", + 1, + false, + ) + .unwrap(); + + let m = &mapped.message; + assert_eq!(m.body_text.as_deref().map(str::trim), Some("plain body")); + let html = m.body_html_sanitized.as_deref().unwrap(); + assert!(html.contains("html body")); + assert!( + !html.contains("