From c0909ceac94d8f457748d3d8b8d714e3f7e826c7 Mon Sep 17 00:00:00 2001 From: fegloff Date: Sun, 2 Feb 2025 23:28:03 -0500 Subject: [PATCH 1/8] update openai prices + add o3-mini model --- src/modules/llms/utils/llmsData.ts | 77 ++++++++++++++++++------------ 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/src/modules/llms/utils/llmsData.ts b/src/modules/llms/utils/llmsData.ts index f20a71b..5d17722 100644 --- a/src/modules/llms/utils/llmsData.ts +++ b/src/modules/llms/utils/llmsData.ts @@ -102,26 +102,26 @@ export const llmData: LLMData = { commands: ['gpto', 'ask', 'chat', 'gpt', 'a'], prefix: ['a. ', '. '], apiSpec: 'https://platform.openai.com/docs/models/gpt-4o', - inputPrice: 0.005, - outputPrice: 0.0015, + inputPrice: 0.0025, + outputPrice: 0.01, maxContextTokens: 128000, chargeType: 'TOKEN', stream: true }, - 'gpt-4': { - provider: 'openai', - name: 'gpt-4', - fullName: 'GPT-4', - botName: 'OpenAIBot', - version: 'gpt-4', - commands: ['gpt4'], - apiSpec: 'https://openai.com/index/gpt-4/', - inputPrice: 0.03, - outputPrice: 0.06, - maxContextTokens: 8192, - chargeType: 'TOKEN', - stream: true - }, + // 'gpt-4': { + // provider: 'openai', + // name: 'gpt-4', + // fullName: 'GPT-4', + // botName: 'OpenAIBot', + // version: 'gpt-4', + // commands: ['gpt4'], + // apiSpec: 'https://openai.com/index/gpt-4/', + // inputPrice: 0.03, + // outputPrice: 0.06, + // maxContextTokens: 8192, + // chargeType: 'TOKEN', + // stream: true + // }, 'gpt-35-turbo': { provider: 'openai', name: 'gpt-35-turbo', @@ -130,8 +130,8 @@ export const llmData: LLMData = { version: 'gpt-3.5-turbo', commands: ['ask35'], apiSpec: 'https://platform.openai.com/docs/models/gpt-3-5-turbo', - inputPrice: 0.0015, - outputPrice: 0.002, + inputPrice: 0.003, + outputPrice: 0.006, maxContextTokens: 4000, chargeType: 'TOKEN', stream: true @@ -162,23 +162,38 @@ export const llmData: LLMData = { apiSpec: 'https://platform.openai.com/docs/models/o1', inputPrice: 0.015, outputPrice: 0.06, - maxContextTokens: 128000, + maxContextTokens: 200000, chargeType: 'TOKEN', stream: false }, - 'o1-mini': { + // 'o1-mini': { + // provider: 'openai', + // name: 'o1-mini', + // fullName: 'O1 Mini', + // botName: 'OpenAIBot', + // version: 'o1-mini', + // commands: ['omini', 'o1m'], + // apiSpec: 'https://platform.openai.com/docs/models/o1', + // inputPrice: 0.0011, + // outputPrice: 0.0044, + // maxContextTokens: 128000, + // chargeType: 'TOKEN', + // stream: true + // }, + 'o3-mini': { provider: 'openai', - name: 'o1-mini', - fullName: 'O1 Mini', + name: 'o3-mini', + fullName: 'O3 Mini', botName: 'OpenAIBot', - version: 'o1-mini-2024-09-12', - commands: ['omini'], + version: 'o3-mini', + commands: ['omini3', 'o3m', 'r'], + prefix: ['r. '], apiSpec: 'https://platform.openai.com/docs/models/o1', - inputPrice: 0.003, - outputPrice: 0.012, - maxContextTokens: 128000, + inputPrice: 0.0011, + outputPrice: 0.0044, + maxContextTokens: 200000, chargeType: 'TOKEN', - stream: false + stream: true }, grok: { provider: 'xai', // using grok through claude api @@ -207,8 +222,8 @@ export const llmData: LLMData = { prefix: ['i. ', ', ', 'd. '], apiSpec: 'https://openai.com/index/dall-e-3/', price: { - '1024x1024': 0.8, - '1024x1792': 0.12, + '1024x1024': 0.04, + '1024x1792': 0.08, '1792x1024': 0.12 } }, @@ -234,7 +249,7 @@ export const llmData: LLMData = { temperature: config.openAi.dalle.completions.temperature, max_completion_tokens: +config.openAi.chatGpt.maxTokens }, - modelOverrides: { o1: { temperature: 1 } } // uses model name, not model version + modelOverrides: { o1: { temperature: 1 }, 'o3-mini': { temperature: undefined } } // uses model name, not model version }, claude: { defaultParameters: { From 1072c6f8a3fa5a21a6a186e4a1caf27df53823b2 Mon Sep 17 00:00:00 2001 From: fegloff Date: Mon, 10 Feb 2025 18:37:03 -0500 Subject: [PATCH 2/8] add deepseek bot + fix deep seek streaming service --- src/bot.ts | 3 + src/modules/llms/api/deepseek.ts | 128 +++++++++++++++++++++++++++++ src/modules/llms/api/openai.ts | 2 +- src/modules/llms/deepSeekBot.ts | 82 ++++++++++++++++++ src/modules/llms/utils/llmsData.ts | 21 +++++ src/modules/llms/utils/types.ts | 2 +- 6 files changed, 236 insertions(+), 2 deletions(-) create mode 100644 src/modules/llms/api/deepseek.ts create mode 100644 src/modules/llms/deepSeekBot.ts diff --git a/src/bot.ts b/src/bot.ts index b150b6e..3d13ce2 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -60,6 +60,7 @@ import { llmModelManager } from './modules/llms/utils/llmModelsManager' import { HmnyBot } from './modules/hmny' import { LumaBot } from './modules/llms/lumaBot' import { XaiBot } from './modules/llms/xaiBot' +import { DeepSeekBot } from './modules/llms/deepSeekBot' Events.EventEmitter.defaultMaxListeners = 30 @@ -216,6 +217,7 @@ const lumaBot = new LumaBot(payments) const claudeBot = new ClaudeBot(payments) const vertexBot = new VertexBot(payments, [llamaAgent]) const xaiBot = new XaiBot(payments) +const deepSeekBot = new DeepSeekBot(payments) const oneCountryBot = new OneCountryBot(payments) const translateBot = new TranslateBot() const telegramPayments = new TelegramPayments(payments) @@ -347,6 +349,7 @@ const PayableBots: Record = { vertexBot: { bot: vertexBot }, lumaBot: { bot: lumaBot }, aixBot: { bot: xaiBot }, + deepSeekBot: { bot: deepSeekBot }, openAiBot: { enabled: (ctx: OnMessageContext) => ctx.session.dalle.isEnabled, bot: openAiBot diff --git a/src/modules/llms/api/deepseek.ts b/src/modules/llms/api/deepseek.ts new file mode 100644 index 0000000..d32df25 --- /dev/null +++ b/src/modules/llms/api/deepseek.ts @@ -0,0 +1,128 @@ +import axios, { type AxiosResponse } from 'axios' +import { type Readable } from 'stream' +import { GrammyError } from 'grammy' +import { pino } from 'pino' + +import config from '../../../config' +import { type OnCallBackQueryData, type OnMessageContext, type ChatConversation } from '../../types' +import { type LlmCompletion } from './llmApi' +import { headersStream } from './helper' +import { LlmModelsEnum } from '../utils/llmModelsManager' +import { type ModelParameters } from '../utils/types' +import { prepareConversation } from './openai' + +const logger = pino({ + name: 'deepSeek - llmsBot', + transport: { + target: 'pino-pretty', + options: { colorize: true } + } +}) + +const API_ENDPOINT = config.llms.apiEndpoint + +export const deepSeekStreamCompletion = async ( + conversation: ChatConversation[], + model = LlmModelsEnum.GPT_35_TURBO, + ctx: OnMessageContext | OnCallBackQueryData, + msgId: number, + limitTokens = true, + parameters?: ModelParameters +): Promise => { + logger.info(`Handling ${model} stream completion`) + parameters = parameters ?? { + system: ctx.session.currentPrompt, + max_tokens: +config.openAi.chatGpt.maxTokens + } + const data = { + model, + stream: true, + system: parameters.system, + max_tokens: limitTokens ? parameters.max_tokens : undefined, + messages: prepareConversation(conversation, model, ctx) // .map(m => { return { content: m.content, role: m.role } }) + } + let wordCount = 0 + let wordCountMinimum = 2 + const url = `${API_ENDPOINT}/deepseek/completions` + if (!ctx.chat?.id) { + throw new Error('Context chat id should not be empty after openAI streaming') + } + + const response: AxiosResponse = await axios.post(url, data, headersStream) + + const completionStream: Readable = response.data + let completion = '' + let outputTokens = '' + let inputTokens = '' + let message = '' + for await (const chunk of completionStream) { + const msg = chunk.toString() + if (msg) { + if (msg.includes('Input Tokens:')) { + const tokenMsg = msg.split('Input Tokens: ')[1] + inputTokens = tokenMsg.split('Output Tokens: ')[0] + outputTokens = tokenMsg.split('Output Tokens: ')[1] + completion += msg.split('Input Tokens: ')[0] + completion = completion.split('Input Tokens: ')[0] + } else if (msg.includes('Output Tokens: ')) { + outputTokens = msg.split('Output Tokens: ')[1] + completion = completion.split('Output Tokens: ')[0] + } else { + wordCount++ + completion += msg + if (wordCount > wordCountMinimum) { + if (wordCountMinimum < 64) { + wordCountMinimum *= 2 + } + completion = completion.replaceAll('...', '') + completion += '...' + wordCount = 0 + if (ctx.chat?.id && message !== completion) { + message = completion + await ctx.api + .editMessageText(ctx.chat?.id, msgId, completion) + .catch(async (e: any) => { + if (e instanceof GrammyError) { + if (e.error_code !== 400) { + throw e + } else { + logger.error(e.message) + } + } else { + throw e + } + }) + } + } + } + } + } + completion = completion.replaceAll('...', '') + await ctx.api + .editMessageText(ctx.chat?.id, msgId, completion) + .catch((e: any) => { + if (e instanceof GrammyError) { + if (e.error_code !== 400) { + throw e + } else { + logger.error(e) + } + } else { + throw e + } + }) + const totalOutputTokens = outputTokens // response.headers['x-openai-output-tokens'] + const totalInputTokens = inputTokens // response.headers['x-openai-input-tokens'] + return { + completion: { + content: completion, + role: 'assistant', + model, + timestamp: Date.now() + }, + usage: parseInt(totalOutputTokens, 10) + parseInt(totalInputTokens, 10), + price: 0, + inputTokens: parseInt(totalInputTokens, 10), + outputTokens: parseInt(totalOutputTokens, 10) + } +} diff --git a/src/modules/llms/api/openai.ts b/src/modules/llms/api/openai.ts index 8cbef0a..e4abe87 100644 --- a/src/modules/llms/api/openai.ts +++ b/src/modules/llms/api/openai.ts @@ -81,7 +81,7 @@ export async function alterGeneratedImg ( type ConversationOutput = Omit -const prepareConversation = (conversation: ChatConversation[], model: string, ctx: OnMessageContext | OnCallBackQueryData): ConversationOutput[] => { +export const prepareConversation = (conversation: ChatConversation[], model: string, ctx: OnMessageContext | OnCallBackQueryData): ConversationOutput[] => { const messages = conversation.filter(c => c.model === model).map(m => { return { content: m.content, role: m.role } }) if (messages.length !== 1 || model === LlmModelsEnum.O1) { return messages diff --git a/src/modules/llms/deepSeekBot.ts b/src/modules/llms/deepSeekBot.ts new file mode 100644 index 0000000..f3b653b --- /dev/null +++ b/src/modules/llms/deepSeekBot.ts @@ -0,0 +1,82 @@ +import { type BotPayments } from '../payment' +import { + type OnMessageContext, + type OnCallBackQueryData, + type ChatConversation +} from '../types' +import { deepSeekStreamCompletion } from './api/deepseek' +import { type LlmCompletion } from './api/llmApi' +import { LlmsBase } from './llmsBase' +import { type ModelVersion } from './utils/llmModelsManager' +import { type ModelParameters } from './utils/types' + +export class DeepSeekBot extends LlmsBase { + private readonly claudeModels: ModelVersion[] + + constructor (payments: BotPayments) { + super(payments, 'deepSeekBot', 'llms') + } + + public getEstimatedPrice (ctx: any): number { + return 0 + } + + public isSupportedEvent ( + ctx: OnMessageContext | OnCallBackQueryData + ): boolean { + const hasCommand = ctx.hasCommand(this.supportedCommands) + + const chatPrefix = this.hasPrefix(ctx.message?.text ?? '') + if (chatPrefix !== '') { + return true + } + return hasCommand + } + + async chatStreamCompletion ( + conversation: ChatConversation[], + model: ModelVersion, + ctx: OnMessageContext | OnCallBackQueryData, + msgId: number, + limitTokens: boolean, + parameters?: ModelParameters): Promise { + if (parameters) { + parameters.system = ctx.session.currentPrompt + } + return await deepSeekStreamCompletion(conversation, model, ctx, msgId, limitTokens, parameters) + } + + async chatCompletion ( + conversation: ChatConversation[], + model: ModelVersion, + ctx: OnMessageContext | OnCallBackQueryData, + hasTools: boolean, + parameters?: ModelParameters + ): Promise { + return { + completion: undefined, + usage: 0, + price: 0, + inputTokens: 0, + outputTokens: 0 + } + } + + public async onEvent (ctx: OnMessageContext | OnCallBackQueryData): Promise { + ctx.transient.analytics.module = this.module + const isSupportedEvent = this.isSupportedEvent(ctx) + if (!isSupportedEvent && ctx.chat?.type !== 'private') { + this.logger.warn(`### unsupported command ${ctx.message?.text}`) + return + } + const model = this.getModelFromContext(ctx) + if (!model) { + this.logger.warn(`### unsupported model for command ${ctx.message?.text}`) + return + } + this.updateSessionModel(ctx, model.version) + + const usesTools = ctx.hasCommand([this.commandsEnum.CTOOL, this.commandsEnum.STOOL]) + await this.onChat(ctx, model.version, usesTools ? false : this.getStreamOption(model.version), usesTools) + } +} diff --git a/src/modules/llms/utils/llmsData.ts b/src/modules/llms/utils/llmsData.ts index 5d17722..1263b2d 100644 --- a/src/modules/llms/utils/llmsData.ts +++ b/src/modules/llms/utils/llmsData.ts @@ -209,6 +209,21 @@ export const llmData: LLMData = { maxContextTokens: 131072, chargeType: 'TOKEN', stream: false + }, + deepSeek: { + provider: 'deepseek', + name: 'gpt-35-turbo', + fullName: 'GPT-3.5 Turbo', + botName: 'deepSeekBot', + version: 'gpt-3.5-turbo', + commands: ['ds'], + prefix: ['ds. '], + apiSpec: 'https://docs.x.ai/api#introduction', + inputPrice: 0.005, + outputPrice: 0.015, + maxContextTokens: 131072, + chargeType: 'TOKEN', + stream: true } }, imageModels: { @@ -269,6 +284,12 @@ export const llmData: LLMData = { max_tokens: +config.openAi.chatGpt.maxTokens } }, + deepseek: { + defaultParameters: { + // system: config.openAi.chatGpt.chatCompletionContext, + max_tokens: +config.openAi.chatGpt.maxTokens + } + }, luma: { defaultParameters: { // system: config.openAi.chatGpt.chatCompletionContext, diff --git a/src/modules/llms/utils/types.ts b/src/modules/llms/utils/types.ts index f88a0fb..6864787 100644 --- a/src/modules/llms/utils/types.ts +++ b/src/modules/llms/utils/types.ts @@ -1,4 +1,4 @@ -export type Provider = 'openai' | 'claude' | 'vertex' | 'xai' | 'luma' +export type Provider = 'openai' | 'claude' | 'vertex' | 'xai' | 'luma' | 'deepseek' export type ChargeType = 'TOKEN' | 'CHAR' export type DalleImageSize = '1024x1024' | '1024x1792' | '1792x1024' From 5da4dc72e4c22841f93bd229c3a59faf666a82bd Mon Sep 17 00:00:00 2001 From: fegloff Date: Wed, 12 Feb 2025 11:22:05 -0500 Subject: [PATCH 3/8] fix deepseek streaming issue + add free deep seek chat model --- src/modules/llms/api/deepseek.ts | 6 ++---- src/modules/llms/utils/llmsData.ts | 16 ++++++++-------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/modules/llms/api/deepseek.ts b/src/modules/llms/api/deepseek.ts index d32df25..ed7d079 100644 --- a/src/modules/llms/api/deepseek.ts +++ b/src/modules/llms/api/deepseek.ts @@ -37,9 +37,8 @@ export const deepSeekStreamCompletion = async ( const data = { model, stream: true, - system: parameters.system, max_tokens: limitTokens ? parameters.max_tokens : undefined, - messages: prepareConversation(conversation, model, ctx) // .map(m => { return { content: m.content, role: m.role } }) + messages: prepareConversation(conversation, model, ctx) } let wordCount = 0 let wordCountMinimum = 2 @@ -47,7 +46,6 @@ export const deepSeekStreamCompletion = async ( if (!ctx.chat?.id) { throw new Error('Context chat id should not be empty after openAI streaming') } - const response: AxiosResponse = await axios.post(url, data, headersStream) const completionStream: Readable = response.data @@ -98,7 +96,7 @@ export const deepSeekStreamCompletion = async ( } } completion = completion.replaceAll('...', '') - await ctx.api + completion !== '' && await ctx.api .editMessageText(ctx.chat?.id, msgId, completion) .catch((e: any) => { if (e instanceof GrammyError) { diff --git a/src/modules/llms/utils/llmsData.ts b/src/modules/llms/utils/llmsData.ts index 1263b2d..6ef0219 100644 --- a/src/modules/llms/utils/llmsData.ts +++ b/src/modules/llms/utils/llmsData.ts @@ -210,18 +210,18 @@ export const llmData: LLMData = { chargeType: 'TOKEN', stream: false }, - deepSeek: { + 'deepseek-chat-free': { provider: 'deepseek', - name: 'gpt-35-turbo', - fullName: 'GPT-3.5 Turbo', + name: 'deepseek-chat-free', + fullName: 'deepseek-chat', botName: 'deepSeekBot', - version: 'gpt-3.5-turbo', + version: 'deepseek-chat', commands: ['ds'], prefix: ['ds. '], - apiSpec: 'https://docs.x.ai/api#introduction', - inputPrice: 0.005, - outputPrice: 0.015, - maxContextTokens: 131072, + apiSpec: 'https://www.deepseek.com/', + inputPrice: 0, + outputPrice: 0, + maxContextTokens: 163840, chargeType: 'TOKEN', stream: true } From b8ea882bf67489cb60f05c3847127b91d863870d Mon Sep 17 00:00:00 2001 From: fegloff Date: Wed, 12 Feb 2025 17:20:46 -0500 Subject: [PATCH 4/8] fix vision command issue --- src/modules/voice-command/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/voice-command/index.ts b/src/modules/voice-command/index.ts index 7d04bb4..e46576d 100644 --- a/src/modules/voice-command/index.ts +++ b/src/modules/voice-command/index.ts @@ -39,7 +39,7 @@ export class VoiceCommand implements PayableBot { } }) this.voiceCommandList = [ - this.commandsEnum.VISION, + // this.commandsEnum.VISION, this.commandsEnum.ASK, this.commandsEnum.DALLE, OpenAISupportedCommands.talk From da3d1e3bef9b3e09dfc174b9c907ed208b7fe0f2 Mon Sep 17 00:00:00 2001 From: fegloff Date: Tue, 25 Mar 2025 23:55:27 -0500 Subject: [PATCH 5/8] update sonnet model + fix deep seek logic + update vertex api call + fix VISION error --- .gitignore | 3 +- src/modules/llms/api/deepseek.ts | 9 ++++-- src/modules/llms/api/openai.ts | 2 +- src/modules/llms/api/vertex.ts | 2 +- src/modules/llms/dalleBot.ts | 44 ++++++++++++++-------------- src/modules/llms/utils/llmsData.ts | 46 ++++++++++++++++++++++++------ src/modules/llms/utils/types.ts | 1 + 7 files changed, 72 insertions(+), 35 deletions(-) diff --git a/.gitignore b/.gitignore index 5b63399..e27b04a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ yarn.lock # Sentry Auth Token .sentryclirc .DS_Store -public/ \ No newline at end of file +public/ +.qodo diff --git a/src/modules/llms/api/deepseek.ts b/src/modules/llms/api/deepseek.ts index ed7d079..0128e00 100644 --- a/src/modules/llms/api/deepseek.ts +++ b/src/modules/llms/api/deepseek.ts @@ -19,7 +19,12 @@ const logger = pino({ } }) -const API_ENDPOINT = config.llms.apiEndpoint +const hasValidContent = (text: string): boolean => { + const trimmed = text.trim() + return trimmed.length > 0 && trimmed !== '\n' && !/^\s+$/.test(trimmed) +} + +const API_ENDPOINT = config.llms.apiEndpoint // 'http://127.0.0.1:5000' // config.llms.apiEndpoint export const deepSeekStreamCompletion = async ( conversation: ChatConversation[], @@ -96,7 +101,7 @@ export const deepSeekStreamCompletion = async ( } } completion = completion.replaceAll('...', '') - completion !== '' && await ctx.api + hasValidContent(completion) && await ctx.api .editMessageText(ctx.chat?.id, msgId, completion) .catch((e: any) => { if (e instanceof GrammyError) { diff --git a/src/modules/llms/api/openai.ts b/src/modules/llms/api/openai.ts index e4abe87..e9b2009 100644 --- a/src/modules/llms/api/openai.ts +++ b/src/modules/llms/api/openai.ts @@ -83,7 +83,7 @@ type ConversationOutput = Omit { const messages = conversation.filter(c => c.model === model).map(m => { return { content: m.content, role: m.role } }) - if (messages.length !== 1 || model === LlmModelsEnum.O1) { + if (messages.length !== 1 || model === LlmModelsEnum.O1 || model.includes('deep')) { return messages } const systemMessage = { diff --git a/src/modules/llms/api/vertex.ts b/src/modules/llms/api/vertex.ts index 4daa3da..16a46ad 100644 --- a/src/modules/llms/api/vertex.ts +++ b/src/modules/llms/api/vertex.ts @@ -74,7 +74,7 @@ export const vertexStreamCompletion = async ( ): Promise => { parameters = parameters ?? { system: ctx.session.currentPrompt, - max_tokens: +config.openAi.chatGpt.maxTokens + max_tokens_to_sample: +config.openAi.chatGpt.maxTokens } const data = { model, diff --git a/src/modules/llms/dalleBot.ts b/src/modules/llms/dalleBot.ts index 33bbaf6..12737c0 100644 --- a/src/modules/llms/dalleBot.ts +++ b/src/modules/llms/dalleBot.ts @@ -9,7 +9,7 @@ import { getMessageExtras, getMinBalance, getPromptPrice, - getUrlFromText, + // getUrlFromText, hasCommandPrefix, MAX_TRIES, PRICE_ADJUSTMENT, @@ -109,10 +109,10 @@ export class DalleBot extends LlmsBase { if (photo && session.isEnabled) { const prompt = ctx.message?.caption ?? ctx.message?.text if ( - prompt && - (ctx.chat?.type === 'private' || - ctx.hasCommand(this.commandsEnum.VISION)) - ) { + prompt && ctx.chat?.type === 'private') { + // (ctx.chat?.type === 'private' || + // ctx.hasCommand(this.commandsEnum.VISION)) + // ) { // && !isNaN(+prompt) return true } @@ -189,23 +189,23 @@ export class DalleBot extends LlmsBase { return } - if (ctx.hasCommand(this.commandsEnum.VISION)) { - const photoUrl = getUrlFromText(ctx) - if (photoUrl) { - const prompt = ctx.match - session.imgRequestQueue.push({ - prompt, - photoUrl, - command: 'vision' // !isNaN(+prompt) ? 'alter' : 'vision' - }) - if (!session.isProcessingQueue) { - session.isProcessingQueue = true - await this.onImgRequestHandler(ctx).then(() => { - session.isProcessingQueue = false - }) - } - } - } + // if (ctx.hasCommand(this.commandsEnum.VISION)) { + // const photoUrl = getUrlFromText(ctx) + // if (photoUrl) { + // const prompt = ctx.match + // session.imgRequestQueue.push({ + // prompt, + // photoUrl, + // command: 'vision' // !isNaN(+prompt) ? 'alter' : 'vision' + // }) + // if (!session.isProcessingQueue) { + // session.isProcessingQueue = true + // await this.onImgRequestHandler(ctx).then(() => { + // session.isProcessingQueue = false + // }) + // } + // } + // } if ( ctx.hasCommand(this.commands) || diff --git a/src/modules/llms/utils/llmsData.ts b/src/modules/llms/utils/llmsData.ts index 6ef0219..f3f32c8 100644 --- a/src/modules/llms/utils/llmsData.ts +++ b/src/modules/llms/utils/llmsData.ts @@ -33,12 +33,12 @@ export const llmData: LLMData = { chargeType: 'CHAR', stream: true }, - 'claude-35-sonnet': { + 'claude-37-sonnet': { provider: 'claude', - name: 'claude-35-sonnet', - fullName: 'Claude Sonnet 3.5', + name: 'claude-37-sonnet', + fullName: 'Claude Sonnet 3.7', botName: 'ClaudeBot', - version: 'claude-3-5-sonnet-20241022', + version: 'claude-3-7-sonnet-latest', commands: ['sonnet', 'claude', 's', 'stool', 'c', 'ctool', 'c0'], prefix: ['s. ', 'c. ', 'c0. '], apiSpec: 'https://www.anthropic.com/news/claude-3-5-sonnet', @@ -48,6 +48,21 @@ export const llmData: LLMData = { chargeType: 'TOKEN', stream: true }, + // 'claude-35-sonnet': { + // provider: 'claude', + // name: 'claude-35-sonnet', + // fullName: 'Claude Sonnet 3.5', + // botName: 'ClaudeBot', + // version: 'claude-3-5-sonnet-20241022', + // commands: ['sonnet', 'claude', 's', 'stool', 'c', 'ctool', 'c0'], + // prefix: ['s. ', 'c. ', 'c0. '], + // apiSpec: 'https://www.anthropic.com/news/claude-3-5-sonnet', + // inputPrice: 0.003, + // outputPrice: 0.015, + // maxContextTokens: 8192, + // chargeType: 'TOKEN', + // stream: true + // }, 'claude-3-opus': { provider: 'claude', name: 'claude-3-opus', @@ -210,17 +225,32 @@ export const llmData: LLMData = { chargeType: 'TOKEN', stream: false }, - 'deepseek-chat-free': { + 'deepseek-r1': { provider: 'deepseek', - name: 'deepseek-chat-free', - fullName: 'deepseek-chat', + name: 'deepseek-r1', + fullName: 'deepseek-r1', botName: 'deepSeekBot', - version: 'deepseek-chat', + version: 'deepseek-r1', commands: ['ds'], prefix: ['ds. '], apiSpec: 'https://www.deepseek.com/', inputPrice: 0, outputPrice: 0, + maxContextTokens: 128000, + chargeType: 'TOKEN', + stream: true + }, + 'deepseek-chat-free': { + provider: 'deepseek', + name: 'deepseek-chat-free', + fullName: 'deepseek-chat', + botName: 'deepSeekBot', + version: 'deepseek-chat', + commands: ['dsf'], + prefix: ['dsf. '], + apiSpec: 'https://www.deepseek.com/', + inputPrice: 0.0008, + outputPrice: 0.0024, maxContextTokens: 163840, chargeType: 'TOKEN', stream: true diff --git a/src/modules/llms/utils/types.ts b/src/modules/llms/utils/types.ts index 6864787..9d8f2ea 100644 --- a/src/modules/llms/utils/types.ts +++ b/src/modules/llms/utils/types.ts @@ -19,6 +19,7 @@ export interface ModelParameters { temperature?: number max_tokens?: number max_completion_tokens?: number + max_tokens_to_sample?: number system?: string } From 35b34ad46457da26ff3285bac5b60022a74dbdb1 Mon Sep 17 00:00:00 2001 From: fegloff Date: Fri, 25 Apr 2025 13:18:28 -0500 Subject: [PATCH 6/8] update openai models including o3 and gpt-4.1 --- src/modules/llms/api/openai.ts | 2 +- src/modules/llms/llmsBase.ts | 2 +- src/modules/llms/openaiBot.ts | 2 +- src/modules/llms/utils/llmsData.ts | 53 +++++++++++++++--------------- 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/modules/llms/api/openai.ts b/src/modules/llms/api/openai.ts index e9b2009..44b0d9c 100644 --- a/src/modules/llms/api/openai.ts +++ b/src/modules/llms/api/openai.ts @@ -83,7 +83,7 @@ type ConversationOutput = Omit { const messages = conversation.filter(c => c.model === model).map(m => { return { content: m.content, role: m.role } }) - if (messages.length !== 1 || model === LlmModelsEnum.O1 || model.includes('deep')) { + if (messages.length !== 1 || model === LlmModelsEnum.O3 || model.includes('deep')) { return messages } const systemMessage = { diff --git a/src/modules/llms/llmsBase.ts b/src/modules/llms/llmsBase.ts index 940c260..8dbf862 100644 --- a/src/modules/llms/llmsBase.ts +++ b/src/modules/llms/llmsBase.ts @@ -410,7 +410,7 @@ export abstract class LlmsBase implements PayableBot { const parameters = this.modelManager.getModelParameters(model) const response = await this.chatCompletion(conversation, model, ctx, usesTools, parameters) if (response.completion) { - if (model === this.modelsEnum.O1) { + if (model === this.modelsEnum.O3) { const msgs = splitTelegramMessage(response.completion.content as string) await ctx.api.editMessageText( ctx.chat.id, diff --git a/src/modules/llms/openaiBot.ts b/src/modules/llms/openaiBot.ts index f40740c..4d31680 100644 --- a/src/modules/llms/openaiBot.ts +++ b/src/modules/llms/openaiBot.ts @@ -88,7 +88,7 @@ export class OpenAIBot extends LlmsBase { usesTools: boolean, parameters?: ModelParameters ): Promise { - return await chatCompletion(conversation, model, ctx, model !== this.modelsEnum.O1, parameters) // limitTokens doesn't apply for o1-preview + return await chatCompletion(conversation, model, ctx, model !== this.modelsEnum.O3, parameters) // limitTokens doesn't apply for o1-preview } hasPrefix (prompt: string): string { diff --git a/src/modules/llms/utils/llmsData.ts b/src/modules/llms/utils/llmsData.ts index f3f32c8..156b4aa 100644 --- a/src/modules/llms/utils/llmsData.ts +++ b/src/modules/llms/utils/llmsData.ts @@ -123,20 +123,21 @@ export const llmData: LLMData = { chargeType: 'TOKEN', stream: true }, - // 'gpt-4': { - // provider: 'openai', - // name: 'gpt-4', - // fullName: 'GPT-4', - // botName: 'OpenAIBot', - // version: 'gpt-4', - // commands: ['gpt4'], - // apiSpec: 'https://openai.com/index/gpt-4/', - // inputPrice: 0.03, - // outputPrice: 0.06, - // maxContextTokens: 8192, - // chargeType: 'TOKEN', - // stream: true - // }, + 'gpt-4-1': { + provider: 'openai', + name: 'gpt-4.1', + fullName: 'GPT-4.1', + botName: 'OpenAIBot', + version: 'gpt-4.1-2025-04-14', + commands: ['gpt41', 'ask41'], + prefix: ['a41. '], + apiSpec: 'https://platform.openai.com/docs/models/gpt-4.1', + inputPrice: 0.002, + outputPrice: 0.008, + maxContextTokens: 32768, + chargeType: 'TOKEN', + stream: true + }, 'gpt-35-turbo': { provider: 'openai', name: 'gpt-35-turbo', @@ -145,9 +146,9 @@ export const llmData: LLMData = { version: 'gpt-3.5-turbo', commands: ['ask35'], apiSpec: 'https://platform.openai.com/docs/models/gpt-3-5-turbo', - inputPrice: 0.003, - outputPrice: 0.006, - maxContextTokens: 4000, + inputPrice: 0.0005, + outputPrice: 0.0015, + maxContextTokens: 4096, chargeType: 'TOKEN', stream: true }, @@ -166,17 +167,17 @@ export const llmData: LLMData = { // chargeType: 'TOKEN', // stream: true // }, - o1: { + o3: { provider: 'openai', - name: 'o1', - fullName: 'O1 Preview', + name: 'o3', + fullName: 'O3', botName: 'OpenAIBot', - version: 'o1-preview', - commands: ['o1', 'ask1'], - prefix: ['o1. '], + version: 'o3-2025-04-16', + commands: ['o3'], + prefix: ['o3. '], apiSpec: 'https://platform.openai.com/docs/models/o1', - inputPrice: 0.015, - outputPrice: 0.06, + inputPrice: 0.01, + outputPrice: 0.04, maxContextTokens: 200000, chargeType: 'TOKEN', stream: false @@ -294,7 +295,7 @@ export const llmData: LLMData = { temperature: config.openAi.dalle.completions.temperature, max_completion_tokens: +config.openAi.chatGpt.maxTokens }, - modelOverrides: { o1: { temperature: 1 }, 'o3-mini': { temperature: undefined } } // uses model name, not model version + modelOverrides: { o3: { temperature: 1 }, 'o3-mini': { temperature: undefined } } // uses model name, not model version }, claude: { defaultParameters: { From 52f5b3f841565c400dcef73eaf44624946b2f347 Mon Sep 17 00:00:00 2001 From: fegloff Date: Wed, 30 Apr 2025 15:37:26 -0500 Subject: [PATCH 7/8] add retry logic for dabatase and telegram connection --- src/bot.ts | 154 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 140 insertions(+), 14 deletions(-) diff --git a/src/bot.ts b/src/bot.ts index 3d13ce2..e6418fd 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -1,3 +1,6 @@ +/* eslint-disable import/first */ +import * as Events from 'events' +Events.EventEmitter.defaultMaxListeners = 30 import { Sentry } from './monitoring/instrument' import express from 'express' import asyncHandler from 'express-async-handler' @@ -41,11 +44,11 @@ import prometheusRegister, { PrometheusMetrics } from './metrics/prometheus' import { chatService, statsService } from './database/services' import { AppDataSource } from './database/datasource' import { autoRetry } from '@grammyjs/auto-retry' -import { run } from '@grammyjs/runner' +import { run, type RunnerHandle } from '@grammyjs/runner' import { runBotHeartBit } from './monitoring/monitoring' import { type BotPaymentLog } from './database/stats.service' import { TelegramPayments } from './modules/telegram_payment' -import * as Events from 'events' + import { ES } from './es' import { hydrateFiles } from '@grammyjs/files' import { VoiceTranslateBot } from './modules/voice-translate' @@ -62,8 +65,6 @@ import { LumaBot } from './modules/llms/lumaBot' import { XaiBot } from './modules/llms/xaiBot' import { DeepSeekBot } from './modules/llms/deepSeekBot' -Events.EventEmitter.defaultMaxListeners = 30 - const logger = pino({ name: 'bot', transport: { @@ -364,7 +365,7 @@ const UtilityBots: Record = { } const executeOrRefund = async (ctx: OnMessageContext, price: number, bot: PayableBot): Promise => { - const refund = (reason?: string): void => {} + const refund = (reason?: string): void => { } await bot.onEvent(ctx, refund).catch((ex: any) => { Sentry.captureException(ex) logger.error(ex?.message ?? 'Unknown error') @@ -704,19 +705,114 @@ app.get('/metrics', asyncHandler(async (req, res): Promise => { async function bootstrap (): Promise { const httpServer = app.listen(config.port, () => { logger.info(`Bot listening on port ${config.port}`) - // bot.start({ - // allowed_updates: ["callback_query"], // Needs to be set for menu middleware, but bot doesn't work with current configuration. - // }); }) - await AppDataSource.initialize() - payments.bootstrap() + // Database connection retry logic + const connectToDatabase = async (maxRetries = 5): Promise => { + let retries = 0 + let connected = false + + while (!connected && retries < maxRetries) { + try { + logger.info(`Database connection attempt ${retries + 1}/${maxRetries}...`) + // Check if already initialized + if (AppDataSource.isInitialized) { + logger.info('Database already initialized') + return true + } + await AppDataSource.initialize() + logger.info('Database initalizated') + connected = true + return true + } catch (error) { + retries++ + logger.error(`Database connection failed: ${(error as Error).message}`) + if (retries >= maxRetries) { + logger.error('Maximum database connection retry attempts reached') + return false + } + // Exponential backoff + const delay = Math.min(1000 * (2 ** retries), 30000) + logger.info(`Waiting ${delay / 1000} seconds before retrying database connection...`) + await new Promise(resolve => setTimeout(resolve, delay)) + } + } + return connected + } - const prometheusMetrics = new PrometheusMetrics() - await prometheusMetrics.bootstrap() + // Connect to database with retries + const dbConnected = await connectToDatabase() + if (!dbConnected) { + logger.error('Failed to connect to database after multiple attempts') + // Continue running but bot functionality will be limited + } - const runner = run(bot) + try { + payments.bootstrap() + } catch (error) { + logger.error(`Payments bootstrap error: ${error}`) + // Continue despite payment initialization errors + } + // Only try to initialize Prometheus metrics if database is connected + let prometheusMetrics: PrometheusMetrics | null = null + if (dbConnected) { + try { + prometheusMetrics = new PrometheusMetrics() + await prometheusMetrics.bootstrap() + } catch (error) { + logger.error(`Prometheus metrics bootstrap error: ${error}`) + // Continue despite metrics initialization errors + } + } else { + logger.warn('Skipping Prometheus metrics initialization due to database connection failure') + } + + // Telegram connection retry logic + const connectToTelegram = async (maxRetries = 5): Promise => { + let retries = 0 + let connected = false + let runner: RunnerHandle | undefined + + logger.info('Starting Telegram bot connection attempt...') + + while (!connected && retries < maxRetries) { + try { + logger.info(`Attempt ${retries + 1}/${maxRetries} to connect to Telegram API...`) + // First test the connection to Telegram's API + await bot.api.getMe() + logger.info('Successfully connected to Telegram API') + // If connection test succeeded, initialize the runner + runner = run(bot) + logger.info('Bot runner started successfully') + connected = true + return runner + } catch (error) { + retries++ + logger.error(`Failed to connect to Telegram: ${(error as Error).message}`) + if (retries >= maxRetries) { + logger.error('Maximum Telegram connection retry attempts reached') + return undefined + } + // Calculate backoff delay + const delay = Math.min(1000 * (2 ** retries), 30000) + logger.info(`Waiting ${delay / 1000} seconds before retry ${retries + 1}...`) + await new Promise(resolve => setTimeout(resolve, delay)) + } + } + return undefined + } + + // Try to connect to Telegram with retry logic + let runner: RunnerHandle | undefined + try { + runner = await connectToTelegram() + } catch (error) { + logger.error(`Error in Telegram connection retry process: ${(error as Error).message}`) + // Don't exit yet, we'll handle this gracefully + } + + // Set up application shutdown handler const stopApplication = async (): Promise => { console.warn('Terminating the bot...') @@ -742,10 +838,12 @@ async function bootstrap (): Promise { } } + // Setup signal handlers process.on('SIGINT', () => { stopApplication().catch(logger.error) }) process.on('SIGTERM', () => { stopApplication().catch(logger.error) }) - if (config.betteruptime.botHeartBitId) { + // Setup heartbeat monitor if configured + if (config.betteruptime.botHeartBitId && runner) { const task = await runBotHeartBit(runner, config.betteruptime.botHeartBitId) const stopHeartBit = (): void => { logger.info('heart bit stopping') @@ -754,6 +852,34 @@ async function bootstrap (): Promise { process.once('SIGINT', stopHeartBit) process.once('SIGTERM', stopHeartBit) } + + // If both database and Telegram had connection issues, schedule a restart + if (!dbConnected || !runner) { + logger.error('Critical services failed to initialize.') + // Schedule a restart after a delay to avoid rapid restart cycles + const restartDelay = 5 * 60 * 1000 // 5 minutes + logger.info(`Scheduling application restart in ${restartDelay / 1000} seconds...`) + setTimeout(() => { + logger.info('Executing scheduled restart after initialization failures') + process.exit(1) // This will trigger a restart by Fly.io + }, restartDelay) + } + + if (dbConnected) { + // Periodically check database connection and attempt to reconnect if needed + setInterval(() => { + void (async () => { + try { + if (!AppDataSource.isInitialized) { + logger.warn('Database connection lost, attempting to reconnect...') + await connectToDatabase(3) // Use fewer retries for periodic checks + } + } catch (error) { + logger.error(`Database monitoring error: ${(error as Error).message}`) + } + })() + }, 5 * 60 * 1000) // Check every minute + } } bootstrap().catch((error) => { From 7824320dbbc58073e389060c90d04c1fe06bcdb9 Mon Sep 17 00:00:00 2001 From: fegloff Date: Wed, 30 Apr 2025 16:15:04 -0500 Subject: [PATCH 8/8] remove db monitor --- src/bot.ts | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/bot.ts b/src/bot.ts index e6418fd..41f5827 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -1,6 +1,7 @@ /* eslint-disable import/first */ import * as Events from 'events' Events.EventEmitter.defaultMaxListeners = 30 + import { Sentry } from './monitoring/instrument' import express from 'express' import asyncHandler from 'express-async-handler' @@ -865,20 +866,26 @@ async function bootstrap (): Promise { }, restartDelay) } - if (dbConnected) { - // Periodically check database connection and attempt to reconnect if needed - setInterval(() => { - void (async () => { - try { - if (!AppDataSource.isInitialized) { - logger.warn('Database connection lost, attempting to reconnect...') - await connectToDatabase(3) // Use fewer retries for periodic checks - } - } catch (error) { - logger.error(`Database monitoring error: ${(error as Error).message}`) - } - })() - }, 5 * 60 * 1000) // Check every minute + try { + payments.bootstrap() + } catch (error) { + logger.error(`Payments bootstrap error: ${error}`) + // Continue despite payment initialization errors + } + + if (dbConnected && AppDataSource.driver) { + try { + // Handle any driver-level errors that bubble up + const driver = AppDataSource.driver as any + // Only attach to events if they're available, don't force it + if (driver.eventEmitter && typeof driver.eventEmitter.on === 'function') { + driver.eventEmitter.on('error', (error: any) => { + logger.error(`Database error event: ${error.message}`) + }) + } + } catch (error) { + logger.warn(`Could not set up database error listener: ${(error as Error).message}`) + } } }