Elizhium/litlyx
1
0
Fork 0
mirror of https://github.com/Litlyx/litlyx synced 2025-12-10 07:48:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add streaming to AI

Browse Source
This commit is contained in:
Emily
2024-12-11 18:32:22 +01:00
parent 23b8f7229a
commit f358bb9bb6
5 changed files with 271 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
dashboard/server/api/ai/[chat_id]/status.ts Normal file
View File

@@ -0,0 +1,17 @@
import { AiChatModel } from "@schema/ai/AiChatSchema";
export default defineEventHandler(async event => {
const data = await getRequestData(event);
if (!data) return;
const { project_id } = data;
if (!event.context.params) return;
const chat_id = event.context.params['chat_id'];
const chat = await AiChatModel.findOne({ _id: chat_id, project_id }, { status: 1, completed: 1 });
if (!chat) return;
return { status: chat.status, completed: chat.completed || false }
});

42
dashboard/server/api/ai/send_message.post.ts
View File

@@ -1,4 +1,4 @@
import { sendMessageOnChat } from "~/server/services/AiService";
import { sendMessageOnChat, updateChatStatus } from "~/server/services/AiService";
import { getAiChatRemainings } from "./chats_remaining";
@@ -15,7 +15,43 @@ export default defineEventHandler(async event => {
const chatsRemaining = await getAiChatRemainings(pid);
if (chatsRemaining <= 0) return setResponseStatus(event, 400, 'CHAT_LIMIT_REACHED');
const response = await sendMessageOnChat(text, pid, chat_id);
const currentStatus: string[] = [];
let responseSent = false;
let targetChatId = '';
await sendMessageOnChat(text, pid, chat_id, {
onChatId: async chat_id => {
if (!responseSent) {
event.node.res.setHeader('Content-Type', 'application/json');
event.node.res.end(JSON.stringify({ chat_id }));
targetChatId = chat_id;
responseSent = true;
}
},
onDelta: async text => {
currentStatus.push(text);
await updateChatStatus(targetChatId, currentStatus.join(''), false);
},
onFunctionName: async name => {
currentStatus.push('[data:FunctionName]');
await updateChatStatus(targetChatId, currentStatus.join(''), false);
},
onFunctionCall: async name => {
currentStatus.push('[data:FunctionCall]');
await updateChatStatus(targetChatId, currentStatus.join(''), false);
},
onFunctionResult: async (name, result) => {
currentStatus.push('[data:FunctionResult]');
await updateChatStatus(targetChatId, currentStatus.join(''), false);
},
onFinish: async calls => {
currentStatus.push('[data:FunctionFinish]');
await updateChatStatus(targetChatId, currentStatus.join(''), false);
}
});
await updateChatStatus(targetChatId, currentStatus.join(''), true);
return response;
});

184
dashboard/server/services/AiService.ts
View File

@@ -9,15 +9,11 @@ import { AiEventsInstance } from '../ai/functions/AI_Events';
import { AiVisitsInstance } from '../ai/functions/AI_Visits';
import { AiComposableChartInstance } from '../ai/functions/AI_ComposableChart';
const { AI_ORG, AI_PROJECT, AI_KEY } = useRuntimeConfig();
const { AI_KEY, AI_ORG, AI_PROJECT } = useRuntimeConfig();
const OPENAI_MODEL: OpenAI.Chat.ChatModel = 'gpt-4o-mini';
const openai = new OpenAI({
organization: AI_ORG,
project: AI_PROJECT,
apiKey: AI_KEY
});
const openai = new OpenAI({ apiKey: AI_KEY, organization: AI_ORG, project: AI_PROJECT });
const tools: OpenAI.Chat.Completions.ChatCompletionTool[] = [
...AiVisitsInstance.getTools(),
@@ -57,6 +53,13 @@ async function setChatTitle(title: string, chat_id?: string) {
await AiChatModel.updateOne({ _id: chat_id }, { title });
}
export async function updateChatStatus(chat_id: string, status: string, completed: boolean) {
await AiChatModel.updateOne({ _id: chat_id }, {
status,
completed
});
}
export function getChartsInMessage(message: OpenAI.Chat.Completions.ChatCompletionMessageParam) {
if (message.role != 'assistant') return [];
@@ -65,13 +68,117 @@ export function getChartsInMessage(message: OpenAI.Chat.Completions.ChatCompleti
return message.tool_calls.filter(e => e.function.name === 'createComposableChart').map(e => e.function.arguments);
}
export async function sendMessageOnChat(text: string, pid: string, initial_chat_id?: string) {
type FunctionCall = { name: string, argsRaw: string[], collecting: boolean, result: any, tool_call_id: string }
type DeltaCallback = (text: string) => any;
type FinishCallback = (functionsCount: number) => any;
type FunctionNameCallback = (name: string) => any;
type FunctionCallCallback = (name: string) => any;
type FunctionResultCallback = (name: string, result: any) => any;
type ElaborateResponseCallbacks = {
onDelta?: DeltaCallback,
onFunctionName?: FunctionNameCallback,
onFunctionCall?: FunctionCallCallback,
onFunctionResult?: FunctionResultCallback,
onFinish?: FinishCallback,
onChatId?: (chat_id: string) => any
}
async function elaborateResponse(messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[], pid: string, chat_id: string, callbacks?: ElaborateResponseCallbacks) {
const responseStream = await openai.beta.chat.completions.stream({ model: OPENAI_MODEL, messages, n: 1, tools });
const functionCalls: FunctionCall[] = [];
let lastFinishReason: "length" | "tool_calls" | "function_call" | "stop" | "content_filter" | null = null;
for await (const part of responseStream) {
const delta = part.choices[0].delta;
const finishReason = part.choices[0].finish_reason;
if (delta.content) await callbacks?.onDelta?.(delta.content);
if (delta.tool_calls) {
const toolCall = delta.tool_calls[0];
if (!toolCall.function) throw Error('Cannot get function from tool_calls');
const functionName = toolCall.function.name;
const functionCall: FunctionCall = functionName ?
{ name: functionName, argsRaw: [], collecting: true, result: null, tool_call_id: toolCall.id as string } :
functionCalls.at(-1) as FunctionCall;
if (functionName) functionCalls.push(functionCall);
if (functionName) await callbacks?.onFunctionName?.(functionName);
if (toolCall.function.arguments) functionCall.argsRaw.push(toolCall.function.arguments);
}
if (finishReason === "tool_calls" && functionCalls.at(-1)?.collecting) {
const functionCall: FunctionCall = functionCalls.at(-1) as FunctionCall;
await callbacks?.onFunctionCall?.(functionCall.name);
const args = JSON.parse(functionCall.argsRaw.join(''));
const functionResult = await functions[functionCall.name]({ project_id: pid, ...args });
functionCall.result = functionResult;
await callbacks?.onFunctionResult?.(functionCall.name, functionResult);
addMessageToChat({
role: 'assistant',
content: delta.content,
refusal: delta.refusal,
tool_calls: [
{
id: functionCall.tool_call_id,
type: 'function',
function: {
name: functionCall.name,
arguments: functionCall.argsRaw.join('')
}
}
],
parsed: null
}, chat_id);
addMessageToChat({
tool_call_id: functionCall.tool_call_id,
role: 'tool',
content: JSON.stringify(functionResult)
}, chat_id);
functionCall.collecting = false;
lastFinishReason = finishReason;
}
}
await callbacks?.onFinish?.(functionCalls.length);
const toolResponseMesages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = functionCalls.map(e => {
return { tool_call_id: e.tool_call_id, role: "tool", content: JSON.stringify(e.result) }
});
if (lastFinishReason == 'tool_calls') return await elaborateResponse([...responseStream.messages, ...toolResponseMesages], pid, chat_id, callbacks);
return responseStream;
}
export async function sendMessageOnChat(text: string, pid: string, initial_chat_id?: string, callbacks?: ElaborateResponseCallbacks) {
const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = []
const chat_id = await createChatIfNotExist(pid, initial_chat_id);
const chatMessages = await getMessagesFromChatId(chat_id);
await callbacks?.onChatId?.(chat_id);
if (chatMessages && chatMessages.length > 0) {
messages.push(...chatMessages);
} else {
@@ -89,32 +196,43 @@ export async function sendMessageOnChat(text: string, pid: string, initial_chat_
messages.push(userMessage);
await addMessageToChat(userMessage, chat_id);
let response = await openai.chat.completions.create({ model: OPENAI_MODEL, messages, n: 1, tools });
const chartsData: string[][] = [];
while ((response.choices[0].message.tool_calls?.length || 0) > 0) {
await addMessageToChat(response.choices[0].message, chat_id);
messages.push(response.choices[0].message);
if (response.choices[0].message.tool_calls) {
console.log('Tools to call', response.choices[0].message.tool_calls.length);
chartsData.push(getChartsInMessage(response.choices[0].message));
for (const toolCall of response.choices[0].message.tool_calls) {
const functionName = toolCall.function.name;
console.log('Calling tool function', functionName);
const functionToCall = functions[functionName];
const functionArgs = JSON.parse(toolCall.function.arguments);
const functionResponse = await functionToCall({ project_id: pid, ...functionArgs });
messages.push({ tool_call_id: toolCall.id, role: "tool", content: JSON.stringify(functionResponse) });
await addMessageToChat({ tool_call_id: toolCall.id, role: "tool", content: JSON.stringify(functionResponse) }, chat_id);
}
}
response = await openai.chat.completions.create({ model: OPENAI_MODEL, messages, n: 1, tools });
try {
const streamResponse = await elaborateResponse(messages, pid, chat_id, callbacks);
await addMessageToChat({ role: 'assistant', refusal: null, content: await streamResponse.finalContent() }, chat_id);
return { content: '', charts: [] };
} catch (ex: any) {
console.error(ex);
return { content: ex.message, charts: [] };
}
await addMessageToChat(response.choices[0].message, chat_id);
await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { ai_messages: 1 } })
return { content: response.choices[0].message.content, charts: chartsData.filter(e => e.length > 0).flat() };
// let response = await openai.chat.completions.create({ model: OPENAI_MODEL, messages, n: 1, tools });
// const chartsData: string[][] = [];
// while ((response.choices[0].message.tool_calls?.length || 0) > 0) {
// await addMessageToChat(response.choices[0].message, chat_id);
// messages.push(response.choices[0].message);
// if (response.choices[0].message.tool_calls) {
// console.log('Tools to call', response.choices[0].message.tool_calls.length);
// chartsData.push(getChartsInMessage(response.choices[0].message));
// for (const toolCall of response.choices[0].message.tool_calls) {
// const functionName = toolCall.function.name;
// console.log('Calling tool function', functionName);
// const functionToCall = functions[functionName];
// const functionArgs = JSON.parse(toolCall.function.arguments);
// const functionResponse = await functionToCall({ project_id: pid, ...functionArgs });
// messages.push({ tool_call_id: toolCall.id, role: "tool", content: JSON.stringify(functionResponse) });
// await addMessageToChat({ tool_call_id: toolCall.id, role: "tool", content: JSON.stringify(functionResponse) }, chat_id);
// }
// }
// response = await openai.chat.completions.create({ model: OPENAI_MODEL, messages, n: 1, tools });
// }
// await addMessageToChat(response.choices[0].message, chat_id);
// await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { ai_messages: 1 } })
// return { content: response.choices[0].message.content, charts: chartsData.filter(e => e.length > 0).flat() };
}