Elizhium/litlyx
1
0
Fork 0
mirror of https://github.com/Litlyx/litlyx synced 2025-12-11 16:28:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

new selfhosted version

Browse Source
This commit is contained in:
antonio
2025-11-28 14:11:51 +01:00
parent afda29997d
commit 951860f67e
1046 changed files with 72586 additions and 574750 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
dashboard/server/services/AggregationService.ts Normal file
View File

@@ -0,0 +1,26 @@
import { getAllDomains } from "../controllers/DomainController";
import { visitController } from "../controllers/VisitController";
import { sessionController } from "../controllers/SessionController";
import { bouncingController } from "../controllers/BouncingController";
import { durationController } from "../controllers/DurationController";
import { ProjectCountModel } from "~/shared/schema/project/ProjectsCounts";
export async function executeAggregationProject(project_id: string, date: Date, override: boolean) {
const domains = await getAllDomains({ project_id, date });
await Promise.all([
visitController.aggregate({ project_id, date, domains, override }),
sessionController.aggregate({ project_id, date, domains, override }),
bouncingController.aggregate({ project_id, date, domains, override }),
durationController.aggregate({ project_id, date, domains, override })
]);
}
export async function executeAggregation(date: Date) {
const targets = await ProjectCountModel.find({ visits: { $gte: 500_000 } }, {}, { lean: true, sort: { visits: -1 } });
for (let i = 0; i < targets.length; i++) {
const target = targets[i];
const project_id = target.project_id.toString();
await executeAggregationProject(project_id, date, true);
}
}

251
dashboard/server/services/AiService.ts
View File

@@ -1,251 +0,0 @@
import OpenAI from "openai";
import { AiChatModel } from '@schema/ai/AiChatSchema';
import { ProjectLimitModel } from '@schema/project/ProjectsLimits';
import { AiEventsInstance } from '../ai/functions/AI_Events';
import { AiVisitsInstance } from '../ai/functions/AI_Visits';
import { AiSessionsInstance } from '../ai/functions/AI_Sessions';
import { AiBillingInstance } from '../ai/functions/AI_Billing';
import { AiSnapshotInstance } from '../ai/functions/AI_Snapshots';
import { AiComposableChartInstance } from '../ai/functions/AI_ComposableChart';
const { AI_KEY, AI_ORG, AI_PROJECT } = useRuntimeConfig();
const OPENAI_MODEL: OpenAI.Chat.ChatModel = 'gpt-4o-mini';
const openai = new OpenAI({ apiKey: AI_KEY, organization: AI_ORG, project: AI_PROJECT });
const tools: OpenAI.Chat.Completions.ChatCompletionTool[] = [
...AiVisitsInstance.getTools(),
...AiEventsInstance.getTools(),
...AiSessionsInstance.getTools(),
...AiBillingInstance.getTools(),
...AiSnapshotInstance.getTools(),
...AiComposableChartInstance.getTools(),
]
const functions: any = {
...AiVisitsInstance.getHandlers(),
...AiEventsInstance.getHandlers(),
...AiSessionsInstance.getHandlers(),
...AiBillingInstance.getHandlers(),
...AiSnapshotInstance.getHandlers(),
...AiComposableChartInstance.getHandlers()
}
async function getMessagesFromChatId(chat_id?: string) {
if (!chat_id) return;
const chatItem = await AiChatModel.findById(chat_id);
if (!chatItem) return;
return chatItem.messages;
}
async function addMessageToChat(message: any, chat_id?: string) {
if (!chat_id) return;
await AiChatModel.updateOne({ _id: chat_id }, { $push: { messages: message } });
}
async function createChatIfNotExist(pid: string, chat_id?: string) {
const chatItem = await AiChatModel.exists({ _id: chat_id });
if (chatItem) return chatItem._id.toString();
const newChatItem = await AiChatModel.create({ messages: [], project_id: pid, title: 'new chat' });
return newChatItem._id.toString();
}
async function setChatTitle(title: string, chat_id?: string) {
if (!chat_id) return;
await AiChatModel.updateOne({ _id: chat_id }, { title });
}
export async function updateChatStatus(chat_id: string, status: string, completed: boolean) {
await AiChatModel.updateOne({ _id: chat_id }, {
status,
completed
});
}
export function getChartsInMessage(message: OpenAI.Chat.Completions.ChatCompletionMessageParam) {
if (message.role != 'assistant') return [];
if (!message.tool_calls) return [];
if (message.tool_calls.length == 0) return [];
return message.tool_calls.filter((e: any) => e.function.name === 'createComposableChart').map((e: any) => e.function.arguments);
}
type FunctionCall = { name: string, argsRaw: string[], collecting: boolean, result: any, tool_call_id: string }
type DeltaCallback = (text: string) => any;
type FinishCallback = (functionsCount: number) => any;
type FunctionNameCallback = (name: string) => any;
type FunctionCallCallback = (name: string) => any;
type FunctionResultCallback = (name: string, result: any) => any;
type ElaborateResponseCallbacks = {
onDelta?: DeltaCallback,
onFunctionName?: FunctionNameCallback,
onFunctionCall?: FunctionCallCallback,
onFunctionResult?: FunctionResultCallback,
onFinish?: FinishCallback,
onChatId?: (chat_id: string) => any
}
async function elaborateResponse(messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[], pid: string, time_offset: number, chat_id: string, callbacks?: ElaborateResponseCallbacks) {
console.log('[ELABORATING RESPONSE]');
console.dir(messages, { depth: Infinity });
const responseStream = await openai.beta.chat.completions.stream({ model: OPENAI_MODEL, messages, n: 1, tools });
const functionCalls: FunctionCall[] = [];
let lastFinishReason: "length" | "tool_calls" | "function_call" | "stop" | "content_filter" | null = null;
for await (const part of responseStream) {
const delta = part.choices[0].delta;
const finishReason = part.choices[0].finish_reason;
if (delta.content) await callbacks?.onDelta?.(delta.content);
if (delta.tool_calls) {
for (const toolCall of delta.tool_calls) {
if (!toolCall.function) throw Error('Cannot get function from tool_calls');
const functionName = toolCall.function.name;
const functionCall: FunctionCall = functionName ?
{ name: functionName, argsRaw: [], collecting: true, result: null, tool_call_id: toolCall.id as string } :
functionCalls.at(-1) as FunctionCall;
if (functionName) functionCalls.push(functionCall);
if (functionName) await callbacks?.onFunctionName?.(functionName);
if (toolCall.function.arguments) functionCall.argsRaw.push(toolCall.function.arguments);
}
}
if (finishReason === "tool_calls" && functionCalls.at(-1)?.collecting) {
for (const functionCall of functionCalls) {
await callbacks?.onFunctionCall?.(functionCall.name);
const args = JSON.parse(functionCall.argsRaw.join(''));
const functionResult = await functions[functionCall.name]({ project_id: pid, time_offset, ...args });
functionCall.result = functionResult;
await callbacks?.onFunctionResult?.(functionCall.name, functionResult);
await addMessageToChat({
role: 'assistant',
content: delta.content,
refusal: delta.refusal,
tool_calls: [
{
id: functionCall.tool_call_id, type: 'function',
function: {
name: functionCall.name, arguments: functionCall.argsRaw.join('')
}
}
]
}, chat_id);
await addMessageToChat({ tool_call_id: functionCall.tool_call_id, role: 'tool', content: JSON.stringify(functionCall.result) }, chat_id);
functionCall.collecting = false;
}
lastFinishReason = finishReason;
}
}
await callbacks?.onFinish?.(functionCalls.length);
const toolResponseMesages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = functionCalls.map(e => {
return { tool_call_id: e.tool_call_id, role: "tool", content: JSON.stringify(e.result) }
});
if (lastFinishReason == 'tool_calls') return await elaborateResponse([...responseStream.messages, ...toolResponseMesages], pid, time_offset, chat_id, callbacks);
return responseStream;
}
export async function sendMessageOnChat(text: string, pid: string, time_offset: number, initial_chat_id?: string, callbacks?: ElaborateResponseCallbacks) {
const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = []
const chat_id = await createChatIfNotExist(pid, initial_chat_id);
const chatMessages = await getMessagesFromChatId(chat_id);
await callbacks?.onChatId?.(chat_id);
if (chatMessages && chatMessages.length > 0) {
messages.push(...chatMessages);
await updateChatStatus(chat_id, '', false);
} else {
const roleMessage: OpenAI.Chat.Completions.ChatCompletionMessageParam = {
role: 'system',
content: "You are an AI Data Analyst and Growth Hacker specialized in helping users analyze data collected within Litlyx and providing strategies to grow their website, app, or business. Your scope is strictly limited to data creation, visualization, and growth-related advice. If a user asks something outside this domain, politely inform them that you are not designed to answer such questions. Today ISO date is " + new Date().toISOString() + " take this in count when the user ask relative dates"
}
messages.push(roleMessage);
await addMessageToChat(roleMessage, chat_id);
await setChatTitle(text.substring(0, 110), chat_id);
}
const userMessage: OpenAI.Chat.Completions.ChatCompletionMessageParam = { role: 'user', content: text }
messages.push(userMessage);
await addMessageToChat(userMessage, chat_id);
try {
const streamResponse = await elaborateResponse(messages, pid, time_offset, chat_id, callbacks);
const finalContent = await streamResponse.finalContent();
await addMessageToChat({ role: 'assistant', refusal: null, content: finalContent }, chat_id);
return { content: finalContent, charts: [] };
} catch (ex: any) {
console.error(ex);
return { content: ex.message, charts: [] };
}
// let response = await openai.chat.completions.create({ model: OPENAI_MODEL, messages, n: 1, tools });
// const chartsData: string[][] = [];
// while ((response.choices[0].message.tool_calls?.length || 0) > 0) {
// await addMessageToChat(response.choices[0].message, chat_id);
// messages.push(response.choices[0].message);
// if (response.choices[0].message.tool_calls) {
// console.log('Tools to call', response.choices[0].message.tool_calls.length);
// chartsData.push(getChartsInMessage(response.choices[0].message));
// for (const toolCall of response.choices[0].message.tool_calls) {
// const functionName = toolCall.function.name;
// console.log('Calling tool function', functionName);
// const functionToCall = functions[functionName];
// const functionArgs = JSON.parse(toolCall.function.arguments);
// const functionResponse = await functionToCall({ project_id: pid, ...functionArgs });
// messages.push({ tool_call_id: toolCall.id, role: "tool", content: JSON.stringify(functionResponse) });
// await addMessageToChat({ tool_call_id: toolCall.id, role: "tool", content: JSON.stringify(functionResponse) }, chat_id);
// }
// }
// response = await openai.chat.completions.create({ model: OPENAI_MODEL, messages, n: 1, tools });
// }
// await addMessageToChat(response.choices[0].message, chat_id);
// await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { ai_messages: 1 } })
// return { content: response.choices[0].message.content, charts: chartsData.filter(e => e.length > 0).flat() };
}

86
dashboard/server/services/ApiService.ts
View File

@@ -1,86 +0,0 @@
import { ApiSettingsModel, TApiSettings } from '@schema/ApiSettingsSchema';
import { EventModel } from '@schema/metrics/EventSchema';
import { VisitModel } from '@schema/metrics/VisitSchema';
import type { H3Event, EventHandlerRequest } from 'h3'
export function checkAuthorization(event: H3Event<EventHandlerRequest>) {
const authorization = getHeader(event, 'Authorization');
if (!authorization) return setResponseStatus(event, 403, 'Authorization is required');
const [type, token] = authorization.split(' ');
if (type != 'Bearer') return setResponseStatus(event, 401, 'Malformed authorization');
return token;
}
export type CheckApiKeyResult = { ok: false } | { ok: true, data: TApiSettings };
export async function checkApiKey(apiKey: string): Promise<CheckApiKeyResult> {
const apiSettings = await ApiSettingsModel.findOne({ apiKey });
if (!apiSettings) return { ok: false }
return { ok: true, data: apiSettings }
}
async function incrementApiUsage(apiKey: string, value: number) {
await ApiSettingsModel.updateOne({ apiKey }, { $inc: { usage: value } });
}
async function checkApiUsage(apiKey: string) {
const data = await ApiSettingsModel.findOne({ apiKey }, { usage: 1 });
if (!data) return false;
if (data.usage > 100000) return false;
return true;
}
export type ApiResult = { ok: true, data: any } | { ok: false, code: number, error: string }
export async function eventsListApi(apiKey: string, project_id: string, rows: string[], limit?: number | string, from?: string, to?: string): Promise<ApiResult> {
const canMakeRequest = await checkApiUsage(apiKey);
if (!canMakeRequest) return { ok: false, code: 429, error: 'Api limit reached (100.000)' }
const projection = Object.fromEntries(rows.map(e => [e, 1]));
const limitNumber = parseInt((limit?.toString() as string));
const limitValue = isNaN(limitNumber) ? 100 : limitNumber;
const events = await EventModel.find({
project_id,
created_at: {
$gte: from || new Date(2023, 0),
$lte: to || new Date(3000, 0)
}
}, { _id: 0, ...projection }, { limit: limitValue });
await incrementApiUsage(apiKey, events.length);
return { ok: true, data: events.map(e => e.toJSON()) }
}
export async function visitsListApi(apiKey: string, project_id: string, rows: string[], limit?: number | string, from?: string, to?: string): Promise<ApiResult> {
const canMakeRequest = await checkApiUsage(apiKey);
if (!canMakeRequest) return { ok: false, code: 429, error: 'Api limit reached (100.000)' }
const projection = Object.fromEntries(rows.map(e => [e, 1]));
const limitNumber = parseInt((limit?.toString() as string));
const limitValue = isNaN(limitNumber) ? 100 : limitNumber;
const visits = await VisitModel.find({
project_id,
created_at: {
$gte: from || new Date(2023, 0),
$lte: to || new Date(3000, 0)
}
}, { _id: 0, ...projection }, { limit: limitValue });
await incrementApiUsage(apiKey, visits.length);
return { ok: true, data: visits.map(e => e.toJSON()) };
}

26
dashboard/server/services/AppsumoService.ts
View File

@@ -1,26 +0,0 @@
import { AppsumoCodeModel } from '@schema/appsumo/AppsumoCodeSchema';
import { AppsumoCodeTryModel } from '@schema/appsumo/AppsumoCodeTrySchema';
export async function canTryAppsumoCode(project_id: string) {
const tries = await AppsumoCodeTryModel.findOne({ project_id });
if (!tries) return true;
if (tries.codes.length >= 30) return false;
return true;
}
export async function useTryAppsumoCode(project_id: string, code: string) {
await AppsumoCodeTryModel.updateOne({ project_id }, { $push: { codes: code } }, { upsert: true });
}
export async function checkAppsumoCode(code: string) {
const target = await AppsumoCodeModel.exists({ code, used_at: { $exists: false } });
return target;
}
export async function useAppsumoCode(project_id: string, code: string) {
await AppsumoCodeTryModel.updateOne({ project_id }, { $push: { valid_codes: code } }, { upsert: true });
await AppsumoCodeModel.updateOne({ code }, { used_at: Date.now() });
}

26
dashboard/server/services/CacheService.ts
View File

@@ -3,18 +3,7 @@ import { createClient } from 'redis';
const runtimeConfig = useRuntimeConfig();
export const DATA_EXPIRE_TIME = 30;
export const TIMELINE_EXPIRE_TIME = 60;
export const COUNTS_EXPIRE_TIME = 10;
export const COUNTS_SESSIONS_EXPIRE_TIME = 60 * 3;
export const EVENT_NAMES_EXPIRE_TIME = 60;
export const EVENT_METADATA_FIELDS_EXPIRE_TIME = 30;
type UseCacheV2Callback<T> = (noStore: () => void, updateExp: (value: number) => void) => Promise<T>
type UseCacheCallback<T> = (noStore: () => void, updateExp: (value: number) => void) => Promise<T>
export class Redis {
@@ -56,18 +45,7 @@ export class Redis {
await this.client.del(key);
}
static async useCache<T extends any>(options: { key: string, exp: number }, action: (noStore: () => void) => Promise<T>): Promise<T> {
const cached = await this.get<T>(options.key);
if (cached) return cached;
let storeResult = true;
const noStore = () => storeResult = false;
const result = await action(noStore);
if (!storeResult) return result;
await this.set(options.key, result, options.exp);
return result;
}
static async useCacheV2<T extends any>(key: string, exp: number, callback: UseCacheV2Callback<T>) {
static async useCache<T extends any>(key: string, exp: number, callback: UseCacheCallback<T>) {
const cached = await this.get<T>(key);
if (cached) return cached;
let expireValue = exp;

16
dashboard/server/services/DataService.ts
View File

@@ -1,16 +0,0 @@
import { VisitModel } from "@schema/metrics/VisitSchema";
import { EventModel } from "@schema/metrics/EventSchema";
import type { Model } from "mongoose";
export type TModelName = keyof typeof allowedModels;
export const allowedModels: Record<string, { model: Model<any> }> = {
'events': {
model: EventModel,
},
'visits': {
model: VisitModel,
}
}

18
dashboard/server/services/EmailServiceHelper.ts
View File

@@ -1,18 +0,0 @@
import { EmailServerInfo } from '@services/EmailService'
const { EMAIL_SECRET } = useRuntimeConfig();
export class EmailServiceHelper {
static async sendEmail(data: EmailServerInfo) {
try {
await $fetch(data.url, {
method: 'POST',
body: JSON.stringify(data.body),
headers: { ...data.headers, 'x-litlyx-token': EMAIL_SECRET }
})
} catch (ex) {
console.error(ex);
}
}
}

225
dashboard/server/services/StripeService.ts
View File

@@ -1,225 +0,0 @@
import { getPlanFromId, getPlanFromTag, PREMIUM_TAG } from '@data/PREMIUM';
import Stripe from 'stripe';
class StripeService {
private stripe?: Stripe;
private privateKey?: string;
private webhookSecret?: string;
public testMode?: boolean;
private disabledMode: boolean = false;
init(privateKey: string, webhookSecret: string, testMode: boolean = false) {
this.privateKey = privateKey;
this.webhookSecret = webhookSecret;
this.stripe = new Stripe(this.privateKey);
this.testMode = testMode;
}
disable() { this.disabledMode = true; }
enable() { this.disabledMode = false; }
isDisabled() { return this.disabledMode; }
parseWebhook(body: any, sig: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
if (!this.webhookSecret) {
console.error('Stripe not initialized')
return;
}
return this.stripe.webhooks.constructEvent(body, sig, this.webhookSecret);
}
async createOnetimePayment(price: string, success_url: string, pid: string, customer?: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const checkout = await this.stripe.checkout.sessions.create({
allow_promotion_codes: true,
payment_method_types: ['card'],
invoice_creation: {
enabled: true,
},
line_items: [
{ price, quantity: 1 }
],
payment_intent_data: {
metadata: {
pid, price
}
},
customer,
success_url,
mode: 'payment'
});
return checkout;
}
async createPayment(price: string, success_url: string, pid: string, customer?: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const checkout = await this.stripe.checkout.sessions.create({
allow_promotion_codes: true,
payment_method_types: ['card'],
line_items: [
{ price, quantity: 1 }
],
subscription_data: {
metadata: { pid },
},
customer,
success_url,
mode: 'subscription'
});
return checkout;
}
async getPriceData(priceId: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const priceData = await this.stripe.prices.retrieve(priceId);
return priceData;
}
async deleteSubscription(subscriptionId: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const subscription = await this.stripe.subscriptions.cancel(subscriptionId);
return subscription;
}
async getSubscription(subscriptionId: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const subscription = await this.stripe.subscriptions.retrieve(subscriptionId);
return subscription;
}
async getAllSubscriptions(customer_id: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const subscriptions = await this.stripe.subscriptions.list({ customer: customer_id });
return subscriptions;
}
async getInvoices(customer_id: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const invoices = await this.stripe?.invoices.list({ customer: customer_id });
return invoices;
}
async getCustomer(customer_id: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const customer = await this.stripe.customers.retrieve(customer_id, { expand: [] })
return customer;
}
async createCustomer(email: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const customer = await this.stripe.customers.create({ email });
return customer;
}
async setCustomerInfo(customer_id: string, address: { line1: string, line2: string, city: string, country: string, postal_code: string, state: string }) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const customer = await this.stripe.customers.update(customer_id, {
address: {
line1: address.line1,
line2: address.line2,
city: address.city,
country: address.country,
postal_code: address.postal_code,
state: address.state
}
})
return customer.id;
}
async deleteCustomer(customer_id: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const { deleted } = await this.stripe.customers.del(customer_id);
return deleted;
}
async createStripeCode(plan: PREMIUM_TAG) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const INCUBATION_COUPON = 'sDD7Weh3';
if (plan === 'INCUBATION') {
await this.stripe.promotionCodes.create({
coupon: INCUBATION_COUPON,
active: true,
code: 'TESTCACCA1',
max_redemptions: 1,
})
return true;
}
return false;
}
async createSubscription(customer_id: string, planId: number) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const PLAN = getPlanFromId(planId);
if (!PLAN) throw Error('Plan not found');
const subscription = await this.stripe.subscriptions.create({
customer: customer_id,
items: [
{ price: this.testMode ? PLAN.PRICE_TEST : PLAN.PRICE, quantity: 1 }
],
});
return subscription;
}
async createOneTimeSubscriptionDummy(customer_id: string, planId: number) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const PLAN = getPlanFromId(planId);
if (!PLAN) throw Error('Plan not found');
const subscription = await this.stripe.subscriptions.create({
customer: customer_id,
items: [
{ price: this.testMode ? PLAN.PRICE_TEST : PLAN.PRICE, quantity: 1 }
],
});
return subscription;
}
async createFreeSubscription(customer_id: string) {
if (this.disabledMode) return;
if (!this.stripe) throw Error('Stripe not initialized');
const FREE_PLAN = getPlanFromTag('FREE');
const subscription = await this.stripe.subscriptions.create({
customer: customer_id,
items: [
{ price: this.testMode ? FREE_PLAN.PRICE_TEST : FREE_PLAN.PRICE, quantity: 1 }
]
});
return subscription;
}
}
const instance = new StripeService();
export default instance;

162
dashboard/server/services/TimelineService.ts
View File

@@ -1,18 +1,23 @@
import { Slice } from "@services/DateService";
import DateService from "@services/DateService";
import type mongoose from "mongoose";
import DateService from '@services/DateService';
import * as fns from 'date-fns'
import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc.js';
dayjs.extend(utc);
export type TimelineAggregationOptions = {
projectId: mongoose.Schema.Types.ObjectId | mongoose.Types.ObjectId,
model: mongoose.Model<any>,
from: string | number,
to: string | number,
from: number,
to: number,
slice: Slice,
timeOffset?: number,
debug?: boolean,
domain?: string
explain?: boolean,
domain?: string,
allowDisk?: boolean,
forced?: boolean
}
export type AdvancedTimelineAggregationOptions = TimelineAggregationOptions & {
@@ -24,7 +29,55 @@ export type AdvancedTimelineAggregationOptions = TimelineAggregationOptions & {
customQueries?: { index: number, query: Record<string, any> }[]
}
export async function executeAdvancedTimelineAggregation<T = {}>(options: AdvancedTimelineAggregationOptions) {
export const granularityMap: Record<Slice, string> = {
hour: 'hour',
day: 'day',
month: 'month',
week: 'week',
year: 'year'
}
export function checkSliceValidity(from: number, to: number, slice: Slice): [false, string] | [true, number] {
const days = fns.differenceInDays(new Date(to), new Date(from));
const [min, max] = DateService.sliceAvailabilityMap[slice];
if (days < min) return [false, 'date gap too small for this slice'];
if (days > max) return [false, 'date gap too big for this slice'];
return [true, days];
}
export function prepareTimelineAggregation(options: TimelineAggregationOptions) {
const granularity = granularityMap[options.slice];
if (!granularity) throw createError({ status: 400, message: 'slice not correct' });
if (!options.forced) {
const [sliceValid, errorOrDays] = checkSliceValidity(options.from, options.to, options.slice);
if (!sliceValid) throw createError({ status: 400, message: errorOrDays });
}
const domainMatch: any = {}
if (options.domain) domainMatch.website = options.domain
let from = new Date(options.from);
let to = new Date(options.to);
if (options.slice === 'month') {
from = dayjs(from).utc().startOf('month').toDate()
to = dayjs(to).utc().startOf('month').toDate()
} else if (options.slice === 'hour') {
// from = dayjs(from).utc().startOf('hour').toDate()
// to = dayjs(to).utc().startOf('hour').toDate()
} else if (options.slice === 'day') {
from = dayjs(from).utc().startOf('day').toDate()
to = dayjs(to).utc().startOf('day').toDate()
}
return { granularity, domainMatch, from, to }
}
export async function executeAdvancedTimelineAggregation<T = {}>(options: AdvancedTimelineAggregationOptions): Promise<any[]> {
options.customMatch = options.customMatch || {};
options.customGroup = options.customGroup || {};
@@ -32,16 +85,8 @@ export async function executeAdvancedTimelineAggregation<T = {}>(options: Advanc
options.customIdGroup = options.customIdGroup || {};
options.customQueries = options.customQueries || [];
const { dateFromParts, granularity } = DateService.getGranularityData(options.slice, '$tmpDate');
if (!dateFromParts) throw Error('Slice is probably not correct');
const [sliceValid, errorOrDays] = checkSliceValidity(options.from, options.to, options.slice);
if (!sliceValid) throw Error(errorOrDays);
const timeOffset = options.timeOffset || 0;
const domainMatch: any = {}
if (options.domain) domainMatch.website = options.domain
const { domainMatch, granularity, from, to } = prepareTimelineAggregation(options);
const aggregation = [
{
@@ -55,51 +100,45 @@ export async function executeAdvancedTimelineAggregation<T = {}>(options: Advanc
...options.customMatch
}
},
{
$addFields: {
tmpDate: {
$dateSubtract: {
startDate: "$created_at",
unit: "minute",
amount: timeOffset
}
}
}
},
{
$addFields: { isoDate: { $dateFromParts: dateFromParts } }
},
{
$group: {
_id: { isoDate: "$isoDate", ...options.customIdGroup },
_id: {
date: {
$dateTrunc: { date: "$created_at", unit: granularity, timezone: "UTC" }
},
...options.customIdGroup
},
count: { $sum: 1 },
...options.customGroup
}
},
{
$densify: {
field: "_id.isoDate",
field: "_id.date",
range: {
step: 1,
unit: granularity,
bounds: 'full'
// [
// new Date(new Date(options.from).getTime() - (timeOffset * 1000 * 60)),
// new Date(new Date(options.to).getTime() - (timeOffset * 1000 * 60) + 1),
// ]
bounds: [from, to]
}
}
},
{
$sort: { "_id.isoDate": 1 }
},
{
$addFields: { count: { $ifNull: ["$count", 0] }, }
},
// {
// $addFields: {
// timestamp: { $toLong: "$_id.date" }
// }
// },
// { $set: { count: { $ifNull: ["$count", 0] } } },
// { $sort: { '_id.date': 1 } },
// {
// $project: {
// _id: 1, count: 1, timestamp: 1, ...options.customProjection
// }
// }
{
$project: {
_id: '$_id.isoDate',
count: '$count',
_id: "$_id.date",
count: { $ifNull: ["$count", 0] },
// timestamp: { $toLong: "$_id.date" },
...options.customProjection
}
}
@@ -111,13 +150,18 @@ export async function executeAdvancedTimelineAggregation<T = {}>(options: Advanc
if (options.customAfterMatch) aggregation.splice(1, 0, options.customAfterMatch);
if (options.debug === true) {
if (options.debug === true || options.explain === true) {
console.log('---------- AGGREAGATION ----------')
console.log(JSON.stringify(aggregation, null, 2));
console.log(getPrettyAggregation(aggregation, 2));
}
const timeline: ({ _id: string, count: number } & T)[] = await options.model.aggregate(aggregation);
if (options.explain) {
const explained: any = await options.model.aggregate(aggregation, { allowDiskUse: options.allowDisk ?? false }).explain('executionStats');
return explained;
}
const timeline: ({ _id: { date: string }, count: number, timestamp: number } & T)[] = await options.model.aggregate(aggregation, {
allowDiskUse: options.allowDisk ?? false
})
return timeline;
@@ -127,21 +171,9 @@ export async function executeTimelineAggregation(options: TimelineAggregationOpt
return executeAdvancedTimelineAggregation(options);
}
/**
* @deprecated use fillAndMergeTimelineAggregationV2
*/
export function fillAndMergeTimelineAggregation(timeline: { _id: string, count: number }[], slice: Slice) {
const filledDates = DateService.fillDates(timeline.map(e => e._id), slice);
const merged = DateService.mergeFilledDates(filledDates, timeline, '_id', slice, { count: 0 });
return merged;
}
// export function fillAndMergeTimelineAggregationV2(timeline: { _id: string, count: number }[], slice: Slice, from: string, to: string) {
// const allDates = DateService.generateDateSlices(slice, new Date(from), new Date(to));
// const merged = DateService.mergeDates(timeline, allDates, slice);
// return merged;
// }
export function fillAndMergeTimelineAggregationV2(timeline: { _id: string, count: number }[], slice: Slice, from: string, to: string) {
const allDates = DateService.generateDateSlices(slice, new Date(from), new Date(to));
const merged = DateService.mergeDates(timeline, allDates, slice);
return merged;
}
export function checkSliceValidity(from: string | number | Date, to: string | number | Date, slice: Slice): [false, string] | [true, number] {
return DateService.canUseSlice(from, to, slice);
}

106
dashboard/server/services/ai/AiService.ts Normal file
View File

@@ -0,0 +1,106 @@
import OpenAI from "openai";
import { AiNewChatModel } from "~/shared/schema/ai/AiNewChatSchema";
import { Types } from 'mongoose';
import { Agent } from "./entities/Agent";
import { InsightAgent } from "./entities/InsightAgent";
export const OPENAI_MODEL: OpenAI.Chat.ChatModel = 'gpt-4o-mini';
export type AiMessage =
OpenAI.Chat.Completions.ChatCompletionMessage |
OpenAI.Chat.Completions.ChatCompletionMessageParam |
OpenAI.Chat.Completions.ChatCompletionAssistantMessageParam |
OpenAI.Chat.Completions.ChatCompletionDeveloperMessageParam;
export type AiTool<T extends string = any> =
OpenAI.Chat.Completions.ChatCompletionTool
& { function: { name: T } }
export type AiPlugin<T extends string = any> = {
name: T,
handler: (...args: any[]) => any,
tool: AiTool<T>
}
export type AiHandleUserMessageOptions = {
pid: string,
text: string,
name: string,
chat_id?: string
}
export class AiService {
private static openai: OpenAI;
static init() {
if (this.openai) return this.openai;
const { AI_KEY, AI_ORG, AI_PROJECT } = useRuntimeConfig();
const openai = new OpenAI({ apiKey: AI_KEY, organization: AI_ORG, project: AI_PROJECT });
this.openai = openai;
return openai;
}
static async generateInsight(pid: string) {
const timestamp = ['month', 'week'];
const data = ['visits', 'referrers', 'browsers', 'devices'];
const agent = new InsightAgent(pid);
const dataType = data[Math.floor(Math.random() * data.length)];
const timeFrame = timestamp[Math.floor(Math.random() * timestamp.length)];
const PROMPT = 'Give me one concise, anomaly-focused insight on [DATA_TYPE] for last [TIME_FRAME], compared to 2 [TIME_FRAME] ago. Respond with only the single insight, in plain text, no fluff, no emojis, no extra wording..'
.replace('[DATA_TYPE]', dataType)
.replace('[TIME_FRAME]', timeFrame)
.replace('[TIME_FRAME]', timeFrame);
const res = await agent.reply(PROMPT, []);
return res.at(-1)?.content ?? 'ERROR_GENERATING_INSIGHT';
}
static async handleUserMessage(options: AiHandleUserMessageOptions) {
let chat;
try {
if (options.chat_id) {
const chatUUID = new Types.ObjectId(options.chat_id);
chat = await AiNewChatModel.findOne({ _id: chatUUID });
}
} catch (ex) {
}
let currentMessages: AiMessage[] = [];
if (!chat) {
chat = await AiNewChatModel.create({
title: options.text.substring(0, 60),
status: 'PROCESSING',
messages: [],
deleted: false,
project_id: options.pid
});
} else {
if (!chat.status.startsWith('COMPLETED') && !chat.status.startsWith('ERRORED')) return;
if (chat.messages.length >= 100) {
await AiNewChatModel.updateOne({ _id: chat._id }, { status: 'ERRORED' });
return chat._id.toString();
}
currentMessages.push(...chat.messages);
}
const agent = new Agent({ documentId: chat._id.toString(), pid: options.pid, userName: options.name });
agent.reply(options.text, currentMessages);
return chat._id.toString();
}
}

52
dashboard/server/services/ai/Plugin.ts Normal file
View File

@@ -0,0 +1,52 @@
import * as fns from 'date-fns'
import dateServiceInstance, { Slice } from '~/shared/services/DateService';
export type AiToolTyped<FunctionName extends string, Args extends string[]> = {
type: 'function',
function: {
name: FunctionName,
description?: string,
parameters: {
type: string,
required: Args[number][],
properties: {
[K in Args[number]]: {
type: string,
description: string,
items?: any
}
}
},
strict?: boolean | null
}
}
export function getFirstAvailableSliceFromDates(from: string, to: string) {
const fromTime = new Date(from).getTime();
const toTime = new Date(to).getTime() + 1000;
const days = fns.differenceInDays(toTime, fromTime);
const keys = Object.keys(dateServiceInstance.sliceAvailabilityMap);
const targetKey = keys.find((key: any) => {
const data = ((dateServiceInstance.sliceAvailabilityMap as any)[key]) as [number, number];
return days > data[0] && days < data[1];
});
return targetKey as Slice;
}
type AiPluginHandlerData<Args extends string[]> = { project_id: string } & { [K in Args[number]]: any }
export class AiPlugin<Name extends string = any, Args extends string[] = any> {
constructor(
public name: Name,
public tool: AiToolTyped<Name, Args>,
public handler: (data: AiPluginHandlerData<Args>) => any
) { }
}

122
dashboard/server/services/ai/entities/Agent.ts Normal file
View File

@@ -0,0 +1,122 @@
import { AiMessage, AiPlugin, AiService, OPENAI_MODEL } from "../AiService";
import { AiNewChatModel } from "~/shared/schema/ai/AiNewChatSchema";
import { visitsPlugins } from "../plugins/VisitsPlugins";
import { sessionsPlugins } from "../plugins/SessionsPlugin";
import { dataPlugins } from "../plugins/DataPlugin";
import { chartPlugins } from "../plugins/ChartPlugin";
import { utmDataPlugins } from "../plugins/UTMData";
import { bouncingRatePlugins } from "../plugins/BouncingRatePlugin";
export const AI_PLUGINS: AiPlugin[] = [
...visitsPlugins,
...sessionsPlugins,
...dataPlugins,
// ...chartPlugins,
...utmDataPlugins,
...bouncingRatePlugins
];
const DEFAULT_PROMPT = `You are an AI analytics agent that transforms structured data from function calls into clear, growth-focused insights, acting like a startup growth analyst explaining results to a founder. You analyze data on visitors, page views, sessions, bounce rates, session duration, and traffic sources, highlighting trends, anomalies, and comparisons in plain language. You identify growth opportunities, funnel bottlenecks, user behavior patterns, and friction points, and suggest concrete, high-impact experiments, quick wins, or growth loops based strictly on the data. Your style is concise, actionable, and easy to understand; you are creative in insights but never in data, you avoid generic advice, and you tailor every suggestion to the dataset provided. Keep initial answers brief unless the user explicitly requests deeper detail, and always end with exactly one specific follow-up question in this format: “Would you like me to analyze [specific aspect]?” Stay strictly within the domain of website and product analytics, respond honestly if something goes beyond your scope, and always prioritize clarity, ROI, and relevance in every response.`
export type AgentConstructorOptions = {
userName: string,
pid: string,
documentId: string
}
export class Agent {
constructor(private options: AgentConstructorOptions) { }
async onStartThinking() {
await AiNewChatModel.updateOne({ _id: this.options.documentId }, { status: `THINKING:Agent` });
}
async onStartFunctionCall() {
await AiNewChatModel.updateOne({ _id: this.options.documentId }, { status: `FUNCTION:Agent` });
}
async onChatCompleted() {
await AiNewChatModel.updateOne({ _id: this.options.documentId }, { status: `COMPLETED` });
}
async onChatErrored(error: string) {
await AiNewChatModel.updateOne({ _id: this.options.documentId }, { status: `ERRORED` });
}
async onNewMessage(message: AiMessage) {
if (message.role === 'system') return;
const messageWithDate = { ...message, created_at: new Date() }
await AiNewChatModel.updateOne({ _id: this.options.documentId }, {
$push: { messages: messageWithDate }
});
}
async processTools(message: AiMessage, chat: AiMessage[]) {
if (message.role != 'assistant') return;
const tool_calls = message.tool_calls;
if (!tool_calls) return;
for (const toolCall of tool_calls) {
const functionName = toolCall.function.name;
const targetFunction = AI_PLUGINS.find(e => e.name === functionName);
if (!targetFunction) return;
const args = JSON.parse(toolCall.function.arguments);
const result = await targetFunction.handler({ ...args, project_id: this.options.pid });
const message: AiMessage = { role: 'tool', tool_call_id: toolCall.id, content: JSON.stringify(result) };
chat.push(message)
await this.onNewMessage(message);
}
}
async reply(userText: string, chat: AiMessage[]) {
chat.push({ role: 'system', content: `Current iso date is: ${new Date().toISOString()}` });
const user_message: AiMessage = {
role: 'user',
content: userText,
name: this.options.userName
};
chat.push(user_message);
await this.onNewMessage(user_message);
await this.onStartThinking();
chat.push({ role: 'system', content: DEFAULT_PROMPT + '. Reply in MD format if possible. Try to make the response short. Do not suggest analytics tools that are not Litlyx.' });
const openai = await AiService.init();
const response = await openai.chat.completions.create({
model: OPENAI_MODEL,
messages: chat,
tools: AI_PLUGINS.map(e => e.tool)
});
const choice = response.choices[0];
if (choice.finish_reason === 'tool_calls') {
await this.onStartFunctionCall();
const chatMessage: AiMessage = { ...choice.message, name: 'Agent' };
chat.push(chatMessage);
await this.onNewMessage(chatMessage);
await this.processTools(chatMessage, chat);
await this.onStartThinking();
const afterToolResponse = await openai.chat.completions.create({ model: OPENAI_MODEL, messages: chat });
const afterToolChatMessage: AiMessage = { ...afterToolResponse.choices[0].message, name: 'Agent' };
chat.push(afterToolChatMessage);
await this.onNewMessage(afterToolChatMessage);
} else {
const chatMessage: AiMessage = { ...choice.message, name: 'Agent' };
chat.push(chatMessage);
await this.onNewMessage(chatMessage);
}
await this.onChatCompleted();
}
}

74
dashboard/server/services/ai/entities/InsightAgent.ts Normal file
View File

@@ -0,0 +1,74 @@
import { AiMessage, AiPlugin, AiService, OPENAI_MODEL } from "../AiService";
import { visitsPlugins } from "../plugins/VisitsPlugins";
import { sessionsPlugins } from "../plugins/SessionsPlugin";
import { dataPlugins } from "../plugins/DataPlugin";
import { utmDataPlugins } from "../plugins/UTMData";
import { bouncingRatePlugins } from "../plugins/BouncingRatePlugin";
export const AI_PLUGINS: AiPlugin[] = [
...visitsPlugins,
...sessionsPlugins,
...dataPlugins,
...utmDataPlugins,
...bouncingRatePlugins
];
const DEFAULT_PROMPT = `No fluff, no emojis, no extra words. Compare with the previous period if possible and output only the single insight.`
export class InsightAgent {
constructor(private pid: string) { }
async processTools(message: AiMessage, chat: AiMessage[]) {
if (message.role != 'assistant') return;
const tool_calls = message.tool_calls;
if (!tool_calls) return;
for (const toolCall of tool_calls) {
const functionName = toolCall.function.name;
const targetFunction = AI_PLUGINS.find(e => e.name === functionName);
if (!targetFunction) return;
const args = JSON.parse(toolCall.function.arguments);
const result = await targetFunction.handler({ ...args, project_id: this.pid });
const message: AiMessage = { role: 'tool', tool_call_id: toolCall.id, content: JSON.stringify(result) };
chat.push(message)
}
}
async reply(userText: string, chat: AiMessage[]) {
chat.push({ role: 'system', content: `Current iso date is: ${new Date().toISOString()}` });
const user_message: AiMessage = {
role: 'user',
content: userText,
};
chat.push(user_message);
chat.push({ role: 'system', content: DEFAULT_PROMPT });
const openai = await AiService.init();
const response = await openai.chat.completions.create({
model: OPENAI_MODEL,
messages: chat,
tools: AI_PLUGINS.map(e => e.tool)
});
const choice = response.choices[0];
if (choice.finish_reason === 'tool_calls') {
const chatMessage: AiMessage = { ...choice.message, name: 'Agent' };
chat.push(chatMessage);
await this.processTools(chatMessage, chat);
const afterToolResponse = await openai.chat.completions.create({ model: OPENAI_MODEL, messages: chat });
const afterToolChatMessage: AiMessage = { ...afterToolResponse.choices[0].message, name: 'Agent' };
chat.push(afterToolChatMessage);
} else {
const chatMessage: AiMessage = { ...choice.message, name: 'Agent' };
chat.push(chatMessage);
}
return chat;
}
}

153
dashboard/server/services/ai/plugins/BouncingRatePlugin.ts Normal file
View File

@@ -0,0 +1,153 @@
import { Types } from "mongoose";
import { AiPlugin, getFirstAvailableSliceFromDates } from "../Plugin";
import { VisitModel } from "~/shared/schema/metrics/VisitSchema";
import { prepareTimelineAggregation } from "../../TimelineService";
const getBouncingRatePlugin = new AiPlugin<'getBouncingRate', ['from', 'to', 'domain', 'limit']>('getBouncingRate',
{
type: 'function',
function: {
name: 'getBouncingRate',
description: 'Gets an array of bouncing rate in the user website on a date range.',
parameters: {
type: 'object',
properties: {
from: { type: 'string', description: 'ISO string of start date' },
to: { type: 'string', description: 'ISO string of end date' },
domain: { type: 'string', description: 'Used only to filter a specific webdomain/website' },
limit: { type: 'number', description: 'Max number of items to return' }
},
required: ['from', 'to']
}
}
},
async (data) => {
const info = prepareTimelineAggregation({
model: VisitModel,
projectId: new Types.ObjectId(data.project_id),
from: new Date(data.from).getTime(),
to: new Date(data.to).getTime(),
slice: getFirstAvailableSliceFromDates(data.from, data.to),
domain: data.domain,
});
const aggregation = [
{
$match: {
project_id: new Types.ObjectId(data.project_id),
created_at: {
$gte: new Date(data.from),
$lte: new Date(data.to)
},
...info.domainMatch,
}
},
{
$project: {
created_at: 1, session: 1
}
},
{
$addFields: {
date: {
$dateTrunc: {
date: "$created_at",
unit: info.granularity,
timezone: "UTC"
}
}
}
},
{
$group: {
_id: {
date: "$date",
session: "$session"
},
pageViews: {
$sum: 1
}
}
},
{
$group: {
_id: {
date: "$_id.date"
},
totalSessions: {
$sum: 1
},
bouncedSessions: {
$sum: { $cond: [{ $eq: ["$pageViews", 1] }, 1, 0] }
}
}
},
{
$project: {
_id: 1,
totalSessions: 1,
bouncedSessions: 1,
bounceRate: {
$cond: [{ $eq: ["$totalSessions", 0] }, 0,
{
$multiply: [
{
$divide: [
"$bouncedSessions",
"$totalSessions"
]
},
100
]
}
]
}
}
},
{
$densify: {
field: "_id.date",
range: {
step: 1,
unit: info.granularity,
bounds: [
info.from,
info.to
]
}
}
},
{
$addFields: {
timestamp: {
$toLong: "$_id.date"
}
}
},
{
$set: {
count: {
$ifNull: ["$bounceRate", 0]
}
}
},
{
$sort: {
"_id.date": 1
}
},
{
$project: { _id: 1, count: 1, timestamp: 1 }
}
] as any[];
const result = await VisitModel.aggregate(aggregation, { allowDiskUse: true });
return result;
}
);
export const bouncingRatePlugins = [
getBouncingRatePlugin
]

63
dashboard/server/services/ai/plugins/ChartPlugin.ts Normal file
View File

@@ -0,0 +1,63 @@
import { AiPlugin } from "../Plugin";
const createChartPlugin = new AiPlugin<'createChart', ['labels', 'title', 'datasets']>('createChart',
{
type: 'function',
function: {
name: 'createChart',
description: 'Creates a chart based on the provided datasets. Used to show a visual rappresentation of data to the user.',
parameters: {
type: 'object',
properties: {
labels: {
type: 'array',
items: { type: 'string' },
description: 'Labels for each data point in the chart'
},
title: {
type: 'string',
description: 'Title of the chart to let user understand what is displaying, not include dates'
},
datasets: {
type: 'array',
description: 'List of datasets',
items: {
type: 'object',
properties: {
chartType: {
type: 'string',
enum: ['line', 'bar'],
description: 'The type of chart to display the dataset, either "line" or "bar"'
},
points: {
type: 'array',
items: { type: 'number' },
description: 'Numerical values for each data point in the chart'
},
color: {
type: 'string',
description: 'Color used to represent the dataset in format "#RRGGBB"'
},
name: {
type: 'string',
description: 'Name of the dataset'
}
},
required: ['points', 'chartType', 'name'],
description: 'Data points and style information for the dataset'
}
}
},
required: ['labels', 'datasets', 'title']
}
}
},
async (data) => {
return { ok: true, chart: true }
}
);
export const chartPlugins = [
createChartPlugin
]

253
dashboard/server/services/ai/plugins/DataPlugin.ts Normal file
View File

@@ -0,0 +1,253 @@
import { Types } from "mongoose";
import { AiPlugin } from "../Plugin";
import { VisitModel } from "~/shared/schema/metrics/VisitSchema";
import { getDomainFromString } from "~/server/utils/getRequestContext";
const getReferrersPlugin = new AiPlugin<'getReferrers', ['from', 'to', 'domain', 'limit']>('getReferrers',
{
type: 'function',
function: {
name: 'getReferrers',
description: 'Gets an array of website referrers (visit sources) on a date range.',
parameters: {
type: 'object',
properties: {
from: { type: 'string', description: 'ISO string of start date' },
to: { type: 'string', description: 'ISO string of end date' },
domain: { type: 'string', description: 'Used only to filter a specific webdomain/website' },
limit: { type: 'number', description: 'Max number of items to return' }
},
required: ['from', 'to']
}
}
},
async (data) => {
const result = await VisitModel.aggregate([
{
$match: {
project_id: new Types.ObjectId(data.project_id),
created_at: {
$gte: new Date(data.from),
$lte: new Date(data.to)
},
website: getDomainFromString(data.domain ?? '*') ?? { $ne: null },
}
},
{ $group: { _id: "$referrer", count: { $sum: 1, } } },
{ $sort: { count: -1 } },
{ $limit: Math.min(data.limit ?? 500, 500) }
]);
return result as { _id: string, count: number }[];
}
);
const getContinentsPlugin = new AiPlugin<'getContinents', ['from', 'to', 'domain', 'limit']>('getContinents',
{
type: 'function',
function: {
name: 'getContinents',
description: 'Gets an array of continents that visited the user website on a date range.',
parameters: {
type: 'object',
properties: {
from: { type: 'string', description: 'ISO string of start date' },
to: { type: 'string', description: 'ISO string of end date' },
domain: { type: 'string', description: 'Used only to filter a specific webdomain/website' },
limit: { type: 'number', description: 'Max number of items to return' }
},
required: ['from', 'to']
}
}
},
async (data) => {
const result = await VisitModel.aggregate([
{
$match: {
project_id: new Types.ObjectId(data.project_id),
created_at: {
$gte: new Date(data.from),
$lte: new Date(data.to)
},
website: getDomainFromString(data.domain ?? '*') ?? { $ne: null },
}
},
{ $group: { _id: "$continent", count: { $sum: 1, } } },
{ $sort: { count: -1 } },
{ $limit: Math.min(data.limit ?? 500, 500) }
]);
return result as { _id: string, count: number }[];
}
);
const getCountriesPlugin = new AiPlugin<'getCountries', ['from', 'to', 'domain', 'limit']>('getCountries',
{
type: 'function',
function: {
name: 'getCountries',
description: 'Gets an array of countries that visited the user website on a date range.',
parameters: {
type: 'object',
properties: {
from: { type: 'string', description: 'ISO string of start date' },
to: { type: 'string', description: 'ISO string of end date' },
domain: { type: 'string', description: 'Used only to filter a specific webdomain/website' },
limit: { type: 'number', description: 'Max number of items to return' }
},
required: ['from', 'to']
}
}
},
async (data) => {
const result = await VisitModel.aggregate([
{
$match: {
project_id: new Types.ObjectId(data.project_id),
created_at: {
$gte: new Date(data.from),
$lte: new Date(data.to)
},
website: getDomainFromString(data.domain ?? '*') ?? { $ne: null },
}
},
{ $group: { _id: "$country", count: { $sum: 1, } } },
{ $sort: { count: -1 } },
{ $limit: Math.min(data.limit ?? 500, 500) }
]);
return result as { _id: string, count: number }[];
}
);
const getPagesPlugin = new AiPlugin<'getPages', ['from', 'to', 'domain', 'limit']>('getPages',
{
type: 'function',
function: {
name: 'getPages',
description: 'Gets an array of most visited pages on a date range.',
parameters: {
type: 'object',
properties: {
from: { type: 'string', description: 'ISO string of start date' },
to: { type: 'string', description: 'ISO string of end date' },
domain: { type: 'string', description: 'Used only to filter a specific webdomain/website' },
limit: { type: 'number', description: 'Max number of items to return' }
},
required: ['from', 'to']
}
}
},
async (data) => {
const result = await VisitModel.aggregate([
{
$match: {
project_id: new Types.ObjectId(data.project_id),
created_at: {
$gte: new Date(data.from),
$lte: new Date(data.to)
},
website: getDomainFromString(data.domain ?? '*') ?? { $ne: null },
}
},
{ $group: { _id: "$page", count: { $sum: 1, } } },
{ $sort: { count: -1 } },
{ $limit: Math.min(data.limit ?? 500, 500) }
]);
return result as { _id: string, count: number }[];
}
);
const getBrowsersPlugin = new AiPlugin<'getBrowsers', ['from', 'to', 'domain', 'limit']>('getBrowsers',
{
type: 'function',
function: {
name: 'getBrowsers',
description: 'Gets an array of browsers that visited the user website on a date range.',
parameters: {
type: 'object',
properties: {
from: { type: 'string', description: 'ISO string of start date' },
to: { type: 'string', description: 'ISO string of end date' },
domain: { type: 'string', description: 'Used only to filter a specific webdomain/website' },
limit: { type: 'number', description: 'Max number of items to return' }
},
required: ['from', 'to']
}
}
},
async (data) => {
const result = await VisitModel.aggregate([
{
$match: {
project_id: new Types.ObjectId(data.project_id),
created_at: {
$gte: new Date(data.from),
$lte: new Date(data.to)
},
website: getDomainFromString(data.domain ?? '*') ?? { $ne: null },
}
},
{ $group: { _id: "$browser", count: { $sum: 1, } } },
{ $sort: { count: -1 } },
{ $limit: Math.min(data.limit ?? 500, 500) }
]);
return result as { _id: string, count: number }[];
}
);
const getDevicesPlugin = new AiPlugin<'getDevices', ['from', 'to', 'domain', 'limit']>('getDevices',
{
type: 'function',
function: {
name: 'getDevices',
description: 'Gets an array of devices that visited the user website on a date range.',
parameters: {
type: 'object',
properties: {
from: { type: 'string', description: 'ISO string of start date' },
to: { type: 'string', description: 'ISO string of end date' },
domain: { type: 'string', description: 'Used only to filter a specific webdomain/website' },
limit: { type: 'number', description: 'Max number of items to return' }
},
required: ['from', 'to']
}
}
},
async (data) => {
const result = await VisitModel.aggregate([
{
$match: {
project_id: new Types.ObjectId(data.project_id),
created_at: {
$gte: new Date(data.from),
$lte: new Date(data.to)
},
website: getDomainFromString(data.domain ?? '*') ?? { $ne: null },
}
},
{ $group: { _id: "$device", count: { $sum: 1, } } },
{ $sort: { count: -1 } },
{ $limit: Math.min(data.limit ?? 500, 500) }
]);
return result as { _id: string, count: number }[];
}
);
export const dataPlugins = [
getReferrersPlugin,
getContinentsPlugin,
getCountriesPlugin,
getPagesPlugin,
getBrowsersPlugin,
getDevicesPlugin
]

52
dashboard/server/services/ai/plugins/SessionsPlugin.ts Normal file
View File

@@ -0,0 +1,52 @@
import { executeAdvancedTimelineAggregation } from "../../TimelineService";
import { Types } from "mongoose";
import { AiPlugin, getFirstAvailableSliceFromDates } from "../Plugin";
import { VisitModel } from "~/shared/schema/metrics/VisitSchema";
const getSessionsTimelinePlugin = new AiPlugin<'getSessionsTimeline', ['from', 'to', 'page', 'domain']>(
'getSessionsTimeline',
{
type: 'function',
function: {
name: 'getSessionsTimeline',
description: 'Gets an array of date and count for sessions (unique visitors) received on a date range.',
parameters: {
type: 'object',
properties: {
from: { type: 'string', description: 'ISO string of start date' },
to: { type: 'string', description: 'ISO string of end date' },
page: { type: 'string', description: 'The page of the visit' },
domain: { type: 'string', description: 'Used only to filter a specific domain/website' }
},
required: ['from', 'to']
}
}
},
async (data) => {
const timelineData = await executeAdvancedTimelineAggregation({
projectId: new Types.ObjectId(data.project_id),
model: VisitModel,
from: new Date(data.from).getTime(),
to: new Date(data.to).getTime(),
slice: getFirstAvailableSliceFromDates(data.from, data.to),
domain: data.domain,
customIdGroup: { count: '$session' },
customQueries: [
{
index: 2,
query: {
$group: { _id: { date: '$_id.date' }, count: { $sum: 1 } }
}
}
]
});
return timelineData;
}
);
export const sessionsPlugins = [
getSessionsTimelinePlugin
]

66
dashboard/server/services/ai/plugins/UTMData.ts Normal file
View File

@@ -0,0 +1,66 @@
import { Types } from "mongoose";
import { AiPlugin } from "../Plugin";
import { VisitModel } from "~/shared/schema/metrics/VisitSchema";
import { getDomainFromString } from "~/server/utils/getRequestContext";
import { UtmKey } from "~/components/complex/line-data/selectors/SelectRefer.vue";
const utmKeys = ["utm_medium", "utm_source", "utm_term", "utm_campaign", "utm_content"]
const plugins: AiPlugin[] = []
for (const utmKey of utmKeys) {
const getUtmPlugin = new AiPlugin<`get_${typeof utmKey}`, ['domain', 'from', 'to', 'limit']>(`get_${utmKey}`,
{
type: 'function',
function: {
name: `get_${utmKey}`,
parameters: {
type: 'object',
properties: {
from: { type: 'string', description: 'ISO string of start date' },
to: { type: 'string', description: 'ISO string of end date' },
domain: { type: 'string', description: 'Used only to filter a specific webdomain/website' },
limit: { type: 'number', description: 'Max number of items to return' }
},
required: ['from', 'to']
},
description: `Gets an array of all the ${utmKey} of the visits in a date range`,
}
},
async ({ domain, from, to, project_id, limit }) => {
const websiteMatch = domain ? { website: domain } : {};
const result = await VisitModel.aggregate([
{
$match: {
project_id: new Types.ObjectId(project_id),
created_at: { $gte: new Date(from), $lte: new Date(to) },
...websiteMatch,
[utmKey]: { $ne: null }
}
},
{
$group: {
_id: `$${utmKey}`,
count: { $sum: 1 }
}
},
{ $sort: { count: -1 } },
{ $limit: Math.min(limit ?? 500, 500) }
]);
return result as { _id: string, count: number }[];
});
plugins.push(getUtmPlugin);
}
export const utmDataPlugins = plugins;

58
dashboard/server/services/ai/plugins/VisitsPlugins.ts Normal file
View File

@@ -0,0 +1,58 @@
import { executeAdvancedTimelineAggregation } from "../../TimelineService";
import { Types } from "mongoose";
import { getFirstAvailableSliceFromDates } from "../Plugin";
import { AiPlugin } from "../Plugin";
import { VisitModel } from "~/shared/schema/metrics/VisitSchema";
const getVisitsTimelinePlugin = new AiPlugin<'getVisitsTimeline', ['from', 'to', 'page', 'domain', 'continent', 'country', 'region', 'city', 'device']>(
'getVisitsTimeline',
{
type: 'function',
function: {
name: 'getVisitsTimeline',
description: 'Gets an array of date and count for visits received on a date range. Can be filtered for domain, continent, country, region, city, devices.',
parameters: {
type: 'object',
properties: {
from: { type: 'string', description: 'ISO string of start date' },
to: { type: 'string', description: 'ISO string of end date' },
page: { type: 'string', description: 'The page of the visit' },
domain: { type: 'string', description: 'Used only to filter a specific domain/website' },
device: { type: 'string', description: 'Used only to filter a specific device' },
continent: { type: 'string', description: 'Used only to filter a specific continent - 2 letters' },
country: { type: 'string', description: 'Used only to filter a specific country - 2 letters' },
region: { type: 'string', description: 'Used only to filter a specific region - 2 letters' },
city: { type: 'string', description: 'Used only to filter a specific city - 2 letters' },
},
required: ['from', 'to']
}
}
},
async (data) => {
const match: Record<string, string> = {}
if (data.device) match.device = data.device;
if (data.continent) match.continent = data.continent;
if (data.country) match.country = data.country;
if (data.region) match.region = data.region;
if (data.city) match.city = data.city;
const timelineData = await executeAdvancedTimelineAggregation({
projectId: new Types.ObjectId(data.project_id),
model: VisitModel,
from: new Date(data.from).getTime(),
to: new Date(data.to).getTime(),
slice: getFirstAvailableSliceFromDates(data.from, data.to),
domain: data.domain,
customMatch: match
});
return timelineData;
}
);
export const visitsPlugins = [
getVisitsTimelinePlugin
]