aggregation optimization

dashboard/.gitignore

@@ -32,3 +32,7 @@ out.pdf
 
 # TESTS - TO REMOVE
 tests
+
+# EXPLAINS MONGODB
+
+explains

dashboard/pages/index.vue

@@ -100,7 +100,8 @@ function goToUpgrade() {
                             Limit reached
                         </div>
                         <div class="poppins text-[#fbbf24]">
-                            Litlyx cannot receive new data as you reached your plan's limit. Resume all the great features and collect even more data with a higher plan.
+                            Litlyx cannot receive new data as you reached your plan's limit. Resume all the great
+                            features and collect even more data with a higher plan.
                         </div>
                     </div>
                     <div>

dashboard/server/api/metrics/[project_id]/timeline/visits.post.ts

@@ -4,6 +4,7 @@ import { getUserProjectFromId } from "~/server/LIVE_DEMO_DATA";
 import DateService from "@services/DateService";
 import { executeTimelineAggregation, fillAndMergeTimelineAggregation } from "~/server/services/TimelineService";
 
+import fs from 'fs';
 
 export default defineEventHandler(async event => {
     const project_id = getRequestProjectId(event);
@@ -20,19 +21,24 @@ export default defineEventHandler(async event => {
     if (!to) return setResponseStatus(event, 400, 'to is required');
     if (!slice) return setResponseStatus(event, 400, 'slice is required');
 
-    return await Redis.useCache({
-        key: `timeline:visits:${project_id}:${slice}:${from || 'none'}:${to || 'none'}`,
-        exp: TIMELINE_EXPIRE_TIME
-    }, async () => {
+    // return await Redis.useCache({
+    //     key: `timeline:visits:${project_id}:${slice}:${from || 'none'}:${to || 'none'}`,
+    //     exp: TIMELINE_EXPIRE_TIME,
+    // }, async () => {
     const timelineData = await executeTimelineAggregation({
         projectId: project._id,
         model: VisitModel,
-            from, to, slice
-        });
-        const timelineFilledMerged = fillAndMergeTimelineAggregation(timelineData, slice);
-        return timelineFilledMerged;
+        from, to, slice,
+        debug: true
     });
 
+    console.log(timelineData);
+    fs.writeFileSync('explains/timeline-visits.json', JSON.stringify(timelineData));
+
+    // const timelineFilledMerged = fillAndMergeTimelineAggregation(timelineData, slice);
+    // return timelineFilledMerged;
+    // });
+
 
 
 

dashboard/server/services/TimelineService.ts

@@ -46,7 +46,7 @@ export async function executeAdvancedTimelineAggregation<T = {}>(options: Advanc
         console.log(JSON.stringify(aggregation, null, 2));
     }
 
-    const timeline: { _id: string, count: number & T }[] = await options.model.aggregate(aggregation);
+    const timeline: { _id: string, count: number & T }[] = await options.model.aggregate(aggregation, { explain: options.debug === true });
 
     return timeline;
 

shared/schema/metrics/EventSchema.ts

@@ -15,7 +15,7 @@ const EventSchema = new Schema<TEvent>({
     metadata: Schema.Types.Mixed,
     session: { type: String },
     flowHash: { type: String },
-    created_at: { type: Date, default: () => Date.now() },
+    created_at: { type: Date, default: () => Date.now(), index: true },
 })
 
 export const EventModel = model<TEvent>('events', EventSchema);

shared/schema/metrics/SessionSchema.ts

@@ -16,7 +16,7 @@ const SessionSchema = new Schema<TSession>({
     flowHash: { type: String },
     duration: { type: Number, required: true, default: 0 },
     updated_at: { type: Date, default: () => Date.now() },
-    created_at: { type: Date, default: () => Date.now() },
+    created_at: { type: Date, default: () => Date.now(), index: true },
 })
 
 export const SessionModel = model<TSession>('sessions', SessionSchema);