Merge branch 'dev'

dashboard/components/dashboard/SessionsLineChart.vue

@@ -1,20 +1,20 @@
 <script lang="ts" setup>
 import { onMounted } from 'vue';
 
+import DateService, { type Slice } from '@services/DateService';
+
 const data = ref<number[]>([]);
 const labels = ref<string[]>([]);
 const ready = ref<boolean>(false);
-const key = ref<string>('0');
 
-const props = defineProps<{ slice: SliceName }>();
+const props = defineProps<{ slice: Slice }>();
 
 async function loadData() {
-    const response = await useTimelineData('sessions', props.slice);
+    const response = await useTimeline('sessions', props.slice);
     if (!response) return;
-    data.value = response.data;
-    labels.value = response.labels;
+    data.value = response.map(e => e.count);
+    labels.value = response.map(e => DateService.getChartLabelFromISO(e._id, navigator.language, props.slice));
     ready.value = true;
-    key.value = Date.now().toString();
 }
 
 onMounted(async () => {

dashboard/components/dashboard/VisitsLineChart.vue

@@ -1,19 +1,18 @@
 <script lang="ts" setup>
 import { onMounted } from 'vue';
+import DateService, { type Slice } from '@services/DateService';
 
 const data = ref<number[]>([]);
 const labels = ref<string[]>([]);
 const ready = ref<boolean>(false);
 
-const props = defineProps<{ slice: SliceName }>();
+const props = defineProps<{ slice: Slice }>();
 
 async function loadData() {
-    const response = await useTimelineDataRaw('visits', props.slice);
+    const response = await useTimeline('visits', props.slice);
     if (!response) return;
-    const fixed = fixMetrics(response, props.slice);
-    console.log(fixed);
-    data.value = fixed.data;
-    labels.value = fixed.labels;
+    data.value = response.map(e => e.count);
+    labels.value = response.map(e => DateService.getChartLabelFromISO(e._id, navigator.language, props.slice));
     ready.value = true;
 }
 

dashboard/components/referrer/ReferrerBarChart.vue

@@ -1,29 +1,19 @@
 <script lang="ts" setup>
 import { onMounted } from 'vue';
+import DateService, { type Slice } from '@services/DateService';
 
 const data = ref<number[]>([]);
 const labels = ref<string[]>([]);
 const ready = ref<boolean>(false);
-const props = defineProps<{ slice: SliceName, referrer: string }>();
 
-const activeProject = useActiveProject();
+const props = defineProps<{ slice: Slice, referrer: string }>();
 
 async function loadData() {
-
-    const response = await $fetch(`/api/metrics/${activeProject.value?._id.toString()}/timeline/referrers`, {
-        method: 'POST',
-        ...signHeaders({ 'Content-Type': 'application/json' }),
-        body: JSON.stringify({ slice: 'day', referrer: props.referrer })
-    });
-
+    const response = await useReferrersTimeline(props.referrer, props.slice);
     if (!response) return;
-
-    const fixed = fixMetrics(response, props.slice);
-    console.log(fixed);
-    data.value = fixed.data;
-    labels.value = fixed.labels;
+    data.value = response.map(e => e.count);
+    labels.value = response.map(e => DateService.getChartLabelFromISO(e._id, navigator.language, props.slice));
     ready.value = true;
-
 }
 
 onMounted(async () => {

dashboard/composables/useDataService.ts

@@ -1,3 +1,5 @@
+import type { Slice } from "@services/DateService";
+import DateService from "@services/DateService";
 import type { MetricsCounts } from "~/server/api/metrics/[project_id]/counts";
 import type { VisitsWebsiteAggregated } from "~/server/api/metrics/[project_id]/data/websites";
 import type { MetricsTimeline } from "~/server/api/metrics/[project_id]/timeline/generic";
@@ -14,6 +16,38 @@ export function useFirstInteractionData() {
     return metricsInfo;
 }
 
+
+export async function useTimelineAdvanced(endpoint: string, slice: Slice, fromDate?: string, toDate?: string, customBody: Object = {}) {
+
+    const { from, to } = DateService.prepareDateRange(
+        fromDate || DateService.getDefaultRange(slice).from,
+        toDate || DateService.getDefaultRange(slice).to,
+        slice
+    );
+
+    const activeProject = useActiveProject();
+    const response = await $fetch(
+        `/api/metrics/${activeProject.value?._id}/timeline/${endpoint}`, {
+        method: 'POST',
+        ...signHeaders({ 'Content-Type': 'application/json' }),
+        body: JSON.stringify({ slice, from, to, ...customBody })
+    });
+
+    return response as { _id: string, count: number }[];
+
+}
+
+
+export async function useTimeline(endpoint: 'visits' | 'sessions' | 'referrers', slice: Slice, fromDate?: string, toDate?: string) {
+    return await useTimelineAdvanced(endpoint, slice, fromDate, toDate, {});
+}
+
+export async function useReferrersTimeline(referrer: string, slice: Slice, fromDate?: string, toDate?: string) {
+    return await useTimelineAdvanced('referrers', slice, fromDate, toDate, { referrer });
+}
+
+
+
 export async function useTimelineDataRaw(timelineEndpointName: string, slice: SliceName) {
     const activeProject = useActiveProject();
 

dashboard/pages/index.vue

@@ -63,7 +63,8 @@ watch(pending, () => {
 
 const selectLabels = [
     { label: 'Hour', value: 'hour' },
-    { label: 'Day', value: 'day' }
+    { label: 'Day', value: 'day' },
+    // { label: 'Month', value: 'month' },
 ];
 
 

dashboard/server/api/metrics/[project_id]/timeline/referrers.post.ts

@@ -1,8 +1,9 @@
 import { getTimeline } from "./generic";
 import { VisitModel } from "@schema/metrics/VisitSchema";
+import DateService from "@services/DateService";
 import { Redis, TIMELINE_EXPIRE_TIME } from "~/server/services/CacheService";
 import { getUserProjectFromId } from "~/server/LIVE_DEMO_DATA";
-
+import { executeAdvancedTimelineAggregation, fillAndMergeTimelineAggregation } from "~/server/services/TimelineService";
 
 export default defineEventHandler(async event => {
     const project_id = getRequestProjectId(event);
@@ -12,19 +13,27 @@ export default defineEventHandler(async event => {
     const project = await getUserProjectFromId(project_id, user);
     if (!project) return;
 
-    const { slice, duration, referrer } = await readBody(event);
+    const { slice, from, to, referrer } = await readBody(event);
 
-    // return await Redis.useCache({ key: `timeline:referrers:${project_id}:${slice}`, exp: TIMELINE_EXPIRE_TIME }, async () => {
-    const timelineReferrers = await getTimeline(VisitModel, project_id, slice, duration,
-        {},
-        {},
-        { referrer: "$_id.referrer" },
-        { referrer: "$referrer" },
-        { referrer }
-    );
-    return timelineReferrers;
-    // });
+    if (!from) return setResponseStatus(event, 400, 'from is required');
+    if (!from) return setResponseStatus(event, 400, 'to is required');
+    if (!from) return setResponseStatus(event, 400, 'slice is required');
 
+    return await Redis.useCache({
+        key: `timeline:referrers:${referrer}:${project_id}:${slice}:${from || 'none'}:${to || 'none'}`,
+        exp: TIMELINE_EXPIRE_TIME
+    }, async () => {
+        const timelineData = await executeAdvancedTimelineAggregation({
+            projectId: project._id,
+            model: VisitModel,
+            from, to, slice,
+            customMatch: {
+                referrer
+            }
+        });
+        const timelineFilledMerged = fillAndMergeTimelineAggregation(timelineData, slice);
+        return timelineFilledMerged;
+    });
 
 
 });

dashboard/server/api/metrics/[project_id]/timeline/sessions.post.ts

@@ -2,7 +2,7 @@ import { getTimeline } from "./generic";
 import { Redis, TIMELINE_EXPIRE_TIME } from "~/server/services/CacheService";
 import { getUserProjectFromId } from "~/server/LIVE_DEMO_DATA";
 import { SessionModel } from "@schema/metrics/SessionSchema";
-
+import { executeTimelineAggregation, fillAndMergeTimelineAggregation } from "~/server/services/TimelineService";
 
 export default defineEventHandler(async event => {
     const project_id = getRequestProjectId(event);
@@ -12,11 +12,24 @@ export default defineEventHandler(async event => {
     const project = await getUserProjectFromId(project_id, user);
     if (!project) return;
 
-    const { slice, duration } = await readBody(event);
 
-    return await Redis.useCache({ key: `timeline:sessions:${project_id}:${slice}`, exp: TIMELINE_EXPIRE_TIME }, async () => {
-        const timelineSessions = await getTimeline(SessionModel, project_id, slice, duration);
-        return timelineSessions;
+    const { slice, from, to } = await readBody(event);
+
+    if (!from) return setResponseStatus(event, 400, 'from is required');
+    if (!from) return setResponseStatus(event, 400, 'to is required');
+    if (!from) return setResponseStatus(event, 400, 'slice is required');
+
+    return await Redis.useCache({
+        key: `timeline:sessions:${project_id}:${slice}:${from || 'none'}:${to || 'none'}`,
+        exp: TIMELINE_EXPIRE_TIME
+    }, async () => {
+        const timelineData = await executeTimelineAggregation({
+            projectId: project._id,
+            model: SessionModel,
+            from, to, slice
+        });
+        const timelineFilledMerged = fillAndMergeTimelineAggregation(timelineData, slice);
+        return timelineFilledMerged;
     });
 
 

dashboard/server/api/metrics/[project_id]/timeline/visits.post.ts

@@ -1,7 +1,8 @@
-import { getTimeline } from "./generic";
 import { VisitModel } from "@schema/metrics/VisitSchema";
 import { Redis, TIMELINE_EXPIRE_TIME } from "~/server/services/CacheService";
 import { getUserProjectFromId } from "~/server/LIVE_DEMO_DATA";
+import DateService from "@services/DateService";
+import { executeTimelineAggregation, fillAndMergeTimelineAggregation } from "~/server/services/TimelineService";
 
 
 export default defineEventHandler(async event => {
@@ -12,13 +13,27 @@ export default defineEventHandler(async event => {
     const project = await getUserProjectFromId(project_id, user);
     if (!project) return;
 
-    const { slice, duration } = await readBody(event);
 
-    return await Redis.useCache({ key: `timeline:visits:${project_id}:${slice}`, exp: TIMELINE_EXPIRE_TIME }, async () => {
-        const timelineVisits = await getTimeline(VisitModel, project_id, slice, duration);
-        return timelineVisits;
+    const { slice, from, to } = await readBody(event);
+
+    if (!from) return setResponseStatus(event, 400, 'from is required');
+    if (!from) return setResponseStatus(event, 400, 'to is required');
+    if (!from) return setResponseStatus(event, 400, 'slice is required');
+
+    return await Redis.useCache({
+        key: `timeline:visits:${project_id}:${slice}:${from || 'none'}:${to || 'none'}`,
+        exp: TIMELINE_EXPIRE_TIME
+    }, async () => {
+        const timelineData = await executeTimelineAggregation({
+            projectId: project._id,
+            model: VisitModel,
+            from, to, slice
+        });
+        const timelineFilledMerged = fillAndMergeTimelineAggregation(timelineData, slice);
+        return timelineFilledMerged;
     });
 
 
 
+
 });

dashboard/server/services/CacheService.ts

@@ -21,6 +21,7 @@ export class Redis {
         url: runtimeConfig.REDIS_URL,
         username: runtimeConfig.REDIS_USERNAME,
         password: runtimeConfig.REDIS_PASSWORD,
+        database: process.dev ? 1 : 0
     });
 
     static async init() {

dashboard/server/services/TimelineService.ts

@@ -0,0 +1,53 @@
+
+import { Slice } from "@services/DateService";
+import DateService from "@services/DateService";
+import type mongoose from "mongoose";
+
+
+export type TimelineAggregationOptions = {
+    projectId: mongoose.Schema.Types.ObjectId,
+    model: mongoose.Model<any>,
+    from: string | number,
+    to: string | number,
+    slice: Slice
+}
+
+export type AdvancedTimelineAggregationOptions = TimelineAggregationOptions & {
+    customMatch?: Record<string, any>
+}
+
+export async function executeAdvancedTimelineAggregation(options: AdvancedTimelineAggregationOptions) {
+
+    options.customMatch = options.customMatch || {};
+
+    const { group, sort, fromParts } = DateService.getQueryDateRange(options.slice);
+
+    const aggregation = [
+        {
+            $match: {
+                project_id: options.projectId,
+                created_at: { $gte: new Date(options.from), $lte: new Date(options.to) },
+                ...options.customMatch
+            }
+        },
+        { $group: { _id: group, count: { $sum: 1 } } },
+        { $sort: sort },
+        { $project: { _id: { $dateFromParts: fromParts }, count: "$count" } }
+    ]
+
+    const timeline: { _id: string, count: number }[] = await options.model.aggregate(aggregation);
+
+    return timeline;
+
+}
+
+export async function executeTimelineAggregation(options: TimelineAggregationOptions) {
+    return executeAdvancedTimelineAggregation(options);
+}
+
+
+export function fillAndMergeTimelineAggregation(timeline: { _id: string, count: number }[], slice: Slice) {
+    const filledDates = DateService.fillDates(timeline.map(e => e._id), slice);
+    const merged = DateService.mergeFilledDates(filledDates, timeline, '_id', slice, { count: 0 });
+    return merged;
+}

shared/package.json

@@ -1,5 +1,6 @@
 {
 	"dependencies": {
+		"dayjs": "^1.11.11",
 		"mongoose": "^8.4.0",
 		"nodemailer": "^6.9.13",
 		"redis": "^4.6.14"

shared/pnpm-lock.yaml

@@ -8,6 +8,9 @@ importers:
 
   .:
     dependencies:
+      dayjs:
+        specifier: ^1.11.11
+        version: 1.11.11
       mongoose:
         specifier: ^8.4.0
         version: 8.4.0
@@ -79,6 +82,9 @@ packages:
     resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
     engines: {node: '>=0.10.0'}
 
+  dayjs@1.11.11:
+    resolution: {integrity: sha512-okzr3f11N6WuqYtZSvm+F776mB41wRZMhKP+hc34YdW+KmtYYK9iqvHSwo2k9FEH3fhGXvOPV6yz2IcSrfRUDg==}
+
   debug@4.3.4:
     resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
     engines: {node: '>=6.0'}
@@ -232,6 +238,8 @@ snapshots:
 
   cluster-key-slot@1.1.2: {}
 
+  dayjs@1.11.11: {}
+
   debug@4.3.4:
     dependencies:
       ms: 2.1.2

shared/services/DateService.ts

@@ -0,0 +1,131 @@
+
+import dayjs from 'dayjs';
+
+export type Slice = keyof typeof slicesData;
+
+const slicesData = {
+    hour: {
+        fromOffset: 1000 * 60 * 60 * 24
+    },
+    day: {
+        fromOffset: 1000 * 60 * 60 * 24 * 7
+    },
+    month: {
+        fromOffset: 1000 * 60 * 60 * 24 * 30 * 12
+    },
+    year: {
+        fromOffset: 1000 * 60 * 60 * 24 * 30 * 12 * 10
+    }
+}
+
+
+class DateService {
+
+    public slicesData = slicesData;
+
+    getChartLabelFromISO(iso: string, locale: string, slice: Slice) {
+        const date = dayjs(iso).locale(locale);
+        if (slice === 'hour') return date.format('HH:mm')
+        if (slice === 'day') return date.format('DD/MM')
+        return date.format();
+    }
+
+    getDefaultRange(slice: Slice) {
+        return {
+            from: new Date(Date.now() - slicesData[slice].fromOffset).toISOString(),
+            to: new Date().toISOString()
+        }
+    }
+
+    getQueryDateRange(slice: Slice) {
+
+        const group: Record<string, any> = {}
+        const sort: Record<string, any> = {}
+        const fromParts: Record<string, any> = {}
+
+        switch (slice) {
+            case 'hour':
+                group.hour = { $hour: '$created_at' }
+                fromParts.hour = "$_id.hour";
+            case 'day':
+                group.day = { $dayOfMonth: '$created_at' }
+                fromParts.day = "$_id.day";
+            case 'month':
+                group.month = { $month: '$created_at' }
+                fromParts.month = "$_id.month";
+            case 'year':
+                group.year = { $year: '$created_at' }
+                fromParts.year = "$_id.year";
+        }
+
+        switch (slice) {
+            case 'year':
+                sort['_id.year'] = 1;
+                break;
+            case 'month':
+                sort['_id.year'] = 1;
+                sort['_id.month'] = 1;
+                break;
+            case 'day':
+                sort['_id.year'] = 1;
+                sort['_id.month'] = 1;
+                sort['_id.day'] = 1;
+                break;
+            case 'hour':
+                sort['_id.year'] = 1;
+                sort['_id.month'] = 1;
+                sort['_id.day'] = 1;
+                sort['_id.hour'] = 1;
+                break;
+        }
+
+        return { group, sort, fromParts }
+    }
+
+    prepareDateRange(from: string, to: string, slice: Slice) {
+
+        let fromDate = dayjs(from).minute(0).second(0).millisecond(0);
+        let toDate = dayjs(to).minute(0).second(0).millisecond(0);
+
+        switch (slice) {
+            case 'day':
+                fromDate = fromDate.hour(0);
+                toDate = toDate.hour(0);
+                break;
+            case 'hour':
+                break;
+        }
+
+        return {
+            from: fromDate.toDate(),
+            to: toDate.toDate()
+        }
+    }
+
+    fillDates(dates: string[], slice: Slice) {
+        const allDates: dayjs.Dayjs[] = [];
+        const firstDate = dayjs(dates.at(0));
+        const lastDate = dayjs(dates.at(-1));
+        let currentDate = firstDate.clone();
+
+        while (currentDate.isBefore(lastDate, slice)) {
+            currentDate = currentDate.add(1, slice);
+            allDates.push(currentDate);
+        }
+        
+        return allDates;
+    }
+
+    mergeFilledDates<T extends Record<string, any>, K extends keyof T>(dates: dayjs.Dayjs[], items: T[], dateField: K, slice: Slice, fillData: Omit<T, K>) {
+        const result = new Array<T>();
+        for (const date of dates) {
+            const item = items.find(e => dayjs(e[dateField]).isSame(date), slice);
+            result.push(item ?? { ...fillData, [dateField]: date.format() } as T);
+        }
+        return result;
+    }
+
+}
+
+const dateServiceInstance = new DateService();
+export default dateServiceInstance;