Elizhium/litlyx
1
0
Fork 0
mirror of https://github.com/Litlyx/litlyx synced 2025-12-13 00:48:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

new selfhosted version

Browse Source
This commit is contained in:
antonio
2025-11-28 14:11:51 +01:00
parent afda29997d
commit 951860f67e
1046 changed files with 72586 additions and 574750 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
dashboard/server/services/TimelineService.ts
View File

@@ -1,18 +1,23 @@
import { Slice } from "@services/DateService";
import DateService from "@services/DateService";
import type mongoose from "mongoose";
import DateService from '@services/DateService';
import * as fns from 'date-fns'
import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc.js';
dayjs.extend(utc);
export type TimelineAggregationOptions = {
projectId: mongoose.Schema.Types.ObjectId | mongoose.Types.ObjectId,
model: mongoose.Model<any>,
from: string | number,
to: string | number,
from: number,
to: number,
slice: Slice,
timeOffset?: number,
debug?: boolean,
domain?: string
explain?: boolean,
domain?: string,
allowDisk?: boolean,
forced?: boolean
}
export type AdvancedTimelineAggregationOptions = TimelineAggregationOptions & {
@@ -24,7 +29,55 @@ export type AdvancedTimelineAggregationOptions = TimelineAggregationOptions & {
customQueries?: { index: number, query: Record<string, any> }[]
}
export async function executeAdvancedTimelineAggregation<T = {}>(options: AdvancedTimelineAggregationOptions) {
export const granularityMap: Record<Slice, string> = {
hour: 'hour',
day: 'day',
month: 'month',
week: 'week',
year: 'year'
}
export function checkSliceValidity(from: number, to: number, slice: Slice): [false, string] | [true, number] {
const days = fns.differenceInDays(new Date(to), new Date(from));
const [min, max] = DateService.sliceAvailabilityMap[slice];
if (days < min) return [false, 'date gap too small for this slice'];
if (days > max) return [false, 'date gap too big for this slice'];
return [true, days];
}
export function prepareTimelineAggregation(options: TimelineAggregationOptions) {
const granularity = granularityMap[options.slice];
if (!granularity) throw createError({ status: 400, message: 'slice not correct' });
if (!options.forced) {
const [sliceValid, errorOrDays] = checkSliceValidity(options.from, options.to, options.slice);
if (!sliceValid) throw createError({ status: 400, message: errorOrDays });
}
const domainMatch: any = {}
if (options.domain) domainMatch.website = options.domain
let from = new Date(options.from);
let to = new Date(options.to);
if (options.slice === 'month') {
from = dayjs(from).utc().startOf('month').toDate()
to = dayjs(to).utc().startOf('month').toDate()
} else if (options.slice === 'hour') {
// from = dayjs(from).utc().startOf('hour').toDate()
// to = dayjs(to).utc().startOf('hour').toDate()
} else if (options.slice === 'day') {
from = dayjs(from).utc().startOf('day').toDate()
to = dayjs(to).utc().startOf('day').toDate()
}
return { granularity, domainMatch, from, to }
}
export async function executeAdvancedTimelineAggregation<T = {}>(options: AdvancedTimelineAggregationOptions): Promise<any[]> {
options.customMatch = options.customMatch || {};
options.customGroup = options.customGroup || {};
@@ -32,16 +85,8 @@ export async function executeAdvancedTimelineAggregation<T = {}>(options: Advanc
options.customIdGroup = options.customIdGroup || {};
options.customQueries = options.customQueries || [];
const { dateFromParts, granularity } = DateService.getGranularityData(options.slice, '$tmpDate');
if (!dateFromParts) throw Error('Slice is probably not correct');
const [sliceValid, errorOrDays] = checkSliceValidity(options.from, options.to, options.slice);
if (!sliceValid) throw Error(errorOrDays);
const timeOffset = options.timeOffset || 0;
const domainMatch: any = {}
if (options.domain) domainMatch.website = options.domain
const { domainMatch, granularity, from, to } = prepareTimelineAggregation(options);
const aggregation = [
{
@@ -55,51 +100,45 @@ export async function executeAdvancedTimelineAggregation<T = {}>(options: Advanc
...options.customMatch
}
},
{
$addFields: {
tmpDate: {
$dateSubtract: {
startDate: "$created_at",
unit: "minute",
amount: timeOffset
}
}
}
},
{
$addFields: { isoDate: { $dateFromParts: dateFromParts } }
},
{
$group: {
_id: { isoDate: "$isoDate", ...options.customIdGroup },
_id: {
date: {
$dateTrunc: { date: "$created_at", unit: granularity, timezone: "UTC" }
},
...options.customIdGroup
},
count: { $sum: 1 },
...options.customGroup
}
},
{
$densify: {
field: "_id.isoDate",
field: "_id.date",
range: {
step: 1,
unit: granularity,
bounds: 'full'
// [
// new Date(new Date(options.from).getTime() - (timeOffset * 1000 * 60)),
// new Date(new Date(options.to).getTime() - (timeOffset * 1000 * 60) + 1),
// ]
bounds: [from, to]
}
}
},
{
$sort: { "_id.isoDate": 1 }
},
{
$addFields: { count: { $ifNull: ["$count", 0] }, }
},
// {
// $addFields: {
// timestamp: { $toLong: "$_id.date" }
// }
// },
// { $set: { count: { $ifNull: ["$count", 0] } } },
// { $sort: { '_id.date': 1 } },
// {
// $project: {
// _id: 1, count: 1, timestamp: 1, ...options.customProjection
// }
// }
{
$project: {
_id: '$_id.isoDate',
count: '$count',
_id: "$_id.date",
count: { $ifNull: ["$count", 0] },
// timestamp: { $toLong: "$_id.date" },
...options.customProjection
}
}
@@ -111,13 +150,18 @@ export async function executeAdvancedTimelineAggregation<T = {}>(options: Advanc
if (options.customAfterMatch) aggregation.splice(1, 0, options.customAfterMatch);
if (options.debug === true) {
if (options.debug === true || options.explain === true) {
console.log('---------- AGGREAGATION ----------')
console.log(JSON.stringify(aggregation, null, 2));
console.log(getPrettyAggregation(aggregation, 2));
}
const timeline: ({ _id: string, count: number } & T)[] = await options.model.aggregate(aggregation);
if (options.explain) {
const explained: any = await options.model.aggregate(aggregation, { allowDiskUse: options.allowDisk ?? false }).explain('executionStats');
return explained;
}
const timeline: ({ _id: { date: string }, count: number, timestamp: number } & T)[] = await options.model.aggregate(aggregation, {
allowDiskUse: options.allowDisk ?? false
})
return timeline;
@@ -127,21 +171,9 @@ export async function executeTimelineAggregation(options: TimelineAggregationOpt
return executeAdvancedTimelineAggregation(options);
}
/**
* @deprecated use fillAndMergeTimelineAggregationV2
*/
export function fillAndMergeTimelineAggregation(timeline: { _id: string, count: number }[], slice: Slice) {
const filledDates = DateService.fillDates(timeline.map(e => e._id), slice);
const merged = DateService.mergeFilledDates(filledDates, timeline, '_id', slice, { count: 0 });
return merged;
}
// export function fillAndMergeTimelineAggregationV2(timeline: { _id: string, count: number }[], slice: Slice, from: string, to: string) {
// const allDates = DateService.generateDateSlices(slice, new Date(from), new Date(to));
// const merged = DateService.mergeDates(timeline, allDates, slice);
// return merged;
// }
export function fillAndMergeTimelineAggregationV2(timeline: { _id: string, count: number }[], slice: Slice, from: string, to: string) {
const allDates = DateService.generateDateSlices(slice, new Date(from), new Date(to));
const merged = DateService.mergeDates(timeline, allDates, slice);
return merged;
}
export function checkSliceValidity(from: string | number | Date, to: string | number | Date, slice: Slice): [false, string] | [true, number] {
return DateService.canUseSlice(from, to, slice);
}