litlyx/dashboard/server/services/AnomalyService.ts

import mongoose from "mongoose";
import { executeTimelineAggregation } from "./TimelineService";
import { VisitModel } from "@schema/metrics/VisitSchema";
import { AnomalyDomainModel } from '@schema/anomalies/AnomalyDomainSchema';
import { AnomalyVisitModel } from '@schema/anomalies/AnomalyVisitSchema';
import { AnomalyEventsModel } from '@schema/anomalies/AnomalyEventsSchema';
import { EventModel } from "@schema/metrics/EventSchema";
import EmailService from "@services/EmailService";

import * as url from 'url';
import { ProjectModel } from "@schema/ProjectSchema";
import { UserModel } from "@schema/UserSchema";

type TAvgInput = { _id: string, count: number }

const anomalyData = { minutes: 0 }

async function anomalyCheckAll() {
    const start = performance.now();
    console.log('START ANOMALY CHECK');
    const projects = await ProjectModel.find({}, { _id: 1 });
    for (const project of projects) {
        await findAnomalies(project.id);
    }
    const end = start - performance.now();
    console.log('END ANOMALY CHECK', end, 'ms');
}

export function anomalyLoop() {
    if (anomalyData.minutes == 60 * 12) {
        anomalyCheckAll();
        anomalyData.minutes = 0;
    }
    setTimeout(() => anomalyLoop(), 1000 * 60);
}


function movingAverageAnomaly(visits: TAvgInput[], windowSize: number, threshold: number): TAvgInput[] {
    const anomalies: TAvgInput[] = [];
    for (let i = windowSize; i < visits.length; i++) {
        const window = visits.slice(i - windowSize, i);
        const mean = window.reduce((a, b) => a + b.count, 0) / window.length;
        const stdDev = Math.sqrt(window.reduce((sum, visit) => sum + Math.pow(visit.count - mean, 2), 0) / window.length);
        const currentVisit = visits[i];
        if (Math.abs(currentVisit.count - mean) > threshold * stdDev) {
            if (currentVisit.count <= mean) continue;
            anomalies.push(currentVisit);
        }
    }
    return anomalies;
}

function getUrlFromString(str: string) {
    const res = str.startsWith('http') ? str : 'http://' + str;
    return res;
}

export async function findAnomalies(project_id: string) {

    const THRESHOLD = 6;
    const WINDOW_SIZE = 14;

    const pid = new mongoose.Types.ObjectId(project_id) as any;

    const from = Date.now() - 1000 * 60 * 60 * 24 * 30;
    const to = Date.now() - 1000 * 60 * 60 * 24;

    const visitsTimelineData = await executeTimelineAggregation({
        projectId: pid,
        model: VisitModel,
        from, to, slice: 'day'
    });

    const eventsTimelineData = await executeTimelineAggregation({
        projectId: pid,
        model: EventModel,
        from, to, slice: 'day'
    });

    const websites: { _id: string, count: number }[] = await VisitModel.aggregate([
        { $match: { project_id: pid, created_at: { $gte: new Date(from), $lte: new Date(to) } }, },
        { $group: { _id: "$website", count: { $sum: 1, } } }
    ]);


    const rootWebsite = websites.reduce((a, e) => {
        return a.count > e.count ? a : e;
    });

    const rootDomain = new url.URL(getUrlFromString(rootWebsite._id)).hostname;

    const detectedWebsites: string[] = [];

    for (const website of websites) {
        const websiteDomain = new url.URL(getUrlFromString(website._id)).hostname;

        if (!websiteDomain.includes(rootDomain)) {
            detectedWebsites.push(website._id);
        }
    }

    const visitAnomalies = movingAverageAnomaly(visitsTimelineData, WINDOW_SIZE, THRESHOLD);
    const eventAnomalies = movingAverageAnomaly(eventsTimelineData, WINDOW_SIZE, THRESHOLD);

    const shouldSendMail = {
        visitsEvents: false,
        domains: false
    }

    for (const visit of visitAnomalies) {
        const anomalyAlreadyExist = await AnomalyVisitModel.findOne({ visitDate: visit._id }, { _id: 1 });
        if (anomalyAlreadyExist) continue;
        await AnomalyVisitModel.create({ project_id: pid, visitDate: visit._id, created_at: Date.now() });
        shouldSendMail.visitsEvents = true;
    }

    for (const event of eventAnomalies) {
        const anomalyAlreadyExist = await AnomalyEventsModel.findOne({ eventDate: event._id }, { _id: 1 });
        if (anomalyAlreadyExist) continue;
        await AnomalyEventsModel.create({ project_id: pid, eventDate: event._id, created_at: Date.now() });
        shouldSendMail.visitsEvents = true;
    }

    for (const website of detectedWebsites) {
        const anomalyAlreadyExist = await AnomalyDomainModel.findOne({ domain: website }, { _id: 1 });
        if (anomalyAlreadyExist) continue;
        await AnomalyDomainModel.create({ project_id: pid, domain: website, created_at: Date.now() });
        shouldSendMail.domains = true;
    }


    const project = await ProjectModel.findById(pid);
    if (!project) return { ok: false, error: 'Cannot find project with id ' + pid.toString() }
    const user = await UserModel.findById(project.owner);
    if (!user) return { ok: false, error: 'Cannot find user with id ' + project.owner.toString() }

    if (shouldSendMail.visitsEvents === true) {
        await EmailService.sendAnomalyVisitsEventsEmail(user.email, project.name);
    }
    if (shouldSendMail.domains === true) {
        await EmailService.sendAnomalyDomainEmail(user.email, project.name);
    }


    return { ok: true };


}