Elizhium/litlyx
1
0
Fork 0
mirror of https://github.com/Litlyx/litlyx synced 2025-12-10 07:48:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

add secutiry

Browse Source
This commit is contained in:
Emily
2024-09-19 15:44:27 +02:00
parent ac7ba7abd3
commit f285e92132
13 changed files with 988 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dashboard/server/api/project/limits_info.ts
View File

@@ -1,7 +1,7 @@
import { ProjectModel, TProject } from "@schema/ProjectSchema";
import { ProjectLimitModel } from "@schema/ProjectsLimits";
import { UserSettingsModel } from "@schema/UserSettings";
import { EVENT_LOG_LIMIT_PERCENT } from '@data/broker/Limits';
import { MAX_LOG_LIMIT_PERCENT } from '@data/broker/Limits';
export default defineEventHandler(async event => {
@@ -25,8 +25,8 @@ export default defineEventHandler(async event => {
return {
total: TOTAL_COUNT,
limit: COUNT_LIMIT,
maxLimit: Math.round(COUNT_LIMIT * EVENT_LOG_LIMIT_PERCENT),
limited: TOTAL_COUNT > COUNT_LIMIT * EVENT_LOG_LIMIT_PERCENT,
maxLimit: Math.round(COUNT_LIMIT * MAX_LOG_LIMIT_PERCENT),
limited: TOTAL_COUNT > COUNT_LIMIT * MAX_LOG_LIMIT_PERCENT,
percent: Math.round(100 / COUNT_LIMIT * TOTAL_COUNT)
}

44
dashboard/server/api/security/list.ts Normal file
View File

@@ -0,0 +1,44 @@
import { getUserProjectFromId } from "~/server/LIVE_DEMO_DATA";
import { AnomalyDomainModel } from '@schema/anomalies/AnomalyDomainSchema';
import { AnomalyEventsModel } from '@schema/anomalies/AnomalyEventsSchema';
import { AnomalyVisitModel } from '@schema/anomalies/AnomalyVisitSchema';
type TSecurityDomainEntry = { type: 'domain', data: { domain: string, created_at: Date } }
type TSecurityVisitEntry = { type: 'visit', data: { visitDate: Date, created_at: Date } }
type TSecurityEventEntry = { type: 'event', data: { eventDate: Date, created_at: Date } }
export type SecutityReport = (TSecurityDomainEntry | TSecurityVisitEntry | TSecurityEventEntry)[];
export default defineEventHandler(async event => {
const project_id = getHeader(event, 'x-pid');
if (!project_id) return;
const user = getRequestUser(event);
const project = await getUserProjectFromId(project_id, user);
if (!project) return;
const visits = await AnomalyVisitModel.find({ project_id }, { _id: 0, project_id: 0 });
const events = await AnomalyEventsModel.find({ project_id }, { _id: 0, project_id: 0 });
const domains = await AnomalyDomainModel.find({ project_id }, { _id: 0, project_id: 0 });
const report: SecutityReport = [];
for (const visit of visits) {
report.push({ type: 'visit', data: visit });
}
for (const event of events) {
report.push({ type: 'event', data: event });
}
for (const domain of domains) {
report.push({ type: 'domain', data: domain });
}
return report.toSorted((a, b) => a.data.created_at.getTime() - b.data.created_at.getTime());
});

152
dashboard/server/services/AnomalyService.ts
View File

@@ -1,152 +0,0 @@
import mongoose from "mongoose";
import { executeTimelineAggregation } from "./TimelineService";
import { VisitModel } from "@schema/metrics/VisitSchema";
import { AnomalyDomainModel } from '@schema/anomalies/AnomalyDomainSchema';
import { AnomalyVisitModel } from '@schema/anomalies/AnomalyVisitSchema';
import { AnomalyEventsModel } from '@schema/anomalies/AnomalyEventsSchema';
import { EventModel } from "@schema/metrics/EventSchema";
import EmailService from "@services/EmailService";
import * as url from 'url';
import { ProjectModel } from "@schema/ProjectSchema";
import { UserModel } from "@schema/UserSchema";
type TAvgInput = { _id: string, count: number }
const anomalyData = { minutes: 0 }
async function anomalyCheckAll() {
const start = performance.now();
console.log('[ANOMALY] START ANOMALY CHECK');
const projects = await ProjectModel.find({}, { _id: 1 });
for (const project of projects) {
await findAnomalies(project.id);
}
const end = performance.now() - start;
console.log('END ANOMALY CHECK', end, 'ms');
}
export function anomalyLoop() {
if (anomalyData.minutes == 60 * 12) {
anomalyCheckAll();
anomalyData.minutes = 0;
}
anomalyData.minutes++;
setTimeout(() => anomalyLoop(), 1000 * 60);
}
function movingAverageAnomaly(visits: TAvgInput[], windowSize: number, threshold: number): TAvgInput[] {
const anomalies: TAvgInput[] = [];
for (let i = windowSize; i < visits.length; i++) {
const window = visits.slice(i - windowSize, i);
const mean = window.reduce((a, b) => a + b.count, 0) / window.length;
const stdDev = Math.sqrt(window.reduce((sum, visit) => sum + Math.pow(visit.count - mean, 2), 0) / window.length);
const currentVisit = visits[i];
if (Math.abs(currentVisit.count - mean) > threshold * stdDev) {
if (currentVisit.count <= mean) continue;
anomalies.push(currentVisit);
}
}
return anomalies;
}
function getUrlFromString(str: string) {
const res = str.startsWith('http') ? str : 'http://' + str;
return res;
}
export async function findAnomalies(project_id: string) {
const THRESHOLD = 6;
const WINDOW_SIZE = 14;
const pid = new mongoose.Types.ObjectId(project_id) as any;
const from = Date.now() - 1000 * 60 * 60 * 24 * 30;
const to = Date.now() - 1000 * 60 * 60 * 24;
const visitsTimelineData = await executeTimelineAggregation({
projectId: pid,
model: VisitModel,
from, to, slice: 'day'
});
const eventsTimelineData = await executeTimelineAggregation({
projectId: pid,
model: EventModel,
from, to, slice: 'day'
});
const websites: { _id: string, count: number }[] = await VisitModel.aggregate([
{ $match: { project_id: pid, created_at: { $gte: new Date(from), $lte: new Date(to) } }, },
{ $group: { _id: "$website", count: { $sum: 1, } } }
]);
const detectedWebsites: string[] = [];
if (websites.length > 0) {
const rootWebsite = websites.reduce((a, e) => {
return a.count > e.count ? a : e;
});
const rootDomain = new url.URL(getUrlFromString(rootWebsite._id)).hostname;
for (const website of websites) {
const websiteDomain = new url.URL(getUrlFromString(website._id)).hostname;
if (websiteDomain === 'localhost') continue;
if (websiteDomain === '127.0.0.1') continue;
if (websiteDomain === '0.0.0.0') continue;
if (!websiteDomain.includes(rootDomain)) { detectedWebsites.push(website._id); }
}
}
const visitAnomalies = movingAverageAnomaly(visitsTimelineData, WINDOW_SIZE, THRESHOLD);
const eventAnomalies = movingAverageAnomaly(eventsTimelineData, WINDOW_SIZE, THRESHOLD);
const shouldSendMail = {
visitsEvents: false,
domains: false
}
for (const visit of visitAnomalies) {
const anomalyAlreadyExist = await AnomalyVisitModel.findOne({ visitDate: visit._id }, { _id: 1 });
if (anomalyAlreadyExist) continue;
await AnomalyVisitModel.create({ project_id: pid, visitDate: visit._id, created_at: Date.now() });
shouldSendMail.visitsEvents = true;
}
for (const event of eventAnomalies) {
const anomalyAlreadyExist = await AnomalyEventsModel.findOne({ eventDate: event._id }, { _id: 1 });
if (anomalyAlreadyExist) continue;
await AnomalyEventsModel.create({ project_id: pid, eventDate: event._id, created_at: Date.now() });
shouldSendMail.visitsEvents = true;
}
for (const website of detectedWebsites) {
const anomalyAlreadyExist = await AnomalyDomainModel.findOne({ domain: website }, { _id: 1 });
if (anomalyAlreadyExist) continue;
await AnomalyDomainModel.create({ project_id: pid, domain: website, created_at: Date.now() });
shouldSendMail.domains = true;
}
const project = await ProjectModel.findById(pid);
if (!project) return { ok: false, error: 'Cannot find project with id ' + pid.toString() }
const user = await UserModel.findById(project.owner);
if (!user) return { ok: false, error: 'Cannot find user with id ' + project.owner.toString() }
if (shouldSendMail.visitsEvents === true) {
await EmailService.sendAnomalyVisitsEventsEmail(user.email, project.name);
}
if (shouldSendMail.domains === true) {
await EmailService.sendAnomalyDomainEmail(user.email, project.name);
}
return { ok: true };
}