add secutiry

dashboard/components/dashboard/TopSection.vue

@@ -31,7 +31,7 @@ function showAnomalyInfoAlert() {
 
 <template>
     <div
-        class="w-full px-6 py-2 lg:py-6 font-bold text-text-sub/40 flex flex-col xl:flex-row text-lg lg:text-2xl gap-2 xl:gap-12">
+        class="w-full px-6 py-2 lg:py-6 font-bold text-text-sub/40 flex flex-col xl:flex-row text-lg gap-2 xl:gap-12 lg:text-2xl">
 
         <div class="flex gap-2 items-center text-text/90 justify-center md:justify-start">
             <div class="animate-pulse w-[1rem] h-[1rem] bg-green-400 rounded-full"> </div>

dashboard/layouts/dashboard.vue

@@ -21,6 +21,7 @@ const sections: Section[] = [
             { label: 'Insights (soon)', to: '#', icon: 'fal fa-lightbulb', disabled: true },
             { label: 'Links (soon)', to: '#', icon: 'fal fa-globe-pointer', disabled: true },
             { label: 'Integrations (soon)', to: '#', icon: 'fal fa-cube', disabled: true },
+            { label: 'Security', to: '/security', icon: 'fal fa-lock' },
             { label: 'Settings', to: '/settings', icon: 'fal fa-gear' },
             {
                 grow: true,

dashboard/pages/security.vue

@@ -0,0 +1,50 @@
+<script setup lang="ts">
+
+definePageMeta({ layout: 'dashboard' });
+const activeProjectId = useActiveProjectId();
+
+const headers = computed(() => {
+    return {
+        'Authorization': authorizationHeaderComputed.value,
+        'x-pid': activeProjectId.data.value || ''
+    }
+});
+
+const reportList = useFetch(`/api/security/list`, { headers });
+
+</script>
+
+
+<template>
+
+    <div class="home w-full h-full px-10 lg:px-0 pt-6 overflow-y-auto">
+
+        <div v-if="reportList.data.value" class="flex flex-col gap-2">
+            <div v-for="entry of reportList.data.value">
+                <div v-if="entry.type === 'event'" class="flex gap-2">
+                    <div class="text-lyx-text-darker">{{ new Date(entry.data.created_at).toLocaleString() }}</div>
+                    <UBadge class="w-[4rem] flex justify-center"> {{ entry.type }} </UBadge>
+                    <div class="text-lyx-text-dark">
+                        Event date: {{ new Date(entry.data.eventDate).toLocaleString() }}
+                    </div>
+                </div>
+                <div v-if="entry.type === 'visit'" class="flex gap-2">
+                    <div class="text-lyx-text-darker">{{ new Date(entry.data.created_at).toLocaleString() }}</div>
+                    <UBadge class="w-[4rem] flex justify-center"> {{ entry.type }} </UBadge>
+                    <div class="text-lyx-text-dark">
+                        Visit date: {{ new Date(entry.data.visitDate).toLocaleString() }}
+                    </div>
+                </div>
+                <div v-if="entry.type === 'domain'" class="flex gap-2">
+                    <div class="text-lyx-text-darker">{{ new Date(entry.data.created_at).toLocaleString() }}</div>
+                    <UBadge class="w-[4rem] flex justify-center"> {{ entry.type }} </UBadge>
+                    <div class="text-lyx-text-dark">
+                        Domain found: {{ entry.data.domain }}
+                    </div>
+                </div>
+            </div>
+        </div>
+
+    </div>
+
+</template>

dashboard/server/api/project/limits_info.ts

@@ -1,7 +1,7 @@
 import { ProjectModel, TProject } from "@schema/ProjectSchema";
 import { ProjectLimitModel } from "@schema/ProjectsLimits";
 import { UserSettingsModel } from "@schema/UserSettings";
-import { EVENT_LOG_LIMIT_PERCENT } from '@data/broker/Limits';
+import { MAX_LOG_LIMIT_PERCENT } from '@data/broker/Limits';
 
 export default defineEventHandler(async event => {
 
@@ -25,8 +25,8 @@ export default defineEventHandler(async event => {
     return {
         total: TOTAL_COUNT,
         limit: COUNT_LIMIT,
-        maxLimit: Math.round(COUNT_LIMIT * EVENT_LOG_LIMIT_PERCENT),
+        maxLimit: Math.round(COUNT_LIMIT * MAX_LOG_LIMIT_PERCENT),
-        limited: TOTAL_COUNT > COUNT_LIMIT * EVENT_LOG_LIMIT_PERCENT,
+        limited: TOTAL_COUNT > COUNT_LIMIT * MAX_LOG_LIMIT_PERCENT,
         percent: Math.round(100 / COUNT_LIMIT * TOTAL_COUNT)
     }
 

dashboard/server/api/security/list.ts

@@ -0,0 +1,44 @@
+import { getUserProjectFromId } from "~/server/LIVE_DEMO_DATA";
+
+import { AnomalyDomainModel } from '@schema/anomalies/AnomalyDomainSchema';
+import { AnomalyEventsModel } from '@schema/anomalies/AnomalyEventsSchema';
+import { AnomalyVisitModel } from '@schema/anomalies/AnomalyVisitSchema';
+
+
+type TSecurityDomainEntry = { type: 'domain', data: { domain: string, created_at: Date } }
+type TSecurityVisitEntry = { type: 'visit', data: { visitDate: Date, created_at: Date } }
+type TSecurityEventEntry = { type: 'event', data: { eventDate: Date, created_at: Date } }
+
+
+
+export type SecutityReport = (TSecurityDomainEntry | TSecurityVisitEntry | TSecurityEventEntry)[];
+
+export default defineEventHandler(async event => {
+
+    const project_id = getHeader(event, 'x-pid');
+    if (!project_id) return;
+
+    const user = getRequestUser(event);
+    const project = await getUserProjectFromId(project_id, user);
+    if (!project) return;
+
+    const visits = await AnomalyVisitModel.find({ project_id }, { _id: 0, project_id: 0 });
+    const events = await AnomalyEventsModel.find({ project_id }, { _id: 0, project_id: 0 });
+    const domains = await AnomalyDomainModel.find({ project_id }, { _id: 0, project_id: 0 });
+
+    const report: SecutityReport = [];
+
+    for (const visit of visits) {
+        report.push({ type: 'visit', data: visit });
+    }
+    for (const event of events) {
+        report.push({ type: 'event', data: event });
+    }
+    for (const domain of domains) {
+        report.push({ type: 'domain', data: domain });
+    }
+
+    return report.toSorted((a, b) => a.data.created_at.getTime() - b.data.created_at.getTime());
+
+
+});

security/.gitignore

@@ -0,0 +1,7 @@
+node_modules
+static
+ecosystem.config.cjs
+dist
+start_dev.js
+package-lock.json
+build_all.bat

security/package.json

@@ -0,0 +1,24 @@
+{
+  "dependencies": {
+    "dayjs": "^1.11.13",
+    "mongoose": "^8.3.2"
+  },
+  "devDependencies": {
+    "@types/node": "^20.12.13",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.5"
+  },
+  "name": "security",
+  "version": "1.0.0",
+  "main": "dist/index.js",
+  "scripts": {
+    "dev": "node scripts/start_dev.js",
+    "compile": "tsc",
+    "build": "node ../scripts/build.js",
+    "build_all": "npm run compile && npm run build"
+  },
+  "keywords": [],
+  "author": "Emily",
+  "license": "MIT",
+  "description": "."
+}

security/pnpm-lock.yaml

@@ -0,0 +1,355 @@
+lockfileVersion: '9.0'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+importers:
+
+  .:
+    dependencies:
+      dayjs:
+        specifier: ^1.11.13
+        version: 1.11.13
+      mongoose:
+        specifier: ^8.3.2
+        version: 8.6.3
+    devDependencies:
+      '@types/node':
+        specifier: ^20.12.13
+        version: 20.16.5
+      ts-node:
+        specifier: ^10.9.2
+        version: 10.9.2(@types/node@20.16.5)(typescript@5.6.2)
+      typescript:
+        specifier: ^5.4.5
+        version: 5.6.2
+
+packages:
+
+  '@cspotcode/source-map-support@0.8.1':
+    resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}
+    engines: {node: '>=12'}
+
+  '@jridgewell/resolve-uri@3.1.2':
+    resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==}
+    engines: {node: '>=6.0.0'}
+
+  '@jridgewell/sourcemap-codec@1.5.0':
+    resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==}
+
+  '@jridgewell/trace-mapping@0.3.9':
+    resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==}
+
+  '@mongodb-js/saslprep@1.1.9':
+    resolution: {integrity: sha512-tVkljjeEaAhCqTzajSdgbQ6gE6f3oneVwa3iXR6csiEwXXOFsiC6Uh9iAjAhXPtqa/XMDHWjjeNH/77m/Yq2dw==}
+
+  '@tsconfig/node10@1.0.11':
+    resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==}
+
+  '@tsconfig/node12@1.0.11':
+    resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==}
+
+  '@tsconfig/node14@1.0.3':
+    resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==}
+
+  '@tsconfig/node16@1.0.4':
+    resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==}
+
+  '@types/node@20.16.5':
+    resolution: {integrity: sha512-VwYCweNo3ERajwy0IUlqqcyZ8/A7Zwa9ZP3MnENWcB11AejO+tLy3pu850goUW2FC/IJMdZUfKpX/yxL1gymCA==}
+
+  '@types/webidl-conversions@7.0.3':
+    resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==}
+
+  '@types/whatwg-url@11.0.5':
+    resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==}
+
+  acorn-walk@8.3.4:
+    resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==}
+    engines: {node: '>=0.4.0'}
+
+  acorn@8.12.1:
+    resolution: {integrity: sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==}
+    engines: {node: '>=0.4.0'}
+    hasBin: true
+
+  arg@4.1.3:
+    resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==}
+
+  bson@6.8.0:
+    resolution: {integrity: sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ==}
+    engines: {node: '>=16.20.1'}
+
+  create-require@1.1.1:
+    resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
+
+  dayjs@1.11.13:
+    resolution: {integrity: sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==}
+
+  debug@4.3.7:
+    resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==}
+    engines: {node: '>=6.0'}
+    peerDependencies:
+      supports-color: '*'
+    peerDependenciesMeta:
+      supports-color:
+        optional: true
+
+  diff@4.0.2:
+    resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==}
+    engines: {node: '>=0.3.1'}
+
+  kareem@2.6.3:
+    resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==}
+    engines: {node: '>=12.0.0'}
+
+  make-error@1.3.6:
+    resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==}
+
+  memory-pager@1.5.0:
+    resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==}
+
+  mongodb-connection-string-url@3.0.1:
+    resolution: {integrity: sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==}
+
+  mongodb@6.8.0:
+    resolution: {integrity: sha512-HGQ9NWDle5WvwMnrvUxsFYPd3JEbqD3RgABHBQRuoCEND0qzhsd0iH5ypHsf1eJ+sXmvmyKpP+FLOKY8Il7jMw==}
+    engines: {node: '>=16.20.1'}
+    peerDependencies:
+      '@aws-sdk/credential-providers': ^3.188.0
+      '@mongodb-js/zstd': ^1.1.0
+      gcp-metadata: ^5.2.0
+      kerberos: ^2.0.1
+      mongodb-client-encryption: '>=6.0.0 <7'
+      snappy: ^7.2.2
+      socks: ^2.7.1
+    peerDependenciesMeta:
+      '@aws-sdk/credential-providers':
+        optional: true
+      '@mongodb-js/zstd':
+        optional: true
+      gcp-metadata:
+        optional: true
+      kerberos:
+        optional: true
+      mongodb-client-encryption:
+        optional: true
+      snappy:
+        optional: true
+      socks:
+        optional: true
+
+  mongoose@8.6.3:
+    resolution: {integrity: sha512-++yRmm7hjMbqVA/8WeiygTnEfrFbiy+OBjQi49GFJIvCQuSYE56myyQWo4j5hbpcHjhHQU8NukMNGTwAWFWjIw==}
+    engines: {node: '>=16.20.1'}
+
+  mpath@0.9.0:
+    resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==}
+    engines: {node: '>=4.0.0'}
+
+  mquery@5.0.0:
+    resolution: {integrity: sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==}
+    engines: {node: '>=14.0.0'}
+
+  ms@2.1.3:
+    resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
+
+  punycode@2.3.1:
+    resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
+    engines: {node: '>=6'}
+
+  sift@17.1.3:
+    resolution: {integrity: sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==}
+
+  sparse-bitfield@3.0.3:
+    resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==}
+
+  tr46@4.1.1:
+    resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==}
+    engines: {node: '>=14'}
+
+  ts-node@10.9.2:
+    resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==}
+    hasBin: true
+    peerDependencies:
+      '@swc/core': '>=1.2.50'
+      '@swc/wasm': '>=1.2.50'
+      '@types/node': '*'
+      typescript: '>=2.7'
+    peerDependenciesMeta:
+      '@swc/core':
+        optional: true
+      '@swc/wasm':
+        optional: true
+
+  typescript@5.6.2:
+    resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==}
+    engines: {node: '>=14.17'}
+    hasBin: true
+
+  undici-types@6.19.8:
+    resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==}
+
+  v8-compile-cache-lib@3.0.1:
+    resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==}
+
+  webidl-conversions@7.0.0:
+    resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==}
+    engines: {node: '>=12'}
+
+  whatwg-url@13.0.0:
+    resolution: {integrity: sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==}
+    engines: {node: '>=16'}
+
+  yn@3.1.1:
+    resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==}
+    engines: {node: '>=6'}
+
+snapshots:
+
+  '@cspotcode/source-map-support@0.8.1':
+    dependencies:
+      '@jridgewell/trace-mapping': 0.3.9
+
+  '@jridgewell/resolve-uri@3.1.2': {}
+
+  '@jridgewell/sourcemap-codec@1.5.0': {}
+
+  '@jridgewell/trace-mapping@0.3.9':
+    dependencies:
+      '@jridgewell/resolve-uri': 3.1.2
+      '@jridgewell/sourcemap-codec': 1.5.0
+
+  '@mongodb-js/saslprep@1.1.9':
+    dependencies:
+      sparse-bitfield: 3.0.3
+
+  '@tsconfig/node10@1.0.11': {}
+
+  '@tsconfig/node12@1.0.11': {}
+
+  '@tsconfig/node14@1.0.3': {}
+
+  '@tsconfig/node16@1.0.4': {}
+
+  '@types/node@20.16.5':
+    dependencies:
+      undici-types: 6.19.8
+
+  '@types/webidl-conversions@7.0.3': {}
+
+  '@types/whatwg-url@11.0.5':
+    dependencies:
+      '@types/webidl-conversions': 7.0.3
+
+  acorn-walk@8.3.4:
+    dependencies:
+      acorn: 8.12.1
+
+  acorn@8.12.1: {}
+
+  arg@4.1.3: {}
+
+  bson@6.8.0: {}
+
+  create-require@1.1.1: {}
+
+  dayjs@1.11.13: {}
+
+  debug@4.3.7:
+    dependencies:
+      ms: 2.1.3
+
+  diff@4.0.2: {}
+
+  kareem@2.6.3: {}
+
+  make-error@1.3.6: {}
+
+  memory-pager@1.5.0: {}
+
+  mongodb-connection-string-url@3.0.1:
+    dependencies:
+      '@types/whatwg-url': 11.0.5
+      whatwg-url: 13.0.0
+
+  mongodb@6.8.0:
+    dependencies:
+      '@mongodb-js/saslprep': 1.1.9
+      bson: 6.8.0
+      mongodb-connection-string-url: 3.0.1
+
+  mongoose@8.6.3:
+    dependencies:
+      bson: 6.8.0
+      kareem: 2.6.3
+      mongodb: 6.8.0
+      mpath: 0.9.0
+      mquery: 5.0.0
+      ms: 2.1.3
+      sift: 17.1.3
+    transitivePeerDependencies:
+      - '@aws-sdk/credential-providers'
+      - '@mongodb-js/zstd'
+      - gcp-metadata
+      - kerberos
+      - mongodb-client-encryption
+      - snappy
+      - socks
+      - supports-color
+
+  mpath@0.9.0: {}
+
+  mquery@5.0.0:
+    dependencies:
+      debug: 4.3.7
+    transitivePeerDependencies:
+      - supports-color
+
+  ms@2.1.3: {}
+
+  punycode@2.3.1: {}
+
+  sift@17.1.3: {}
+
+  sparse-bitfield@3.0.3:
+    dependencies:
+      memory-pager: 1.5.0
+
+  tr46@4.1.1:
+    dependencies:
+      punycode: 2.3.1
+
+  ts-node@10.9.2(@types/node@20.16.5)(typescript@5.6.2):
+    dependencies:
+      '@cspotcode/source-map-support': 0.8.1
+      '@tsconfig/node10': 1.0.11
+      '@tsconfig/node12': 1.0.11
+      '@tsconfig/node14': 1.0.3
+      '@tsconfig/node16': 1.0.4
+      '@types/node': 20.16.5
+      acorn: 8.12.1
+      acorn-walk: 8.3.4
+      arg: 4.1.3
+      create-require: 1.1.1
+      diff: 4.0.2
+      make-error: 1.3.6
+      typescript: 5.6.2
+      v8-compile-cache-lib: 3.0.1
+      yn: 3.1.1
+
+  typescript@5.6.2: {}
+
+  undici-types@6.19.8: {}
+
+  v8-compile-cache-lib@3.0.1: {}
+
+  webidl-conversions@7.0.0: {}
+
+  whatwg-url@13.0.0:
+    dependencies:
+      tr46: 4.1.1
+      webidl-conversions: 7.0.0
+
+  yn@3.1.1: {}

security/security-report.json

@@ -0,0 +1,386 @@
+[
+    {
+        "visits": [],
+        "events": [
+            {
+                "_id": "2024-09-06T00:00:00.000Z",
+                "count": 519
+            }
+        ],
+        "dns": [
+            "www.animetion.space",
+            "irc.matejc.com",
+            "157.90.230.224",
+            "animetion.space"
+        ],
+        "pid": "6637a30dd655db96d4c49976"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "tracking.veliu.net",
+            "testlitlyx.ddns.net:3000",
+            "vps751.basicserver.io:3000",
+            "192.168.1.30:3000",
+            "devtes.kayaisland.com",
+            "www.versicherungshelden-nrw.de",
+            "versicherungshelden-nrw.de",
+            "3.84.164.35",
+            "94.130.182.52",
+            "192.168.137.1:3000",
+            "103.120.203.10:3000",
+            "veliu-litlyx.cjcbee.easypanel.host",
+            "172.26.224.1:3000",
+            "lit.cordes-hosting.net",
+            "192.168.1.188:3000",
+            "litlyx.cutesquirrels.pt",
+            "tracking-litlyx-dashboard.cjcbee.easypanel.host",
+            "10.0.0.179:3000"
+        ],
+        "pid": "6643cd08a1854e3b81722ab5"
+    },
+    {
+        "visits": [
+            {
+                "_id": "2024-09-09T00:00:00.000Z",
+                "count": 307
+            }
+        ],
+        "events": [],
+        "dns": [],
+        "pid": "66464179387eb6d32bab65a4"
+    },
+    {
+        "visits": [
+            {
+                "_id": "2024-09-14T00:00:00.000Z",
+                "count": 24
+            }
+        ],
+        "events": [],
+        "dns": [],
+        "pid": "66684e8e154a0d23bff5e8bc"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "adclick.g.doubleclick.net",
+            "aax-eu.amazon-adsystem.com",
+            "www.mybestlazio.me",
+            "mybestlazio.dstage.it",
+            "track.adform.net",
+            "www-eaglepictures-com.translate.goog"
+        ],
+        "pid": "6668520c154a0d23bff5ee9d"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "rpr.penglyhome.uk",
+            "www-nationalgeographic-it.translate.goog",
+            "cc.bingj.com",
+            "t.widebuy.co.kr"
+        ],
+        "pid": "6668540a154a0d23bff5f46c"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "niagagaramcemerlang.co.id",
+            "ngc-rf6m4hvrl-deikaze16s-projects.vercel.app"
+        ],
+        "pid": "666a4ddf8b2366c9cc760115"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "deckx-c58snhqcp-davide-lanottes-projects.vercel.app",
+            "deckx-app.vercel.app"
+        ],
+        "pid": "6671b2cf35115c204ee2a126"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "webcache.googleusercontent.com"
+        ],
+        "pid": "66739a9635115c204ee2c529"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "devmirza-profile-3j7k7adhh-devmirzas-projects.vercel.app",
+            "devmirza-profile-ghqlw730h-devmirzas-projects.vercel.app"
+        ],
+        "pid": "667710e92e282316a8b11ebf"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "ntp.sarc-iitb.org",
+            "ilpsummer.sarc-iitb.org",
+            "team24.sarc-iitb.org",
+            "www.anamorate.com",
+            "ybform.sarc-iitb.org",
+            "asmp.sarc-iitb.org",
+            "alumination.sarc-iitb.org",
+            "oldasmp.sarc-iitb.org",
+            "sarc-iitb.org",
+            "pmp.sarc-iitb.org",
+            "anamorate.com",
+            "ilp.sarc-iitb.org",
+            "team23.sarc-iitb.org",
+            "centralised.sarc-iitb.org",
+            "sarcasm.sarc-iitb.org",
+            "68.178.169.196"
+        ],
+        "pid": "6677b9ed2e282316a8b19d7c"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "admin.forceget.com",
+            "test-admin.forceget.com.tr"
+        ],
+        "pid": "667970322e282316a8b28de8"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "10.4.6.218:3001"
+        ],
+        "pid": "667aaf152e282316a8b2c02c"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "naimaroma-it.translate.goog",
+            "www-naimaroma-it.translate.goog"
+        ],
+        "pid": "667ad0b72e282316a8b2c6e6"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "portfolio-website-itk86pkb3-uo1428s-projects.vercel.app",
+            "krvtcweb.vercel.app"
+        ],
+        "pid": "667b8ef32e282316a8b2d568"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "memorable-sheets-009380-0472127d2.framer.app",
+            "memorable-sheets-009380-474612544.framer.app"
+        ],
+        "pid": "6696d2426ecd27bb3a069d35"
+    },
+    {
+        "visits": [
+            {
+                "_id": "2024-09-03T00:00:00.000Z",
+                "count": 133
+            }
+        ],
+        "events": [],
+        "dns": [
+            "uclusion.com"
+        ],
+        "pid": "66993552aa427ae32a5ee11e"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "portfolio-k53bw684g-harjot-singhs-projects-448f632c.vercel.app",
+            "www.harjot.co",
+            "portfolio-2js7slsc8-harjot-singhs-projects-448f632c.vercel.app",
+            "cloudflare-cors-anywhere.harjjotsinghh.workers.dev"
+        ],
+        "pid": "66a76256aa427ae32a61b074"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "devnav.vercel.app"
+        ],
+        "pid": "66ac5feaaa427ae32a621b86"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "192.168.0.253:5000",
+            "github.baitao.pics"
+        ],
+        "pid": "66ac95d3aa427ae32a622737"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "20.0.2.22:8080",
+            "20.0.0.90",
+            "20.0.0.95",
+            "20.0.0.103"
+        ],
+        "pid": "66acaf51aa427ae32a623465"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "t3-agile-pulse.fly.dev"
+        ],
+        "pid": "66ae06f8aa427ae32a62a092"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "bradenhirschi-hju0603fo-bradenhirschi29s-projects.vercel.app"
+        ],
+        "pid": "66b664e5507bcfb0999b3df6"
+    },
+    {
+        "visits": [
+            {
+                "_id": "2024-09-13T00:00:00.000Z",
+                "count": 52
+            }
+        ],
+        "events": [],
+        "dns": [],
+        "pid": "66b9afc1507bcfb0999b725a"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "mqtt.primebiz.net",
+            "gigharbor1.netlify.app",
+            "drgaylesmith.primebiz.com",
+            "drgaylesmith.com",
+            "corporatebriefs.com",
+            "dbgeeks.com",
+            "primebiz.biz",
+            "bankbriefs.com",
+            "www.primebiz.com",
+            "rp4",
+            "ionos.primebiz.net",
+            "100.70.223.117",
+            "primebiz.net",
+            "www.primebiz.net",
+            "www.bankbriefs.com",
+            "sunny000.github.io",
+            "birdwingmast.com",
+            "www.drgaylesmith.com",
+            "74.208.98.61",
+            "www.primebiz.biz",
+            "www.birdwingmast.com",
+            "www.corporatebriefs.com",
+            "primebiz.com"
+        ],
+        "pid": "66ba310d507bcfb0999b9a69"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "erbox.web.localhost"
+        ],
+        "pid": "66c6513b507bcfb0999cc5c1"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "khazarchat.com"
+        ],
+        "pid": "66c6a805507bcfb0999ccd28"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "getsoon.me",
+            "13.37.227.196",
+            "estimate.getsoon.me"
+        ],
+        "pid": "66c79881507bcfb0999cdf7a"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "15.157.46.223"
+        ],
+        "pid": "66d0930a507bcfb0999dbafb"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "check-list.endymion.tech:8081",
+            "demo3.endymion.tech",
+            "demo1.endymion.tech",
+            "cube-demo.endymion.tech",
+            "duck-demo.endymion.tech",
+            "20.54.81.38:8081",
+            "demo2.endymion.tech"
+        ],
+        "pid": "66d201a048ad8047c37d9115"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "bdch-front-2px7mvasy-byte-crafts.vercel.app",
+            "bdch-front-jxpazisb5-byte-crafts.vercel.app",
+            "bdch-front-at2itzcdn-byte-crafts.vercel.app",
+            "bdch-front-7gn91c4vt-byte-crafts.vercel.app",
+            "bdch-front-end-three.vercel.app",
+            "bdch-front-9kt6t1j5g-byte-crafts.vercel.app"
+        ],
+        "pid": "66e2509741eed9eef436b00e"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "dev.bearback4now.com"
+        ],
+        "pid": "66e2b2d541eed9eef436c21a"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "musikversicherung.webflow.io"
+        ],
+        "pid": "66e6cc6c4bf99edbca06d00a"
+    },
+    {
+        "visits": [],
+        "events": [],
+        "dns": [
+            "socialpage-99ztjqsul-qaahirstewarts-projects.vercel.app",
+            "socialpage.bio"
+        ],
+        "pid": "66e8a59fb2138459c595e9c5"
+    }
+]

security/src/Aggregations.ts

@@ -0,0 +1,24 @@
+import DateService, { Slice } from '@services/DateService';
+import { Model, Types } from "mongoose";
+
+
+export async function getAggregation(model: Model<any>, pid: Types.ObjectId, from: number, to: number, slice: Slice) {
+
+    const { group, sort, fromParts } = DateService.getQueryDateRange(slice);
+
+    const result = model.aggregate([
+        {
+            $match: {
+                project_id: pid,
+                created_at: { $gte: new Date(from), $lte: new Date(to) },
+            }
+        },
+        { $group: { _id: group, count: { $sum: 1 } } },
+        { $sort: sort },
+        { $project: { _id: { $dateFromParts: fromParts }, count: "$count" } }
+    ]);
+
+    return result;
+
+}
+

security/src/AnomalyService.ts

@@ -1,38 +1,48 @@
 import mongoose from "mongoose";
-import { executeTimelineAggregation } from "./TimelineService";
-import { VisitModel } from "@schema/metrics/VisitSchema";
 import { AnomalyDomainModel } from '@schema/anomalies/AnomalyDomainSchema';
 import { AnomalyVisitModel } from '@schema/anomalies/AnomalyVisitSchema';
 import { AnomalyEventsModel } from '@schema/anomalies/AnomalyEventsSchema';
 import { EventModel } from "@schema/metrics/EventSchema";
+import { VisitModel } from '@schema/metrics/VisitSchema'
 import EmailService from "@services/EmailService";
-
 import * as url from 'url';
 import { ProjectModel } from "@schema/ProjectSchema";
 import { UserModel } from "@schema/UserSchema";
+import { getAggregation } from "./Aggregations";
 
 type TAvgInput = { _id: string, count: number }
 
+export type AnomalyReport = {
+    visits: TAvgInput[],
+    events: TAvgInput[],
+    dns: string[],
+    pid: string
+}
+
+export type AnomalyCallback = (report: AnomalyReport) => any;
+
 const anomalyData = { minutes: 0 }
 
-async function anomalyCheckAll() {
+export async function anomalyCheckAll(callback: AnomalyCallback) {
     const start = performance.now();
     console.log('[ANOMALY] START ANOMALY CHECK');
     const projects = await ProjectModel.find({}, { _id: 1 });
+    let i = 0;
     for (const project of projects) {
-        await findAnomalies(project.id);
+        console.log('Project:', i++, '/', projects.length);
+        await findAnomalies(project.id, callback);
     }
     const end = performance.now() - start;
     console.log('END ANOMALY CHECK', end, 'ms');
 }
 
-export function anomalyLoop() {
+export function anomalyLoop(callback: AnomalyCallback) {
     if (anomalyData.minutes == 60 * 12) {
-        anomalyCheckAll();
+        anomalyCheckAll(callback);
         anomalyData.minutes = 0;
     }
     anomalyData.minutes++;
-    setTimeout(() => anomalyLoop(), 1000 * 60);
+    setTimeout(() => anomalyLoop(callback), 1000 * 60);
 }
 
 
@@ -56,7 +66,7 @@ function getUrlFromString(str: string) {
     return res;
 }
 
-export async function findAnomalies(project_id: string) {
+export async function findAnomalies(project_id: string, callback: AnomalyCallback) {
 
     const THRESHOLD = 6;
     const WINDOW_SIZE = 14;
@@ -66,17 +76,10 @@ export async function findAnomalies(project_id: string) {
     const from = Date.now() - 1000 * 60 * 60 * 24 * 30;
     const to = Date.now() - 1000 * 60 * 60 * 24;
 
-    const visitsTimelineData = await executeTimelineAggregation({
+    const visitsTimelineData = await getAggregation(VisitModel, pid, from, to, 'day');
-        projectId: pid,
+
-        model: VisitModel,
+    const eventsTimelineData = await getAggregation(EventModel, pid, from, to, 'day');
-        from, to, slice: 'day'
-    });
 
-    const eventsTimelineData = await executeTimelineAggregation({
-        projectId: pid,
-        model: EventModel,
-        from, to, slice: 'day'
-    });
 
     const websites: { _id: string, count: number }[] = await VisitModel.aggregate([
         { $match: { project_id: pid, created_at: { $gte: new Date(from), $lte: new Date(to) } }, },
@@ -102,51 +105,52 @@ export async function findAnomalies(project_id: string) {
         }
     }
 
-
     const visitAnomalies = movingAverageAnomaly(visitsTimelineData, WINDOW_SIZE, THRESHOLD);
     const eventAnomalies = movingAverageAnomaly(eventsTimelineData, WINDOW_SIZE, THRESHOLD);
 
-    const shouldSendMail = {
+    const report: AnomalyReport = {
-        visitsEvents: false,
+        visits: [],
-        domains: false
+        events: [],
+        dns: [],
+        pid: project_id
     }
 
     for (const visit of visitAnomalies) {
         const anomalyAlreadyExist = await AnomalyVisitModel.findOne({ visitDate: visit._id }, { _id: 1 });
         if (anomalyAlreadyExist) continue;
         await AnomalyVisitModel.create({ project_id: pid, visitDate: visit._id, created_at: Date.now() });
-        shouldSendMail.visitsEvents = true;
+        report.visits.push(visit);
     }
 
     for (const event of eventAnomalies) {
         const anomalyAlreadyExist = await AnomalyEventsModel.findOne({ eventDate: event._id }, { _id: 1 });
         if (anomalyAlreadyExist) continue;
         await AnomalyEventsModel.create({ project_id: pid, eventDate: event._id, created_at: Date.now() });
-        shouldSendMail.visitsEvents = true;
+        report.events.push(event);
     }
 
     for (const website of detectedWebsites) {
         const anomalyAlreadyExist = await AnomalyDomainModel.findOne({ domain: website }, { _id: 1 });
         if (anomalyAlreadyExist) continue;
         await AnomalyDomainModel.create({ project_id: pid, domain: website, created_at: Date.now() });
-        shouldSendMail.domains = true;
+        report.dns.push(website);
     }
 
+    // const project = await ProjectModel.findById(pid);
+    // if (!project) return { ok: false, error: 'Cannot find project with id ' + pid.toString() }
+    // const user = await UserModel.findById(project.owner);
+    // if (!user) return { ok: false, error: 'Cannot find user with id ' + project.owner.toString() }
 
-    const project = await ProjectModel.findById(pid);
+    // if (shouldSendMail.visitsEvents === true) {
-    if (!project) return { ok: false, error: 'Cannot find project with id ' + pid.toString() }
+    //     await EmailService.sendAnomalyVisitsEventsEmail(user.email, project.name);
-    const user = await UserModel.findById(project.owner);
+    // }
-    if (!user) return { ok: false, error: 'Cannot find user with id ' + project.owner.toString() }
+    // if (shouldSendMail.domains === true) {
-
+    //     await EmailService.sendAnomalyDomainEmail(user.email, project.name);
-    if (shouldSendMail.visitsEvents === true) {
+    // }
-        await EmailService.sendAnomalyVisitsEventsEmail(user.email, project.name);
-    }
-    if (shouldSendMail.domains === true) {
-        await EmailService.sendAnomalyDomainEmail(user.email, project.name);
-    }
 
 
-    return { ok: true };
+    callback(report);
+    return report;
 
 
 }

security/src/index.ts

@@ -0,0 +1,18 @@
+import { anomalyCheckAll, AnomalyReport } from "./AnomalyService";
+import { connectDatabase } from '@services/DatabaseService'
+import { requireEnv } from '@utils/requireEnv'
+
+connectDatabase(requireEnv('MONGO_CONNECTION_STRING'));
+
+
+import fs from 'fs';
+
+const reports: AnomalyReport[] = [];
+
+anomalyCheckAll(report => {
+    if (report.visits.length > 0 || report.events.length > 0 || report.dns.length > 0) {
+        reports.push(report);
+    }
+}).then(e => {
+    fs.writeFileSync('security-report.json', JSON.stringify(reports));
+});

security/tsconfig.json

@@ -0,0 +1,32 @@
+{
+    "compilerOptions": {
+        "module": "NodeNext",
+        "target": "ESNext",
+        "esModuleInterop": true,
+        "outDir": "dist",
+        "skipLibCheck": true,
+        "paths": {
+            "@schema/*": [
+                "../shared/schema/*"
+            ],
+            "@services/*": [
+                "../shared/services/*"
+            ],
+            "@data/*": [
+                "../shared/data/*"
+            ],
+            "@functions/*": [
+                "../shared/functions/*"
+            ],
+            "@utils/*": [
+                "../shared/utils/*"
+            ]
+        }
+    },
+    "include": [
+        "src/**/*.ts"
+    ],
+    "exclude": [
+        "node_modules"
+    ]
+}