add new service + fix docs links

broker/.gitignore

@@ -3,4 +3,5 @@ static
 ecosystem.config.cjs
 dist
 scripts/start_dev.js
-package-lock.json
+package-lock.json
+build_all.bat

broker/ecosystem.config.example.cjs

@@ -1,18 +1,16 @@
 module.exports = {
     apps: [
         {
-            name: 'QueueBroker',
+            name: 'Producer',
             port: '3000',
-            exec_mode: 'cluster',
-            instances: '2',
-            script: './dist/broker/src/index.js',
+            exec_mode: 'fork',
+            script: './dist/producer/src/index.js',
             env: {
-                EMAIL_SERVICE: "",
-                EMAIL_HOST: "",
-                EMAIL_USER: "",
-                EMAIL_PASS: "",
                 PORT: "",
-                MONGO_CONNECTION_STRING: ""
+                REDIS_URL: "",
+                REDIS_USERNAME: "",
+                REDIS_PASSWORD: "",
+                STREAM_NAME: ""
             }
         }
     ]

broker/package.json

@@ -4,6 +4,7 @@
     "express": "^4.19.2",
     "mongoose": "^8.3.2",
     "nodemailer": "^6.9.13",
+    "redis": "^4.6.14",
     "ua-parser-js": "^1.0.37"
   },
   "devDependencies": {

broker/pnpm-lock.yaml

@@ -20,6 +20,9 @@ importers:
       nodemailer:
         specifier: ^6.9.13
         version: 6.9.13
+      redis:
+        specifier: ^4.6.14
+        version: 4.6.14
       ua-parser-js:
         specifier: ^1.0.37
         version: 1.0.38
@@ -79,6 +82,35 @@ packages:
     resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
     engines: {node: '>=14'}
 
+  '@redis/bloom@1.2.0':
+    resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/client@1.5.16':
+    resolution: {integrity: sha512-X1a3xQ5kEMvTib5fBrHKh6Y+pXbeKXqziYuxOUo1ojQNECg4M5Etd1qqyhMap+lFUOAh8S7UYevgJHOm4A+NOg==}
+    engines: {node: '>=14'}
+
+  '@redis/graph@1.1.1':
+    resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/json@1.0.6':
+    resolution: {integrity: sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/search@1.1.6':
+    resolution: {integrity: sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/time-series@1.0.5':
+    resolution: {integrity: sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
   '@tsconfig/node10@1.0.11':
     resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==}
 
@@ -206,6 +238,10 @@ packages:
     resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==}
     engines: {node: '>= 0.4'}
 
+  cluster-key-slot@1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
+    engines: {node: '>=0.10.0'}
+
   color-convert@2.0.1:
     resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
     engines: {node: '>=7.0.0'}
@@ -330,6 +366,10 @@ packages:
   function-bind@1.1.2:
     resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
 
+  generic-pool@3.9.0:
+    resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==}
+    engines: {node: '>= 4'}
+
   get-intrinsic@1.2.4:
     resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}
     engines: {node: '>= 0.4'}
@@ -549,6 +589,9 @@ packages:
     resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==}
     engines: {node: '>= 0.8'}
 
+  redis@4.6.14:
+    resolution: {integrity: sha512-GrNg/e33HtsQwNXL7kJT+iNFPSwE1IPmd7wzV3j4f2z0EYxZfZE7FVTmUysgAtqQQtg5NXF5SNLR9OdO/UHOfw==}
+
   safe-buffer@5.2.1:
     resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
 
@@ -707,6 +750,9 @@ packages:
     resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==}
     engines: {node: '>=12'}
 
+  yallist@4.0.0:
+    resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
+
   yn@3.1.1:
     resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==}
     engines: {node: '>=6'}
@@ -742,6 +788,32 @@ snapshots:
   '@pkgjs/parseargs@0.11.0':
     optional: true
 
+  '@redis/bloom@1.2.0(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/client@1.5.16':
+    dependencies:
+      cluster-key-slot: 1.1.2
+      generic-pool: 3.9.0
+      yallist: 4.0.0
+
+  '@redis/graph@1.1.1(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/json@1.0.6(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/search@1.1.6(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/time-series@1.0.5(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
   '@tsconfig/node10@1.0.11': {}
 
   '@tsconfig/node12@1.0.11': {}
@@ -881,6 +953,8 @@ snapshots:
       get-intrinsic: 1.2.4
       set-function-length: 1.2.2
 
+  cluster-key-slot@1.1.2: {}
+
   color-convert@2.0.1:
     dependencies:
       color-name: 1.1.4
@@ -1015,6 +1089,8 @@ snapshots:
 
   function-bind@1.1.2: {}
 
+  generic-pool@3.9.0: {}
+
   get-intrinsic@1.2.4:
     dependencies:
       es-errors: 1.3.0
@@ -1206,6 +1282,15 @@ snapshots:
       iconv-lite: 0.4.24
       unpipe: 1.0.0
 
+  redis@4.6.14:
+    dependencies:
+      '@redis/bloom': 1.2.0(@redis/client@1.5.16)
+      '@redis/client': 1.5.16
+      '@redis/graph': 1.1.1(@redis/client@1.5.16)
+      '@redis/json': 1.0.6(@redis/client@1.5.16)
+      '@redis/search': 1.1.6(@redis/client@1.5.16)
+      '@redis/time-series': 1.0.5(@redis/client@1.5.16)
+
   safe-buffer@5.2.1: {}
 
   safer-buffer@2.1.2: {}
@@ -1377,4 +1462,6 @@ snapshots:
       string-width: 5.1.2
       strip-ansi: 7.1.0
 
+  yallist@4.0.0: {}
+
   yn@3.1.1: {}

broker/src/StreamLoopController.ts

@@ -0,0 +1,127 @@
+
+import { RedisStreamService } from '@services/RedisStreamService';
+import { requireEnv } from '../../shared/utilts/requireEnv';
+import { EventModel } from '@schema/metrics/EventSchema';
+import { SessionModel } from '@schema/metrics/SessionSchema';
+import { ProjectModel } from '@schema/ProjectSchema';
+import { ProjectLimitModel } from '@schema/ProjectsLimits';
+import { ProjectCountModel } from '@schema/ProjectsCounts';
+import { EVENT_LOG_LIMIT_PERCENT } from '@data/broker/Limits';
+import { checkLimitsForEmail } from './Controller';
+import { lookup } from './lookup';
+import { UAParser } from 'ua-parser-js';
+import { VisitModel } from '@schema/metrics/VisitSchema';
+
+
+export async function startStreamLoop() {
+
+    await RedisStreamService.connect();
+
+    await RedisStreamService.startReadingLoop({
+        streamName: requireEnv('STREAM_NAME'),
+        delay: { base: 100, empty: 5000 },
+        readBlock: 2500
+    }, processStreamEvent);
+}
+
+
+
+
+async function processStreamEvent(data: Record<string, string>) {
+    try {
+        const eventType = data._type;
+        if (!eventType) return;
+
+
+
+        const { pid, sessionHash } = data;
+
+        const project = await ProjectModel.exists({ _id: pid });
+        if (!project) return;
+
+        if (eventType === 'event') return await process_event(data, sessionHash);
+        if (eventType === 'keep_alive') return await process_keep_alive(data, sessionHash);
+        if (eventType === 'visit') return await process_visit(data, sessionHash);
+
+    } catch (ex: any) {
+        console.error('ERROR PROCESSING STREAM EVENT', ex.message);
+    }
+}
+
+
+async function process_visit(data: Record<string, string>, sessionHash: string) {
+
+    const { pid, ip, website, page, referrer, userAgent } = data;
+
+    const projectLimits = await ProjectLimitModel.findOne({ project_id: pid });
+    if (!projectLimits) return;
+
+    const TOTAL_COUNT = projectLimits.events + projectLimits.visits;
+    const COUNT_LIMIT = projectLimits.limit;
+    if ((TOTAL_COUNT * EVENT_LOG_LIMIT_PERCENT) > COUNT_LIMIT) return;
+    await checkLimitsForEmail(projectLimits);
+
+    let referrerParsed;
+    try {
+        referrerParsed = new URL(referrer);
+    } catch (ex) {
+        referrerParsed = { hostname: referrer };
+    }
+
+    const geoLocation = lookup(ip);
+
+    const userAgentParsed = UAParser(userAgent);
+
+    const visit = new VisitModel({
+        project_id: pid, website, page, referrer: referrerParsed.hostname,
+        browser: userAgentParsed.browser.name || 'NO_BROWSER',
+        os: userAgentParsed.os.name || 'NO_OS',
+        device: userAgentParsed.device.type,
+        continent: geoLocation[0],
+        country: geoLocation[1],
+    });
+
+    await visit.save();
+
+    await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'visits': 1 } }, { upsert: true });
+    await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { 'visits': 1 } });
+
+}
+
+async function process_keep_alive(data: Record<string, string>, sessionHash: string) {
+
+    const { pid, instant } = data;
+
+    if (instant == "true") {
+        await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
+            $inc: { duration: 0 },
+            updated_at: Date.now()
+        }, { upsert: true });
+    } else {
+        await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
+            $inc: { duration: 1 },
+            updated_at: Date.now()
+        }, { upsert: true });
+    }
+
+}
+
+
+async function process_event(data: Record<string, string>, sessionHash: string) {
+
+    const { name, metadata, pid } = data;
+
+    let metadataObject;
+    try {
+        if (metadata) metadataObject = JSON.parse(metadata);
+    } catch (ex) {
+        metadataObject = { error: 'Error parsing metadata' }
+    }
+
+    const event = new EventModel({ project_id: pid, name, metadata: metadataObject });
+    await event.save();
+
+    await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'events': 1 } }, { upsert: true });
+    await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { 'events': 1 } });
+
+}

broker/src/index.ts

@@ -3,6 +3,7 @@ import cors from 'cors';
 
 import { requireEnv } from '../../shared/utilts/requireEnv';
 import { connectDatabase } from '@services/DatabaseService';
+import { startStreamLoop } from './StreamLoopController';
 
 const app = express();
 app.use(cors());
@@ -11,7 +12,7 @@ connectDatabase(requireEnv('MONGO_CONNECTION_STRING'));
 
 import HealthRouter from './routes/HealthRouter';
 app.use('/health', HealthRouter);
-import V1Router from './routes/v1/Router';
-app.use('/v1', V1Router);
 
 app.listen(requireEnv('PORT'), () => console.log(`Listening on port ${requireEnv('PORT')}`));
+
+startStreamLoop();

broker/src/routes/v1/Router.ts

@@ -1,140 +0,0 @@
-
-import { Router, json } from "express";
-import { createSessionHash, getIPFromRequest } from "../../utils/Utils";
-
-import { SessionModel } from "@schema/metrics/SessionSchema";
-import { EVENT_LOG_LIMIT_PERCENT } from '@data/broker/Limits';
-import { EventType } from '@data/broker/EventType';
-import { lookup } from "../../lookup";
-import { UAParser } from "ua-parser-js";
-import { getDeviceFromScreenSize } from "../../ScreenSize";
-import { VisitModel } from "@schema/metrics/VisitSchema";
-import { EventModel } from "@schema/metrics/EventSchema";
-import { ProjectCountModel } from "@schema/ProjectsCounts";
-import { checkLimitsForEmail } from "../../Controller";
-import { ProjectLimitModel } from "@schema/ProjectsLimits";
-import { ProjectModel } from "@schema/ProjectSchema";
-
-const router = Router();
-
-const allowAnyType = () => true;
-const jsonOptions = { limit: '10mb', type: allowAnyType }
-
-router.post('/keep_alive', json(jsonOptions), async (req, res) => {
-    try {
-
-        const ip = getIPFromRequest(req);
-
-        const { pid, website, userAgent, instant } = req.body;
-
-        const sessionHash = createSessionHash(website, ip, userAgent);
-
-        if (instant == true) {
-            await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
-                $inc: { duration: 0 },
-                updated_at: Date.now()
-            }, { upsert: true });
-        } else {
-            await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
-                $inc: { duration: 1 },
-                updated_at: Date.now()
-            }, { upsert: true });
-        }
-
-        return res.sendStatus(200);
-
-    } catch (ex) {
-        console.error(ex);
-        return res.status(500).json({ error: 'ERROR' });
-    }
-});
-
-
-router.post('/metrics/push', json(jsonOptions), async (req, res) => {
-
-    try {
-
-        const { pid } = req.body;
-
-        const projectExist = await ProjectModel.exists({ _id: pid });
-        if (!projectExist) return res.status(400).json({ error: 'Project not exist' });
-
-        const projectLimits = await ProjectLimitModel.findOne({ project_id: pid });
-
-        if (!projectLimits) return res.status(400).json({ error: 'No limits found' });
-
-        const TOTAL_COUNT = projectLimits.events + projectLimits.visits;
-        const COUNT_LIMIT = projectLimits.limit;
-        if ((TOTAL_COUNT * EVENT_LOG_LIMIT_PERCENT) > COUNT_LIMIT) {
-            return res.status(200).json({ error: 'Limit reached' });
-        };
-        await checkLimitsForEmail(projectLimits);
-
-
-
-        const ip = getIPFromRequest(req);
-
-        const { type } = req.body;
-
-        if (type === null || type === undefined) return res.status(400).json({ error: 'type is required' });
-        if (typeof type !== 'number') return res.status(400).json({ error: 'type must be a number' });
-        if (type < 0) return res.status(400).json({ error: 'type must be positive' });
-
-        if (type === EventType.VISIT) {
-            const { website, page, referrer, screenWidth, screenHeight, userAgent } = req.body;
-            let referrerParsed;
-            try {
-                referrerParsed = new URL(referrer);
-            } catch (ex) {
-                referrerParsed = { hostname: referrer };
-            }
-
-            const geoLocation = lookup(ip);
-
-            const userAgentParsed = UAParser(userAgent);
-
-            const device = getDeviceFromScreenSize(screenWidth, screenHeight);
-
-            const visit = new VisitModel({
-                project_id: pid, website, page, referrer: referrerParsed.hostname,
-                browser: userAgentParsed.browser.name || 'NO_BROWSER',
-                os: userAgentParsed.os.name || 'NO_OS',
-                device,
-                continent: geoLocation[0],
-                country: geoLocation[1],
-            });
-
-            await visit.save();
-
-        } else {
-            const { name, metadata } = req.body;
-            let metadataObject;
-            try {
-                if (metadata) metadataObject = JSON.parse(metadata);
-            } catch (ex) {
-                metadataObject = { error: 'Error parsing metadata' }
-            }
-
-            const event = new EventModel({ project_id: pid, name, metadata: metadataObject });
-            await event.save();
-        }
-
-
-        const fieldToInc = type === EventType.VISIT ? 'visits' : 'events';
-
-        await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { [fieldToInc]: 1 } }, { upsert: true });
-
-        await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { [fieldToInc]: 1 } });
-
-
-
-        return res.sendStatus(200);
-
-    } catch (ex) {
-        console.error(ex);
-        return res.status(500).json({ error: 'ERROR' });
-    }
-});
-
-
-export default router;