add new service + fix docs links

.gitignore

@@ -1 +1,2 @@
-steps
+steps
+PROCESS_EVENT

PROCESS_EVENT.md

@@ -0,0 +1,6 @@
+
+
+
+LIB ---> Producer ---> Save to Redis stream
+
+Broker ---> Read from redis stream ---> Process event ---> Save to DB

broker/.gitignore

@@ -3,4 +3,5 @@ static
 ecosystem.config.cjs
 dist
 scripts/start_dev.js
-package-lock.json
+package-lock.json
+build_all.bat

broker/ecosystem.config.example.cjs

@@ -1,18 +1,16 @@
 module.exports = {
     apps: [
         {
-            name: 'QueueBroker',
+            name: 'Producer',
             port: '3000',
-            exec_mode: 'cluster',
+            exec_mode: 'fork',
-            instances: '2',
+            script: './dist/producer/src/index.js',
-            script: './dist/broker/src/index.js',
             env: {
-                EMAIL_SERVICE: "",
-                EMAIL_HOST: "",
-                EMAIL_USER: "",
-                EMAIL_PASS: "",
                 PORT: "",
-                MONGO_CONNECTION_STRING: ""
+                REDIS_URL: "",
+                REDIS_USERNAME: "",
+                REDIS_PASSWORD: "",
+                STREAM_NAME: ""
             }
         }
     ]

broker/package.json

@@ -4,6 +4,7 @@
     "express": "^4.19.2",
     "mongoose": "^8.3.2",
     "nodemailer": "^6.9.13",
+    "redis": "^4.6.14",
     "ua-parser-js": "^1.0.37"
   },
   "devDependencies": {

broker/pnpm-lock.yaml

@@ -20,6 +20,9 @@ importers:
       nodemailer:
         specifier: ^6.9.13
         version: 6.9.13
+      redis:
+        specifier: ^4.6.14
+        version: 4.6.14
       ua-parser-js:
         specifier: ^1.0.37
         version: 1.0.38
@@ -79,6 +82,35 @@ packages:
     resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
     engines: {node: '>=14'}
 
+  '@redis/bloom@1.2.0':
+    resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/client@1.5.16':
+    resolution: {integrity: sha512-X1a3xQ5kEMvTib5fBrHKh6Y+pXbeKXqziYuxOUo1ojQNECg4M5Etd1qqyhMap+lFUOAh8S7UYevgJHOm4A+NOg==}
+    engines: {node: '>=14'}
+
+  '@redis/graph@1.1.1':
+    resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/json@1.0.6':
+    resolution: {integrity: sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/search@1.1.6':
+    resolution: {integrity: sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/time-series@1.0.5':
+    resolution: {integrity: sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
   '@tsconfig/node10@1.0.11':
     resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==}
 
@@ -206,6 +238,10 @@ packages:
     resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==}
     engines: {node: '>= 0.4'}
 
+  cluster-key-slot@1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
+    engines: {node: '>=0.10.0'}
+
   color-convert@2.0.1:
     resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
     engines: {node: '>=7.0.0'}
@@ -330,6 +366,10 @@ packages:
   function-bind@1.1.2:
     resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
 
+  generic-pool@3.9.0:
+    resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==}
+    engines: {node: '>= 4'}
+
   get-intrinsic@1.2.4:
     resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}
     engines: {node: '>= 0.4'}
@@ -549,6 +589,9 @@ packages:
     resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==}
     engines: {node: '>= 0.8'}
 
+  redis@4.6.14:
+    resolution: {integrity: sha512-GrNg/e33HtsQwNXL7kJT+iNFPSwE1IPmd7wzV3j4f2z0EYxZfZE7FVTmUysgAtqQQtg5NXF5SNLR9OdO/UHOfw==}
+
   safe-buffer@5.2.1:
     resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
 
@@ -707,6 +750,9 @@ packages:
     resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==}
     engines: {node: '>=12'}
 
+  yallist@4.0.0:
+    resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
+
   yn@3.1.1:
     resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==}
     engines: {node: '>=6'}
@@ -742,6 +788,32 @@ snapshots:
   '@pkgjs/parseargs@0.11.0':
     optional: true
 
+  '@redis/bloom@1.2.0(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/client@1.5.16':
+    dependencies:
+      cluster-key-slot: 1.1.2
+      generic-pool: 3.9.0
+      yallist: 4.0.0
+
+  '@redis/graph@1.1.1(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/json@1.0.6(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/search@1.1.6(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/time-series@1.0.5(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
   '@tsconfig/node10@1.0.11': {}
 
   '@tsconfig/node12@1.0.11': {}
@@ -881,6 +953,8 @@ snapshots:
       get-intrinsic: 1.2.4
       set-function-length: 1.2.2
 
+  cluster-key-slot@1.1.2: {}
+
   color-convert@2.0.1:
     dependencies:
       color-name: 1.1.4
@@ -1015,6 +1089,8 @@ snapshots:
 
   function-bind@1.1.2: {}
 
+  generic-pool@3.9.0: {}
+
   get-intrinsic@1.2.4:
     dependencies:
       es-errors: 1.3.0
@@ -1206,6 +1282,15 @@ snapshots:
       iconv-lite: 0.4.24
       unpipe: 1.0.0
 
+  redis@4.6.14:
+    dependencies:
+      '@redis/bloom': 1.2.0(@redis/client@1.5.16)
+      '@redis/client': 1.5.16
+      '@redis/graph': 1.1.1(@redis/client@1.5.16)
+      '@redis/json': 1.0.6(@redis/client@1.5.16)
+      '@redis/search': 1.1.6(@redis/client@1.5.16)
+      '@redis/time-series': 1.0.5(@redis/client@1.5.16)
+
   safe-buffer@5.2.1: {}
 
   safer-buffer@2.1.2: {}
@@ -1377,4 +1462,6 @@ snapshots:
       string-width: 5.1.2
       strip-ansi: 7.1.0
 
+  yallist@4.0.0: {}
+
   yn@3.1.1: {}

broker/src/StreamLoopController.ts

@@ -0,0 +1,127 @@
+
+import { RedisStreamService } from '@services/RedisStreamService';
+import { requireEnv } from '../../shared/utilts/requireEnv';
+import { EventModel } from '@schema/metrics/EventSchema';
+import { SessionModel } from '@schema/metrics/SessionSchema';
+import { ProjectModel } from '@schema/ProjectSchema';
+import { ProjectLimitModel } from '@schema/ProjectsLimits';
+import { ProjectCountModel } from '@schema/ProjectsCounts';
+import { EVENT_LOG_LIMIT_PERCENT } from '@data/broker/Limits';
+import { checkLimitsForEmail } from './Controller';
+import { lookup } from './lookup';
+import { UAParser } from 'ua-parser-js';
+import { VisitModel } from '@schema/metrics/VisitSchema';
+
+
+export async function startStreamLoop() {
+
+    await RedisStreamService.connect();
+
+    await RedisStreamService.startReadingLoop({
+        streamName: requireEnv('STREAM_NAME'),
+        delay: { base: 100, empty: 5000 },
+        readBlock: 2500
+    }, processStreamEvent);
+}
+
+
+
+
+async function processStreamEvent(data: Record<string, string>) {
+    try {
+        const eventType = data._type;
+        if (!eventType) return;
+
+
+
+        const { pid, sessionHash } = data;
+
+        const project = await ProjectModel.exists({ _id: pid });
+        if (!project) return;
+
+        if (eventType === 'event') return await process_event(data, sessionHash);
+        if (eventType === 'keep_alive') return await process_keep_alive(data, sessionHash);
+        if (eventType === 'visit') return await process_visit(data, sessionHash);
+
+    } catch (ex: any) {
+        console.error('ERROR PROCESSING STREAM EVENT', ex.message);
+    }
+}
+
+
+async function process_visit(data: Record<string, string>, sessionHash: string) {
+
+    const { pid, ip, website, page, referrer, userAgent } = data;
+
+    const projectLimits = await ProjectLimitModel.findOne({ project_id: pid });
+    if (!projectLimits) return;
+
+    const TOTAL_COUNT = projectLimits.events + projectLimits.visits;
+    const COUNT_LIMIT = projectLimits.limit;
+    if ((TOTAL_COUNT * EVENT_LOG_LIMIT_PERCENT) > COUNT_LIMIT) return;
+    await checkLimitsForEmail(projectLimits);
+
+    let referrerParsed;
+    try {
+        referrerParsed = new URL(referrer);
+    } catch (ex) {
+        referrerParsed = { hostname: referrer };
+    }
+
+    const geoLocation = lookup(ip);
+
+    const userAgentParsed = UAParser(userAgent);
+
+    const visit = new VisitModel({
+        project_id: pid, website, page, referrer: referrerParsed.hostname,
+        browser: userAgentParsed.browser.name || 'NO_BROWSER',
+        os: userAgentParsed.os.name || 'NO_OS',
+        device: userAgentParsed.device.type,
+        continent: geoLocation[0],
+        country: geoLocation[1],
+    });
+
+    await visit.save();
+
+    await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'visits': 1 } }, { upsert: true });
+    await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { 'visits': 1 } });
+
+}
+
+async function process_keep_alive(data: Record<string, string>, sessionHash: string) {
+
+    const { pid, instant } = data;
+
+    if (instant == "true") {
+        await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
+            $inc: { duration: 0 },
+            updated_at: Date.now()
+        }, { upsert: true });
+    } else {
+        await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
+            $inc: { duration: 1 },
+            updated_at: Date.now()
+        }, { upsert: true });
+    }
+
+}
+
+
+async function process_event(data: Record<string, string>, sessionHash: string) {
+
+    const { name, metadata, pid } = data;
+
+    let metadataObject;
+    try {
+        if (metadata) metadataObject = JSON.parse(metadata);
+    } catch (ex) {
+        metadataObject = { error: 'Error parsing metadata' }
+    }
+
+    const event = new EventModel({ project_id: pid, name, metadata: metadataObject });
+    await event.save();
+
+    await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'events': 1 } }, { upsert: true });
+    await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { 'events': 1 } });
+
+}

broker/src/index.ts

@@ -3,6 +3,7 @@ import cors from 'cors';
 
 import { requireEnv } from '../../shared/utilts/requireEnv';
 import { connectDatabase } from '@services/DatabaseService';
+import { startStreamLoop } from './StreamLoopController';
 
 const app = express();
 app.use(cors());
@@ -11,7 +12,7 @@ connectDatabase(requireEnv('MONGO_CONNECTION_STRING'));
 
 import HealthRouter from './routes/HealthRouter';
 app.use('/health', HealthRouter);
-import V1Router from './routes/v1/Router';
-app.use('/v1', V1Router);
 
 app.listen(requireEnv('PORT'), () => console.log(`Listening on port ${requireEnv('PORT')}`));
+
+startStreamLoop();

broker/src/routes/v1/Router.ts

@@ -1,140 +0,0 @@
-
-import { Router, json } from "express";
-import { createSessionHash, getIPFromRequest } from "../../utils/Utils";
-
-import { SessionModel } from "@schema/metrics/SessionSchema";
-import { EVENT_LOG_LIMIT_PERCENT } from '@data/broker/Limits';
-import { EventType } from '@data/broker/EventType';
-import { lookup } from "../../lookup";
-import { UAParser } from "ua-parser-js";
-import { getDeviceFromScreenSize } from "../../ScreenSize";
-import { VisitModel } from "@schema/metrics/VisitSchema";
-import { EventModel } from "@schema/metrics/EventSchema";
-import { ProjectCountModel } from "@schema/ProjectsCounts";
-import { checkLimitsForEmail } from "../../Controller";
-import { ProjectLimitModel } from "@schema/ProjectsLimits";
-import { ProjectModel } from "@schema/ProjectSchema";
-
-const router = Router();
-
-const allowAnyType = () => true;
-const jsonOptions = { limit: '10mb', type: allowAnyType }
-
-router.post('/keep_alive', json(jsonOptions), async (req, res) => {
-    try {
-
-        const ip = getIPFromRequest(req);
-
-        const { pid, website, userAgent, instant } = req.body;
-
-        const sessionHash = createSessionHash(website, ip, userAgent);
-
-        if (instant == true) {
-            await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
-                $inc: { duration: 0 },
-                updated_at: Date.now()
-            }, { upsert: true });
-        } else {
-            await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
-                $inc: { duration: 1 },
-                updated_at: Date.now()
-            }, { upsert: true });
-        }
-
-        return res.sendStatus(200);
-
-    } catch (ex) {
-        console.error(ex);
-        return res.status(500).json({ error: 'ERROR' });
-    }
-});
-
-
-router.post('/metrics/push', json(jsonOptions), async (req, res) => {
-
-    try {
-
-        const { pid } = req.body;
-
-        const projectExist = await ProjectModel.exists({ _id: pid });
-        if (!projectExist) return res.status(400).json({ error: 'Project not exist' });
-
-        const projectLimits = await ProjectLimitModel.findOne({ project_id: pid });
-
-        if (!projectLimits) return res.status(400).json({ error: 'No limits found' });
-
-        const TOTAL_COUNT = projectLimits.events + projectLimits.visits;
-        const COUNT_LIMIT = projectLimits.limit;
-        if ((TOTAL_COUNT * EVENT_LOG_LIMIT_PERCENT) > COUNT_LIMIT) {
-            return res.status(200).json({ error: 'Limit reached' });
-        };
-        await checkLimitsForEmail(projectLimits);
-
-
-
-        const ip = getIPFromRequest(req);
-
-        const { type } = req.body;
-
-        if (type === null || type === undefined) return res.status(400).json({ error: 'type is required' });
-        if (typeof type !== 'number') return res.status(400).json({ error: 'type must be a number' });
-        if (type < 0) return res.status(400).json({ error: 'type must be positive' });
-
-        if (type === EventType.VISIT) {
-            const { website, page, referrer, screenWidth, screenHeight, userAgent } = req.body;
-            let referrerParsed;
-            try {
-                referrerParsed = new URL(referrer);
-            } catch (ex) {
-                referrerParsed = { hostname: referrer };
-            }
-
-            const geoLocation = lookup(ip);
-
-            const userAgentParsed = UAParser(userAgent);
-
-            const device = getDeviceFromScreenSize(screenWidth, screenHeight);
-
-            const visit = new VisitModel({
-                project_id: pid, website, page, referrer: referrerParsed.hostname,
-                browser: userAgentParsed.browser.name || 'NO_BROWSER',
-                os: userAgentParsed.os.name || 'NO_OS',
-                device,
-                continent: geoLocation[0],
-                country: geoLocation[1],
-            });
-
-            await visit.save();
-
-        } else {
-            const { name, metadata } = req.body;
-            let metadataObject;
-            try {
-                if (metadata) metadataObject = JSON.parse(metadata);
-            } catch (ex) {
-                metadataObject = { error: 'Error parsing metadata' }
-            }
-
-            const event = new EventModel({ project_id: pid, name, metadata: metadataObject });
-            await event.save();
-        }
-
-
-        const fieldToInc = type === EventType.VISIT ? 'visits' : 'events';
-
-        await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { [fieldToInc]: 1 } }, { upsert: true });
-
-        await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { [fieldToInc]: 1 } });
-
-
-
-        return res.sendStatus(200);
-
-    } catch (ex) {
-        console.error(ex);
-        return res.status(500).json({ error: 'ERROR' });
-    }
-});
-
-
-export default router;

dashboard/pages/index.vue

@@ -31,7 +31,7 @@ function copyScript() {
         return [
             '<script defer ',
             `data-project="${activeProject.value?._id}" `,
-            'src="https://cdn.jsdelivr.net/npm/litlyx/browser/litlyx.js"></',
+            'src="https://cdn.jsdelivr.net/gh/litlyx/litlyx-js/browser/litlyx.js"></',
             'script>'
         ].join('')
     }
@@ -211,7 +211,7 @@ const selectLabelsEvents = [
                         <div class="text-[.9rem] text-[#acacac] lg:w-min">
                             {{ `
                             <script defer data-project="${activeProject?._id}"
-                                src="https://cdn.jsdelivr.net/npm/litlyx/browser/litlyx.js"></script>` }}
+                                src="https://cdn.jsdelivr.net/gh/litlyx/litlyx-js/browser/litlyx.js"></script>` }}
                         </div>
                     </div>
                 </div>

docs/docs/getstarted/import.md

@@ -31,7 +31,7 @@ import { Lit } from 'litlyx';
 If your are using vanilla js you can find pretty intresting this approach too
 
 ```html
-<script defer data-project="project_id_here" src="https://cdn.jsdelivr.net/npm/litlyx@1.1.0/browser/litlyx.js"></script>
+<script defer data-project="project_id_here" src="https://cdn.jsdelivr.net/gh/litlyx/litlyx-js/browser/litlyx.js"></script>
 ```
 
   </TabItem>

docs/docs/intro.md

@@ -24,7 +24,7 @@ Place this line in your project (recommended below the body tag in index.html):
 
 
 ```html
-<script defer data-project="project_id_here" src="https://cdn.jsdelivr.net/npm/litlyx@1.1.0/browser/litlyx.js"></script>
+<script defer data-project="project_id_here" src="https://cdn.jsdelivr.net/gh/litlyx/litlyx-js/browser/litlyx.js"></script>
 ```
 
 This script collects various data from your websites, including `page visits`, `referrers`, `page routes`, `operating systems (OS)`, `browsers`, `countries`, `unique users`, `average session times`, and `real-time user access`. All data is gathered and displayed in real-time in your Litlyx dashboard.

landing/nuxt.config.ts

@@ -7,7 +7,7 @@ export default defineNuxtConfig({
     head: {
       script: [
         {
-          src: 'https://cdn.jsdelivr.net/npm/litlyx@1.0.5/browser/litlyx.js',
+          src: 'https://cdn.jsdelivr.net/gh/litlyx/litlyx-js/browser/litlyx.js',
           'data-project': '6643cd08a1854e3b81722ab5',
           defer: true
         }

producer/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+start_dev.js
+dist

producer/ecosystem.config.cjs

@@ -0,0 +1,17 @@
+module.exports = {
+    apps: [
+        {
+            name: 'Producer',
+            port: '3000',
+            exec_mode: 'fork',
+            script: './dist/producer/src/index.js',
+            env: {
+                PORT: "8088",
+                REDIS_URL: "redis://litlyx.com",
+                REDIS_USERNAME: "default",
+                REDIS_PASSWORD: "cameriera",
+                STREAM_NAME: "lib-events"
+            }
+        }
+    ]
+}

producer/ecosystem.config.example.cjs

@@ -0,0 +1,23 @@
+module.exports = {
+    apps: [
+        {
+            name: 'QueueBroker',
+            port: '3000',
+            exec_mode: 'cluster',
+            instances: '2',
+            script: './dist/broker/src/index.js',
+            env: {
+                EMAIL_SERVICE: "",
+                EMAIL_HOST: "",
+                EMAIL_USER: "",
+                EMAIL_PASS: "",
+                PORT: "",
+                MONGO_CONNECTION_STRING: "",
+                REDIS_URL: "",
+                REDIS_USERNAME: "",
+                REDIS_PASSWORD: "",
+                STREAM_NAME: ""
+            }
+        }
+    ]
+}

producer/package.json

@@ -0,0 +1,29 @@
+{
+	"dependencies": {
+		"cors": "^2.8.5",
+		"express": "^4.19.2",
+		"ioredis": "^5.4.1",
+		"redis": "^4.6.14",
+		"remove": "^0.1.5"
+	},
+	"devDependencies": {
+		"@types/cors": "^2.8.17",
+		"@types/express": "^4.17.21",
+		"@types/node": "^20.14.2",
+		"glob": "^10.4.1",
+		"typescript": "^5.4.5"
+	},
+	"name": "litlyx-producer",
+	"version": "1.0.0",
+	"main": "dist/index.js",
+	"scripts": {
+		"dev": "node scripts/start_dev.js",
+		"compile": "tsc",
+		"build": "ts-node scripts/build.ts",
+		"build_all": "scripts/build_all.bat"
+	},
+	"keywords": [],
+	"author": "Emily",
+	"license": "MIT",
+	"description": "Producer for Litlyx - Saves events to redis stream."
+}

producer/scripts/build.ts

@@ -0,0 +1,17 @@
+
+import fs from 'fs';
+import { globSync } from 'glob';
+const tsconfigContent = fs.readFileSync('tsconfig.json', 'utf8');
+const tsconfigObject = JSON.parse(tsconfigContent);
+const paths = tsconfigObject.compilerOptions.paths;
+const filesList = globSync('dist/**/*.js');
+filesList.forEach(file => {
+    let raw = fs.readFileSync(file, 'utf8');
+    for (const path in paths) {
+        const deep = (file.match(/\\|\//g) || []).length;
+        const pathText = path.replace('*', '');
+        const toReplaceText = new RegExp(`"${pathText}(.*?)"`, 'g');
+        raw = raw.replace(toReplaceText, `"${new Array(deep - 2).fill('../').join('')}${paths[path][0].replace('*', '')}${'$1'}"`);
+    }
+    fs.writeFileSync(file, raw);
+});

producer/src/index.ts

@@ -0,0 +1,69 @@
+import { requireEnv } from "../../shared/utilts/requireEnv";
+import { RedisStreamService } from "@services/RedisStreamService";
+
+import crypto from 'crypto';
+import express from 'express';
+import cors from 'cors';
+
+const app = express();
+app.use(cors());
+
+const allowAnyType = () => true;
+const jsonOptions = { limit: '5mb', type: allowAnyType }
+
+const streamName = requireEnv('STREAM_NAME');
+
+
+function getIPFromRequest(req: express.Request) {
+    const ip = req.header('X-Real-IP') || req.header('X-Forwarded-For') || '0.0.0.0';
+    return ip;
+}
+
+
+export function createSessionHash(website: string, ip: string, userAgent: string) {
+    const dailySalt = new Date().toLocaleDateString('it-IT');
+    const sessionClean = dailySalt + website + ip + userAgent;
+    const sessionHash = crypto.createHash('md5').update(sessionClean).digest("hex");
+    return sessionHash;
+}
+
+app.post('/event', express.json(jsonOptions), async (req, res) => {
+    try {
+        const ip = getIPFromRequest(req);
+        const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent);
+        await RedisStreamService.addToStream(streamName, { ...req.body, _type: 'event', sessionHash, ip });
+        return res.sendStatus(200);
+    } catch (ex: any) {
+        return res.status(500).json({ error: ex.message });
+    }
+});
+
+app.post('/visit', express.json(jsonOptions), async (req, res) => {
+    try {
+        const ip = getIPFromRequest(req);
+        const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent);
+        await RedisStreamService.addToStream(streamName, { ...req.body, _type: 'visit', sessionHash, ip });
+        return res.sendStatus(200);
+    } catch (ex: any) {
+        return res.status(500).json({ error: ex.message });
+    }
+});
+
+app.post('/keep_alive', express.json(jsonOptions), async (req, res) => {
+    try {
+        const ip = getIPFromRequest(req);
+        const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent);
+        await RedisStreamService.addToStream(streamName, { ...req.body, _type: 'keep_alive', sessionHash, ip });
+        return res.sendStatus(200);
+    } catch (ex: any) {
+        return res.status(500).json({ error: ex.message });
+    }
+});
+
+async function main() {
+    await RedisStreamService.connect();
+    app.listen(requireEnv("PORT"), () => console.log(`Listening on port ${requireEnv("PORT")}`));
+}
+
+main();
+

producer/tsconfig.json

@@ -0,0 +1,28 @@
+{
+    "compilerOptions": {
+        "module": "NodeNext",
+        "target": "ESNext",
+        "esModuleInterop": true,
+        "outDir": "dist",
+        "paths": {
+            "@schema/*": [
+                "../shared/schema/*"
+            ],
+            "@services/*": [
+                "../shared/services/*"
+            ],
+            "@data/*": [
+                "../shared/data/*"
+            ],
+            "@functions/*": [
+                "../shared/functions/*"
+            ]
+        }
+    },
+    "include": [
+        "src/**/*.ts"
+    ],
+    "exclude": [
+        "node_modules"
+    ]
+}

readme.md

@@ -48,7 +48,7 @@ npm i litlyx
 Or import it directly into your JavaScript code:
 
 ```html
-<script defer data-project="project_id_here" src="https://cdn.jsdelivr.net/npm/litlyx/browser/litlyx.js"></script>
+<script defer data-project="project_id_here" src="https://cdn.jsdelivr.net/gh/litlyx/litlyx-js/browser/litlyx.js"></script>
 ```
 
 Importing Litlyx with a direct script already tracks 10 KPIs such as page visits, browsers, devices, OS, real-time online users, and many more.

shared/package.json

@@ -1,7 +1,8 @@
 {
 	"dependencies": {
 		"mongoose": "^8.4.0",
-		"nodemailer": "^6.9.13"
+		"nodemailer": "^6.9.13",
+		"redis": "^4.6.14"
 	},
 	"devDependencies": {
 		"@types/node": "^20.12.13",

shared/pnpm-lock.yaml

@@ -14,6 +14,9 @@ importers:
       nodemailer:
         specifier: ^6.9.13
         version: 6.9.13
+      redis:
+        specifier: ^4.6.14
+        version: 4.6.14
     devDependencies:
       '@types/node':
         specifier: ^20.12.13
@@ -27,6 +30,35 @@ packages:
   '@mongodb-js/saslprep@1.1.7':
     resolution: {integrity: sha512-dCHW/oEX0KJ4NjDULBo3JiOaK5+6axtpBbS+ao2ZInoAL9/YRQLhXzSNAFz7hP4nzLkIqsfYAK/PDE3+XHny0Q==}
 
+  '@redis/bloom@1.2.0':
+    resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/client@1.5.16':
+    resolution: {integrity: sha512-X1a3xQ5kEMvTib5fBrHKh6Y+pXbeKXqziYuxOUo1ojQNECg4M5Etd1qqyhMap+lFUOAh8S7UYevgJHOm4A+NOg==}
+    engines: {node: '>=14'}
+
+  '@redis/graph@1.1.1':
+    resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/json@1.0.6':
+    resolution: {integrity: sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/search@1.1.6':
+    resolution: {integrity: sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/time-series@1.0.5':
+    resolution: {integrity: sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
   '@types/node@20.12.13':
     resolution: {integrity: sha512-gBGeanV41c1L171rR7wjbMiEpEI/l5XFQdLLfhr/REwpgDy/4U8y89+i8kRiLzDyZdOkXh+cRaTetUnCYutoXA==}
 
@@ -43,6 +75,10 @@ packages:
     resolution: {integrity: sha512-w2IquM5mYzYZv6rs3uN2DZTOBe2a0zXLj53TGDqwF4l6Sz/XsISrisXOJihArF9+BZ6Cq/GjVht7Sjfmri7ytQ==}
     engines: {node: '>=16.20.1'}
 
+  cluster-key-slot@1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
+    engines: {node: '>=0.10.0'}
+
   debug@4.3.4:
     resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
     engines: {node: '>=6.0'}
@@ -52,6 +88,10 @@ packages:
       supports-color:
         optional: true
 
+  generic-pool@3.9.0:
+    resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==}
+    engines: {node: '>= 4'}
+
   kareem@2.6.3:
     resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==}
     engines: {node: '>=12.0.0'}
@@ -115,6 +155,9 @@ packages:
     resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
     engines: {node: '>=6'}
 
+  redis@4.6.14:
+    resolution: {integrity: sha512-GrNg/e33HtsQwNXL7kJT+iNFPSwE1IPmd7wzV3j4f2z0EYxZfZE7FVTmUysgAtqQQtg5NXF5SNLR9OdO/UHOfw==}
+
   sift@17.1.3:
     resolution: {integrity: sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==}
 
@@ -136,12 +179,41 @@ packages:
     resolution: {integrity: sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==}
     engines: {node: '>=16'}
 
+  yallist@4.0.0:
+    resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
+
 snapshots:
 
   '@mongodb-js/saslprep@1.1.7':
     dependencies:
       sparse-bitfield: 3.0.3
 
+  '@redis/bloom@1.2.0(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/client@1.5.16':
+    dependencies:
+      cluster-key-slot: 1.1.2
+      generic-pool: 3.9.0
+      yallist: 4.0.0
+
+  '@redis/graph@1.1.1(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/json@1.0.6(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/search@1.1.6(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
+  '@redis/time-series@1.0.5(@redis/client@1.5.16)':
+    dependencies:
+      '@redis/client': 1.5.16
+
   '@types/node@20.12.13':
     dependencies:
       undici-types: 5.26.5
@@ -158,10 +230,14 @@ snapshots:
 
   bson@6.7.0: {}
 
+  cluster-key-slot@1.1.2: {}
+
   debug@4.3.4:
     dependencies:
       ms: 2.1.2
 
+  generic-pool@3.9.0: {}
+
   kareem@2.6.3: {}
 
   memory-pager@1.5.0: {}
@@ -212,6 +288,15 @@ snapshots:
 
   punycode@2.3.1: {}
 
+  redis@4.6.14:
+    dependencies:
+      '@redis/bloom': 1.2.0(@redis/client@1.5.16)
+      '@redis/client': 1.5.16
+      '@redis/graph': 1.1.1(@redis/client@1.5.16)
+      '@redis/json': 1.0.6(@redis/client@1.5.16)
+      '@redis/search': 1.1.6(@redis/client@1.5.16)
+      '@redis/time-series': 1.0.5(@redis/client@1.5.16)
+
   sift@17.1.3: {}
 
   sparse-bitfield@3.0.3:
@@ -230,3 +315,5 @@ snapshots:
     dependencies:
       tr46: 4.1.1
       webidl-conversions: 7.0.0
+
+  yallist@4.0.0: {}

shared/services/RedisStreamService.ts

@@ -0,0 +1,59 @@
+import { createClient } from 'redis';
+import { requireEnv } from '../utilts/requireEnv';
+
+export type ReadingLoopOptions = {
+    delay?: {
+        base?: number,
+        empty?: number
+    },
+    readBlock?: number,
+    streamName: string
+}
+
+export class RedisStreamService {
+
+    private static client = createClient({
+        url: requireEnv("REDIS_URL"),
+        username: requireEnv("REDIS_USERNAME"),
+        password: requireEnv("REDIS_PASSWORD"),
+    });
+
+    static async connect() {
+        await this.client.connect();
+    }
+
+    private static async readingLoop(options: ReadingLoopOptions, processFunction: (content: Record<string, string>) => Promise<any>) {
+        const result = await this.readFromStream(options.streamName, options.readBlock || 2500);
+        if (!result) {
+            await new Promise(r => setTimeout(r, options.delay?.empty || 5000));
+            setTimeout(() => this.readingLoop(options, processFunction), 1);
+            return;
+        }
+        await processFunction(result);
+        await new Promise(r => setTimeout(r, options.delay?.base || 100));
+        setTimeout(() => this.readingLoop(options, processFunction), 1);
+        return;
+    }
+
+    static startReadingLoop(options: ReadingLoopOptions, processFunction: (content: Record<string, string>) => Promise<any>) {
+        this.readingLoop(options, processFunction)
+    }
+
+    private static async readFromStream(streamName: string, readBlock: number) {
+        const result = await this.client.xRead({ id: '0', key: streamName }, { COUNT: 1, BLOCK: readBlock });
+        if (!result) return;
+        if (result.length == 0) return;
+        if (!result[0].messages) return;
+        if (result[0].messages.length == 0) return;
+        const message = result[0].messages[0];
+        await this.client.xDel(streamName, message.id);
+        const content = message.message;
+        return content;
+    }
+
+    static async addToStream(streamName: string, data: Record<string, string>) {
+        const result = await this.client.xAdd(streamName, "*", data);
+        return result;
+    }
+
+}