Services rewrite

broker/src/Controller.ts

@@ -2,7 +2,7 @@ import { ProjectModel } from "@schema/ProjectSchema";
 import { UserModel } from "@schema/UserSchema";
 import { LimitNotifyModel } from "@schema/broker/LimitNotifySchema";
 import EmailService from '@services/EmailService';
-import { requireEnv } from "../../shared/utilts/requireEnv";
+import { requireEnv } from "@utils/requireEnv";
 import { TProjectLimit } from "@schema/ProjectsLimits";
 
 if (process.env.EMAIL_SERVICE) {

broker/src/StreamLoopController.ts

@@ -1,6 +1,6 @@
 
 import { RedisStreamService } from '@services/RedisStreamService';
-import { requireEnv } from '../../shared/utilts/requireEnv';
+import { requireEnv } from '@utils/requireEnv';
 import { EventModel } from '@schema/metrics/EventSchema';
 import { SessionModel } from '@schema/metrics/SessionSchema';
 import { ProjectModel } from '@schema/ProjectSchema';

broker/src/index.ts

@@ -1,7 +1,7 @@
 import express from 'express';
 import cors from 'cors';
 
-import { requireEnv } from '../../shared/utilts/requireEnv';
+import { requireEnv } from '@utils/requireEnv';
 import { connectDatabase } from '@services/DatabaseService';
 import { startStreamLoop } from './StreamLoopController';
 

broker/tsconfig.json

@@ -21,6 +21,9 @@
             ],
             "@functions/*": [
                 "../shared/functions/*"
+            ],
+            "@utils/*": [
+                "../shared/utils/*"
             ]
         }
     },

consumer-database/.gitignore

@@ -0,0 +1,8 @@
+node_modules
+static
+ecosystem.config.cjs
+dist
+scripts/start_dev.js
+package-lock.json
+build_all.bat
+tests

consumer-database/ecosystem.config.example.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+    apps: [
+        {
+            name: 'QueueBroker',
+            port: '3999',
+            exec_mode: 'fork',
+            script: './dist/consumer-database/src/index.js',
+            env: {
+                MONGO_CONNECTION_STRING: "",
+                REDIS_URL: "",
+                REDIS_USERNAME: "",
+                REDIS_PASSWORD: "",
+                STREAM_NAME: "",
+                GROUP_NAME: ""
+            }
+        }
+    ]
+}

consumer-database/package.json

@@ -0,0 +1,27 @@
+{
+  "dependencies": {
+    "mongoose": "^8.3.2",
+    "redis": "^4.6.14",
+    "ua-parser-js": "^1.0.37"
+  },
+  "devDependencies": {
+    "@types/node": "^20.12.13",
+    "@types/ua-parser-js": "^0.7.39",
+    "glob": "^10.4.1",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.5"
+  },
+  "name": "litlyx-queue-broker",
+  "version": "1.0.0",
+  "main": "dist/index.js",
+  "scripts": {
+    "dev": "node scripts/start_dev.js",
+    "compile": "tsc",
+    "build": "node ../scripts/build.js",
+    "build_all": "npm run compile && npm run build"
+  },
+  "keywords": [],
+  "author": "Emily",
+  "license": "MIT",
+  "description": "Database Consumer - Saves events to database."
+}

consumer-database/pnpm-lock.yaml

@@ -0,0 +1,716 @@
+lockfileVersion: '9.0'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+importers:
+
+  .:
+    dependencies:
+      mongoose:
+        specifier: ^8.3.2
+        version: 8.6.3
+      redis:
+        specifier: ^4.6.14
+        version: 4.7.0
+      ua-parser-js:
+        specifier: ^1.0.37
+        version: 1.0.39
+    devDependencies:
+      '@types/node':
+        specifier: ^20.12.13
+        version: 20.16.5
+      '@types/ua-parser-js':
+        specifier: ^0.7.39
+        version: 0.7.39
+      glob:
+        specifier: ^10.4.1
+        version: 10.4.5
+      ts-node:
+        specifier: ^10.9.2
+        version: 10.9.2(@types/node@20.16.5)(typescript@5.6.2)
+      typescript:
+        specifier: ^5.4.5
+        version: 5.6.2
+
+packages:
+
+  '@cspotcode/source-map-support@0.8.1':
+    resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}
+    engines: {node: '>=12'}
+
+  '@isaacs/cliui@8.0.2':
+    resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
+    engines: {node: '>=12'}
+
+  '@jridgewell/resolve-uri@3.1.2':
+    resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==}
+    engines: {node: '>=6.0.0'}
+
+  '@jridgewell/sourcemap-codec@1.5.0':
+    resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==}
+
+  '@jridgewell/trace-mapping@0.3.9':
+    resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==}
+
+  '@mongodb-js/saslprep@1.1.9':
+    resolution: {integrity: sha512-tVkljjeEaAhCqTzajSdgbQ6gE6f3oneVwa3iXR6csiEwXXOFsiC6Uh9iAjAhXPtqa/XMDHWjjeNH/77m/Yq2dw==}
+
+  '@pkgjs/parseargs@0.11.0':
+    resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
+    engines: {node: '>=14'}
+
+  '@redis/bloom@1.2.0':
+    resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/client@1.6.0':
+    resolution: {integrity: sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==}
+    engines: {node: '>=14'}
+
+  '@redis/graph@1.1.1':
+    resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/json@1.0.7':
+    resolution: {integrity: sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/search@1.2.0':
+    resolution: {integrity: sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/time-series@1.1.0':
+    resolution: {integrity: sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@tsconfig/node10@1.0.11':
+    resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==}
+
+  '@tsconfig/node12@1.0.11':
+    resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==}
+
+  '@tsconfig/node14@1.0.3':
+    resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==}
+
+  '@tsconfig/node16@1.0.4':
+    resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==}
+
+  '@types/node@20.16.5':
+    resolution: {integrity: sha512-VwYCweNo3ERajwy0IUlqqcyZ8/A7Zwa9ZP3MnENWcB11AejO+tLy3pu850goUW2FC/IJMdZUfKpX/yxL1gymCA==}
+
+  '@types/ua-parser-js@0.7.39':
+    resolution: {integrity: sha512-P/oDfpofrdtF5xw433SPALpdSchtJmY7nsJItf8h3KXqOslkbySh8zq4dSWXH2oTjRvJ5PczVEoCZPow6GicLg==}
+
+  '@types/webidl-conversions@7.0.3':
+    resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==}
+
+  '@types/whatwg-url@11.0.5':
+    resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==}
+
+  acorn-walk@8.3.4:
+    resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==}
+    engines: {node: '>=0.4.0'}
+
+  acorn@8.12.1:
+    resolution: {integrity: sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==}
+    engines: {node: '>=0.4.0'}
+    hasBin: true
+
+  ansi-regex@5.0.1:
+    resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
+    engines: {node: '>=8'}
+
+  ansi-regex@6.1.0:
+    resolution: {integrity: sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==}
+    engines: {node: '>=12'}
+
+  ansi-styles@4.3.0:
+    resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
+    engines: {node: '>=8'}
+
+  ansi-styles@6.2.1:
+    resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==}
+    engines: {node: '>=12'}
+
+  arg@4.1.3:
+    resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==}
+
+  balanced-match@1.0.2:
+    resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
+
+  brace-expansion@2.0.1:
+    resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==}
+
+  bson@6.8.0:
+    resolution: {integrity: sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ==}
+    engines: {node: '>=16.20.1'}
+
+  cluster-key-slot@1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
+    engines: {node: '>=0.10.0'}
+
+  color-convert@2.0.1:
+    resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
+    engines: {node: '>=7.0.0'}
+
+  color-name@1.1.4:
+    resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
+
+  create-require@1.1.1:
+    resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
+
+  cross-spawn@7.0.3:
+    resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==}
+    engines: {node: '>= 8'}
+
+  debug@4.3.7:
+    resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==}
+    engines: {node: '>=6.0'}
+    peerDependencies:
+      supports-color: '*'
+    peerDependenciesMeta:
+      supports-color:
+        optional: true
+
+  diff@4.0.2:
+    resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==}
+    engines: {node: '>=0.3.1'}
+
+  eastasianwidth@0.2.0:
+    resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==}
+
+  emoji-regex@8.0.0:
+    resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
+
+  emoji-regex@9.2.2:
+    resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
+
+  foreground-child@3.3.0:
+    resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==}
+    engines: {node: '>=14'}
+
+  generic-pool@3.9.0:
+    resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==}
+    engines: {node: '>= 4'}
+
+  glob@10.4.5:
+    resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==}
+    hasBin: true
+
+  is-fullwidth-code-point@3.0.0:
+    resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}
+    engines: {node: '>=8'}
+
+  isexe@2.0.0:
+    resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
+
+  jackspeak@3.4.3:
+    resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==}
+
+  kareem@2.6.3:
+    resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==}
+    engines: {node: '>=12.0.0'}
+
+  lru-cache@10.4.3:
+    resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==}
+
+  make-error@1.3.6:
+    resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==}
+
+  memory-pager@1.5.0:
+    resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==}
+
+  minimatch@9.0.5:
+    resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==}
+    engines: {node: '>=16 || 14 >=14.17'}
+
+  minipass@7.1.2:
+    resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}
+    engines: {node: '>=16 || 14 >=14.17'}
+
+  mongodb-connection-string-url@3.0.1:
+    resolution: {integrity: sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==}
+
+  mongodb@6.8.0:
+    resolution: {integrity: sha512-HGQ9NWDle5WvwMnrvUxsFYPd3JEbqD3RgABHBQRuoCEND0qzhsd0iH5ypHsf1eJ+sXmvmyKpP+FLOKY8Il7jMw==}
+    engines: {node: '>=16.20.1'}
+    peerDependencies:
+      '@aws-sdk/credential-providers': ^3.188.0
+      '@mongodb-js/zstd': ^1.1.0
+      gcp-metadata: ^5.2.0
+      kerberos: ^2.0.1
+      mongodb-client-encryption: '>=6.0.0 <7'
+      snappy: ^7.2.2
+      socks: ^2.7.1
+    peerDependenciesMeta:
+      '@aws-sdk/credential-providers':
+        optional: true
+      '@mongodb-js/zstd':
+        optional: true
+      gcp-metadata:
+        optional: true
+      kerberos:
+        optional: true
+      mongodb-client-encryption:
+        optional: true
+      snappy:
+        optional: true
+      socks:
+        optional: true
+
+  mongoose@8.6.3:
+    resolution: {integrity: sha512-++yRmm7hjMbqVA/8WeiygTnEfrFbiy+OBjQi49GFJIvCQuSYE56myyQWo4j5hbpcHjhHQU8NukMNGTwAWFWjIw==}
+    engines: {node: '>=16.20.1'}
+
+  mpath@0.9.0:
+    resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==}
+    engines: {node: '>=4.0.0'}
+
+  mquery@5.0.0:
+    resolution: {integrity: sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==}
+    engines: {node: '>=14.0.0'}
+
+  ms@2.1.3:
+    resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
+
+  package-json-from-dist@1.0.0:
+    resolution: {integrity: sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==}
+
+  path-key@3.1.1:
+    resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
+    engines: {node: '>=8'}
+
+  path-scurry@1.11.1:
+    resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==}
+    engines: {node: '>=16 || 14 >=14.18'}
+
+  punycode@2.3.1:
+    resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
+    engines: {node: '>=6'}
+
+  redis@4.7.0:
+    resolution: {integrity: sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==}
+
+  shebang-command@2.0.0:
+    resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==}
+    engines: {node: '>=8'}
+
+  shebang-regex@3.0.0:
+    resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
+    engines: {node: '>=8'}
+
+  sift@17.1.3:
+    resolution: {integrity: sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==}
+
+  signal-exit@4.1.0:
+    resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}
+    engines: {node: '>=14'}
+
+  sparse-bitfield@3.0.3:
+    resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==}
+
+  string-width@4.2.3:
+    resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
+    engines: {node: '>=8'}
+
+  string-width@5.1.2:
+    resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==}
+    engines: {node: '>=12'}
+
+  strip-ansi@6.0.1:
+    resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==}
+    engines: {node: '>=8'}
+
+  strip-ansi@7.1.0:
+    resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==}
+    engines: {node: '>=12'}
+
+  tr46@4.1.1:
+    resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==}
+    engines: {node: '>=14'}
+
+  ts-node@10.9.2:
+    resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==}
+    hasBin: true
+    peerDependencies:
+      '@swc/core': '>=1.2.50'
+      '@swc/wasm': '>=1.2.50'
+      '@types/node': '*'
+      typescript: '>=2.7'
+    peerDependenciesMeta:
+      '@swc/core':
+        optional: true
+      '@swc/wasm':
+        optional: true
+
+  typescript@5.6.2:
+    resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==}
+    engines: {node: '>=14.17'}
+    hasBin: true
+
+  ua-parser-js@1.0.39:
+    resolution: {integrity: sha512-k24RCVWlEcjkdOxYmVJgeD/0a1TiSpqLg+ZalVGV9lsnr4yqu0w7tX/x2xX6G4zpkgQnRf89lxuZ1wsbjXM8lw==}
+    hasBin: true
+
+  undici-types@6.19.8:
+    resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==}
+
+  v8-compile-cache-lib@3.0.1:
+    resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==}
+
+  webidl-conversions@7.0.0:
+    resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==}
+    engines: {node: '>=12'}
+
+  whatwg-url@13.0.0:
+    resolution: {integrity: sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==}
+    engines: {node: '>=16'}
+
+  which@2.0.2:
+    resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
+    engines: {node: '>= 8'}
+    hasBin: true
+
+  wrap-ansi@7.0.0:
+    resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==}
+    engines: {node: '>=10'}
+
+  wrap-ansi@8.1.0:
+    resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==}
+    engines: {node: '>=12'}
+
+  yallist@4.0.0:
+    resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
+
+  yn@3.1.1:
+    resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==}
+    engines: {node: '>=6'}
+
+snapshots:
+
+  '@cspotcode/source-map-support@0.8.1':
+    dependencies:
+      '@jridgewell/trace-mapping': 0.3.9
+
+  '@isaacs/cliui@8.0.2':
+    dependencies:
+      string-width: 5.1.2
+      string-width-cjs: string-width@4.2.3
+      strip-ansi: 7.1.0
+      strip-ansi-cjs: strip-ansi@6.0.1
+      wrap-ansi: 8.1.0
+      wrap-ansi-cjs: wrap-ansi@7.0.0
+
+  '@jridgewell/resolve-uri@3.1.2': {}
+
+  '@jridgewell/sourcemap-codec@1.5.0': {}
+
+  '@jridgewell/trace-mapping@0.3.9':
+    dependencies:
+      '@jridgewell/resolve-uri': 3.1.2
+      '@jridgewell/sourcemap-codec': 1.5.0
+
+  '@mongodb-js/saslprep@1.1.9':
+    dependencies:
+      sparse-bitfield: 3.0.3
+
+  '@pkgjs/parseargs@0.11.0':
+    optional: true
+
+  '@redis/bloom@1.2.0(@redis/client@1.6.0)':
+    dependencies:
+      '@redis/client': 1.6.0
+
+  '@redis/client@1.6.0':
+    dependencies:
+      cluster-key-slot: 1.1.2
+      generic-pool: 3.9.0
+      yallist: 4.0.0
+
+  '@redis/graph@1.1.1(@redis/client@1.6.0)':
+    dependencies:
+      '@redis/client': 1.6.0
+
+  '@redis/json@1.0.7(@redis/client@1.6.0)':
+    dependencies:
+      '@redis/client': 1.6.0
+
+  '@redis/search@1.2.0(@redis/client@1.6.0)':
+    dependencies:
+      '@redis/client': 1.6.0
+
+  '@redis/time-series@1.1.0(@redis/client@1.6.0)':
+    dependencies:
+      '@redis/client': 1.6.0
+
+  '@tsconfig/node10@1.0.11': {}
+
+  '@tsconfig/node12@1.0.11': {}
+
+  '@tsconfig/node14@1.0.3': {}
+
+  '@tsconfig/node16@1.0.4': {}
+
+  '@types/node@20.16.5':
+    dependencies:
+      undici-types: 6.19.8
+
+  '@types/ua-parser-js@0.7.39': {}
+
+  '@types/webidl-conversions@7.0.3': {}
+
+  '@types/whatwg-url@11.0.5':
+    dependencies:
+      '@types/webidl-conversions': 7.0.3
+
+  acorn-walk@8.3.4:
+    dependencies:
+      acorn: 8.12.1
+
+  acorn@8.12.1: {}
+
+  ansi-regex@5.0.1: {}
+
+  ansi-regex@6.1.0: {}
+
+  ansi-styles@4.3.0:
+    dependencies:
+      color-convert: 2.0.1
+
+  ansi-styles@6.2.1: {}
+
+  arg@4.1.3: {}
+
+  balanced-match@1.0.2: {}
+
+  brace-expansion@2.0.1:
+    dependencies:
+      balanced-match: 1.0.2
+
+  bson@6.8.0: {}
+
+  cluster-key-slot@1.1.2: {}
+
+  color-convert@2.0.1:
+    dependencies:
+      color-name: 1.1.4
+
+  color-name@1.1.4: {}
+
+  create-require@1.1.1: {}
+
+  cross-spawn@7.0.3:
+    dependencies:
+      path-key: 3.1.1
+      shebang-command: 2.0.0
+      which: 2.0.2
+
+  debug@4.3.7:
+    dependencies:
+      ms: 2.1.3
+
+  diff@4.0.2: {}
+
+  eastasianwidth@0.2.0: {}
+
+  emoji-regex@8.0.0: {}
+
+  emoji-regex@9.2.2: {}
+
+  foreground-child@3.3.0:
+    dependencies:
+      cross-spawn: 7.0.3
+      signal-exit: 4.1.0
+
+  generic-pool@3.9.0: {}
+
+  glob@10.4.5:
+    dependencies:
+      foreground-child: 3.3.0
+      jackspeak: 3.4.3
+      minimatch: 9.0.5
+      minipass: 7.1.2
+      package-json-from-dist: 1.0.0
+      path-scurry: 1.11.1
+
+  is-fullwidth-code-point@3.0.0: {}
+
+  isexe@2.0.0: {}
+
+  jackspeak@3.4.3:
+    dependencies:
+      '@isaacs/cliui': 8.0.2
+    optionalDependencies:
+      '@pkgjs/parseargs': 0.11.0
+
+  kareem@2.6.3: {}
+
+  lru-cache@10.4.3: {}
+
+  make-error@1.3.6: {}
+
+  memory-pager@1.5.0: {}
+
+  minimatch@9.0.5:
+    dependencies:
+      brace-expansion: 2.0.1
+
+  minipass@7.1.2: {}
+
+  mongodb-connection-string-url@3.0.1:
+    dependencies:
+      '@types/whatwg-url': 11.0.5
+      whatwg-url: 13.0.0
+
+  mongodb@6.8.0:
+    dependencies:
+      '@mongodb-js/saslprep': 1.1.9
+      bson: 6.8.0
+      mongodb-connection-string-url: 3.0.1
+
+  mongoose@8.6.3:
+    dependencies:
+      bson: 6.8.0
+      kareem: 2.6.3
+      mongodb: 6.8.0
+      mpath: 0.9.0
+      mquery: 5.0.0
+      ms: 2.1.3
+      sift: 17.1.3
+    transitivePeerDependencies:
+      - '@aws-sdk/credential-providers'
+      - '@mongodb-js/zstd'
+      - gcp-metadata
+      - kerberos
+      - mongodb-client-encryption
+      - snappy
+      - socks
+      - supports-color
+
+  mpath@0.9.0: {}
+
+  mquery@5.0.0:
+    dependencies:
+      debug: 4.3.7
+    transitivePeerDependencies:
+      - supports-color
+
+  ms@2.1.3: {}
+
+  package-json-from-dist@1.0.0: {}
+
+  path-key@3.1.1: {}
+
+  path-scurry@1.11.1:
+    dependencies:
+      lru-cache: 10.4.3
+      minipass: 7.1.2
+
+  punycode@2.3.1: {}
+
+  redis@4.7.0:
+    dependencies:
+      '@redis/bloom': 1.2.0(@redis/client@1.6.0)
+      '@redis/client': 1.6.0
+      '@redis/graph': 1.1.1(@redis/client@1.6.0)
+      '@redis/json': 1.0.7(@redis/client@1.6.0)
+      '@redis/search': 1.2.0(@redis/client@1.6.0)
+      '@redis/time-series': 1.1.0(@redis/client@1.6.0)
+
+  shebang-command@2.0.0:
+    dependencies:
+      shebang-regex: 3.0.0
+
+  shebang-regex@3.0.0: {}
+
+  sift@17.1.3: {}
+
+  signal-exit@4.1.0: {}
+
+  sparse-bitfield@3.0.3:
+    dependencies:
+      memory-pager: 1.5.0
+
+  string-width@4.2.3:
+    dependencies:
+      emoji-regex: 8.0.0
+      is-fullwidth-code-point: 3.0.0
+      strip-ansi: 6.0.1
+
+  string-width@5.1.2:
+    dependencies:
+      eastasianwidth: 0.2.0
+      emoji-regex: 9.2.2
+      strip-ansi: 7.1.0
+
+  strip-ansi@6.0.1:
+    dependencies:
+      ansi-regex: 5.0.1
+
+  strip-ansi@7.1.0:
+    dependencies:
+      ansi-regex: 6.1.0
+
+  tr46@4.1.1:
+    dependencies:
+      punycode: 2.3.1
+
+  ts-node@10.9.2(@types/node@20.16.5)(typescript@5.6.2):
+    dependencies:
+      '@cspotcode/source-map-support': 0.8.1
+      '@tsconfig/node10': 1.0.11
+      '@tsconfig/node12': 1.0.11
+      '@tsconfig/node14': 1.0.3
+      '@tsconfig/node16': 1.0.4
+      '@types/node': 20.16.5
+      acorn: 8.12.1
+      acorn-walk: 8.3.4
+      arg: 4.1.3
+      create-require: 1.1.1
+      diff: 4.0.2
+      make-error: 1.3.6
+      typescript: 5.6.2
+      v8-compile-cache-lib: 3.0.1
+      yn: 3.1.1
+
+  typescript@5.6.2: {}
+
+  ua-parser-js@1.0.39: {}
+
+  undici-types@6.19.8: {}
+
+  v8-compile-cache-lib@3.0.1: {}
+
+  webidl-conversions@7.0.0: {}
+
+  whatwg-url@13.0.0:
+    dependencies:
+      tr46: 4.1.1
+      webidl-conversions: 7.0.0
+
+  which@2.0.2:
+    dependencies:
+      isexe: 2.0.0
+
+  wrap-ansi@7.0.0:
+    dependencies:
+      ansi-styles: 4.3.0
+      string-width: 4.2.3
+      strip-ansi: 6.0.1
+
+  wrap-ansi@8.1.0:
+    dependencies:
+      ansi-styles: 6.2.1
+      string-width: 5.1.2
+      strip-ansi: 7.1.0
+
+  yallist@4.0.0: {}
+
+  yn@3.1.1: {}

consumer-database/scripts/GeoLite2-Country-Locations-en.csv

@@ -0,0 +1,253 @@
+geoname_id,locale_code,continent_code,continent_name,country_iso_code,country_name,is_in_european_union
+49518,en,AF,Africa,RW,Rwanda,0
+51537,en,AF,Africa,SO,Somalia,0
+69543,en,AS,Asia,YE,Yemen,0
+99237,en,AS,Asia,IQ,Iraq,0
+102358,en,AS,Asia,SA,"Saudi Arabia",0
+130758,en,AS,Asia,IR,Iran,0
+146669,en,EU,Europe,CY,Cyprus,1
+149590,en,AF,Africa,TZ,Tanzania,0
+163843,en,AS,Asia,SY,Syria,0
+174982,en,AS,Asia,AM,Armenia,0
+192950,en,AF,Africa,KE,Kenya,0
+203312,en,AF,Africa,CD,"DR Congo",0
+223816,en,AF,Africa,DJ,Djibouti,0
+226074,en,AF,Africa,UG,Uganda,0
+239880,en,AF,Africa,CF,"Central African Republic",0
+241170,en,AF,Africa,SC,Seychelles,0
+248816,en,AS,Asia,JO,Jordan,0
+272103,en,AS,Asia,LB,Lebanon,0
+285570,en,AS,Asia,KW,Kuwait,0
+286963,en,AS,Asia,OM,Oman,0
+289688,en,AS,Asia,QA,Qatar,0
+290291,en,AS,Asia,BH,Bahrain,0
+290557,en,AS,Asia,AE,"United Arab Emirates",0
+294640,en,AS,Asia,IL,Israel,0
+298795,en,AS,Asia,TR,Türkiye,0
+337996,en,AF,Africa,ET,Ethiopia,0
+338010,en,AF,Africa,ER,Eritrea,0
+357994,en,AF,Africa,EG,Egypt,0
+366755,en,AF,Africa,SD,Sudan,0
+390903,en,EU,Europe,GR,Greece,1
+433561,en,AF,Africa,BI,Burundi,0
+453733,en,EU,Europe,EE,Estonia,1
+458258,en,EU,Europe,LV,Latvia,1
+587116,en,AS,Asia,AZ,Azerbaijan,0
+597427,en,EU,Europe,LT,Lithuania,1
+607072,en,EU,Europe,SJ,"Svalbard and Jan Mayen",0
+614540,en,AS,Asia,GE,Georgia,0
+617790,en,EU,Europe,MD,Moldova,0
+630336,en,EU,Europe,BY,Belarus,0
+660013,en,EU,Europe,FI,Finland,1
+661882,en,EU,Europe,AX,"Åland Islands",1
+690791,en,EU,Europe,UA,Ukraine,0
+718075,en,EU,Europe,MK,"North Macedonia",0
+719819,en,EU,Europe,HU,Hungary,1
+732800,en,EU,Europe,BG,Bulgaria,1
+783754,en,EU,Europe,AL,Albania,0
+798544,en,EU,Europe,PL,Poland,1
+798549,en,EU,Europe,RO,Romania,1
+831053,en,EU,Europe,XK,Kosovo,0
+878675,en,AF,Africa,ZW,Zimbabwe,0
+895949,en,AF,Africa,ZM,Zambia,0
+921929,en,AF,Africa,KM,Comoros,0
+927384,en,AF,Africa,MW,Malawi,0
+932692,en,AF,Africa,LS,Lesotho,0
+933860,en,AF,Africa,BW,Botswana,0
+934292,en,AF,Africa,MU,Mauritius,0
+934841,en,AF,Africa,SZ,Eswatini,0
+935317,en,AF,Africa,RE,Réunion,1
+953987,en,AF,Africa,ZA,"South Africa",0
+1024031,en,AF,Africa,YT,Mayotte,1
+1036973,en,AF,Africa,MZ,Mozambique,0
+1062947,en,AF,Africa,MG,Madagascar,0
+1149361,en,AS,Asia,AF,Afghanistan,0
+1168579,en,AS,Asia,PK,Pakistan,0
+1210997,en,AS,Asia,BD,Bangladesh,0
+1218197,en,AS,Asia,TM,Turkmenistan,0
+1220409,en,AS,Asia,TJ,Tajikistan,0
+1227603,en,AS,Asia,LK,"Sri Lanka",0
+1252634,en,AS,Asia,BT,Bhutan,0
+1269750,en,AS,Asia,IN,India,0
+1282028,en,AS,Asia,MV,Maldives,0
+1282588,en,AS,Asia,IO,"British Indian Ocean Territory",0
+1282988,en,AS,Asia,NP,Nepal,0
+1327865,en,AS,Asia,MM,Myanmar,0
+1512440,en,AS,Asia,UZ,Uzbekistan,0
+1522867,en,AS,Asia,KZ,Kazakhstan,0
+1527747,en,AS,Asia,KG,Kyrgyzstan,0
+1546748,en,AN,Antarctica,TF,"French Southern Territories",0
+1547314,en,AN,Antarctica,HM,"Heard and McDonald Islands",0
+1547376,en,AS,Asia,CC,"Cocos (Keeling) Islands",0
+1559582,en,OC,Oceania,PW,Palau,0
+1562822,en,AS,Asia,VN,Vietnam,0
+1605651,en,AS,Asia,TH,Thailand,0
+1643084,en,AS,Asia,ID,Indonesia,0
+1655842,en,AS,Asia,LA,Laos,0
+1668284,en,AS,Asia,TW,Taiwan,0
+1694008,en,AS,Asia,PH,Philippines,0
+1733045,en,AS,Asia,MY,Malaysia,0
+1814991,en,AS,Asia,CN,China,0
+1819730,en,AS,Asia,HK,"Hong Kong",0
+1820814,en,AS,Asia,BN,Brunei,0
+1821275,en,AS,Asia,MO,Macao,0
+1831722,en,AS,Asia,KH,Cambodia,0
+1835841,en,AS,Asia,KR,"South Korea",0
+1861060,en,AS,Asia,JP,Japan,0
+1873107,en,AS,Asia,KP,"North Korea",0
+1880251,en,AS,Asia,SG,Singapore,0
+1899402,en,OC,Oceania,CK,"Cook Islands",0
+1966436,en,OC,Oceania,TL,Timor-Leste,0
+2017370,en,EU,Europe,RU,Russia,0
+2029969,en,AS,Asia,MN,Mongolia,0
+2077456,en,OC,Oceania,AU,Australia,0
+2078138,en,OC,Oceania,CX,"Christmas Island",0
+2080185,en,OC,Oceania,MH,"Marshall Islands",0
+2081918,en,OC,Oceania,FM,"Federated States of Micronesia",0
+2088628,en,OC,Oceania,PG,"Papua New Guinea",0
+2103350,en,OC,Oceania,SB,"Solomon Islands",0
+2110297,en,OC,Oceania,TV,Tuvalu,0
+2110425,en,OC,Oceania,NR,Nauru,0
+2134431,en,OC,Oceania,VU,Vanuatu,0
+2139685,en,OC,Oceania,NC,"New Caledonia",0
+2155115,en,OC,Oceania,NF,"Norfolk Island",0
+2186224,en,OC,Oceania,NZ,"New Zealand",0
+2205218,en,OC,Oceania,FJ,Fiji,0
+2215636,en,AF,Africa,LY,Libya,0
+2233387,en,AF,Africa,CM,Cameroon,0
+2245662,en,AF,Africa,SN,Senegal,0
+2260494,en,AF,Africa,CG,"Congo Republic",0
+2264397,en,EU,Europe,PT,Portugal,1
+2275384,en,AF,Africa,LR,Liberia,0
+2287781,en,AF,Africa,CI,"Ivory Coast",0
+2300660,en,AF,Africa,GH,Ghana,0
+2309096,en,AF,Africa,GQ,"Equatorial Guinea",0
+2328926,en,AF,Africa,NG,Nigeria,0
+2361809,en,AF,Africa,BF,"Burkina Faso",0
+2363686,en,AF,Africa,TG,Togo,0
+2372248,en,AF,Africa,GW,Guinea-Bissau,0
+2378080,en,AF,Africa,MR,Mauritania,0
+2395170,en,AF,Africa,BJ,Benin,0
+2400553,en,AF,Africa,GA,Gabon,0
+2403846,en,AF,Africa,SL,"Sierra Leone",0
+2410758,en,AF,Africa,ST,"São Tomé and Príncipe",0
+2411586,en,EU,Europe,GI,Gibraltar,0
+2413451,en,AF,Africa,GM,Gambia,0
+2420477,en,AF,Africa,GN,Guinea,0
+2434508,en,AF,Africa,TD,Chad,0
+2440476,en,AF,Africa,NE,Niger,0
+2453866,en,AF,Africa,ML,Mali,0
+2461445,en,AF,Africa,EH,"Western Sahara",0
+2464461,en,AF,Africa,TN,Tunisia,0
+2510769,en,EU,Europe,ES,Spain,1
+2542007,en,AF,Africa,MA,Morocco,0
+2562770,en,EU,Europe,MT,Malta,1
+2589581,en,AF,Africa,DZ,Algeria,0
+2622320,en,EU,Europe,FO,"Faroe Islands",0
+2623032,en,EU,Europe,DK,Denmark,1
+2629691,en,EU,Europe,IS,Iceland,0
+2635167,en,EU,Europe,GB,"United Kingdom",0
+2658434,en,EU,Europe,CH,Switzerland,0
+2661886,en,EU,Europe,SE,Sweden,1
+2750405,en,EU,Europe,NL,"The Netherlands",1
+2782113,en,EU,Europe,AT,Austria,1
+2802361,en,EU,Europe,BE,Belgium,1
+2921044,en,EU,Europe,DE,Germany,1
+2960313,en,EU,Europe,LU,Luxembourg,1
+2963597,en,EU,Europe,IE,Ireland,1
+2993457,en,EU,Europe,MC,Monaco,0
+3017382,en,EU,Europe,FR,France,1
+3041565,en,EU,Europe,AD,Andorra,0
+3042058,en,EU,Europe,LI,Liechtenstein,0
+3042142,en,EU,Europe,JE,Jersey,0
+3042225,en,EU,Europe,IM,"Isle of Man",0
+3042362,en,EU,Europe,GG,Guernsey,0
+3057568,en,EU,Europe,SK,Slovakia,1
+3077311,en,EU,Europe,CZ,Czechia,1
+3144096,en,EU,Europe,NO,Norway,0
+3164670,en,EU,Europe,VA,"Vatican City",0
+3168068,en,EU,Europe,SM,"San Marino",0
+3175395,en,EU,Europe,IT,Italy,1
+3190538,en,EU,Europe,SI,Slovenia,1
+3194884,en,EU,Europe,ME,Montenegro,0
+3202326,en,EU,Europe,HR,Croatia,1
+3277605,en,EU,Europe,BA,"Bosnia and Herzegovina",0
+3351879,en,AF,Africa,AO,Angola,0
+3355338,en,AF,Africa,NA,Namibia,0
+3370751,en,AF,Africa,SH,"Saint Helena",0
+3371123,en,AN,Antarctica,BV,"Bouvet Island",0
+3374084,en,NA,"North America",BB,Barbados,0
+3374766,en,AF,Africa,CV,"Cabo Verde",0
+3378535,en,SA,"South America",GY,Guyana,0
+3381670,en,SA,"South America",GF,"French Guiana",1
+3382998,en,SA,"South America",SR,Suriname,0
+3424932,en,NA,"North America",PM,"Saint Pierre and Miquelon",0
+3425505,en,NA,"North America",GL,Greenland,0
+3437598,en,SA,"South America",PY,Paraguay,0
+3439705,en,SA,"South America",UY,Uruguay,0
+3469034,en,SA,"South America",BR,Brazil,0
+3474414,en,SA,"South America",FK,"Falkland Islands",0
+3474415,en,AN,Antarctica,GS,"South Georgia and the South Sandwich Islands",0
+3489940,en,NA,"North America",JM,Jamaica,0
+3508796,en,NA,"North America",DO,"Dominican Republic",0
+3562981,en,NA,"North America",CU,Cuba,0
+3570311,en,NA,"North America",MQ,Martinique,1
+3572887,en,NA,"North America",BS,Bahamas,0
+3573345,en,NA,"North America",BM,Bermuda,0
+3573511,en,NA,"North America",AI,Anguilla,0
+3573591,en,NA,"North America",TT,"Trinidad and Tobago",0
+3575174,en,NA,"North America",KN,"St Kitts and Nevis",0
+3575830,en,NA,"North America",DM,Dominica,0
+3576396,en,NA,"North America",AG,"Antigua and Barbuda",0
+3576468,en,NA,"North America",LC,"Saint Lucia",0
+3576916,en,NA,"North America",TC,"Turks and Caicos Islands",0
+3577279,en,NA,"North America",AW,Aruba,0
+3577718,en,NA,"North America",VG,"British Virgin Islands",0
+3577815,en,NA,"North America",VC,"St Vincent and Grenadines",0
+3578097,en,NA,"North America",MS,Montserrat,0
+3578421,en,NA,"North America",MF,"Saint Martin",1
+3578476,en,NA,"North America",BL,"Saint Barthélemy",0
+3579143,en,NA,"North America",GP,Guadeloupe,1
+3580239,en,NA,"North America",GD,Grenada,0
+3580718,en,NA,"North America",KY,"Cayman Islands",0
+3582678,en,NA,"North America",BZ,Belize,0
+3585968,en,NA,"North America",SV,"El Salvador",0
+3595528,en,NA,"North America",GT,Guatemala,0
+3608932,en,NA,"North America",HN,Honduras,0
+3617476,en,NA,"North America",NI,Nicaragua,0
+3624060,en,NA,"North America",CR,"Costa Rica",0
+3625428,en,SA,"South America",VE,Venezuela,0
+3658394,en,SA,"South America",EC,Ecuador,0
+3686110,en,SA,"South America",CO,Colombia,0
+3703430,en,NA,"North America",PA,Panama,0
+3723988,en,NA,"North America",HT,Haiti,0
+3865483,en,SA,"South America",AR,Argentina,0
+3895114,en,SA,"South America",CL,Chile,0
+3923057,en,SA,"South America",BO,Bolivia,0
+3932488,en,SA,"South America",PE,Peru,0
+3996063,en,NA,"North America",MX,Mexico,0
+4030656,en,OC,Oceania,PF,"French Polynesia",0
+4030699,en,OC,Oceania,PN,"Pitcairn Islands",0
+4030945,en,OC,Oceania,KI,Kiribati,0
+4031074,en,OC,Oceania,TK,Tokelau,0
+4032283,en,OC,Oceania,TO,Tonga,0
+4034749,en,OC,Oceania,WF,"Wallis and Futuna",0
+4034894,en,OC,Oceania,WS,Samoa,0
+4036232,en,OC,Oceania,NU,Niue,0
+4041468,en,OC,Oceania,MP,"Northern Mariana Islands",0
+4043988,en,OC,Oceania,GU,Guam,0
+4566966,en,NA,"North America",PR,"Puerto Rico",0
+4796775,en,NA,"North America",VI,"U.S. Virgin Islands",0
+5854968,en,OC,Oceania,UM,"U.S. Outlying Islands",0
+5880801,en,OC,Oceania,AS,"American Samoa",0
+6251999,en,NA,"North America",CA,Canada,0
+6252001,en,NA,"North America",US,"United States",0
+6254930,en,AS,Asia,PS,Palestine,0
+6255147,en,AS,Asia,,,0
+6255148,en,EU,Europe,,,0
+6290252,en,EU,Europe,RS,Serbia,0
+6697173,en,AN,Antarctica,AQ,Antarctica,0
+7609695,en,NA,"North America",SX,"Sint Maarten",0
+7626836,en,NA,"North America",CW,Curaçao,0
+7626844,en,NA,"North America",BQ,"Bonaire, Sint Eustatius, and Saba",0
+7909807,en,AF,Africa,SS,"South Sudan",0

consumer-database/scripts/create_database.ts

@@ -0,0 +1,36 @@
+import fs from 'fs';
+
+function createIpDatabase() {
+    const data = fs.readFileSync('GeoLite2-Country-Blocks-IPv4.csv', 'utf8');
+    const rows = data.split('\n');
+    rows.splice(0, 1);
+    rows.splice(-1);
+    const parsed: [number, string, number][] = [];
+    for (const row of rows) {
+        const lineData = row.trim().split(',');
+        parsed.push([
+            parseInt(lineData[0].split('.')[0]),
+            lineData[0],
+            parseInt(lineData[1] || '0')
+        ]);
+    }
+    fs.writeFileSync('../dist/ipv4-db.json', JSON.stringify(parsed));
+
+}
+
+function createCountryDatabase() {
+    const data = fs.readFileSync('GeoLite2-Country-Locations-en.csv', 'utf8');
+    const rows = data.split('\n');
+    rows.splice(0, 1);
+    rows.splice(-1);
+    const parsed: [number, string, string][] = [];
+    for (const row of rows) {
+        const lineData = row.trim().split(',');
+        parsed.push([parseInt(lineData[0]), lineData[2], lineData[4]]);
+    }
+    fs.writeFileSync('../dist/countries-db.json', JSON.stringify(parsed));
+
+}
+
+createIpDatabase();
+createCountryDatabase();

consumer-database/src/index.ts

@@ -0,0 +1,112 @@
+
+import { requireEnv } from '@utils/requireEnv';
+import { connectDatabase } from '@services/DatabaseService';
+import { RedisStreamService } from '@services/RedisStreamService';
+import { ProjectModel } from "@schema/ProjectSchema";
+import { VisitModel } from "@schema/metrics/VisitSchema";
+import { SessionModel } from "@schema/metrics/SessionSchema";
+import { EventModel } from "@schema/metrics/EventSchema";
+import { lookup } from './lookup';
+import { UAParser } from 'ua-parser-js';
+
+connectDatabase(requireEnv('MONGO_CONNECTION_STRING'));
+main();
+
+async function main() {
+
+    await RedisStreamService.connect();
+
+    const stream_name = requireEnv('STREAM_NAME');
+    const group_name = requireEnv('GROUP_NAME') as any; // Checks are inside "startReadingLoop"
+
+    await RedisStreamService.startReadingLoop({
+        stream_name, group_name, consumer_name: `CONSUMER_${process.env.NODE_APP_INSTANCE}`
+    }, processStreamEntry);
+
+}
+
+async function processStreamEntry(data: Record<string, string>) {
+    try {
+        const eventType = data._type;
+        if (!eventType) return;
+
+        const { pid, sessionHash } = data;
+
+        const project = await ProjectModel.exists({ _id: pid });
+        if (!project) return;
+
+
+        if (eventType === 'event') return await process_event(data, sessionHash);
+        if (eventType === 'keep_alive') return await process_keep_alive(data, sessionHash);
+        if (eventType === 'visit') return await process_visit(data, sessionHash);
+
+    } catch (ex: any) {
+        console.error('ERROR PROCESSING STREAM EVENT', ex.message);
+    }
+}
+
+async function process_visit(data: Record<string, string>, sessionHash: string) {
+
+    const { pid, ip, website, page, referrer, userAgent, flowHash } = data;
+
+    let referrerParsed;
+    try {
+        referrerParsed = new URL(referrer);
+    } catch (ex) {
+        referrerParsed = { hostname: referrer };
+    }
+
+    const geoLocation = lookup(ip);
+
+    const userAgentParsed = UAParser(userAgent);
+
+    const device = userAgentParsed.device.type;
+
+    await VisitModel.create({
+        project_id: pid, website, page, referrer: referrerParsed.hostname,
+        browser: userAgentParsed.browser.name || 'NO_BROWSER',
+        os: userAgentParsed.os.name || 'NO_OS',
+        device: device ? device : (userAgentParsed.browser.name ? 'desktop' : undefined),
+        session: sessionHash,
+        flowHash,
+        continent: geoLocation[0],
+        country: geoLocation[1],
+    });
+
+}
+
+async function process_keep_alive(data: Record<string, string>, sessionHash: string) {
+
+    const { pid, instant, flowHash } = data;
+
+    if (instant == "true") {
+        await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
+            $inc: { duration: 0 },
+            flowHash,
+            updated_at: Date.now()
+        }, { upsert: true });
+    } else {
+        await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
+            $inc: { duration: 1 },
+            flowHash,
+            updated_at: Date.now()
+        }, { upsert: true });
+    }
+
+}
+
+async function process_event(data: Record<string, string>, sessionHash: string) {
+
+    const { name, metadata, pid, flowHash } = data;
+
+    let metadataObject;
+    try {
+        if (metadata) metadataObject = JSON.parse(metadata);
+    } catch (ex) {
+        metadataObject = { error: 'Error parsing metadata' }
+    }
+
+    const event = new EventModel({ project_id: pid, name, flowHash, metadata: metadataObject, session: sessionHash });
+    await event.save();
+
+}

consumer-database/src/lookup.ts

@@ -0,0 +1,42 @@
+import fs from 'fs';
+
+const ipsData = JSON.parse(fs.readFileSync('./dist/ipv4-db.json', 'utf8'));
+const countriesData = JSON.parse(fs.readFileSync('./dist/countries-db.json', 'utf8'));
+
+function inRange(ip: string, cidr: string) {
+    const [subnet, mask] = cidr.split('/');
+    const ipBytes = ip.split('.').map(Number);
+    const subnetBytes = subnet.split('.').map(Number);
+
+    const ipInt = (ipBytes[0] << 24) | (ipBytes[1] << 16) | (ipBytes[2] << 8) | ipBytes[3];
+    const subnetInt = (subnetBytes[0] << 24) | (subnetBytes[1] << 16) | (subnetBytes[2] << 8) | subnetBytes[3];
+
+    const maskInt = 0xffffffff << (32 - parseInt(mask));
+
+    return (ipInt & maskInt) === (subnetInt & maskInt);
+}
+
+function getCountryFromId(id: number) {
+    for (const country of countriesData) {
+        if (country[0] == id) {
+            return country;
+        }
+    }
+}
+
+export function lookup(ip: string) {
+    try {
+        const startPiece = parseInt(ip.split('.')[0]);
+        for (const target of ipsData) {
+            const matchingStartPiece = target[0] == startPiece;
+            if (!matchingStartPiece) continue;
+            if (!inRange(ip, target[1])) continue;
+            const country = getCountryFromId(target[2]);
+            return [country[1], country[2]];
+        }
+        return ['??', '??'];
+    } catch (ex) {
+        console.error('ERROR DURING LOOKUP', ex);
+        return ['??', '??'];
+    }
+}

consumer-database/tsconfig.json

@@ -0,0 +1,32 @@
+{
+    "compilerOptions": {
+        "baseUrl": ".",
+        "module": "NodeNext",
+        "target": "ESNext",
+        "esModuleInterop": true,
+        "outDir": "dist",
+        "paths": {
+            "@schema/*": [
+                "../shared/schema/*"
+            ],
+            "@services/*": [
+                "../shared/services/*"
+            ],
+            "@data/*": [
+                "../shared/data/*"
+            ],
+            "@functions/*": [
+                "../shared/functions/*"
+            ],
+            "@utils/*": [
+                "../shared/utils/*"
+            ]
+        }
+    },
+    "include": [
+        "src/**/*.ts"
+    ],
+    "exclude": [
+        "node_modules"
+    ]
+}

dashboard/components/dashboard/EventsChart.vue

@@ -77,7 +77,7 @@ const chartData = ref<ChartData<'doughnut'>>({
 
 const { doughnutChartProps, doughnutChartRef } = useDoughnutChart({ chartData: chartData, options: chartOptions });
 
-const activeProject = useActiveProject();
+const activeProjectId = useActiveProjectId();
 
 const { safeSnapshotDates } = useSnapshot();
 
@@ -102,12 +102,14 @@ const headers = computed(() => {
     return {
         'x-from': safeSnapshotDates.value.from,
         'x-to': safeSnapshotDates.value.to,
-        Authorization: authorizationHeaderComputed.value,
+        'Authorization': authorizationHeaderComputed.value,
-        limit: "10"
+        'x-schema': 'events',
+        'x-limit': "10",
+        'x-pid': activeProjectId.data.value || ''
     }
 });
 
-const eventsData = useFetch(`/api/metrics/${activeProject.value?._id}/data/events`, {
+const eventsData = useFetch(`/api/data/query`, {
     method: 'POST', headers, lazy: true, immediate: false, transform: transformResponse
 });
 

dashboard/components/settings/billing.vue

@@ -8,8 +8,11 @@ const activeProject = useActiveProject();
 definePageMeta({ layout: 'dashboard' });
 
 const { data: planData, refresh: planRefresh, pending: planPending } = useFetch('/api/project/plan', {
-    ...signHeaders(),
+    ...signHeaders(), lazy: true
-    lazy: true
+});
+
+const { data: customerAddress, refresh: refreshCustomerAddress } = useFetch(`/api/pay/${activeProject.value?._id.toString()}/customer_info`, {
+    ...signHeaders(), lazy: true
 });
 
 const percent = computed(() => {
@@ -43,8 +46,7 @@ const prettyExpireDate = computed(() => {
 
 
 const { data: invoices, refresh: invoicesRefresh, pending: invoicesPending } = useFetch(`/api/pay/${activeProject.value?._id.toString()}/invoices`, {
-    ...signHeaders(),
+    ...signHeaders(), lazy: true
-    lazy: true
 })
 
 function openInvoice(link: string) {
@@ -65,25 +67,50 @@ function getPremiumPrice(type: number) {
     return (PLAN.COST / 100).toFixed(2).replace('.', ',')
 }
 
-
-watch(activeProject, () => {
-    invoicesRefresh();
-    planRefresh();
-})
-
-
 const entries: SettingsTemplateEntry[] = [
-    // { id: 'info', title: 'Billing informations', text: 'Manage billing informations for this project' },
+    { id: 'info', title: 'Billing informations', text: 'Manage billing informations for this project' },
     { id: 'plan', title: 'Current plan', text: 'Manage current plat for this project' },
     { id: 'usage', title: 'Usage', text: 'Show usage of current project' },
     { id: 'invoices', title: 'Invoices', text: 'Manage invoices of current project' },
 ]
 
+watch(customerAddress, () => {
+    console.log('UPDATE',customerAddress.value)
+    if (!customerAddress.value) return;
+    currentBillingInfo.value = customerAddress.value;
+});
 
 const currentBillingInfo = ref<any>({
-    address: ''
+    line1: '',
+    line2: '',
+    city: '',
+    country: '',
+    postal_code: '',
+    state: ''
 });
 
+const { createAlert } = useAlert()
+
+async function saveBillingInfo() {
+    
+    try {
+    const res = await $fetch(`/api/pay/${activeProject.value?._id.toString()}/update_customer`, {
+        method: 'POST',
+        ...signHeaders({
+            'Content-Type': 'application/json'
+        }),
+        body: JSON.stringify(currentBillingInfo.value)
+    });
+
+    createAlert('Customer updated','Customer updated successfully', 'far fa-check', 5000);
+
+} catch(ex) {
+    createAlert('Error updating customer','An error occurred while updating the customer', 'far fa-error', 8000);
+}
+
+}
+
+
 const { visible } = usePricingDrawer();
 
 </script>
@@ -97,6 +124,35 @@ const { visible } = usePricingDrawer();
         </div>
 
         <SettingsTemplate v-if="!invoicesPending && !planPending" :entries="entries">
+            <template #info>
+                <div>
+                    <div class="flex flex-col gap-2">
+                        <LyxUiInput class="px-2 py-1" placeholder="Address line 1" v-model="currentBillingInfo.line1">
+                        </LyxUiInput>
+                        <LyxUiInput class="px-2 py-1" placeholder="Address line 2" v-model="currentBillingInfo.line2">
+                        </LyxUiInput>
+                        <div class="flex gap-2 w-full">
+                            <LyxUiInput class="px-2 py-1 w-full" placeholder="Country"
+                                v-model="currentBillingInfo.country">
+                            </LyxUiInput>
+                            <LyxUiInput class="px-2 py-1 w-full" placeholder="Postal code"
+                                v-model="currentBillingInfo.postal_code">
+                            </LyxUiInput>
+                        </div>
+                        <div class="flex gap-2 w-full">
+                            <LyxUiInput class="px-2 py-1 w-full" placeholder="City" v-model="currentBillingInfo.city">
+                            </LyxUiInput>
+                            <LyxUiInput class="px-2 py-1 w-full" placeholder="State" v-model="currentBillingInfo.state">
+                            </LyxUiInput>
+                        </div>
+                    </div>
+                    <div class="mt-5 flex justify-end">
+                        <LyxUiButton type="primary" @click="saveBillingInfo">
+                            Save
+                        </LyxUiButton>
+                    </div>
+                </div>
+            </template>
             <template #plan>
                 <LyxUiCard v-if="planData" class="flex flex-col w-full">
                     <div class="flex flex-col gap-6 px-8 grow">

dashboard/pages/settings.vue

@@ -3,6 +3,8 @@
 definePageMeta({ layout: 'dashboard' });
 
 
+const activeProject = useActiveProject();
+
 const items = [
     { label: 'General', slot: 'general' },
     { label: 'Members', slot: 'members' },
@@ -18,32 +20,18 @@ const items = [
 
         <CustomTab :items="items" class="mt-8">
             <template #general>
-                <SettingsGeneral></SettingsGeneral>
+                <SettingsGeneral :key="activeProject?._id.toString()"></SettingsGeneral>
             </template>
             <template #members>
-                <SettingsMembers></SettingsMembers>
+                <SettingsMembers :key="activeProject?._id.toString()"></SettingsMembers>
             </template>
             <template #billing>
-                <SettingsBilling></SettingsBilling>
+                <SettingsBilling :key="activeProject?._id.toString()"></SettingsBilling>
             </template>
             <template #account>
-                <SettingsAccount></SettingsAccount>
+                <SettingsAccount :key="activeProject?._id.toString()"></SettingsAccount>
             </template>
         </CustomTab>
 
-        <!-- <UTabs :items="items" class="mt-8">
-            <template #general>
-                <SettingsGeneral></SettingsGeneral>
-            </template>
-            <template #members>
-                <SettingsMembers></SettingsMembers>
-            </template>
-            <template #billing>
-                <SettingsBilling></SettingsBilling>
-            </template>
-            <template #account>
-                <SettingsAccount></SettingsAccount>
-            </template>
-        </UTabs> -->
     </div>
 </template>

dashboard/server/LIVE_DEMO_DATA.ts

@@ -4,19 +4,22 @@ import { LITLYX_PROJECT_ID } from '@data/LITLYX'
 import { hasAccessToProject } from "./utils/hasAccessToProject";
 
 export async function getUserProjectFromId(project_id: string, user: AuthContext | undefined, allowGuest: boolean = true) {
-    if (project_id == LITLYX_PROJECT_ID) {
+    if (!project_id) return;
-        const project = await ProjectModel.findOne({ _id: project_id });
+
-        return project;
+    if (project_id === LITLYX_PROJECT_ID) {
-    } else {
+        return await ProjectModel.findOne({ _id: project_id });
-        if (!user?.logged) return;
-        if (!project_id) return;
-        const project = await ProjectModel.findById(project_id);
-        if (!project) return;
-        const [hasAccess, role] = await hasAccessToProject(user.id, project_id, project);
-        if (!hasAccess) return;
-        if (role === 'GUEST' && !allowGuest) return false;
-        return project;
     }
 
+    if (!user || !user.logged) return;
+
+    const project = await ProjectModel.findById(project_id);
+    if (!project) return;
+
+    const [hasAccess, role] = await hasAccessToProject(user.id, project_id, project);
+    if (!hasAccess) return;
+
+    if (role === 'GUEST' && !allowGuest) return false;
+
+    return project;
 
 }

dashboard/server/api/data/query.ts

@@ -0,0 +1,65 @@
+import { EventModel } from "@schema/metrics/EventSchema";
+import { getUserProjectFromId } from "~/server/LIVE_DEMO_DATA";
+import { Redis } from "~/server/services/CacheService";
+
+import type { Model } from "mongoose";
+
+
+const allowedModels: Record<string, { model: Model<any>, field: string }> = {
+    'events': {
+        model: EventModel,
+        field: 'name'
+    }
+}
+
+type TModelName = keyof typeof allowedModels;
+
+export default defineEventHandler(async event => {
+    const project_id = getHeader(event, 'x-pid');
+    if (!project_id) return;
+
+    const user = getRequestUser(event);
+    const project = await getUserProjectFromId(project_id, user);
+    if (!project) return;
+
+    const from = getRequestHeader(event, 'x-from');
+    const to = getRequestHeader(event, 'x-to');
+
+    if (!from || !to) return setResponseStatus(event, 400, 'x-from and x-to are required');
+
+    const schemaName = getRequestHeader(event, 'x-schema');
+    if (!schemaName) return setResponseStatus(event, 400, 'x-schema is required');
+
+    if (!Object.keys(allowedModels).includes(schemaName)) return setResponseStatus(event, 400, 'x-schema value is not valid');
+
+    const limitHeader = getRequestHeader(event, 'x-query-limit');
+    const limitNumber = parseInt(limitHeader || '10');
+    const limit = isNaN(limitNumber) ? 10 : limitNumber;
+
+    const cacheKey = `${schemaName}:${project_id}:${from}:${to}`;
+    const cacheExp = 60;
+
+    return await Redis.useCacheV2(cacheKey, cacheExp, async (noStore, updateExp) => {
+
+        const { model } = allowedModels[schemaName as TModelName];
+
+        const result = await model.aggregate([
+            {
+                $match: {
+                    project_id: project._id,
+                    created_at: {
+                        $gte: new Date(from),
+                        $lte: new Date(to)
+                    }
+                }
+            },
+            { $group: { _id: "$name", count: { $sum: 1, } } },
+            { $sort: { count: -1 } },
+            { $limit: limit }
+        ]);
+
+        return result;
+
+    });
+
+});

dashboard/server/api/metrics/[project_id]/timeline/events_stacked.post.ts

@@ -2,7 +2,8 @@ import { EventModel } from "@schema/metrics/EventSchema";
 import { getTimeline } from "./generic";
 import { Redis, TIMELINE_EXPIRE_TIME } from "~/server/services/CacheService";
 import { getUserProjectFromId } from "~/server/LIVE_DEMO_DATA";
-import { executeAdvancedTimelineAggregation } from "~/server/services/TimelineService";
+import { executeAdvancedTimelineAggregation, fillAndMergeTimelineAggregationV2 } from "~/server/services/TimelineService";
+import DateService from '@services/DateService';
 
 export default defineEventHandler(async event => {
     const project_id = getRequestProjectId(event);
@@ -29,6 +30,9 @@ export default defineEventHandler(async event => {
             customIdGroup: { name: '$name' },
         })
 
+        // const filledDates = DateService.createBetweenDates(from, to, slice);
+        // const merged = DateService.mergeFilledDates(filledDates.dates, timelineStackedEvents, '_id', slice, { count: 0, name: '' });
+
         return timelineStackedEvents;
     });
 

dashboard/server/api/pay/[project_id]/create.post.ts

@@ -27,7 +27,7 @@ export default defineEventHandler(async event => {
         return setResponseStatus(event, 400, 'Plan not exist');
     }
 
-    const checkout = await StripeService.cretePayment(
+    const checkout = await StripeService.createPayment(
         StripeService.testMode ? PLAN.PRICE_TEST : PLAN.PRICE,
         'https://dashboard.litlyx.com/payment_ok',
         project_id,

dashboard/server/api/pay/[project_id]/customer_info.ts

@@ -0,0 +1,21 @@
+import { getUserProjectFromId } from "~/server/LIVE_DEMO_DATA";
+import StripeService from '~/server/services/StripeService';
+
+
+export default defineEventHandler(async event => {
+
+    const project_id = getRequestProjectId(event);
+    if (!project_id) return;
+
+    const user = getRequestUser(event);
+    const project = await getUserProjectFromId(project_id, user, false);
+    if (!project) return;
+
+    if (!project.customer_id) return;
+
+    const customer = await StripeService.getCustomer(project.customer_id);
+    if (customer?.deleted) return;
+    
+    return customer?.address;
+
+});

dashboard/server/api/pay/[project_id]/update_customer.post.ts

@@ -0,0 +1,21 @@
+import { getUserProjectFromId } from "~/server/LIVE_DEMO_DATA";
+import StripeService from '~/server/services/StripeService';
+
+
+export default defineEventHandler(async event => {
+
+    const project_id = getRequestProjectId(event);
+    if (!project_id) return setResponseStatus(event, 400, 'Cannot get project_id');
+
+    const user = getRequestUser(event);
+    const project = await getUserProjectFromId(project_id, user, false);
+    if (!project) return setResponseStatus(event, 400, 'Cannot get user from project_id');
+
+    if (!project.customer_id) return setResponseStatus(event, 400, 'Project has no customer_id');
+
+    const body = await readBody(event);
+    const res = await StripeService.setCustomerInfo(project.customer_id, body);
+
+    return { ok: true, data: res }
+
+});

dashboard/server/services/CacheService.ts

@@ -13,6 +13,8 @@ export const EVENT_NAMES_EXPIRE_TIME = 60;
 
 export const EVENT_METADATA_FIELDS_EXPIRE_TIME = 30;
 
+type UseCacheV2Callback<T> = (noStore: () => void, updateExp: (value: number) => void) => Promise<T>
+
 
 export class Redis {
 
@@ -65,4 +67,17 @@ export class Redis {
         return result;
     }
 
+    static async useCacheV2<T extends any>(key: string, exp: number, callback: UseCacheV2Callback<T>) {
+        const cached = await this.get<T>(key);
+        if (cached) return cached;
+        let expireValue = exp;
+        let shouldStore = true;
+        const noStore = () => shouldStore = false;
+        const updateExp = (newExp: number) => expireValue = newExp;
+        const result = await callback(noStore, updateExp);
+        if (!shouldStore) return result;
+        await this.set(key, result, expireValue);
+        return result;
+    }
+
 }

dashboard/server/services/StripeService.ts

@@ -56,7 +56,7 @@ class StripeService {
         return checkout;
     }
 
-    async cretePayment(price: string, success_url: string, pid: string, customer?: string) {
+    async createPayment(price: string, success_url: string, pid: string, customer?: string) {
         if (this.disabledMode) return;
         if (!this.stripe) throw Error('Stripe not initialized');
 
@@ -126,6 +126,22 @@ class StripeService {
         return customer;
     }
 
+    async setCustomerInfo(customer_id: string, address: { line1: string, line2: string, city: string, country: string, postal_code: string, state: string }) {
+        if (this.disabledMode) return;
+        if (!this.stripe) throw Error('Stripe not initialized');
+        const customer = await this.stripe.customers.update(customer_id, {
+            address: {
+                line1: address.line1,
+                line2: address.line2,
+                city: address.city,
+                country: address.country,
+                postal_code: address.postal_code,
+                state: address.state
+            }
+        })
+        return customer.id;
+    }
+
     async deleteCustomer(customer_id: string) {
         if (this.disabledMode) return;
         if (!this.stripe) throw Error('Stripe not initialized');

dashboard/server/services/TimelineService.ts

@@ -46,7 +46,7 @@ export async function executeAdvancedTimelineAggregation<T = {}>(options: Advanc
         console.log(JSON.stringify(aggregation, null, 2));
     }
 
-    const timeline: { _id: string, count: number & T }[] = await options.model.aggregate(aggregation);
+    const timeline: ({ _id: string, count: number } & T)[] = await options.model.aggregate(aggregation);
 
     return timeline;
 

dashboard/server/utils/hasAccessToProject.ts

@@ -2,10 +2,10 @@ import { ProjectModel, TProject } from "@schema/ProjectSchema";
 import { TeamMemberModel } from "@schema/TeamMemberSchema";
 
 export async function hasAccessToProject(user_id: string, project_id: string, project?: TProject) {
-    const targetProject = project || await ProjectModel.findById(project_id, { owner: true });
+    const targetProject = project ?? await ProjectModel.findById(project_id, { owner: true });
     if (!targetProject) return [false, 'NONE'];
     if (targetProject.owner.toString() === user_id) return [true, 'OWNER'];
-    const members = await TeamMemberModel.find({ project_id });
+    const isGuest = await TeamMemberModel.exists({ project_id, user_id });
-    if (members.map(e => e.user_id.toString()).includes(user_id)) return [true, 'GUEST'];
+    if (isGuest) return [true, 'GUEST'];
     return [false, 'NONE'];
 }

producer/package.json

@@ -20,7 +20,7 @@
 	"scripts": {
 		"dev": "node scripts/start_dev.js",
 		"compile": "tsc",
-		"build": "ts-node scripts/build.ts",
+		"build": "node ../scripts/build.js",
 		"build_all": "npm run compile && npm run build",
 		"docker-build": "docker build -t litlyx-producer -f Dockerfile ../",
 		"docker-inspect": "docker run -it litlyx-producer sh"

producer/scripts/build.ts

@@ -1,17 +0,0 @@
-
-import fs from 'fs';
-import { globSync } from 'glob';
-const tsconfigContent = fs.readFileSync('tsconfig.json', 'utf8');
-const tsconfigObject = JSON.parse(tsconfigContent);
-const paths = tsconfigObject.compilerOptions.paths;
-const filesList = globSync('dist/**/*.js');
-filesList.forEach(file => {
-    let raw = fs.readFileSync(file, 'utf8');
-    for (const path in paths) {
-        const deep = (file.match(/\\|\//g) || []).length;
-        const pathText = path.replace('*', '');
-        const toReplaceText = new RegExp(`"${pathText}(.*?)"`, 'g');
-        raw = raw.replace(toReplaceText, `"${new Array(deep - 2).fill('../').join('')}${paths[path][0].replace('*', '')}${'$1'}"`);
-    }
-    fs.writeFileSync(file, raw);
-});

producer/src/deprecated.ts

@@ -1,6 +1,6 @@
 import { Router, json } from "express";
 import { createSessionHash, getIPFromRequest } from "./utils";
-import { requireEnv } from "../../shared/utilts/requireEnv";
+import { requireEnv } from "@utils/requireEnv";
 import { RedisStreamService } from "@services/RedisStreamService";
 
 const router = Router();

producer/src/index.ts

@@ -1,4 +1,4 @@
-import { requireEnv } from "../../shared/utilts/requireEnv";
+import { requireEnv } from "@utils/requireEnv";
 import { RedisStreamService } from "@services/RedisStreamService";
 
 import express from 'express';

producer/tsconfig.json

@@ -16,6 +16,9 @@
             ],
             "@functions/*": [
                 "../shared/functions/*"
+            ],
+            "@utils/*": [
+                "../shared/utils/*"
             ]
         }
     },

scripts/.gitignore

@@ -0,0 +1 @@
+node_modules

scripts/build.js

@@ -0,0 +1,28 @@
+
+
+const { globSync } = require('glob');
+const fs = require('fs');
+const path = require('path');
+
+const tsConfigPath = path.join(process.cwd(), 'tsconfig.json');
+const tsconfigContent = fs.readFileSync(tsConfigPath, 'utf8');
+
+const tsconfigObject = JSON.parse(tsconfigContent);
+const paths = tsconfigObject.compilerOptions.paths;
+
+const filesList = globSync('dist/**/*.js');
+
+filesList.forEach(file => {
+    let raw = fs.readFileSync(file, 'utf8');
+    for (const path in paths) {
+        const deep = (file.match(/\\|\//g) || []).length;
+        const pathText = path.replace('*', '');
+        const toReplaceText = new RegExp(`"${pathText}(.*?)"`, 'g');
+        try {
+            raw = raw.replace(toReplaceText, `"${new Array(deep - 2).fill('../').join('')}${paths[path][0].replace('*', '')}${'$1'}"`);
+        } catch (ex) {
+            console.log({ deep, pathText, toReplaceText, path })
+        }
+    }
+    fs.writeFileSync(file, raw);
+});

scripts/package.json

@@ -0,0 +1,6 @@
+{
+	"dependencies": {},
+	"devDependencies": {
+		"glob": "^11.0.0"
+	}
+}

scripts/pnpm-lock.yaml

@@ -0,0 +1,298 @@
+lockfileVersion: '9.0'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+importers:
+
+  .:
+    dependencies:
+      glob:
+        specifier: ^11.0.0
+        version: 11.0.0
+    devDependencies:
+      '@types/node':
+        specifier: ^22.5.5
+        version: 22.5.5
+
+packages:
+
+  '@isaacs/cliui@8.0.2':
+    resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
+    engines: {node: '>=12'}
+
+  '@pkgjs/parseargs@0.11.0':
+    resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
+    engines: {node: '>=14'}
+
+  '@types/node@22.5.5':
+    resolution: {integrity: sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==}
+
+  ansi-regex@5.0.1:
+    resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
+    engines: {node: '>=8'}
+
+  ansi-regex@6.1.0:
+    resolution: {integrity: sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==}
+    engines: {node: '>=12'}
+
+  ansi-styles@4.3.0:
+    resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
+    engines: {node: '>=8'}
+
+  ansi-styles@6.2.1:
+    resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==}
+    engines: {node: '>=12'}
+
+  balanced-match@1.0.2:
+    resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
+
+  brace-expansion@2.0.1:
+    resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==}
+
+  color-convert@2.0.1:
+    resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
+    engines: {node: '>=7.0.0'}
+
+  color-name@1.1.4:
+    resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
+
+  cross-spawn@7.0.3:
+    resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==}
+    engines: {node: '>= 8'}
+
+  eastasianwidth@0.2.0:
+    resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==}
+
+  emoji-regex@8.0.0:
+    resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
+
+  emoji-regex@9.2.2:
+    resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
+
+  foreground-child@3.3.0:
+    resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==}
+    engines: {node: '>=14'}
+
+  glob@11.0.0:
+    resolution: {integrity: sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g==}
+    engines: {node: 20 || >=22}
+    hasBin: true
+
+  is-fullwidth-code-point@3.0.0:
+    resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}
+    engines: {node: '>=8'}
+
+  isexe@2.0.0:
+    resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
+
+  jackspeak@4.0.1:
+    resolution: {integrity: sha512-cub8rahkh0Q/bw1+GxP7aeSe29hHHn2V4m29nnDlvCdlgU+3UGxkZp7Z53jLUdpX3jdTO0nJZUDl3xvbWc2Xog==}
+    engines: {node: 20 || >=22}
+
+  lru-cache@11.0.1:
+    resolution: {integrity: sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==}
+    engines: {node: 20 || >=22}
+
+  minimatch@10.0.1:
+    resolution: {integrity: sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==}
+    engines: {node: 20 || >=22}
+
+  minipass@7.1.2:
+    resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}
+    engines: {node: '>=16 || 14 >=14.17'}
+
+  package-json-from-dist@1.0.0:
+    resolution: {integrity: sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==}
+
+  path-key@3.1.1:
+    resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
+    engines: {node: '>=8'}
+
+  path-scurry@2.0.0:
+    resolution: {integrity: sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==}
+    engines: {node: 20 || >=22}
+
+  shebang-command@2.0.0:
+    resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==}
+    engines: {node: '>=8'}
+
+  shebang-regex@3.0.0:
+    resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
+    engines: {node: '>=8'}
+
+  signal-exit@4.1.0:
+    resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}
+    engines: {node: '>=14'}
+
+  string-width@4.2.3:
+    resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
+    engines: {node: '>=8'}
+
+  string-width@5.1.2:
+    resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==}
+    engines: {node: '>=12'}
+
+  strip-ansi@6.0.1:
+    resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==}
+    engines: {node: '>=8'}
+
+  strip-ansi@7.1.0:
+    resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==}
+    engines: {node: '>=12'}
+
+  undici-types@6.19.8:
+    resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==}
+
+  which@2.0.2:
+    resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
+    engines: {node: '>= 8'}
+    hasBin: true
+
+  wrap-ansi@7.0.0:
+    resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==}
+    engines: {node: '>=10'}
+
+  wrap-ansi@8.1.0:
+    resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==}
+    engines: {node: '>=12'}
+
+snapshots:
+
+  '@isaacs/cliui@8.0.2':
+    dependencies:
+      string-width: 5.1.2
+      string-width-cjs: string-width@4.2.3
+      strip-ansi: 7.1.0
+      strip-ansi-cjs: strip-ansi@6.0.1
+      wrap-ansi: 8.1.0
+      wrap-ansi-cjs: wrap-ansi@7.0.0
+
+  '@pkgjs/parseargs@0.11.0':
+    optional: true
+
+  '@types/node@22.5.5':
+    dependencies:
+      undici-types: 6.19.8
+
+  ansi-regex@5.0.1: {}
+
+  ansi-regex@6.1.0: {}
+
+  ansi-styles@4.3.0:
+    dependencies:
+      color-convert: 2.0.1
+
+  ansi-styles@6.2.1: {}
+
+  balanced-match@1.0.2: {}
+
+  brace-expansion@2.0.1:
+    dependencies:
+      balanced-match: 1.0.2
+
+  color-convert@2.0.1:
+    dependencies:
+      color-name: 1.1.4
+
+  color-name@1.1.4: {}
+
+  cross-spawn@7.0.3:
+    dependencies:
+      path-key: 3.1.1
+      shebang-command: 2.0.0
+      which: 2.0.2
+
+  eastasianwidth@0.2.0: {}
+
+  emoji-regex@8.0.0: {}
+
+  emoji-regex@9.2.2: {}
+
+  foreground-child@3.3.0:
+    dependencies:
+      cross-spawn: 7.0.3
+      signal-exit: 4.1.0
+
+  glob@11.0.0:
+    dependencies:
+      foreground-child: 3.3.0
+      jackspeak: 4.0.1
+      minimatch: 10.0.1
+      minipass: 7.1.2
+      package-json-from-dist: 1.0.0
+      path-scurry: 2.0.0
+
+  is-fullwidth-code-point@3.0.0: {}
+
+  isexe@2.0.0: {}
+
+  jackspeak@4.0.1:
+    dependencies:
+      '@isaacs/cliui': 8.0.2
+    optionalDependencies:
+      '@pkgjs/parseargs': 0.11.0
+
+  lru-cache@11.0.1: {}
+
+  minimatch@10.0.1:
+    dependencies:
+      brace-expansion: 2.0.1
+
+  minipass@7.1.2: {}
+
+  package-json-from-dist@1.0.0: {}
+
+  path-key@3.1.1: {}
+
+  path-scurry@2.0.0:
+    dependencies:
+      lru-cache: 11.0.1
+      minipass: 7.1.2
+
+  shebang-command@2.0.0:
+    dependencies:
+      shebang-regex: 3.0.0
+
+  shebang-regex@3.0.0: {}
+
+  signal-exit@4.1.0: {}
+
+  string-width@4.2.3:
+    dependencies:
+      emoji-regex: 8.0.0
+      is-fullwidth-code-point: 3.0.0
+      strip-ansi: 6.0.1
+
+  string-width@5.1.2:
+    dependencies:
+      eastasianwidth: 0.2.0
+      emoji-regex: 9.2.2
+      strip-ansi: 7.1.0
+
+  strip-ansi@6.0.1:
+    dependencies:
+      ansi-regex: 5.0.1
+
+  strip-ansi@7.1.0:
+    dependencies:
+      ansi-regex: 6.1.0
+
+  undici-types@6.19.8: {}
+
+  which@2.0.2:
+    dependencies:
+      isexe: 2.0.0
+
+  wrap-ansi@7.0.0:
+    dependencies:
+      ansi-styles: 4.3.0
+      string-width: 4.2.3
+      strip-ansi: 6.0.1
+
+  wrap-ansi@8.1.0:
+    dependencies:
+      ansi-styles: 6.2.1
+      string-width: 5.1.2
+      strip-ansi: 7.1.0

shared/services/RedisStreamService.ts

@@ -1,16 +1,21 @@
 import { createClient } from 'redis';
-import { requireEnv } from '../utilts/requireEnv';
+import { requireEnv } from '../utils/requireEnv';
 
 export type ReadingLoopOptions = {
-    delay?: {
+    stream_name: string,
-        base?: number,
+    group_name: ConsumerGroup,
-        empty?: number
+    consumer_name: string
-    },
-    readBlock?: number,
-    streamName: string,
-    consumer?: string
 }
 
+
+type xReadGroupMessage = { id: string, message: { [x: string]: string } }
+type xReadGgroupResult = { name: string, messages: xReadGroupMessage[] }[] | null
+
+const consumerGroups = ['DATABASE', 'LIMITS'] as const;
+
+type ConsumerGroup = typeof consumerGroups[number];
+
+
 export class RedisStreamService {
 
     private static processed = 0;
@@ -23,65 +28,69 @@ export class RedisStreamService {
     });
 
     static async connect() {
-        console.log('RedisStreamService DEV_MODE=', process.env.DEV_MODE === 'true');
         await this.client.connect();
+    }
 
+    static async readFromStream(stream_name: string, group_name: string, consumer_name: string, process_function: (content: Record<string, string>) => Promise<any>) {
+
+        const result: xReadGgroupResult = await this.client.xReadGroup(group_name, consumer_name, [{ key: stream_name, id: '>' }], { COUNT: 5, BLOCK: 10000 });
+
+        if (!result) {
+            setTimeout(() => this.readFromStream(stream_name, group_name, consumer_name, process_function), 10);
+            return;
+        }
+
+        for (const entry of result) {
+            console.log(`[${group_name}-${consumer_name}]`, 'Processing', entry.messages.length, 'messages');
+            for (const messageData of entry.messages) {
+                await process_function(messageData.message);
+                await this.client.xAck(stream_name, group_name, messageData.id);
+                await this.client.set(`ACK:${group_name}`, messageData.id);
+                RedisStreamService.processed++;
+            }
+        }
+
+        await this.trimStream(stream_name);
+
+        setTimeout(() => this.readFromStream(stream_name, group_name, consumer_name, process_function), 10);
+        return;
 
     }
 
-    private static async readingLoop(options: ReadingLoopOptions, processFunction: (content: Record<string, string>) => Promise<any>) {
+    private static async trimStream(stream_name: string) {
-        const result = await this.readFromStream(options.streamName, options.readBlock || 2500, options.consumer || 'base_consumer');
+
-        if (!result) {
+        let lastMessageAck = '0';
-            await new Promise(r => setTimeout(r, options.delay?.empty || 5000));
+
-            setTimeout(() => this.readingLoop(options, processFunction), 1);
+        for (const consumerGroup of consumerGroups) {
-            return;
+            const lastAck = await this.client.get(`ACK:${consumerGroup}`);
+            if (!lastAck) continue;
+            if (lastAck > lastMessageAck) lastMessageAck = lastAck;
         }
-        try {
+
-            await processFunction(result);
+        await this.client.xTrim(stream_name, 'MINID', lastMessageAck as any);
-        } catch (ex) {
+
-            console.error('Error on processing function');
-        }
-        RedisStreamService.processed++;
-        await new Promise(r => setTimeout(r, options.delay?.base || 100));
-        setTimeout(() => this.readingLoop(options, processFunction), 1);
-        return;
     }
 
     static async startReadingLoop(options: ReadingLoopOptions, processFunction: (content: Record<string, string>) => Promise<any>) {
 
+        if (!consumerGroups.includes(options.group_name)) return console.error('GROUP NAME NOT ALLOWED');
+
         setInterval(() => {
             if (RedisStreamService.processed > 0) {
-                console.log('Processed:', (RedisStreamService.processed / 30).toFixed(2), '/s');
+                console.log('Processed:', (RedisStreamService.processed / 10).toFixed(2), '/s');
                 RedisStreamService.processed = 0;
             }
-        }, 30_000)
+        }, 10_000);
+
+        console.log('Start reading loop')
 
         try {
-            console.log('Start reading loop');
+            await this.client.xGroupCreate(options.stream_name, options.group_name, '0', { MKSTREAM: true });
-            await this.client.xGroupCreate(options.streamName, 'broker', '0', { MKSTREAM: true, });
-            console.log('Reading loop started');
         } catch (ex) {
-            console.error(ex);
+            console.log('Group', options.group_name, 'already exist');
         }
 
-        this.readingLoop(options, processFunction)
+        this.readFromStream(options.stream_name, options.group_name, options.consumer_name, processFunction);
-    }
-
-    private static async readFromStream(streamName: string, readBlock: number, consumer: string) {
-        const result = await this.client.xReadGroup(
-            'broker', consumer,
-            [{ key: streamName, id: '>' }],
-            { COUNT: 1, BLOCK: readBlock }
-        );
-        if (!result) return;
-        if (result.length == 0) return;
-        if (!result[0].messages) return;
-        if (result[0].messages.length == 0) return;
-        const message = result[0].messages[0];
-        await this.client.xAck(streamName, 'broker', message.id);
-        await this.client.xDel(streamName, message.id);
-        const content = message.message;
-        return content;
     }
 
     static async addToStream(streamName: string, data: Record<string, string>) {