Services rewrite

consumer-database/.gitignore

@@ -0,0 +1,8 @@
+node_modules
+static
+ecosystem.config.cjs
+dist
+scripts/start_dev.js
+package-lock.json
+build_all.bat
+tests

consumer-database/ecosystem.config.example.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+    apps: [
+        {
+            name: 'QueueBroker',
+            port: '3999',
+            exec_mode: 'fork',
+            script: './dist/consumer-database/src/index.js',
+            env: {
+                MONGO_CONNECTION_STRING: "",
+                REDIS_URL: "",
+                REDIS_USERNAME: "",
+                REDIS_PASSWORD: "",
+                STREAM_NAME: "",
+                GROUP_NAME: ""
+            }
+        }
+    ]
+}

consumer-database/package.json

@@ -0,0 +1,27 @@
+{
+  "dependencies": {
+    "mongoose": "^8.3.2",
+    "redis": "^4.6.14",
+    "ua-parser-js": "^1.0.37"
+  },
+  "devDependencies": {
+    "@types/node": "^20.12.13",
+    "@types/ua-parser-js": "^0.7.39",
+    "glob": "^10.4.1",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.5"
+  },
+  "name": "litlyx-queue-broker",
+  "version": "1.0.0",
+  "main": "dist/index.js",
+  "scripts": {
+    "dev": "node scripts/start_dev.js",
+    "compile": "tsc",
+    "build": "node ../scripts/build.js",
+    "build_all": "npm run compile && npm run build"
+  },
+  "keywords": [],
+  "author": "Emily",
+  "license": "MIT",
+  "description": "Database Consumer - Saves events to database."
+}

consumer-database/pnpm-lock.yaml

@@ -0,0 +1,716 @@
+lockfileVersion: '9.0'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+importers:
+
+  .:
+    dependencies:
+      mongoose:
+        specifier: ^8.3.2
+        version: 8.6.3
+      redis:
+        specifier: ^4.6.14
+        version: 4.7.0
+      ua-parser-js:
+        specifier: ^1.0.37
+        version: 1.0.39
+    devDependencies:
+      '@types/node':
+        specifier: ^20.12.13
+        version: 20.16.5
+      '@types/ua-parser-js':
+        specifier: ^0.7.39
+        version: 0.7.39
+      glob:
+        specifier: ^10.4.1
+        version: 10.4.5
+      ts-node:
+        specifier: ^10.9.2
+        version: 10.9.2(@types/node@20.16.5)(typescript@5.6.2)
+      typescript:
+        specifier: ^5.4.5
+        version: 5.6.2
+
+packages:
+
+  '@cspotcode/source-map-support@0.8.1':
+    resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}
+    engines: {node: '>=12'}
+
+  '@isaacs/cliui@8.0.2':
+    resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
+    engines: {node: '>=12'}
+
+  '@jridgewell/resolve-uri@3.1.2':
+    resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==}
+    engines: {node: '>=6.0.0'}
+
+  '@jridgewell/sourcemap-codec@1.5.0':
+    resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==}
+
+  '@jridgewell/trace-mapping@0.3.9':
+    resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==}
+
+  '@mongodb-js/saslprep@1.1.9':
+    resolution: {integrity: sha512-tVkljjeEaAhCqTzajSdgbQ6gE6f3oneVwa3iXR6csiEwXXOFsiC6Uh9iAjAhXPtqa/XMDHWjjeNH/77m/Yq2dw==}
+
+  '@pkgjs/parseargs@0.11.0':
+    resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
+    engines: {node: '>=14'}
+
+  '@redis/bloom@1.2.0':
+    resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/client@1.6.0':
+    resolution: {integrity: sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==}
+    engines: {node: '>=14'}
+
+  '@redis/graph@1.1.1':
+    resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/json@1.0.7':
+    resolution: {integrity: sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/search@1.2.0':
+    resolution: {integrity: sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/time-series@1.1.0':
+    resolution: {integrity: sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@tsconfig/node10@1.0.11':
+    resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==}
+
+  '@tsconfig/node12@1.0.11':
+    resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==}
+
+  '@tsconfig/node14@1.0.3':
+    resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==}
+
+  '@tsconfig/node16@1.0.4':
+    resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==}
+
+  '@types/node@20.16.5':
+    resolution: {integrity: sha512-VwYCweNo3ERajwy0IUlqqcyZ8/A7Zwa9ZP3MnENWcB11AejO+tLy3pu850goUW2FC/IJMdZUfKpX/yxL1gymCA==}
+
+  '@types/ua-parser-js@0.7.39':
+    resolution: {integrity: sha512-P/oDfpofrdtF5xw433SPALpdSchtJmY7nsJItf8h3KXqOslkbySh8zq4dSWXH2oTjRvJ5PczVEoCZPow6GicLg==}
+
+  '@types/webidl-conversions@7.0.3':
+    resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==}
+
+  '@types/whatwg-url@11.0.5':
+    resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==}
+
+  acorn-walk@8.3.4:
+    resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==}
+    engines: {node: '>=0.4.0'}
+
+  acorn@8.12.1:
+    resolution: {integrity: sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==}
+    engines: {node: '>=0.4.0'}
+    hasBin: true
+
+  ansi-regex@5.0.1:
+    resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
+    engines: {node: '>=8'}
+
+  ansi-regex@6.1.0:
+    resolution: {integrity: sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==}
+    engines: {node: '>=12'}
+
+  ansi-styles@4.3.0:
+    resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
+    engines: {node: '>=8'}
+
+  ansi-styles@6.2.1:
+    resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==}
+    engines: {node: '>=12'}
+
+  arg@4.1.3:
+    resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==}
+
+  balanced-match@1.0.2:
+    resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
+
+  brace-expansion@2.0.1:
+    resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==}
+
+  bson@6.8.0:
+    resolution: {integrity: sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ==}
+    engines: {node: '>=16.20.1'}
+
+  cluster-key-slot@1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
+    engines: {node: '>=0.10.0'}
+
+  color-convert@2.0.1:
+    resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
+    engines: {node: '>=7.0.0'}
+
+  color-name@1.1.4:
+    resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
+
+  create-require@1.1.1:
+    resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
+
+  cross-spawn@7.0.3:
+    resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==}
+    engines: {node: '>= 8'}
+
+  debug@4.3.7:
+    resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==}
+    engines: {node: '>=6.0'}
+    peerDependencies:
+      supports-color: '*'
+    peerDependenciesMeta:
+      supports-color:
+        optional: true
+
+  diff@4.0.2:
+    resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==}
+    engines: {node: '>=0.3.1'}
+
+  eastasianwidth@0.2.0:
+    resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==}
+
+  emoji-regex@8.0.0:
+    resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
+
+  emoji-regex@9.2.2:
+    resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
+
+  foreground-child@3.3.0:
+    resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==}
+    engines: {node: '>=14'}
+
+  generic-pool@3.9.0:
+    resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==}
+    engines: {node: '>= 4'}
+
+  glob@10.4.5:
+    resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==}
+    hasBin: true
+
+  is-fullwidth-code-point@3.0.0:
+    resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}
+    engines: {node: '>=8'}
+
+  isexe@2.0.0:
+    resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
+
+  jackspeak@3.4.3:
+    resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==}
+
+  kareem@2.6.3:
+    resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==}
+    engines: {node: '>=12.0.0'}
+
+  lru-cache@10.4.3:
+    resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==}
+
+  make-error@1.3.6:
+    resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==}
+
+  memory-pager@1.5.0:
+    resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==}
+
+  minimatch@9.0.5:
+    resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==}
+    engines: {node: '>=16 || 14 >=14.17'}
+
+  minipass@7.1.2:
+    resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}
+    engines: {node: '>=16 || 14 >=14.17'}
+
+  mongodb-connection-string-url@3.0.1:
+    resolution: {integrity: sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==}
+
+  mongodb@6.8.0:
+    resolution: {integrity: sha512-HGQ9NWDle5WvwMnrvUxsFYPd3JEbqD3RgABHBQRuoCEND0qzhsd0iH5ypHsf1eJ+sXmvmyKpP+FLOKY8Il7jMw==}
+    engines: {node: '>=16.20.1'}
+    peerDependencies:
+      '@aws-sdk/credential-providers': ^3.188.0
+      '@mongodb-js/zstd': ^1.1.0
+      gcp-metadata: ^5.2.0
+      kerberos: ^2.0.1
+      mongodb-client-encryption: '>=6.0.0 <7'
+      snappy: ^7.2.2
+      socks: ^2.7.1
+    peerDependenciesMeta:
+      '@aws-sdk/credential-providers':
+        optional: true
+      '@mongodb-js/zstd':
+        optional: true
+      gcp-metadata:
+        optional: true
+      kerberos:
+        optional: true
+      mongodb-client-encryption:
+        optional: true
+      snappy:
+        optional: true
+      socks:
+        optional: true
+
+  mongoose@8.6.3:
+    resolution: {integrity: sha512-++yRmm7hjMbqVA/8WeiygTnEfrFbiy+OBjQi49GFJIvCQuSYE56myyQWo4j5hbpcHjhHQU8NukMNGTwAWFWjIw==}
+    engines: {node: '>=16.20.1'}
+
+  mpath@0.9.0:
+    resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==}
+    engines: {node: '>=4.0.0'}
+
+  mquery@5.0.0:
+    resolution: {integrity: sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==}
+    engines: {node: '>=14.0.0'}
+
+  ms@2.1.3:
+    resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
+
+  package-json-from-dist@1.0.0:
+    resolution: {integrity: sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==}
+
+  path-key@3.1.1:
+    resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
+    engines: {node: '>=8'}
+
+  path-scurry@1.11.1:
+    resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==}
+    engines: {node: '>=16 || 14 >=14.18'}
+
+  punycode@2.3.1:
+    resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
+    engines: {node: '>=6'}
+
+  redis@4.7.0:
+    resolution: {integrity: sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==}
+
+  shebang-command@2.0.0:
+    resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==}
+    engines: {node: '>=8'}
+
+  shebang-regex@3.0.0:
+    resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
+    engines: {node: '>=8'}
+
+  sift@17.1.3:
+    resolution: {integrity: sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==}
+
+  signal-exit@4.1.0:
+    resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}
+    engines: {node: '>=14'}
+
+  sparse-bitfield@3.0.3:
+    resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==}
+
+  string-width@4.2.3:
+    resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
+    engines: {node: '>=8'}
+
+  string-width@5.1.2:
+    resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==}
+    engines: {node: '>=12'}
+
+  strip-ansi@6.0.1:
+    resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==}
+    engines: {node: '>=8'}
+
+  strip-ansi@7.1.0:
+    resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==}
+    engines: {node: '>=12'}
+
+  tr46@4.1.1:
+    resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==}
+    engines: {node: '>=14'}
+
+  ts-node@10.9.2:
+    resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==}
+    hasBin: true
+    peerDependencies:
+      '@swc/core': '>=1.2.50'
+      '@swc/wasm': '>=1.2.50'
+      '@types/node': '*'
+      typescript: '>=2.7'
+    peerDependenciesMeta:
+      '@swc/core':
+        optional: true
+      '@swc/wasm':
+        optional: true
+
+  typescript@5.6.2:
+    resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==}
+    engines: {node: '>=14.17'}
+    hasBin: true
+
+  ua-parser-js@1.0.39:
+    resolution: {integrity: sha512-k24RCVWlEcjkdOxYmVJgeD/0a1TiSpqLg+ZalVGV9lsnr4yqu0w7tX/x2xX6G4zpkgQnRf89lxuZ1wsbjXM8lw==}
+    hasBin: true
+
+  undici-types@6.19.8:
+    resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==}
+
+  v8-compile-cache-lib@3.0.1:
+    resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==}
+
+  webidl-conversions@7.0.0:
+    resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==}
+    engines: {node: '>=12'}
+
+  whatwg-url@13.0.0:
+    resolution: {integrity: sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==}
+    engines: {node: '>=16'}
+
+  which@2.0.2:
+    resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
+    engines: {node: '>= 8'}
+    hasBin: true
+
+  wrap-ansi@7.0.0:
+    resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==}
+    engines: {node: '>=10'}
+
+  wrap-ansi@8.1.0:
+    resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==}
+    engines: {node: '>=12'}
+
+  yallist@4.0.0:
+    resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
+
+  yn@3.1.1:
+    resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==}
+    engines: {node: '>=6'}
+
+snapshots:
+
+  '@cspotcode/source-map-support@0.8.1':
+    dependencies:
+      '@jridgewell/trace-mapping': 0.3.9
+
+  '@isaacs/cliui@8.0.2':
+    dependencies:
+      string-width: 5.1.2
+      string-width-cjs: string-width@4.2.3
+      strip-ansi: 7.1.0
+      strip-ansi-cjs: strip-ansi@6.0.1
+      wrap-ansi: 8.1.0
+      wrap-ansi-cjs: wrap-ansi@7.0.0
+
+  '@jridgewell/resolve-uri@3.1.2': {}
+
+  '@jridgewell/sourcemap-codec@1.5.0': {}
+
+  '@jridgewell/trace-mapping@0.3.9':
+    dependencies:
+      '@jridgewell/resolve-uri': 3.1.2
+      '@jridgewell/sourcemap-codec': 1.5.0
+
+  '@mongodb-js/saslprep@1.1.9':
+    dependencies:
+      sparse-bitfield: 3.0.3
+
+  '@pkgjs/parseargs@0.11.0':
+    optional: true
+
+  '@redis/bloom@1.2.0(@redis/client@1.6.0)':
+    dependencies:
+      '@redis/client': 1.6.0
+
+  '@redis/client@1.6.0':
+    dependencies:
+      cluster-key-slot: 1.1.2
+      generic-pool: 3.9.0
+      yallist: 4.0.0
+
+  '@redis/graph@1.1.1(@redis/client@1.6.0)':
+    dependencies:
+      '@redis/client': 1.6.0
+
+  '@redis/json@1.0.7(@redis/client@1.6.0)':
+    dependencies:
+      '@redis/client': 1.6.0
+
+  '@redis/search@1.2.0(@redis/client@1.6.0)':
+    dependencies:
+      '@redis/client': 1.6.0
+
+  '@redis/time-series@1.1.0(@redis/client@1.6.0)':
+    dependencies:
+      '@redis/client': 1.6.0
+
+  '@tsconfig/node10@1.0.11': {}
+
+  '@tsconfig/node12@1.0.11': {}
+
+  '@tsconfig/node14@1.0.3': {}
+
+  '@tsconfig/node16@1.0.4': {}
+
+  '@types/node@20.16.5':
+    dependencies:
+      undici-types: 6.19.8
+
+  '@types/ua-parser-js@0.7.39': {}
+
+  '@types/webidl-conversions@7.0.3': {}
+
+  '@types/whatwg-url@11.0.5':
+    dependencies:
+      '@types/webidl-conversions': 7.0.3
+
+  acorn-walk@8.3.4:
+    dependencies:
+      acorn: 8.12.1
+
+  acorn@8.12.1: {}
+
+  ansi-regex@5.0.1: {}
+
+  ansi-regex@6.1.0: {}
+
+  ansi-styles@4.3.0:
+    dependencies:
+      color-convert: 2.0.1
+
+  ansi-styles@6.2.1: {}
+
+  arg@4.1.3: {}
+
+  balanced-match@1.0.2: {}
+
+  brace-expansion@2.0.1:
+    dependencies:
+      balanced-match: 1.0.2
+
+  bson@6.8.0: {}
+
+  cluster-key-slot@1.1.2: {}
+
+  color-convert@2.0.1:
+    dependencies:
+      color-name: 1.1.4
+
+  color-name@1.1.4: {}
+
+  create-require@1.1.1: {}
+
+  cross-spawn@7.0.3:
+    dependencies:
+      path-key: 3.1.1
+      shebang-command: 2.0.0
+      which: 2.0.2
+
+  debug@4.3.7:
+    dependencies:
+      ms: 2.1.3
+
+  diff@4.0.2: {}
+
+  eastasianwidth@0.2.0: {}
+
+  emoji-regex@8.0.0: {}
+
+  emoji-regex@9.2.2: {}
+
+  foreground-child@3.3.0:
+    dependencies:
+      cross-spawn: 7.0.3
+      signal-exit: 4.1.0
+
+  generic-pool@3.9.0: {}
+
+  glob@10.4.5:
+    dependencies:
+      foreground-child: 3.3.0
+      jackspeak: 3.4.3
+      minimatch: 9.0.5
+      minipass: 7.1.2
+      package-json-from-dist: 1.0.0
+      path-scurry: 1.11.1
+
+  is-fullwidth-code-point@3.0.0: {}
+
+  isexe@2.0.0: {}
+
+  jackspeak@3.4.3:
+    dependencies:
+      '@isaacs/cliui': 8.0.2
+    optionalDependencies:
+      '@pkgjs/parseargs': 0.11.0
+
+  kareem@2.6.3: {}
+
+  lru-cache@10.4.3: {}
+
+  make-error@1.3.6: {}
+
+  memory-pager@1.5.0: {}
+
+  minimatch@9.0.5:
+    dependencies:
+      brace-expansion: 2.0.1
+
+  minipass@7.1.2: {}
+
+  mongodb-connection-string-url@3.0.1:
+    dependencies:
+      '@types/whatwg-url': 11.0.5
+      whatwg-url: 13.0.0
+
+  mongodb@6.8.0:
+    dependencies:
+      '@mongodb-js/saslprep': 1.1.9
+      bson: 6.8.0
+      mongodb-connection-string-url: 3.0.1
+
+  mongoose@8.6.3:
+    dependencies:
+      bson: 6.8.0
+      kareem: 2.6.3
+      mongodb: 6.8.0
+      mpath: 0.9.0
+      mquery: 5.0.0
+      ms: 2.1.3
+      sift: 17.1.3
+    transitivePeerDependencies:
+      - '@aws-sdk/credential-providers'
+      - '@mongodb-js/zstd'
+      - gcp-metadata
+      - kerberos
+      - mongodb-client-encryption
+      - snappy
+      - socks
+      - supports-color
+
+  mpath@0.9.0: {}
+
+  mquery@5.0.0:
+    dependencies:
+      debug: 4.3.7
+    transitivePeerDependencies:
+      - supports-color
+
+  ms@2.1.3: {}
+
+  package-json-from-dist@1.0.0: {}
+
+  path-key@3.1.1: {}
+
+  path-scurry@1.11.1:
+    dependencies:
+      lru-cache: 10.4.3
+      minipass: 7.1.2
+
+  punycode@2.3.1: {}
+
+  redis@4.7.0:
+    dependencies:
+      '@redis/bloom': 1.2.0(@redis/client@1.6.0)
+      '@redis/client': 1.6.0
+      '@redis/graph': 1.1.1(@redis/client@1.6.0)
+      '@redis/json': 1.0.7(@redis/client@1.6.0)
+      '@redis/search': 1.2.0(@redis/client@1.6.0)
+      '@redis/time-series': 1.1.0(@redis/client@1.6.0)
+
+  shebang-command@2.0.0:
+    dependencies:
+      shebang-regex: 3.0.0
+
+  shebang-regex@3.0.0: {}
+
+  sift@17.1.3: {}
+
+  signal-exit@4.1.0: {}
+
+  sparse-bitfield@3.0.3:
+    dependencies:
+      memory-pager: 1.5.0
+
+  string-width@4.2.3:
+    dependencies:
+      emoji-regex: 8.0.0
+      is-fullwidth-code-point: 3.0.0
+      strip-ansi: 6.0.1
+
+  string-width@5.1.2:
+    dependencies:
+      eastasianwidth: 0.2.0
+      emoji-regex: 9.2.2
+      strip-ansi: 7.1.0
+
+  strip-ansi@6.0.1:
+    dependencies:
+      ansi-regex: 5.0.1
+
+  strip-ansi@7.1.0:
+    dependencies:
+      ansi-regex: 6.1.0
+
+  tr46@4.1.1:
+    dependencies:
+      punycode: 2.3.1
+
+  ts-node@10.9.2(@types/node@20.16.5)(typescript@5.6.2):
+    dependencies:
+      '@cspotcode/source-map-support': 0.8.1
+      '@tsconfig/node10': 1.0.11
+      '@tsconfig/node12': 1.0.11
+      '@tsconfig/node14': 1.0.3
+      '@tsconfig/node16': 1.0.4
+      '@types/node': 20.16.5
+      acorn: 8.12.1
+      acorn-walk: 8.3.4
+      arg: 4.1.3
+      create-require: 1.1.1
+      diff: 4.0.2
+      make-error: 1.3.6
+      typescript: 5.6.2
+      v8-compile-cache-lib: 3.0.1
+      yn: 3.1.1
+
+  typescript@5.6.2: {}
+
+  ua-parser-js@1.0.39: {}
+
+  undici-types@6.19.8: {}
+
+  v8-compile-cache-lib@3.0.1: {}
+
+  webidl-conversions@7.0.0: {}
+
+  whatwg-url@13.0.0:
+    dependencies:
+      tr46: 4.1.1
+      webidl-conversions: 7.0.0
+
+  which@2.0.2:
+    dependencies:
+      isexe: 2.0.0
+
+  wrap-ansi@7.0.0:
+    dependencies:
+      ansi-styles: 4.3.0
+      string-width: 4.2.3
+      strip-ansi: 6.0.1
+
+  wrap-ansi@8.1.0:
+    dependencies:
+      ansi-styles: 6.2.1
+      string-width: 5.1.2
+      strip-ansi: 7.1.0
+
+  yallist@4.0.0: {}
+
+  yn@3.1.1: {}

consumer-database/scripts/GeoLite2-Country-Locations-en.csv

@@ -0,0 +1,253 @@
+geoname_id,locale_code,continent_code,continent_name,country_iso_code,country_name,is_in_european_union
+49518,en,AF,Africa,RW,Rwanda,0
+51537,en,AF,Africa,SO,Somalia,0
+69543,en,AS,Asia,YE,Yemen,0
+99237,en,AS,Asia,IQ,Iraq,0
+102358,en,AS,Asia,SA,"Saudi Arabia",0
+130758,en,AS,Asia,IR,Iran,0
+146669,en,EU,Europe,CY,Cyprus,1
+149590,en,AF,Africa,TZ,Tanzania,0
+163843,en,AS,Asia,SY,Syria,0
+174982,en,AS,Asia,AM,Armenia,0
+192950,en,AF,Africa,KE,Kenya,0
+203312,en,AF,Africa,CD,"DR Congo",0
+223816,en,AF,Africa,DJ,Djibouti,0
+226074,en,AF,Africa,UG,Uganda,0
+239880,en,AF,Africa,CF,"Central African Republic",0
+241170,en,AF,Africa,SC,Seychelles,0
+248816,en,AS,Asia,JO,Jordan,0
+272103,en,AS,Asia,LB,Lebanon,0
+285570,en,AS,Asia,KW,Kuwait,0
+286963,en,AS,Asia,OM,Oman,0
+289688,en,AS,Asia,QA,Qatar,0
+290291,en,AS,Asia,BH,Bahrain,0
+290557,en,AS,Asia,AE,"United Arab Emirates",0
+294640,en,AS,Asia,IL,Israel,0
+298795,en,AS,Asia,TR,Türkiye,0
+337996,en,AF,Africa,ET,Ethiopia,0
+338010,en,AF,Africa,ER,Eritrea,0
+357994,en,AF,Africa,EG,Egypt,0
+366755,en,AF,Africa,SD,Sudan,0
+390903,en,EU,Europe,GR,Greece,1
+433561,en,AF,Africa,BI,Burundi,0
+453733,en,EU,Europe,EE,Estonia,1
+458258,en,EU,Europe,LV,Latvia,1
+587116,en,AS,Asia,AZ,Azerbaijan,0
+597427,en,EU,Europe,LT,Lithuania,1
+607072,en,EU,Europe,SJ,"Svalbard and Jan Mayen",0
+614540,en,AS,Asia,GE,Georgia,0
+617790,en,EU,Europe,MD,Moldova,0
+630336,en,EU,Europe,BY,Belarus,0
+660013,en,EU,Europe,FI,Finland,1
+661882,en,EU,Europe,AX,"Åland Islands",1
+690791,en,EU,Europe,UA,Ukraine,0
+718075,en,EU,Europe,MK,"North Macedonia",0
+719819,en,EU,Europe,HU,Hungary,1
+732800,en,EU,Europe,BG,Bulgaria,1
+783754,en,EU,Europe,AL,Albania,0
+798544,en,EU,Europe,PL,Poland,1
+798549,en,EU,Europe,RO,Romania,1
+831053,en,EU,Europe,XK,Kosovo,0
+878675,en,AF,Africa,ZW,Zimbabwe,0
+895949,en,AF,Africa,ZM,Zambia,0
+921929,en,AF,Africa,KM,Comoros,0
+927384,en,AF,Africa,MW,Malawi,0
+932692,en,AF,Africa,LS,Lesotho,0
+933860,en,AF,Africa,BW,Botswana,0
+934292,en,AF,Africa,MU,Mauritius,0
+934841,en,AF,Africa,SZ,Eswatini,0
+935317,en,AF,Africa,RE,Réunion,1
+953987,en,AF,Africa,ZA,"South Africa",0
+1024031,en,AF,Africa,YT,Mayotte,1
+1036973,en,AF,Africa,MZ,Mozambique,0
+1062947,en,AF,Africa,MG,Madagascar,0
+1149361,en,AS,Asia,AF,Afghanistan,0
+1168579,en,AS,Asia,PK,Pakistan,0
+1210997,en,AS,Asia,BD,Bangladesh,0
+1218197,en,AS,Asia,TM,Turkmenistan,0
+1220409,en,AS,Asia,TJ,Tajikistan,0
+1227603,en,AS,Asia,LK,"Sri Lanka",0
+1252634,en,AS,Asia,BT,Bhutan,0
+1269750,en,AS,Asia,IN,India,0
+1282028,en,AS,Asia,MV,Maldives,0
+1282588,en,AS,Asia,IO,"British Indian Ocean Territory",0
+1282988,en,AS,Asia,NP,Nepal,0
+1327865,en,AS,Asia,MM,Myanmar,0
+1512440,en,AS,Asia,UZ,Uzbekistan,0
+1522867,en,AS,Asia,KZ,Kazakhstan,0
+1527747,en,AS,Asia,KG,Kyrgyzstan,0
+1546748,en,AN,Antarctica,TF,"French Southern Territories",0
+1547314,en,AN,Antarctica,HM,"Heard and McDonald Islands",0
+1547376,en,AS,Asia,CC,"Cocos (Keeling) Islands",0
+1559582,en,OC,Oceania,PW,Palau,0
+1562822,en,AS,Asia,VN,Vietnam,0
+1605651,en,AS,Asia,TH,Thailand,0
+1643084,en,AS,Asia,ID,Indonesia,0
+1655842,en,AS,Asia,LA,Laos,0
+1668284,en,AS,Asia,TW,Taiwan,0
+1694008,en,AS,Asia,PH,Philippines,0
+1733045,en,AS,Asia,MY,Malaysia,0
+1814991,en,AS,Asia,CN,China,0
+1819730,en,AS,Asia,HK,"Hong Kong",0
+1820814,en,AS,Asia,BN,Brunei,0
+1821275,en,AS,Asia,MO,Macao,0
+1831722,en,AS,Asia,KH,Cambodia,0
+1835841,en,AS,Asia,KR,"South Korea",0
+1861060,en,AS,Asia,JP,Japan,0
+1873107,en,AS,Asia,KP,"North Korea",0
+1880251,en,AS,Asia,SG,Singapore,0
+1899402,en,OC,Oceania,CK,"Cook Islands",0
+1966436,en,OC,Oceania,TL,Timor-Leste,0
+2017370,en,EU,Europe,RU,Russia,0
+2029969,en,AS,Asia,MN,Mongolia,0
+2077456,en,OC,Oceania,AU,Australia,0
+2078138,en,OC,Oceania,CX,"Christmas Island",0
+2080185,en,OC,Oceania,MH,"Marshall Islands",0
+2081918,en,OC,Oceania,FM,"Federated States of Micronesia",0
+2088628,en,OC,Oceania,PG,"Papua New Guinea",0
+2103350,en,OC,Oceania,SB,"Solomon Islands",0
+2110297,en,OC,Oceania,TV,Tuvalu,0
+2110425,en,OC,Oceania,NR,Nauru,0
+2134431,en,OC,Oceania,VU,Vanuatu,0
+2139685,en,OC,Oceania,NC,"New Caledonia",0
+2155115,en,OC,Oceania,NF,"Norfolk Island",0
+2186224,en,OC,Oceania,NZ,"New Zealand",0
+2205218,en,OC,Oceania,FJ,Fiji,0
+2215636,en,AF,Africa,LY,Libya,0
+2233387,en,AF,Africa,CM,Cameroon,0
+2245662,en,AF,Africa,SN,Senegal,0
+2260494,en,AF,Africa,CG,"Congo Republic",0
+2264397,en,EU,Europe,PT,Portugal,1
+2275384,en,AF,Africa,LR,Liberia,0
+2287781,en,AF,Africa,CI,"Ivory Coast",0
+2300660,en,AF,Africa,GH,Ghana,0
+2309096,en,AF,Africa,GQ,"Equatorial Guinea",0
+2328926,en,AF,Africa,NG,Nigeria,0
+2361809,en,AF,Africa,BF,"Burkina Faso",0
+2363686,en,AF,Africa,TG,Togo,0
+2372248,en,AF,Africa,GW,Guinea-Bissau,0
+2378080,en,AF,Africa,MR,Mauritania,0
+2395170,en,AF,Africa,BJ,Benin,0
+2400553,en,AF,Africa,GA,Gabon,0
+2403846,en,AF,Africa,SL,"Sierra Leone",0
+2410758,en,AF,Africa,ST,"São Tomé and Príncipe",0
+2411586,en,EU,Europe,GI,Gibraltar,0
+2413451,en,AF,Africa,GM,Gambia,0
+2420477,en,AF,Africa,GN,Guinea,0
+2434508,en,AF,Africa,TD,Chad,0
+2440476,en,AF,Africa,NE,Niger,0
+2453866,en,AF,Africa,ML,Mali,0
+2461445,en,AF,Africa,EH,"Western Sahara",0
+2464461,en,AF,Africa,TN,Tunisia,0
+2510769,en,EU,Europe,ES,Spain,1
+2542007,en,AF,Africa,MA,Morocco,0
+2562770,en,EU,Europe,MT,Malta,1
+2589581,en,AF,Africa,DZ,Algeria,0
+2622320,en,EU,Europe,FO,"Faroe Islands",0
+2623032,en,EU,Europe,DK,Denmark,1
+2629691,en,EU,Europe,IS,Iceland,0
+2635167,en,EU,Europe,GB,"United Kingdom",0
+2658434,en,EU,Europe,CH,Switzerland,0
+2661886,en,EU,Europe,SE,Sweden,1
+2750405,en,EU,Europe,NL,"The Netherlands",1
+2782113,en,EU,Europe,AT,Austria,1
+2802361,en,EU,Europe,BE,Belgium,1
+2921044,en,EU,Europe,DE,Germany,1
+2960313,en,EU,Europe,LU,Luxembourg,1
+2963597,en,EU,Europe,IE,Ireland,1
+2993457,en,EU,Europe,MC,Monaco,0
+3017382,en,EU,Europe,FR,France,1
+3041565,en,EU,Europe,AD,Andorra,0
+3042058,en,EU,Europe,LI,Liechtenstein,0
+3042142,en,EU,Europe,JE,Jersey,0
+3042225,en,EU,Europe,IM,"Isle of Man",0
+3042362,en,EU,Europe,GG,Guernsey,0
+3057568,en,EU,Europe,SK,Slovakia,1
+3077311,en,EU,Europe,CZ,Czechia,1
+3144096,en,EU,Europe,NO,Norway,0
+3164670,en,EU,Europe,VA,"Vatican City",0
+3168068,en,EU,Europe,SM,"San Marino",0
+3175395,en,EU,Europe,IT,Italy,1
+3190538,en,EU,Europe,SI,Slovenia,1
+3194884,en,EU,Europe,ME,Montenegro,0
+3202326,en,EU,Europe,HR,Croatia,1
+3277605,en,EU,Europe,BA,"Bosnia and Herzegovina",0
+3351879,en,AF,Africa,AO,Angola,0
+3355338,en,AF,Africa,NA,Namibia,0
+3370751,en,AF,Africa,SH,"Saint Helena",0
+3371123,en,AN,Antarctica,BV,"Bouvet Island",0
+3374084,en,NA,"North America",BB,Barbados,0
+3374766,en,AF,Africa,CV,"Cabo Verde",0
+3378535,en,SA,"South America",GY,Guyana,0
+3381670,en,SA,"South America",GF,"French Guiana",1
+3382998,en,SA,"South America",SR,Suriname,0
+3424932,en,NA,"North America",PM,"Saint Pierre and Miquelon",0
+3425505,en,NA,"North America",GL,Greenland,0
+3437598,en,SA,"South America",PY,Paraguay,0
+3439705,en,SA,"South America",UY,Uruguay,0
+3469034,en,SA,"South America",BR,Brazil,0
+3474414,en,SA,"South America",FK,"Falkland Islands",0
+3474415,en,AN,Antarctica,GS,"South Georgia and the South Sandwich Islands",0
+3489940,en,NA,"North America",JM,Jamaica,0
+3508796,en,NA,"North America",DO,"Dominican Republic",0
+3562981,en,NA,"North America",CU,Cuba,0
+3570311,en,NA,"North America",MQ,Martinique,1
+3572887,en,NA,"North America",BS,Bahamas,0
+3573345,en,NA,"North America",BM,Bermuda,0
+3573511,en,NA,"North America",AI,Anguilla,0
+3573591,en,NA,"North America",TT,"Trinidad and Tobago",0
+3575174,en,NA,"North America",KN,"St Kitts and Nevis",0
+3575830,en,NA,"North America",DM,Dominica,0
+3576396,en,NA,"North America",AG,"Antigua and Barbuda",0
+3576468,en,NA,"North America",LC,"Saint Lucia",0
+3576916,en,NA,"North America",TC,"Turks and Caicos Islands",0
+3577279,en,NA,"North America",AW,Aruba,0
+3577718,en,NA,"North America",VG,"British Virgin Islands",0
+3577815,en,NA,"North America",VC,"St Vincent and Grenadines",0
+3578097,en,NA,"North America",MS,Montserrat,0
+3578421,en,NA,"North America",MF,"Saint Martin",1
+3578476,en,NA,"North America",BL,"Saint Barthélemy",0
+3579143,en,NA,"North America",GP,Guadeloupe,1
+3580239,en,NA,"North America",GD,Grenada,0
+3580718,en,NA,"North America",KY,"Cayman Islands",0
+3582678,en,NA,"North America",BZ,Belize,0
+3585968,en,NA,"North America",SV,"El Salvador",0
+3595528,en,NA,"North America",GT,Guatemala,0
+3608932,en,NA,"North America",HN,Honduras,0
+3617476,en,NA,"North America",NI,Nicaragua,0
+3624060,en,NA,"North America",CR,"Costa Rica",0
+3625428,en,SA,"South America",VE,Venezuela,0
+3658394,en,SA,"South America",EC,Ecuador,0
+3686110,en,SA,"South America",CO,Colombia,0
+3703430,en,NA,"North America",PA,Panama,0
+3723988,en,NA,"North America",HT,Haiti,0
+3865483,en,SA,"South America",AR,Argentina,0
+3895114,en,SA,"South America",CL,Chile,0
+3923057,en,SA,"South America",BO,Bolivia,0
+3932488,en,SA,"South America",PE,Peru,0
+3996063,en,NA,"North America",MX,Mexico,0
+4030656,en,OC,Oceania,PF,"French Polynesia",0
+4030699,en,OC,Oceania,PN,"Pitcairn Islands",0
+4030945,en,OC,Oceania,KI,Kiribati,0
+4031074,en,OC,Oceania,TK,Tokelau,0
+4032283,en,OC,Oceania,TO,Tonga,0
+4034749,en,OC,Oceania,WF,"Wallis and Futuna",0
+4034894,en,OC,Oceania,WS,Samoa,0
+4036232,en,OC,Oceania,NU,Niue,0
+4041468,en,OC,Oceania,MP,"Northern Mariana Islands",0
+4043988,en,OC,Oceania,GU,Guam,0
+4566966,en,NA,"North America",PR,"Puerto Rico",0
+4796775,en,NA,"North America",VI,"U.S. Virgin Islands",0
+5854968,en,OC,Oceania,UM,"U.S. Outlying Islands",0
+5880801,en,OC,Oceania,AS,"American Samoa",0
+6251999,en,NA,"North America",CA,Canada,0
+6252001,en,NA,"North America",US,"United States",0
+6254930,en,AS,Asia,PS,Palestine,0
+6255147,en,AS,Asia,,,0
+6255148,en,EU,Europe,,,0
+6290252,en,EU,Europe,RS,Serbia,0
+6697173,en,AN,Antarctica,AQ,Antarctica,0
+7609695,en,NA,"North America",SX,"Sint Maarten",0
+7626836,en,NA,"North America",CW,Curaçao,0
+7626844,en,NA,"North America",BQ,"Bonaire, Sint Eustatius, and Saba",0
+7909807,en,AF,Africa,SS,"South Sudan",0

consumer-database/scripts/create_database.ts

@@ -0,0 +1,36 @@
+import fs from 'fs';
+
+function createIpDatabase() {
+    const data = fs.readFileSync('GeoLite2-Country-Blocks-IPv4.csv', 'utf8');
+    const rows = data.split('\n');
+    rows.splice(0, 1);
+    rows.splice(-1);
+    const parsed: [number, string, number][] = [];
+    for (const row of rows) {
+        const lineData = row.trim().split(',');
+        parsed.push([
+            parseInt(lineData[0].split('.')[0]),
+            lineData[0],
+            parseInt(lineData[1] || '0')
+        ]);
+    }
+    fs.writeFileSync('../dist/ipv4-db.json', JSON.stringify(parsed));
+
+}
+
+function createCountryDatabase() {
+    const data = fs.readFileSync('GeoLite2-Country-Locations-en.csv', 'utf8');
+    const rows = data.split('\n');
+    rows.splice(0, 1);
+    rows.splice(-1);
+    const parsed: [number, string, string][] = [];
+    for (const row of rows) {
+        const lineData = row.trim().split(',');
+        parsed.push([parseInt(lineData[0]), lineData[2], lineData[4]]);
+    }
+    fs.writeFileSync('../dist/countries-db.json', JSON.stringify(parsed));
+
+}
+
+createIpDatabase();
+createCountryDatabase();

consumer-database/src/index.ts

@@ -0,0 +1,112 @@
+
+import { requireEnv } from '@utils/requireEnv';
+import { connectDatabase } from '@services/DatabaseService';
+import { RedisStreamService } from '@services/RedisStreamService';
+import { ProjectModel } from "@schema/ProjectSchema";
+import { VisitModel } from "@schema/metrics/VisitSchema";
+import { SessionModel } from "@schema/metrics/SessionSchema";
+import { EventModel } from "@schema/metrics/EventSchema";
+import { lookup } from './lookup';
+import { UAParser } from 'ua-parser-js';
+
+connectDatabase(requireEnv('MONGO_CONNECTION_STRING'));
+main();
+
+async function main() {
+
+    await RedisStreamService.connect();
+
+    const stream_name = requireEnv('STREAM_NAME');
+    const group_name = requireEnv('GROUP_NAME') as any; // Checks are inside "startReadingLoop"
+
+    await RedisStreamService.startReadingLoop({
+        stream_name, group_name, consumer_name: `CONSUMER_${process.env.NODE_APP_INSTANCE}`
+    }, processStreamEntry);
+
+}
+
+async function processStreamEntry(data: Record<string, string>) {
+    try {
+        const eventType = data._type;
+        if (!eventType) return;
+
+        const { pid, sessionHash } = data;
+
+        const project = await ProjectModel.exists({ _id: pid });
+        if (!project) return;
+
+
+        if (eventType === 'event') return await process_event(data, sessionHash);
+        if (eventType === 'keep_alive') return await process_keep_alive(data, sessionHash);
+        if (eventType === 'visit') return await process_visit(data, sessionHash);
+
+    } catch (ex: any) {
+        console.error('ERROR PROCESSING STREAM EVENT', ex.message);
+    }
+}
+
+async function process_visit(data: Record<string, string>, sessionHash: string) {
+
+    const { pid, ip, website, page, referrer, userAgent, flowHash } = data;
+
+    let referrerParsed;
+    try {
+        referrerParsed = new URL(referrer);
+    } catch (ex) {
+        referrerParsed = { hostname: referrer };
+    }
+
+    const geoLocation = lookup(ip);
+
+    const userAgentParsed = UAParser(userAgent);
+
+    const device = userAgentParsed.device.type;
+
+    await VisitModel.create({
+        project_id: pid, website, page, referrer: referrerParsed.hostname,
+        browser: userAgentParsed.browser.name || 'NO_BROWSER',
+        os: userAgentParsed.os.name || 'NO_OS',
+        device: device ? device : (userAgentParsed.browser.name ? 'desktop' : undefined),
+        session: sessionHash,
+        flowHash,
+        continent: geoLocation[0],
+        country: geoLocation[1],
+    });
+
+}
+
+async function process_keep_alive(data: Record<string, string>, sessionHash: string) {
+
+    const { pid, instant, flowHash } = data;
+
+    if (instant == "true") {
+        await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
+            $inc: { duration: 0 },
+            flowHash,
+            updated_at: Date.now()
+        }, { upsert: true });
+    } else {
+        await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
+            $inc: { duration: 1 },
+            flowHash,
+            updated_at: Date.now()
+        }, { upsert: true });
+    }
+
+}
+
+async function process_event(data: Record<string, string>, sessionHash: string) {
+
+    const { name, metadata, pid, flowHash } = data;
+
+    let metadataObject;
+    try {
+        if (metadata) metadataObject = JSON.parse(metadata);
+    } catch (ex) {
+        metadataObject = { error: 'Error parsing metadata' }
+    }
+
+    const event = new EventModel({ project_id: pid, name, flowHash, metadata: metadataObject, session: sessionHash });
+    await event.save();
+
+}

consumer-database/src/lookup.ts

@@ -0,0 +1,42 @@
+import fs from 'fs';
+
+const ipsData = JSON.parse(fs.readFileSync('./dist/ipv4-db.json', 'utf8'));
+const countriesData = JSON.parse(fs.readFileSync('./dist/countries-db.json', 'utf8'));
+
+function inRange(ip: string, cidr: string) {
+    const [subnet, mask] = cidr.split('/');
+    const ipBytes = ip.split('.').map(Number);
+    const subnetBytes = subnet.split('.').map(Number);
+
+    const ipInt = (ipBytes[0] << 24) | (ipBytes[1] << 16) | (ipBytes[2] << 8) | ipBytes[3];
+    const subnetInt = (subnetBytes[0] << 24) | (subnetBytes[1] << 16) | (subnetBytes[2] << 8) | subnetBytes[3];
+
+    const maskInt = 0xffffffff << (32 - parseInt(mask));
+
+    return (ipInt & maskInt) === (subnetInt & maskInt);
+}
+
+function getCountryFromId(id: number) {
+    for (const country of countriesData) {
+        if (country[0] == id) {
+            return country;
+        }
+    }
+}
+
+export function lookup(ip: string) {
+    try {
+        const startPiece = parseInt(ip.split('.')[0]);
+        for (const target of ipsData) {
+            const matchingStartPiece = target[0] == startPiece;
+            if (!matchingStartPiece) continue;
+            if (!inRange(ip, target[1])) continue;
+            const country = getCountryFromId(target[2]);
+            return [country[1], country[2]];
+        }
+        return ['??', '??'];
+    } catch (ex) {
+        console.error('ERROR DURING LOOKUP', ex);
+        return ['??', '??'];
+    }
+}

consumer-database/tsconfig.json

@@ -0,0 +1,32 @@
+{
+    "compilerOptions": {
+        "baseUrl": ".",
+        "module": "NodeNext",
+        "target": "ESNext",
+        "esModuleInterop": true,
+        "outDir": "dist",
+        "paths": {
+            "@schema/*": [
+                "../shared/schema/*"
+            ],
+            "@services/*": [
+                "../shared/services/*"
+            ],
+            "@data/*": [
+                "../shared/data/*"
+            ],
+            "@functions/*": [
+                "../shared/functions/*"
+            ],
+            "@utils/*": [
+                "../shared/utils/*"
+            ]
+        }
+    },
+    "include": [
+        "src/**/*.ts"
+    ],
+    "exclude": [
+        "node_modules"
+    ]
+}