From 628e471cec6c24ae7668e52595ee490355a42982 Mon Sep 17 00:00:00 2001 From: Emily Date: Wed, 18 Sep 2024 17:57:25 +0200 Subject: [PATCH] update consumers --- .../ecosystem.config.example.cjs | 3 +- consumer-database/package.json | 5 +- consumer-limits/.gitignore | 8 + consumer-limits/ecosystem.config.example.cjs | 19 + consumer-limits/package.json | 27 + consumer-limits/pnpm-lock.yaml | 716 ++++++++++++++++++ consumer-limits/src/EmailController.ts | 67 ++ consumer-limits/src/index.ts | 79 ++ consumer-limits/tsconfig.json | 32 + shared/data/broker/Limits.ts | 2 +- 10 files changed, 953 insertions(+), 5 deletions(-) create mode 100644 consumer-limits/.gitignore create mode 100644 consumer-limits/ecosystem.config.example.cjs create mode 100644 consumer-limits/package.json create mode 100644 consumer-limits/pnpm-lock.yaml create mode 100644 consumer-limits/src/EmailController.ts create mode 100644 consumer-limits/src/index.ts create mode 100644 consumer-limits/tsconfig.json diff --git a/consumer-database/ecosystem.config.example.cjs b/consumer-database/ecosystem.config.example.cjs index 567c32b..7ef1d28 100644 --- a/consumer-database/ecosystem.config.example.cjs +++ b/consumer-database/ecosystem.config.example.cjs @@ -1,8 +1,7 @@ module.exports = { apps: [ { - name: 'QueueBroker', - port: '3999', + name: 'consumer-database', exec_mode: 'fork', script: './dist/consumer-database/src/index.js', env: { diff --git a/consumer-database/package.json b/consumer-database/package.json index 63a02ec..f599387 100644 --- a/consumer-database/package.json +++ b/consumer-database/package.json @@ -11,14 +11,15 @@ "ts-node": "^10.9.2", "typescript": "^5.4.5" }, - "name": "litlyx-queue-broker", + "name": "consumer-database", "version": "1.0.0", "main": "dist/index.js", "scripts": { "dev": "node scripts/start_dev.js", "compile": "tsc", "build": "node ../scripts/build.js", - "build_all": "npm run compile && npm run build" + "create_db": "cd scripts && ts-node create_database.ts", + "build_all": "npm run compile && npm run build && npm run create_db" }, "keywords": [], "author": "Emily", diff --git a/consumer-limits/.gitignore b/consumer-limits/.gitignore new file mode 100644 index 0000000..08c1985 --- /dev/null +++ b/consumer-limits/.gitignore @@ -0,0 +1,8 @@ +node_modules +static +ecosystem.config.cjs +dist +scripts/start_dev.js +package-lock.json +build_all.bat +tests \ No newline at end of file diff --git a/consumer-limits/ecosystem.config.example.cjs b/consumer-limits/ecosystem.config.example.cjs new file mode 100644 index 0000000..61bae21 --- /dev/null +++ b/consumer-limits/ecosystem.config.example.cjs @@ -0,0 +1,19 @@ +module.exports = { + apps: [ + { + name: 'consumer-limits', + exec_mode: 'fork', + script: './dist/consumer-limits/src/index.js', + env: { + EMAIL_SERVICE: "", + BREVO_API_KEY: "", + MONGO_CONNECTION_STRING: "", + REDIS_URL: "", + REDIS_USERNAME: "", + REDIS_PASSWORD: "", + STREAM_NAME: "", + GROUP_NAME: "" + } + } + ] +} \ No newline at end of file diff --git a/consumer-limits/package.json b/consumer-limits/package.json new file mode 100644 index 0000000..9098316 --- /dev/null +++ b/consumer-limits/package.json @@ -0,0 +1,27 @@ +{ + "dependencies": { + "mongoose": "^8.3.2", + "redis": "^4.6.14", + "ua-parser-js": "^1.0.37" + }, + "devDependencies": { + "@types/node": "^20.12.13", + "@types/ua-parser-js": "^0.7.39", + "glob": "^10.4.1", + "ts-node": "^10.9.2", + "typescript": "^5.4.5" + }, + "name": "consumer-limits", + "version": "1.0.0", + "main": "dist/index.js", + "scripts": { + "dev": "node scripts/start_dev.js", + "compile": "tsc", + "build": "node ../scripts/build.js", + "build_all": "npm run compile && npm run build" + }, + "keywords": [], + "author": "Emily", + "license": "MIT", + "description": "Database Consumer - Saves events to database." +} \ No newline at end of file diff --git a/consumer-limits/pnpm-lock.yaml b/consumer-limits/pnpm-lock.yaml new file mode 100644 index 0000000..0172519 --- /dev/null +++ b/consumer-limits/pnpm-lock.yaml @@ -0,0 +1,716 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + mongoose: + specifier: ^8.3.2 + version: 8.6.3 + redis: + specifier: ^4.6.14 + version: 4.7.0 + ua-parser-js: + specifier: ^1.0.37 + version: 1.0.39 + devDependencies: + '@types/node': + specifier: ^20.12.13 + version: 20.16.5 + '@types/ua-parser-js': + specifier: ^0.7.39 + version: 0.7.39 + glob: + specifier: ^10.4.1 + version: 10.4.5 + ts-node: + specifier: ^10.9.2 + version: 10.9.2(@types/node@20.16.5)(typescript@5.6.2) + typescript: + specifier: ^5.4.5 + version: 5.6.2 + +packages: + + '@cspotcode/source-map-support@0.8.1': + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.0': + resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==} + + '@jridgewell/trace-mapping@0.3.9': + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + + '@mongodb-js/saslprep@1.1.9': + resolution: {integrity: sha512-tVkljjeEaAhCqTzajSdgbQ6gE6f3oneVwa3iXR6csiEwXXOFsiC6Uh9iAjAhXPtqa/XMDHWjjeNH/77m/Yq2dw==} + + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + + '@redis/bloom@1.2.0': + resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/client@1.6.0': + resolution: {integrity: sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==} + engines: {node: '>=14'} + + '@redis/graph@1.1.1': + resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/json@1.0.7': + resolution: {integrity: sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/search@1.2.0': + resolution: {integrity: sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/time-series@1.1.0': + resolution: {integrity: sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@tsconfig/node10@1.0.11': + resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==} + + '@tsconfig/node12@1.0.11': + resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==} + + '@tsconfig/node14@1.0.3': + resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==} + + '@tsconfig/node16@1.0.4': + resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + + '@types/node@20.16.5': + resolution: {integrity: sha512-VwYCweNo3ERajwy0IUlqqcyZ8/A7Zwa9ZP3MnENWcB11AejO+tLy3pu850goUW2FC/IJMdZUfKpX/yxL1gymCA==} + + '@types/ua-parser-js@0.7.39': + resolution: {integrity: sha512-P/oDfpofrdtF5xw433SPALpdSchtJmY7nsJItf8h3KXqOslkbySh8zq4dSWXH2oTjRvJ5PczVEoCZPow6GicLg==} + + '@types/webidl-conversions@7.0.3': + resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + + '@types/whatwg-url@11.0.5': + resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==} + + acorn-walk@8.3.4: + resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==} + engines: {node: '>=0.4.0'} + + acorn@8.12.1: + resolution: {integrity: sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==} + engines: {node: '>=0.4.0'} + hasBin: true + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-regex@6.1.0: + resolution: {integrity: sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==} + engines: {node: '>=12'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + ansi-styles@6.2.1: + resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} + engines: {node: '>=12'} + + arg@4.1.3: + resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + + brace-expansion@2.0.1: + resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==} + + bson@6.8.0: + resolution: {integrity: sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ==} + engines: {node: '>=16.20.1'} + + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + create-require@1.1.1: + resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + + cross-spawn@7.0.3: + resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} + engines: {node: '>= 8'} + + debug@4.3.7: + resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + diff@4.0.2: + resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} + engines: {node: '>=0.3.1'} + + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + + foreground-child@3.3.0: + resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==} + engines: {node: '>=14'} + + generic-pool@3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + + glob@10.4.5: + resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} + hasBin: true + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + + kareem@2.6.3: + resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==} + engines: {node: '>=12.0.0'} + + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + + make-error@1.3.6: + resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} + + memory-pager@1.5.0: + resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + + minimatch@9.0.5: + resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} + engines: {node: '>=16 || 14 >=14.17'} + + minipass@7.1.2: + resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} + engines: {node: '>=16 || 14 >=14.17'} + + mongodb-connection-string-url@3.0.1: + resolution: {integrity: sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==} + + mongodb@6.8.0: + resolution: {integrity: sha512-HGQ9NWDle5WvwMnrvUxsFYPd3JEbqD3RgABHBQRuoCEND0qzhsd0iH5ypHsf1eJ+sXmvmyKpP+FLOKY8Il7jMw==} + engines: {node: '>=16.20.1'} + peerDependencies: + '@aws-sdk/credential-providers': ^3.188.0 + '@mongodb-js/zstd': ^1.1.0 + gcp-metadata: ^5.2.0 + kerberos: ^2.0.1 + mongodb-client-encryption: '>=6.0.0 <7' + snappy: ^7.2.2 + socks: ^2.7.1 + peerDependenciesMeta: + '@aws-sdk/credential-providers': + optional: true + '@mongodb-js/zstd': + optional: true + gcp-metadata: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + snappy: + optional: true + socks: + optional: true + + mongoose@8.6.3: + resolution: {integrity: sha512-++yRmm7hjMbqVA/8WeiygTnEfrFbiy+OBjQi49GFJIvCQuSYE56myyQWo4j5hbpcHjhHQU8NukMNGTwAWFWjIw==} + engines: {node: '>=16.20.1'} + + mpath@0.9.0: + resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==} + engines: {node: '>=4.0.0'} + + mquery@5.0.0: + resolution: {integrity: sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==} + engines: {node: '>=14.0.0'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + package-json-from-dist@1.0.0: + resolution: {integrity: sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + redis@4.7.0: + resolution: {integrity: sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + sift@17.1.3: + resolution: {integrity: sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + sparse-bitfield@3.0.3: + resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-ansi@7.1.0: + resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} + engines: {node: '>=12'} + + tr46@4.1.1: + resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==} + engines: {node: '>=14'} + + ts-node@10.9.2: + resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} + hasBin: true + peerDependencies: + '@swc/core': '>=1.2.50' + '@swc/wasm': '>=1.2.50' + '@types/node': '*' + typescript: '>=2.7' + peerDependenciesMeta: + '@swc/core': + optional: true + '@swc/wasm': + optional: true + + typescript@5.6.2: + resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==} + engines: {node: '>=14.17'} + hasBin: true + + ua-parser-js@1.0.39: + resolution: {integrity: sha512-k24RCVWlEcjkdOxYmVJgeD/0a1TiSpqLg+ZalVGV9lsnr4yqu0w7tX/x2xX6G4zpkgQnRf89lxuZ1wsbjXM8lw==} + hasBin: true + + undici-types@6.19.8: + resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} + + v8-compile-cache-lib@3.0.1: + resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} + + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + + whatwg-url@13.0.0: + resolution: {integrity: sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==} + engines: {node: '>=16'} + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + + yn@3.1.1: + resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} + engines: {node: '>=6'} + +snapshots: + + '@cspotcode/source-map-support@0.8.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.1.0 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.0': {} + + '@jridgewell/trace-mapping@0.3.9': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.0 + + '@mongodb-js/saslprep@1.1.9': + dependencies: + sparse-bitfield: 3.0.3 + + '@pkgjs/parseargs@0.11.0': + optional: true + + '@redis/bloom@1.2.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/client@1.6.0': + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + + '@redis/graph@1.1.1(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/json@1.0.7(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/search@1.2.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/time-series@1.1.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@tsconfig/node10@1.0.11': {} + + '@tsconfig/node12@1.0.11': {} + + '@tsconfig/node14@1.0.3': {} + + '@tsconfig/node16@1.0.4': {} + + '@types/node@20.16.5': + dependencies: + undici-types: 6.19.8 + + '@types/ua-parser-js@0.7.39': {} + + '@types/webidl-conversions@7.0.3': {} + + '@types/whatwg-url@11.0.5': + dependencies: + '@types/webidl-conversions': 7.0.3 + + acorn-walk@8.3.4: + dependencies: + acorn: 8.12.1 + + acorn@8.12.1: {} + + ansi-regex@5.0.1: {} + + ansi-regex@6.1.0: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@6.2.1: {} + + arg@4.1.3: {} + + balanced-match@1.0.2: {} + + brace-expansion@2.0.1: + dependencies: + balanced-match: 1.0.2 + + bson@6.8.0: {} + + cluster-key-slot@1.1.2: {} + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + create-require@1.1.1: {} + + cross-spawn@7.0.3: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + debug@4.3.7: + dependencies: + ms: 2.1.3 + + diff@4.0.2: {} + + eastasianwidth@0.2.0: {} + + emoji-regex@8.0.0: {} + + emoji-regex@9.2.2: {} + + foreground-child@3.3.0: + dependencies: + cross-spawn: 7.0.3 + signal-exit: 4.1.0 + + generic-pool@3.9.0: {} + + glob@10.4.5: + dependencies: + foreground-child: 3.3.0 + jackspeak: 3.4.3 + minimatch: 9.0.5 + minipass: 7.1.2 + package-json-from-dist: 1.0.0 + path-scurry: 1.11.1 + + is-fullwidth-code-point@3.0.0: {} + + isexe@2.0.0: {} + + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + + kareem@2.6.3: {} + + lru-cache@10.4.3: {} + + make-error@1.3.6: {} + + memory-pager@1.5.0: {} + + minimatch@9.0.5: + dependencies: + brace-expansion: 2.0.1 + + minipass@7.1.2: {} + + mongodb-connection-string-url@3.0.1: + dependencies: + '@types/whatwg-url': 11.0.5 + whatwg-url: 13.0.0 + + mongodb@6.8.0: + dependencies: + '@mongodb-js/saslprep': 1.1.9 + bson: 6.8.0 + mongodb-connection-string-url: 3.0.1 + + mongoose@8.6.3: + dependencies: + bson: 6.8.0 + kareem: 2.6.3 + mongodb: 6.8.0 + mpath: 0.9.0 + mquery: 5.0.0 + ms: 2.1.3 + sift: 17.1.3 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - gcp-metadata + - kerberos + - mongodb-client-encryption + - snappy + - socks + - supports-color + + mpath@0.9.0: {} + + mquery@5.0.0: + dependencies: + debug: 4.3.7 + transitivePeerDependencies: + - supports-color + + ms@2.1.3: {} + + package-json-from-dist@1.0.0: {} + + path-key@3.1.1: {} + + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.2 + + punycode@2.3.1: {} + + redis@4.7.0: + dependencies: + '@redis/bloom': 1.2.0(@redis/client@1.6.0) + '@redis/client': 1.6.0 + '@redis/graph': 1.1.1(@redis/client@1.6.0) + '@redis/json': 1.0.7(@redis/client@1.6.0) + '@redis/search': 1.2.0(@redis/client@1.6.0) + '@redis/time-series': 1.1.0(@redis/client@1.6.0) + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + sift@17.1.3: {} + + signal-exit@4.1.0: {} + + sparse-bitfield@3.0.3: + dependencies: + memory-pager: 1.5.0 + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.1.0 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-ansi@7.1.0: + dependencies: + ansi-regex: 6.1.0 + + tr46@4.1.1: + dependencies: + punycode: 2.3.1 + + ts-node@10.9.2(@types/node@20.16.5)(typescript@5.6.2): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + '@tsconfig/node10': 1.0.11 + '@tsconfig/node12': 1.0.11 + '@tsconfig/node14': 1.0.3 + '@tsconfig/node16': 1.0.4 + '@types/node': 20.16.5 + acorn: 8.12.1 + acorn-walk: 8.3.4 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + typescript: 5.6.2 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + + typescript@5.6.2: {} + + ua-parser-js@1.0.39: {} + + undici-types@6.19.8: {} + + v8-compile-cache-lib@3.0.1: {} + + webidl-conversions@7.0.0: {} + + whatwg-url@13.0.0: + dependencies: + tr46: 4.1.1 + webidl-conversions: 7.0.0 + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.1 + string-width: 5.1.2 + strip-ansi: 7.1.0 + + yallist@4.0.0: {} + + yn@3.1.1: {} diff --git a/consumer-limits/src/EmailController.ts b/consumer-limits/src/EmailController.ts new file mode 100644 index 0000000..c871b63 --- /dev/null +++ b/consumer-limits/src/EmailController.ts @@ -0,0 +1,67 @@ +import { ProjectModel } from "@schema/ProjectSchema"; +import { UserModel } from "@schema/UserSchema"; +import { LimitNotifyModel } from "@schema/broker/LimitNotifySchema"; +import EmailService from '@services/EmailService'; +import { requireEnv } from "@utils/requireEnv"; +import { TProjectLimit } from "@schema/ProjectsLimits"; + +if (process.env.EMAIL_SERVICE) { + EmailService.init(requireEnv('BREVO_API_KEY')); +} + +export async function checkLimitsForEmail(projectCounts: TProjectLimit) { + + const project_id = projectCounts.project_id; + const hasNotifyEntry = await LimitNotifyModel.findOne({ project_id }); + if (!hasNotifyEntry) { + await LimitNotifyModel.create({ project_id, limit1: false, limit2: false, limit3: false }) + } + + if ((projectCounts.visits + projectCounts.events) >= (projectCounts.limit)) { + + const notify = await LimitNotifyModel.findOne({ project_id }); + if (notify && notify.limit3 === true) return; + + const project = await ProjectModel.findById(project_id); + if (!project) return; + + const owner = await UserModel.findById(project.owner); + if (!owner) return; + + if (process.env.EMAIL_SERVICE) await EmailService.sendLimitEmailMax(owner.email, project.name); + await LimitNotifyModel.updateOne({ project_id: projectCounts.project_id }, { limit1: true, limit2: true, limit3: true }); + + } else if ((projectCounts.visits + projectCounts.events) >= (projectCounts.limit * 0.9)) { + + const notify = await LimitNotifyModel.findOne({ project_id }); + if (notify && notify.limit2 === true) return; + + const project = await ProjectModel.findById(project_id); + if (!project) return; + + const owner = await UserModel.findById(project.owner); + if (!owner) return; + + if (process.env.EMAIL_SERVICE) await EmailService.sendLimitEmail90(owner.email, project.name); + await LimitNotifyModel.updateOne({ project_id: projectCounts.project_id }, { limit1: true, limit2: true, limit3: false }); + + } else if ((projectCounts.visits + projectCounts.events) >= (projectCounts.limit * 0.5)) { + + const notify = await LimitNotifyModel.findOne({ project_id }); + if (notify && notify.limit1 === true) return; + + const project = await ProjectModel.findById(project_id); + if (!project) return; + + const owner = await UserModel.findById(project.owner); + if (!owner) return; + + if (process.env.EMAIL_SERVICE) await EmailService.sendLimitEmail50(owner.email, project.name); + await LimitNotifyModel.updateOne({ project_id: projectCounts.project_id }, { limit1: true, limit2: false, limit3: false }); + + } + + + + +} diff --git a/consumer-limits/src/index.ts b/consumer-limits/src/index.ts new file mode 100644 index 0000000..47c40c1 --- /dev/null +++ b/consumer-limits/src/index.ts @@ -0,0 +1,79 @@ + +import { requireEnv } from '@utils/requireEnv'; +import { connectDatabase } from '@services/DatabaseService'; +import { RedisStreamService } from '@services/RedisStreamService'; +import { ProjectModel } from "@schema/ProjectSchema"; +import { SessionModel } from "@schema/metrics/SessionSchema"; +import { ProjectLimitModel } from '@schema/ProjectsLimits'; +import { ProjectCountModel } from '@schema/ProjectsCounts'; +import { MAX_LOG_LIMIT_PERCENT } from '@data/broker/Limits'; +import { checkLimitsForEmail } from './EmailController'; + +connectDatabase(requireEnv('MONGO_CONNECTION_STRING')); +main(); + +async function main() { + + await RedisStreamService.connect(); + + const stream_name = requireEnv('STREAM_NAME'); + const group_name = requireEnv('GROUP_NAME') as any; // Checks are inside "startReadingLoop" + + await RedisStreamService.startReadingLoop({ + stream_name, group_name, consumer_name: `CONSUMER_${process.env.NODE_APP_INSTANCE}` + }, processStreamEntry); + +} + +async function checkLimits(project_id: string) { + const projectLimits = await ProjectLimitModel.findOne({ project_id }); + if (!projectLimits) return false; + const TOTAL_COUNT = projectLimits.events + projectLimits.visits; + const COUNT_LIMIT = projectLimits.limit; + if ((TOTAL_COUNT) > COUNT_LIMIT * MAX_LOG_LIMIT_PERCENT) return false; + await checkLimitsForEmail(projectLimits); + return true; +} + + +async function processStreamEntry(data: Record) { + try { + const eventType = data._type; + if (!eventType) return; + + const { pid, sessionHash } = data; + + const project = await ProjectModel.exists({ _id: pid }); + if (!project) return; + + const canLog = await checkLimits(pid); + if (!canLog) return; + + if (eventType === 'event') return await process_event(data, sessionHash); + if (eventType === 'keep_alive') return await process_keep_alive(data, sessionHash); + if (eventType === 'visit') return await process_visit(data, sessionHash); + + } catch (ex: any) { + console.error('ERROR PROCESSING STREAM EVENT', ex.message); + } +} + +async function process_visit(data: Record, sessionHash: string) { + const { pid } = data; + await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'visits': 1 } }, { upsert: true }); + await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { 'visits': 1 } }); + +} + +async function process_keep_alive(data: Record, sessionHash: string) { + const { pid } = data; + const existingSession = await SessionModel.findOne({ project_id: pid, session: sessionHash }, { _id: 1 }); + if (existingSession) return; + await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'sessions': 1 } }, { upsert: true }); +} + +async function process_event(data: Record, sessionHash: string) { + const { pid } = data; + await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'events': 1 } }, { upsert: true }); + await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { 'events': 1 } }); +} \ No newline at end of file diff --git a/consumer-limits/tsconfig.json b/consumer-limits/tsconfig.json new file mode 100644 index 0000000..667c65e --- /dev/null +++ b/consumer-limits/tsconfig.json @@ -0,0 +1,32 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "module": "NodeNext", + "target": "ESNext", + "esModuleInterop": true, + "outDir": "dist", + "paths": { + "@schema/*": [ + "../shared/schema/*" + ], + "@services/*": [ + "../shared/services/*" + ], + "@data/*": [ + "../shared/data/*" + ], + "@functions/*": [ + "../shared/functions/*" + ], + "@utils/*": [ + "../shared/utils/*" + ] + } + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} \ No newline at end of file diff --git a/shared/data/broker/Limits.ts b/shared/data/broker/Limits.ts index c2e2fb9..fd28d88 100644 --- a/shared/data/broker/Limits.ts +++ b/shared/data/broker/Limits.ts @@ -2,4 +2,4 @@ // Default: 1.01 // ((events + visits) * VALUE) > limit -export const EVENT_LOG_LIMIT_PERCENT = 1.01; \ No newline at end of file +export const MAX_LOG_LIMIT_PERCENT = 1.01; \ No newline at end of file