diff --git a/consumer/package.json b/consumer/package.json index 97b4ad4..4212f42 100644 --- a/consumer/package.json +++ b/consumer/package.json @@ -2,9 +2,12 @@ "dependencies": { "axios": "^1.7.9", "express": "^4.19.2", + "mongoose": "^8.9.5", + "redis": "^4.7.0", "ua-parser-js": "^1.0.37" }, "devDependencies": { + "@types/express": "^5.0.0", "@types/node": "^20.12.13", "@types/ua-parser-js": "^0.7.39", "ts-node": "^10.9.2", diff --git a/consumer/pnpm-lock.yaml b/consumer/pnpm-lock.yaml index 4d15a9d..35af407 100644 --- a/consumer/pnpm-lock.yaml +++ b/consumer/pnpm-lock.yaml @@ -14,10 +14,19 @@ importers: express: specifier: ^4.19.2 version: 4.21.2 + mongoose: + specifier: ^8.9.5 + version: 8.9.5 + redis: + specifier: ^4.7.0 + version: 4.7.0 ua-parser-js: specifier: ^1.0.37 version: 1.0.39 devDependencies: + '@types/express': + specifier: ^5.0.0 + version: 5.0.0 '@types/node': specifier: ^20.12.13 version: 20.16.5 @@ -47,6 +56,38 @@ packages: '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + '@mongodb-js/saslprep@1.1.9': + resolution: {integrity: sha512-tVkljjeEaAhCqTzajSdgbQ6gE6f3oneVwa3iXR6csiEwXXOFsiC6Uh9iAjAhXPtqa/XMDHWjjeNH/77m/Yq2dw==} + + '@redis/bloom@1.2.0': + resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/client@1.6.0': + resolution: {integrity: sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==} + engines: {node: '>=14'} + + '@redis/graph@1.1.1': + resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/json@1.0.7': + resolution: {integrity: sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/search@1.2.0': + resolution: {integrity: sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/time-series@1.1.0': + resolution: {integrity: sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==} + peerDependencies: + '@redis/client': ^1.0.0 + '@tsconfig/node10@1.0.11': resolution: {integrity: sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==} @@ -59,12 +100,48 @@ packages: '@tsconfig/node16@1.0.4': resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + '@types/body-parser@1.19.5': + resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} + + '@types/connect@3.4.38': + resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + + '@types/express-serve-static-core@5.0.6': + resolution: {integrity: sha512-3xhRnjJPkULekpSzgtoNYYcTWgEZkp4myc+Saevii5JPnHNvHMRlBSHDbs7Bh1iPPoVTERHEZXyhyLbMEsExsA==} + + '@types/express@5.0.0': + resolution: {integrity: sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==} + + '@types/http-errors@2.0.4': + resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==} + + '@types/mime@1.3.5': + resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} + '@types/node@20.16.5': resolution: {integrity: sha512-VwYCweNo3ERajwy0IUlqqcyZ8/A7Zwa9ZP3MnENWcB11AejO+tLy3pu850goUW2FC/IJMdZUfKpX/yxL1gymCA==} + '@types/qs@6.9.18': + resolution: {integrity: sha512-kK7dgTYDyGqS+e2Q4aK9X3D7q234CIZ1Bv0q/7Z5IwRDoADNU81xXJK/YVyLbLTZCoIwUoDoffFeF+p/eIklAA==} + + '@types/range-parser@1.2.7': + resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} + + '@types/send@0.17.4': + resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==} + + '@types/serve-static@1.15.7': + resolution: {integrity: sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==} + '@types/ua-parser-js@0.7.39': resolution: {integrity: sha512-P/oDfpofrdtF5xw433SPALpdSchtJmY7nsJItf8h3KXqOslkbySh8zq4dSWXH2oTjRvJ5PczVEoCZPow6GicLg==} + '@types/webidl-conversions@7.0.3': + resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + + '@types/whatwg-url@11.0.5': + resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==} + accepts@1.3.8: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} @@ -94,6 +171,10 @@ packages: resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + bson@6.10.2: + resolution: {integrity: sha512-5afhLTjqDSA3akH56E+/2J6kTDuSIlBxyXPdQslj9hcIgOUE378xdOfZvC/9q3LifJNI6KR/juZ+d0NRNYBwXg==} + engines: {node: '>=16.20.1'} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -106,6 +187,10 @@ packages: resolution: {integrity: sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA==} engines: {node: '>= 0.4'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -136,6 +221,15 @@ packages: supports-color: optional: true + debug@4.4.0: + resolution: {integrity: sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -218,6 +312,10 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + generic-pool@3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + get-intrinsic@1.2.7: resolution: {integrity: sha512-VW6Pxhsrk0KAOqs3WEd0klDiF/+V7gQOpAvY1jVU/LHmaD/kQO4523aiJuikX/QAKYiW6x8Jh+RJej1almdtCA==} engines: {node: '>= 0.4'} @@ -253,6 +351,10 @@ packages: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} + kareem@2.6.3: + resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==} + engines: {node: '>=12.0.0'} + make-error@1.3.6: resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} @@ -264,6 +366,9 @@ packages: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} + memory-pager@1.5.0: + resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + merge-descriptors@1.0.3: resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} @@ -284,6 +389,48 @@ packages: engines: {node: '>=4'} hasBin: true + mongodb-connection-string-url@3.0.2: + resolution: {integrity: sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==} + + mongodb@6.12.0: + resolution: {integrity: sha512-RM7AHlvYfS7jv7+BXund/kR64DryVI+cHbVAy9P61fnb1RcWZqOW1/Wj2YhqMCx+MuYhqTRGv7AwHBzmsCKBfA==} + engines: {node: '>=16.20.1'} + peerDependencies: + '@aws-sdk/credential-providers': ^3.188.0 + '@mongodb-js/zstd': ^1.1.0 || ^2.0.0 + gcp-metadata: ^5.2.0 + kerberos: ^2.0.1 + mongodb-client-encryption: '>=6.0.0 <7' + snappy: ^7.2.2 + socks: ^2.7.1 + peerDependenciesMeta: + '@aws-sdk/credential-providers': + optional: true + '@mongodb-js/zstd': + optional: true + gcp-metadata: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + snappy: + optional: true + socks: + optional: true + + mongoose@8.9.5: + resolution: {integrity: sha512-SPhOrgBm0nKV3b+IIHGqpUTOmgVL5Z3OO9AwkFEmvOZznXTvplbomstCnPOGAyungtRXE5pJTgKpKcZTdjeESg==} + engines: {node: '>=16.20.1'} + + mpath@0.9.0: + resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==} + engines: {node: '>=4.0.0'} + + mquery@5.0.0: + resolution: {integrity: sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==} + engines: {node: '>=14.0.0'} + ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} @@ -316,6 +463,10 @@ packages: proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + qs@6.13.0: resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==} engines: {node: '>=0.6'} @@ -328,6 +479,9 @@ packages: resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} engines: {node: '>= 0.8'} + redis@4.7.0: + resolution: {integrity: sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -361,6 +515,12 @@ packages: resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} engines: {node: '>= 0.4'} + sift@17.1.3: + resolution: {integrity: sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==} + + sparse-bitfield@3.0.3: + resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + statuses@2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} @@ -369,6 +529,10 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + tr46@5.0.0: + resolution: {integrity: sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==} + engines: {node: '>=18'} + ts-node@10.9.2: resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} hasBin: true @@ -414,6 +578,17 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + + whatwg-url@14.1.0: + resolution: {integrity: sha512-jlf/foYIKywAt3x/XWKZ/3rz8OSJPiWktjmk891alJUEjiVxKX9LEO92qH3hv4aJ0mN3MWPvGMCy8jQi95xK4w==} + engines: {node: '>=18'} + + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} @@ -433,6 +608,36 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.0 + '@mongodb-js/saslprep@1.1.9': + dependencies: + sparse-bitfield: 3.0.3 + + '@redis/bloom@1.2.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/client@1.6.0': + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + + '@redis/graph@1.1.1(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/json@1.0.7(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/search@1.2.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + + '@redis/time-series@1.1.0(@redis/client@1.6.0)': + dependencies: + '@redis/client': 1.6.0 + '@tsconfig/node10@1.0.11': {} '@tsconfig/node12@1.0.11': {} @@ -441,12 +646,60 @@ snapshots: '@tsconfig/node16@1.0.4': {} + '@types/body-parser@1.19.5': + dependencies: + '@types/connect': 3.4.38 + '@types/node': 20.16.5 + + '@types/connect@3.4.38': + dependencies: + '@types/node': 20.16.5 + + '@types/express-serve-static-core@5.0.6': + dependencies: + '@types/node': 20.16.5 + '@types/qs': 6.9.18 + '@types/range-parser': 1.2.7 + '@types/send': 0.17.4 + + '@types/express@5.0.0': + dependencies: + '@types/body-parser': 1.19.5 + '@types/express-serve-static-core': 5.0.6 + '@types/qs': 6.9.18 + '@types/serve-static': 1.15.7 + + '@types/http-errors@2.0.4': {} + + '@types/mime@1.3.5': {} + '@types/node@20.16.5': dependencies: undici-types: 6.19.8 + '@types/qs@6.9.18': {} + + '@types/range-parser@1.2.7': {} + + '@types/send@0.17.4': + dependencies: + '@types/mime': 1.3.5 + '@types/node': 20.16.5 + + '@types/serve-static@1.15.7': + dependencies: + '@types/http-errors': 2.0.4 + '@types/node': 20.16.5 + '@types/send': 0.17.4 + '@types/ua-parser-js@0.7.39': {} + '@types/webidl-conversions@7.0.3': {} + + '@types/whatwg-url@11.0.5': + dependencies: + '@types/webidl-conversions': 7.0.3 + accepts@1.3.8: dependencies: mime-types: 2.1.35 @@ -489,6 +742,8 @@ snapshots: transitivePeerDependencies: - supports-color + bson@6.10.2: {} + bytes@3.1.2: {} call-bind-apply-helpers@1.0.1: @@ -501,6 +756,8 @@ snapshots: call-bind-apply-helpers: 1.0.1 get-intrinsic: 1.2.7 + cluster-key-slot@1.1.2: {} + combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 @@ -521,6 +778,10 @@ snapshots: dependencies: ms: 2.0.0 + debug@4.4.0: + dependencies: + ms: 2.1.3 + delayed-stream@1.0.0: {} depd@2.0.0: {} @@ -615,6 +876,8 @@ snapshots: function-bind@1.1.2: {} + generic-pool@3.9.0: {} + get-intrinsic@1.2.7: dependencies: call-bind-apply-helpers: 1.0.1 @@ -657,12 +920,16 @@ snapshots: ipaddr.js@1.9.1: {} + kareem@2.6.3: {} + make-error@1.3.6: {} math-intrinsics@1.1.0: {} media-typer@0.3.0: {} + memory-pager@1.5.0: {} + merge-descriptors@1.0.3: {} methods@1.1.2: {} @@ -675,6 +942,44 @@ snapshots: mime@1.6.0: {} + mongodb-connection-string-url@3.0.2: + dependencies: + '@types/whatwg-url': 11.0.5 + whatwg-url: 14.1.0 + + mongodb@6.12.0: + dependencies: + '@mongodb-js/saslprep': 1.1.9 + bson: 6.10.2 + mongodb-connection-string-url: 3.0.2 + + mongoose@8.9.5: + dependencies: + bson: 6.10.2 + kareem: 2.6.3 + mongodb: 6.12.0 + mpath: 0.9.0 + mquery: 5.0.0 + ms: 2.1.3 + sift: 17.1.3 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - gcp-metadata + - kerberos + - mongodb-client-encryption + - snappy + - socks + - supports-color + + mpath@0.9.0: {} + + mquery@5.0.0: + dependencies: + debug: 4.4.0 + transitivePeerDependencies: + - supports-color + ms@2.0.0: {} ms@2.1.3: {} @@ -698,6 +1003,8 @@ snapshots: proxy-from-env@1.1.0: {} + punycode@2.3.1: {} + qs@6.13.0: dependencies: side-channel: 1.1.0 @@ -711,6 +1018,15 @@ snapshots: iconv-lite: 0.4.24 unpipe: 1.0.0 + redis@4.7.0: + dependencies: + '@redis/bloom': 1.2.0(@redis/client@1.6.0) + '@redis/client': 1.6.0 + '@redis/graph': 1.1.1(@redis/client@1.6.0) + '@redis/json': 1.0.7(@redis/client@1.6.0) + '@redis/search': 1.2.0(@redis/client@1.6.0) + '@redis/time-series': 1.1.0(@redis/client@1.6.0) + safe-buffer@5.2.1: {} safer-buffer@2.1.2: {} @@ -772,10 +1088,20 @@ snapshots: side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 + sift@17.1.3: {} + + sparse-bitfield@3.0.3: + dependencies: + memory-pager: 1.5.0 + statuses@2.0.1: {} toidentifier@1.0.1: {} + tr46@5.0.0: + dependencies: + punycode: 2.3.1 + ts-node@10.9.2(@types/node@20.16.5)(typescript@5.6.2): dependencies: '@cspotcode/source-map-support': 0.8.1 @@ -813,4 +1139,13 @@ snapshots: vary@1.1.2: {} + webidl-conversions@7.0.0: {} + + whatwg-url@14.1.0: + dependencies: + tr46: 5.0.0 + webidl-conversions: 7.0.0 + + yallist@4.0.0: {} + yn@3.1.1: {} diff --git a/consumer/src/Metrics.ts b/consumer/src/Metrics.ts new file mode 100644 index 0000000..61a77b6 --- /dev/null +++ b/consumer/src/Metrics.ts @@ -0,0 +1,48 @@ + +import { Router } from 'express'; +import { RedisStreamService } from './shared/services/RedisStreamService'; +import { requireEnv } from './shared/utils/requireEnv'; + +const stream_name = requireEnv('STREAM_NAME'); + +export class MetricsManager { + + private static processTime = new Map(); + + static onProcess(id: string, time: number) { + const target = this.processTime.get(id); + if (!target) { + this.processTime.set(id, [time]); + } else { + target.push(time); + if (target.length > 1000) target.splice(0, target.length - 1000); + } + } + + static get() { + return Array.from(this.processTime.entries()); + } + +} + +export const metricsRouter = Router(); + +metricsRouter.get('/queue', async (req, res) => { + try { + const size = await RedisStreamService.getQueueInfo(stream_name); + res.json({ size }); + } catch (ex) { + console.error(ex); + res.status(500).json({ error: ex.message }); + } +}) + +metricsRouter.get('/durations', async (req, res) => { + try { + const durations = MetricsManager.get(); + res.json({ durations }); + } catch (ex) { + console.error(ex); + res.status(500).json({ error: ex.message }); + } +}) \ No newline at end of file diff --git a/consumer/src/index.ts b/consumer/src/index.ts index 076955f..7c98428 100644 --- a/consumer/src/index.ts +++ b/consumer/src/index.ts @@ -13,27 +13,19 @@ import express from 'express'; import { ProjectLimitModel } from './shared/schema/project/ProjectsLimits'; import { ProjectCountModel } from './shared/schema/project/ProjectsCounts'; +import { MetricsManager, metricsRouter } from './Metrics'; const app = express(); -let durations: number[] = []; - -app.get('/status', async (req, res) => { - try { - return res.json({ status: 'ALIVE', durations }) - } catch (ex) { - console.error(ex); - return res.setStatus(500).json({ error: ex.message }); - } -}) +app.use('/metrics', metricsRouter); app.listen(process.env.PORT); connectDatabase(requireEnv('MONGO_CONNECTION_STRING')); main(); - +const CONSUMER_NAME = `CONSUMER_${process.env.NODE_APP_INSTANCE || 'DEFAULT'}` async function main() { @@ -43,7 +35,7 @@ async function main() { const group_name = requireEnv('GROUP_NAME') as any; // Checks are inside "startReadingLoop" await RedisStreamService.startReadingLoop({ - stream_name, group_name, consumer_name: `CONSUMER_${process.env.NODE_APP_INSTANCE || 'DEFAULT'}` + stream_name, group_name, consumer_name: CONSUMER_NAME }, processStreamEntry); } @@ -73,18 +65,13 @@ async function processStreamEntry(data: Record) { await process_visit(data, sessionHash); } - // console.log('Entry processed in', duration, 'ms'); - } catch (ex: any) { console.error('ERROR PROCESSING STREAM EVENT', ex.message); } const duration = Date.now() - start; - durations.push(duration); - if (durations.length > 1000) { - durations = durations.splice(500); - } + MetricsManager.onProcess(CONSUMER_NAME, duration); } diff --git a/consumer/src/shared/services/RedisStreamService.ts b/consumer/src/shared/services/RedisStreamService.ts index 81a5ff7..a3e3928 100644 --- a/consumer/src/shared/services/RedisStreamService.ts +++ b/consumer/src/shared/services/RedisStreamService.ts @@ -29,9 +29,20 @@ export class RedisStreamService { await this.client.connect(); } + + static async getQueueInfo(stream_name: string) { + try { + const size = await this.client.xLen(stream_name); + return size; + } catch (ex) { + console.error(ex); + return 0; + } + } + static async readFromStream(stream_name: string, group_name: string, consumer_name: string, process_function: (content: Record) => Promise) { - const result: xReadGgroupResult = await this.client.xReadGroup(group_name, consumer_name, [{ key: stream_name, id: '>' }], { COUNT: 5, BLOCK: 10000 }); + const result: xReadGgroupResult = await this.client.xReadGroup(group_name, consumer_name, [{ key: stream_name, id: '>' }], { COUNT: 5, BLOCK: 2000 }); if (!result) { setTimeout(() => this.readFromStream(stream_name, group_name, consumer_name, process_function), 10); diff --git a/shared_global/pnpm-lock.yaml b/shared_global/pnpm-lock.yaml index 576d8fe..9c4d653 100644 --- a/shared_global/pnpm-lock.yaml +++ b/shared_global/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@types/node': specifier: ^22.10.10 version: 22.10.10 + '@types/redis': + specifier: ^4.0.11 + version: 4.0.11 date-fns: specifier: ^4.1.0 version: 4.1.0 @@ -61,6 +64,10 @@ packages: '@types/node@22.10.10': resolution: {integrity: sha512-X47y/mPNzxviAGY5TcYPtYL8JsY3kAq2n8fMmKoRCxq/c4v4pyGNCzM2R6+M5/umG4ZfHuT+sgqDYqWc9rJ6ww==} + '@types/redis@4.0.11': + resolution: {integrity: sha512-bI+gth8La8Wg/QCR1+V1fhrL9+LZUSWfcqpOj2Kc80ZQ4ffbdL173vQd5wovmoV9i071FU9oP2g6etLuEwb6Rg==} + deprecated: This is a stub types definition. redis provides its own type definitions, so you do not need this installed. + '@types/webidl-conversions@7.0.3': resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} @@ -213,6 +220,10 @@ snapshots: dependencies: undici-types: 6.20.0 + '@types/redis@4.0.11': + dependencies: + redis: 4.7.0 + '@types/webidl-conversions@7.0.3': {} '@types/whatwg-url@11.0.5': diff --git a/shared_global/services/RedisStreamService.ts b/shared_global/services/RedisStreamService.ts index 81a5ff7..a3e3928 100644 --- a/shared_global/services/RedisStreamService.ts +++ b/shared_global/services/RedisStreamService.ts @@ -29,9 +29,20 @@ export class RedisStreamService { await this.client.connect(); } + + static async getQueueInfo(stream_name: string) { + try { + const size = await this.client.xLen(stream_name); + return size; + } catch (ex) { + console.error(ex); + return 0; + } + } + static async readFromStream(stream_name: string, group_name: string, consumer_name: string, process_function: (content: Record) => Promise) { - const result: xReadGgroupResult = await this.client.xReadGroup(group_name, consumer_name, [{ key: stream_name, id: '>' }], { COUNT: 5, BLOCK: 10000 }); + const result: xReadGgroupResult = await this.client.xReadGroup(group_name, consumer_name, [{ key: stream_name, id: '>' }], { COUNT: 5, BLOCK: 2000 }); if (!result) { setTimeout(() => this.readFromStream(stream_name, group_name, consumer_name, process_function), 10);