diff --git a/dashboard/app.config.ts b/dashboard/app.config.ts new file mode 100644 index 0000000..9402715 --- /dev/null +++ b/dashboard/app.config.ts @@ -0,0 +1,9 @@ + + +export default defineAppConfig({ + ui: { + notifications: { + position: 'top-0 bottom-[unset]' + } + } +}) \ No newline at end of file diff --git a/dashboard/app.vue b/dashboard/app.vue index 0d2fbe6..31f811b 100644 --- a/dashboard/app.vue +++ b/dashboard/app.vue @@ -69,6 +69,7 @@ const { drawerVisible, hideDrawer, drawerClasses } = useDrawer(); + diff --git a/dashboard/components/CustomTab.vue b/dashboard/components/CustomTab.vue index 9ee1c63..1797514 100644 --- a/dashboard/components/CustomTab.vue +++ b/dashboard/components/CustomTab.vue @@ -54,7 +54,7 @@ onMounted(() => {
+
\ No newline at end of file diff --git a/dashboard/components/admin/Backend.vue b/dashboard/components/admin/Backend.vue index 510d792..6d20d3d 100644 --- a/dashboard/components/admin/Backend.vue +++ b/dashboard/components/admin/Backend.vue @@ -9,7 +9,23 @@ const avgDuration = computed(() => { return (backendData.value.durations.durations.reduce((a: any, e: any) => a + parseInt(e[1]), 0) / backendData.value.durations.durations.length); }) -const labels = new Array(650).fill('-'); +const labels = computed(() => { + if (!backendData?.value?.durations) return []; + + const sizes = new Map(); + + for (const e of backendData.value.durations.durations) { + if (!sizes.has(e[0])) { + sizes.set(e[0], 0); + } else { + const data = sizes.get(e[0]) ?? 0; + sizes.set(e[0], data + 1); + } + } + + const max = Array.from(sizes.values()).reduce((a, e) => a > e ? a : e, 0); + return new Array(max).fill('-'); +}); const durationsDatasets = computed(() => { if (!backendData?.value?.durations) return []; @@ -26,7 +42,7 @@ const durationsDatasets = computed(() => { datasets.push({ points: consumerDurations.map((e: any) => { - return 1000 / parseInt(e[1]) + return 1000 / parseInt(e[1]) }), color: colors[i], chartType: 'line', @@ -45,7 +61,7 @@ const durationsDatasets = computed(() => {
-
+
Queue size: {{ backendData.queue?.size || 'ERROR' }}
diff --git a/dashboard/components/dashboard/ActionableChart.vue b/dashboard/components/dashboard/ActionableChart.vue index 73f4ea3..40f033b 100644 --- a/dashboard/components/dashboard/ActionableChart.vue +++ b/dashboard/components/dashboard/ActionableChart.vue @@ -373,8 +373,9 @@ const legendClasses = ref([
-
Unique visitors is greater than visits.
-
This can indicate bot traffic.
+
Unique visitors are higher than total visits
+
which often means bots (automated scripts or crawlers)
+
are inflating the numbers.
diff --git a/dashboard/components/dialog/shields/AddDomain.vue b/dashboard/components/dialog/shields/AddDomain.vue new file mode 100644 index 0000000..6b74c3e --- /dev/null +++ b/dashboard/components/dialog/shields/AddDomain.vue @@ -0,0 +1,67 @@ + + + \ No newline at end of file diff --git a/dashboard/components/dialog/shields/DeleteDomain.vue b/dashboard/components/dialog/shields/DeleteDomain.vue new file mode 100644 index 0000000..dac4e7c --- /dev/null +++ b/dashboard/components/dialog/shields/DeleteDomain.vue @@ -0,0 +1,56 @@ + + + \ No newline at end of file diff --git a/dashboard/components/shields/Addresses.vue b/dashboard/components/shields/Addresses.vue new file mode 100644 index 0000000..e36db64 --- /dev/null +++ b/dashboard/components/shields/Addresses.vue @@ -0,0 +1,101 @@ + + + \ No newline at end of file diff --git a/dashboard/components/shields/Domains.vue b/dashboard/components/shields/Domains.vue new file mode 100644 index 0000000..d03cf76 --- /dev/null +++ b/dashboard/components/shields/Domains.vue @@ -0,0 +1,101 @@ + + + \ No newline at end of file diff --git a/dashboard/layouts/dashboard.vue b/dashboard/layouts/dashboard.vue index dd73741..c3aa52b 100644 --- a/dashboard/layouts/dashboard.vue +++ b/dashboard/layouts/dashboard.vue @@ -19,6 +19,7 @@ const sections: Section[] = [ { label: 'Web Analytics', to: '/', icon: 'fal fa-table-layout' }, { label: 'Custom Events', to: '/events', icon: 'fal fa-square-bolt' }, { label: 'Members', to: '/members', icon: 'fal fa-users' }, + { label: 'Shields', to: '/shields', icon: 'fal fa-shield' }, { label: 'Ask AI', to: '/analyst', icon: 'fal fa-sparkles' }, // { label: 'Security', to: '/security', icon: 'fal fa-shield', disabled: selfhosted }, diff --git a/dashboard/pages/shields.vue b/dashboard/pages/shields.vue new file mode 100644 index 0000000..a3ea3fd --- /dev/null +++ b/dashboard/pages/shields.vue @@ -0,0 +1,38 @@ + + + \ No newline at end of file diff --git a/dashboard/server/api/auth/google_login.post.ts b/dashboard/server/api/auth/google_login.post.ts index 1b784cb..841456d 100644 --- a/dashboard/server/api/auth/google_login.post.ts +++ b/dashboard/server/api/auth/google_login.post.ts @@ -59,6 +59,13 @@ export default defineEventHandler(async event => { const savedUser = await newUser.save(); + + setImmediate(() => { + const emailData = EmailService.getEmailServerInfo('brevolist_add', { email: payload.email as string }); + EmailServiceHelper.sendEmail(emailData); + }); + + setImmediate(() => { console.log('SENDING WELCOME EMAIL TO', payload.email); if (!payload.email) return; diff --git a/dashboard/server/api/auth/register.post.ts b/dashboard/server/api/auth/register.post.ts index c1bcaf8..88006f0 100644 --- a/dashboard/server/api/auth/register.post.ts +++ b/dashboard/server/api/auth/register.post.ts @@ -34,6 +34,11 @@ export default defineEventHandler(async event => { await RegisterModel.create({ email, password: hashedPassword }); + setImmediate(() => { + const emailData = EmailService.getEmailServerInfo('brevolist_add', { email }); + EmailServiceHelper.sendEmail(emailData); + }); + setImmediate(() => { const emailData = EmailService.getEmailServerInfo('confirm', { target: email, link: `https://dashboard.litlyx.com/api/auth/confirm_email?register_code=${jwt}` }); EmailServiceHelper.sendEmail(emailData); diff --git a/dashboard/server/api/project/members/accept.post.ts b/dashboard/server/api/project/members/accept.post.ts index 0692dc0..bb062cb 100644 --- a/dashboard/server/api/project/members/accept.post.ts +++ b/dashboard/server/api/project/members/accept.post.ts @@ -12,7 +12,12 @@ export default defineEventHandler(async event => { console.log({ project_id, user_id: data.user.id }); - const member = await TeamMemberModel.findOne({ project_id, user_id: data.user.id }); + const member = await TeamMemberModel.findOne({ + project_id, $or: [ + { user_id: data.user.id }, + { email: data.user.user.email } + ] + }); if (!member) return setResponseStatus(event, 400, 'member not found'); member.pending = false; diff --git a/dashboard/server/api/shields/domains/add.post.ts b/dashboard/server/api/shields/domains/add.post.ts new file mode 100644 index 0000000..fe4210b --- /dev/null +++ b/dashboard/server/api/shields/domains/add.post.ts @@ -0,0 +1,21 @@ +import { DomainWhitelistModel } from "~/shared/schema/shields/DomainWhitelistSchema"; + +export default defineEventHandler(async event => { + + const data = await getRequestData(event, [], ['OWNER']); + if (!data) return; + + const body = await readBody(event); + const { domain } = body; + + if (domain.trim().length == 0) return setResponseStatus(event, 400, 'Domain is required'); + + const whitelist = await DomainWhitelistModel.updateOne({ + project_id: data.project_id + }, + { $push: { domains: domain } }, + { upsert: true } + ); + + return { ok: true }; +}); \ No newline at end of file diff --git a/dashboard/server/api/shields/domains/delete.delete.ts b/dashboard/server/api/shields/domains/delete.delete.ts new file mode 100644 index 0000000..d8b02d0 --- /dev/null +++ b/dashboard/server/api/shields/domains/delete.delete.ts @@ -0,0 +1,18 @@ +import { DomainWhitelistModel } from "~/shared/schema/shields/DomainWhitelistSchema"; + +export default defineEventHandler(async event => { + + const data = await getRequestData(event, [], ['OWNER']); + if (!data) return; + + const body = await readBody(event); + const { domain } = body; + + const removal = await DomainWhitelistModel.updateOne({ + project_id: data.project_id + }, + { $pull: { domains: domain } }, + ); + + return { ok: removal.modifiedCount == 1 }; +}); \ No newline at end of file diff --git a/dashboard/server/api/shields/domains/list.ts b/dashboard/server/api/shields/domains/list.ts new file mode 100644 index 0000000..3e1cc7d --- /dev/null +++ b/dashboard/server/api/shields/domains/list.ts @@ -0,0 +1,10 @@ +import { DomainWhitelistModel } from "~/shared/schema/shields/DomainWhitelistSchema"; + +export default defineEventHandler(async event => { + const data = await getRequestData(event, [], ['OWNER']); + if (!data) return; + const whitelist = await DomainWhitelistModel.findOne({ project_id: data.project_id }); + if (!whitelist) return []; + const domains = whitelist.domains; + return domains; +}); \ No newline at end of file diff --git a/dashboard/server/api/shields/ip/add.post.ts b/dashboard/server/api/shields/ip/add.post.ts new file mode 100644 index 0000000..1862791 --- /dev/null +++ b/dashboard/server/api/shields/ip/add.post.ts @@ -0,0 +1,11 @@ +import { AddressBlacklistModel } from "~/shared/schema/shields/AddressBlacklistSchema"; + +export default defineEventHandler(async event => { + const data = await getRequestData(event, [], ['OWNER']); + if (!data) return; + const body = await readBody(event); + const { address, description } = body; + if (address.trim().length == 0) return setResponseStatus(event, 400, 'Address is required'); + const result = await AddressBlacklistModel.updateOne({ project_id: data.project_id, address }, { description }, { upsert: true }); + return { ok: result.acknowledged }; +}); \ No newline at end of file diff --git a/dashboard/server/api/shields/ip/delete.delete.ts b/dashboard/server/api/shields/ip/delete.delete.ts new file mode 100644 index 0000000..8e2ecd7 --- /dev/null +++ b/dashboard/server/api/shields/ip/delete.delete.ts @@ -0,0 +1,14 @@ +import { AddressBlacklistModel } from "~/shared/schema/shields/AddressBlacklistSchema"; + +export default defineEventHandler(async event => { + + const data = await getRequestData(event, [], ['OWNER']); + if (!data) return; + + const body = await readBody(event); + const { address } = body; + + const removal = await AddressBlacklistModel.deleteOne({ project_id: data.project_id, address }); + + return { ok: removal.deletedCount == 1 }; +}); \ No newline at end of file diff --git a/dashboard/server/api/shields/ip/list.ts b/dashboard/server/api/shields/ip/list.ts new file mode 100644 index 0000000..be3f828 --- /dev/null +++ b/dashboard/server/api/shields/ip/list.ts @@ -0,0 +1,8 @@ +import { AddressBlacklistModel } from "~/shared/schema/shields/AddressBlacklistSchema"; + +export default defineEventHandler(async event => { + const data = await getRequestData(event, [], ['OWNER']); + if (!data) return; + const blacklist = await AddressBlacklistModel.find({ project_id: data.project_id }); + return blacklist.map(e => e.toJSON()); +}); \ No newline at end of file diff --git a/email/src/services/email.ts b/email/src/services/email.ts index 636cb30..c58beef 100644 --- a/email/src/services/email.ts +++ b/email/src/services/email.ts @@ -55,6 +55,7 @@ export class EmailService { try { await this.apiContacts.createContact({ email }); await this.apiContacts.addContactToList(12, { emails: [email] }) + return true; } catch (ex) { console.error('ERROR ADDING CONTACT', ex); return false; diff --git a/email/src/services/server.ts b/email/src/services/server.ts index 573c602..a42b020 100644 --- a/email/src/services/server.ts +++ b/email/src/services/server.ts @@ -58,7 +58,7 @@ app.post('/send/invite/noaccount', express.json(), async (req, res) => { } }); -app.post('/brevolist/add', express.json(), async (req, res) => { +app.post('/send/brevolist/add', express.json(), async (req, res) => { try { const { email } = req.body; const ok = await EmailService.createContact(email); diff --git a/producer/package.json b/producer/package.json index 1001e02..12c20e9 100644 --- a/producer/package.json +++ b/producer/package.json @@ -2,6 +2,7 @@ "dependencies": { "cors": "^2.8.5", "express": "^4.19.2", + "mongoose": "^8.12.1", "redis": "^4.7.0" }, "devDependencies": { diff --git a/producer/pnpm-lock.yaml b/producer/pnpm-lock.yaml index d26ff92..381789a 100644 --- a/producer/pnpm-lock.yaml +++ b/producer/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: express: specifier: ^4.19.2 version: 4.19.2 + mongoose: + specifier: ^8.12.1 + version: 8.12.1 redis: specifier: ^4.7.0 version: 4.7.0 @@ -50,6 +53,9 @@ packages: '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + '@mongodb-js/saslprep@1.2.0': + resolution: {integrity: sha512-+ywrb0AqkfaYuhHs6LxKWgqbh3I72EpEgESCw37o+9qPx9WTCkgDm2B+eMrwehGtHBWHFU4GXvnSCNiFhhausg==} + '@redis/bloom@1.2.0': resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} peerDependencies: @@ -127,6 +133,12 @@ packages: '@types/serve-static@1.15.7': resolution: {integrity: sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==} + '@types/webidl-conversions@7.0.3': + resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + + '@types/whatwg-url@11.0.5': + resolution: {integrity: sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==} + accepts@1.3.8: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} @@ -150,6 +162,10 @@ packages: resolution: {integrity: sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + bson@6.10.3: + resolution: {integrity: sha512-MTxGsqgYTwfshYWTRdmZRC+M7FnG1b4y7RO7p2k3X24Wq0yv1m77Wsj0BzlPzd/IowgESfsruQCUToa7vbOpPQ==} + engines: {node: '>=16.20.1'} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -192,6 +208,15 @@ packages: supports-color: optional: true + debug@4.4.0: + resolution: {integrity: sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -290,6 +315,10 @@ packages: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} + kareem@2.6.3: + resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==} + engines: {node: '>=12.0.0'} + make-error@1.3.6: resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} @@ -297,6 +326,9 @@ packages: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} + memory-pager@1.5.0: + resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + merge-descriptors@1.0.1: resolution: {integrity: sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==} @@ -317,6 +349,48 @@ packages: engines: {node: '>=4'} hasBin: true + mongodb-connection-string-url@3.0.2: + resolution: {integrity: sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==} + + mongodb@6.14.2: + resolution: {integrity: sha512-kMEHNo0F3P6QKDq17zcDuPeaywK/YaJVCEQRzPF3TOM/Bl9MFg64YE5Tu7ifj37qZJMhwU1tl2Ioivws5gRG5Q==} + engines: {node: '>=16.20.1'} + peerDependencies: + '@aws-sdk/credential-providers': ^3.188.0 + '@mongodb-js/zstd': ^1.1.0 || ^2.0.0 + gcp-metadata: ^5.2.0 + kerberos: ^2.0.1 + mongodb-client-encryption: '>=6.0.0 <7' + snappy: ^7.2.2 + socks: ^2.7.1 + peerDependenciesMeta: + '@aws-sdk/credential-providers': + optional: true + '@mongodb-js/zstd': + optional: true + gcp-metadata: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + snappy: + optional: true + socks: + optional: true + + mongoose@8.12.1: + resolution: {integrity: sha512-UW22y8QFVYmrb36hm8cGncfn4ARc/XsYWQwRTaj0gxtQk1rDuhzDO1eBantS+hTTatfAIS96LlRCJrcNHvW5+Q==} + engines: {node: '>=16.20.1'} + + mpath@0.9.0: + resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==} + engines: {node: '>=4.0.0'} + + mquery@5.0.0: + resolution: {integrity: sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==} + engines: {node: '>=14.0.0'} + ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} @@ -349,6 +423,10 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + qs@6.11.0: resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==} engines: {node: '>=0.6'} @@ -389,6 +467,12 @@ packages: resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} engines: {node: '>= 0.4'} + sift@17.1.3: + resolution: {integrity: sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==} + + sparse-bitfield@3.0.3: + resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + statuses@2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} @@ -397,6 +481,10 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + tr46@5.1.0: + resolution: {integrity: sha512-IUWnUK7ADYR5Sl1fZlO1INDUhVhatWl7BtJWsIhwJ0UAK7ilzzIa8uIqOO/aYVWHZPJkKbEL+362wrzoeRF7bw==} + engines: {node: '>=18'} + ts-node@10.9.2: resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} hasBin: true @@ -438,6 +526,14 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + + whatwg-url@14.2.0: + resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} + engines: {node: '>=18'} + yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} @@ -460,6 +556,10 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.4.15 + '@mongodb-js/saslprep@1.2.0': + dependencies: + sparse-bitfield: 3.0.3 + '@redis/bloom@1.2.0(@redis/client@1.6.0)': dependencies: '@redis/client': 1.6.0 @@ -544,6 +644,12 @@ snapshots: '@types/node': 20.14.2 '@types/send': 0.17.4 + '@types/webidl-conversions@7.0.3': {} + + '@types/whatwg-url@11.0.5': + dependencies: + '@types/webidl-conversions': 7.0.3 + accepts@1.3.8: dependencies: mime-types: 2.1.35 @@ -576,6 +682,8 @@ snapshots: transitivePeerDependencies: - supports-color + bson@6.10.3: {} + bytes@3.1.2: {} call-bind@1.0.7: @@ -609,6 +717,10 @@ snapshots: dependencies: ms: 2.0.0 + debug@4.4.0: + dependencies: + ms: 2.1.3 + define-data-property@1.1.4: dependencies: es-define-property: 1.0.0 @@ -731,10 +843,14 @@ snapshots: ipaddr.js@1.9.1: {} + kareem@2.6.3: {} + make-error@1.3.6: {} media-typer@0.3.0: {} + memory-pager@1.5.0: {} + merge-descriptors@1.0.1: {} methods@1.1.2: {} @@ -747,6 +863,44 @@ snapshots: mime@1.6.0: {} + mongodb-connection-string-url@3.0.2: + dependencies: + '@types/whatwg-url': 11.0.5 + whatwg-url: 14.2.0 + + mongodb@6.14.2: + dependencies: + '@mongodb-js/saslprep': 1.2.0 + bson: 6.10.3 + mongodb-connection-string-url: 3.0.2 + + mongoose@8.12.1: + dependencies: + bson: 6.10.3 + kareem: 2.6.3 + mongodb: 6.14.2 + mpath: 0.9.0 + mquery: 5.0.0 + ms: 2.1.3 + sift: 17.1.3 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - gcp-metadata + - kerberos + - mongodb-client-encryption + - snappy + - socks + - supports-color + + mpath@0.9.0: {} + + mquery@5.0.0: + dependencies: + debug: 4.4.0 + transitivePeerDependencies: + - supports-color + ms@2.0.0: {} ms@2.1.3: {} @@ -770,6 +924,8 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 + punycode@2.3.1: {} + qs@6.11.0: dependencies: side-channel: 1.0.6 @@ -841,10 +997,20 @@ snapshots: get-intrinsic: 1.2.4 object-inspect: 1.13.1 + sift@17.1.3: {} + + sparse-bitfield@3.0.3: + dependencies: + memory-pager: 1.5.0 + statuses@2.0.1: {} toidentifier@1.0.1: {} + tr46@5.1.0: + dependencies: + punycode: 2.3.1 + ts-node@10.9.2(@types/node@20.14.2)(typescript@5.4.5): dependencies: '@cspotcode/source-map-support': 0.8.1 @@ -880,6 +1046,13 @@ snapshots: vary@1.1.2: {} + webidl-conversions@7.0.0: {} + + whatwg-url@14.2.0: + dependencies: + tr46: 5.1.0 + webidl-conversions: 7.0.0 + yallist@4.0.0: {} yn@3.1.1: {} diff --git a/producer/src/controller.ts b/producer/src/controller.ts new file mode 100644 index 0000000..6c4d1ab --- /dev/null +++ b/producer/src/controller.ts @@ -0,0 +1,13 @@ +import { DomainWhitelistModel } from "./shared/schema/shields/DomainWhitelistSchema"; + +export async function isAllowedToLog(project_id: string, website: string) { + const whitelist = await DomainWhitelistModel.findOne({ project_id }, { website: 1 }); + if (!whitelist) return; + const allowedDomains = whitelist.domains; + for (const allowedDomain of allowedDomains) { + const regexpDomain = new RegExp(allowedDomain.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*')); + const result = website.match(regexpDomain); + if (result != null) return true; + } + return false; +} \ No newline at end of file diff --git a/producer/src/deprecated.ts b/producer/src/deprecated.ts index 7380242..62b520a 100644 --- a/producer/src/deprecated.ts +++ b/producer/src/deprecated.ts @@ -2,6 +2,7 @@ import { Router, json } from "express"; import { createSessionHash, getIPFromRequest } from "./utils"; import { requireEnv } from "./shared/utils/requireEnv"; import { RedisStreamService } from "./shared/services/RedisStreamService"; +import { isAllowedToLog } from "./controller"; const router = Router(); @@ -14,6 +15,10 @@ router.post('/keep_alive', json(jsonOptions), async (req, res) => { try { const ip = getIPFromRequest(req); const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent); + + const allowed = await isAllowedToLog(req.body.pid, req.body.website); + if (!allowed) return res.status(400); + await RedisStreamService.addToStream(streamName, { ...req.body, _type: 'keep_alive', sessionHash, ip, instant: req.body.instant + '', @@ -32,6 +37,9 @@ router.post('/metrics/push', json(jsonOptions), async (req, res) => { const ip = getIPFromRequest(req); const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent); + const allowed = await isAllowedToLog(req.body.pid, req.body.website); + if (!allowed) return res.status(400); + const { type } = req.body; if (type === 0) { diff --git a/producer/src/index.ts b/producer/src/index.ts index 88a70c5..a641515 100644 --- a/producer/src/index.ts +++ b/producer/src/index.ts @@ -14,17 +14,31 @@ const jsonOptions = { limit: '25kb', type: allowAnyType } const streamName = requireEnv('STREAM_NAME'); import DeprecatedRouter from "./deprecated"; +import { isAllowedToLog } from "./controller"; +import { connectDatabase } from "./shared/services/DatabaseService"; app.use('/v1', DeprecatedRouter); app.post('/event', express.json(jsonOptions), async (req, res) => { try { + + const startTime = Date.now(); + const ip = getIPFromRequest(req); const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent); const flowHash = createFlowSessionHash(req.body.pid, ip, req.body.userAgent); + + const allowed = await isAllowedToLog(req.body.pid, req.body.website); + if (!allowed) return res.status(400); + await RedisStreamService.addToStream(streamName, { ...req.body, _type: 'event', sessionHash, ip, flowHash, timestamp: Date.now() }); + + const duration = Date.now() - startTime; + + await RedisStreamService.METRICS_PRODUCER_onProcess(process.env.NODE_APP_INSTANCE, duration); + return res.sendStatus(200); } catch (ex: any) { return res.status(500).json({ error: ex.message }); @@ -33,10 +47,22 @@ app.post('/event', express.json(jsonOptions), async (req, res) => { app.post('/visit', express.json(jsonOptions), async (req, res) => { try { + + const startTime = Date.now(); + const ip = getIPFromRequest(req); const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent); const flowHash = createFlowSessionHash(req.body.pid, ip, req.body.userAgent); + + const allowed = await isAllowedToLog(req.body.pid, req.body.website); + if (!allowed) return res.status(400); + await RedisStreamService.addToStream(streamName, { ...req.body, _type: 'visit', sessionHash, ip, flowHash, timestamp: Date.now() }); + + const duration = Date.now() - startTime; + + await RedisStreamService.METRICS_PRODUCER_onProcess(process.env.NODE_APP_INSTANCE, duration); + return res.sendStatus(200); } catch (ex: any) { return res.status(500).json({ error: ex.message }); @@ -45,14 +71,26 @@ app.post('/visit', express.json(jsonOptions), async (req, res) => { app.post('/keep_alive', express.json(jsonOptions), async (req, res) => { try { + + const startTime = Date.now(); + const ip = getIPFromRequest(req); const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent); const flowHash = createFlowSessionHash(req.body.pid, ip, req.body.userAgent); + + const allowed = await isAllowedToLog(req.body.pid, req.body.website); + if (!allowed) return res.status(400); + await RedisStreamService.addToStream(streamName, { ...req.body, _type: 'keep_alive', sessionHash, ip, instant: req.body.instant + '', flowHash, timestamp: Date.now() }); + + const duration = Date.now() - startTime; + + await RedisStreamService.METRICS_PRODUCER_onProcess(process.env.NODE_APP_INSTANCE, duration); + return res.sendStatus(200); } catch (ex: any) { return res.status(500).json({ error: ex.message }); @@ -61,6 +99,7 @@ app.post('/keep_alive', express.json(jsonOptions), async (req, res) => { async function main() { const PORT = requireEnv("PORT"); + await connectDatabase(process.env.MONGO_CONNECTION_STRING); await RedisStreamService.connect(); app.listen(PORT, () => console.log(`Listening on port ${PORT}`)); } diff --git a/scripts/producer/deploy.ts b/scripts/producer/deploy.ts index 7e14195..4f4a496 100644 --- a/scripts/producer/deploy.ts +++ b/scripts/producer/deploy.ts @@ -4,7 +4,7 @@ import path from 'path'; import child from 'child_process'; import { createZip } from '../helpers/zip-helper'; import { DeployHelper } from '../helpers/deploy-helper'; -import { REMOTE_HOST_TESTMODE } from '../.config'; +import { DATABASE_CONNECTION_STRING_PRODUCTION, DATABASE_CONNECTION_STRING_TESTMODE, REMOTE_HOST_TESTMODE } from '../.config'; const TMP_PATH = path.join(__dirname, '../../tmp'); const LOCAL_PATH = path.join(__dirname, '../../producer'); @@ -37,7 +37,9 @@ async function main() { if (MODE === 'testmode') { const ecosystemContent = fs.readFileSync(LOCAL_PATH + '/ecosystem.config.js', 'utf8'); const REDIS_URL = ecosystemContent.match(/REDIS_URL: ["'](.*?)["']/)[1]; - const devContent = ecosystemContent.replace(REDIS_URL, `redis://${REMOTE_HOST_TESTMODE}`); + const devContent = ecosystemContent + .replace(REDIS_URL, `redis://${REMOTE_HOST_TESTMODE}`) + .replace(DATABASE_CONNECTION_STRING_PRODUCTION, `redis://${DATABASE_CONNECTION_STRING_TESTMODE}`); archive.append(Buffer.from(devContent), { name: '/ecosystem.config.js' }); } else { archive.file(LOCAL_PATH + '/ecosystem.config.js', { name: '/ecosystem.config.js' }) diff --git a/scripts/producer/shared.ts b/scripts/producer/shared.ts index cde34a6..33d76aa 100644 --- a/scripts/producer/shared.ts +++ b/scripts/producer/shared.ts @@ -11,3 +11,8 @@ helper.copy('utils/requireEnv.ts'); helper.create('services'); helper.copy('services/RedisStreamService.ts'); +helper.copy('services/DatabaseService.ts'); + +helper.create('schema'); +helper.create('schema/shields'); +helper.copy('schema/shields/DomainWhitelistSchema.ts'); diff --git a/shared_global/schema/shields/AddressBlacklistSchema.ts b/shared_global/schema/shields/AddressBlacklistSchema.ts new file mode 100644 index 0000000..001b5ff --- /dev/null +++ b/shared_global/schema/shields/AddressBlacklistSchema.ts @@ -0,0 +1,18 @@ +import { model, Schema, Types } from 'mongoose'; + +export type TAddressBlacklistSchema = { + _id: Schema.Types.ObjectId, + project_id: Schema.Types.ObjectId, + address: string, + description: string, + created_at: Date +} + +const AddressBlacklistSchema = new Schema({ + project_id: { type: Types.ObjectId, index: 1 }, + address: { type: String, required: true }, + description: { type: String }, + created_at: { type: Date, default: () => Date.now() }, +}); + +export const AddressBlacklistModel = model('address_blacklists', AddressBlacklistSchema); diff --git a/shared_global/schema/shields/DomainWhitelistSchema.ts b/shared_global/schema/shields/DomainWhitelistSchema.ts new file mode 100644 index 0000000..afb5bc8 --- /dev/null +++ b/shared_global/schema/shields/DomainWhitelistSchema.ts @@ -0,0 +1,16 @@ +import { model, Schema, Types } from 'mongoose'; + +export type TDomainWhitelistSchema = { + _id: Schema.Types.ObjectId, + project_id: Schema.Types.ObjectId, + domains: string[], + created_at: Date +} + +const DomainWhitelistSchema = new Schema({ + project_id: { type: Types.ObjectId, index: 1 }, + domains: [{ type: String, required: true }], + created_at: { type: Date, default: () => Date.now() }, +}); + +export const DomainWhitelistModel = model('domain_whitelists', DomainWhitelistSchema); diff --git a/shared_global/services/EmailService.ts b/shared_global/services/EmailService.ts index b7dc6f6..91caf58 100644 --- a/shared_global/services/EmailService.ts +++ b/shared_global/services/EmailService.ts @@ -9,7 +9,8 @@ const templateMap = { limit_90: '/limit/90', limit_max: '/limit/max', invite_project: '/invite', - invite_project_noaccount: '/invite/noaccount' + invite_project_noaccount: '/invite/noaccount', + brevolist_add: '/brevolist/add' } as const; export type EmailTemplate = keyof typeof templateMap; @@ -27,6 +28,7 @@ type EmailData = | { template: 'limit_max', data: { target: string, projectName: string } } | { template: 'invite_project', data: { target: string, projectName: string, link: string } } | { template: 'invite_project_noaccount', data: { target: string, projectName: string, link: string } } + | { template: 'brevolist_add', data: { email: string } } export class EmailService { static getEmailServerInfo(template: T, data: Extract['data']): EmailServerInfo { diff --git a/shared_global/services/RedisStreamService.ts b/shared_global/services/RedisStreamService.ts index 6cd3a93..d247d40 100644 --- a/shared_global/services/RedisStreamService.ts +++ b/shared_global/services/RedisStreamService.ts @@ -26,6 +26,7 @@ export class RedisStreamService { private static METRICS_MAX_ENTRIES = 1000; + private static METRICS_MAX_ENTRIES_PRODUCER = 1000; static async METRICS_onProcess(id: string, time: number) { const key = `___dev_metrics`; @@ -39,6 +40,18 @@ export class RedisStreamService { return data.map(e => e.split(':')) as [string, string][]; } + static async METRICS_PRODUCER_onProcess(id: string, time: number) { + const key = `___dev_metrics_producer`; + await this.client.lPush(key, `${id}:${time.toString()}`); + await this.client.lTrim(key, 0, this.METRICS_MAX_ENTRIES_PRODUCER - 1); + } + + static async METRICS_PRODUCER_get() { + const key = `___dev_metrics_producer`; + const data = await this.client.lRange(key, 0, -1); + return data.map(e => e.split(':')) as [string, string][]; + } + static async connect() { await this.client.connect();