diff --git a/consumer/Dockerfile b/consumer/Dockerfile index 3ef504d..543c91b 100644 --- a/consumer/Dockerfile +++ b/consumer/Dockerfile @@ -1,38 +1,28 @@ -# Start with a minimal Node.js base image + FROM node:21-alpine as base -# Install pnpm globally with caching to avoid reinstalling if nothing has changed RUN npm i -g pnpm -# Set the working directory WORKDIR /home/app -# Copy only package-related files to leverage caching +COPY --link ./package.json ./tsconfig.json ./pnpm-lock.yaml ./ COPY --link ./scripts/package.json ./scripts/pnpm-lock.yaml ./scripts/ -COPY --link ./shared/package.json ./shared/pnpm-lock.yaml ./shared/ COPY --link ./consumer/package.json ./consumer/pnpm-lock.yaml ./consumer/ -# Install dependencies for each package +RUN pnpm install +RUN pnpm install --filter consumer + WORKDIR /home/app/scripts -RUN pnpm install --frozen-lockfile +RUN pnpm install -WORKDIR /home/app/shared -RUN pnpm install --frozen-lockfile - -WORKDIR /home/app/consumer -RUN pnpm install --frozen-lockfile - -# Now copy the rest of the source files WORKDIR /home/app COPY --link ../scripts ./scripts COPY --link ../shared ./shared COPY --link ../consumer ./consumer -# Build the consumer WORKDIR /home/app/consumer -RUN pnpm run build_all +RUN pnpm run build -# Start the application CMD ["node", "/home/app/consumer/dist/consumer/src/index.js"] \ No newline at end of file diff --git a/consumer/ecosystem.config.example.cjs b/consumer/ecosystem.config.example.cjs index 8261ff3..95efba9 100644 --- a/consumer/ecosystem.config.example.cjs +++ b/consumer/ecosystem.config.example.cjs @@ -2,15 +2,19 @@ module.exports = { apps: [ { name: 'consumer', - exec_mode: 'fork', + port: '3031', + exec_mode: 'cluster', + instances: '2', script: './dist/consumer/src/index.js', env: { - MONGO_CONNECTION_STRING: "", + EMAIL_SERVICE: '', + BREVO_API_KEY: '', + MONGO_CONNECTION_STRING: '', REDIS_URL: "", REDIS_USERNAME: "", REDIS_PASSWORD: "", STREAM_NAME: "", - GROUP_NAME: "" + GROUP_NAME: '' } } ] diff --git a/consumer/package.json b/consumer/package.json index 5ef3b6a..81974de 100644 --- a/consumer/package.json +++ b/consumer/package.json @@ -1,8 +1,6 @@ { "dependencies": { - "@getbrevo/brevo": "^2.2.0", - "mongoose": "^8.3.2", - "redis": "^4.6.14", + "express": "^4.19.2", "ua-parser-js": "^1.0.37" }, "devDependencies": { @@ -11,17 +9,17 @@ "ts-node": "^10.9.2", "typescript": "^5.4.5" }, - "name": "consumer-database", + "name": "consumer", "version": "1.0.0", "main": "dist/index.js", "scripts": { "dev": "node scripts/start_dev.js", "compile": "tsc", - "build": "node ../scripts/build.js", + "build_project": "node ../scripts/build.js", + "build": "npm run compile && npm run build_project && npm run create_db", "create_db": "cd scripts && ts-node create_database.ts", - "build_all": "npm run compile && npm run build && npm run create_db", "docker-build": "docker build -t litlyx-consumer -f Dockerfile ../", - "docker-inspect": "docker run -it litlyx-consumer sh" + "docker-inspect": "docker run -it litlyx-consumer sh" }, "keywords": [], "author": "Emily", diff --git a/consumer/src/EmailController.ts b/consumer/src/EmailController.ts index ab55729..58926f0 100644 --- a/consumer/src/EmailController.ts +++ b/consumer/src/EmailController.ts @@ -1,9 +1,9 @@ -import { ProjectModel } from "@schema/ProjectSchema"; +import { ProjectModel } from "@schema/project/ProjectSchema"; import { UserModel } from "@schema/UserSchema"; import { LimitNotifyModel } from "@schema/broker/LimitNotifySchema"; import EmailService from '@services/EmailService'; import { requireEnv } from "@utils/requireEnv"; -import { TProjectLimit } from "@schema/ProjectsLimits"; +import { TProjectLimit } from "@schema/project/ProjectsLimits"; if (process.env.EMAIL_SERVICE) { EmailService.init(requireEnv('BREVO_API_KEY')); diff --git a/consumer/src/LimitChecker.ts b/consumer/src/LimitChecker.ts index 1ee6ba9..9eda8a8 100644 --- a/consumer/src/LimitChecker.ts +++ b/consumer/src/LimitChecker.ts @@ -1,6 +1,6 @@ -import { ProjectLimitModel } from '@schema/ProjectsLimits'; +import { ProjectLimitModel } from '@schema/project/ProjectsLimits'; import { MAX_LOG_LIMIT_PERCENT } from '@data/broker/Limits'; import { checkLimitsForEmail } from './EmailController'; diff --git a/consumer/src/index.ts b/consumer/src/index.ts index 3c466cb..d54f739 100644 --- a/consumer/src/index.ts +++ b/consumer/src/index.ts @@ -2,20 +2,39 @@ import { requireEnv } from '@utils/requireEnv'; import { connectDatabase } from '@services/DatabaseService'; import { RedisStreamService } from '@services/RedisStreamService'; -import { ProjectModel } from "@schema/ProjectSchema"; +import { ProjectModel } from "@schema/project/ProjectSchema"; import { VisitModel } from "@schema/metrics/VisitSchema"; import { SessionModel } from "@schema/metrics/SessionSchema"; import { EventModel } from "@schema/metrics/EventSchema"; import { lookup } from './lookup'; import { UAParser } from 'ua-parser-js'; import { checkLimits } from './LimitChecker'; +import express from 'express'; -import { ProjectLimitModel } from '@schema/ProjectsLimits'; -import { ProjectCountModel } from '@schema/ProjectsCounts'; +import { ProjectLimitModel } from '@schema/project/ProjectsLimits'; +import { ProjectCountModel } from '@schema/project/ProjectsCounts'; + + +const app = express(); + +let durations: number[] = []; + +app.get('/status', async (req, res) => { + try { + return res.json({ status: 'ALIVE', durations }) + } catch (ex) { + console.error(ex); + return res.setStatus(500).json({ error: ex.message }); + } +}) + +app.listen(process.env.PORT); connectDatabase(requireEnv('MONGO_CONNECTION_STRING')); main(); + + async function main() { await RedisStreamService.connect(); @@ -30,9 +49,10 @@ async function main() { } async function processStreamEntry(data: Record) { - try { - const start = Date.now(); + const start = Date.now(); + + try { const eventType = data._type; if (!eventType) return; @@ -53,18 +73,24 @@ async function processStreamEntry(data: Record) { await process_visit(data, sessionHash); } - const duration = Date.now() - start; - // console.log('Entry processed in', duration, 'ms'); } catch (ex: any) { console.error('ERROR PROCESSING STREAM EVENT', ex.message); } + + const duration = Date.now() - start; + + durations.push(duration); + if (durations.length > 1000) { + durations = durations.splice(500); + } + } async function process_visit(data: Record, sessionHash: string) { - const { pid, ip, website, page, referrer, userAgent, flowHash } = data; + const { pid, ip, website, page, referrer, userAgent, flowHash, timestamp } = data; let referrerParsed; try { @@ -89,6 +115,7 @@ async function process_visit(data: Record, sessionHash: string) flowHash, continent: geoLocation[0], country: geoLocation[1], + created_at: new Date(parseInt(timestamp)) }), ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'visits': 1 } }, { upsert: true }), ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { 'visits': 1 } }) @@ -98,7 +125,7 @@ async function process_visit(data: Record, sessionHash: string) async function process_keep_alive(data: Record, sessionHash: string) { - const { pid, instant, flowHash } = data; + const { pid, instant, flowHash, timestamp } = data; const existingSession = await SessionModel.findOne({ project_id: pid, session: sessionHash }, { _id: 1 }); if (!existingSession) { @@ -123,7 +150,7 @@ async function process_keep_alive(data: Record, sessionHash: str async function process_event(data: Record, sessionHash: string) { - const { name, metadata, pid, flowHash } = data; + const { name, metadata, pid, flowHash, timestamp } = data; let metadataObject; try { @@ -133,7 +160,10 @@ async function process_event(data: Record, sessionHash: string) } await Promise.all([ - EventModel.create({ project_id: pid, name, flowHash, metadata: metadataObject, session: sessionHash }), + EventModel.create({ + project_id: pid, name, flowHash, metadata: metadataObject, session: sessionHash, + created_at: new Date(parseInt(timestamp)) + }), ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'events': 1 } }, { upsert: true }), ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { 'events': 1 } }) ]); diff --git a/consumer/tsconfig.json b/consumer/tsconfig.json index 667c65e..2ec5c72 100644 --- a/consumer/tsconfig.json +++ b/consumer/tsconfig.json @@ -1,27 +1,10 @@ { + "extends": "../tsconfig.json", "compilerOptions": { - "baseUrl": ".", "module": "NodeNext", "target": "ESNext", "esModuleInterop": true, - "outDir": "dist", - "paths": { - "@schema/*": [ - "../shared/schema/*" - ], - "@services/*": [ - "../shared/services/*" - ], - "@data/*": [ - "../shared/data/*" - ], - "@functions/*": [ - "../shared/functions/*" - ], - "@utils/*": [ - "../shared/utils/*" - ] - } + "outDir": "dist" }, "include": [ "src/**/*.ts" diff --git a/dashboard/Dockerfile b/dashboard/Dockerfile index 6dfe4c8..3e988a8 100644 --- a/dashboard/Dockerfile +++ b/dashboard/Dockerfile @@ -1,50 +1,31 @@ -# Start with a minimal Node.js base image + FROM node:21-alpine AS base -# Create a distinct build environment FROM base AS build -# Install pnpm globally with caching to avoid reinstalling if nothing has changed RUN npm i -g pnpm -# Set the working directory WORKDIR /home/app -# Copy only package-related files to leverage caching +COPY --link ./package.json ./tsconfig.json ./pnpm-lock.yaml ./ COPY --link ./dashboard/package.json ./dashboard/pnpm-lock.yaml ./dashboard/ -COPY --link ./lyx-ui/package.json ./lyx-ui/pnpm-lock.yaml ./lyx-ui/ -COPY --link ./shared/package.json ./shared/pnpm-lock.yaml ./shared/ -# Install dependencies for each package -WORKDIR /home/app/lyx-ui -RUN pnpm install --frozen-lockfile +RUN pnpm install +RUN pnpm install --filter dashboard -# WORKDIR /home/app/shared -# RUN pnpm install --frozen-lockfile - -WORKDIR /home/app/dashboard -RUN pnpm install --frozen-lockfile - -# Now copy the rest of the source files WORKDIR /home/app COPY --link ./dashboard ./dashboard -COPY --link ./lyx-ui ./lyx-ui COPY --link ./shared ./shared -# Build the dashboard WORKDIR /home/app/dashboard RUN pnpm run build -# Use a smaller base image for the final production build FROM node:21-alpine AS production -# Set the working directory for the production container WORKDIR /home/app -# Copy the built application from the build stage COPY --from=build /home/app/dashboard/.output /home/app/.output -# Start the application CMD ["node", "/home/app/.output/server/index.mjs"] \ No newline at end of file diff --git a/dashboard/app.vue b/dashboard/app.vue index 21ef47b..21c5ea1 100644 --- a/dashboard/app.vue +++ b/dashboard/app.vue @@ -10,7 +10,7 @@ const { alerts, closeAlert } = useAlert(); const { showDialog, closeDialog, dialogComponent, dialogParams, dialogStyle, dialogClosable } = useCustomDialog(); -const { visible } = usePricingDrawer(); +const { drawerVisible, hideDrawer, drawerClasses } = useDrawer(); @@ -18,10 +18,10 @@ const { visible } = usePricingDrawer();
- - - + + + @@ -70,6 +70,8 @@ const { visible } = usePricingDrawer(); + + @@ -78,18 +80,18 @@ const { visible } = usePricingDrawer(); diff --git a/dashboard/assets/scss/main.scss b/dashboard/assets/scss/main.scss index a3abc31..bbdfd57 100644 --- a/dashboard/assets/scss/main.scss +++ b/dashboard/assets/scss/main.scss @@ -1,10 +1,11 @@ +@use './utilities.scss'; +@use './colors.scss'; + @import url('https://fonts.googleapis.com/css2?family=Nunito:ital,wght@0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;0,1000;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900;1,1000&display=swap'); @import url('https://fonts.cdnfonts.com/css/brockmann'); @import url('https://fonts.googleapis.com/css2?family=Inter:wght@100..900&display=swap'); @import '../font-awesome/css/all.css'; -@import './utilities.scss'; -@import './colors.scss'; @import url('https://fonts.cdnfonts.com/css/geometric-sans-serif-v1'); @import url('https://fonts.googleapis.com/css2?family=Manrope:wght@200..800&display=swap'); diff --git a/dashboard/components/CVerticalNavigation.vue b/dashboard/components/CVerticalNavigation.vue index 107a744..91831fc 100644 --- a/dashboard/components/CVerticalNavigation.vue +++ b/dashboard/components/CVerticalNavigation.vue @@ -105,8 +105,6 @@ const { data: maxProjects } = useFetch("/api/user/max_projects", { }); -const pricingDrawer = usePricingDrawer(); -