From e9312355332bc20eccef64f48ea90891683a3f53 Mon Sep 17 00:00:00 2001 From: Emily Date: Fri, 31 Jan 2025 18:10:22 +0100 Subject: [PATCH] removed shared --- consumer/.gitignore | 11 +- consumer/src/shared/data/broker/Limits.ts | 5 - consumer/src/shared/schema/UserSchema.ts | 38 ------- .../shared/schema/broker/LimitNotifySchema.ts | 18 ---- .../src/shared/schema/metrics/EventSchema.ts | 22 ---- .../shared/schema/metrics/SessionSchema.ts | 23 ---- .../src/shared/schema/metrics/VisitSchema.ts | 45 -------- .../shared/schema/project/ProjectSchema.ts | 26 ----- .../shared/schema/project/ProjectsCounts.ts | 22 ---- .../shared/schema/project/ProjectsLimits.ts | 26 ----- .../src/shared/services/DatabaseService.ts | 10 -- consumer/src/shared/services/EmailService.ts | 35 ------ .../src/shared/services/RedisStreamService.ts | 101 ------------------ consumer/src/shared/utils/requireEnv.ts | 9 -- 14 files changed, 6 insertions(+), 385 deletions(-) delete mode 100644 consumer/src/shared/data/broker/Limits.ts delete mode 100644 consumer/src/shared/schema/UserSchema.ts delete mode 100644 consumer/src/shared/schema/broker/LimitNotifySchema.ts delete mode 100644 consumer/src/shared/schema/metrics/EventSchema.ts delete mode 100644 consumer/src/shared/schema/metrics/SessionSchema.ts delete mode 100644 consumer/src/shared/schema/metrics/VisitSchema.ts delete mode 100644 consumer/src/shared/schema/project/ProjectSchema.ts delete mode 100644 consumer/src/shared/schema/project/ProjectsCounts.ts delete mode 100644 consumer/src/shared/schema/project/ProjectsLimits.ts delete mode 100644 consumer/src/shared/services/DatabaseService.ts delete mode 100644 consumer/src/shared/services/EmailService.ts delete mode 100644 consumer/src/shared/services/RedisStreamService.ts delete mode 100644 consumer/src/shared/utils/requireEnv.ts diff --git a/consumer/.gitignore b/consumer/.gitignore index 08c1985..e2a45ec 100644 --- a/consumer/.gitignore +++ b/consumer/.gitignore @@ -1,8 +1,9 @@ + node_modules -static + ecosystem.config.cjs -dist +ecosystem.config.js + scripts/start_dev.js -package-lock.json -build_all.bat -tests \ No newline at end of file +dist +src/shared \ No newline at end of file diff --git a/consumer/src/shared/data/broker/Limits.ts b/consumer/src/shared/data/broker/Limits.ts deleted file mode 100644 index fd28d88..0000000 --- a/consumer/src/shared/data/broker/Limits.ts +++ /dev/null @@ -1,5 +0,0 @@ - - -// Default: 1.01 -// ((events + visits) * VALUE) > limit -export const MAX_LOG_LIMIT_PERCENT = 1.01; \ No newline at end of file diff --git a/consumer/src/shared/schema/UserSchema.ts b/consumer/src/shared/schema/UserSchema.ts deleted file mode 100644 index d312393..0000000 --- a/consumer/src/shared/schema/UserSchema.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { model, Schema, Types } from 'mongoose'; - -export type TUser = { - email: string, - name: string, - given_name: string, - locale: string, - picture: string, - created_at: Date, - google_tokens?: { - refresh_token?: string; - expiry_date?: number; - access_token?: string; - token_type?: string; - id_token?: string; - scope?: string; - } -} - -const UserSchema = new Schema({ - email: { type: String, unique: true, index: 1 }, - name: String, - given_name: String, - locale: String, - picture: String, - google_tokens: { - refresh_token: String, - expiry_date: Number, - access_token: String, - token_type: String, - id_token: String, - scope: String - }, - created_at: { type: Date, default: () => Date.now() } -}) - -export const UserModel = model('users', UserSchema); - diff --git a/consumer/src/shared/schema/broker/LimitNotifySchema.ts b/consumer/src/shared/schema/broker/LimitNotifySchema.ts deleted file mode 100644 index c1ec3d6..0000000 --- a/consumer/src/shared/schema/broker/LimitNotifySchema.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { model, Schema, Types } from 'mongoose'; - -export type TLimitNotify = { - _id: Schema.Types.ObjectId, - project_id: Schema.Types.ObjectId, - limit1: boolean, - limit2: boolean, - limit3: boolean -} - -const LimitNotifySchema = new Schema({ - project_id: { type: Types.ObjectId, index: 1 }, - limit1: { type: Boolean }, - limit2: { type: Boolean }, - limit3: { type: Boolean } -}); - -export const LimitNotifyModel = model('limit_notifies', LimitNotifySchema); \ No newline at end of file diff --git a/consumer/src/shared/schema/metrics/EventSchema.ts b/consumer/src/shared/schema/metrics/EventSchema.ts deleted file mode 100644 index 17ffc2b..0000000 --- a/consumer/src/shared/schema/metrics/EventSchema.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { model, Schema, Types } from 'mongoose'; - -export type TEvent = { - project_id: Schema.Types.ObjectId, - name: string, - metadata: Record, - session: string, - flowHash: string, - created_at: Date -} - -const EventSchema = new Schema({ - project_id: { type: Types.ObjectId, index: 1 }, - name: { type: String, required: true, index: 1 }, - metadata: Schema.Types.Mixed, - session: { type: String, index: 1 }, - flowHash: { type: String }, - created_at: { type: Date, default: () => Date.now(), index: true }, -}) - -export const EventModel = model('events', EventSchema); - diff --git a/consumer/src/shared/schema/metrics/SessionSchema.ts b/consumer/src/shared/schema/metrics/SessionSchema.ts deleted file mode 100644 index 152e534..0000000 --- a/consumer/src/shared/schema/metrics/SessionSchema.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { model, Schema, Types } from 'mongoose'; - - -export type TSession = { - project_id: Schema.Types.ObjectId, - session: string, - flowHash: string, - duration: number, - updated_at: Date, - created_at: Date, -} - -const SessionSchema = new Schema({ - project_id: { type: Types.ObjectId, index: 1 }, - session: { type: String, required: true, index: 1 }, - flowHash: { type: String }, - duration: { type: Number, required: true, default: 0 }, - updated_at: { type: Date, default: () => Date.now() }, - created_at: { type: Date, default: () => Date.now(), index: true }, -}) - -export const SessionModel = model('sessions', SessionSchema); - diff --git a/consumer/src/shared/schema/metrics/VisitSchema.ts b/consumer/src/shared/schema/metrics/VisitSchema.ts deleted file mode 100644 index 8981baa..0000000 --- a/consumer/src/shared/schema/metrics/VisitSchema.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { model, Schema } from 'mongoose'; - -export type TVisit = { - project_id: Schema.Types.ObjectId, - - browser: string, - os: string, - - continent: string, - country: string, - - session: string, - flowHash: string, - device: string, - - website: string, - page: string, - referrer: string, - - created_at: Date -} - -const VisitSchema = new Schema({ - project_id: { type: Schema.Types.ObjectId, index: true }, - - browser: { type: String, required: true }, - os: { type: String, required: true }, - - continent: { type: String }, - country: { type: String }, - - session: { type: String, index: true }, - flowHash: { type: String }, - device: { type: String }, - - website: { type: String, required: true, index: true }, - page: { type: String, required: true }, - referrer: { type: String, required: true }, - created_at: { type: Date, default: () => Date.now() }, -}) - -VisitSchema.index({ project_id: 1, created_at: -1 }); - -export const VisitModel = model('visits', VisitSchema); - diff --git a/consumer/src/shared/schema/project/ProjectSchema.ts b/consumer/src/shared/schema/project/ProjectSchema.ts deleted file mode 100644 index 5470139..0000000 --- a/consumer/src/shared/schema/project/ProjectSchema.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { model, Schema, Types } from 'mongoose'; - -export type TProject = { - _id: Schema.Types.ObjectId, - owner: Schema.Types.ObjectId, - name: string, - premium: boolean, - premium_type: number, - customer_id: string, - subscription_id: string, - premium_expire_at: Date, - created_at: Date -} - -const ProjectSchema = new Schema({ - owner: { type: Types.ObjectId, index: 1 }, - name: { type: String, required: true }, - premium: { type: Boolean, default: false }, - premium_type: { type: Number, default: 0 }, - customer_id: { type: String, required: true }, - subscription_id: { type: String, required: true }, - premium_expire_at: { type: Date, required: true }, - created_at: { type: Date, default: () => Date.now() }, -}) - -export const ProjectModel = model('projects', ProjectSchema); diff --git a/consumer/src/shared/schema/project/ProjectsCounts.ts b/consumer/src/shared/schema/project/ProjectsCounts.ts deleted file mode 100644 index bf4124b..0000000 --- a/consumer/src/shared/schema/project/ProjectsCounts.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { model, Schema, Types } from 'mongoose'; - -export type TProjectCount = { - _id: Schema.Types.ObjectId, - project_id: Schema.Types.ObjectId, - events: number, - visits: number, - sessions: number, - lastRecheck?: Date, - updated_at: Date -} - -const ProjectCountSchema = new Schema({ - project_id: { type: Types.ObjectId, index: true, unique: true }, - events: { type: Number, required: true, default: 0 }, - visits: { type: Number, required: true, default: 0 }, - sessions: { type: Number, required: true, default: 0 }, - lastRecheck: { type: Date }, - updated_at: { type: Date } -}, { timestamps: { updatedAt: 'updated_at' } }); - -export const ProjectCountModel = model('project_counts', ProjectCountSchema); \ No newline at end of file diff --git a/consumer/src/shared/schema/project/ProjectsLimits.ts b/consumer/src/shared/schema/project/ProjectsLimits.ts deleted file mode 100644 index b619225..0000000 --- a/consumer/src/shared/schema/project/ProjectsLimits.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { model, Schema, Types } from 'mongoose'; - -export type TProjectLimit = { - _id: Schema.Types.ObjectId, - project_id: Schema.Types.ObjectId, - events: number, - visits: number, - ai_messages: number, - limit: number, - ai_limit: number, - billing_expire_at: Date, - billing_start_at: Date, -} - -const ProjectLimitSchema = new Schema({ - project_id: { type: Types.ObjectId, index: true, unique: true }, - events: { type: Number, required: true, default: 0 }, - visits: { type: Number, required: true, default: 0 }, - ai_messages: { type: Number, required: true, default: 0 }, - limit: { type: Number, required: true }, - ai_limit: { type: Number, required: true }, - billing_start_at: { type: Date, required: true }, - billing_expire_at: { type: Date, required: true }, -}); - -export const ProjectLimitModel = model('project_limits', ProjectLimitSchema); \ No newline at end of file diff --git a/consumer/src/shared/services/DatabaseService.ts b/consumer/src/shared/services/DatabaseService.ts deleted file mode 100644 index 74c73ed..0000000 --- a/consumer/src/shared/services/DatabaseService.ts +++ /dev/null @@ -1,10 +0,0 @@ - -import mongoose from "mongoose"; - -export async function connectDatabase(connectionString: string) { - await mongoose.connect(connectionString); -} - -export async function disconnectDatabase() { - await mongoose.disconnect(); -} diff --git a/consumer/src/shared/services/EmailService.ts b/consumer/src/shared/services/EmailService.ts deleted file mode 100644 index 30759f2..0000000 --- a/consumer/src/shared/services/EmailService.ts +++ /dev/null @@ -1,35 +0,0 @@ -const templateMap = { - confirm: '/confirm', - welcome: '/welcome', - purchase: '/purchase', - reset_password: '/reset_password', - anomaly_domain: '/anomaly/domain', - anomaly_visits_events: '/anomaly_visits_events', - limit_50: '/limit/50', - limit_90: '/limit/90', - limit_max: '/limit/max', -} as const; - -export type EmailTemplate = keyof typeof templateMap; -export type EmailServerInfo = { url: string, body: Record, headers: Record }; - -type EmailData = - | { template: 'confirm', data: { target: string, link: string } } - | { template: 'welcome', data: { target: string } } - | { template: 'purchase', data: { target: string, projectName: string } } - | { template: 'reset_password', data: { target: string, newPassword: string } } - | { template: 'anomaly_domain', data: { target: string, projectName: string, domains: string[] } } - | { template: 'anomaly_visits_events', data: { target: string, projectName: string, data: any[] } } - | { template: 'limit_50', data: { target: string, projectName: string } } - | { template: 'limit_90', data: { target: string, projectName: string } } - | { template: 'limit_max', data: { target: string, projectName: string } }; - -export class EmailService { - static getEmailServerInfo(template: T, data: Extract['data']): EmailServerInfo { - return { - url: `https://mail-service.litlyx.com/send${templateMap[template]}`, - body: data, - headers: { 'Content-Type': 'application/json' } - }; - } -} \ No newline at end of file diff --git a/consumer/src/shared/services/RedisStreamService.ts b/consumer/src/shared/services/RedisStreamService.ts deleted file mode 100644 index a3e3928..0000000 --- a/consumer/src/shared/services/RedisStreamService.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { createClient } from 'redis'; -import { requireEnv } from '../utils/requireEnv'; - -export type ReadingLoopOptions = { - stream_name: string, - group_name: ConsumerGroup, - consumer_name: string -} - - -type xReadGroupMessage = { id: string, message: { [x: string]: string } } -type xReadGgroupResult = { name: string, messages: xReadGroupMessage[] }[] | null - -const consumerGroups = ['DATABASE'] as const; - -type ConsumerGroup = typeof consumerGroups[number]; - - -export class RedisStreamService { - - private static client = createClient({ - url: requireEnv("REDIS_URL"), - username: requireEnv("REDIS_USERNAME"), - password: requireEnv("REDIS_PASSWORD"), - database: process.env.DEV_MODE === 'true' ? 1 : 0 - }); - - static async connect() { - await this.client.connect(); - } - - - static async getQueueInfo(stream_name: string) { - try { - const size = await this.client.xLen(stream_name); - return size; - } catch (ex) { - console.error(ex); - return 0; - } - } - - static async readFromStream(stream_name: string, group_name: string, consumer_name: string, process_function: (content: Record) => Promise) { - - const result: xReadGgroupResult = await this.client.xReadGroup(group_name, consumer_name, [{ key: stream_name, id: '>' }], { COUNT: 5, BLOCK: 2000 }); - - if (!result) { - setTimeout(() => this.readFromStream(stream_name, group_name, consumer_name, process_function), 10); - return; - } - - for (const entry of result) { - for (const messageData of entry.messages) { - await process_function(messageData.message); - await this.client.xAck(stream_name, group_name, messageData.id); - await this.client.set(`ACK:${group_name}`, messageData.id); - } - } - - await this.trimStream(stream_name); - - setTimeout(() => this.readFromStream(stream_name, group_name, consumer_name, process_function), 10); - return; - - } - - private static async trimStream(stream_name: string) { - - let lastMessageAck = '0'; - - for (const consumerGroup of consumerGroups) { - const lastAck = await this.client.get(`ACK:${consumerGroup}`); - if (!lastAck) continue; - if (lastAck > lastMessageAck) lastMessageAck = lastAck; - } - - await this.client.xTrim(stream_name, 'MINID', lastMessageAck as any); - - } - - static async startReadingLoop(options: ReadingLoopOptions, processFunction: (content: Record) => Promise) { - - if (!consumerGroups.includes(options.group_name)) return console.error('GROUP NAME NOT ALLOWED'); - - console.log('Start reading loop') - - try { - await this.client.xGroupCreate(options.stream_name, options.group_name, '0', { MKSTREAM: true }); - } catch (ex) { - console.log('Group', options.group_name, 'already exist'); - } - - this.readFromStream(options.stream_name, options.group_name, options.consumer_name, processFunction); - } - - static async addToStream(streamName: string, data: Record) { - const result = await this.client.xAdd(streamName, "*", { ...data, timestamp: Date.now().toString() }); - return result; - } - -} diff --git a/consumer/src/shared/utils/requireEnv.ts b/consumer/src/shared/utils/requireEnv.ts deleted file mode 100644 index a6497e0..0000000 --- a/consumer/src/shared/utils/requireEnv.ts +++ /dev/null @@ -1,9 +0,0 @@ - -export function requireEnv(name: string, errorMessage?: string) { - if (!process.env[name]) { - console.error(errorMessage || `ENV variable ${name} is required`); - return process.exit(1); - } - console.log('requireEnv', name, process.env[name]); - return process.env[name] as string; -} \ No newline at end of file