Elizhium/litlyx
1
0
Fork 0
mirror of https://github.com/Litlyx/litlyx synced 2025-12-10 15:58:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

add new service + fix docs links

Browse Source
This commit is contained in:
Emily
2024-06-09 19:00:44 +02:00
parent 2720222048
commit 127be91506
25 changed files with 2241 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
broker/src/StreamLoopController.ts Normal file
View File

@@ -0,0 +1,127 @@
import { RedisStreamService } from '@services/RedisStreamService';
import { requireEnv } from '../../shared/utilts/requireEnv';
import { EventModel } from '@schema/metrics/EventSchema';
import { SessionModel } from '@schema/metrics/SessionSchema';
import { ProjectModel } from '@schema/ProjectSchema';
import { ProjectLimitModel } from '@schema/ProjectsLimits';
import { ProjectCountModel } from '@schema/ProjectsCounts';
import { EVENT_LOG_LIMIT_PERCENT } from '@data/broker/Limits';
import { checkLimitsForEmail } from './Controller';
import { lookup } from './lookup';
import { UAParser } from 'ua-parser-js';
import { VisitModel } from '@schema/metrics/VisitSchema';
export async function startStreamLoop() {
await RedisStreamService.connect();
await RedisStreamService.startReadingLoop({
streamName: requireEnv('STREAM_NAME'),
delay: { base: 100, empty: 5000 },
readBlock: 2500
}, processStreamEvent);
}
async function processStreamEvent(data: Record<string, string>) {
try {
const eventType = data._type;
if (!eventType) return;
const { pid, sessionHash } = data;
const project = await ProjectModel.exists({ _id: pid });
if (!project) return;
if (eventType === 'event') return await process_event(data, sessionHash);
if (eventType === 'keep_alive') return await process_keep_alive(data, sessionHash);
if (eventType === 'visit') return await process_visit(data, sessionHash);
} catch (ex: any) {
console.error('ERROR PROCESSING STREAM EVENT', ex.message);
}
}
async function process_visit(data: Record<string, string>, sessionHash: string) {
const { pid, ip, website, page, referrer, userAgent } = data;
const projectLimits = await ProjectLimitModel.findOne({ project_id: pid });
if (!projectLimits) return;
const TOTAL_COUNT = projectLimits.events + projectLimits.visits;
const COUNT_LIMIT = projectLimits.limit;
if ((TOTAL_COUNT * EVENT_LOG_LIMIT_PERCENT) > COUNT_LIMIT) return;
await checkLimitsForEmail(projectLimits);
let referrerParsed;
try {
referrerParsed = new URL(referrer);
} catch (ex) {
referrerParsed = { hostname: referrer };
}
const geoLocation = lookup(ip);
const userAgentParsed = UAParser(userAgent);
const visit = new VisitModel({
project_id: pid, website, page, referrer: referrerParsed.hostname,
browser: userAgentParsed.browser.name || 'NO_BROWSER',
os: userAgentParsed.os.name || 'NO_OS',
device: userAgentParsed.device.type,
continent: geoLocation[0],
country: geoLocation[1],
});
await visit.save();
await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'visits': 1 } }, { upsert: true });
await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { 'visits': 1 } });
}
async function process_keep_alive(data: Record<string, string>, sessionHash: string) {
const { pid, instant } = data;
if (instant == "true") {
await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
$inc: { duration: 0 },
updated_at: Date.now()
}, { upsert: true });
} else {
await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
$inc: { duration: 1 },
updated_at: Date.now()
}, { upsert: true });
}
}
async function process_event(data: Record<string, string>, sessionHash: string) {
const { name, metadata, pid } = data;
let metadataObject;
try {
if (metadata) metadataObject = JSON.parse(metadata);
} catch (ex) {
metadataObject = { error: 'Error parsing metadata' }
}
const event = new EventModel({ project_id: pid, name, metadata: metadataObject });
await event.save();
await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { 'events': 1 } }, { upsert: true });
await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { 'events': 1 } });
}

5
broker/src/index.ts
View File

@@ -3,6 +3,7 @@ import cors from 'cors';
import { requireEnv } from '../../shared/utilts/requireEnv';
import { connectDatabase } from '@services/DatabaseService';
import { startStreamLoop } from './StreamLoopController';
const app = express();
app.use(cors());
@@ -11,7 +12,7 @@ connectDatabase(requireEnv('MONGO_CONNECTION_STRING'));
import HealthRouter from './routes/HealthRouter';
app.use('/health', HealthRouter);
import V1Router from './routes/v1/Router';
app.use('/v1', V1Router);
app.listen(requireEnv('PORT'), () => console.log(`Listening on port ${requireEnv('PORT')}`));
startStreamLoop();

140
broker/src/routes/v1/Router.ts
View File

@@ -1,140 +0,0 @@
import { Router, json } from "express";
import { createSessionHash, getIPFromRequest } from "../../utils/Utils";
import { SessionModel } from "@schema/metrics/SessionSchema";
import { EVENT_LOG_LIMIT_PERCENT } from '@data/broker/Limits';
import { EventType } from '@data/broker/EventType';
import { lookup } from "../../lookup";
import { UAParser } from "ua-parser-js";
import { getDeviceFromScreenSize } from "../../ScreenSize";
import { VisitModel } from "@schema/metrics/VisitSchema";
import { EventModel } from "@schema/metrics/EventSchema";
import { ProjectCountModel } from "@schema/ProjectsCounts";
import { checkLimitsForEmail } from "../../Controller";
import { ProjectLimitModel } from "@schema/ProjectsLimits";
import { ProjectModel } from "@schema/ProjectSchema";
const router = Router();
const allowAnyType = () => true;
const jsonOptions = { limit: '10mb', type: allowAnyType }
router.post('/keep_alive', json(jsonOptions), async (req, res) => {
try {
const ip = getIPFromRequest(req);
const { pid, website, userAgent, instant } = req.body;
const sessionHash = createSessionHash(website, ip, userAgent);
if (instant == true) {
await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
$inc: { duration: 0 },
updated_at: Date.now()
}, { upsert: true });
} else {
await SessionModel.updateOne({ project_id: pid, session: sessionHash, }, {
$inc: { duration: 1 },
updated_at: Date.now()
}, { upsert: true });
}
return res.sendStatus(200);
} catch (ex) {
console.error(ex);
return res.status(500).json({ error: 'ERROR' });
}
});
router.post('/metrics/push', json(jsonOptions), async (req, res) => {
try {
const { pid } = req.body;
const projectExist = await ProjectModel.exists({ _id: pid });
if (!projectExist) return res.status(400).json({ error: 'Project not exist' });
const projectLimits = await ProjectLimitModel.findOne({ project_id: pid });
if (!projectLimits) return res.status(400).json({ error: 'No limits found' });
const TOTAL_COUNT = projectLimits.events + projectLimits.visits;
const COUNT_LIMIT = projectLimits.limit;
if ((TOTAL_COUNT * EVENT_LOG_LIMIT_PERCENT) > COUNT_LIMIT) {
return res.status(200).json({ error: 'Limit reached' });
};
await checkLimitsForEmail(projectLimits);
const ip = getIPFromRequest(req);
const { type } = req.body;
if (type === null || type === undefined) return res.status(400).json({ error: 'type is required' });
if (typeof type !== 'number') return res.status(400).json({ error: 'type must be a number' });
if (type < 0) return res.status(400).json({ error: 'type must be positive' });
if (type === EventType.VISIT) {
const { website, page, referrer, screenWidth, screenHeight, userAgent } = req.body;
let referrerParsed;
try {
referrerParsed = new URL(referrer);
} catch (ex) {
referrerParsed = { hostname: referrer };
}
const geoLocation = lookup(ip);
const userAgentParsed = UAParser(userAgent);
const device = getDeviceFromScreenSize(screenWidth, screenHeight);
const visit = new VisitModel({
project_id: pid, website, page, referrer: referrerParsed.hostname,
browser: userAgentParsed.browser.name || 'NO_BROWSER',
os: userAgentParsed.os.name || 'NO_OS',
device,
continent: geoLocation[0],
country: geoLocation[1],
});
await visit.save();
} else {
const { name, metadata } = req.body;
let metadataObject;
try {
if (metadata) metadataObject = JSON.parse(metadata);
} catch (ex) {
metadataObject = { error: 'Error parsing metadata' }
}
const event = new EventModel({ project_id: pid, name, metadata: metadataObject });
await event.save();
}
const fieldToInc = type === EventType.VISIT ? 'visits' : 'events';
await ProjectCountModel.updateOne({ project_id: pid }, { $inc: { [fieldToInc]: 1 } }, { upsert: true });
await ProjectLimitModel.updateOne({ project_id: pid }, { $inc: { [fieldToInc]: 1 } });
return res.sendStatus(200);
} catch (ex) {
console.error(ex);
return res.status(500).json({ error: 'ERROR' });
}
});
export default router;