update shields

dashboard/components/dialog/shields/AddAddress.vue

@@ -0,0 +1,80 @@
+<script lang="ts" setup>
+
+const emit = defineEmits(['success', 'cancel']);
+
+const address = ref<string>('');
+const description = ref<string>('');
+
+
+const { data: currentIP } = useFetch<any>('https://api.ipify.org/?format=json');
+
+
+const canAddAddress = computed(() => {
+    return address.value.trim().length > 0;
+})
+
+async function addAddress() {
+    if (!canAddAddress.value) return;
+    try {
+        const res = await $fetch('/api/shields/ip/add', {
+            method: 'POST',
+            headers: useComputedHeaders({}).value,
+            body: JSON.stringify({ address: address.value, description: description.value })
+        });
+        address.value = '';
+        emit('success');
+    } catch (ex: any) {
+        alert(ex.message);
+        emit('cancel');
+    }
+}
+
+</script>
+
+<template>
+    <UModal :ui="{
+        strategy: 'override',
+        overlay: {
+            background: 'bg-lyx-background/85'
+        },
+        background: 'dark:bg-lyx-widget bg-lyx-lightmode-widget-light',
+        ring: 'border-solid border-[1px] border-[#262626]'
+    }">
+        <div class="h-full flex flex-col gap-2 p-4">
+
+            <div class="flex flex-col gap-3">
+
+                <div class="font-semibold text-[1.1rem]"> Add IP to Block List </div>
+
+                <div class="flex flex-col gap-2 dark:text-lyx-text-dark text-lyx-lightmode-text-dark">
+                    <div> Your current IP address is: {{ currentIP?.ip || '...' }} </div>
+                    <div> Copy and Paste your IP address in the box below or enter a custom address </div>
+                </div>           
+
+                <div class="flex flex-col gap-2">
+                    <div class="font-medium"> IP Address </div>
+                    <LyxUiInput class="px-2 py-1" v-model="address" placeholder="127.0.0.1"></LyxUiInput>
+                </div>
+
+                <div class="flex flex-col gap-2">
+                    <div class="font-medium"> Description (optional) </div>
+                    <LyxUiInput class="px-2 py-1" v-model="description" placeholder="e.g. localhost or office">
+                    </LyxUiInput>
+                </div>
+
+                <div class="flex flex-col gap-2 dark:text-lyx-text-dark text-lyx-lightmode-text-dark">
+                    <div> Once added, we will start rejecting traffic from this IP within a few minutes.</div>
+                </div>
+
+                <div class="flex">
+                    <LyxUiButton class="w-full text-center" :disabled="!canAddAddress" @click="addAddress()"
+                        type="primary">
+                        Add IP Address
+                    </LyxUiButton>
+                </div>
+            </div>
+
+        </div>
+    </UModal>
+
+</template>

dashboard/components/dialog/shields/AddDomain.vue

@@ -50,7 +50,7 @@ async function addDomain() {
                             subdomains.
                         </div>
                     </div>
-                    <div> NB: Once added, we will start rejecting traffic from non-matching hostnames within a few
+                    <div> NB: Once added, we will start allowing traffic only from matching hostnames within a few
                         minutes.</div>
                 </div>
 

dashboard/components/dialog/shields/DeleteAddress.vue

@@ -0,0 +1,56 @@
+<script lang="ts" setup>
+
+const emit = defineEmits(['success', 'cancel']);
+
+const props = defineProps<{ address: string }>();
+
+async function deleteAddress() {
+    if (!props.address) return;
+    try {
+        const res = await $fetch('/api/shields/ip/delete', {
+            method: 'DELETE',
+            headers: useComputedHeaders({}).value,
+            body: JSON.stringify({ address: props.address })
+        });
+        emit('success');
+    } catch (ex: any) {
+        alert(ex.message);
+        emit('cancel');
+    }
+}
+
+</script>
+
+<template>
+    <UModal :ui="{
+        strategy: 'override',
+        overlay: {
+            background: 'bg-lyx-background/85'
+        },
+        background: 'dark:bg-lyx-widget bg-lyx-lightmode-widget-light',
+        ring: 'border-solid border-[1px] border-[#262626]'
+    }">
+        <div class="h-full flex flex-col gap-2 p-4">
+
+            <div class="flex flex-col gap-3">
+
+                <div class="font-semibold text-[1.1rem]"> IP Address delete </div>
+
+                <div> Are you sure to delete the blacklisted IP Address
+                    <span class="font-semibold">{{ props.address }}</span>
+                </div>
+
+                <div class="flex justify-end gap-2">
+                    <LyxUiButton type="secondary" @click="emit('cancel')">
+                        Cancel
+                    </LyxUiButton>
+                    <LyxUiButton @click="deleteAddress()" type="danger">
+                        Delete
+                    </LyxUiButton>
+                </div>
+            </div>
+
+        </div>
+    </UModal>
+
+</template>

dashboard/components/shields/Addresses.vue

@@ -1,10 +1,8 @@
 <script lang="ts" setup>
-import { DialogShieldsAddDomain, DialogShieldsDeleteDomain } from '#components';
+import { DialogShieldsDeleteAddress, DialogShieldsAddAddress } from '#components';
 
 definePageMeta({ layout: 'dashboard' });
 
-const { project } = useProject();
-
 const { data: blackAddresses, refresh: refreshAddresses, pending: pendingAddresses } = useFetch('/api/shields/ip/list', {
     headers: useComputedHeaders({})
 });
@@ -12,16 +10,16 @@ const { data: blackAddresses, refresh: refreshAddresses, pending: pendingAddress
 const toast = useToast()
 const modal = useModal();
 
-function showAddDomainModal() {
-    modal.open(DialogShieldsAddDomain, {
+function showAddAddressModal() {
+    modal.open(DialogShieldsAddAddress, {
         onSuccess: () => {
             refreshAddresses();
             modal.close();
 
             toast.add({
-                id: 'shield_domain_add_success',
+                id: 'shield_address_add_success',
                 title: 'Success',
-                description: 'Whitelist updated with the new domain',
+                description: 'Blacklist updated with the new address',
                 timeout: 5000
             });
 
@@ -32,16 +30,16 @@ function showAddDomainModal() {
     })
 }
 
-function showDeleteDomainModal(domain: string) {
-    modal.open(DialogShieldsDeleteDomain, {
-        domain,
+function showDeleteAddressModal(address: string) {
+    modal.open(DialogShieldsDeleteAddress, {
+        address,
         onSuccess: () => {
             refreshAddresses();
             modal.close();
             toast.add({
-                id: 'shield_domain_remove_success',
+                id: 'shield_address_remove_success',
                 title: 'Deleted',
-                description: 'Whitelist domain deleted successfully',
+                description: 'Blacklist address deleted successfully',
                 timeout: 5000
             });
 
@@ -59,15 +57,15 @@ function showDeleteDomainModal(domain: string) {
     <div class="py-4 flex">
         <LyxUiCard class="w-full mx-2">
             <div>
-                <div class="text-[1.2rem] font-semibold"> IP Block Llist </div>
+                <div class="text-[1.2rem] font-semibold"> IP Block List </div>
                 <div class="dark:text-lyx-text-dark text-lyx-lightmode-text-dark">
-                   Reject incoming traffic from specific IP addresses
+                    Reject incoming traffic from specific IP addresses
                 </div>
             </div>
             <LyxUiSeparator class="my-3"></LyxUiSeparator>
 
             <div class="flex justify-end pb-3">
-                <LyxUiButton type="primary" @click="showAddDomainModal()"> Add Domain </LyxUiButton>
+                <LyxUiButton type="primary" @click="showAddAddressModal()"> Add IP Address </LyxUiButton>
             </div>
 
             <div class="flex justify-center pb-8 text-[1.2rem]" v-if="pendingAddresses">
@@ -86,12 +84,14 @@ function showDeleteDomainModal(domain: string) {
 
             <div v-if="!pendingAddresses && blackAddresses && blackAddresses.length > 0"
                 class="grid grid-cols-[auto_auto_auto_auto] px-10">
-                <div class="col-span-3">Domain</div>
-                <div>Actions</div>
+                <div> Domain </div>
+                <div class="col-span-2"> Description </div>
+                <div> Actions </div>
                 <LyxUiSeparator class="col-span-4 my-3"></LyxUiSeparator>
-                <template v-for="domain of blackAddresses">
-                    <div class="col-span-3 mb-3">{{ domain }}</div>
-                    <div> <i @click="showDeleteDomainModal(domain)"
+                <template v-for="entry of blackAddresses">
+                    <div class="mb-2"> {{ entry.address }} </div>
+                    <div class="col-span-2">{{ entry.description || 'No description' }}</div>
+                    <div> <i @click="showDeleteAddressModal(entry.address)"
                             class="far fa-trash cursor-pointer hover:text-lyx-text-dark"></i> </div>
                 </template>
             </div>

dashboard/components/shields/Bots.vue

@@ -0,0 +1,47 @@
+<script lang="ts" setup>
+
+definePageMeta({ layout: 'dashboard' });
+
+const { data: botOptions, refresh: refreshBotOptions, pending: pendingBotOptions } = useFetch('/api/shields/bots/options', {
+    headers: useComputedHeaders({})
+});
+
+async function onChange(newValue: boolean) {
+    await $fetch('/api/shields/bots/update_options', {
+        method: 'POST',
+        headers: useComputedHeaders({ custom: { 'Content-Type': 'application/json' } }).value,
+        body: JSON.stringify({ block: newValue })
+    })
+    await refreshBotOptions();
+}
+
+</script>
+
+<template>
+
+    <div class="py-4 flex">
+        <LyxUiCard class="w-full mx-2">
+            <div>
+                <div class="text-[1.2rem] font-semibold"> Block bot traffic </div>
+                <div class="dark:text-lyx-text-dark text-lyx-lightmode-text-dark">
+                    Automatically block unwanted bot and crawler traffic to protect your site from spam, scrapers, and
+                    unnecessary server load.
+                </div>
+            </div>
+
+            <LyxUiSeparator class="my-3"></LyxUiSeparator>
+
+            <div class="flex justify-center pb-8 text-[1.2rem]" v-if="pendingBotOptions">
+                <i class="fas fa-loader animate-spin"></i>
+            </div>
+
+            <div v-if="!pendingBotOptions && botOptions">
+                <div class="flex gap-2">
+                    <UToggle :modelValue="botOptions.block" @change="onChange"></UToggle>
+                    <div> Enable bot protection </div>
+                </div>
+            </div>
+
+        </LyxUiCard>
+    </div>
+</template>

dashboard/components/shields/Domains.vue

@@ -3,8 +3,6 @@ import { DialogShieldsAddDomain, DialogShieldsDeleteDomain } from '#components';
 
 definePageMeta({ layout: 'dashboard' });
 
-const { project } = useProject();
-
 const { data: allowedDomains, refresh: refreshDomains, pending: pendingDomains } = useFetch('/api/shields/domains/list', {
     headers: useComputedHeaders({})
 });
@@ -61,7 +59,7 @@ function showDeleteDomainModal(domain: string) {
             <div>
                 <div class="text-[1.2rem] font-semibold"> Domains allow list </div>
                 <div class="dark:text-lyx-text-dark text-lyx-lightmode-text-dark">
-                    Accept incoming traffic only from familiar domains
+                    Accept incoming traffic only from familiar domains.
                 </div>
             </div>
             <LyxUiSeparator class="my-3"></LyxUiSeparator>

dashboard/pages/shields.vue

@@ -7,8 +7,9 @@ const selfhosted = useSelfhosted();
 const items = [
     { label: 'Domains', slot: 'domains', tab: 'domains' },
     { label: 'IP Addresses', slot: 'ipaddresses', tab: 'ipaddresses' },
-    { label: 'Countries', slot: 'countries', tab: 'countries' },
-    { label: 'Pages', slot: 'pages', tab: 'pages' },
+    { label: 'Bot traffic', slot: 'bots', tab: 'bots' },
+    // { label: 'Countries', slot: 'countries', tab: 'countries' },
+    // { label: 'Pages', slot: 'pages', tab: 'pages' },
 ]
 
 </script>
@@ -23,14 +24,10 @@ const items = [
                 <ShieldsDomains></ShieldsDomains>
             </template>
             <template #ipaddresses>
-                <div class="flex items-center justify-center py-20 text-[1.2rem]"> Coming soon </div>
-                <!-- <ShieldsAddresses></ShieldsAddresses> -->
+                <ShieldsAddresses></ShieldsAddresses>
             </template>
-            <template #countries>
-                <div class="flex items-center justify-center py-20 text-[1.2rem]"> Coming soon </div>
-            </template>
-            <template #pages>
-                <div class="flex items-center justify-center py-20 text-[1.2rem]"> Coming soon </div>
+            <template #bots>
+                <ShieldsBots></ShieldsBots>
             </template>
         </CustomTab>
 

dashboard/server/api/shields/bots/options.ts

@@ -0,0 +1,9 @@
+import { BotTrafficOptionModel } from "~/shared/schema/shields/BotTrafficOptionSchema";
+
+export default defineEventHandler(async event => {
+    const data = await getRequestData(event, [], ['OWNER']);
+    if (!data) return;
+    const result = await BotTrafficOptionModel.findOne({ project_id: data.project_id });
+    if (!result) return { block: false };
+    return { block: result.block }
+});

dashboard/server/api/shields/bots/update_options.post.ts

@@ -0,0 +1,14 @@
+import { BotTrafficOptionModel } from "~/shared/schema/shields/BotTrafficOptionSchema";
+
+export default defineEventHandler(async event => {
+    const data = await getRequestData(event, [], ['OWNER']);
+    if (!data) return;
+    const body = await readBody(event);
+    const { block } = body;
+
+    if (block != true && block != false)
+        return setResponseStatus(event, 400, 'block is required and must be true or false');
+
+    const result = await BotTrafficOptionModel.updateOne({ project_id: data.project_id }, { block }, { upsert: true });
+    return { ok: result.acknowledged };
+});

dashboard/server/api/shields/countries/add.post.ts

@@ -0,0 +1,11 @@
+import { CountryBlacklistModel } from "~/shared/schema/shields/CountryBlacklistSchema";
+
+export default defineEventHandler(async event => {
+    const data = await getRequestData(event, [], ['OWNER']);
+    if (!data) return;
+    const body = await readBody(event);
+    const { country, description } = body;
+    if (country.trim().length == 0) return setResponseStatus(event, 400, 'Country is required');
+    const result = await CountryBlacklistModel.updateOne({ project_id: data.project_id, country }, { description }, { upsert: true });
+    return { ok: result.acknowledged };
+});

dashboard/server/api/shields/countries/delete.delete.ts

@@ -0,0 +1,14 @@
+import { CountryBlacklistModel } from "~/shared/schema/shields/CountryBlacklistSchema";
+
+export default defineEventHandler(async event => {
+
+    const data = await getRequestData(event, [], ['OWNER']);
+    if (!data) return;
+
+    const body = await readBody(event);
+    const { country } = body;
+
+    const removal = await CountryBlacklistModel.deleteOne({ project_id: data.project_id, country });
+
+    return { ok: removal.deletedCount == 1 };
+});

dashboard/server/api/shields/countries/list.ts

@@ -0,0 +1,8 @@
+import { CountryBlacklistModel } from "~/shared/schema/shields/CountryBlacklistSchema";
+
+export default defineEventHandler(async event => {
+    const data = await getRequestData(event, [], ['OWNER']);
+    if (!data) return;
+    const blacklist = await CountryBlacklistModel.find({ project_id: data.project_id });
+    return blacklist.map(e => e.toJSON());
+});

producer/src/controller.ts

@@ -1,13 +1,637 @@
 import { DomainWhitelistModel } from "./shared/schema/shields/DomainWhitelistSchema";
+import { AddressBlacklistModel } from "./shared/schema/shields/AddressBlacklistSchema";
+import { BotTrafficOptionModel } from "./shared/schema/shields/BotTrafficOptionSchema";
+
+
+const BOT_PATTERNS = [
+    "Googlebot\\/",
+    "Googlebot-Mobile",
+    "Googlebot-Image",
+    "Googlebot-News",
+    "Googlebot-Video",
+    "AdsBot-Google([^-]|$)",
+    "AdsBot-Google-Mobile",
+    "Feedfetcher-Google",
+    "Mediapartners-Google",
+    "Mediapartners \\(Googlebot\\)",
+    "APIs-Google",
+    "Google-InspectionTool",
+    "Storebot-Google",
+    "GoogleOther",
+    "bingbot",
+    "Slurp",
+    "[wW]get",
+    "LinkedInBot",
+    "Python-urllib",
+    "python-requests",
+    "aiohttp",
+    "httpx",
+    "libwww-perl",
+    "httpunit",
+    "Nutch",
+    "Go-http-client",
+    "phpcrawl",
+    "msnbot",
+    "jyxobot",
+    "FAST-WebCrawler",
+    "FAST Enterprise Crawler",
+    "BIGLOTRON",
+    "Teoma",
+    "convera",
+    "seekbot",
+    "Gigabot",
+    "Gigablast",
+    "exabot",
+    "ia_archiver",
+    "GingerCrawler",
+    "webmon ",
+    "HTTrack",
+    "grub\\.org",
+    "UsineNouvelleCrawler",
+    "antibot",
+    "netresearchserver",
+    "speedy",
+    "fluffy",
+    "findlink",
+    "msrbot",
+    "panscient",
+    "yacybot",
+    "AISearchBot",
+    "ips-agent",
+    "tagoobot",
+    "MJ12bot",
+    "woriobot",
+    "yanga",
+    "buzzbot",
+    "mlbot",
+    "yandex\\.com\\/bots",
+    "purebot",
+    "Linguee Bot",
+    "CyberPatrol",
+    "voilabot",
+    "Baiduspider",
+    "citeseerxbot",
+    "spbot",
+    "twengabot",
+    "postrank",
+    "Turnitin",
+    "scribdbot",
+    "page2rss",
+    "sitebot",
+    "linkdex",
+    "Adidxbot",
+    "ezooms",
+    "dotbot",
+    "Mail\\.RU_Bot",
+    "discobot",
+    "heritrix",
+    "findthatfile",
+    "europarchive\\.org",
+    "NerdByNature\\.Bot",
+    "(sistrix|SISTRIX) [cC]rawler",
+    "Ahrefs(Bot|SiteAudit)",
+    "fuelbot",
+    "CrunchBot",
+    "IndeedBot",
+    "mappydata",
+    "woobot",
+    "ZoominfoBot",
+    "PrivacyAwareBot",
+    "Multiviewbot",
+    "SWIMGBot",
+    "Grobbot",
+    "eright",
+    "Apercite",
+    "semanticbot",
+    "Aboundex",
+    "domaincrawler",
+    "wbsearchbot",
+    "summify",
+    "CCBot",
+    "edisterbot",
+    "SeznamBot",
+    "ec2linkfinder",
+    "gslfbot",
+    "aiHitBot",
+    "intelium_bot",
+    "facebookexternalhit",
+    "Yeti",
+    "RetrevoPageAnalyzer",
+    "lb-spider",
+    "Sogou",
+    "lssbot",
+    "careerbot",
+    "wotbox",
+    "wocbot",
+    "ichiro",
+    "DuckDuckBot",
+    "lssrocketcrawler",
+    "drupact",
+    "webcompanycrawler",
+    "acoonbot",
+    "openindexspider",
+    "gnam gnam spider",
+    "web-archive-net\\.com\\.bot",
+    "backlinkcrawler",
+    "coccoc",
+    "integromedb",
+    "content crawler spider",
+    "toplistbot",
+    "it2media-domain-crawler",
+    "ip-web-crawler\\.com",
+    "siteexplorer\\.info",
+    "elisabot",
+    "proximic",
+    "changedetection",
+    "arabot",
+    "WeSEE:Search",
+    "niki-bot",
+    "CrystalSemanticsBot",
+    "rogerbot",
+    "360Spider",
+    "psbot",
+    "InterfaxScanBot",
+    "CC Metadata Scaper",
+    "g00g1e\\.net",
+    "GrapeshotCrawler",
+    "urlappendbot",
+    "brainobot",
+    "fr-crawler",
+    "binlar",
+    "SimpleCrawler",
+    "Twitterbot",
+    "cXensebot",
+    "smtbot",
+    "bnf\\.fr_bot",
+    "A6-Indexer",
+    "ADmantX",
+    "Facebot",
+    "OrangeBot\\/",
+    "memorybot",
+    "AdvBot",
+    "MegaIndex",
+    "SemanticScholarBot",
+    "ltx71",
+    "nerdybot",
+    "xovibot",
+    "BUbiNG",
+    "Qwantify",
+    "archive\\.org_bot",
+    "Applebot",
+    "TweetmemeBot",
+    "crawler4j",
+    "findxbot",
+    "S[eE][mM]rushBot",
+    "yoozBot",
+    "lipperhey",
+    "Y!J",
+    "Domain Re-Animator Bot",
+    "AddThis",
+    "Screaming Frog SEO Spider",
+    "MetaURI",
+    "Scrapy",
+    "Livelap[bB]ot",
+    "OpenHoseBot",
+    "CapsuleChecker",
+    "collection@infegy\\.com",
+    "IstellaBot",
+    "DeuSu\\/",
+    "betaBot",
+    "Cliqzbot\\/",
+    "MojeekBot\\/",
+    "netEstate NE Crawler",
+    "SafeSearch microdata crawler",
+    "Gluten Free Crawler\\/",
+    "Sonic",
+    "Sysomos",
+    "Trove",
+    "deadlinkchecker",
+    "Slack-ImgProxy",
+    "Embedly",
+    "RankActiveLinkBot",
+    "iskanie",
+    "SafeDNSBot",
+    "SkypeUriPreview",
+    "Veoozbot",
+    "Slackbot",
+    "redditbot",
+    "datagnionbot",
+    "Google-Adwords-Instant",
+    "adbeat_bot",
+    "WhatsApp",
+    "contxbot",
+    "pinterest\\.com\\/bot",
+    "electricmonk",
+    "GarlikCrawler",
+    "BingPreview\\/",
+    "vebidoobot",
+    "FemtosearchBot",
+    "Yahoo Link Preview",
+    "MetaJobBot",
+    "DomainStatsBot",
+    "mindUpBot",
+    "Daum\\/",
+    "Jugendschutzprogramm-Crawler",
+    "Xenu Link Sleuth",
+    "Pcore-HTTP",
+    "moatbot",
+    "KosmioBot",
+    "[pP]ingdom",
+    "AppInsights",
+    "PhantomJS",
+    "Gowikibot",
+    "PiplBot",
+    "Discordbot",
+    "TelegramBot",
+    "Jetslide",
+    "newsharecounts",
+    "James BOT",
+    "Bark[rR]owler",
+    "TinEye",
+    "SocialRankIOBot",
+    "trendictionbot",
+    "Ocarinabot",
+    "epicbot",
+    "Primalbot",
+    "DuckDuckGo-Favicons-Bot",
+    "GnowitNewsbot",
+    "Leikibot",
+    "LinkArchiver",
+    "YaK\\/",
+    "PaperLiBot",
+    "Digg Deeper",
+    "dcrawl",
+    "Snacktory",
+    "AndersPinkBot",
+    "Fyrebot",
+    "EveryoneSocialBot",
+    "Mediatoolkitbot",
+    "Luminator-robots",
+    "ExtLinksBot",
+    "SurveyBot",
+    "NING\\/",
+    "okhttp",
+    "Nuzzel",
+    "omgili",
+    "PocketParser",
+    "YisouSpider",
+    "um-LN",
+    "ToutiaoSpider",
+    "MuckRack",
+    "Jamie's Spider",
+    "AHC\\/",
+    "NetcraftSurveyAgent",
+    "Laserlikebot",
+    "^Apache-HttpClient",
+    "AppEngine-Google",
+    "Jetty",
+    "Upflow",
+    "Thinklab",
+    "Traackr\\.com",
+    "Twurly",
+    "Mastodon",
+    "http_get",
+    "DnyzBot",
+    "botify",
+    "007ac9 Crawler",
+    "BehloolBot",
+    "BrandVerity",
+    "check_http",
+    "BDCbot",
+    "ZumBot",
+    "EZID",
+    "ICC-Crawler",
+    "ArchiveBot",
+    "^LCC ",
+    "filterdb\\.iss\\.net\\/crawler",
+    "BLP_bbot",
+    "BomboraBot",
+    "Buck\\/",
+    "Companybook-Crawler",
+    "Genieo",
+    "magpie-crawler",
+    "MeltwaterNews",
+    "Moreover",
+    "newspaper\\/",
+    "ScoutJet",
+    "(^| )sentry\\/",
+    "StorygizeBot",
+    "UptimeRobot",
+    "OutclicksBot",
+    "seoscanners",
+    "Hatena",
+    "Google Web Preview",
+    "MauiBot",
+    "AlphaBot",
+    "SBL-BOT",
+    "IAS crawler",
+    "adscanner",
+    "Netvibes",
+    "acapbot",
+    "Baidu-YunGuanCe",
+    "bitlybot",
+    "blogmuraBot",
+    "Bot\\.AraTurka\\.com",
+    "bot-pge\\.chlooe\\.com",
+    "BoxcarBot",
+    "BTWebClient",
+    "ContextAd Bot",
+    "Digincore bot",
+    "Disqus",
+    "Feedly",
+    "Fetch\\/",
+    "Fever",
+    "Flamingo_SearchEngine",
+    "FlipboardProxy",
+    "g2reader-bot",
+    "G2 Web Services",
+    "imrbot",
+    "K7MLWCBot",
+    "Kemvibot",
+    "Landau-Media-Spider",
+    "linkapediabot",
+    "vkShare",
+    "Siteimprove\\.com",
+    "BLEXBot\\/",
+    "DareBoost",
+    "ZuperlistBot\\/",
+    "Miniflux\\/",
+    "Feedspot",
+    "Diffbot\\/",
+    "SEOkicks",
+    "tracemyfile",
+    "Nimbostratus-Bot",
+    "zgrab",
+    "PR-CY\\.RU",
+    "AdsTxtCrawler",
+    "Datafeedwatch",
+    "Zabbix",
+    "TangibleeBot",
+    "google-xrawler",
+    "axios",
+    "Amazon CloudFront",
+    "Pulsepoint",
+    "CloudFlare-AlwaysOnline",
+    "Cloudflare-Healthchecks",
+    "Cloudflare-Traffic-Manager",
+    "CloudFlare-Prefetch",
+    "Cloudflare-SSLDetector",
+    "https:\\/\\/developers\\.cloudflare\\.com\\/security-center\\/",
+    "Google-Structured-Data-Testing-Tool",
+    "WordupInfoSearch",
+    "WebDataStats",
+    "HttpUrlConnection",
+    "ZoomBot",
+    "VelenPublicWebCrawler",
+    "MoodleBot",
+    "jpg-newsbot",
+    "outbrain",
+    "W3C_Validator",
+    "Validator\\.nu",
+    "W3C-checklink",
+    "W3C-mobileOK",
+    "W3C_I18n-Checker",
+    "FeedValidator",
+    "W3C_CSS_Validator",
+    "W3C_Unicorn",
+    "Google-PhysicalWeb",
+    "Blackboard",
+    "ICBot\\/",
+    "BazQux",
+    "Twingly",
+    "Rivva",
+    "Experibot",
+    "awesomecrawler",
+    "Dataprovider\\.com",
+    "GroupHigh\\/",
+    "theoldreader\\.com",
+    "AnyEvent",
+    "Uptimebot\\.org",
+    "Nmap Scripting Engine",
+    "2ip\\.ru",
+    "Clickagy",
+    "Caliperbot",
+    "MBCrawler",
+    "online-webceo-bot",
+    "B2B Bot",
+    "AddSearchBot",
+    "Google Favicon",
+    "HubSpot",
+    "Chrome-Lighthouse",
+    "HeadlessChrome",
+    "CheckMarkNetwork\\/",
+    "www\\.uptime\\.com",
+    "Streamline3Bot\\/",
+    "serpstatbot\\/",
+    "MixnodeCache\\/",
+    "^curl",
+    "SimpleScraper",
+    "RSSingBot",
+    "Jooblebot",
+    "fedoraplanet",
+    "Friendica",
+    "NextCloud",
+    "Tiny Tiny RSS",
+    "RegionStuttgartBot",
+    "Bytespider",
+    "Datanyze",
+    "Google-Site-Verification",
+    "TrendsmapResolver",
+    "tweetedtimes",
+    "NTENTbot",
+    "Gwene",
+    "SimplePie",
+    "SearchAtlas",
+    "Superfeedr",
+    "feedbot",
+    "UT-Dorkbot",
+    "Amazonbot",
+    "SerendeputyBot",
+    "Eyeotabot",
+    "officestorebot",
+    "Neticle Crawler",
+    "SurdotlyBot",
+    "LinkisBot",
+    "AwarioSmartBot",
+    "AwarioRssBot",
+    "RyteBot",
+    "FreeWebMonitoring SiteChecker",
+    "AspiegelBot",
+    "NAVER Blog Rssbot",
+    "zenback bot",
+    "SentiBot",
+    "Domains Project\\/",
+    "Pandalytics",
+    "VKRobot",
+    "bidswitchbot",
+    "tigerbot",
+    "NIXStatsbot",
+    "Atom Feed Robot",
+    "[Cc]urebot",
+    "PagePeeker\\/",
+    "Vigil\\/",
+    "rssbot\\/",
+    "startmebot\\/",
+    "JobboerseBot",
+    "seewithkids",
+    "NINJA bot",
+    "Cutbot",
+    "BublupBot",
+    "BrandONbot",
+    "RidderBot",
+    "Taboolabot",
+    "Dubbotbot",
+    "FindITAnswersbot",
+    "infoobot",
+    "Refindbot",
+    "BlogTraffic\\/\\d\\.\\d+ Feed-Fetcher",
+    "SeobilityBot",
+    "Cincraw",
+    "Dragonbot",
+    "VoluumDSP-content-bot",
+    "FreshRSS",
+    "BitBot",
+    "^PHP-Curl-Class",
+    "Google-Certificates-Bridge",
+    "centurybot",
+    "Viber",
+    "e\\.ventures Investment Crawler",
+    "evc-batch",
+    "PetalBot",
+    "virustotal",
+    "(^| )PTST\\/",
+    "minicrawler",
+    "Cookiebot",
+    "trovitBot",
+    "seostar\\.co",
+    "IonCrawl",
+    "Uptime-Kuma",
+    "Seekport",
+    "FreshpingBot",
+    "Feedbin",
+    "CriteoBot",
+    "Snap URL Preview Service",
+    "Better Uptime Bot",
+    "RuxitSynthetic",
+    "Google-Read-Aloud",
+    "Valve\\/Steam",
+    "OdklBot\\/",
+    "GPTBot",
+    "ChatGPT-User",
+    "OAI-SearchBot",
+    "YandexRenderResourcesBot\\/",
+    "LightspeedSystemsCrawler",
+    "ev-crawler\\/",
+    "BitSightBot\\/",
+    "woorankreview\\/",
+    "Google-Safety",
+    "AwarioBot",
+    "DataForSeoBot",
+    "Linespider",
+    "WellKnownBot",
+    "A Patent Crawler",
+    "StractBot",
+    "search\\.marginalia\\.nu",
+    "YouBot",
+    "Nicecrawler",
+    "Neevabot",
+    "BrightEdge Crawler",
+    "SiteCheckerBotCrawler",
+    "TombaPublicWebCrawler",
+    "CrawlyProjectCrawler",
+    "KomodiaBot",
+    "KStandBot",
+    "CISPA Webcrawler",
+    "MTRobot",
+    "hyscore\\.io",
+    "AlexandriaOrgBot",
+    "2ip bot",
+    "Yellowbrandprotectionbot",
+    "SEOlizer",
+    "vuhuvBot",
+    "INETDEX-BOT",
+    "Synapse",
+    "t3versionsBot",
+    "deepnoc",
+    "Cocolyzebot",
+    "hypestat",
+    "ReverseEngineeringBot",
+    "sempi\\.tech",
+    "Iframely",
+    "MetaInspector",
+    "node-fetch",
+    "l9explore",
+    "python-opengraph",
+    "OpenGraphCheck",
+    "developers\\.google\\.com\\/\\+\\/web\\/snippet",
+    "SenutoBot",
+    "MaCoCu",
+    "NewsBlur",
+    "inoreader",
+    "NetSystemsResearch",
+    "PageThing",
+    "WordPress\\/",
+    "PhxBot",
+    "ImagesiftBot",
+    "Expanse",
+    "InternetMeasurement",
+    "^BW\\/",
+    "GeedoBot",
+    "Audisto Crawler",
+    "PerplexityBot\\/",
+    "[cC]laude[bB]ot",
+    "Monsidobot",
+    "GroupMeBot",
+    "Vercelbot",
+    "vercel-screenshot",
+    "facebookcatalog\\/",
+    "meta-externalagent\\/",
+    "meta-externalfetcher\\/",
+    "AcademicBotRTU",
+    "KeybaseBot",
+    "Lemmy",
+    "CookieHubScan",
+    "Hydrozen\\.io",
+    "HTTP Banner Detection",
+    "SummalyBot",
+    "MicrosoftPreview\\/",
+    "GeedoProductSearch",
+    "TikTokSpider"
+]
+
+function isBot(userAgent: string) {
+    for (const pattern of BOT_PATTERNS) {
+        const regexp = new RegExp(pattern);
+        const result = userAgent.match(regexp);
+        if (result != null) return true;
+    }
+    return false;
+}
+
+export async function isAllowedToLog(project_id: string, website: string, ip: string, userAgent: string) {
+
+    const blacklistData = await AddressBlacklistModel.find({ project_id }, { address: 1 });
+    for (const blacklistedData of blacklistData) {
+        if (blacklistedData.address == ip) return false;
+    }
+
+    const botOptions = await BotTrafficOptionModel.findOne({ project_id }, { block: 1 });
+    if (botOptions && botOptions.block) {
+        const isbot = isBot(userAgent);
+        if (isbot) return false;
+    }
+
+    const whitelist = await DomainWhitelistModel.findOne({ project_id }, { domains: 1 });
+    if (!whitelist) return true;
+    if (!whitelist.domains) return true;
+    if (whitelist.domains.length == 0) return true;
 
-export async function isAllowedToLog(project_id: string, website: string) {
-    const whitelist = await DomainWhitelistModel.findOne({ project_id }, { website: 1 });
-    if (!whitelist) return;
     const allowedDomains = whitelist.domains;
     for (const allowedDomain of allowedDomains) {
         const regexpDomain = new RegExp(allowedDomain.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*'));
         const result = website.match(regexpDomain);
         if (result != null) return true;
     }
+
     return false;
 }

producer/src/deprecated.ts

@@ -16,8 +16,8 @@ router.post('/keep_alive', json(jsonOptions), async (req, res) => {
         const ip = getIPFromRequest(req);
         const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent);
 
-        const allowed = await isAllowedToLog(req.body.pid, req.body.website);
-        if (!allowed) return res.status(400);
+        const allowed = await isAllowedToLog(req.body.pid, req.body.website, ip, req.body.userAgent);
+        if (!allowed) return res.sendStatus(400);
 
         await RedisStreamService.addToStream(streamName, {
             ...req.body, _type: 'keep_alive', sessionHash, ip,
@@ -37,8 +37,8 @@ router.post('/metrics/push', json(jsonOptions), async (req, res) => {
         const ip = getIPFromRequest(req);
         const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent);
 
-        const allowed = await isAllowedToLog(req.body.pid, req.body.website);
-        if (!allowed) return res.status(400);
+        const allowed = await isAllowedToLog(req.body.pid, req.body.website, ip, req.body.userAgent);
+        if (!allowed) return res.sendStatus(400);
 
         const { type } = req.body;
 

producer/src/index.ts

@@ -27,8 +27,8 @@ app.post('/event', express.json(jsonOptions), async (req, res) => {
         const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent);
         const flowHash = createFlowSessionHash(req.body.pid, ip, req.body.userAgent);
 
-        const allowed = await isAllowedToLog(req.body.pid, req.body.website);
-        if (!allowed) return res.status(400);
+        const allowed = await isAllowedToLog(req.body.pid, req.body.website, ip, req.body.userAgent);
+        if (!allowed) return res.sendStatus(400);
 
         await RedisStreamService.addToStream(streamName, {
             ...req.body, _type: 'event', sessionHash, ip, flowHash,
@@ -54,8 +54,8 @@ app.post('/visit', express.json(jsonOptions), async (req, res) => {
         const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent);
         const flowHash = createFlowSessionHash(req.body.pid, ip, req.body.userAgent);
 
-        const allowed = await isAllowedToLog(req.body.pid, req.body.website);
-        if (!allowed) return res.status(400);
+        const allowed = await isAllowedToLog(req.body.pid, req.body.website, ip, req.body.userAgent);
+        if (!allowed) return res.sendStatus(400);
 
         await RedisStreamService.addToStream(streamName, { ...req.body, _type: 'visit', sessionHash, ip, flowHash, timestamp: Date.now() });
 
@@ -78,8 +78,8 @@ app.post('/keep_alive', express.json(jsonOptions), async (req, res) => {
         const sessionHash = createSessionHash(req.body.website, ip, req.body.userAgent);
         const flowHash = createFlowSessionHash(req.body.pid, ip, req.body.userAgent);
 
-        const allowed = await isAllowedToLog(req.body.pid, req.body.website);
-        if (!allowed) return res.status(400);
+        const allowed = await isAllowedToLog(req.body.pid, req.body.website, ip, req.body.userAgent);
+        if (!allowed) return res.sendStatus(400);
 
         await RedisStreamService.addToStream(streamName, {
             ...req.body, _type: 'keep_alive', sessionHash, ip,

scripts/producer/shared.ts

@@ -15,4 +15,8 @@ helper.copy('services/DatabaseService.ts');
 
 helper.create('schema');
 helper.create('schema/shields');
+helper.copy('schema/shields/AddressBlacklistSchema.ts');
+helper.copy('schema/shields/BotTrafficOptionSchema.ts');
+helper.copy('schema/shields/CountryBlacklistSchema.ts');
 helper.copy('schema/shields/DomainWhitelistSchema.ts');
+helper.copy('schema/shields/PageBlacklistSchema.ts');

shared_global/schema/shields/BotTrafficOptionSchema.ts

@@ -0,0 +1,16 @@
+import { model, Schema, Types } from 'mongoose';
+
+export type TBotTrafficOptionSchema = {
+    _id: Schema.Types.ObjectId,
+    project_id: Schema.Types.ObjectId,
+    block: boolean,
+    created_at: Date
+}
+
+const BotTrafficOptionSchema = new Schema<TBotTrafficOptionSchema>({
+    project_id: { type: Types.ObjectId, index: 1 },
+    block: { type: Boolean, required: true },
+    created_at: { type: Date, default: () => Date.now() },
+});
+
+export const BotTrafficOptionModel = model<TBotTrafficOptionSchema>('bot_traffic_options', BotTrafficOptionSchema);

shared_global/schema/shields/CountryBlacklistSchema.ts

@@ -0,0 +1,18 @@
+import { model, Schema, Types } from 'mongoose';
+
+export type TCountryBlacklistSchema = {
+    _id: Schema.Types.ObjectId,
+    project_id: Schema.Types.ObjectId,
+    country: string,
+    description: string,
+    created_at: Date
+}
+
+const CountryBlacklistSchema = new Schema<TCountryBlacklistSchema>({
+    project_id: { type: Types.ObjectId, index: 1 },
+    country: { type: String, required: true },
+    description: { type: String },
+    created_at: { type: Date, default: () => Date.now() },
+});
+
+export const CountryBlacklistModel = model<TCountryBlacklistSchema>('country_blacklists', CountryBlacklistSchema);

shared_global/schema/shields/PageBlacklistSchema.ts

@@ -0,0 +1,18 @@
+import { model, Schema, Types } from 'mongoose';
+
+export type TPageBlacklistSchema = {
+    _id: Schema.Types.ObjectId,
+    project_id: Schema.Types.ObjectId,
+    page: string,
+    description:string,
+    created_at: Date
+}
+
+const CountryBlacklistSchema = new Schema<TPageBlacklistSchema>({
+    project_id: { type: Types.ObjectId, index: 1 },
+    page: { type: String, required: true },
+    description: { type: String },
+    created_at: { type: Date, default: () => Date.now() },
+});
+
+export const CountryBlacklistModel = model<TPageBlacklistSchema>('country_blacklists', CountryBlacklistSchema);