"Googlebot\\/",
"Googlebot-Mobile",
"Googlebot-Image",
"Googlebot-News",
"Googlebot-Video",
"AdsBot-Google([^-]|$)",
"AdsBot-Google-Mobile",
"Feedfetcher-Google",
"Mediapartners-Google",
"Mediapartners \\(Googlebot\\)",
"APIs-Google",
"Google-InspectionTool",
"Storebot-Google",
"GoogleOther",
"bingbot",
"Slurp",
"[wW]get",
"LinkedInBot",
"Python-urllib",
"python-requests",
"aiohttp",
"httpx",
"libwww-perl",
"httpunit",
"Nutch",
"Go-http-client",
"phpcrawl",
"msnbot",
"jyxobot",
"FAST-WebCrawler",
"FAST Enterprise Crawler",
"BIGLOTRON",
"Teoma",
"convera",
"seekbot",
"Gigabot",
"Gigablast",
"exabot",
"ia_archiver",
"GingerCrawler",
"webmon ",
"HTTrack",
"grub\\.org",
"UsineNouvelleCrawler",
"antibot",
"netresearchserver",
"speedy",
"fluffy",
"findlink",
"msrbot",
"panscient",
"yacybot",
"AISearchBot",
"ips-agent",
"tagoobot",
"MJ12bot",
"woriobot",
"yanga",
"buzzbot",
"mlbot",
"yandex\\.com\\/bots",
"purebot",
"Linguee Bot",
"CyberPatrol",
"voilabot",
"Baiduspider",
"citeseerxbot",
"spbot",
"twengabot",
"postrank",
"Turnitin",
"scribdbot",
"page2rss",
"sitebot",
"linkdex",
"Adidxbot",
"ezooms",
"dotbot",
"Mail\\.RU_Bot",
"discobot",
"heritrix",
"findthatfile",
"europarchive\\.org",
"NerdByNature\\.Bot",
"(sistrix|SISTRIX) [cC]rawler",
"Ahrefs(Bot|SiteAudit)",
"fuelbot",
"CrunchBot",
"IndeedBot",
"mappydata",
"woobot",
"ZoominfoBot",
"PrivacyAwareBot",
"Multiviewbot",
"SWIMGBot",
"Grobbot",
"eright",
"Apercite",
"semanticbot",
"Aboundex",
"domaincrawler",
"wbsearchbot",
"summify",
"CCBot",
"edisterbot",
"SeznamBot",
"ec2linkfinder",
"gslfbot",
"aiHitBot",
"intelium_bot",
"facebookexternalhit",
"Yeti",
"RetrevoPageAnalyzer",
"lb-spider",
"Sogou",
"lssbot",
"careerbot",
"wotbox",
"wocbot",
"ichiro",
"DuckDuckBot",
"lssrocketcrawler",
"drupact",
"webcompanycrawler",
"acoonbot",
"openindexspider",
"gnam gnam spider",
"web-archive-net\\.com\\.bot",
"backlinkcrawler",
"coccoc",
"integromedb",
"content crawler spider",
"toplistbot",
"it2media-domain-crawler",
"ip-web-crawler\\.com",
"siteexplorer\\.info",
"elisabot",
"proximic",
"changedetection",
"arabot",
"WeSEE:Search",
"niki-bot",
"CrystalSemanticsBot",
"rogerbot",
"360Spider",
"psbot",
"InterfaxScanBot",
"CC Metadata Scaper",
"g00g1e\\.net",
"GrapeshotCrawler",
"urlappendbot",
"brainobot",
"fr-crawler",
"binlar",
"SimpleCrawler",
"Twitterbot",
"cXensebot",
"smtbot",
"bnf\\.fr_bot",
"A6-Indexer",
"ADmantX",
"Facebot",
"OrangeBot\\/",
"memorybot",
"AdvBot",
"MegaIndex",
"SemanticScholarBot",
"ltx71",
"nerdybot",
"xovibot",
"BUbiNG",
"Qwantify",
"archive\\.org_bot",
"Applebot",
"TweetmemeBot",
"crawler4j",
"findxbot",
"S[eE][mM]rushBot",
"yoozBot",
"lipperhey",
"Y!J",
"Domain Re-Animator Bot",
"AddThis",
"Screaming Frog SEO Spider",
"MetaURI",
"Scrapy",
"Livelap[bB]ot",
"OpenHoseBot",
"CapsuleChecker",
"collection@infegy\\.com",
"IstellaBot",
"DeuSu\\/",
"betaBot",
"Cliqzbot\\/",
"MojeekBot\\/",
"netEstate NE Crawler",
"SafeSearch microdata crawler",
"Gluten Free Crawler\\/",
"Sonic",
"Sysomos",
"Trove",
"deadlinkchecker",
"Slack-ImgProxy",
"Embedly",
"RankActiveLinkBot",
"iskanie",
"SafeDNSBot",
"SkypeUriPreview",
"Veoozbot",
"Slackbot",
"redditbot",
"datagnionbot",
"Google-Adwords-Instant",
"adbeat_bot",
"WhatsApp",
"contxbot",
"pinterest\\.com\\/bot",
"electricmonk",
"GarlikCrawler",
"BingPreview\\/",
"vebidoobot",
"FemtosearchBot",
"Yahoo Link Preview",
"MetaJobBot",
"DomainStatsBot",
"mindUpBot",
"Daum\\/",
"Jugendschutzprogramm-Crawler",
"Xenu Link Sleuth",
"Pcore-HTTP",
"moatbot",
"KosmioBot",
"[pP]ingdom",
"AppInsights",
"PhantomJS",
"Gowikibot",
"PiplBot",
"Discordbot",
"TelegramBot",
"Jetslide",
"newsharecounts",
"James BOT",
"Bark[rR]owler",
"TinEye",
"SocialRankIOBot",
"trendictionbot",
"Ocarinabot",
"epicbot",
"Primalbot",
"DuckDuckGo-Favicons-Bot",
"GnowitNewsbot",
"Leikibot",
"LinkArchiver",
"YaK\\/",
"PaperLiBot",
"Digg Deeper",
"dcrawl",
"Snacktory",
"AndersPinkBot",
"Fyrebot",
"EveryoneSocialBot",
"Mediatoolkitbot",
"Luminator-robots",
"ExtLinksBot",
"SurveyBot",
"NING\\/",
"okhttp",
"Nuzzel",
"omgili",
"PocketParser",
"YisouSpider",
"um-LN",
"ToutiaoSpider",
"MuckRack",
"Jamie's Spider",
"AHC\\/",
"NetcraftSurveyAgent",
"Laserlikebot",
"^Apache-HttpClient",
"AppEngine-Google",
"Jetty",
"Upflow",
"Thinklab",
"Traackr\\.com",
"Twurly",
"Mastodon",
"http_get",
"DnyzBot",
"botify",
"007ac9 Crawler",
"BehloolBot",
"BrandVerity",
"check_http",
"BDCbot",
"ZumBot",
"EZID",
"ICC-Crawler",
"ArchiveBot",
"^LCC ",
"filterdb\\.iss\\.net\\/crawler",
"BLP_bbot",
"BomboraBot",
"Buck\\/",
"Companybook-Crawler",
"Genieo",
"magpie-crawler",
"MeltwaterNews",
"Moreover",
"newspaper\\/",
"ScoutJet",
"(^| )sentry\\/",
"StorygizeBot",
"UptimeRobot",
"OutclicksBot",
"seoscanners",
"Hatena",
"Google Web Preview",
"MauiBot",
"AlphaBot",
"SBL-BOT",
"IAS crawler",
"adscanner",
"Netvibes",
"acapbot",
"Baidu-YunGuanCe",
"bitlybot",
"blogmuraBot",
"Bot\\.AraTurka\\.com",
"bot-pge\\.chlooe\\.com",
"BoxcarBot",
"BTWebClient",
"ContextAd Bot",
"Digincore bot",
"Disqus",
"Feedly",
"Fetch\\/",
"Fever",
"Flamingo_SearchEngine",
"FlipboardProxy",
"g2reader-bot",
"G2 Web Services",
"imrbot",
"K7MLWCBot",
"Kemvibot",
"Landau-Media-Spider",
"linkapediabot",
"vkShare",
"Siteimprove\\.com",
"BLEXBot\\/",
"DareBoost",
"ZuperlistBot\\/",
"Miniflux\\/",
"Feedspot",
"Diffbot\\/",
"SEOkicks",
"tracemyfile",
"Nimbostratus-Bot",
"zgrab",
"PR-CY\\.RU",
"AdsTxtCrawler",
"Datafeedwatch",
"Zabbix",
"TangibleeBot",
"google-xrawler",
"axios",
"Amazon CloudFront",
"Pulsepoint",
"CloudFlare-AlwaysOnline",
"Cloudflare-Healthchecks",
"Cloudflare-Traffic-Manager",
"CloudFlare-Prefetch",
"Cloudflare-SSLDetector",
"https:\\/\\/developers\\.cloudflare\\.com\\/security-center\\/",
"Google-Structured-Data-Testing-Tool",
"WordupInfoSearch",
"WebDataStats",
"HttpUrlConnection",
"ZoomBot",
"VelenPublicWebCrawler",
"MoodleBot",
"jpg-newsbot",
"outbrain",
"W3C_Validator",
"Validator\\.nu",
"W3C-checklink",
"W3C-mobileOK",
"W3C_I18n-Checker",
"FeedValidator",
"W3C_CSS_Validator",
"W3C_Unicorn",
"Google-PhysicalWeb",
"Blackboard",
"ICBot\\/",
"BazQux",
"Twingly",
"Rivva",
"Experibot",
"awesomecrawler",
"Dataprovider\\.com",
"GroupHigh\\/",
"theoldreader\\.com",
"AnyEvent",
"Uptimebot\\.org",
"Nmap Scripting Engine",
"2ip\\.ru",
"Clickagy",
"Caliperbot",
"MBCrawler",
"online-webceo-bot",
"B2B Bot",
"AddSearchBot",
"Google Favicon",
"HubSpot",
"Chrome-Lighthouse",
"HeadlessChrome",
"CheckMarkNetwork\\/",
"www\\.uptime\\.com",
"Streamline3Bot\\/",
"serpstatbot\\/",
"MixnodeCache\\/",
"^curl",
"SimpleScraper",
"RSSingBot",
"Jooblebot",
"fedoraplanet",
"Friendica",
"NextCloud",
"Tiny Tiny RSS",
"RegionStuttgartBot",
"Bytespider",
"Datanyze",
"Google-Site-Verification",
"TrendsmapResolver",
"tweetedtimes",
"NTENTbot",
"Gwene",
"SimplePie",
"SearchAtlas",
"Superfeedr",
"feedbot",
"UT-Dorkbot",
"Amazonbot",
"SerendeputyBot",
"Eyeotabot",
"officestorebot",
"Neticle Crawler",
"SurdotlyBot",
"LinkisBot",
"AwarioSmartBot",
"AwarioRssBot",
"RyteBot",
"FreeWebMonitoring SiteChecker",
"AspiegelBot",
"NAVER Blog Rssbot",
"zenback bot",
"SentiBot",
"Domains Project\\/",
"Pandalytics",
"VKRobot",
"bidswitchbot",
"tigerbot",
"NIXStatsbot",
"Atom Feed Robot",
"[Cc]urebot",
"PagePeeker\\/",
"Vigil\\/",
"rssbot\\/",
"startmebot\\/",
"JobboerseBot",
"seewithkids",
"NINJA bot",
"Cutbot",
"BublupBot",
"BrandONbot",
"RidderBot",
"Taboolabot",
"Dubbotbot",
"FindITAnswersbot",
"infoobot",
"Refindbot",
"BlogTraffic\\/\\d\\.\\d+ Feed-Fetcher",
"SeobilityBot",
"Cincraw",
"Dragonbot",
"VoluumDSP-content-bot",
"FreshRSS",
"BitBot",
"^PHP-Curl-Class",
"Google-Certificates-Bridge",
"centurybot",
"Viber",
"e\\.ventures Investment Crawler",
"evc-batch",
"PetalBot",
"virustotal",
"(^| )PTST\\/",
"minicrawler",
"Cookiebot",
"trovitBot",
"seostar\\.co",
"IonCrawl",
"Uptime-Kuma",
"Seekport",
"FreshpingBot",
"Feedbin",
"CriteoBot",
"Snap URL Preview Service",
"Better Uptime Bot",
"RuxitSynthetic",
"Google-Read-Aloud",
"Valve\\/Steam",
"OdklBot\\/",
"GPTBot",
"ChatGPT-User",
"OAI-SearchBot",
"YandexRenderResourcesBot\\/",
"LightspeedSystemsCrawler",
"ev-crawler\\/",
"BitSightBot\\/",
"woorankreview\\/",
"Google-Safety",
"AwarioBot",
"DataForSeoBot",
"Linespider",
"WellKnownBot",
"A Patent Crawler",
"StractBot",
"search\\.marginalia\\.nu",
"YouBot",
"Nicecrawler",
"Neevabot",
"BrightEdge Crawler",
"SiteCheckerBotCrawler",
"TombaPublicWebCrawler",
"CrawlyProjectCrawler",
"KomodiaBot",
"KStandBot",
"CISPA Webcrawler",
"MTRobot",
"hyscore\\.io",
"AlexandriaOrgBot",
"2ip bot",
"Yellowbrandprotectionbot",
"SEOlizer",
"vuhuvBot",
"INETDEX-BOT",
"Synapse",
"t3versionsBot",
"deepnoc",
"Cocolyzebot",
"hypestat",
"ReverseEngineeringBot",
"sempi\\.tech",
"Iframely",
"MetaInspector",
"node-fetch",
"l9explore",
"python-opengraph",
"OpenGraphCheck",
"developers\\.google\\.com\\/\\+\\/web\\/snippet",
"SenutoBot",
"MaCoCu",
"NewsBlur",
"inoreader",
"NetSystemsResearch",
"PageThing",
"WordPress\\/",
"PhxBot",
"ImagesiftBot",
"Expanse",
"InternetMeasurement",
"^BW\\/",
"GeedoBot",
"Audisto Crawler",
"PerplexityBot\\/",
"[cC]laude[bB]ot",
"Monsidobot",
"GroupMeBot",
"Vercelbot",
"vercel-screenshot",
"facebookcatalog\\/",
"meta-externalagent\\/",
"meta-externalfetcher\\/",
"AcademicBotRTU",
"KeybaseBot",
"Lemmy",
"CookieHubScan",
"Hydrozen\\.io",
"HTTP Banner Detection",
"SummalyBot",
"MicrosoftPreview\\/",
"GeedoProductSearch",
"TikTokSpider"