From 782022754c396f53590a7673035480857307c605 Mon Sep 17 00:00:00 2001 From: Lewis Dale Date: Mon, 8 Jan 2024 08:11:53 +0000 Subject: [PATCH] Blog all of the DarkAgent spiders. Go away, nasty robots --- src/robots.txt | 2327 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 2326 insertions(+), 1 deletion(-) diff --git a/src/robots.txt b/src/robots.txt index 4f00592..29739e4 100644 --- a/src/robots.txt +++ b/src/robots.txt @@ -1,5 +1,2330 @@ Sitemap: https://lewisdale.dev/sitemap.xml +User-agent: ChatGPT-User +Disallow: / + +User-agent: dotbot +Disallow: / + +User-agent: bingpreview +Disallow: / + +User-agent: siteauditbot +Disallow: / + +User-agent: yahoo-slurp +Disallow: / + +User-agent: semrushbot-ct +Disallow: / + +User-agent: semrushbot +Disallow: / + +User-agent: gptbot +Disallow: / + +User-agent: googlebot +Disallow: / + +User-agent: yeti +Disallow: / + +User-agent: bingbot +Disallow: / + +User-agent: barkrowler +Disallow: / + +User-agent: bytespider +Disallow: / + +User-agent: dataforseobot +Disallow: / + +User-agent: rogerbot +Disallow: / + +User-agent: applebot +Disallow: / + +User-agent: seznambot +Disallow: / + +User-agent: pinterestbot +Disallow: / + +User-agent: google-inspectiontool +Disallow: / + +User-agent: discordbot +Disallow: / + +User-agent: dark-visitor +Disallow: / + +User-agent: ahrefssiteaudit +Disallow: / + +User-agent: archive-org-bot +Disallow: / + +User-agent: youbot +Disallow: / + +User-agent: semrushbot-si +Disallow: / + +User-agent: facebookbot +Disallow: / + +User-agent: amazonbot +Disallow: / + +User-agent: googlebot-image +Disallow: / + +User-agent: mj12bot +Disallow: / + +User-agent: awariosmartbot +Disallow: / + +User-agent: ccbot +Disallow: / + +User-agent: proximic +Disallow: / + +User-agent: cohere-ai +Disallow: / + +User-agent: semrushbot-ba +Disallow: / + +User-agent: chatgpt-user +Disallow: / + +User-agent: perplexitybot +Disallow: / + +User-agent: zoombot +Disallow: / + +User-agent: blexbot +Disallow: / + +User-agent: yandexbot +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +User-agent: omgilibot +Disallow: / + +User-agent: telegrambot +Disallow: / + +User-agent: twitterbot +Disallow: / + +User-agent: petalbot +Disallow: / + +User-agent: semrushbot-swa +Disallow: / + +User-agent: trendictionbot +Disallow: / + +User-agent: slackbot-linkexpanding +Disallow: / + +User-agent: screaming-frog-seo-spider +Disallow: / + +User-agent: snapchat +Disallow: / + +User-agent: google-extended +Disallow: / + +User-agent: scrapy +Disallow: / + +User-agent: linkfluence +Disallow: / + +User-agent: ahrefsbot +Disallow: / + +User-agent: yandexvideo +Disallow: / + +User-agent: trove +Disallow: / + +User-agent: google-structured-data-testing-tool +Disallow: / + +User-agent: garlikcrawler +Disallow: / + +User-agent: nuzzel +Disallow: / + +User-agent: spbot +Disallow: / + +User-agent: jobboersebot +Disallow: / + +User-agent: the-intraformant +Disallow: / + +User-agent: bitlybot +Disallow: / + +User-agent: siteimprove-com +Disallow: / + +User-agent: bunnyslippers +Disallow: / + +User-agent: score3 +Disallow: / + +User-agent: scrubby +Disallow: / + +User-agent: mastodon +Disallow: / + +User-agent: netants +Disallow: / + +User-agent: msnbot +Disallow: / + +User-agent: grapeshotcrawler +Disallow: / + +User-agent: nimbostratus-bot +Disallow: / + +User-agent: spinn3r +Disallow: / + +User-agent: privacyawarebot +Disallow: / + +User-agent: dragonbot +Disallow: / + +User-agent: epicbot +Disallow: / + +User-agent: nmap-scripting-engine +Disallow: / + +User-agent: gnowitnewsbot +Disallow: / + +User-agent: everyonesocialbot +Disallow: / + +User-agent: flaming-attackbot +Disallow: / + +User-agent: youdaobot +Disallow: / + +User-agent: findxbot +Disallow: / + +User-agent: zeus-link-scout +Disallow: / + +User-agent: nerdybot +Disallow: / + +User-agent: webcopier-v-2-2 +Disallow: / + +User-agent: cherrypicker +Disallow: / + +User-agent: linkisbot +Disallow: / + +User-agent: toplistbot +Disallow: / + +User-agent: serpstatbot +Disallow: / + +User-agent: digincore-bot +Disallow: / + +User-agent: lssrocketcrawler +Disallow: / + +User-agent: mlbot +Disallow: / + +User-agent: finditanswersbot +Disallow: / + +User-agent: baiduspider-news +Disallow: / + +User-agent: checkmarknetwork +Disallow: / + +User-agent: simplescraper +Disallow: / + +User-agent: verticalsearch +Disallow: / + +User-agent: crawler4j +Disallow: / + +User-agent: seokicks +Disallow: / + +User-agent: ubicrawler +Disallow: / + +User-agent: mediatoolkitbot +Disallow: / + +User-agent: brightedge-crawler +Disallow: / + +User-agent: spotter +Disallow: / + +User-agent: crescent +Disallow: / + +User-agent: streamline3bot +Disallow: / + +User-agent: disqus +Disallow: / + +User-agent: feedfetcher-google +Disallow: / + +User-agent: findthatfile +Disallow: / + +User-agent: binlar +Disallow: / + +User-agent: online-webceo-bot +Disallow: / + +User-agent: yandeximages +Disallow: / + +User-agent: megaindex-ru +Disallow: / + +User-agent: offline-explorer +Disallow: / + +User-agent: vigil +Disallow: / + +User-agent: brandverity +Disallow: / + +User-agent: nicecrawler +Disallow: / + +User-agent: blogmurabot +Disallow: / + +User-agent: feedbot +Disallow: / + +User-agent: swiftbot +Disallow: / + +User-agent: google-certificates-bridge +Disallow: / + +User-agent: heritrix +Disallow: / + +User-agent: officestorebot +Disallow: / + +User-agent: odklbot +Disallow: / + +User-agent: datafeedwatch +Disallow: / + +User-agent: careerbot +Disallow: / + +User-agent: mixnodecache +Disallow: / + +User-agent: vkshare +Disallow: / + +User-agent: cegbfeieh +Disallow: / + +User-agent: ip-web-crawler-com +Disallow: / + +User-agent: seokicks-robot +Disallow: / + +User-agent: twengabot +Disallow: / + +User-agent: extractorpro +Disallow: / + +User-agent: ask-n-read +Disallow: / + +User-agent: vsw +Disallow: / + +User-agent: googlebot-video +Disallow: / + +User-agent: facebot +Disallow: / + +User-agent: sonic +Disallow: / + +User-agent: retrevopageanalyzer +Disallow: / + +User-agent: url-control +Disallow: / + +User-agent: backdoorbot +Disallow: / + +User-agent: rankflex +Disallow: / + +User-agent: kstandbot +Disallow: / + +User-agent: icbot +Disallow: / + +User-agent: mister-pix +Disallow: / + +User-agent: hypestat +Disallow: / + +User-agent: googleother +Disallow: / + +User-agent: fast +Disallow: / + +User-agent: eyeotabot +Disallow: / + +User-agent: memorybot +Disallow: / + +User-agent: gsa-crawler +Disallow: / + +User-agent: imrbot +Disallow: / + +User-agent: fairad-client +Disallow: / + +User-agent: anyevent +Disallow: / + +User-agent: yandexrenderresourcesbot +Disallow: / + +User-agent: searchmetricsbot +Disallow: / + +User-agent: cocolyzebot +Disallow: / + +User-agent: startmebot +Disallow: / + +User-agent: mappydata +Disallow: / + +User-agent: snacktory +Disallow: / + +User-agent: 5emerue +Disallow: / + +User-agent: bublupbot +Disallow: / + +User-agent: buzzbot +Disallow: / + +User-agent: google-adwords-instant +Disallow: / + +User-agent: zoominfobot +Disallow: / + +User-agent: grouphigh +Disallow: / + +User-agent: ms-search-4-0-robot +Disallow: / + +User-agent: virustotal +Disallow: / + +User-agent: tweetedtimes +Disallow: / + +User-agent: meltawer +Disallow: / + +User-agent: fedoraplanet +Disallow: / + +User-agent: mediapartners-google +Disallow: / + +User-agent: k7mlwcbot +Disallow: / + +User-agent: g2-web-services +Disallow: / + +User-agent: vci +Disallow: / + +User-agent: pulsepoint +Disallow: / + +User-agent: szukacz +Disallow: / + +User-agent: bookmark-search-tool +Disallow: / + +User-agent: sistrix-crawler +Disallow: / + +User-agent: feedly +Disallow: / + +User-agent: openfind +Disallow: / + +User-agent: aihitbot +Disallow: / + +User-agent: pagepeeker +Disallow: / + +User-agent: uptimerobot +Disallow: / + +User-agent: datagnionbot +Disallow: / + +User-agent: surveybot +Disallow: / + +User-agent: zyborg +Disallow: / + +User-agent: adscanner +Disallow: / + +User-agent: iron33 +Disallow: / + +User-agent: landau-media-spider +Disallow: / + +User-agent: crunchbot +Disallow: / + +User-agent: bot-pge-chlooe-com +Disallow: / + +User-agent: stractbot +Disallow: / + +User-agent: domainstatsbot +Disallow: / + +User-agent: xenu-link-sleuth +Disallow: / + +User-agent: adequat +Disallow: / + +User-agent: veoozbot +Disallow: / + +User-agent: zao +Disallow: / + +User-agent: content-crawler-spider +Disallow: / + +User-agent: netresearchserver +Disallow: / + +User-agent: wbsearchbot +Disallow: / + +User-agent: ev-crawler +Disallow: / + +User-agent: linkextractorpro +Disallow: / + +User-agent: awesomecrawler +Disallow: / + +User-agent: usasearch +Disallow: / + +User-agent: aspiegelbot +Disallow: / + +User-agent: jamies-spider +Disallow: / + +User-agent: yandex +Disallow: / + +User-agent: meltwaternews +Disallow: / + +User-agent: uptime-kuma +Disallow: / + +User-agent: netsystemsresearch +Disallow: / + +User-agent: coexel +Disallow: / + +User-agent: cherrypickerelite +Disallow: / + +User-agent: yak +Disallow: / + +User-agent: um-ln +Disallow: / + +User-agent: pingdom +Disallow: / + +User-agent: rukicrawler +Disallow: / + +User-agent: siteexplorer-info +Disallow: / + +User-agent: freewebmonitoring-sitechecker +Disallow: / + +User-agent: rivva +Disallow: / + +User-agent: diffbot +Disallow: / + +User-agent: tagoobot +Disallow: / + +User-agent: linkpadbot +Disallow: / + +User-agent: claude-web +Disallow: / + +User-agent: seoscanners +Disallow: / + +User-agent: fast-enterprise-crawler +Disallow: / + +User-agent: seewithkids +Disallow: / + +User-agent: purebot +Disallow: / + +User-agent: cutbot +Disallow: / + +User-agent: aboundex +Disallow: / + +User-agent: doc +Disallow: / + +User-agent: baidu-yunguance +Disallow: / + +User-agent: robozilla +Disallow: / + +User-agent: lkxscan +Disallow: / + +User-agent: a-patent-crawler +Disallow: / + +User-agent: gluten-free-crawler +Disallow: / + +User-agent: psbot +Disallow: / + +User-agent: 007ac9-crawler +Disallow: / + +User-agent: webmasterworldforumbot +Disallow: / + +User-agent: moget +Disallow: / + +User-agent: webbandit +Disallow: / + +User-agent: deepnoc +Disallow: / + +User-agent: wellknownbot +Disallow: / + +User-agent: jikespider +Disallow: / + +User-agent: adbeat-bot +Disallow: / + +User-agent: sentibot +Disallow: / + +User-agent: flipboardproxy +Disallow: / + +User-agent: newsnow +Disallow: / + +User-agent: download-ninja +Disallow: / + +User-agent: vuhuvbot +Disallow: / + +User-agent: boxcarbot +Disallow: / + +User-agent: emailwolf +Disallow: / + +User-agent: ezooms +Disallow: / + +User-agent: tweetmemebot +Disallow: / + +User-agent: lipperhey +Disallow: / + +User-agent: digimind +Disallow: / + +User-agent: woriobot +Disallow: / + +User-agent: page2rss +Disallow: / + +User-agent: sitecheckerbotcrawler +Disallow: / + +User-agent: adstxtcrawler +Disallow: / + +User-agent: backlinkcrawler +Disallow: / + +User-agent: grapeshot +Disallow: / + +User-agent: komodiabot +Disallow: / + +User-agent: trendybuzz +Disallow: / + +User-agent: brandonbot +Disallow: / + +User-agent: dow-jones-searchbot +Disallow: / + +User-agent: gigablast +Disallow: / + +User-agent: netcraftsurveyagent +Disallow: / + +User-agent: xovibot +Disallow: / + +User-agent: builtbottough +Disallow: / + +User-agent: webcopier-v3-2a +Disallow: / + +User-agent: minicrawler +Disallow: / + +User-agent: metainspector +Disallow: / + +User-agent: woobot +Disallow: / + +User-agent: iframely +Disallow: / + +User-agent: webzip +Disallow: / + +User-agent: qwam-content-intelligence +Disallow: / + +User-agent: appinsights +Disallow: / + +User-agent: apercite +Disallow: / + +User-agent: outclicksbot +Disallow: / + +User-agent: webdatastats +Disallow: / + +User-agent: botrighthere +Disallow: / + +User-agent: bomborabot +Disallow: / + +User-agent: www-collector-e +Disallow: / + +User-agent: xenus +Disallow: / + +User-agent: regionstuttgartbot +Disallow: / + +User-agent: cosmos +Disallow: / + +User-agent: centurybot +Disallow: / + +User-agent: senutobot +Disallow: / + +User-agent: deadlinkchecker +Disallow: / + +User-agent: daum +Disallow: / + +User-agent: redditbot +Disallow: / + +User-agent: cloudflare-alwaysonline +Disallow: / + +User-agent: pinterest +Disallow: / + +User-agent: storebot-google +Disallow: / + +User-agent: linkwalker +Disallow: / + +User-agent: citeseerxbot +Disallow: / + +User-agent: biglotron +Disallow: / + +User-agent: feedspot +Disallow: / + +User-agent: rssbot +Disallow: / + +User-agent: orangebot +Disallow: / + +User-agent: istellabot +Disallow: / + +User-agent: smtbot +Disallow: / + +User-agent: k2spider +Disallow: / + +User-agent: friendica +Disallow: / + +User-agent: jetty +Disallow: / + +User-agent: nixstatsbot +Disallow: / + +User-agent: slackbot +Disallow: / + +User-agent: betabot +Disallow: / + +User-agent: google-read-aloud +Disallow: / + +User-agent: sosospider +Disallow: / + +User-agent: linkarchiver +Disallow: / + +User-agent: sindup +Disallow: / + +User-agent: tocrawl +Disallow: / + +User-agent: ut-dorkbot +Disallow: / + +User-agent: pandalytics +Disallow: / + +User-agent: qwantify +Disallow: / + +User-agent: jpg-newsbot +Disallow: / + +User-agent: opengraphcheck +Disallow: / + +User-agent: daumoa +Disallow: / + +User-agent: bullseye +Disallow: / + +User-agent: w3c-validator +Disallow: / + +User-agent: cyberpatrol +Disallow: / + +User-agent: wotbox +Disallow: / + +User-agent: freshpingbot +Disallow: / + +User-agent: acoonbot +Disallow: / + +User-agent: blackboard +Disallow: / + +User-agent: googlebot-mobile +Disallow: / + +User-agent: hubspot +Disallow: / + +User-agent: acunetix-security-scanner +Disallow: / + +User-agent: hyscore-io +Disallow: / + +User-agent: antibot +Disallow: / + +User-agent: lssbot +Disallow: / + +User-agent: harvest +Disallow: / + +User-agent: intelium-bot +Disallow: / + +User-agent: thenomad +Disallow: / + +User-agent: jugendschutzprogramm-crawler +Disallow: / + +User-agent: blowfish +Disallow: / + +User-agent: thinklab +Disallow: / + +User-agent: web-image-collector +Disallow: / + +User-agent: ninja-bot +Disallow: / + +User-agent: cision +Disallow: / + +User-agent: woorankreview +Disallow: / + +User-agent: converacrawler +Disallow: / + +User-agent: uptimebot-org +Disallow: / + +User-agent: webcapture-2-0 +Disallow: / + +User-agent: googlebot-news +Disallow: / + +User-agent: teleport +Disallow: / + +User-agent: flamingo-searchengine +Disallow: / + +User-agent: queryn-metasearch +Disallow: / + +User-agent: searchatlas +Disallow: / + +User-agent: viber +Disallow: / + +User-agent: gaisbot +Disallow: / + +User-agent: gslfbot +Disallow: / + +User-agent: google-site-verification +Disallow: / + +User-agent: ezid +Disallow: / + +User-agent: elisabot +Disallow: / + +User-agent: velenpublicwebcrawler +Disallow: / + +User-agent: url-spider-pro +Disallow: / + +User-agent: jennybot +Disallow: / + +User-agent: semrushbot-sa +Disallow: / + +User-agent: adequat-systems +Disallow: / + +User-agent: trovitbot +Disallow: / + +User-agent: changedetection +Disallow: / + +User-agent: dcrawl +Disallow: / + +User-agent: fr-crawler +Disallow: / + +User-agent: twingly +Disallow: / + +User-agent: botalot +Disallow: / + +User-agent: netmechanic +Disallow: / + +User-agent: alexibot +Disallow: / + +User-agent: feedvalidator +Disallow: / + +User-agent: getright +Disallow: / + +User-agent: opinion-tracker +Disallow: / + +User-agent: websauger +Disallow: / + +User-agent: postrank +Disallow: / + +User-agent: t3versionsbot +Disallow: / + +User-agent: voilabot +Disallow: / + +User-agent: datanyze +Disallow: / + +User-agent: taboolabot +Disallow: / + +User-agent: evc-batch +Disallow: / + +User-agent: mojeekbot +Disallow: / + +User-agent: baiduspider-image +Disallow: / + +User-agent: keyword-density +Disallow: / + +User-agent: telesoft +Disallow: / + +User-agent: nutch +Disallow: / + +User-agent: cxensebot +Disallow: / + +User-agent: web-archive-net-com-bot +Disallow: / + +User-agent: msiecrawler +Disallow: / + +User-agent: openhosebot +Disallow: / + +User-agent: dataprovider-com +Disallow: / + +User-agent: nextcloud +Disallow: / + +User-agent: behloolbot +Disallow: / + +User-agent: irlbot +Disallow: / + +User-agent: ioncrawl +Disallow: / + +User-agent: coccocbot +Disallow: / + +User-agent: riddler +Disallow: / + +User-agent: urly-warning +Disallow: / + +User-agent: discobot +Disallow: / + +User-agent: refindbot +Disallow: / + +User-agent: contextad-bot +Disallow: / + +User-agent: fast-webcrawler +Disallow: / + +User-agent: coccoc +Disallow: / + +User-agent: usinenouvellecrawler +Disallow: / + +User-agent: yahoo-link-preview +Disallow: / + +User-agent: adsbot-google +Disallow: / + +User-agent: emailcollector +Disallow: / + +User-agent: seostar-co +Disallow: / + +User-agent: outbrain +Disallow: / + +User-agent: g2reader-bot +Disallow: / + +User-agent: aisearchbot +Disallow: / + +User-agent: infoseek +Disallow: / + +User-agent: addthis +Disallow: / + +User-agent: anderspinkbot +Disallow: / + +User-agent: obot +Disallow: / + +User-agent: cc-metadata-scaper +Disallow: / + +User-agent: nerdbynature-bot +Disallow: / + +User-agent: pr-cy-ru +Disallow: / + +User-agent: emailsiphon +Disallow: / + +User-agent: sitebot +Disallow: / + +User-agent: openfind-data-gatherer +Disallow: / + +User-agent: zumbot +Disallow: / + +User-agent: sitecheck-internetseer-com +Disallow: / + +User-agent: freshrss +Disallow: / + +User-agent: zeus-32297-webster-pro-v2-9-win32 +Disallow: / + +User-agent: scoutjet +Disallow: / + +User-agent: infoobot +Disallow: / + +User-agent: niki-bot +Disallow: / + +User-agent: drupact +Disallow: / + +User-agent: wasalive-bot +Disallow: / + +User-agent: crescent-internet-toolpak-http-ole-control-v-1-0 +Disallow: / + +User-agent: summify +Disallow: / + +User-agent: hatena +Disallow: / + +User-agent: kosmiobot +Disallow: / + +User-agent: serendeputybot +Disallow: / + +User-agent: miniflux +Disallow: / + +User-agent: suzuran +Disallow: / + +User-agent: sogou-web-spider +Disallow: / + +User-agent: awariobot +Disallow: / + +User-agent: tombapublicwebcrawler +Disallow: / + +User-agent: bnf-fr-bot +Disallow: / + +User-agent: siteimprove +Disallow: / + +User-agent: skypeuripreview +Disallow: / + +User-agent: urlappendbot +Disallow: / + +User-agent: sogou +Disallow: / + +User-agent: voltron +Disallow: / + +User-agent: baiduspider-video +Disallow: / + +User-agent: youmag +Disallow: / + +User-agent: google-xrawler +Disallow: / + +User-agent: turnitinbot +Disallow: / + +User-agent: eright +Disallow: / + +User-agent: claritybot +Disallow: / + +User-agent: metauri +Disallow: / + +User-agent: linkscan +Disallow: / + +User-agent: moodlebot +Disallow: / + +User-agent: website-quester +Disallow: / + +User-agent: duckduckbot +Disallow: / + +User-agent: ltx71 +Disallow: / + +User-agent: 360spider +Disallow: / + +User-agent: repomonkey +Disallow: / + +User-agent: amisoftware +Disallow: / + +User-agent: bitbot +Disallow: / + +User-agent: xenu +Disallow: / + +User-agent: rankactivelinkbot +Disallow: / + +User-agent: baiduspider +Disallow: / + +User-agent: laserlikebot +Disallow: / + +User-agent: coccocbot-web +Disallow: / + +User-agent: asknread-com +Disallow: / + +User-agent: feedbin +Disallow: / + +User-agent: seostats +Disallow: / + +User-agent: naverbot +Disallow: / + +User-agent: tiny-tiny-rss +Disallow: / + +User-agent: lightspeedsystemscrawler +Disallow: / + +User-agent: teleportpro +Disallow: / + +User-agent: interfaxscanbot +Disallow: / + +User-agent: ichiro +Disallow: / + +User-agent: domaincrawler +Disallow: / + +User-agent: pocketparser +Disallow: / + +User-agent: google-web-preview +Disallow: / + +User-agent: aqua-products +Disallow: / + +User-agent: 5erue +Disallow: / + +User-agent: addsearchbot +Disallow: / + +User-agent: sogou-inst-spider +Disallow: / + +User-agent: extlinksbot +Disallow: / + +User-agent: primalbot +Disallow: / + +User-agent: trendeo +Disallow: / + +User-agent: b2b-bot +Disallow: / + +User-agent: crawlyprojectcrawler +Disallow: / + +User-agent: w3c-unicorn +Disallow: / + +User-agent: companybook-crawler +Disallow: / + +User-agent: crystalsemanticsbot +Disallow: / + +User-agent: surdotlybot +Disallow: / + +User-agent: bot-araturka-com +Disallow: / + +User-agent: netestate-ne-crawler +Disallow: / + +User-agent: jetbot +Disallow: / + +User-agent: augure +Disallow: / + +User-agent: w3c-css-validator +Disallow: / + +User-agent: livelapbot +Disallow: / + +User-agent: seekportbot +Disallow: / + +User-agent: edisterbot +Disallow: / + +User-agent: bidswitchbot +Disallow: / + +User-agent: slack-imgproxy +Disallow: / + +User-agent: brainobot +Disallow: / + +User-agent: ips-agent +Disallow: / + +User-agent: mytwip +Disallow: / + +User-agent: rssingbot +Disallow: / + +User-agent: auramundi +Disallow: / + +User-agent: pagething +Disallow: / + +User-agent: titan +Disallow: / + +User-agent: zeus +Disallow: / + +User-agent: deusu +Disallow: / + +User-agent: gnam-gnam-spider +Disallow: / + +User-agent: mindupbot +Disallow: / + +User-agent: gowikibot +Disallow: / + +User-agent: e-ventures-investment-crawler +Disallow: / + +User-agent: true-robot +Disallow: / + +User-agent: validator-nu +Disallow: / + +User-agent: ec2linkfinder +Disallow: / + +User-agent: alphabot +Disallow: / + +User-agent: zuperlistbot +Disallow: / + +User-agent: linespider +Disallow: / + +User-agent: linkedinbot +Disallow: / + +User-agent: copyrightcheck +Disallow: / + +User-agent: sbl-bot +Disallow: / + +User-agent: knowings +Disallow: / + +User-agent: proxem +Disallow: / + +User-agent: whatsapp +Disallow: / + +User-agent: 2ip-ru +Disallow: / + +User-agent: yanga +Disallow: / + +User-agent: sysomos +Disallow: / + +User-agent: vecteurplus +Disallow: / + +User-agent: black-hole +Disallow: / + +User-agent: ellisphere +Disallow: / + +User-agent: semanticscholarbot +Disallow: / + +User-agent: tineye-bot +Disallow: / + +User-agent: neticle-crawler +Disallow: / + +User-agent: socialrankiobot +Disallow: / + +User-agent: oracle-ultra-search +Disallow: / + +User-agent: webmon +Disallow: / + +User-agent: metajobbot +Disallow: / + +User-agent: nextgensearchbot +Disallow: / + +User-agent: w3c-mobileok +Disallow: / + +User-agent: electricmonk +Disallow: / + +User-agent: searchpreview +Disallow: / + +User-agent: seekbot +Disallow: / + +User-agent: npbot +Disallow: / + +User-agent: www-uptime-com +Disallow: / + +User-agent: ridderbot +Disallow: / + +User-agent: embedly +Disallow: / + +User-agent: dittospyder +Disallow: / + +User-agent: vebidoobot +Disallow: / + +User-agent: leikibot +Disallow: / + +User-agent: kenjin-spider +Disallow: / + +User-agent: icc-crawler +Disallow: / + +User-agent: spankbot +Disallow: / + +User-agent: exabot +Disallow: / + +User-agent: superbot +Disallow: / + +User-agent: mata-hari +Disallow: / + +User-agent: sogou-spider2 +Disallow: / + +User-agent: a6-indexer +Disallow: / + +User-agent: wordupinfosearch +Disallow: / + +User-agent: james-bot +Disallow: / + +User-agent: httrack-3-0 +Disallow: / + +User-agent: tangibleebot +Disallow: / + +User-agent: fyrebot +Disallow: / + +User-agent: snap-url-preview-service +Disallow: / + +User-agent: jetslide +Disallow: / + +User-agent: radiation-retriever-1-1 +Disallow: / + +User-agent: www-mechanize +Disallow: / + +User-agent: advbot +Disallow: / + +User-agent: arabot +Disallow: / + +User-agent: webster-pro +Disallow: / + +User-agent: spanner +Disallow: / + +User-agent: grub-client +Disallow: / + +User-agent: safesearch-microdata-crawler +Disallow: / + +User-agent: blp-bbot +Disallow: / + +User-agent: rma +Disallow: / + +User-agent: genieo +Disallow: / + +User-agent: 2ip-bot +Disallow: / + +User-agent: dubbotbot +Disallow: / + +User-agent: piplbot +Disallow: / + +User-agent: bazqux +Disallow: / + +User-agent: jooblebot +Disallow: / + +User-agent: perman +Disallow: / + +User-agent: speedy +Disallow: / + +User-agent: facebookexternalhit +Disallow: / + +User-agent: httrack +Disallow: / + +User-agent: acapbot +Disallow: / + +User-agent: kbcrawl +Disallow: / + +User-agent: admantx +Disallow: / + +User-agent: turingos +Disallow: / + +User-agent: prowebwalker +Disallow: / + +User-agent: ocarinabot +Disallow: / + +User-agent: contxbot +Disallow: / + +User-agent: swimgbot +Disallow: / + +User-agent: alexabot +Disallow: / + +User-agent: google-physicalweb +Disallow: / + +User-agent: sitesucker +Disallow: / + +User-agent: google-favicon +Disallow: / + +User-agent: etaospider +Disallow: / + +User-agent: capsulechecker +Disallow: / + +User-agent: talkwater +Disallow: / + +User-agent: mbcrawler +Disallow: / + +User-agent: tigerbot +Disallow: / + +User-agent: lb-spider +Disallow: / + +User-agent: linkapediabot +Disallow: / + +User-agent: seekport-crawler +Disallow: / + +User-agent: propowerbot +Disallow: / + +User-agent: foobot +Disallow: / + +User-agent: yacybot +Disallow: / + +User-agent: ruxitsynthetic +Disallow: / + +User-agent: chrome-lighthouse +Disallow: / + +User-agent: teoma +Disallow: / + +User-agent: ntentbot +Disallow: / + +User-agent: lnspiderguy +Disallow: / + +User-agent: webenhancer +Disallow: / + +User-agent: 008 +Disallow: / + +User-agent: miixpc +Disallow: / + +User-agent: femtosearchbot +Disallow: / + +User-agent: vkrobot +Disallow: / + +User-agent: europarchive-org +Disallow: / + +User-agent: moreover +Disallow: / + +User-agent: msnbot-media +Disallow: / + +User-agent: linko +Disallow: / + +User-agent: voluumdsp-content-bot +Disallow: / + +User-agent: synthesio +Disallow: / + +User-agent: winello +Disallow: / + +User-agent: luminator-robots +Disallow: / + +User-agent: cheesebot +Disallow: / + +User-agent: turnitin +Disallow: / + +User-agent: nimblecrawler +Disallow: / + +User-agent: reverseengineeringbot +Disallow: / + +User-agent: experibot +Disallow: / + +User-agent: scribdbot +Disallow: / + +User-agent: zite +Disallow: / + +User-agent: dareboost +Disallow: / + +User-agent: trendkite-akashic-crawler +Disallow: / + +User-agent: ias-crawler +Disallow: / + +User-agent: google-safety +Disallow: / + +User-agent: erocrawler +Disallow: / + +User-agent: gwene +Disallow: / + +User-agent: magpie-crawler +Disallow: / + +User-agent: safednsbot +Disallow: / + +User-agent: zgrab +Disallow: / + +User-agent: lexibot +Disallow: / + +User-agent: gigabot +Disallow: / + +User-agent: yoozbot +Disallow: / + +User-agent: rytebot +Disallow: / + +User-agent: clickagy +Disallow: / + +User-agent: digg-deeper +Disallow: / + +User-agent: curebot +Disallow: / + +User-agent: caliperbot +Disallow: / + +User-agent: twurly +Disallow: / + +User-agent: megaindex +Disallow: / + +User-agent: simplepie +Disallow: / + +User-agent: theoldreader-com +Disallow: / + +User-agent: cispa-webcrawler +Disallow: / + +User-agent: jamesbot +Disallow: / + +User-agent: buck +Disallow: / + +User-agent: lwp-trivial +Disallow: / + +User-agent: haosouspider +Disallow: / + +User-agent: repomonkey-bait-tackle +Disallow: / + +User-agent: b2w +Disallow: / + +User-agent: libweb +Disallow: / + +User-agent: corporama +Disallow: / + +User-agent: blekkobot +Disallow: / + +User-agent: linkdex +Disallow: / + +User-agent: webcopier +Disallow: / + +User-agent: vci-webviewer-vci-webviewer-win32 +Disallow: / + +User-agent: webreaper +Disallow: / + +User-agent: fluffy +Disallow: / + +User-agent: newspaper +Disallow: / + +User-agent: asterias +Disallow: / + +User-agent: humanlinks +Disallow: / + +User-agent: fever +Disallow: / + +User-agent: updownerbot +Disallow: / + +User-agent: bubing +Disallow: / + +User-agent: newsharecounts +Disallow: / + +User-agent: linguee-bot +Disallow: / + +User-agent: toutiaospider +Disallow: / + +User-agent: netvibes +Disallow: / + +User-agent: webauto +Disallow: / + +User-agent: fuelbot +Disallow: / + +User-agent: alexandriaorgbot +Disallow: / + +User-agent: newzbin +Disallow: / + +User-agent: integromedb +Disallow: / + +User-agent: tineye +Disallow: / + +User-agent: botify +Disallow: / + +User-agent: semanticbot +Disallow: / + +User-agent: webcompanycrawler +Disallow: / + +User-agent: it2media-domain-crawler +Disallow: / + +User-agent: dnyzbot +Disallow: / + +User-agent: inoreader +Disallow: / + +User-agent: cookiebot +Disallow: / + +User-agent: jyxobot +Disallow: / + +User-agent: w3c-i18n-checker +Disallow: / + +User-agent: wiseguys-robot +Disallow: / + +User-agent: orthogaffe +Disallow: / + +User-agent: upflow +Disallow: / + +User-agent: paperlibot +Disallow: / + +User-agent: linkcheck-by-siteimprove-com +Disallow: / + +User-agent: findlink +Disallow: / + +User-agent: cherrypickerse +Disallow: / + +User-agent: bitsightbot +Disallow: / + +User-agent: criteobot +Disallow: / + +User-agent: convera +Disallow: / + +User-agent: domain-re-animator-bot +Disallow: / + +User-agent: larbin +Disallow: / + +User-agent: grobbot +Disallow: / + +User-agent: panscient +Disallow: / + +User-agent: ms-search-6-0-robot +Disallow: / + +User-agent: g00g1e-net +Disallow: / + +User-agent: tracemyfile +Disallow: / + +User-agent: meltwater +Disallow: / + +User-agent: xenus-link-sleuth-1-1c +Disallow: / + +User-agent: mtrobot +Disallow: / + +User-agent: israbot +Disallow: / + +User-agent: semrushbot-bm +Disallow: / + +User-agent: cliqzbot +Disallow: / + +User-agent: webstripper +Disallow: / + +User-agent: bdcbot +Disallow: / + +User-agent: multiviewbot +Disallow: / + +User-agent: atom-feed-robot +Disallow: / + +User-agent: adsbot-google-mobile +Disallow: / + +User-agent: naver-blog-rssbot +Disallow: / + +User-agent: gingercrawler +Disallow: / + +User-agent: cincraw +Disallow: / + +User-agent: iskanie +Disallow: / + +User-agent: neevabot +Disallow: / + +User-agent: w3c-checklink +Disallow: / + +User-agent: trendsmapresolver +Disallow: / + +User-agent: synapse +Disallow: / + +User-agent: zabbix +Disallow: / + +User-agent: btwebclient +Disallow: / + +User-agent: msrbot +Disallow: / + +User-agent: inetdex-bot +Disallow: / + +User-agent: seobilitybot +Disallow: / + +User-agent: sitesnagger +Disallow: / + +User-agent: wocbot +Disallow: / + +User-agent: storygizebot +Disallow: / + +User-agent: better-uptime-bot +Disallow: / + +User-agent: yahoo-pipes-1-0 +Disallow: / + +User-agent: moatbot +Disallow: / + +User-agent: superfeedr +Disallow: / + +User-agent: sempi-tech +Disallow: / + +User-agent: copernic +Disallow: / + +User-agent: duckduckgo-favicons-bot +Disallow: / + +User-agent: indeedbot +Disallow: / + +User-agent: zealbot +Disallow: / + +User-agent: kemvibot +Disallow: / + +User-agent: mauibot +Disallow: / + +User-agent: deepcrawl +Disallow: / + +User-agent: scoop-it +Disallow: / + +User-agent: winhttrack +Disallow: / + +User-agent: muckrack +Disallow: / + +User-agent: mention +Disallow: / + +User-agent: mail-ru-bot +Disallow: / + +User-agent: leadbox +Disallow: / + +User-agent: seolizer +Disallow: / + +User-agent: acunetix-web-vulnerability-scanner +Disallow: / + +User-agent: awariorssbot +Disallow: / + +User-agent: openindexspider +Disallow: / + +User-agent: slurp +Disallow: / + +User-agent: yisouspider +Disallow: / + +User-agent: search-marginalia-nu +Disallow: / + +User-agent: openbot +Disallow: / + +User-agent: adidxbot +Disallow: / + +User-agent: macocu +Disallow: / + +User-agent: zenback-bot +Disallow: / + +User-agent: yellowbrandprotectionbot +Disallow: / + +User-agent: simplecrawler +Disallow: / + +User-agent: nicerspro +Disallow: / + +User-agent: infonavirobot +Disallow: / + +User-agent: traackr-com +Disallow: / + +User-agent: linkdexbot +Disallow: / + +User-agent: truliabot +Disallow: / + +User-agent: newsblur +Disallow: / + +User-agent: grub-org +Disallow: / + User-Agent: Bytespider Disallow: / @@ -10,4 +2335,4 @@ User-Agent: * Allow: / Disallow: /assets/ Disallow: /wp-content/ -Host: https://lewisdale.dev \ No newline at end of file +Host: https://lewisdale.dev