From 1956bb83be8fc9381c62b516b6f1fd6c6feddb3e Mon Sep 17 00:00:00 2001 From: Lewis Dale Date: Tue, 9 Jan 2024 08:06:57 +0000 Subject: [PATCH] Revert robots.txt - accidentally blocking archive.org among others --- src/robots.txt | 2322 ------------------------------------------------ 1 file changed, 2322 deletions(-) diff --git a/src/robots.txt b/src/robots.txt index 29739e4..6e766a8 100644 --- a/src/robots.txt +++ b/src/robots.txt @@ -3,2328 +3,6 @@ Sitemap: https://lewisdale.dev/sitemap.xml User-agent: ChatGPT-User Disallow: / -User-agent: dotbot -Disallow: / - -User-agent: bingpreview -Disallow: / - -User-agent: siteauditbot -Disallow: / - -User-agent: yahoo-slurp -Disallow: / - -User-agent: semrushbot-ct -Disallow: / - -User-agent: semrushbot -Disallow: / - -User-agent: gptbot -Disallow: / - -User-agent: googlebot -Disallow: / - -User-agent: yeti -Disallow: / - -User-agent: bingbot -Disallow: / - -User-agent: barkrowler -Disallow: / - -User-agent: bytespider -Disallow: / - -User-agent: dataforseobot -Disallow: / - -User-agent: rogerbot -Disallow: / - -User-agent: applebot -Disallow: / - -User-agent: seznambot -Disallow: / - -User-agent: pinterestbot -Disallow: / - -User-agent: google-inspectiontool -Disallow: / - -User-agent: discordbot -Disallow: / - -User-agent: dark-visitor -Disallow: / - -User-agent: ahrefssiteaudit -Disallow: / - -User-agent: archive-org-bot -Disallow: / - -User-agent: youbot -Disallow: / - -User-agent: semrushbot-si -Disallow: / - -User-agent: facebookbot -Disallow: / - -User-agent: amazonbot -Disallow: / - -User-agent: googlebot-image -Disallow: / - -User-agent: mj12bot -Disallow: / - -User-agent: awariosmartbot -Disallow: / - -User-agent: ccbot -Disallow: / - -User-agent: proximic -Disallow: / - -User-agent: cohere-ai -Disallow: / - -User-agent: semrushbot-ba -Disallow: / - -User-agent: chatgpt-user -Disallow: / - -User-agent: perplexitybot -Disallow: / - -User-agent: zoombot -Disallow: / - -User-agent: blexbot -Disallow: / - -User-agent: yandexbot -Disallow: / - -User-agent: anthropic-ai -Disallow: / - -User-agent: omgilibot -Disallow: / - -User-agent: telegrambot -Disallow: / - -User-agent: twitterbot -Disallow: / - -User-agent: petalbot -Disallow: / - -User-agent: semrushbot-swa -Disallow: / - -User-agent: trendictionbot -Disallow: / - -User-agent: slackbot-linkexpanding -Disallow: / - -User-agent: screaming-frog-seo-spider -Disallow: / - -User-agent: snapchat -Disallow: / - -User-agent: google-extended -Disallow: / - -User-agent: scrapy -Disallow: / - -User-agent: linkfluence -Disallow: / - -User-agent: ahrefsbot -Disallow: / - -User-agent: yandexvideo -Disallow: / - -User-agent: trove -Disallow: / - -User-agent: google-structured-data-testing-tool -Disallow: / - -User-agent: garlikcrawler -Disallow: / - -User-agent: nuzzel -Disallow: / - -User-agent: spbot -Disallow: / - -User-agent: jobboersebot -Disallow: / - -User-agent: the-intraformant -Disallow: / - -User-agent: bitlybot -Disallow: / - -User-agent: siteimprove-com -Disallow: / - -User-agent: bunnyslippers -Disallow: / - -User-agent: score3 -Disallow: / - -User-agent: scrubby -Disallow: / - -User-agent: mastodon -Disallow: / - -User-agent: netants -Disallow: / - -User-agent: msnbot -Disallow: / - -User-agent: grapeshotcrawler -Disallow: / - -User-agent: nimbostratus-bot -Disallow: / - -User-agent: spinn3r -Disallow: / - -User-agent: privacyawarebot -Disallow: / - -User-agent: dragonbot -Disallow: / - -User-agent: epicbot -Disallow: / - -User-agent: nmap-scripting-engine -Disallow: / - -User-agent: gnowitnewsbot -Disallow: / - -User-agent: everyonesocialbot -Disallow: / - -User-agent: flaming-attackbot -Disallow: / - -User-agent: youdaobot -Disallow: / - -User-agent: findxbot -Disallow: / - -User-agent: zeus-link-scout -Disallow: / - -User-agent: nerdybot -Disallow: / - -User-agent: webcopier-v-2-2 -Disallow: / - -User-agent: cherrypicker -Disallow: / - -User-agent: linkisbot -Disallow: / - -User-agent: toplistbot -Disallow: / - -User-agent: serpstatbot -Disallow: / - -User-agent: digincore-bot -Disallow: / - -User-agent: lssrocketcrawler -Disallow: / - -User-agent: mlbot -Disallow: / - -User-agent: finditanswersbot -Disallow: / - -User-agent: baiduspider-news -Disallow: / - -User-agent: checkmarknetwork -Disallow: / - -User-agent: simplescraper -Disallow: / - -User-agent: verticalsearch -Disallow: / - -User-agent: crawler4j -Disallow: / - -User-agent: seokicks -Disallow: / - -User-agent: ubicrawler -Disallow: / - -User-agent: mediatoolkitbot -Disallow: / - -User-agent: brightedge-crawler -Disallow: / - -User-agent: spotter -Disallow: / - -User-agent: crescent -Disallow: / - -User-agent: streamline3bot -Disallow: / - -User-agent: disqus -Disallow: / - -User-agent: feedfetcher-google -Disallow: / - -User-agent: findthatfile -Disallow: / - -User-agent: binlar -Disallow: / - -User-agent: online-webceo-bot -Disallow: / - -User-agent: yandeximages -Disallow: / - -User-agent: megaindex-ru -Disallow: / - -User-agent: offline-explorer -Disallow: / - -User-agent: vigil -Disallow: / - -User-agent: brandverity -Disallow: / - -User-agent: nicecrawler -Disallow: / - -User-agent: blogmurabot -Disallow: / - -User-agent: feedbot -Disallow: / - -User-agent: swiftbot -Disallow: / - -User-agent: google-certificates-bridge -Disallow: / - -User-agent: heritrix -Disallow: / - -User-agent: officestorebot -Disallow: / - -User-agent: odklbot -Disallow: / - -User-agent: datafeedwatch -Disallow: / - -User-agent: careerbot -Disallow: / - -User-agent: mixnodecache -Disallow: / - -User-agent: vkshare -Disallow: / - -User-agent: cegbfeieh -Disallow: / - -User-agent: ip-web-crawler-com -Disallow: / - -User-agent: seokicks-robot -Disallow: / - -User-agent: twengabot -Disallow: / - -User-agent: extractorpro -Disallow: / - -User-agent: ask-n-read -Disallow: / - -User-agent: vsw -Disallow: / - -User-agent: googlebot-video -Disallow: / - -User-agent: facebot -Disallow: / - -User-agent: sonic -Disallow: / - -User-agent: retrevopageanalyzer -Disallow: / - -User-agent: url-control -Disallow: / - -User-agent: backdoorbot -Disallow: / - -User-agent: rankflex -Disallow: / - -User-agent: kstandbot -Disallow: / - -User-agent: icbot -Disallow: / - -User-agent: mister-pix -Disallow: / - -User-agent: hypestat -Disallow: / - -User-agent: googleother -Disallow: / - -User-agent: fast -Disallow: / - -User-agent: eyeotabot -Disallow: / - -User-agent: memorybot -Disallow: / - -User-agent: gsa-crawler -Disallow: / - -User-agent: imrbot -Disallow: / - -User-agent: fairad-client -Disallow: / - -User-agent: anyevent -Disallow: / - -User-agent: yandexrenderresourcesbot -Disallow: / - -User-agent: searchmetricsbot -Disallow: / - -User-agent: cocolyzebot -Disallow: / - -User-agent: startmebot -Disallow: / - -User-agent: mappydata -Disallow: / - -User-agent: snacktory -Disallow: / - -User-agent: 5emerue -Disallow: / - -User-agent: bublupbot -Disallow: / - -User-agent: buzzbot -Disallow: / - -User-agent: google-adwords-instant -Disallow: / - -User-agent: zoominfobot -Disallow: / - -User-agent: grouphigh -Disallow: / - -User-agent: ms-search-4-0-robot -Disallow: / - -User-agent: virustotal -Disallow: / - -User-agent: tweetedtimes -Disallow: / - -User-agent: meltawer -Disallow: / - -User-agent: fedoraplanet -Disallow: / - -User-agent: mediapartners-google -Disallow: / - -User-agent: k7mlwcbot -Disallow: / - -User-agent: g2-web-services -Disallow: / - -User-agent: vci -Disallow: / - -User-agent: pulsepoint -Disallow: / - -User-agent: szukacz -Disallow: / - -User-agent: bookmark-search-tool -Disallow: / - -User-agent: sistrix-crawler -Disallow: / - -User-agent: feedly -Disallow: / - -User-agent: openfind -Disallow: / - -User-agent: aihitbot -Disallow: / - -User-agent: pagepeeker -Disallow: / - -User-agent: uptimerobot -Disallow: / - -User-agent: datagnionbot -Disallow: / - -User-agent: surveybot -Disallow: / - -User-agent: zyborg -Disallow: / - -User-agent: adscanner -Disallow: / - -User-agent: iron33 -Disallow: / - -User-agent: landau-media-spider -Disallow: / - -User-agent: crunchbot -Disallow: / - -User-agent: bot-pge-chlooe-com -Disallow: / - -User-agent: stractbot -Disallow: / - -User-agent: domainstatsbot -Disallow: / - -User-agent: xenu-link-sleuth -Disallow: / - -User-agent: adequat -Disallow: / - -User-agent: veoozbot -Disallow: / - -User-agent: zao -Disallow: / - -User-agent: content-crawler-spider -Disallow: / - -User-agent: netresearchserver -Disallow: / - -User-agent: wbsearchbot -Disallow: / - -User-agent: ev-crawler -Disallow: / - -User-agent: linkextractorpro -Disallow: / - -User-agent: awesomecrawler -Disallow: / - -User-agent: usasearch -Disallow: / - -User-agent: aspiegelbot -Disallow: / - -User-agent: jamies-spider -Disallow: / - -User-agent: yandex -Disallow: / - -User-agent: meltwaternews -Disallow: / - -User-agent: uptime-kuma -Disallow: / - -User-agent: netsystemsresearch -Disallow: / - -User-agent: coexel -Disallow: / - -User-agent: cherrypickerelite -Disallow: / - -User-agent: yak -Disallow: / - -User-agent: um-ln -Disallow: / - -User-agent: pingdom -Disallow: / - -User-agent: rukicrawler -Disallow: / - -User-agent: siteexplorer-info -Disallow: / - -User-agent: freewebmonitoring-sitechecker -Disallow: / - -User-agent: rivva -Disallow: / - -User-agent: diffbot -Disallow: / - -User-agent: tagoobot -Disallow: / - -User-agent: linkpadbot -Disallow: / - -User-agent: claude-web -Disallow: / - -User-agent: seoscanners -Disallow: / - -User-agent: fast-enterprise-crawler -Disallow: / - -User-agent: seewithkids -Disallow: / - -User-agent: purebot -Disallow: / - -User-agent: cutbot -Disallow: / - -User-agent: aboundex -Disallow: / - -User-agent: doc -Disallow: / - -User-agent: baidu-yunguance -Disallow: / - -User-agent: robozilla -Disallow: / - -User-agent: lkxscan -Disallow: / - -User-agent: a-patent-crawler -Disallow: / - -User-agent: gluten-free-crawler -Disallow: / - -User-agent: psbot -Disallow: / - -User-agent: 007ac9-crawler -Disallow: / - -User-agent: webmasterworldforumbot -Disallow: / - -User-agent: moget -Disallow: / - -User-agent: webbandit -Disallow: / - -User-agent: deepnoc -Disallow: / - -User-agent: wellknownbot -Disallow: / - -User-agent: jikespider -Disallow: / - -User-agent: adbeat-bot -Disallow: / - -User-agent: sentibot -Disallow: / - -User-agent: flipboardproxy -Disallow: / - -User-agent: newsnow -Disallow: / - -User-agent: download-ninja -Disallow: / - -User-agent: vuhuvbot -Disallow: / - -User-agent: boxcarbot -Disallow: / - -User-agent: emailwolf -Disallow: / - -User-agent: ezooms -Disallow: / - -User-agent: tweetmemebot -Disallow: / - -User-agent: lipperhey -Disallow: / - -User-agent: digimind -Disallow: / - -User-agent: woriobot -Disallow: / - -User-agent: page2rss -Disallow: / - -User-agent: sitecheckerbotcrawler -Disallow: / - -User-agent: adstxtcrawler -Disallow: / - -User-agent: backlinkcrawler -Disallow: / - -User-agent: grapeshot -Disallow: / - -User-agent: komodiabot -Disallow: / - -User-agent: trendybuzz -Disallow: / - -User-agent: brandonbot -Disallow: / - -User-agent: dow-jones-searchbot -Disallow: / - -User-agent: gigablast -Disallow: / - -User-agent: netcraftsurveyagent -Disallow: / - -User-agent: xovibot -Disallow: / - -User-agent: builtbottough -Disallow: / - -User-agent: webcopier-v3-2a -Disallow: / - -User-agent: minicrawler -Disallow: / - -User-agent: metainspector -Disallow: / - -User-agent: woobot -Disallow: / - -User-agent: iframely -Disallow: / - -User-agent: webzip -Disallow: / - -User-agent: qwam-content-intelligence -Disallow: / - -User-agent: appinsights -Disallow: / - -User-agent: apercite -Disallow: / - -User-agent: outclicksbot -Disallow: / - -User-agent: webdatastats -Disallow: / - -User-agent: botrighthere -Disallow: / - -User-agent: bomborabot -Disallow: / - -User-agent: www-collector-e -Disallow: / - -User-agent: xenus -Disallow: / - -User-agent: regionstuttgartbot -Disallow: / - -User-agent: cosmos -Disallow: / - -User-agent: centurybot -Disallow: / - -User-agent: senutobot -Disallow: / - -User-agent: deadlinkchecker -Disallow: / - -User-agent: daum -Disallow: / - -User-agent: redditbot -Disallow: / - -User-agent: cloudflare-alwaysonline -Disallow: / - -User-agent: pinterest -Disallow: / - -User-agent: storebot-google -Disallow: / - -User-agent: linkwalker -Disallow: / - -User-agent: citeseerxbot -Disallow: / - -User-agent: biglotron -Disallow: / - -User-agent: feedspot -Disallow: / - -User-agent: rssbot -Disallow: / - -User-agent: orangebot -Disallow: / - -User-agent: istellabot -Disallow: / - -User-agent: smtbot -Disallow: / - -User-agent: k2spider -Disallow: / - -User-agent: friendica -Disallow: / - -User-agent: jetty -Disallow: / - -User-agent: nixstatsbot -Disallow: / - -User-agent: slackbot -Disallow: / - -User-agent: betabot -Disallow: / - -User-agent: google-read-aloud -Disallow: / - -User-agent: sosospider -Disallow: / - -User-agent: linkarchiver -Disallow: / - -User-agent: sindup -Disallow: / - -User-agent: tocrawl -Disallow: / - -User-agent: ut-dorkbot -Disallow: / - -User-agent: pandalytics -Disallow: / - -User-agent: qwantify -Disallow: / - -User-agent: jpg-newsbot -Disallow: / - -User-agent: opengraphcheck -Disallow: / - -User-agent: daumoa -Disallow: / - -User-agent: bullseye -Disallow: / - -User-agent: w3c-validator -Disallow: / - -User-agent: cyberpatrol -Disallow: / - -User-agent: wotbox -Disallow: / - -User-agent: freshpingbot -Disallow: / - -User-agent: acoonbot -Disallow: / - -User-agent: blackboard -Disallow: / - -User-agent: googlebot-mobile -Disallow: / - -User-agent: hubspot -Disallow: / - -User-agent: acunetix-security-scanner -Disallow: / - -User-agent: hyscore-io -Disallow: / - -User-agent: antibot -Disallow: / - -User-agent: lssbot -Disallow: / - -User-agent: harvest -Disallow: / - -User-agent: intelium-bot -Disallow: / - -User-agent: thenomad -Disallow: / - -User-agent: jugendschutzprogramm-crawler -Disallow: / - -User-agent: blowfish -Disallow: / - -User-agent: thinklab -Disallow: / - -User-agent: web-image-collector -Disallow: / - -User-agent: ninja-bot -Disallow: / - -User-agent: cision -Disallow: / - -User-agent: woorankreview -Disallow: / - -User-agent: converacrawler -Disallow: / - -User-agent: uptimebot-org -Disallow: / - -User-agent: webcapture-2-0 -Disallow: / - -User-agent: googlebot-news -Disallow: / - -User-agent: teleport -Disallow: / - -User-agent: flamingo-searchengine -Disallow: / - -User-agent: queryn-metasearch -Disallow: / - -User-agent: searchatlas -Disallow: / - -User-agent: viber -Disallow: / - -User-agent: gaisbot -Disallow: / - -User-agent: gslfbot -Disallow: / - -User-agent: google-site-verification -Disallow: / - -User-agent: ezid -Disallow: / - -User-agent: elisabot -Disallow: / - -User-agent: velenpublicwebcrawler -Disallow: / - -User-agent: url-spider-pro -Disallow: / - -User-agent: jennybot -Disallow: / - -User-agent: semrushbot-sa -Disallow: / - -User-agent: adequat-systems -Disallow: / - -User-agent: trovitbot -Disallow: / - -User-agent: changedetection -Disallow: / - -User-agent: dcrawl -Disallow: / - -User-agent: fr-crawler -Disallow: / - -User-agent: twingly -Disallow: / - -User-agent: botalot -Disallow: / - -User-agent: netmechanic -Disallow: / - -User-agent: alexibot -Disallow: / - -User-agent: feedvalidator -Disallow: / - -User-agent: getright -Disallow: / - -User-agent: opinion-tracker -Disallow: / - -User-agent: websauger -Disallow: / - -User-agent: postrank -Disallow: / - -User-agent: t3versionsbot -Disallow: / - -User-agent: voilabot -Disallow: / - -User-agent: datanyze -Disallow: / - -User-agent: taboolabot -Disallow: / - -User-agent: evc-batch -Disallow: / - -User-agent: mojeekbot -Disallow: / - -User-agent: baiduspider-image -Disallow: / - -User-agent: keyword-density -Disallow: / - -User-agent: telesoft -Disallow: / - -User-agent: nutch -Disallow: / - -User-agent: cxensebot -Disallow: / - -User-agent: web-archive-net-com-bot -Disallow: / - -User-agent: msiecrawler -Disallow: / - -User-agent: openhosebot -Disallow: / - -User-agent: dataprovider-com -Disallow: / - -User-agent: nextcloud -Disallow: / - -User-agent: behloolbot -Disallow: / - -User-agent: irlbot -Disallow: / - -User-agent: ioncrawl -Disallow: / - -User-agent: coccocbot -Disallow: / - -User-agent: riddler -Disallow: / - -User-agent: urly-warning -Disallow: / - -User-agent: discobot -Disallow: / - -User-agent: refindbot -Disallow: / - -User-agent: contextad-bot -Disallow: / - -User-agent: fast-webcrawler -Disallow: / - -User-agent: coccoc -Disallow: / - -User-agent: usinenouvellecrawler -Disallow: / - -User-agent: yahoo-link-preview -Disallow: / - -User-agent: adsbot-google -Disallow: / - -User-agent: emailcollector -Disallow: / - -User-agent: seostar-co -Disallow: / - -User-agent: outbrain -Disallow: / - -User-agent: g2reader-bot -Disallow: / - -User-agent: aisearchbot -Disallow: / - -User-agent: infoseek -Disallow: / - -User-agent: addthis -Disallow: / - -User-agent: anderspinkbot -Disallow: / - -User-agent: obot -Disallow: / - -User-agent: cc-metadata-scaper -Disallow: / - -User-agent: nerdbynature-bot -Disallow: / - -User-agent: pr-cy-ru -Disallow: / - -User-agent: emailsiphon -Disallow: / - -User-agent: sitebot -Disallow: / - -User-agent: openfind-data-gatherer -Disallow: / - -User-agent: zumbot -Disallow: / - -User-agent: sitecheck-internetseer-com -Disallow: / - -User-agent: freshrss -Disallow: / - -User-agent: zeus-32297-webster-pro-v2-9-win32 -Disallow: / - -User-agent: scoutjet -Disallow: / - -User-agent: infoobot -Disallow: / - -User-agent: niki-bot -Disallow: / - -User-agent: drupact -Disallow: / - -User-agent: wasalive-bot -Disallow: / - -User-agent: crescent-internet-toolpak-http-ole-control-v-1-0 -Disallow: / - -User-agent: summify -Disallow: / - -User-agent: hatena -Disallow: / - -User-agent: kosmiobot -Disallow: / - -User-agent: serendeputybot -Disallow: / - -User-agent: miniflux -Disallow: / - -User-agent: suzuran -Disallow: / - -User-agent: sogou-web-spider -Disallow: / - -User-agent: awariobot -Disallow: / - -User-agent: tombapublicwebcrawler -Disallow: / - -User-agent: bnf-fr-bot -Disallow: / - -User-agent: siteimprove -Disallow: / - -User-agent: skypeuripreview -Disallow: / - -User-agent: urlappendbot -Disallow: / - -User-agent: sogou -Disallow: / - -User-agent: voltron -Disallow: / - -User-agent: baiduspider-video -Disallow: / - -User-agent: youmag -Disallow: / - -User-agent: google-xrawler -Disallow: / - -User-agent: turnitinbot -Disallow: / - -User-agent: eright -Disallow: / - -User-agent: claritybot -Disallow: / - -User-agent: metauri -Disallow: / - -User-agent: linkscan -Disallow: / - -User-agent: moodlebot -Disallow: / - -User-agent: website-quester -Disallow: / - -User-agent: duckduckbot -Disallow: / - -User-agent: ltx71 -Disallow: / - -User-agent: 360spider -Disallow: / - -User-agent: repomonkey -Disallow: / - -User-agent: amisoftware -Disallow: / - -User-agent: bitbot -Disallow: / - -User-agent: xenu -Disallow: / - -User-agent: rankactivelinkbot -Disallow: / - -User-agent: baiduspider -Disallow: / - -User-agent: laserlikebot -Disallow: / - -User-agent: coccocbot-web -Disallow: / - -User-agent: asknread-com -Disallow: / - -User-agent: feedbin -Disallow: / - -User-agent: seostats -Disallow: / - -User-agent: naverbot -Disallow: / - -User-agent: tiny-tiny-rss -Disallow: / - -User-agent: lightspeedsystemscrawler -Disallow: / - -User-agent: teleportpro -Disallow: / - -User-agent: interfaxscanbot -Disallow: / - -User-agent: ichiro -Disallow: / - -User-agent: domaincrawler -Disallow: / - -User-agent: pocketparser -Disallow: / - -User-agent: google-web-preview -Disallow: / - -User-agent: aqua-products -Disallow: / - -User-agent: 5erue -Disallow: / - -User-agent: addsearchbot -Disallow: / - -User-agent: sogou-inst-spider -Disallow: / - -User-agent: extlinksbot -Disallow: / - -User-agent: primalbot -Disallow: / - -User-agent: trendeo -Disallow: / - -User-agent: b2b-bot -Disallow: / - -User-agent: crawlyprojectcrawler -Disallow: / - -User-agent: w3c-unicorn -Disallow: / - -User-agent: companybook-crawler -Disallow: / - -User-agent: crystalsemanticsbot -Disallow: / - -User-agent: surdotlybot -Disallow: / - -User-agent: bot-araturka-com -Disallow: / - -User-agent: netestate-ne-crawler -Disallow: / - -User-agent: jetbot -Disallow: / - -User-agent: augure -Disallow: / - -User-agent: w3c-css-validator -Disallow: / - -User-agent: livelapbot -Disallow: / - -User-agent: seekportbot -Disallow: / - -User-agent: edisterbot -Disallow: / - -User-agent: bidswitchbot -Disallow: / - -User-agent: slack-imgproxy -Disallow: / - -User-agent: brainobot -Disallow: / - -User-agent: ips-agent -Disallow: / - -User-agent: mytwip -Disallow: / - -User-agent: rssingbot -Disallow: / - -User-agent: auramundi -Disallow: / - -User-agent: pagething -Disallow: / - -User-agent: titan -Disallow: / - -User-agent: zeus -Disallow: / - -User-agent: deusu -Disallow: / - -User-agent: gnam-gnam-spider -Disallow: / - -User-agent: mindupbot -Disallow: / - -User-agent: gowikibot -Disallow: / - -User-agent: e-ventures-investment-crawler -Disallow: / - -User-agent: true-robot -Disallow: / - -User-agent: validator-nu -Disallow: / - -User-agent: ec2linkfinder -Disallow: / - -User-agent: alphabot -Disallow: / - -User-agent: zuperlistbot -Disallow: / - -User-agent: linespider -Disallow: / - -User-agent: linkedinbot -Disallow: / - -User-agent: copyrightcheck -Disallow: / - -User-agent: sbl-bot -Disallow: / - -User-agent: knowings -Disallow: / - -User-agent: proxem -Disallow: / - -User-agent: whatsapp -Disallow: / - -User-agent: 2ip-ru -Disallow: / - -User-agent: yanga -Disallow: / - -User-agent: sysomos -Disallow: / - -User-agent: vecteurplus -Disallow: / - -User-agent: black-hole -Disallow: / - -User-agent: ellisphere -Disallow: / - -User-agent: semanticscholarbot -Disallow: / - -User-agent: tineye-bot -Disallow: / - -User-agent: neticle-crawler -Disallow: / - -User-agent: socialrankiobot -Disallow: / - -User-agent: oracle-ultra-search -Disallow: / - -User-agent: webmon -Disallow: / - -User-agent: metajobbot -Disallow: / - -User-agent: nextgensearchbot -Disallow: / - -User-agent: w3c-mobileok -Disallow: / - -User-agent: electricmonk -Disallow: / - -User-agent: searchpreview -Disallow: / - -User-agent: seekbot -Disallow: / - -User-agent: npbot -Disallow: / - -User-agent: www-uptime-com -Disallow: / - -User-agent: ridderbot -Disallow: / - -User-agent: embedly -Disallow: / - -User-agent: dittospyder -Disallow: / - -User-agent: vebidoobot -Disallow: / - -User-agent: leikibot -Disallow: / - -User-agent: kenjin-spider -Disallow: / - -User-agent: icc-crawler -Disallow: / - -User-agent: spankbot -Disallow: / - -User-agent: exabot -Disallow: / - -User-agent: superbot -Disallow: / - -User-agent: mata-hari -Disallow: / - -User-agent: sogou-spider2 -Disallow: / - -User-agent: a6-indexer -Disallow: / - -User-agent: wordupinfosearch -Disallow: / - -User-agent: james-bot -Disallow: / - -User-agent: httrack-3-0 -Disallow: / - -User-agent: tangibleebot -Disallow: / - -User-agent: fyrebot -Disallow: / - -User-agent: snap-url-preview-service -Disallow: / - -User-agent: jetslide -Disallow: / - -User-agent: radiation-retriever-1-1 -Disallow: / - -User-agent: www-mechanize -Disallow: / - -User-agent: advbot -Disallow: / - -User-agent: arabot -Disallow: / - -User-agent: webster-pro -Disallow: / - -User-agent: spanner -Disallow: / - -User-agent: grub-client -Disallow: / - -User-agent: safesearch-microdata-crawler -Disallow: / - -User-agent: blp-bbot -Disallow: / - -User-agent: rma -Disallow: / - -User-agent: genieo -Disallow: / - -User-agent: 2ip-bot -Disallow: / - -User-agent: dubbotbot -Disallow: / - -User-agent: piplbot -Disallow: / - -User-agent: bazqux -Disallow: / - -User-agent: jooblebot -Disallow: / - -User-agent: perman -Disallow: / - -User-agent: speedy -Disallow: / - -User-agent: facebookexternalhit -Disallow: / - -User-agent: httrack -Disallow: / - -User-agent: acapbot -Disallow: / - -User-agent: kbcrawl -Disallow: / - -User-agent: admantx -Disallow: / - -User-agent: turingos -Disallow: / - -User-agent: prowebwalker -Disallow: / - -User-agent: ocarinabot -Disallow: / - -User-agent: contxbot -Disallow: / - -User-agent: swimgbot -Disallow: / - -User-agent: alexabot -Disallow: / - -User-agent: google-physicalweb -Disallow: / - -User-agent: sitesucker -Disallow: / - -User-agent: google-favicon -Disallow: / - -User-agent: etaospider -Disallow: / - -User-agent: capsulechecker -Disallow: / - -User-agent: talkwater -Disallow: / - -User-agent: mbcrawler -Disallow: / - -User-agent: tigerbot -Disallow: / - -User-agent: lb-spider -Disallow: / - -User-agent: linkapediabot -Disallow: / - -User-agent: seekport-crawler -Disallow: / - -User-agent: propowerbot -Disallow: / - -User-agent: foobot -Disallow: / - -User-agent: yacybot -Disallow: / - -User-agent: ruxitsynthetic -Disallow: / - -User-agent: chrome-lighthouse -Disallow: / - -User-agent: teoma -Disallow: / - -User-agent: ntentbot -Disallow: / - -User-agent: lnspiderguy -Disallow: / - -User-agent: webenhancer -Disallow: / - -User-agent: 008 -Disallow: / - -User-agent: miixpc -Disallow: / - -User-agent: femtosearchbot -Disallow: / - -User-agent: vkrobot -Disallow: / - -User-agent: europarchive-org -Disallow: / - -User-agent: moreover -Disallow: / - -User-agent: msnbot-media -Disallow: / - -User-agent: linko -Disallow: / - -User-agent: voluumdsp-content-bot -Disallow: / - -User-agent: synthesio -Disallow: / - -User-agent: winello -Disallow: / - -User-agent: luminator-robots -Disallow: / - -User-agent: cheesebot -Disallow: / - -User-agent: turnitin -Disallow: / - -User-agent: nimblecrawler -Disallow: / - -User-agent: reverseengineeringbot -Disallow: / - -User-agent: experibot -Disallow: / - -User-agent: scribdbot -Disallow: / - -User-agent: zite -Disallow: / - -User-agent: dareboost -Disallow: / - -User-agent: trendkite-akashic-crawler -Disallow: / - -User-agent: ias-crawler -Disallow: / - -User-agent: google-safety -Disallow: / - -User-agent: erocrawler -Disallow: / - -User-agent: gwene -Disallow: / - -User-agent: magpie-crawler -Disallow: / - -User-agent: safednsbot -Disallow: / - -User-agent: zgrab -Disallow: / - -User-agent: lexibot -Disallow: / - -User-agent: gigabot -Disallow: / - -User-agent: yoozbot -Disallow: / - -User-agent: rytebot -Disallow: / - -User-agent: clickagy -Disallow: / - -User-agent: digg-deeper -Disallow: / - -User-agent: curebot -Disallow: / - -User-agent: caliperbot -Disallow: / - -User-agent: twurly -Disallow: / - -User-agent: megaindex -Disallow: / - -User-agent: simplepie -Disallow: / - -User-agent: theoldreader-com -Disallow: / - -User-agent: cispa-webcrawler -Disallow: / - -User-agent: jamesbot -Disallow: / - -User-agent: buck -Disallow: / - -User-agent: lwp-trivial -Disallow: / - -User-agent: haosouspider -Disallow: / - -User-agent: repomonkey-bait-tackle -Disallow: / - -User-agent: b2w -Disallow: / - -User-agent: libweb -Disallow: / - -User-agent: corporama -Disallow: / - -User-agent: blekkobot -Disallow: / - -User-agent: linkdex -Disallow: / - -User-agent: webcopier -Disallow: / - -User-agent: vci-webviewer-vci-webviewer-win32 -Disallow: / - -User-agent: webreaper -Disallow: / - -User-agent: fluffy -Disallow: / - -User-agent: newspaper -Disallow: / - -User-agent: asterias -Disallow: / - -User-agent: humanlinks -Disallow: / - -User-agent: fever -Disallow: / - -User-agent: updownerbot -Disallow: / - -User-agent: bubing -Disallow: / - -User-agent: newsharecounts -Disallow: / - -User-agent: linguee-bot -Disallow: / - -User-agent: toutiaospider -Disallow: / - -User-agent: netvibes -Disallow: / - -User-agent: webauto -Disallow: / - -User-agent: fuelbot -Disallow: / - -User-agent: alexandriaorgbot -Disallow: / - -User-agent: newzbin -Disallow: / - -User-agent: integromedb -Disallow: / - -User-agent: tineye -Disallow: / - -User-agent: botify -Disallow: / - -User-agent: semanticbot -Disallow: / - -User-agent: webcompanycrawler -Disallow: / - -User-agent: it2media-domain-crawler -Disallow: / - -User-agent: dnyzbot -Disallow: / - -User-agent: inoreader -Disallow: / - -User-agent: cookiebot -Disallow: / - -User-agent: jyxobot -Disallow: / - -User-agent: w3c-i18n-checker -Disallow: / - -User-agent: wiseguys-robot -Disallow: / - -User-agent: orthogaffe -Disallow: / - -User-agent: upflow -Disallow: / - -User-agent: paperlibot -Disallow: / - -User-agent: linkcheck-by-siteimprove-com -Disallow: / - -User-agent: findlink -Disallow: / - -User-agent: cherrypickerse -Disallow: / - -User-agent: bitsightbot -Disallow: / - -User-agent: criteobot -Disallow: / - -User-agent: convera -Disallow: / - -User-agent: domain-re-animator-bot -Disallow: / - -User-agent: larbin -Disallow: / - -User-agent: grobbot -Disallow: / - -User-agent: panscient -Disallow: / - -User-agent: ms-search-6-0-robot -Disallow: / - -User-agent: g00g1e-net -Disallow: / - -User-agent: tracemyfile -Disallow: / - -User-agent: meltwater -Disallow: / - -User-agent: xenus-link-sleuth-1-1c -Disallow: / - -User-agent: mtrobot -Disallow: / - -User-agent: israbot -Disallow: / - -User-agent: semrushbot-bm -Disallow: / - -User-agent: cliqzbot -Disallow: / - -User-agent: webstripper -Disallow: / - -User-agent: bdcbot -Disallow: / - -User-agent: multiviewbot -Disallow: / - -User-agent: atom-feed-robot -Disallow: / - -User-agent: adsbot-google-mobile -Disallow: / - -User-agent: naver-blog-rssbot -Disallow: / - -User-agent: gingercrawler -Disallow: / - -User-agent: cincraw -Disallow: / - -User-agent: iskanie -Disallow: / - -User-agent: neevabot -Disallow: / - -User-agent: w3c-checklink -Disallow: / - -User-agent: trendsmapresolver -Disallow: / - -User-agent: synapse -Disallow: / - -User-agent: zabbix -Disallow: / - -User-agent: btwebclient -Disallow: / - -User-agent: msrbot -Disallow: / - -User-agent: inetdex-bot -Disallow: / - -User-agent: seobilitybot -Disallow: / - -User-agent: sitesnagger -Disallow: / - -User-agent: wocbot -Disallow: / - -User-agent: storygizebot -Disallow: / - -User-agent: better-uptime-bot -Disallow: / - -User-agent: yahoo-pipes-1-0 -Disallow: / - -User-agent: moatbot -Disallow: / - -User-agent: superfeedr -Disallow: / - -User-agent: sempi-tech -Disallow: / - -User-agent: copernic -Disallow: / - -User-agent: duckduckgo-favicons-bot -Disallow: / - -User-agent: indeedbot -Disallow: / - -User-agent: zealbot -Disallow: / - -User-agent: kemvibot -Disallow: / - -User-agent: mauibot -Disallow: / - -User-agent: deepcrawl -Disallow: / - -User-agent: scoop-it -Disallow: / - -User-agent: winhttrack -Disallow: / - -User-agent: muckrack -Disallow: / - -User-agent: mention -Disallow: / - -User-agent: mail-ru-bot -Disallow: / - -User-agent: leadbox -Disallow: / - -User-agent: seolizer -Disallow: / - -User-agent: acunetix-web-vulnerability-scanner -Disallow: / - -User-agent: awariorssbot -Disallow: / - -User-agent: openindexspider -Disallow: / - -User-agent: slurp -Disallow: / - -User-agent: yisouspider -Disallow: / - -User-agent: search-marginalia-nu -Disallow: / - -User-agent: openbot -Disallow: / - -User-agent: adidxbot -Disallow: / - -User-agent: macocu -Disallow: / - -User-agent: zenback-bot -Disallow: / - -User-agent: yellowbrandprotectionbot -Disallow: / - -User-agent: simplecrawler -Disallow: / - -User-agent: nicerspro -Disallow: / - -User-agent: infonavirobot -Disallow: / - -User-agent: traackr-com -Disallow: / - -User-agent: linkdexbot -Disallow: / - -User-agent: truliabot -Disallow: / - -User-agent: newsblur -Disallow: / - -User-agent: grub-org -Disallow: / - User-Agent: Bytespider Disallow: /