# robots.txt https://www.westfalen-blatt.de User-agent: * Disallow: /api/ # Disallow indexation of content copiers and other unwanted robots User-agent: UbiCrawler Disallow: / User-agent: DOC Disallow: / User-agent: Zao Disallow: / User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: libwww Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / User-agent: grub-client Disallow: / User-agent: k2spider Disallow: / User-agent: NPBot Disallow: / User-agent: WebReaper Disallow: / User-agent: CrystalSemanticsBot Disallow: / User-agent: CompSpyBot Disallow: / User-agent: Curious George Disallow: / User-agent: CybEye.com Disallow: / User-agent: DoCoMo Disallow: / User-agent: ExB Language Crawler Disallow: / User-agent: Ezooms Disallow: / User-agent: Flamingo_SearchEngine Disallow: / User-agent: Genieo Disallow: / User-agent: Genio Disallow: / User-agent: LWNutch Disallow: / User-agent: LexxeBot Disallow: / User-agent: OpenWebIndex Disallow: / User-agent: RediffNewsBot Disallow: / User-agent: SEOENGWorldBot Disallow: / User-agent: Scanmine Disallow: / User-agent: ShopWiki Disallow: / User-agent: ShowyouBot Disallow: / User-agent: Sosospider Disallow: / User-agent: WocBot Disallow: / User-agent: Yeti Disallow: / User-agent: YoudaoBot Disallow: / User-agent: daumoa Disallow: / User-agent: gsa-crawler Disallow: / User-agent: libcrawl Disallow: / User-agent: linkdex Disallow: / User-agent: magpie-crawler Disallow: / User-agent: repparser Disallow: / User-agent: rogerbot Disallow: / User-agent: sindice-site-manager Disallow: / User-agent: sogou spider Disallow: / User-agent: sogou Disallow: / User-agent: woriobot Disallow: / User-agent: yacybot Disallow: / User-agent: yolinkBot Disallow: / User-agent: UnisterBot Disallow: / User-agent: JikeSpider Disallow: / User-agent: Orthogaffe Disallow: / User-agent: wget Disallow: / User-agent: Yandex Disallow: / user-agent: AhrefsBot disallow: / User-Agent: trendictionbot Disallow: / User-agent: SeznamBot Disallow: / User-agent: search:robot Disallow: / User-agent: MJ12bot Disallow: / User-agent: XoviBot Disallow: / User-agent: xovi Disallow: / User-agent: CCBot Disallow: / User-agent: ChatGPT-User Disallow: / User-agent: GPTBot Disallow: / User-agent: Cutbot Disallow: / # http://linkfluence.net/ User-agent: Kraken Disallow: / # Yahoo User-agent: Slurp Crawl-delay: 10 User-agent: bingbot Crawl-delay: 2 max-image-preview:large Sitemap: https://www.westfalen-blatt.de/sitemaps/sitemap-index.xml Sitemap: https://www.westfalen-blatt.de/sitemaps/newssitemap-index.xml