# Robots.txt file for tomshw.it
# Last updated: 2026-08-12

# Search Engines and Good Bots
User-agent: *
User-agent: Applebot
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: AdsBot-Google
User-agent: Mediapartners-Google
User-agent: Bingbot
User-agent: msnbot
User-agent: DuckDuckBot
User-agent: Slurp
User-agent: Yandex
User-agent: YandexImages
User-agent: YandexBot
User-agent: AmazonAdBot
Allow: /

# System and API paths
Disallow: /cms/
Disallow: /nova-api/

# Dynamic content and filters
Disallow: /*?*keyword=
Disallow: /*?*min_rating=
Disallow: /*?*platform=
Disallow: /*?*genre=
Disallow: /*?*year=
Disallow: /*?*s=
Disallow: /*?*page=
Disallow: /*?*vertical=

# Taxonomy pages and content sections are intentionally NOT listed here.
# They serve X-Robots-Tag: noindex (see the noindex middleware) or a 404, and a
# Disallow would stop Google from ever reading it: the pages would stay indexed
# without content. Do not re-add /tag/, /brand/, /brand_tag/, /product/,
# /tipo-prodotto/ or /notizie-*/ until they are out of the index.

# Legacy and utility pages
Disallow: /img_vedi.php
Disallow: /ricerca/
Disallow: /codici-sconto/
Disallow: /cont/
Disallow: /software.php
Disallow: /network.php
Disallow: /amp_validated_url

# Social Media Bots
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: WhatsApp
Allow: /

# Allow Adasta bot
User-agent: grapeshot
Allow: /

# Allowed SEO Tools with rate limit
User-agent: AhrefsBot
Crawl-delay: 15
Allow: /

# Block known malicious, aggressive and other SEO bots
User-agent: Acunetix
User-agent: ChinaClaw
User-agent: DotBot
User-agent: FHscan
User-agent: MJ12Bot
User-agent: MauiBot
User-agent: NPBot
User-agent: NPBot-1/2.0
User-agent: Teleport
User-agent: magpie-crawler
User-agent: PiplBot
User-agent: Exabot
User-agent: HTTrack
User-agent: WebCopier
User-agent: WebReaper
User-agent: WebStripper
User-agent: WebZIP
User-agent: Wget
User-agent: Xenu
User-agent: Zao
User-agent: Zeus
User-agent: ZyBorg
User-agent: Screaming Frog SEO Spider
User-agent: rogerbot
User-agent: Bytespider
User-agent: SeznamBot
User-agent: Linguee
User-agent: ia_archiver
User-agent: Baiduspider
User-agent: proximic
User-agent: 360Spider
User-agent: SemrushBot
Disallow: /

# XML Sitemaps
Sitemap: https://www.tomshw.it/sitemap.xml
Sitemap: https://www.tomshw.it/google-news-sitemap.xml