# robots.txt — clean, GSC-friendly, crawl-safe
# Purpose: allow indexing of core URLs while reducing crawl pressure
# Note: robots.txt does NOT stop abusive bots — use WAF / rate limiting for that

# =========================
# 1) Default rules (apply to all crawlers)
# =========================
User-agent: *

# Block crawl-space / internal parameters
Disallow: /*?rText=
Disallow: /*&rText=
Disallow: /*?page=
Disallow: /*&page=
Disallow: /*?orderBy=
Disallow: /*&orderBy=
Disallow: /*?pais=
Disallow: /*&pais=
Disallow: /*__hstc=

# Block sensitive paths
Disallow: /login

Allow: /

# =========================
# 2) Googlebot (explicit — no crawl-delay, Google ignores it anyway)
# =========================
User-agent: Googlebot
Disallow: /*?rText=
Disallow: /*&rText=
Disallow: /*?page=
Disallow: /*&page=
Disallow: /*?orderBy=
Disallow: /*&orderBy=
Disallow: /*?pais=
Disallow: /*&pais=
Disallow: /*__hstc=
Disallow: /login
Allow: /

# =========================
# 3) Bingbot (crawl-delay supported)
# =========================
User-agent: Bingbot
Crawl-delay: 2
Disallow: /*?rText=
Disallow: /*&rText=
Disallow: /*?page=
Disallow: /*&page=
Disallow: /*?orderBy=
Disallow: /*&orderBy=
Disallow: /*?pais=
Disallow: /*&pais=
Disallow: /*__hstc=
Disallow: /login
Allow: /

# =========================
# 4) Applebot (crawl-delay supported)
# =========================
User-agent: Applebot
Crawl-delay: 2
Disallow: /*?rText=
Disallow: /*&rText=
Disallow: /*?page=
Disallow: /*&page=
Disallow: /*?orderBy=
Disallow: /*&orderBy=
Disallow: /*?pais=
Disallow: /*&pais=
Disallow: /*__hstc=
Disallow: /login
Allow: /

# =========================
# 5) AI / training-related crawlers — blocked
# =========================
User-agent: Applebot-Extended
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Amazonbot
Disallow: /

# =========================
# 6) SEO crawlers / scrapers — blocked
# =========================
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: duggmirror
Disallow: /

User-agent: asterias
Disallow: /

User-agent: BackDoorBot
Disallow: /

User-agent: BlowFish
Disallow: /

User-agent: BotALot
Disallow: /

User-agent: BuiltBotTough
Disallow: /

User-agent: BunnySlippers
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebCopier
Disallow: /

# =========================
# 7) Sitemaps
# =========================
Sitemap: https://www.seminuevos.com/sitemap-blog.xml
Sitemap: https://www.seminuevos.com/sitemap-dealers.xml
Sitemap: https://www.seminuevos.com/sitemap-fichas.xml
Sitemap: https://www.seminuevos.com/sitemap-long-tail-seo-1.xml
Sitemap: https://www.seminuevos.com/sitemap-long-tail-seo-2.xml
Sitemap: https://www.seminuevos.com/sitemap-other.xml
Sitemap: https://www.seminuevos.com/sitemap_01.xml
Sitemap: https://www.seminuevos.com/sitemap_02.xml
Sitemap: https://www.seminuevos.com/sitemap_03.xml
Sitemap: https://www.seminuevos.com/sitemap_04.xml
Sitemap: https://www.seminuevos.com/sitemap_05.xml
Sitemap: https://www.seminuevos.com/sitemap_06.xml
Sitemap: https://www.seminuevos.com/sitemap_07.xml
Sitemap: https://www.seminuevos.com/sitemap_08.xml
Sitemap: https://www.seminuevos.com/sitemap_09.xml
Sitemap: https://www.seminuevos.com/sitemap_10.xml
Sitemap: https://www.seminuevos.com/sitemap_11.xml
Sitemap: https://www.seminuevos.com/sitemap_12.xml
Sitemap: https://www.seminuevos.com/sitemap_13.xml
Sitemap: https://www.seminuevos.com/sitemap_14.xml
Sitemap: https://www.seminuevos.com/sitemap_15.xml