# robots.txt — HostGrade.pl
# Last updated: 2026-04-17
# AI crawlers are explicitly ALLOWED for GEO (Generative Engine Optimisation)

# -----------------------------------------------------------------
# Default: all crawlers
# -----------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/
# Content usage preferences (draft-romm-aipref-contentsignals) are now
# served via the X-Content-Signal HTTP response header on every page —
# Google PSI's strict RFC 9309 parser flagged the in-robots directive as
# "Unknown", even though content-signal-aware crawlers (Cloudflare, OpenAI)
# accept both channels.

# -----------------------------------------------------------------
# OpenAI (ChatGPT / GPT-4o training + browsing)
# -----------------------------------------------------------------
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Anthropic (Claude)
# -----------------------------------------------------------------
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Perplexity AI
# -----------------------------------------------------------------
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Google (Gemini training + extended AI)
# -----------------------------------------------------------------
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Apple (Siri, Apple Intelligence)
# -----------------------------------------------------------------
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Common Crawl (LLM training datasets)
# -----------------------------------------------------------------
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Cohere AI
# -----------------------------------------------------------------
User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Amazon (Alexa AI / Bedrock)
# -----------------------------------------------------------------
User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# ByteDance (TikTok AI / Doubao)
# -----------------------------------------------------------------
User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Diffbot (structured web intelligence)
# -----------------------------------------------------------------
User-agent: Diffbot
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Timpi (decentralised AI index)
# -----------------------------------------------------------------
User-agent: Timpibot
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Omgili / Omgilibot (media intelligence)
# -----------------------------------------------------------------
User-agent: Omgili
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

User-agent: Omgilibot
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Google (standard web index — explicit for clarity)
# -----------------------------------------------------------------
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /out/
Disallow: /cdn-cgi/
Disallow: /_astro/

# -----------------------------------------------------------------
# Sitemap index + individual sitemaps
# -----------------------------------------------------------------
Sitemap: https://hostgrade.pl/sitemap-index.xml
Sitemap: https://hostgrade.pl/sitemap-pages.xml
Sitemap: https://hostgrade.pl/sitemap-providers.xml
Sitemap: https://hostgrade.pl/sitemap-knowledge.xml