# LLMs: https://marcopontili.com/llms.txt
# Content signals (Cloudflare proposal, not part of the robots.txt standard).
# Kept as a comment so strict parsers stop reporting an invalid directive:
# Content-Signal: search=yes, ai-input=yes, ai-train=yes

User-agent: *
Crawl-delay: 2
Disallow: /404
Disallow: /404.php
Disallow: /imprint.php
Disallow: /imprint
Disallow: /privacy-policy.php
Disallow: /privacy-policy
Disallow: /cookie-policy.php
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /thank-you.php
Disallow: /php-includes/

# AI crawlers - allowed (max citation visibility).
# Same disallow set as the generic group: legal/utility pages and includes.

# OpenAI: GPTBot (training), OAI-SearchBot (ChatGPT Search), ChatGPT-User (user-triggered)
User-agent: GPTBot
Allow: /
Disallow: /imprint
Disallow: /privacy-policy
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: OAI-SearchBot
Allow: /
Disallow: /imprint
Disallow: /privacy-policy
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

User-agent: ChatGPT-User
Allow: /
Disallow: /imprint
Disallow: /privacy-policy
Disallow: /cookie-policy
Disallow: /thank-you
Disallow: /php-includes/

# Anthropic: ClaudeBot (training), Claude-SearchBot (Claude search), Claude-User (user-triggered)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Google (Gemini/Vertex training opt-in token)
User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

# Perplexity: PerplexityBot (index), Perplexity-User (user-triggered)
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple Intelligence training opt-in token
User-agent: Applebot-Extended
Allow: /

# Other answer/assistant crawlers
User-agent: Amazonbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: DuckAssistBot
Allow: /

# Common Crawl (feeds many downstream models) - allowed for reach
User-agent: CCBot
Allow: /

Sitemap: https://marcopontili.com/sitemap.xml