# robots.txt — scrollvault.net
# Updated 2026-05-06: explicit AI/LLM crawler allowlist for citation traffic.

# Default — all crawlers welcome except /data/ and /scripts/.
User-agent: *
Allow: /
Disallow: /data/
Disallow: /scripts/
Disallow: /_ops/
Disallow: /api/
Disallow: /tools/commander-bracket/scryfall-proxy.php
Disallow: /tools/manabase/deck-proxy.php

# Explicitly welcomed: AI/LLM crawlers.
# Captures AI Overview citations, ChatGPT/Claude/Perplexity referrals,
# and AI-search-result inclusion. Same disallows apply.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Bytespider
User-agent: cohere-ai
Allow: /
Disallow: /data/
Disallow: /scripts/
Disallow: /_ops/
Disallow: /api/
Disallow: /tools/commander-bracket/scryfall-proxy.php
Disallow: /tools/manabase/deck-proxy.php

Sitemap: https://scrollvault.net/sitemap.xml

# AI agent content — structured Markdown for LLM consumption
# llms.txt: site index with page descriptions
# llms-full.txt: complete FAQ and guide content in one file