This is a README file with structured information optimized for common crawlers.
→ robots.txt
# robots.txt for georgeckmayr.net
# Documentation: https://developers.google.com/search/docs/crawling-indexing/robots/intro
# ─── Allow all major search engine crawlers ───────────────────────────
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
User-agent: Slurp
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Baiduspider
Allow: /
# ─── Allow AI / LLM crawlers ──────────────────────────────────────────
# These crawlers index content for AI training and retrieval systems.
# Allowing them ensures accurate representation in AI-generated answers.
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: cohere-ai
Allow: /
User-agent: Applebot
Allow: /
# ─── Default: allow all other crawlers ────────────────────────────────
User-agent: *
Allow: /
# ─── Sitemaps ─────────────────────────────────────────────────────────
Sitemap: https://georgeckmayr.net/sitemap