# Pilot Instructional Center - Robots.txt
# Generated: 2026-09-01T23:08:39.255Z

# ============================================
# SEARCH ENGINES & DEFAULT CRAWL RULES
# ============================================
User-agent: *
Allow: /

# Allow static assets and Astro build files
Allow: /_astro/
Allow: /*.css$
Allow: /*.js$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.gif$
Allow: /*.ico$

# Security & Privacy - Prevent crawling of internal endpoints & build caches
Disallow: /admin/
Disallow: /private/
Disallow: /dashboard/
Disallow: /api/
Disallow: /auth/
Disallow: /tmp/
Disallow: /cache/
Disallow: /backup/

# Conversion & Non-Indexable System Pages
Disallow: /*-confirmation
Disallow: /*-confirmation/
Disallow: /thank-you*
Disallow: /thankyou*
Disallow: /success*
Disallow: /submit*
Disallow: /process*
Disallow: /handler*

# Query Parameter Crawl Budget Control
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*source=
Disallow: /*?*campaign=

# ============================================
# AI & LLM CRAWLERS / SEARCH AGENTS
# ============================================
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Cohere-ai
Allow: /

# ============================================
# SITEMAP
# ============================================
Sitemap: https://pilotinstructionalcenter.com/sitemap-index.xml