# robots.txt for www.cdalawnservice.com (Aspen Lawn Care)
# Configured for MAXIMUM AI VISIBILITY per AI_Visibility_Crawler_Readiness_Standard_2.
# Net-new at migration (the live site has no robots.txt today — a pure, additive infra add).
#
# NOTE: robots.txt must return HTTP 200 (a 404 = "allow everything"; a 5xx pauses Googlebot).
# AI visibility also depends on the host/WAF — audit it after cutover and un-block
# OAI-SearchBot, Claude-SearchBot, PerplexityBot, Bingbot, Applebot. Stealth crawlers
# (Perplexity's undeclared bot, Bytespider) need a host/WAF block — robots.txt alone won't stop them.

User-agent: *
Allow: /

Allow: /*.css$
Allow: /*.js$
Allow: /*.webp$
Allow: /*.avif$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.svg$

# (/thank-you is NOT Disallowed: it carries meta robots="noindex" and must stay crawlable so the
#  bot can SEE the noindex; it is simply excluded from sitemap.xml.)

# ── OpenAI — allow all three (training + search + user-initiated) ──
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# ── Anthropic — allow all three (training + search + user-initiated) ──
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Google generative AI (training + Gemini grounding); Googlebot itself is governed by the * block
User-agent: Google-Extended
Allow: /

# Apple (Siri / Spotlight + Apple Intelligence training)
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Microsoft / Bing (also powers Copilot grounding)
User-agent: Bingbot
Allow: /

# Perplexity (declared bot; stealth crawler documented — host/WAF-block that separately)
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Meta AI / Llama training
User-agent: meta-externalagent
Allow: /

# Common Crawl (feeds many open models)
User-agent: CCBot
Allow: /

# ── Block: ByteDance / Bytespider (documented history of ignoring robots.txt; host/WAF block too) ──
User-agent: Bytespider
Disallow: /

Sitemap: https://www.cdalawnservice.com/sitemap.xml