# https://www.robotstxt.org/robotstxt.html
#
# Policy (ops/nginx/README.md, "AI crawler policy"): search engines and AI
# search / assistant fetchers are welcome, AI training crawlers are not.
# This file is the policy. Cloudflare's managed robots.txt stays OFF: it
# prepends its own groups, disallows the assistant fetchers too, and enforces
# all of it at the edge.
User-agent: *
# Content Signals (contentsignals.org): search and answer-grounding may use
# the pages, model training may not.
Content-Signal: search=yes,ai-train=no,use=reference
# Signed-in and form-only pages: every facility page links to
# /daycare-portal?claim=<id>, which made ~230k crawlable URLs of one login
# shell. The bare /daycare-portal (the provider landing page) stays crawlable,
# and so does /api/: Google needs it to render client-side pages.
Disallow: /daycare-portal?
Disallow: /daycare-portal/
Disallow: /alerts
Disallow: /profile
Disallow: /account
Disallow: /legal/daycare-injury?

# AI training crawlers: they copy the corpus and send nobody. The search and
# assistant agents of the same companies (OAI-SearchBot, ChatGPT-User,
# Claude-SearchBot, Claude-User, PerplexityBot, Perplexity-User, Applebot,
# DuckAssistBot, MistralAI-User) are deliberately not listed here.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Diffbot
User-agent: omgili
User-agent: cohere-ai
User-agent: MistralAI-Training
Disallow: /

# Backlink-index crawlers: declared, well behaved, and worth nothing to this
# site (no Semrush or SE Ranking account). SERankingBacklinksBot pulled about
# 1,000 facility pages an hour on 2026-10-03.
User-agent: SemrushBot
User-agent: SERankingBacklinksBot
Disallow: /

Sitemap: https://daycarealert.com/sitemap.xml