# AI training crawlers: blocked (owner decision, 2 Oct 2026). Search and answer crawlers
# (Googlebot, Bingbot, OAI-SearchBot, ChatGPT-User, PerplexityBot, Claude-SearchBot, Claude-User,
# Applebot) are NOT listed, so the rule below lets them in and the site stays findable and citable.
# Meta-ExternalFetcher is not listed either since 14 Oct 2026: it fetches a page when someone asks Meta's
# assistant about it; Meta lists training for Meta-ExternalAgent, not for it (ADR-0028 amendment). No group disallows /mcp: it is an API that
# assistants call for a user, not a page, and it answers GET with 405 and noindex (ADR-0032).
# Blocking Google-Extended does not affect Google Search or AI Overviews. Unblocking it is an open owner decision.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: TikTokSpider
User-agent: PetalBot
User-agent: FacebookBot
User-agent: ProRataInc
Disallow: /

# Google's AdSense crawler ignores the "*" group below and only obeys a group that names it. Without
# this it followed every /go/ partner redirect (hundreds of hits on day one). It can still read pages.
User-agent: Mediapartners-Google
Disallow: /go/
Disallow: /api/

# Content-Signal (contentsignals.org) states the same policy in words for crawlers that read it:
# search and answers yes, training no. Crawlers that don't know the line ignore it.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /go/
Disallow: /api/

Sitemap: https://landlordlookout.co.uk/sitemap-index.xml