# TLM Marine — robots.txt
# Last updated: 2026-06-15
# We explicitly welcome AI/LLM crawlers for citation and discovery.
# Form-submission endpoint is excluded; assets are public.

# ---------------------------------------------------------------
# Default rules for all crawlers
# ---------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /send.php

# ---------------------------------------------------------------
# AI / LLM crawlers — explicitly allowed
# These bots train on or cite content in AI answer engines.
# Keeping them whitelisted improves visibility in ChatGPT,
# Claude, Perplexity, Gemini, Copilot, and Apple Intelligence.
# ---------------------------------------------------------------

# OpenAI — training crawler (ChatGPT, GPT models)
User-agent: GPTBot
Allow: /
Disallow: /send.php

# OpenAI — real-time fetch when a ChatGPT user opens a URL
User-agent: ChatGPT-User
Allow: /

# OpenAI — ChatGPT Search index
User-agent: OAI-SearchBot
Allow: /
Disallow: /send.php

# Anthropic — Claude training crawler
User-agent: ClaudeBot
Allow: /
Disallow: /send.php

# Anthropic — real-time fetch for Claude users
User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity — search index
User-agent: PerplexityBot
Allow: /
Disallow: /send.php

# Perplexity — real-time fetch for Perplexity users
User-agent: Perplexity-User
Allow: /

# Google — Gemini / Vertex AI training (separate opt-in from Googlebot)
User-agent: Google-Extended
Allow: /
Disallow: /send.php

# Common Crawl — base dataset for most open LLMs
User-agent: CCBot
Allow: /
Disallow: /send.php

# Microsoft Bing — powers Copilot answer engine
User-agent: Bingbot
Allow: /
Disallow: /send.php

# Apple — Apple Intelligence training opt-in
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /
Disallow: /send.php

# DuckDuckGo Assist
User-agent: DuckAssistBot
Allow: /

# Cohere — model training
User-agent: cohere-ai
Allow: /
Disallow: /send.php

# Meta / Llama — answer engine + AI training
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /send.php

User-agent: FacebookBot
Allow: /

# Diffbot — entity graph used by many AI systems
User-agent: Diffbot
Allow: /

# Amazon — Alexa, Bedrock, Rufus
User-agent: Amazonbot
Allow: /

# Mistral — model training
User-agent: MistralAI-User
Allow: /

# You.com
User-agent: YouBot
Allow: /

# ---------------------------------------------------------------
# Sitemap
# ---------------------------------------------------------------
Sitemap: https://tlm-marine.com/sitemap.xml