# robots.txt for Pipette.bio
# Controls how search engines crawl and index pages

User-agent: *
Allow: /

# Block non-public areas
Disallow: /app/
Disallow: /admin/
Disallow: /api/

# Allow all use case pages (explicit for clarity)
Allow: /use-cases/
Allow: /use-cases/*.html

# Allow blog pages
Allow: /blog/
Allow: /blog/*.html

# Block any temporary or draft directories (if you have them)
Disallow: /drafts/
Disallow: /temp/
Disallow: /_archive/

# Sitemap location
Sitemap: https://pipette.bio/sitemap.xml

# LLM-readable site summary
# See https://llmstxt.org for spec
LLMs-txt: https://pipette.bio/llms.txt

# Crawl delay (polite to server)
Crawl-delay: 1

# Specific bot rules
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Block AI scrapers if you don't want your content used for training
# (Optional - uncomment if you want to block them)
# User-agent: GPTBot
# Disallow: /
# 
# User-agent: ChatGPT-User
# Disallow: /
# 
# User-agent: CCBot
# Disallow: /
# 
# User-agent: anthropic-ai
# Disallow: /
# 
# User-agent: Claude-Web
# Disallow: /

# Block problematic bots (optional)
# User-agent: MJ12bot
# Disallow: /
# 
# User-agent: AhrefsBot
# Disallow: /
# 
# User-agent: SemrushBot
# Disallow: /

# End of file