# Main crawlers (Google, Bing, etc.)
User-agent: *
Allow: /
Allow: /docs
Allow: /docs/*

# Disallow API routes and internal Next.js files
Disallow: /api/
Disallow: /_next/

# Disallow search and filter URLs with query params (avoid duplicate content)
Disallow: /*?filter=*
Disallow: /*?s=*

# Specific bot configurations
User-agent: Googlebot
Allow: /
Allow: /docs/
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /
Disallow: /_next/

User-agent: Bingbot
Allow: /
Crawl-delay: 0

User-agent: Slurp
Allow: /
Crawl-delay: 1

# AI Crawlers - Allow access to public content
User-agent: GPTBot
Allow: /docs/

User-agent: ChatGPT-User
Allow: /docs/

User-agent: CCBot
Allow: /docs/

User-agent: anthropic-ai
Allow: /docs/

User-agent: Claude-Web
Allow: /docs/

# Social Media Crawlers - Allow for link previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

# Bad bots and scrapers - Block completely
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Baiduspider
Disallow: /

# Sitemap location
Sitemap: https://agenticants.ai/sitemap.xml

# Host (preferred domain)
Host: https://agenticants.ai