# TickDB Robots.txt
# Updated: 2026-04-14

# Search Engine Crawlers (indexing)
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: BingPreview
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Baiduspider
User-agent: Sogou
User-agent: 360Spider
User-agent: Yandex
User-agent: Amazonbot
Crawl-delay: 1
Allow: /markets/
Allow: /en/markets/
Allow: /zh-tw/markets/
Allow: /blog/
Allow: /api/blog/
Allow: /api/public/
Allow: /help
Allow: /careers
Allow: /contact
Allow: /partners
Allow: /privacy
Allow: /terms
Allow: /security
Allow: /cookies
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /login
Disallow: /payment
Disallow: /verify-email

# AI Crawlers (allow indexing & AI training)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: cohere-ai
User-agent: Bytespider
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
Crawl-delay: 2
Allow: /markets/
Allow: /en/markets/
Allow: /zh-tw/markets/
Allow: /blog/
Allow: /api/blog/
Allow: /api/public/
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /login
Disallow: /payment
Disallow: /verify-email

# General Crawlers
User-agent: *
Crawl-delay: 3
Allow: /markets/
Allow: /en/markets/
Allow: /zh-tw/markets/
Allow: /blog/
Allow: /api/blog/
Allow: /api/public/
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /login
Disallow: /payment
Disallow: /verify-email

Sitemap: https://tickdb.ai/sitemap.xml
Sitemap: https://tickdb.ai/sitemap-blog.xml