User-agent: *
Allow: /

# 优化爬取频率
Crawl-delay: 1

# 禁止访问的目录
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /_astro/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /src/
Disallow: /vendor/

# 允许访问重要资源
Allow: /_astro/*.css
Allow: /_astro/*.js
Allow: /_astro/*.webp
Allow: /_astro/*.jpg
Allow: /_astro/*.png

# Sitemap位置
Sitemap: https://ipflex.ink/sitemap-index.xml
Sitemap: https://ipflex.ink/rss.xml
# LLM discoverability
Sitemap: https://ipflex.ink/llms.txt
Sitemap: https://ipflex.ink/llms-full.txt

# 指定主域名
Host: https://ipflex.ink

# 清理缓存指令
Clean-param: utm_source&utm_medium&utm_campaign&utm_term&utm_content

# 特殊爬虫规则
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot  
Allow: /
Crawl-delay: 1

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# 社交媒体爬虫
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# SEO工具爬虫
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /