# robots.txt for SF999.com - 找私服传奇发布站
# 遵循百度爬虫协议

User-agent: *
Allow: /
Allow: /chuanqi/
Allow: /SF999/
Allow: /news/
Allow: /rank/

# 禁止抓取
Disallow: /admin/
Disallow: /login/
Disallow: /register/
Disallow: /user/
Disallow: /api/
Disallow: /*.php$
Disallow: /*.asp$
Disallow: /*.jsp$

# 百度爬虫专用规则
User-agent: Baiduspider
Allow: /
Allow: /chuanqi/
Allow: /SF999/
Allow: /news/
Allow: /rank/
Crawl-delay: 1

# Google爬虫
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# 360爬虫
User-agent: 360Spider
Allow: /

# 搜狗爬虫
User-agent: Sogou web spider
Allow: /

# 必应爬虫
User-agent: bingbot
Allow: /

# 站点地图位置
Sitemap: https://www.SF999.com/sitemap.xml
Sitemap: https://www.SF999.com/sitemap_news.xml