38 lines
629 B
Plaintext
38 lines
629 B
Plaintext
# Robots.txt for License Manager
|
|
|
|
# 百度爬虫特殊配置
|
|
User-agent: Baiduspider
|
|
Allow: /
|
|
Crawl-delay: 1
|
|
|
|
# 360搜索爬虫
|
|
User-agent: 360Spider
|
|
Allow: /
|
|
Crawl-delay: 1
|
|
|
|
# 搜狗爬虫
|
|
User-agent: Sogou web spider
|
|
Allow: /
|
|
Crawl-delay: 1
|
|
|
|
# 其他搜索引擎
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
# 允许搜索引擎索引所有内容
|
|
Allow: /guide/
|
|
Allow: /en/
|
|
Allow: /docs-blog/
|
|
|
|
# 阻止索引一些不重要的路径
|
|
Disallow: /node_modules/
|
|
Disallow: /.vitepress/
|
|
Disallow: /.github/
|
|
Disallow: /dist/
|
|
|
|
# XML Sitemap
|
|
Sitemap: https://docs.lm.cedar-v.com/sitemap.xml
|
|
|
|
# HTML Sitemap
|
|
Sitemap: https://docs.lm.cedar-v.com/sitemap.html
|