100 lines
1.5 KiB
Plaintext
100 lines
1.5 KiB
Plaintext
User-agent: MJ12bot
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: AdsBot*
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Mediapartners*
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: *SemrushBot*
|
|
|
|
Disallow: /
|
|
|
|
User-agent: BLEXBot
|
|
|
|
Disallow: /
|
|
|
|
User-agent: DataForSeoBot
|
|
|
|
Disallow: /
|
|
|
|
User-agent: PetalBot
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: *
|
|
|
|
Disallow: /api/*
|
|
Disallow: /avatars
|
|
Disallow: /user/*
|
|
Disallow: /*/*/src/commit/*
|
|
Disallow: /*/*/commit/*
|
|
Disallow: /*/*/*/refs/*
|
|
Disallow: /*/*/*/star
|
|
Disallow: /*/*/*/watch
|
|
Disallow: /*/*/labels
|
|
Disallow: /*/*/activity/*
|
|
Disallow: /vendor/*
|
|
Disallow: /swagger.*.json
|
|
|
|
Disallow: /*?lang=
|
|
|
|
Disallow: */pulse
|
|
Disallow: */tree/*
|
|
Disallow: */blob/*
|
|
Disallow: */wiki/
|
|
Disallow: /gist/
|
|
Disallow: /oembed
|
|
Disallow: */forks
|
|
Disallow: */stars
|
|
Disallow: */download
|
|
Disallow: */revisions
|
|
Disallow: */issues/new
|
|
Disallow: */issues/search
|
|
Disallow: */commits/
|
|
Disallow: */commits/*?author
|
|
Disallow: */commits/*?path
|
|
Disallow: */branches
|
|
Disallow: */tags
|
|
Disallow: */contributors
|
|
Disallow: */comments
|
|
Disallow: */tarball/
|
|
Disallow: */zipball/
|
|
Disallow: */raw/*
|
|
Disallow: */followers
|
|
Disallow: */following
|
|
Disallow: */archive/
|
|
Disallow: */blame/
|
|
Disallow: */watchers
|
|
Disallow: */network
|
|
Disallow: */graphs
|
|
Disallow: */raw/
|
|
Disallow: */compare/
|
|
Disallow: */cache/
|
|
Disallow: */blame/
|
|
Disallow: */network
|
|
Disallow: */graphs
|
|
Disallow: */raw/
|
|
Disallow: */compare/
|
|
Disallow: */cache/
|
|
Disallow: /.git/
|
|
Disallow: */.git/
|
|
Disallow: /*.git$
|
|
Disallow: /*/sitemap.xml
|
|
Disallow: /search/advanced
|
|
Disallow: /search
|
|
Disallow: */search
|
|
Disallow: /*q=
|
|
Disallow: /*.atom
|
|
Disallow: /assets/
|
|
|
|
Crawl-delay: 3
|