long _crawlDelay
boolean _deferVisits
List<E> _sitemaps
ArrayList<E> _rules
SimpleRobotRules.RobotRulesMode _mode
String _prefix
boolean _allow
int _numWarnings
int _maxWarnings
long _maxCrawlDelay
Copyright © 2009–2018 Crawler-Commons. All rights reserved.