long _crawlDelay
boolean _deferVisits
List<E> _sitemaps
ArrayList<E> _rules
SimpleRobotRules.RobotRulesMode _mode
String _prefix
boolean _allow
int _numWarnings
String error
Copyright © 2009–2017 Crawler-Commons. All rights reserved.