diff --git a/doc/javadoc/allclasses-frame.html b/doc/javadoc/allclasses-frame.html
new file mode 100644
index 0000000..e3f731d
--- /dev/null
+++ b/doc/javadoc/allclasses-frame.html
@@ -0,0 +1,101 @@
+
+
+
+
+
+
+All Classes (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+All Classes
+
+All Classes (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+All Classes
+
+Constant Field Values (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Constant Field Values
+
+Contents
+
+
+
+
+crawlercommons.fetcher.*
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+crawlercommons.robots.*
+
+
+
+
+
+
+
+
+
+
+
+
+crawlercommons.url.*
+
+
+
+
+
+
+crawlercommons.url.EffectiveTldFinder
+
+
+
+public static final java.lang.String COMMENT "//"
+
+
+public static final char DOT 46
+
+
+public static final java.lang.String DOT_REGEX "\\."
+
+
+public static final java.lang.String ETLD_DATA "/effective_tld_names.dat"
+
+
+public static final java.lang.String EXCEPTION "!"
+
+
+public static final java.lang.String WILD_CARD "*."
+
+
+
+
+
+
+
+
+AbortedFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetchException
+ crawlercommons.fetcher.AbortedFetchException
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class AbortedFetchException extends BaseFetchException
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class crawlercommons.fetcher.BaseFetchException
+
+compareToBase , equals , getCause , getLocalizedMessage , getMessage , getStackTrace , getUrl , hashCode , initCause , printStackTrace , printStackTrace , printStackTrace , readBaseFields , setStackTrace , toString , writeBaseFields
+
+
+
+Methods inherited from class java.lang.Throwable
+
+fillInStackTrace
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, finalize, getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+AbortedFetchException
+
+public AbortedFetchException ()
+
+
+
+AbortedFetchException
+
+public AbortedFetchException (java.lang.String url,
+ AbortedFetchReason abortReason)
+
+
+
+AbortedFetchException
+
+public AbortedFetchException (java.lang.String url,
+ java.lang.String msg,
+ AbortedFetchReason abortReason)
+
+
+
+
+
+
+getAbortReason
+
+public AbortedFetchReason getAbortReason ()
+
+
+
+
+
+
+AbortedFetchReason (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ AbortedFetchReason >
+ crawlercommons.fetcher.AbortedFetchReason
+
+
+All Implemented Interfaces: java.io.Serializable, java.lang.Comparable<AbortedFetchReason >
+
+
+public enum AbortedFetchReason extends java.lang.Enum<AbortedFetchReason >
+
+
+
+
+
+
+
+
+
+
+
+
+
+Method Summary
+
+
+static AbortedFetchReason valueOf
+
+
+static AbortedFetchReason [] values
+
+
+
+Methods inherited from class java.lang.Enum
+
+clone, compareTo, equals, finalize, getDeclaringClass, hashCode, name, ordinal, toString, valueOf
+
+
+
+Methods inherited from class java.lang.Object
+
+getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Enum Constant Detail
+
+
+
+SLOW_RESPONSE_RATE
+
+public static final AbortedFetchReason SLOW_RESPONSE_RATE
+
+
+
+
+
+INVALID_MIMETYPE
+
+public static final AbortedFetchReason INVALID_MIMETYPE
+
+
+
+
+
+INTERRUPTED
+
+public static final AbortedFetchReason INTERRUPTED
+
+
+
+
+
+CONTENT_SIZE
+
+public static final AbortedFetchReason CONTENT_SIZE
+
+
+
+
+
+
+
+
+values
+
+public static AbortedFetchReason [] values ()
+
+Returns an array containing the constants of this enum type, in
+the order they are declared. This method may be used to iterate
+over the constants as follows:
+
+for (AbortedFetchReason c : AbortedFetchReason.values())
+ System.out.println(c);
+
+
+
+
+Returns: an array containing the constants of this enum type, in
+the order they are declared
+
+
+valueOf
+
+public static AbortedFetchReason valueOf (java.lang.String name)
+
+Returns the enum constant of this type with the specified name.
+The string must match exactly an identifier used to declare an
+enum constant in this type. (Extraneous whitespace characters are
+not permitted.)
+
+
+Parameters: name - the name of the enum constant to be returned.
+Returns: the enum constant with the specified name
+ Throws:
+java.lang.IllegalArgumentException - if this enum type has no constant
+with the specified name
+java.lang.NullPointerException - if the argument is null
+
+
+
+BadProtocolFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetchException
+ crawlercommons.fetcher.BadProtocolFetchException
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class BadProtocolFetchException extends BaseFetchException
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class crawlercommons.fetcher.BaseFetchException
+
+compareToBase , equals , getCause , getLocalizedMessage , getMessage , getStackTrace , getUrl , hashCode , initCause , printStackTrace , printStackTrace , printStackTrace , readBaseFields , setStackTrace , toString , writeBaseFields
+
+
+
+Methods inherited from class java.lang.Throwable
+
+fillInStackTrace
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, finalize, getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+BadProtocolFetchException
+
+public BadProtocolFetchException ()
+
+
+
+BadProtocolFetchException
+
+public BadProtocolFetchException (java.lang.String url)
+
+
+
+
+BaseFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetchException
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+Direct Known Subclasses: AbortedFetchException , BadProtocolFetchException , HttpFetchException , IOFetchException , RedirectFetchException , UrlFetchException
+
+public abstract class BaseFetchException extends java.lang.Exception
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Throwable
+
+fillInStackTrace
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, finalize, getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+BaseFetchException
+
+protected BaseFetchException ()
+
+
+
+BaseFetchException
+
+protected BaseFetchException (java.lang.String url)
+
+
+
+BaseFetchException
+
+protected BaseFetchException (java.lang.String url,
+ java.lang.String msg)
+
+
+
+BaseFetchException
+
+protected BaseFetchException (java.lang.String url,
+ java.lang.Exception e)
+
+
+
+BaseFetchException
+
+protected BaseFetchException (java.lang.String url,
+ java.lang.String msg,
+ java.lang.Exception e)
+
+
+
+
+
+
+getUrl
+
+public java.lang.String getUrl ()
+
+
+
+
+
+compareToBase
+
+protected int compareToBase (BaseFetchException e)
+
+
+
+
+
+equals
+
+public boolean equals (java.lang.Object obj)
+
+
+Overrides: equals in class java.lang.Object
+
+
+
+
+getCause
+
+public java.lang.Throwable getCause ()
+
+
+Overrides: getCause in class java.lang.Throwable
+
+
+
+
+getLocalizedMessage
+
+public java.lang.String getLocalizedMessage ()
+
+
+Overrides: getLocalizedMessage in class java.lang.Throwable
+
+
+
+
+getMessage
+
+public java.lang.String getMessage ()
+
+
+Overrides: getMessage in class java.lang.Throwable
+
+
+
+
+getStackTrace
+
+public java.lang.StackTraceElement[] getStackTrace ()
+
+
+Overrides: getStackTrace in class java.lang.Throwable
+
+
+
+
+hashCode
+
+public int hashCode ()
+
+
+Overrides: hashCode in class java.lang.Object
+
+
+
+
+initCause
+
+public java.lang.Throwable initCause (java.lang.Throwable cause)
+
+
+Overrides: initCause in class java.lang.Throwable
+
+
+
+
+printStackTrace
+
+public void printStackTrace ()
+
+
+Overrides: printStackTrace in class java.lang.Throwable
+
+
+
+
+printStackTrace
+
+public void printStackTrace (java.io.PrintStream s)
+
+
+Overrides: printStackTrace in class java.lang.Throwable
+
+
+
+
+printStackTrace
+
+public void printStackTrace (java.io.PrintWriter s)
+
+
+Overrides: printStackTrace in class java.lang.Throwable
+
+
+
+
+setStackTrace
+
+public void setStackTrace (java.lang.StackTraceElement[] stackTrace)
+
+
+Overrides: setStackTrace in class java.lang.Throwable
+
+
+
+
+toString
+
+public java.lang.String toString ()
+
+
+Overrides: toString in class java.lang.Throwable
+
+
+
+
+readBaseFields
+
+protected void readBaseFields (java.io.DataInput input)
+ throws java.io.IOException
+
+
+
+Throws:
+java.io.IOException
+
+
+writeBaseFields
+
+protected void writeBaseFields (java.io.DataOutput output)
+ throws java.io.IOException
+
+
+
+Throws:
+java.io.IOException
+
+
+
+BaseFetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetcher
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+Direct Known Subclasses: BaseHttpFetcher , SimpleFileFetcher
+
+public abstract class BaseFetcher extends java.lang.Objectimplements java.io.Serializable
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+DEFAULT_MAX_CONTENT_SIZE
+
+public static final int DEFAULT_MAX_CONTENT_SIZE
+
+
+See Also: Constant Field Values
+
+
+_maxContentSizes
+
+protected java.util.Map<java.lang.String,java.lang.Integer> _maxContentSizes
+
+
+
+
+
+_defaultMaxContentSize
+
+protected int _defaultMaxContentSize
+
+
+
+
+
+_validMimeTypes
+
+protected java.util.Set<java.lang.String> _validMimeTypes
+
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+BaseFetcher
+
+public BaseFetcher ()
+
+
+
+
+
+
+setDefaultMaxContentSize
+
+public void setDefaultMaxContentSize (int defaultMaxContentSize)
+
+
+
+
+
+
+
+getDefaultMaxContentSize
+
+public int getDefaultMaxContentSize ()
+
+
+
+
+
+
+
+setMaxContentSize
+
+public void setMaxContentSize (java.lang.String mimeType,
+ int maxContentSize)
+
+
+
+
+
+
+
+getMaxContentSize
+
+public int getMaxContentSize (java.lang.String mimeType)
+
+
+
+
+
+
+
+getValidMimeTypes
+
+public java.util.Set<java.lang.String> getValidMimeTypes ()
+
+
+
+
+
+
+
+setValidMimeTypes
+
+public void setValidMimeTypes (java.util.Set<java.lang.String> validMimeTypes)
+
+
+
+
+
+
+
+addValidMimeTypes
+
+public void addValidMimeTypes (java.util.Set<java.lang.String> validMimeTypes)
+
+
+
+
+
+
+
+addValidMimeType
+
+public void addValidMimeType (java.lang.String validMimeType)
+
+
+
+
+
+
+
+get
+
+public FetchedResult get (java.lang.String url)
+ throws BaseFetchException
+
+
+
+
+
+Throws:
+BaseFetchException
+
+
+getMimeTypeFromContentType
+
+protected static java.lang.String getMimeTypeFromContentType (java.lang.String contentType)
+
+
+
+
+
+
+
+get
+
+public abstract FetchedResult get (java.lang.String url,
+ Payload payload)
+ throws BaseFetchException
+
+Get the content stored in the resource referenced by
+
+
+
+
+Parameters: url - payload -
+Returns:
+ Throws:
+BaseFetchException
+
+
+abort
+
+public abstract void abort ()
+
+Terminate any async request being processed.
+
+
+
+
+
+
+
+
+EncodingUtils.ExpandedResult (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.EncodingUtils.ExpandedResult
+
+
+Enclosing class: EncodingUtils
+
+public static class EncodingUtils.ExpandedResult extends java.lang.Object
+
+
+
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+EncodingUtils.ExpandedResult
+
+public EncodingUtils.ExpandedResult (byte[] expanded,
+ boolean isTruncated)
+
+
+
+
+
+
+getExpanded
+
+public byte[] getExpanded ()
+
+
+
+
+
+setExpanded
+
+public void setExpanded (byte[] expanded)
+
+
+
+
+
+isTruncated
+
+public boolean isTruncated ()
+
+
+
+
+
+setTruncated
+
+public void setTruncated (boolean isTruncated)
+
+
+
+
+
+
+EncodingUtils (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.EncodingUtils
+
+
+public class EncodingUtils extends java.lang.Object
+
+
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+EncodingUtils
+
+public EncodingUtils ()
+
+
+
+
+
+
+processGzipEncoded
+
+public static byte[] processGzipEncoded (byte[] compressed)
+ throws java.io.IOException
+
+
+
+Throws:
+java.io.IOException
+
+
+processGzipEncoded
+
+public static EncodingUtils.ExpandedResult processGzipEncoded (byte[] compressed,
+ int sizeLimit)
+ throws java.io.IOException
+
+
+
+Throws:
+java.io.IOException
+
+
+processDeflateEncoded
+
+public static byte[] processDeflateEncoded (byte[] content)
+ throws java.io.IOException
+
+
+
+Throws:
+java.io.IOException
+
+
+processDeflateEncoded
+
+public static byte[] processDeflateEncoded (byte[] compressed,
+ int sizeLimit)
+ throws java.io.IOException
+
+
+
+Throws:
+java.io.IOException
+
+
+
+FetchedResult (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.FetchedResult
+
+
+public class FetchedResult extends java.lang.Object
+
+
+
+
+
+
+
+
+
+
+Constructor Summary
+
+FetchedResult Payload payload,
+ java.lang.String newBaseUrl,
+ int numRedirects,
+ java.lang.String hostAddress)
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+FetchedResult
+
+public FetchedResult (java.lang.String baseUrl,
+ java.lang.String redirectedUrl,
+ long fetchTime,
+ org.apache.tika.metadata.Metadata headers,
+ byte[] content,
+ java.lang.String contentType,
+ int responseRate,
+ Payload payload,
+ java.lang.String newBaseUrl,
+ int numRedirects,
+ java.lang.String hostAddress)
+
+
+
+
+
+
+getPayload
+
+public Payload getPayload ()
+
+
+
+
+
+setPayload
+
+public void setPayload (Payload payload)
+
+
+
+
+
+getBaseUrl
+
+public java.lang.String getBaseUrl ()
+
+
+
+
+
+getFetchedUrl
+
+public java.lang.String getFetchedUrl ()
+
+
+
+
+
+getFetchTime
+
+public long getFetchTime ()
+
+
+
+
+
+getContent
+
+public byte[] getContent ()
+
+
+
+
+
+getContentLength
+
+public int getContentLength ()
+
+
+
+
+
+getContentType
+
+public java.lang.String getContentType ()
+
+
+
+
+
+getResponseRate
+
+public int getResponseRate ()
+
+
+
+
+
+getHeaders
+
+public org.apache.tika.metadata.Metadata getHeaders ()
+
+
+
+
+
+getNewBaseUrl
+
+public java.lang.String getNewBaseUrl ()
+
+
+
+
+
+getNumRedirects
+
+public int getNumRedirects ()
+
+
+
+
+
+getHostAddress
+
+public java.lang.String getHostAddress ()
+
+
+
+
+
+
+HttpFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetchException
+ crawlercommons.fetcher.HttpFetchException
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class HttpFetchException extends BaseFetchException
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+Constructor Summary
+
+HttpFetchException
+
+HttpFetchException
+
+
+
+
+
+
+Methods inherited from class crawlercommons.fetcher.BaseFetchException
+
+compareToBase , equals , getCause , getLocalizedMessage , getStackTrace , getUrl , hashCode , initCause , printStackTrace , printStackTrace , printStackTrace , readBaseFields , setStackTrace , toString , writeBaseFields
+
+
+
+Methods inherited from class java.lang.Throwable
+
+fillInStackTrace
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, finalize, getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+HttpFetchException
+
+public HttpFetchException ()
+
+
+
+HttpFetchException
+
+public HttpFetchException (java.lang.String url,
+ java.lang.String msg,
+ int httpStatus,
+ org.apache.tika.metadata.Metadata httpHeaders)
+
+
+
+
+
+
+getHttpStatus
+
+public int getHttpStatus ()
+
+
+
+
+
+getHttpHeaders
+
+public org.apache.tika.metadata.Metadata getHttpHeaders ()
+
+
+
+
+
+getMessage
+
+public java.lang.String getMessage ()
+
+
+Overrides: getMessage BaseFetchException
+
+
+
+
+
+IOFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetchException
+ crawlercommons.fetcher.IOFetchException
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class IOFetchException extends BaseFetchException
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class crawlercommons.fetcher.BaseFetchException
+
+compareToBase , equals , getCause , getLocalizedMessage , getMessage , getStackTrace , getUrl , hashCode , initCause , printStackTrace , printStackTrace , printStackTrace , readBaseFields , setStackTrace , toString , writeBaseFields
+
+
+
+Methods inherited from class java.lang.Throwable
+
+fillInStackTrace
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, finalize, getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+IOFetchException
+
+public IOFetchException ()
+
+
+
+IOFetchException
+
+public IOFetchException (java.lang.String url,
+ java.io.IOException e)
+
+
+
+
+Payload (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.Payload
+
+
+All Implemented Interfaces: java.util.Map<java.lang.String,java.lang.Object>
+
+
+public class Payload extends java.lang.Objectimplements java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+
+
+
+
+
+Nested Class Summary
+
+
+
+Nested classes/interfaces inherited from interface java.util.Map
+
+java.util.Map.Entry<K,V>
+
+
+
+
+
+
+
+
+Constructor Summary
+
+Payload
+
+
+
+
+
+
+
+Method Summary
+
+
+ void clear
+
+
+ boolean containsKey
+
+
+ boolean containsValue
+
+
+ java.util.Set<java.util.Map.Entry<java.lang.String,java.lang.Object>> entrySet
+
+
+ boolean equals
+
+
+ java.lang.Object get
+
+
+ int hashCode
+
+
+ boolean isEmpty
+
+
+ java.util.Set<java.lang.String> keySet
+
+
+ java.lang.Object put
+
+
+ void putAll
+
+
+ java.lang.Object remove
+
+
+ int size
+
+
+ java.util.Collection<java.lang.Object> values
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, finalize, getClass, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+Payload
+
+public Payload ()
+
+
+
+
+
+
+clear
+
+public void clear ()
+
+
+Specified by: clear in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+containsKey
+
+public boolean containsKey (java.lang.Object key)
+
+
+Specified by: containsKey in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+containsValue
+
+public boolean containsValue (java.lang.Object value)
+
+
+Specified by: containsValue in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+entrySet
+
+public java.util.Set<java.util.Map.Entry<java.lang.String,java.lang.Object>> entrySet ()
+
+
+Specified by: entrySet in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+equals
+
+public boolean equals (java.lang.Object o)
+
+
+Specified by: equals in interface java.util.Map<java.lang.String,java.lang.Object>Overrides: equals in class java.lang.Object
+
+
+
+
+get
+
+public java.lang.Object get (java.lang.Object key)
+
+
+Specified by: get in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+hashCode
+
+public int hashCode ()
+
+
+Specified by: hashCode in interface java.util.Map<java.lang.String,java.lang.Object>Overrides: hashCode in class java.lang.Object
+
+
+
+
+isEmpty
+
+public boolean isEmpty ()
+
+
+Specified by: isEmpty in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+keySet
+
+public java.util.Set<java.lang.String> keySet ()
+
+
+Specified by: keySet in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+put
+
+public java.lang.Object put (java.lang.String key,
+ java.lang.Object value)
+
+
+Specified by: put in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+putAll
+
+public void putAll (java.util.Map<? extends java.lang.String,? extends java.lang.Object> m)
+
+
+Specified by: putAll in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+remove
+
+public java.lang.Object remove (java.lang.Object key)
+
+
+Specified by: remove in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+size
+
+public int size ()
+
+
+Specified by: size in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+values
+
+public java.util.Collection<java.lang.Object> values ()
+
+
+Specified by: values in interface java.util.Map<java.lang.String,java.lang.Object>
+
+
+
+
+
+RedirectFetchException.RedirectExceptionReason (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ RedirectFetchException.RedirectExceptionReason >
+ crawlercommons.fetcher.RedirectFetchException.RedirectExceptionReason
+
+
+All Implemented Interfaces: java.io.Serializable, java.lang.Comparable<RedirectFetchException.RedirectExceptionReason >
+
+
+Enclosing class: RedirectFetchException
+
+public static enum RedirectFetchException.RedirectExceptionReason extends java.lang.Enum<RedirectFetchException.RedirectExceptionReason >
+
+
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Enum
+
+clone, compareTo, equals, finalize, getDeclaringClass, hashCode, name, ordinal, toString, valueOf
+
+
+
+Methods inherited from class java.lang.Object
+
+getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Enum Constant Detail
+
+
+
+TOO_MANY_REDIRECTS
+
+public static final RedirectFetchException.RedirectExceptionReason TOO_MANY_REDIRECTS
+
+
+
+
+
+PERM_REDIRECT_DISALLOWED
+
+public static final RedirectFetchException.RedirectExceptionReason PERM_REDIRECT_DISALLOWED
+
+
+
+
+
+TEMP_REDIRECT_DISALLOWED
+
+public static final RedirectFetchException.RedirectExceptionReason TEMP_REDIRECT_DISALLOWED
+
+
+
+
+
+
+
+
+values
+
+public static RedirectFetchException.RedirectExceptionReason [] values ()
+
+Returns an array containing the constants of this enum type, in
+the order they are declared. This method may be used to iterate
+over the constants as follows:
+
+for (RedirectFetchException.RedirectExceptionReason c : RedirectFetchException.RedirectExceptionReason.values())
+ System.out.println(c);
+
+
+
+
+Returns: an array containing the constants of this enum type, in
+the order they are declared
+
+
+valueOf
+
+public static RedirectFetchException.RedirectExceptionReason valueOf (java.lang.String name)
+
+Returns the enum constant of this type with the specified name.
+The string must match exactly an identifier used to declare an
+enum constant in this type. (Extraneous whitespace characters are
+not permitted.)
+
+
+Parameters: name - the name of the enum constant to be returned.
+Returns: the enum constant with the specified name
+ Throws:
+java.lang.IllegalArgumentException - if this enum type has no constant
+with the specified name
+java.lang.NullPointerException - if the argument is null
+
+
+
+RedirectFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetchException
+ crawlercommons.fetcher.RedirectFetchException
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class RedirectFetchException extends BaseFetchException
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+Methods inherited from class crawlercommons.fetcher.BaseFetchException
+
+compareToBase , equals , getCause , getLocalizedMessage , getMessage , getStackTrace , getUrl , hashCode , initCause , printStackTrace , printStackTrace , printStackTrace , readBaseFields , setStackTrace , toString , writeBaseFields
+
+
+
+Methods inherited from class java.lang.Throwable
+
+fillInStackTrace
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, finalize, getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+RedirectFetchException
+
+public RedirectFetchException ()
+
+
+
+RedirectFetchException
+
+public RedirectFetchException (java.lang.String url,
+ java.lang.String redirectedUrl,
+ RedirectFetchException.RedirectExceptionReason reason)
+
+
+
+
+
+
+getRedirectedUrl
+
+public java.lang.String getRedirectedUrl ()
+
+
+
+
+
+getReason
+
+public RedirectFetchException.RedirectExceptionReason getReason ()
+
+
+
+
+
+
+UrlFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetchException
+ crawlercommons.fetcher.UrlFetchException
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class UrlFetchException extends BaseFetchException
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class crawlercommons.fetcher.BaseFetchException
+
+compareToBase , equals , getCause , getLocalizedMessage , getMessage , getStackTrace , getUrl , hashCode , initCause , printStackTrace , printStackTrace , printStackTrace , readBaseFields , setStackTrace , toString , writeBaseFields
+
+
+
+Methods inherited from class java.lang.Throwable
+
+fillInStackTrace
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, finalize, getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+UrlFetchException
+
+public UrlFetchException ()
+
+
+
+UrlFetchException
+
+public UrlFetchException (java.lang.String url,
+ java.lang.String msg)
+
+
+
+
+Uses of Class crawlercommons.fetcher.AbortedFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.fetcher.AbortedFetchException
+
+
+Uses of Class crawlercommons.fetcher.AbortedFetchReason (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.BadProtocolFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.fetcher.BadProtocolFetchException
+
+
+Uses of Class crawlercommons.fetcher.BaseFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.BaseFetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.EncodingUtils.ExpandedResult (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.EncodingUtils (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.fetcher.EncodingUtils
+
+
+Uses of Class crawlercommons.fetcher.FetchedResult (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.HttpFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.fetcher.HttpFetchException
+
+
+Uses of Class crawlercommons.fetcher.IOFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.fetcher.IOFetchException
+
+
+Uses of Class crawlercommons.fetcher.Payload (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Constructors in crawlercommons.fetcher with parameters of type Payload
+
+
+FetchedResult Payload payload,
+ java.lang.String newBaseUrl,
+ int numRedirects,
+ java.lang.String hostAddress)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.RedirectFetchException.RedirectExceptionReason (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.RedirectFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.fetcher.RedirectFetchException
+
+
+Uses of Class crawlercommons.fetcher.UrlFetchException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.fetcher.UrlFetchException
+
+
+SimpleFileFetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher.file
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetcher
+ crawlercommons.fetcher.file.SimpleFileFetcher
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class SimpleFileFetcher extends BaseFetcher
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+Method Summary
+
+
+ void abort
+
+
+ FetchedResult get Payload payload)
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+SimpleFileFetcher
+
+public SimpleFileFetcher ()
+
+
+
+
+
+
+get
+
+public FetchedResult get (java.lang.String url,
+ Payload payload)
+ throws BaseFetchException
+
+Description copied from class: BaseFetcher Get the content stored in the resource referenced by
+
+
+Specified by: get BaseFetcher
+
+
+Returns:
+ Throws:
+BaseFetchException
+
+
+abort
+
+public void abort ()
+
+Description copied from class: BaseFetcher Terminate any async request being processed.
+
+
+Specified by: abort BaseFetcher
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.file.SimpleFileFetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.fetcher.file.SimpleFileFetcher
+
+
+crawlercommons.fetcher.file (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+crawlercommons.fetcher.file
+
+
+
+
+
diff --git a/doc/javadoc/crawlercommons/fetcher/file/package-summary.html b/doc/javadoc/crawlercommons/fetcher/file/package-summary.html
new file mode 100644
index 0000000..2c7781f
--- /dev/null
+++ b/doc/javadoc/crawlercommons/fetcher/file/package-summary.html
@@ -0,0 +1,157 @@
+
+
+
+
+
+
+crawlercommons.fetcher.file (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+Package crawlercommons.fetcher.file
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher.file Class Hierarchy (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Hierarchy For Package crawlercommons.fetcher.file
+
+
+
+Package Hierarchies: All Packages
+
+Class Hierarchy
+
+
+java.lang.Object
+crawlercommons.fetcher.BaseFetcher
+
+
+Uses of Package crawlercommons.fetcher.file (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Package
+
+No usage of crawlercommons.fetcher.file
+
+
+BaseHttpFetcher.RedirectMode (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher.http
+
+
+java.lang.Object
+ BaseHttpFetcher.RedirectMode >
+ crawlercommons.fetcher.http.BaseHttpFetcher.RedirectMode
+
+
+All Implemented Interfaces: java.io.Serializable, java.lang.Comparable<BaseHttpFetcher.RedirectMode >
+
+
+Enclosing class: BaseHttpFetcher
+
+public static enum BaseHttpFetcher.RedirectMode extends java.lang.Enum<BaseHttpFetcher.RedirectMode >
+
+
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Enum
+
+clone, compareTo, equals, finalize, getDeclaringClass, hashCode, name, ordinal, toString, valueOf
+
+
+
+Methods inherited from class java.lang.Object
+
+getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Enum Constant Detail
+
+
+
+FOLLOW_ALL
+
+public static final BaseHttpFetcher.RedirectMode FOLLOW_ALL
+
+
+
+
+
+FOLLOW_TEMP
+
+public static final BaseHttpFetcher.RedirectMode FOLLOW_TEMP
+
+
+
+
+
+FOLLOW_NONE
+
+public static final BaseHttpFetcher.RedirectMode FOLLOW_NONE
+
+
+
+
+
+
+
+
+values
+
+public static BaseHttpFetcher.RedirectMode [] values ()
+
+Returns an array containing the constants of this enum type, in
+the order they are declared. This method may be used to iterate
+over the constants as follows:
+
+for (BaseHttpFetcher.RedirectMode c : BaseHttpFetcher.RedirectMode.values())
+ System.out.println(c);
+
+
+
+
+Returns: an array containing the constants of this enum type, in
+the order they are declared
+
+
+valueOf
+
+public static BaseHttpFetcher.RedirectMode valueOf (java.lang.String name)
+
+Returns the enum constant of this type with the specified name.
+The string must match exactly an identifier used to declare an
+enum constant in this type. (Extraneous whitespace characters are
+not permitted.)
+
+
+Parameters: name - the name of the enum constant to be returned.
+Returns: the enum constant with the specified name
+ Throws:
+java.lang.IllegalArgumentException - if this enum type has no constant
+with the specified name
+java.lang.NullPointerException - if the argument is null
+
+
+
+BaseHttpFetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher.http
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetcher
+ crawlercommons.fetcher.http.BaseHttpFetcher
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+Direct Known Subclasses: SimpleHttpFetcher
+
+public abstract class BaseHttpFetcher extends BaseFetcher
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+Methods inherited from class crawlercommons.fetcher.BaseFetcher
+
+abort , addValidMimeType , addValidMimeTypes , get , get , getDefaultMaxContentSize , getMaxContentSize , getMimeTypeFromContentType , getValidMimeTypes , setDefaultMaxContentSize , setMaxContentSize , setValidMimeTypes
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+NO_MIN_RESPONSE_RATE
+
+public static final int NO_MIN_RESPONSE_RATE
+
+
+See Also: Constant Field Values
+
+
+NO_REDIRECTS
+
+public static final int NO_REDIRECTS
+
+
+See Also: Constant Field Values
+
+
+DEFAULT_MIN_RESPONSE_RATE
+
+public static final int DEFAULT_MIN_RESPONSE_RATE
+
+
+See Also: Constant Field Values
+
+
+DEFAULT_MAX_CONNECTIONS_PER_HOST
+
+public static final int DEFAULT_MAX_CONNECTIONS_PER_HOST
+
+
+See Also: Constant Field Values
+
+
+DEFAULT_MAX_REDIRECTS
+
+public static final int DEFAULT_MAX_REDIRECTS
+
+
+See Also: Constant Field Values
+
+
+DEFAULT_ACCEPT_LANGUAGE
+
+public static final java.lang.String DEFAULT_ACCEPT_LANGUAGE
+
+
+See Also: Constant Field Values
+
+
+DEFAULT_REDIRECT_MODE
+
+public static final BaseHttpFetcher.RedirectMode DEFAULT_REDIRECT_MODE
+
+
+
+
+
+_maxThreads
+
+protected int _maxThreads
+
+
+
+
+
+_userAgent
+
+protected UserAgent _userAgent
+
+
+
+
+
+_maxRedirects
+
+protected int _maxRedirects
+
+
+
+
+
+_maxConnectionsPerHost
+
+protected int _maxConnectionsPerHost
+
+
+
+
+
+_minResponseRate
+
+protected int _minResponseRate
+
+
+
+
+
+_acceptLanguage
+
+protected java.lang.String _acceptLanguage
+
+
+
+
+
+_redirectMode
+
+protected BaseHttpFetcher.RedirectMode _redirectMode
+
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+BaseHttpFetcher
+
+public BaseHttpFetcher (int maxThreads,
+ UserAgent userAgent)
+
+
+
+
+
+
+getMaxThreads
+
+public int getMaxThreads ()
+
+
+
+
+
+getUserAgent
+
+public UserAgent getUserAgent ()
+
+
+
+
+
+setMaxConnectionsPerHost
+
+public void setMaxConnectionsPerHost (int maxConnectionsPerHost)
+
+
+
+
+
+getMaxConnectionsPerHost
+
+public int getMaxConnectionsPerHost ()
+
+
+
+
+
+setMinResponseRate
+
+public void setMinResponseRate (int minResponseRate)
+
+
+
+
+
+getMinResponseRate
+
+public int getMinResponseRate ()
+
+Return the minimum response rate. If the speed at which bytes are being returned
+ from the server drops below this, the fetch of that page will be aborted.
+
+
+
+Returns: bytes/second
+
+
+setAcceptLanguage
+
+public void setAcceptLanguage (java.lang.String acceptLanguage)
+
+
+
+
+
+getAcceptLanguage
+
+public java.lang.String getAcceptLanguage ()
+
+
+
+
+
+setMaxRedirects
+
+public void setMaxRedirects (int maxRedirects)
+
+
+
+
+
+getMaxRedirects
+
+public int getMaxRedirects ()
+
+
+
+
+
+setRedirectMode
+
+public void setRedirectMode (BaseHttpFetcher.RedirectMode mode)
+
+
+
+
+
+getRedirectMode
+
+public BaseHttpFetcher.RedirectMode getRedirectMode ()
+
+
+
+
+
+
+SimpleHttpFetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher.http
+
+
+java.lang.Object
+ crawlercommons.fetcher.BaseFetcher
+ crawlercommons.fetcher.http.BaseHttpFetcher
+ crawlercommons.fetcher.http.SimpleHttpFetcher
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class SimpleHttpFetcher extends BaseHttpFetcher
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+Nested Class Summary
+
+
+
+Fields inherited from class crawlercommons.fetcher.http.BaseHttpFetcher
+
+_acceptLanguage , _maxConnectionsPerHost , _maxRedirects , _maxThreads , _minResponseRate , _redirectMode , _userAgent , DEFAULT_ACCEPT_LANGUAGE , DEFAULT_MAX_CONNECTIONS_PER_HOST , DEFAULT_MAX_REDIRECTS , DEFAULT_MIN_RESPONSE_RATE , DEFAULT_REDIRECT_MODE , NO_MIN_RESPONSE_RATE , NO_REDIRECTS
+
+
+
+Methods inherited from class crawlercommons.fetcher.http.BaseHttpFetcher
+
+getAcceptLanguage , getMaxConnectionsPerHost , getMaxRedirects , getMaxThreads , getMinResponseRate , getRedirectMode , getUserAgent , setAcceptLanguage , setMaxConnectionsPerHost , setMaxRedirects , setMinResponseRate , setRedirectMode
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+SimpleHttpFetcher
+
+public SimpleHttpFetcher (UserAgent userAgent)
+
+
+
+SimpleHttpFetcher
+
+public SimpleHttpFetcher (int maxThreads,
+ UserAgent userAgent)
+
+
+
+
+
+
+getHttpVersion
+
+public org.apache.http.HttpVersion getHttpVersion ()
+
+
+
+
+
+setHttpVersion
+
+public void setHttpVersion (org.apache.http.HttpVersion httpVersion)
+
+
+
+
+
+getSocketTimeout
+
+public int getSocketTimeout ()
+
+
+
+
+
+setSocketTimeout
+
+public void setSocketTimeout (int socketTimeoutInMs)
+
+
+
+
+
+getConnectionTimeout
+
+public int getConnectionTimeout ()
+
+
+
+
+
+setConnectionTimeout
+
+public void setConnectionTimeout (int connectionTimeoutInMs)
+
+
+
+
+
+getMaxRetryCount
+
+public int getMaxRetryCount ()
+
+
+
+
+
+setMaxRetryCount
+
+public void setMaxRetryCount (int maxRetryCount)
+
+
+
+
+
+get
+
+public FetchedResult get (java.lang.String url,
+ Payload payload)
+ throws BaseFetchException
+
+Description copied from class: BaseFetcher Get the content stored in the resource referenced by
+
+
+Specified by: get BaseFetcher
+
+
+Returns:
+ Throws:
+BaseFetchException
+
+
+fetch
+
+public FetchedResult fetch (java.lang.String url)
+ throws BaseFetchException
+
+
+
+Throws:
+BaseFetchException
+
+
+fetch
+
+public FetchedResult fetch (org.apache.http.client.methods.HttpRequestBase request,
+ java.lang.String url,
+ Payload payload)
+ throws BaseFetchException
+
+
+
+Throws:
+BaseFetchException
+
+
+abort
+
+public void abort ()
+
+Description copied from class: BaseFetcher Terminate any async request being processed.
+
+
+Specified by: abort BaseFetcher
+
+
+
+
+
+UserAgent (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher.http
+
+
+java.lang.Object
+ crawlercommons.fetcher.http.UserAgent
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class UserAgent extends java.lang.Objectimplements java.io.Serializable
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+
+Constructor Summary
+
+UserAgent
+
+UserAgent
+
+UserAgent
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+DEFAULT_BROWSER_VERSION
+
+public static final java.lang.String DEFAULT_BROWSER_VERSION
+
+
+See Also: Constant Field Values
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+UserAgent
+
+public UserAgent (java.lang.String agentName,
+ java.lang.String emailAddress,
+ java.lang.String webAddress)
+
+
+
+UserAgent
+
+public UserAgent (java.lang.String agentName,
+ java.lang.String emailAddress,
+ java.lang.String webAddress,
+ java.lang.String browserVersion)
+
+
+
+UserAgent
+
+public UserAgent (java.lang.String agentName,
+ java.lang.String emailAddress,
+ java.lang.String webAddress,
+ java.lang.String browserVersion,
+ java.lang.String crawlerVersion)
+
+
+
+
+
+
+getAgentName
+
+public java.lang.String getAgentName ()
+
+
+
+
+
+
+
+getUserAgentString
+
+public java.lang.String getUserAgentString ()
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.http.BaseHttpFetcher.RedirectMode (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.http.BaseHttpFetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.fetcher.http.SimpleHttpFetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.fetcher.http.SimpleHttpFetcher
+
+
+Uses of Class crawlercommons.fetcher.http.UserAgent (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher.http (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+crawlercommons.fetcher.http
+
+
+
+
+
+
+
+
diff --git a/doc/javadoc/crawlercommons/fetcher/http/package-summary.html b/doc/javadoc/crawlercommons/fetcher/http/package-summary.html
new file mode 100644
index 0000000..d2cfe75
--- /dev/null
+++ b/doc/javadoc/crawlercommons/fetcher/http/package-summary.html
@@ -0,0 +1,179 @@
+
+
+
+
+
+
+crawlercommons.fetcher.http (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+Package crawlercommons.fetcher.http
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher.http Class Hierarchy (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Hierarchy For Package crawlercommons.fetcher.http
+
+
+
+Package Hierarchies: All Packages
+
+Class Hierarchy
+
+
+java.lang.Object
+crawlercommons.fetcher.BaseFetcher crawlercommons.fetcher.http.UserAgent
+
+
+Enum Hierarchy
+
+
+java.lang.Object
+java.lang.Enum<E> (implements java.lang.Comparable<T>, java.io.Serializable)
+
+
+
+
+Uses of Package crawlercommons.fetcher.http (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Package
+
+
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+crawlercommons.fetcher
+
+
+
+
+
+
+
+
+
+
+
diff --git a/doc/javadoc/crawlercommons/fetcher/package-summary.html b/doc/javadoc/crawlercommons/fetcher/package-summary.html
new file mode 100644
index 0000000..08fdd62
--- /dev/null
+++ b/doc/javadoc/crawlercommons/fetcher/package-summary.html
@@ -0,0 +1,229 @@
+
+
+
+
+
+
+crawlercommons.fetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+Package crawlercommons.fetcher
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+crawlercommons.fetcher Class Hierarchy (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Hierarchy For Package crawlercommons.fetcher
+
+
+
+Package Hierarchies: All Packages
+
+Class Hierarchy
+
+
+
+Enum Hierarchy
+
+
+java.lang.Object
+java.lang.Enum<E> (implements java.lang.Comparable<T>, java.io.Serializable)
+
+
+
+
+Uses of Package crawlercommons.fetcher (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Package
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+BaseRobotRules (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.robots
+
+
+java.lang.Object
+ crawlercommons.robots.BaseRobotRules
+
+
+Direct Known Subclasses: SimpleRobotRules
+
+public abstract class BaseRobotRules extends java.lang.Object
+
+
+
+Result from parsing a single robots.txt file - which means we
+ get a set of rules, and a crawl-delay.
+
+
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+UNSET_CRAWL_DELAY
+
+public static final long UNSET_CRAWL_DELAY
+
+
+See Also: Constant Field Values
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+BaseRobotRules
+
+public BaseRobotRules ()
+
+
+
+
+
+
+isAllowed
+
+public abstract boolean isAllowed (java.lang.String url)
+
+
+
+
+
+isAllowAll
+
+public abstract boolean isAllowAll ()
+
+
+
+
+
+isAllowNone
+
+public abstract boolean isAllowNone ()
+
+
+
+
+
+getCrawlDelay
+
+public long getCrawlDelay ()
+
+
+
+
+
+setCrawlDelay
+
+public void setCrawlDelay (long crawlDelay)
+
+
+
+
+
+isDeferVisits
+
+public boolean isDeferVisits ()
+
+
+
+
+
+setDeferVisits
+
+public void setDeferVisits (boolean deferVisits)
+
+
+
+
+
+addSitemap
+
+public void addSitemap (java.lang.String sitemap)
+
+
+
+
+
+getSitemaps
+
+public java.util.List<java.lang.String> getSitemaps ()
+
+
+
+
+
+
+BaseRobotsParser (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.robots
+
+
+java.lang.Object
+ crawlercommons.robots.BaseRobotsParser
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+Direct Known Subclasses: SimpleRobotRulesParser
+
+public abstract class BaseRobotsParser extends java.lang.Objectimplements java.io.Serializable
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+
+
+Method Summary
+
+
+abstract BaseRobotRules failedFetch
+
+
+abstract BaseRobotRules parseContent , and return rules appropriate for
+ processing paths by
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+BaseRobotsParser
+
+public BaseRobotsParser ()
+
+
+
+
+
+
+parseContent
+
+public abstract BaseRobotRules parseContent (java.lang.String url,
+ byte[] content,
+ java.lang.String contentType,
+ java.lang.String robotName)
+
+Parse the robots.txt file in , and return rules appropriate for
+ processing paths by
+
+
+
+
+Parameters: url - URL that content was fetched from (for reporting purposes)content - raw bytes from the site's robots.txt filecontentType - HTTP response header (mime-type)robotName - name of crawler, to be used when processing file contents
+ (just the name portion, w/o version or other details)
+Returns: robot rules.
+
+
+failedFetch
+
+public abstract BaseRobotRules failedFetch (int httpStatusCode)
+
+The fetch of robots.txt failed, so return rules appropriate give the
+ HTTP status code.
+
+
+
+
+Parameters: httpStatusCode - a failure status code (NOT 2xx)
+Returns: robot rules
+
+
+
+RobotUtils (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.robots
+
+
+java.lang.Object
+ crawlercommons.robots.RobotUtils
+
+
+public class RobotUtils extends java.lang.Object
+
+
+
+
+
+
+
+
+
+
+Constructor Summary
+
+RobotUtils
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+RobotUtils
+
+public RobotUtils ()
+
+
+
+
+
+
+createFetcher
+
+public static BaseHttpFetcher createFetcher (BaseHttpFetcher fetcher)
+
+
+
+
+
+createFetcher
+
+public static BaseHttpFetcher createFetcher (UserAgent userAgent,
+ int maxThreads)
+
+
+
+
+
+getMaxFetchTime
+
+public static long getMaxFetchTime ()
+
+
+
+
+
+getRobotRules
+
+public static BaseRobotRules getRobotRules (BaseHttpFetcher fetcher,
+ BaseRobotsParser parser,
+ java.net.URL robotsUrl)
+
+Externally visible, static method for use in tools and for testing.
+ Fetch the indicated robots.txt file, parse it, and generate rules.
+
+
+Parameters: fetcher - Fetcher for downloading robots.txt filerobotsUrl - URL to robots.txt file
+Returns: Robot rules
+
+
+
+SimpleRobotRules.RobotRule (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.robots
+
+
+java.lang.Object
+ crawlercommons.robots.SimpleRobotRules.RobotRule
+
+
+Enclosing class: SimpleRobotRules
+
+protected class SimpleRobotRules.RobotRule extends java.lang.Object
+
+
+
+Single rule that maps from a path prefix to an allow flag.
+
+
+
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+SimpleRobotRules.RobotRule
+
+public SimpleRobotRules.RobotRule (java.lang.String prefix,
+ boolean allow)
+
+
+
+SimpleRobotRules.RobotRule
+
+public SimpleRobotRules.RobotRule (java.util.regex.Pattern pattern,
+ boolean allow)
+
+
+
+
+SimpleRobotRules.RobotRulesMode (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.robots
+
+
+java.lang.Object
+ SimpleRobotRules.RobotRulesMode >
+ crawlercommons.robots.SimpleRobotRules.RobotRulesMode
+
+
+All Implemented Interfaces: java.io.Serializable, java.lang.Comparable<SimpleRobotRules.RobotRulesMode >
+
+
+Enclosing class: SimpleRobotRules
+
+public static enum SimpleRobotRules.RobotRulesMode extends java.lang.Enum<SimpleRobotRules.RobotRulesMode >
+
+
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Enum
+
+clone, compareTo, equals, finalize, getDeclaringClass, hashCode, name, ordinal, toString, valueOf
+
+
+
+Methods inherited from class java.lang.Object
+
+getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Enum Constant Detail
+
+
+
+ALLOW_ALL
+
+public static final SimpleRobotRules.RobotRulesMode ALLOW_ALL
+
+
+
+
+
+ALLOW_NONE
+
+public static final SimpleRobotRules.RobotRulesMode ALLOW_NONE
+
+
+
+
+
+ALLOW_SOME
+
+public static final SimpleRobotRules.RobotRulesMode ALLOW_SOME
+
+
+
+
+
+
+
+
+values
+
+public static SimpleRobotRules.RobotRulesMode [] values ()
+
+Returns an array containing the constants of this enum type, in
+the order they are declared. This method may be used to iterate
+over the constants as follows:
+
+for (SimpleRobotRules.RobotRulesMode c : SimpleRobotRules.RobotRulesMode.values())
+ System.out.println(c);
+
+
+
+
+Returns: an array containing the constants of this enum type, in
+the order they are declared
+
+
+valueOf
+
+public static SimpleRobotRules.RobotRulesMode valueOf (java.lang.String name)
+
+Returns the enum constant of this type with the specified name.
+The string must match exactly an identifier used to declare an
+enum constant in this type. (Extraneous whitespace characters are
+not permitted.)
+
+
+Parameters: name - the name of the enum constant to be returned.
+Returns: the enum constant with the specified name
+ Throws:
+java.lang.IllegalArgumentException - if this enum type has no constant
+with the specified name
+java.lang.NullPointerException - if the argument is null
+
+
+
+SimpleRobotRules (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.robots
+
+
+java.lang.Object
+ crawlercommons.robots.BaseRobotRules
+ crawlercommons.robots.SimpleRobotRules
+
+
+public class SimpleRobotRules extends BaseRobotRules
+
+
+
+Result from parsing a single robots.txt file - which means we
+ get a set of rules, and a crawl-delay.
+
+
+
+
+
+
+
+
+
+
+
+
+Method Summary
+
+
+ void addRule
+
+
+ void clearRules
+
+
+ boolean isAllowAll
+
+
+ boolean isAllowed
+
+
+ boolean isAllowNone
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+SimpleRobotRules
+
+public SimpleRobotRules ()
+
+
+
+SimpleRobotRules
+
+public SimpleRobotRules (SimpleRobotRules.RobotRulesMode mode)
+
+
+
+
+
+
+clearRules
+
+public void clearRules ()
+
+
+
+
+
+addRule
+
+public void addRule (java.lang.String prefix,
+ boolean allow)
+
+
+
+
+
+isAllowed
+
+public boolean isAllowed (java.lang.String url)
+
+
+Specified by: isAllowed BaseRobotRules
+
+
+
+
+isAllowAll
+
+public boolean isAllowAll ()
+
+Is our ruleset set up to allow all access?
+
+
+Specified by: isAllowAll BaseRobotRules
+
+
+Returns: true if all URLs are allowed.
+
+
+isAllowNone
+
+public boolean isAllowNone ()
+
+Is our ruleset set up to disallow all access?
+
+
+Specified by: isAllowNone BaseRobotRules
+
+
+Returns: true if no URLs are allowed.
+
+
+
+SimpleRobotRulesParser (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.robots
+
+
+java.lang.Object
+ crawlercommons.robots.BaseRobotsParser
+ crawlercommons.robots.SimpleRobotRulesParser
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class SimpleRobotRulesParser extends BaseRobotsParser
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+
+
+Method Summary
+
+
+ BaseRobotRules failedFetch
+
+
+ int getNumWarnings
+
+
+ BaseRobotRules parseContent , and return rules appropriate for
+ processing paths by
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+SimpleRobotRulesParser
+
+public SimpleRobotRulesParser ()
+
+
+
+
+
+
+failedFetch
+
+public BaseRobotRules failedFetch (int httpStatusCode)
+
+Description copied from class: BaseRobotsParser The fetch of robots.txt failed, so return rules appropriate give the
+ HTTP status code.
+
+
+Specified by: failedFetch BaseRobotsParser
+
+Parameters: httpStatusCode - a failure status code (NOT 2xx)
+Returns: robot rules
+
+
+parseContent
+
+public BaseRobotRules parseContent (java.lang.String url,
+ byte[] content,
+ java.lang.String contentType,
+ java.lang.String robotName)
+
+Description copied from class: BaseRobotsParser Parse the robots.txt file in , and return rules appropriate for
+ processing paths by
+
+
+Specified by: parseContent BaseRobotsParser
+
+Parameters: url - URL that content was fetched from (for reporting purposes)content - raw bytes from the site's robots.txt filecontentType - HTTP response header (mime-type)robotName - name of crawler, to be used when processing file contents
+ (just the name portion, w/o version or other details)
+Returns: robot rules.
+
+
+getNumWarnings
+
+public int getNumWarnings ()
+
+
+
+
+
+
+Uses of Class crawlercommons.robots.BaseRobotRules (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Methods in crawlercommons.robots that return BaseRobotRules
+
+
+
+ BaseRobotRules SimpleRobotRulesParser. failedFetch
+
+
+abstract BaseRobotRules BaseRobotsParser. failedFetch
+
+
+static BaseRobotRules RobotUtils. getRobotRules BaseHttpFetcher fetcher,
+ BaseRobotsParser parser,
+ java.net.URL robotsUrl)
+
+
+ BaseRobotRules SimpleRobotRulesParser. parseContent
+
+
+abstract BaseRobotRules BaseRobotsParser. parseContent , and return rules appropriate for
+ processing paths by
+
+
+
+
+Uses of Class crawlercommons.robots.BaseRobotsParser (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.robots.RobotUtils (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.robots.RobotUtils
+
+
+Uses of Class crawlercommons.robots.SimpleRobotRules.RobotRule (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.robots.SimpleRobotRules.RobotRule
+
+
+Uses of Class crawlercommons.robots.SimpleRobotRules.RobotRulesMode (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.robots.SimpleRobotRules (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.robots.SimpleRobotRules
+
+
+Uses of Class crawlercommons.robots.SimpleRobotRulesParser (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.robots.SimpleRobotRulesParser
+
+
+crawlercommons.robots (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+crawlercommons.robots
+
+
+
+
+
+
+
+
diff --git a/doc/javadoc/crawlercommons/robots/package-summary.html b/doc/javadoc/crawlercommons/robots/package-summary.html
new file mode 100644
index 0000000..7d7abe0
--- /dev/null
+++ b/doc/javadoc/crawlercommons/robots/package-summary.html
@@ -0,0 +1,189 @@
+
+
+
+
+
+
+crawlercommons.robots (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+Package crawlercommons.robots
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+crawlercommons.robots Class Hierarchy (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Hierarchy For Package crawlercommons.robots
+
+
+
+Package Hierarchies: All Packages
+
+Class Hierarchy
+
+
+
+Enum Hierarchy
+
+
+java.lang.Object
+java.lang.Enum<E> (implements java.lang.Comparable<T>, java.io.Serializable)
+
+
+
+
+Uses of Package crawlercommons.robots (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Package
+
+
+
+
+
+
+
+
+
+AbstractSiteMap.SitemapType (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.sitemaps
+
+
+java.lang.Object
+ AbstractSiteMap.SitemapType >
+ crawlercommons.sitemaps.AbstractSiteMap.SitemapType
+
+
+All Implemented Interfaces: java.io.Serializable, java.lang.Comparable<AbstractSiteMap.SitemapType >
+
+
+Enclosing class: AbstractSiteMap
+
+public static enum AbstractSiteMap.SitemapType extends java.lang.Enum<AbstractSiteMap.SitemapType >
+
+
+
+Various Sitemap types
+
+
+
+
+
+
+
+
+
+Enum Constant Summary
+
+ATOM
+
+INDEX
+
+RSS
+
+TEXT
+
+XML
+
+
+
+
+
+
+Methods inherited from class java.lang.Enum
+
+clone, compareTo, equals, finalize, getDeclaringClass, hashCode, name, ordinal, toString, valueOf
+
+
+
+Methods inherited from class java.lang.Object
+
+getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Enum Constant Detail
+
+
+
+INDEX
+
+public static final AbstractSiteMap.SitemapType INDEX
+
+
+
+
+
+XML
+
+public static final AbstractSiteMap.SitemapType XML
+
+
+
+
+
+ATOM
+
+public static final AbstractSiteMap.SitemapType ATOM
+
+
+
+
+
+RSS
+
+public static final AbstractSiteMap.SitemapType RSS
+
+
+
+
+
+TEXT
+
+public static final AbstractSiteMap.SitemapType TEXT
+
+
+
+
+
+
+
+
+values
+
+public static AbstractSiteMap.SitemapType [] values ()
+
+Returns an array containing the constants of this enum type, in
+the order they are declared. This method may be used to iterate
+over the constants as follows:
+
+for (AbstractSiteMap.SitemapType c : AbstractSiteMap.SitemapType.values())
+ System.out.println(c);
+
+
+
+
+Returns: an array containing the constants of this enum type, in
+the order they are declared
+
+
+valueOf
+
+public static AbstractSiteMap.SitemapType valueOf (java.lang.String name)
+
+Returns the enum constant of this type with the specified name.
+The string must match exactly an identifier used to declare an
+enum constant in this type. (Extraneous whitespace characters are
+not permitted.)
+
+
+Parameters: name - the name of the enum constant to be returned.
+Returns: the enum constant with the specified name
+ Throws:
+java.lang.IllegalArgumentException - if this enum type has no constant
+with the specified name
+java.lang.NullPointerException - if the argument is null
+
+
+
+AbstractSiteMap (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.sitemaps
+
+
+java.lang.Object
+ crawlercommons.sitemaps.AbstractSiteMap
+
+
+Direct Known Subclasses: SiteMap , SiteMapIndex
+
+public abstract class AbstractSiteMap extends java.lang.Object
+
+
+
+SiteMap or SiteMapIndex
+
+
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+AbstractSiteMap
+
+public AbstractSiteMap ()
+
+
+
+
+
+
+getFullDateFormat
+
+public static java.text.DateFormat getFullDateFormat ()
+
+
+
+
+
+isIndex
+
+public boolean isIndex ()
+
+
+
+
+
+getUrl
+
+public java.net.URL getUrl ()
+
+
+
+Returns: the URL of the Sitemap
+
+
+getType
+
+public AbstractSiteMap.SitemapType getType ()
+
+
+
+Returns: the Sitemap type
+
+
+isProcessed
+
+public boolean isProcessed ()
+
+
+
+Returns: true if the Sitemap has been processed i.e it contains at least
+ one SiteMapURL
+
+
+setLastModified
+
+public void setLastModified (java.util.Date lastModified)
+
+
+Parameters: lastModified - - the lastModified to set
+
+
+setLastModified
+
+public void setLastModified (java.lang.String lastModified)
+
+
+Parameters: lastModified - - the lastModified to set
+
+
+getLastModified
+
+public java.util.Date getLastModified ()
+
+
+
+Returns: the lastModified date of the Sitemap
+
+
+convertToDate
+
+public static java.util.Date convertToDate (java.lang.String date)
+
+Convert the given date (given in an acceptable DateFormat), null if the
+ date is not in the correct format.
+
+
+Parameters: date - - the date to be parsed
+Returns: the Date equivalent
+
+
+
+SiteMap (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.sitemaps
+
+
+java.lang.Object
+ crawlercommons.sitemaps.AbstractSiteMap
+ crawlercommons.sitemaps.SiteMap
+
+
+public class SiteMap extends AbstractSiteMap
+
+
+
+
+
+
+
+
+
+Nested Class Summary
+
+
+
+
+Constructor Summary
+
+SiteMap
+
+SiteMap
+
+SiteMap
+
+SiteMap
+
+SiteMap
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+SiteMap
+
+public SiteMap ()
+
+
+
+SiteMap
+
+public SiteMap (java.net.URL url)
+
+
+
+SiteMap
+
+public SiteMap (java.lang.String url)
+
+
+
+SiteMap
+
+public SiteMap (java.net.URL url,
+ java.util.Date lastModified)
+
+
+
+SiteMap
+
+public SiteMap (java.lang.String url,
+ java.lang.String lastModified)
+
+
+
+
+
+
+getSiteMapUrls
+
+public java.util.Collection<SiteMapURL > getSiteMapUrls ()
+
+
+
+Returns: the Collection of SitemapUrls in this Sitemap.
+
+
+toString
+
+public java.lang.String toString ()
+
+
+Overrides: toString in class java.lang.Object
+
+
+
+
+getBaseUrl
+
+public java.lang.String getBaseUrl ()
+
+
+
+Returns: the baseUrl for this Sitemap.
+
+
+addSiteMapUrl
+
+public void addSiteMapUrl (SiteMapURL url)
+
+
+Parameters: url - The SitemapUrl to be added to the Sitemap.
+
+
+isIndex
+
+public boolean isIndex ()
+
+
+Overrides: isIndex AbstractSiteMap
+
+
+
+
+
+SiteMapIndex (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.sitemaps
+
+
+java.lang.Object
+ crawlercommons.sitemaps.AbstractSiteMap
+ crawlercommons.sitemaps.SiteMapIndex
+
+
+public class SiteMapIndex extends AbstractSiteMap
+
+
+
+
+
+
+
+
+
+Nested Class Summary
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+SiteMapIndex
+
+public SiteMapIndex ()
+
+
+
+SiteMapIndex
+
+public SiteMapIndex (java.net.URL url)
+
+
+
+
+
+
+getSitemaps
+
+public java.util.Collection<AbstractSiteMap > getSitemaps ()
+
+
+
+Returns: a Collection of Sitemaps in this Sitemap Index.
+
+
+getSitemap
+
+public AbstractSiteMap getSitemap (java.net.URL url)
+
+Returns the Sitemap that has the given URL. Returns null if the URL
+ cannot be found.
+
+
+Parameters: url - - The Sitemap's URL
+Returns: SiteMap corresponding to the URL or null
+
+
+hasUnprocessedSitemap
+
+public boolean hasUnprocessedSitemap ()
+
+
+
+Returns: true if there are Sitemaps in this index that have not been
+ processed yet, false otherwise.
+
+
+nextUnprocessedSitemap
+
+public AbstractSiteMap nextUnprocessedSitemap ()
+
+
+
+Returns: an unprocessed Sitemap or null if no unprocessed Sitemaps could
+ be found.
+
+
+toString
+
+public java.lang.String toString ()
+
+
+Overrides: toString in class java.lang.Object
+
+
+
+
+isIndex
+
+public boolean isIndex ()
+
+
+Overrides: isIndex AbstractSiteMap
+
+
+
+
+
+SiteMapParser (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.sitemaps
+
+
+java.lang.Object
+ crawlercommons.sitemaps.SiteMapParser
+
+
+public class SiteMapParser extends java.lang.Object
+
+
+
+
+
+
+
+
+
+Field Summary
+
+
+static org.slf4j.Logger LOG
+
+
+static int MAX_BYTES_ALLOWED
+
+
+
+
+
+
+
+Method Summary
+
+
+ AbstractSiteMap parseSiteMap AbstractSiteMap sitemap)
+
+
+ AbstractSiteMap parseSiteMap
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+LOG
+
+public static final org.slf4j.Logger LOG
+
+
+
+
+
+MAX_BYTES_ALLOWED
+
+public static int MAX_BYTES_ALLOWED
+
+Sitemap docs must be limited to 10MB (10,485,760 bytes)
+
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+SiteMapParser
+
+public SiteMapParser ()
+
+
+
+
+
+
+parseSiteMap
+
+public AbstractSiteMap parseSiteMap (java.lang.String contentType,
+ byte[] content,
+ AbstractSiteMap sitemap)
+ throws UnknownFormatException ,
+ java.io.IOException
+
+Returned a processed copy of an unprocessed sitemap object, i.e. transfer the value of
+ getLastModified and sets the original sitemap to processed.
+
+
+
+Throws:
+UnknownFormatException java.io.IOException
+
+
+parseSiteMap
+
+public AbstractSiteMap parseSiteMap (java.lang.String contentType,
+ byte[] content,
+ java.net.URL url)
+ throws UnknownFormatException ,
+ java.io.IOException
+
+Returns a SiteMap or SiteMapIndex given a content type, byte content and
+ the URL of a sitemap
+
+
+
+Throws:
+UnknownFormatException java.io.IOException
+
+
+
+SiteMapURL.ChangeFrequency (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.sitemaps
+
+
+java.lang.Object
+ SiteMapURL.ChangeFrequency >
+ crawlercommons.sitemaps.SiteMapURL.ChangeFrequency
+
+
+All Implemented Interfaces: java.io.Serializable, java.lang.Comparable<SiteMapURL.ChangeFrequency >
+
+
+Enclosing class: SiteMapURL
+
+public static enum SiteMapURL.ChangeFrequency extends java.lang.Enum<SiteMapURL.ChangeFrequency >
+
+
+
+Allowed change frequencies
+
+
+
+
+
+
+
+
+
+
+
+
+Methods inherited from class java.lang.Enum
+
+clone, compareTo, equals, finalize, getDeclaringClass, hashCode, name, ordinal, toString, valueOf
+
+
+
+Methods inherited from class java.lang.Object
+
+getClass, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Enum Constant Detail
+
+
+
+ALWAYS
+
+public static final SiteMapURL.ChangeFrequency ALWAYS
+
+
+
+
+
+HOURLY
+
+public static final SiteMapURL.ChangeFrequency HOURLY
+
+
+
+
+
+DAILY
+
+public static final SiteMapURL.ChangeFrequency DAILY
+
+
+
+
+
+WEEKLY
+
+public static final SiteMapURL.ChangeFrequency WEEKLY
+
+
+
+
+
+MONTHLY
+
+public static final SiteMapURL.ChangeFrequency MONTHLY
+
+
+
+
+
+YEARLY
+
+public static final SiteMapURL.ChangeFrequency YEARLY
+
+
+
+
+
+NEVER
+
+public static final SiteMapURL.ChangeFrequency NEVER
+
+
+
+
+
+
+
+
+values
+
+public static SiteMapURL.ChangeFrequency [] values ()
+
+Returns an array containing the constants of this enum type, in
+the order they are declared. This method may be used to iterate
+over the constants as follows:
+
+for (SiteMapURL.ChangeFrequency c : SiteMapURL.ChangeFrequency.values())
+ System.out.println(c);
+
+
+
+
+Returns: an array containing the constants of this enum type, in
+the order they are declared
+
+
+valueOf
+
+public static SiteMapURL.ChangeFrequency valueOf (java.lang.String name)
+
+Returns the enum constant of this type with the specified name.
+The string must match exactly an identifier used to declare an
+enum constant in this type. (Extraneous whitespace characters are
+not permitted.)
+
+
+Parameters: name - the name of the enum constant to be returned.
+Returns: the enum constant with the specified name
+ Throws:
+java.lang.IllegalArgumentException - if this enum type has no constant
+with the specified name
+java.lang.NullPointerException - if the argument is null
+
+
+
+SiteMapURL (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.sitemaps
+
+
+java.lang.Object
+ crawlercommons.sitemaps.SiteMapURL
+
+
+public class SiteMapURL extends java.lang.Object
+
+
+
+The SitemapUrl class represents a URL found in a Sitemap.
+
+
+
+
+Author: fmccown
+
+
+
+
+
+
+
+
+
+
+
+Constructor Summary
+
+SiteMapURL
+
+SiteMapURL
+
+SiteMapURL
+
+SiteMapURL SiteMapURL.ChangeFrequency changeFreq,
+ double priority)
+
+
+
+
+
+
+
+Method Summary
+
+
+ SiteMapURL.ChangeFrequency getChangeFrequency
+
+
+ java.util.Date getLastModified
+
+
+ double getPriority
+
+
+ java.net.URL getUrl
+
+
+ void setChangeFrequency SiteMapURL.ChangeFrequency changeFreq)
+
+
+ void setChangeFrequency
+
+
+ void setLastModified
+
+
+ void setLastModified
+
+
+ void setPriority
+
+
+ void setPriority
+
+
+ void setUrl
+
+
+ void setUrl
+
+
+ java.lang.String toString
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+SiteMapURL
+
+public SiteMapURL (java.lang.String url)
+
+
+
+SiteMapURL
+
+public SiteMapURL (java.net.URL url)
+
+
+
+SiteMapURL
+
+public SiteMapURL (java.lang.String url,
+ java.lang.String lastModified,
+ java.lang.String changeFreq,
+ java.lang.String priority)
+
+
+
+SiteMapURL
+
+public SiteMapURL (java.net.URL url,
+ java.util.Date lastModified,
+ SiteMapURL.ChangeFrequency changeFreq,
+ double priority)
+
+
+
+
+
+
+getUrl
+
+public java.net.URL getUrl ()
+
+Return the URL.
+
+
+
+Returns: URL
+
+
+setUrl
+
+public void setUrl (java.net.URL url)
+
+Set the URL.
+
+
+Parameters: url -
+
+
+setUrl
+
+public void setUrl (java.lang.String url)
+
+Set the URL.
+
+
+Parameters: url -
+
+
+getLastModified
+
+public java.util.Date getLastModified ()
+
+Return when this URL was last modified.
+
+
+
+Returns: last modified date
+
+
+setLastModified
+
+public void setLastModified (java.lang.String lastModified)
+
+Set when this URL was last modified.
+
+
+Parameters: lastModified -
+
+
+setLastModified
+
+public void setLastModified (java.util.Date lastModified)
+
+Set when this URL was last modified.
+
+
+Parameters: lastModified -
+
+
+getPriority
+
+public double getPriority ()
+
+Return this URL's priority (a value between [0.0 - 1.0]).
+
+
+
+Returns: URL's priority (a value between [0.0 - 1.0])
+
+
+setPriority
+
+public void setPriority (double priority)
+
+Set the URL's priority to a value between [0.0 - 1.0] (0.0 is used if the
+ given priority is out of range).
+
+
+Parameters: priority -
+
+
+setPriority
+
+public void setPriority (java.lang.String priority)
+
+Set the URL's priority to a value between [0.0 - 1.0] (0.0 is used if the
+ given priority is out of range).
+
+
+Parameters: priority -
+
+
+getChangeFrequency
+
+public SiteMapURL.ChangeFrequency getChangeFrequency ()
+
+Return the URL's change frequency
+
+
+
+Returns: the URL's change frequency
+
+
+setChangeFrequency
+
+public void setChangeFrequency (SiteMapURL.ChangeFrequency changeFreq)
+
+Set the URL's change frequency
+
+
+Parameters: changeFreq -
+
+
+setChangeFrequency
+
+public void setChangeFrequency (java.lang.String changeFreq)
+
+Set the URL's change frequency
+
+
+Parameters: changeFreq -
+
+
+toString
+
+public java.lang.String toString ()
+
+
+Overrides: toString in class java.lang.Object
+
+
+
+
+
+UnknownFormatException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.sitemaps
+
+
+java.lang.Object
+ crawlercommons.sitemaps.UnknownFormatException
+
+
+All Implemented Interfaces: java.io.Serializable
+
+
+public class UnknownFormatException extends java.lang.Exception
+
+
+
+
+See Also: Serialized Form
+
+
+
+
+
+
+
+
+
+
+
+Method Summary
+
+
+ java.lang.String getError
+
+
+
+Methods inherited from class java.lang.Throwable
+
+fillInStackTrace, getCause, getLocalizedMessage, getMessage, getStackTrace, initCause, printStackTrace, printStackTrace, printStackTrace, setStackTrace, toString
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+UnknownFormatException
+
+public UnknownFormatException ()
+
+
+
+UnknownFormatException
+
+public UnknownFormatException (java.lang.String err)
+
+
+
+
+
+
+getError
+
+public java.lang.String getError ()
+
+
+
+
+
+
+Uses of Class crawlercommons.sitemaps.AbstractSiteMap.SitemapType (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.sitemaps.AbstractSiteMap (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.sitemaps.SiteMap (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.sitemaps.SiteMap
+
+
+Uses of Class crawlercommons.sitemaps.SiteMapIndex (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.sitemaps.SiteMapIndex
+
+
+Uses of Class crawlercommons.sitemaps.SiteMapParser (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.sitemaps.SiteMapParser
+
+
+Uses of Class crawlercommons.sitemaps.SiteMapURL.ChangeFrequency (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.sitemaps.SiteMapURL (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.sitemaps.UnknownFormatException (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+crawlercommons.sitemaps (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+crawlercommons.sitemaps
+
+
+
+
+
+
+
+
+
+
+
diff --git a/doc/javadoc/crawlercommons/sitemaps/package-summary.html b/doc/javadoc/crawlercommons/sitemaps/package-summary.html
new file mode 100644
index 0000000..1cc290e
--- /dev/null
+++ b/doc/javadoc/crawlercommons/sitemaps/package-summary.html
@@ -0,0 +1,205 @@
+
+
+
+
+
+
+crawlercommons.sitemaps (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+Package crawlercommons.sitemaps
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+crawlercommons.sitemaps Class Hierarchy (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Hierarchy For Package crawlercommons.sitemaps
+
+
+
+Package Hierarchies: All Packages
+
+Class Hierarchy
+
+
+
+Enum Hierarchy
+
+
+java.lang.Object
+java.lang.Enum<E> (implements java.lang.Comparable<T>, java.io.Serializable)
+
+
+
+
+Uses of Package crawlercommons.sitemaps (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Package
+
+
+
+
+
+
+
+
+
+EffectiveTldFinder.EffectiveTLD (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.url
+
+
+java.lang.Object
+ crawlercommons.url.EffectiveTldFinder.EffectiveTLD
+
+
+Enclosing class: EffectiveTldFinder
+
+public static class EffectiveTldFinder.EffectiveTLD extends java.lang.Object
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Method Summary
+
+
+ java.lang.String getDomain
+
+
+ boolean isException
+
+
+ boolean isWild
+
+
+ java.lang.String toString
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+EffectiveTldFinder.EffectiveTLD
+
+public EffectiveTldFinder.EffectiveTLD (java.lang.String line)
+
+
+
+
+
+
+getDomain
+
+public java.lang.String getDomain ()
+
+
+
+
+
+isWild
+
+public boolean isWild ()
+
+
+
+
+
+isException
+
+public boolean isException ()
+
+
+
+
+
+toString
+
+public java.lang.String toString ()
+
+
+Overrides: toString in class java.lang.Object
+
+
+
+
+
+EffectiveTldFinder (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.url
+
+
+java.lang.Object
+ crawlercommons.url.EffectiveTldFinder
+
+
+public class EffectiveTldFinder extends java.lang.Object
+
+
+
+Given a URL's hostname, there are determining the actual domain requires
+ knowledge of the various domain registrars and their assignment policies.
+ The best publicly available knowledge of this is maintained by the Mozilla
+ developers; this class uses their data file format. For more information, see
+
+
+ This class just needs "effective_tld_names.dat" in the classpath. If you want
+ to configure it with other data, call EffectiveTldFinder.getInstance.initialize(is)
+ and have at it.
+
+
+
+
+
+
+
+
+
+
+
+
+Field Summary
+
+
+static java.lang.String COMMENT
+
+
+static char DOT
+
+
+static java.lang.String DOT_REGEX
+
+
+static java.lang.String ETLD_DATA
+
+
+static java.lang.String EXCEPTION
+
+
+static java.lang.String WILD_CARD
+
+
+
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+ETLD_DATA
+
+public static final java.lang.String ETLD_DATA
+
+
+See Also: Constant Field Values
+
+
+COMMENT
+
+public static final java.lang.String COMMENT
+
+
+See Also: Constant Field Values
+
+
+DOT_REGEX
+
+public static final java.lang.String DOT_REGEX
+
+
+See Also: Constant Field Values
+
+
+EXCEPTION
+
+public static final java.lang.String EXCEPTION
+
+
+See Also: Constant Field Values
+
+
+WILD_CARD
+
+public static final java.lang.String WILD_CARD
+
+
+See Also: Constant Field Values
+
+
+DOT
+
+public static final char DOT
+
+
+See Also: Constant Field Values
+
+
+
+
+
+getInstance
+
+public static EffectiveTldFinder getInstance ()
+
+
+
+
+
+initialize
+
+public boolean initialize (java.io.InputStream effective_tld_data_stream)
+
+
+
+
+
+getEffectiveTLDs
+
+public static java.util.Map<java.lang.String,EffectiveTldFinder.EffectiveTLD > getEffectiveTLDs ()
+
+
+
+
+
+getEffectiveTLD
+
+public static EffectiveTldFinder.EffectiveTLD getEffectiveTLD (java.lang.String hostname)
+
+
+Parameters: hostname -
+Returns: the Effective TLD
+
+
+getAssignedDomain
+
+public static java.lang.String getAssignedDomain (java.lang.String hostname)
+
+This method uses the effective TLD to determine which component of
+ a FQDN is the NIC-assigned domain name.
+
+
+Parameters: hostname -
+Returns: the NIC-assigned domain name
+
+
+isConfigured
+
+public boolean isConfigured ()
+
+
+
+
+
+
+PaidLevelDomain (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons.url
+
+
+java.lang.Object
+ crawlercommons.url.PaidLevelDomain
+
+
+public class PaidLevelDomain extends java.lang.Object
+
+
+
+Routines to extract the PLD (paid-level domain, as per the IRLbot paper) from a hostname or URL.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Method Summary
+
+
+static java.lang.String getPLD
+
+
+static java.lang.String getPLD
+
+
+
+Methods inherited from class java.lang.Object
+
+clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
+
+
+
+
+
+
+
+
+Constructor Detail
+
+
+
+PaidLevelDomain
+
+public PaidLevelDomain ()
+
+
+
+
+
+
+getPLD
+
+public static java.lang.String getPLD (java.lang.String hostname)
+
+Extract the PLD (paid-level domain) from the hostname. If the format isn't recognized,
+ the original hostname is returned.
+
+
+Parameters: hostname - - hostname from URL, e.g. www.domain.com.it
+Returns: - PLD, e.g. domain.com.it
+
+
+getPLD
+
+public static java.lang.String getPLD (java.net.URL url)
+
+Extract the PLD (paid-level domain) from the URL.
+
+
+Parameters: url - - Valid URL, e.g. http://www.domain.com.it
+Returns: - PLD e.g. domain.com.it
+
+
+
+Uses of Class crawlercommons.url.EffectiveTldFinder.EffectiveTLD (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.url.EffectiveTldFinder (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+
+
+
+
+
+
+
+
+
+
+
+
+Uses of Class crawlercommons.url.PaidLevelDomain (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Class
+
+No usage of crawlercommons.url.PaidLevelDomain
+
+
+crawlercommons.url (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+crawlercommons.url
+
+
+
+
+
diff --git a/doc/javadoc/crawlercommons/url/package-summary.html b/doc/javadoc/crawlercommons/url/package-summary.html
new file mode 100644
index 0000000..92a08b2
--- /dev/null
+++ b/doc/javadoc/crawlercommons/url/package-summary.html
@@ -0,0 +1,166 @@
+
+
+
+
+
+
+crawlercommons.url (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+Package crawlercommons.url
+
+
+
+
+
+Class Summary
+
+EffectiveTldFinder Given a URL's hostname, there are determining the actual domain requires
+ knowledge of the various domain registrars and their assignment policies.
+
+
+EffectiveTldFinder.EffectiveTLD
+
+
+PaidLevelDomain Routines to extract the PLD (paid-level domain, as per the IRLbot paper) from a hostname or URL.
+
+
+
+
+
+
+
+
+crawlercommons.url Class Hierarchy (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Hierarchy For Package crawlercommons.url
+
+
+
+Package Hierarchies: All Packages
+
+Class Hierarchy
+
+
+
+Uses of Package crawlercommons.url (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Uses of Package
+
+
+
+
+
+
+
+
+
+Deprecated List (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Deprecated API
+
+Contents
+
+
+API Help (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+How This API Document Is Organized
+
+This API (Application Programming Interface) document has pages corresponding to the items in the navigation bar, described as follows.
+Overview
+
+
+
+The Overview page is the front page of this API document and provides a list of all packages with a summary for each. This page can also contain an overall description of the set of packages.
+
+Package
+
+
+
+Each package has a page that contains a list of its classes and interfaces, with a summary for each. This page can contain four categories:
+Interfaces (italic) Classes Enums Exceptions Errors Annotation Types
+
+
+Class/Interface
+
+
+
+Each class, interface, nested class and nested interface has its own separate page. Each of these pages has three sections consisting of a class/interface description, summary tables, and detailed member descriptions:
+Class inheritance diagram Direct Subclasses All Known Subinterfaces All Known Implementing Classes Class/interface declaration Class/interface description
+
+
Nested Class Summary Field Summary Constructor Summary Method Summary
+
+
Field Detail Constructor Detail Method Detail
+Each summary entry contains the first sentence from the detailed description for that item. The summary entries are alphabetical, while the detailed descriptions are in the order they appear in the source code. This preserves the logical groupings established by the programmer.
+
+
+Annotation Type
+
+
+
+Each annotation type has its own separate page with the following sections:
+Annotation Type declaration Annotation Type description Required Element Summary Optional Element Summary Element Detail
+
+
+
+Enum
+
+
+
+Each enum has its own separate page with the following sections:
+Enum declaration Enum description Enum Constant Summary Enum Constant Detail
+
+
+Use
+
+Each documented package, class and interface has its own Use page. This page describes what packages, classes, methods, constructors and fields use any part of the given class or package. Given a class or interface A, its Use page includes subclasses of A, fields declared as A, methods that return A, and methods and constructors with parameters of type A. You can access this page by first going to the package, class or interface, then clicking on the "Use" link in the navigation bar.
+
+Tree (Class Hierarchy)
+
+There is a Class Hierarchy page for all packages, plus a hierarchy for each package. Each hierarchy page contains a list of classes and a list of interfaces. The classes are organized by inheritance structure starting with java.lang.Object. The interfaces do not inherit from java.lang.Object.
+When viewing the Overview page, clicking on "Tree" displays the hierarchy for all packages. When viewing a particular package, class or interface page, clicking "Tree" displays the hierarchy for only that package.
+
+
+Deprecated API
+
+The Deprecated API page lists all of the API that have been deprecated. A deprecated API is not recommended for use, generally due to improvements, and a replacement API is usually given. Deprecated APIs may be removed in future implementations.
+
+Index
+
+The Index contains an alphabetic list of all classes, interfaces, constructors, methods, and fields.
+
+Prev/Next
+These links take you to the next or previous class, interface, package, or related page.
+Frames/No Frames
+These links show and hide the HTML frames. All pages are available with or without frames.
+
+
+Serialized Form
+Each serializable or externalizable class has a description of its serialization fields and methods. This information is of interest to re-implementors, not to developers using the API. While there is no link in the navigation bar, you can get to this information by going to any serialized class and clicking "Serialized Form" in the "See also" section of the class description.
+
+
+Constant Field Values
+The Constant Field Values page lists the static final fields and their values.
+
+
+
+This help file applies to API documentation generated using the standard doclet.
+
+
+Index (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+A B C D E F G H I K L M N P R S T U V W _
+A
+
+abort() BaseFetcher
+Terminate any async request being processed.
+ abort() SimpleFileFetcher
+
+ abort() SimpleHttpFetcher
+
+ AbortedFetchException crawlercommons.fetcher AbortedFetchException() AbortedFetchException
+
+ AbortedFetchException(String, AbortedFetchReason) AbortedFetchException
+
+ AbortedFetchException(String, String, AbortedFetchReason) AbortedFetchException
+
+ AbortedFetchReason crawlercommons.fetcher AbstractSiteMap crawlercommons.sitemaps SiteMap or SiteMapIndex AbstractSiteMap() AbstractSiteMap
+
+ AbstractSiteMap.SitemapType crawlercommons.sitemaps Various Sitemap types addRule(String, boolean) SimpleRobotRules
+
+ addSitemap(String) BaseRobotRules
+
+ addSiteMapUrl(SiteMapURL) SiteMap
+
+ addValidMimeType(String) BaseFetcher
+
+ addValidMimeTypes(Set<String>) BaseFetcher
+
+
+
+B
+
+BadProtocolFetchException crawlercommons.fetcher BadProtocolFetchException() BadProtocolFetchException
+
+ BadProtocolFetchException(String) BadProtocolFetchException
+
+ BaseFetcher crawlercommons.fetcher BaseFetcher() BaseFetcher
+
+ BaseFetchException crawlercommons.fetcher BaseFetchException() BaseFetchException
+
+ BaseFetchException(String) BaseFetchException
+
+ BaseFetchException(String, String) BaseFetchException
+
+ BaseFetchException(String, Exception) BaseFetchException
+
+ BaseFetchException(String, String, Exception) BaseFetchException
+
+ BaseHttpFetcher crawlercommons.fetcher.http BaseHttpFetcher(int, UserAgent) BaseHttpFetcher
+
+ BaseHttpFetcher.RedirectMode crawlercommons.fetcher.http BaseRobotRules crawlercommons.robots Result from parsing a single robots.txt file - which means we
+ get a set of rules, and a crawl-delay. BaseRobotRules() BaseRobotRules
+
+ BaseRobotsParser crawlercommons.robots BaseRobotsParser() BaseRobotsParser
+
+
+
+C
+
+clear() Payload
+
+ clearRules() SimpleRobotRules
+
+ COMMENT EffectiveTldFinder
+
+ compareToBase(BaseFetchException) BaseFetchException
+
+ containsKey(Object) Payload
+
+ containsValue(Object) Payload
+
+ convertToDate(String) AbstractSiteMap
+Convert the given date (given in an acceptable DateFormat), null if the
+ date is not in the correct format.
+ crawlercommons.fetcher crawlercommons.fetcher.file crawlercommons.fetcher.http crawlercommons.robots crawlercommons.sitemaps crawlercommons.url createFetcher(BaseHttpFetcher) RobotUtils
+
+ createFetcher(UserAgent, int) RobotUtils
+
+
+
+D
+
+DEFAULT_ACCEPT_LANGUAGE BaseHttpFetcher
+
+ DEFAULT_BROWSER_VERSION UserAgent
+
+ DEFAULT_MAX_CONNECTIONS_PER_HOST BaseHttpFetcher
+
+ DEFAULT_MAX_CONTENT_SIZE BaseFetcher
+
+ DEFAULT_MAX_REDIRECTS BaseHttpFetcher
+
+ DEFAULT_MIN_RESPONSE_RATE BaseHttpFetcher
+
+ DEFAULT_REDIRECT_MODE BaseHttpFetcher
+
+ DOT EffectiveTldFinder
+
+ DOT_REGEX EffectiveTldFinder
+
+
+
+E
+
+EffectiveTldFinder crawlercommons.url Given a URL's hostname, there are determining the actual domain requires
+ knowledge of the various domain registrars and their assignment policies. EffectiveTldFinder.EffectiveTLD crawlercommons.url EffectiveTldFinder.EffectiveTLD(String) EffectiveTldFinder.EffectiveTLD
+
+ EncodingUtils crawlercommons.fetcher EncodingUtils() EncodingUtils
+
+ EncodingUtils.ExpandedResult crawlercommons.fetcher EncodingUtils.ExpandedResult(byte[], boolean) EncodingUtils.ExpandedResult
+
+ entrySet() Payload
+
+ equals(Object) BaseFetchException
+
+ equals(Object) Payload
+
+ ETLD_DATA EffectiveTldFinder
+
+ EXCEPTION EffectiveTldFinder
+
+
+
+F
+
+failedFetch(int) BaseRobotsParser
+The fetch of robots.txt failed, so return rules appropriate give the
+ HTTP status code.
+ failedFetch(int) SimpleRobotRulesParser
+
+ fetch(String) SimpleHttpFetcher
+
+ fetch(HttpRequestBase, String, Payload) SimpleHttpFetcher
+
+ FetchedResult crawlercommons.fetcher FetchedResult(String, String, long, Metadata, byte[], String, int, Payload, String, int, String) FetchedResult
+
+
+
+G
+
+get(String) BaseFetcher
+
+ get(String, Payload) BaseFetcher
+Get the content stored in the resource referenced by
+get(String, Payload) SimpleFileFetcher
+
+ get(String, Payload) SimpleHttpFetcher
+
+ get(Object) Payload
+
+ getAbortReason() AbortedFetchException
+
+ getAcceptLanguage() BaseHttpFetcher
+
+ getAgentName() UserAgent
+
+ getAssignedDomain(String) EffectiveTldFinder
+This method uses the effective TLD to determine which component of
+ a FQDN is the NIC-assigned domain name.
+ getBaseUrl() FetchedResult
+
+ getBaseUrl() SiteMap
+
+ getCause() BaseFetchException
+
+ getChangeFrequency() SiteMapURL
+Return the URL's change frequency
+ getConnectionTimeout() SimpleHttpFetcher
+
+ getContent() FetchedResult
+
+ getContentLength() FetchedResult
+
+ getContentType() FetchedResult
+
+ getCrawlDelay() BaseRobotRules
+
+ getDefaultMaxContentSize() BaseFetcher
+
+ getDomain() EffectiveTldFinder.EffectiveTLD
+
+ getEffectiveTLD(String) EffectiveTldFinder
+
+ getEffectiveTLDs() EffectiveTldFinder
+
+ getError() UnknownFormatException
+
+ getExpanded() EncodingUtils.ExpandedResult
+
+ getFetchedUrl() FetchedResult
+
+ getFetchTime() FetchedResult
+
+ getFullDateFormat() AbstractSiteMap
+
+ getHeaders() FetchedResult
+
+ getHostAddress() FetchedResult
+
+ getHttpHeaders() HttpFetchException
+
+ getHttpStatus() HttpFetchException
+
+ getHttpVersion() SimpleHttpFetcher
+
+ getInstance() EffectiveTldFinder
+
+ getLastModified() AbstractSiteMap
+
+ getLastModified() SiteMapURL
+Return when this URL was last modified.
+ getLocalizedMessage() BaseFetchException
+
+ getMaxConnectionsPerHost() BaseHttpFetcher
+
+ getMaxContentSize(String) BaseFetcher
+
+ getMaxFetchTime() RobotUtils
+
+ getMaxRedirects() BaseHttpFetcher
+
+ getMaxRetryCount() SimpleHttpFetcher
+
+ getMaxThreads() BaseHttpFetcher
+
+ getMessage() BaseFetchException
+
+ getMessage() HttpFetchException
+
+ getMimeTypeFromContentType(String) BaseFetcher
+
+ getMinResponseRate() BaseHttpFetcher
+Return the minimum response rate.
+ getNewBaseUrl() FetchedResult
+
+ getNumRedirects() FetchedResult
+
+ getNumWarnings() SimpleRobotRulesParser
+
+ getPayload() FetchedResult
+
+ getPLD(String) PaidLevelDomain
+Extract the PLD (paid-level domain) from the hostname.
+ getPLD(URL) PaidLevelDomain
+Extract the PLD (paid-level domain) from the URL.
+ getPriority() SiteMapURL
+Return this URL's priority (a value between [0.0 - 1.0]).
+ getReason() RedirectFetchException
+
+ getRedirectedUrl() RedirectFetchException
+
+ getRedirectMode() BaseHttpFetcher
+
+ getResponseRate() FetchedResult
+
+ getRobotRules(BaseHttpFetcher, BaseRobotsParser, URL) RobotUtils
+Externally visible, static method for use in tools and for testing.
+ getSitemap(URL) SiteMapIndex
+Returns the Sitemap that has the given URL.
+ getSitemaps() BaseRobotRules
+
+ getSitemaps() SiteMapIndex
+
+ getSiteMapUrls() SiteMap
+
+ getSocketTimeout() SimpleHttpFetcher
+
+ getStackTrace() BaseFetchException
+
+ getType() AbstractSiteMap
+
+ getUrl() BaseFetchException
+
+ getUrl() AbstractSiteMap
+
+ getUrl() SiteMapURL
+Return the URL.
+ getUserAgent() BaseHttpFetcher
+
+ getUserAgentString() UserAgent
+
+ getValidMimeTypes() BaseFetcher
+
+
+
+H
+
+hashCode() BaseFetchException
+
+ hashCode() Payload
+
+ hasUnprocessedSitemap() SiteMapIndex
+
+ HttpFetchException crawlercommons.fetcher HttpFetchException() HttpFetchException
+
+ HttpFetchException(String, String, int, Metadata) HttpFetchException
+
+
+
+I
+
+initCause(Throwable) BaseFetchException
+
+ initialize(InputStream) EffectiveTldFinder
+
+ IOFetchException crawlercommons.fetcher IOFetchException() IOFetchException
+
+ IOFetchException(String, IOException) IOFetchException
+
+ isAllowAll() BaseRobotRules
+
+ isAllowAll() SimpleRobotRules
+Is our ruleset set up to allow all access?
+ isAllowed(String) BaseRobotRules
+
+ isAllowed(String) SimpleRobotRules
+
+ isAllowNone() BaseRobotRules
+
+ isAllowNone() SimpleRobotRules
+Is our ruleset set up to disallow all access?
+ isConfigured() EffectiveTldFinder
+
+ isDeferVisits() BaseRobotRules
+
+ isEmpty() Payload
+
+ isException() EffectiveTldFinder.EffectiveTLD
+
+ isIndex() AbstractSiteMap
+
+ isIndex() SiteMap
+
+ isIndex() SiteMapIndex
+
+ isProcessed() AbstractSiteMap
+
+ isTruncated() EncodingUtils.ExpandedResult
+
+ isWild() EffectiveTldFinder.EffectiveTLD
+
+
+
+K
+
+keySet() Payload
+
+
+
+L
+
+LOG SiteMapParser
+
+
+
+M
+
+MAX_BYTES_ALLOWED SiteMapParser
+Sitemap docs must be limited to 10MB (10,485,760 bytes)
+
+
+N
+
+nextUnprocessedSitemap() SiteMapIndex
+
+ NO_MIN_RESPONSE_RATE BaseHttpFetcher
+
+ NO_REDIRECTS BaseHttpFetcher
+
+
+
+P
+
+PaidLevelDomain crawlercommons.url Routines to extract the PLD (paid-level domain, as per the IRLbot paper) from a hostname or URL. PaidLevelDomain() PaidLevelDomain
+
+ parseContent(String, byte[], String, String) BaseRobotsParser
+Parse the robots.txt file in , and return rules appropriate for
+ processing paths by
+parseContent(String, byte[], String, String) SimpleRobotRulesParser
+
+ parseSiteMap(String, byte[], AbstractSiteMap) SiteMapParser
+Returned a processed copy of an unprocessed sitemap object, i.e.
+ parseSiteMap(String, byte[], URL) SiteMapParser
+Returns a SiteMap or SiteMapIndex given a content type, byte content and
+ the URL of a sitemap
+ Payload crawlercommons.fetcher Payload() Payload
+
+ printStackTrace() BaseFetchException
+
+ printStackTrace(PrintStream) BaseFetchException
+
+ printStackTrace(PrintWriter) BaseFetchException
+
+ processDeflateEncoded(byte[]) EncodingUtils
+
+ processDeflateEncoded(byte[], int) EncodingUtils
+
+ processGzipEncoded(byte[]) EncodingUtils
+
+ processGzipEncoded(byte[], int) EncodingUtils
+
+ put(String, Object) Payload
+
+ putAll(Map<? extends String, ? extends Object>) Payload
+
+
+
+R
+
+readBaseFields(DataInput) BaseFetchException
+
+ RedirectFetchException crawlercommons.fetcher RedirectFetchException() RedirectFetchException
+
+ RedirectFetchException(String, String, RedirectFetchException.RedirectExceptionReason) RedirectFetchException
+
+ RedirectFetchException.RedirectExceptionReason crawlercommons.fetcher remove(Object) Payload
+
+ RobotUtils crawlercommons.robots RobotUtils() RobotUtils
+
+
+
+S
+
+setAcceptLanguage(String) BaseHttpFetcher
+
+ setChangeFrequency(SiteMapURL.ChangeFrequency) SiteMapURL
+Set the URL's change frequency
+ setChangeFrequency(String) SiteMapURL
+Set the URL's change frequency
+ setConnectionTimeout(int) SimpleHttpFetcher
+
+ setCrawlDelay(long) BaseRobotRules
+
+ setDefaultMaxContentSize(int) BaseFetcher
+
+ setDeferVisits(boolean) BaseRobotRules
+
+ setExpanded(byte[]) EncodingUtils.ExpandedResult
+
+ setHttpVersion(HttpVersion) SimpleHttpFetcher
+
+ setLastModified(Date) AbstractSiteMap
+
+ setLastModified(String) AbstractSiteMap
+
+ setLastModified(String) SiteMapURL
+Set when this URL was last modified.
+ setLastModified(Date) SiteMapURL
+Set when this URL was last modified.
+ setMaxConnectionsPerHost(int) BaseHttpFetcher
+
+ setMaxContentSize(String, int) BaseFetcher
+
+ setMaxRedirects(int) BaseHttpFetcher
+
+ setMaxRetryCount(int) SimpleHttpFetcher
+
+ setMinResponseRate(int) BaseHttpFetcher
+
+ setPayload(Payload) FetchedResult
+
+ setPriority(double) SiteMapURL
+Set the URL's priority to a value between [0.0 - 1.0] (0.0 is used if the
+ given priority is out of range).
+ setPriority(String) SiteMapURL
+Set the URL's priority to a value between [0.0 - 1.0] (0.0 is used if the
+ given priority is out of range).
+ setRedirectMode(BaseHttpFetcher.RedirectMode) BaseHttpFetcher
+
+ setSocketTimeout(int) SimpleHttpFetcher
+
+ setStackTrace(StackTraceElement[]) BaseFetchException
+
+ setTruncated(boolean) EncodingUtils.ExpandedResult
+
+ setUrl(URL) SiteMapURL
+Set the URL.
+ setUrl(String) SiteMapURL
+Set the URL.
+ setValidMimeTypes(Set<String>) BaseFetcher
+
+ SimpleFileFetcher crawlercommons.fetcher.file SimpleFileFetcher() SimpleFileFetcher
+
+ SimpleHttpFetcher crawlercommons.fetcher.http SimpleHttpFetcher(UserAgent) SimpleHttpFetcher
+
+ SimpleHttpFetcher(int, UserAgent) SimpleHttpFetcher
+
+ SimpleRobotRules crawlercommons.robots Result from parsing a single robots.txt file - which means we
+ get a set of rules, and a crawl-delay. SimpleRobotRules() SimpleRobotRules
+
+ SimpleRobotRules(SimpleRobotRules.RobotRulesMode) SimpleRobotRules
+
+ SimpleRobotRules.RobotRule crawlercommons.robots Single rule that maps from a path prefix to an allow flag. SimpleRobotRules.RobotRule(String, boolean) SimpleRobotRules.RobotRule
+
+ SimpleRobotRules.RobotRule(Pattern, boolean) SimpleRobotRules.RobotRule
+
+ SimpleRobotRules.RobotRulesMode crawlercommons.robots SimpleRobotRulesParser crawlercommons.robots SimpleRobotRulesParser() SimpleRobotRulesParser
+
+ SiteMap crawlercommons.sitemaps SiteMap() SiteMap
+
+ SiteMap(URL) SiteMap
+
+ SiteMap(String) SiteMap
+
+ SiteMap(URL, Date) SiteMap
+
+ SiteMap(String, String) SiteMap
+
+ SiteMapIndex crawlercommons.sitemaps SiteMapIndex() SiteMapIndex
+
+ SiteMapIndex(URL) SiteMapIndex
+
+ SiteMapParser crawlercommons.sitemaps SiteMapParser() SiteMapParser
+
+ SiteMapURL crawlercommons.sitemaps The SitemapUrl class represents a URL found in a Sitemap. SiteMapURL(String) SiteMapURL
+
+ SiteMapURL(URL) SiteMapURL
+
+ SiteMapURL(String, String, String, String) SiteMapURL
+
+ SiteMapURL(URL, Date, SiteMapURL.ChangeFrequency, double) SiteMapURL
+
+ SiteMapURL.ChangeFrequency crawlercommons.sitemaps Allowed change frequencies size() Payload
+
+
+
+T
+
+toString() BaseFetchException
+
+ toString() SiteMap
+
+ toString() SiteMapIndex
+
+ toString() SiteMapURL
+
+ toString() EffectiveTldFinder.EffectiveTLD
+
+
+
+U
+
+UnknownFormatException crawlercommons.sitemaps UnknownFormatException() UnknownFormatException
+
+ UnknownFormatException(String) UnknownFormatException
+
+ UNSET_CRAWL_DELAY BaseRobotRules
+
+ UrlFetchException crawlercommons.fetcher UrlFetchException() UrlFetchException
+
+ UrlFetchException(String, String) UrlFetchException
+
+ UserAgent crawlercommons.fetcher.http UserAgent(String, String, String) UserAgent
+
+ UserAgent(String, String, String, String) UserAgent
+
+ UserAgent(String, String, String, String, String) UserAgent
+
+
+
+V
+
+valueOf(String) AbortedFetchReason
+Returns the enum constant of this type with the specified name.
+ valueOf(String) BaseHttpFetcher.RedirectMode
+Returns the enum constant of this type with the specified name.
+ valueOf(String) RedirectFetchException.RedirectExceptionReason
+Returns the enum constant of this type with the specified name.
+ valueOf(String) SimpleRobotRules.RobotRulesMode
+Returns the enum constant of this type with the specified name.
+ valueOf(String) AbstractSiteMap.SitemapType
+Returns the enum constant of this type with the specified name.
+ valueOf(String) SiteMapURL.ChangeFrequency
+Returns the enum constant of this type with the specified name.
+ values() AbortedFetchReason
+Returns an array containing the constants of this enum type, in
+the order they are declared.
+ values() BaseHttpFetcher.RedirectMode
+Returns an array containing the constants of this enum type, in
+the order they are declared.
+ values() Payload
+
+ values() RedirectFetchException.RedirectExceptionReason
+Returns an array containing the constants of this enum type, in
+the order they are declared.
+ values() SimpleRobotRules.RobotRulesMode
+Returns an array containing the constants of this enum type, in
+the order they are declared.
+ values() AbstractSiteMap.SitemapType
+Returns an array containing the constants of this enum type, in
+the order they are declared.
+ values() SiteMapURL.ChangeFrequency
+Returns an array containing the constants of this enum type, in
+the order they are declared.
+
+
+W
+
+WILD_CARD EffectiveTldFinder
+
+ writeBaseFields(DataOutput) BaseFetchException
+
+
+
+_
+
+_acceptLanguage BaseHttpFetcher
+
+ _defaultMaxContentSize BaseFetcher
+
+ _maxConnectionsPerHost BaseHttpFetcher
+
+ _maxContentSizes BaseFetcher
+
+ _maxRedirects BaseHttpFetcher
+
+ _maxThreads BaseHttpFetcher
+
+ _minResponseRate BaseHttpFetcher
+
+ _redirectMode BaseHttpFetcher
+
+ _userAgent BaseHttpFetcher
+
+ _validMimeTypes BaseFetcher
+
+
+A B C D E F G H I K L M N P R S T U V W _
+
+
+
+crawlercommons 0.2-SNAPSHOT API
+
+
+
+
+
+
+
+
+<![CDATA[
+<H2>
+Frame Alert</H2>
+
+<P>
+This document is designed to be viewed using the frames feature. If you see this message, you are using a non-frame-capable web client.
+<BR>
+Link to<A HREF="overview-summary.html">Non-frame version.</A>
+]]>
+
+
diff --git a/doc/javadoc/overview-frame.html b/doc/javadoc/overview-frame.html
new file mode 100644
index 0000000..a59de77
--- /dev/null
+++ b/doc/javadoc/overview-frame.html
@@ -0,0 +1,52 @@
+
+
+
+
+
+
+Overview List (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
diff --git a/doc/javadoc/overview-summary.html b/doc/javadoc/overview-summary.html
new file mode 100644
index 0000000..ab2f352
--- /dev/null
+++ b/doc/javadoc/overview-summary.html
@@ -0,0 +1,176 @@
+
+
+
+
+
+
+Overview (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+crawlercommons 0.2-SNAPSHOT API
+
+
+
+
+
+
+
+Class Hierarchy (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Hierarchy For All Packages
+
+
+Package Hierarchies: crawlercommons.fetcher , crawlercommons.fetcher.file , crawlercommons.fetcher.http , crawlercommons.robots , crawlercommons.sitemaps , crawlercommons.url
+
+Class Hierarchy
+
+
+
+Enum Hierarchy
+
+
+java.lang.Object
+java.lang.Enum<E> (implements java.lang.Comparable<T>, java.io.Serializable)
+
+
+
+
+Serialized Form (crawlercommons 0.2-SNAPSHOT API)
+
+
+
+
+
+
+
+
+
+
+Serialized Form
+
+
+
+
+Package crawlercommons.fetcher
+
+
+
+
+
+
+
+
+
+Serialized Fields
+
+
+
+_abortReason
+
+AbortedFetchReason _abortReason
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Serialized Fields
+
+
+
+_maxContentSizes
+
+java.util.Map<K,V> _maxContentSizes
+
+
+
+
+
+_defaultMaxContentSize
+
+int _defaultMaxContentSize
+
+
+
+
+
+_validMimeTypes
+
+java.util.Set<E> _validMimeTypes
+
+
+
+
+
+
+
+
+
+
+
+
+Serialized Fields
+
+
+
+_url
+
+java.lang.String _url
+
+
+
+
+
+_exception
+
+java.lang.Exception _exception
+
+
+
+
+
+
+
+
+
+
+
+
+Serialized Fields
+
+
+
+_httpStatus
+
+int _httpStatus
+
+
+
+
+
+_httpHeaders
+
+org.apache.tika.metadata.Metadata _httpHeaders
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Serialized Fields
+
+
+
+_redirectedUrl
+
+java.lang.String _redirectedUrl
+
+
+
+
+
+_reason
+
+RedirectFetchException.RedirectExceptionReason _reason
+
+
+
+
+
+
+
+
+
+
+
+
+Package crawlercommons.fetcher.file
+
+
+
+
+
+
+
+
+
+Package crawlercommons.fetcher.http
+
+
+
+
+
+
+
+
+
+Serialized Fields
+
+
+
+_maxThreads
+
+int _maxThreads
+
+
+
+
+
+_userAgent
+
+UserAgent _userAgent
+
+
+
+
+
+_maxRedirects
+
+int _maxRedirects
+
+
+
+
+
+_maxConnectionsPerHost
+
+int _maxConnectionsPerHost
+
+
+
+
+
+_minResponseRate
+
+int _minResponseRate
+
+
+
+
+
+_acceptLanguage
+
+java.lang.String _acceptLanguage
+
+
+
+
+
+_redirectMode
+
+BaseHttpFetcher.RedirectMode _redirectMode
+
+
+
+
+
+
+
+
+
+
+
+
+Serialized Fields
+
+
+
+_httpVersion
+
+org.apache.http.HttpVersion _httpVersion
+
+
+
+
+
+_socketTimeout
+
+int _socketTimeout
+
+
+
+
+
+_connectionTimeout
+
+int _connectionTimeout
+
+
+
+
+
+_maxRetryCount
+
+int _maxRetryCount
+
+
+
+
+
+
+
+
+
+
+
+
+Serialized Fields
+
+
+
+_agentName
+
+java.lang.String _agentName
+
+
+
+
+
+_emailAddress
+
+java.lang.String _emailAddress
+
+
+
+
+
+_webAddress
+
+java.lang.String _webAddress
+
+
+
+
+
+_browserVersion
+
+java.lang.String _browserVersion
+
+
+
+
+
+_crawlerVersion
+
+java.lang.String _crawlerVersion
+
+
+
+
+
+
+
+Package crawlercommons.robots
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Serialized Fields
+
+
+
+_numWarnings
+
+int _numWarnings
+
+
+
+
+
+
+
+Package crawlercommons.sitemaps
+
+
+
+
+
+
+
+
+
+Serialized Fields
+
+
+
+error
+
+java.lang.String error
+
+
+
+
+
+
+
java.lang.Throwable
+ crawlercommons.fetcher.BaseFetcher +