class HTTPContentDigest extends Processor {

    @SuppressWarnings("unused")
    private static final long serialVersionUID = 3L;

    private static Logger logger =
        Logger.getLogger(HTTPContentDigest.class.getName());

    
    {
        setStripRegex("");
    }
    public String getStripRegex() {
        return (String) kp.get("stripRegex");
    }
    /**
     * A regular expression that matches those portions of downloaded documents
     * that need to be ignored when calculating the content digest. Segments
     * matching this expression will be rewritten with the blank character for
     * the content digest.
     */
    public void setStripRegex(String regex) {
        kp.put("stripRegex",regex);
    }

    {
        setMaxSizeToDigest(1*1024*1024L); // 1MB
    }
    public long getMaxSizeToDigest() {
        return (Long) kp.get("maxSizeToDigest");
    }
    /** Maximum file size for - longer files will be ignored. -1 = unlimited*/
    public void setMaxSizeToDigest(long threshold) {
        kp.put("maxSizeToDigest",threshold);
    }
    
    private static final String SHA1 = "SHA1";

    
    /**
     * Constructor.
     */
    public HTTPContentDigest() {
    }

    
    protected boolean shouldProcess(CrawlURI uri) {
        if (!uri.getContentType().startsWith("text")) {
            return false;
        }
        
        long maxSize = getMaxSizeToDigest();
        if ((maxSize > - 1) && (maxSize < uri.getContentSize())) {
            return false;
        }
        
        return true;
    }

    protected void innerProcess(CrawlURI curi) throws InterruptedException {<FILL_FUNCTION_BODY>}
}

        // Ok, if we got this far we need to calculate the content digest. 
        // Get the regex
        String regex = getStripRegex();
        
        // Get a replay of the document character seq.
        ReplayCharSequence cs = null;
        try {
           cs = curi.getRecorder().getContentReplayCharSequence();
           // Create a MessageDigest 
           MessageDigest digest = null;
           try {
               digest = MessageDigest.getInstance(SHA1);
           } catch (NoSuchAlgorithmException e1) {
               e1.printStackTrace();
               return;
           }

           digest.reset();

           String s = null;

           if (StringUtils.isEmpty(regex)) {
               s = cs.toString();
           } else {
               // Process the document
               Matcher m = TextUtils.getMatcher(regex, cs);
               s = m.replaceAll(" ");
               TextUtils.recycleMatcher(m);
           }
           digest.update(s.getBytes());
           // Get the new digest value
           byte[] newDigestValue = digest.digest();
           // Save new digest value
           curi.setContentDigest(SHA1, newDigestValue);
           
        } catch (Exception e) {
            curi.getNonFatalFailures().add(e);
            logger.warning("Failed get of replay char sequence " +
                curi.toString() + " " + e.getMessage() + " " +
                Thread.currentThread().getName());
            return; // Can't proceed if this happens.
        }
    
<methods>public non-sealed void <init>() ,public void doCheckpoint(org.archive.checkpointing.Checkpoint) throws java.io.IOException,public void finishCheckpoint(org.archive.checkpointing.Checkpoint) ,public static java.lang.String flattenVia(org.archive.modules.CrawlURI) ,public java.lang.String getBeanName() ,public boolean getEnabled() ,public org.archive.spring.KeyedProperties getKeyedProperties() ,public static long getRecordedSize(org.archive.modules.CrawlURI) ,public org.archive.modules.deciderules.DecideRule getShouldProcessRule() ,public long getURICount() ,public static boolean hasHttpAuthenticationCredential(org.archive.modules.CrawlURI) ,public boolean isRunning() ,public static boolean isSuccess(org.archive.modules.CrawlURI) ,public org.archive.modules.ProcessResult process(org.archive.modules.CrawlURI) throws java.lang.InterruptedException,public java.lang.String report() ,public void setBeanName(java.lang.String) ,public void setEnabled(boolean) ,public void setRecoveryCheckpoint(org.archive.checkpointing.Checkpoint) ,public void setShouldProcessRule(org.archive.modules.deciderules.DecideRule) ,public void start() ,public void startCheckpoint(org.archive.checkpointing.Checkpoint) ,public void stop() <variables>protected java.lang.String beanName,protected boolean isRunning,protected org.archive.spring.KeyedProperties kp,protected org.archive.checkpointing.Checkpoint recoveryCheckpoint,protected java.util.concurrent.atomic.AtomicLong uriCount