@Override
public Document toDocument(Stage stage, ICrawlData crawlData) {
Document doc = new Document();
String ref = StringUtil.truncateWithHash(
crawlData.getReference(), 1024, "!");
doc.put(FIELD_REFERENCE, ref);
if (!Objects.equals(ref, crawlData.getReference())) {
doc.put(FIELD_REFERENCE_EXCESSIVE, crawlData.getReference());
}
doc.put(FIELD_PARENT_ROOT_REFERENCE,
crawlData.getParentRootReference());
doc.put(FIELD_IS_ROOT_PARENT_REFERENCE,
crawlData.isRootParentReference());
doc.put(FIELD_CRAWL_STATE, crawlData.getState().toString());
doc.put(FIELD_META_CHECKSUM, crawlData.getMetaChecksum());
doc.put(FIELD_CONTENT_CHECKSUM, crawlData.getContentChecksum());
doc.put(FIELD_IS_VALID, crawlData.getState().isGoodState());
doc.put(FIELD_STAGE, stage.toString());
if (crawlData.getContentType() != null) {
doc.put(FIELD_CONTENT_TYPE, crawlData.getContentType().toString());
}
doc.put(FIELD_CRAWL_DATE, crawlData.getCrawlDate());
return doc;
}