@Override public RawRecordMetadata createMetadata(Text shardTableName, Text metadataTableName, Text loadDatesTableName, Text shardIndexTableName, Text shardReverseIndexTableName, boolean frequency) { return new EventMetadata(shardTableName, metadataTableName, loadDatesTableName, shardIndexTableName, shardReverseIndexTableName, frequency); } }
@Override public void addEvent(IngestHelperInterface helper, RawRecordContainer event, Multimap<String,NormalizedContentInterface> fields, long loadTimeInMillis) { addEvent(helper, event, fields, this.frequency, INCLUDE_LOAD_DATES, loadTimeInMillis); }
@Override public Multimap<BulkIngestKey,Value> getBulkMetadata() { Multimap<BulkIngestKey,Value> bulkData = HashMultimap.create(); addIndexedFieldToMetadata(bulkData, eventFieldsInfo); addIndexedFieldToMetadata(bulkData, termFrequencyFieldsInfo); addIndexedFieldToMetadata(bulkData, indexedFieldsInfo); addIndexedFieldToMetadata(bulkData, reverseIndexedFieldsInfo); addFrequenciesToMetadata(bulkData); addIndexedFieldToMetadata(bulkData, dataTypeFieldsInfo); addIndexedFieldToMetadata(bulkData, normalizedFieldsInfo); addIndexedFieldToMetadata(bulkData, this.compositeFieldsInfo); addIndexedFieldToMetadata(bulkData, this.compositeSeparators); addToLoadDates(bulkData, this.indexedFieldsLoadDateCounts); addToLoadDates(bulkData, this.reverseIndexedFieldsLoadDateCounts); return bulkData; }
long countDelta = getCountDelta(helper); addEventField(helper, event, fieldName, fieldDelta, frequency); updateForIndexedField(helper, event, fields, countDelta, loadDateStr, NO_TOKEN_DESIGNATOR, fieldName); updateForReverseIndexedField(helper, event, fields, countDelta, loadDateStr, NO_TOKEN_DESIGNATOR, fieldName); shouldWriteDataType = true; log.debug(fieldName + " is normalized"); updateMetadata(this.normalizedFieldsInfo, helper, event, fields, fieldName); update(helper.getDataTypes(fieldName), event, fields.get(fieldName), "", 0, null, this.dataTypeFieldsInfo, null); } else { log.debug(fieldName + " apparently has no data type"); addTokenizedContent(helper, event, fields, countDelta, loadDateStr);
protected void addToLoadDates(Multimap<BulkIngestKey,Value> results, MetadataCounterGroup countsGroup) { if (loadDatesTableName != null) { for (MetadataCounterGroup.CountAndKeyComponents entry : countsGroup.getEntries()) { Long count = entry.getCount(); Key k = new Key(new Text(entry.getRowId()), countsGroup.getColumnFamily(), new Text(entry.getDate() + DELIMITER + entry.getDataType())); addToResults(results, count, k, loadDatesTableName); } } }
protected void addEventField(IngestHelperInterface helper, RawRecordContainer event, String fieldName, long countDelta, boolean frequency) { // if only indexing this field, then do not add to event and frequency maps if (helper.isIndexOnlyField(fieldName)) { log.debug(fieldName + " is indexonly, not adding to event"); return; } if (helper.isCompositeField(fieldName) && !helper.isOverloadedCompositeField(fieldName)) { log.debug(fieldName + " is a composite, not adding to event"); return; } log.debug("createOrUpdate for " + fieldName); eventFieldsInfo.createOrUpdate(fieldName, event.getDataType().outputName(), MetadataWithMostRecentDate.IGNORED_NORMALIZER_CLASS, event.getDate()); if (frequency) { addToFrequencyCounts(event, fieldName, countDelta); } }
protected void addFrequenciesToMetadata(Multimap<BulkIngestKey,Value> results) { if (frequency) { for (MetadataCounterGroup.CountAndKeyComponents entry : frequencyCounts.getEntries()) { Long count = entry.getCount(); Key k = new Key(new Text(entry.getRowId()), frequencyCounts.getColumnFamily(), new Text(entry.getDataType() + DELIMITER + entry.getDate()), DateHelper.parse(entry.getDate()).getTime()); addToResults(results, count, k, this.metadataTableName); } } }
@Override public RawRecordMetadata createMetadata(Text shardTableName, Text metadataTableName, Text loadDatesTableName, Text shardIndexTableName, Text shardReverseIndexTableName, boolean frequency) { return new EventMetadata(shardTableName, metadataTableName, loadDatesTableName, shardIndexTableName, shardReverseIndexTableName, frequency); } }
@Override public void addEvent(IngestHelperInterface helper, RawRecordContainer event, Multimap<String,NormalizedContentInterface> fields) { addEvent(helper, event, fields, this.frequency, INCLUDE_LOAD_DATES, System.currentTimeMillis()); }
@Test public void testCreatesReverseIndexedField() throws IOException { setupMocks(); helper.addReverseIndexedField(FIELD_NAME_TO_COUNT); RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); assertFieldNameCountEquals(1L, RINDEX_TABLE_NAME, FIELD_NAME_TO_COUNT, eventMetadata); Assert.assertFalse(assertContainsKey(eventMetadata, INDEX_TABLE_NAME, FIELD_NAME_TO_COUNT)); assertNonIndexedFieldNameIsMissing(eventMetadata); EasyMock.verify(event); }
@Override public void addEventWithoutLoadDates(IngestHelperInterface helper, RawRecordContainer event, Multimap<String,NormalizedContentInterface> fields) { addEvent(helper, event, fields, this.frequency, EXCLUDE_LOAD_DATES, System.currentTimeMillis()); }
@Test public void testCountsTwo() throws IOException { setupMocks(); helper.addIndexedField(FIELD_NAME_TO_COUNT); RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); assertFieldNameCountEquals(2L, INDEX_TABLE_NAME, FIELD_NAME_TO_COUNT, eventMetadata); EasyMock.verify(event); }
@Test public void testCreatesBoth() throws IOException { setupMocks(); helper.addReverseIndexedField(FIELD_NAME_TO_COUNT); helper.addIndexedField(FIELD_NAME_TO_COUNT); RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); assertFieldNameCountEquals(1L, INDEX_TABLE_NAME, FIELD_NAME_TO_COUNT, eventMetadata); assertFieldNameCountEquals(1L, RINDEX_TABLE_NAME, FIELD_NAME_TO_COUNT, eventMetadata); assertNonIndexedFieldNameIsMissing(eventMetadata); EasyMock.verify(event); }
@Test public void testCreatesIndexedField() throws IOException { setupMocks(); helper.addIndexedField(FIELD_NAME_TO_COUNT); RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); assertFieldNameCountEquals(1L, INDEX_TABLE_NAME, FIELD_NAME_TO_COUNT, eventMetadata); Assert.assertFalse(assertContainsKey(eventMetadata, RINDEX_TABLE_NAME, FIELD_NAME_TO_COUNT)); assertNonIndexedFieldNameIsMissing(eventMetadata); EasyMock.verify(event); }