/** * Generates the entity id for this imported line using the source from the import descriptor. * Called within the produce() method. * * @param fields One line of input text split on the column delimiter. * @param context The context used by the produce() method. * @return The EntityId for the data that gets imported by this line. */ protected EntityId getEntityId(List<String> fields, FijiTableContext context) { //TODO(FIJIMRLIB-3) Extend this to support composite row key ids String rowkey = fields.get(mFieldMap.get(getEntityIdSource())); return context.getEntityId(rowkey); }
/** * Reads a single news article, and writes its contents to a new fiji row, * indexed by the article's name (A string consisting of the parent folder, and * this article's hash), and the a priori categorization of this article. * * @param key The fully qualified path to the current file we're reading. * @param value The raw data to insert into this column. * @param context The context to write to. * @throws IOException if there is an error. */ @Override public void produce(Text key, Text value, FijiTableContext context) throws IOException { Path qualifiedPath = new Path(key.toString()); // Category is specified on the containing folder. String category = qualifiedPath.getParent().getName(); // Name is the concatenation of category and file name. String name = category + "." + qualifiedPath.getName(); // write name, category, and raw article. EntityId entity = context.getEntityId(name); context.put(entity, FAMILY, ARTICLE_NAME_QUALIFIER, name); context.put(entity, FAMILY, CATEGORY_QUALIFIER, category); context.put(entity, FAMILY, RAW_ARTICLE_QUALIFIER, value.toString()); } }
final EntityId eid = context.getEntityId(entityIdStr);
final EntityId user = context.getEntityId(firstName + "," + lastName);
/** {@inheritDoc} */ @Override public void produce(Text value, FijiTableContext context) throws IOException { Map<Field, String> fieldMap; try { fieldMap = CommonLogParser.get().parseCommonLog(value.toString()); } catch (ParseException pe) { reject(value, context, "Unable to parse row: " + value.toString()); return; } Field entityIdSource = Field.valueOf(getEntityIdSource()); EntityId eid = context.getEntityId(fieldMap.get(entityIdSource)); for (FijiColumnName fijiColumnName : getDestinationColumns()) { Field source = Field.valueOf(getSource(fijiColumnName)); String fieldValue = fieldMap.get(source); if (fieldValue != null) { // TODO(FIJIMRLIB-12) Add some ability to use timestamps derived from the log file. context.put(eid, fijiColumnName.getFamily(), fijiColumnName.getQualifier(), fieldValue); } else { reject(value, context, "Log file missing field: " + source); } } } }
return; final EntityId eid = context.getEntityId(entityIdSource); String source = getSource(fijiColumnName); String fieldValue = getFromPath(gson, source);
/** {@inheritDoc} */ @Override public void produce(ImmutableBytesWritable hbaseRowKey, Result hbaseRow, FijiTableContext context) throws IOException { EntityId entity = context.getEntityId(Bytes.toString(hbaseRowKey.get())); for (ColumnDescriptor columnDescriptor : mColumnDescriptors) { KeyValue keyValue = hbaseRow.getColumnLatest(columnDescriptor.getHBaseFamilyBytes(), columnDescriptor.getHBaseQualifierBytes()); if (null == keyValue) { // No data in this HTable column, skip it. continue; } // Convert the HBase cell to a Fiji cell. DecodedCell<?> fijiCell = decodeHBaseCell(columnDescriptor, keyValue.getValue()); // Write it at the same timestamp as the HBase cell. final String family = columnDescriptor.getFijiColumnName().getFamily(); final String qualifier = columnDescriptor.getFijiColumnName().getQualifier(); context.put(entity, family, qualifier, keyValue.getTimestamp(), fijiCell.getData()); } }