/** * Sets the column to retrieve values from. * * @param colName the column to use. * @return this builder instance. */ public Builder withColumn(FijiColumnName colName) { if (!colName.isFullyQualified()) { throw new IllegalArgumentException("Must specify a fully-qualified column, not a map."); } mColumn = colName; return this; }
/** {@inheritDoc} */ @Override public int compareTo(FijiColumnName otherObj) { final int comparison = this.mFamily.compareTo(otherObj.mFamily); if (0 == comparison) { return (this.isFullyQualified() ? mQualifier : "") .compareTo(otherObj.isFullyQualified() ? otherObj.getQualifier() : ""); } else { return comparison; } } }
/** {@inheritDoc} */ @Override public void setConf(Configuration conf) { super.setConf(conf); HadoopConfigurator.configure(this); // Validate that they are either both families or both columns. if (mInputColumn.isFullyQualified() != mOutputColumn.isFullyQualified()) { throw new RuntimeException( "Input and output must both be a specific column, or both be a family"); } }
/** * Initializes a new FijiCell. * * @param column Name of the column in which this cell exists. * @param timestamp Timestamp of this cell in milliseconds since the epoch. * @param decodedCell Decoded cell content. */ private FijiCell( final FijiColumnName column, final long timestamp, final DecodedCell<T> decodedCell ) { Preconditions.checkArgument(column.isFullyQualified(), "Cannot create a FijiCell without a fully qualified column. Found family: %s", column.getName()); mColumn = column; mTimestamp = timestamp; mDecodedCell = decodedCell; }
/** * Initialize the family and qualifier instance variables. */ private void initializeInputColumn() { mInputColumn = new FijiColumnName(getInputColumn()); if (!mInputColumn.isFullyQualified()) { throw new RuntimeException("getInputColumn() must contain a colon (':')"); } }
/** * Given a FijiColumnName, return a sensible Hive column name. For a fully qualified column, * return the qualifier. Otherwise if it's a map type family, return the family name. * * @param fijiColumnName to generate the Hive column name for. * @return default Hive column name corresponding to the Fiji column. */ protected static String getDefaultHiveColumnName(FijiColumnName fijiColumnName) { if (fijiColumnName.isFullyQualified()) { return fijiColumnName.getQualifier(); } return fijiColumnName.getFamily(); }
/** * Constructs a row filter that excludes rows that have no data in <code>columnName</code>. * * @param columnName The column family:qualifier of interest. */ public HasColumnDataRowFilter(String columnName) { if (null == columnName) { throw new IllegalArgumentException("columnName is required"); } FijiColumnName fijiColName = FijiColumnName.create(columnName); if (!fijiColName.isFullyQualified()) { throw new IllegalArgumentException("Cannot use an unqualified column family."); } mFamily = fijiColName.getFamily(); mQualifier = fijiColName.getQualifier(); }
/** * Sets the family. * * @param family The family. */ @HadoopConf(key=CONF_EXPORT_FAMILY) protected void setFamily(String family) { FijiColumnName name = new FijiColumnName(family); if (name.isFullyQualified()) { throw new RuntimeException("Expected an unqualified map type family. " + "Requested family was: " + name.getName()); } mFamily = family; }
/** * Parses an input string into a ColumnDescriptor. * * <p>The input string must be formatted as:</p> * * <code>hbase-family:hbase-qualifier:hbase-type:fiji-family:fiji-qualifier</code> * * @param input The input string. * @return A parsed ColumnDescriptor. * @throws IOException If the input string cannot be parsed. */ public static ColumnDescriptor parse(String input) throws IOException { String[] parts = StringUtils.splitByWholeSeparatorPreserveAllTokens(input, ":", 4); if (parts.length < 4) { throw new IOException("Expected at least 4 colon-separated fields, but got " + parts.length + " while parsing " + input); } FijiColumnName fijiColumnName = new FijiColumnName(parts[3]); if (!fijiColumnName.isFullyQualified()) { throw new IOException("Output fiji columns must have a family and qualifier."); } return new ColumnDescriptor(parts[0], parts[1], parts[2], fijiColumnName); } }
if (column.isFullyQualified()) {
/** * Reports whether a column exists. * * @param column Column name. * @return whether the specified column exists. */ public boolean exists(FijiColumnName column) { final FamilyLayout fLayout = mFamilyMap.get(column.getFamily()); if (fLayout == null) { // Family does not exist: return false; } if (fLayout.isMapType()) { // This is a map-type family, we don't need to validate the qualifier: return true; } // This is a group-type family: if (!column.isFullyQualified()) { // No column qualifier, the group-type family exists: return true; } // Validate the qualifier: return fLayout.getColumnMap().containsKey(column.getQualifier()); }
/** {@inheritDoc} */ @Override protected void setup() throws IOException { Preconditions.checkArgument((mColumnURIFlag != null) && !mColumnURIFlag.isEmpty(), "Specify a target table to write synthesized data to with " + "--table=fiji://hbase-address/fiji-instance/table"); mColumnURI = FijiURI.newBuilder(mColumnURIFlag).build(); Preconditions.checkArgument(mColumnURI.getTable() != null, "No table specified in target URI '{}'. " + "Specify a target table to write synthesized data to with " + "--table=fiji://hbase-address/fiji-instance/table", mColumnURI); Preconditions.checkArgument(mColumnURI.getColumns().size() == 1, "Invalid target column '{}', specify exactly one column in URI with " + "--target=fiji://hbase-address/fiji-instance/table/family:qualifier", mColumnURI); Preconditions.checkArgument(mColumnURI.getColumns().get(0).isFullyQualified(), "Missing column qualifier in '{}', specify exactly one column in URI with " + "--target=fiji://hbase-address/fiji-instance/table/family:qualifier", mColumnURI); mFiji = Fiji.Factory.open(mColumnURI, getConf()); mTable = mFiji.openTable(mColumnURI.getTable()); }
/** * Reports the request the applies for the specified column, or null if no such request exists. * * <p> * If the column belongs to a map-type family and there is a request for the entire family, * the request for the entire family is reported. * </p> * * @param columnName Requested column or family. * @return the request that applies for the specified column or family, * or null if no such request exists. */ public Column getRequestForColumn(final FijiColumnName columnName) { final Column column = getColumn(columnName); if (column != null) { return column; } if (columnName.isFullyQualified()) { return getColumn(columnName.getFamily(), null); } return null; }
/** * Get the list of {@code KeyValue}s in a {@code Result} belonging to a column request. * * <p> * This method will filter extra version from the result if necessary. * </p> * * @param columnRequest of the column whose {@code KeyValues} to return. * @param translator for the table. * @param result the scan results. * @return the {@code KeyValue}s for the column. */ private static List<KeyValue> getColumnKeyValues( final Column columnRequest, final HBaseColumnNameTranslator translator, final Result result ) { if (result.isEmpty()) { return ImmutableList.of(); } final FijiColumnName column = columnRequest.getColumnName(); final List<KeyValue> keyValues = Arrays.asList(result.raw()); if (column.isFullyQualified()) { return getQualifiedColumnKeyValues(columnRequest, translator, keyValues); } else { return getFamilyKeyValues(columnRequest, translator, keyValues); } }
/** * Builds the Hive type from the schema for a particular cell within Hive. * * @param fijiColumnName that specifices which column to build the Hive type from. * @param fijiTableLayout where the column resides. * @param schemaTable of the Fiji instance for type mappings * @return String representing the corresponding Hive type for a cell within the table's layout. * @throws IOException if there is an issue retrieving the schema of a particular column. */ protected static String getHiveType(FijiColumnName fijiColumnName, FijiTableLayout fijiTableLayout, FijiSchemaTable schemaTable) throws IOException { CellSchema cellSchema = fijiTableLayout.getCellSchema(fijiColumnName); Schema schema = getSchemaFromCellSchema(cellSchema, schemaTable); String hiveType = ""; if (null != schema) { hiveType = convertSchemaToHiveType(schema); if (!fijiColumnName.isFullyQualified()) { hiveType = "MAP<STRING, " + hiveType + ">"; } } else { // Null schemas are probably indicative of a counter, but they are currently unsupported // within the Fiji Hive Adapter. Log a warning, and let the user fix it. LOG.warn(fijiColumnName.toString() + " has a null schema and is unsupported within Hive."); } return hiveType; }
/** {@inheritDoc} */ @Override public void storeToConf(KeyValueStoreConfiguration conf) throws IOException { if (null == mTableUri) { throw new IOException("Required attribute not set: table URI"); } if (null == mColumn) { throw new IOException("Required attribute not set: column"); } if (!mColumn.isFullyQualified()) { throw new IOException("Column must be fully qualified"); } conf.set(CONF_TABLE_URI, mTableUri.toString()); conf.set(CONF_COLUMN, mColumn.toString()); conf.setLong(CONF_MIN_TS, mMinTs); conf.setLong(CONF_MAX_TS, mMaxTs); conf.setInt(CONF_CACHE_SIZE, mMaxObjectsToCache); if (null != mReaderSchema) { conf.set(CONF_READER_SCHEMA, mReaderSchema.toString()); } }
/** * Initializes the list of associated column family FijiPagers for this FijiRowData. * * @param fijiDataRequest the data request for this FijiRowData. * @return map of families to their associated qualifier pagers. */ private Map<String, FijiPager> getFijiQualifierPagers(FijiDataRequest fijiDataRequest) { Map<String, FijiPager> fijiQualifierPagers = Maps.newHashMap(); for (FijiDataRequest.Column column : fijiDataRequest.getColumns()) { if (column.isPagingEnabled() && !column.getColumnName().isFullyQualified()) { // Only include pagers for column families. try { LOG.info("Paging enabled for column family: {}", column.getColumnName()); FijiPager fijiPager = mRowData.getPager(column.getFamily()); fijiQualifierPagers.put(column.getFamily(), fijiPager); } catch (FijiColumnPagingNotEnabledException e) { LOG.warn("Paging not enabled for column family: {}", column.getColumnName()); } } } return fijiQualifierPagers; }
/** * Initializes the list of associated fully qualified cell FijiPagers for this FijiRowData. Any * non fully qualified cell paging configuration will be ignored. * * @param fijiDataRequest the data request for this FijiRowData. * @param fijiRowData the fijiRowData to generate the pagers from. * @return map of FijiColumnNames to their associated cell pagers. */ private static Map<FijiColumnName, FijiPager> getFijiCellPagers(FijiDataRequest fijiDataRequest, FijiRowData fijiRowData) { Map<FijiColumnName, FijiPager> fijiCellPagers = Maps.newHashMap(); for (FijiDataRequest.Column column : fijiDataRequest.getColumns()) { if (column.isPagingEnabled() && column.getColumnName().isFullyQualified()) { // Only include pagers for fully qualified cells try { LOG.info("Paging enabled for column: {}", column.getColumnName()); FijiPager fijiPager = fijiRowData.getPager(column.getFamily(), column.getQualifier()); fijiCellPagers.put(column.getColumnName(), fijiPager); } catch (FijiColumnPagingNotEnabledException e) { LOG.warn("Paging not enabled for column: {}", column.getColumnName()); } } } return fijiCellPagers; }
/** {@inheritDoc} */ @Override public void produce(FijiRowData input, ProducerContext context) throws IOException { if (!mInputColumn.isFullyQualified()) { // Copy the entire family. for (String qualifier : input.getQualifiers(mInputColumn.getFamily())) { FijiColumnName sourceColumn = new FijiColumnName(mInputColumn.getFamily(), qualifier); produceAllVersions(input, context, sourceColumn); } } else { // Copy just a specific column. produceAllVersions(input, context, mInputColumn); } }
/** * Produces all data from a given column name into the output column. * * @param input The input row. * @param context The producer context used to write. * @param columnName The column to read from. * @throws IOException If there is an IO error. */ private void produceAllVersions( FijiRowData input, ProducerContext context, FijiColumnName columnName) throws IOException { for (long timestamp : input.getTimestamps(columnName.getFamily(), columnName.getQualifier())) { // Read the data from the input column. Object data = input.getValue( mInputColumn.getFamily(), columnName.getQualifier(), timestamp); // Write the data to the output column. if (!mOutputColumn.isFullyQualified()) { context.put(columnName.getQualifier(), timestamp, data); } else { context.put(timestamp, data); } } } }