public ValueOnlyColumnBuilder() { setColumnFamily("P"); }
public ProfileHBaseMapper() { setRowKeyBuilder(new SaltyRowKeyBuilder()); setColumnBuilder(new ValueOnlyColumnBuilder()); }
/** * Creates the ColumnBuilder to use in accessing the profile data. * @param global The global configuration. */ private ColumnBuilder getColumnBuilder(Map<String, Object> global) { ColumnBuilder columnBuilder; String columnFamily = PROFILER_COLUMN_FAMILY.get(global, String.class); columnBuilder = new ValueOnlyColumnBuilder(columnFamily); return columnBuilder; }
/** * Writes a set of measurements to HBase. * * @param iterator The measurements to write. * @return The number of measurements written to HBase. */ @Override public Iterator<Integer> call(Iterator<ProfileMeasurementAdapter> iterator) throws Exception { int count = 0; LOG.debug("About to write profile measurement(s) to HBase"); // do not open hbase connection, if nothing to write List<ProfileMeasurementAdapter> measurements = IteratorUtils.toList(iterator); if(measurements.size() > 0) { // open an HBase connection Configuration config = HBaseConfiguration.create(); try (HBaseClient client = new HBaseClient(tableProvider, config, tableName)) { for (ProfileMeasurementAdapter adapter : measurements) { ProfileMeasurement m = adapter.toProfileMeasurement(); client.addMutation(rowKeyBuilder.rowKey(m), columnBuilder.columns(m), durability); } count = client.mutate(); } catch (IOException e) { LOG.error("Unable to open connection to HBase", e); throw new RuntimeException(e); } } LOG.debug("{} profile measurement(s) written to HBase", count); return IteratorUtils.singletonIterator(count); }
/** * Build the row key. * @param profile The name of the profile. * @param entity The name of the entity. * @param period The measure period * @param groups The groups. * @return The HBase row key. */ public byte[] rowKey(String profile, String entity, long period, List<Object> groups) { // row key = salt + prefix + group(s) + time byte[] salt = getSalt(period, saltDivisor); byte[] prefixKey = prefixKey(profile, entity); byte[] groupKey = groupKey(groups); byte[] timeKey = timeKey(period); int capacity = salt.length + prefixKey.length + groupKey.length + timeKey.length; return ByteBuffer .allocate(capacity) .put(salt) .put(prefixKey) .put(groupKey) .put(timeKey) .array(); }
/** * Creates the ColumnBuilder to use in accessing the profile data. * @param global The global configuration. */ private RowKeyBuilder getRowKeyBuilder(Map<String, Object> global) { Integer saltDivisor = PROFILER_SALT_DIVISOR.get(global, Integer.class); return new SaltyRowKeyBuilder(saltDivisor, getPeriodDurationInMillis(global), TimeUnit.MILLISECONDS); }
/** * Defines the HBase row key that will be used when writing the data from a * tuple to HBase. * * @param tuple The tuple to map to HBase. */ @Override public byte[] rowKey(Tuple tuple) { ProfileMeasurement measurement = (ProfileMeasurement) tuple.getValueByField("measurement"); return rowKeyBuilder.rowKey(measurement); }
/** * Defines the columnar structure that will be used when writing the data * from a tuple to HBase. * * @param tuple The tuple to map to HBase. */ @Override public ColumnList columns(Tuple tuple) { ProfileMeasurement measurement = (ProfileMeasurement) tuple.getValueByField("measurement"); return columnBuilder.columns(measurement); }
/** * Build the row key. * @param profile The name of the profile. * @param entity The name of the entity. * @param period The period in which the measurement was taken. * @param groups The groups. * @return The HBase row key. */ public byte[] rowKey(String profile, String entity, ProfilePeriod period, List<Object> groups) { return rowKey(profile, entity, period.getPeriod(), groups); }
/** * Builds the 'time' portion of the row key * @param period The ProfilePeriod in which the ProfileMeasurement was taken. */ private static byte[] timeKey(ProfilePeriod period) { return timeKey(period.getPeriod()); }
/** * Calculates a salt value that is used as part of the row key. * * The salt is calculated as 'md5(period) % N' where N is a configurable value that ideally * is close to the number of nodes in the Hbase cluster. * * @param period The period in which a profile measurement is taken. */ public static byte[] getSalt(ProfilePeriod period, int saltDivisor) { return getSalt(period.getPeriod(), saltDivisor); }
@Override public ColumnList columns(ProfileMeasurement measurement) { ColumnList cols = new ColumnList(); cols.addColumn(columnFamilyBytes, getColumnQualifier("value"), SerDeUtils.toBytes(measurement.getProfileValue())); return cols; }
public HBaseWriterFunction(Properties properties) { tableName = HBASE_TABLE_NAME.get(properties, String.class); durability = HBASE_WRITE_DURABILITY.get(properties, Durability.class); // row key builder int saltDivisor = HBASE_SALT_DIVISOR.get(properties, Integer.class); int periodDuration = PERIOD_DURATION.get(properties, Integer.class); TimeUnit periodDurationUnits = TimeUnit.valueOf(PERIOD_DURATION_UNITS.get(properties, String.class)); rowKeyBuilder = new SaltyRowKeyBuilder(saltDivisor, periodDuration, periodDurationUnits); // column builder String columnFamily = HBASE_COLUMN_FAMILY.get(properties, String.class); columnBuilder = new ValueOnlyColumnBuilder(columnFamily); // hbase table provider String providerImpl = HBASE_TABLE_PROVIDER.get(properties, String.class); tableProvider = createTableProvider(providerImpl); }
/** * Creates the ColumnBuilder to use in accessing the profile data. * @param global The global configuration. */ private ColumnBuilder getColumnBuilder(Map<String, Object> global) { String columnFamily = PROFILER_COLUMN_FAMILY.get(global, String.class); return new ValueOnlyColumnBuilder(columnFamily); }
/** * Creates the ColumnBuilder to use in accessing the profile data. * @param global The global configuration. */ private RowKeyBuilder getRowKeyBuilder(Map<String, Object> global) { // how long is the profile period? long duration = PROFILER_PERIOD.get(global, Long.class); LOG.debug("profiler client: {}={}", PROFILER_PERIOD, duration); // which units are used to define the profile period? String configuredUnits = PROFILER_PERIOD_UNITS.get(global, String.class); TimeUnit units = TimeUnit.valueOf(configuredUnits); LOG.debug("profiler client: {}={}", PROFILER_PERIOD_UNITS, units); // what is the salt divisor? Integer saltDivisor = PROFILER_SALT_DIVISOR.get(global, Integer.class); LOG.debug("profiler client: {}={}", PROFILER_SALT_DIVISOR, saltDivisor); return new SaltyRowKeyBuilder(saltDivisor, duration, units); }
/** * Builds a list of row keys necessary to retrieve a profile's measurements over * a time horizon. * <p> * This method is useful when attempting to read ProfileMeasurements stored in HBase. * * @param profile The name of the profile. * @param entity The name of the entity. * @param groups The group(s) used to sort the profile data. * @param periods The profile measurement periods to compute the rowkeys for * @return All of the row keys necessary to retrieve the profile measurements. */ @Override public List<byte[]> rowKeys(String profile, String entity, List<Object> groups, Iterable<ProfilePeriod> periods) { List<byte[]> ret = new ArrayList<>(); for(ProfilePeriod period : periods) { ret.add(rowKey(profile, entity, period, groups)); } return ret; }
public ValueOnlyColumnBuilder(String columnFamily) { setColumnFamily(columnFamily); }
/** * Calculates a salt value that is used as part of the row key. * * The salt is calculated as 'md5(period) % N' where N is a configurable value that ideally * is close to the number of nodes in the Hbase cluster. * * @param period The period * @param saltDivisor The salt divisor */ public static byte[] getSalt(long period, int saltDivisor) { try { // an MD5 is 16 bytes aka 128 bits MessageDigest digest = MessageDigest.getInstance("MD5"); byte[] hash = digest.digest(timeKey(period)); int salt = Bytes.toShort(hash) % saltDivisor; return Bytes.toBytes(salt); } catch(NoSuchAlgorithmException e) { throw new RuntimeException(e); } }
/** * Builds a list of row keys necessary to retrieve profile measurements over * a time horizon. * * @param profile The name of the profile. * @param entity The name of the entity. * @param groups The group(s) used to sort the profile data. * @param start When the time horizon starts in epoch milliseconds. * @param end When the time horizon ends in epoch milliseconds. * @return All of the row keys necessary to retrieve the profile measurements. */ @Override public List<byte[]> rowKeys(String profile, String entity, List<Object> groups, long start, long end) { // be forgiving of out-of-order start and end times; order is critical to this algorithm long max = Math.max(start, end); start = Math.min(start, end); end = max; // find the starting period and advance until the end time is reached return ProfilePeriod.visitPeriods( start , end , periodDurationMillis , TimeUnit.MILLISECONDS , Optional.empty() , period -> rowKey(profile, entity, period, groups) ); }
/** * Builds the row key for a given profile measurement. * @param m The profile measurement. * @return The HBase row key. */ @Override public byte[] rowKey(ProfileMeasurement m) { return rowKey(m.getProfileName(), m.getEntity(), m.getPeriod(), m.getGroups()); }