@VisibleForTesting static long timeForPartitionKey(PartitionKey key) { int year = (Integer) key.getField(FIELD_YEAR); int month = (Integer) key.getField(FIELD_MONTH) - 1; int day = (Integer) key.getField(FIELD_DAY); int hour = (Integer) key.getField(FIELD_HOUR); int minute = (Integer) key.getField(FIELD_MINUTE); Calendar calendar = Calendar.getInstance(); calendar.clear(); //noinspection MagicConstant calendar.set(year, month, day, hour, minute); return calendar.getTimeInMillis(); }
/** * Validates the partition key against the partitioning. */ private static void validatePartitionKey(PartitionKey key, Partitioning partitioning) { if (!partitioning.getFields().keySet().equals(key.getFields().keySet())) { throw new IllegalArgumentException(String.format( "Partition key is invalid: It contains fields %s, but the partitioning requires %s", key.getFields().keySet(), partitioning.getFields().keySet())); } for (Map.Entry<String, FieldType> entry : partitioning.getFields().entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); if (fieldValue == null) { throw new IllegalArgumentException( String.format("Incomplete partition key: value for field '%s' is missing", fieldName)); } try { fieldType.validate(fieldValue); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format( "Invalid partition key: Value for field '%s' is incompatible with the partitioning: %s", fieldName, e.getMessage())); } } }
@VisibleForTesting static long timeForPartitionKey(PartitionKey key) { int year = (Integer) key.getField(FIELD_YEAR); int month = (Integer) key.getField(FIELD_MONTH) - 1; int day = (Integer) key.getField(FIELD_DAY); int hour = (Integer) key.getField(FIELD_HOUR); int minute = (Integer) key.getField(FIELD_MINUTE); Calendar calendar = Calendar.getInstance(); calendar.clear(); //noinspection MagicConstant calendar.set(year, month, day, hour, minute); return calendar.getTimeInMillis(); }
/** * Validates the partition key against the partitioning. */ private static void validatePartitionKey(PartitionKey key, Partitioning partitioning) { if (!partitioning.getFields().keySet().equals(key.getFields().keySet())) { throw new IllegalArgumentException(String.format( "Partition key is invalid: It contains fields %s, but the partitioning requires %s", key.getFields().keySet(), partitioning.getFields().keySet())); } for (Map.Entry<String, FieldType> entry : partitioning.getFields().entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); if (fieldValue == null) { throw new IllegalArgumentException( String.format("Incomplete partition key: value for field '%s' is missing", fieldName)); } try { fieldType.validate(fieldValue); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format( "Invalid partition key: Value for field '%s' is incompatible with the partitioning: %s", fieldName, e.getMessage())); } } }
public static String getOutputPath(PartitionKey key, Partitioning partitioning) { validatePartitionKey(key, partitioning); StringBuilder builder = new StringBuilder(); String sep = ""; for (String fieldName : partitioning.getFields().keySet()) { builder.append(sep).append(key.getField(fieldName).toString()); sep = "/"; } return builder.toString(); }
public static String getOutputPath(PartitionKey key, Partitioning partitioning) { validatePartitionKey(key, partitioning); StringBuilder builder = new StringBuilder(); String sep = ""; for (String fieldName : partitioning.getFields().keySet()) { builder.append(sep).append(key.getField(fieldName).toString()); sep = "/"; } return builder.toString(); }
/** * Match this filter against a partition key. The key matches iff it matches all conditions. * * @throws java.lang.IllegalArgumentException if one of the field types in the partition key are incompatible */ public boolean match(PartitionKey partitionKey) { for (Map.Entry<String, Condition<? extends Comparable>> condition : getConditions().entrySet()) { Comparable value = partitionKey.getField(condition.getKey()); if (value == null || !condition.getValue().match(value)) { return false; } } return true; }
/** * Validates the partition key against the partitioning and gererates the row key for that partition key. */ @VisibleForTesting static byte[] generateRowKey(PartitionKey key, Partitioning partitioning) { validatePartitionKey(key, partitioning); // validate partition key, convert values, and compute size of output Map<String, FieldType> partitionFields = partitioning.getFields(); int totalSize = partitionFields.size() - 1; // one \0 between each of the fields ArrayList<byte[]> values = Lists.newArrayListWithCapacity(partitionFields.size()); for (Map.Entry<String, FieldType> entry : partitionFields.entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); byte[] bytes = FieldTypes.toBytes(fieldValue, fieldType); totalSize += bytes.length; values.add(bytes); } byte[] rowKey = new byte[totalSize]; int offset = 0; for (byte[] bytes : values) { System.arraycopy(bytes, 0, rowKey, offset, bytes.length); offset += bytes.length + 1; // this leaves a \0 byte after the value } return rowKey; }
/** * Validates the partition key against the partitioning and gererates the row key for that partition key. */ @VisibleForTesting static byte[] generateRowKey(PartitionKey key, Partitioning partitioning) { validatePartitionKey(key, partitioning); // validate partition key, convert values, and compute size of output Map<String, FieldType> partitionFields = partitioning.getFields(); int totalSize = partitionFields.size() - 1; // one \0 between each of the fields ArrayList<byte[]> values = Lists.newArrayListWithCapacity(partitionFields.size()); for (Map.Entry<String, FieldType> entry : partitionFields.entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); byte[] bytes = FieldTypes.toBytes(fieldValue, fieldType); totalSize += bytes.length; values.add(bytes); } byte[] rowKey = new byte[totalSize]; int offset = 0; for (byte[] bytes : values) { System.arraycopy(bytes, 0, rowKey, offset, bytes.length); offset += bytes.length + 1; // this leaves a \0 byte after the value } return rowKey; }
@Override public Return accept(PartitionDetail partitionDetail) { String sField = (String) partitionDetail.getPartitionKey().getField("s"); if (!allowedSField.equals(sField)) { return Return.SKIP; } int iField = (int) partitionDetail.getPartitionKey().getField("i"); if (stopOnI != null && stopOnI.equals(iField)) { return Return.STOP; } return Return.ACCEPT; } }
private static List<PartitionFilter> addUpperFilters(String[] fields, int position, PartitionKey keyUpper, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int upper = (Integer) keyUpper.getField(fieldName); // only add the filter if this condition is satisfiable. For example, not for hour<0 or month<1 if (isSatisfiableUpperBound(fieldName, upper)) { filters.add(supplier.get().addRangeCondition(fieldName, null, upper).build()); } return addUpperFilters(fields, position + 1, keyUpper, filters, nextSupplier(supplier, fieldName, upper)); }
private static List<PartitionFilter> addUpperFilters(String[] fields, int position, PartitionKey keyUpper, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int upper = (Integer) keyUpper.getField(fieldName); // only add the filter if this condition is satisfiable. For example, not for hour<0 or month<1 if (isSatisfiableUpperBound(fieldName, upper)) { filters.add(supplier.get().addRangeCondition(fieldName, null, upper).build()); } return addUpperFilters(fields, position + 1, keyUpper, filters, nextSupplier(supplier, fieldName, upper)); }
private static List<PartitionFilter> addLowerFilters(String[] fields, int position, PartitionKey keyLower, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int lower = (Integer) keyLower.getField(fieldName); // if this field is at the finest granularity (minutes), we must include its value in the range // otherwise we exclude it from the range and descend into the next finer granularity with a value // constraints on the current field name. For example: // - for hour:15/minute:10, we add a filter for hour>=16 (excluding 15) and descent for hour=15 // - now the remaining field is minute:10, we add a filter for minute>=10 (including 10) int lowerBound = position == fields.length - 1 ? lower : lower + 1; // only add the filter if this condition is satisfiable. For example, not for hour>=24 or month>=13 if (isSatisfiableLowerBound(fieldName, lowerBound)) { filters.add(supplier.get().addRangeCondition(fieldName, lowerBound, null).build()); } return addLowerFilters(fields, position + 1, keyLower, filters, nextSupplier(supplier, fieldName, lower)); }
private static List<PartitionFilter> addLowerFilters(String[] fields, int position, PartitionKey keyLower, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int lower = (Integer) keyLower.getField(fieldName); // if this field is at the finest granularity (minutes), we must include its value in the range // otherwise we exclude it from the range and descend into the next finer granularity with a value // constraints on the current field name. For example: // - for hour:15/minute:10, we add a filter for hour>=16 (excluding 15) and descent for hour=15 // - now the remaining field is minute:10, we add a filter for minute>=10 (including 10) int lowerBound = position == fields.length - 1 ? lower : lower + 1; // only add the filter if this condition is satisfiable. For example, not for hour>=24 or month>=13 if (isSatisfiableLowerBound(fieldName, lowerBound)) { filters.add(supplier.get().addRangeCondition(fieldName, lowerBound, null).build()); } return addLowerFilters(fields, position + 1, keyLower, filters, nextSupplier(supplier, fieldName, lower)); }
@Test public void testBuilderGetter() { PartitionKey key = PartitionKey.builder() .addField("a", "value") .addField("b", 1L) .addField("c", -17) .addField("d", true) .addIntField("e", 42) .addLongField("f", 15) .addStringField("g", "ghijk") .build(); Assert.assertEquals("value", key.getField("a")); Assert.assertEquals(1L, key.getField("b")); Assert.assertEquals(-17, key.getField("c")); Assert.assertEquals(true, key.getField("d")); Assert.assertEquals(42, key.getField("e")); Assert.assertEquals(15L, key.getField("f")); Assert.assertEquals("ghijk", key.getField("g")); }
actualIFields.add((Integer) consumedPartition.getPartitionKey().getField("i"));
actualIFields.add((Integer) consumedPartition.getPartitionKey().getField("i"));