@Override public PartitionFilter.Builder get() { return PartitionFilter.builder(); } };
/** * Logs a warning if the partition filter contains a field that is not part of the partitioning. */ private void warnIfInvalidPartitionFilter(PartitionFilter filter, Partitioning partitioning) { if (null == filter) { return; } for (Map.Entry<String, PartitionFilter.Condition<? extends Comparable>> entry : filter.getConditions().entrySet()) { if (!partitioning.getFields().containsKey(entry.getKey())) { LOG.warn("Partition filter cannot match any partitions in dataset '{}' because it contains field '{}' " + "that is not a valid partitioning field", getName(), entry.getKey()); } } }
String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); PartitionFilter.Condition<? extends Comparable> condition = filter.getCondition(fieldName); if (condition == null) { break; // this field is not present; we can't include any more fields in the start key
PartitionFilter filter = PartitionFilter.builder() .addValueCondition("year", 2012) .addRangeCondition("month", 4, 7) .build(); Assert.assertEquals(4, filter.getConditions().size()); validateCondition(filter, "year", true, Partitioning.FieldType.INT, 2012, null, 2011, 2013); validateCondition(filter, "month", false, Partitioning.FieldType.INT, 4, 5, 6, null, 3, 7, 8); Assert.assertTrue(filter.match(PartitionKey .builder() .addField("month", 4) Assert.assertFalse(filter.match(PartitionKey .builder() .addField("month", 7) Assert.assertFalse(filter.match(PartitionKey .builder() .addField("month", 4) Assert.assertTrue(filter.match(PartitionKey .builder() .addField("day", "tue")
@Test public void testBuilderNullRange() { PartitionFilter filter = PartitionFilter.builder() .addValueCondition("a", 1) .<Long>addRangeCondition("x", null, null) .build(); Assert.assertEquals(1, filter.getConditions().size()); // only the one for "a" Assert.assertNull(filter.getCondition("x")); }
@Test(expected = IllegalArgumentException.class) public void testIncompatibleMatch() { PartitionFilter filter = PartitionFilter.builder() .addValueCondition("year", 2012) .addRangeCondition("month", 4, 7) .addValueCondition("market", "asia") .build(); // field of incompatible type filter.match(PartitionKey .builder() .addField("month", "january") .addField("market", "latin") .addField("year", 2012) .build()); }
continue; if (filter != null && !filter.match(key)) { continue;
/** * Create the PartitionFilter. * * @throws java.lang.IllegalStateException if no fields have been added */ public PartitionFilter build() { if (map.isEmpty()) { throw new IllegalStateException("Partition filter cannot be empty."); } return new PartitionFilter(map); }
@Test public void testPartitionsForTimeRange() throws Exception { for (Object[] test : rangeTests) { try { long start = test[0] instanceof Long ? (Long) test[0] : DATE_FORMAT.parse((String) test[0]).getTime(); long stop = test[1] instanceof Long ? (Long) test[1] : DATE_FORMAT.parse((String) test[1]).getTime(); List<PartitionFilter> filters = TimePartitionedFileSetDataset.partitionFiltersForTimeRange(start, stop); //Assert.assertEquals(test.length - 2, filters.size()); Set<String> expectedSet = Sets.newHashSet(); for (int i = 2; i < test.length; i++) { expectedSet.add((String) test[i]); } Set<String> actualSet = Sets.newHashSet(); for (PartitionFilter filter : filters) { actualSet.add(filter == null ? null : filter.toString()); } Assert.assertEquals(expectedSet, actualSet); } catch (Throwable t) { throw new Exception("Failed for range " + test[0] + "..." + test[1], t); } } }
continue; if (filter != null && !filter.match(key)) { continue;
@Override public PartitionFilter.Builder get() { return PartitionFilter.builder(); } };
/** * Logs a warning if the partition filter contains a field that is not part of the partitioning. */ private void warnIfInvalidPartitionFilter(PartitionFilter filter, Partitioning partitioning) { if (null == filter) { return; } for (Map.Entry<String, PartitionFilter.Condition<? extends Comparable>> entry : filter.getConditions().entrySet()) { if (!partitioning.getFields().containsKey(entry.getKey())) { LOG.warn("Partition filter cannot match any partitions in dataset '{}' because it contains field '{}' " + "that is not a valid partitioning field", getName(), entry.getKey()); } } }
@Override public void initialize(MapReduceTaskContext<byte[], Put> context) throws Exception { InputContext inputContext = context.getInputContext(); Preconditions.checkArgument(PARTITIONED.equals(inputContext.getInputName())); Preconditions.checkArgument(inputContext instanceof PartitionedFileSetInputContext); this.pfsInputcontext = (PartitionedFileSetInputContext) inputContext; Preconditions.checkNotNull(pfsInputcontext.getInputPartitionKey()); Preconditions.checkArgument( pfsInputcontext.getInputPartitionKeys().contains(pfsInputcontext.getInputPartitionKey()) ); Map<String, String> dsArguments = RuntimeArguments.extractScope(Scope.DATASET, PARTITIONED, context.getRuntimeArguments()); PartitionFilter inputPartitionFilter = PartitionedFileSetArguments.getInputPartitionFilter(dsArguments); Preconditions.checkNotNull(inputPartitionFilter); // verify that the partition matches the partition filter for this MapReduce Preconditions.checkArgument(inputPartitionFilter.match(pfsInputcontext.getInputPartitionKey())); }
String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); PartitionFilter.Condition<? extends Comparable> condition = filter.getCondition(fieldName); if (condition == null) { break; // this field is not present; we can't include any more fields in the start key
private static <T extends Comparable<T>> void addSingleConditionFilters(List<PartitionFilter> filters, String field, T[][] conditions) { for (T[] condition : conditions) { filters.add(addCondition(PartitionFilter.builder(), field, condition).build()); } }
/** * Match this filter against a partition key. The key matches iff it matches all conditions. * * @throws java.lang.IllegalArgumentException if one of the field types in the partition key are incompatible */ public boolean match(PartitionKey partitionKey) { for (Map.Entry<String, Condition<? extends Comparable>> condition : getConditions().entrySet()) { Comparable value = partitionKey.getField(condition.getKey()); if (value == null || !condition.getValue().match(value)) { return false; } } return true; }
String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); PartitionFilter.Condition<? extends Comparable> condition = filter.getCondition(fieldName); if (condition == null) { break; // this field is not present; we can't include any more fields in the stop key
public void deleteMatchingPartitionsByTime(long upperLimit) throws IOException { if (upperLimit > 0 && upperLimit < Long.MAX_VALUE) { PartitionFilter filter = PartitionFilter.builder().addRangeCondition(SNAPSHOT_FIELD, null, upperLimit).build(); Set<PartitionDetail> partitions = files.getPartitions(filter); for (PartitionDetail partition : partitions) { files.dropPartition(partition.getPartitionKey()); } } }
String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); PartitionFilter.Condition<? extends Comparable> condition = filter.getCondition(fieldName); if (condition == null) { break; // this field is not present; we can't include any more fields in the stop key
@Test(expected = IllegalStateException.class) public void testBuilderEmpty() { PartitionFilter.builder().build(); }