private static List<PartitionFilter> addUpperFilters(String[] fields, int position, PartitionKey keyUpper, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int upper = (Integer) keyUpper.getField(fieldName); // only add the filter if this condition is satisfiable. For example, not for hour<0 or month<1 if (isSatisfiableUpperBound(fieldName, upper)) { filters.add(supplier.get().addRangeCondition(fieldName, null, upper).build()); } return addUpperFilters(fields, position + 1, keyUpper, filters, nextSupplier(supplier, fieldName, upper)); }
private static List<PartitionFilter> addUpperFilters(String[] fields, int position, PartitionKey keyUpper, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int upper = (Integer) keyUpper.getField(fieldName); // only add the filter if this condition is satisfiable. For example, not for hour<0 or month<1 if (isSatisfiableUpperBound(fieldName, upper)) { filters.add(supplier.get().addRangeCondition(fieldName, null, upper).build()); } return addUpperFilters(fields, position + 1, keyUpper, filters, nextSupplier(supplier, fieldName, upper)); }
private static List<PartitionFilter> addLowerFilters(String[] fields, int position, PartitionKey keyLower, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int lower = (Integer) keyLower.getField(fieldName); // if this field is at the finest granularity (minutes), we must include its value in the range // otherwise we exclude it from the range and descend into the next finer granularity with a value // constraints on the current field name. For example: // - for hour:15/minute:10, we add a filter for hour>=16 (excluding 15) and descent for hour=15 // - now the remaining field is minute:10, we add a filter for minute>=10 (including 10) int lowerBound = position == fields.length - 1 ? lower : lower + 1; // only add the filter if this condition is satisfiable. For example, not for hour>=24 or month>=13 if (isSatisfiableLowerBound(fieldName, lowerBound)) { filters.add(supplier.get().addRangeCondition(fieldName, lowerBound, null).build()); } return addLowerFilters(fields, position + 1, keyLower, filters, nextSupplier(supplier, fieldName, lower)); }
@Test public void testSetGetInputPartitionFilter() throws Exception { Map<String, String> arguments = new HashMap<>(); PartitionFilter filter = PartitionFilter.builder() .addRangeCondition("i", 30, 40) .addValueCondition("l", 17L) .addValueCondition("s", "x") .build(); PartitionedFileSetArguments.setInputPartitionFilter(arguments, filter); Assert.assertEquals(filter, PartitionedFileSetArguments.getInputPartitionFilter(arguments)); arguments = new HashMap<>(); filter = PartitionFilter.builder() .addRangeCondition("i", 30, 40) .addValueCondition("s", "x") .build(); PartitionedFileSetArguments.setInputPartitionFilter(arguments, filter); Assert.assertEquals(filter, PartitionedFileSetArguments.getInputPartitionFilter(arguments)); arguments = new HashMap<>(); filter = PartitionFilter.ALWAYS_MATCH; PartitionedFileSetArguments.setInputPartitionFilter(arguments, filter); Assert.assertEquals(filter, PartitionedFileSetArguments.getInputPartitionFilter(arguments)); }
inputArgs, PartitionFilter.builder().addRangeCondition("x", "na", "nx").build()); Map<String, String> outputArgs = new HashMap<>(); PartitionKey outputKey = PartitionKey.builder().addStringField("x", "xx").build();
private static List<PartitionFilter> addLowerFilters(String[] fields, int position, PartitionKey keyLower, List<PartitionFilter> filters, Supplier<PartitionFilter.Builder> supplier) { if (position >= fields.length) { return filters; } String fieldName = fields[position]; int lower = (Integer) keyLower.getField(fieldName); // if this field is at the finest granularity (minutes), we must include its value in the range // otherwise we exclude it from the range and descend into the next finer granularity with a value // constraints on the current field name. For example: // - for hour:15/minute:10, we add a filter for hour>=16 (excluding 15) and descent for hour=15 // - now the remaining field is minute:10, we add a filter for minute>=10 (including 10) int lowerBound = position == fields.length - 1 ? lower : lower + 1; // only add the filter if this condition is satisfiable. For example, not for hour>=24 or month>=13 if (isSatisfiableLowerBound(fieldName, lowerBound)) { filters.add(supplier.get().addRangeCondition(fieldName, lowerBound, null).build()); } return addLowerFilters(fields, position + 1, keyLower, filters, nextSupplier(supplier, fieldName, lower)); }
@Test(expected = IllegalArgumentException.class) public void testIncompatibleMatch() { PartitionFilter filter = PartitionFilter.builder() .addValueCondition("year", 2012) .addRangeCondition("month", 4, 7) .addValueCondition("market", "asia") .build(); // field of incompatible type filter.match(PartitionKey .builder() .addField("month", "january") .addField("market", "latin") .addField("year", 2012) .build()); }
@Test(expected = IllegalStateException.class) public void testBuilderEmpty() { PartitionFilter.builder().build(); }
private static <T1 extends Comparable<T1>, T2 extends Comparable<T2>, T3 extends Comparable<T3>> void addThreeConditionFilters(List<PartitionFilter> filters, String field1, T1[][] conditions1, String field2, T2[][] conditions2, String field3, T3[][] conditions3) { for (T1[] cond1 : conditions1) { for (T2[] cond2 : conditions2) { for (T3[] cond3 : conditions3) { filters.add(addCondition(addCondition(addCondition( PartitionFilter.builder(), field1, cond1), field2, cond2), field3, cond3).build()); } } } }
@Test(expected = IllegalArgumentException.class) public void testBuilderNullValue() { PartitionFilter.builder().<Long>addValueCondition("x", null).build(); }
@Test(expected = IllegalArgumentException.class) public void testBuilderEmptyField2() { PartitionFilter.builder().addRangeCondition("", 1, 2).build(); }
@Test(expected = IllegalArgumentException.class) public void testBuilderEmptyField() { PartitionFilter.builder().addValueCondition("", 1).build(); }
@Test(expected = IllegalArgumentException.class) public void testBuilderDuplicateField() { PartitionFilter.builder() .addValueCondition("x", 10) .addValueCondition("y", 10L) .addRangeCondition("x", 14, 15).build(); }
private static <T extends Comparable<T>> void addSingleConditionFilters(List<PartitionFilter> filters, String field, T[][] conditions) { for (T[] condition : conditions) { filters.add(addCondition(PartitionFilter.builder(), field, condition).build()); } }
@Test(expected = IllegalArgumentException.class) public void testBuilderNullField2() { PartitionFilter.builder().addRangeCondition(null, 1, 2).build(); }
@Test(expected = IllegalArgumentException.class) public void testBuilderNullField() { PartitionFilter.builder().addValueCondition(null, 1).build(); }
public void deleteMatchingPartitionsByTime(long upperLimit) throws IOException { if (upperLimit > 0 && upperLimit < Long.MAX_VALUE) { PartitionFilter filter = PartitionFilter.builder().addRangeCondition(SNAPSHOT_FIELD, null, upperLimit).build(); Set<PartitionDetail> partitions = files.getPartitions(filter); for (PartitionDetail partition : partitions) { files.dropPartition(partition.getPartitionKey()); } } }
private static <T1 extends Comparable<T1>, T2 extends Comparable<T2>> void addTwoConditionFilters(List<PartitionFilter> filters, String field1, T1[][] conditions1, String field2, T2[][] conditions2) { for (T1[] cond1 : conditions1) { for (T2[] cond2 : conditions2) { filters.add(addCondition(addCondition(PartitionFilter.builder(), field1, cond1), field2, cond2).build()); } } }
@Test public void testBuilderNullRange() { PartitionFilter filter = PartitionFilter.builder() .addValueCondition("a", 1) .<Long>addRangeCondition("x", null, null) .build(); Assert.assertEquals(1, filter.getConditions().size()); // only the one for "a" Assert.assertNull(filter.getCondition("x")); }
@Override public void apply() throws Exception { // this should succeed without error (but log a warning) Assert.assertEquals(Collections.EMPTY_SET, pfs.getPartitions(PartitionFilter.builder().addValueCondition("me-not-there", 42).build())); } });