@Override public SearchArgument call() { return create(conf, SerializationUtilities.deserializeExpression(sargString)); } })
private static SearchArgument getSearchArgumentFromExpression(Configuration conf, String sargString) { try { return isSargsCacheEnabled(conf)? getSargsCache(conf).get(sargString, () -> create(conf, SerializationUtilities.deserializeExpression(sargString))) : create(conf, SerializationUtilities.deserializeExpression(sargString)); } catch (ExecutionException exception) { throw new RuntimeException(exception); } }
ExprNodeGenericFuncDesc filterExpr = SerializationUtilities.deserializeExpression(filterExprSerialized); LOG.info("Kafka trimmer working on Filter tree {}", filterExpr.getExprString()); Callable<List<KafkaInputSplit>>
private static SearchArgument getSearchArgumentFromExpression(final Configuration conf, final String sargString) { try { return isSargsCacheEnabled(conf)? getSargsCache(conf).get(sargString, new Callable<SearchArgument>() { @Override public SearchArgument call() { return create(conf, SerializationUtilities.deserializeExpression(sargString)); } }) : create(conf, SerializationUtilities.deserializeExpression(sargString)); } catch (ExecutionException exception) { throw new RuntimeException(exception); } }
ExprNodeGenericFuncDesc filterExpr = SerializationUtilities.deserializeExpression(filterExprSerialized);
@Test public void TestFloatSarg() throws Exception { String serialAst = "AQEAamF2YS51dGlsLkFycmF5TGlz9AECAQFvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLnBsYW4uRXh" + "wck5vZGVDb2x1bW5EZXPjAQFmbPQAAAFiaWdvcuMBAm9yZy5hcGFjaGUuaGFkb29wLmhpdmUuc2VyZG" + "UyLnR5cGVpbmZvLlByaW1pdGl2ZVR5cGVJbmbvAQFmbG9h9AEDb3JnLmFwYWNoZS5oYWRvb3AuaGl2Z" + "S5xbC5wbGFuLkV4cHJOb2RlQ29uc3RhbnREZXPjAQECBwQ/jMzNAQRvcmcuYXBhY2hlLmhhZG9vcC5o" + "aXZlLnFsLnVkZi5nZW5lcmljLkdlbmVyaWNVREZPUEVxdWHsAQAAAYI9AUVRVUHMAQVvcmcuYXBhY2h" + "lLmhhZG9vcC5pby5Cb29sZWFuV3JpdGFibOUBAAABAgEBYm9vbGVh7g=="; SearchArgument sarg = new ConvertAstToSearchArg(conf, SerializationUtilities.deserializeExpression(serialAst)) .buildSearchArgument(); assertEquals("leaf-0", sarg.getExpression().toString()); assertEquals(1, sarg.getLeaves().size()); PredicateLeaf leaf = sarg.getLeaves().get(0); assertEquals(PredicateLeaf.Type.FLOAT, leaf.getType()); assertEquals("(EQUALS flt " + ((Number) 1.1f).doubleValue() + ")", leaf.toString()); }
@Test public void TestDateSarg() throws Exception { String serialAst = "AQEAamF2YS51dGlsLkFycmF5TGlz9AECAQFvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLnBsYW4uRXh" + "wck5vZGVDb2x1bW5EZXPjAQFk9AAAAWJpZ29y4wECb3JnLmFwYWNoZS5oYWRvb3AuaGl2ZS5zZXJkZT" + "IudHlwZWluZm8uUHJpbWl0aXZlVHlwZUluZu8BAWRhdOUBA29yZy5hcGFjaGUuaGFkb29wLmhpdmUuc" + "WwucGxhbi5FeHByTm9kZUNvbnN0YW50RGVz4wEBAgEBc3RyaW7nAwEyMDE1LTA1LTC1AQRvcmcuYXBh" + "Y2hlLmhhZG9vcC5oaXZlLnFsLnVkZi5nZW5lcmljLkdlbmVyaWNVREZPUEVxdWHsAQAAAYI9AUVRVUH" + "MAQVvcmcuYXBhY2hlLmhhZG9vcC5pby5Cb29sZWFuV3JpdGFibOUBAAABAgEBYm9vbGVh7g=="; SearchArgument sarg = new ConvertAstToSearchArg(conf, SerializationUtilities.deserializeExpression(serialAst)) .buildSearchArgument(); assertEquals("leaf-0", sarg.getExpression().toString()); assertEquals(1, sarg.getLeaves().size()); PredicateLeaf leaf = sarg.getLeaves().get(0); assertEquals(PredicateLeaf.Type.DATE, leaf.getType()); assertEquals("(EQUALS dt 2015-05-05)", leaf.toString()); }
@Test public void TestBigintSarg() throws Exception { String serialAst = "AQEAamF2YS51dGlsLkFycmF5TGlz9AECAQFvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLnBsYW4uRXh" + "wck5vZGVDb2x1bW5EZXPjAQFi6QAAAWJpZ29y4wECb3JnLmFwYWNoZS5oYWRvb3AuaGl2ZS5zZXJkZT" + "IudHlwZWluZm8uUHJpbWl0aXZlVHlwZUluZu8BAWJpZ2lu9AEDb3JnLmFwYWNoZS5oYWRvb3AuaGl2Z" + "S5xbC5wbGFuLkV4cHJOb2RlQ29uc3RhbnREZXPjAQECBwnywAEBBG9yZy5hcGFjaGUuaGFkb29wLmhp" + "dmUucWwudWRmLmdlbmVyaWMuR2VuZXJpY1VERk9QRXF1YewBAAABgj0BRVFVQcwBBW9yZy5hcGFjaGU" + "uaGFkb29wLmlvLkJvb2xlYW5Xcml0YWJs5QEAAAECAQFib29sZWHu"; SearchArgument sarg = new ConvertAstToSearchArg(conf, SerializationUtilities.deserializeExpression(serialAst)) .buildSearchArgument(); assertEquals("leaf-0", sarg.getExpression().toString()); assertEquals(1, sarg.getLeaves().size()); PredicateLeaf leaf = sarg.getLeaves().get(0); assertEquals(PredicateLeaf.Type.LONG, leaf.getType()); assertEquals("(EQUALS bi 12345)", leaf.toString()); }
@Test public void TestDoubleSarg() throws Exception { String serialAst = "AQEAamF2YS51dGlsLkFycmF5TGlz9AECAQFvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLnBsYW4uRXh" + "wck5vZGVDb2x1bW5EZXPjAQFkYuwAAAFiaWdvcuMBAm9yZy5hcGFjaGUuaGFkb29wLmhpdmUuc2VyZG" + "UyLnR5cGVpbmZvLlByaW1pdGl2ZVR5cGVJbmbvAQFkb3VibOUBA29yZy5hcGFjaGUuaGFkb29wLmhpd" + "mUucWwucGxhbi5FeHByTm9kZUNvbnN0YW50RGVz4wEBAgcKQAGZmZmZmZoBBG9yZy5hcGFjaGUuaGFk" + "b29wLmhpdmUucWwudWRmLmdlbmVyaWMuR2VuZXJpY1VERk9QRXF1YewBAAABgj0BRVFVQcwBBW9yZy5" + "hcGFjaGUuaGFkb29wLmlvLkJvb2xlYW5Xcml0YWJs5QEAAAECAQFib29sZWHu"; SearchArgument sarg = new ConvertAstToSearchArg(conf, SerializationUtilities.deserializeExpression(serialAst)) .buildSearchArgument(); assertEquals("leaf-0", sarg.getExpression().toString()); assertEquals(1, sarg.getLeaves().size()); PredicateLeaf leaf = sarg.getLeaves().get(0); assertEquals(PredicateLeaf.Type.FLOAT, leaf.getType()); assertEquals("(EQUALS dbl 2.2)", leaf.toString()); } }
@Test public void TestTimestampSarg() throws Exception { String serialAst = "AQEAamF2YS51dGlsLkFycmF5TGlz9AECAQFvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLn" + "BsYW4uRXhwck5vZGVDb2x1bW5EZXPjAQF08wAAAWJpZ29y4wECb3JnLmFwYWNoZS5o" + "YWRvb3AuaGl2ZS5zZXJkZTIudHlwZWluZm8uUHJpbWl0aXZlVHlwZUluZu8BAXRpbW" + "VzdGFt8AEDb3JnLmFwYWNoZS5oYWRvb3AuaGl2ZS5xbC5wbGFuLkV4cHJOb2RlQ29u" + "c3RhbnREZXPjAQECAQFzdHJpbucDATIwMTUtMDMtMTcgMTI6MzQ6NbYBBG9yZy5hcG" + "FjaGUuaGFkb29wLmhpdmUucWwudWRmLmdlbmVyaWMuR2VuZXJpY1VERk9QRXF1YewB" + "AAABgj0BRVFVQcwBBW9yZy5hcGFjaGUuaGFkb29wLmlvLkJvb2xlYW5Xcml0YWJs5Q" + "EAAAECAQFib29sZWHu"; SearchArgument sarg = new ConvertAstToSearchArg(conf, SerializationUtilities.deserializeExpression(serialAst)) .buildSearchArgument(); assertEquals("leaf-0", sarg.getExpression().toString()); assertEquals(1, sarg.getLeaves().size()); PredicateLeaf leaf = sarg.getLeaves().get(0); assertEquals(PredicateLeaf.Type.TIMESTAMP, leaf.getType()); assertEquals("(EQUALS ts 2015-03-17 12:34:56.0)", leaf.toString()); }
@Test public void TestDecimalSarg() throws Exception { String serialAst = "AQEAamF2YS51dGlsLkFycmF5TGlz9AECAQFvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLnBsYW4uRXh" + "wck5vZGVDb2x1bW5EZXPjAQFkZeMAAAFiaWdvcuMBAm9yZy5hcGFjaGUuaGFkb29wLmhpdmUuc2VyZG" + "UyLnR5cGVpbmZvLkRlY2ltYWxUeXBlSW5m7wEUAAFkZWNpbWHsAQNvcmcuYXBhY2hlLmhhZG9vcC5oa" + "XZlLnFsLnBsYW4uRXhwck5vZGVDb25zdGFudERlc+MBAQRvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnNl" + "cmRlMi50eXBlaW5mby5QcmltaXRpdmVUeXBlSW5m7wEBaW70AvYBAQVvcmcuYXBhY2hlLmhhZG9vcC5" + "oaXZlLnFsLnVkZi5nZW5lcmljLkdlbmVyaWNVREZPUEVxdWHsAQAAAYI9AUVRVUHMAQZvcmcuYXBhY2" + "hlLmhhZG9vcC5pby5Cb29sZWFuV3JpdGFibOUBAAABBAEBYm9vbGVh7g=="; SearchArgument sarg = new ConvertAstToSearchArg(conf, SerializationUtilities.deserializeExpression(serialAst)) .buildSearchArgument(); assertEquals("leaf-0", sarg.getExpression().toString()); assertEquals(1, sarg.getLeaves().size()); PredicateLeaf leaf = sarg.getLeaves().get(0); assertEquals(PredicateLeaf.Type.DECIMAL, leaf.getType()); assertEquals("(EQUALS dec 123)", leaf.toString()); }
@Test public void TestCharSarg() throws Exception { String serialAst = "AQEAamF2YS51dGlsLkFycmF5TGlz9AECAQFvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLnBsYW4uRXh" + "wck5vZGVDb2x1bW5EZXPjAQFj6AAAAWJpZ29y4wECb3JnLmFwYWNoZS5oYWRvb3AuaGl2ZS5zZXJkZT" + "IudHlwZWluZm8uQ2hhclR5cGVJbmbvARQBY2hh8gEDb3JnLmFwYWNoZS5oYWRvb3AuaGl2ZS5xbC5wb" + "GFuLkV4cHJOb2RlQ29uc3RhbnREZXPjAQEEb3JnLmFwYWNoZS5oYWRvb3AuaGl2ZS5zZXJkZTIudHlw" + "ZWluZm8uUHJpbWl0aXZlVHlwZUluZu8BAXN0cmlu5wMBY2hhciAgICAgoAEFb3JnLmFwYWNoZS5oYWR" + "vb3AuaGl2ZS5xbC51ZGYuZ2VuZXJpYy5HZW5lcmljVURGT1BFcXVh7AEAAAGCPQFFUVVBzAEGb3JnLm" + "FwYWNoZS5oYWRvb3AuaW8uQm9vbGVhbldyaXRhYmzlAQAAAQQBAWJvb2xlYe4="; SearchArgument sarg = new ConvertAstToSearchArg(conf, SerializationUtilities.deserializeExpression(serialAst)) .buildSearchArgument(); assertEquals("leaf-0", sarg.getExpression().toString()); assertEquals(1, sarg.getLeaves().size()); PredicateLeaf leaf = sarg.getLeaves().get(0); assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); assertEquals("(EQUALS ch char )", leaf.toString()); }
@Test public void TestVarcharSarg() throws Exception { String serialAst = "AQEAamF2YS51dGlsLkFycmF5TGlz9AECAQFvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLnBsYW4uRXh" + "wck5vZGVDb2x1bW5EZXPjAQF24wAAAWJpZ29y4wECb3JnLmFwYWNoZS5oYWRvb3AuaGl2ZS5zZXJkZT" + "IudHlwZWluZm8uVmFyY2hhclR5cGVJbmbvAcgBAXZhcmNoYfIBA29yZy5hcGFjaGUuaGFkb29wLmhpd" + "mUucWwucGxhbi5FeHByTm9kZUNvbnN0YW50RGVz4wEBBG9yZy5hcGFjaGUuaGFkb29wLmhpdmUuc2Vy" + "ZGUyLnR5cGVpbmZvLlByaW1pdGl2ZVR5cGVJbmbvAQFzdHJpbucDAXZhcmlhYmzlAQVvcmcuYXBhY2h" + "lLmhhZG9vcC5oaXZlLnFsLnVkZi5nZW5lcmljLkdlbmVyaWNVREZPUEVxdWHsAQAAAYI9AUVRVUHMAQ" + "ZvcmcuYXBhY2hlLmhhZG9vcC5pby5Cb29sZWFuV3JpdGFibOUBAAABBAEBYm9vbGVh7g=="; SearchArgument sarg = new ConvertAstToSearchArg(conf, SerializationUtilities.deserializeExpression(serialAst)) .buildSearchArgument(); assertEquals("leaf-0", sarg.getExpression().toString()); assertEquals(1, sarg.getLeaves().size()); PredicateLeaf leaf = sarg.getLeaves().get(0); assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); assertEquals("(EQUALS vc variable)", leaf.toString()); }
@Test public void TestBooleanSarg() throws Exception { String serialAst = "AQEAamF2YS51dGlsLkFycmF5TGlz9AECAQFvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLnBsYW4uRXh" + "wck5vZGVHZW5lcmljRnVuY0Rlc+MBAQABAgECb3JnLmFwYWNoZS5oYWRvb3AuaGl2ZS5xbC5wbGFuLk" + "V4cHJOb2RlQ29sdW1uRGVz4wEBYrEAAAFib29sb3LjAQNvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnNlc" + "mRlMi50eXBlaW5mby5QcmltaXRpdmVUeXBlSW5m7wEBYm9vbGVh7gEEb3JnLmFwYWNoZS5oYWRvb3Au" + "aGl2ZS5xbC5wbGFuLkV4cHJOb2RlQ29uc3RhbnREZXPjAQEDCQUBAQVvcmcuYXBhY2hlLmhhZG9vcC5" + "oaXZlLnFsLnVkZi5nZW5lcmljLkdlbmVyaWNVREZPUEVxdWHsAQAAAYI9AUVRVUHMAQZvcmcuYXBhY2" + "hlLmhhZG9vcC5pby5Cb29sZWFuV3JpdGFibOUBAAABAwkBAgEBYrIAAAgBAwkBB29yZy5hcGFjaGUua" + "GFkb29wLmhpdmUucWwudWRmLmdlbmVyaWMuR2VuZXJpY1VERk9QQW7kAQEGAQAAAQMJ"; SearchArgument sarg = new ConvertAstToSearchArg(conf, SerializationUtilities.deserializeExpression(serialAst)) .buildSearchArgument(); assertEquals("(and leaf-0 leaf-1)", sarg.getExpression().toString()); assertEquals(2, sarg.getLeaves().size()); PredicateLeaf leaf = sarg.getLeaves().get(0); assertEquals(PredicateLeaf.Type.BOOLEAN, leaf.getType()); assertEquals("(EQUALS b1 true)", leaf.toString()); leaf = sarg.getLeaves().get(1); assertEquals(PredicateLeaf.Type.BOOLEAN, leaf.getType()); assertEquals("(EQUALS b2 true)", leaf.toString()); }
actual = kafkaScanTrimmer.computeOptimizedScan(SerializationUtilities .deserializeExpression(SerializationUtilities.serializeExpression(falseFilter))); Assert.assertTrue(actual.isEmpty()); actual = kafkaScanTrimmer.computeOptimizedScan(SerializationUtilities .deserializeExpression(SerializationUtilities.serializeExpression(falseFilter2))); Assert.assertTrue(actual.isEmpty()); actual = kafkaScanTrimmer.computeOptimizedScan(SerializationUtilities .deserializeExpression(SerializationUtilities.serializeExpression(filter3))); Assert.assertTrue(actual.isEmpty()); actual = kafkaScanTrimmer.computeOptimizedScan(SerializationUtilities .deserializeExpression(SerializationUtilities.serializeExpression(filter4))); Assert.assertTrue(actual.isEmpty());
@Test public void computeOptimizedScanOrAndCombinedFilter() { KafkaScanTrimmer kafkaScanTrimmer = new KafkaScanTrimmer(fullHouse, null); // partition = 0 and 30 <= offset < 35 or partition = 3 and 35 <= offset < 75 or (partition = 0 and offset = 40) ExprNodeGenericFuncDesc part1 = and(Lists.newArrayList(greaterThanEq(Lists.newArrayList(offsetColumn, thirtyLong)), eq(Lists.newArrayList(partitionColumn, zeroInt)), lessThan(Lists.newArrayList(offsetColumn, thirtyFiveLong)))); ExprNodeGenericFuncDesc part2 = and(Lists.newArrayList(greaterThanEq(Lists.newArrayList(offsetColumn, thirtyFiveLong)), eq(Lists.newArrayList(partitionColumn, threeInt)), lessThan(Lists.newArrayList(offsetColumn, seventyFiveLong)))); ExprNodeGenericFuncDesc part3 = and(Lists.newArrayList(eq(Lists.newArrayList(offsetColumn, fortyLong)), eq(Lists.newArrayList(partitionColumn, zeroInt)))); ExprNodeGenericFuncDesc orExpression = or(Lists.newArrayList(part1, part2, part3)); assertNotNull(orExpression); Map actual = kafkaScanTrimmer.computeOptimizedScan(SerializationUtilities .deserializeExpression(SerializationUtilities.serializeExpression(orExpression))); TopicPartition tpZero = new TopicPartition(topic, 0); TopicPartition toThree = new TopicPartition(topic, 3); KafkaInputSplit split1 = new KafkaInputSplit(topic, 0, 30, 41, PATH); KafkaInputSplit split2 = new KafkaInputSplit(topic, 3, 35, 75, PATH); Map expected = ImmutableMap.of(tpZero, split1, toThree, split2); Assert.assertEquals(expected, actual); }
@Test public void testSerializeTimestamp() { Timestamp ts = Timestamp.ofEpochMilli(1374554702000L, 123456); ExprNodeConstantDesc constant = new ExprNodeConstantDesc(ts); List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(1); children.add(constant); ExprNodeGenericFuncDesc desc = new ExprNodeGenericFuncDesc(TypeInfoFactory.timestampTypeInfo, new GenericUDFFromUtcTimestamp(), children); assertEquals(desc.getExprString(), SerializationUtilities.deserializeExpression( SerializationUtilities.serializeExpression(desc)).getExprString()); }
@Test public void computeOptimizedScanPartitionOrAndCombinedFilter() { KafkaScanTrimmer kafkaScanTrimmer = new KafkaScanTrimmer(fullHouse, null); // partition = 1 or (partition >2 and <= 3) ExprNodeGenericFuncDesc eq = eq(Lists.newArrayList(partitionColumn, ConstantExprBuilder.build(1))); ExprNodeGenericFuncDesc lessEq = lessThanEq(Lists.newArrayList(partitionColumn, ConstantExprBuilder.build(3))); ExprNodeGenericFuncDesc greater = greaterThan(Lists.newArrayList(partitionColumn, ConstantExprBuilder.build(2))); ExprNodeGenericFuncDesc orNode = or(Lists.newArrayList(and(Lists.newArrayList(lessEq, greater)), eq)); Map actual = kafkaScanTrimmer.computeOptimizedScan(SerializationUtilities .deserializeExpression(SerializationUtilities.serializeExpression(orNode))); Map expected = Maps.filterValues(fullHouse, tp -> Objects.requireNonNull(tp).getPartition() == 1 || tp.getPartition() == 3); Assert.assertEquals(expected, actual); assertNotNull(orNode); }
@Test public void computeOptimizedScanPartitionBinaryOpFilter() { KafkaScanTrimmer kafkaScanTrimmer = new KafkaScanTrimmer(fullHouse, null); int partitionId = 2; ExprNodeDesc constant = ConstantExprBuilder.build(partitionId); final List<ExprNodeDesc> children = Lists.newArrayList(partitionColumn, constant); ExprNodeGenericFuncDesc node = eq(children); assertNotNull(node); Map actual = kafkaScanTrimmer.computeOptimizedScan(SerializationUtilities .deserializeExpression(SerializationUtilities.serializeExpression(node))); Map expected = Maps.filterValues(fullHouse, tp -> Objects.requireNonNull(tp).getPartition() == partitionId); Assert.assertEquals(expected, actual); ExprNodeGenericFuncDesc lessNode = lessThan(children); assertNotNull(lessNode); actual = kafkaScanTrimmer.computeOptimizedScan(SerializationUtilities .deserializeExpression(SerializationUtilities.serializeExpression(lessNode))); expected = Maps.filterValues(fullHouse, tp -> Objects.requireNonNull(tp).getPartition() < partitionId); Assert.assertEquals(expected, actual); ExprNodeGenericFuncDesc lessEqNode = lessThanEq(children); assertNotNull(lessEqNode); actual = kafkaScanTrimmer.computeOptimizedScan(SerializationUtilities .deserializeExpression(SerializationUtilities.serializeExpression(lessEqNode))); expected = Maps.filterValues(fullHouse, tp -> Objects.requireNonNull(tp).getPartition() <= partitionId); Assert.assertEquals(expected, actual); }
if (filterExprSerialized != null) { ExprNodeGenericFuncDesc filterExpr = SerializationUtilities.deserializeExpression(filterExprSerialized); PhoenixPredicateDecomposer predicateDecomposer = PhoenixPredicateDecomposer.create(Arrays.asList(jobConf.get(serdeConstants.LIST_COLUMNS).split(",")));