public static SearchArgument.Builder newBuilder() { return newBuilder(null); }
Integer colId = nameIdMap.get(pl.getColumnName()); String newColName = RecordReaderImpl.encodeTranslatedSargColumn(rootColumn, colId); SearchArgumentFactory.setPredicateLeafColumn(pl, newColName);
static SearchArgument getSearchArgument(Configuration conf) { String searchArgumentKryo = conf.get(SEARCH_ARGUMENT); if (searchArgumentKryo == null) { return null; } return SearchArgumentFactory.create(searchArgumentKryo); }
static void setSearchArgument(Reader.Options options, List<OrcProto.Type> types, Configuration conf, boolean isOriginal) { String columnNamesString = conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR); if (columnNamesString == null) { LOG.debug("No ORC pushdown predicate - no column names"); options.searchArgument(null, null); return; } SearchArgument sarg = SearchArgumentFactory.createFromConf(conf); if (sarg == null) { LOG.debug("No ORC pushdown predicate"); options.searchArgument(null, null); return; } LOG.info("ORC pushdown predicate: " + sarg); options.searchArgument(sarg, getSargColumnNames( columnNamesString.split(","), types, options.getInclude(), isOriginal)); }
public static SearchArgument createFromConf(Configuration conf) { String sargString = null; if ((sargString = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR)) != null) { return create(Utilities.deserializeExpression(sargString)); } else if ((sargString = conf.get(SARG_PUSHDOWN)) != null) { return create(sargString); } return null; } }
/** * Builds the expression and leaf list from the original predicate. * @param expression the expression to translate. */ ConvertAstToSearchArg(Configuration conf, ExprNodeGenericFuncDesc expression) { this.conf = conf; builder = SearchArgumentFactory.newBuilder(conf); parse(expression); }
public FilterCompat.Filter setFilter(final JobConf conf) { String serializedPushdown = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR); String columnNamesString = conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR); if (serializedPushdown == null || columnNamesString == null || serializedPushdown.isEmpty() || columnNamesString.isEmpty()) { return null; } FilterPredicate p = SearchArgumentFactory.create(Utilities.deserializeExpression(serializedPushdown)) .toFilterPredicate(); if (p != null) { LOG.debug("Predicate filter for parquet is " + p.toString()); ParquetInputFormat.setFilterPredicate(conf, p); return FilterCompat.get(p); } else { LOG.debug("No predicate filter can be generated for " + TableScanDesc.FILTER_EXPR_CONF_STR + " with the value of " + serializedPushdown); return null; } }
Integer colId = nameIdMap.get(pl.getColumnName()); String newColName = RecordReaderImpl.encodeTranslatedSargColumn(rootColumn, colId); SearchArgumentFactory.setPredicateLeafColumn(pl, newColName);
/** * Builds the expression and leaf list from the original predicate. * @param expression the expression to translate. */ ConvertAstToSearchArg(Configuration conf, ExprNodeGenericFuncDesc expression) { this.conf = conf; builder = SearchArgumentFactory.newBuilder(conf); parse(expression); }
@Test public void testTimestampSerialization() throws Exception { // There is a kryo which after serialize/deserialize, // Timestamp becomes Date. We get around this issue in // SearchArgumentImpl.getLiteral. Once kryo fixed the issue // We can simplify SearchArgumentImpl.getLiteral Timestamp now = new Timestamp(new java.util.Date().getTime()); SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.TIMESTAMP, now) .end() .build(); String serializedSarg = TestInputOutputFormat.toKryo(sarg); SearchArgument sarg2 = ConvertAstToSearchArg.create(serializedSarg); Field literalField = PredicateLeafImpl.class.getDeclaredField("literal"); literalField.setAccessible(true); assertTrue(literalField.get(sarg2.getLeaves().get(0)) instanceof java.util.Date); Timestamp ts = (Timestamp)sarg2.getLeaves().get(0).getLiteral(); assertEquals(ts, now); }
@Test public void testFilterBetween() { MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 bCol; }"); SearchArgument sarg = SearchArgumentFactory.newBuilder() .between("bCol", PredicateLeaf.Type.LONG, 1L, 5L) .build(); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(lteq(bCol, 5), not(lt(bCol, 1)))"; assertEquals(expected, p.toString()); sarg = SearchArgumentFactory.newBuilder() .between("bCol", PredicateLeaf.Type.LONG, 5L, 1L) .build(); p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); expected = "and(lteq(bCol, 1), not(lt(bCol, 5)))"; assertEquals(expected, p.toString()); sarg = SearchArgumentFactory.newBuilder() .between("bCol", PredicateLeaf.Type.LONG, 1L, 1L) .build(); p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); expected = "and(lteq(bCol, 1), not(lt(bCol, 1)))"; assertEquals(expected, p.toString()); } }
@Test(expected = IllegalArgumentException.class) public void testBadLiteral() throws Exception { SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.LONG, "hi") .end() .build(); }
@Test(expected = IllegalArgumentException.class) public void testBadLiteralList() throws Exception { SearchArgumentFactory.newBuilder() .startAnd() .in("x", PredicateLeaf.Type.STRING, "hi", 23) .end() .build(); } }
@Test public void testBuilderFloat() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.LONG, 22L) .lessThan("x1", PredicateLeaf.Type.LONG, 22L) .lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()) .equals("z", PredicateLeaf.Type.FLOAT, new Double(0.22)) .equals("z1", PredicateLeaf.Type.FLOAT, new Double(0.22)) .end() .build(); assertEquals("leaf-0 = (LESS_THAN x 22), " + "leaf-1 = (LESS_THAN x1 22), " + "leaf-2 = (LESS_THAN_EQUALS y hi ), " + "leaf-3 = (EQUALS z 0.22), " + "leaf-4 = (EQUALS z1 0.22), " + "expr = (and leaf-0 leaf-1 leaf-2 leaf-3 leaf-4)", sarg.toString()); }
@Test public void testBuilder() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.LONG, 10L) .lessThanEquals("y", PredicateLeaf.Type.STRING, "hi") .equals("z", PredicateLeaf.Type.FLOAT, 1.0) .end() .build(); assertEquals("leaf-0 = (LESS_THAN x 10), " + "leaf-1 = (LESS_THAN_EQUALS y hi), " + "leaf-2 = (EQUALS z 1.0), " + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString()); sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.LONG, 10L, 20L) .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, "stinger") .end() .end() .build(); assertEquals("leaf-0 = (IS_NULL x), " + "leaf-1 = (BETWEEN y 10 20), " + "leaf-2 = (IN z 1 2 3), " + "leaf-3 = (NULL_SAFE_EQUALS a stinger), " + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString()); }
SearchArgument.Builder b = SearchArgumentFactory.newBuilder(); b = b.startAnd(); for (Predicate predicate : conjunctPredicates) {
if(keyInterval.getMinKey() != null) { RecordIdentifier k = keyInterval.getMinKey(); b = SearchArgumentFactory.newBuilder(); RecordIdentifier k = keyInterval.getMaxKey(); if(b == null) { b = SearchArgumentFactory.newBuilder();
@Test public void testFilterFloatColumns() { MessageType schema = MessageTypeParser.parseMessageType("message test { required float a; required int32 b; }"); SearchArgument sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() .isNull("a", PredicateLeaf.Type.FLOAT) .between("a", PredicateLeaf.Type.FLOAT, 10.2, 20.3) .in("b", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .end() .end() .build(); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(and(not(eq(a, null)), not(and(lteq(a, 20.3), not(lt(a, 10.2))))), not(or(or(eq(b, 1), eq(b, 2)), eq(b, 3))))"; assertEquals(expected, p.toString()); }
@Test public void testFilterColumnsThatDoNoExistOnSchema() { MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 a; required binary stinger; }"); SearchArgument sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() .isNull("a", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.LONG, 10L, 20L) // Column will be removed from filter .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) // Column will be removed from filter .nullSafeEquals("a", PredicateLeaf.Type.STRING, "stinger") .end() .end() .build(); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(not(eq(a, null)), not(eq(a, Binary{\"stinger\"})))"; assertEquals(expected, p.toString()); }
serde = new OrcSerde(); SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("z", PredicateLeaf.Type.STRING, new String("foo"))