.startAnd() .startNot() .lessThan("int1", PredicateLeaf.Type.LONG, 300000L) .end() .lessThan("int1", PredicateLeaf.Type.LONG, 600000L) .end() .build(); RecordReader rows = reader.rowsOptions(new Reader.Options() .range(0L, Long.MAX_VALUE) .startAnd() .lessThan("int1", PredicateLeaf.Type.LONG, 0L) .end() .build(); rows = reader.rowsOptions(new Reader.Options() .range(0L, Long.MAX_VALUE) .startOr() .lessThan("int1", PredicateLeaf.Type.LONG, 300L * 100) .startNot() .lessThan("int1", PredicateLeaf.Type.LONG, 300L * 3400) .end() .end() .build(); rows = reader.rowsOptions(new Reader.Options() .range(0L, Long.MAX_VALUE)
ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) expression; if (columnDesc.getTypeString().equals("boolean")) { builder.equals(columnDesc.getColumn(), PredicateLeaf.Type.BOOLEAN, true); return; builder.literal(SearchArgument.TruthValue.YES_NO_NULL); return; builder.startOr(); addChildren(expr); builder.end(); } else if (op == GenericUDFOPAnd.class) { builder.startAnd(); addChildren(expr); builder.end(); } else if (op == GenericUDFOPNot.class) { builder.startNot(); addChildren(expr); builder.end(); } else if (op == GenericUDFOPEqual.class) { createLeaf(PredicateLeaf.Operator.EQUALS, expr); } else if (op == GenericUDFOPNotEqual.class) { builder.startNot(); createLeaf(PredicateLeaf.Operator.EQUALS, expr); builder.end(); } else if (op == GenericUDFOPEqualNS.class) { createLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, expr); } else if (op == GenericUDFOPGreaterThan.class) {
String columnName = getColumnName(expression, variable); if (columnName == null) { builder.literal(SearchArgument.TruthValue.YES_NO_NULL); return; builder.literal(SearchArgument.TruthValue.YES_NO_NULL); return; builder.startNot(); switch (operator) { case IS_NULL: builder.isNull(columnName, type); break; case EQUALS: builder.equals(columnName, type, findLiteral(conf, expression, type)); break; case NULL_SAFE_EQUALS: builder.nullSafeEquals(columnName, type, findLiteral(conf, expression, type)); break; case LESS_THAN: builder.lessThan(columnName, type, findLiteral(conf, expression, type)); break; case LESS_THAN_EQUALS: builder.lessThanEquals(columnName, type, findLiteral(conf, expression, type)); break; case IN: builder.in(columnName, type, getLiteralList(expression, type, variable + 1));
sargBuilder.startNot(); sargBuilder.lessThan(col, literalPair.second, literalPair.first); sargBuilder.end(); } else { sargBuilder.lessThan(col, literalPair.second, literalPair.first); sargBuilder.startNot(); sargBuilder.lessThanEquals(col, literalPair.second, literalPair.first); sargBuilder.end(); } else { sargBuilder.lessThanEquals(col, literalPair.second, literalPair.first); sargBuilder.lessThanEquals(col, literalPair.second, literalPair.first); } else { sargBuilder.startNot(); sargBuilder.lessThanEquals(col, literalPair.second, literalPair.first); sargBuilder.end(); sargBuilder.lessThan(col, literalPair.second, literalPair.first); } else { sargBuilder.startNot(); sargBuilder.lessThan(col, literalPair.second, literalPair.first); sargBuilder.end(); sargBuilder.equals(col, literalPair.second, literalPair.first); return null; case NOT_EQUALS:
switch (expr.getOpType()) { case OP_AND: builder.startAnd(); buildSearchArgument(lhs, builder); buildSearchArgument(rhs, builder); builder.end(); break; case OP_OR: builder.startOr(); buildSearchArgument(lhs, builder); buildSearchArgument(rhs, builder); builder.end(); break; case OP_EQ: builder.equals(getColumnName(lhs), getExpressionValue(rhs)); break; case OP_NE: builder.startNot(); builder.equals(getColumnName(lhs), getExpressionValue(rhs)); builder.end(); break; case OP_LT: builder.lessThan(getColumnName(lhs), getExpressionValue(rhs)); break; case OP_LE: builder.lessThanEquals(getColumnName(lhs), getExpressionValue(rhs)); break; case OP_GT: builder.startNot();
@VisibleForTesting SearchArgument getSearchArgument(Expression expr) { if (expr == null) { return null; } Builder builder = SearchArgumentFactory.newBuilder(); boolean beginWithAnd = !(expr.getOpType().equals(OpType.OP_AND) || expr.getOpType().equals(OpType.OP_OR) || expr.getOpType().equals(OpType.OP_NOT)); if (beginWithAnd) { builder.startAnd(); } buildSearchArgument(expr, builder); if (beginWithAnd) { builder.end(); } SearchArgument sArg = builder.build(); return sArg; }
ORCSearchArgumentGenerator(final List<String> columnNames) { super(true); this.columnNames = columnNames; sargBuilder = SearchArgumentFactory.newBuilder(); sargBuilder.startAnd(); }
ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) expression; if (columnDesc.getTypeString().equals("boolean")) { builder.equals(columnDesc.getColumn(), PredicateLeaf.Type.BOOLEAN, true); return; builder.literal(SearchArgument.TruthValue.YES_NO_NULL); return; builder.startOr(); addChildren(expr); builder.end(); } else if (op == GenericUDFOPAnd.class) { builder.startAnd(); addChildren(expr); builder.end(); } else if (op == GenericUDFOPNot.class) { builder.startNot(); addChildren(expr); builder.end(); } else if (op == GenericUDFOPEqual.class) { createLeaf(PredicateLeaf.Operator.EQUALS, expr); } else if (op == GenericUDFOPNotEqual.class) { builder.startNot(); createLeaf(PredicateLeaf.Operator.EQUALS, expr); builder.end(); } else if (op == GenericUDFOPEqualNS.class) { createLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, expr); } else if (op == GenericUDFOPGreaterThan.class) {
String columnName = getColumnName(expression, variable); if (columnName == null) { builder.literal(SearchArgument.TruthValue.YES_NO_NULL); return; builder.literal(SearchArgument.TruthValue.YES_NO_NULL); return; builder.startNot(); switch (operator) { case IS_NULL: builder.isNull(columnName, boxType.type); break; case EQUALS: builder.equals(columnName, boxType.type, findLiteral(conf, expression, boxType)); break; case NULL_SAFE_EQUALS: builder.nullSafeEquals(columnName, boxType.type, findLiteral(conf, expression, boxType)); break; case LESS_THAN: builder.lessThan(columnName, boxType.type, findLiteral(conf, expression, boxType)); break; case LESS_THAN_EQUALS: builder.lessThanEquals(columnName, boxType.type, findLiteral(conf, expression, boxType)); break; case IN: builder.in(columnName, boxType.type, getLiteralList(expression, boxType, variable + 1));
RecordIdentifier k = keyInterval.getMinKey(); b = SearchArgumentFactory.newBuilder(); b.startAnd() //not(ot < 7) -> ot >=7 .startNot().lessThan("originalTransaction", PredicateLeaf.Type.LONG, k.getWriteId()).end(); b.startNot().lessThan( "bucket", PredicateLeaf.Type.LONG, minBucketProp).end(); b.startNot().lessThan("rowId", PredicateLeaf.Type.LONG, minRowId).end(); b.end(); b = SearchArgumentFactory.newBuilder(); b.startAnd().lessThanEquals( "originalTransaction", PredicateLeaf.Type.LONG, k.getWriteId()); b.lessThanEquals("bucket", PredicateLeaf.Type.LONG, maxBucketProp); b.lessThanEquals("rowId", PredicateLeaf.Type.LONG, maxRowId); b.end(); deleteEventSarg = b.build(); LOG.info("deleteReader SARG(" + deleteEventSarg + ") "); deleteEventReaderOptions.searchArgument(deleteEventSarg,
SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.DATE, Date.valueOf("1970-1-11")) .lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()) .equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")) .end() .build(); MessageType schema = MessageTypeParser.parseMessageType("message test {" + " required int32 x; required binary y; required binary z;}"); .startNot() .startOr() .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")) .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()) .end() .end() .build(); schema = MessageTypeParser.parseMessageType("message test {" + " optional int32 x; required binary y; required int32 z;" +
SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.DATE, Date.valueOf("2005-3-12")) .lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()) .equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")) .end() .build(); MessageType schema = MessageTypeParser.parseMessageType("message test {" + " required int32 x; required binary y; required binary z;}"); .startNot() .startOr() .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")) .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()) .end() .end() .build(); schema = MessageTypeParser.parseMessageType("message test {" + " optional int32 x; required binary y; required int32 z;" +
SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.DATE, Date.valueOf("1970-1-11")) .lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()) .equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")) .end() .build(); assertEquals("leaf-0 = (LESS_THAN x 1970-01-11), " + "leaf-1 = (LESS_THAN_EQUALS y hi ), " + .startNot() .startOr() .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")) .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()) .end() .end() .build(); assertEquals("leaf-0 = (IS_NULL x), " + "leaf-1 = (BETWEEN y 10 20), " +
@Test public void testFilterBetween() { MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 bCol; }"); SearchArgument sarg = SearchArgumentFactory.newBuilder() .between("bCol", PredicateLeaf.Type.LONG, 1L, 5L) .build(); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(lteq(bCol, 5), not(lt(bCol, 1)))"; assertEquals(expected, p.toString()); sarg = SearchArgumentFactory.newBuilder() .between("bCol", PredicateLeaf.Type.LONG, 5L, 1L) .build(); p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); expected = "and(lteq(bCol, 1), not(lt(bCol, 5)))"; assertEquals(expected, p.toString()); sarg = SearchArgumentFactory.newBuilder() .between("bCol", PredicateLeaf.Type.LONG, 1L, 1L) .build(); p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); expected = "and(lteq(bCol, 1), not(lt(bCol, 1)))"; assertEquals(expected, p.toString()); } }
SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.DATE, Date.valueOf("2005-3-12")) .lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()) .equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")) .end() .build(); assertEquals("leaf-0 = (LESS_THAN x 2005-03-12), " + "leaf-1 = (LESS_THAN_EQUALS y hi ), " + .startNot() .startOr() .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")) .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()) .end() .end() .build(); assertEquals("leaf-0 = (IS_NULL x), " + "leaf-1 = (BETWEEN y 10 20), " +
@Test public void testBuilder() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.LONG, 10L) .lessThanEquals("y", PredicateLeaf.Type.STRING, "hi") .equals("z", PredicateLeaf.Type.FLOAT, 1.0) .end() .build(); assertEquals("leaf-0 = (LESS_THAN x 10), " + "leaf-1 = (LESS_THAN_EQUALS y hi), " + "leaf-2 = (EQUALS z 1.0), " + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString()); sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.LONG, 10L, 20L) .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, "stinger") .end() .end() .build(); assertEquals("leaf-0 = (IS_NULL x), " + "leaf-1 = (BETWEEN y 10 20), " + "leaf-2 = (IN z 1 2 3), " + "leaf-3 = (NULL_SAFE_EQUALS a stinger), " + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString()); }