/** * Creates a {@link DataSourceReader} to scan the data from this data source. * * If this method fails (by throwing an exception), the action will fail and no Spark job will be * submitted. * * @param schema the user specified schema. * @param options the options for the returned data source reader, which is an immutable * case-insensitive string-to-string map. * * By default this method throws {@link UnsupportedOperationException}, implementations should * override this method to handle user specified schema. */ default DataSourceReader createReader(StructType schema, DataSourceOptions options) { String name; if (this instanceof DataSourceRegister) { name = ((DataSourceRegister) this).shortName(); } else { name = this.getClass().getName(); } throw new UnsupportedOperationException(name + " does not support user specified schema"); }
@Override public Filter[] pushFilters(Filter[] filters) { Filter[] supported = Arrays.stream(filters).filter(f -> { if (f instanceof GreaterThan) { GreaterThan gt = (GreaterThan) f; return gt.attribute().equals("i") && gt.value() instanceof Integer; } else { return false; } }).toArray(Filter[]::new); Filter[] unsupported = Arrays.stream(filters).filter(f -> { if (f instanceof GreaterThan) { GreaterThan gt = (GreaterThan) f; return !gt.attribute().equals("i") || !(gt.value() instanceof Integer); } else { return true; } }).toArray(Filter[]::new); this.filters = supported; return unsupported; }
if (filter instanceof EqualTo) { final EqualTo equalTo = (EqualTo) filter; if (equalTo.attribute().equals(SchemaToStructTypeConverter.GROUP)) { LOGGER.info("Filter {} specifies that {} should be {}", filter, SchemaToStructTypeConverter.GROUP, equalTo.value()); return Collections.singleton((String) equalTo.value()); if (or.left() instanceof EqualTo && or.right() instanceof EqualTo && ((EqualTo) or.left()).attribute().equals(SchemaToStructTypeConverter.GROUP) && ((EqualTo) or.right()).attribute().equals(SchemaToStructTypeConverter.GROUP)) { final Set<String> groups = new HashSet<>(); groups.add((String) ((EqualTo) or.left()).value()); groups.add((String) ((EqualTo) or.right()).value()); LOGGER.info("Filter {} specifies that {} should be {} or {}", filter, SchemaToStructTypeConverter.GROUP, ((EqualTo) or.left()).value(), ((EqualTo) or.right()).value()); return groups; if (in.attribute().equals(SchemaToStructTypeConverter.GROUP)) { final Set<String> groups = new HashSet<>(); for (final Object o : in.values()) { groups.add((String) o); StringUtils.join(in.values(), ',')); return groups;
case IS_NULL: IsNull isNullFilter = (IsNull) filter; return isNull(isNullFilter.attribute()); return notNull(notNullFilter.attribute()); return lessThan(lt.attribute(), convertLiteral(lt.value())); return lessThanOrEqual(ltEq.attribute(), convertLiteral(ltEq.value())); return greaterThan(gt.attribute(), convertLiteral(gt.value())); return greaterThanOrEqual(gtEq.attribute(), convertLiteral(gtEq.value())); Preconditions.checkNotNull(eq.value(), "Expression is always false (eq is not null-safe): " + filter); return equal(eq.attribute(), convertLiteral(eq.value())); } else { EqualNullSafe eq = (EqualNullSafe) filter; if (eq.value() == null) { return isNull(eq.attribute()); } else { return equal(eq.attribute(), convertLiteral(eq.value())); In inFilter = (In) filter; Expression in = alwaysFalse(); for (Object value : inFilter.values()) { in = or(in, equal(inFilter.attribute(), convertLiteral(value)));
final Predicate<?> isEqual = new IsEqual(equalNullSafe.value()); final Set<String> relevantGroups = getGroupsFromFilter(filter); if (null != relevantGroups) { map.get(group).add(new TupleAdaptedPredicate<>(isEqual, new String[]{equalNullSafe.attribute()})); LOGGER.debug("Converted {} to IsEqual ({})", filter, equalNullSafe.attribute()); } else if (filter instanceof GreaterThan) { final GreaterThan greaterThan = (GreaterThan) filter; final Predicate<?> isMoreThan = new IsMoreThan((Comparable<?>) greaterThan.value(), false); final Set<String> relevantGroups = getGroupsFromFilter(filter); if (null != relevantGroups) { map.get(group).add(new TupleAdaptedPredicate<>(isMoreThan, new String[]{greaterThan.attribute()})); LOGGER.debug("Converted {} to isMoreThan ({})", filter, greaterThan.attribute()); } else if (filter instanceof GreaterThanOrEqual) { final GreaterThanOrEqual greaterThan = (GreaterThanOrEqual) filter; final Predicate<?> isMoreThan = new IsMoreThan((Comparable<?>) greaterThan.value(), true); final Set<String> relevantGroups = getGroupsFromFilter(filter); if (null != relevantGroups) { map.get(group).add(new TupleAdaptedPredicate<>(isMoreThan, new String[]{greaterThan.attribute()})); LOGGER.debug("Converted {} to IsMoreThan ({})", filter, greaterThan.attribute()); } else if (filter instanceof LessThan) { final LessThan lessThan = (LessThan) filter; final Predicate<?> isLessThan = new IsLessThan((Comparable<?>) lessThan.value(), false); final Set<String> relevantGroups = getGroupsFromFilter(filter);
private Set<String> getGroupsFromFilter(final Filter filter) { if (filter instanceof EqualTo) { return getGroupsThatHaveProperty(((EqualTo) filter).attribute()); } else if (filter instanceof EqualNullSafe) { return getGroupsThatHaveProperty(((EqualNullSafe) filter).attribute()); } else if (filter instanceof GreaterThan) { return getGroupsThatHaveProperty(((GreaterThan) filter).attribute()); } else if (filter instanceof GreaterThanOrEqual) { return getGroupsThatHaveProperty(((GreaterThanOrEqual) filter).attribute()); } else if (filter instanceof LessThan) { return getGroupsThatHaveProperty(((LessThan) filter).attribute()); } else if (filter instanceof LessThanOrEqual) { return getGroupsThatHaveProperty(((LessThanOrEqual) filter).attribute()); } else if (filter instanceof In) { return getGroupsThatHaveProperty(((In) filter).attribute()); } else if (filter instanceof IsNull) { return getGroupsThatHaveProperty(((IsNotNull) filter).attribute()); } else if (filter instanceof And) { final And and = (And) filter; final Set<String> groups = new HashSet<>(); final Set<String> leftGroups = getGroupsFromFilter(and.left()); final Set<String> rightGroups = getGroupsFromFilter(and.right()); if (null != leftGroups) { groups.addAll(leftGroups);
if (filter instanceof EqualTo) { final EqualTo equalTo = (EqualTo) filter; final String attribute = equalTo.attribute(); if (attribute.equals(SchemaToStructTypeConverter.VERTEX_COL_NAME)) { LOGGER.info("Setting operation to GetRDDOfElements"); operation = new GetRDDOfElements.Builder() .input(new EntitySeed(equalTo.value())) .view(clonedView) .build(); LOGGER.info("Setting operation to GetRDDOfElements"); operation = new GetRDDOfElements.Builder() .input(new EntitySeed(equalTo.value())) .view(clonedView) .build();
String baseTable = value != null ? value.toString() : ""; String dmls = JdbcExtendedUtils. getInsertOrPutString(fullyQualifiedName, schema, false, false); value = parameters.get(ExternalStoreUtils.DEPENDENT_RELATIONS()); String[] dependentRelations = value != null
@Override public Filter[] pushFilters(Filter[] filters) { Filter[] supported = Arrays.stream(filters).filter(f -> { if (f instanceof GreaterThan) { GreaterThan gt = (GreaterThan) f; return gt.attribute().equals("i") && gt.value() instanceof Integer; } else { return false; } }).toArray(Filter[]::new); Filter[] unsupported = Arrays.stream(filters).filter(f -> { if (f instanceof GreaterThan) { GreaterThan gt = (GreaterThan) f; return !gt.attribute().equals("i") || !(gt.value() instanceof Integer); } else { return true; } }).toArray(Filter[]::new); this.filters = supported; return unsupported; }
/** * Creates a {@link DataSourceReader} to scan the data from this data source. * * If this method fails (by throwing an exception), the action will fail and no Spark job will be * submitted. * * @param schema the user specified schema. * @param options the options for the returned data source reader, which is an immutable * case-insensitive string-to-string map. * * By default this method throws {@link UnsupportedOperationException}, implementations should * override this method to handle user specified schema. */ default DataSourceReader createReader(StructType schema, DataSourceOptions options) { String name; if (this instanceof DataSourceRegister) { name = ((DataSourceRegister) this).shortName(); } else { name = this.getClass().getName(); } throw new UnsupportedOperationException(name + " does not support user specified schema"); }
@Override public List<InputPartition<InternalRow>> planInputPartitions() { List<InputPartition<InternalRow>> res = new ArrayList<>(); Integer lowerBound = null; for (Filter filter : filters) { if (filter instanceof GreaterThan) { GreaterThan f = (GreaterThan) filter; if ("i".equals(f.attribute()) && f.value() instanceof Integer) { lowerBound = (Integer) f.value(); break; } } } if (lowerBound == null) { res.add(new JavaAdvancedInputPartition(0, 5, requiredSchema)); res.add(new JavaAdvancedInputPartition(5, 10, requiredSchema)); } else if (lowerBound < 4) { res.add(new JavaAdvancedInputPartition(lowerBound + 1, 5, requiredSchema)); res.add(new JavaAdvancedInputPartition(5, 10, requiredSchema)); } else if (lowerBound < 9) { res.add(new JavaAdvancedInputPartition(lowerBound + 1, 10, requiredSchema)); } return res; } }
@Override public List<InputPartition<InternalRow>> planInputPartitions() { List<InputPartition<InternalRow>> res = new ArrayList<>(); Integer lowerBound = null; for (Filter filter : filters) { if (filter instanceof GreaterThan) { GreaterThan f = (GreaterThan) filter; if ("i".equals(f.attribute()) && f.value() instanceof Integer) { lowerBound = (Integer) f.value(); break; } } } if (lowerBound == null) { res.add(new JavaAdvancedInputPartition(0, 5, requiredSchema)); res.add(new JavaAdvancedInputPartition(5, 10, requiredSchema)); } else if (lowerBound < 4) { res.add(new JavaAdvancedInputPartition(lowerBound + 1, 5, requiredSchema)); res.add(new JavaAdvancedInputPartition(5, 10, requiredSchema)); } else if (lowerBound < 9) { res.add(new JavaAdvancedInputPartition(lowerBound + 1, 10, requiredSchema)); } return res; } }