@Override public PlanNode visitUnnest(UnnestNode node, RewriteContext<Set<Symbol>> context) { List<Symbol> replicateSymbols = node.getReplicateSymbols().stream() .filter(context.get()::contains) .collect(toImmutableList()); Optional<Symbol> ordinalitySymbol = node.getOrdinalitySymbol(); if (ordinalitySymbol.isPresent() && !context.get().contains(ordinalitySymbol.get())) { ordinalitySymbol = Optional.empty(); } Map<Symbol, List<Symbol>> unnestSymbols = node.getUnnestSymbols(); ImmutableSet.Builder<Symbol> expectedInputs = ImmutableSet.<Symbol>builder() .addAll(replicateSymbols) .addAll(unnestSymbols.keySet()); PlanNode source = context.rewrite(node.getSource(), expectedInputs.build()); return new UnnestNode(node.getId(), source, replicateSymbols, unnestSymbols, ordinalitySymbol); }
private static Optional<Set<Symbol>> getNonConstantSymbols(Set<ColumnHandle> columnHandles, Map<ColumnHandle, Symbol> assignments, Set<ColumnHandle> globalConstants) { // Strip off the constants from the partitioning columns (since those are not required for translation) Set<ColumnHandle> constantsStrippedPartitionColumns = columnHandles.stream() .filter(column -> !globalConstants.contains(column)) .collect(toImmutableSet()); ImmutableSet.Builder<Symbol> builder = ImmutableSet.builder(); for (ColumnHandle column : constantsStrippedPartitionColumns) { Symbol translated = assignments.get(column); if (translated == null) { return Optional.empty(); } builder.add(translated); } return Optional.of(builder.build()); }
/** * Returns an immutable set containing each of {@code elements}, minus duplicates, in the order * each appears first in the source iterator. * * @throws NullPointerException if any of {@code elements} is null */ public static <E> ImmutableSet<E> copyOf(Iterator<? extends E> elements) { // We special-case for 0 or 1 elements, but anything further is madness. if (!elements.hasNext()) { return of(); } E first = elements.next(); if (!elements.hasNext()) { return of(first); } else { return new ImmutableSet.Builder<E>().add(first).addAll(elements).build(); } }
Set<Expression> groupByExpressions = ImmutableSet.copyOf(analysis.getGroupByExpressions(node)); Iterable<Expression> inputs = Iterables.concat(groupByExpressions, arguments.build()); subPlan = handleSubqueries(subPlan, node, inputs); if (!Iterables.isEmpty(inputs)) { // avoid an empty projection if the only aggregation is COUNT (which has no arguments) subPlan = project(subPlan, inputs); List<Set<FieldId>> columnOnlyGroupingSets = ImmutableList.of(ImmutableSet.of()); if (node.getGroupBy().isPresent()) { columnOnlyGroupingSets = enumerateGroupingSets(groupingSetAnalysis); if (node.getGroupBy().get().isDistinct()) { Optional<Symbol> groupIdSymbol = Optional.empty(); ImmutableMap.Builder<Symbol, Aggregation> aggregationsBuilder = ImmutableMap.builder(); boolean needPostProjectionCoercion = false; for (FunctionCall aggregate : analysis.getAggregates(node)) { ImmutableSet.Builder<Integer> globalGroupingSets = ImmutableSet.builder(); for (int i = 0; i < groupingSets.size(); i++) { if (groupingSets.get(i).isEmpty()) { globalGroupingSets.add(i); globalGroupingSets.build()),
Map<ColumnHandle, NullableValue> fixedValues = TupleDomain.extractFixedValues(tupleDomain).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !indexableColumns.contains(entry.getKey())) .filter(entry -> !entry.getValue().isNull()) // strip nulls since meaningless in index join lookups .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); Set<String> lookupColumnNames = ImmutableSet.<String>builder() .addAll(handleToNames(ImmutableList.copyOf(indexableColumns))) .addAll(handleToNames(ImmutableList.copyOf(fixedValues.keySet()))) .build(); if (!indexedData.getIndexedTable(tpchTableHandle.getTableName(), tpchTableHandle.getScaleFactor(), lookupColumnNames).isPresent()) { return Optional.empty();
FunctionCall call = aggregation.getCall(); if (call.isDistinct() && !call.getFilter().isPresent() && !aggregation.getMask().isPresent()) { Set<Symbol> inputs = call.getArguments().stream() .map(Symbol::from) .collect(toSet()); marker = context.getSymbolAllocator().newSymbol(Iterables.getLast(inputs).getName(), BOOLEAN, "distinct"); markers.put(inputs, marker); ImmutableSet.Builder<Symbol> distinctSymbols = ImmutableSet.<Symbol>builder() .addAll(parent.getGroupingKeys()) .addAll(inputs); parent.getGroupIdSymbol().ifPresent(distinctSymbols::add); subPlan, marker, ImmutableList.copyOf(distinctSymbols.build()), Optional.empty()); newAggregations, parent.getGroupingSets(), ImmutableList.of(), parent.getStep(), parent.getHashSymbol(),
public static Optional<HiveBucketFilter> getHiveBucketFilter(Table table, TupleDomain<ColumnHandle> effectivePredicate) if (!table.getStorage().getBucketProperty().isPresent()) { return Optional.empty(); if (!bindings.isPresent()) { return Optional.empty(); OptionalInt singleBucket = getHiveBucket(table, bindings.get()); if (singleBucket.isPresent()) { return Optional.of(new HiveBucketFilter(ImmutableSet.of(singleBucket.getAsInt()))); Optional<Domain> domain = effectivePredicate.getDomains().get().entrySet().stream() .filter(entry -> ((HiveColumnHandle) entry.getKey()).getName().equals(BUCKET_COLUMN_NAME)) .findFirst() .map(Entry::getValue); if (!domain.isPresent()) { ValueSet values = domain.get().getValues(); ImmutableSet.Builder<Integer> builder = ImmutableSet.builder(); int bucketCount = table.getStorage().getBucketProperty().get().getBucketCount(); for (int i = 0; i < bucketCount; i++) { if (values.containsValue((long) i)) { builder.add(i); return Optional.of(new HiveBucketFilter(builder.build()));
ImmutableMap.Builder<StageId, StageLinkage> stageLinkages) ImmutableList.Builder<SqlStageExecution> stages = ImmutableList.builder(); schedulerStats); stages.add(stage); if (partitioningHandle.equals(SOURCE_DISTRIBUTION)) { Entry<PlanNodeId, SplitSource> entry = Iterables.getOnlyElement(plan.getSplitSources().entrySet()); PlanNodeId planNodeId = entry.getKey(); SplitSource splitSource = entry.getValue(); bucketToPartition = Optional.of(new int[1]); bucketToPartition = Optional.of(new int[1]); childStagesBuilder.add(childStage); Set<SqlStageExecution> childStages = childStagesBuilder.build(); stage.addStateChangeListener(newState -> { if (newState.isDone()) { return stages.build();
if (node.getFilter().isPresent()) { expectedFilterInputs = ImmutableSet.<Symbol>builder() .addAll(SymbolsExtractor.extractUnique(node.getFilter().get())) .addAll(context.get()) .build(); ImmutableSet.Builder<Symbol> leftInputsBuilder = ImmutableSet.builder(); leftInputsBuilder.addAll(context.get()).addAll(Iterables.transform(node.getCriteria(), JoinNode.EquiJoinClause::getLeft)); if (node.getLeftHashSymbol().isPresent()) { leftInputsBuilder.add(node.getLeftHashSymbol().get()); leftInputsBuilder.addAll(expectedFilterInputs); Set<Symbol> leftInputs = leftInputsBuilder.build(); ImmutableSet.Builder<Symbol> rightInputsBuilder = ImmutableSet.builder(); rightInputsBuilder.addAll(context.get()).addAll(Iterables.transform(node.getCriteria(), JoinNode.EquiJoinClause::getRight)); if (node.getRightHashSymbol().isPresent()) { rightInputsBuilder.add(node.getRightHashSymbol().get()); rightInputsBuilder.addAll(expectedFilterInputs); Set<Symbol> rightInputs = rightInputsBuilder.build(); .filter(context.get()::contains) .distinct() .collect(toImmutableList());
private void setLocalityGroups(Map<String, Object> tableProperties, AccumuloTable table) { Optional<Map<String, Set<String>>> groups = AccumuloTableProperties.getLocalityGroups(tableProperties); if (!groups.isPresent()) { LOG.debug("No locality groups to set"); return; } ImmutableMap.Builder<String, Set<Text>> localityGroupsBuilder = ImmutableMap.builder(); for (Map.Entry<String, Set<String>> g : groups.get().entrySet()) { ImmutableSet.Builder<Text> familyBuilder = ImmutableSet.builder(); // For each configured column for this locality group for (String col : g.getValue()) { // Locate the column family mapping via the Handle // We already validated this earlier, so it'll exist AccumuloColumnHandle handle = table.getColumns() .stream() .filter(x -> x.getName().equals(col)) .collect(Collectors.toList()) .get(0); familyBuilder.add(new Text(handle.getFamily().get())); } localityGroupsBuilder.put(g.getKey(), familyBuilder.build()); } Map<String, Set<Text>> localityGroups = localityGroupsBuilder.build(); LOG.debug("Setting locality groups: {}", localityGroups); tableManager.setLocalityGroups(table.getFullTableName(), localityGroups); }
@Override public List<String> listSchemaNames(Session session, String catalogName) { Optional<CatalogMetadata> catalog = getOptionalCatalogMetadata(session, catalogName); ImmutableSet.Builder<String> schemaNames = ImmutableSet.builder(); if (catalog.isPresent()) { CatalogMetadata catalogMetadata = catalog.get(); ConnectorSession connectorSession = session.toConnectorSession(catalogMetadata.getConnectorId()); for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); metadata.listSchemaNames(connectorSession).stream() .map(schema -> schema.toLowerCase(Locale.ENGLISH)) .forEach(schemaNames::add); } } return ImmutableList.copyOf(schemaNames.build()); }
@Override protected Void visitLambdaExpression(LambdaExpression node, Set<String> lambdaArgumentNames) { return process(node.getBody(), ImmutableSet.<String>builder() .addAll(lambdaArgumentNames) .addAll(node.getArguments().stream() .map(LambdaArgumentDeclaration::getName) .map(Identifier::getValue) .collect(toImmutableSet())) .build()); } }
public OnHeapPollingCache(Iterable<Map.Entry<K, V>> entries) { if (entries == null) { immutableMap = ImmutableMap.of(); immutableReverseMap = ImmutableMap.of(); } else { ImmutableSet.Builder<V> setOfValuesBuilder = ImmutableSet.builder(); ImmutableMap.Builder<K, V> mapBuilder = ImmutableMap.builder(); for (Map.Entry<K, V> entry : entries) { setOfValuesBuilder.add(entry.getValue()); mapBuilder.put(entry.getKey(), entry.getValue()); } final Set<V> setOfValues = setOfValuesBuilder.build(); immutableMap = mapBuilder.build(); immutableReverseMap = ImmutableMap.copyOf( Maps.asMap( setOfValues, val -> immutableMap .keySet() .stream() .filter(key -> { V retVal = immutableMap.get(key); return retVal != null && retVal.equals(val); }) .collect(Collectors.toList()) ) ); } }
this.writeChecksumBuilder = writeValidation.map(validation -> createWriteChecksumBuilder(includedColumns)); this.rowGroupStatisticsValidation = writeValidation.map(validation -> validation.createWriteStatisticsBuilder(includedColumns)); this.stripeStatisticsValidation = writeValidation.map(validation -> validation.createWriteStatisticsBuilder(includedColumns)); this.fileStatisticsValidation = writeValidation.map(validation -> validation.createWriteStatisticsBuilder(includedColumns)); this.systemMemoryUsage = systemMemoryUsage.newAggregatedMemoryContext(); ImmutableSet.Builder<Integer> presentColumns = ImmutableSet.builder(); ImmutableMap.Builder<Integer, Type> presentColumnsAndTypes = ImmutableMap.builder(); OrcType root = types.get(0); for (Map.Entry<Integer, Type> entry : includedColumns.entrySet()) { presentColumns.add(entry.getKey()); presentColumnsAndTypes.put(entry.getKey(), entry.getValue()); this.presentColumns = presentColumns.build(); ImmutableList.Builder<StripeInformation> stripes = ImmutableList.builder(); ImmutableList.Builder<Long> stripeFilePositions = ImmutableList.builder(); if (predicate.matches(numberOfRows, getStatisticsByColumnOrdinal(root, fileStats))) { this.stripes = stripes.build(); this.stripeFilePositions = stripeFilePositions.build(); .map(StripeInfo::getStripe) .mapToLong(StripeInformation::getNumberOfRows) .sum(); this.userMetadata = ImmutableMap.copyOf(Maps.transformValues(userMetadata, Slices::copyOf));
private BugCheckerInfo(Class<? extends BugChecker> checker, BugPattern pattern) { this( checker, pattern.name(), ImmutableSet.<String>builder().add(pattern.name()).add(pattern.altNames()).build(), pattern.summary(), pattern.severity(), createLinkUrl(pattern), Stream.of(pattern.suppressionAnnotations()).anyMatch(a -> isSuppressWarnings(a)), Stream.of(pattern.suppressionAnnotations()) .filter(a -> !isSuppressWarnings(a)) .collect(toImmutableSet()), ImmutableSet.copyOf(pattern.tags()), pattern.disableable()); }
public Set<String> getAllTables(String schema) throws SchemaNotFoundException { ImmutableSet.Builder<String> builder = ImmutableSet.builder(); builder.addAll(ImmutableList.copyOf(client.getDatabase(schema).listCollectionNames()).stream() .filter(name -> !name.equals(schemaCollection)) .filter(name -> !SYSTEM_TABLES.contains(name)) .collect(toSet())); builder.addAll(getTableMetadataNames(schema)); return builder.build(); }
@Override public PlanNode visitLateralJoin(LateralJoinNode node, RewriteContext<Set<Symbol>> context) { PlanNode subquery = context.rewrite(node.getSubquery(), context.get()); // remove unused lateral nodes if (intersection(ImmutableSet.copyOf(subquery.getOutputSymbols()), context.get()).isEmpty() && isScalar(subquery)) { return context.rewrite(node.getInput(), context.get()); } // prune not used correlation symbols Set<Symbol> subquerySymbols = SymbolsExtractor.extractUnique(subquery); List<Symbol> newCorrelation = node.getCorrelation().stream() .filter(subquerySymbols::contains) .collect(toImmutableList()); Set<Symbol> inputContext = ImmutableSet.<Symbol>builder() .addAll(context.get()) .addAll(newCorrelation) .build(); PlanNode input = context.rewrite(node.getInput(), inputContext); // remove unused lateral nodes if (intersection(ImmutableSet.copyOf(input.getOutputSymbols()), inputContext).isEmpty() && isScalar(input)) { return subquery; } return new LateralJoinNode(node.getId(), input, subquery, newCorrelation, node.getType(), node.getOriginSubquery()); } }
processedInputPositions = inputOperator.getInputPositions(); OperatorStats outputOperator = requireNonNull(getLast(operators, null)); outputDataSize = outputOperator.getOutputDataSize(); outputPositions = outputOperator.getOutputPositions(); .map(OperatorStats::getPhysicalWrittenDataSize) .mapToLong(DataSize::toBytes) .sum(); ImmutableSet.Builder<BlockedReason> builder = ImmutableSet.builder(); if (operator.getBlockedReason().isPresent()) { builder.add(operator.getBlockedReason().get()); succinctNanos(totalBlockedTime), blockedMonitor != null, builder.build(), rawInputDataSize.convertToMostSuccinctDataSize(), rawInputPositions, outputPositions, succinctBytes(physicalWrittenDataSize), ImmutableList.copyOf(transform(operatorContexts, OperatorContext::getOperatorStats)));
@Override public PlanNode visitSpatialJoin(SpatialJoinNode node, RewriteContext<Set<Symbol>> context) { Set<Symbol> requiredInputs = ImmutableSet.<Symbol>builder() .addAll(SymbolsExtractor.extractUnique(node.getFilter())) .addAll(context.get()) .build(); ImmutableSet.Builder<Symbol> leftInputs = ImmutableSet.builder(); node.getLeftPartitionSymbol().map(leftInputs::add); ImmutableSet.Builder<Symbol> rightInputs = ImmutableSet.builder(); node.getRightPartitionSymbol().map(rightInputs::add); PlanNode left = context.rewrite(node.getLeft(), leftInputs.addAll(requiredInputs).build()); PlanNode right = context.rewrite(node.getRight(), rightInputs.addAll(requiredInputs).build()); List<Symbol> outputSymbols = node.getOutputSymbols().stream() .filter(context.get()::contains) .distinct() .collect(toImmutableList()); return new SpatialJoinNode(node.getId(), node.getType(), left, right, outputSymbols, node.getFilter(), node.getLeftPartitionSymbol(), node.getRightPartitionSymbol(), node.getKdbTree()); }