@Override public DatasetSplit apply(SplitWork input) { return input.getSplit(); }}).toList(); return new ElasticsearchSubScan(
@Override public SplitWork apply(DatasetSplit split) { return new SplitWork(split, nodeMap, affinityType); }}); }
public static Iterator<SplitWork> transform(final TableMetadata dataset, ExecutionNodeMap nodeMap, DistributionAffinity affinityType){ return transform(dataset.getSplits(), nodeMap, affinityType); }
@Override public int compareTo(CompleteWork o) { return Long.compare(getTotalBytes(), o.getTotalBytes()); }
@Override public Iterator<SplitWork> getSplits(ExecutionNodeMap nodeMap) { return SplitWork.transform(dataset.getSplits(), nodeMap, getDistributionAffinity()); }
@Override public HBaseSubScanSpec apply(SplitWork input) { return toSubScan(input.getSplit()); }}).toList();
private final Wrapper newSplitWrapper(double cost, int minWidth, int maxWidth, List<EndpointAffinity> epAffs, ExecutionNodeMap execNodeMap) { PhysicalOperator root = Mockito.mock(PhysicalOperator.class); Fragment fragment = Mockito.mock(Fragment.class); when(fragment.getRoot()).thenReturn(root); final Wrapper fragmentWrapper = new Wrapper(fragment, 1); final Stats stats = fragmentWrapper.getStats(); stats.addCost(cost); stats.addMinWidth(minWidth); stats.addMaxWidth(maxWidth); DatasetSplit dataSplit = new DatasetSplit(); Iterable<Affinity> affinities = Iterables.transform( epAffs, new Function<EndpointAffinity, Affinity>() { @Override public Affinity apply(@Nullable EndpointAffinity endpointAffinity) { return new Affinity() .setFactor(endpointAffinity.getAffinity()) .setHost(endpointAffinity.getEndpoint() .getAddress()); } } ); dataSplit.setAffinitiesList(Lists.newArrayList(affinities)); SplitWork splitWork = new SplitWork(dataSplit, execNodeMap, DistributionAffinity.HARD); GroupScan groupScan = Mockito.mock(GroupScan.class); when(groupScan.getDistributionAffinity()).thenReturn(DistributionAffinity.HARD); stats.addSplits(groupScan, ImmutableList.<CompleteWork>of(splitWork)); return fragmentWrapper; }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final BatchSchema schema = cachedRelDataType == null ? getDataset().getSchema(): BatchSchema.fromCalciteRowType(cachedRelDataType); // Create an abridged version of the splits to save network bytes. List<DatasetSplit> splits = work.stream().map( workSplit -> ProtostuffUtil.copy(workSplit.getSplit()) .setExtendedProperty(convertToScanXAttr(workSplit.getSplit().getExtendedProperty())) ).collect(Collectors.toList()); return new ParquetSubScan(dataset.getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), filter == null ? null : filter.getConditions(), dataset.getStoragePluginId(), columns, dataset.getReadDefinition().getPartitionColumnsList(), globalDictionaryEncodedColumns, dataset.getReadDefinition().getExtendedProperty()); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { List<DatasetSplit> splits = new ArrayList<>(work.size()); BatchSchema schema = getDataset().getSchema(); for(SplitWork split : work){ splits.add(split.getSplit()); } boolean storageImpersonationEnabled = dataset.getStoragePluginId().getCapabilities().getCapability(SourceCapabilities.STORAGE_IMPERSONATION); String userName = storageImpersonationEnabled ? getUserName() : ImpersonationUtil.getProcessUserName(); final ReadDefinition readDefinition = dataset.getReadDefinition(); return new HiveSubScan(splits, userName, schema, dataset.getName().getPathComponents(), filter, dataset.getStoragePluginId(), columns, readDefinition.getPartitionColumnsList()); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final List<DatasetSplit> splits = new ArrayList<>(work.size()); final BatchSchema schema = getDataset().getSchema(); for(SplitWork split : work){ splits.add(split.getSplit()); } return new EasySubScan( getDataset().getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), dataset.getStoragePluginId(), columns, getDataset().getReadDefinition().getPartitionColumnsList(), getDataset().getReadDefinition().getExtendedProperty()); }