private List<String> getSourceInterestList(List<InputSpec> inputSpecList) { List<String> sourcesOfInterest = Collections.emptyList(); if (inputSpecList != null) { boolean alreadyFound = false; for (InputSpec inputSpec : inputSpecList) { if (LlapTezUtils.isSourceOfInterest(inputSpec.getInputDescriptor().getClassName())) { if (!alreadyFound) { alreadyFound = true; sourcesOfInterest = new LinkedList<>(); } sourcesOfInterest.add(inputSpec.getSourceVertexName()); } } } return sourcesOfInterest; }
private static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) { IOSpecProto.Builder builder = IOSpecProto.newBuilder(); if (inputSpec.getSourceVertexName() != null) { builder.setConnectedVertexName(inputSpec.getSourceVertexName()); } if (inputSpec.getInputDescriptor() != null) { builder.setIoDescriptor(convertToProto(inputSpec.getInputDescriptor())); } builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount()); return builder.build(); }
private void verifyInputSpecAndProto(InputSpec inputSpec, IOSpecProto inputSpecProto) { assertEquals(inputSpec.getPhysicalEdgeCount(), inputSpecProto.getPhysicalEdgeCount()); assertEquals(inputSpec.getSourceVertexName(), inputSpecProto.getConnectedVertexName()); assertEquals(inputSpec.getInputDescriptor().getClassName(), inputSpecProto.getIoDescriptor().getClassName()); assertEquals(inputSpec.getInputDescriptor().getUserPayload().getPayload(), inputSpecProto.getIoDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer()); }
private List<String> getSourceInterestList(List<InputSpec> inputSpecList) { List<String> sourcesOfInterest = Collections.emptyList(); if (inputSpecList != null) { boolean alreadyFound = false; for (InputSpec inputSpec : inputSpecList) { if (LlapTezUtils.isSourceOfInterest(inputSpec.getInputDescriptor().getClassName())) { if (!alreadyFound) { alreadyFound = true; sourcesOfInterest = new LinkedList<>(); } sourcesOfInterest.add(inputSpec.getSourceVertexName()); } } } return sourcesOfInterest; }
private LogicalInput createInput(InputSpec inputSpec, InputContext inputContext) throws TezException { InputDescriptor inputDesc = inputSpec.getInputDescriptor(); Input input = ReflectionUtils.createClazzInstance(inputDesc.getClassName(), new Class[]{InputContext.class, Integer.TYPE}, new Object[]{inputContext, inputSpec.getPhysicalEdgeCount()}); if (!(input instanceof LogicalInput)) { throw new TezUncheckedException(inputDesc.getClass().getName() + " is not a sub-type of LogicalInput." + " Only LogicalInput sub-types supported by LogicalIOProcessor."); } return (LogicalInput) input; }
private InputContext createInputContext(Map<String, LogicalInput> inputMap, InputSpec inputSpec, int inputIndex) { InputContext inputContext = new TezInputContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), inputSpec.getSourceVertexName(), taskSpec.getVertexParallelism(), taskSpec.getTaskAttemptID(), inputIndex, inputSpec.getInputDescriptor().getUserPayload(), this, serviceConsumerMetadata, envMap, initialMemoryDistributor, inputSpec.getInputDescriptor(), inputMap, inputReadyTracker, objectRegistry, ExecutionContext, memAvailable, sharedExecutor); return inputContext; }