public static List<RootInputLeafOutput< InputDescriptor, InputInitializerDescriptor>> getVertexInputs(Vertex wx) { return wx.getInputs(); } }
private static void setUserPayload(EntityDescriptor<?> entity, UserPayload payload) { if (payload != null) { entity.setUserPayload(payload); } }
@Override public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return context.getDestinationVertexNumTasks(); }
@Override public void onVertexStarted(Map<String, List<Integer>> completions) { int numTasks = context.getVertexNumTasks(context.getVertexName()); List<VertexManagerPluginContext.TaskWithLocationHint> scheduledTasks = new ArrayList<VertexManagerPluginContext.TaskWithLocationHint>(numTasks); for (int i = 0; i < numTasks; ++i) { scheduledTasks.add(new VertexManagerPluginContext.TaskWithLocationHint(new Integer(i), null)); } context.scheduleVertexTasks(scheduledTasks); }
private void setupQuickStart(TezEdgeProperty edgeProp, Vertex v) throws IOException { if (!edgeProp.isSlowStart()) { Configuration pluginConf = new Configuration(false); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName()); pluginConf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0); pluginConf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, 0); UserPayload payload = TezUtils.createUserPayloadFromConf(pluginConf); desc.setUserPayload(payload); v.setVertexManagerPlugin(desc); } }
private long hiveInputRecordsFromOtherVertices(String vertexName) { List<Vertex> inputVerticesList = dag.getVertex(vertexName).getInputVertices(); long result = 0; for (Vertex inputVertex : inputVerticesList) { long inputVertexRecords = hiveInputRecordsFromTezCounters(vertexName, inputVertex.getName()); if (inputVertexRecords < 0) { inputVertexRecords = hiveInputRecordsFromHiveCounters(inputVertex.getName()); } result += inputVertexRecords; } return result; }
private static ProcessorDescriptor convertProcessorDescriptorFromProto( EntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertPayloadFromProto(proto); ProcessorDescriptor pd = ProcessorDescriptor.create(className); setUserPayload(pd, payload); return pd; }
private static OutputDescriptor convertOutputDescriptorFromProto( EntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertPayloadFromProto(proto); OutputDescriptor od = OutputDescriptor.create(className); setUserPayload(od, payload); return od; }
private static InputDescriptor convertInputDescriptorFromProto( EntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertPayloadFromProto(proto); InputDescriptor id = InputDescriptor.create(className); setUserPayload(id, payload); return id; }
private VertexExecutionContext createVertexExecutionContext(BaseWork work) { VertexExecutionContext vertexExecutionContext = VertexExecutionContext.createExecuteInContainers(true); if (work.getLlapMode()) { vertexExecutionContext = VertexExecutionContext .create(TezSessionState.LLAP_SERVICE, TezSessionState.LLAP_SERVICE, TezSessionState.LLAP_SERVICE); } if (work.getUberMode()) { vertexExecutionContext = VertexExecutionContext.createExecuteInAm(true); } return vertexExecutionContext; }
@Override public Edge answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); return Edge.create((Vertex)args[1], (Vertex)args[2], mock(EdgeProperty.class)); } });
@Override public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return context.getSourceVertexNumTasks(); }
/** * Adds any necessary resources that must be localized in each vertex to the DAG. */ void addExtraResourcesToDag(TezSessionState session, DAG dag, String[] inputOutputJars, Map<String,LocalResource> inputOutputLocalResources) throws Exception { if (!session.hasResources(inputOutputJars)) { if (null != inputOutputLocalResources) { dag.addTaskLocalFiles(inputOutputLocalResources); } } }
@Override public void onVertexStarted(Map<String, List<Integer>> completions) { int numTasks = context.getVertexNumTasks(context.getVertexName()); List<VertexManagerPluginContext.TaskWithLocationHint> scheduledTasks = new ArrayList<VertexManagerPluginContext.TaskWithLocationHint>(numTasks); for (int i = 0; i < numTasks; ++i) { scheduledTasks.add(new VertexManagerPluginContext.TaskWithLocationHint(new Integer(i), null)); } context.scheduleVertexTasks(scheduledTasks); }
private long hiveInputRecordsFromOtherVertices(String vertexName) { List<Vertex> inputVerticesList = dag.getVertex(vertexName).getInputVertices(); long result = 0; for (Vertex inputVertex : inputVerticesList) { long inputVertexRecords = hiveInputRecordsFromTezCounters(vertexName, inputVertex.getName()); if (inputVertexRecords < 0) { inputVertexRecords = hiveInputRecordsFromHiveCounters(inputVertex.getName()); } result += inputVertexRecords; } return result; }
private VertexExecutionContext createVertexExecutionContext(BaseWork work) { VertexExecutionContext vertexExecutionContext = VertexExecutionContext.createExecuteInContainers(true); if (work.getLlapMode()) { vertexExecutionContext = VertexExecutionContext .create(TezSessionState.LLAP_SERVICE, TezSessionState.LLAP_SERVICE, TezSessionState.LLAP_SERVICE); } if (work.getUberMode()) { vertexExecutionContext = VertexExecutionContext.createExecuteInAm(true); } return vertexExecutionContext; }
@Override public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return context.getDestinationVertexNumTasks(); }
@Override public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return context.getSourceVertexNumTasks(); }
public static List<RootInputLeafOutput< InputDescriptor, InputInitializerDescriptor>> getVertexInputs(Vertex wx) { return wx.getInputs(); } }
@Override public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, Map<Integer, List<Integer>> mapDestTaskIndices) { List<Integer> outputIndices = Collections.singletonList(sourceTaskIndex); for (int i = 0; i < context.getDestinationVertexNumTasks(); i++) { mapDestTaskIndices.put(i, outputIndices); } }