private void setupAutoReducerParallelism(TezEdgeProperty edgeProp, Vertex v) throws IOException { if (edgeProp.isAutoReduce()) { Configuration pluginConf = new Configuration(false); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName()); pluginConf.setBoolean( ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL, true); pluginConf.setInt(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM, edgeProp.getMinReducer()); pluginConf.setLong( ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE, edgeProp.getInputSizePerReducer()); UserPayload payload = TezUtils.createUserPayloadFromConf(pluginConf); desc.setUserPayload(payload); v.setVertexManagerPlugin(desc); } }
VertexManagerPluginDescriptor.create(NoOpVertexManager.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap(out.toByteArray()))), dagUgi, this, appContext, stateChangeNotifier); return; .getVertexManagerPlugin()); LOG.info("Setting user vertex manager plugin: " + pluginDesc.getClassName() + " on vertex: " + getLogIdentifier()); vertexManager = new VertexManager(pluginDesc, dagUgi, this, appContext, stateChangeNotifier); } else { + logIdentifier); vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()), dagUgi, this, appContext, stateChangeNotifier); } else if (hasBipartite && !hasCustom) { + logIdentifier); vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(ImmediateStartVertexManager.class.getName()), dagUgi, this, appContext, stateChangeNotifier);
public static VertexManagerPluginDescriptor convertVertexManagerPluginDescriptorFromDAGPlan( TezEntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertTezUserPayloadFromDAGPlan(proto); VertexManagerPluginDescriptor vmpd = VertexManagerPluginDescriptor.create(className); setUserPayload(vmpd, payload); return vmpd; }
public VertexManager(VertexManagerPluginDescriptor pluginDesc, UserGroupInformation dagUgi, Vertex managedVertex, AppContext appContext, StateChangeNotifier stateChangeNotifier) throws TezException { checkNotNull(pluginDesc, "pluginDesc is null"); checkNotNull(managedVertex, "managedVertex is null"); checkNotNull(appContext, "appContext is null"); checkNotNull(stateChangeNotifier, "notifier is null"); this.pluginDesc = pluginDesc; this.dagUgi = dagUgi; this.managedVertex = managedVertex; this.appContext = appContext; this.stateChangeNotifier = stateChangeNotifier; // don't specify the size of rootInputInitEventQueue, otherwise it will fail when addAll this.rootInputInitEventQueue = new LinkedBlockingQueue<TezEvent>(); pluginContext = new VertexManagerPluginContextImpl(); Preconditions.checkArgument(pluginDesc != null); payload = pluginDesc.getUserPayload(); pluginFailed = new AtomicBoolean(false); plugin = ReflectionUtils.createClazzInstance(pluginDesc.getClassName(), new Class[] { VertexManagerPluginContext.class }, new Object[] { pluginContext }); execService = appContext.getExecService(); eventQueue = new LinkedBlockingQueue<VertexManagerEvent>(); eventInFlight = new AtomicBoolean(false); }
public static VertexManagerPluginDescriptor create(String className) { return new VertexManagerPluginDescriptor(className); } }
OneToOneProcessor.class.getName()).setUserPayload(procPayload)); oneToOneVertex.setVertexManagerPlugin( VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()));
private void setupQuickStart(TezEdgeProperty edgeProp, Vertex v) throws IOException { if (!edgeProp.isSlowStart()) { Configuration pluginConf = new Configuration(false); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName()); pluginConf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0); pluginConf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, 0); UserPayload payload = TezUtils.createUserPayloadFromConf(pluginConf); desc.setUserPayload(payload); v.setVertexManagerPlugin(desc); } }
private void setupAutoReducerParallelism(TezEdgeProperty edgeProp, Vertex v) throws IOException { if (edgeProp.isAutoReduce()) { Configuration pluginConf = new Configuration(false); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName()); pluginConf.setBoolean( ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL, true); pluginConf.setInt(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM, edgeProp.getMinReducer()); pluginConf.setLong( ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE, edgeProp.getInputSizePerReducer()); UserPayload payload = TezUtils.createUserPayloadFromConf(pluginConf); desc.setUserPayload(payload); v.setVertexManagerPlugin(desc); } }
DataOutputBuffer dob = new DataOutputBuffer(); vertexConf.write(dob); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create( CustomPartitionVertex.class.getName()); byte[] userPayloadBytes = dob.getData(); ByteBuffer userPayload = ByteBuffer.wrap(userPayloadBytes); desc.setUserPayload(UserPayload.create(userPayload)); w.setVertexManagerPlugin(desc); break;
DataOutputBuffer dob = new DataOutputBuffer(); vertexConf.write(dob); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create( CustomPartitionVertex.class.getName()); byte[] userPayloadBytes = dob.getData(); ByteBuffer userPayload = ByteBuffer.wrap(userPayloadBytes); desc.setUserPayload(UserPayload.create(userPayload)); w.setVertexManagerPlugin(desc); break;
vertexConf.write(dob); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(CustomPartitionVertex.class.getName()); byte[] userPayloadBytes = dob.getData(); ByteBuffer userPayload = ByteBuffer.wrap(userPayloadBytes); desc.setUserPayload(UserPayload.create(userPayload)); w.setVertexManagerPlugin(desc); break;
vertexConf.write(dob); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(CustomPartitionVertex.class.getName()); byte[] userPayloadBytes = dob.getData(); ByteBuffer userPayload = ByteBuffer.wrap(userPayloadBytes); desc.setUserPayload(UserPayload.create(userPayload)); w.setVertexManagerPlugin(desc); break;
VertexManagerPluginDescriptor.create(CustomPartitionVertex.class.getName()); vertexConf.write(dob); byte[] userPayload = dob.getData(); desc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); mergeVx.setVertexManagerPlugin(desc); return mergeVx;
VertexManagerPluginDescriptor.create(CustomPartitionVertex.class.getName()); vertexConf.write(dob); byte[] userPayload = dob.getData(); desc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); mergeVx.setVertexManagerPlugin(desc); return mergeVx;
CartesianProductConfig cpConfig = new CartesianProductConfig(crossProductSources); v.setVertexManagerPlugin( VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName()) .setUserPayload(cpConfig.toUserPayload(new TezConfiguration(conf))));
public VertexManagerPluginDescriptor build() { VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create( FairShuffleVertexManager.class.getName()); try { return desc.setUserPayload(TezUtils.createUserPayloadFromConf( this.conf)); } catch (IOException e) { throw new TezUncheckedException(e); } } }
public VertexManagerPluginDescriptor build() { VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create( RootInputVertexManager.class.getName()); try { return desc.setUserPayload(TezUtils .createUserPayloadFromConf(this.conf)); } catch (IOException e) { throw new TezUncheckedException(e); } } }
public VertexManagerPluginDescriptor build() { VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create( ShuffleVertexManager.class.getName()); try { return desc.setUserPayload(TezUtils.createUserPayloadFromConf(this.conf)); } catch (IOException e) { throw new TezUncheckedException(e); } } }
private void setupAutoReducerParallelism(TezEdgeProperty edgeProp, Vertex v) throws IOException { if (edgeProp.isAutoReduce()) { Configuration pluginConf = new Configuration(false); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName()); pluginConf.setBoolean( ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL, true); pluginConf.setInt(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM, edgeProp.getMinReducer()); pluginConf.setLong( ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE, edgeProp.getInputSizePerReducer()); UserPayload payload = TezUtils.createUserPayloadFromConf(pluginConf); desc.setUserPayload(payload); v.setVertexManagerPlugin(desc); } }
DataOutputBuffer dob = new DataOutputBuffer(); vertexConf.write(dob); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create( CustomPartitionVertex.class.getName()); byte[] userPayloadBytes = dob.getData(); ByteBuffer userPayload = ByteBuffer.wrap(userPayloadBytes); desc.setUserPayload(UserPayload.create(userPayload)); w.setVertexManagerPlugin(desc); break;