case BROADCAST_EDGE: UnorderedKVEdgeConfig et1Conf = UnorderedKVEdgeConfig .newBuilder(keyClass, valClass) .setFromConfiguration(conf) .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null) case ONE_TO_ONE_EDGE: UnorderedKVEdgeConfig et4Conf = UnorderedKVEdgeConfig .newBuilder(keyClass, valClass) .setFromConfiguration(conf) .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null)
case BROADCAST_EDGE: UnorderedKVEdgeConfig et1Conf = UnorderedKVEdgeConfig .newBuilder(keyClass, valClass) .setFromConfiguration(conf) .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null)
private DAG createDAG(int numGenTasks, int totalSourceDataSize, int numFetcherTasks) { int bytesPerSource = totalSourceDataSize / numGenTasks; LOG.info("DataPerSourceTask(bytes)=" + bytesPerSource); ByteBuffer payload = ByteBuffer.allocate(4); payload.putInt(0, bytesPerSource); Vertex broadcastVertex = Vertex.create("DataGen", ProcessorDescriptor.create(InputGenProcessor.class.getName()) .setUserPayload(UserPayload.create(payload)), numGenTasks); Vertex fetchVertex = Vertex.create("FetchVertex", ProcessorDescriptor.create(InputFetchProcessor.class.getName()), numFetcherTasks); UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig.newBuilder(NullWritable.class .getName(), IntWritable.class.getName()).setCompression(false, null, null).build(); DAG dag = DAG.create("BroadcastLoadGen"); dag.addVertex(broadcastVertex).addVertex(fetchVertex).addEdge( Edge.create(broadcastVertex, fetchVertex, edgeConf.createDefaultBroadcastEdgeProperty())); return dag; }
case BROADCAST_EDGE: UnorderedKVEdgeConfig et1Conf = UnorderedKVEdgeConfig .newBuilder(keyClass, valClass) .setFromConfiguration(conf) .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null)
.newBuilder(Text.class.getName(), TextLongPair.class.getName()) .setFromConfiguration(tezConf).build();
} else { UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName()).build(); cpEdgeProperty = edgeConf.createDefaultCustomEdgeProperty(cpEdgeManager); UnorderedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName()).build(); broadcastEdgeProperty = broadcastEdgeConf.createDefaultBroadcastEdgeProperty();
.newBuilder(Text.class.getName(), IntWritable.class.getName()) .setFromConfiguration(tezConf).build();
.newBuilder(Text.class.getName(), TextLongPair.class.getName()) .setFromConfiguration(tezConf).build();
.newBuilder(Text.class.getName(), NullWritable.class.getName()) .setFromConfiguration(tezConf) .build();