/** * Gets the unique ID for the next operator in the graph. The ID is of the following format: * jobName-jobId-opCode-nextOpNum; * * @param opCode the {@link OpCode} of the next operator * @return the unique ID for the next operator in the graph */ public String getNextOpId(OpCode opCode) { return getNextOpId(opCode, null); }
/** * Gets the unique ID for the next operator in the graph. The ID is of the following format: * jobName-jobId-opCode-nextOpNum; * * @param opCode the {@link OpCode} of the next operator * @return the unique ID for the next operator in the graph */ public String getNextOpId(OpCode opCode) { return getNextOpId(opCode, null); }
@Override public void sendTo(OutputStream<M> outputStream) { String opId = this.streamAppDesc.getNextOpId(OpCode.SEND_TO); OutputOperatorSpec<M> op = OperatorSpecs.createSendToOperatorSpec( (OutputStreamImpl<M>) outputStream, opId); this.operatorSpec.registerNextOperatorSpec(op); }
@Override public void sink(SinkFunction<? super M> sinkFn) { String opId = this.streamAppDesc.getNextOpId(OpCode.SINK); SinkOperatorSpec<M> op = OperatorSpecs.createSinkOperatorSpec(sinkFn, opId); this.operatorSpec.registerNextOperatorSpec(op); }
@Override public void sendTo(OutputStream<M> outputStream) { String opId = this.streamAppDesc.getNextOpId(OpCode.SEND_TO); OutputOperatorSpec<M> op = OperatorSpecs.createSendToOperatorSpec( (OutputStreamImpl<M>) outputStream, opId); this.operatorSpec.registerNextOperatorSpec(op); }
@Override public void sendTo(OutputStream<M> outputStream) { String opId = this.streamAppDesc.getNextOpId(OpCode.SEND_TO); OutputOperatorSpec<M> op = OperatorSpecs.createSendToOperatorSpec( (OutputStreamImpl<M>) outputStream, opId); this.operatorSpec.registerNextOperatorSpec(op); }
@Override public MessageStream<M> filter(FilterFunction<? super M> filterFn) { String opId = this.streamAppDesc.getNextOpId(OpCode.FILTER); StreamOperatorSpec<M, M> op = OperatorSpecs.createFilterOperatorSpec(filterFn, opId); this.operatorSpec.registerNextOperatorSpec(op); return new MessageStreamImpl<>(this.streamAppDesc, op); }
@Override public MessageStream<M> merge(Collection<? extends MessageStream<? extends M>> otherStreams) { if (otherStreams.isEmpty()) return this; String opId = this.streamAppDesc.getNextOpId(OpCode.MERGE); StreamOperatorSpec<M, M> op = OperatorSpecs.createMergeOperatorSpec(opId); this.operatorSpec.registerNextOperatorSpec(op); otherStreams.forEach(other -> ((MessageStreamImpl<M>) other).getOperatorSpec().registerNextOperatorSpec(op)); return new MessageStreamImpl<>(this.streamAppDesc, op); }
@Override public <K, V> void sendTo(Table<KV<K, V>> table) { String opId = this.streamAppDesc.getNextOpId(OpCode.SEND_TO); SendToTableOperatorSpec<K, V> op = OperatorSpecs.createSendToTableOperatorSpec(((TableImpl) table).getTableSpec(), opId); this.operatorSpec.registerNextOperatorSpec(op); }
@Override public MessageStream<M> merge(Collection<? extends MessageStream<? extends M>> otherStreams) { if (otherStreams.isEmpty()) return this; String opId = this.streamAppDesc.getNextOpId(OpCode.MERGE); StreamOperatorSpec<M, M> op = OperatorSpecs.createMergeOperatorSpec(opId); this.operatorSpec.registerNextOperatorSpec(op); otherStreams.forEach(other -> ((MessageStreamImpl<M>) other).getOperatorSpec().registerNextOperatorSpec(op)); return new MessageStreamImpl<>(this.streamAppDesc, op); }
@Override public <K, WV> MessageStream<WindowPane<K, WV>> window(Window<M, K, WV> window, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.WINDOW, userDefinedId); OperatorSpec<M, WindowPane<K, WV>> op = OperatorSpecs.createWindowOperatorSpec((WindowInternal<M, K, WV>) window, opId); this.operatorSpec.registerNextOperatorSpec(op); return new MessageStreamImpl<>(this.streamAppDesc, op); }
@Override public <TM> MessageStream<TM> map(MapFunction<? super M, ? extends TM> mapFn) { String opId = this.streamAppDesc.getNextOpId(OpCode.MAP); StreamOperatorSpec<M, TM> op = OperatorSpecs.createMapOperatorSpec(mapFn, opId); this.operatorSpec.registerNextOperatorSpec(op); return new MessageStreamImpl<>(this.streamAppDesc, op); }
@Override public MessageStream<M> merge(Collection<? extends MessageStream<? extends M>> otherStreams) { if (otherStreams.isEmpty()) return this; String opId = this.streamAppDesc.getNextOpId(OpCode.MERGE); StreamOperatorSpec<M, M> op = OperatorSpecs.createMergeOperatorSpec(opId); this.operatorSpec.registerNextOperatorSpec(op); otherStreams.forEach(other -> ((MessageStreamImpl<M>) other).getOperatorSpec().registerNextOperatorSpec(op)); return new MessageStreamImpl<>(this.streamAppDesc, op); }
@Override public <K, V> void sendTo(Table<KV<K, V>> table) { String opId = this.streamAppDesc.getNextOpId(OpCode.SEND_TO); SendToTableOperatorSpec<K, V> op = OperatorSpecs.createSendToTableOperatorSpec(((TableImpl) table).getTableSpec(), opId); this.operatorSpec.registerNextOperatorSpec(op); }
@Override public <TM> MessageStream<TM> flatMap(FlatMapFunction<? super M, ? extends TM> flatMapFn) { String opId = this.streamAppDesc.getNextOpId(OpCode.FLAT_MAP); StreamOperatorSpec<M, TM> op = OperatorSpecs.createFlatMapOperatorSpec(flatMapFn, opId); this.operatorSpec.registerNextOperatorSpec(op); return new MessageStreamImpl<>(this.streamAppDesc, op); }
@Override public <K, WV> MessageStream<WindowPane<K, WV>> window(Window<M, K, WV> window, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.WINDOW, userDefinedId); OperatorSpec<M, WindowPane<K, WV>> op = OperatorSpecs.createWindowOperatorSpec((WindowInternal<M, K, WV>) window, opId); this.operatorSpec.registerNextOperatorSpec(op); return new MessageStreamImpl<>(this.streamAppDesc, op); }
@Override public <K, V> void sendTo(Table<KV<K, V>> table) { String opId = this.streamAppDesc.getNextOpId(OpCode.SEND_TO); SendToTableOperatorSpec<K, V> op = OperatorSpecs.createSendToTableOperatorSpec(((TableImpl) table).getTableId(), opId); this.operatorSpec.registerNextOperatorSpec(op); }
@Override public MessageStream<M> broadcast(Serde<M> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.BROADCAST, userDefinedId); IntermediateMessageStreamImpl<M> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, true); BroadcastOperatorSpec<M> broadcastOperatorSpec = OperatorSpecs.createBroadCastOperatorSpec(intermediateStream.getOutputStream(), opId); this.operatorSpec.registerNextOperatorSpec(broadcastOperatorSpec); return intermediateStream; }
@Override public <K, R extends KV, JM> MessageStream<JM> join(Table<R> table, StreamTableJoinFunction<? extends K, ? super M, ? super R, ? extends JM> joinFn) { String opId = this.streamAppDesc.getNextOpId(OpCode.JOIN); TableSpec tableSpec = ((TableImpl) table).getTableSpec(); StreamTableJoinOperatorSpec<K, M, R, JM> joinOpSpec = OperatorSpecs.createStreamTableJoinOperatorSpec( tableSpec, (StreamTableJoinFunction<K, M, R, JM>) joinFn, opId); this.operatorSpec.registerNextOperatorSpec(joinOpSpec); return new MessageStreamImpl<>(this.streamAppDesc, joinOpSpec); }
@Test(expected = SamzaException.class) public void testGetNextOpIdRejectsDuplicates() { HashMap<String, String> configMap = new HashMap<>(); configMap.put(JobConfig.JOB_NAME(), "jobName"); configMap.put(JobConfig.JOB_ID(), "1234"); Config config = new MapConfig(configMap); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, config); assertEquals("jobName-1234-join-customName", streamAppDesc.getNextOpId(OpCode.JOIN, "customName")); streamAppDesc.getNextOpId(OpCode.JOIN, "customName"); // should throw }