static TaskStream fromTuple(Tuple input) { return new TaskStream(input.getSourceTask(), input.getSourceGlobalStreamid()); } }
@Override public void execute(Tuple tuple) { Object obj = tuple.getValue(0); long count = tuple.getLong(1); int source = tuple.getSourceTask(); Map<Integer, Long> subCounts = counts.get(obj); if (subCounts == null) { subCounts = new HashMap<Integer, Long>(); counts.put(obj, subCounts); } // Update the current count for this object subCounts.put(source, count); // Output the sum of all the known counts so for this key long sum = 0; for (Long val : subCounts.values()) { sum += val; } collector.emit(new Values(obj, sum)); }
private void processWatermark(Tuple input) { long watermark = ((Watermark) input.getValue(0)).getTimestamp(); // emit watermark to downstream tasks collector.emit(Common.WATERMARK_STREAM_ID, new Values(input.getValue(0))); Integer taskId = input.getSourceTask(); Long taskWatermark = upstreamWatermarks.get(taskId); if (taskWatermark == null) { upstreamWatermarks.put(taskId, watermark); } else if (watermark > taskWatermark) { upstreamWatermarks.put(taskId, watermark); } // todo: needs optimization to update watermark, a bit slow for now long minWatermark = Collections.min(upstreamWatermarks.values()); if (minWatermark > currentWatermark) { currentWatermark = minWatermark; LOG.debug("Updating current watermark to {}({})", currentWatermark, TimeUtils.format((int) (currentWatermark / 1000L))); } checkEventTimeWindows(); }
obj.put("comp", input.getSourceComponent()); obj.put("stream", input.getSourceStreamId()); obj.put("task", input.getSourceTask()); obj.put("tuple", input.getValues()); sendToSubprocess(obj.toString());
public byte[] serialize(Tuple tuple) { try { _outputter.reset(); WritableUtils.writeVInt(_dataOutputter, tuple.getSourceTask()); WritableUtils.writeVInt(_dataOutputter, _ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId())); tuple.getMessageId().serialize(_dataOutputter); _kryo.serializeInto(tuple.getValues(), _outputter); return _outputter.toByteArray(); } catch (IOException e) { throw new RuntimeException(e); } }
protected void processBatchTuple(Tuple batchEvent) { String stream = batchEvent.getSourceStreamId(); if (stream.equals(TransactionCommon.BARRIER_STREAM_ID)) { Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) batchEvent.getValue(0); BatchSnapshot snapshot = (BatchSnapshot) val.getSecond().get(0); if (currentBatchTracker.receiveBarrier(batchEvent.getSourceTask())) currentBatchTracker.expectedTupleCount += snapshot.getTupleCount(); else LOG.warn("Received expired or unexpected barrier={} from task-{}", snapshot, batchEvent.getSourceTask()); // LOG.debug("Received batch, stream={}, batchId={}, sourceTask={}, values={}", stream, currentBatchTracker.bactchId, // batchEvent.getSourceTask(), snapshot); LOG.debug("currentBatchTracker={}\n processingBatches={}\n PendingBatches={}\n intraPendingBatches={}", currentBatchTracker, processingBatches, batchCache, intraBatchCache); /* * if (currentBatchTracker.isAllBarriersReceived()) { if (currentBatchTracker.checkFinish()) finishCurrentBatch(); else * outputCollector.fail(batchEvent); } */ } else { for (Object value : batchEvent.getValues()) { Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value; TupleImplExt tuple = new TupleImplExt(topologyContext, val.getSecond(), val.getFirst(), ((TupleImplExt) batchEvent)); boltExecutor.execute(tuple); } currentBatchTracker.incrementReceivedCount(batchEvent.getValues().size()); } if (currentBatchTracker.checkFinish()) { finishCurrentBatch(); } }
public IPersistentMap meta() { if(_meta==null) { _meta = new PersistentArrayMap( new Object[] { makeKeyword("stream"), getSourceStreamId(), makeKeyword("component"), getSourceComponent(), makeKeyword("task"), getSourceTask()}); } return _meta; }
tradeCustomer = (TradeCustomer) input.getValue(1); } catch (Exception e) { LOG.error(input.getSourceComponent() + " " + input.getSourceTask() + " " + input.getSourceStreamId() + " target " + input); throw new RuntimeException(e);
int sourceTask = input.getSourceTask(); int uptime = (Integer) input.getValue(0); TaskStatus executorStatus = new TaskStatus();
private BoltMsg createBoltMessage(Tuple input, String genId) { BoltMsg boltMsg = new BoltMsg(); boltMsg.setId(genId); boltMsg.setComp(input.getSourceComponent()); boltMsg.setStream(input.getSourceStreamId()); boltMsg.setTask(input.getSourceTask()); boltMsg.setTuple(input.getValues()); return boltMsg; }
public void processTuple(Tuple tuple) { int taskId = tuple.getSourceTask();
LOG.info("Unknown source stream, " + stream_id + " from task-" + input.getSourceTask()); return;
output.writeInt(tuple.getSourceTask(), true); output.writeInt(_ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()), true);
static TaskStream fromTuple(Tuple input) { return new TaskStream(input.getSourceTask(), input.getSourceGlobalStreamid()); } }
public void execute(Tuple input) { if (_exception != null) { throw new RuntimeException(_exception); } //just need an id String genId = Long.toString(_rand.nextLong()); _inputs.put(genId, input); try { BoltMsg boltMsg = new BoltMsg(); boltMsg.setId(genId); boltMsg.setComp(input.getSourceComponent()); boltMsg.setStream(input.getSourceStreamId()); boltMsg.setTask(input.getSourceTask()); boltMsg.setTuple(input.getValues()); _pendingWrites.put(boltMsg); } catch(InterruptedException e) { throw new RuntimeException("Error during multilang processing", e); } }
public void process(Tuple input) { int sourceTask = input.getSourceTask(); int uptime = (Integer) input.getValue(0);
public void processBatchTuple(Tuple batchEvent) { String stream = batchEvent.getSourceStreamId(); if (stream.equals(TransactionCommon.BARRIER_STREAM_ID)) { Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) batchEvent.getValue(0); BatchSnapshot snapshot = (BatchSnapshot) val.getSecond().get(1); currentBatchTracker.receiveBarrier(batchEvent.getSourceTask()); currentBatchTracker.expectedTupleCount += snapshot.getTupleCount(); LOG.debug("Received batch, stream={}, batchGroupId={}, sourceTask={}, values={}", stream, currentBatchTracker.bactchGroupId, batchEvent.getSourceTask(), snapshot); LOG.debug("currentBatchTracker={}, processingBatches={}, pendingBatches={}", currentBatchTracker, processingBatches, batchCache); } else { for (Object value : batchEvent.getValues()) { /*List<Object> firstTupleValue = ((Pair<MessageId, List<Object>>) value).getSecond(); BatchGroupId batchGroupId = (BatchGroupId) firstTupleValue.get(0); if (!batchGroupId.equals(currentBatchTracker.bactchGroupId)) { LOG.warn("batchgroupid-{} is not equal to the once of current batch tracker-{}!", batchGroupId, currentBatchTracker.bactchGroupId); }*/ Pair<MessageId, List<Object>> val = (Pair<MessageId, List<Object>>) value; val.getSecond().remove(0); TupleImplExt tuple = new TupleImplExt(topologyContext, val.getSecond(), val.getFirst(), ((TupleImplExt) batchEvent)); boltExecutor.execute(tuple); } currentBatchTracker.incrementReceivedCount(batchEvent.getValues().size()); } if (currentBatchTracker.checkFinish()) { finishCurrentBatch(); } }
@Override public String format(Tuple tuple) { Fields schema = context.getComponentOutputFields(tuple.getSourceComponent(), tuple.getSourceStreamId()); String values = ""; for (int i=0; i<tuple.size(); i++) { if (i != 0) values += ", "; values += String.format("%s=%s", schema.get(i), tuple.getValue(i)); } return String.format(TEMPLATE, tuple.getSourceComponent(), tuple.getSourceTask(), tuple.getSourceStreamId(), tuple.getMessageId().toString(), values); }
public byte[] serialize(Tuple tuple) { try { _kryoOut.clear(); _kryoOut.writeInt(tuple.getSourceTask(), true); _kryoOut.writeInt(_ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()), true); tuple.getMessageId().serialize(_kryoOut); _kryo.serializeInto(tuple.getValues(), _kryoOut); return _kryoOut.toBytes(); } catch (IOException e) { throw new RuntimeException(e); } }
private BoltMsg createBoltMessage(Tuple input, String genId) { BoltMsg boltMsg = new BoltMsg(); boltMsg.setId(genId); boltMsg.setComp(input.getSourceComponent()); boltMsg.setStream(input.getSourceStreamId()); boltMsg.setTask(input.getSourceTask()); boltMsg.setTuple(input.getValues()); return boltMsg; }