public void addTuplesBatch(Object batchId, List<TridentTuple> tuples) { LOG.debug("Adding tuples to window-manager for batch: [{}]", batchId); List<WindowsStore.Entry> entries = new ArrayList<>(); for (int i = 0; i < tuples.size(); i++) { String key = keyOf(batchId); TridentTuple tridentTuple = tuples.get(i); entries.add(new WindowsStore.Entry(key + i, tridentTuple.select(inputFields))); } // tuples should be available in store before they are added to window manager windowStore.putAll(entries); for (int i = 0; i < tuples.size(); i++) { String key = keyOf(batchId); TridentTuple tridentTuple = tuples.get(i); addToWindowManager(i, key, tridentTuple); } }
public List<TridentTuple> getTridentTuples(List<TridentBatchTuple> tridentBatchTuples) { List<TridentTuple> resultTuples = new ArrayList<>(); List<String> keys = new ArrayList<>(); for (TridentBatchTuple tridentBatchTuple : tridentBatchTuples) { TridentTuple tuple = collectTridentTupleOrKey(tridentBatchTuple, keys); if (tuple != null) { resultTuples.add(tuple); } } if (keys.size() > 0) { Iterable<Object> storedTupleValues = windowStore.get(keys); for (Object storedTupleValue : storedTupleValues) { TridentTuple tridentTuple = freshOutputFactory.create((List<Object>) storedTupleValue); resultTuples.add(tridentTuple); } } return resultTuples; }
public String keyOf(Object batchId) { return windowTupleTaskId + getBatchTxnId(batchId) + WindowsStore.KEY_SEPARATOR; }
for (String key : allEntriesIterable) { if (key.startsWith(windowTupleTaskId)) { int tupleIndexValue = lastPart(key); String batchId = secondLastPart(key); LOG.debug("Received tuple with batch [{}] and tuple index [{}]", batchId, tupleIndexValue); windowManager.add(new TridentBatchTuple(batchId, System.currentTimeMillis(), tupleIndexValue)); int i = 0; for (Object triggerObject : triggerObjects) { int id = lastPart(triggerKeys.get(i++)); if (!triggersToBeIgnored.contains(id)) { LOG.info("Adding pending trigger value [{}]", triggerObject);
public TridentTuple collectTridentTupleOrKey(TridentBatchTuple tridentBatchTuple, List<String> keys) { if (tridentBatchTuple.tridentTuple != null) { return tridentBatchTuple.tridentTuple; } keys.add(tupleKey(tridentBatchTuple)); return null; }
@Override public void prepare(Map<String, Object> topoConf, TopologyContext context, TridentContext tridentContext) { this.topologyContext = context; List<TridentTuple.Factory> parents = tridentContext.getParentTupleFactories(); if (parents.size() != 1) { throw new RuntimeException("Aggregation related operation can only have one parent"); } Long maxTuplesCacheSize = getWindowTuplesCacheSize(topoConf); this.tridentContext = tridentContext; collector = new FreshCollector(tridentContext); projection = new TridentTupleView.ProjectionFactory(parents.get(0), inputFields); windowStore = windowStoreFactory.create(topoConf); windowTaskId = windowId + WindowsStore.KEY_SEPARATOR + topologyContext.getThisTaskId() + WindowsStore.KEY_SEPARATOR; windowTriggerInprocessId = getWindowTriggerInprocessIdPrefix(windowTaskId); tridentWindowManager = storeTuplesInStore ? new StoreBasedTridentWindowManager(windowConfig, windowTaskId, windowStore, aggregator, tridentContext.getDelegateCollector(), maxTuplesCacheSize, inputFields) : new InMemoryTridentWindowManager(windowConfig, windowTaskId, windowStore, aggregator, tridentContext.getDelegateCollector()); tridentWindowManager.prepare(); }
for (String key : allEntriesIterable) { if (key.startsWith(windowTupleTaskId)) { int tupleIndexValue = lastPart(key); String batchId = secondLastPart(key); LOG.debug("Received tuple with batch [{}] and tuple index [{}]", batchId, tupleIndexValue); windowManager.add(new TridentBatchTuple(batchId, System.currentTimeMillis(), tupleIndexValue)); int i=0; for (Object triggerObject : triggerObjects) { int id = lastPart(triggerKeys.get(i++)); if(!triggersToBeIgnored.contains(id)) { LOG.info("Adding pending trigger value [{}]", triggerObject);
public void onTuplesExpired(List<TridentBatchTuple> expiredTuples) { if (maxCachedTuplesSize != null) { currentCachedTuplesSize.addAndGet(-expiredTuples.size()); } List<String> keys = new ArrayList<>(); for (TridentBatchTuple expiredTuple : expiredTuples) { keys.add(tupleKey(expiredTuple)); } windowStore.removeAll(keys); }
@Override public void prepare(Map stormConf, TopologyContext context, TridentContext tridentContext) { this.topologyContext = context; List<TridentTuple.Factory> parents = tridentContext.getParentTupleFactories(); if (parents.size() != 1) { throw new RuntimeException("Aggregation related operation can only have one parent"); } Long maxTuplesCacheSize = getWindowTuplesCacheSize(stormConf); this.tridentContext = tridentContext; collector = new FreshCollector(tridentContext); projection = new TridentTupleView.ProjectionFactory(parents.get(0), inputFields); windowStore = windowStoreFactory.create(stormConf); windowTaskId = windowId + WindowsStore.KEY_SEPARATOR + topologyContext.getThisTaskId() + WindowsStore.KEY_SEPARATOR; windowTriggerInprocessId = getWindowTriggerInprocessIdPrefix(windowTaskId); tridentWindowManager = storeTuplesInStore ? new StoreBasedTridentWindowManager(windowConfig, windowTaskId, windowStore, aggregator, tridentContext.getDelegateCollector(), maxTuplesCacheSize, inputFields) : new InMemoryTridentWindowManager(windowConfig, windowTaskId, windowStore, aggregator, tridentContext.getDelegateCollector()); tridentWindowManager.prepare(); }
public void addTuplesBatch(Object batchId, List<TridentTuple> tuples) { LOG.debug("Adding tuples to window-manager for batch: [{}]", batchId); List<WindowsStore.Entry> entries = new ArrayList<>(); for (int i = 0; i < tuples.size(); i++) { String key = keyOf(batchId); TridentTuple tridentTuple = tuples.get(i); entries.add(new WindowsStore.Entry(key+i, tridentTuple.select(inputFields))); } // tuples should be available in store before they are added to window manager windowStore.putAll(entries); for (int i = 0; i < tuples.size(); i++) { String key = keyOf(batchId); TridentTuple tridentTuple = tuples.get(i); addToWindowManager(i, key, tridentTuple); } }
public TridentTuple collectTridentTupleOrKey(TridentBatchTuple tridentBatchTuple, List<String> keys) { if (tridentBatchTuple.tridentTuple != null) { return tridentBatchTuple.tridentTuple; } keys.add(tupleKey(tridentBatchTuple)); return null; }
public String keyOf(Object batchId) { return windowTupleTaskId + getBatchTxnId(batchId) + WindowsStore.KEY_SEPARATOR; }
public List<TridentTuple> getTridentTuples(List<TridentBatchTuple> tridentBatchTuples) { List<TridentTuple> resultTuples = new ArrayList<>(); List<String> keys = new ArrayList<>(); for (TridentBatchTuple tridentBatchTuple : tridentBatchTuples) { TridentTuple tuple = collectTridentTupleOrKey(tridentBatchTuple, keys); if(tuple != null) { resultTuples.add(tuple); } } if(keys.size() > 0) { Iterable<Object> storedTupleValues = windowStore.get(keys); for (Object storedTupleValue : storedTupleValues) { TridentTuple tridentTuple = freshOutputFactory.create((List<Object>) storedTupleValue); resultTuples.add(tridentTuple); } } return resultTuples; }
public void onTuplesExpired(List<TridentBatchTuple> expiredTuples) { if (maxCachedTuplesSize != null) { currentCachedTuplesSize.addAndGet(-expiredTuples.size()); } List<String> keys = new ArrayList<>(); for (TridentBatchTuple expiredTuple : expiredTuples) { keys.add(tupleKey(expiredTuple)); } windowStore.removeAll(keys); }