private void preInitialize() { LOG.debug("Getting current trigger count for this component/task"); // get trigger count value from store Object result = windowStore.get(windowTriggerCountId); Integer currentCount = 0; if (result == null) { LOG.info("No current trigger count in windows store."); } else { currentCount = (Integer) result + 1; } windowStore.put(windowTriggerCountId, currentCount); triggerId.set(currentCount); }
List<String> triggerKeys = new ArrayList<>(); Iterable<String> allEntriesIterable = windowStore.getAllKeys(); for (String key : allEntriesIterable) { if (key.startsWith(windowTupleTaskId)) { Iterable<Object> attemptedTriggers = windowStore.get(attemptedTriggerKeys); for (Object attemptedTrigger : attemptedTriggers) { triggersToBeIgnored.addAll((List<Integer>) attemptedTrigger); Iterable<Object> triggerObjects = windowStore.get(triggerKeys); int i = 0; for (Object triggerObject : triggerObjects) {
@Override public void putAll(Collection<Entry> entries) { for (Entry entry : entries) { _put(entry.key, entry.value); } if (backingStore != null) { backingStore.putAll(entries); } }
@Override public void cleanup() { windowsStore.shutdown(); } }
public void onTuplesExpired(List<TridentBatchTuple> expiredTuples) { if (maxCachedTuplesSize != null) { currentCachedTuplesSize.addAndGet(-expiredTuples.size()); } List<String> keys = new ArrayList<>(); for (TridentBatchTuple expiredTuple : expiredTuples) { keys.add(tupleKey(expiredTuple)); } windowStore.removeAll(keys); }
private void _remove(String key) { Object oldValue = store.remove(key); if (oldValue != null) { decrementSize(); if (backingStore != null) { backingStore.remove(key); } } }
@Override public Object get(String key) { Object value = store.get(key); if (value == null && backingStore != null) { value = backingStore.get(key); } return value; }
@Override public void put(String key, Object value) { _put(key, value); if (backingStore != null) { backingStore.put(key, value); } }
@Override public Iterable<String> getAllKeys() { if (backingStore != null) { return backingStore.getAllKeys(); } final Enumeration<String> storeEnumeration = store.keys(); final Iterator<String> resultIterator = new Iterator<String>() { @Override public boolean hasNext() { return storeEnumeration.hasMoreElements(); } @Override public String next() { return storeEnumeration.nextElement(); } @Override public void remove() { throw new UnsupportedOperationException("remove operation is not supported as it is immutable."); } }; return new Iterable<String>() { @Override public Iterator<String> iterator() { return resultIterator; } }; }
@Override public void shutdown() { store.clear(); if (backingStore != null) { backingStore.shutdown(); } }
@Override public void removeAll(Collection<String> keys) { for (String key : keys) { _remove(key); } if (backingStore != null) { backingStore.removeAll(keys); } }
@Override public void remove(String key) { _remove(key); if (backingStore != null) { backingStore.remove(key); } }
public List<TridentTuple> getTridentTuples(List<TridentBatchTuple> tridentBatchTuples) { List<TridentTuple> resultTuples = new ArrayList<>(); List<String> keys = new ArrayList<>(); for (TridentBatchTuple tridentBatchTuple : tridentBatchTuples) { TridentTuple tuple = collectTridentTupleOrKey(tridentBatchTuple, keys); if (tuple != null) { resultTuples.add(tuple); } } if (keys.size() > 0) { Iterable<Object> storedTupleValues = windowStore.get(keys); for (Object storedTupleValue : storedTupleValues) { TridentTuple tridentTuple = freshOutputFactory.create((List<Object>) storedTupleValue); resultTuples.add(tridentTuple); } } return resultTuples; }
@Override public void put(String key, Object value) { _put(key, value); if(backingStore != null) { backingStore.put(key, value); } }
@Override public Iterable<String> getAllKeys() { if(backingStore != null) { return backingStore.getAllKeys(); } final Enumeration<String> storeEnumeration = store.keys(); final Iterator<String> resultIterator = new Iterator<String>() { @Override public boolean hasNext() { return storeEnumeration.hasMoreElements(); } @Override public String next() { return storeEnumeration.nextElement(); } @Override public void remove() { throw new UnsupportedOperationException("remove operation is not supported as it is immutable."); } }; return new Iterable<String>() { @Override public Iterator<String> iterator() { return resultIterator; } }; }
pendingTriggerIds = (List<Integer>) windowStore.get(inprocessTriggerKey(batchTxnId)); if (pendingTriggerIds != null) { for (Integer pendingTriggerId : pendingTriggerIds) { triggerKeys.add(triggerKey(pendingTriggerId)); triggerValues = windowStore.get(triggerKeys); windowStore.put(inprocessTriggerKey(batchTxnId), pendingTriggerIds);
List<String> triggerKeys = new ArrayList<>(); Iterable<String> allEntriesIterable = windowStore.getAllKeys(); for (String key : allEntriesIterable) { if (key.startsWith(windowTupleTaskId)) { Iterable<Object> attemptedTriggers = windowStore.get(attemptedTriggerKeys); for (Object attemptedTrigger : attemptedTriggers) { triggersToBeIgnored.addAll((List<Integer>) attemptedTrigger); Iterable<Object> triggerObjects = windowStore.get(triggerKeys); int i=0; for (Object triggerObject : triggerObjects) {
public void shutdown() { try { LOG.info("window manager [{}] is being shutdown", windowManager); windowManager.shutdown(); } finally { LOG.info("window store [{}] is being shutdown", windowStore); windowStore.shutdown(); } }
@Override public void updateState(WindowsState state, List<TridentTuple> tuples, TridentCollector collector) { Long currentTxId = state.getCurrentTxId(); LOG.debug("Removing triggers using WindowStateUpdater, txnId: [{}] ", currentTxId); for (TridentTuple tuple : tuples) { try { Object fieldValue = tuple.getValueByField(WindowTridentProcessor.TRIGGER_FIELD_NAME); if (!(fieldValue instanceof WindowTridentProcessor.TriggerInfo)) { throw new IllegalClassException(WindowTridentProcessor.TriggerInfo.class, fieldValue.getClass()); } WindowTridentProcessor.TriggerInfo triggerInfo = (WindowTridentProcessor.TriggerInfo) fieldValue; String triggerCompletedKey = WindowTridentProcessor.getWindowTriggerInprocessIdPrefix(triggerInfo.windowTaskId) + currentTxId; LOG.debug("Removing trigger key [{}] and trigger completed key [{}] from store: [{}]", triggerInfo, triggerCompletedKey, windowsStore); windowsStore.removeAll(Lists.newArrayList(triggerInfo.generateTriggerKey(), triggerCompletedKey)); } catch (Exception ex) { LOG.warn(ex.getMessage()); collector.reportError(ex); throw new FailedException(ex); } } }
public void addTuplesBatch(Object batchId, List<TridentTuple> tuples) { LOG.debug("Adding tuples to window-manager for batch: [{}]", batchId); List<WindowsStore.Entry> entries = new ArrayList<>(); for (int i = 0; i < tuples.size(); i++) { String key = keyOf(batchId); TridentTuple tridentTuple = tuples.get(i); entries.add(new WindowsStore.Entry(key + i, tridentTuple.select(inputFields))); } // tuples should be available in store before they are added to window manager windowStore.putAll(entries); for (int i = 0; i < tuples.size(); i++) { String key = keyOf(batchId); TridentTuple tridentTuple = tuples.get(i); addToWindowManager(i, key, tridentTuple); } }