@SuppressWarnings("deprecation") private static <T extends AutoCloseable> TimeCacheMap<String, T> fileCacheMap(Map<String, Object> conf) { return new TimeCacheMap<>(ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_FILE_COPY_EXPIRATION_SECS), 600), (id, stream) -> { try { stream.close(); } catch (Exception e) { throw new RuntimeException(e); } }); }
/** * Constructs a TimeCacheMap instance with a blobstore timeout and no callback function. * * @param conf the config to use * @return the newly created TimeCacheMap */ @SuppressWarnings("deprecation") private static TimeCacheMap<String, Iterator<String>> makeBlobListCacheMap(Map<String, Object> conf) { return new TimeCacheMap<>(ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_BLOBSTORE_EXPIRATION_SECS), 600)); }
/** * Constructs a TimeCacheMap instance with a blob store timeout whose expiration callback invokes cancel on the value held by an expired * entry when that value is an AtomicOutputStream and calls close otherwise. * * @param conf the config to use * @return the newly created map */ @SuppressWarnings("deprecation") private static <T extends AutoCloseable> TimeCacheMap<String, T> makeBlobCacheMap(Map<String, Object> conf) { return new TimeCacheMap<>(ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_BLOBSTORE_EXPIRATION_SECS), 600), (id, stream) -> { try { if (stream instanceof AtomicOutputStream) { ((AtomicOutputStream) stream).cancel(); } else { stream.close(); } } catch (Exception e) { throw new RuntimeException(e); } }); }
@Override public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) { _fieldLocations = new HashMap<String, GlobalStreamId>(); _collector = collector; int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue(); _pending = new TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>>(timeout, new ExpireCallback()); _numSources = context.getThisSources().size(); Set<String> idFields = null; for (GlobalStreamId source : context.getThisSources().keySet()) { Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId()); Set<String> setFields = new HashSet<String>(fields.toList()); if (idFields == null) { idFields = setFields; } else { idFields.retainAll(setFields); } for (String outfield : _outFields) { for (String sourcefield : fields) { if (outfield.equals(sourcefield)) { _fieldLocations.put(outfield, source); } } } } _idFields = new Fields(new ArrayList<String>(idFields)); if (_fieldLocations.size() != _outFields.size()) { throw new RuntimeException("Cannot find all outfields among sources"); } }
public void prepare(Map<String, Object> config, TopologyContext context, OutputCollector collector) { TimeCacheMap.ExpiredCallback<Object, TrackingInfo> callback = null; if (_delegate instanceof TimeoutCallback) { callback = new TimeoutItems(); } _tracked = new TimeCacheMap<>(context.maxTopologyMessageTimeout(), callback); _collector = collector; _delegate.prepare(config, context, new OutputCollector(new CoordinatedOutputCollector(collector))); for (String component : Utils.get(context.getThisTargets(), Constants.COORDINATED_STREAM_ID, new HashMap<String, Grouping>()) .keySet()) { for (Integer task : context.getComponentTasks(component)) { _countOutTasks.add(task); } } if (!_sourceArgs.isEmpty()) { _numSourceReports = 0; for (Entry<String, SourceArgs> entry : _sourceArgs.entrySet()) { if (entry.getValue().singleCount) { _numSourceReports += 1; } else { _numSourceReports += context.getComponentTasks(entry.getKey()).size(); } } } }
/** * Invoked once immediately after construction * @param storm_conf Storm configuration */ @Override public void prepare(Map storm_conf) { int timeout = Utils.getInt(storm_conf.get(Config.STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS)); cachedGroups = new TimeCacheMap<>(timeout); }
@Override public void prepare(Map conf, TopologyContext context, OutputCollector collector) { _fieldLocations = new HashMap<String, GlobalStreamId>(); _collector = collector; int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue(); _pending = new TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>>(timeout, new ExpireCallback()); _numSources = context.getThisSources().size(); Set<String> idFields = null; for (GlobalStreamId source : context.getThisSources().keySet()) { Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId()); Set<String> setFields = new HashSet<String>(fields.toList()); if (idFields == null) idFields = setFields; else idFields.retainAll(setFields); for (String outfield : _outFields) { for (String sourcefield : fields) { if (outfield.equals(sourcefield)) { _fieldLocations.put(outfield, source); } } } } _idFields = new Fields(new ArrayList<String>(idFields)); if (_fieldLocations.size() != _outFields.size()) { throw new RuntimeException("Cannot find all outfields among sources"); } }
public void prepare(Map config, TopologyContext context, OutputCollector collector) { TimeCacheMap.ExpiredCallback<Object, TrackingInfo> callback = null; if(_delegate instanceof TimeoutCallback) { callback = new TimeoutItems(); } _tracked = new TimeCacheMap<>(context.maxTopologyMessageTimeout(), callback); _collector = collector; _delegate.prepare(config, context, new OutputCollector(new CoordinatedOutputCollector(collector))); for(String component: Utils.get(context.getThisTargets(), Constants.COORDINATED_STREAM_ID, new HashMap<String, Grouping>()) .keySet()) { for(Integer task: context.getComponentTasks(component)) { _countOutTasks.add(task); } } if(!_sourceArgs.isEmpty()) { _numSourceReports = 0; for(Entry<String, SourceArgs> entry: _sourceArgs.entrySet()) { if(entry.getValue().singleCount) { _numSourceReports+=1; } else { _numSourceReports+=context.getComponentTasks(entry.getKey()).size(); } } } }