@Override public void fail(Object msgId) { final KestrelSourceId sourceId = (KestrelSourceId) msgId; final KestrelClientInfo info = _kestrels.get(sourceId.index); // see not above about why this works with blacklisting strategy try { if (info.client != null) { final HashSet<Long> xids = new HashSet<Long>(); xids.add(sourceId.id); info.client.abort(_queueName, xids); } } catch (final TException e) { blacklist(info, e); } }
public void tryEachKestrelUntilBufferFilled() { for (int i = 0; i < _kestrels.size(); i++) { final int index = (_emitIndex + i) % _kestrels.size(); if (bufferKestrelGet(index)) { _emitIndex = index; break; } } _emitIndex = (_emitIndex + 1) % _kestrels.size(); }
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(getOutputFields()); } }
public static void main(String[] args) throws Exception { final TopologyBuilder builder = new TopologyBuilder(); final KestrelThriftSpout spout = new KestrelThriftSpout("localhost", 2229, "test", new StringScheme()); builder.setSpout("spout", spout).setDebug(true); builder.setBolt("bolt", new FailEveryOther()) .shuffleGrouping("spout"); final LocalCluster cluster = new LocalCluster(); final Config conf = new Config(); cluster.submitTopology("test", conf, builder.createTopology()); Thread.sleep(600000); } }
@Override public void nextTuple() { if (_emitBuffer.isEmpty()) tryEachKestrelUntilBufferFilled(); if (countTriples % 1000 == 0 && countTriples != lastEmit) { LOG.debug("Number of triples emitted: " + countTriples); LOG.debug("Number of empty iterations: " + emptyIterations); emptyIterations = 0; lastEmit = countTriples; } final EmitItem item = _emitBuffer.poll(); if (item != null) { countTriples += 1; _collector.emit(item.tuple, item.sourceId); } else { // If buffer is still empty here, then every kestrel Q is also // empty. emptyIterations++; Utils.sleep(10); } }
public static void main(String[] args) throws Exception { final TopologyBuilder builder = new TopologyBuilder(); final KestrelThriftSpout spout = new KestrelThriftSpout("localhost", 2229, "test", new StringScheme()); builder.setSpout("spout", spout).setDebug(true); builder.setBolt("bolt", new FailEveryOther()) .shuffleGrouping("spout"); final LocalCluster cluster = new LocalCluster(); final Config conf = new Config(); cluster.submitTopology("test", conf, builder.createTopology()); Thread.sleep(600000); } }
@Override public void nextTuple() { if (_emitBuffer.isEmpty()) tryEachKestrelUntilBufferFilled(); if (countTriples % 1000 == 0 && countTriples != lastEmit) { LOG.debug("Number of triples emitted: " + countTriples); LOG.debug("Number of empty iterations: " + emptyIterations); emptyIterations = 0; lastEmit = countTriples; } final EmitItem item = _emitBuffer.poll(); if (item != null) { countTriples += 1; _collector.emit(item.tuple, item.sourceId); } else { // If buffer is still empty here, then every kestrel Q is also // empty. emptyIterations++; Utils.sleep(10); } }
@Override public void fail(Object msgId) { final KestrelSourceId sourceId = (KestrelSourceId) msgId; final KestrelClientInfo info = _kestrels.get(sourceId.index); // see not above about why this works with blacklisting strategy try { if (info.client != null) { final HashSet<Long> xids = new HashSet<Long>(); xids.add(sourceId.id); info.client.abort(_queueName, xids); } } catch (final TException e) { blacklist(info, e); } }
public void tryEachKestrelUntilBufferFilled() { for (int i = 0; i < _kestrels.size(); i++) { final int index = (_emitIndex + i) % _kestrels.size(); if (bufferKestrelGet(index)) { _emitIndex = index; break; } } _emitIndex = (_emitIndex + 1) % _kestrels.size(); }
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(getOutputFields()); } }
@Override public void ack(Object msgId) { final KestrelSourceId sourceId = (KestrelSourceId) msgId; final KestrelClientInfo info = _kestrels.get(sourceId.index); // if the transaction didn't exist, it just returns false. so this code // works // even if client gets blacklisted, disconnects, and kestrel puts the // item // back on the queue try { if (info.client != null) { final HashSet<Long> xids = new HashSet<Long>(); xids.add(sourceId.id); info.client.confirm(_queueName, xids); } } catch (final TException e) { blacklist(info, e); } }
@Override public void ack(Object msgId) { final KestrelSourceId sourceId = (KestrelSourceId) msgId; final KestrelClientInfo info = _kestrels.get(sourceId.index); // if the transaction didn't exist, it just returns false. so this code // works // even if client gets blacklisted, disconnects, and kestrel puts the // item // back on the queue try { if (info.client != null) { final HashSet<Long> xids = new HashSet<Long>(); xids.add(sourceId.id); info.client.confirm(_queueName, xids); } } catch (final TException e) { blacklist(info, e); } }