@Override public Iterable<List<Object>> deserialize(final byte[] ser) { List<Object> o = scheme.deserialize(ser); if (o == null) return null; else return Arrays.asList(o); }
@Override public Fields getOutputFields() { return scheme.getOutputFields(); } }
@Override public Fields getOutputFields() { return scheme.getOutputFields(); } }
@Override public Iterable<List<Object>> deserialize(final byte[] ser) { List<Object> o = scheme.deserialize(ser); if (o == null) return null; else return Arrays.asList(o); }
@Override public Fields getOutputFields() { return scheme.getOutputFields(); } }
@Override public Iterable<List<Object>> deserialize(final byte[] ser) { List<Object> o = scheme.deserialize(ser); if (o == null) { return null; } else { return Arrays.asList(o); } }
@Override public Fields getOutputFields() { return scheme.getOutputFields(); } }
@Override public Iterable<List<Object>> deserialize(final byte[] ser) { List<Object> o = scheme.deserialize(ser); if(o == null) return null; else return Arrays.asList(o); }
public Fields getOutputFields() { return _scheme.getOutputFields(); }
private void getSomeMoreTuples() { if (this.tuples.size() > HOLD_ITEMS - MAX_ITEMS_PER_QUEUE / 2) return; int clientIndex = 0; for (final KestrelServerSpec clientSpec : this.clients) { try { final KestrelThriftClient client = clientSpec.getValidClient(); final List<Item> ret = client .get(this.queue, MAX_ITEMS_PER_QUEUE, TIMEOUT, TIMEOUT * MAX_ITEMS_PER_QUEUE); readTotal += ret.size(); logger.debug("Read total: " + readTotal); final Set<Long> ids = new HashSet<Long>(); for (final Item item : ret) { final long kestrelId = item.get_id(); ids.add(kestrelId); final List<Object> deserialize = scheme.deserialize(item.get_data()); if (deserialize == null) continue; // we silently skip null items final EmitItem e = new EmitItem(deserialize, new KestrelSourceId(clientIndex, kestrelId)); this.tuples.add(e); } // We immediately confirm ALL items. This is the thing that // makes it unreliable! client.confirm(this.queue, ids); } catch (final TException e) { } clientIndex++; } }
public Fields getOutputFields() { return _scheme.getOutputFields(); }
private void getSomeMoreTuples() { if (this.tuples.size() > HOLD_ITEMS - MAX_ITEMS_PER_QUEUE / 2) return; int clientIndex = 0; for (final KestrelServerSpec clientSpec : this.clients) { try { final KestrelThriftClient client = clientSpec.getValidClient(); final List<Item> ret = client .get(this.queue, MAX_ITEMS_PER_QUEUE, TIMEOUT, TIMEOUT * MAX_ITEMS_PER_QUEUE); readTotal += ret.size(); logger.debug("Read total: " + readTotal); final Set<Long> ids = new HashSet<Long>(); for (final Item item : ret) { final long kestrelId = item.get_id(); ids.add(kestrelId); final List<Object> deserialize = scheme.deserialize(item.get_data()); if (deserialize == null) continue; // we silently skip null items final EmitItem e = new EmitItem(deserialize, new KestrelSourceId(clientIndex, kestrelId)); this.tuples.add(e); } // We immediately confirm ALL items. This is the thing that // makes it unreliable! client.confirm(this.queue, ids); } catch (final TException e) { } clientIndex++; } }
/** * Declares the output fields of this spout according to the provided * {@link backtype.storm.spout.Scheme}. */ @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(serialisationScheme.getOutputFields()); }
final List<Object> retItems = _scheme.deserialize(item.get_data());
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(scheme.getOutputFields()); }
final List<Object> retItems = _scheme.deserialize(item.get_data());
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(scheme.getOutputFields()); }
/** * Emits the next message from the queue as a tuple. * * <p>If no message is ready to emit, this will wait a short time * ({@link #WAIT_FOR_NEXT_MESSAGE}) for one to arrive on the queue, * to avoid a tight loop in the spout worker.</p> */ @Override public void nextTuple() { if (amqpConsumer != null) { try { final QueueingConsumer.Delivery delivery = amqpConsumer.nextDelivery(WAIT_FOR_NEXT_MESSAGE); if (delivery == null) return; final long deliveryTag = delivery.getEnvelope().getDeliveryTag(); final byte[] message = delivery.getBody(); collector.emit(serialisationScheme.deserialize(message), deliveryTag); /* * TODO what to do about malformed messages? Skip? * Avoid infinite retry! * Maybe we should output them on a separate stream. */ } catch (ShutdownSignalException e) { log.warn("AMQP connection dropped, will attempt to reconnect..."); Utils.sleep(WAIT_AFTER_SHUTDOWN_SIGNAL); reconnect(); } catch (InterruptedException e) { // interrupted while waiting for message, big deal } } }