@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getRecords() == null) ? 0 : getRecords().hashCode()); hashCode = prime * hashCode + ((getNextShardIterator() == null) ? 0 : getNextShardIterator().hashCode()); hashCode = prime * hashCode + ((getMillisBehindLatest() == null) ? 0 : getMillisBehindLatest().hashCode()); return hashCode; }
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getRecords() != null) sb.append("Records: ").append(getRecords()).append(","); if (getNextShardIterator() != null) sb.append("NextShardIterator: ").append(getNextShardIterator()).append(","); if (getMillisBehindLatest() != null) sb.append("MillisBehindLatest: ").append(getMillisBehindLatest()); sb.append("}"); return sb.toString(); }
LOG.debug("Records size from fetchNewRecords is {}", records.size()); shardIteratorPerShard.put(shardId, getRecordsResult.getNextShardIterator()); if (records.size() == 0) { LOG.warn("No records returned from kinesis. Hence sleeping for 1 second");
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof GetRecordsResult == false) return false; GetRecordsResult other = (GetRecordsResult) obj; if (other.getRecords() == null ^ this.getRecords() == null) return false; if (other.getRecords() != null && other.getRecords().equals(this.getRecords()) == false) return false; if (other.getNextShardIterator() == null ^ this.getNextShardIterator() == null) return false; if (other.getNextShardIterator() != null && other.getNextShardIterator().equals(this.getNextShardIterator()) == false) return false; if (other.getMillisBehindLatest() == null ^ this.getMillisBehindLatest() == null) return false; if (other.getMillisBehindLatest() != null && other.getMillisBehindLatest().equals(this.getMillisBehindLatest()) == false) return false; return true; }
LOG.debug("Records size from fetchFailedRecords is {}", records.size()); shardIteratorPerFailedMessage.put(kinesisMessageId, getRecordsResult.getNextShardIterator()); if (records.size() == 0) { LOG.warn("No records returned from kinesis. Hence sleeping for 1 second");
shardIterator = recordsResult.getNextShardIterator();
shardIterator = recordsResult.getNextShardIterator(); // will be null if the shard has been closed ); if (recordsResult != null) { shardIterator = recordsResult.getNextShardIterator(); // will be null if the shard has been closed rescheduleRunnable(fetchDelayMillis); } else {
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getRecords() == null) ? 0 : getRecords().hashCode()); hashCode = prime * hashCode + ((getNextShardIterator() == null) ? 0 : getNextShardIterator().hashCode()); hashCode = prime * hashCode + ((getMillisBehindLatest() == null) ? 0 : getMillisBehindLatest().hashCode()); return hashCode; }
/** * Returns a string representation of this object; useful for testing and * debugging. * * @return A string representation of this object. * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getRecords() != null) sb.append("Records: " + getRecords() + ","); if (getNextShardIterator() != null) sb.append("NextShardIterator: " + getNextShardIterator() + ","); if (getMillisBehindLatest() != null) sb.append("MillisBehindLatest: " + getMillisBehindLatest()); sb.append("}"); return sb.toString(); }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof GetRecordsResult == false) return false; GetRecordsResult other = (GetRecordsResult) obj; if (other.getRecords() == null ^ this.getRecords() == null) return false; if (other.getRecords() != null && other.getRecords().equals(this.getRecords()) == false) return false; if (other.getNextShardIterator() == null ^ this.getNextShardIterator() == null) return false; if (other.getNextShardIterator() != null && other.getNextShardIterator().equals(this.getNextShardIterator()) == false) return false; if (other.getMillisBehindLatest() == null ^ this.getMillisBehindLatest() == null) return false; if (other.getMillisBehindLatest() != null && other.getMillisBehindLatest().equals(this.getMillisBehindLatest()) == false) return false; return true; } }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getRecords() == null) ? 0 : getRecords().hashCode()); hashCode = prime * hashCode + ((getNextShardIterator() == null) ? 0 : getNextShardIterator().hashCode()); hashCode = prime * hashCode + ((getMillisBehindLatest() == null) ? 0 : getMillisBehindLatest().hashCode()); return hashCode; }
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getRecords() != null) sb.append("Records: ").append(getRecords()).append(","); if (getNextShardIterator() != null) sb.append("NextShardIterator: ").append(getNextShardIterator()).append(","); if (getMillisBehindLatest() != null) sb.append("MillisBehindLatest: ").append(getMillisBehindLatest()); sb.append("}"); return sb.toString(); }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof GetRecordsResult == false) return false; GetRecordsResult other = (GetRecordsResult) obj; if (other.getRecords() == null ^ this.getRecords() == null) return false; if (other.getRecords() != null && other.getRecords().equals(this.getRecords()) == false) return false; if (other.getNextShardIterator() == null ^ this.getNextShardIterator() == null) return false; if (other.getNextShardIterator() != null && other.getNextShardIterator().equals(this.getNextShardIterator()) == false) return false; if (other.getMillisBehindLatest() == null ^ this.getMillisBehindLatest() == null) return false; if (other.getMillisBehindLatest() != null && other.getMillisBehindLatest().equals(this.getMillisBehindLatest()) == false) return false; return true; }
@Override public GetRecordsResult accept() { nextIterator = result.getNextShardIterator(); if (!CollectionUtils.isNullOrEmpty(result.getRecords())) { lastKnownSequenceNumber = Iterables.getLast(result.getRecords()).getSequenceNumber(); } if (nextIterator == null) { isShardEndReached = true; } return getResult(); }
@Override public Records getNext(int maxNumberOfRecords) throws AmazonClientException, ResourceNotFoundException, InvalidArgumentException { if (shardIterator == null) { LOG.debug(this + " Null shardIterator for " + shardId + ". This can happen if shard is closed."); return Records.empty(true); } final ImmutableList.Builder<Record> records = new ImmutableList.Builder<>(); try { final GetRecordsRequest request = new GetRecordsRequest(); request.setShardIterator(shardIterator); request.setLimit(maxNumberOfRecords); final GetRecordsResult result = safeGetRecords(request); for (Record rec : result.getRecords()) { records.add(rec); positionInShard = ShardPosition.afterSequenceNumber(rec.getSequenceNumber()); } if (LOG.isDebugEnabled()) { LOG.debug(this + " fetched " + result.getRecords().size() + " records from Kinesis (requested " + maxNumberOfRecords + ")."); } shardIterator = result.getNextShardIterator(); } catch (AmazonClientException e) { // We'll treat this equivalent to fetching 0 records - the spout drives the retry as part of nextTuple() // We don't sleep here - we can continue processing ack/fail on the spout thread. LOG.error(this + "Caught exception when fetching records for " + shardId, e); } return new Records(records.build(), shardIterator == null); }
shardIteratorId = recordsResult.getNextShardIterator();
attributesHolder.remove(); if (result != null) { this.shardIterator = result.getNextShardIterator();
attributesHolder.remove(); if (result != null) { this.shardIterator = result.getNextShardIterator();
/** * Gets records from Kinesis and deaggregates them if needed. * * @return list of deaggregated records * @throws TransientKinesisException - in case of recoverable situation */ public GetKinesisRecordsResult getRecords( final String shardIterator, final String streamName, final String shardId, final Integer limit) throws TransientKinesisException { return wrapExceptions( () -> { GetRecordsResult response = kinesis.getRecords( new GetRecordsRequest().withShardIterator(shardIterator).withLimit(limit)); return new GetKinesisRecordsResult( UserRecord.deaggregate(response.getRecords()), response.getNextShardIterator(), response.getMillisBehindLatest(), streamName, shardId); }); }
public String processNextIterator(String iterator) { GetRecordsRequest getRequest = new GetRecordsRequest(); getRequest.setLimit(1000); getRequest.setShardIterator(iterator); // call "get" operation and get everything in this shard range GetRecordsResult getResponse = client.getRecords(getRequest); iterator = getResponse.getNextShardIterator(); List<Record> records = getResponse.getRecords(); processResponseRecords(records); return iterator; }