void initialize (int myTaskIndex, int totalTasks) { deactivated = false; lastCommitTime = System.currentTimeMillis(); kinesisConnection.initialize(); zkConnection.initialize(); List<Shard> shards = kinesisConnection.getShardsForStream(kinesisConfig.getStreamName()); LOG.info("myTaskIndex is " + myTaskIndex); LOG.info("totalTasks is " + totalTasks); int i = myTaskIndex; while (i < shards.size()) { LOG.info("Shard id " + shards.get(i).getShardId() + " assigned to task " + myTaskIndex); toEmitPerShard.put(shards.get(i).getShardId(), new LinkedList<Record>()); i += totalTasks; } initializeFetchedSequenceNumbers(); refreshShardIteratorsForNewRecords(); }
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getShardId() != null) sb.append("ShardId: ").append(getShardId()).append(","); if (getParentShardId() != null) sb.append("ParentShardId: ").append(getParentShardId()).append(","); if (getAdjacentParentShardId() != null) sb.append("AdjacentParentShardId: ").append(getAdjacentParentShardId()).append(","); if (getHashKeyRange() != null) sb.append("HashKeyRange: ").append(getHashKeyRange()).append(","); if (getSequenceNumberRange() != null) sb.append("SequenceNumberRange: ").append(getSequenceNumberRange()); sb.append("}"); return sb.toString(); }
List<Shard> getShardsForStream (String stream) { DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(stream); List<Shard> shards = new ArrayList<>(); String exclusiveStartShardId = null; do { describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); DescribeStreamResult describeStreamResult = kinesisClient.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) { exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); } else { exclusiveStartShardId = null; } } while ( exclusiveStartShardId != null ); LOG.info("Number of shards for stream " + stream + " are " + shards.size()); return shards; }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getParentShardId() == null) ? 0 : getParentShardId().hashCode()); hashCode = prime * hashCode + ((getAdjacentParentShardId() == null) ? 0 : getAdjacentParentShardId().hashCode()); hashCode = prime * hashCode + ((getHashKeyRange() == null) ? 0 : getHashKeyRange().hashCode()); hashCode = prime * hashCode + ((getSequenceNumberRange() == null) ? 0 : getSequenceNumberRange().hashCode()); return hashCode; }
/** * Marshall the given parameter object. */ public void marshall(Shard shard, ProtocolMarshaller protocolMarshaller) { if (shard == null) { throw new SdkClientException("Invalid argument passed to marshall(...)"); } try { protocolMarshaller.marshall(shard.getShardId(), SHARDID_BINDING); protocolMarshaller.marshall(shard.getParentShardId(), PARENTSHARDID_BINDING); protocolMarshaller.marshall(shard.getAdjacentParentShardId(), ADJACENTPARENTSHARDID_BINDING); protocolMarshaller.marshall(shard.getHashKeyRange(), HASHKEYRANGE_BINDING); protocolMarshaller.marshall(shard.getSequenceNumberRange(), SEQUENCENUMBERRANGE_BINDING); } catch (Exception e) { throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e); } }
return false; Shard other = (Shard) obj; if (other.getShardId() == null ^ this.getShardId() == null) return false; if (other.getShardId() != null && other.getShardId().equals(this.getShardId()) == false) return false; if (other.getParentShardId() == null ^ this.getParentShardId() == null)
/** * Returns a string representation of this object; useful for testing and * debugging. * * @return A string representation of this object. * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getShardId() != null) sb.append("ShardId: " + getShardId() + ","); if (getParentShardId() != null) sb.append("ParentShardId: " + getParentShardId() + ","); if (getAdjacentParentShardId() != null) sb.append("AdjacentParentShardId: " + getAdjacentParentShardId() + ","); if (getHashKeyRange() != null) sb.append("HashKeyRange: " + getHashKeyRange() + ","); if (getSequenceNumberRange() != null) sb.append("SequenceNumberRange: " + getSequenceNumberRange()); sb.append("}"); return sb.toString(); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getParentShardId() == null) ? 0 : getParentShardId().hashCode()); hashCode = prime * hashCode + ((getAdjacentParentShardId() == null) ? 0 : getAdjacentParentShardId().hashCode()); hashCode = prime * hashCode + ((getHashKeyRange() == null) ? 0 : getHashKeyRange().hashCode()); hashCode = prime * hashCode + ((getSequenceNumberRange() == null) ? 0 : getSequenceNumberRange().hashCode()); return hashCode; }
Shard other = (Shard) obj; if (other.getShardId() == null ^ this.getShardId() == null) return false; if (other.getShardId() != null && other.getShardId().equals(this.getShardId()) == false) return false; if (other.getParentShardId() == null ^ this.getParentShardId() == null)
public String getShardId() { return shard.getShardId(); }
@JsonProperty("shardID") protected String getShardId() { return this.shard.getShardId(); }
public void update(List<Shard> shards) { if (shards == null || shards.isEmpty()) { return; } this.shards.addAll(shards); Shard lastShard = shards.get(shards.size() - 1); if (lastShardId == null || lastShardId.compareTo(lastShard.getShardId()) < 0) { lastShardId = lastShard.getShardId(); } } }
private void updateLastFetchedShardId(List<Shard> shards) { if (shards.size() > 0) { Shard lastShard = shards.get(shards.size() - 1); lastFetchedShardId = lastShard.getShardId(); } }
private void updateLastFetchedShardId(List<Shard> shards) { if (shards.size() > 0) { Shard lastShard = shards.get(shards.size() - 1); lastFetchedShardId = lastShard.getShardId(); } }
public void marshall(Shard shard, AwsJsonWriter jsonWriter) throws Exception { jsonWriter.beginObject(); if (shard.getShardId() != null) { String shardId = shard.getShardId(); jsonWriter.name("ShardId"); jsonWriter.value(shardId); } if (shard.getParentShardId() != null) { String parentShardId = shard.getParentShardId(); jsonWriter.name("ParentShardId"); jsonWriter.value(parentShardId); } if (shard.getAdjacentParentShardId() != null) { String adjacentParentShardId = shard.getAdjacentParentShardId(); jsonWriter.name("AdjacentParentShardId"); jsonWriter.value(adjacentParentShardId); } if (shard.getHashKeyRange() != null) { HashKeyRange hashKeyRange = shard.getHashKeyRange(); jsonWriter.name("HashKeyRange"); HashKeyRangeJsonMarshaller.getInstance().marshall(hashKeyRange, jsonWriter); } if (shard.getSequenceNumberRange() != null) { SequenceNumberRange sequenceNumberRange = shard.getSequenceNumberRange(); jsonWriter.name("SequenceNumberRange"); SequenceNumberRangeJsonMarshaller.getInstance().marshall(sequenceNumberRange, jsonWriter); } jsonWriter.endObject(); }
public static ShardHashInfo getOpenShard(AmazonKinesisClient kinesisClient, String streamName, String shardId) throws Exception { Shard s = getShard(kinesisClient, streamName, shardId); if (!s.getShardId().equals(shardId)) { throw new Exception(String.format("Shard %s not found in Stream %s", shardId, streamName)); } else { return new ShardHashInfo(streamName, s); } }
/** * This method is called in setup method of the operator */ public void create(){ holdingBuffer = new ArrayBlockingQueue<Pair<String, Record>>(bufferSize); boolean defaultSelect = (shardIds == null) || (shardIds.size() == 0); final List<Shard> pms = KinesisUtil.getInstance().getShardList(streamName); for (final Shard shId: pms) { if((shardIds.contains(shId.getShardId()) || defaultSelect) && !closedShards.contains(shId)) { simpleConsumerThreads.add(shId); } } }
private void activeAtTimestamp(Shard shard, Instant startTimestamp) { prepareShard( shard, "timestampIterator-" + shard.getShardId(), ShardIteratorType.AT_TIMESTAMP, startTimestamp); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getShardId() == null) ? 0 : getShardId().hashCode()); hashCode = prime * hashCode + ((getParentShardId() == null) ? 0 : getParentShardId().hashCode()); hashCode = prime * hashCode + ((getAdjacentParentShardId() == null) ? 0 : getAdjacentParentShardId().hashCode()); hashCode = prime * hashCode + ((getHashKeyRange() == null) ? 0 : getHashKeyRange().hashCode()); hashCode = prime * hashCode + ((getSequenceNumberRange() == null) ? 0 : getSequenceNumberRange().hashCode()); return hashCode; }
@Override public void startStream() { this.streamNames = this.config.getStreams(); for (final String stream : streamNames) { DescribeStreamResult describeStreamResult = client.describeStream(stream); if( "ACTIVE".equals(describeStreamResult.getStreamDescription().getStreamStatus())) { List<Shard> shardList = describeStreamResult.getStreamDescription().getShards(); for( Shard shard : shardList ) { executor.submit(new KinesisPersistReaderTask(this, stream, shard.getShardId())); } } } }