public PutRecordsRequestEntry unmarshall(JsonUnmarshallerContext context) throws Exception { PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry(); if (context.testExpression("Data", targetDepth)) { context.nextToken(); putRecordsRequestEntry.setData(context.getUnmarshaller(java.nio.ByteBuffer.class).unmarshall(context)); putRecordsRequestEntry.setExplicitHashKey(context.getUnmarshaller(String.class).unmarshall(context)); putRecordsRequestEntry.setPartitionKey(context.getUnmarshaller(String.class).unmarshall(context));
PutRecordsRequestEntry record = new PutRecordsRequestEntry().withData(ByteBuffer.wrap(baos.toByteArray())); record.setPartitionKey(partitionKey); } else { record.setPartitionKey(Integer.toString(randomParitionKeyGenerator.nextInt()));
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getData() == null) ? 0 : getData().hashCode()); hashCode = prime * hashCode + ((getExplicitHashKey() == null) ? 0 : getExplicitHashKey().hashCode()); hashCode = prime * hashCode + ((getPartitionKey() == null) ? 0 : getPartitionKey().hashCode()); return hashCode; }
@Override protected void runOnce() throws Exception { ClickEvent event = inputQueue.take(); String partitionKey = event.getSessionId(); ByteBuffer data = ByteBuffer.wrap( event.getPayload().getBytes("UTF-8")); recordsPut.getAndIncrement(); addEntry(new PutRecordsRequestEntry() .withPartitionKey(partitionKey) .withData(data)); }
@Override public List<byte[]> sendBatch(String streamName, List<byte[]> data) { if (data == null || data.isEmpty()) { return Collections.emptyList(); } final PutRecordsRequest request = new PutRecordsRequest(); request.setStreamName(streamName); final List<PutRecordsRequestEntry> records = new ArrayList<PutRecordsRequestEntry>(data.size()); for (final byte[] d : data) { final String partKey = StringUtils.isBlank(this.partitionKey) ? UUID.randomUUID().toString() : this.partitionKey; final PutRecordsRequestEntry r = new PutRecordsRequestEntry(); r.setData(ByteBuffer.wrap(d)); r.setPartitionKey(partKey); records.add(r); } request.setRecords(records); request.getRequestClientOptions().appendUserAgent(userAgent); final PutRecordsResult result = client.putRecords(request); final int size = result.getRecords().size(); final List<byte[]> failures = new ArrayList<byte[]>(result.getFailedRecordCount()); for (int i = 0; i < size; i++) { if (result.getRecords().get(i).getErrorCode() != null) { // always retry failed record failures.add(data.get(i)); } } return failures; }
List<PutRecordsRequestEntry> requestRecords = new ArrayList<>(); for(KinesisRecord data : buffer) { PutRecordsRequestEntry record = new PutRecordsRequestEntry(); record.setData(data.data()); record.setPartitionKey(data.partitionKey()); requestRecords.add(record); if (responseEntry.getErrorCode() == null) { sentRecords.add(index); totalBytesSent += record.getData().limit(); } else { logger.trace("{}:{} Record {} returned error code {}: {}", flow.getId(), buffer, index, responseEntry.getErrorCode(),
protected void addEntry(PutRecordsRequestEntry entry) { int newDataSize = dataSize + entry.getData().remaining() + entry.getPartitionKey().length(); if (newDataSize <= 5 * 1024 * 1024 && entries.size() < 500) { dataSize = newDataSize; entries.add(entry); } else { flush(); dataSize = 0; addEntry(entry); } } }
/** * <p> * Determines which shard in the stream the data record is assigned to. Partition keys are Unicode strings with a * maximum length limit of 256 characters for each key. Amazon Kinesis Data Streams uses the partition key as input * to a hash function that maps the partition key and associated data to a specific shard. Specifically, an MD5 hash * function is used to map partition keys to 128-bit integer values and to map associated data records to shards. As * a result of this hashing mechanism, all data records with the same partition key map to the same shard within the * stream. * </p> * * @param partitionKey * Determines which shard in the stream the data record is assigned to. Partition keys are Unicode strings * with a maximum length limit of 256 characters for each key. Amazon Kinesis Data Streams uses the partition * key as input to a hash function that maps the partition key and associated data to a specific shard. * Specifically, an MD5 hash function is used to map partition keys to 128-bit integer values and to map * associated data records to shards. As a result of this hashing mechanism, all data records with the same * partition key map to the same shard within the stream. * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsRequestEntry withPartitionKey(String partitionKey) { setPartitionKey(partitionKey); return this; }
/** * <p> * The data blob to put into the record, which is base64-encoded when the blob is serialized. When the data blob * (the payload before base64-encoding) is added to the partition key size, the total size must not exceed the * maximum record size (1 MB). * </p> * <p> * The AWS SDK for Java performs a Base64 encoding on this field before sending this request to the AWS service. * Users of the SDK should not perform Base64 encoding on this field. * </p> * <p> * Warning: ByteBuffers returned by the SDK are mutable. Changes to the content or position of the byte buffer will * be seen by all objects that have a reference to this object. It is recommended to call ByteBuffer.duplicate() or * ByteBuffer.asReadOnlyBuffer() before using or reading from the buffer. This behavior will be changed in a future * major version of the SDK. * </p> * * @param data * The data blob to put into the record, which is base64-encoded when the blob is serialized. When the data * blob (the payload before base64-encoding) is added to the partition key size, the total size must not * exceed the maximum record size (1 MB). * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsRequestEntry withData(java.nio.ByteBuffer data) { setData(data); return this; }
for(PutRecordsRequestEntry record:records) String partitionKey = record.getPartitionKey(); assertFalse("There are duplicated partition keys", partitionKeys.contains(partitionKey)); partitionKeys.add(partitionKey);
long totalSize = putRecordsRequest.getRecords().stream().mapToLong(record -> record.getData().limit()).sum(); LOGGER.info("Sending {} records ({} bytes)", putRecordsRequest.getRecords().size(), totalSize);
public Observable<PutRecordsResultEntry> play(LocalDateTime start, @Nullable LocalDateTime end) { return playableObjects(start, end) .onBackpressureBuffer() .observeOn(Schedulers.io()) .flatMap(this::objectToPayloads) .map(ByteBuffer::wrap) .lift(new OperatorBufferKinesisBatch(MAX_KINESIS_BATCH_SIZE, MAX_KINESIS_BATCH_WEIGHT)) .onBackpressureBuffer() .map(byteBuffers -> byteBuffers.stream() .map(buffer -> new PutRecordsRequestEntry() .withData(buffer) .withPartitionKey(UUID.randomUUID().toString())) .collect(toList())) .map(entries -> new PutRecordsRequest() .withStreamName(vcrConfiguration.targetStream) .withRecords(entries)) .observeOn(Schedulers.io()) .flatMap(putRecordsRequest -> Observable.create((Observable.OnSubscribe<List<PutRecordsResult>>) os -> { os.onStart(); kinesisWriter.submit(() -> { os.onNext(putWithRetry(putRecordsRequest).orElse(Collections.<PutRecordsResult>emptyList())); os.onCompleted(); }); })) .flatMap(Observable::from) .flatMap(putRecordsResult -> Observable.from(putRecordsResult.getRecords())) .doOnNext(result -> LOGGER.debug("Wrote record. Seq {}, shard {}", result.getSequenceNumber(), result.getShardId())); }
@Test public void test() throws Exception { PutRecordsRequest putRecordsRequest = new PutRecordsRequest(); putRecordsRequest.setStreamName("stream name"); List<PutRecordsRequestEntry> records = new ArrayList<PutRecordsRequestEntry>(); for (int i = 0; i < 10; i++) { PutRecordsRequestEntry record = new PutRecordsRequestEntry(); String randomStr = RandomStringUtils.random(8 * 1024); record.setData(ByteBuffer.wrap(randomStr.getBytes(StringUtils.UTF8))); record.setPartitionKey("partition key"); records.add(record); } putRecordsRequest.setRecords(records); PutRecordsRequestMarshaller marshaller = new PutRecordsRequestMarshaller(); Request<PutRecordsRequest> request = marshaller.marshall(putRecordsRequest); assertEquals("content encoding", "gzip", request.getHeaders().get("Content-Encoding")); byte[] content = IOUtils.toByteArray(request.getContent()); assertEquals("content length", request.getHeaders().get("Content-Length"), String.valueOf(content.length)); GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(content)); String str = IOUtils.toString(gis); assertTrue("content is compressed", content.length < str.length()); Map<String, String> map = JsonUtils.jsonToMap(str); assertEquals("StreamName", "stream name", map.get("StreamName")); } }
public Optional<PutRecordsRequest> put(PutRecordsRequestEntry entry) { int newRequestSize = requestSize + entry.getData().remaining() + entry.getPartitionKey().length(); if (entries.size() < maxCount && newRequestSize <= maxSize) { requestSize = newRequestSize; entries.add(entry); return Optional.empty(); } else { Optional<PutRecordsRequest> ret = flush(); put(entry); return ret; } }
/** * <p> * Determines which shard in the stream the data record is assigned to. Partition keys are Unicode strings with a * maximum length limit of 256 characters for each key. Amazon Kinesis Data Streams uses the partition key as input * to a hash function that maps the partition key and associated data to a specific shard. Specifically, an MD5 hash * function is used to map partition keys to 128-bit integer values and to map associated data records to shards. As * a result of this hashing mechanism, all data records with the same partition key map to the same shard within the * stream. * </p> * * @param partitionKey * Determines which shard in the stream the data record is assigned to. Partition keys are Unicode strings * with a maximum length limit of 256 characters for each key. Amazon Kinesis Data Streams uses the partition * key as input to a hash function that maps the partition key and associated data to a specific shard. * Specifically, an MD5 hash function is used to map partition keys to 128-bit integer values and to map * associated data records to shards. As a result of this hashing mechanism, all data records with the same * partition key map to the same shard within the stream. * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsRequestEntry withPartitionKey(String partitionKey) { setPartitionKey(partitionKey); return this; }
/** * <p> * The data blob to put into the record, which is base64-encoded when the blob is serialized. When the data blob * (the payload before base64-encoding) is added to the partition key size, the total size must not exceed the * maximum record size (1 MB). * </p> * <p> * The AWS SDK for Java performs a Base64 encoding on this field before sending this request to the AWS service. * Users of the SDK should not perform Base64 encoding on this field. * </p> * <p> * Warning: ByteBuffers returned by the SDK are mutable. Changes to the content or position of the byte buffer will * be seen by all objects that have a reference to this object. It is recommended to call ByteBuffer.duplicate() or * ByteBuffer.asReadOnlyBuffer() before using or reading from the buffer. This behavior will be changed in a future * major version of the SDK. * </p> * * @param data * The data blob to put into the record, which is base64-encoded when the blob is serialized. When the data * blob (the payload before base64-encoding) is added to the partition key size, the total size must not * exceed the maximum record size (1 MB). * @return Returns a reference to this object so that method calls can be chained together. */ public PutRecordsRequestEntry withData(java.nio.ByteBuffer data) { setData(data); return this; }
public void storeBatchInline(List<Event> events, int offset, int limit) { PutRecordsRequestEntry[] records = new PutRecordsRequestEntry[limit]; for (int i = 0; i < limit; i++) { Event event = events.get(offset + i); PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry() .withData(getBuffer(event)) .withPartitionKey(event.project() + "|" + event.collection()); records[i] = putRecordsRequestEntry; } try { PutRecordsResult putRecordsResult = kinesis.putRecords(new PutRecordsRequest() .withRecords(records) .withStreamName(config.getEventStoreStreamName())); if (putRecordsResult.getFailedRecordCount() > 0) { for (PutRecordsResultEntry resultEntry : putRecordsResult.getRecords()) { resultEntry.getErrorMessage(); } } } catch (ResourceNotFoundException e) { try { createAndWaitForStreamToBecomeAvailable(kinesis, config.getEventStoreStreamName(), 1); } catch (Exception e1) { throw new RuntimeException("Couldn't send event to Amazon Kinesis", e); } } }
private void addRecord(T tuple) { try { Pair<String, V> keyValue = tupleToKeyValue(tuple); PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry(); putRecordsEntry.setData(ByteBuffer.wrap(getRecord(keyValue.second))); putRecordsEntry.setPartitionKey(keyValue.first); putRecordsRequestEntryList.add(putRecordsEntry); } catch (AmazonClientException e) { throw new RuntimeException(e); } }
public PutRecordsRequestEntry unmarshall(JsonUnmarshallerContext context) throws Exception { AwsJsonReader reader = context.getReader(); if (!reader.isContainer()) { reader.skipValue(); return null; } PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry(); reader.beginObject(); while (reader.hasNext()) { String name = reader.nextName(); if (name.equals("Data")) { putRecordsRequestEntry.setData(ByteBufferJsonUnmarshaller.getInstance() .unmarshall(context)); } else if (name.equals("ExplicitHashKey")) { putRecordsRequestEntry.setExplicitHashKey(StringJsonUnmarshaller.getInstance() .unmarshall(context)); } else if (name.equals("PartitionKey")) { putRecordsRequestEntry.setPartitionKey(StringJsonUnmarshaller.getInstance() .unmarshall(context)); } else { reader.skipValue(); } } reader.endObject(); return putRecordsRequestEntry; }
PutRecordsRequestEntry record = new PutRecordsRequestEntry().withData(ByteBuffer.wrap(baos.toByteArray())); record.setPartitionKey(partitionKey); } else { record.setPartitionKey(Integer.toString(randomParitionKeyGenerator.nextInt()));