PutRecordsRequestEntry record = new PutRecordsRequestEntry().withData(ByteBuffer.wrap(baos.toByteArray()));
public PutRecordsRequestEntry unmarshall(JsonUnmarshallerContext context) throws Exception { PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
@Override protected void runOnce() throws Exception { ClickEvent event = inputQueue.take(); String partitionKey = event.getSessionId(); ByteBuffer data = ByteBuffer.wrap( event.getPayload().getBytes("UTF-8")); recordsPut.getAndIncrement(); addEntry(new PutRecordsRequestEntry() .withPartitionKey(partitionKey) .withData(data)); }
@Override public List<byte[]> sendBatch(String streamName, List<byte[]> data) { if (data == null || data.isEmpty()) { return Collections.emptyList(); } final PutRecordsRequest request = new PutRecordsRequest(); request.setStreamName(streamName); final List<PutRecordsRequestEntry> records = new ArrayList<PutRecordsRequestEntry>(data.size()); for (final byte[] d : data) { final String partKey = StringUtils.isBlank(this.partitionKey) ? UUID.randomUUID().toString() : this.partitionKey; final PutRecordsRequestEntry r = new PutRecordsRequestEntry(); r.setData(ByteBuffer.wrap(d)); r.setPartitionKey(partKey); records.add(r); } request.setRecords(records); request.getRequestClientOptions().appendUserAgent(userAgent); final PutRecordsResult result = client.putRecords(request); final int size = result.getRecords().size(); final List<byte[]> failures = new ArrayList<byte[]>(result.getFailedRecordCount()); for (int i = 0; i < size; i++) { if (result.getRecords().get(i).getErrorCode() != null) { // always retry failed record failures.add(data.get(i)); } } return failures; }
public PutRecordsRequestEntry unmarshall(JsonUnmarshallerContext context) throws Exception { AwsJsonReader reader = context.getReader(); if (!reader.isContainer()) { reader.skipValue(); return null; } PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry(); reader.beginObject(); while (reader.hasNext()) { String name = reader.nextName(); if (name.equals("Data")) { putRecordsRequestEntry.setData(ByteBufferJsonUnmarshaller.getInstance() .unmarshall(context)); } else if (name.equals("ExplicitHashKey")) { putRecordsRequestEntry.setExplicitHashKey(StringJsonUnmarshaller.getInstance() .unmarshall(context)); } else if (name.equals("PartitionKey")) { putRecordsRequestEntry.setPartitionKey(StringJsonUnmarshaller.getInstance() .unmarshall(context)); } else { reader.skipValue(); } } reader.endObject(); return putRecordsRequestEntry; }
@Test public void test() throws Exception { PutRecordsRequest putRecordsRequest = new PutRecordsRequest(); putRecordsRequest.setStreamName("stream name"); List<PutRecordsRequestEntry> records = new ArrayList<PutRecordsRequestEntry>(); for (int i = 0; i < 10; i++) { PutRecordsRequestEntry record = new PutRecordsRequestEntry(); String randomStr = RandomStringUtils.random(8 * 1024); record.setData(ByteBuffer.wrap(randomStr.getBytes(StringUtils.UTF8))); record.setPartitionKey("partition key"); records.add(record); } putRecordsRequest.setRecords(records); PutRecordsRequestMarshaller marshaller = new PutRecordsRequestMarshaller(); Request<PutRecordsRequest> request = marshaller.marshall(putRecordsRequest); assertEquals("content encoding", "gzip", request.getHeaders().get("Content-Encoding")); byte[] content = IOUtils.toByteArray(request.getContent()); assertEquals("content length", request.getHeaders().get("Content-Length"), String.valueOf(content.length)); GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(content)); String str = IOUtils.toString(gis); assertTrue("content is compressed", content.length < str.length()); Map<String, String> map = JsonUtils.jsonToMap(str); assertEquals("StreamName", "stream name", map.get("StreamName")); } }
private void addRecord(T tuple) { try { Pair<String, V> keyValue = tupleToKeyValue(tuple); PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry(); putRecordsEntry.setData(ByteBuffer.wrap(getRecord(keyValue.second))); putRecordsEntry.setPartitionKey(keyValue.first); putRecordsRequestEntryList.add(putRecordsEntry); } catch (AmazonClientException e) { throw new RuntimeException(e); } }
private void addRecord(T tuple) { try { Pair<String, V> keyValue = tupleToKeyValue(tuple); PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry(); putRecordsEntry.setData(ByteBuffer.wrap(getRecord(keyValue.second))); putRecordsEntry.setPartitionKey(keyValue.first); putRecordsRequestEntryList.add(putRecordsEntry); } catch (AmazonClientException e) { throw new RuntimeException(e); } }
public Observable<PutRecordsResultEntry> play(LocalDateTime start, @Nullable LocalDateTime end) { return playableObjects(start, end) .onBackpressureBuffer() .observeOn(Schedulers.io()) .flatMap(this::objectToPayloads) .map(ByteBuffer::wrap) .lift(new OperatorBufferKinesisBatch(MAX_KINESIS_BATCH_SIZE, MAX_KINESIS_BATCH_WEIGHT)) .onBackpressureBuffer() .map(byteBuffers -> byteBuffers.stream() .map(buffer -> new PutRecordsRequestEntry() .withData(buffer) .withPartitionKey(UUID.randomUUID().toString())) .collect(toList())) .map(entries -> new PutRecordsRequest() .withStreamName(vcrConfiguration.targetStream) .withRecords(entries)) .observeOn(Schedulers.io()) .flatMap(putRecordsRequest -> Observable.create((Observable.OnSubscribe<List<PutRecordsResult>>) os -> { os.onStart(); kinesisWriter.submit(() -> { os.onNext(putWithRetry(putRecordsRequest).orElse(Collections.<PutRecordsResult>emptyList())); os.onCompleted(); }); })) .flatMap(Observable::from) .flatMap(putRecordsResult -> Observable.from(putRecordsResult.getRecords())) .doOnNext(result -> LOGGER.debug("Wrote record. Seq {}, shard {}", result.getSequenceNumber(), result.getShardId())); }
PutRecordsRequestEntry record = new PutRecordsRequestEntry().withData(ByteBuffer.wrap(baos.toByteArray()));
List<PutRecordsRequestEntry> requestRecords = new ArrayList<>(); for(KinesisRecord data : buffer) { PutRecordsRequestEntry record = new PutRecordsRequestEntry(); record.setData(data.data()); record.setPartitionKey(data.partitionKey());
private void generateRecords() { // Create dummy message int recordNo = 1; while (recordNo <= sendCount) { String dataStr = "Record_" + recordNo; PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry(); putRecordsEntry.setData(ByteBuffer.wrap(dataStr.getBytes())); putRecordsEntry.setPartitionKey(dataStr); putRecordsRequestEntryList.add(putRecordsEntry); if ( (putRecordsRequestEntryList.size() == batchSize) || (recordNo == sendCount )) { PutRecordsRequest putRecordsRequest = new PutRecordsRequest(); putRecordsRequest.setStreamName(streamName); putRecordsRequest.setRecords(putRecordsRequestEntryList); client.putRecords(putRecordsRequest); putRecordsRequestEntryList.clear(); } recordNo++; } }
PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry() .withData(getBuffer(event)) .withPartitionKey(event.project() + "|" + event.collection());
.withRecords(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"))); .containsExactlyInAnyOrder(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"));
public void storeBatchInline(List<Event> events, int offset, int limit) { PutRecordsRequestEntry[] records = new PutRecordsRequestEntry[limit]; for (int i = 0; i < limit; i++) { Event event = events.get(offset + i); PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry() .withData(getBuffer(event)) .withPartitionKey(event.project() + "|" + event.collection()); records[i] = putRecordsRequestEntry; } try { PutRecordsResult putRecordsResult = kinesis.putRecords(new PutRecordsRequest() .withRecords(records) .withStreamName(config.getEventStoreStreamName())); if (putRecordsResult.getFailedRecordCount() > 0) { for (PutRecordsResultEntry resultEntry : putRecordsResult.getRecords()) { resultEntry.getErrorMessage(); } } } catch (ResourceNotFoundException e) { try { createAndWaitForStreamToBecomeAvailable(kinesis, config.getEventStoreStreamName(), 1); } catch (Exception e1) { throw new RuntimeException("Couldn't send event to Amazon Kinesis", e); } } }
.withRecords(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"))); .containsExactlyInAnyOrder(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey")); .withRecords(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"))); assertThat(((PutRecordsRequest) putRecordsFailure.getRequest()).getStreamName()).isEqualTo("myStream"); assertThat(((PutRecordsRequest) putRecordsFailure.getRequest()).getRecords()) .containsExactlyInAnyOrder(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"));
public PutRecordsRequestEntry unmarshall(JsonUnmarshallerContext context) throws Exception { PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
for (String row : partition) { allRecords.add( new PutRecordsRequestEntry() .withData(ByteBuffer.wrap(row.getBytes(StandardCharsets.UTF_8))) .withPartitionKey(Integer.toString(row.hashCode())));