@Override protected void runOnce() throws Exception { ClickEvent event = inputQueue.take(); String partitionKey = event.getSessionId(); ByteBuffer data = ByteBuffer.wrap( event.getPayload().getBytes("UTF-8")); recordsPut.getAndIncrement(); addEntry(new PutRecordsRequestEntry() .withPartitionKey(partitionKey) .withData(data)); }
public Observable<PutRecordsResultEntry> play(LocalDateTime start, @Nullable LocalDateTime end) { return playableObjects(start, end) .onBackpressureBuffer() .observeOn(Schedulers.io()) .flatMap(this::objectToPayloads) .map(ByteBuffer::wrap) .lift(new OperatorBufferKinesisBatch(MAX_KINESIS_BATCH_SIZE, MAX_KINESIS_BATCH_WEIGHT)) .onBackpressureBuffer() .map(byteBuffers -> byteBuffers.stream() .map(buffer -> new PutRecordsRequestEntry() .withData(buffer) .withPartitionKey(UUID.randomUUID().toString())) .collect(toList())) .map(entries -> new PutRecordsRequest() .withStreamName(vcrConfiguration.targetStream) .withRecords(entries)) .observeOn(Schedulers.io()) .flatMap(putRecordsRequest -> Observable.create((Observable.OnSubscribe<List<PutRecordsResult>>) os -> { os.onStart(); kinesisWriter.submit(() -> { os.onNext(putWithRetry(putRecordsRequest).orElse(Collections.<PutRecordsResult>emptyList())); os.onCompleted(); }); })) .flatMap(Observable::from) .flatMap(putRecordsResult -> Observable.from(putRecordsResult.getRecords())) .doOnNext(result -> LOGGER.debug("Wrote record. Seq {}, shard {}", result.getSequenceNumber(), result.getShardId())); }
PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry() .withData(getBuffer(event)) .withPartitionKey(event.project() + "|" + event.collection()); records[i] = putRecordsRequestEntry;
.withRecords(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"))); .containsExactlyInAnyOrder(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"));
public void storeBatchInline(List<Event> events, int offset, int limit) { PutRecordsRequestEntry[] records = new PutRecordsRequestEntry[limit]; for (int i = 0; i < limit; i++) { Event event = events.get(offset + i); PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry() .withData(getBuffer(event)) .withPartitionKey(event.project() + "|" + event.collection()); records[i] = putRecordsRequestEntry; } try { PutRecordsResult putRecordsResult = kinesis.putRecords(new PutRecordsRequest() .withRecords(records) .withStreamName(config.getEventStoreStreamName())); if (putRecordsResult.getFailedRecordCount() > 0) { for (PutRecordsResultEntry resultEntry : putRecordsResult.getRecords()) { resultEntry.getErrorMessage(); } } } catch (ResourceNotFoundException e) { try { createAndWaitForStreamToBecomeAvailable(kinesis, config.getEventStoreStreamName(), 1); } catch (Exception e1) { throw new RuntimeException("Couldn't send event to Amazon Kinesis", e); } } }
.withRecords(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"))); .containsExactlyInAnyOrder(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey")); .withRecords(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"))); .containsExactlyInAnyOrder(new PutRecordsRequestEntry() .withData(ByteBuffer.wrap("test".getBytes())) .withPartitionKey("testKey"));
new PutRecordsRequestEntry() .withData(ByteBuffer.wrap(row.getBytes(StandardCharsets.UTF_8))) .withPartitionKey(Integer.toString(row.hashCode())));