/** * Convenient method to return the low-level unprocessed items. * * @see BatchWriteItemApi#batchWriteItemUnprocessed(Map) * @see BatchWriteItemSpec#withUnprocessedItems(Map) */ public Map<String, List<WriteRequest>> getUnprocessedItems() { return result.getUnprocessedItems(); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getUnprocessedItems() == null) ? 0 : getUnprocessedItems().hashCode()); hashCode = prime * hashCode + ((getItemCollectionMetrics() == null) ? 0 : getItemCollectionMetrics().hashCode()); hashCode = prime * hashCode + ((getConsumedCapacity() == null) ? 0 : getConsumedCapacity().hashCode()); return hashCode; }
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getUnprocessedItems() != null) sb.append("UnprocessedItems: ").append(getUnprocessedItems()).append(","); if (getItemCollectionMetrics() != null) sb.append("ItemCollectionMetrics: ").append(getItemCollectionMetrics()).append(","); if (getConsumedCapacity() != null) sb.append("ConsumedCapacity: ").append(getConsumedCapacity()); sb.append("}"); return sb.toString(); }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof BatchWriteItemResult == false) return false; BatchWriteItemResult other = (BatchWriteItemResult) obj; if (other.getUnprocessedItems() == null ^ this.getUnprocessedItems() == null) return false; if (other.getUnprocessedItems() != null && other.getUnprocessedItems().equals(this.getUnprocessedItems()) == false) return false; if (other.getItemCollectionMetrics() == null ^ this.getItemCollectionMetrics() == null) return false; if (other.getItemCollectionMetrics() != null && other.getItemCollectionMetrics().equals(this.getItemCollectionMetrics()) == false) return false; if (other.getConsumedCapacity() == null ^ this.getConsumedCapacity() == null) return false; if (other.getConsumedCapacity() != null && other.getConsumedCapacity().equals(this.getConsumedCapacity()) == false) return false; return true; }
/** * Helper method to handle unprocessed items items * @param session process session * @param keysToFlowFileMap map of flow db primary key to flow file * @param table dynamodb table * @param hashKeyName the hash key name * @param hashKeyValueType the hash key value * @param rangeKeyName the range key name * @param rangeKeyValueType range key value * @param outcome the write outcome */ protected void handleUnprocessedItems(final ProcessSession session, Map<ItemKeys, FlowFile> keysToFlowFileMap, final String table, final String hashKeyName, final String hashKeyValueType, final String rangeKeyName, final String rangeKeyValueType, BatchWriteItemOutcome outcome) { BatchWriteItemResult result = outcome.getBatchWriteItemResult(); // Handle unprocessed items List<WriteRequest> unprocessedItems = result.getUnprocessedItems().get(table); if ( unprocessedItems != null && unprocessedItems.size() > 0 ) { for ( WriteRequest request : unprocessedItems) { Map<String,AttributeValue> item = getRequestItem(request); Object hashKeyValue = getValue(item, hashKeyName, hashKeyValueType); Object rangeKeyValue = getValue(item, rangeKeyName, rangeKeyValueType); sendUnprocessedToUnprocessedRelationship(session, keysToFlowFileMap, hashKeyValue, rangeKeyValue); } } }
return failedBatch; pendingItems = result.getUnprocessedItems();
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getUnprocessedItems() == null) ? 0 : getUnprocessedItems().hashCode()); hashCode = prime * hashCode + ((getItemCollectionMetrics() == null) ? 0 : getItemCollectionMetrics().hashCode()); hashCode = prime * hashCode + ((getConsumedCapacity() == null) ? 0 : getConsumedCapacity().hashCode()); return hashCode; }
/** * Returns a string representation of this object; useful for testing and * debugging. * * @return A string representation of this object. * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getUnprocessedItems() != null) sb.append("UnprocessedItems: " + getUnprocessedItems() + ","); if (getItemCollectionMetrics() != null) sb.append("ItemCollectionMetrics: " + getItemCollectionMetrics() + ","); if (getConsumedCapacity() != null) sb.append("ConsumedCapacity: " + getConsumedCapacity()); sb.append("}"); return sb.toString(); }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof BatchWriteItemResult == false) return false; BatchWriteItemResult other = (BatchWriteItemResult) obj; if (other.getUnprocessedItems() == null ^ this.getUnprocessedItems() == null) return false; if (other.getUnprocessedItems() != null && other.getUnprocessedItems().equals(this.getUnprocessedItems()) == false) return false; if (other.getItemCollectionMetrics() == null ^ this.getItemCollectionMetrics() == null) return false; if (other.getItemCollectionMetrics() != null && other.getItemCollectionMetrics().equals(this.getItemCollectionMetrics()) == false) return false; if (other.getConsumedCapacity() == null ^ this.getConsumedCapacity() == null) return false; if (other.getConsumedCapacity() != null && other.getConsumedCapacity().equals(this.getConsumedCapacity()) == false) return false; return true; } }
/** * Continue trying to process the batch until it finishes or an exception * occurs. */ private FailedBatch callUntilCompletion(Map<String, List<WriteRequest>> batch) { BatchWriteItemResult result = null; int retries = 0; FailedBatch failedBatch = null; while (true) { try { result = db.batchWriteItem(applyBatchOperationUserAgent( new BatchWriteItemRequest().withRequestItems(batch))); } catch (final Exception e) { failedBatch = new FailedBatch(); failedBatch.setUnprocessedItems(batch); failedBatch.setException(e); return failedBatch; } retries++; batch = result.getUnprocessedItems(); if (batch.size() > 0) { pauseExponentially(retries); } else { break; } } return failedBatch; }
/** * Convenient method to return the low-level unprocessed items. * * @see BatchWriteItemApi#batchWriteItemUnprocessed(Map) * @see BatchWriteItemSpec#withUnprocessedItems(Map) */ public Map<String, List<WriteRequest>> getUnprocessedItems() { return result.getUnprocessedItems(); }
@Test public void testWriteOneBatchWithEntityTooLarge() { Map<String, List<WriteRequest>> batchMap = new HashMap<String, List<WriteRequest>>(); List<WriteRequest> batchList = new ArrayList<WriteRequest>(); WriteRequest wr1 = new WriteRequest(); WriteRequest wr2 = new WriteRequest(); WriteRequest wr3 = new WriteRequest(); batchList.add(wr1); batchList.add(wr2); batchList.add(wr3); batchMap.put("testTable", batchList); EasyMock.reset(mockClient); AmazonServiceException ase = new AmazonServiceException("TestException"); ase.setErrorCode("Request entity too large"); BatchWriteItemResult mockResult = EasyMock.createMock(BatchWriteItemResult.class); EasyMock.reset(mockResult); EasyMock.expect(mockResult.getUnprocessedItems()).andReturn( new HashMap<String, List<WriteRequest>>()).times(2); // Will cause batches to be split and re-tried EasyMock.expect(mockClient.batchWriteItem(anyObject(BatchWriteItemRequest.class))) .andThrow(ase); EasyMock.expect(mockClient.batchWriteItem(anyObject(BatchWriteItemRequest.class))) .andReturn(mockResult); EasyMock.expect(mockClient.batchWriteItem(anyObject(BatchWriteItemRequest.class))) .andReturn(mockResult); EasyMock.replay(mockClient, mockResult); List<FailedBatch> result = mapper.writeOneBatch(batchMap); assertEquals(result.size(), 0); EasyMock.verify(mockClient); }
@Override public void onSuccess( final BatchWriteItemRequest request, final BatchWriteItemResult result) { retryAsync(result.getUnprocessedItems()); if (futureMap.remove(request) == null) { LOGGER.warn(" Unable to delete BatchWriteRequest from futuresMap "); } } });
private List<Map<String, AttributeValue>> unproccessedItems(BatchWriteItemResult result, Map<WriteRequest, Map<String, AttributeValue>> requestMap) { Collection<List<WriteRequest>> items = result.getUnprocessedItems().values(); List<Map<String, AttributeValue>> unprocessed = new ArrayList<Map<String, AttributeValue>>(); // retrieve the unprocessed items for (List<WriteRequest> list : items) { for (WriteRequest request : list) { unprocessed.add(requestMap.get(request)); } } return unprocessed; }
private List<Map<String, AttributeValue>> unproccessedItems(BatchWriteItemResult result, Map<WriteRequest, Map<String, AttributeValue>> requestMap) { Collection<List<WriteRequest>> items = result.getUnprocessedItems().values(); List<Map<String, AttributeValue>> unprocessed = new ArrayList<Map<String, AttributeValue>>(); // retrieve the unprocessed items for (List<WriteRequest> list : items) { for (WriteRequest request : list) { unprocessed.add(requestMap.get(request)); } } return unprocessed; }
private void writeUntilDone(List<WriteRequest> requests) { List<WriteRequest> remainingRequests = requests; BatchWriteItemResult result; do { result = runBatchWriteRequest(remainingRequests); remainingRequests = result.getUnprocessedItems().get(tableName); } while (remainingRequests!= null && remainingRequests.isEmpty()); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getUnprocessedItems() == null) ? 0 : getUnprocessedItems().hashCode()); hashCode = prime * hashCode + ((getItemCollectionMetrics() == null) ? 0 : getItemCollectionMetrics().hashCode()); hashCode = prime * hashCode + ((getConsumedCapacity() == null) ? 0 : getConsumedCapacity().hashCode()); return hashCode; }
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getUnprocessedItems() != null) sb.append("UnprocessedItems: ").append(getUnprocessedItems()).append(","); if (getItemCollectionMetrics() != null) sb.append("ItemCollectionMetrics: ").append(getItemCollectionMetrics()).append(","); if (getConsumedCapacity() != null) sb.append("ConsumedCapacity: ").append(getConsumedCapacity()); sb.append("}"); return sb.toString(); }
private void batchWrite(Map<String, List<WriteRequest>> items) { if (items == null || items.isEmpty()) { return; } try { BatchWriteItemResult result = client().batchWriteItem(new BatchWriteItemRequest(). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL).withRequestItems(items)); if (result == null) { return; } logger.debug("batchWrite() CC: {}", result.getConsumedCapacity()); if (result.getUnprocessedItems() != null && !result.getUnprocessedItems().isEmpty()) { Thread.sleep(1000); logger.warn("UNPROCESSED {0}", result.getUnprocessedItems().size()); batchWrite(result.getUnprocessedItems()); } } catch (Exception e) { logger.error(null, e); } }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof BatchWriteItemResult == false) return false; BatchWriteItemResult other = (BatchWriteItemResult) obj; if (other.getUnprocessedItems() == null ^ this.getUnprocessedItems() == null) return false; if (other.getUnprocessedItems() != null && other.getUnprocessedItems().equals(this.getUnprocessedItems()) == false) return false; if (other.getItemCollectionMetrics() == null ^ this.getItemCollectionMetrics() == null) return false; if (other.getItemCollectionMetrics() != null && other.getItemCollectionMetrics().equals(this.getItemCollectionMetrics()) == false) return false; if (other.getConsumedCapacity() == null ^ this.getConsumedCapacity() == null) return false; if (other.getConsumedCapacity() != null && other.getConsumedCapacity().equals(this.getConsumedCapacity()) == false) return false; return true; }