private void addOperationToBatch(DynamicTableEntity entity, IndexedRecord record) throws IOException { if (latestPartitionKey == null || latestPartitionKey.isEmpty()) { latestPartitionKey = entity.getPartitionKey(); } // we reached the threshold for batch OR changed PartitionKey if (batchOperationsCount == 100 || !entity.getPartitionKey().equals(latestPartitionKey)) { processBatch(); latestPartitionKey = entity.getPartitionKey(); } TableOperation to = getTableOperation(entity); batchOperations.add(to); batchRecords.add(record); batchOperationsCount++; latestPartitionKey = entity.getPartitionKey(); }
private void addOperationToBatch(DynamicTableEntity entity, IndexedRecord record) throws IOException { if (latestPartitionKey == null || latestPartitionKey.isEmpty()) { latestPartitionKey = entity.getPartitionKey(); } // we reached the threshold for batch OR changed PartitionKey if (batchOperationsCount == 100 || !entity.getPartitionKey().equals(latestPartitionKey)) { processBatch(); latestPartitionKey = entity.getPartitionKey(); } TableOperation to = getTableOperation(entity); batchOperations.add(to); batchRecords.add(record); batchOperationsCount++; latestPartitionKey = entity.getPartitionKey(); }
@Override public Object convertToAvro(DynamicTableEntity value) { try { if (COL_PARITION_KEY.equals(f.name()) || COL_PARITION_KEY.equals(mappedName)) { return value.getPartitionKey(); } if (COL_ROW_KEY.equals(f.name()) || COL_ROW_KEY.equals(mappedName)) { return value.getRowKey(); } if (f.name().equals(COL_TIMESTAMP) || mappedName.equals(COL_TIMESTAMP)) { // should be set to DT // but... String pattern = f.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); if (pattern != null && !pattern.isEmpty()) return new SimpleDateFormat(pattern).format(value.getTimestamp()); else return value.getTimestamp(); } if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) { return null; } return value.getProperties().get(mappedName).getValueAsString(); } catch (Exception e) { LOGGER.error(i18nMessages.getMessage("error.ConversionError", e)); throw new ComponentException(e); } } }
@Override public Object convertToAvro(DynamicTableEntity value) { try { if (COL_PARITION_KEY.equals(f.name()) || COL_PARITION_KEY.equals(mappedName)) { return value.getPartitionKey(); } if (COL_ROW_KEY.equals(f.name()) || COL_ROW_KEY.equals(mappedName)) { return value.getRowKey(); } if (f.name().equals(COL_TIMESTAMP) || mappedName.equals(COL_TIMESTAMP)) { // should be set to DT // but... String pattern = f.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); if (pattern != null && !pattern.isEmpty()) return new SimpleDateFormat(pattern).format(value.getTimestamp()); else return value.getTimestamp(); } if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) { return null; } return value.getProperties().get(mappedName).getValueAsString(); } catch (Exception e) { LOGGER.error(i18nMessages.getMessage("error.ConversionError", e)); throw new ComponentException(e); } } }
@NotNull private static TableEntity getTableEntity(@NotNull String tableName, @NotNull DynamicTableEntity dte) { String partitionKey = Strings.nullToEmpty(dte.getPartitionKey()); String rowKey = Strings.nullToEmpty(dte.getRowKey()); String eTag = Strings.nullToEmpty(dte.getEtag());
assertEquals(firstIteration.get(m).getPartitionKey(), secondIteration.get(m).getPartitionKey()); assertEquals(firstIteration.get(m).getRowKey(), secondIteration.get(m).getRowKey()); assertEquals(firstIteration.get(m).getProperties().size(), secondIteration.get(m).getProperties().size());
@Test public void testTableOperationRetrieveJsonNoMetadataFail() { // set custom property resolver this.options.setPropertyResolver(new CustomPropertyResolver()); try { this.table.execute(TableOperation.retrieve(this.ent.getPartitionKey(), this.ent.getRowKey(), Class1.class), this.options, null); fail("Invalid property resolver should throw"); } catch (StorageException e) { assertEquals("Failed to parse property 'fooint' with value '1234' as type 'Edm.Guid'", e.getMessage()); } }
@Test public void testTableOperationRetrieveJsonNoMetadataResolverFail() { // set custom property resolver which throws this.options.setPropertyResolver(new ThrowingPropertyResolver()); try { this.table.execute(TableOperation.retrieve(this.ent.getPartitionKey(), this.ent.getRowKey(), Class1.class), this.options, null); fail("Invalid property resolver should throw"); } catch (StorageException e) { assertEquals( "The custom property resolver delegate threw an exception. Check the inner exception for more details.", e.getMessage()); assertTrue(e.getCause().getClass() == IllegalArgumentException.class); } }
entity.getPartitionKey() + " " + entity.getRowKey()