private Status readEntity(String key, Map<String, ByteIterator> result) { try { // firstly, retrieve the entity to be deleted TableOperation retrieveOp = TableOperation.retrieve(partitionKey, key, DynamicTableEntity.class); DynamicTableEntity entity = cloudTable.execute(retrieveOp).getResultAsType(); HashMap<String, EntityProperty> properties = entity.getProperties(); for (Entry<String, EntityProperty> entry: properties.entrySet()) { String fieldName = entry.getKey(); ByteIterator fieldVal = new ByteArrayByteIterator(entry.getValue().getValueAsByteArray()); result.put(fieldName, fieldVal); } return Status.OK; } catch (Exception e) { return Status.ERROR; } }
private Status insertOrUpdate(String key, Map<String, ByteIterator> values) { HashMap<String, EntityProperty> properties = new HashMap<String, EntityProperty>(); for (Entry<String, ByteIterator> entry : values.entrySet()) { String fieldName = entry.getKey(); byte[] fieldVal = entry.getValue().toArray(); properties.put(fieldName, new EntityProperty(fieldVal)); } DynamicTableEntity entity = new DynamicTableEntity(partitionKey, key, properties); TableOperation insertOrReplace = TableOperation.insertOrReplace(entity); try { cloudTable.execute(insertOrReplace); return Status.OK; } catch (Exception e) { return Status.ERROR; } }
/** * Deletes the table from the storage service, using the specified request options and operation context. * * @param options * A {@link TableRequestOptions} object that specifies any additional options for the request. Specifying * <code>null</code> will use the default request options from the associated service client ( * {@link CloudTableClient}). * @param opContext * An {@link OperationContext} object that represents the context for the current operation. This object * is used to track requests to the storage service, and to provide additional runtime information about * the operation. * * @throws StorageException * If a storage service error occurred during the operation. */ @DoesServiceRequest public void delete(TableRequestOptions options, OperationContext opContext) throws StorageException { if (opContext == null) { opContext = new OperationContext(); } opContext.initialize(); options = TableRequestOptions.populateAndApplyDefaults(options, this.tableServiceClient); Utility.assertNotNullOrEmpty("tableName", this.name); final DynamicTableEntity tableEntry = new DynamicTableEntity(); tableEntry.getProperties().put(TableConstants.TABLE_NAME, new EntityProperty(this.name)); TableOperation deleteOp = new TableOperation(tableEntry, TableOperationType.DELETE); deleteOp.execute(this.tableServiceClient, TableConstants.TABLES_SERVICE_TABLES_NAME, options, opContext); }
@Before public void tableODataTestBeforeMethodSetUp() throws StorageException, URISyntaxException { this.table = TableTestHelper.getRandomTableReference(); this.table.createIfNotExists(); final CloudTableClient tClient = TableTestHelper.createCloudTableClient(); this.options = TableRequestOptions.populateAndApplyDefaults(this.options, tClient); this.options.setTablePayloadFormat(TablePayloadFormat.JsonNoMetadata); // Insert Entity this.ent = new DynamicTableEntity(); this.ent.setPartitionKey("jxscl_odata"); this.ent.setRowKey(UUID.randomUUID().toString()); this.ent.getProperties().put("foo2", new EntityProperty("bar2")); this.ent.getProperties().put("foo", new EntityProperty("bar")); this.ent.getProperties().put("fooint", new EntityProperty(1234)); this.table.execute(TableOperation.insert(this.ent), this.options, null); }
@Override public Object convertToAvro(DynamicTableEntity value) { try { if (COL_PARITION_KEY.equals(f.name()) || COL_PARITION_KEY.equals(mappedName)) { return value.getPartitionKey(); } if (COL_ROW_KEY.equals(f.name()) || COL_ROW_KEY.equals(mappedName)) { return value.getRowKey(); } if (f.name().equals(COL_TIMESTAMP) || mappedName.equals(COL_TIMESTAMP)) { // should be set to DT // but... String pattern = f.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); if (pattern != null && !pattern.isEmpty()) return new SimpleDateFormat(pattern).format(value.getTimestamp()); else return value.getTimestamp(); } if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) { return null; } return value.getProperties().get(mappedName).getValueAsString(); } catch (Exception e) { LOGGER.error(i18nMessages.getMessage("error.ConversionError", e)); throw new ComponentException(e); } } }
assertEquals(ent.getProperties().size(), 4); assertEquals(ent.getProperties().get("A").getValueAsString(), randEnt.getA()); assertEquals(ent.getProperties().get("B").getValueAsString(), randEnt.getB()); assertEquals(ent.getProperties().get("C").getValueAsString(), randEnt.getC()); assertTrue(Arrays.equals(ent.getProperties().get("D").getValueAsByteArray(), randEnt.getD())); firstIteration.add(ent); assertEquals(ent.getProperties().size(), 4); assertEquals(ent.getProperties().get("A").getValueAsString(), randEnt.getA()); assertEquals(ent.getProperties().get("B").getValueAsString(), randEnt.getB()); assertEquals(ent.getProperties().get("C").getValueAsString(), randEnt.getC()); assertTrue(Arrays.equals(ent.getProperties().get("D").getValueAsByteArray(), randEnt.getD())); secondIteration.add(ent); assertEquals(firstIteration.get(m).getPartitionKey(), secondIteration.get(m).getPartitionKey()); assertEquals(firstIteration.get(m).getRowKey(), secondIteration.get(m).getRowKey()); assertEquals(firstIteration.get(m).getProperties().size(), secondIteration.get(m).getProperties().size()); assertEquals(firstIteration.get(m).getProperties().get("A").getValueAsString(), secondIteration.get(m) .getProperties().get("A").getValueAsString()); assertEquals(firstIteration.get(m).getProperties().get("B").getValueAsString(), secondIteration.get(m) .getProperties().get("B").getValueAsString()); assertEquals(firstIteration.get(m).getProperties().get("C").getValueAsString(), secondIteration.get(m) .getProperties().get("C").getValueAsString()); assertTrue(Arrays.equals(firstIteration.get(m).getProperties().get("D").getValueAsByteArray(), secondIteration.get(m).getProperties().get("D").getValueAsByteArray()));
private DynamicTableEntity createDynamicEntityFromInputRecord(IndexedRecord indexedRecord, Schema schema) { DynamicTableEntity entity = new DynamicTableEntity(); HashMap<String, EntityProperty> entityProps = new HashMap<>(); for (Field f : schema.getFields()) { entity.setPartitionKey((String) indexedRecord.get(f.pos())); } else if (sName.equals(rowKey)) { entity.setRowKey((String) indexedRecord.get(f.pos())); } else if (mName.equals(AzureStorageTableProperties.TABLE_TIMESTAMP)) { entity.setEtag("*"); entity.setProperties(entityProps); return entity;
@Override public Object convertToAvro(DynamicTableEntity value) { try { if (COL_TIMESTAMP.equals(f.name()) || COL_TIMESTAMP.equals(mappedName)) { return value.getTimestamp(); } if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) { return null; } return value.getProperties().get(mappedName).getValueAsDate(); } catch (Exception e) { LOGGER.error(i18nMessages.getMessage("error.ConversionError", e)); throw new ComponentException(e); } } }
assertNotNull("Property D", retrievedEntity.getProperties().get("D")); assertTrue(Arrays.equals(baseEntity.getD(), retrievedEntity.getProperties().get("D").getValueAsByteArray())); retrievedEntity.getProperties().remove("D"); retrievedEntity.setEtag(oldEtag);
@Test public void testTableOperationRetrieveJsonNoMetadataFail() { // set custom property resolver this.options.setPropertyResolver(new CustomPropertyResolver()); try { this.table.execute(TableOperation.retrieve(this.ent.getPartitionKey(), this.ent.getRowKey(), Class1.class), this.options, null); fail("Invalid property resolver should throw"); } catch (StorageException e) { assertEquals("Failed to parse property 'fooint' with value '1234' as type 'Edm.Guid'", e.getMessage()); } }
/** * Constructs a {@link DynamicTableEntity} instance using the specified property map. * * @param properties * A <code>java.util.HashMap</code> containing a map of <code>String</code> property names to * {@link EntityProperty} data typed values to store in the new {@link DynamicTableEntity}. */ public DynamicTableEntity(final HashMap<String, EntityProperty> properties) { this.setProperties(properties); }
private void addOperationToBatch(DynamicTableEntity entity, IndexedRecord record) throws IOException { if (latestPartitionKey == null || latestPartitionKey.isEmpty()) { latestPartitionKey = entity.getPartitionKey(); } // we reached the threshold for batch OR changed PartitionKey if (batchOperationsCount == 100 || !entity.getPartitionKey().equals(latestPartitionKey)) { processBatch(); latestPartitionKey = entity.getPartitionKey(); } TableOperation to = getTableOperation(entity); batchOperations.add(to); batchRecords.add(record); batchOperationsCount++; latestPartitionKey = entity.getPartitionKey(); }
/** * Deletes the table from the storage service, using the specified request options and operation context. * * @param options * A {@link TableRequestOptions} object that specifies any additional options for the request. Specifying * <code>null</code> will use the default request options from the associated service client ( * {@link CloudTableClient}). * @param opContext * An {@link OperationContext} object that represents the context for the current operation. This object * is used to track requests to the storage service, and to provide additional runtime information about * the operation. * * @throws StorageException * If a storage service error occurred during the operation. */ @DoesServiceRequest public void delete(TableRequestOptions options, OperationContext opContext) throws StorageException { if (opContext == null) { opContext = new OperationContext(); } opContext.initialize(); options = TableRequestOptions.populateAndApplyDefaults(options, this.tableServiceClient); Utility.assertNotNullOrEmpty("tableName", this.name); final DynamicTableEntity tableEntry = new DynamicTableEntity(); tableEntry.getProperties().put(TableConstants.TABLE_NAME, new EntityProperty(this.name)); TableOperation deleteOp = new TableOperation(tableEntry, TableOperationType.DELETE); deleteOp.execute(this.tableServiceClient, TableConstants.TABLES_SERVICE_TABLES_NAME, options, opContext); }
@Test public void testInsertEntityWithNumericProperty() throws StorageException { TableRequestOptions options = new TableRequestOptions(); options.setTablePayloadFormat(TablePayloadFormat.Json); DynamicTableEntity ref = new DynamicTableEntity(); String propName = ""; for (int m = 0; m < 255; m++) { propName = propName.concat(Integer.toString(m % 9)); } ref.getProperties().put(propName, new EntityProperty("test")); ref.setPartitionKey("jxscl_odata"); ref.setRowKey(UUID.randomUUID().toString()); try { this.table.execute(TableOperation.insert(ref), options, null); fail(); } catch (TableServiceException ex) { assertEquals(ex.getMessage(), "Bad Request"); assertTrue(ex.getExtendedErrorInformation().getErrorMessage().startsWith("The property name is invalid.")); assertEquals(ex.getExtendedErrorInformation().getErrorCode(), "PropertyNameInvalid"); } }
@Override public Object convertToAvro(DynamicTableEntity value) { try { if (COL_PARITION_KEY.equals(f.name()) || COL_PARITION_KEY.equals(mappedName)) { return value.getPartitionKey(); } if (COL_ROW_KEY.equals(f.name()) || COL_ROW_KEY.equals(mappedName)) { return value.getRowKey(); } if (f.name().equals(COL_TIMESTAMP) || mappedName.equals(COL_TIMESTAMP)) { // should be set to DT // but... String pattern = f.getProp(SchemaConstants.TALEND_COLUMN_PATTERN); if (pattern != null && !pattern.isEmpty()) return new SimpleDateFormat(pattern).format(value.getTimestamp()); else return value.getTimestamp(); } if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) { return null; } return value.getProperties().get(mappedName).getValueAsString(); } catch (Exception e) { LOGGER.error(i18nMessages.getMessage("error.ConversionError", e)); throw new ComponentException(e); } } }
entity.getPartitionKey() + " " + entity.getRowKey() + "\t" + entity.getProperties().get("Email") .getValueAsString() + "\t" + entity.getProperties().get("PhoneNumber") .getValueAsString() + "\t" + entity.getProperties().get("Id") .getValueAsUUID());
private DynamicTableEntity createDynamicEntityFromInputRecord(IndexedRecord indexedRecord, Schema schema) { DynamicTableEntity entity = new DynamicTableEntity(); HashMap<String, EntityProperty> entityProps = new HashMap<>(); for (Field f : schema.getFields()) { entity.setPartitionKey((String) indexedRecord.get(f.pos())); } else if (sName.equals(rowKey)) { entity.setRowKey((String) indexedRecord.get(f.pos())); } else if (mName.equals(AzureStorageTableProperties.TABLE_TIMESTAMP)) { entity.setEtag("*"); entity.setProperties(entityProps); return entity;
@Override public Object convertToAvro(DynamicTableEntity value) { try { if (COL_TIMESTAMP.equals(f.name()) || COL_TIMESTAMP.equals(mappedName)) { return value.getTimestamp(); } if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) { return null; } return value.getProperties().get(mappedName).getValueAsLong(); } catch (Exception e) { LOGGER.error(i18nMessages.getMessage("error.ConversionError", e)); throw new ComponentException(e); } } }
@Test public void testTableOperationRetrieveJsonNoMetadataResolverFail() { // set custom property resolver which throws this.options.setPropertyResolver(new ThrowingPropertyResolver()); try { this.table.execute(TableOperation.retrieve(this.ent.getPartitionKey(), this.ent.getRowKey(), Class1.class), this.options, null); fail("Invalid property resolver should throw"); } catch (StorageException e) { assertEquals( "The custom property resolver delegate threw an exception. Check the inner exception for more details.", e.getMessage()); assertTrue(e.getCause().getClass() == IllegalArgumentException.class); } }
/** * Constructs a {@link DynamicTableEntity} instance using the specified property map. * * @param properties * A <code>java.util.HashMap</code> containing a map of <code>String</code> property names to * {@link EntityProperty} data typed values to store in the new {@link DynamicTableEntity}. */ public DynamicTableEntity(final HashMap<String, EntityProperty> properties) { this.setProperties(properties); }