public static Notification forPartitions(DatasetId datasetId, Collection<? extends PartitionKey> partitionKeys) { Map<String, String> properties = new HashMap<>(); properties.put(DATASET_ID, datasetId.toString()); properties.put(NUM_PARTITIONS, Integer.toString(partitionKeys.size())); properties.put(PARTITION_KEYS, GSON.toJson(partitionKeys)); return new Notification(Notification.Type.PARTITION, properties); }
public static Notification forPartitions(DatasetId datasetId, Collection<? extends PartitionKey> partitionKeys) { Map<String, String> properties = new HashMap<>(); properties.put(DATASET_ID, datasetId.toString()); properties.put(NUM_PARTITIONS, Integer.toString(partitionKeys.size())); properties.put(PARTITION_KEYS, GSON.toJson(partitionKeys)); return new Notification(Notification.Type.PARTITION, properties); }
private int getPartitionsCount(List<Notification> notifications) { int partitionsCount = 0; for (Notification notification : notifications) { if (!notification.getNotificationType().equals(Notification.Type.PARTITION)) { continue; } String datasetId = notification.getProperties().get(Notification.DATASET_ID); if (!dataset.toString().equals(datasetId)) { continue; } String numPartitionsString = notification.getProperties().get(Notification.NUM_PARTITIONS); if (numPartitionsString != null) { partitionsCount += Integer.parseInt(numPartitionsString); } } return partitionsCount; }
private int getPartitionsCount(List<Notification> notifications) { int partitionsCount = 0; for (Notification notification : notifications) { if (!notification.getNotificationType().equals(Notification.Type.PARTITION)) { continue; } String datasetId = notification.getProperties().get(Notification.DATASET_ID); if (!dataset.toString().equals(datasetId)) { continue; } String numPartitionsString = notification.getProperties().get(Notification.NUM_PARTITIONS); if (numPartitionsString != null) { partitionsCount += Integer.parseInt(numPartitionsString); } } return partitionsCount; }
futures.put(datasetId.toString(), future);
protected QueryHandle doDisableExploreDataset(DatasetId datasetInstance, DatasetSpecification spec) throws ExploreException { String body = spec == null ? null : GSON.toJson(new DisableExploreParameters(spec)); String endpoint = spec == null ? "disable" : "disable-internal"; HttpResponse response = doPost(String.format("namespaces/%s/data/explore/datasets/%s/%s", datasetInstance.getNamespace(), datasetInstance.getEntityName(), endpoint), body, null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot disable explore on dataset %s. Reason: %s", datasetInstance.toString(), response)); }
protected QueryHandle doUpdateExploreDataset(DatasetId datasetInstance, DatasetSpecification oldSpec, DatasetSpecification newSpec) throws ExploreException { HttpResponse response = doPost(String.format("namespaces/%s/data/explore/datasets/%s/update", datasetInstance.getNamespace(), datasetInstance.getEntityName()), GSON.toJson(new UpdateExploreParameters(oldSpec, newSpec)), null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot update explore on dataset %s. Reason: %s", datasetInstance.toString(), response)); }
protected QueryHandle doUpdateExploreDataset(DatasetId datasetInstance, DatasetSpecification oldSpec, DatasetSpecification newSpec) throws ExploreException { HttpResponse response = doPost(String.format("namespaces/%s/data/explore/datasets/%s/update", datasetInstance.getNamespace(), datasetInstance.getEntityName()), GSON.toJson(new UpdateExploreParameters(oldSpec, newSpec)), null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot update explore on dataset %s. Reason: %s", datasetInstance.toString(), response)); }
protected QueryHandle doEnableExploreDataset(DatasetId datasetInstance, DatasetSpecification spec, boolean truncating) throws ExploreException { String body = spec == null ? null : GSON.toJson(new EnableExploreParameters(spec, truncating)); String endpoint = spec == null ? "enable" : "enable-internal"; HttpResponse response = doPost(String.format("namespaces/%s/data/explore/datasets/%s/%s", datasetInstance.getNamespace(), datasetInstance.getEntityName(), endpoint), body, null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot enable explore on dataset %s. Reason: %s", datasetInstance.toString(), response)); }
protected QueryHandle doEnableExploreDataset(DatasetId datasetInstance, DatasetSpecification spec, boolean truncating) throws ExploreException { String body = spec == null ? null : GSON.toJson(new EnableExploreParameters(spec, truncating)); String endpoint = spec == null ? "enable" : "enable-internal"; HttpResponse response = doPost(String.format("namespaces/%s/data/explore/datasets/%s/%s", datasetInstance.getNamespace(), datasetInstance.getEntityName(), endpoint), body, null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot enable explore on dataset %s. Reason: %s", datasetInstance.toString(), response)); }
protected QueryHandle doDisableExploreDataset(DatasetId datasetInstance, DatasetSpecification spec) throws ExploreException { String body = spec == null ? null : GSON.toJson(new DisableExploreParameters(spec)); String endpoint = spec == null ? "disable" : "disable-internal"; HttpResponse response = doPost(String.format("namespaces/%s/data/explore/datasets/%s/%s", datasetInstance.getNamespace(), datasetInstance.getEntityName(), endpoint), body, null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot disable explore on dataset %s. Reason: %s", datasetInstance.toString(), response)); }
private QueryHandle doPartitionOperation(DatasetId datasetId, DatasetSpecification spec, PartitionKey key, String endpoint, String operationName, Map<String, String> additionalArguments) throws ExploreException { Map<String, String> args = new HashMap<>(additionalArguments); PartitionedFileSetArguments.setOutputPartitionKey(args, key); String tableName = ExploreProperties.getExploreTableName(spec.getProperties()); String databaseName = ExploreProperties.getExploreDatabaseName(spec.getProperties()); if (tableName != null) { args.put(ExploreProperties.PROPERTY_EXPLORE_TABLE_NAME, tableName); } if (databaseName != null) { args.put(ExploreProperties.PROPERTY_EXPLORE_DATABASE_NAME, databaseName); } HttpResponse response = doPost(String.format("namespaces/%s/data/explore/datasets/%s/%s", datasetId.getNamespace(), datasetId.getEntityName(), endpoint), GSON.toJson(args), null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot %s partition with key %s in dataset %s. Reason: %s", operationName, key, datasetId.toString(), response)); }
private QueryHandle doPartitionOperation(DatasetId datasetId, DatasetSpecification spec, PartitionKey key, String endpoint, String operationName, Map<String, String> additionalArguments) throws ExploreException { Map<String, String> args = new HashMap<>(additionalArguments); PartitionedFileSetArguments.setOutputPartitionKey(args, key); String tableName = ExploreProperties.getExploreTableName(spec.getProperties()); String databaseName = ExploreProperties.getExploreDatabaseName(spec.getProperties()); if (tableName != null) { args.put(ExploreProperties.PROPERTY_EXPLORE_TABLE_NAME, tableName); } if (databaseName != null) { args.put(ExploreProperties.PROPERTY_EXPLORE_DATABASE_NAME, databaseName); } HttpResponse response = doPost(String.format("namespaces/%s/data/explore/datasets/%s/%s", datasetId.getNamespace(), datasetId.getEntityName(), endpoint), GSON.toJson(args), null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot %s partition with key %s in dataset %s. Reason: %s", operationName, key, datasetId.toString(), response)); }
@Test public void testWildCardEntities() throws Exception { // in-mem authorizer does not support wildcard privileges so we can't actually enforce after grant. // this test is just to check that our grant apis work fine. After grant we list privileges to check for existence DatasetId datasetId = new DatasetId("ns", "wildcard"); String wildcardEntityStar = datasetId.toString().replace("wildcard", "*"); client.grant(Authorizable.fromString(wildcardEntityStar), admin, ImmutableSet.of(Action.ADMIN)); String wildcardEntityQuestion = datasetId.toString().replace("wildcard", "someSt?"); client.grant(Authorizable.fromString(wildcardEntityQuestion), admin, ImmutableSet.of(Action.ADMIN)); Set<Privilege> privileges = client.listPrivileges(admin); Assert.assertTrue(privileges.contains(new Privilege(Authorizable.fromString(wildcardEntityStar), Action.ADMIN))); Assert.assertTrue(privileges.contains(new Privilege(Authorizable.fromString(wildcardEntityQuestion), Action.ADMIN))); // revoke privilege from the wildcard entity and it should not exist anymore client.revoke(Authorizable.fromString(wildcardEntityQuestion), admin, ImmutableSet.of(Action.ADMIN)); privileges = client.listPrivileges(admin); Assert.assertFalse(privileges.contains(new Privilege(Authorizable.fromString(wildcardEntityQuestion), Action.ADMIN))); // revoke all should work too String authFromPrivilege = null; for (Privilege privilege : privileges) { if (privilege.getAuthorizable().getEntityType().equals(EntityType.DATASET)) { authFromPrivilege = privilege.getAuthorizable().toString(); } } client.revoke(Authorizable.fromString(authFromPrivilege)); privileges = client.listPrivileges(admin); Assert.assertFalse(privileges.contains(new Privilege(Authorizable.fromString(wildcardEntityStar), Action.ADMIN))); }
Assert.assertArrayEquals(aliceEntity.toString().getBytes(StandardCharsets.UTF_8), token.getIdentifier()); Assert.assertArrayEquals(aliceEntity.toString().getBytes(StandardCharsets.UTF_8), token.getPassword()); Assert.assertEquals(new Text("entity"), token.getKind()); Assert.assertEquals(new Text("service"), token.getService());