@Override public boolean hasNext() { return index < pSpec.getPartitions().size(); }
private void getPartitionsWithVals(List<String> filters, int expectedPartitions) throws TException { // get partitions from "trusted" API List<Partition> partitions = client.listPartitions(dbName, tblName, filters, (short) -1); GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); projectSpec.setFieldList(Arrays.asList("sd.location")); request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_VALUES); request.getFilterSpec().setFilters(filters); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); Assert.assertNotNull(response); if (expectedPartitions > 0) { PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); Assert.assertNotNull(partitionSpecWithSharedSD); Assert.assertEquals("Invalid number of partitions returned", expectedPartitions, partitionSpecWithSharedSD.getPartitionsSize()); verifyLocations(partitions, partitionSpecWithSharedSD.getSd(), partitionSpecWithSharedSD.getPartitions()); } else { Assert.assertTrue( "Partition spec should have been empty since filter doesn't match with any partitions", response.getPartitionSpec().isEmpty()); } }
@Override public void remove() { pSpec.getPartitions().remove(index); }
private void getPartitionsWithNames(List<String> names, int expectedPartitionCount) throws TException { GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); projectSpec.setFieldList(Arrays.asList("sd.location")); request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_NAMES); request.getFilterSpec().setFilters(names); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); Assert.assertNotNull(response); if (expectedPartitionCount > 0) { PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); Assert.assertNotNull(partitionSpecWithSharedSD); Assert.assertEquals("Invalid number of partitions returned", expectedPartitionCount, partitionSpecWithSharedSD.getPartitionsSize()); List<Partition> origPartitions = client.getPartitionsByNames(dbName, tblName, names); verifyLocations(origPartitions, partitionSpecWithSharedSD.getSd(), partitionSpecWithSharedSD.getPartitions()); } else { Assert.assertTrue( "Partition spec should have been empty since filter doesn't match with any partitions", response.getPartitionSpec().isEmpty()); } }
public Object getFieldValue(_Fields field) { switch (field) { case PARTITIONS: return getPartitions(); case SD: return getSd(); } throw new IllegalStateException(); }
@Override public void setCreateTime(long time) { pSpec.getPartitions().get(index).setCreateTime((int)time); }
@Override public Map<String, String> getParameters() { return pSpec.getPartitions().get(index).getParameters(); }
@Override public void setParameters(Map<String, String> parameters) { pSpec.getPartitions().get(index).setParameters(parameters); }
@Override public void putToParameters(String key, String value) { pSpec.getPartitions().get(index).putToParameters(key, value); }
@Override public String getLocation() { return pSpec.getSd().getLocation() + pSpec.getPartitions().get(index).getRelativePath(); }
@Override public Partition getCurrent() { PartitionWithoutSD partWithoutSD = pSpec.getPartitions().get(index); StorageDescriptor partSD = new StorageDescriptor(pSpec.getSd()); partSD.setLocation(partSD.getLocation() + partWithoutSD.getRelativePath()); Partition p = new Partition( partWithoutSD.getValues(), partitionSpecWithSharedSDProxy.partitionSpec.getDbName(), partitionSpecWithSharedSDProxy.partitionSpec.getTableName(), partWithoutSD.getCreateTime(), partWithoutSD.getLastAccessTime(), partSD, partWithoutSD.getParameters() ); p.setCatName(partitionSpecWithSharedSDProxy.partitionSpec.getCatName()); return p; }
private void validateBasic(GetPartitionsResponse response) throws TException { Assert.assertNotNull("Response is null", response); Assert.assertNotNull("Returned partition spec is null", response.getPartitionSpec()); Assert.assertEquals(1, response.getPartitionSpecSize()); PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); Assert.assertNotNull(partitionSpecWithSharedSD.getSd()); StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); Assert.assertEquals("Root location should be set to table location", tbl.getSd().getLocation(), sharedSD.getLocation()); List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); Assert.assertEquals(origPartitions.size(), partitionWithoutSDS.size()); for (int i = 0; i < origPartitions.size(); i++) { Partition origPartition = origPartitions.get(i); PartitionWithoutSD returnedPartitionWithoutSD = partitionWithoutSDS.get(i); Assert.assertEquals(String.format("Location returned for Partition %d is not correct", i), origPartition.getSd().getLocation(), sharedSD.getLocation() + returnedPartitionWithoutSD.getRelativePath()); } }
PartitionSpec ps = result.get(0); assertThat(ps.getRootPath(), is((String)null)); List<PartitionWithoutSD> partitions = ps.getSharedSDPartitionSpec().getPartitions(); assertThat(partitions.size(), is(1)); PartitionWithoutSD partition = partitions.get(0);
@Test public void testPartitionProjectionExcludeParameters() throws Throwable { GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); projectSpec .setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values")); projectSpec.setExcludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%"); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); Assert.assertNotNull("All the partitions should be returned in sharedSD spec", partitionSpecWithSharedSD); PartitionListComposingSpec partitionListComposingSpec = response.getPartitionSpec().get(0).getPartitionList(); Assert.assertNull("Partition list composing spec should be null", partitionListComposingSpec); for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) { Assert.assertFalse("excluded parameter key is found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1")); Assert.assertFalse("excluded parameter key is found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2")); } }
@Test public void testPartitionProjectionIncludeParameters() throws Throwable { GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); projectSpec .setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values")); projectSpec.setIncludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%"); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); Assert.assertNotNull("All the partitions should be returned in sharedSD spec", partitionSpecWithSharedSD); PartitionListComposingSpec partitionListComposingSpec = response.getPartitionSpec().get(0).getPartitionList(); Assert.assertNull("Partition list composing spec should be null since all the " + "partitions are expected to be in sharedSD spec", partitionListComposingSpec); for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) { Assert.assertTrue("included parameter key is not found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1")); Assert.assertTrue("included parameter key is not found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2")); Assert.assertEquals("Additional parameters returned other than inclusion keys", 2, retPartion.getParameters().size()); } }
@Test public void testPartitionProjectionIncludeExcludeParameters() throws Throwable { GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); projectSpec .setFieldList(Arrays.asList("dbName", "tableName", "catName", "parameters", "values")); // test parameter key inclusion using setIncludeParamKeyPattern projectSpec.setIncludeParamKeyPattern(EXCLUDE_KEY_PREFIX + "%"); projectSpec.setExcludeParamKeyPattern("%key1%"); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); Assert.assertNotNull("All the partitions should be returned in sharedSD spec", partitionSpecWithSharedSD); PartitionListComposingSpec partitionListComposingSpec = response.getPartitionSpec().get(0).getPartitionList(); Assert.assertNull("Partition list composing spec should be null since all the " + "partitions are expected to be in sharedSD spec", partitionListComposingSpec); for (PartitionWithoutSD retPartion : partitionSpecWithSharedSD.getPartitions()) { Assert.assertFalse("excluded parameter key is found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key1")); Assert.assertTrue("included parameter key is not found in the response", retPartion.getParameters().containsKey(EXCLUDE_KEY_PREFIX + "key2")); Assert.assertEquals("Additional parameters returned other than inclusion keys", 1, retPartion.getParameters().size()); } }
PartitionSpec ps = result.get(0); assertThat(ps.getRootPath(), is(tbl.getSd().getLocation())); List<PartitionWithoutSD> partitions = ps.getSharedSDPartitionSpec().getPartitions(); assertThat(partitions.size(), is(1)); PartitionWithoutSD partition = partitions.get(0);
@Test public void testPartitionProjectionAllSingleValuedFields() throws Throwable { GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); List<String> projectedFields = Arrays .asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", "sd.serdeInfo.name", "sd.serdeInfo.serializationLib"/*, "sd.serdeInfo.serdeType"*/); //TODO directSQL does not support serdeType, serializerClass and deserializerClass in serdeInfo projectSpec.setFieldList(projectedFields); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); Assert.assertEquals(1, response.getPartitionSpec().size()); PartitionSpec partitionSpec = response.getPartitionSpec().get(0); Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName()); Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName()); PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); Assert.assertNotNull(sharedSD); List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); Assert.assertNotNull(partitionWithoutSDS); Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size()); comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0); }
List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); Assert.assertNotNull(partitionWithoutSDS); Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size()); partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); Assert.assertNotNull(partitionWithoutSDS); Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size());
Assert.assertEquals(origSd, sharedSDCopy); List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); Assert.assertNotNull(partitionWithoutSDS); Assert.assertEquals("Unexpected number of partitions returned",