public Object getFieldValue(_Fields field) { switch (field) { case PARTITIONS: return getPartitions(); case SD: return getSd(); } throw new IllegalStateException(); }
@Override public String getLocation() { return pSpec.getSd().getLocation() + pSpec.getPartitions().get(index).getRelativePath(); }
@Override public void setRootLocation(String rootLocation) throws MetaException { partitionSpec.setRootPath(rootLocation); partitionSpec.getSharedSDPartitionSpec().getSd().setLocation(rootLocation); }
public PartitionSpecWithSharedSDProxy(PartitionSpec partitionSpec) throws MetaException { assert partitionSpec.isSetSharedSDPartitionSpec(); if (partitionSpec.getSharedSDPartitionSpec().getSd() == null) { throw new MetaException("The shared storage descriptor must be set."); } this.partitionSpec = partitionSpec; }
@Override public Partition getCurrent() { PartitionWithoutSD partWithoutSD = pSpec.getPartitions().get(index); StorageDescriptor partSD = new StorageDescriptor(pSpec.getSd()); partSD.setLocation(partSD.getLocation() + partWithoutSD.getRelativePath()); Partition p = new Partition( partWithoutSD.getValues(), partitionSpecWithSharedSDProxy.partitionSpec.getDbName(), partitionSpecWithSharedSDProxy.partitionSpec.getTableName(), partWithoutSD.getCreateTime(), partWithoutSD.getLastAccessTime(), partSD, partWithoutSD.getParameters() ); p.setCatName(partitionSpecWithSharedSDProxy.partitionSpec.getCatName()); return p; }
private void validateBasic(GetPartitionsResponse response) throws TException { Assert.assertNotNull("Response is null", response); Assert.assertNotNull("Returned partition spec is null", response.getPartitionSpec()); Assert.assertEquals(1, response.getPartitionSpecSize()); PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); Assert.assertNotNull(partitionSpecWithSharedSD.getSd()); StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); Assert.assertEquals("Root location should be set to table location", tbl.getSd().getLocation(), sharedSD.getLocation()); List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); Assert.assertEquals(origPartitions.size(), partitionWithoutSDS.size()); for (int i = 0; i < origPartitions.size(); i++) { Partition origPartition = origPartitions.get(i); PartitionWithoutSD returnedPartitionWithoutSD = partitionWithoutSDS.get(i); Assert.assertEquals(String.format("Location returned for Partition %d is not correct", i), origPartition.getSd().getLocation(), sharedSD.getLocation() + returnedPartitionWithoutSD.getRelativePath()); } }
@Test public void testPartitionProjectionAllSingleValuedFields() throws Throwable { GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); List<String> projectedFields = Arrays .asList("dbName", "tableName", "createTime", "lastAccessTime", "sd.location", "sd.inputFormat", "sd.outputFormat", "sd.compressed", "sd.numBuckets", "sd.serdeInfo.name", "sd.serdeInfo.serializationLib"/*, "sd.serdeInfo.serdeType"*/); //TODO directSQL does not support serdeType, serializerClass and deserializerClass in serdeInfo projectSpec.setFieldList(projectedFields); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); Assert.assertEquals(1, response.getPartitionSpec().size()); PartitionSpec partitionSpec = response.getPartitionSpec().get(0); Assert.assertTrue("DbName is not set", partitionSpec.isSetDbName()); Assert.assertTrue("tableName is not set", partitionSpec.isSetTableName()); PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); Assert.assertNotNull(sharedSD); List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); Assert.assertNotNull(partitionWithoutSDS); Assert.assertEquals(partitionWithoutSDS.size(), origPartitions.size()); comparePartitionForSingleValuedFields(projectedFields, sharedSD, partitionWithoutSDS, 0); }
@Test public void testParameterExpansion() throws TException { GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); projectSpec.setFieldList(Arrays.asList("sd.cols", "sd.serdeInfo")); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); Assert.assertNotNull("sd.cols were requested but was not returned", sharedSD.getCols()); Assert.assertEquals("Returned serdeInfo does not match with original serdeInfo", origPartitions.get(0).getSd().getCols(), sharedSD.getCols()); Assert .assertNotNull("sd.serdeInfo were requested but was not returned", sharedSD.getSerdeInfo()); Assert.assertEquals("Returned serdeInfo does not match with original serdeInfo", origPartitions.get(0).getSd().getSerdeInfo(), sharedSD.getSerdeInfo()); }
@Test public void testNestedMultiValuedFieldProjection() throws TException { GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); projectSpec.setFieldList(Arrays.asList("sd.cols.name", "sd.cols.type")); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); Assert.assertNotNull("sd.cols were requested but was not returned", sharedSD.getCols()); for (FieldSchema col : sharedSD.getCols()) { Assert.assertTrue("sd.cols.name was requested but was not returned", col.isSetName()); Assert.assertTrue("sd.cols.type was requested but was not returned", col.isSetType()); Assert.assertFalse("sd.cols.comment was not requested but was returned", col.isSetComment()); } }
PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); Assert.assertNotNull(partitionSpecWithSharedSD.getSd()); StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); Assert.assertEquals("Root location should be set to table location", tbl.getSd().getLocation(), sharedSD.getLocation()); response.getPartitionSpec().get(0).getTableName()); Assert.assertTrue("sd.cols was requested but was not found in the returned response", partitionSpecWithSharedSD.getSd().isSetCols()); List<FieldSchema> origSdCols = createdPartitions.get(0).getSd().getCols(); Assert.assertEquals("Size of the requested sd.cols should be same", origSdCols.size(), partitionSpecWithSharedSD.getSd().getCols().size()); for (int i = 0; i < origSdCols.size(); i++) { FieldSchema origFs = origSdCols.get(i); FieldSchema returnedFs = partitionSpecWithSharedSD.getSd().getCols().get(i); Assert.assertEquals("Field schemas returned different than expected", origFs, returnedFs);
Assert.assertNotNull("sd was requested but not returned", partitionSpecWithSharedSD.getSd()); Assert.assertEquals("shared SD should have table location", table.getSd().getLocation(), partitionSpecWithSharedSD.getSd().getLocation()); List<List<String>> expectedVals = new ArrayList<>(2); expectedVals.add(Arrays.asList("p1", "PaloAlto"));
PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); Assert.assertNotNull(sharedSD); List<PartitionWithoutSD> partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions(); partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); sharedSD = partitionSpecWithSharedSD.getSd(); Assert.assertNotNull(sharedSD); partitionWithoutSDS = partitionSpecWithSharedSD.getPartitions();
private void getPartitionsWithVals(List<String> filters, int expectedPartitions) throws TException { // get partitions from "trusted" API List<Partition> partitions = client.listPartitions(dbName, tblName, filters, (short) -1); GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); projectSpec.setFieldList(Arrays.asList("sd.location")); request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_VALUES); request.getFilterSpec().setFilters(filters); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); Assert.assertNotNull(response); if (expectedPartitions > 0) { PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); Assert.assertNotNull(partitionSpecWithSharedSD); Assert.assertEquals("Invalid number of partitions returned", expectedPartitions, partitionSpecWithSharedSD.getPartitionsSize()); verifyLocations(partitions, partitionSpecWithSharedSD.getSd(), partitionSpecWithSharedSD.getPartitions()); } else { Assert.assertTrue( "Partition spec should have been empty since filter doesn't match with any partitions", response.getPartitionSpec().isEmpty()); } }
private void getPartitionsWithNames(List<String> names, int expectedPartitionCount) throws TException { GetPartitionsRequest request = getGetPartitionsRequest(); GetPartitionsProjectionSpec projectSpec = request.getProjectionSpec(); projectSpec.setFieldList(Arrays.asList("sd.location")); request.getFilterSpec().setFilterMode(PartitionFilterMode.BY_NAMES); request.getFilterSpec().setFilters(names); GetPartitionsResponse response = client.getPartitionsWithSpecs(request); Assert.assertNotNull(response); if (expectedPartitionCount > 0) { PartitionSpecWithSharedSD partitionSpecWithSharedSD = response.getPartitionSpec().get(0).getSharedSDPartitionSpec(); Assert.assertNotNull(partitionSpecWithSharedSD); Assert.assertEquals("Invalid number of partitions returned", expectedPartitionCount, partitionSpecWithSharedSD.getPartitionsSize()); List<Partition> origPartitions = client.getPartitionsByNames(dbName, tblName, names); verifyLocations(origPartitions, partitionSpecWithSharedSD.getSd(), partitionSpecWithSharedSD.getPartitions()); } else { Assert.assertTrue( "Partition spec should have been empty since filter doesn't match with any partitions", response.getPartitionSpec().isEmpty()); } }
PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); Assert.assertNotNull(sharedSD);
PartitionSpecWithSharedSD partitionSpecWithSharedSD = partitionSpec.getSharedSDPartitionSpec(); Assert.assertEquals(origPartitions.size(), partitionSpecWithSharedSD.getPartitions().size()); StorageDescriptor sharedSD = partitionSpecWithSharedSD.getSd(); for (int i = 0; i < origPartitions.size(); i++) { Partition origPartition = origPartitions.get(i);
public Object getFieldValue(_Fields field) { switch (field) { case PARTITIONS: return getPartitions(); case SD: return getSd(); } throw new IllegalStateException(); }
public Object getFieldValue(_Fields field) { switch (field) { case PARTITIONS: return getPartitions(); case SD: return getSd(); } throw new IllegalStateException(); }
public Object getFieldValue(_Fields field) { switch (field) { case PARTITIONS: return getPartitions(); case SD: return getSd(); } throw new IllegalStateException(); }
@Override public String getLocation() { return pSpec.getSd().getLocation() + pSpec.getPartitions().get(index).getRelativePath(); }