/** * Gets the storage handler. * * @return the storage handler */ public String getStorageHandler() { return this.sd .getParameters() .get(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE); }
static void setSDParameters(String SD_PARAMS, boolean convertMapNullsToEmptyStrings, PersistenceManager pm, TreeMap<Long, StorageDescriptor> sds, String sdIds) throws MetaException { String queryText; queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + SD_PARAMS + "" + " where \"SD_ID\" in (" + sdIds + ") and \"PARAM_KEY\" is not null" + " order by \"SD_ID\" asc"; loopJoinOrderedResult(pm, sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { t.putToParameters((String)fields[1], extractSqlClob(fields[2])); }}); // Perform conversion of null map values for (StorageDescriptor t : sds.values()) { t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } }
Map<String, String> parameters = tTable.getSd().getParameters(); String sf = parameters!=null?parameters.get(SERIALIZATION_FORMAT) : null; if (sf != null) {
Map<String, String> parameters = tTable.getSd().getParameters(); String sf = parameters!=null?parameters.get(SERIALIZATION_FORMAT) : null; if (sf != null) {
private PartitionWrapper makePartitionWrapper(Partition part, SharedCache sharedCache) { Partition partCopy = part.deepCopy(); PartitionWrapper wrapper; if (part.getSd() != null) { byte[] sdHash = MetaStoreServerUtils.hashStorageDescriptor(part.getSd(), md); StorageDescriptor sd = part.getSd(); sharedCache.increSd(sd, sdHash); partCopy.setSd(null); wrapper = new PartitionWrapper(partCopy, sdHash, sd.getLocation(), sd.getParameters()); } else { wrapper = new PartitionWrapper(partCopy, null, null, null); } return wrapper; } }
public Object getFieldValue(_Fields field) { switch (field) { case COLS: return getCols(); case LOCATION: return getLocation(); case INPUT_FORMAT: return getInputFormat(); case OUTPUT_FORMAT: return getOutputFormat(); case COMPRESSED: return isCompressed(); case NUM_BUCKETS: return getNumBuckets(); case SERDE_INFO: return getSerdeInfo(); case BUCKET_COLS: return getBucketCols(); case SORT_COLS: return getSortCols(); case PARAMETERS: return getParameters(); case SKEWED_INFO: return getSkewedInfo(); case STORED_AS_SUB_DIRECTORIES: return isStoredAsSubDirectories(); } throw new IllegalStateException(); }
public StorageDescriptorWrapper(StorageDescriptor storageDescriptor) { sd = storageDescriptor; location = storageDescriptor.getLocation(); inputFormat = storageDescriptor.getInputFormat(); outputFormat = storageDescriptor.getOutputFormat(); compressed = storageDescriptor.isCompressed(); numBuckets = storageDescriptor.getNumBuckets(); serDeInfo = new SerDeInfoWrapper(storageDescriptor.getSerdeInfo()); if (sd.getSortCols() != null) { sortCols = Lists.newArrayList(); for (Order order : sd.getSortCols()) { sortCols.add(new OrderWrapper(order)); } } parameters = storageDescriptor.getParameters(); if (sd.getCols() != null) { this.columns = Lists.newArrayList(); for (FieldSchema fieldSchema : sd.getCols()) { this.columns.add(new FieldSchemaWrapper(fieldSchema)); } } }
private static State getStorageProps(StorageDescriptor sd) { State storageProps = new State(); for (Map.Entry<String, String> entry : sd.getParameters().entrySet()) { storageProps.setProp(entry.getKey(), entry.getValue()); } if (sd.isSetLocation()) { storageProps.setProp(HiveConstants.LOCATION, sd.getLocation()); } if (sd.isSetInputFormat()) { storageProps.setProp(HiveConstants.INPUT_FORMAT, sd.getInputFormat()); } if (sd.isSetOutputFormat()) { storageProps.setProp(HiveConstants.OUTPUT_FORMAT, sd.getOutputFormat()); } if (sd.isSetCompressed()) { storageProps.setProp(HiveConstants.COMPRESSED, sd.isCompressed()); } if (sd.isSetNumBuckets()) { storageProps.setProp(HiveConstants.NUM_BUCKETS, sd.getNumBuckets()); } if (sd.isSetBucketCols()) { for (String bucketColumn : sd.getBucketCols()) { storageProps.appendToListProp(HiveConstants.BUCKET_COLUMNS, bucketColumn); } } if (sd.isSetStoredAsSubDirectories()) { storageProps.setProp(HiveConstants.STORED_AS_SUB_DIRS, sd.isStoredAsSubDirectories()); } return storageProps; }
sd.setOutputFormat(RCFileOutputFormat.class.getName()); sd.setParameters(new HashMap<String, String>()); sd.getParameters().put("test_param_1", "Use this for comments etc");
private void verifyPartitionSharedSD(Table table, String name, List<String> values, int index) throws Exception { Partition part = client.getPartition(table.getDbName(), table.getTableName(), name); Assert.assertNotNull(part); Assert.assertEquals(table.getTableName(), part.getTableName()); List<String> partValues = part.getValues(); Assert.assertEquals(values.size(), partValues.size()); Assert.assertTrue(partValues.containsAll(values)); Assert.assertEquals(table.getDbName(), part.getDbName()); Assert.assertEquals(DEFAULT_CREATE_TIME, part.getLastAccessTime()); Assert.assertEquals(DEFAULT_PARAM_VALUE + index, part.getParameters().get(DEFAULT_PARAM_KEY + index)); Assert.assertFalse(part.getParameters().keySet().contains(table.getParameters().keySet())); StorageDescriptor sd = part.getSd(); Assert.assertNotNull(sd); Assert.assertEquals("TestInputFormat", sd.getInputFormat()); Assert.assertEquals("TestOutputFormat", sd.getOutputFormat()); Assert.assertEquals("sharedSDPartSerde", sd.getSerdeInfo().getName()); Assert.assertEquals("testSDParamValue", sd.getParameters().get("testSDParamKey")); Assert.assertEquals( metaStore.getWarehouseRoot() + "/" + TABLE_NAME + "/sharedSDTest/partwithoutsd" + index, sd.getLocation()); Assert.assertTrue(metaStore.isPathExists(new Path(sd.getLocation()))); }
Assert.assertEquals( "The parameter map of the partition's storage descriptor should contain the partsdkey - partsdvalue pair.", "partsdvalue" + index, sd.getParameters().get("partsdkey" + index)); Assert.assertEquals("The parameter's location is not correct.", metaStore.getWarehouseRoot() + "/" + TABLE_NAME + "/" + name, sd.getLocation());
Assert.assertEquals( "The parameter map of the partition's storage descriptor should contain the partsdkey - partsdvalue pair.", "partsdvalue" + index, sd.getParameters().get("partsdkey" + index)); Assert.assertEquals("The parameter's location is not correct.", metaStore.getWarehouseRoot() + "/" + TABLE_NAME + "/" + name, sd.getLocation());
private TableWrapper createTableWrapper(String catName, String dbName, String tblName, Table tbl) { TableWrapper wrapper; Table tblCopy = tbl.deepCopy(); tblCopy.setCatName(normalizeIdentifier(catName)); tblCopy.setDbName(normalizeIdentifier(dbName)); tblCopy.setTableName(normalizeIdentifier(tblName)); if (tblCopy.getPartitionKeys() != null) { for (FieldSchema fs : tblCopy.getPartitionKeys()) { fs.setName(normalizeIdentifier(fs.getName())); } } if (tbl.getSd() != null) { byte[] sdHash = MetaStoreServerUtils.hashStorageDescriptor(tbl.getSd(), md); StorageDescriptor sd = tbl.getSd(); increSd(sd, sdHash); tblCopy.setSd(null); wrapper = new TableWrapper(tblCopy, sdHash, sd.getLocation(), sd.getParameters()); } else { wrapper = new TableWrapper(tblCopy, null, null, null); } return wrapper; }
sd.getSortCols().isEmpty()); Assert.assertTrue("Per default the storage descriptor parameters should be empty.", sd.getParameters().isEmpty()); SerDeInfo serdeInfo = sd.getSerdeInfo(); Assert.assertNotNull("The serdeInfo attribute should not be null.", serdeInfo);
private void updateTableObj(Table newTable, SharedCache sharedCache) { byte[] sdHash = getSdHash(); // Remove old table object's sd hash if (sdHash != null) { sharedCache.decrSd(sdHash); } Table tblCopy = newTable.deepCopy(); if (tblCopy.getPartitionKeys() != null) { for (FieldSchema fs : tblCopy.getPartitionKeys()) { fs.setName(StringUtils.normalizeIdentifier(fs.getName())); } } setTable(tblCopy); if (tblCopy.getSd() != null) { sdHash = MetaStoreServerUtils.hashStorageDescriptor(tblCopy.getSd(), md); StorageDescriptor sd = tblCopy.getSd(); sharedCache.increSd(sd, sdHash); tblCopy.setSd(null); setSdHash(sdHash); setLocation(sd.getLocation()); setParameters(sd.getParameters()); } else { setSdHash(null); setLocation(null); setParameters(null); } }
/** * Converts a storage descriptor to a db-backed storage descriptor. It points the * storage descriptor's column descriptor to the one passed as an argument, * so it does not create a new mcolumn descriptor object. * @param sd the storage descriptor to wrap in a db-backed object * @param mcd the db-backed column descriptor * @return the db-backed storage descriptor object * @throws MetaException */ private MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd, MColumnDescriptor mcd) throws MetaException { if (sd == null) { return null; } return new MStorageDescriptor(mcd, sd .getLocation(), sd.getInputFormat(), sd.getOutputFormat(), sd .isCompressed(), sd.getNumBuckets(), convertToMSerDeInfo(sd .getSerdeInfo()), sd.getBucketCols(), convertToMOrders(sd.getSortCols()), sd.getParameters(), (null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo().getSkewedColNames(), convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() .getSkewedColValues()), covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() .getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories()); }
assertEquals(tbl2.getSd().getNumBuckets(), 1); assertEquals("Use this for comments etc", tbl2.getSd().getParameters() .get("test_param_1")); assertEquals("name", tbl2.getSd().getBucketCols().get(0));
break; case "sd.parameters": validateMap(origPartition.getSd().getParameters(), sharedSD.getParameters()); break; case "sd.skewedInfo":
assertEquals("Use this for comments etc", tbl2.getSd().getParameters() .get("test_param_1")); assertEquals("name", tbl2.getSd().getBucketCols().get(0));
Assert.assertEquals("Storage descriptor bucket cols", 0, createdSd.getBucketCols().size()); Assert.assertEquals("Storage descriptor sort cols", 0, createdSd.getSortCols().size()); Assert.assertEquals("Storage descriptor parameters", 0, createdSd.getParameters().size()); Assert.assertFalse("Storage descriptor stored as subdir", createdSd.isStoredAsSubDirectories());