public synchronized void increSd(StorageDescriptor sd, byte[] sdHash) { ByteArrayWrapper byteArray = new ByteArrayWrapper(sdHash); if (sdCache.containsKey(byteArray)) { sdCache.get(byteArray).refCount++; } else { StorageDescriptor sdToCache = sd.deepCopy(); sdToCache.setLocation(null); sdToCache.setParameters(null); sdCache.put(byteArray, new StorageDescriptorWrapper(sdToCache, 1)); } }
static void setSDParameters(String SD_PARAMS, boolean convertMapNullsToEmptyStrings, PersistenceManager pm, TreeMap<Long, StorageDescriptor> sds, String sdIds) throws MetaException { String queryText; queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + SD_PARAMS + "" + " where \"SD_ID\" in (" + sdIds + ") and \"PARAM_KEY\" is not null" + " order by \"SD_ID\" asc"; loopJoinOrderedResult(pm, sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { t.putToParameters((String)fields[1], extractSqlClob(fields[2])); }}); // Perform conversion of null map values for (StorageDescriptor t : sds.values()) { t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } }
private static StorageDescriptor makeStorageDescriptor(String tableName, List<Column> columns, Storage storage) { if (storage.isSkewed()) { throw new IllegalArgumentException("Writing to skewed table/partition is not supported"); } SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName(tableName); serdeInfo.setSerializationLib(storage.getStorageFormat().getSerDeNullable()); serdeInfo.setParameters(storage.getSerdeParameters()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(emptyToNull(storage.getLocation())); sd.setCols(columns.stream() .map(ThriftMetastoreUtil::toMetastoreApiFieldSchema) .collect(toList())); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable()); sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable()); sd.setParameters(ImmutableMap.of()); Optional<HiveBucketProperty> bucketProperty = storage.getBucketProperty(); if (bucketProperty.isPresent()) { sd.setNumBuckets(bucketProperty.get().getBucketCount()); sd.setBucketCols(bucketProperty.get().getBucketedBy()); if (!bucketProperty.get().getSortedBy().isEmpty()) { sd.setSortCols(bucketProperty.get().getSortedBy().stream() .map(column -> new Order(column.getColumnName(), column.getOrder().getHiveOrder())) .collect(toList())); } } return sd; }
static Table assemble(TableWrapper wrapper, SharedCache sharedCache) { Table t = wrapper.getTable().deepCopy(); if (wrapper.getSdHash() != null) { StorageDescriptor sdCopy = sharedCache.getSdFromCache(wrapper.getSdHash()).deepCopy(); if (sdCopy.getBucketCols() == null) { sdCopy.setBucketCols(Collections.emptyList()); } if (sdCopy.getSortCols() == null) { sdCopy.setSortCols(Collections.emptyList()); } if (sdCopy.getSkewedInfo() == null) { sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters()); t.setSd(sdCopy); } return t; }
static Partition assemble(PartitionWrapper wrapper, SharedCache sharedCache) { Partition p = wrapper.getPartition().deepCopy(); if (wrapper.getSdHash() != null) { StorageDescriptor sdCopy = sharedCache.getSdFromCache(wrapper.getSdHash()).deepCopy(); if (sdCopy.getBucketCols() == null) { sdCopy.setBucketCols(Collections.emptyList()); } if (sdCopy.getSortCols() == null) { sdCopy.setSortCols(Collections.emptyList()); } if (sdCopy.getSkewedInfo() == null) { sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters()); p.setSd(sdCopy); } return p; }
private static StorageDescriptor getStorageDescriptor(HiveRegistrationUnit unit) { State props = unit.getStorageProps(); StorageDescriptor sd = new StorageDescriptor(); sd.setParameters(getParameters(props)); sd.setCols(getFieldSchemas(unit)); if (unit.getLocation().isPresent()) { sd.setLocation(unit.getLocation().get()); } if (unit.getInputFormat().isPresent()) { sd.setInputFormat(unit.getInputFormat().get()); } if (unit.getOutputFormat().isPresent()) { sd.setOutputFormat(unit.getOutputFormat().get()); } if (unit.getIsCompressed().isPresent()) { sd.setCompressed(unit.getIsCompressed().get()); } if (unit.getNumBuckets().isPresent()) { sd.setNumBuckets(unit.getNumBuckets().get()); } if (unit.getBucketColumns().isPresent()) { sd.setBucketCols(unit.getBucketColumns().get()); } if (unit.getIsStoredAsSubDirs().isPresent()) { sd.setStoredAsSubDirectories(unit.getIsStoredAsSubDirs().get()); } sd.setSerdeInfo(getSerDeInfo(unit)); return sd; }
sd.setBucketCols(new ArrayList<String>()); sd.setCols(new ArrayList<FieldSchema>()); sd.setParameters(new HashMap<String, String>()); sd.setSortCols(new ArrayList<Order>()); sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.setBucketCols(new ArrayList<String>()); sd.setCols(new ArrayList<FieldSchema>()); sd.setParameters(new HashMap<String, String>()); sd.setSortCols(new ArrayList<Order>()); sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.setParameters(new HashMap<String, String>()); sd.setSkewedInfo(new SkewedInfo(new ArrayList<String>(), new ArrayList<List<String>>(), new HashMap<List<String>, String>()));
private void addSd(ArrayList<FieldSchema> cols, Table tbl) { StorageDescriptor sd = new StorageDescriptor(); sd.setCols(cols); sd.setCompressed(false); sd.setNumBuckets(1); sd.setParameters(new HashMap<String, String>()); sd.setBucketCols(new ArrayList<String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().getParameters() .put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.setSortCols(new ArrayList<Order>()); sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); sd.setInputFormat(HiveInputFormat.class.getName()); sd.setOutputFormat(HiveOutputFormat.class.getName()); tbl.setSd(sd); }
private static void createTable(String tableName, String tablePerm) throws Exception { Table tbl = new Table(); tbl.setDbName(DATABASE); tbl.setTableName(tableName); StorageDescriptor sd = new StorageDescriptor(); sd.setCols(ColumnHolder.colMapping.get(tableName)); tbl.setSd(sd); sd.setParameters(new HashMap<String, String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.getSerdeInfo().setSerializationLib( org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName()); tbl.setPartitionKeys(ColumnHolder.partitionCols); hmsc.createTable(tbl); Path path = new Path(warehousedir, tableName); FileSystem fs = path.getFileSystem(hiveConf); fs.setPermission(path, new FsPermission(tablePerm)); }
private StorageDescriptor buildSD(String location) { StorageDescriptor sd = new StorageDescriptor(); sd.setInputFormat("TestInputFormat"); sd.setOutputFormat("TestOutputFormat"); sd.setCols(getYearPartCol()); sd.setCompressed(false); Map<String, String> parameters = new HashMap<>(); parameters.put("testSDParamKey", "testSDParamValue"); sd.setParameters(parameters); sd.setLocation(location); SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName("sharedSDPartSerde"); sd.setSerdeInfo(serdeInfo); return sd; }
params1.put("key", "value"); sd1.setCols(cols1); sd1.setParameters(params1); sd1.setLocation("loc1"); part1.setSd(sd1); params2.put("key", "value"); sd2.setCols(cols2); sd2.setParameters(params2); sd2.setLocation("loc2"); part2.setSd(sd2); params3.put("key2", "value2"); sd3.setCols(cols3); sd3.setParameters(params3); sd3.setLocation("loc3"); part3.setSd(sd3); newParams1.put("key", "value"); newSd1.setCols(newCols1); newSd1.setParameters(params1); newSd1.setLocation("loc1new"); newPart1.setSd(newSd1);
sd.setParameters(new HashMap<String, String>()); sd.getParameters().put("test_param_1", "Use this for comments etc");
params1.put("key", "value"); sd1.setCols(cols1); sd1.setParameters(params1); sd1.setLocation("loc1"); tbl1.setSd(sd1); params2.put("key", "value"); sd2.setCols(cols2); sd2.setParameters(params2); sd2.setLocation("loc2"); tbl2.setSd(sd2); params3.put("key2", "value2"); sd3.setCols(cols3); sd3.setParameters(params3); sd3.setLocation("loc3"); tbl3.setSd(sd3); newParams1.put("key", "value"); newSd1.setCols(newCols1); newSd1.setParameters(params1); newSd1.setLocation("loc1"); newTbl1.setSd(newSd1);
viewSd.setCols(viewCols); viewSd.setCompressed(false); viewSd.setParameters(new HashMap<>()); viewSd.setSerdeInfo(new SerDeInfo()); viewSd.getSerdeInfo().setParameters(new HashMap<>());
unsetParameters(); } else { setParameters((Map<String,String>)value);
private static StorageDescriptor makeStorageDescriptor(String tableName, HiveStorageFormat format, Path targetPath, List<FieldSchema> columns) { SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName(tableName); serdeInfo.setSerializationLib(format.getSerDe()); serdeInfo.setParameters(ImmutableMap.of()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(targetPath.toString()); sd.setCols(columns); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(format.getInputFormat()); sd.setOutputFormat(format.getOutputFormat()); sd.setParameters(ImmutableMap.of()); return sd; } }
static Table assemble(TableWrapper wrapper, SharedCache sharedCache) { Table t = wrapper.getTable().deepCopy(); if (wrapper.getSdHash() != null) { StorageDescriptor sdCopy = sharedCache.getSdFromCache(wrapper.getSdHash()).deepCopy(); if (sdCopy.getBucketCols() == null) { sdCopy.setBucketCols(Collections.emptyList()); } if (sdCopy.getSortCols() == null) { sdCopy.setSortCols(Collections.emptyList()); } if (sdCopy.getSkewedInfo() == null) { sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters()); t.setSd(sdCopy); } return t; }
static Partition assemble(PartitionWrapper wrapper, SharedCache sharedCache) { Partition p = wrapper.getPartition().deepCopy(); if (wrapper.getSdHash() != null) { StorageDescriptor sdCopy = sharedCache.getSdFromCache(wrapper.getSdHash()).deepCopy(); if (sdCopy.getBucketCols() == null) { sdCopy.setBucketCols(Collections.emptyList()); } if (sdCopy.getSortCols() == null) { sdCopy.setSortCols(Collections.emptyList()); } if (sdCopy.getSkewedInfo() == null) { sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters()); p.setSd(sdCopy); } return p; }