public StorageDescriptor deepCopy() { return new StorageDescriptor(this); }
/** * Performs a deep copy on <i>other</i>. */ public PartitionSpecWithSharedSD(PartitionSpecWithSharedSD other) { if (other.isSetPartitions()) { List<PartitionWithoutSD> __this__partitions = new ArrayList<PartitionWithoutSD>(other.partitions.size()); for (PartitionWithoutSD other_element : other.partitions) { __this__partitions.add(new PartitionWithoutSD(other_element)); } this.partitions = __this__partitions; } if (other.isSetSd()) { this.sd = new StorageDescriptor(other.sd); } }
public HCatTable(String dbName, String tableName) { this.dbName = StringUtils.isBlank(dbName)? Warehouse.DEFAULT_DATABASE_NAME : dbName; this.tableName = tableName; this.sd = new StorageDescriptor(); this.sd.setInputFormat(DEFAULT_INPUT_FORMAT_CLASS); this.sd.setOutputFormat(DEFAULT_OUTPUT_FORMAT_CLASS); this.sd.setSerdeInfo(new SerDeInfo()); this.sd.getSerdeInfo().setSerializationLib(DEFAULT_SERDE_CLASS); this.sd.getSerdeInfo().setParameters(new HashMap<String, String>()); this.sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); // Default serialization format. }
private static StorageDescriptor makeStorageDescriptor(String tableName, List<Column> columns, Storage storage) { if (storage.isSkewed()) { throw new IllegalArgumentException("Writing to skewed table/partition is not supported"); } SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName(tableName); serdeInfo.setSerializationLib(storage.getStorageFormat().getSerDeNullable()); serdeInfo.setParameters(storage.getSerdeParameters()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(emptyToNull(storage.getLocation())); sd.setCols(columns.stream() .map(ThriftMetastoreUtil::toMetastoreApiFieldSchema) .collect(toList())); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable()); sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable()); sd.setParameters(ImmutableMap.of()); Optional<HiveBucketProperty> bucketProperty = storage.getBucketProperty(); if (bucketProperty.isPresent()) { sd.setNumBuckets(bucketProperty.get().getBucketCount()); sd.setBucketCols(bucketProperty.get().getBucketedBy()); if (!bucketProperty.get().getSortedBy().isEmpty()) { sd.setSortCols(bucketProperty.get().getSortedBy().stream() .map(column -> new Order(column.getColumnName(), column.getOrder().getHiveOrder())) .collect(toList())); } } return sd; }
protected StorageDescriptor buildSd() throws MetaException { StorageDescriptor sd = new StorageDescriptor(getCols(), location, inputFormat, outputFormat, compressed, numBuckets, buildSerde(), bucketCols, sortCols, storageDescriptorParams); sd.setStoredAsSubDirectories(storedAsSubDirectories); if (skewedColNames != null) { SkewedInfo skewed = new SkewedInfo(skewedColNames, skewedColValues, skewedColValueLocationMaps); sd.setSkewedInfo(skewed); } return sd; }
private StorageDescriptor createFakeSd(String location) { return new StorageDescriptor(null, location, null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); }
public HCatPartition(HCatTable hcatTable, Map<String, String> partitionKeyValues, String location) throws HCatException { this.hcatTable = hcatTable; this.tableName = hcatTable.getTableName(); this.dbName = hcatTable.getDbName(); this.sd = new StorageDescriptor(hcatTable.getSd()); this.sd.setLocation(location); this.columns = getColumns(this.sd); this.createTime = (int)(System.currentTimeMillis()/1000); this.lastAccessTime = -1; this.values = new ArrayList<String>(hcatTable.getPartCols().size()); for (HCatFieldSchema partField : hcatTable.getPartCols()) { if (!partitionKeyValues.containsKey(partField.getName())) { throw new HCatException("Missing value for partition-key \'" + partField.getName() + "\' in table: " + hcatTable.getDbName() + "." + hcatTable.getTableName()); } else { values.add(partitionKeyValues.get(partField.getName())); } } }
private Table createTestTbl(String dbName, String tblName, String tblOwner, List<FieldSchema> cols, List<FieldSchema> ptnCols) { String serdeLocation = "file:/tmp"; Map<String, String> serdeParams = new HashMap<>(); Map<String, String> tblParams = new HashMap<>(); SerDeInfo serdeInfo = new SerDeInfo("serde", "seriallib", new HashMap<>()); StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serdeInfo, null, null, serdeParams); sd.setStoredAsSubDirectories(false); Table tbl = new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, ptnCols, tblParams, null, null, TableType.MANAGED_TABLE.toString()); tbl.setCatName(DEFAULT_CATALOG_NAME); return tbl; }
private static void populatePartitions(HiveMetaStoreClient hmsc, Table table, List<String> blurbs) throws Exception { for (int i=0; i< nDates; ++i) { for (String blurb : blurbs) { StorageDescriptor sd = new StorageDescriptor(table.getSd()); // Add partitions located in the table-directory (i.e. default). List<String> values = Arrays.asList(datePrefix + i, blurb); sd.setLocation(getPartitionPath(table, values)); hmsc.add_partition(new Partition(values, dbName, tableName, 0, 0, sd, null)); } } }
private Partition getTestPartition(Table table) throws HiveException { Partition partition = new Partition(table, ImmutableMap.of("partition_key", "1"), null); StorageDescriptor sd = new StorageDescriptor(); sd.setSerdeInfo(new SerDeInfo("avro", AvroSerDe.class.getName(), null)); sd.setCols(Lists.newArrayList(new FieldSchema("foo", "int", null))); partition.getTPartition().setSd(sd); return partition; } }
@Test(expected = MetaException.class) public void testAddPartitionSpecWithSharedSDInvalidSD() throws Exception { Table table = createTable(); PartitionWithoutSD partition = buildPartitionWithoutSD(Lists.newArrayList("2002"), 0); partition.setRelativePath("year2002"); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(table.getSd().getLocation() + "/nullLocationTest/"); PartitionSpecProxy partitionSpecProxy = buildPartitionSpecWithSharedSD(Lists.newArrayList(partition), sd); client.add_partitions_pspec(partitionSpecProxy); }
@Override public Partition getCurrent() { PartitionWithoutSD partWithoutSD = pSpec.getPartitions().get(index); StorageDescriptor partSD = new StorageDescriptor(pSpec.getSd()); partSD.setLocation(partSD.getLocation() + partWithoutSD.getRelativePath()); Partition p = new Partition( partWithoutSD.getValues(), partitionSpecWithSharedSDProxy.partitionSpec.getDbName(), partitionSpecWithSharedSDProxy.partitionSpec.getTableName(), partWithoutSD.getCreateTime(), partWithoutSD.getLastAccessTime(), partSD, partWithoutSD.getParameters() ); p.setCatName(partitionSpecWithSharedSDProxy.partitionSpec.getCatName()); return p; }
private void addSd(ArrayList<FieldSchema> cols, Table tbl) { StorageDescriptor sd = new StorageDescriptor(); sd.setCols(cols); sd.setCompressed(false); sd.setNumBuckets(1); sd.setParameters(new HashMap<String, String>()); sd.setBucketCols(new ArrayList<String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().getParameters() .put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.setSortCols(new ArrayList<Order>()); sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); sd.setInputFormat(HiveInputFormat.class.getName()); sd.setOutputFormat(HiveOutputFormat.class.getName()); tbl.setSd(sd); }
private StorageDescriptor buildSD(String location) { StorageDescriptor sd = new StorageDescriptor(); sd.setInputFormat("TestInputFormat"); sd.setOutputFormat("TestOutputFormat"); sd.setCols(getYearPartCol()); sd.setCompressed(false); Map<String, String> parameters = new HashMap<>(); parameters.put("testSDParamKey", "testSDParamValue"); sd.setParameters(parameters); sd.setLocation(location); SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName("sharedSDPartSerde"); sd.setSerdeInfo(serdeInfo); return sd; }
public Partition addTestPartition(Table tbl, List<String> values, int createTime) throws Exception { StorageDescriptor partitionSd = new StorageDescriptor(); if (StringUtils.isNotBlank(tbl.getSd().getLocation())) { partitionSd.setLocation(tbl.getSd().getLocation() + values); } else { partitionSd.setLocation("/tmp/" + tbl.getTableName() + "/part1"); } partitionSd.setSerdeInfo( new SerDeInfo("name", "serializationLib", ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy"))); partitionSd.setCols(tbl.getPartitionKeys()); Partition partition = new Partition(values, tbl.getDbName(), tbl.getTableName(), 1, 1, partitionSd, new HashMap<String, String>()); partition.setCreateTime(createTime); return this.getLocalMetastoreClient().add_partition(partition); }
public static Table getTestTable(String dbName, String tableName) { Table table = new Table(); table.setDbName(dbName); table.setTableName(tableName); table.setTableType(TableType.EXTERNAL_TABLE.name()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation("/tmp/test"); table.setSd(sd); return table; }
private static void addPartition(IMetaStoreClient client, Table tbl , List<String> partValues) throws IOException, TException { Partition part = new Partition(); part.setDbName(tbl.getDbName()); part.setTableName(tbl.getTableName()); StorageDescriptor sd = new StorageDescriptor(tbl.getSd()); sd.setLocation(sd.getLocation() + Path.SEPARATOR + makePartPath(tbl.getPartitionKeys(), partValues)); part.setSd(sd); part.setValues(partValues); client.add_partition(part); }
@Test public void testCreateTableDefaultValuesView() throws Exception { Table table = new Table(); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<>(); table.setDbName(DEFAULT_DATABASE); table.setTableName("test_table_2"); table.setTableType("VIRTUAL_VIEW"); cols.add(new FieldSchema("column_name", "int", null)); sd.setCols(cols); sd.setSerdeInfo(new SerDeInfo()); table.setSd(sd); client.createTable(table); Table createdTable = client.getTable(table.getDbName(), table.getTableName()); // No location should be created for views Assert.assertNull("Storage descriptor location should be null", createdTable.getSd().getLocation()); }
@Test public void testCreateTableDefaultLocationInSpecificDatabase() throws Exception { Table table = new Table(); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<>(); table.setDbName(OTHER_DATABASE); table.setTableName("test_table_2"); cols.add(new FieldSchema("column_name", "int", null)); sd.setCols(cols); sd.setSerdeInfo(new SerDeInfo()); table.setSd(sd); client.createTable(table); Table createdTable = client.getTable(table.getDbName(), table.getTableName()); Assert.assertEquals("Storage descriptor location", metaStore.getWarehouseRoot() + "/" + table.getDbName() + ".db/" + table.getTableName(), createdTable.getSd().getLocation()); }
private StorageDescriptor convertToStorageDescriptor( MStorageDescriptor msd, boolean noFS) throws MetaException { if (msd == null) { return null; } List<MFieldSchema> mFieldSchemas = msd.getCD() == null ? null : msd.getCD().getCols(); StorageDescriptor sd = new StorageDescriptor(noFS ? null : convertToFieldSchemas(mFieldSchemas), msd.getLocation(), msd.getInputFormat(), msd.getOutputFormat(), msd .isCompressed(), msd.getNumBuckets(), convertToSerDeInfo(msd .getSerDeInfo(), true), convertList(msd.getBucketCols()), convertToOrders(msd .getSortCols()), convertMap(msd.getParameters())); SkewedInfo skewedInfo = new SkewedInfo(convertList(msd.getSkewedColNames()), convertToSkewedValues(msd.getSkewedColValues()), covertToSkewedMap(msd.getSkewedColValueLocationMaps())); sd.setSkewedInfo(skewedInfo); sd.setStoredAsSubDirectories(msd.isStoredAsSubDirectories()); return sd; }