private SerDeInfo getSerDeInfo() { if (!sd.isSetSerdeInfo()) { sd.setSerdeInfo(new SerDeInfo()); } return sd.getSerdeInfo(); }
public HCatTable(String dbName, String tableName) { this.dbName = StringUtils.isBlank(dbName)? Warehouse.DEFAULT_DATABASE_NAME : dbName; this.tableName = tableName; this.sd = new StorageDescriptor(); this.sd.setInputFormat(DEFAULT_INPUT_FORMAT_CLASS); this.sd.setOutputFormat(DEFAULT_OUTPUT_FORMAT_CLASS); this.sd.setSerdeInfo(new SerDeInfo()); this.sd.getSerdeInfo().setSerializationLib(DEFAULT_SERDE_CLASS); this.sd.getSerdeInfo().setParameters(new HashMap<String, String>()); this.sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); // Default serialization format. }
private static StorageDescriptor makeStorageDescriptor(String tableName, List<Column> columns, Storage storage) { if (storage.isSkewed()) { throw new IllegalArgumentException("Writing to skewed table/partition is not supported"); } SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName(tableName); serdeInfo.setSerializationLib(storage.getStorageFormat().getSerDeNullable()); serdeInfo.setParameters(storage.getSerdeParameters()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(emptyToNull(storage.getLocation())); sd.setCols(columns.stream() .map(ThriftMetastoreUtil::toMetastoreApiFieldSchema) .collect(toList())); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable()); sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable()); sd.setParameters(ImmutableMap.of()); Optional<HiveBucketProperty> bucketProperty = storage.getBucketProperty(); if (bucketProperty.isPresent()) { sd.setNumBuckets(bucketProperty.get().getBucketCount()); sd.setBucketCols(bucketProperty.get().getBucketedBy()); if (!bucketProperty.get().getSortedBy().isEmpty()) { sd.setSortCols(bucketProperty.get().getSortedBy().stream() .map(column -> new Order(column.getColumnName(), column.getOrder().getHiveOrder())) .collect(toList())); } } return sd; }
private Partition getTestPartition(Table table) throws HiveException { Partition partition = new Partition(table, ImmutableMap.of("partition_key", "1"), null); StorageDescriptor sd = new StorageDescriptor(); sd.setSerdeInfo(new SerDeInfo("avro", AvroSerDe.class.getName(), null)); sd.setCols(Lists.newArrayList(new FieldSchema("foo", "int", null))); partition.getTPartition().setSd(sd); return partition; } }
@Test(expected = MetaException.class) public void testAddPartitionsEmptySerdeInfo() throws Exception { createTable(); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); partition.getSd().setSerdeInfo(null); client.add_partitions(Lists.newArrayList(partition)); }
@Test(expected = MetaException.class) public void testCreateTableInvalidStorageDescriptorNullSerdeInfo() throws Exception { Table table = getNewTable(); table.getSd().setSerdeInfo(null); client.createTable(table); }
private StorageDescriptor newStorageDescriptor(String location, List<Order> sortCols) { StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<FieldSchema>(2); cols.add(new FieldSchema("a", "varchar(25)", "still no comment")); cols.add(new FieldSchema("b", "int", "comment")); sd.setCols(cols); sd.setLocation(location); sd.setInputFormat(MockInputFormat.class.getName()); sd.setOutputFormat(MockOutputFormat.class.getName()); sd.setNumBuckets(1); SerDeInfo serde = new SerDeInfo(); serde.setSerializationLib(LazySimpleSerDe.class.getName()); sd.setSerdeInfo(serde); List<String> bucketCols = new ArrayList<String>(1); bucketCols.add("a"); sd.setBucketCols(bucketCols); if (sortCols != null) { sd.setSortCols(sortCols); } return sd; }
@Test(expected = MetaException.class) public void testAddPartitionEmptySerdeInfo() throws Exception { createTable(); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); partition.getSd().setSerdeInfo(null); client.add_partition(partition); }
private void addSd(ArrayList<FieldSchema> cols, Table tbl) { StorageDescriptor sd = new StorageDescriptor(); sd.setCols(cols); sd.setCompressed(false); sd.setNumBuckets(1); sd.setParameters(new HashMap<String, String>()); sd.setBucketCols(new ArrayList<String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().getParameters() .put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.setSortCols(new ArrayList<Order>()); sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); sd.setInputFormat(HiveInputFormat.class.getName()); sd.setOutputFormat(HiveOutputFormat.class.getName()); tbl.setSd(sd); }
private StorageDescriptor buildSD(String location) { StorageDescriptor sd = new StorageDescriptor(); sd.setInputFormat("TestInputFormat"); sd.setOutputFormat("TestOutputFormat"); sd.setCols(getYearPartCol()); sd.setCompressed(false); Map<String, String> parameters = new HashMap<>(); parameters.put("testSDParamKey", "testSDParamValue"); sd.setParameters(parameters); sd.setLocation(location); SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName("sharedSDPartSerde"); sd.setSerdeInfo(serdeInfo); return sd; }
@Test(expected = MetaException.class) public void testAlterTableInvalidStorageDescriptorNullSerdeInfo() throws Exception { Table originalTable = testTables[0]; Table newTable = originalTable.deepCopy(); newTable.getSd().setSerdeInfo(null); client.alter_table(originalTable.getDbName(), originalTable.getTableName(), newTable); }
private static void createTable(String tableName, String tablePerm) throws Exception { Table tbl = new Table(); tbl.setDbName(DATABASE); tbl.setTableName(tableName); StorageDescriptor sd = new StorageDescriptor(); sd.setCols(ColumnHolder.colMapping.get(tableName)); tbl.setSd(sd); sd.setParameters(new HashMap<String, String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.setInputFormat(org.apache.hadoop.hive.ql.io.RCFileInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.RCFileOutputFormat.class.getName()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.getSerdeInfo().setSerializationLib( org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class.getName()); tbl.setPartitionKeys(ColumnHolder.partitionCols); hmsc.createTable(tbl); Path path = new Path(warehousedir, tableName); FileSystem fs = path.getFileSystem(hiveConf); fs.setPermission(path, new FsPermission(tablePerm)); }
public Partition addTestPartition(Table tbl, List<String> values, int createTime) throws Exception { StorageDescriptor partitionSd = new StorageDescriptor(); if (StringUtils.isNotBlank(tbl.getSd().getLocation())) { partitionSd.setLocation(tbl.getSd().getLocation() + values); } else { partitionSd.setLocation("/tmp/" + tbl.getTableName() + "/part1"); } partitionSd.setSerdeInfo( new SerDeInfo("name", "serializationLib", ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy"))); partitionSd.setCols(tbl.getPartitionKeys()); Partition partition = new Partition(values, tbl.getDbName(), tbl.getTableName(), 1, 1, partitionSd, new HashMap<String, String>()); partition.setCreateTime(createTime); return this.getLocalMetastoreClient().add_partition(partition); }
private Table getTable(String dbName, String tblName, String typeName) throws NoSuchObjectException, MetaException, TException, AlreadyExistsException, InvalidObjectException { msc.dropTable(dbName, tblName); silentDropDatabase(dbName); msc.dropType(typeName); Type typ1 = new Type(); typ1.setName(typeName); typ1.setFields(new ArrayList<FieldSchema>(1)); typ1.getFields().add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, "")); msc.createType(typ1); Table tbl = new Table(); tbl.setDbName(dbName); tbl.setTableName(tblName); StorageDescriptor sd = new StorageDescriptor(); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tblName); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); sd.setInputFormat(HiveInputFormat.class.getName()); sd.setOutputFormat(HiveOutputFormat.class.getName()); tbl.setSd(sd); sd.setCols(typ1.getFields()); sd.setSerdeInfo(new SerDeInfo()); return tbl; }
private Table createPartitionedTable(String catName, String dbName, String tableName) throws Exception { try { db.dropTable(catName, dbName, tableName); Table table = new Table(); table.setCatName(catName); table.setDbName(dbName); table.setTableName(tableName); FieldSchema col1 = new FieldSchema("key", "string", ""); FieldSchema col2 = new FieldSchema("value", "int", ""); FieldSchema col3 = new FieldSchema("city", "string", ""); StorageDescriptor sd = new StorageDescriptor(); sd.setSerdeInfo(new SerDeInfo()); sd.setInputFormat(TextInputFormat.class.getCanonicalName()); sd.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); sd.setCols(Arrays.asList(col1, col2)); table.setPartitionKeys(Arrays.asList(col3)); table.setSd(sd); db.createTable(table); return db.getTable(catName, dbName, tableName); } catch (Exception exception) { fail("Unable to drop and create table " + StatsUtils .getFullyQualifiedTableName(dbName, tableName) + " because " + StringUtils .stringifyException(exception)); throw exception; } }
@Test public void testCreateTableDefaultValuesView() throws Exception { Table table = new Table(); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<>(); table.setDbName(DEFAULT_DATABASE); table.setTableName("test_table_2"); table.setTableType("VIRTUAL_VIEW"); cols.add(new FieldSchema("column_name", "int", null)); sd.setCols(cols); sd.setSerdeInfo(new SerDeInfo()); table.setSd(sd); client.createTable(table); Table createdTable = client.getTable(table.getDbName(), table.getTableName()); // No location should be created for views Assert.assertNull("Storage descriptor location should be null", createdTable.getSd().getLocation()); }
@Test public void testCreateTableDefaultLocationInSpecificDatabase() throws Exception { Table table = new Table(); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<>(); table.setDbName(OTHER_DATABASE); table.setTableName("test_table_2"); cols.add(new FieldSchema("column_name", "int", null)); sd.setCols(cols); sd.setSerdeInfo(new SerDeInfo()); table.setSd(sd); client.createTable(table); Table createdTable = client.getTable(table.getDbName(), table.getTableName()); Assert.assertEquals("Storage descriptor location", metaStore.getWarehouseRoot() + "/" + table.getDbName() + ".db/" + table.getTableName(), createdTable.getSd().getLocation()); }
private static Partition makePartitionObject(String dbName, String tblName, List<String> ptnVals, Table tbl, String ptnLocationSuffix) throws MetaException { Partition part4 = new Partition(); part4.setDbName(dbName); part4.setTableName(tblName); part4.setValues(ptnVals); part4.setParameters(new HashMap<>()); part4.setSd(tbl.getSd().deepCopy()); part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy()); part4.getSd().setLocation(tbl.getSd().getLocation() + ptnLocationSuffix); MetaStoreServerUtils.updatePartitionStatsFast(part4, tbl, warehouse, false, false, null, true); return part4; }
private void add_partition(HiveMetaStoreClient client, Table table, List<String> vals, String location) throws TException { Partition part = new Partition(); part.setDbName(table.getDbName()); part.setTableName(table.getTableName()); part.setValues(vals); part.setParameters(new HashMap<>()); part.setSd(table.getSd().deepCopy()); part.getSd().setSerdeInfo(table.getSd().getSerdeInfo()); part.getSd().setLocation(table.getSd().getLocation() + location); client.add_partition(part); }
private void addPartition(HiveMetaStoreClient client, Table table, List<String> vals, String location) throws TException { Partition part = new Partition(); part.setDbName(table.getDbName()); part.setTableName(table.getTableName()); part.setValues(vals); part.setParameters(new HashMap<String, String>()); part.setSd(table.getSd().deepCopy()); part.getSd().setSerdeInfo(table.getSd().getSerdeInfo()); part.getSd().setLocation(table.getSd().getLocation() + location); client.add_partition(part); } }