public SerDeInfo deepCopy() { return new SerDeInfo(this); }
@JsonCreator public SerDeInfoWrapper(@JsonProperty("name") String name, @JsonProperty("serializationLib") String serializationLib, @JsonProperty("parameters") Map<String, String> parameters) { this.name = name; this.serializationLib = serializationLib; this.parameters = parameters; this.serDeInfo = new SerDeInfo(name, serializationLib, parameters); }
private StorageDescriptor createFakeSd(String location) { return new StorageDescriptor(null, location, null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); }
private SerDeInfo getSerDeInfo() { if (!sd.isSetSerdeInfo()) { sd.setSerdeInfo(new SerDeInfo()); } return sd.getSerdeInfo(); }
protected SerDeInfo buildSerde() { SerDeInfo serDeInfo = new SerDeInfo(serdeName, serdeLib, serdeParams); if (serdeDescription != null) serDeInfo.setDescription(serdeDescription); if (serdeSerializerClass != null) serDeInfo.setSerializerClass(serdeSerializerClass); if (serdeDeserializerClass != null) serDeInfo.setDeserializerClass(serdeDeserializerClass); if (serdeType != null) serDeInfo.setSerdeType(serdeType); return serDeInfo; }
public HCatTable(String dbName, String tableName) { this.dbName = StringUtils.isBlank(dbName)? Warehouse.DEFAULT_DATABASE_NAME : dbName; this.tableName = tableName; this.sd = new StorageDescriptor(); this.sd.setInputFormat(DEFAULT_INPUT_FORMAT_CLASS); this.sd.setOutputFormat(DEFAULT_OUTPUT_FORMAT_CLASS); this.sd.setSerdeInfo(new SerDeInfo()); this.sd.getSerdeInfo().setSerializationLib(DEFAULT_SERDE_CLASS); this.sd.getSerdeInfo().setParameters(new HashMap<String, String>()); this.sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); // Default serialization format. }
private static StorageDescriptor makeStorageDescriptor(String tableName, List<Column> columns, Storage storage) { if (storage.isSkewed()) { throw new IllegalArgumentException("Writing to skewed table/partition is not supported"); } SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName(tableName); serdeInfo.setSerializationLib(storage.getStorageFormat().getSerDeNullable()); serdeInfo.setParameters(storage.getSerdeParameters()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(emptyToNull(storage.getLocation())); sd.setCols(columns.stream() .map(ThriftMetastoreUtil::toMetastoreApiFieldSchema) .collect(toList())); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable()); sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable()); sd.setParameters(ImmutableMap.of()); Optional<HiveBucketProperty> bucketProperty = storage.getBucketProperty(); if (bucketProperty.isPresent()) { sd.setNumBuckets(bucketProperty.get().getBucketCount()); sd.setBucketCols(bucketProperty.get().getBucketedBy()); if (!bucketProperty.get().getSortedBy().isEmpty()) { sd.setSortCols(bucketProperty.get().getSortedBy().stream() .map(column -> new Order(column.getColumnName(), column.getOrder().getHiveOrder())) .collect(toList())); } } return sd; }
private static SerDeInfo getSerDeInfo(HiveRegistrationUnit unit) { State props = unit.getSerDeProps(); SerDeInfo si = new SerDeInfo(); si.setParameters(getParameters(props)); si.setName(unit.getTableName()); if (unit.getSerDeType().isPresent()) { si.setSerializationLib(unit.getSerDeType().get()); } return si; }
private Table createTestTbl(String dbName, String tblName, String tblOwner, List<FieldSchema> cols, List<FieldSchema> ptnCols) { String serdeLocation = "file:/tmp"; Map<String, String> serdeParams = new HashMap<>(); Map<String, String> tblParams = new HashMap<>(); SerDeInfo serdeInfo = new SerDeInfo("serde", "seriallib", new HashMap<>()); StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serdeInfo, null, null, serdeParams); sd.setStoredAsSubDirectories(false); Table tbl = new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, ptnCols, tblParams, null, null, TableType.MANAGED_TABLE.toString()); tbl.setCatName(DEFAULT_CATALOG_NAME); return tbl; }
@Test(expected = AlreadyExistsException.class) public void duplicateSerde() throws TException { String serdeName = uniqueSerdeName(); SerDeInfo serDeInfo = new SerDeInfo(serdeName, "x", Collections.emptyMap()); client.addSerDe(serDeInfo); client.addSerDe(serDeInfo); }
private static void createTable(HiveMetaStoreClient hmsc, boolean enablePartitionGrouping) throws Exception { List<FieldSchema> columns = new ArrayList<>(); columns.add(new FieldSchema("foo", "string", "")); columns.add(new FieldSchema("bar", "string", "")); List<FieldSchema> partColumns = new ArrayList<>(); partColumns.add(new FieldSchema("dt", "string", "")); partColumns.add(new FieldSchema("blurb", "string", "")); SerDeInfo serdeInfo = new SerDeInfo("LBCSerDe", "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", new HashMap<>()); StorageDescriptor storageDescriptor = new StorageDescriptor(columns, null, "org.apache.hadoop.hive.ql.io.RCFileInputFormat", "org.apache.hadoop.hive.ql.io.RCFileOutputFormat", false, 0, serdeInfo, null, null, null); Map<String, String> tableParameters = new HashMap<>(); tableParameters.put("hive.hcatalog.partition.spec.grouping.enabled", enablePartitionGrouping? "true":"false"); Table table = new Table(tableName, dbName, "", 0, 0, 0, storageDescriptor, partColumns, tableParameters, "", "", ""); hmsc.createTable(table); Assert.assertTrue("Table " + dbName + "." + tableName + " does not exist", hmsc.tableExists(dbName, tableName)); }
private Partition getTestPartition(Table table) throws HiveException { Partition partition = new Partition(table, ImmutableMap.of("partition_key", "1"), null); StorageDescriptor sd = new StorageDescriptor(); sd.setSerdeInfo(new SerDeInfo("avro", AvroSerDe.class.getName(), null)); sd.setCols(Lists.newArrayList(new FieldSchema("foo", "int", null))); partition.getTPartition().setSd(sd); return partition; } }
@Test(expected = NoSuchObjectException.class) public void mapSerDeNoSuchSchema() throws TException { SerDeInfo serDeInfo = new SerDeInfo(uniqueSerdeName(), "lib", Collections.emptyMap()); client.mapSchemaVersionToSerde(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, uniqueSchemaName(), 1, serDeInfo.getName()); }
private void addSd(ArrayList<FieldSchema> cols, Table tbl) { StorageDescriptor sd = new StorageDescriptor(); sd.setCols(cols); sd.setCompressed(false); sd.setNumBuckets(1); sd.setParameters(new HashMap<String, String>()); sd.setBucketCols(new ArrayList<String>()); sd.setSerdeInfo(new SerDeInfo()); sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().getParameters() .put(serdeConstants.SERIALIZATION_FORMAT, "1"); sd.setSortCols(new ArrayList<Order>()); sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName()); sd.setInputFormat(HiveInputFormat.class.getName()); sd.setOutputFormat(HiveOutputFormat.class.getName()); tbl.setSd(sd); }
private StorageDescriptor buildSD(String location) { StorageDescriptor sd = new StorageDescriptor(); sd.setInputFormat("TestInputFormat"); sd.setOutputFormat("TestOutputFormat"); sd.setCols(getYearPartCol()); sd.setCompressed(false); Map<String, String> parameters = new HashMap<>(); parameters.put("testSDParamKey", "testSDParamValue"); sd.setParameters(parameters); sd.setLocation(location); SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName("sharedSDPartSerde"); sd.setSerdeInfo(serdeInfo); return sd; }
@Test(expected = NoSuchObjectException.class) public void mapSerDeNoSuchSchemaVersion() throws TException { SerDeInfo serDeInfo = new SerDeInfo(uniqueSerdeName(), "lib", Collections.emptyMap()); ISchema schema = new ISchemaBuilder() .setSchemaType(SchemaType.AVRO) .setName(uniqueSchemaName()) .build(); client.createISchema(schema); client.mapSchemaVersionToSerde(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, schema.getName(), 3, serDeInfo.getName()); }
public Partition addTestPartition(Table tbl, List<String> values, int createTime) throws Exception { StorageDescriptor partitionSd = new StorageDescriptor(); if (StringUtils.isNotBlank(tbl.getSd().getLocation())) { partitionSd.setLocation(tbl.getSd().getLocation() + values); } else { partitionSd.setLocation("/tmp/" + tbl.getTableName() + "/part1"); } partitionSd.setSerdeInfo( new SerDeInfo("name", "serializationLib", ImmutableMap.of(HiveAvroSerDeManager.SCHEMA_URL, "/tmp/dummy"))); partitionSd.setCols(tbl.getPartitionKeys()); Partition partition = new Partition(values, tbl.getDbName(), tbl.getTableName(), 1, 1, partitionSd, new HashMap<String, String>()); partition.setCreateTime(createTime); return this.getLocalMetastoreClient().add_partition(partition); }
@Test public void testCreateTableDefaultValuesView() throws Exception { Table table = new Table(); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<>(); table.setDbName(DEFAULT_DATABASE); table.setTableName("test_table_2"); table.setTableType("VIRTUAL_VIEW"); cols.add(new FieldSchema("column_name", "int", null)); sd.setCols(cols); sd.setSerdeInfo(new SerDeInfo()); table.setSd(sd); client.createTable(table); Table createdTable = client.getTable(table.getDbName(), table.getTableName()); // No location should be created for views Assert.assertNull("Storage descriptor location should be null", createdTable.getSd().getLocation()); }
@Test public void testCreateTableDefaultLocationInSpecificDatabase() throws Exception { Table table = new Table(); StorageDescriptor sd = new StorageDescriptor(); List<FieldSchema> cols = new ArrayList<>(); table.setDbName(OTHER_DATABASE); table.setTableName("test_table_2"); cols.add(new FieldSchema("column_name", "int", null)); sd.setCols(cols); sd.setSerdeInfo(new SerDeInfo()); table.setSd(sd); client.createTable(table); Table createdTable = client.getTable(table.getDbName(), table.getTableName()); Assert.assertEquals("Storage descriptor location", metaStore.getWarehouseRoot() + "/" + table.getDbName() + ".db/" + table.getTableName(), createdTable.getSd().getLocation()); }
@Test public void addSerde() throws TException { String serdeName = uniqueSerdeName(); SerDeInfo serDeInfo = new SerDeInfo(serdeName, "serdeLib", Collections.singletonMap("a", "b")); serDeInfo.setSerializerClass("serializer"); serDeInfo.setDeserializerClass("deserializer"); serDeInfo.setDescription("description"); serDeInfo.setSerdeType(SerdeType.SCHEMA_REGISTRY); client.addSerDe(serDeInfo); serDeInfo = client.getSerDe(serdeName); Assert.assertEquals(serdeName, serDeInfo.getName()); Assert.assertEquals("serdeLib", serDeInfo.getSerializationLib()); Assert.assertEquals(1, serDeInfo.getParametersSize()); Assert.assertEquals("b", serDeInfo.getParameters().get("a")); Assert.assertEquals("serializer", serDeInfo.getSerializerClass()); Assert.assertEquals("deserializer", serDeInfo.getDeserializerClass()); Assert.assertEquals("description", serDeInfo.getDescription()); Assert.assertEquals(SerdeType.SCHEMA_REGISTRY, serDeInfo.getSerdeType()); }