/** * Getter for SerDe parameters. * @return The SerDe parameters. */ public Map<String, String> getSerdeParams() { return this.sd.getSerdeInfo().getParameters(); }
/** * Gets the serde. * * @return the serde */ public String getSerDe() { return this.sd.getSerdeInfo().getSerializationLib(); }
private SerDeInfo getSerdeInfo() { return tTable.getSd().getSerdeInfo(); }
StorageDescriptorKey(String baseLocation, StorageDescriptor sd) { this.sd = sd; this.baseLocation = baseLocation; if (sd == null) { hashCode = Objects.hashCode(baseLocation); } else { // use the baseLocation provided instead of sd.getLocation() hashCode = Objects.hash(sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getSerializationLib(), sd.getInputFormat(), sd.getOutputFormat(), baseLocation, sd.getCols()); } }
static StorerInfo extractStorerInfo(StorageDescriptor sd, Map<String, String> properties) throws IOException { Properties hcatProperties = new Properties(); for (String key : properties.keySet()) { hcatProperties.put(key, properties.get(key)); } // also populate with StorageDescriptor->SerDe.Parameters for (Map.Entry<String, String> param : sd.getSerdeInfo().getParameters().entrySet()) { hcatProperties.put(param.getKey(), param.getValue()); } return new StorerInfo( sd.getInputFormat(), sd.getOutputFormat(), sd.getSerdeInfo().getSerializationLib(), properties.get(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE), hcatProperties); }
private boolean isAvro(Table table) { return AvroSerDe.class.getName().equals(table.getSd().getSerdeInfo().getSerializationLib()); } }
public static boolean isAvroTableWithExternalSchema(Table table) { if (table.getSd().getSerdeInfo().getSerializationLib().equals(AVRO_SERDE_CLASSNAME)) { String schemaUrl = table.getParameters().get(AVRO_SCHEMA_URL_PROPERTY); if (schemaUrl != null && !schemaUrl.isEmpty()) { return true; } } return false; }
public void clearSerDeInfo() { tTable.getSd().getSerdeInfo().getParameters().clear(); }
public HCatTable(String dbName, String tableName) { this.dbName = StringUtils.isBlank(dbName)? Warehouse.DEFAULT_DATABASE_NAME : dbName; this.tableName = tableName; this.sd = new StorageDescriptor(); this.sd.setInputFormat(DEFAULT_INPUT_FORMAT_CLASS); this.sd.setOutputFormat(DEFAULT_OUTPUT_FORMAT_CLASS); this.sd.setSerdeInfo(new SerDeInfo()); this.sd.getSerdeInfo().setSerializationLib(DEFAULT_SERDE_CLASS); this.sd.getSerdeInfo().setParameters(new HashMap<String, String>()); this.sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); // Default serialization format. }
public static Class<? extends Deserializer> getDeserializerClass( Configuration conf, org.apache.hadoop.hive.metastore.api.Table table) throws Exception { String lib = table.getSd().getSerdeInfo().getSerializationLib(); return lib == null ? null : conf.getClassByName(lib).asSubclass(Deserializer.class); }
private SerDeInfo getSerDeInfo() { if (!sd.isSetSerdeInfo()) { sd.setSerdeInfo(new SerDeInfo()); } return sd.getSerdeInfo(); }
public Optional<String> getFileFormat() { String serdeLib = this.hivePartition.getTPartition().getSd().getSerdeInfo().getSerializationLib(); for (HiveSerDeWrapper.BuiltInHiveSerDe hiveSerDe : HiveSerDeWrapper.BuiltInHiveSerDe.values()) { if (hiveSerDe.toString().equalsIgnoreCase(serdeLib)) { return Optional.fromNullable(hiveSerDe.name()); } } return Optional.<String>absent(); }
public static boolean isAvroTableWithSchemaSet(org.apache.hadoop.hive.metastore.api.Table table) { if (table.getParameters() == null) { return false; } StorageDescriptor storageDescriptor = table.getSd(); if (storageDescriptor == null) { throw new PrestoException(HIVE_INVALID_METADATA, "Table does not contain a storage descriptor: " + table); } SerDeInfo serdeInfo = storageDescriptor.getSerdeInfo(); if (serdeInfo == null) { throw new PrestoException(HIVE_INVALID_METADATA, "Table storage descriptor is missing SerDe info"); } return serdeInfo.getSerializationLib() != null && table.getParameters().get(AVRO_SCHEMA_URL_KEY) != null && serdeInfo.getSerializationLib().equals(AVRO.getSerDe()); }
protected Table getDummyTable() throws SemanticException { Path dummyPath = createDummyFile(); Table desc = new Table(DUMMY_DATABASE, DUMMY_TABLE); desc.getTTable().getSd().setLocation(dummyPath.toString()); desc.getTTable().getSd().getSerdeInfo().setSerializationLib(NullStructSerDe.class.getName()); desc.setInputFormatClass(NullRowsInputFormat.class); desc.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); return desc; }
private Table getDummyTable() throws SemanticException { Path dummyPath = createDummyFile(); Table desc = new Table(DUMMY_DATABASE, DUMMY_TABLE); desc.getTTable().getSd().setLocation(dummyPath.toString()); desc.getTTable().getSd().getSerdeInfo().setSerializationLib(NullStructSerDe.class.getName()); desc.setInputFormatClass(NullRowsInputFormat.class); desc.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); return desc; }
public static void fromMetastoreApiStorageDescriptor(StorageDescriptor storageDescriptor, Storage.Builder builder, String tablePartitionName) { SerDeInfo serdeInfo = storageDescriptor.getSerdeInfo(); if (serdeInfo == null) { throw new PrestoException(HIVE_INVALID_METADATA, "Table storage descriptor is missing SerDe info"); } builder.setStorageFormat(StorageFormat.createNullable(serdeInfo.getSerializationLib(), storageDescriptor.getInputFormat(), storageDescriptor.getOutputFormat())) .setLocation(nullToEmpty(storageDescriptor.getLocation())) .setBucketProperty(HiveBucketProperty.fromStorageDescriptor(storageDescriptor, tablePartitionName)) .setSkewed(storageDescriptor.isSetSkewedInfo() && storageDescriptor.getSkewedInfo().isSetSkewedColNames() && !storageDescriptor.getSkewedInfo().getSkewedColNames().isEmpty()) .setSerdeParameters(serdeInfo.getParameters() == null ? ImmutableMap.of() : serdeInfo.getParameters()); }
private Partition createPartition(List<String> vals, Table table) throws MetaException { return new PartitionBuilder() .inTable(table) .setValues(vals) .addPartParam("key1", "S1") .addPartParam("key2", "S2") .addPartParam(EXCLUDE_KEY_PREFIX + "key1", "e1") .addPartParam(EXCLUDE_KEY_PREFIX + "key2", "e2") .setBucketCols(table.getSd().getBucketCols()) .setSortCols(table.getSd().getSortCols()) .setSerdeName(table.getSd().getSerdeInfo().getName()) .setSerdeLib(table.getSd().getSerdeInfo().getSerializationLib()) .setSerdeParams(table.getSd().getSerdeInfo().getParameters()) .build(conf); }
private static Partition makePartitionObject(String dbName, String tblName, List<String> ptnVals, Table tbl, String ptnLocationSuffix) throws MetaException { Partition part4 = new Partition(); part4.setDbName(dbName); part4.setTableName(tblName); part4.setValues(ptnVals); part4.setParameters(new HashMap<>()); part4.setSd(tbl.getSd().deepCopy()); part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy()); part4.getSd().setLocation(tbl.getSd().getLocation() + ptnLocationSuffix); MetaStoreServerUtils.updatePartitionStatsFast(part4, tbl, warehouse, false, false, null, true); return part4; }
private void add_partition(HiveMetaStoreClient client, Table table, List<String> vals, String location) throws TException { Partition part = new Partition(); part.setDbName(table.getDbName()); part.setTableName(table.getTableName()); part.setValues(vals); part.setParameters(new HashMap<>()); part.setSd(table.getSd().deepCopy()); part.getSd().setSerdeInfo(table.getSd().getSerdeInfo()); part.getSd().setLocation(table.getSd().getLocation() + location); client.add_partition(part); }
private void addPartition(HiveMetaStoreClient client, Table table, List<String> vals, String location) throws TException { Partition part = new Partition(); part.setDbName(table.getDbName()); part.setTableName(table.getTableName()); part.setValues(vals); part.setParameters(new HashMap<String, String>()); part.setSd(table.getSd().deepCopy()); part.getSd().setSerdeInfo(table.getSd().getSerdeInfo()); part.getSd().setLocation(table.getSd().getLocation() + location); client.add_partition(part); } }