public static StorageFormat fromHiveStorageFormat(HiveStorageFormat hiveStorageFormat) { return new StorageFormat(hiveStorageFormat.getSerDe(), hiveStorageFormat.getInputFormat(), hiveStorageFormat.getOutputFormat()); }
private static HiveStorageFormat extractHiveStorageFormat(Table table) { StorageFormat storageFormat = table.getStorage().getStorageFormat(); String outputFormat = storageFormat.getOutputFormat(); String serde = storageFormat.getSerDe(); for (HiveStorageFormat format : HiveStorageFormat.values()) { if (format.getOutputFormat().equals(outputFormat) && format.getSerDe().equals(serde)) { return format; } } throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, format("Output format %s with SerDe %s is not supported", outputFormat, serde)); }
throws Exception HiveOutputFormat<?, ?> outputFormat = newInstance(storageFormat.getOutputFormat(), HiveOutputFormat.class); @SuppressWarnings("deprecation") SerDe serDe = newInstance(storageFormat.getSerDe(), SerDe.class);
.setStorageFormat(StorageFormat.create(hiveStorageFormat.getSerDe(), hiveStorageFormat.getInputFormat(), hiveStorageFormat.getOutputFormat())) .setBucketProperty(bucketProperty) .setSerdeParameters(ImmutableMap.of());
private static HiveStorageFormat extractHiveStorageFormat(Table table) { StorageDescriptor descriptor = table.getSd(); if (descriptor == null) { throw new PrestoException(HIVE_INVALID_METADATA, "Table is missing storage descriptor"); } SerDeInfo serdeInfo = descriptor.getSerdeInfo(); if (serdeInfo == null) { throw new PrestoException(HIVE_INVALID_METADATA, "Table storage descriptor is missing SerDe info"); } String outputFormat = descriptor.getOutputFormat(); String serializationLib = serdeInfo.getSerializationLib(); for (HiveStorageFormat format : HiveStorageFormat.values()) { if (format.getOutputFormat().equals(outputFormat) && format.getSerDe().equals(serializationLib)) { return format; } } throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, format("Output format %s with SerDe %s is not supported", outputFormat, serializationLib)); }
outputFormat = tableStorageFormat.getOutputFormat(); serDe = tableStorageFormat.getSerDe(); target = locationService.targetPath(locationHandle, partitionName); write = locationService.writePath(locationHandle, partitionName).orElse(target); outputFormat = tableStorageFormat.getOutputFormat(); serDe = tableStorageFormat.getSerDe();
public RecordFormatWriter(File targetFile, List<String> columnNames, List<Type> columnTypes, HiveCompressionCodec compressionCodec, HiveStorageFormat format) { JobConf config = new JobConf(conf); configureCompression(config, compressionCodec); List<DataColumn> dataColumns = new ArrayList<>(columnNames.size()); for (int i = 0; i < columnNames.size(); i++) { dataColumns.add(new DataColumn(columnNames.get(i), columnTypes.get(i), HiveType.toHiveType(columnTypes.get(i)))); } recordWriter = new HiveRecordWriter( "test_schema", "test_table", null, compressionCodec != HiveCompressionCodec.NONE, true, dataColumns, format.getOutputFormat(), format.getSerDe(), createSchema(format, columnNames, columnTypes), targetFile.getName(), targetFile.getParent(), targetFile.toString(), TYPE_MANAGER, config); }
private static StorageDescriptor makeStorageDescriptor(String tableName, HiveStorageFormat format, Path targetPath, List<FieldSchema> columns) { SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName(tableName); serdeInfo.setSerializationLib(format.getSerDe()); serdeInfo.setParameters(ImmutableMap.of()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(targetPath.toString()); sd.setCols(columns); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(format.getInputFormat()); sd.setOutputFormat(format.getOutputFormat()); sd.setParameters(ImmutableMap.of()); return sd; } }
throws Exception HiveOutputFormat<?, ?> outputFormat = newInstance(storageFormat.getOutputFormat(), HiveOutputFormat.class); @SuppressWarnings("deprecation") SerDe serDe = newInstance(storageFormat.getSerDe(), SerDe.class);
info.setLocation(location); info.setInputFormat(hiveStorageFormat.getInputFormat()); info.setOutputFormat(hiveStorageFormat.getOutputFormat()); info.setOwner(handle.getTableOwner()); info.setParameters(ImmutableMap.<String, String>of());
.setStorageFormat(StorageFormat.create(hiveStorageFormat.getSerDe(), hiveStorageFormat.getInputFormat(), hiveStorageFormat.getOutputFormat())) .setBucketProperty(bucketProperty) .setSerdeParameters(ImmutableMap.of());
sd.setSerdeInfo(serdeInfo); sd.setInputFormat(hiveStorageFormat.getInputFormat()); sd.setOutputFormat(hiveStorageFormat.getOutputFormat()); sd.setParameters(ImmutableMap.of());