/** * Create a work unit to materialize a table / view to a target table using a staging table in between. * @param dataset {@link HiveDataset} for the source table. * @param storageFormat format in which target table should be written. * @param destinationTable {@link StageableTableMetadata} specifying staging and target tables metadata. */ public static HiveWorkUnit viewMaterializationWorkUnit(HiveDataset dataset, HiveConverterUtils.StorageFormat storageFormat, StageableTableMetadata destinationTable, @Nullable String partitionName) { HiveWorkUnit workUnit = new HiveWorkUnit(dataset); workUnit.setProp(MATERIALIZER_MODE_KEY, MaterializerMode.TABLE_MATERIALIZATION.name()); workUnit.setProp(STORAGE_FORMAT_KEY, storageFormat.name()); workUnit.setProp(STAGEABLE_TABLE_METADATA_KEY, HiveSource.GENERICS_AWARE_GSON.toJson(destinationTable)); if (!Strings.isNullOrEmpty(partitionName)) { workUnit.setPartitionName(partitionName); } TaskUtils.setTaskFactoryClass(workUnit, HiveMaterializerTaskFactory.class); return workUnit; }
/** * Create a work unit to materialize a query to a target table using a staging table in between. * @param query the query to materialize. * @param storageFormat format in which target table should be written. * @param destinationTable {@link StageableTableMetadata} specifying staging and target tables metadata. */ public static WorkUnit queryResultMaterializationWorkUnit(String query, HiveConverterUtils.StorageFormat storageFormat, StageableTableMetadata destinationTable) { WorkUnit workUnit = new WorkUnit(); workUnit.setProp(MATERIALIZER_MODE_KEY, MaterializerMode.QUERY_RESULT_MATERIALIZATION.name()); workUnit.setProp(STORAGE_FORMAT_KEY, storageFormat.name()); workUnit.setProp(QUERY_RESULT_TO_MATERIALIZE_KEY, query); workUnit.setProp(STAGEABLE_TABLE_METADATA_KEY, HiveSource.GENERICS_AWARE_GSON.toJson(destinationTable)); TaskUtils.setTaskFactoryClass(workUnit, HiveMaterializerTaskFactory.class); HiveTask.disableHiveWatermarker(workUnit); return workUnit; }
/** * Create a work unit to materialize a table / view to a target table using a staging table in between. * @param dataset {@link HiveDataset} for the source table. * @param storageFormat format in which target table should be written. * @param destinationTable {@link StageableTableMetadata} specifying staging and target tables metadata. */ public static HiveWorkUnit viewMaterializationWorkUnit(HiveDataset dataset, HiveConverterUtils.StorageFormat storageFormat, StageableTableMetadata destinationTable, @Nullable String partitionName) { HiveWorkUnit workUnit = new HiveWorkUnit(dataset); workUnit.setProp(MATERIALIZER_MODE_KEY, MaterializerMode.TABLE_MATERIALIZATION.name()); workUnit.setProp(STORAGE_FORMAT_KEY, storageFormat.name()); workUnit.setProp(STAGEABLE_TABLE_METADATA_KEY, HiveSource.GENERICS_AWARE_GSON.toJson(destinationTable)); if (!Strings.isNullOrEmpty(partitionName)) { workUnit.setPartitionName(partitionName); } TaskUtils.setTaskFactoryClass(workUnit, HiveMaterializerTaskFactory.class); return workUnit; }
/** * Create a work unit to materialize a query to a target table using a staging table in between. * @param query the query to materialize. * @param storageFormat format in which target table should be written. * @param destinationTable {@link StageableTableMetadata} specifying staging and target tables metadata. */ public static WorkUnit queryResultMaterializationWorkUnit(String query, HiveConverterUtils.StorageFormat storageFormat, StageableTableMetadata destinationTable) { WorkUnit workUnit = new WorkUnit(); workUnit.setProp(MATERIALIZER_MODE_KEY, MaterializerMode.QUERY_RESULT_MATERIALIZATION.name()); workUnit.setProp(STORAGE_FORMAT_KEY, storageFormat.name()); workUnit.setProp(QUERY_RESULT_TO_MATERIALIZE_KEY, query); workUnit.setProp(STAGEABLE_TABLE_METADATA_KEY, HiveSource.GENERICS_AWARE_GSON.toJson(destinationTable)); TaskUtils.setTaskFactoryClass(workUnit, HiveMaterializerTaskFactory.class); HiveTask.disableHiveWatermarker(workUnit); return workUnit; }
private HiveConverterUtils.StorageFormat getOutputStorageFormat(State state) { return HiveConverterUtils.StorageFormat.valueOf(state.getProp(OUTPUT_STORAGE_FORMAT, HiveConverterUtils.StorageFormat.TEXT_FILE.name())); }
private HiveConverterUtils.StorageFormat getOutputStorageFormat(State state) { return HiveConverterUtils.StorageFormat.valueOf(state.getProp(OUTPUT_STORAGE_FORMAT, HiveConverterUtils.StorageFormat.TEXT_FILE.name())); }