/** * Instantiates a new hcat partition info. * @param partitionSchema the partition schema * @param storageHandler the storage handler * @param location the location * @param hcatProperties hcat-specific properties at the partition * @param jobProperties the job properties * @param tableInfo the table information */ public PartInfo(HCatSchema partitionSchema, HiveStorageHandler storageHandler, String location, Properties hcatProperties, Map<String, String> jobProperties, HCatTableInfo tableInfo) { this.partitionSchema = partitionSchema; this.location = location; this.hcatProperties = hcatProperties; this.jobProperties = jobProperties; this.tableInfo = tableInfo; this.storageHandlerClassName = storageHandler.getClass().getName(); this.inputFormatClassName = storageHandler.getInputFormatClass().getName(); this.serdeClassName = storageHandler.getSerDeClass().getName(); this.outputFormatClassName = storageHandler.getOutputFormatClass().getName(); }
/** * Gets the output format instance. * @param context the job context * @return the output format instance * @throws IOException */ protected OutputFormat<WritableComparable<?>, HCatRecord> getOutputFormat(JobContext context) throws IOException { OutputJobInfo jobInfo = getJobInfo(context.getConfiguration()); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(context.getConfiguration(), jobInfo.getTableInfo().getStorerInfo()); // Always configure storage handler with jobproperties/jobconf before calling any methods on it configureOutputStorageHandler(context); if (storageHandler instanceof FosterStorageHandler) { return new FileOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } else { return new DefaultOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } }
/** * Setter for StorageHandler class. */ public HCatTable storageHandler(String storageHandler) throws HCatException { this.tblProps.put( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, storageHandler); LOG.warn("HiveStorageHandlers can't be reliably instantiated on the client-side. " + "Attempting to derive Input/OutputFormat settings from StorageHandler, on best effort: "); try { HiveStorageHandler sh = HiveUtils.getStorageHandler(getConf(), storageHandler); this.sd.setInputFormat(sh.getInputFormatClass().getName()); this.sd.setOutputFormat(sh.getOutputFormatClass().getName()); this.sd.getSerdeInfo().setSerializationLib(sh.getSerDeClass().getName()); } catch (HiveException e) { LOG.warn("Could not derive Input/OutputFormat and SerDe settings from storageHandler. " + "These values need to be set explicitly.", e); } return this; }
final public Class<? extends OutputFormat> getOutputFormatClass() { if (outputFormatClass == null) { try { String className = tTable.getSd().getOutputFormat(); Class<?> c; if (className == null) { if (getStorageHandler() == null) { return null; } c = getStorageHandler().getOutputFormatClass(); } else { c = Class.forName(className, true, Utilities.getSessionSpecifiedClassLoader()); } // Replace FileOutputFormat for backward compatibility outputFormatClass = HiveFileFormatUtils.getOutputFormatSubstitute(c); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } return outputFormatClass; }
public static Map<String, String> getInputJobProperties(HiveStorageHandler storageHandler, InputJobInfo inputJobInfo) { Properties props = inputJobInfo.getTableInfo().getStorerInfo().getProperties(); props.put(serdeConstants.SERIALIZATION_LIB,storageHandler.getSerDeClass().getName()); TableDesc tableDesc = new TableDesc(storageHandler.getInputFormatClass(), storageHandler.getOutputFormatClass(),props); if (tableDesc.getJobProperties() == null) { tableDesc.setJobProperties(new HashMap<String, String>()); } Properties mytableProperties = tableDesc.getProperties(); mytableProperties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME,inputJobInfo.getDatabaseName()+ "." + inputJobInfo.getTableName()); Map<String, String> jobProperties = new HashMap<String, String>(); try { Map<String, String> properties = tableDesc.getJobProperties(); LinkedList<InputJobInfo> inputJobInfos = (LinkedList<InputJobInfo>) HCatUtil.deserialize( properties.get(HCatConstants.HCAT_KEY_JOB_INFO)); if (inputJobInfos == null) { inputJobInfos = new LinkedList<>(); } inputJobInfos.add(inputJobInfo); properties.put(HCatConstants.HCAT_KEY_JOB_INFO, HCatUtil.serialize(inputJobInfos)); storageHandler.configureInputJobProperties(tableDesc, jobProperties); } catch (IOException e) { throw new IllegalStateException( "Failed to configure StorageHandler", e); } return jobProperties; }
final public Class<? extends OutputFormat> getOutputFormatClass() { if (outputFormatClass == null) { try { String className = tTable.getSd().getOutputFormat(); Class<?> c; if (className == null) { if (getStorageHandler() == null) { return null; } c = getStorageHandler().getOutputFormatClass(); } else { c = Class.forName(className, true, Utilities.getSessionSpecifiedClassLoader()); } // Replace FileOutputFormat for backward compatibility outputFormatClass = HiveFileFormatUtils.getOutputFormatSubstitute(c); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } return outputFormatClass; }
ReflectionUtils.newInstance(storageHandler.getOutputFormatClass(), currTaskContext.getJobConf());
out_class = storageHandler.getOutputFormatClass(); } else { out_class = JavaUtils.loadClass(crtViewDesc.getOutputFormat());
out_class = storageHandler.getOutputFormatClass(); } else { out_class = JavaUtils.loadClass(crtTblDesc.getOutputFormat());
out_class = storageHandler.getOutputFormatClass(); } else { out_class = JavaUtils.loadClass(crtTblDesc.getOutputFormat());
/** * Instantiates a new hcat partition info. * @param partitionSchema the partition schema * @param storageHandler the storage handler * @param location the location * @param hcatProperties hcat-specific properties at the partition * @param jobProperties the job properties * @param tableInfo the table information */ public PartInfo(HCatSchema partitionSchema, HiveStorageHandler storageHandler, String location, Properties hcatProperties, Map<String, String> jobProperties, HCatTableInfo tableInfo) { this.partitionSchema = partitionSchema; this.location = location; this.hcatProperties = hcatProperties; this.jobProperties = jobProperties; this.tableInfo = tableInfo; this.storageHandlerClassName = storageHandler.getClass().getName(); this.inputFormatClassName = storageHandler.getInputFormatClass().getName(); this.serdeClassName = storageHandler.getSerDeClass().getName(); this.outputFormatClassName = storageHandler.getOutputFormatClass().getName(); }
/** * Instantiates a new hcat partition info. * @param partitionSchema the partition schema * @param storageHandler the storage handler * @param location the location * @param hcatProperties hcat-specific properties at the partition * @param jobProperties the job properties * @param tableInfo the table information */ public PartInfo(HCatSchema partitionSchema, HiveStorageHandler storageHandler, String location, Properties hcatProperties, Map<String, String> jobProperties, HCatTableInfo tableInfo) { this.partitionSchema = partitionSchema; this.location = location; this.hcatProperties = hcatProperties; this.jobProperties = jobProperties; this.tableInfo = tableInfo; this.storageHandlerClassName = storageHandler.getClass().getName(); this.inputFormatClassName = storageHandler.getInputFormatClass().getName(); this.serdeClassName = storageHandler.getSerDeClass().getName(); this.outputFormatClassName = storageHandler.getOutputFormatClass().getName(); }
/** * Instantiates a new hcat partition info. * @param partitionSchema the partition schema * @param storageHandler the storage handler * @param location the location * @param hcatProperties hcat-specific properties at the partition * @param jobProperties the job properties * @param tableInfo the table information */ public PartInfo(HCatSchema partitionSchema, HiveStorageHandler storageHandler, String location, Properties hcatProperties, Map<String, String> jobProperties, HCatTableInfo tableInfo) { this.partitionSchema = partitionSchema; this.location = location; this.hcatProperties = hcatProperties; this.jobProperties = jobProperties; this.tableInfo = tableInfo; this.storageHandlerClassName = storageHandler.getClass().getName(); this.inputFormatClassName = storageHandler.getInputFormatClass().getName(); this.serdeClassName = storageHandler.getSerDeClass().getName(); this.outputFormatClassName = storageHandler.getOutputFormatClass().getName(); }
@Override protected OutputFormat<WritableComparable<?>, HCatRecord> getOutputFormat(JobContext context) throws IOException { OutputJobInfo jobInfo = getJobInfo(context.getConfiguration()); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(context.getConfiguration(), jobInfo.getTableInfo().getStorerInfo()); // Always configure storage handler with jobproperties/jobconf before // calling any methods on it configureOutputStorageHandler(context); if (storageHandler instanceof FosterStorageHandler) { return new CrunchFileOutputFormatContainer( ReflectionUtils.newInstance(storageHandler.getOutputFormatClass(), context.getConfiguration())); } else { return new CrunchDefaultOutputFormatContainer( ReflectionUtils.newInstance(storageHandler.getOutputFormatClass(), context.getConfiguration())); } } }
/** * Gets the output format instance. * @param context the job context * @return the output format instance * @throws IOException */ protected OutputFormat<WritableComparable<?>, HCatRecord> getOutputFormat(JobContext context) throws IOException { OutputJobInfo jobInfo = getJobInfo(context.getConfiguration()); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(context.getConfiguration(), jobInfo.getTableInfo().getStorerInfo()); // Always configure storage handler with jobproperties/jobconf before calling any methods on it configureOutputStorageHandler(context); if (storageHandler instanceof FosterStorageHandler) { return new FileOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } else { return new DefaultOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } }
/** * Gets the output format instance. * @param context the job context * @return the output format instance * @throws IOException */ protected OutputFormat<WritableComparable<?>, HCatRecord> getOutputFormat(JobContext context) throws IOException { OutputJobInfo jobInfo = getJobInfo(context.getConfiguration()); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(context.getConfiguration(), jobInfo.getTableInfo().getStorerInfo()); // Always configure storage handler with jobproperties/jobconf before calling any methods on it configureOutputStorageHandler(context); if (storageHandler instanceof FosterStorageHandler) { return new FileOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } else { return new DefaultOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } }
/** * Setter for StorageHandler class. */ public HCatTable storageHandler(String storageHandler) throws HCatException { this.tblProps.put( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, storageHandler); LOG.warn("HiveStorageHandlers can't be reliably instantiated on the client-side. " + "Attempting to derive Input/OutputFormat settings from StorageHandler, on best effort: "); try { HiveStorageHandler sh = HiveUtils.getStorageHandler(getConf(), storageHandler); this.sd.setInputFormat(sh.getInputFormatClass().getName()); this.sd.setOutputFormat(sh.getOutputFormatClass().getName()); this.sd.getSerdeInfo().setSerializationLib(sh.getSerDeClass().getName()); } catch (HiveException e) { LOG.warn("Could not derive Input/OutputFormat and SerDe settings from storageHandler. " + "These values need to be set explicitly.", e); } return this; }
/** * Setter for StorageHandler class. */ public HCatTable storageHandler(String storageHandler) throws HCatException { this.tblProps.put( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, storageHandler); LOG.warn("HiveStorageHandlers can't be reliably instantiated on the client-side. " + "Attempting to derive Input/OutputFormat settings from StorageHandler, on best effort: "); try { HiveStorageHandler sh = HiveUtils.getStorageHandler(getConf(), storageHandler); this.sd.setInputFormat(sh.getInputFormatClass().getName()); this.sd.setOutputFormat(sh.getOutputFormatClass().getName()); this.sd.getSerdeInfo().setSerializationLib(sh.getSerDeClass().getName()); } catch (HiveException e) { LOG.warn("Could not derive Input/OutputFormat and SerDe settings from storageHandler. " + "These values need to be set explicitly.", e); } return this; }
/** * Setter for StorageHandler class. */ public HCatTable storageHandler(String storageHandler) throws HCatException { this.tblProps.put( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, storageHandler); LOG.warn("HiveStorageHandlers can't be reliably instantiated on the client-side. " + "Attempting to derive Input/OutputFormat settings from StorageHandler, on best effort: "); try { HiveStorageHandler sh = HiveUtils.getStorageHandler(getConf(), storageHandler); this.sd.setInputFormat(sh.getInputFormatClass().getName()); this.sd.setOutputFormat(sh.getOutputFormatClass().getName()); this.sd.getSerdeInfo().setSerializationLib(sh.getSerDeClass().getName()); } catch (HiveException e) { LOG.warn("Could not derive Input/OutputFormat and SerDe settings from storageHandler. " + "These values need to be set explicitly.", e); } return this; }
final public Class<? extends OutputFormat> getOutputFormatClass() { if (outputFormatClass == null) { try { String className = tTable.getSd().getOutputFormat(); Class<?> c; if (className == null) { if (getStorageHandler() == null) { return null; } c = getStorageHandler().getOutputFormatClass(); } else { c = Class.forName(className, true, Utilities.getSessionSpecifiedClassLoader()); } // Replace FileOutputFormat for backward compatibility outputFormatClass = HiveFileFormatUtils.getOutputFormatSubstitute(c); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } return outputFormatClass; }