Refine search
private AbstractSerDe initializeSerde(Configuration conf, Properties props) throws SerDeException { String serdeName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEFETCHOUTPUTSERDE); Class<? extends AbstractSerDe> serdeClass; try { serdeClass = Class.forName(serdeName, true, JavaUtils.getClassLoader()).asSubclass(AbstractSerDe.class); } catch (ClassNotFoundException e) { throw new SerDeException(e); } // cast only needed for Hadoop 0.17 compatibility AbstractSerDe serde = ReflectionUtil.newInstance(serdeClass, null); Properties serdeProps = new Properties(); if (serde instanceof DelimitedJSONSerDe) { serdeProps.put(SERIALIZATION_FORMAT, props.getProperty(SERIALIZATION_FORMAT)); serdeProps.put(SERIALIZATION_NULL_FORMAT, props.getProperty(SERIALIZATION_NULL_FORMAT)); } SerDeUtils.initializeSerDe(serde, conf, serdeProps, null); return serde; }
/** * Create an object for the given class and initialize it from conf * @param theClass class of which an object is created * @param conf Configuration * @return a new object */ @SuppressWarnings("unchecked") public static <T> T newInstance(Class<T> theClass, Configuration conf) { T result; try { Constructor<?> ctor = CONSTRUCTOR_CACHE.getIfPresent(theClass); if (ctor == null) { ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY); ctor.setAccessible(true); CONSTRUCTOR_CACHE.put(theClass, ctor); } result = (T)ctor.newInstance(); } catch (Exception e) { throw new RuntimeException(e); } setConf(result, conf); return result; }
/** * Check and set 'configuration' if necessary. * * @param theObject object for which to set configuration * @param conf Configuration */ public static void setConf(Object theObject, Configuration conf) { if (conf != null) { if (theObject instanceof Configurable) { ((Configurable) theObject).setConf(conf); } setJobConf(theObject, conf); } }
public Deserializer getDeserializer(Configuration conf, boolean ignoreError) throws Exception { Deserializer de = ReflectionUtil.newInstance( getDeserializerClass().asSubclass(Deserializer.class), conf); if (ignoreError) { SerDeUtils.initializeSerDeWithoutErrorCheck(de, conf, properties, null); } else { SerDeUtils.initializeSerDe(de, conf, properties, null); } return de; }
static AbstractSerDe createDelegate(String className) { final Class<? extends AbstractSerDe> clazz; try { //noinspection unchecked clazz = (Class<? extends AbstractSerDe>) Class.forName(className); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } // we are not setting conf thus null is okay return ReflectionUtil.newInstance(clazz, null); }
public Deserializer getDeserializer(Configuration conf, boolean ignoreError) throws Exception { Deserializer de = ReflectionUtil.newInstance( getDeserializerClass().asSubclass(Deserializer.class), conf); if (ignoreError) { SerDeUtils.initializeSerDeWithoutErrorCheck(de, conf, properties, null); } else { SerDeUtils.initializeSerDe(de, conf, properties, null); } return de; }
private static HiveOutputFormat<?, ?> getHiveOutputFormat( Configuration conf, Class<? extends OutputFormat> outputClass) throws HiveException { OutputFormat<?, ?> outputFormat = ReflectionUtil.newInstance(outputClass, conf); if (!(outputFormat instanceof HiveOutputFormat)) { outputFormat = new HivePassThroughOutputFormat(outputFormat); } return (HiveOutputFormat<?, ?>) outputFormat; }
/** * Return a deserializer object corresponding to the partitionDesc. */ public Deserializer getDeserializer(Configuration conf) throws Exception { Properties schema = getProperties(); String clazzName = getDeserializerClassName(); Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(clazzName) .asSubclass(Deserializer.class), conf); SerDeUtils.initializeSerDe(deserializer, conf, getTableDesc().getProperties(), schema); return deserializer; }
public static Deserializer getDeserializer(Configuration conf, org.apache.hadoop.hive.metastore.api.Table table, boolean skipConfError, String lib) throws MetaException { try { Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(lib). asSubclass(Deserializer.class), conf); if (skipConfError) { SerDeUtils.initializeSerDeWithoutErrorCheck(deserializer, conf, MetaStoreUtils.getTableMetadata(table), null); } else { SerDeUtils.initializeSerDe(deserializer, conf, MetaStoreUtils.getTableMetadata(table), null); } return deserializer; } catch (RuntimeException e) { throw e; } catch (Throwable e) { LOG.error("error in initSerDe: " + e.getClass().getName() + " " + e.getMessage(), e); throw new MetaException(e.getClass().getName() + " " + e.getMessage()); } }
@SuppressWarnings("unchecked") public static InputFormat getInputFormatFromCache( Class<? extends InputFormat> inputFormatClass, Configuration conf) throws IOException { if (Configurable.class.isAssignableFrom(inputFormatClass) || JobConfigurable.class.isAssignableFrom(inputFormatClass)) { return ReflectionUtil.newInstance(inputFormatClass, conf); } // TODO: why is this copy-pasted from HiveInputFormat? InputFormat format = inputFormats.get(inputFormatClass.getName()); if (format == null) { try { format = ReflectionUtil.newInstance(inputFormatClass, conf); inputFormats.put(inputFormatClass.getName(), format); } catch (Exception e) { throw new IOException("Cannot create an instance of InputFormat class " + inputFormatClass.getName() + " as specified in mapredWork!", e); } } return format; }
/** * Check and set 'configuration' if necessary. * * @param theObject object for which to set configuration * @param conf Configuration */ public static void setConf(Object theObject, Configuration conf) { if (conf != null) { if (theObject instanceof Configurable) { ((Configurable) theObject).setConf(conf); } setJobConf(theObject, conf); } }
/** * Create an object for the given class and initialize it from conf * @param theClass class of which an object is created * @param conf Configuration * @return a new object */ @SuppressWarnings("unchecked") public static <T> T newInstance(Class<T> theClass, Configuration conf) { T result; try { Constructor<?> ctor = CONSTRUCTOR_CACHE.getIfPresent(theClass); if (ctor == null) { ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY); ctor.setAccessible(true); CONSTRUCTOR_CACHE.put(theClass, ctor); } result = (T)ctor.newInstance(); } catch (Exception e) { throw new RuntimeException(e); } setConf(result, conf); return result; }
/** * Return a deserializer object corresponding to the partitionDesc. */ public Deserializer getDeserializer(Configuration conf) throws Exception { Properties schema = getProperties(); String clazzName = getDeserializerClassName(); Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(clazzName) .asSubclass(Deserializer.class), conf); SerDeUtils.initializeSerDe(deserializer, conf, getTableDesc().getProperties(), schema); return deserializer; }
public Deserializer getDeserializer(Configuration conf, boolean ignoreError) throws Exception { Deserializer de = ReflectionUtil.newInstance( getDeserializerClass().asSubclass(Deserializer.class), conf); if (ignoreError) { SerDeUtils.initializeSerDeWithoutErrorCheck(de, conf, properties, null); } else { SerDeUtils.initializeSerDe(de, conf, properties, null); } return de; }
private static HiveOutputFormat<?, ?> getHiveOutputFormat( Configuration conf, Class<? extends OutputFormat> outputClass) throws HiveException { OutputFormat<?, ?> outputFormat = ReflectionUtil.newInstance(outputClass, conf); if (!(outputFormat instanceof HiveOutputFormat)) { outputFormat = new HivePassThroughOutputFormat(outputFormat); } return (HiveOutputFormat<?, ?>) outputFormat; }
/** * Check and set 'configuration' if necessary. * * @param theObject object for which to set configuration * @param conf Configuration */ public static void setConf(Object theObject, Configuration conf) { if (conf != null) { if (theObject instanceof Configurable) { ((Configurable) theObject).setConf(conf); } setJobConf(theObject, conf); } }
/** * Create an object for the given class and initialize it from conf * @param theClass class of which an object is created * @param conf Configuration * @return a new object */ @SuppressWarnings("unchecked") public static <T> T newInstance(Class<T> theClass, Configuration conf) { T result; try { Constructor<?> ctor = CONSTRUCTOR_CACHE.getIfPresent(theClass); if (ctor == null) { ctor = theClass.getDeclaredConstructor(EMPTY_ARRAY); ctor.setAccessible(true); CONSTRUCTOR_CACHE.put(theClass, ctor); } result = (T)ctor.newInstance(); } catch (Exception e) { throw new RuntimeException(e); } setConf(result, conf); return result; }
public static AbstractSerDe getSpillSerDe(byte alias, TableDesc[] spillTableDesc, JoinDesc conf, boolean noFilter) { TableDesc desc = getSpillTableDesc(alias, spillTableDesc, conf, noFilter); if (desc == null) { return null; } AbstractSerDe sd = (AbstractSerDe) ReflectionUtil.newInstance(desc.getDeserializerClass(), null); try { SerDeUtils.initializeSerDe(sd, null, desc.getProperties(), null); } catch (SerDeException e) { e.printStackTrace(); return null; } return sd; }
Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(lib). asSubclass(Deserializer.class), conf); if (skipConfError) { SerDeUtils.initializeSerDeWithoutErrorCheck(deserializer, conf, MetaStoreUtils.getTableMetadata(table), null); } else { SerDeUtils.initializeSerDe(deserializer, conf, MetaStoreUtils.getTableMetadata(table), null);
/** NOTE: this no longer wraps the IF for LLAP. Call wrapForLlap manually if needed. */ public static InputFormat<WritableComparable, Writable> getInputFormatFromCache( Class inputFormatClass, JobConf job) throws IOException { InputFormat<WritableComparable, Writable> instance = inputFormats.get(inputFormatClass); if (instance == null) { try { instance = (InputFormat<WritableComparable, Writable>) ReflectionUtil .newInstance(inputFormatClass, job); // HBase input formats are not thread safe today. See HIVE-8808. String inputFormatName = inputFormatClass.getName().toLowerCase(); if (!inputFormatName.contains("hbase")) { inputFormats.put(inputFormatClass, instance); } } catch (Exception e) { throw new IOException("Cannot create an instance of InputFormat class " + inputFormatClass.getName() + " as specified in mapredWork!", e); } } return instance; }