public String getDeserializerClassName() { Properties schema = getProperties(); String clazzName = schema.getProperty(serdeConstants.SERIALIZATION_LIB); if (clazzName == null) { throw new IllegalStateException("Property " + serdeConstants.SERIALIZATION_LIB + " cannot be null"); } return clazzName; }
public String getDeserializerClassName() { Properties schema = getProperties(); String clazzName = schema.getProperty(serdeConstants.SERIALIZATION_LIB); if (clazzName == null) { throw new IllegalStateException("Property " + serdeConstants.SERIALIZATION_LIB + " cannot be null"); } return clazzName; }
@Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public String getTableName() { return getProperties().getProperty(hive_metastoreConstants.META_TABLE_NAME); }
/** * @return the serdeClassName */ @Explain(displayName = "serde", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public String getSerdeClassName() { return getProperties().getProperty(serdeConstants.SERIALIZATION_LIB); }
/** * @return the serdeClassName */ @Explain(displayName = "serde", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public String getSerdeClassName() { return getProperties().getProperty(serdeConstants.SERIALIZATION_LIB); }
@Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public String getTableName() { return getProperties().getProperty(hive_metastoreConstants.META_TABLE_NAME); }
@Explain(displayName = "properties", explainLevels = { Level.EXTENDED }) public Map getPropertiesExplain() { return PlanUtils.getPropertiesExplain(getProperties()); }
private VectorPartitionContext(PartitionDesc partDesc) { this.partDesc = partDesc; TableDesc td = partDesc.getTableDesc(); // Use table properties in case of unpartitioned tables, // and the union of table properties and partition properties, with partition // taking precedence, in the case of partitioned tables Properties overlayedProps = SerDeUtils.createOverlayedProperties(td.getProperties(), partDesc.getProperties()); Map<String, String> partSpec = partDesc.getPartSpec(); tableName = String.valueOf(overlayedProps.getProperty("name")); partName = String.valueOf(partSpec); }
@Explain(displayName = "properties", explainLevels = { Level.EXTENDED }) public Map getPropertiesExplain() { return HiveStringUtils.getPropertiesExplain(getProperties()); }
private VectorPartitionContext(PartitionDesc partDesc) { this.partDesc = partDesc; TableDesc td = partDesc.getTableDesc(); // Use table properties in case of unpartitioned tables, // and the union of table properties and partition properties, with partition // taking precedence, in the case of partitioned tables Properties overlayedProps = SerDeUtils.createOverlayedProperties(td.getProperties(), partDesc.getProperties()); Map<String, String> partSpec = partDesc.getPartSpec(); tableName = String.valueOf(overlayedProps.getProperty("name")); partName = String.valueOf(partSpec); }
private PartitionDesc changePartitionToMetadataOnly(PartitionDesc desc, Path path) { if (desc == null) { return null; } boolean isEmpty = false; try { isEmpty = Utilities.isEmptyPath(physicalContext.getConf(), path); } catch (IOException e) { LOG.error("Cannot determine if the table is empty", e); } desc.setInputFileFormatClass( isEmpty ? ZeroRowsInputFormat.class : OneNullRowInputFormat.class); desc.setOutputFileFormatClass(HiveIgnoreKeyTextOutputFormat.class); desc.getProperties().setProperty(serdeConstants.SERIALIZATION_LIB, NullStructSerDe.class.getName()); return desc; }
private PartitionDesc changePartitionToMetadataOnly(PartitionDesc desc, Path path) { if (desc == null) return null; boolean isEmpty = false; try { isEmpty = Utilities.isEmptyPath(physicalContext.getConf(), path); } catch (IOException e) { LOG.error("Cannot determine if the table is empty", e); } desc.setInputFileFormatClass( isEmpty ? ZeroRowsInputFormat.class : OneNullRowInputFormat.class); desc.setOutputFileFormatClass(HiveIgnoreKeyTextOutputFormat.class); desc.getProperties().setProperty(serdeConstants.SERIALIZATION_LIB, NullStructSerDe.class.getName()); return desc; }
private boolean needConversion(TableDesc tableDesc, List<PartitionDesc> partDescs) { Class<?> tableSerDe = tableDesc.getDeserializerClass(); SerDeSpec spec = AnnotationUtils.getAnnotation(tableSerDe, SerDeSpec.class); if (null == spec) { // Serde may not have this optional annotation defined in which case be conservative // and say conversion is needed. return true; } String[] schemaProps = spec.schemaProps(); Properties tableProps = tableDesc.getProperties(); for (PartitionDesc partitionDesc : partDescs) { if (!tableSerDe.getName().equals(partitionDesc.getDeserializerClassName())) { return true; } Properties partProps = partitionDesc.getProperties(); for (String schemaProp : schemaProps) { if (!org.apache.commons.lang3.StringUtils.equals( tableProps.getProperty(schemaProp), partProps.getProperty(schemaProp))) { return true; } } } return false; }
/** * Return a deserializer object corresponding to the partitionDesc. */ public Deserializer getDeserializer(Configuration conf) throws Exception { Properties schema = getProperties(); String clazzName = getDeserializerClassName(); Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(clazzName) .asSubclass(Deserializer.class), conf); SerDeUtils.initializeSerDe(deserializer, conf, getTableDesc().getProperties(), schema); return deserializer; }
/** * Return a deserializer object corresponding to the partitionDesc. */ public Deserializer getDeserializer(Configuration conf) throws Exception { Properties schema = getProperties(); String clazzName = getDeserializerClassName(); Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(clazzName) .asSubclass(Deserializer.class), conf); SerDeUtils.initializeSerDe(deserializer, conf, getTableDesc().getProperties(), schema); return deserializer; }
@SuppressWarnings("rawtypes") private static Path createDummyFileForEmptyPartition(Path path, JobConf job, PartitionDesc partDesc, Path hiveScratchDir) throws Exception { String strPath = path.toString(); // The input file does not exist, replace it by a empty file if (partDesc.getTableDesc().isNonNative()) { // if this isn't a hive table we can't create an empty file for it. return path; } Properties props = SerDeUtils.createOverlayedProperties( partDesc.getTableDesc().getProperties(), partDesc.getProperties()); HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, partDesc); boolean oneRow = partDesc.getInputFileFormatClass() == OneNullRowInputFormat.class; Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job, props, oneRow); LOG.info("Changed input file {} to empty file {} ({})", strPath, newPath, oneRow); return newPath; }
@Override public int hashCode() { final int prime = 31; int result = 1; result = result * prime + (getInputFileFormatClass() == null ? 0 : getInputFileFormatClass().hashCode()); result = result * prime + (getOutputFileFormatClass() == null ? 0 : getOutputFileFormatClass().hashCode()); result = result * prime + (getProperties() == null ? 0 : getProperties().hashCode()); result = result * prime + (getTableDesc() == null ? 0 : getTableDesc().hashCode()); result = result * prime + (getPartSpec() == null ? 0 : getPartSpec().hashCode()); result = result * prime + (getVectorPartitionDesc() == null ? 0 : getVectorPartitionDesc().hashCode()); return result; }
@Override public int hashCode() { final int prime = 31; int result = 1; result = result * prime + (getInputFileFormatClass() == null ? 0 : getInputFileFormatClass().hashCode()); result = result * prime + (getOutputFileFormatClass() == null ? 0 : getOutputFileFormatClass().hashCode()); result = result * prime + (getProperties() == null ? 0 : getProperties().hashCode()); result = result * prime + (getTableDesc() == null ? 0 : getTableDesc().hashCode()); result = result * prime + (getPartSpec() == null ? 0 : getPartSpec().hashCode()); result = result * prime + (getVectorPartitionDesc() == null ? 0 : getVectorPartitionDesc().hashCode()); return result; }
public void initEmptyInputChildren(List<Operator<?>> children, Configuration hconf) throws SerDeException, Exception { setChildOperators(children); Map<String, Configuration> tableNameToConf = cloneConfsForNestedColPruning(hconf); for (Operator<?> child : children) { TableScanOperator tsOp = (TableScanOperator) child; StructObjectInspector soi = null; PartitionDesc partDesc = conf.getAliasToPartnInfo().get(tsOp.getConf().getAlias()); Configuration newConf = tableNameToConf.get(partDesc.getTableDesc().getTableName()); Deserializer serde = partDesc.getTableDesc().getDeserializer(); partDesc.setProperties(partDesc.getProperties()); MapOpCtx opCtx = new MapOpCtx(tsOp.getConf().getAlias(), child, partDesc); StructObjectInspector tableRowOI = (StructObjectInspector) serde.getObjectInspector(); initObjectInspector(newConf, opCtx, tableRowOI); soi = opCtx.rowObjectInspector; child.getParentOperators().add(this); childrenOpToOI.put(child, soi); } }
public void initEmptyInputChildren(List<Operator<?>> children, Configuration hconf) throws SerDeException, Exception { setChildOperators(children); Map<String, Configuration> tableNameToConf = cloneConfsForNestedColPruning(hconf); for (Operator<?> child : children) { TableScanOperator tsOp = (TableScanOperator) child; StructObjectInspector soi = null; PartitionDesc partDesc = conf.getAliasToPartnInfo().get(tsOp.getConf().getAlias()); Configuration newConf = tableNameToConf.get(partDesc.getTableDesc().getTableName()); Deserializer serde = partDesc.getTableDesc().getDeserializer(); partDesc.setProperties(partDesc.getProperties()); MapOpCtx opCtx = new MapOpCtx(tsOp.getConf().getAlias(), child, partDesc); StructObjectInspector tableRowOI = (StructObjectInspector) serde.getObjectInspector(); initObjectInspector(newConf, opCtx, tableRowOI); soi = opCtx.rowObjectInspector; child.getParentOperators().add(this); childrenOpToOI.put(child, soi); } }