public static VectorPartitionDesc createVectorizedInputFileFormat(String inputFileFormatClassName, boolean isInputFileFormatSelfDescribing, TypeInfo[] dataTypeInfos) { return new VectorPartitionDesc( inputFileFormatClassName, isInputFileFormatSelfDescribing, VectorMapOperatorReadType.VECTORIZED_INPUT_FILE_FORMAT, dataTypeInfos); }
@Override public boolean equals(Object o) { if (o instanceof VectorPartitionDesc) { VectorPartitionDesc other = (VectorPartitionDesc) o; return Strings.nullToEmpty(getInputFileFormatClassName()).equals( Strings.nullToEmpty(other.getInputFileFormatClassName())) && Strings.nullToEmpty(getRowDeserializerClassName()).equals( Strings.nullToEmpty(other.getRowDeserializerClassName())) && getVectorDeserializeType() == other.getVectorDeserializeType() && getVectorMapOperatorReadType() == other.getVectorMapOperatorReadType() && getIsInputFileFormatSelfDescribing() == other.getIsInputFileFormatSelfDescribing() && Arrays.equals(getDataTypeInfos(), other.getDataTypeInfos()); } return false; }
VectorPartitionDesc.createVectorizedInputFileFormat( inputFileFormatClassName, Utilities.isInputFileFormatSelfDescribing(pd), VectorPartitionDesc.createVectorizedInputFileFormat( inputFileFormatClassName, Utilities.isInputFileFormatSelfDescribing(pd), VectorPartitionDesc.createVectorDeserialize( inputFileFormatClassName, VectorDeserializeType.LAZY_SIMPLE, dataTypeInfos), vectorPartitionDescMap); VectorPartitionDesc.createVectorDeserialize( inputFileFormatClassName, VectorDeserializeType.LAZY_BINARY, dataTypeInfos), vectorPartitionDescMap); addVectorPartitionDesc( pd, VectorPartitionDesc.createRowDeserialize( inputFileFormatClassName, Utilities.isInputFileFormatSelfDescribing(pd),
TypeInfo[] dataTypeInfos = vectorPartDesc.getDataTypeInfos(); switch (vectorPartDesc.getVectorDeserializeType()) { case LAZY_SIMPLE: "Unexpected vector deserialize row type " + vectorPartDesc.getVectorDeserializeType().name());
if (vectorPartDesc.getIsInputFileFormatSelfDescribing()) { vectorPartDesc.setDataTypeInfos(nextDataTypeInfoList);
if (isVectorized) { part.setVectorPartitionDesc( VectorPartitionDesc.createVectorizedInputFileFormat("MockInputFileFormatClassName", false, null));
@Override public PartitionDesc clone() { PartitionDesc ret = new PartitionDesc(); ret.inputFileFormatClass = inputFileFormatClass; ret.outputFileFormatClass = outputFileFormatClass; if (properties != null) { Properties newProp = new Properties(); Enumeration<Object> keysProp = properties.keys(); while (keysProp.hasMoreElements()) { Object key = keysProp.nextElement(); newProp.put(key, properties.get(key)); } ret.setProperties(newProp); } ret.tableDesc = (TableDesc) tableDesc.clone(); // The partition spec is not present if (partSpec != null) { ret.partSpec = new LinkedHashMap<>(partSpec); } if (vectorPartitionDesc != null) { ret.vectorPartitionDesc = vectorPartitionDesc.clone(); } return ret; }
public void init(Configuration hconf) throws Exception { VectorPartitionDesc vectorPartDesc = partDesc.getVectorPartitionDesc(); partDeserializer = partDesc.getDeserializer(hconf); if (partDeserializer instanceof OrcSerde) { // UNDONE: We need to get the table schema inspector from self-describing Input File // Formats like ORC. Modify the ORC serde instead? For now, this works. partRawRowObjectInspector = (StructObjectInspector) OrcStruct.createObjectInspector(tableStructTypeInfo); } else { partRawRowObjectInspector = (StructObjectInspector) partDeserializer.getObjectInspector(); } TypeInfo[] dataTypeInfos = vectorPartDesc.getDataTypeInfos(); vectorAssign = new VectorAssignRow(); // Initialize with data type conversion parameters. readerColumnCount = vectorAssign.initConversion(dataTypeInfos, tableRowTypeInfos, dataColumnsToIncludeTruncated); } }
VectorPartitionDesc vecPartDesc2 = other.getVectorPartitionDesc(); return (vecPartDesc1 == null && vecPartDesc2 == null) || (vecPartDesc1 != null && vecPartDesc1.equals(vecPartDesc2));
if (!vectorPartDesc.getIsInputFileFormatSelfDescribing()) { final int nextDataTypeInfoSize = nextDataTypeInfoList.size(); if (nextDataTypeInfoSize > tableDataTypeInfoList.size()) {
VectorPartitionDesc.createVectorizedInputFileFormat( inputFileFormatClassName, Utilities.isInputFileFormatSelfDescribing(pd))); VectorPartitionDesc.createVectorizedInputFileFormat( inputFileFormatClassName, Utilities.isInputFileFormatSelfDescribing(pd))); VectorPartitionDesc.createVectorDeserialize( inputFileFormatClassName, VectorDeserializeType.LAZY_SIMPLE)); VectorPartitionDesc.createVectorDeserialize( inputFileFormatClassName, VectorDeserializeType.LAZY_BINARY)); VectorPartitionDesc.createRowDeserialize( inputFileFormatClassName, Utilities.isInputFileFormatSelfDescribing(pd),
TypeInfo[] dataTypeInfos = vectorPartDesc.getDataTypeInfos(); switch (vectorPartDesc.getVectorDeserializeType()) { case LAZY_SIMPLE: "Unexpected vector deserialize row type " + vectorPartDesc.getVectorDeserializeType().name());
@Override public PartitionDesc clone() { PartitionDesc ret = new PartitionDesc(); ret.inputFileFormatClass = inputFileFormatClass; ret.outputFileFormatClass = outputFileFormatClass; if (properties != null) { ret.setProperties((Properties) properties.clone()); } ret.tableDesc = (TableDesc) tableDesc.clone(); // The partition spec is not present if (partSpec != null) { ret.partSpec = new LinkedHashMap<>(partSpec); } if (vectorPartitionDesc != null) { ret.vectorPartitionDesc = vectorPartitionDesc.clone(); } return ret; }
public void init(Configuration hconf) throws Exception { VectorPartitionDesc vectorPartDesc = partDesc.getVectorPartitionDesc(); partDeserializer = partDesc.getDeserializer(hconf); if (partDeserializer instanceof OrcSerde) { // UNDONE: We need to get the table schema inspector from self-describing Input File // Formats like ORC. Modify the ORC serde instead? For now, this works. partRawRowObjectInspector = (StructObjectInspector) OrcStruct.createObjectInspector(tableStructTypeInfo); } else { partRawRowObjectInspector = (StructObjectInspector) partDeserializer.getObjectInspector(); } TypeInfo[] dataTypeInfos = vectorPartDesc.getDataTypeInfos(); vectorAssign = new VectorAssignRow(); // Initialize with data type conversion parameters. readerColumnCount = vectorAssign.initConversion(dataTypeInfos, tableRowTypeInfos, dataColumnsToIncludeTruncated); } }
VectorPartitionDesc vecPartDesc2 = other.getVectorPartitionDesc(); return (vecPartDesc1 == null && vecPartDesc2 == null) || (vecPartDesc1 != null && vecPartDesc1.equals(vecPartDesc2));
@Override public int hashCode() { final int prime = 31; int result = 1; result = result * prime + (getInputFileFormatClassName() == null ? 0 : getInputFileFormatClassName().hashCode()); result = result * prime + (getRowDeserializerClassName() == null ? 0 : getRowDeserializerClassName().hashCode()); result = result * prime + (getVectorDeserializeType() == null ? 0 : getVectorDeserializeType().hashCode()); result = result * prime + (getVectorMapOperatorReadType() == null ? 0 : getVectorMapOperatorReadType().hashCode()); result = result * prime + Boolean.valueOf(getIsInputFileFormatSelfDescribing()).hashCode(); result = result * prime + Arrays.hashCode(getDataTypeInfos()); return result; }
public static VectorPartitionDesc createRowDeserialize(String inputFileFormatClassName, boolean isInputFileFormatSelfDescribing, String rowDeserializerClassName, TypeInfo[] dataTypeInfos) { return new VectorPartitionDesc(rowDeserializerClassName, isInputFileFormatSelfDescribing, inputFileFormatClassName, dataTypeInfos); }
@Override public int hashCode() { final int prime = 31; int result = 1; result = result * prime + (getInputFileFormatClassName() == null ? 0 : getInputFileFormatClassName().hashCode()); result = result * prime + (getRowDeserializerClassName() == null ? 0 : getRowDeserializerClassName().hashCode()); result = result * prime + (getVectorDeserializeType() == null ? 0 : getVectorDeserializeType().hashCode()); result = result * prime + (getVectorMapOperatorReadType() == null ? 0 : getVectorMapOperatorReadType().hashCode()); result = result * prime + Boolean.valueOf(getIsInputFileFormatSelfDescribing()).hashCode(); result = result * prime + Arrays.hashCode(getDataTypeInfos()); return result; }
public static VectorPartitionDesc createVectorDeserialize(String inputFileFormatClassName, VectorDeserializeType vectorDeserializeType, TypeInfo[] dataTypeInfos) { return new VectorPartitionDesc(inputFileFormatClassName, vectorDeserializeType, dataTypeInfos); }
@Override public boolean equals(Object o) { if (o instanceof VectorPartitionDesc) { VectorPartitionDesc other = (VectorPartitionDesc) o; return Strings.nullToEmpty(getInputFileFormatClassName()).equals( Strings.nullToEmpty(other.getInputFileFormatClassName())) && Strings.nullToEmpty(getRowDeserializerClassName()).equals( Strings.nullToEmpty(other.getRowDeserializerClassName())) && getVectorDeserializeType() == other.getVectorDeserializeType() && getVectorMapOperatorReadType() == other.getVectorMapOperatorReadType() && getIsInputFileFormatSelfDescribing() == other.getIsInputFileFormatSelfDescribing() && Arrays.equals(getDataTypeInfos(), other.getDataTypeInfos()); } return false; }