private PartitionDesc extractSinglePartSpec(CombineHiveInputSplit hsplit) throws IOException { PartitionDesc part = null; Map<Map<Path,PartitionDesc>, Map<Path,PartitionDesc>> cache = new HashMap<>(); for (Path path : hsplit.getPaths()) { PartitionDesc otherPart = HiveFileFormatUtils.getFromPathRecursively( pathToPartInfo, path, cache); LOG.debug("Found spec for " + path + " " + otherPart + " from " + pathToPartInfo); if (part == null) { part = otherPart; } else if (otherPart != part) { // Assume we should have the exact same object. // TODO: we could also compare the schema and SerDe, and pass only those to the call // instead; most of the time these would be the same and LLAP IO can handle that. LOG.warn("Multiple partitions found; not going to pass a part spec to LLAP IO: {" + part.getPartSpec() + "} and {" + otherPart.getPartSpec() + "}"); return null; } } return part; }
private PartitionDesc extractSinglePartSpec(CombineHiveInputSplit hsplit) throws IOException { PartitionDesc part = null; Map<Map<Path,PartitionDesc>, Map<Path,PartitionDesc>> cache = new HashMap<>(); for (Path path : hsplit.getPaths()) { PartitionDesc otherPart = HiveFileFormatUtils.getPartitionDescFromPathRecursively( pathToPartInfo, path, cache); LOG.debug("Found spec for " + path + " " + otherPart + " from " + pathToPartInfo); if (part == null) { part = otherPart; } else if (otherPart != part) { // Assume we should have the exact same object. // TODO: we could also compare the schema and SerDe, and pass only those to the call // instead; most of the time these would be the same and LLAP IO can handle that. LOG.warn("Multiple partitions found; not going to pass a part spec to LLAP IO: {" + part.getPartSpec() + "} and {" + otherPart.getPartSpec() + "}"); return null; } } return part; }
private VectorPartitionContext(PartitionDesc partDesc) { this.partDesc = partDesc; TableDesc td = partDesc.getTableDesc(); // Use table properties in case of unpartitioned tables, // and the union of table properties and partition properties, with partition // taking precedence, in the case of partitioned tables Properties overlayedProps = SerDeUtils.createOverlayedProperties(td.getProperties(), partDesc.getProperties()); Map<String, String> partSpec = partDesc.getPartSpec(); tableName = String.valueOf(overlayedProps.getProperty("name")); partName = String.valueOf(partSpec); }
private VectorPartitionContext(PartitionDesc partDesc) { this.partDesc = partDesc; TableDesc td = partDesc.getTableDesc(); // Use table properties in case of unpartitioned tables, // and the union of table properties and partition properties, with partition // taking precedence, in the case of partitioned tables Properties overlayedProps = SerDeUtils.createOverlayedProperties(td.getProperties(), partDesc.getProperties()); Map<String, String> partSpec = partDesc.getPartSpec(); tableName = String.valueOf(overlayedProps.getProperty("name")); partName = String.valueOf(partSpec); }
Path p = it.next(); PartitionDesc desc = work.getPathToPartitionInfo().get(p); Map<String, String> spec = desc.getPartSpec(); if (spec == null) { throw new IllegalStateException("No partition spec found in dynamic pruning");
private Object[] createPartValue(PartitionDesc partDesc, StructObjectInspector partOI) { Map<String, String> partSpec = partDesc.getPartSpec(); List<? extends StructField> fields = partOI.getAllStructFieldRefs(); Object[] partValues = new Object[fields.size()]; for (int i = 0; i < partValues.length; i++) { StructField field = fields.get(i); String value = partSpec.get(field.getFieldName()); ObjectInspector oi = field.getFieldObjectInspector(); partValues[i] = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi).convert(value); } return partValues; }
Path p = it.next(); PartitionDesc desc = work.getPathToPartitionInfo().get(p); Map<String, String> spec = desc.getPartSpec(); Preconditions.checkNotNull(spec, "No partition spec found in dynamic pruning");
Path p = it.next(); PartitionDesc desc = work.getPathToPartitionInfo().get(p); Map<String, String> spec = desc.getPartSpec(); if (spec == null) { throw new IllegalStateException("No partition spec found in dynamic pruning");
private Object[] createPartValue(PartitionDesc partDesc, StructObjectInspector partOI) { Map<String, String> partSpec = partDesc.getPartSpec(); List<? extends StructField> fields = partOI.getAllStructFieldRefs(); Object[] partValues = new Object[fields.size()]; for (int i = 0; i < partValues.length; i++) { StructField field = fields.get(i); String value = partSpec.get(field.getFieldName()); ObjectInspector oi = field.getFieldObjectInspector(); partValues[i] = ObjectInspectorConverters.getConverter( PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi).convert(value); } return partValues; }
private void processAlias(MapWork work, Path path, Collection<String> aliasesAffected, Set<String> aliases) { // the aliases that are allowed to map to a null scan. Collection<String> allowed = aliasesAffected.stream() .filter(a -> aliases.contains(a)).collect(Collectors.toList()); if (!allowed.isEmpty()) { PartitionDesc partDesc = work.getPathToPartitionInfo().get(path).clone(); PartitionDesc newPartition = changePartitionToMetadataOnly(partDesc, path); // Prefix partition with something to avoid it being a hidden file. Path fakePath = new Path(NullScanFileSystem.getBase() + newPartition.getTableName() + "/part" + encode(newPartition.getPartSpec())); StringInternUtils.internUriStringsInPath(fakePath); work.addPathToPartitionInfo(fakePath, newPartition); work.addPathToAlias(fakePath, new ArrayList<>(allowed)); aliasesAffected.removeAll(allowed); if (aliasesAffected.isEmpty()) { work.removePathToAlias(path); work.removePathToPartitionInfo(path); } } }
Path p = it.next(); PartitionDesc desc = work.getPathToPartitionInfo().get(p); Map<String, String> spec = desc.getPartSpec(); if (spec == null) { throw new AssertionException("No partition spec found in dynamic pruning");
PartitionDesc partDesc = mapWork.getPathToPartitionInfo().get(paths.next()); if (partDesc != null) { LinkedHashMap<String, String> partSpec = partDesc.getPartSpec(); if (partSpec != null && !partSpec.isEmpty()) { partitionColumnCount = partSpec.size();
public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx, PartitionDesc partDesc, Object[] partitionValues) { LinkedHashMap<String, String> partSpec = partDesc.getPartSpec(); for (int i = 0; i < vrbCtx.partitionColumnCount; i++) { Object objectValue; if (partSpec == null) { // For partition-less table, initialize partValue to empty string. // We can have partition-less table even if we have partition keys // when there is only only partition selected and the partition key is not // part of the projection/include list. objectValue = null; } else { String key = vrbCtx.rowColumnNames[vrbCtx.dataColumnCount + i]; // Create a Standard java object Inspector TypeInfo partColTypeInfo = vrbCtx.rowColumnTypeInfos[vrbCtx.dataColumnCount + i]; ObjectInspector objectInspector = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(partColTypeInfo); objectValue = ObjectInspectorConverters. getConverter(PrimitiveObjectInspectorFactory. javaStringObjectInspector, objectInspector). convert(partSpec.get(key)); if (partColTypeInfo instanceof CharTypeInfo) { objectValue = ((HiveChar) objectValue).getStrippedValue(); } } partitionValues[i] = objectValue; } }
private void processAlias(MapWork work, Path path, ArrayList<String> aliasesAffected, ArrayList<String> aliases) { // the aliases that are allowed to map to a null scan. ArrayList<String> allowed = new ArrayList<String>(); for (String alias : aliasesAffected) { if (aliases.contains(alias)) { allowed.add(alias); } } if (allowed.size() > 0) { PartitionDesc partDesc = work.getPathToPartitionInfo().get(path).clone(); PartitionDesc newPartition = changePartitionToMetadataOnly(partDesc, path); // Prefix partition with something to avoid it being a hidden file. Path fakePath = new Path(NullScanFileSystem.getBase() + newPartition.getTableName() + "/part" + encode(newPartition.getPartSpec())); StringInternUtils.internUriStringsInPath(fakePath); work.addPathToPartitionInfo(fakePath, newPartition); work.addPathToAlias(fakePath, new ArrayList<>(allowed)); aliasesAffected.removeAll(allowed); if (aliasesAffected.isEmpty()) { work.removePathToAlias(path); work.removePathToPartitionInfo(path); } } }
public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx, PartitionDesc partDesc, Object[] partitionValues) { LinkedHashMap<String, String> partSpec = partDesc.getPartSpec(); for (int i = 0; i < vrbCtx.partitionColumnCount; i++) { Object objectValue; if (partSpec == null) { // For partition-less table, initialize partValue to empty string. // We can have partition-less table even if we have partition keys // when there is only only partition selected and the partition key is not // part of the projection/include list. objectValue = null; } else { String key = vrbCtx.rowColumnNames[vrbCtx.dataColumnCount + i]; // Create a Standard java object Inspector TypeInfo partColTypeInfo = vrbCtx.rowColumnTypeInfos[vrbCtx.dataColumnCount + i]; ObjectInspector objectInspector = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(partColTypeInfo); objectValue = ObjectInspectorConverters. getConverter(PrimitiveObjectInspectorFactory. javaStringObjectInspector, objectInspector). convert(partSpec.get(key)); if (partColTypeInfo instanceof CharTypeInfo) { objectValue = ((HiveChar) objectValue).getStrippedValue(); } } partitionValues[i] = objectValue; } }
@Override public int hashCode() { final int prime = 31; int result = 1; result = result * prime + (getInputFileFormatClass() == null ? 0 : getInputFileFormatClass().hashCode()); result = result * prime + (getOutputFileFormatClass() == null ? 0 : getOutputFileFormatClass().hashCode()); result = result * prime + (getProperties() == null ? 0 : getProperties().hashCode()); result = result * prime + (getTableDesc() == null ? 0 : getTableDesc().hashCode()); result = result * prime + (getPartSpec() == null ? 0 : getPartSpec().hashCode()); result = result * prime + (getVectorPartitionDesc() == null ? 0 : getVectorPartitionDesc().hashCode()); return result; }
@Override public int hashCode() { final int prime = 31; int result = 1; result = result * prime + (getInputFileFormatClass() == null ? 0 : getInputFileFormatClass().hashCode()); result = result * prime + (getOutputFileFormatClass() == null ? 0 : getOutputFileFormatClass().hashCode()); result = result * prime + (getProperties() == null ? 0 : getProperties().hashCode()); result = result * prime + (getTableDesc() == null ? 0 : getTableDesc().hashCode()); result = result * prime + (getPartSpec() == null ? 0 : getPartSpec().hashCode()); result = result * prime + (getVectorPartitionDesc() == null ? 0 : getVectorPartitionDesc().hashCode()); return result; }
Map<String, String> partSpec1 = getPartSpec(); Map<String, String> partSpec2 = other.getPartSpec(); cond = (partSpec1 == null && partSpec2 == null) || (partSpec1 != null && partSpec1.equals(partSpec2));
@Override public PartitionDesc read(Kryo kryo, Input input, Class<PartitionDesc> type) { PartitionDesc partitionDesc = super.read(kryo, input, type); // The set methods in PartitionDesc intern the any duplicate strings which is why we call them // during de-serialization partitionDesc.setBaseFileName(partitionDesc.getBaseFileName()); partitionDesc.setPartSpec(partitionDesc.getPartSpec()); partitionDesc.setInputFileFormatClass(partitionDesc.getInputFileFormatClass()); partitionDesc.setOutputFileFormatClass(partitionDesc.getOutputFileFormatClass()); return partitionDesc; } }
Map<String, String> partSpec1 = getPartSpec(); Map<String, String> partSpec2 = other.getPartSpec(); cond = (partSpec1 == null && partSpec2 == null) || (partSpec1 != null && partSpec1.equals(partSpec2));