private ValidWriteIdList extractValidWriteIdList() { if (currDesc.getTableName() == null || !org.apache.commons.lang.StringUtils.isBlank(currDesc.getTableName())) { String txnString = job.get(ValidWriteIdList.VALID_WRITEIDS_KEY); LOG.debug("FetchOperator get writeIdStr: " + txnString); return txnString == null ? new ValidReaderWriteIdList() : new ValidReaderWriteIdList(txnString); } return null; // not fetching from a table directly but from a temp location }
String tableName = conf.getPathToPartitionInfo().get(e.getKey()).getTableName(); if (tableNameToConf.containsKey(tableName)) { continue; if (!tableNameToConf.containsKey(pd.getTableName())) { tableNameToConf.put(pd.getTableName(), hconf); if (!tableNameToConf.containsKey(pd.getTableName())) { tableNameToConf.put(pd.getTableName(), hconf);
String tableName = conf.getPathToPartitionInfo().get(e.getKey()).getTableName(); for (String alias: aliases) { Operator<?> rootOp = conf.getAliasToWork().get(alias); if (!tableNameToConf.containsKey(pd.getTableName())) { tableNameToConf.put(pd.getTableName(), hconf); if (!tableNameToConf.containsKey(pd.getTableName())) { tableNameToConf.put(pd.getTableName(), hconf);
private void processCurrPathForMmWriteIds(InputFormat inputFormat, List<Path> dirs, List<Path> dirsWithOriginals) throws IOException { if (inputFormat instanceof HiveInputFormat) { dirs.add(currPath); // No need to process here. } ValidWriteIdList validWriteIdList; if (AcidUtils.isInsertOnlyTable(currDesc.getTableDesc().getProperties())) { validWriteIdList = extractValidWriteIdList(); } else { validWriteIdList = null; // non-MM case } if (validWriteIdList != null) { Utilities.FILE_OP_LOGGER.info("Processing " + currDesc.getTableName() + " for MM paths"); } HiveInputFormat.processPathsForMmRead( Lists.newArrayList(currPath), job, validWriteIdList, dirs, dirsWithOriginals); }
private void processAlias(MapWork work, Path path, Collection<String> aliasesAffected, Set<String> aliases) { // the aliases that are allowed to map to a null scan. Collection<String> allowed = aliasesAffected.stream() .filter(a -> aliases.contains(a)).collect(Collectors.toList()); if (!allowed.isEmpty()) { PartitionDesc partDesc = work.getPathToPartitionInfo().get(path).clone(); PartitionDesc newPartition = changePartitionToMetadataOnly(partDesc, path); // Prefix partition with something to avoid it being a hidden file. Path fakePath = new Path(NullScanFileSystem.getBase() + newPartition.getTableName() + "/part" + encode(newPartition.getPartSpec())); StringInternUtils.internUriStringsInPath(fakePath); work.addPathToPartitionInfo(fakePath, newPartition); work.addPathToAlias(fakePath, new ArrayList<>(allowed)); aliasesAffected.removeAll(allowed); if (aliasesAffected.isEmpty()) { work.removePathToAlias(path); work.removePathToPartitionInfo(path); } } }
private void processAlias(MapWork work, Path path, ArrayList<String> aliasesAffected, ArrayList<String> aliases) { // the aliases that are allowed to map to a null scan. ArrayList<String> allowed = new ArrayList<String>(); for (String alias : aliasesAffected) { if (aliases.contains(alias)) { allowed.add(alias); } } if (allowed.size() > 0) { PartitionDesc partDesc = work.getPathToPartitionInfo().get(path).clone(); PartitionDesc newPartition = changePartitionToMetadataOnly(partDesc, path); // Prefix partition with something to avoid it being a hidden file. Path fakePath = new Path(NullScanFileSystem.getBase() + newPartition.getTableName() + "/part" + encode(newPartition.getPartSpec())); StringInternUtils.internUriStringsInPath(fakePath); work.addPathToPartitionInfo(fakePath, newPartition); work.addPathToAlias(fakePath, new ArrayList<>(allowed)); aliasesAffected.removeAll(allowed); if (aliasesAffected.isEmpty()) { work.removePathToAlias(path); work.removePathToPartitionInfo(path); } } }
if (LOG.isInfoEnabled()) { LOG.info("Resetting VectorMapOperatorReadType from " + old + " for partition " + part.getTableName() + " " + part.getPartSpec());
private void processAlias(MapWork work, String path, ArrayList<String> aliasesAffected, ArrayList<String> aliases) { // the aliases that are allowed to map to a null scan. ArrayList<String> allowed = new ArrayList<String>(); for (String alias : aliasesAffected) { if (aliases.contains(alias)) { allowed.add(alias); } } if (allowed.size() > 0) { work.setUseOneNullRowInputFormat(true); PartitionDesc partDesc = work.getPathToPartitionInfo().get(path).clone(); PartitionDesc newPartition = changePartitionToMetadataOnly(partDesc); Path fakePath = new Path(physicalContext.getContext().getMRTmpPath() + newPartition.getTableName() + encode(newPartition.getPartSpec())); work.getPathToPartitionInfo().put(fakePath.getName(), newPartition); work.getPathToAliases().put(fakePath.getName(), new ArrayList<String>(allowed)); aliasesAffected.removeAll(allowed); if (aliasesAffected.isEmpty()) { work.getPathToAliases().remove(path); work.getPathToPartitionInfo().remove(path); } } }