@Override public int compareTo(CompactionInfo o) { return getFullPartitionName().compareTo(o.getFullPartitionName()); } public String toString() {
@Override public int hashCode() { int result = 17; result = 31 * result + this.getFullPartitionName().hashCode(); return result; }
/** * Get the partition being compacted. * @param ci compaction info returned from the compaction queue * @return metastore partition, or null if there is not partition in this compaction info * @throws Exception if underlying calls throw, or if the partition name resolves to more than * one partition. */ protected Partition resolvePartition(CompactionInfo ci) throws Exception { if (ci.partName != null) { List<Partition> parts; try { parts = getPartitionsByNames(ci); if (parts == null || parts.size() == 0) { // The partition got dropped before we went looking for it. return null; } } catch (Exception e) { LOG.error("Unable to find partition " + ci.getFullPartitionName() + ", " + e.getMessage()); throw e; } if (parts.size() != 1) { LOG.error(ci.getFullPartitionName() + " does not refer to a single partition. " + parts); throw new MetaException("Too many partitions for : " + ci.getFullPartitionName()); } return parts.get(0); } else { return null; } }
/** * Get the partition being compacted. * @param ci compaction info returned from the compaction queue * @return metastore partition, or null if there is not partition in this compaction info * @throws Exception if underlying calls throw, or if the partition name resolves to more than * one partition. */ protected Partition resolvePartition(CompactionInfo ci) throws Exception { if (ci.partName != null) { List<Partition> parts = null; try { parts = rs.getPartitionsByNames(ci.dbname, ci.tableName, Collections.singletonList(ci.partName)); if (parts == null || parts.size() == 0) { // The partition got dropped before we went looking for it. return null; } } catch (Exception e) { LOG.error("Unable to find partition " + ci.getFullPartitionName() + ", " + e.getMessage()); throw e; } if (parts.size() != 1) { LOG.error(ci.getFullPartitionName() + " does not refer to a single partition. " + parts); throw new MetaException("Too many partitions for : " + ci.getFullPartitionName()); } return parts.get(0); } else { return null; } }
LOG.debug("Found too many aborted transactions for " + ci.getFullPartitionName() + ", " + "initiating major compaction"); return CompactionType.MAJOR; return determineCompactionType(ci, writeIds, sd, tblproperties); } else { LOG.info("Going to initiate as user " + runAs + " for " + ci.getFullPartitionName()); UserGroupInformation ugi = UserGroupInformation.createProxyUser(runAs, UserGroupInformation.getLoginUser()); } catch (IOException exception) { LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " + ci.getFullPartitionName(), exception);
LOG.debug("Found too many aborted transactions for " + ci.getFullPartitionName() + ", " + "initiating major compaction"); return CompactionType.MAJOR; } catch (IOException exception) { LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " + ci.getFullPartitionName(), exception);
if(!ci.getFullPartitionName().equals(lastCompactedEntity)) { lastCompactedEntity = ci.getFullPartitionName(); rc = new RetentionCounters(MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED), getFailedCompactionRetention(),
private void clean(CompactionInfo ci) throws MetaException { LOG.info("Starting cleaning for " + ci.getFullPartitionName()); try { Table t = resolveTable(ci); if (p == null) { LOG.info("Unable to find partition " + ci.getFullPartitionName() + ", assuming it was dropped"); txnHandler.markCleaned(ci); removeFiles(location, txnList); } else { LOG.info("Cleaning as user " + ci.runAs + " for " + ci.getFullPartitionName()); UserGroupInformation ugi = UserGroupInformation.createProxyUser(ci.runAs, UserGroupInformation.getLoginUser()); } catch (IOException exception) { LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " + ci.getFullPartitionName(), exception);
"checking to see if we should compact any of them"); for (CompactionInfo ci : potentials) { LOG.info("Checking to see if we should compact " + ci.getFullPartitionName()); try { Table t = resolveTable(ci); ci.getFullPartitionName() + " so we will not initiate another compaction"); continue; LOG.warn("Will not initiate compaction for " + ci.getFullPartitionName() + " since last " + HiveConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " attempts to compact it failed."); txnHandler.markFailed(ci); LOG.info("Can't find partition " + ci.getFullPartitionName() + ", assuming it has been dropped and moving on."); continue;
p = resolvePartition(ci); if (p == null && ci.partName != null) { LOG.info("Unable to find partition " + ci.getFullPartitionName() + ", assuming it was dropped and moving on."); txnHandler.markCleaned(ci); final StringBuilder jobName = new StringBuilder(name); jobName.append("-compactor-"); jobName.append(ci.getFullPartitionName()); ci.getFullPartitionName()); } catch (IOException exception) { LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " + ci.getFullPartitionName(), exception);
LOG.warn(parsedDeltas.size() + " delta files found for " + ci.getFullPartitionName() + " located at " + sd.getLocation() + "! This is likely a sign of misconfiguration, " + "especially if this message repeats. Check that compaction is running properly. Check for any " +
"checking to see if we should compact any of them"); for (CompactionInfo ci : potentials) { LOG.info("Checking to see if we should compact " + ci.getFullPartitionName()); try { Table t = resolveTable(ci); ci.getFullPartitionName() + " so we will not initiate another compaction"); continue; LOG.warn("Will not initiate compaction for " + ci.getFullPartitionName() + " since last " + HiveConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " attempts to compact it failed."); txnHandler.markFailed(ci); LOG.info("Can't find partition " + ci.getFullPartitionName() + ", assuming it has been dropped and moving on."); continue;
LOG.warn(parsedDeltas.size() + " delta files found for " + ci.getFullPartitionName() + " located at " + sd.getLocation() + "! This is likely a sign of misconfiguration, " + "especially if this message repeats. Check that compaction is running properly. Check for any " +
p = resolvePartition(ci); if (p == null && ci.partName != null) { LOG.info("Unable to find partition " + ci.getFullPartitionName() + ", assuming it was dropped and moving on."); msc.markCleaned(CompactionInfo.compactionInfoToStruct(ci)); final StringBuilder jobName = new StringBuilder(workerName); jobName.append("-compactor-"); jobName.append(ci.getFullPartitionName()); LOG.info("Starting " + ci.type.toString() + " compaction for " + ci.getFullPartitionName() + " in " + JavaUtils.txnIdToString(compactorTxnId)); final StatsUpdater su = StatsUpdater.init(ci, msc.findColumnsWithStats( CompactionInfo.compactionInfoToStruct(ci)), conf, } catch (IOException exception) { LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " + ci.getFullPartitionName(), exception);
if (p == null) { LOG.info("Unable to find partition " + ci.getFullPartitionName() + ", assuming it was dropped." + idWatermark(ci)); txnHandler.markCleaned(ci); removeFiles(location, validWriteIdList, ci); } else { LOG.info("Cleaning as user " + ci.runAs + " for " + ci.getFullPartitionName()); UserGroupInformation ugi = UserGroupInformation.createProxyUser(ci.runAs, UserGroupInformation.getLoginUser()); } catch (IOException exception) { LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " + ci.getFullPartitionName() + idWatermark(ci), exception);
@Override public int compareTo(CompactionInfo o) { return getFullPartitionName().compareTo(o.getFullPartitionName()); } public String toString() {
@Override public int compareTo(CompactionInfo o) { return getFullPartitionName().compareTo(o.getFullPartitionName()); } public String toString() {
/** * Get the partition being compacted. * @param ci compaction info returned from the compaction queue * @return metastore partition, or null if there is not partition in this compaction info * @throws Exception if underlying calls throw, or if the partition name resolves to more than * one partition. */ protected Partition resolvePartition(CompactionInfo ci) throws Exception { if (ci.partName != null) { List<Partition> parts = null; try { parts = rs.getPartitionsByNames(ci.dbname, ci.tableName, Collections.singletonList(ci.partName)); if (parts == null || parts.size() == 0) { // The partition got dropped before we went looking for it. return null; } } catch (Exception e) { LOG.error("Unable to find partition " + ci.getFullPartitionName() + ", " + e.getMessage()); throw e; } if (parts.size() != 1) { LOG.error(ci.getFullPartitionName() + " does not refer to a single partition"); throw new MetaException("Too many partitions"); } return parts.get(0); } else { return null; } }
private void requestCompaction(CompactionInfo ci, String runAs, CompactionType type) throws MetaException { String s = "Requesting " + type.toString() + " compaction for " + ci.getFullPartitionName(); LOG.info(s); CompactionRequest rqst = new CompactionRequest(ci.dbname, ci.tableName, type); if (ci.partName != null) rqst.setPartitionname(ci.partName); rqst.setRunas(runAs); txnHandler.compact(rqst); }
private CompactionType checkForCompaction(final CompactionInfo ci, final ValidTxnList txns, final StorageDescriptor sd, final String runAs) throws IOException, InterruptedException { // If it's marked as too many aborted, we already know we need to compact if (ci.tooManyAborts) { LOG.debug("Found too many aborted transactions for " + ci.getFullPartitionName() + ", " + "initiating major compaction"); return CompactionType.MAJOR; } if (runJobAsSelf(runAs)) { return determineCompactionType(ci, txns, sd); } else { LOG.info("Going to initiate as user " + runAs); UserGroupInformation ugi = UserGroupInformation.createProxyUser(runAs, UserGroupInformation.getLoginUser()); return ugi.doAs(new PrivilegedExceptionAction<CompactionType>() { @Override public CompactionType run() throws Exception { return determineCompactionType(ci, txns, sd); } }); } }