compactId2LockMap.put(ci.id, findRelatedLocks(ci, locksResponse)); compactId2CompactInfoMap.put(ci.id, ci); Set<Long> currentLocks = buildCurrentLockSet(locksResponse); List<Long> expiredLocks = new ArrayList<Long>(); List<Long> compactionsCleaned = new ArrayList<Long>(); clean(compactId2CompactInfoMap.get(queueEntry.getKey())); } else {
private void clean(CompactionInfo ci) throws MetaException { LOG.info("Starting cleaning for " + ci.getFullPartitionName()); try { Table t = resolveTable(ci); if (t == null) { p = resolvePartition(ci); if (p == null) { StorageDescriptor sd = resolveStorageDescriptor(t, p); final String location = sd.getLocation(); new ValidReadTxnList(new long[0], ci.highestTxnId) : new ValidReadTxnList(); if (runJobAsSelf(ci.runAs)) { removeFiles(location, txnList); } else { LOG.info("Cleaning as user " + ci.runAs + " for " + ci.getFullPartitionName());
long minOpenTxnId = txnHandler.findMinOpenTxnId(); for(CompactionInfo compactionInfo : txnHandler.findReadyToClean()) { clean(compactionInfo, minOpenTxnId);
private static void runCompactorThread(HiveConf hiveConf, CompactorThreadType type) throws Exception { AtomicBoolean stop = new AtomicBoolean(true); CompactorThread t = null; switch (type) { case INITIATOR: t = new Initiator(); break; case WORKER: t = new Worker(); break; case CLEANER: t = new Cleaner(); break; default: throw new IllegalArgumentException("Unknown type: " + type); } t.setThreadId((int) t.getId()); t.setConf(hiveConf); AtomicBoolean looped = new AtomicBoolean(); t.init(stop, looped); t.run(); }
private void startThread(char type, boolean stopAfterOne, AtomicBoolean looped) throws Exception { TxnDbUtil.setConfValues(conf); CompactorThread t = null; switch (type) { case 'i': t = new Initiator(); break; case 'w': t = new Worker(); break; case 'c': t = new Cleaner(); break; default: throw new RuntimeException("Huh? Unknown thread type."); } t.setThreadId((int) t.getId()); t.setConf(conf); stop.set(stopAfterOne); t.init(stop, looped); if (stopAfterOne) t.run(); else t.start(); }
private void clean(CompactionInfo ci, long minOpenTxnGLB) throws MetaException { LOG.info("Starting cleaning for " + ci); try { Table t = resolveTable(ci); if (t == null) { idWatermark(ci)); txnHandler.markCleaned(ci); return; p = resolvePartition(ci); if (p == null) { ", assuming it was dropped." + idWatermark(ci)); txnHandler.markCleaned(ci); return; StorageDescriptor sd = resolveStorageDescriptor(t, p); final String location = sd.getLocation(); ValidTxnList validTxnList = if (runJobAsSelf(ci.runAs)) { removeFiles(location, validWriteIdList, ci); } else { LOG.info("Cleaning as user " + ci.runAs + " for " + ci.getFullPartitionName()); } catch (IOException exception) { LOG.error("Could not clean up file-system handles for UGI: " + ugi + " for " + ci.getFullPartitionName() + idWatermark(ci), exception);
compactId2LockMap.put(ci.id, findRelatedLocks(ci, locksResponse)); compactId2CompactInfoMap.put(ci.id, ci); Set<Long> currentLocks = buildCurrentLockSet(locksResponse); List<Long> expiredLocks = new ArrayList<Long>(); List<Long> compactionsCleaned = new ArrayList<Long>(); clean(compactId2CompactInfoMap.get(queueEntry.getKey())); } else {
private void clean(CompactionInfo ci) throws MetaException { LOG.info("Starting cleaning for " + ci.getFullPartitionName()); try { Table t = resolveTable(ci); if (t == null) { p = resolvePartition(ci); if (p == null) { StorageDescriptor sd = resolveStorageDescriptor(t, p); final String location = sd.getLocation(); if (runJobAsSelf(ci.runAs)) { removeFiles(location, txnList); } else { LOG.info("Cleaning as user " + ci.runAs);