Path baseDir = null; if (entries.get(BASE_DIR) != null) baseDir = new Path(entries.get(BASE_DIR)); StringableList tmpDeltaDirs = new StringableList(entries.get(DELTA_DIRS)); Path[] deltaDirs = tmpDeltaDirs.toArray(new Path[tmpDeltaDirs.size()]); StringableList dirsToSearch = new StringableList(entries.get(DIRS_TO_SEARCH)); Map<Integer, BucketTracker> splitToBucketMap = new HashMap<Integer, BucketTracker>(); for (Path dir : dirsToSearch) {
StringableList(String s) { String[] parts = s.split(":", 2); // read that many chars int numElements = Integer.parseInt(parts[0]); s = parts[1]; for (int i = 0; i < numElements; i++) { parts = s.split(":", 2); int len = Integer.parseInt(parts[0]); String val = parts[1].substring(0, len); s = parts[1].substring(len); add(new Path(val)); } }
StringableList dirsToSearch = new StringableList(); Path baseDir = null; if (isMajor) { dirsToSearch.add(stat.getPath()); LOG.debug("Adding original file " + stat.getPath().toString() + " to dirs to search"); dirsToSearch.add(baseDir); StringableList deltaDirs = new StringableList(); long minTxn = Long.MAX_VALUE; long maxTxn = Long.MIN_VALUE; for (AcidUtils.ParsedDelta delta : parsedDeltas) { LOG.debug("Adding delta " + delta.getPath() + " to directories to search"); dirsToSearch.add(delta.getPath()); deltaDirs.add(delta.getPath()); minTxn = Math.min(minTxn, delta.getMinTransaction()); maxTxn = Math.max(maxTxn, delta.getMaxTransaction()); job.set(DELTA_DIRS, deltaDirs.toString()); job.set(DIRS_TO_SEARCH, dirsToSearch.toString()); job.setLong(MIN_TXN, minTxn); job.setLong(MAX_TXN, maxTxn);
Path baseDir = null; if (entries.get(BASE_DIR) != null) baseDir = new Path(entries.get(BASE_DIR)); StringableList tmpDeltaDirs = new StringableList(entries.get(DELTA_DIRS)); Path[] deltaDirs = tmpDeltaDirs.toArray(new Path[tmpDeltaDirs.size()]); StringableList dirsToSearch = new StringableList(entries.get(DIRS_TO_SEARCH)); Map<Integer, BucketTracker> splitToBucketMap = new HashMap<Integer, BucketTracker>(); for (Path dir : dirsToSearch) {
StringableList(String s) { String[] parts = s.split(":", 2); // read that many chars int numElements = Integer.valueOf(parts[0]); s = parts[1]; for (int i = 0; i < numElements; i++) { parts = s.split(":", 2); int len = Integer.valueOf(parts[0]); String val = parts[1].substring(0, len); s = parts[1].substring(len); add(new Path(val)); } }
StringableList dirsToSearch = new StringableList(); Path baseDir = null; if (ci.isMajorCompaction()) { dirsToSearch.add(path); LOG.debug("Adding original file " + path + " to dirs to search"); dirsToSearch.add(baseDir);
StringableList dirsToSearch = new StringableList(); Path baseDir = null; if (ci.isMajorCompaction()) { Path path = stat.getFileStatus().getPath(); dirsToSearch.add(path); LOG.debug("Adding original file " + path + " to dirs to search"); dirsToSearch.add(baseDir);
job.setBoolean(IS_MAJOR, compactionType == CompactionType.MAJOR); if(dirsToSearch == null) { dirsToSearch = new StringableList(); StringableList deltaDirs = new StringableList(); for (AcidUtils.ParsedDelta delta : parsedDeltas) { LOG.debug("Adding delta " + delta.getPath() + " to directories to search"); dirsToSearch.add(delta.getPath()); deltaDirs.add(delta.getPath()); minTxn = Math.min(minTxn, delta.getMinWriteId()); maxTxn = Math.max(maxTxn, delta.getMaxWriteId()); job.set(DELTA_DIRS, deltaDirs.toString()); job.set(DIRS_TO_SEARCH, dirsToSearch.toString()); job.setLong(MIN_TXN, minTxn); job.setLong(MAX_TXN, maxTxn);
job.setBoolean(IS_MAJOR, compactionType == CompactionType.MAJOR); if(dirsToSearch == null) { dirsToSearch = new StringableList(); StringableList deltaDirs = new StringableList(); long minTxn = Long.MAX_VALUE; long maxTxn = Long.MIN_VALUE; for (AcidUtils.ParsedDelta delta : parsedDeltas) { LOG.debug("Adding delta " + delta.getPath() + " to directories to search"); dirsToSearch.add(delta.getPath()); deltaDirs.add(delta.getPath()); minTxn = Math.min(minTxn, delta.getMinTransaction()); maxTxn = Math.max(maxTxn, delta.getMaxTransaction()); job.set(DELTA_DIRS, deltaDirs.toString()); job.set(DIRS_TO_SEARCH, dirsToSearch.toString()); job.setLong(MIN_TXN, minTxn); job.setLong(MAX_TXN, maxTxn);
Path baseDir = null; if (entries.get(BASE_DIR) != null) baseDir = new Path(entries.get(BASE_DIR)); StringableList tmpDeltaDirs = new StringableList(entries.get(DELTA_DIRS)); Path[] deltaDirs = tmpDeltaDirs.toArray(new Path[tmpDeltaDirs.size()]); StringableList dirsToSearch = new StringableList(entries.get(DIRS_TO_SEARCH)); Map<Integer, BucketTracker> splitToBucketMap = new HashMap<Integer, BucketTracker>(); for (Path dir : dirsToSearch) {
@Test public void stringableList() throws Exception { // Empty list case CompactorMR.StringableList ls = new CompactorMR.StringableList(); String s = ls.toString(); Assert.assertEquals("0:", s); ls = new CompactorMR.StringableList(s); Assert.assertEquals(0, ls.size()); ls = new CompactorMR.StringableList(); ls.add(new Path("/tmp")); ls.add(new Path("/usr")); s = ls.toString(); Assert.assertTrue("Expected 2:4:/tmp4:/usr or 2:4:/usr4:/tmp, got " + s, "2:4:/tmp4:/usr".equals(s) || "2:4:/usr4:/tmp".equals(s)); ls = new CompactorMR.StringableList(s); Assert.assertEquals(2, ls.size()); boolean sawTmp = false, sawUsr = false; for (Path p : ls) { if ("/tmp".equals(p.toString())) sawTmp = true; else if ("/usr".equals(p.toString())) sawUsr = true; else Assert.fail("Unexpected path " + p.toString()); } Assert.assertTrue(sawTmp); Assert.assertTrue(sawUsr); }
StringableList(String s) { String[] parts = s.split(":", 2); // read that many chars int numElements = Integer.parseInt(parts[0]); s = parts[1]; for (int i = 0; i < numElements; i++) { parts = s.split(":", 2); int len = Integer.parseInt(parts[0]); String val = parts[1].substring(0, len); s = parts[1].substring(len); add(new Path(val)); } }