throw new IOException( "Cannot find mid point using data files, too many " + mapFiles.size()); tmpDir = createTmpDir(context, tabletDirectory); mapFiles = reduceFiles(context, context.getHadoopConf(), prevEndRow, endRow, mapFiles, maxToOpen, tmpDir, 0); long t2 = System.currentTimeMillis(); numKeys = countIndexEntries(context, prevEndRow, endRow, mapFiles, tmpDir == null ? useIndex : false, readers); prevEndRow, endRow, mapFiles); return findMidPoint(context, tabletDirectory, prevEndRow, endRow, origMapFiles, minSplit, false); cleanupIndexOp(tmpDir, context.getVolumeManager(), readers);
@Override public void run() { Map<KeyExtent,Long> estimatedSizes = null; try { estimatedSizes = FileUtil.estimateSizes(context, entry.getKey(), mapFileSizes.get(entry.getKey()), extentsOf(entry.getValue())); } catch (IOException e) { log.warn("Failed to estimate map file sizes {}", e.getMessage()); } if (estimatedSizes == null) { // estimation failed, do a simple estimation estimatedSizes = new TreeMap<>(); long estSize = (long) (mapFileSizes.get(entry.getKey()) / (double) entry.getValue().size()); for (TabletLocation tl : entry.getValue()) estimatedSizes.put(tl.tablet_extent, estSize); } List<AssignmentInfo> assignmentInfoList = new ArrayList<>(estimatedSizes.size()); for (Entry<KeyExtent,Long> entry2 : estimatedSizes.entrySet()) assignmentInfoList.add(new AssignmentInfo(entry2.getKey(), entry2.getValue())); ais.put(entry.getKey(), assignmentInfoList); } };
keys = FileUtil.findMidPoint(context, tabletDirectory, extent.getPrevEndRow(), extent.getEndRow(), FileUtil.toPathStrings(files), .25); } catch (IOException e) { log.error("Failed to find midpoint {}", e.getMessage()); Key lastKey = (Key) FileUtil.findLastKey(context, files); lastRow = lastKey.getRow(); } else {
tmpDir = createTmpDir(context, tabletDir); mapFiles = reduceFiles(context, context.getHadoopConf(), prevEndRow, endRow, mapFiles, maxToOpen, tmpDir, 0); long t2 = System.currentTimeMillis(); numKeys = countIndexEntries(context, prevEndRow, endRow, mapFiles, true, readers); cleanupIndexOp(tmpDir, context.getVolumeManager(), readers);
Map<FileRef,FileUtil.FileInfo> firstAndLastRows = FileUtil.tryToGetFirstAndLastRows(context, getDatafileManager().getFiles()); Text tsp = new Text(sp); splitPoint = new SplitRowSpec( FileUtil.estimatePercentageLTE(context, tabletDirectory, extent.getPrevEndRow(), extent.getEndRow(), FileUtil.toPathStrings(getDatafileManager().getFiles()), tsp), tsp);
public static SortedMap<Double,Key> findMidPoint(ServerContext context, String tabletDir, Text prevEndRow, Text endRow, Collection<String> mapFiles, double minSplit) throws IOException { return findMidPoint(context, tabletDir, prevEndRow, endRow, mapFiles, minSplit, true); }
return reduceFiles(context, conf, prevEndRow, endRow, outFiles, maxFiles, tmpDir, pass + 1);
Map<FileRef,FileUtil.FileInfo> firstAndLastRows = FileUtil.tryToGetFirstAndLastRows( getTabletServer().getFileSystem(), getTabletServer().getConfiguration(), getDatafileManager().getFiles()); Text tsp = new Text(sp); splitPoint = new SplitRowSpec( FileUtil.estimatePercentageLTE(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(), extent.getPrevEndRow(), extent.getEndRow(), FileUtil.toPathStrings(getDatafileManager().getFiles()), tsp), tsp);
public static SortedMap<Double,Key> findMidPoint(VolumeManager fs, AccumuloConfiguration acuConf, Text prevEndRow, Text endRow, Collection<String> mapFiles, double minSplit) throws IOException { return findMidPoint(fs, acuConf, prevEndRow, endRow, mapFiles, minSplit, true); }
return reduceFiles(acuConf, conf, fs, prevEndRow, endRow, outFiles, maxFiles, tmpDir, pass + 1);
keys = FileUtil.findMidPoint(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(), extent.getPrevEndRow(), extent.getEndRow(), FileUtil.toPathStrings(files), .25); } catch (IOException e) { log.error("Failed to find midpoint " + e.getMessage()); Key lastKey = (Key) FileUtil.findLastKey(getTabletServer().getFileSystem(), getTabletServer().getConfiguration(), files); lastRow = lastKey.getRow();
@Override public void run() { Map<KeyExtent,Long> estimatedSizes = null; try { estimatedSizes = FileUtil.estimateSizes(acuConf, entry.getKey(), mapFileSizes.get(entry.getKey()), extentsOf(entry.getValue()), conf, vm); } catch (IOException e) { log.warn("Failed to estimate map file sizes {}", e.getMessage()); } if (estimatedSizes == null) { // estimation failed, do a simple estimation estimatedSizes = new TreeMap<>(); long estSize = (long) (mapFileSizes.get(entry.getKey()) / (double) entry.getValue().size()); for (TabletLocation tl : entry.getValue()) estimatedSizes.put(tl.tablet_extent, estSize); } List<AssignmentInfo> assignmentInfoList = new ArrayList<>(estimatedSizes.size()); for (Entry<KeyExtent,Long> entry2 : estimatedSizes.entrySet()) assignmentInfoList.add(new AssignmentInfo(entry2.getKey(), entry2.getValue())); ais.put(entry.getKey(), assignmentInfoList); } };
tmpDir = createTmpDir(acuconf, fs); mapFiles = reduceFiles(acuconf, conf, fs, prevEndRow, endRow, mapFiles, maxToOpen, tmpDir, 0); long t2 = System.currentTimeMillis(); numKeys = countIndexEntries(acuconf, prevEndRow, endRow, mapFiles, true, conf, fs, readers); cleanupIndexOp(tmpDir, fs, readers);
throw new IOException( "Cannot find mid point using data files, too many " + mapFiles.size()); tmpDir = createTmpDir(acuConf, fs); mapFiles = reduceFiles(acuConf, conf, fs, prevEndRow, endRow, mapFiles, maxToOpen, tmpDir, 0); long t2 = System.currentTimeMillis(); numKeys = countIndexEntries(acuConf, prevEndRow, endRow, mapFiles, tmpDir == null ? useIndex : false, conf, fs, readers); + " for " + mapFiles); return findMidPoint(fs, acuConf, prevEndRow, endRow, origMapFiles, minSplit, false); cleanupIndexOp(tmpDir, fs, readers);