private FileStatus[] getFileList(List<Path> logDirs, PathFilter filter) throws IOException { return getFileList(conf, logDirs, filter); }
public List<FileStatus> getWALsToSplit(ServerName serverName, boolean splitMeta) throws IOException { List<Path> logDirs = master.getMasterWalManager().getLogDirs(Collections.singleton(serverName)); FileStatus[] fileStatuses = SplitLogManager.getFileList(this.conf, logDirs, splitMeta ? META_FILTER : NON_META_FILTER); LOG.info("size of WALs of {} is {}, isMeta: {}", serverName, fileStatuses.length, splitMeta); return Lists.newArrayList(fileStatuses); }
@VisibleForTesting public static List<Path> split(Path rootDir, Path logDir, Path oldLogDir, FileSystem walFS, Configuration conf, final WALFactory factory) throws IOException { final FileStatus[] logfiles = SplitLogManager.getFileList(conf, Collections.singletonList(logDir), null); List<Path> splits = new ArrayList<>(); if (ArrayUtils.isNotEmpty(logfiles)) { for (FileStatus logfile: logfiles) { WALSplitter s = new WALSplitter(factory, conf, rootDir, walFS, null, null); if (s.splitLogFile(logfile, null)) { finishSplitLogFile(rootDir, oldLogDir, logfile.getPath(), conf); if (s.outputSink.splits != null) { splits.addAll(s.outputSink.splits); } } } } if (!walFS.delete(logDir, true)) { throw new IOException("Unable to delete src dir: " + logDir); } return splits; }
TaskBatch batch = null; long startTime = 0; FileStatus[] logfiles = getFileList(logDirs, filter); if (logfiles.length != 0) { status.setStatus("Checking directory contents...");
private FileStatus[] getFileList(List<Path> logDirs, PathFilter filter) throws IOException { return getFileList(conf, logDirs, filter); }
public static List<Path> split(Path rootDir, Path logDir, Path oldLogDir, FileSystem fs, Configuration conf, final WALFactory factory) throws IOException { final FileStatus[] logfiles = SplitLogManager.getFileList(conf, Collections.singletonList(logDir), null); List<Path> splits = new ArrayList<Path>(); if (logfiles != null && logfiles.length > 0) { for (FileStatus logfile: logfiles) { WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, null, null, RecoveryMode.LOG_SPLITTING); if (s.splitLogFile(logfile, null)) { finishSplitLogFile(rootDir, oldLogDir, logfile.getPath(), conf); if (s.outputSink.splits != null) { splits.addAll(s.outputSink.splits); } } } } if (!fs.delete(logDir, true)) { throw new IOException("Unable to delete src dir: " + logDir); } return splits; }
MonitoredTask status = TaskMonitor.get().createStatus( "Doing distributed log split in " + logDirs); FileStatus[] logfiles = getFileList(logDirs); status.setStatus("Checking directory contents..."); LOG.debug("Scheduling batch of logs to split");
MonitoredTask status = TaskMonitor.get().createStatus("Doing distributed log split in " + logDirs + " for serverName=" + serverNames); FileStatus[] logfiles = getFileList(logDirs, filter); status.setStatus("Checking directory contents..."); SplitLogCounters.tot_mgr_log_split_batch_start.incrementAndGet();