/** * Gets the latest index file created for a job suite (if one exists). * @param suiteWorkdir suite working directory * @param suiteId suite unique ID (ID of the root job) * @return file the index file */ public static File getSuiteIndexFile( String suiteWorkdir, String suiteId) { return new File(suiteWorkdir + File.separator + "latest" + File.separator + FileUtil.toSafeFileName(suiteId) + ".index"); } }
@Override public void delete() { File fileToDelete = null; try { fileToDelete = metaFile; FileUtil.delete(fileToDelete); fileToDelete = refFile; FileUtil.delete(fileToDelete); fileToDelete = contentFile; FileUtil.delete(fileToDelete); } catch (IOException e) { LOG.error("Could not delete commit file: " + fileToDelete, e); } }
private void deleteEmptyOldDirs(File parentDir) { final long someTimeAgo = System.currentTimeMillis() - (DateUtils.MILLIS_PER_SECOND * EMPTY_DIRS_SECONDS_LIMIT); Date date = new Date(someTimeAgo); int dirCount = FileUtil.deleteEmptyDirs(parentDir, date); if (LOG.isDebugEnabled()) { LOG.debug("Deleted " + dirCount + " empty directories under " + parentDir); } }
@Override public final void backup(final String suiteId, final Date backupDate) throws IOException { resolveDirsIfNeeded(); String date = new SimpleDateFormat( "yyyyMMddHHmmssSSSS").format(backupDate); File progressFile = getLogFile(suiteId); File backupDir = FileUtil.createDateDirs( new File(logdirBackupBase), backupDate); backupDir = new File(backupDir, "logs"); if (!backupDir.exists()) { try { FileUtils.forceMkdir(backupDir); } catch (IOException e) { throw new JEFException("Cannot create backup directory: " + backupDir, e); } } File backupFile = new File( backupDir + "/" + date + "__" + FileUtil.toSafeFileName(suiteId) + LOG_SUFFIX); if (progressFile.exists()) { FileUtil.moveFile(progressFile, backupFile); } }
File backupDir; try { backupDir = FileUtil.createDateDirs( new File(jobdirBackupBase), backupDate); } catch (IOException e) { + FileUtil.toSafeFileName(suiteName) + "__" + FileUtil.toSafeFileName(jobId) + ".job");
@Override protected long getInitialQueueDocCount() { final MutableLong fileCount = new MutableLong(); // --- Additions and Deletions --- FileUtil.visitAllFiles( new File(queue.getDirectory()), new IFileVisitor() { @Override public void visit(File file) { fileCount.increment(); } }, REF_FILTER); return fileCount.longValue(); }
@Override public final void backup( final String suiteName, final String jobId, final Date backupDate) throws IOException { File progressFile = getStatusFile(suiteName, jobId); File backupFile = getBackupFile(suiteName, jobId, backupDate); if (progressFile.exists()) { FileUtil.moveFile(progressFile, backupFile); } }
String suiteName = FileUtil.fromSafeFileName( FilenameUtils.getBaseName(suiteIndex.getPath())); XMLConfiguration xml = new XMLConfiguration();
File backupDir = new File(getWorkdir() + File.separator + "backup"); try { backupDir = FileUtil.createDateDirs(backupDir, backupDate); } catch (IOException e) { throw new JobException("Could not create backup directory for " date + "__" + FileUtil.toSafeFileName(getId()) + ".index"); if (!indexFile.renameTo(backupFile)) { throw new IOException("Could not create backup file: "
FileUtil.visitAllFiles( new File(queue.getDirectory()), new IFileVisitor() { @Override
/** * Gets the file used to store the job progress. * @param suiteName name space given to the job progress * @param jobId the job unique name * @return file used to store the job process */ private File getStatusFile(final String suiteName, final String jobId) { resolveDirsIfNeeded(); return new File(jobdirLatest + "/" + FileUtil.toSafeFileName(suiteName) + "__" + FileUtil.toSafeFileName(jobId) + ".job"); } /**
@Override public void delete() { try { FileUtil.delete(refFile); } catch (IOException e) { LOG.error("Could not delete commit file: " + refFile, e); } }
protected void execute(JobStatusUpdater statusUpdater, JobSuite suite, ICrawlDataStore crawlDataStore) { //--- Process start/queued references ---------------------------------- LOG.info(getId() + ": Crawling references..."); ImporterPipelineContext contextPrototype = new ImporterPipelineContext(this, crawlDataStore); processReferences(statusUpdater, suite, contextPrototype); if (!isStopped()) { handleOrphans(crawlDataStore, statusUpdater, suite); } ICommitter committer = getCrawlerConfig().getCommitter(); if (committer != null) { LOG.info(getId() + ": Crawler " + (isStopped() ? "stopping" : "finishing") + ": committing documents."); committer.commit(); } LOG.info(getId() + ": " + processedCount + " reference(s) processed."); LOG.debug(getId() + ": Removing empty directories"); FileUtil.deleteEmptyDirs(getCrawlerDownloadDir()); if (!isStopped()) { fireCrawlerEvent(CrawlerEvent.CRAWLER_FINISHED, null, this); } else { fireCrawlerEvent(CrawlerEvent.CRAWLER_STOPPED, null, this); } LOG.info(getId() + ": Crawler " + (isStopped() ? "stopped." : "completed.")); }
/** * Gets the log file used by this log manager. * @param suiteId log file suiteId * @return log file */ public File getLogFile(final String suiteId) { if (suiteId == null) { return null; } resolveDirsIfNeeded(); return new File(logdirLatest + "/" + FileUtil.toSafeFileName(suiteId) + LOG_SUFFIX); }
@Override public final void remove(final String suiteName, final String jobId) throws IOException { File file = getStatusFile(suiteName, jobId); FileUtil.delete(file); }
@Override public final Appender createAppender(final String suiteId) throws IOException { resolveDirsIfNeeded(); return new FileAppender(new PatternLayout(LAYOUT_PATTERN), logdirLatest + "/" + FileUtil.toSafeFileName(suiteId) + LOG_SUFFIX); }
public synchronized void stopMonitoring() { monitoring = false; if (stopFile.exists()) { try { FileUtil.delete(stopFile); } catch (IOException e) { throw new JEFException( "Cannot delete stop file: " + stopFile, e); } } }
/*default*/ File getSuiteStopFile() { return new File(getWorkdir() + File.separator + "latest" + File.separator + FileUtil.toSafeFileName(getId()) + ".stop"); } /*default*/ List<IJobLifeCycleListener> getJobLifeCycleListeners() {
b.append("d."); b.append(FileUtil.toSafeFileName(segPart));
String fileName = StringUtils.stripToEmpty( FileUtil.toSafeFileName(fileNamePrefix)) + fileBaseName + StringUtils.stripToEmpty( FileUtil.toSafeFileName(fileNameSuffix)) + "_" + rollCount + ".json"; if (compress) {