protected boolean doesFileExist(Path filePath, ReplicationTarget target) throws IOException { if (!fs.exists(filePath)) { log.warn("Received work request for {} and {}, but the file doesn't exist", filePath, target); return false; } return true; }
public static boolean isInitialized(VolumeManager fs, SiteConfiguration siteConfig, Configuration hadoopConf) throws IOException { for (String baseDir : VolumeConfiguration.getVolumeUris(siteConfig, hadoopConf)) { if (fs.exists(new Path(baseDir, ServerConstants.INSTANCE_ID_DIR)) || fs.exists(new Path(baseDir, ServerConstants.VERSION_DIR))) return true; } return false; }
/** * Looks for write-ahead logs in recovery directories. * * @return map of log uuids to paths */ protected Map<UUID,Path> getSortedWALogs() throws IOException { Map<UUID,Path> result = new HashMap<>(); for (String dir : ServerConstants.getRecoveryDirs(context)) { Path recoveryDir = new Path(dir); if (fs.exists(recoveryDir)) { for (FileStatus status : fs.listStatus(recoveryDir)) { try { UUID logId = path2uuid(status.getPath()); result.put(logId, status.getPath()); } catch (IllegalArgumentException iae) { log.debug("Ignoring file " + status.getPath() + " because it doesn't look like a uuid"); } } } } return result; } }
public void recover(VolumeManager fs, KeyExtent extent, List<LogEntry> logEntries, Set<String> tabletFiles, MutationReceiver mutationReceiver) throws IOException { List<Path> recoveryLogs = new ArrayList<>(); List<LogEntry> sorted = new ArrayList<>(logEntries); Collections.sort(sorted, (e1, e2) -> (int) (e1.timestamp - e2.timestamp)); for (LogEntry entry : sorted) { Path recovery = null; Path finished = RecoveryPath.getRecoveryPath(fs.getFullPath(FileType.WAL, entry.filename)); finished = SortedLogState.getFinishedMarkerPath(finished); TabletServer.log.debug("Looking for " + finished); if (fs.exists(finished)) { recovery = finished.getParent(); } if (recovery == null) throw new IOException( "Unable to find recovery files for extent " + extent + " logEntry: " + entry); recoveryLogs.add(recovery); } logger.recover(fs, extent, recoveryLogs, tabletFiles, mutationReceiver); }
public static void renameReplacement(VolumeManager fs, FileRef tmpDatafile, FileRef newDatafile) throws IOException { if (fs.exists(newDatafile.path())) { log.error("Target map file already exist " + newDatafile, new Exception()); throw new IllegalStateException("Target map file already exist " + newDatafile); } DatafileManager.rename(fs, tmpDatafile.path(), newDatafile.path()); }
String expectedCompactedFile = path.substring(0, path.lastIndexOf("/delete+")) + "/" + filename.split("\\+")[1]; if (fs.exists(new Path(expectedCompactedFile))) {
if (!fs.exists(new Path(tableInfo.exportDir, oldFileName))) { throw new AcceptableThriftTableOperationException(tableInfo.tableId.canonicalID(), tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
@Override @SuppressWarnings("unchecked") public void run() { try { if (!fs.exists(path)) { missing.incrementAndGet(); Mutation m = new Mutation(key.getRow()); m.putDelete(key.getColumnFamily(), key.getColumnQualifier()); if (writer != null) { writer.addMutation(m); System.out.println("Reference " + path + " removed from " + key.getRow()); } else { System.out.println("File " + path + " is missing"); } } else { synchronized (processing) { cache.put(path, path); } } } catch (Exception e) { exceptionRef.compareAndSet(null, e); } finally { synchronized (processing) { processing.remove(path); processing.notify(); } } } }
private static Path createNewBulkDir(ServerContext context, VolumeManager fs, String sourceDir, Table.ID tableId) throws IOException { Path tempPath = fs.matchingFileSystem(new Path(sourceDir), ServerConstants.getTablesDirs(context)); if (tempPath == null) throw new IOException(sourceDir + " is not in a volume configured for Accumulo"); String tableDir = tempPath.toString(); if (tableDir == null) throw new IOException(sourceDir + " is not in a volume configured for Accumulo"); Path directory = new Path(tableDir + "/" + tableId); fs.mkdirs(directory); // only one should be able to create the lock file // the purpose of the lock file is to avoid a race // condition between the call to fs.exists() and // fs.mkdirs()... if only hadoop had a mkdir() function // that failed when the dir existed UniqueNameAllocator namer = context.getUniqueNameAllocator(); while (true) { Path newBulkDir = new Path(directory, Constants.BULK_PREFIX + namer.getNextName()); if (fs.exists(newBulkDir)) // sanity check throw new IOException("Dir exist when it should not " + newBulkDir); if (fs.mkdirs(newBulkDir)) return newBulkDir; log.warn("Failed to create {} for unknown reason", newBulkDir); sleepUninterruptibly(3, TimeUnit.SECONDS); } }
private long setAndGetCreatedTime(Path file, String tableId) throws IOException, MutationsRejectedException { long createdTime; if (fs.exists(file)) { createdTime = fs.getFileStatus(file).getModificationTime(); } else { createdTime = System.currentTimeMillis(); } Status status = Status.newBuilder().setCreatedTime(createdTime).build(); Mutation m = new Mutation(new Text(ReplicationSection.getRowPrefix() + file)); m.put(MetadataSchema.ReplicationSection.COLF, new Text(tableId), ProtobufUtil.toValue(status)); replicationWriter.addMutation(m); replicationWriter.flush(); return createdTime; } }
VolumeManager fs = master.getFileSystem(); if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT))) return new CleanUpBulkImport(tableId, source, bulk, error); while ((line = in.readLine()) != null) { Path path = new Path(line); if (!fs.exists(new Path(error, path.getName()))) failures.put(new FileRef(line, path), line); Path dest = new Path(error, orig.getName()); if (fs.exists(dest)) continue;
lowDirectory = Constants.DEFAULT_TABLET_LOCATION; Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory); if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath)) { FileSystem pathFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem(); return lowDirectoryPath.makeQualified(pathFs.getUri(), pathFs.getWorkingDirectory()) lowDirectory = "/" + Constants.GENERATED_TABLET_DIRECTORY_PREFIX + namer.getNextName(); Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory); if (fs.exists(lowDirectoryPath)) throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath); if (fs.mkdirs(lowDirectoryPath)) {
++status.current.deleted; } else if (fs.exists(fullPath)) {
if (master.getFileSystem().exists(SortedLogState.getFinishedMarkerPath(dest))) { synchronized (this) { closeTasksQueued.remove(sortId);
if (fs.exists(path)) { fs.delete(path);
if (getFileSystem().exists(new Path(outputFileName))) { getFileSystem().deleteRecursively(new Path(outputFileName));
if (tablet.getTabletServer().getFileSystem().exists(newDatafile.path())) { log.error("Target map file already exist " + newDatafile, new Exception()); throw new IllegalStateException("Target map file already exist " + newDatafile);
tablet.getTabletServer().getFileSystem().deleteRecursively(tmpDatafile.path()); } else { if (tablet.getTabletServer().getFileSystem().exists(newDatafile.path())) { log.warn("Target map file already exist {}", newDatafile); tablet.getTabletServer().getFileSystem().deleteRecursively(newDatafile.path());