@Override public SpillDirectory getSpillSubdir(String id) throws UserException { ArrayList<String> currentSpillDirs = Lists.newArrayList(spillDirs); while (!currentSpillDirs.isEmpty()) { // pick a random spill directory final int index = ThreadLocalRandom.current().nextInt(currentSpillDirs.size()); final String spillDir = currentSpillDirs.get(index); final Path spillDirPath = new Path(spillDir); if (isHealthy(spillDirPath)) { try { //TODO: track number of spills created in 'spillDir' FileSystem fileSystem = spillDirPath.getFileSystem(SPILLING_CONFIG); final Path spillSubdir = new Path(spillDirPath, id); return new SpillDirectory(spillSubdir, fileSystem); } catch (IOException e) { // Ignore this 'spillDir'. Still consider the others } } // Hm... 'spillDir' didn't work out. Let's consider the others currentSpillDirs.remove(index); } // TODO: withContextParameters() throw UserException.dataWriteError() .message("Failed to spill to disk. Please check space availability") .addContext("spill id", id) .addContext("all spill locations", spillDirs.toString()) .build(logger); }
public SpillFile getSpillFile(String fileName) throws RuntimeException { try { final SpillDirectory spillDirectory = spillService.getSpillSubdir(id); return new SpillFile(spillDirectory.getFileSystem(), new Path(spillDirectory.getSpillDirPath(), fileName)); } catch (UserException e) { throw UserException.dataWriteError(e) .addContext("for %s spill id %s", caller, id) .addContext("Caller", caller) .build(logger); } }
final Path path = new Path(tempDir.getAbsolutePath(), "dremioSerializable"); tempDir.deleteOnExit(); final SpillDirectory spillDirectory = new SpillDirectory(path, fs);
final Path path = new Path(tempDir.getAbsolutePath(), "dremioSerializable"); tempDir.deleteOnExit(); final SpillDirectory spillDirectory = new SpillDirectory(path, fs);