/** * Create the spill service * @param config Configuration for the spill service, containing items such as the spill path(s), * number of I/O completion threads, etc. */ public SpillServiceImpl(DremioConfig config, SpillServiceOptions options, final Provider<SchedulerService> schedulerService) { this.spillDirs = new ArrayList<>(config.getStringList(DremioConfig.SPILLING_PATH_STRING)); this.spillStreams = new HashMap<>(); this.options = options; this.schedulerService = schedulerService; // Option values set at start minDiskSpace = 0; minDiskSpacePercentage = 0; healthCheckInterval = 0; spillSweepInterval = 0; spillSweepThreshold = 0; healthCheckEnabled = false; for (String spillDir : this.spillDirs) { try { final Path spillDirPath = new Path(spillDir); final FileSystem fileSystem = spillDirPath.getFileSystem(SPILLING_CONFIG); healthCheckEnabled = healthCheckEnabled || isHealthCheckEnabled(fileSystem.getUri().getScheme()); } catch (Exception e) {} } // healthySpillDirs set at start() this.healthySpillDirs = Lists.newArrayList(); }
@Override public SpillDirectory getSpillSubdir(String id) throws UserException { ArrayList<String> currentSpillDirs = Lists.newArrayList(spillDirs); while (!currentSpillDirs.isEmpty()) { // pick a random spill directory final int index = ThreadLocalRandom.current().nextInt(currentSpillDirs.size()); final String spillDir = currentSpillDirs.get(index); final Path spillDirPath = new Path(spillDir); if (isHealthy(spillDirPath)) { try { //TODO: track number of spills created in 'spillDir' FileSystem fileSystem = spillDirPath.getFileSystem(SPILLING_CONFIG); final Path spillSubdir = new Path(spillDirPath, id); return new SpillDirectory(spillSubdir, fileSystem); } catch (IOException e) { // Ignore this 'spillDir'. Still consider the others } } // Hm... 'spillDir' didn't work out. Let's consider the others currentSpillDirs.remove(index); } // TODO: withContextParameters() throw UserException.dataWriteError() .message("Failed to spill to disk. Please check space availability") .addContext("spill id", id) .addContext("all spill locations", spillDirs.toString()) .build(logger); }
when(config.getStringList(DremioConfig.SPILLING_PATH_STRING)).thenReturn(ImmutableList.of(spillDir.getPath())); final NoopScheduler schedulerService = new NoopScheduler(); final SpillService spillService = new SpillServiceImpl(config, new TestSpillServiceOptions(), new Provider<SchedulerService>() { @Override
final SpillService spillService = new SpillServiceImpl(DremioConfig.create(null, config), new DefaultSpillServiceOptions(), new Provider<SchedulerService>() { @Override
final SpillService spillService = new SpillServiceImpl(DremioConfig.create(null, config), new DefaultSpillServiceOptions(), new Provider<SchedulerService>() { @Override
boolean outOfMemory = false; final SchedulerService schedulerService = Mockito.mock(SchedulerService.class); final SpillService spillService = new SpillServiceImpl(DremioConfig.create(null, config), new DefaultSpillServiceOptions(), new Provider<SchedulerService>() { @Override
final DremioConfig dremioConfig = DremioConfig.create(null, config); final SchedulerService schedulerService = Mockito.mock(SchedulerService.class); final SpillService spillService = new SpillServiceImpl(dremioConfig, new SpillServiceOptionsImpl(options), new Provider<SchedulerService>() { @Override
registry.bind(SpillService.class, new SpillServiceImpl( config, new SpillServiceOptionsImpl(registry.provider(SabotContext.class)),
)); registry.bind(SpillService.class, new SpillServiceImpl( dremioConfig, new SpillServiceOptionsImpl(registry.provider(SabotContext.class)),