public PartitionManager(NodeControllerService ncs) { this.ncs = ncs; this.availablePartitionMap = new HashMap<>(); this.deallocatableRegistry = new DefaultDeallocatableRegistry(); this.fileFactory = new WorkspaceFileFactory(deallocatableRegistry, ncs.getIoManager()); failedJobsCache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.MINUTES).build(); }
public ResultPartitionManager(NodeControllerService ncs, Executor executor, int availableMemory, long resultTTL, long resultSweepThreshold) { super(resultTTL); this.ncs = ncs; this.executor = executor; deallocatableRegistry = new DefaultDeallocatableRegistry(); fileFactory = new WorkspaceFileFactory(deallocatableRegistry, ncs.getIoManager()); if (availableMemory >= ResultMemoryManager.getPageSize()) { resultMemoryManager = new ResultMemoryManager(availableMemory); } else { resultMemoryManager = null; } partitionResultStateMap = new HashMap<>(); executor.execute(new ResultStateSweeper(this, resultSweepThreshold, LOGGER)); }
private static void checkNcStore(NodeControllerService nc) throws Exception { List<IODeviceHandle> ioDevices = nc.getIoManager().getIODevices(); int expectedPartitionNum = -1; for (IODeviceHandle ioDevice : ioDevices) { File[] dataDirs = ioDevice.getMount().listFiles(); for (File dataDir : dataDirs) { String dirName = dataDir.getName(); if (!dirName.equals(StorageConstants.STORAGE_ROOT_DIR_NAME)) { // Skips non-storage directories. continue; } int numPartitions = getNumResidentPartitions(dataDir.listFiles()); if (expectedPartitionNum < 0) { // Sets the expected number of partitions to the number of partitions on the first io device. expectedPartitionNum = numPartitions; } else { // Checks whether the number of partitions of the current io device is expected. if (expectedPartitionNum != numPartitions) { throw new Exception("Non-uniform data distribution on io devices: " + dataDir.getAbsolutePath() + " number of partitions: " + numPartitions + " expected number of partitions: " + expectedPartitionNum); } } } } }
public IHyracksTaskContext createTestContext(JobId jobId, int partition, boolean withMessaging) throws HyracksDataException { IHyracksTaskContext ctx = TestUtils.create(KB32, ExecutionTestUtil.integrationUtil.ncs[0].getIoManager()); if (withMessaging) { TaskUtil.put(HyracksConstants.KEY_MESSAGE, new VSizeFrame(ctx), ctx); } IHyracksJobletContext jobletCtx = Mockito.mock(IHyracksJobletContext.class); JobEventListenerFactory factory = new JobEventListenerFactory(new TxnId(jobId.getId()), true); Mockito.when(jobletCtx.getJobletEventListenerFactory()).thenReturn(factory); Mockito.when(jobletCtx.getServiceContext()).thenReturn(ExecutionTestUtil.integrationUtil.ncs[0].getContext()); Mockito.when(jobletCtx.getJobId()).thenReturn(jobId); ctx = Mockito.spy(ctx); Mockito.when(ctx.getJobletContext()).thenReturn(jobletCtx); Mockito.when(ctx.getIoManager()).thenReturn(ExecutionTestUtil.integrationUtil.ncs[0].getIoManager()); TaskAttemptId taskId = new TaskAttemptId(new TaskId(new ActivityId(new OperatorDescriptorId(0), 0), partition), 0); Mockito.when(ctx.getTaskAttemptId()).thenReturn(taskId); return ctx; }