ioMan = new IOManagerAsync(); networkBufferPool1 = new NetworkBufferPool(100, PAGE_SIZE); networkBufferPool2 = new NetworkBufferPool(100, PAGE_SIZE); BufferPool pool1 = networkBufferPool1.createBufferPool(100, 100); BufferPool pool2 = networkBufferPool2.createBufferPool(100, 100); networkBufferPool1.destroyAllBufferPools(); networkBufferPool1.destroy(); networkBufferPool2.destroyAllBufferPools(); networkBufferPool2.destroy();
@Override public int getMemorySegmentSize() { return networkBufferPool.getMemorySegmentSize(); }
private void returnMemorySegment(MemorySegment segment) { assert Thread.holdsLock(availableMemorySegments); numberOfRequestedMemorySegments--; networkBufferPool.recycle(segment); }
networkBufferPool.destroyAllBufferPools(); networkBufferPool.destroy();
@Override public BufferPool createBufferPool(int numRequiredBuffers, int maxUsedBuffers) throws IOException { return createBufferPool(numRequiredBuffers, maxUsedBuffers, Optional.empty()); }
final NetworkBufferPool bufferPool = new NetworkBufferPool(bufferPoolSize, segmentSize);
private void recycleMemorySegments(List<MemorySegment> segments, int size) throws IOException { synchronized (factoryLock) { numTotalRequiredBuffers -= size; availableMemorySegments.addAll(segments); // note: if this fails, we're fine for the buffer pool since we already recycled the segments redistributeBuffers(); } }
redistributeBuffers(); } catch (IOException e) { try { destroyBufferPool(localBufferPool); } catch (IOException inner) { e.addSuppressed(inner);
redistributeBuffers(); } catch (Throwable t) { this.numTotalRequiredBuffers -= numRequiredBuffers; redistributeBuffers(); } catch (IOException inner) { t.addSuppressed(inner); recycleMemorySegments(segments, numRequiredBuffers); } catch (IOException inner) { e.addSuppressed(inner);
tryReleaseMemory(numRequiredBuffers); redistributeBuffers(); } catch (Throwable t) { this.numTotalRequiredBuffers -= numRequiredBuffers; redistributeBuffers(); } catch (IOException inner) { t.addSuppressed(inner); recycleMemorySegments(segments, numRequiredBuffers); } catch (IOException inner) { e.addSuppressed(inner);
@Override public void notifyBufferPoolOwnerReleased() { checkState(owner != null, "The buffer owner should not be null."); try { networkBufferPool.destroyBufferPool(this); } catch (IOException e) { ExceptionUtils.rethrow(e); } }
private MemorySegment requestMemorySegment(boolean isBlocking) throws InterruptedException, IOException { synchronized (availableMemorySegments) { returnExcessMemorySegments(); boolean askToRecycle = owner.isPresent(); // fill availableMemorySegments with at least one element, wait if required while (availableMemorySegments.isEmpty()) { if (isDestroyed) { throw new IllegalStateException("Buffer pool is destroyed."); } if (numberOfRequestedMemorySegments < currentPoolSize) { final MemorySegment segment = networkBufferPool.requestMemorySegment(); if (segment != null) { numberOfRequestedMemorySegments++; return segment; } } if (askToRecycle) { owner.get().releaseMemory(1); } if (isBlocking) { availableMemorySegments.wait(2000); } else { return null; } } return availableMemorySegments.poll(); } }
/** * The exclusive segments are recycled to network buffer pool directly when input channel is released. * * @param segments The exclusive segments need to be recycled */ public void returnExclusiveSegments(List<MemorySegment> segments) throws IOException { networkBufferPool.recycleMemorySegments(segments); }
tryReleaseMemory(numRequiredBuffers); redistributeBuffers(); } catch (IOException e) { try { tryDestroyBufferPool(localBufferPool, false); } catch (IOException inner) { e.addSuppressed(inner);
networkBufferPool.destroyAllBufferPools(); networkBufferPool.destroy();
@Override public BufferPool createBufferPool(int numRequiredBuffers, int maxUsedBuffers) throws IOException { return createBufferPool(numRequiredBuffers, maxUsedBuffers, Optional.empty()); }
public NetworkEnvironment( int numBuffers, int memorySegmentSize, int partitionRequestInitialBackoff, int partitionRequestMaxBackoff, int networkBuffersPerChannel, int extraNetworkBuffersPerGate, boolean enableCreditBased) { this( new NetworkBufferPool(numBuffers, memorySegmentSize), new LocalConnectionManager(), new ResultPartitionManager(), new TaskEventDispatcher(), new KvStateRegistry(), null, null, IOManager.IOMode.SYNC, partitionRequestInitialBackoff, partitionRequestMaxBackoff, networkBuffersPerChannel, extraNetworkBuffersPerGate, enableCreditBased); }
@Override public void destroyBufferPool(BufferPool bufferPool) throws IOException { if (!(bufferPool instanceof LocalBufferPool)) { throw new IllegalArgumentException("bufferPool is no LocalBufferPool"); } synchronized (factoryLock) { if (allBufferPools.remove(bufferPool)) { numTotalRequiredBuffers -= bufferPool.getNumberOfRequiredMemorySegments(); redistributeBuffers(); } } }
redistributeBuffers(); } catch (IOException e) { try { destroyBufferPool(localBufferPool); } catch (IOException inner) { e.addSuppressed(inner);
redistributeBuffers(); } catch (Throwable t) { this.numTotalRequiredBuffers -= numRequiredBuffers; redistributeBuffers(); } catch (IOException inner) { t.addSuppressed(inner); recycleMemorySegments(segments, numRequiredBuffers); } catch (IOException inner) { e.addSuppressed(inner);