@Test(timeout=10000) public void testInterruptInMemory() throws Exception { final int FETCHER = 2; IFileWrappedMapOutput<Text,Text> immo = spy(new InMemoryMapOutput<Text,Text>( job, id, mm, 100, null, true)); when(mm.reserve(any(TaskAttemptID.class), anyLong(), anyInt()))
byte[] mapOutputBytes1 = writeMapOutput(conf, map1); byte[] mapOutputBytes2 = writeMapOutput(conf, map2); InMemoryMapOutput<Text, Text> mapOutput1 = new InMemoryMapOutput<Text, Text>( conf, mapId1, mergeManager, mapOutputBytes1.length, null, true); InMemoryMapOutput<Text, Text> mapOutput2 = new InMemoryMapOutput<Text, Text>( conf, mapId2, mergeManager, mapOutputBytes2.length, null, true); System.arraycopy(mapOutputBytes1, 0, mapOutput1.getMemory(), 0, mapOutputBytes1.length); System.arraycopy(mapOutputBytes2, 0, mapOutput2.getMemory(), 0, mapOutputBytes2.length); byte[] mapOutputBytes3 = writeMapOutput(conf, map3); byte[] mapOutputBytes4 = writeMapOutput(conf, map4); InMemoryMapOutput<Text, Text> mapOutput3 = new InMemoryMapOutput<Text, Text>( conf, mapId3, mergeManager, mapOutputBytes3.length, null, true); InMemoryMapOutput<Text, Text> mapOutput4 = new InMemoryMapOutput<Text, Text>( conf, mapId4, mergeManager, mapOutputBytes4.length, null, true); System.arraycopy(mapOutputBytes3, 0, mapOutput3.getMemory(), 0, mapOutputBytes3.length); System.arraycopy(mapOutputBytes4, 0, mapOutput4.getMemory(), 0, mapOutputBytes4.length);
@Override public void merge(List<InMemoryMapOutput<Text, Text>> inputs) throws IOException { synchronized (this) { numMerges.incrementAndGet(); for (InMemoryMapOutput<Text, Text> input : inputs) { manager.unreserve(input.getSize()); } } try { mergeStart.await(); mergeComplete.await(); } catch (InterruptedException e) { } catch (BrokenBarrierException e) { } } }
@SuppressWarnings("unchecked") @Test(timeout=10000) public void testCopyFromHostWithRetryUnreserve() throws Exception { InMemoryMapOutput<Text, Text> immo = mock(InMemoryMapOutput.class); Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(jobWithRetry, id, ss, mm, r, metrics, except, key, connection); String replyHash = SecureShuffleUtils.generateHash(encHash.getBytes(), key); when(connection.getResponseCode()).thenReturn(200); when(connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH)) .thenReturn(replyHash); ShuffleHeader header = new ShuffleHeader(map1ID.toString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.write(new DataOutputStream(bout)); ByteArrayInputStream in = new ByteArrayInputStream(bout.toByteArray()); when(connection.getInputStream()).thenReturn(in); when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_NAME)) .thenReturn(ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); when(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_VERSION)) .thenReturn(ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); // Verify that unreserve occurs if an exception happens after shuffle // buffer is reserved. when(mm.reserve(any(TaskAttemptID.class), anyLong(), anyInt())) .thenReturn(immo); doThrow(new IOException("forced error")).when(immo).shuffle( any(MapHost.class), any(InputStream.class), anyLong(), anyLong(), any(ShuffleClientMetrics.class), any(Reporter.class)); underTest.copyFromHost(host); verify(immo).abort(); }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
public synchronized void closeInMemoryMergedFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMergedMapOutputs.add(mapOutput); LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() + ", inMemoryMergedMapOutputs.size() -> " + inMemoryMergedMapOutputs.size()); }
when(connection.getInputStream()).thenReturn(in); IFileWrappedMapOutput<Text,Text> mapOut = new InMemoryMapOutput<Text, Text>( job, map1ID, mm, 8, null, true ); IFileWrappedMapOutput<Text,Text> mapOut2 = new InMemoryMapOutput<Text, Text>( job, map2ID, mm, 10, null, true );
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
public synchronized void closeInMemoryMergedFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMergedMapOutputs.add(mapOutput); LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() + ", inMemoryMergedMapOutputs.size() -> " + inMemoryMergedMapOutputs.size()); }
/** * Unconditional Reserve is used by the Memory-to-Memory thread * @return */ private synchronized InMemoryMapOutput<K, V> unconditionalReserve( TaskAttemptID mapId, long requestedSize, boolean primaryMapOutput) { usedMemory += requestedSize; return new InMemoryMapOutput<K,V>(jobConf, mapId, this, (int)requestedSize, codec, primaryMapOutput); }
public synchronized void closeInMemoryMergedFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMergedMapOutputs.add(mapOutput); LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() + ", inMemoryMergedMapOutputs.size() -> " + inMemoryMergedMapOutputs.size()); }
/** * Unconditional Reserve is used by the Memory-to-Memory thread * @return */ private synchronized InMemoryMapOutput<K, V> unconditionalReserve( TaskAttemptID mapId, long requestedSize, boolean primaryMapOutput) { usedMemory += requestedSize; return new InMemoryMapOutput<K,V>(jobConf, mapId, this, (int)requestedSize, codec, primaryMapOutput); }
boolean mergePhaseFinished = false; if (inMemoryMapOutputs.size() > 0) { TaskID mapId = inMemoryMapOutputs.get(0).getMapId().getTaskID(); inMemToDiskBytes = createInMemorySegments(inMemoryMapOutputs, memDiskSegments,
public synchronized void closeInMemoryMergedFile(InMemoryMapOutput<K,V> mapOutput) { inMemoryMergedMapOutputs.add(mapOutput); LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() + ", inMemoryMergedMapOutputs.size() -> " + inMemoryMergedMapOutputs.size()); }