@Override protected MergeThread<InMemoryMapOutput<Text, Text>, Text, Text> createInMemoryMerger() { mergeThread = new TestMergeThread(this, getExceptionReporter()); return mergeThread; }
public Void answer(InvocationOnMock ignore) throws IOException { in.close(); return null; } }).when(connection).disconnect();
@Override protected void openConnection(URL url) throws IOException { if (null == connection || renewConnection) { super.openConnection(url); } // already 'opened' the mocked connection return; } }
@Test public void testCopyFromHostConnectionRejected() throws Exception { when(connection.getResponseCode()) .thenReturn(Fetcher.TOO_MANY_REQ_STATUS_CODE); Fetcher<Text, Text> fetcher = new FakeFetcher<>(job, id, ss, mm, r, metrics, except, key, connection); fetcher.copyFromHost(host); Assert.assertEquals("No host failure is expected.", ss.hostFailureCount(host.getHostName()), 0); Assert.assertEquals("No fetch failure is expected.", ss.fetchFailureCount(map1ID), 0); Assert.assertEquals("No fetch failure is expected.", ss.fetchFailureCount(map2ID), 0); verify(ss).penalize(eq(host), anyLong()); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); }
private void verifyReservedMapOutputType(MergeManagerImpl<Text, Text> mgr, long size, String expectedShuffleMode) throws IOException { final TaskAttemptID mapId = TaskAttemptID.forName("attempt_0_1_m_1_1"); final MapOutput<Text, Text> mapOutput = mgr.reserve(mapId, size, 1); assertEquals("Shuffled bytes: " + size, expectedShuffleMode, mapOutput.getDescription()); mgr.unreserve(size); }
@Override public void merge(List<InMemoryMapOutput<Text, Text>> inputs) throws IOException { synchronized (this) { numMerges.incrementAndGet(); for (InMemoryMapOutput<Text, Text> input : inputs) { manager.unreserve(input.getSize()); } } try { mergeStart.await(); mergeComplete.await(); } catch (InterruptedException e) { } catch (BrokenBarrierException e) { } } }
@Override public int getMapCompletionEvents() throws IOException, InterruptedException { return super.getMapCompletionEvents(); }
@Test public void testZeroShuffleMemoryLimitPercent() throws Exception { final JobConf jobConf = new JobConf(); jobConf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f); final MergeManagerImpl<Text, Text> mgr = new MergeManagerImpl<>(null, jobConf, mock(LocalFileSystem.class), null, null, null, null, null, null, null, null, null, null, new MROutputFiles()); verifyReservedMapOutputType(mgr, 10L, "DISK"); } }
@Test public void testUncompressed() throws IOException { testMergeShouldReturnProperProgress(getUncompressedSegments()); }
@Test public void testCompressed() throws IOException { testMergeShouldReturnProperProgress(getCompressedSegments()); }
@Override public int read() throws IOException { int ret = super.read(); if (ret != -1) { return ret; } return freeze(); }
private Segment<Text, Text> getUncompressedSegment(int i) throws IOException { return new Segment<Text, Text>(getReader(i, false), false); }
public int getNumMerges() { return mergeThread.getNumMerges(); } }
public StubbedMergeManager(JobConf conf, ExceptionReporter reporter, CyclicBarrier mergeStart, CyclicBarrier mergeComplete) { super(null, conf, mock(LocalFileSystem.class), null, null, null, null, null, null, null, null, reporter, null, mock(MapOutputFile.class)); mergeThread.setSyncBarriers(mergeStart, mergeComplete); }
private List<Segment<Text, Text>> getCompressedSegments() throws IOException { List<Segment<Text, Text>> segments = new ArrayList<Segment<Text, Text>>(); for (int i = 0; i < 2; i++) { segments.add(getCompressedSegment(i)); } return segments; }
private void fillOutput(InMemoryMapOutput<Text, Text> output) throws IOException { BoundedByteArrayOutputStream stream = output.getArrayStream(); int count = stream.getLimit(); for (int i=0; i < count; ++i) { stream.write(i); } }
@Override public int read(byte[] b) throws IOException { int ret = super.read(b); if (ret != -1) { return ret; } return freeze(); }
public Void answer(InvocationOnMock ignore) throws IOException { in.close(); return null; } }).when(connection).disconnect();
private Segment<Text, Text> getCompressedSegment(int i) throws IOException { return new Segment<Text, Text>(getReader(i, true), false, 3000l); }
@Override public int read(byte[] b, int off, int len) throws IOException { int ret = super.read(b, off, len); if (ret != -1) { return ret; } return freeze(); }