private List<Segment<Text, Text>> getCompressedSegments() throws IOException { List<Segment<Text, Text>> segments = new ArrayList<Segment<Text, Text>>(); for (int i = 0; i < 2; i++) { segments.add(getCompressedSegment(i)); } return segments; }
@Test public void testCompressed() throws IOException { testMergeShouldReturnProperProgress(getCompressedSegments()); }
private Segment<Text, Text> getCompressedSegment(int i) throws IOException { return new Segment<Text, Text>(getReader(i, true), false, 3000l); }
@Test public void testUncompressed() throws IOException { testMergeShouldReturnProperProgress(getUncompressedSegments()); }
@SuppressWarnings("unchecked") private Reader<Text, Text> getReader(int i, boolean isCompressedInput) throws IOException { Reader<Text, Text> readerMock = mock(Reader.class); when(readerMock.getLength()).thenReturn(30l); when(readerMock.getPosition()).thenReturn(0l).thenReturn(10l).thenReturn( 20l); when( readerMock.nextRawKey(any(DataInputBuffer.class))) .thenAnswer(getKeyAnswer("Segment" + i, isCompressedInput)); doAnswer(getValueAnswer("Segment" + i)).when(readerMock).nextRawValue( any(DataInputBuffer.class)); return readerMock; }
Map<String, String> map2 = new TreeMap<String, String>(); map1.put("banana", "pretty good"); byte[] mapOutputBytes1 = writeMapOutput(conf, map1); byte[] mapOutputBytes2 = writeMapOutput(conf, map2); InMemoryMapOutput<Text, Text> mapOutput1 = new InMemoryMapOutput<Text, Text>( conf, mapId1, mergeManager, mapOutputBytes1.length, null, true); Map<String, String> map4 = new TreeMap<String, String>(); map4.put("banana", "bla"); byte[] mapOutputBytes3 = writeMapOutput(conf, map3); byte[] mapOutputBytes4 = writeMapOutput(conf, map4); InMemoryMapOutput<Text, Text> mapOutput3 = new InMemoryMapOutput<Text, Text>( conf, mapId3, mergeManager, mapOutputBytes3.length, null, true); while (iterator.hasNext()) { CompressAwarePath next = iterator.next(); readOnDiskMapOutput(conf, fs, next, keys, values); paths.add(next); readOnDiskMapOutput(conf, fs, mergeManager.onDiskMapOutputs.iterator().next(), keys, values); Assert.assertEquals(keys, Arrays.asList("apple", "apple", "banana", "banana", "carrot", "carrot")); Assert.assertEquals(values, Arrays.asList("awesome", "disgusting", "pretty good", "bla", "amazing", "delicious"));
private List<Segment<Text, Text>> getUncompressedSegments() throws IOException { List<Segment<Text, Text>> segments = new ArrayList<Segment<Text, Text>>(); for (int i = 0; i < 2; i++) { segments.add(getUncompressedSegment(i)); } return segments; }
@Test public void testEncryptedMerger() throws Throwable { jobConf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true); conf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); TokenCache.setEncryptedSpillKey(new byte[16], credentials); UserGroupInformation.getCurrentUser().addCredentials(credentials); testInMemoryAndOnDiskMerger(); }
Progress mergePhase = new Progress(); RawKeyValueIterator mergeQueue = Merger.merge(conf, fs, keyClass, valueClass, segments, 2, tmpDir, comparator, getReporter(), readsCounter, writesCounter, mergePhase); final float epsilon = 0.00001f;
private Segment<Text, Text> getUncompressedSegment(int i) throws IOException { return new Segment<Text, Text>(getReader(i, false), false); }