@Override public String[] getLocations() throws IOException, InterruptedException { List<String> locations = fragment.getOwnerNodeNames(); return locations.toArray(new String[locations.size()]); }
private static FileSplit getSplit(BlockMap blockMap, Path path, long start, long end) { DirectInputFragment f = blockMap.get(start, end); List<String> owners = f.getOwnerNodeNames(); FileSplit split = new FileSplit( path, start, end - start, owners.toArray(new String[owners.size()])); return split; }
fragment.getOffset(), fragment.getSize(), fragment.getOwnerNodeNames()));
/** * per prefered size. */ @Test public void pref_size() { BlockBuilder builder = new BlockBuilder(); builder.add(400, "a"); List<DirectInputFragment> results = builder.compute(10, 80, true, true); assertThat(results.size(), is(5)); assertThat(results.get(0).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(1).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(2).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(3).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(4).getOwnerNodeNames(), hasItem("a")); }
@Override public void write(DataOutput out) throws IOException { DirectInputGroup groupCopy = group; WritableUtils.writeString(out, groupCopy.containerPath); WritableUtils.writeString(out, groupCopy.dataType.getName()); WritableUtils.writeString(out, groupCopy.formatClass.getName()); if (groupCopy.filterClass == null) { out.writeBoolean(false); } else { out.writeBoolean(true); WritableUtils.writeString(out, groupCopy.filterClass.getName()); } DirectInputFragment fragmentCopy = fragment; WritableUtils.writeString(out, fragmentCopy.getPath()); WritableUtils.writeVLong(out, fragmentCopy.getOffset()); WritableUtils.writeVLong(out, fragmentCopy.getSize()); List<String> ownerNodeNames = fragmentCopy.getOwnerNodeNames(); WritableUtils.writeStringArray(out, ownerNodeNames.toArray(new String[ownerNodeNames.size()])); Map<String, String> attributes = fragmentCopy.getAttributes(); WritableUtils.writeVInt(out, attributes.size()); for (Map.Entry<String, String> entry : attributes.entrySet()) { WritableUtils.writeString(out, entry.getKey()); WritableUtils.writeString(out, entry.getValue()); } }
begin, end - begin, fragment.getOwnerNodeNames()));
fragment.getOffset(), fragment.getSize(), fragment.getOwnerNodeNames()));
/** * tail blocks are too small. */ @Test public void ignore_little_locality() { BlockBuilder builder = new BlockBuilder(); builder.add(100, "a"); builder.add(1, "b", "c", "d"); List<DirectInputFragment> results = builder.compute(50, 200, true, true); assertThat(results.size(), is(1)); assertThat(results.get(0).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(0).getOwnerNodeNames(), not(hasItem("b"))); }
/** * per prefered size with block join. */ @Test public void pref_size_with_join() { BlockBuilder builder = new BlockBuilder(); builder.add(100, "a"); builder.add(100, "a"); builder.add(100, "a"); builder.add(100, "a"); List<DirectInputFragment> results = builder.compute(10, 80, true, true); assertThat(results.size(), is(5)); assertThat(results.get(0).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(1).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(2).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(3).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(4).getOwnerNodeNames(), hasItem("a")); }
/** * per prefered size without block join. */ @Test public void pref_size_without_join() { BlockBuilder builder = new BlockBuilder(); builder.add(100, "a"); builder.add(100, "b"); builder.add(100, "c"); builder.add(100, "d"); List<DirectInputFragment> results = builder.compute(10, 80, true, true); assertThat(results.size(), is(4)); assertThat(results.get(0).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(0).getSize(), is(100L)); assertThat(results.get(1).getOwnerNodeNames(), hasItem("b")); assertThat(results.get(1).getSize(), is(100L)); assertThat(results.get(2).getOwnerNodeNames(), hasItem("c")); assertThat(results.get(2).getSize(), is(100L)); assertThat(results.get(3).getOwnerNodeNames(), hasItem("d")); assertThat(results.get(3).getSize(), is(100L)); }
/** * simple testing. */ @Test public void simple() { BlockBuilder builder = new BlockBuilder(); builder.add(100, "a"); List<DirectInputFragment> results = builder.compute(50, 100, true, true); assertThat(results.size(), is(1)); assertThat(find(results, 0).getOwnerNodeNames(), hasItem("a")); }
/** * head and edge blocks are too small. */ @Test public void edge_too_small() { BlockBuilder builder = new BlockBuilder(); builder.add(10, "b"); builder.add(10, "c"); builder.add(400, "a"); builder.add(10, "b"); builder.add(10, "c"); List<DirectInputFragment> results = builder.compute(50, 200, true, true); assertThat(results.size(), is(2)); assertThat(results.get(0).getOwnerNodeNames(), hasItem("a")); assertThat(results.get(1).getOwnerNodeNames(), hasItem("a")); }
/** * tail blocks are too small. */ @Test public void tail_too_small() { BlockBuilder builder = new BlockBuilder(); builder.add(100, "a"); builder.add(10, "b"); builder.add(10, "c"); builder.add(10, "d"); List<DirectInputFragment> results = builder.compute(50, 200, true, true); assertThat(results.size(), is(1)); assertThat(results.get(0).getOwnerNodeNames(), hasItem("a")); }
/** * head blocks are too small. */ @Test public void head_too_small() { BlockBuilder builder = new BlockBuilder(); builder.add(10, "b"); builder.add(10, "c"); builder.add(10, "d"); builder.add(100, "a"); List<DirectInputFragment> results = builder.compute(50, 200, true, true); assertThat(results.size(), is(1)); assertThat(results.get(0).getOwnerNodeNames(), hasItem("a")); }
/** * block information is sparse. */ @Test public void sparse() { BlockBuilder builder = new BlockBuilder(); builder.seek(5); builder.add(10, "a"); builder.seek(5); builder.add(10, "a"); builder.seek(5); builder.add(10, "a"); builder.seek(5); builder.add(10, "a"); builder.seek(5); builder.add(10, "a"); builder.seek(5); List<DirectInputFragment> results = builder.compute(50, 100, true, true); assertThat(results.size(), is(1)); assertThat(find(results, 0).getOwnerNodeNames(), hasItem("a")); }