if (isLocal(host, isw.getSplit().getHostnames())) {
@Override public boolean equals(Object obj) { if (obj == this) { return true; } else if (obj instanceof FileInputSplit && super.equals(obj)) { FileInputSplit other = (FileInputSplit) obj; return this.start == other.start && this.length == other.length && (this.file == null ? other.file == null : (other.file != null && this.file.equals(other.file))); } else { return false; } }
hosts.add(splitHosts[rand.nextInt(splitHosts.length)]); splits.add(new LocatableInputSplit(i, hosts.toArray(stringArray))); hosts.clear();
splits.add(new LocatableInputSplit(splitCnt++, threeLocalHosts)); splits.add(new LocatableInputSplit(splitCnt++, twoLocalHosts)); splits.add(new LocatableInputSplit(splitCnt++, oneLocalHost)); splits.add(new LocatableInputSplit(splitCnt++, noLocalHost)); assertTrue(Arrays.equals(is.getHostnames(), oneLocalHost)); } else if (host.equals(localHosts[1])) { assertTrue(Arrays.equals(is.getHostnames(), twoLocalHosts)); } else if (host.equals(localHosts[2])) { assertTrue(Arrays.equals(is.getHostnames(), threeLocalHosts)); } else { assertTrue(Arrays.equals(is.getHostnames(), noLocalHost));
@Override public void run() { LocatableInputSplit split; while ((split = ia.getNextInputSplit(null, 0)) != null) { splitsRetrieved.incrementAndGet(); sumOfIds.addAndGet(split.getSplitNumber()); } } };
@Override public void run() { LocatableInputSplit split; while ((split = ia.getNextInputSplit("testhost", 0)) != null) { splitsRetrieved.incrementAndGet(); sumOfIds.addAndGet(split.getSplitNumber()); } } };
public String[] getHostNames() { return split.getHostnames(); }
@Override public boolean equals(Object obj) { if (obj == this) { return true; } else if (obj instanceof FileInputSplit && super.equals(obj)) { FileInputSplit other = (FileInputSplit) obj; return this.start == other.start && this.length == other.length && (this.file == null ? other.file == null : (other.file != null && this.file.equals(other.file))); } else { return false; } }
splits.add(new LocatableInputSplit(splitCnt++, "host1")); splits.add(new LocatableInputSplit(splitCnt++, "host2")); splits.add(new LocatableInputSplit(splitCnt++, "remoteHost"));
@Override public void run() { final String threadHost = hosts[(int) (Math.random() * hosts.length)]; LocatableInputSplit split; while ((split = ia.getNextInputSplit(threadHost, 0)) != null) { splitsRetrieved.incrementAndGet(); sumOfIds.addAndGet(split.getSplitNumber()); } } };
if (isLocal(host, isw.getSplit().getHostnames())) {
@Override public boolean equals(Object obj) { if (obj == this) { return true; } else if (obj instanceof FileInputSplit && super.equals(obj)) { FileInputSplit other = (FileInputSplit) obj; return this.start == other.start && this.length == other.length && (this.file == null ? other.file == null : (other.file != null && this.file.equals(other.file))); } else { return false; } }
splits.add(new LocatableInputSplit(i, hosts[i%hosts.length]));
@Override public void inputSplitsAssigned(int taskId, List<InputSplit> inputSplits) { for (InputSplit inputSplit : inputSplits) { boolean found = false; for (AssigningInputSplit split : unassigned) { if (split.getSplit().equals(inputSplit)) { unassigned.remove(split); found = true; break; } } if (!found) { throw new FlinkRuntimeException("InputSplit not found for " + inputSplit.getSplitNumber()); } } }
splits.add(new LocatableInputSplit(i, hosts[i%3]));
splits.add(new LocatableInputSplit(i, "testhost"));
splits.add(new LocatableInputSplit(i, hosts[i % hosts.length]));
splits.add(new LocatableInputSplit(i, "testhost"));
splits.add(new LocatableInputSplit(i, hosts[i%3]));
@Test public void testSerialSplitAssignmentMixedLocalHost() { try { final String[] hosts = { "host1", "host1", "host1", "host2", "host2", "host3" }; final int NUM_SPLITS = 10 * hosts.length; // load some splits Set<LocatableInputSplit> splits = new HashSet<LocatableInputSplit>(); for (int i = 0; i < NUM_SPLITS; i++) { splits.add(new LocatableInputSplit(i, hosts[i % hosts.length])); } // get all available splits LocatableInputSplitAssigner ia = new LocatableInputSplitAssigner(splits); InputSplit is = null; int i = 0; while ((is = ia.getNextInputSplit(hosts[i++ % hosts.length], 0)) != null) { assertTrue(splits.remove(is)); } // check we had all assertTrue(splits.isEmpty()); assertNull(ia.getNextInputSplit("anotherHost", 0)); assertEquals(0, ia.getNumberOfRemoteAssignments()); assertEquals(NUM_SPLITS, ia.getNumberOfLocalAssignments()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }