@Test public void testCopyFromHostConnectionRejected() throws Exception { when(connection.getResponseCode()) .thenReturn(Fetcher.TOO_MANY_REQ_STATUS_CODE); Fetcher<Text, Text> fetcher = new FakeFetcher<>(job, id, ss, mm, r, metrics, except, key, connection); fetcher.copyFromHost(host); Assert.assertEquals("No host failure is expected.", ss.hostFailureCount(host.getHostName()), 0); Assert.assertEquals("No fetch failure is expected.", ss.fetchFailureCount(map1ID), 0); Assert.assertEquals("No fetch failure is expected.", ss.fetchFailureCount(map2ID), 0); verify(ss).penalize(eq(host), anyLong()); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map1ID)); verify(ss).putBackKnownMapOutput(any(MapHost.class), eq(map2ID)); }
scheduler.hostFailed(host1.getHostName());
/** check if hit timeout of retry, if not, throw an exception and start a * new round of retry.*/ private void checkTimeoutOrRetry(MapHost host, IOException ioe) throws IOException { // First time to retry. long currentTime = Time.monotonicNow(); if (retryStartTime == 0) { retryStartTime = currentTime; } // Retry is not timeout, let's do retry with throwing an exception. if (currentTime - retryStartTime < this.fetchRetryTimeout) { LOG.warn("Shuffle output from " + host.getHostName() + " failed, retry it.", ioe); throw ioe; } else { // timeout, prepare to be failed. LOG.warn("Timeout for copying MapOutput with retry on host " + host + "after " + fetchRetryTimeout + " milliseconds."); } }
/** check if hit timeout of retry, if not, throw an exception and start a * new round of retry.*/ private void checkTimeoutOrRetry(MapHost host, IOException ioe) throws IOException { // First time to retry. long currentTime = Time.monotonicNow(); if (retryStartTime == 0) { retryStartTime = currentTime; } // Retry is not timeout, let's do retry with throwing an exception. if (currentTime - retryStartTime < this.fetchRetryTimeout) { LOG.warn("Shuffle output from " + host.getHostName() + " failed, retry it.", ioe); throw ioe; } else { // timeout, prepare to be failed. LOG.warn("Timeout for copying MapOutput with retry on host " + host + "after " + fetchRetryTimeout + " milliseconds."); } }
/** check if hit timeout of retry, if not, throw an exception and start a * new round of retry.*/ private void checkTimeoutOrRetry(MapHost host, IOException ioe) throws IOException { // First time to retry. long currentTime = Time.monotonicNow(); if (retryStartTime == 0) { retryStartTime = currentTime; } // Retry is not timeout, let's do retry with throwing an exception. if (currentTime - retryStartTime < this.fetchRetryTimeout) { LOG.warn("Shuffle output from " + host.getHostName() + " failed, retry it.", ioe); throw ioe; } else { // timeout, prepare to be failed. LOG.warn("Timeout for copying MapOutput with retry on host " + host + "after " + fetchRetryTimeout + " milliseconds."); } }
private DataInputStream openShuffleUrl(MapHost host, Set<TaskAttemptID> remaining, URL url) { DataInputStream input = null; try { setupConnectionsWithRetry(host, remaining, url); if (stopped) { abortConnect(host, remaining); } else { input = new DataInputStream(connection.getInputStream()); } } catch (IOException ie) { boolean connectExcpt = ie instanceof ConnectException; ioErrs.increment(1); LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie); // If connect did not succeed, just mark all the maps as failed, // indirectly penalizing the host scheduler.hostFailed(host.getHostName()); for(TaskAttemptID left: remaining) { scheduler.copyFailed(left, host, false, connectExcpt); } } return input; }
failureCounts.put(mapId, new IntWritable(1)); String hostname = host.getHostName(); IntWritable hostFailedNum = hostFailures.get(hostname);
failureCounts.put(mapId, new IntWritable(1)); String hostname = host.getHostName(); if (hostFailures.containsKey(hostname)) { IntWritable x = hostFailures.get(hostname);
failureCounts.put(mapId, new IntWritable(1)); String hostname = host.getHostName(); IntWritable hostFailedNum = hostFailures.get(hostname);
failureCounts.put(mapId, new IntWritable(1)); String hostname = host.getHostName(); IntWritable hostFailedNum = hostFailures.get(hostname);
failureCounts.put(mapId, new IntWritable(1)); String hostname = host.getHostName(); IntWritable hostFailedNum = hostFailures.get(hostname);
/** check if hit timeout of retry, if not, throw an exception and start a * new round of retry.*/ private void checkTimeoutOrRetry(MapHost host, IOException ioe) throws IOException { // First time to retry. long currentTime = Time.monotonicNow(); if (retryStartTime == 0) { retryStartTime = currentTime; } // Retry is not timeout, let's do retry with throwing an exception. if (currentTime - retryStartTime < this.fetchRetryTimeout) { LOG.warn("Shuffle output from " + host.getHostName() + " failed, retry it.", ioe); throw ioe; } else { // timeout, prepare to be failed. LOG.warn("Timeout for copying MapOutput with retry on host " + host + "after " + fetchRetryTimeout + " milliseconds."); } }
private DataInputStream openShuffleUrl(MapHost host, Set<TaskAttemptID> remaining, URL url) { DataInputStream input = null; try { setupConnectionsWithRetry(url); if (stopped) { abortConnect(host, remaining); } else { input = new DataInputStream(connection.getInputStream()); } } catch (IOException ie) { boolean connectExcpt = ie instanceof ConnectException; ioErrs.increment(1); LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie); // If connect did not succeed, just mark all the maps as failed, // indirectly penalizing the host scheduler.hostFailed(host.getHostName()); for(TaskAttemptID left: remaining) { scheduler.copyFailed(left, host, false, connectExcpt); } // Add back all the remaining maps, WITHOUT marking them as failed for(TaskAttemptID left: remaining) { scheduler.putBackKnownMapOutput(host, left); } } return input; }
private DataInputStream openShuffleUrl(MapHost host, Set<TaskAttemptID> remaining, URL url) { DataInputStream input = null; try { setupConnectionsWithRetry(host, remaining, url); if (stopped) { abortConnect(host, remaining); } else { input = new DataInputStream(connection.getInputStream()); } } catch (IOException ie) { boolean connectExcpt = ie instanceof ConnectException; ioErrs.increment(1); LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie); // If connect did not succeed, just mark all the maps as failed, // indirectly penalizing the host scheduler.hostFailed(host.getHostName()); for(TaskAttemptID left: remaining) { scheduler.copyFailed(left, host, false, connectExcpt); } // Add back all the remaining maps, WITHOUT marking them as failed for(TaskAttemptID left: remaining) { scheduler.putBackKnownMapOutput(host, left); } } return input; }
) throws IOException { failureCounts.remove(mapId); hostFailures.remove(host.getHostName()); int mapIndex = mapId.getTaskID().getId();
) throws IOException { failureCounts.remove(mapId); hostFailures.remove(host.getHostName()); int mapIndex = mapId.getTaskID().getId();
) throws IOException { failureCounts.remove(mapId); hostFailures.remove(host.getHostName()); int mapIndex = mapId.getTaskID().getId();
private DataInputStream openShuffleUrl(MapHost host, Set<TaskAttemptID> remaining, URL url) { DataInputStream input = null; try { setupConnectionsWithRetry(host, remaining, url); if (stopped) { abortConnect(host, remaining); } else { input = new DataInputStream(connection.getInputStream()); } } catch (IOException ie) { boolean connectExcpt = ie instanceof ConnectException; ioErrs.increment(1); LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie); // If connect did not succeed, just mark all the maps as failed, // indirectly penalizing the host scheduler.hostFailed(host.getHostName()); for(TaskAttemptID left: remaining) { scheduler.copyFailed(left, host, false, connectExcpt); } // Add back all the remaining maps, WITHOUT marking them as failed for(TaskAttemptID left: remaining) { scheduler.putBackKnownMapOutput(host, left); } } return input; }
) throws IOException { failureCounts.remove(mapId); hostFailures.remove(host.getHostName()); int mapIndex = mapId.getTaskID().getId();
) throws IOException { failureCounts.remove(mapId); hostFailures.remove(host.getHostName()); int mapIndex = mapId.getTaskID().getId();