/** * Clean up the child procress and socket. * @throws IOException */ void cleanup() throws IOException { serverSocket.close(); try { downlink.close(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } }
/** * Process all of the keys and values. Start up the application if we haven't * started it yet. */ public void reduce(K2 key, Iterator<V2> values, OutputCollector<K3, V3> output, Reporter reporter ) throws IOException { isOk = false; startApplication(output, reporter); downlink.reduceKey(key); while (values.hasNext()) { downlink.reduceValue(values.next()); } if(skipping) { //flush the streams on every record input if running in skip mode //so that we don't buffer other records surrounding a bad record. downlink.flush(); } isOk = true; }
/** * Abort the application and wait for it to finish. * @param t the exception that signalled the problem * @throws IOException A wrapper around the exception that was passed in */ void abort(Throwable t) throws IOException { LOG.info("Aborting because of " + StringUtils.stringifyException(t)); try { downlink.abort(); downlink.flush(); } catch (IOException e) { // IGNORE cleanup problems } try { handler.waitForFinish(); } catch (Throwable ignored) { process.destroy(); } IOException wrapper = new IOException("pipe child exception"); wrapper.initCause(t); throw wrapper; }
downlink.runMap(reporter.getInputSplit(), job.getNumReduceTasks(), isJavaInput); boolean skipping = job.getBoolean("mapred.skip.on", false); downlink.setInputTypes(key.getClass().getName(), value.getClass().getName()); downlink.mapItem(key, value); if(skipping) { downlink.flush(); downlink.endOfInput();
application.getDownlink().endOfInput(); } else { application.getDownlink().abort();
/** * Wait for authentication response. * @throws IOException * @throws InterruptedException */ void waitForAuthentication() throws IOException, InterruptedException { downlink.flush(); LOG.debug("Waiting for authentication response"); handler.waitForAuthentication(); }
@SuppressWarnings("unchecked") private void startApplication(OutputCollector<K3, V3> output, Reporter reporter) throws IOException { if (application == null) { try { LOG.info("starting application"); application = new Application<K2, V2, K3, V3>( job, null, output, reporter, (Class<? extends K3>) job.getOutputKeyClass(), (Class<? extends V3>) job.getOutputValueClass()); downlink = application.getDownlink(); } catch (InterruptedException ie) { throw new RuntimeException("interrupted", ie); } int reduce=0; downlink.runReduce(reduce, Submitter.getIsJavaRecordWriter(job)); } }
downlink.runMap(reporter.getInputSplit(), job.getNumReduceTasks(), isJavaInput); boolean skipping = job.getBoolean(MRJobConfig.SKIP_RECORDS, false); downlink.setInputTypes(key.getClass().getName(), value.getClass().getName()); downlink.mapItem(key, value); if(skipping) { downlink.flush(); downlink.endOfInput();
application.getDownlink().endOfInput(); } else { application.getDownlink().abort();
/** * Wait for the application to finish * @return did the application finish correctly? * @throws Throwable */ boolean waitForFinish() throws Throwable { downlink.flush(); return handler.waitForFinish(); }
@SuppressWarnings("unchecked") private void startApplication(OutputCollector<K3, V3> output, Reporter reporter) throws IOException { if (application == null) { try { LOG.info("starting application"); application = new Application<K2, V2, K3, V3>( job, null, output, reporter, (Class<? extends K3>) job.getOutputKeyClass(), (Class<? extends V3>) job.getOutputValueClass()); downlink = application.getDownlink(); } catch (InterruptedException ie) { throw new RuntimeException("interrupted", ie); } int reduce=0; downlink.runReduce(reduce, Submitter.getIsJavaRecordWriter(job)); } }
downlink.runMap(reporter.getInputSplit(), job.getNumReduceTasks(), isJavaInput); boolean skipping = job.getBoolean("mapred.skip.on", false); downlink.setInputTypes(key.getClass().getName(), value.getClass().getName()); downlink.mapItem(key, value); if(skipping) { downlink.flush(); downlink.endOfInput();
/** * Process all of the keys and values. Start up the application if we haven't * started it yet. */ public void reduce(K2 key, Iterator<V2> values, OutputCollector<K3, V3> output, Reporter reporter ) throws IOException { isOk = false; startApplication(output, reporter); downlink.reduceKey(key); while (values.hasNext()) { downlink.reduceValue(values.next()); } if(skipping) { //flush the streams on every record input if running in skip mode //so that we don't buffer other records surrounding a bad record. downlink.flush(); } isOk = true; }
/** * Abort the application and wait for it to finish. * @param t the exception that signalled the problem * @throws IOException A wrapper around the exception that was passed in */ void abort(Throwable t) throws IOException { LOG.info("Aborting because of " + StringUtils.stringifyException(t)); try { downlink.abort(); downlink.flush(); } catch (IOException e) { // IGNORE cleanup problems } try { handler.waitForFinish(); } catch (Throwable ignored) { process.destroy(); } IOException wrapper = new IOException("pipe child exception"); wrapper.initCause(t); throw wrapper; }
application.getDownlink().endOfInput(); } else { application.getDownlink().abort();
/** * Wait for authentication response. * @throws IOException * @throws InterruptedException */ void waitForAuthentication() throws IOException, InterruptedException { downlink.flush(); LOG.debug("Waiting for authentication response"); handler.waitForAuthentication(); }
/** * Clean up the child procress and socket. * @throws IOException */ void cleanup() throws IOException { serverSocket.close(); try { downlink.close(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } }
@SuppressWarnings("unchecked") private void startApplication(OutputCollector<K3, V3> output, Reporter reporter) throws IOException { if (application == null) { try { LOG.info("starting application"); application = new Application<K2, V2, K3, V3>( job, null, output, reporter, (Class<? extends K3>) job.getOutputKeyClass(), (Class<? extends V3>) job.getOutputValueClass()); downlink = application.getDownlink(); } catch (InterruptedException ie) { throw new RuntimeException("interrupted", ie); } int reduce=0; downlink.runReduce(reduce, Submitter.getIsJavaRecordWriter(job)); } }
downlink.runMap(reporter.getInputSplit(), job.getNumReduceTasks(), isJavaInput); boolean skipping = job.getBoolean(MRJobConfig.SKIP_RECORDS, false); downlink.setInputTypes(key.getClass().getName(), value.getClass().getName()); downlink.mapItem(key, value); if(skipping) { downlink.flush(); downlink.endOfInput();