+ zkQuorum + "', '-batchSize 1000');"); if (pigServer.executeBatch().get(0).getStatus() != JOB_STATUS.COMPLETED) { throw new RuntimeException("Job failed", pigServer.executeBatch() .get(0).getException());
if (job.getStatus() == JOB_STATUS.COMPLETED) { return job.getResults(); } else if (job.getStatus() == JOB_STATUS.FAILED && job.getException() != null) { Exception e = job.getException(); int errCode = 1066; String msg = "Unable to open iterator for alias " + id } else { throw new IOException("Job terminated with anomalous status " + job.getStatus().toString());
@Override public Long doInPig(PigServer pigServer) throws ExecException, IOException { pigServer.setBatchOn(); pigServer.registerScript(script.getResource() .getInputStream()); List<ExecJob> executeBatch = pigServer.executeBatch(); long count = 0; for (ExecJob execJob : executeBatch) { count = (Long) execJob.getResults().next().get(0); LOG.debug( "Pig Script Exec job result for total events: {}", count); } return count; } });
private void saveStats(List<ExecJob> execs, StepContribution contribution) throws Exception { if (CollectionUtils.isEmpty(execs) || contribution == null) { return; } for (ExecJob execJob : execs) { PigStats stats = execJob.getStatistics(); // embedded pig contains no stats and further more throws Exceptions // use CDH3 compatible comparison if (stats != null && !stats.getClass().getName().contains("EmbeddedPigStats")) { // compute the input stats manually List<InputStats> inputStats = stats.getInputStats(); for (InputStats is : inputStats) { for (int i = 0; i < safeLongToInt(is.getNumberRecords()); i++) { contribution.incrementReadCount(); } } contribution.incrementWriteCount(safeLongToInt(stats.getRecordWritten())); // Skip information not available yet // workaround: query the internal map/reduce jobs ? //contribution.incrementReadSkipCount(safeLongToInt(count.getValue())); // //for (int i = 0; i < safeLongToInt(count.getValue()); i++) { // contribution.incrementWriteSkipCount(); //} } } }
if (pigServer.executeBatch().get(0).getStatus() != JOB_STATUS.COMPLETED) { throw new RuntimeException("Job failed", pigServer.executeBatch() .get(0).getException());
+ zkQuorum + "', '-batchSize 1000');"); if (pigServer.executeBatch().get(0).getStatus() != JOB_STATUS.COMPLETED) { throw new RuntimeException("Job failed", pigServer.executeBatch() .get(0).getException());
if (pigServer.executeBatch().get(0).getStatus() != JOB_STATUS.COMPLETED) { throw new RuntimeException("Job failed", pigServer.executeBatch() .get(0).getException());
+ zkQuorum + "', '-batchSize 1000');"); if (pigServer.executeBatch().get(0).getStatus() != JOB_STATUS.COMPLETED) { throw new RuntimeException("Job failed", pigServer.executeBatch() .get(0).getException());