conf.setOutputValueClass(BytesWritable.class); conf.setJarByClass(getClass()); conf.setReduceSpeculativeExecution(false); FileInputFormat.setInputPaths(conf, inputPath); conf.set("final.output.dir", outputDir.toString());
/** * Some properties cannot be set using the new API. However, we know internally that the configuration Hadoop uses is really just the old JobConf which * exposes the methods we want. In particular, we have to turn off speculative execution since we are loading data and don't want Hadoop to spawn many * speculative tasks that will load duplicate data. */ protected void setReduceSpeculativeExecution(Configuration conf, boolean value) { if (conf instanceof org.apache.hadoop.mapred.JobConf) { org.apache.hadoop.mapred.JobConf jobConf = (org.apache.hadoop.mapred.JobConf) conf; jobConf.setReduceSpeculativeExecution(value); } }
/** * Turn speculative execution on or off for this job. * * @param speculativeExecution <code>true</code> if speculative execution * should be turned on, else <code>false</code>. */ public void setSpeculativeExecution(boolean speculativeExecution) { setMapSpeculativeExecution(speculativeExecution); setReduceSpeculativeExecution(speculativeExecution); }
/** * Turn speculative execution on or off for this job. * * @param speculativeExecution <code>true</code> if speculative execution * should be turned on, else <code>false</code>. */ public void setSpeculativeExecution(boolean speculativeExecution) { setMapSpeculativeExecution(speculativeExecution); setReduceSpeculativeExecution(speculativeExecution); }
/** * Turn speculative execution on or off for this job. * * @param speculativeExecution <code>true</code> if speculative execution * should be turned on, else <code>false</code>. */ public void setSpeculativeExecution(boolean speculativeExecution) { setMapSpeculativeExecution(speculativeExecution); setReduceSpeculativeExecution(speculativeExecution); }
/** * Turn speculative execution on or off for this job for reduce tasks. * * @param speculativeExecution <code>true</code> if speculative execution * should be turned on for reduce tasks, * else <code>false</code>. */ public void setReduceSpeculativeExecution(boolean speculativeExecution) { ensureState(JobState.DEFINE); conf.setReduceSpeculativeExecution(speculativeExecution); }
/** * Turn speculative execution on or off for this job for reduce tasks. * * @param speculativeExecution <code>true</code> if speculative execution * should be turned on for reduce tasks, * else <code>false</code>. */ public void setReduceSpeculativeExecution(boolean speculativeExecution) { ensureState(JobState.DEFINE); conf.setReduceSpeculativeExecution(speculativeExecution); }
/** * Turn speculative execution on or off for this job for reduce tasks. * * @param speculativeExecution <code>true</code> if speculative execution * should be turned on for reduce tasks, * else <code>false</code>. */ public void setReduceSpeculativeExecution(boolean speculativeExecution) { ensureState(JobState.DEFINE); conf.setReduceSpeculativeExecution(speculativeExecution); }
/** * Turn speculative execution on or off for this job. * * @param speculativeExecution <code>true</code> if speculative execution * should be turned on, else <code>false</code>. */ public void setSpeculativeExecution(boolean speculativeExecution) { setMapSpeculativeExecution(speculativeExecution); setReduceSpeculativeExecution(speculativeExecution); }
/** * Turn speculative execution on or off for this job. * * @param speculativeExecution <code>true</code> if speculative execution * should be turned on, else <code>false</code>. */ public void setSpeculativeExecution(boolean speculativeExecution) { setMapSpeculativeExecution(speculativeExecution); setReduceSpeculativeExecution(speculativeExecution); }
/** * Turn speculative execution on or off for this job. * * @param speculativeExecution <code>true</code> if speculative execution * should be turned on, else <code>false</code>. */ public void setSpeculativeExecution(boolean speculativeExecution) { setMapSpeculativeExecution(speculativeExecution); setReduceSpeculativeExecution(speculativeExecution); }
private static DBConfiguration setOutput(Job job, JobConf jobConf, String tableName) throws IOException { job.setOutputFormatClass(DBOutputFormat.class); jobConf.setReduceSpeculativeExecution(false); DBConfiguration dbConf = new DBConfiguration(job.getConfiguration()); dbConf.setOutputTableName(tableName); return dbConf; } }
public RetrieverJob(Retriever retriever) throws IOException { super(retriever.repository.getConf(), retriever.repository.configuredString("rr.conf")); this.retriever = retriever; inputformat = new QueryInputFormat(retriever.repository); path = conf.get("retriever.tempdir", "") + UUID.randomUUID().toString(); conf.setReduceSpeculativeExecution(false); setJob(); }
public RetrieverJob(Retriever retriever) throws IOException { super(retriever.repository.getConf(), retriever.repository.configuredString("rr.conf")); this.retriever = retriever; inputformat = new QueryInputFormat(retriever.repository); path = conf.get("retriever.tempdir", "") + UUID.randomUUID().toString(); conf.setReduceSpeculativeExecution(false); setJob(); }
private static DBConfiguration setOutput(JobConf job, String tableName) { job.setOutputFormat(DBOutputFormat.class); job.setReduceSpeculativeExecution(false); DBConfiguration dbConf = new DBConfiguration(job); dbConf.setOutputTableName(tableName); return dbConf; }
private static DBConfiguration setOutput(JobConf job, String tableName) { job.setOutputFormat(DBOutputFormat.class); job.setReduceSpeculativeExecution(false); DBConfiguration dbConf = new DBConfiguration(job); dbConf.setOutputTableName(tableName); return dbConf; }
private static DBConfiguration setOutput(JobConf job, String tableName) { job.setOutputFormat(DBOutputFormat.class); job.setReduceSpeculativeExecution(false); DBConfiguration dbConf = new DBConfiguration(job); dbConf.setOutputTableName(tableName); return dbConf; }
private static DBConfiguration setOutput(JobConf job, String tableName) { job.setOutputFormat(DBOutputFormat.class); job.setReduceSpeculativeExecution(false); DBConfiguration dbConf = new DBConfiguration(job); dbConf.setOutputTableName(tableName); return dbConf; }
private static DBConfiguration setOutput(JobConf job, String tableName) { job.setOutputFormat(DBOutputFormat.class); job.setReduceSpeculativeExecution(false); DBConfiguration dbConf = new DBConfiguration(job); dbConf.setOutputTableName(tableName); return dbConf; }
assertTrue(conf.getReduceSpeculativeExecution()); assertTrue(conf.getSpeculativeExecution()); conf.setReduceSpeculativeExecution(false); assertTrue(conf.getSpeculativeExecution());