public static RecordWriter createParquetWriter(Path target, JobConf conf, Properties properties, boolean compress, ConnectorSession session) throws IOException, ReflectiveOperationException conf.setLong(ParquetOutputFormat.BLOCK_SIZE, getParquetWriterBlockSize(session).toBytes()); conf.setLong(ParquetOutputFormat.PAGE_SIZE, getParquetWriterPageSize(session).toBytes());
private void setupExternalCacheConfig(boolean isPpd, String paths) { FileInputFormat.setInputPaths(conf, paths); conf.set(ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname, "ETL"); conf.setLong(HiveConf.ConfVars.MAPREDMINSPLITSIZE.varname, 1000); conf.setLong(HiveConf.ConfVars.MAPREDMAXSPLITSIZE.varname, 5000); conf.setBoolean(ConfVars.HIVE_ORC_MS_FOOTER_CACHE_PPD.varname, isPpd); conf.setBoolean(ConfVars.HIVEOPTINDEXFILTER.varname, isPpd); }
JobConf job = new JobConf(new Configuration(), TheJob.class); job.setLong("Param Name",longValue)
conf.setLong(OrcConf.STRIPE_SIZE.getHiveConfName(), newStripeSize); conf.setLong(HiveConf.ConfVars.MAPREDMAXSPLITSIZE.varname, newMaxSplitSize); conf.setLong(OrcConf.STRIPE_SIZE.getHiveConfName(), oldDefaultStripeSize); } else { conf.setLong(HiveConf.ConfVars.MAPREDMAXSPLITSIZE.varname, oldMaxSplitSize); } else {
private void setNutchOptions(JobConf job) throws URISyntaxException { job.setLong("pages", options.getNumPages() ); job.setLong("slotpages", options.getNumSlotPages()); Utils.shareLinkZipfCore(options, job); Utils.shareWordZipfCore(options, job); job.set(NUTCH_WORK_DIR_PARAM_NAME , options.getResultPath().toString()); // job.set(DataOptions.URL_MAP_NAME, paths.getPath(DataPaths.URLS).toString()); }
@Test (timeout=10000) public void testTaskProgress() throws Exception { JobConf job = new JobConf(); job.setLong(MRJobConfig.TASK_PROGRESS_REPORT_INTERVAL, 1000); Task task = new DummyTask(); task.setConf(job); DummyTaskReporter reporter = new DummyTaskReporter(task); Thread t = new Thread(reporter); t.start(); Thread.sleep(2100); task.setTaskDone(); reporter.resetDoneFlag(); t.join(); Assert.assertEquals(statusUpdateTimes, 2); }
conf.setLong(MRJobConfig.TASK_LOCAL_WRITE_LIMIT_BYTES, limit); LocalFileSystem localFS = FileSystem.getLocal(conf); Path tmpPath = new Path(TEST_DIR + "/testBytesWrittenLimit-tmpFile-"
JobConf conf = new JobConf(); conf.setStrings(MRConfig.LOCAL_DIR, "file://" + tmpPath); conf.setLong(MRJobConfig.JOB_SINGLE_DISK_LIMIT_BYTES, 1024L); conf.setBoolean(MRJobConfig.JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED, true); conf.setBoolean(MRJobConfig.JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED, true); conf.setLong(MRJobConfig.JOB_SINGLE_DISK_LIMIT_BYTES, -1L); testScratchDirLimit(false, conf); data.delete();
/** * Ensure that M/R 1.x applications can get and set task virtual memory with * old property names */ @SuppressWarnings("deprecation") @Test (timeout = 10000) public void testDeprecatedPropertyNameForTaskVmem() { JobConf configuration = new JobConf(); configuration.setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, 1024); configuration.setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, 1024); Assert.assertEquals(1024, configuration.getMemoryForMapTask()); Assert.assertEquals(1024, configuration.getMemoryForReduceTask()); // Make sure new property names aren't broken by the old ones configuration.setLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, 1025); configuration.setLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, 1025); Assert.assertEquals(1025, configuration.getMemoryForMapTask()); Assert.assertEquals(1025, configuration.getMemoryForReduceTask()); configuration.setMemoryForMapTask(2048); configuration.setMemoryForReduceTask(2048); Assert.assertEquals(2048, configuration.getLong( JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, -1)); Assert.assertEquals(2048, configuration.getLong( JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, -1)); // Make sure new property names aren't broken by the old ones Assert.assertEquals(2048, configuration.getLong( JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, -1)); Assert.assertEquals(2048, configuration.getLong( JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, -1)); }
@Test public void testLargeMemoryLimits() throws Exception { final JobConf conf = new JobConf(); // Xmx in production conf.setLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, 8L * 1024 * 1024 * 1024); // M1 = Xmx fraction for map outputs conf.setFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 1.0f); // M2 = max M1 fraction for a single maple output conf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.95f); // M3 = M1 fraction at which in memory merge is triggered conf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 1.0f); // M4 = M1 fraction of map outputs remaining in memory for a reduce conf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 1.0f); final MergeManagerImpl<Text, Text> mgr = new MergeManagerImpl<Text, Text>( null, conf, mock(LocalFileSystem.class), null, null, null, null, null, null, null, null, null, null, new MROutputFiles()); assertTrue("Large shuffle area unusable: " + mgr.memoryLimit, mgr.memoryLimit > Integer.MAX_VALUE); final long maxInMemReduce = mgr.getMaxInMemReduceLimit(); assertTrue("Large in-memory reduce area unusable: " + maxInMemReduce, maxInMemReduce > Integer.MAX_VALUE); assertEquals("maxSingleShuffleLimit to be capped at Integer.MAX_VALUE", Integer.MAX_VALUE, mgr.maxSingleShuffleLimit); verifyReservedMapOutputType(mgr, 10L, "MEMORY"); verifyReservedMapOutputType(mgr, 1L + Integer.MAX_VALUE, "DISK"); }
JobConf conf = new JobConf(); conf.setFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 1.0f); conf.setLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, TOTAL_MEM_BYTES); conf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.8f); conf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.9f);