/** * @param igniteInstanceName Ignite instance name. * @return Hadoop configuration. */ public HadoopConfiguration hadoopConfiguration(String igniteInstanceName) { HadoopConfiguration cfg = new HadoopConfiguration(); cfg.setMaxParallelTasks(3); return cfg; }
/** * Copy constructor. * * @param cfg Configuration to copy. */ public HadoopConfiguration(HadoopConfiguration cfg) { // Preserve alphabetic order. // TODO: IGNITE-404: Uncomment when fixed. //extExecution = cfg.isExternalExecution(); finishedJobInfoTtl = cfg.getFinishedJobInfoTtl(); planner = cfg.getMapReducePlanner(); maxParallelTasks = cfg.getMaxParallelTasks(); maxTaskQueueSize = cfg.getMaxTaskQueueSize(); libNames = cfg.getNativeLibraryNames(); }
/** * Initializes default hadoop configuration. * * @param cfg Hadoop configuration. */ private void initializeDefaults(HadoopConfiguration cfg) { if (cfg.getMapReducePlanner() == null) cfg.setMapReducePlanner(new IgniteHadoopWeightedMapReducePlanner()); }
/** {@inheritDoc} */ @Override protected HadoopConfiguration createHadoopConfiguration() { HadoopConfiguration hadoopCfg = new HadoopConfiguration(); // Use weighted planner with default settings: IgniteHadoopWeightedMapReducePlanner planner = new IgniteHadoopWeightedMapReducePlanner(); hadoopCfg.setMapReducePlanner(planner); return hadoopCfg; } }
/** {@inheritDoc} */ @Override public void onKernalStart() throws IgniteCheckedException { super.onKernalStart(); jobTracker = ctx.jobTracker(); exec = new HadoopExecutorService(log, ctx.kernalContext().igniteInstanceName(), ctx.configuration().getMaxParallelTasks(), ctx.configuration().getMaxTaskQueueSize()); }
/** {@inheritDoc} */ @Override public HadoopClassLoader commonClassLoader() { HadoopClassLoader res = ldr; if (res == null) { synchronized (this) { res = ldr; if (res == null) { String[] libNames = null; if (ctx != null && ctx.config().getHadoopConfiguration() != null) libNames = ctx.config().getHadoopConfiguration().getNativeLibraryNames(); res = new HadoopClassLoader(null, "hadoop-common", libNames, this); ldr = res; } } } return res; }
/** * @return Map-reduce planner. */ public HadoopMapReducePlanner planner() { return cfg.getMapReducePlanner(); }
/** {@inheritDoc} */ @Override public HadoopConfiguration hadoopConfiguration(String igniteInstanceName) { HadoopConfiguration cfg = super.hadoopConfiguration(igniteInstanceName); cfg.setMapReducePlanner(new HadoopTestRoundRobinMrPlanner()); // TODO: IGNITE-404: Uncomment when fixed. //cfg.setExternalExecution(false); return cfg; }
if (ctx.configuration().getFinishedJobInfoTtl() > 0) { ExpiryPolicy finishedJobPlc = new ModifiedExpiryPolicy( new Duration(MILLISECONDS, ctx.configuration().getFinishedJobInfoTtl()));
/** {@inheritDoc} */ @Override public void start() throws IgniteCheckedException { if (ctx.isDaemon()) return; HadoopConfiguration cfg = ctx.config().getHadoopConfiguration(); if (cfg == null) cfg = new HadoopConfiguration(); else cfg = new HadoopConfiguration(cfg); initializeDefaults(cfg); hctx = new HadoopContext( ctx, cfg, new HadoopJobTracker(), new HadoopEmbeddedTaskExecutor(), // TODO: IGNITE-404: Uncomment when fixed. //cfg.isExternalExecution() ? new HadoopExternalTaskExecutor() : new HadoopEmbeddedTaskExecutor(), new HadoopShuffle()); for (HadoopComponent c : hctx.components()) c.start(hctx); hadoop = new HadoopImpl(this); ctx.addNodeAttribute(HadoopAttributes.NAME, new HadoopAttributes(cfg)); }
/** {@inheritDoc} */ @Override public HadoopConfiguration hadoopConfiguration(String igniteInstanceName) { HadoopConfiguration cfg = super.hadoopConfiguration(igniteInstanceName); cfg.setMaxParallelTasks(5); // TODO: IGNITE-404: Uncomment when fixed. //cfg.setExternalExecution(false); return cfg; }
job = jobInfo.createJob(jobCls, jobId, log, ctx.configuration().getNativeLibraryNames(), ctx.kernalContext().hadoopHelper());
/** * Constructor. * * @param cfg Configuration. */ public HadoopAttributes(HadoopConfiguration cfg) { assert cfg != null; assert cfg.getMapReducePlanner() != null; plannerCls = cfg.getMapReducePlanner().getClass().getName(); // TODO: IGNITE-404: Get from configuration when fixed. extExec = false; maxParallelTasks = cfg.getMaxParallelTasks(); maxTaskQueueSize = cfg.getMaxTaskQueueSize(); libNames = cfg.getNativeLibraryNames(); // Cores count already passed in other attributes, we add it here for convenience. cores = Runtime.getRuntime().availableProcessors(); }
/** * Create data transfer object for hadoop configuration. * * @param src Hadoop configuration. */ public VisorHadoopConfiguration(HadoopConfiguration src) { planner = compactClass(src.getMapReducePlanner()); // TODO: IGNITE-404: Uncomment when fixed. //extExecution = cfg.isExternalExecution(); finishedJobInfoTtl = src.getFinishedJobInfoTtl(); maxParallelTasks = src.getMaxParallelTasks(); maxTaskQueueSize = src.getMaxTaskQueueSize(); libNames = U.sealList(src.getNativeLibraryNames()); }
/** * Copy constructor. * * @param cfg Configuration to copy. */ public HadoopConfiguration(HadoopConfiguration cfg) { // Preserve alphabetic order. // TODO: IGNITE-404: Uncomment when fixed. //extExecution = cfg.isExternalExecution(); finishedJobInfoTtl = cfg.getFinishedJobInfoTtl(); planner = cfg.getMapReducePlanner(); maxParallelTasks = cfg.getMaxParallelTasks(); maxTaskQueueSize = cfg.getMaxTaskQueueSize(); libNames = cfg.getNativeLibraryNames(); }
/** * Create data transfer object for hadoop configuration. * * @param src Hadoop configuration. */ public VisorHadoopConfiguration(HadoopConfiguration src) { planner = compactClass(src.getMapReducePlanner()); // TODO: IGNITE-404: Uncomment when fixed. //extExecution = cfg.isExternalExecution(); finishedJobInfoTtl = src.getFinishedJobInfoTtl(); maxParallelTasks = src.getMaxParallelTasks(); maxTaskQueueSize = src.getMaxTaskQueueSize(); libNames = U.sealList(src.getNativeLibraryNames()); }