session.addAppMasterLocalFiles(DagUtils.createTezLrMap(null, newResources)); session.addAppMasterLocalFiles( DagUtils.getResourcesUpdatableForAm(resources.localizedResources));
/** * Ensures that the Tez Session is open and the AM has all necessary jars configured. */ void updateSession(TezSessionState session, JobConf jobConf, Path scratchDir, String[] inputOutputJars, Map<String,LocalResource> extraResources) throws Exception { final boolean missingLocalResources = !session .hasResources(inputOutputJars); TezClient client = session.getSession(); // TODO null can also mean that this operation was interrupted. Should we really try to re-create the session in that case ? if (client == null) { // can happen if the user sets the tez flag after the session was // established LOG.info("Tez session hasn't been created yet. Opening session"); session.open(conf, inputOutputJars); } else { LOG.info("Session is already open"); // Ensure the open session has the necessary resources (StorageHandler) if (missingLocalResources) { LOG.info("Tez session missing resources," + " adding additional necessary resources"); client.addAppMasterLocalFiles(extraResources); } session.refreshLocalResourcesFromConf(conf); } }
try { sessionState.getSession().addAppMasterLocalFiles(resourceMap); dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) {
DAGClient submit(JobConf conf, DAG dag, Path scratchDir, LocalResource appJarLr, TezSessionState sessionState, List<LocalResource> additionalLr, String[] inputOutputJars, Map<String,LocalResource> inputOutputLocalResources) throws Exception { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG); DAGClient dagClient = null; Map<String, LocalResource> resourceMap = new HashMap<String, LocalResource>(); if (additionalLr != null) { for (LocalResource lr: additionalLr) { if (lr.getType() == LocalResourceType.FILE) { // TEZ AM will only localize FILE (no script operators in the AM) resourceMap.put(utils.getBaseName(lr), lr); } } } try { // ready to start execution on the cluster sessionState.getSession().addAppMasterLocalFiles(resourceMap); dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening..."); // close the old one, but keep the tmp files around TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars, true); console.printInfo("Session re-established."); dagClient = sessionState.getSession().submitDAG(dag); } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG); return dagClient; }
/** * Ensures that the Tez Session is open and the AM has all necessary jars configured. */ void updateSession(TezSessionState session, JobConf jobConf, Path scratchDir, String[] inputOutputJars, Map<String,LocalResource> extraResources) throws Exception { final boolean missingLocalResources = !session .hasResources(inputOutputJars); if (!session.isOpen()) { // can happen if the user sets the tez flag after the session was // established LOG.info("Tez session hasn't been created yet. Opening session"); session.open(conf, inputOutputJars); } else { LOG.info("Session is already open"); // Ensure the open session has the necessary resources (StorageHandler) if (missingLocalResources) { LOG.info("Tez session missing resources," + " adding additional necessary resources"); session.getSession().addAppMasterLocalFiles(extraResources); } session.refreshLocalResourcesFromConf(conf); } }