private void createUdfContext() { udfContext = UDFContext.getUDFContext(); udfContext.addJobConf(conf); }
private void resetUDFContext() { UDFContext.getUDFContext().addJobConf(null); }
public static void setupUDFContext(Configuration job) throws IOException { UDFContext udfc = UDFContext.getUDFContext(); udfc.addJobConf(job); // don't deserialize in front-end if (udfc.isUDFConfEmpty()) { udfc.deserialize(); } }
private void init(PhysicalPlan pp, POStore poStore) throws IOException { poStore.setStoreImpl(new FetchPOStoreImpl(pigContext)); poStore.setUp(); TaskAttemptID taskAttemptID = HadoopShims.getNewTaskAttemptID(); //Fetch mode needs to explicitly set the task id which is otherwise done by Hadoop conf.setInt(MRConfiguration.JOB_APPLICATION_ATTEMPT_ID, taskAttemptID.getId()); if (!PlanHelper.getPhysicalOperators(pp, POStream.class).isEmpty()) { MapRedUtil.setupStreamingDirsConfSingle(poStore, pigContext, conf); } String currentTime = Long.toString(System.currentTimeMillis()); conf.set("pig.script.submitted.timestamp", currentTime); conf.set("pig.job.submitted.timestamp", currentTime); PhysicalOperator.setReporter(new FetchProgressableReporter()); SchemaTupleBackend.initialize(conf, pigContext); UDFContext udfContext = UDFContext.getUDFContext(); udfContext.addJobConf(conf); udfContext.setClientSystemProps(pigContext.getProperties()); udfContext.serialize(conf); PigMapReduce.sJobConfInternal.set(conf); Utils.setDefaultTimeZone(conf); boolean aggregateWarning = "true".equalsIgnoreCase(conf.get("aggregate.warning")); PigStatusReporter pigStatusReporter = PigStatusReporter.getInstance(); pigStatusReporter.setContext(new FetchTaskContext(new FetchContext())); PigHadoopLogger pigHadoopLogger = PigHadoopLogger.getInstance(); pigHadoopLogger.setReporter(pigStatusReporter); pigHadoopLogger.setAggregate(aggregateWarning); PhysicalOperator.setPigLogger(pigHadoopLogger); }
/** * Runs the fetch task by executing chain of calls on the PhysicalPlan from the leaf * up to the LoadFunc * * @param pp - Physical plan * @return SimpleFetchPigStats instance representing the fetched result * @throws IOException */ public PigStats launchPig(PhysicalPlan pp) throws IOException { try { POStore poStore = (POStore) pp.getLeaves().get(0); init(pp, poStore); // run fetch runPipeline(poStore); UDFFinishVisitor udfFinisher = new UDFFinishVisitor(pp, new DependencyOrderWalker<PhysicalOperator, PhysicalPlan>(pp)); udfFinisher.visit(); return PigStats.start(new EmptyPigStats(pigContext, poStore)); } finally { UDFContext.getUDFContext().addJobConf(null); pigContext.getProperties().remove(PigImplConstants.CONVERTED_TO_FETCH); } }
UDFContext.getUDFContext().addJobConf(conf); UDFContext.getUDFContext().deserialize();
UDFContext.getUDFContext().addJobConf(null); return vertex;
PhysicalOperator root = expPhysicalPlan.getLeaves().get(0); try { UDFContext.getUDFContext().addJobConf(ConfigurationUtil.toConfiguration(pc.getProperties(), true)); PigHadoopLogger pigHadoopLogger = PigHadoopLogger.getInstance(); PhysicalOperator.setPigLogger(pigHadoopLogger); val = root.getNext(root.getResultType()).result; restoreDefaultTimeZone(); UDFContext.getUDFContext().addJobConf(null); } catch (ExecException e) { throw new FrontendException(e); UDFContext.getUDFContext().addJobConf(ConfigurationUtil.toConfiguration(pc.getProperties(), true)); setDefaultTimeZone(); val = udf.getEvalFunc().exec(null); restoreDefaultTimeZone(); UDFContext.getUDFContext().addJobConf(null); } catch (IOException e) { throw new FrontendException(e);