/** * creates PigServer in LOCAL mode. * http://pig.apache.org/docs/r0.12.0/perf.html#error-handling * @param stopOnFailure equivalent of "-stop_on_failure" command line arg, setting to 'true' makes * debugging easier */ public static PigServer createPigServer(boolean stopOnFailure, Properties p) throws ExecException { Path workDir = new Path(System.getProperty("test.tmp.dir", "target" + File.separator + "test" + File.separator + "tmp")); String testId = "HCatBaseTest_" + System.currentTimeMillis(); p.put("mapred.local.dir", workDir + File.separator + testId + File.separator + "mapred" + File.separator + "local"); p.put("mapred.system.dir", workDir + File.separator + testId + File.separator + "mapred" + File.separator + "system"); p.put("mapreduce.jobtracker.staging.root.dir", workDir + File.separator + testId + File.separator + "mapred" + File.separator + "staging"); p.put("mapred.temp.dir", workDir + File.separator + testId + File.separator + "mapred" + File.separator + "temp"); p.put("pig.temp.dir", workDir + File.separator + testId + File.separator + "pig" + File.separator + "temp"); if(stopOnFailure) { p.put("stop.on.failure", Boolean.TRUE.toString()); return new PigServer(ExecType.LOCAL, p); } return new PigServer(ExecType.LOCAL, p); } }
@Before public void setUp() throws Exception { conf = getTestClusterConfig(); conf.set(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); // Set CURRENT_SCN to confirm that it's ignored conf.set(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(System.currentTimeMillis()+QueryConstants.MILLIS_IN_DAY)); pigServer = new PigServer(ExecType.LOCAL, conf); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); conn = DriverManager.getConnection(getUrl(), props); zkQuorum = LOCALHOST + JDBC_PROTOCOL_SEPARATOR + getZKClientPort(conf); tupleFactory = TupleFactory.getInstance(); }
private PigServer getPigServer() throws ExecException { if (pigServer == null) { pigServer = new PigServer(pigContext, false); } return pigServer; }
@Override public PigServer run() throws Exception { return new PigServer(ctx, true); } });
PigServer getPigServer(String execType) throws IOException { return new PigServer(execType, properties); } }
public Grunt(BufferedReader in, PigContext pigContext) throws ExecException { this.in = in; this.pig = new PigServer(pigContext); if (in != null) { parser = new GruntParser(in, pig); } }
public static void main(String[] args) { try { PigServer pigServer = new PigServer("local"); runQuery(pigServer); } catch (Exception e) { logger.error("Caught exception:", e); } }
public static void main(String[] args) { try { PigServer pigServer = new PigServer("local"); runQuery(pigServer); } catch (Exception e) { logger.error("Caught exception:", e); } }
import java.io.IOException; import org.apache.pig.PigServer; public class WordCount { public static void main(String[] args) { PigServer pigServer = new PigServer(); try { pigServer.registerJar("/mylocation/tokenize.jar"); runMyQuery(pigServer, "myinput.txt"; } catch (IOException e) { e.printStackTrace(); } } public static void runMyQuery(PigServer pigServer, String inputFile) throws IOException { pigServer.registerQuery("A = load '" + inputFile + "' using TextLoader();"); pigServer.registerQuery("B = foreach A generate flatten(tokenize($0));"); pigServer.registerQuery("C = group B by $1;"); pigServer.registerQuery("D = foreach C generate flatten(group), COUNT(B.$0);"); pigServer.store("D", "myoutput"); } }
/** * Explain this pipeline. Results will be printed to stdout. * @throws IOException if explain fails. */ public void explain() throws IOException { if (queries.isEmpty()) { LOG.info("No bound query to explain"); return; } PigServer pigServer = new PigServer(scriptContext.getPigContext(), false); registerQueryForDiagnostics(pigServer, queries.get(0)); pigServer.explain(null, System.out); }
@Override public int[] executeScript( String pigScript, ExecutionMode mode, Properties properties ) throws IOException, org.apache.pig.tools.pigscript.parser.ParseException { ClassLoader cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader( getClass().getClassLoader() ); try { PigServer pigServer = new PigServer( getExecType( mode ), properties ); GruntParser grunt = new GruntParser( new StringReader( pigScript ), pigServer ); grunt.setInteractive( false ); return grunt.parseStopOnError( false ); } finally { Thread.currentThread().setContextClassLoader( cl ); } } }
/** * Run illustrate for this pipeline. Results will be printed to stdout. * @throws IOException if illustrate fails. */ public void illustrate() throws IOException { if (queries.isEmpty()) { LOG.info("No bound query to illustrate"); return; } PigServer pigServer = new PigServer(scriptContext.getPigContext(), false); registerQueryForDiagnostics(pigServer, queries.get(0)); pigServer.getExamples(null); }
@Override public int[] executeScript( String pigScript, ExecutionMode mode, Properties properties ) throws IOException, org.apache.pig.tools.pigscript.parser.ParseException { ClassLoader cl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader( getClass().getClassLoader() ); try { PigServer pigServer = new PigServer( getExecType( mode ), properties ); GruntParser grunt = new GruntParser( new StringReader( pigScript ) ); grunt.setInteractive( false ); grunt.setParams( pigServer ); return grunt.parseStopOnError( false ); } finally { Thread.currentThread().setContextClassLoader( cl ); } }
/** * Describe the schema of an alias in this pipeline. * Results will be printed to stdout. * @param alias to be described * @throws IOException if describe fails. */ public void describe(String alias) throws IOException { if (queries.isEmpty()) { LOG.info("No bound query to describe"); return; } PigServer pigServer = new PigServer(scriptContext.getPigContext(), false); registerQueryForDiagnostics(pigServer, queries.get(0)); pigServer.dumpSchema(alias); }
/** * Register a jar for use in Pig. Once this is done this jar will be * registered for <b>all subsequent</b> Pig pipelines in this script. * If you wish to register it for only a single Pig pipeline, use * register within that definition. * @param jarfile Path of jar to include. * @throws IOException if the indicated jarfile cannot be found. */ public static void registerJar(String jarfile) throws IOException { LOG.info("Register jar: "+ jarfile); ScriptPigContext ctx = getScriptContext(); PigServer pigServer = new PigServer(ctx.getPigContext(), false); pigServer.registerJar(jarfile); }
@Override protected void init() throws InitUnitException { try { Configuration configuration = gatherConfigs(); PigContext pigContext = new PigContext(ExecType.MAPREDUCE, configuration); pigServer = new PigServer(pigContext); } catch (PigException e) { throw new InitUnitException(e); } }
/** * Set a variable for use in Pig Latin. This set * will then be present for <b>all subsequent</b> Pig pipelines defined in this * script. If you wish to set it for only a single Pig pipeline, use * set within that definition. * @param var variable to set * @param value to set it to */ public static void set(String var, String value) throws IOException { ScriptPigContext ctx = getScriptContext(); PigServer pigServer = new PigServer(ctx.getPigContext(), false); pigServer.getPigContext().getProperties().setProperty(var, value); }
System.setProperty(PySystemState.PYTHON_CACHEDIR_SKIP, "false"); PigServer pigServer = new PigServer(pigContext, false);
@Override public PigStats call() throws Exception { LOG.info("Query to run:\n" + query); PigContext pc = scriptContext.getPigContext(); ScriptState scriptState = pc.getExecutionEngine().instantiateScriptState(); ScriptState.start(scriptState); ScriptState.get().setScript(query); ScriptState.get().registerListener(adaptor); PigServer pigServer = new PigServer(ctx, true); GruntParser grunt = new GruntParser(new StringReader(query), pigServer); grunt.setInteractive(false); try { grunt.parseStopOnError(false); } catch (ParseException e) { throw new IOException("Failed to parse script", e); } return PigStats.get(); } }
private PigStats exec(String query) throws IOException { LOG.info("Query to run:\n" + query); List<PigProgressNotificationListener> listeners = ScriptState.get().getAllListeners(); PigContext pc = scriptContext.getPigContext(); String scriptName = new File(ScriptState.get().getFileName()).getName(); ScriptState scriptState = pc.getExecutionEngine().instantiateScriptState(); scriptState.setFileName(scriptName); ScriptState.start(scriptState); ScriptState.get().setScript(query); for (PigProgressNotificationListener listener : listeners) { ScriptState.get().registerListener(listener); } PigServer pigServer = new PigServer(scriptContext.getPigContext(), false); if (!pigServer.getPigContext().getProperties().containsKey("jobName")) { pigServer.setJobName(scriptName); } GruntParser grunt = new GruntParser(new StringReader(query), pigServer); grunt.setInteractive(false); try { grunt.parseStopOnError(false); } catch (ParseException e) { throw new IOException("Failed to parse script " + e.getMessage(), e); } return PigStats.get(); }