public static void initialize(Configuration jConf) throws IOException { initialize(jConf, Utils.isLocal(jConf)); }
private static String addSingleFileToDistributedCache( PigContext pigContext, Configuration conf, String filename, String prefix) throws IOException { if (!pigContext.inIllustrator && !FileLocalizer.fileExists(filename, pigContext)) { throw new IOException( "Internal error: skew join partition file " + filename + " does not exist"); } String symlink = filename; // XXX Hadoop currently doesn't support distributed cache in local mode. // This line will be removed after the support is added by Hadoop team. if (!Utils.isLocal(pigContext, conf)) { symlink = prefix + "_" + Integer.toString(System.identityHashCode(filename)) + "_" + Long.toString(System.currentTimeMillis()); filename = filename + "#" + symlink; setupDistributedCache(pigContext, conf, new String[] { filename }, false); } return symlink; }
@Override public void visitMergeJoin(POMergeJoin join) throws VisitorException { // XXX Hadoop currently doesn't support distributed cache in local mode. // This line will be removed after the support is added if (Utils.isLocal(pigContext, conf)) return; String indexFile = join.getIndexFile(); // merge join may not use an index file if (indexFile == null) return; try { String symlink = addSingleFileToDistributedCache(pigContext, conf, indexFile, "indexfile_"); join.setIndexFile(symlink); } catch (IOException e) { String msg = "Internal error. Distributed cache could not " + "be set up for merge join index file"; throw new VisitorException(msg, e); } }
@Override public void visitMergeCoGroup(POMergeCogroup mergeCoGrp) throws VisitorException { // XXX Hadoop currently doesn't support distributed cache in local mode. // This line will be removed after the support is added if (Utils.isLocal(pigContext, conf)) return; String indexFile = mergeCoGrp.getIndexFileName(); if (indexFile == null) throw new VisitorException("No index file"); try { String symlink = addSingleFileToDistributedCache(pigContext, conf, indexFile, "indexfile_mergecogrp_"); mergeCoGrp.setIndexFileName(symlink); } catch (IOException e) { String msg = "Internal error. Distributed cache could not " + "be set up for merge cogrp index file"; throw new VisitorException(msg, e); } } }
if (Utils.isLocal(pigContext, conf)) return;
if(!Utils.isLocal(conf)) { fs.setWorkingDirectory(jobcontext.getWorkingDirectory());
if(Utils.isLocal(pigContext, conf)) { ConfigurationUtil.replaceConfigForLocalMode(conf);