/** * This method deserializes the Base64 encoded String into an Object instance. * * @param string * @return an Object */ public static <T> T deserializeBase64( String string, Configuration conf, Class<T> type ) throws IOException { return deserializeBase64( string, conf, type, true ); }
/** * This method deserializes the Base64 encoded String into an Object instance. * * @param string * @return an Object */ public static <T> T deserializeBase64( String string, Configuration conf, Class<T> type ) throws IOException { return deserializeBase64( string, conf, type, true ); }
private List<Map<String, String>> getConfigs( JobConf job ) throws IOException { return (List<Map<String, String>>) HadoopUtil.deserializeBase64( job.get( "cascading.multiinputformats" ), job, ArrayList.class, true ); }
private List<Map<String, String>> getConfigs( JobConf job ) throws IOException { return (List<Map<String, String>>) HadoopUtil.deserializeBase64( job.get( "cascading.multiinputformats" ), job, ArrayList.class, true ); }
public static Map<Integer, Fields> getFields( Configuration conf, String property ) throws IOException { String value = conf.getRaw( property ); if( value == null || value.isEmpty() ) return Collections.emptyMap(); Map<String, Fields> map = deserializeBase64( value, conf, Map.class, true ); Map<Integer, Fields> result = new HashMap<>(); for( Map.Entry<String, Fields> entry : map.entrySet() ) result.put( Integer.parseInt( entry.getKey() ), entry.getValue() ); return result; }
public static Map<Integer, Fields> getFields( Configuration conf, String property ) throws IOException { String value = conf.getRaw( property ); if( value == null || value.isEmpty() ) return Collections.emptyMap(); Map<String, Fields> map = deserializeBase64( value, conf, Map.class, true ); Map<Integer, Fields> result = new HashMap<>(); for( Map.Entry<String, Fields> entry : map.entrySet() ) result.put( Integer.parseInt( entry.getKey() ), entry.getValue() ); return result; }
public static Comparator[] getFieldComparatorsFrom( Configuration conf, String name ) { String value = conf.get( name ); if( value == null ) return new Comparator[ conf.getInt( name + ".size", 1 ) ]; try { return HadoopUtil.deserializeBase64( value, conf, Fields.class ).getComparators(); } catch( IOException exception ) { throw new CascadingException( "unable to deserialize comparators for: " + name ); } }
public static Comparator[] getFieldComparatorsFrom( Configuration conf, String name ) { String value = conf.get( name ); if( value == null ) return new Comparator[ conf.getInt( name + ".size", 1 ) ]; try { return HadoopUtil.deserializeBase64( value, conf, Fields.class ).getComparators(); } catch( IOException exception ) { throw new CascadingException( "unable to deserialize comparators for: " + name ); } }
private TezConfiguration getSourceConf( FlowProcess<TezConfiguration> flowProcess, TezConfiguration conf, String property ) { Map<String, String> priorConf; try { priorConf = (Map<String, String>) HadoopUtil.deserializeBase64( property, conf, HashMap.class, true ); } catch( IOException exception ) { throw new FlowException( "unable to deserialize properties", exception ); } return flowProcess.mergeMapIntoConfig( conf, priorConf ); }
private JobConf getSourceConf( HadoopFlowProcess flowProcess, JobConf conf, String property ) { Map<String, String> priorConf; try { priorConf = (Map<String, String>) HadoopUtil.deserializeBase64( property, conf, HashMap.class, true ); } catch( IOException exception ) { throw new FlowException( "unable to deserialize properties", exception ); } return flowProcess.mergeMapIntoConfig( conf, priorConf ); }
private TezConfiguration getSourceConf( FlowProcess<TezConfiguration> flowProcess, TezConfiguration conf, String property ) { Map<String, String> priorConf; try { priorConf = (Map<String, String>) HadoopUtil.deserializeBase64( property, conf, HashMap.class, true ); } catch( IOException exception ) { throw new FlowException( "unable to deserialize properties", exception ); } return flowProcess.mergeMapIntoConfig( conf, priorConf ); }
private JobConf getSourceConf( HadoopFlowProcess flowProcess, JobConf conf, String property ) { Map<String, String> priorConf; try { priorConf = (Map<String, String>) HadoopUtil.deserializeBase64( property, conf, HashMap.class, true ); } catch( IOException exception ) { throw new FlowException( "unable to deserialize properties", exception ); } return flowProcess.mergeMapIntoConfig( conf, priorConf ); }
public SolrRecordWriter(JobConf conf, String name, Progressable progress) throws IOException { // Copy Solr core directory from HDFS to temp local location. Path sourcePath = new Path(conf.get(SOLR_CORE_PATH_KEY)); String coreName = sourcePath.getName(); String tmpDir = System.getProperty("java.io.tmpdir"); File localSolrCore = new File(tmpDir, "cascading.solr-" + UUID.randomUUID() + "/" + coreName); FileSystem sourceFS = sourcePath.getFileSystem(conf); sourceFS.copyToLocalFile(sourcePath, new Path(localSolrCore.getAbsolutePath())); // Figure out where ultimately the results need to wind up. _outputPath = new Path(FileOutputFormat.getTaskOutputPath(conf, name), "index"); _outputFS = _outputPath.getFileSystem(conf); // Get the set of fields we're indexing. Fields sinkFields = HadoopUtil.deserializeBase64(conf.get(SINK_FIELDS_KEY), conf, Fields.class); int maxSegments = conf.getInt(MAX_SEGMENTS_KEY, DEFAULT_MAX_SEGMENTS); String dataDirPropertyName = conf.get(DATA_DIR_PROPERTY_NAME_KEY); // Set up local Solr home. File localSolrHome = SolrSchemeUtil.makeTempSolrHome(localSolrCore); // This is where data will wind up, inside of an index subdir. _localIndexDir = new File(localSolrHome, "data"); _keepAliveHook = new HadoopKeepAliveHook(progress); _solrWriter = new SolrWriter(_keepAliveHook, sinkFields, dataDirPropertyName, _localIndexDir.getAbsolutePath(), localSolrCore, maxSegments) { }; }
mapNodeState = readStateFromDistCache( jobConf, jobConf.get( FlowStep.CASCADING_FLOW_STEP_ID ), "map" ); flowNode = deserializeBase64( mapNodeState, jobConf, BaseFlowNode.class );
reduceNodeState = readStateFromDistCache( jobConf, jobConf.get( FlowStep.CASCADING_FLOW_STEP_ID ), "reduce" ); flowNode = deserializeBase64( reduceNodeState, jobConf, BaseFlowNode.class );
mapNodeState = readStateFromDistCache( jobConf, jobConf.get( FlowStep.CASCADING_FLOW_STEP_ID ), "map" ); flowNode = deserializeBase64( mapNodeState, jobConf, BaseFlowNode.class );
reduceNodeState = readStateFromDistCache( jobConf, jobConf.get( FlowStep.CASCADING_FLOW_STEP_ID ), "reduce" ); flowNode = deserializeBase64( reduceNodeState, jobConf, BaseFlowNode.class );
@Override public void initialize() throws Exception { configuration = new TezConfiguration( TezUtils.createConfFromUserPayload( getContext().getUserPayload() ) ); TezUtil.setMRProperties( getContext(), configuration, true ); try { HadoopUtil.initLog4j( configuration ); LOG.info( "cascading version: {}", configuration.get( "cascading.version", "" ) ); currentProcess = new Hadoop2TezFlowProcess( new FlowSession(), getContext(), configuration ); flowNode = deserializeBase64( configuration.getRaw( FlowNode.CASCADING_FLOW_NODE ), configuration, BaseFlowNode.class ); LOG.info( "flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal() ); logMemory( LOG, "flow node id: " + flowNode.getID() + ", mem on start" ); } catch( Throwable throwable ) { if( throwable instanceof CascadingException ) throw (CascadingException) throwable; throw new FlowException( "internal error during processor configuration", throwable ); } }
@Override public void initialize() throws Exception { configuration = new TezConfiguration( TezUtils.createConfFromUserPayload( getContext().getUserPayload() ) ); TezUtil.setMRProperties( getContext(), configuration, true ); try { HadoopUtil.initLog4j( configuration ); LOG.info( "cascading version: {}", configuration.get( "cascading.version", "" ) ); currentProcess = new Hadoop2TezFlowProcess( new FlowSession(), getContext(), configuration ); flowNode = deserializeBase64( configuration.getRaw( FlowNode.CASCADING_FLOW_NODE ), configuration, BaseFlowNode.class ); LOG.info( "flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal() ); logMemory( LOG, "flow node id: " + flowNode.getID() + ", mem on start" ); } catch( Throwable throwable ) { if( throwable instanceof CascadingException ) throw (CascadingException) throwable; throw new FlowException( "internal error during processor configuration", throwable ); } }