tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString()); conf.stripHiddenConfigurations(tezConfig);
private UserPayload createIntermediatePayloadInput( TezConfiguration config, EdgeValues edgeValues ) { config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, edgeValues.keyClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, edgeValues.valueClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS, edgeValues.keyComparatorClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, edgeValues.keyPartitionerClassName ); setWorkingDirectory( config ); return getPayload( config ); }
private UserPayload createIntermediatePayloadOutput( TezConfiguration config, EdgeValues edgeValues ) { config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, edgeValues.keyClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, edgeValues.valueClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS, edgeValues.keyComparatorClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, edgeValues.keyPartitionerClassName ); setWorkingDirectory( config ); return getPayload( config ); }
tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString()); conf.stripHiddenConfigurations(tezConfig);
private UserPayload createIntermediatePayloadInput( TezConfiguration config, EdgeValues edgeValues ) { config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, edgeValues.keyClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, edgeValues.valueClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS, edgeValues.keyComparatorClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, edgeValues.keyPartitionerClassName ); setWorkingDirectory( config ); return getPayload( config ); }
private UserPayload createIntermediatePayloadOutput( TezConfiguration config, EdgeValues edgeValues ) { config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, edgeValues.keyClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, edgeValues.valueClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS, edgeValues.keyComparatorClassName ); config.set( TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, edgeValues.keyPartitionerClassName ); setWorkingDirectory( config ); return getPayload( config ); }
@Override protected void setConfigProperty( TezConfiguration config, Object key, Object value ) { // don't let these objects pass, even though toString is called below. if( value instanceof Class || value instanceof Configuration || value == null ) return; config.set( key.toString(), value.toString() ); }
@Override protected void setConfigProperty( TezConfiguration config, Object key, Object value ) { // don't let these objects pass, even though toString is called below. if( value instanceof Class || value instanceof Configuration || value == null ) return; config.set( key.toString(), value.toString() ); }
@Override public void init(TezConfiguration tezConf, YarnConfiguration yarnConf) { this.conf = tezConf; tezConf.set("fs.defaultFS", "file:///"); // Tez libs already in the client's classpath this.conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true); this.conf.set(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, localModeDAGSchedulerClassName); isSession = tezConf.getBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT); // disable web service for local mode. this.conf.setBoolean(TezConfiguration.TEZ_AM_WEBSERVICE_ENABLE, false); }
protected void initConfig( Map<Object, Object> properties, TezConfiguration parentConfig ) { if( properties != null ) parentConfig = createConfig( properties, parentConfig ); if( parentConfig == null ) // this is ok, getJobConf will pass a default parent in return; flowConf = new TezConfiguration( parentConfig ); // prevent local values from being shared flowConf.set( "fs.http.impl", HttpFileSystem.class.getName() ); flowConf.set( "fs.https.impl", HttpFileSystem.class.getName() ); UserGroupInformation.setConfiguration( flowConf ); flowStagingPath = createStagingRoot(); }
protected void initConfig( Map<Object, Object> properties, TezConfiguration parentConfig ) { if( properties != null ) parentConfig = createConfig( properties, parentConfig ); if( parentConfig == null ) // this is ok, getJobConf will pass a default parent in return; flowConf = new TezConfiguration( parentConfig ); // prevent local values from being shared flowConf.set( "fs.http.impl", HttpFileSystem.class.getName() ); flowConf.set( "fs.https.impl", HttpFileSystem.class.getName() ); UserGroupInformation.setConfiguration( flowConf ); flowStagingPath = createStagingRoot(); }
public static TezConfiguration createTezConf( Map<Object, Object> properties, TezConfiguration defaultJobconf ) { TezConfiguration jobConf = defaultJobconf == null ? new TezConfiguration() : new TezConfiguration( defaultJobconf ); if( properties == null ) return jobConf; Set<Object> keys = new HashSet<Object>( properties.keySet() ); // keys will only be grabbed if both key/value are String, so keep orig keys if( properties instanceof Properties ) keys.addAll( ( (Properties) properties ).stringPropertyNames() ); for( Object key : keys ) { Object value = properties.get( key ); if( value == null && properties instanceof Properties && key instanceof String ) value = ( (Properties) properties ).getProperty( (String) key ); if( value == null ) // don't stuff null values continue; // don't let these objects pass, even though toString is called below. if( value instanceof Class || value instanceof TezConfiguration ) continue; jobConf.set( key.toString(), value.toString() ); } return jobConf; }
public static TezConfiguration createTezConf( Map<Object, Object> properties, TezConfiguration defaultJobconf ) { TezConfiguration jobConf = defaultJobconf == null ? new TezConfiguration() : new TezConfiguration( defaultJobconf ); if( properties == null ) return jobConf; Set<Object> keys = new HashSet<Object>( properties.keySet() ); // keys will only be grabbed if both key/value are String, so keep orig keys if( properties instanceof Properties ) keys.addAll( ( (Properties) properties ).stringPropertyNames() ); for( Object key : keys ) { Object value = properties.get( key ); if( value == null && properties instanceof Properties && key instanceof String ) value = ( (Properties) properties ).getProperty( (String) key ); if( value == null ) // don't stuff null values continue; // don't let these objects pass, even though toString is called below. if( value instanceof Class || value instanceof TezConfiguration ) continue; jobConf.set( key.toString(), value.toString() ); } return jobConf; }
private Path prepareEnsureStagingDir( TezConfiguration workingConf ) throws IOException { String stepStagingPath = createStepStagingPath(); workingConf.set( TezConfiguration.TEZ_AM_STAGING_DIR, stepStagingPath ); Path stagingDir = new Path( stepStagingPath ); FileSystem fileSystem = FileSystem.get( workingConf ); stagingDir = fileSystem.makeQualified( stagingDir ); TokenCache.obtainTokensForNamenodes( new Credentials(), new Path[]{stagingDir}, workingConf ); TezClientUtils.ensureStagingDirExists( workingConf, stagingDir ); if( fileSystem.getScheme().startsWith( "file:/" ) ) new File( stagingDir.toUri() ).mkdirs(); return stagingDir; }
private Path prepareEnsureStagingDir( TezConfiguration workingConf ) throws IOException { String stepStagingPath = createStepStagingPath(); workingConf.set( TezConfiguration.TEZ_AM_STAGING_DIR, stepStagingPath ); Path stagingDir = new Path( stepStagingPath ); FileSystem fileSystem = FileSystem.get( workingConf ); stagingDir = fileSystem.makeQualified( stagingDir ); TokenCache.obtainTokensForNamenodes( new Credentials(), new Path[]{stagingDir}, workingConf ); TezClientUtils.ensureStagingDirExists( workingConf, stagingDir ); if( fileSystem.getScheme().startsWith( "file:/" ) ) new File( stagingDir.toUri() ).mkdirs(); return stagingDir; }
private TezConfiguration getDAGAMConfFromMRConf() { TezConfiguration finalConf = new TezConfiguration(this.tezConf); Map<String, String> mrParamToDAGParamMap = DeprecatedKeys .getMRToDAGParamMap(); for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) { if (finalConf.get(entry.getKey()) != null) { finalConf.set(entry.getValue(), finalConf.get(entry.getKey())); finalConf.unset(entry.getKey()); if (LOG.isDebugEnabled()) { LOG.debug("MR->DAG Translating MR key: " + entry.getKey() + " to Tez key: " + entry.getValue() + " with value " + finalConf.get(entry.getValue())); } } } return finalConf; }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { TezConfiguration jobConf = new TezConfiguration( getConfiguration() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); int nodeNum = jobConf.getInt( "cascading.flow.node.num", 0 ); String partname = String.format( "-%05d-%05d-", stepNum, nodeNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new Hadoop2TezFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { TezConfiguration jobConf = new TezConfiguration( getConfiguration() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); int nodeNum = jobConf.getInt( "cascading.flow.node.num", 0 ); String partname = String.format( "-%05d-%05d-", stepNum, nodeNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new Hadoop2TezFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
private EdgeProperty createEdgeProperty( EdgeValues edgeValues ) { TezConfiguration outputConfig = new TezConfiguration( edgeValues.getConfig() ); outputConfig.set( "cascading.node.sink", FlowElements.id( edgeValues.getFlowElement() ) ); outputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) ); addFields( outputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() ); addFields( outputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() ); addFields( outputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() ); UserPayload outputPayload = createIntermediatePayloadOutput( outputConfig, edgeValues ); TezConfiguration inputConfig = new TezConfiguration( edgeValues.getConfig() ); inputConfig.set( "cascading.node.source", FlowElements.id( edgeValues.getFlowElement() ) ); inputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) ); addFields( inputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() ); addFields( inputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() ); addFields( inputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() ); UserPayload inputPayload = createIntermediatePayloadInput( inputConfig, edgeValues ); return EdgeProperty.create( edgeValues.getMovementType(), edgeValues.getSourceType(), edgeValues.getSchedulingType(), OutputDescriptor.create( edgeValues.getOutputClassName() ).setUserPayload( outputPayload ), InputDescriptor.create( edgeValues.getInputClassName() ).setUserPayload( inputPayload ) ); }