/** * Find the data interface for the step with the specified name. * * @param name the step name * @return the step data interface */ public StepDataInterface findDataInterface( String name ) { if ( steps == null ) { return null; } for ( int i = 0; i < steps.size(); i++ ) { StepMetaDataCombi sid = steps.get( i ); StepInterface rt = sid.step; if ( rt.getStepname().equalsIgnoreCase( name ) ) { return sid.data; } } return null; }
/** * Find the run thread for the step with the specified name. * * @param stepname the step name * @return a StepInterface object corresponding to the run thread for the specified step */ public StepInterface findRunThread( String stepname ) { if ( steps == null ) { return null; } for ( int i = 0; i < steps.size(); i++ ) { StepMetaDataCombi sid = steps.get( i ); StepInterface step = sid.step; if ( step.getStepname().equalsIgnoreCase( stepname ) ) { return step; } } return null; }
/** * Find the executing step copy for the step with the specified name and copy number * * @param stepname the step name * @param copynr * @return the executing step found or null if no copy could be found. */ public StepInterface findStepInterface( String stepname, int copyNr ) { if ( steps == null ) { return null; } for ( int i = 0; i < steps.size(); i++ ) { StepMetaDataCombi sid = steps.get( i ); StepInterface stepInterface = sid.step; if ( stepInterface.getStepname().equalsIgnoreCase( stepname ) && sid.copy == copyNr ) { return stepInterface; } } return null; }
/** * Find the available executing step copies for the step with the specified name * * @param stepname the step name * @param copynr * @return the list of executing step copies found or null if no steps are available yet (incorrect usage) */ public List<StepInterface> findStepInterfaces( String stepname ) { if ( steps == null ) { return null; } List<StepInterface> list = new ArrayList<>(); for ( int i = 0; i < steps.size(); i++ ) { StepMetaDataCombi sid = steps.get( i ); StepInterface stepInterface = sid.step; if ( stepInterface.getStepname().equalsIgnoreCase( stepname ) ) { list.add( stepInterface ); } } return list; }
/** * Find the base steps for the step with the specified name. * * @param stepname the step name * @return the list of base steps for the specified step */ public List<StepInterface> findBaseSteps( String stepname ) { List<StepInterface> baseSteps = new ArrayList<>(); if ( steps == null ) { return baseSteps; } for ( int i = 0; i < steps.size(); i++ ) { StepMetaDataCombi sid = steps.get( i ); StepInterface stepInterface = sid.step; if ( stepInterface.getStepname().equalsIgnoreCase( stepname ) ) { baseSteps.add( stepInterface ); } } return baseSteps; }
/** * Gets the run thread for the step with the specified name and copy number. * * @param name the step name * @param copy the copy number * @return a StepInterface object corresponding to the run thread for the specified step */ public StepInterface getRunThread( String name, int copy ) { if ( steps == null ) { return null; } for ( int i = 0; i < steps.size(); i++ ) { StepMetaDataCombi sid = steps.get( i ); StepInterface step = sid.step; if ( step.getStepname().equalsIgnoreCase( name ) && step.getCopy() == copy ) { return step; } } return null; }
/** * Logs a summary message for the specified step. * * @param si the step interface */ public void logSummary( StepInterface si ) { log.logBasic( si.getStepname(), BaseMessages.getString( PKG, "Trans.Log.FinishedProcessing", String.valueOf( si .getLinesInput() ), String.valueOf( si.getLinesOutput() ), String.valueOf( si.getLinesRead() ) ) + BaseMessages .getString( PKG, "Trans.Log.FinishedProcessing2", String.valueOf( si.getLinesWritten() ), String.valueOf( si .getLinesUpdated() ), String.valueOf( si.getErrors() ) ) ); }
/** * Asks all steps to stop but doesn't wait around for it to happen. This is a special method for use with mappings. */ private void killAllNoWait() { if ( steps == null ) { return; } for ( int i = 0; i < steps.size(); i++ ) { StepMetaDataCombi sid = steps.get( i ); StepInterface step = sid.step; if ( log.isDebug() ) { log.logDebug( BaseMessages.getString( PKG, "Trans.Log.LookingAtStep" ) + step.getStepname() ); } step.stopAll(); try { Thread.sleep( 20 ); } catch ( Exception e ) { log.logError( BaseMessages.getString( PKG, "Trans.Log.TransformationErrors" ) + e.toString() ); return; } } }
if ( seconds != 0 ) { if ( step.getErrors() == 0 ) { log.logBasic( BaseMessages.getString( PKG, "Trans.Log.ProcessSuccessfullyInfo", step.getStepname(), "." + step .getCopy(), String.valueOf( proc ), String.valueOf( ( proc / seconds ) ) ) ); } else { log.logError( BaseMessages.getString( PKG, "Trans.Log.ProcessErrorInfo", step.getStepname(), "." + step .getCopy(), String.valueOf( step.getErrors() ), String.valueOf( proc ), String.valueOf( proc / seconds ) ) ); log.logBasic( BaseMessages.getString( PKG, "Trans.Log.ProcessSuccessfullyInfo", step.getStepname(), "." + step .getCopy(), String.valueOf( proc ), seconds != 0 ? String.valueOf( ( proc / seconds ) ) : "-" ) ); } else { log.logError( BaseMessages.getString( PKG, "Trans.Log.ProcessErrorInfo2", step.getStepname(), "." + step .getCopy(), String.valueOf( step.getErrors() ), String.valueOf( proc ), String.valueOf( seconds ) ) );
tLogString.append( "Steps after sort: " ).append( "\n" ); for ( StepMetaDataCombi combi : steps ) { tLogString.append( combi.step.getStepname() ).append( "\n" );
public Optional<Result> execute( List<RowMetaAndData> rows ) throws KettleException { if ( rows.isEmpty() || stopped ) { return Optional.empty(); } Trans subtrans = this.createSubtrans(); running.add( subtrans ); parentTrans.addActiveSubTransformation( subTransName, subtrans ); // Pass parameter values passParametersToTrans( subtrans, rows.get( 0 ) ); Result result = new Result(); result.setRows( rows ); subtrans.setPreviousResult( result ); subtrans.prepareExecution( this.parentTrans.getArguments() ); List<RowMetaAndData> rowMetaAndData = new ArrayList<>(); subtrans.getSteps().stream() .filter( c -> c.step.getStepname().equalsIgnoreCase( subStep ) ) .findFirst() .ifPresent( c -> c.step.addRowListener( new RowAdapter() { @Override public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) { rowMetaAndData.add( new RowMetaAndData( rowMeta, row ) ); } } ) ); subtrans.startThreads(); subtrans.waitUntilFinished(); updateStatuses( subtrans ); running.remove( subtrans ); Result subtransResult = subtrans.getResult(); subtransResult.setRows( rowMetaAndData ); return Optional.of( subtransResult ); }
log.logDebug( BaseMessages.getString( PKG, "Trans.Log.LookingAtStep" ) + sid.step.getStepname() );
public synchronized void updateAll( StepInterface baseStep ) { // Proc: nr of lines processed: input + output! this.stepname = baseStep.getStepname(); this.copy = baseStep.getCopy(); this.linesRead = linesRead + baseStep.getLinesRead(); this.linesWritten = linesWritten + baseStep.getLinesWritten(); this.linesInput = linesInput + baseStep.getLinesInput(); this.linesOutput = linesOutput + baseStep.getLinesOutput(); this.linesUpdated = linesUpdated + baseStep.getLinesUpdated(); this.linesRejected = linesRejected + baseStep.getLinesRejected(); this.errors = errors + baseStep.getErrors(); this.accumlatedRuntime = accumlatedRuntime + baseStep.getRuntime(); this.statusDescription = baseStep.getStatus().getDescription(); long in_proc = Math.max( linesInput, linesRead ); long out_proc = Math.max( linesOutput + linesUpdated, linesWritten + linesRejected ); float lapsed = ( (float) accumlatedRuntime ) / 1000; double in_speed = 0; double out_speed = 0; if ( lapsed != 0 ) { in_speed = Math.floor( 10 * ( in_proc / lapsed ) ) / 10; out_speed = Math.floor( 10 * ( out_proc / lapsed ) ) / 10; } double speedNumber = ( in_speed > out_speed ? in_speed : out_speed ); this.seconds = Math.floor( ( lapsed * 10 ) + 0.5 ) / 10; this.speed = lapsed == 0 ? "-" : " " + speedDf.format( speedNumber ); this.priority = baseStep.isRunning() ? " " + baseStep.rowsetInputSize() + "/" + baseStep.rowsetOutputSize() : "-"; this.stopped = baseStep.isStopped(); this.paused = baseStep.isPaused(); }
@Test public void testSafeStop() { when( stepMock.isSafeStopped() ).thenReturn( false ); when( stepMock.getStepname() ).thenReturn( "stepName" ); trans.setSteps( of( combi( stepMock, data, stepMeta ) ) ); Result result = trans.getResult(); assertFalse( result.isSafeStop() ); when( stepMock.isSafeStopped() ).thenReturn( true ); result = trans.getResult(); assertTrue( result.isSafeStop() ); }
rowSet.setThreadNameFromToCopy( sourceStep.getStepname(), 0, mappingStepname, 0 );
if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_READ ) ) ) { result.setNrLinesRead( result.getNrLinesRead() + step.getLinesRead() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_INPUT ) ) ) { result.setNrLinesInput( result.getNrLinesInput() + step.getLinesInput() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_WRITTEN ) ) ) { result.setNrLinesWritten( result.getNrLinesWritten() + step.getLinesWritten() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_OUTPUT ) ) ) { result.setNrLinesOutput( result.getNrLinesOutput() + step.getLinesOutput() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_UPDATED ) ) ) { result.setNrLinesUpdated( result.getNrLinesUpdated() + step.getLinesUpdated() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_REJECTED ) ) ) { result.setNrLinesRejected( result.getNrLinesRejected() + step.getLinesRejected() );
logDetailed( "Finished running step [" + step.getStepname() + "(" + step.getCopy() + ")]." );
public void run() { // Set the internal variables also on the initialization thread! // ((BaseStep)combi.step).setInternalVariables(); if ( !doIt ) { // An extension point plugin decided we should not initialize the step. // Logging, error handling, finished flag... should all be handled in the extension point. // return; } try { combi.step.getLogChannel().snap( Metrics.METRIC_STEP_INIT_START ); if ( combi.step.init( combi.meta, combi.data ) ) { combi.data.setStatus( StepExecutionStatus.STATUS_IDLE ); ok = true; } else { combi.step.setErrors( 1 ); log.logError( BaseMessages.getString( PKG, "Trans.Log.ErrorInitializingStep", combi.step.getStepname() ) ); } } catch ( Throwable e ) { log.logError( BaseMessages.getString( PKG, "Trans.Log.ErrorInitializingStep", combi.step.getStepname() ) ); log.logError( Const.getStackTracker( e ) ); } finally { combi.step.getLogChannel().snap( Metrics.METRIC_STEP_INIT_STOP ); } finished = true; }
public void setConnectorSteps( StepInterface[] targetSteps, List<MappingValueRename> inputValueRenames, List<MappingValueRename> outputValueRenames ) { for ( int i = 0; i < targetSteps.length; i++ ) { // OK, before we leave, make sure there is a rowset that covers the path to this target step. // We need to create a new RowSet and add it to the Input RowSets of the target step // BlockingRowSet rowSet = new BlockingRowSet( getTransMeta().getSizeRowset() ); // This is always a single copy, but for source and target... // rowSet.setThreadNameFromToCopy( getStepname(), 0, targetSteps[i].getStepname(), 0 ); // Make sure to connect it to both sides... // addRowSetToOutputRowSets( rowSet ); // Add the row set to the target step as input. // This will appropriately drain the buffer as data comes in. // However, as an exception, we can't attach it to another mapping step. // We need to attach it to the appropriate mapping input step. // The main problem is that we can't do it here since we don't know that the other step has initialized properly // yet. // This method is called during init() and we can't tell for sure it's done already. // As such, we'll simply grab the remaining row sets at the Mapping#processRow() level and assign them to a // Mapping Input step. // targetSteps[i].addRowSetToInputRowSets( rowSet ); } data.inputValueRenames = inputValueRenames; data.outputValueRenames = outputValueRenames; data.targetSteps = targetSteps; }
logDetailed( "Finished running step [" + step.getStepname() + "(" + step.getCopy() + ")]." ); rowData[index++] = step.getStepname();