public MQTTClientBuilder withStep( StepInterface step ) { this.logChannel = step.getLogChannel(); this.stepName = step.getStepMeta().getName(); return this; }
public synchronized void updateAll( StepInterface baseStep ) { // Proc: nr of lines processed: input + output! this.stepname = baseStep.getStepname(); this.copy = baseStep.getCopy(); this.linesRead = linesRead + baseStep.getLinesRead(); this.linesWritten = linesWritten + baseStep.getLinesWritten(); this.linesInput = linesInput + baseStep.getLinesInput(); this.linesOutput = linesOutput + baseStep.getLinesOutput(); this.linesUpdated = linesUpdated + baseStep.getLinesUpdated(); this.linesRejected = linesRejected + baseStep.getLinesRejected(); this.errors = errors + baseStep.getErrors(); this.accumlatedRuntime = accumlatedRuntime + baseStep.getRuntime(); this.statusDescription = baseStep.getStatus().getDescription(); long in_proc = Math.max( linesInput, linesRead ); long out_proc = Math.max( linesOutput + linesUpdated, linesWritten + linesRejected ); float lapsed = ( (float) accumlatedRuntime ) / 1000; double in_speed = 0; double out_speed = 0; if ( lapsed != 0 ) { in_speed = Math.floor( 10 * ( in_proc / lapsed ) ) / 10; out_speed = Math.floor( 10 * ( out_proc / lapsed ) ) / 10; } double speedNumber = ( in_speed > out_speed ? in_speed : out_speed ); this.seconds = Math.floor( ( lapsed * 10 ) + 0.5 ) / 10; this.speed = lapsed == 0 ? "-" : " " + speedDf.format( speedNumber ); this.priority = baseStep.isRunning() ? " " + baseStep.rowsetInputSize() + "/" + baseStep.rowsetOutputSize() : "-"; this.stopped = baseStep.isStopped(); this.paused = baseStep.isPaused(); }
/** * Logs a summary message for the specified step. * * @param si the step interface */ public void logSummary( StepInterface si ) { log.logBasic( si.getStepname(), BaseMessages.getString( PKG, "Trans.Log.FinishedProcessing", String.valueOf( si .getLinesInput() ), String.valueOf( si.getLinesOutput() ), String.valueOf( si.getLinesRead() ) ) + BaseMessages .getString( PKG, "Trans.Log.FinishedProcessing2", String.valueOf( si.getLinesWritten() ), String.valueOf( si .getLinesUpdated() ), String.valueOf( si.getErrors() ) ) ); }
/** * Checks whether the transformation steps are running lookup. * * @return a boolean array associated with the step list, indicating whether that step is running a lookup. */ public boolean[] getTransStepIsRunningLookup() { if ( steps == null ) { return null; } boolean[] tResult = new boolean[ steps.size() ]; for ( int i = 0; i < steps.size(); i++ ) { StepMetaDataCombi sid = steps.get( i ); tResult[ i ] = ( sid.step.isRunning() || sid.step.getStatus() != StepExecutionStatus.STATUS_FINISHED ); } return tResult; }
baseStep.addRowListener( new RowAdapter() { public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException { try {
public void run() { try { step.setRunning( true ); step.getLogChannel().snap( Metrics.METRIC_STEP_EXECUTION_START ); while ( step.processRow( meta, data ) ) { if ( step.isStopped() ) { break; e.printStackTrace(); } finally { step.setErrors( 1 ); step.stopAll(); step.dispose( meta, data ); step.getLogChannel().snap( Metrics.METRIC_STEP_EXECUTION_STOP ); try { long li = step.getLinesInput(); long lo = step.getLinesOutput(); long lr = step.getLinesRead(); long lw = step.getLinesWritten(); long lu = step.getLinesUpdated(); long lj = step.getLinesRejected(); long e = step.getErrors(); if ( li > 0 || lo > 0 || lr > 0 || lw > 0 || lu > 0 || lj > 0 || e > 0 ) { log.logBasic( BaseMessages.getString( PKG, "BaseStep.Log.SummaryInfo", String.valueOf( li ), step.markStop();
StepInterface step = e.getValue(); if ( step.getStatus() != StepExecutionStatus.STATUS_FINISHED ) { logDetailed( "Finished running step [" + step.getStepname() + "(" + step.getCopy() + ")]." ); rowData[index++] = step.getStepname(); rowData[index++] = step.getStepID(); rowData[index++] = step.getLinesInput(); rowData[index++] = step.getLinesOutput(); rowData[index++] = step.getLinesRead(); rowData[index++] = step.getLinesUpdated(); rowData[index++] = step.getLinesWritten(); rowData[index++] = step.getLinesRejected(); rowData[index++] = step.getRuntime();
StepInterface step = sid.step; result.setNrErrors( result.getNrErrors() + sid.step.getErrors() ); result.getResultFiles().putAll( step.getResultFiles() ); if ( step.isSafeStopped() ) { result.setSafeStop( step.isSafeStopped() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_READ ) ) ) { result.setNrLinesRead( result.getNrLinesRead() + step.getLinesRead() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_INPUT ) ) ) { result.setNrLinesInput( result.getNrLinesInput() + step.getLinesInput() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_WRITTEN ) ) ) { result.setNrLinesWritten( result.getNrLinesWritten() + step.getLinesWritten() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_OUTPUT ) ) ) { result.setNrLinesOutput( result.getNrLinesOutput() + step.getLinesOutput() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_UPDATED ) ) ) { result.setNrLinesUpdated( result.getNrLinesUpdated() + step.getLinesUpdated() ); if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_REJECTED ) ) ) { result.setNrLinesRejected( result.getNrLinesRejected() + step.getLinesRejected() );
value = new Long( combi.step.getTrans().getBatchId() ); break; case CHANNEL_ID: value = combi.step.getLogChannel().getLogChannelId(); break; case LOG_DATE: break; case TRANSNAME: value = combi.step.getTrans().getName(); break; case STEPNAME: break; case LINES_READ: value = new Long( combi.step.getLinesRead() ); break; case LINES_WRITTEN: value = new Long( combi.step.getLinesWritten() ); break; case LINES_UPDATED: value = new Long( combi.step.getLinesUpdated() ); break; case LINES_INPUT: value = new Long( combi.step.getLinesInput() ); break; case LINES_OUTPUT: value = new Long( combi.step.getLinesOutput() ); break;
new StepPerformanceSnapShot( seqNr, getBatchId(), new Date(), getName(), stepMeta.getName(), step.getCopy(), step.getLinesRead(), step.getLinesWritten(), step.getLinesInput(), step.getLinesOutput(), step .getLinesUpdated(), step.getLinesRejected(), step.getErrors() ); snapShot.diff( previous, step.rowsetInputSize(), step.rowsetOutputSize() ); snapShotList.add( snapShot );
if ( outputStep != null ) { rowCollector = new OutputCollectorRowListener( output, outClassK, outClassV, reporter, debug ); outputStep.addRowListener( rowCollector ); StepMetaInterface inputStepMeta = inputStep.getStepMeta().getStepMetaInterface();
step.initializeVariablesFrom( this ); step.setUsingThreadPriorityManagment( transMeta.isUsingThreadPriorityManagment() ); step.setRepository( repository ); step.setMetaStore( metaStore ); List<String> partitionIDs = stepMeta.getStepPartitioningMeta().getPartitionSchema().getPartitionIDs(); if ( partitionIDs != null && partitionIDs.size() > 0 ) { step.setPartitionID( partitionIDs.get( c ) ); // Pass the partition ID LogChannelInterface logChannel = combi.step.getLogChannel(); logChannel.setLogLevel( logLevel ); logChannel.setGatheringMetrics( log.isGatheringMetrics() ); StepMetaDataCombi combi = steps.get( s ); if ( combi.stepMeta.isDoingErrorHandling() ) { combi.step.identifyErrorOutput(); baseStep.setPartitioned( stepMeta.isPartitioned() ); baseStep.setRepartitioning( StepPartitioningMeta.PARTITIONING_METHOD_NONE ); baseStep.setRepartitioning( nextStepPartitioningMeta.getMethodType() ); StepPartitioningMeta targetStepPartitioningMeta = baseStep.getStepMeta().getTargetStepPartitioningMeta(); if ( targetStepPartitioningMeta != null ) { baseStep.setRepartitioning( targetStepPartitioningMeta.getMethodType() ); combi.step.dispose( combi.meta, combi.data );
@Test public void doesNotWriteRowOnTimeWhenStopped() throws KettleException, InterruptedException { TransMeta transMeta = new TransMeta( getClass().getResource( "safe-stop.ktr" ).getPath() ); Trans trans = new Trans( transMeta ); trans.prepareExecution( new String[] {} ); trans.getSteps().get( 1 ).step.addRowListener( new RowAdapter() { @Override public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException { trans.safeStop(); } } ); trans.startThreads(); trans.waitUntilFinished(); assertEquals( 1, trans.getSteps().get( 0 ).step.getLinesWritten() ); assertEquals( 1, trans.getSteps().get( 1 ).step.getLinesRead() ); } }
public static double getProcessCount( Context actualContext, Scriptable actualObject, Object[] ArgList, Function FunctionContext ) { if ( ArgList.length == 1 ) { try { Object scmO = actualObject.get( "_step_", actualObject ); StepInterface scm = (StepInterface) Context.jsToJava( scmO, StepInterface.class ); String strType = Context.toString( ArgList[0] ).toLowerCase(); if ( strType.equals( "i" ) ) { return scm.getLinesInput(); } else if ( strType.equals( "o" ) ) { return scm.getLinesOutput(); } else if ( strType.equals( "r" ) ) { return scm.getLinesRead(); } else if ( strType.equals( "u" ) ) { return scm.getLinesUpdated(); } else if ( strType.equals( "w" ) ) { return scm.getLinesWritten(); } else if ( strType.equals( "e" ) ) { return scm.getLinesRejected(); } else { return 0; } } catch ( Exception e ) { // throw Context.reportRuntimeError(e.toString()); return 0; } } else { throw Context.reportRuntimeError( "The function call getProcessCount requires 1 argument." ); } }
for ( int i = 0; i < stepInterfaces.size(); i++ ) { StepInterface look = stepInterfaces.get( i ); if ( look.getCopy() == copyNr ) { step = look; step.addRowListener( rowListener ); && step.isRunning() && !trans.isFinished() && !trans.isStopped() ) { step.removeRowListener( rowListener );
public Optional<Result> execute( List<RowMetaAndData> rows ) throws KettleException { if ( rows.isEmpty() || stopped ) { return Optional.empty(); } Trans subtrans = this.createSubtrans(); running.add( subtrans ); parentTrans.addActiveSubTransformation( subTransName, subtrans ); // Pass parameter values passParametersToTrans( subtrans, rows.get( 0 ) ); Result result = new Result(); result.setRows( rows ); subtrans.setPreviousResult( result ); subtrans.prepareExecution( this.parentTrans.getArguments() ); List<RowMetaAndData> rowMetaAndData = new ArrayList<>(); subtrans.getSteps().stream() .filter( c -> c.step.getStepname().equalsIgnoreCase( subStep ) ) .findFirst() .ifPresent( c -> c.step.addRowListener( new RowAdapter() { @Override public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) { rowMetaAndData.add( new RowMetaAndData( rowMeta, row ) ); } } ) ); subtrans.startThreads(); subtrans.waitUntilFinished(); updateStatuses( subtrans ); running.remove( subtrans ); Result subtransResult = subtrans.getResult(); subtransResult.setRows( rowMetaAndData ); return Optional.of( subtransResult ); }
if ( ( baseStep.isRunning() ) || baseStep.getStatus() != StepExecutionStatus.STATUS_EMPTY ) { StepStatus stepStatus = new StepStatus( baseStep ); transStatus.getStepStatusList().add( stepStatus ); for ( int i = 0; i < trans.nrSteps(); i++ ) { StepInterface step = trans.getRunThread( i ); if ( ( step.isRunning() ) || step.getStatus() != StepExecutionStatus.STATUS_EMPTY ) { StepStatus stepStatus = new StepStatus( step ); boolean snif = false; String htmlString = ""; if ( step.isRunning() && !step.isStopped() && !step.isPaused() ) { snif = true; String sniffLink = + convertContextPath( SniffStepServlet.CONTEXT_PATH ) + "?trans=" + URLEncoder.encode( transName, "UTF-8" ) + "&id=" + URLEncoder.encode( id, "UTF-8" ) + "&lines=50" + "©nr=" + step.getCopy() + "&type=" + SniffStepServlet.TYPE_OUTPUT + "&step=" + URLEncoder.encode( step.getStepname(), "UTF-8" ) + "\">" + Encode.forHtml( stepStatus.getStepname() ) + "</a>"; stepStatus.setStepname( sniffLink );
public void run() { // Set the internal variables also on the initialization thread! // ((BaseStep)combi.step).setInternalVariables(); if ( !doIt ) { // An extension point plugin decided we should not initialize the step. // Logging, error handling, finished flag... should all be handled in the extension point. // return; } try { combi.step.getLogChannel().snap( Metrics.METRIC_STEP_INIT_START ); if ( combi.step.init( combi.meta, combi.data ) ) { combi.data.setStatus( StepExecutionStatus.STATUS_IDLE ); ok = true; } else { combi.step.setErrors( 1 ); log.logError( BaseMessages.getString( PKG, "Trans.Log.ErrorInitializingStep", combi.step.getStepname() ) ); } } catch ( Throwable e ) { log.logError( BaseMessages.getString( PKG, "Trans.Log.ErrorInitializingStep", combi.step.getStepname() ) ); log.logError( Const.getStackTracker( e ) ); } finally { combi.step.getLogChannel().snap( Metrics.METRIC_STEP_INIT_STOP ); } finished = true; }
StepMetaDataCombi sid = steps.get( i ); StepInterface step = sid.step; long proc = step.getProcessed(); if ( seconds != 0 ) { if ( step.getErrors() == 0 ) { log.logBasic( BaseMessages.getString( PKG, "Trans.Log.ProcessSuccessfullyInfo", step.getStepname(), "." + step .getCopy(), String.valueOf( proc ), String.valueOf( ( proc / seconds ) ) ) ); } else { log.logError( BaseMessages.getString( PKG, "Trans.Log.ProcessErrorInfo", step.getStepname(), "." + step .getCopy(), String.valueOf( step.getErrors() ), String.valueOf( proc ), String.valueOf( proc / seconds ) ) ); if ( step.getErrors() == 0 ) { log.logBasic( BaseMessages.getString( PKG, "Trans.Log.ProcessSuccessfullyInfo", step.getStepname(), "." + step .getCopy(), String.valueOf( proc ), seconds != 0 ? String.valueOf( ( proc / seconds ) ) : "-" ) ); } else { log.logError( BaseMessages.getString( PKG, "Trans.Log.ProcessErrorInfo2", step.getStepname(), "." + step .getCopy(), String.valueOf( step.getErrors() ), String.valueOf( proc ), String.valueOf( seconds ) ) );
while ( once || ( rowSet.size() > 0 && !stepDone ) ) { once = false; stepDone = !combi.step.processRow( combi.meta, combi.data ); if ( combi.step.getErrors() > 0 ) { return false; List<RowSet> rowSets = combi.step.getInputRowSets(); stepDone = !combi.step.processRow( combi.meta, combi.data ); if ( combi.step.getErrors() > 0 ) { return false; stepDone = !combi.step.processRow( combi.meta, combi.data ); if ( combi.step.getErrors() > 0 ) { return false; combi.step.batchComplete();