/** * * @param errorMsg * Message to send to rejected row if enabled * @return If should stop processing after having problems with a file */ public boolean failAfterBadFile( String errorMsg ) { if ( getStepMeta().isDoingErrorHandling() && data.filename != null && !data.rejectedFiles.containsKey( data.filename ) ) { data.rejectedFiles.put( data.filename, true ); rejectCurrentFile( errorMsg ); } return !meta.errorHandling.errorIgnored || !meta.errorHandling.skipBadFiles; }
public boolean isSendingErrorRowsToStep( StepMeta targetStep ) { return ( isDoingErrorHandling() && stepErrorMeta.getTargetStep().equals( targetStep ) ); }
@Override public void identifyErrorOutput() { if ( stepMeta.isDoingErrorHandling() ) { StepErrorMeta stepErrorMeta = stepMeta.getStepErrorMeta(); outputRowSetsLock.writeLock().lock(); try { for ( int rowsetNr = 0; rowsetNr < outputRowSets.size(); rowsetNr++ ) { RowSet outputRowSet = outputRowSets.get( rowsetNr ); if ( outputRowSet.getDestinationStepName().equalsIgnoreCase( stepErrorMeta.getTargetStep().getName() ) ) { // This is the rowset to move! // errorRowSet = outputRowSet; outputRowSets.remove( rowsetNr ); return; } } } finally { outputRowSetsLock.writeLock().unlock(); } } }
/** * * @param errorMsg * Message to send to rejected row if enabled * @return If should stop processing after having problems with a file */ private boolean failAfterBadFile( String errorMsg ) { if ( getStepMeta().isDoingErrorHandling() && data.filename != null && !data.rejectedFiles.containsKey( data.filename ) ) { data.rejectedFiles.put( data.filename, true ); rejectCurrentFile( errorMsg ); } return !meta.isErrorIgnored() || !meta.isSkipBadFiles() || data.isLastFile; }
private void inputError( String errorMsg ) { if ( getStepMeta().isDoingErrorHandling() ) { sendErrorRow( errorMsg ); } else { incrementErrors(); } }
public void setFlags() { wlIgnoreFlagField.setEnabled( wErrorIgnored.getSelection() ); wIgnoreFlagField.setEnabled( wErrorIgnored.getSelection() ); DatabaseMeta databaseMeta = transMeta.findDatabase( wConnection.getText() ); boolean hasErrorHandling = transMeta.findStep( stepname ).isDoingErrorHandling(); // Can't use batch yet when grabbing auto-generated keys... // Only enable batch option when not returning keys. // If we are on PostgreSQL (and look-a-likes), error handling is not supported. (PDI-366) // boolean enableBatch = wBatch.getSelection() && !transMeta.isUsingUniqueConnections(); enableBatch = enableBatch && !( databaseMeta != null && databaseMeta.supportsErrorHandlingOnBatchUpdates() && hasErrorHandling ); }
@Before public void setUp() throws Exception { current = Locale.getDefault(); Locale.setDefault( Locale.UK ); helper = StepMockUtil.getStepMockHelper( SelectValuesMeta.class, "SelectValues_LocaleHandling_Test" ); when( helper.stepMeta.isDoingErrorHandling() ).thenReturn( true ); step = new SelectValues( helper.stepMeta, helper.stepDataInterface, 1, helper.transMeta, helper.trans ); step = spy( step ); // Dec 28, 2015 Calendar calendar = Calendar.getInstance(); calendar.set( 2015, Calendar.DECEMBER, 28, 0, 0, 0 ); doReturn( new Object[] { calendar.getTime() } ).doReturn( null ) .when( step ).getRow(); }
@Before public void setUp() throws Exception { helper = StepMockUtil.getStepMockHelper( SelectValuesMeta.class, "SelectValuesTest" ); when( helper.stepMeta.isDoingErrorHandling() ).thenReturn( true ); step = new SelectValues( helper.stepMeta, helper.stepDataInterface, 1, helper.transMeta, helper.trans ); step = spy( step ); doReturn( inputRow ).when( step ).getRow(); doNothing().when( step ) .putError( any( RowMetaInterface.class ), any( Object[].class ), anyLong(), anyString(), anyString(), anyString() ); RowMeta inputRowMeta = new RowMeta(); inputRowMeta.addValueMeta( new ValueMetaString( SELECTED_FIELD ) ); step.setInputRowMeta( inputRowMeta ); }
singleThreaderData.batchCount++; if ( getStepMeta().isDoingErrorHandling() ) { singleThreaderData.errorBuffer.add( row );
@Test public void testSetConnectorSteps() throws Exception { when( stepMockHelper.transMeta.getSizeRowset() ).thenReturn( 1 ); MappingInputData mappingInputData = new MappingInputData(); MappingInput mappingInput = new MappingInput( stepMockHelper.stepMeta, mappingInputData, 0, stepMockHelper.transMeta, stepMockHelper.trans ); mappingInput.init( stepMockHelper.initStepMetaInterface, mappingInputData ); ValidatorData validatorData = new ValidatorData(); Validator previousStep = new Validator( stepMockHelper.stepMeta, validatorData, 0, stepMockHelper.transMeta, stepMockHelper.trans ); when( stepMockHelper.stepMeta.isDoingErrorHandling() ).thenReturn( true ); StepErrorMeta stepErrorMeta = mock( StepErrorMeta.class ); when( stepErrorMeta.getTargetStep() ).thenReturn( stepMockHelper.stepMeta ); when( stepMockHelper.stepMeta.getName() ).thenReturn( stepName ); when( stepMockHelper.stepMeta.getStepErrorMeta() ).thenReturn( stepErrorMeta ); StepInterface[] si = new StepInterface[] { previousStep }; mappingInput.setConnectorSteps( si, Collections.<MappingValueRename>emptyList(), stepName ); assertEquals( previousStep.getOutputRowSets().size(), 0 ); }
@Test public void testHandleOpenFileException() throws Exception { final String content = new StringBuilder() .append( "123" ).append( '\n' ).append( "333\n" ).toString(); final String virtualFile = createVirtualFile( "pdi-16697.txt", content ); TextFileInputMeta meta = createMetaObject( field( "col1" ) ); meta.inputFields[ 0 ].setType( 1 ); meta.errorHandling.errorIgnored = true; meta.errorHandling.skipBadFiles = true; TextFileInputData data = createDataObject( virtualFile, ";", "col1" ); data.dataErrorLineHandler = Mockito.mock( FileErrorHandler.class ); TestTextFileInput textFileInput = StepMockUtil.getStep( TestTextFileInput.class, TextFileInputMeta.class, "test" ); StepMeta stepMeta = textFileInput.getStepMeta(); Mockito.doReturn( true ).when( stepMeta ).isDoingErrorHandling(); List<Object[]> output = TransTestingUtil.execute( textFileInput, meta, data, 0, false ); deleteVfsFile( virtualFile ); assertEquals( 1, data.rejectedFiles.size() ); assertEquals( 0, textFileInput.getErrors() ); }
private boolean execOneIteration() { boolean more = false; try { more = getData().executor.oneIteration(); if ( getData().executor.isStopped() || getData().executor.getErrors() > 0 ) { return handleError(); } } catch ( Exception e ) { setErrors( 1L ); stopAll(); logError( BaseMessages.getString( PKG, "SingleThreader.Log.ErrorOccurredInSubTransformation" ) ); return false; } finally { if ( getStepMeta().isDoingErrorHandling() ) { getData().errorBuffer.clear(); } } return more; }
private boolean handleError() throws KettleStepException { SingleThreaderData singleThreaderData = getData(); if ( getStepMeta().isDoingErrorHandling() ) { int lastLogLine = KettleLogStore.getLastBufferLineNr(); StringBuffer logText = KettleLogStore.getAppender().getBuffer( singleThreaderData.mappingTrans.getLogChannelId(), false, singleThreaderData.lastLogLine ); singleThreaderData.lastLogLine = lastLogLine; for ( Object[] row : singleThreaderData.errorBuffer ) { putError( getInputRowMeta(), row, 1L, logText.toString(), null, "STR-001" ); } singleThreaderData.executor.clearError(); return true; // continue } else { setErrors( 1 ); stopAll(); logError( BaseMessages.getString( PKG, "SingleThreader.Log.ErrorOccurredInSubTransformation" ) ); return false; // stop running } }
if ( !getStepMeta().isDoingErrorHandling() ) { stopErrorExecution( e ); return false; if ( getStepMeta().isDoingErrorHandling() ) { sendErrorRow( e.toString() ); } else {
private void ok() { if ( Utils.isEmpty( wStepname.getText() ) ) { return; } stepname = wStepname.getText(); // return value getInfo( input ); if ( input.getDatabaseMeta() == null ) { MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_ERROR ); mb.setMessage( BaseMessages.getString( PKG, "TableOutputDialog.ConnectionError.DialogMessage" ) ); mb.setText( BaseMessages.getString( PKG, "System.Dialog.Error.Title" ) ); mb.open(); return; } // PDI-6211 : Show a warning in case batch processing is not truly supported in combination // with error handling... // Show it *every* time the user OK's the dialog. // DatabaseMeta databaseMeta = input.getDatabaseMeta(); boolean supportsBatchErrorHandling = databaseMeta != null && databaseMeta.supportsErrorHandlingOnBatchUpdates(); boolean hasErrorHandling = stepMeta.isDoingErrorHandling(); if ( !supportsBatchErrorHandling && hasErrorHandling ) { MessageBox mb = new MessageBox( shell, SWT.OK | SWT.ICON_WARNING ); mb.setMessage( BaseMessages.getString( PKG, "TableOutput.Warning.ErrorHandlingIsNotFullySupportedWithBatchProcessing" ) ); mb.setText( BaseMessages.getString( PKG, "TableOutput.Warning" ) ); mb.open(); } dispose(); }
if ( getStepMeta().isDoingErrorHandling() ) { sendToErrorRow = true; errorMessage = e.toString();
if ( getStepMeta().isDoingErrorHandling() ) { sendToErrorRow = true; errorMessage = e.toString();
if ( getStepMeta().isDoingErrorHandling() ) { sendToErrorRow = true; errorMessage = e.toString();
if ( getStepMeta().isDoingErrorHandling() ) { rejectRow( item.getItemId(), item.getFailureMessage() );
if ( transHopMeta.getFromStep().isDoingErrorHandling() ) { StepErrorMeta stepErrorMeta = fromStepMeta.getStepErrorMeta();