private void putRowToRowSet( RowSet rs, RowMetaInterface rowMeta, Object[] row ) { RowMetaInterface toBeSent; RowMetaInterface metaFromRs = rs.getRowMeta(); if ( metaFromRs == null ) { // RowSet is not initialised so far toBeSent = rowMeta.clone(); } else { // use the existing toBeSent = metaFromRs; } while ( !rs.putRow( toBeSent, row ) ) { if ( isStopped() && !safeStopped.get() ) { return; } } }
private void assertRowSetMatches( String msg, Object[] expectedRow, RowSet outputRowSet ) { Object[] actualRow = outputRowSet.getRow(); Assert.assertEquals( msg + ". Output row is of an unexpected length", expectedRow.length, outputRowSet.getRowMeta() .size() ); for ( int i = 0; i < expectedRow.length; i++ ) { Assert.assertEquals( msg + ". Unexpected output value at index " + i, expectedRow[i], actualRow[i] ); } } }
private StepMockHelper<DatabaseLookupMeta, DatabaseLookupData> createMockHelper() { StepMockHelper<DatabaseLookupMeta, DatabaseLookupData> mockHelper = new StepMockHelper<DatabaseLookupMeta, DatabaseLookupData>( "test DatabaseLookup", DatabaseLookupMeta.class, DatabaseLookupData.class ); when( mockHelper.logChannelInterfaceFactory.create( any(), any( LoggingObjectInterface.class ) ) ) .thenReturn( mockHelper.logChannelInterface ); when( mockHelper.trans.isRunning() ).thenReturn( true ); RowMeta inputRowMeta = new RowMeta(); RowSet rowSet = mock( RowSet.class ); when( rowSet.getRowWait( anyLong(), any( TimeUnit.class ) ) ).thenReturn( new Object[ 0 ] ).thenReturn( null ); when( rowSet.getRowMeta() ).thenReturn( inputRowMeta ); when( mockHelper.trans.findRowSet( anyString(), anyInt(), anyString(), anyInt() ) ).thenReturn( rowSet ); when( mockHelper.transMeta.findNextSteps( Matchers.any( StepMeta.class ) ) ) .thenReturn( Collections.singletonList( mock( StepMeta.class ) ) ); when( mockHelper.transMeta.findPreviousSteps( any( StepMeta.class ), anyBoolean() ) ) .thenReturn( Collections.singletonList( mock( StepMeta.class ) ) ); return mockHelper; }
private boolean verifyOutput( Object[][] expectedRows, RowSet outputRowSet ) throws KettleValueException { if ( expectedRows.length == outputRowSet.size() ) { for ( Object[] expectedRow : expectedRows ) { Object[] row = outputRowSet.getRow(); if ( expectedRow.length == outputRowSet.getRowMeta().size() ) { for ( int j = 0; j < expectedRow.length; j++ ) { if ( !expectedRow[j].equals( outputRowSet.getRowMeta().getString( row, j ) ) ) { return false; } } return true; } } } return false; }
@Test public void testDummyTransDoesntWriteOutputWithoutInputRow() throws KettleException { DummyTrans dummy = new DummyTrans( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans ); dummy.init( stepMockHelper.initStepMetaInterface, stepMockHelper.initStepDataInterface ); RowSet rowSet = stepMockHelper.getMockInputRowSet(); RowMetaInterface inputRowMeta = mock( RowMetaInterface.class ); when( rowSet.getRowMeta() ).thenReturn( inputRowMeta ); dummy.addRowSetToInputRowSets( rowSet ); RowSet outputRowSet = mock( RowSet.class ); dummy.addRowSetToOutputRowSets( outputRowSet ); dummy.processRow( stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface ); verify( inputRowMeta, never() ).cloneRow( any( Object[].class ) ); verify( outputRowSet, never() ).putRow( any( RowMetaInterface.class ), any( Object[].class ) ); }
private RowSet mockInputRowSet() { ValueMetaString valueMeta = new ValueMetaString( "Value" ); valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING ); valueMeta.setStorageMetadata( new ValueMetaString( "Value" ) ); RowMeta inputRowMeta = new RowMeta(); inputRowMeta.addValueMeta( valueMeta ); RowSet inputRowSet = smh.getMockInputRowSet( new Object[][] { { " Value ".getBytes() } } ); doReturn( inputRowMeta ).when( inputRowSet ).getRowMeta(); return inputRowSet; }
@Test public void testOutputIsMuchBiggerThanInputDoesntThrowArrayIndexOutOfBounds() throws KettleException { RegexEval regexEval = new RegexEval( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans ); when( stepMockHelper.processRowsStepMetaInterface.isAllowCaptureGroupsFlagSet() ).thenReturn( true ); String[] outFields = new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k" }; when( stepMockHelper.processRowsStepMetaInterface.getFieldName() ).thenReturn( outFields ); when( stepMockHelper.processRowsStepMetaInterface.getMatcher() ).thenReturn( "\\.+" ); stepMockHelper.processRowsStepDataInterface.pattern = Pattern.compile( "(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)" ); Object[] inputRow = new Object[] {}; RowSet inputRowSet = stepMockHelper.getMockInputRowSet( inputRow ); RowMetaInterface mockInputRowMeta = mock( RowMetaInterface.class ); RowMetaInterface mockOutputRoMeta = mock( RowMetaInterface.class ); when( mockOutputRoMeta.size() ).thenReturn( outFields.length ); when( mockInputRowMeta.size() ).thenReturn( 0 ); when( inputRowSet.getRowMeta() ).thenReturn( mockInputRowMeta ); when( mockInputRowMeta.clone() ).thenReturn( mockOutputRoMeta ); when( mockInputRowMeta.isNull( any( Object[].class ), anyInt() ) ).thenReturn( true ); regexEval.addRowSetToInputRowSets( inputRowSet ); regexEval.init( stepMockHelper.initStepMetaInterface, stepMockHelper.initStepDataInterface ); regexEval .processRow( stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface ); } }
@Test public void testDummyTransWritesOutputWithInputRow() throws KettleException { DummyTrans dummy = new DummyTrans( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans ); dummy.init( stepMockHelper.initStepMetaInterface, stepMockHelper.initStepDataInterface ); Object[] row = new Object[] { "abcd" }; RowSet rowSet = stepMockHelper.getMockInputRowSet( row ); RowMetaInterface inputRowMeta = mock( RowMetaInterface.class ); when( inputRowMeta.clone() ).thenReturn( inputRowMeta ); when( rowSet.getRowMeta() ).thenReturn( inputRowMeta ); dummy.addRowSetToInputRowSets( rowSet ); RowSet outputRowSet = mock( RowSet.class ); dummy.addRowSetToOutputRowSets( outputRowSet ); when( outputRowSet.putRow( inputRowMeta, row ) ).thenReturn( true ); dummy.processRow( stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface ); verify( outputRowSet, times( 1 ) ).putRow( inputRowMeta, row ); } }
Mockito.doReturn( null ).when( input ).getRowFrom( rowset ); Mockito.when( input.getRowFrom( rowset ) ).thenReturn( obj1, obj2, null ); Mockito.doReturn( rwi ).when( rowset ).getRowMeta(); Mockito.when( rwi.getString( obj2, 0 ) ).thenReturn( "filename1", "filename2" ); List<Object[]> output = TransTestingUtil.execute( input, meta, data, 0, false );
/** * Create ExcelWriterStep object and mock some of its required data * * @param fileType * @throws Exception */ private void setupStepMock( String fileType ) throws Exception { step = new ExcelWriterStep( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans ); step.init( stepMockHelper.initStepMetaInterface, stepMockHelper.initStepDataInterface ); List<Object[]> rows = createRowData(); String[] outFields = new String[] { "col 1", "col 2", "col 3", "col 4" }; RowSet inputRowSet = stepMockHelper.getMockInputRowSet( rows ); RowMetaInterface inputRowMeta = createRowMeta(); inputRowSet.setRowMeta( inputRowMeta ); RowMetaInterface mockOutputRowMeta = mock( RowMetaInterface.class ); when( mockOutputRowMeta.size() ).thenReturn( outFields.length ); when( inputRowSet.getRowMeta() ).thenReturn( inputRowMeta ); step.addRowSetToInputRowSets( inputRowSet ); step.setInputRowMeta( inputRowMeta ); step.addRowSetToOutputRowSets( inputRowSet ); }
Object[] expectedRow = new Object[] { null, null, d3, d4 }; Assert.assertEquals( "Output row is of an unexpected length", expectedRow.length, outputRowSet.getRowMeta().size() );
private RowSet mockLookupRowSet( boolean binary ) { final int storageType = binary ? ValueMetaInterface.STORAGE_TYPE_BINARY_STRING : ValueMetaInterface.STORAGE_TYPE_NORMAL; Object[][] data = { { "Value1", "1" }, { "Value2", "2" } }; if ( binary ) { convertDataToBinary( data ); } RowSet lookupRowSet = smh.getMockInputRowSet( data ); doReturn( "Lookup" ).when( lookupRowSet ).getOriginStepName(); doReturn( "StreamLookup" ).when( lookupRowSet ).getDestinationStepName(); RowMeta lookupRowMeta = new RowMeta(); ValueMetaString valueMeta = new ValueMetaString( "Value" ); valueMeta.setStorageType( storageType ); valueMeta.setStorageMetadata( new ValueMetaString() ); lookupRowMeta.addValueMeta( valueMeta ); ValueMetaString idMeta = new ValueMetaString( "Id" ); idMeta.setStorageType( storageType ); idMeta.setStorageMetadata( new ValueMetaString() ); lookupRowMeta.addValueMeta( idMeta ); doReturn( lookupRowMeta ).when( lookupRowSet ).getRowMeta(); return lookupRowSet; }
@Test public void outputRowMetasAreNotSharedAmongSeveralStreams() throws Exception { RowSet rs1 = new SingleRowRowSet(); RowSet rs2 = new SingleRowRowSet(); when( mockHelper.trans.isRunning() ).thenReturn( true ); BaseStep baseStep = new BaseStep( mockHelper.stepMeta, mockHelper.stepDataInterface, 0, mockHelper.transMeta, mockHelper.trans ); baseStep.setStopped( false ); baseStep.setRepartitioning( StepPartitioningMeta.PARTITIONING_METHOD_NONE ); baseStep.setOutputRowSets( Arrays.asList( rs1, rs2 ) ); for ( RowSet rowSet : baseStep.getOutputRowSets() ) { assertNull( "RowMeta should be null, since no calls were done", rowSet.getRowMeta() ); } RowMetaInterface rowMeta = new RowMeta(); rowMeta.addValueMeta( new ValueMetaString( "string" ) ); rowMeta.addValueMeta( new ValueMetaInteger( "integer" ) ); baseStep.putRow( rowMeta, new Object[] { "a", 1 } ); RowMetaInterface meta1 = rs1.getRowMeta(); RowMetaInterface meta2 = rs2.getRowMeta(); assertNotNull( meta1 ); assertNotNull( meta2 ); // content is same for ( ValueMetaInterface meta : meta1.getValueMetaList() ) { assertTrue( meta.getName(), meta2.exists( meta ) ); } // whereas instances differ assertFalse( meta1 == meta2 ); }
@Test public void test() throws KettleException { KettleEnvironment.init(); NullIf step = new NullIf( smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans ); step.init( smh.initStepMetaInterface, smh.stepDataInterface ); step.setInputRowMeta( getInputRowMeta() ); step.addRowSetToInputRowSets( mockInputRowSet() ); step.addRowSetToOutputRowSets( new QueueRowSet() ); boolean hasMoreRows; do { hasMoreRows = step.processRow( mockProcessRowMeta(), smh.processRowsStepDataInterface ); } while ( hasMoreRows ); RowSet outputRowSet = step.getOutputRowSets().get( 0 ); Object[] actualRow = outputRowSet.getRow(); Object[] expectedRow = new Object[] { "value1", null, "value3" }; Assert.assertEquals( "Output row is of an unexpected length", expectedRow.length, outputRowSet.getRowMeta().size() ); for ( int i = 0; i < expectedRow.length; i++ ) { Assert.assertEquals( "Unexpected output value at index " + i, expectedRow[i], actualRow[i] ); } }
private RowSet mockDataRowSet( boolean binary ) { final int storageType = binary ? ValueMetaInterface.STORAGE_TYPE_BINARY_STRING : ValueMetaInterface.STORAGE_TYPE_NORMAL; Object[][] data = { { "Name1", "1" }, { "Name2", "2" } }; if ( binary ) { convertDataToBinary( data ); } RowSet dataRowSet = smh.getMockInputRowSet( data ); RowMeta dataRowMeta = new RowMeta(); ValueMetaString valueMeta = new ValueMetaString( "Name" ); valueMeta.setStorageType( storageType ); valueMeta.setStorageMetadata( new ValueMetaString() ); dataRowMeta.addValueMeta( valueMeta ); ValueMetaString idMeta = new ValueMetaString( "Id" ); idMeta.setStorageType( storageType ); idMeta.setStorageMetadata( new ValueMetaString() ); dataRowMeta.addValueMeta( idMeta ); doReturn( dataRowMeta ).when( dataRowSet ).getRowMeta(); return dataRowSet; }
valueMeta.setStorageType( ValueMetaInterface.TYPE_STRING ); rowMetaInterface.addValueMeta( valueMeta ); when( lookupRowSet.getRowMeta() ).thenReturn( rowMetaInterface ); fuzzyMatch.setInputRowMeta( rowMetaInterface.clone() ); data.outputRowMeta = rowMetaInterface.clone();
private void doTest( boolean memoryPreservationActive, boolean binaryLookupStream, boolean binaryDataStream ) throws KettleException { StreamLookup step = new StreamLookup( smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans ); step.init( smh.initStepMetaInterface, smh.initStepDataInterface ); step.addRowSetToInputRowSets( mockLookupRowSet( binaryLookupStream ) ); step.addRowSetToInputRowSets( mockDataRowSet( binaryDataStream ) ); step.addRowSetToOutputRowSets( new QueueRowSet() ); StreamLookupMeta meta = mockProcessRowMeta( memoryPreservationActive ); StreamLookupData data = new StreamLookupData(); data.readLookupValues = true; RowSet outputRowSet = step.getOutputRowSets().get( 0 ); // Process rows and collect output int rowNumber = 0; String[] expectedOutput = { "Name", "", "Value" }; while ( step.processRow( meta, data ) ) { Object[] rowData = outputRowSet.getRow(); if ( rowData != null ) { RowMetaInterface rowMeta = outputRowSet.getRowMeta(); Assert.assertEquals( "Output row is of wrong size", 3, rowMeta.size() ); rowNumber++; // Verify output for ( int valueIndex = 0; valueIndex < rowMeta.size(); valueIndex++ ) { String expectedValue = expectedOutput[valueIndex] + rowNumber; Object actualValue = rowMeta.getValueMeta( valueIndex ).convertToNormalStorageType( rowData[valueIndex] ); Assert.assertEquals( "Unexpected value at row " + rowNumber + " position " + valueIndex, expectedValue, actualValue ); } } } Assert.assertEquals( "Incorrect output row number", 2, rowNumber ); }
private RowMetaAndData readStartDate() throws KettleException { if ( log.isDetailed() ) { logDetailed( "Reading from step [" + data.infoStream.getStepname() + "]" ); } RowMetaInterface parametersMeta = new RowMeta(); Object[] parametersData = new Object[] {}; RowSet rowSet = findInputRowSet( data.infoStream.getStepname() ); if ( rowSet != null ) { Object[] rowData = getRowFrom( rowSet ); // rows are originating from "lookup_from" while ( rowData != null ) { parametersData = RowDataUtil.addRowData( parametersData, parametersMeta.size(), rowData ); parametersMeta.addRowMeta( rowSet.getRowMeta() ); rowData = getRowFrom( rowSet ); // take all input rows if needed! } if ( parametersMeta.size() == 0 ) { throw new KettleException( "Expected to read parameters from step [" + data.infoStream.getStepname() + "] but none were found." ); } } else { throw new KettleException( "Unable to find rowset to read from, perhaps step [" + data.infoStream.getStepname() + "] doesn't exist. (or perhaps you are trying a preview?)" ); } RowMetaAndData parameters = new RowMetaAndData( parametersMeta, parametersData ); return parameters; }
@Test public void testSplitFields() throws KettleException { FieldSplitter step = new FieldSplitter( smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans ); step.init( smh.initStepMetaInterface, smh.stepDataInterface ); step.setInputRowMeta( getInputRowMeta() ); step.addRowSetToInputRowSets( mockInputRowSet() ); step.addRowSetToOutputRowSets( new QueueRowSet() ); boolean hasMoreRows; do { hasMoreRows = step.processRow( mockProcessRowMeta(), smh.processRowsStepDataInterface ); } while ( hasMoreRows ); RowSet outputRowSet = step.getOutputRowSets().get( 0 ); Object[] actualRow = outputRowSet.getRow(); Object[] expectedRow = new Object[] { "before", null, "b=b", "after" }; assertEquals( "Output row is of an unexpected length", expectedRow.length, outputRowSet.getRowMeta().size() ); for ( int i = 0; i < expectedRow.length; i++ ) { assertEquals( "Unexpected output value at index " + i, expectedRow[i], actualRow[i] ); } }
RowMetaInterface rm = rs.getRowMeta();