private void assertRowSetMatches( String msg, Object[] expectedRow, RowSet outputRowSet ) { Object[] actualRow = outputRowSet.getRow(); Assert.assertEquals( msg + ". Output row is of an unexpected length", expectedRow.length, outputRowSet.getRowMeta() .size() ); for ( int i = 0; i < expectedRow.length; i++ ) { Assert.assertEquals( msg + ". Unexpected output value at index " + i, expectedRow[i], actualRow[i] ); } } }
private void putRowToRowSet( RowSet rs, RowMetaInterface rowMeta, Object[] row ) { RowMetaInterface toBeSent; RowMetaInterface metaFromRs = rs.getRowMeta(); if ( metaFromRs == null ) { // RowSet is not initialised so far toBeSent = rowMeta.clone(); } else { // use the existing toBeSent = metaFromRs; } while ( !rs.putRow( toBeSent, row ) ) { if ( isStopped() && !safeStopped.get() ) { return; } } }
/** * Finds the RowSet between two steps (or copies of steps). * * @param from the name of the "from" step * @param fromcopy the copy number of the "from" step * @param to the name of the "to" step * @param tocopy the copy number of the "to" step * @return the row set, or null if none found */ public RowSet findRowSet( String from, int fromcopy, String to, int tocopy ) { // Start with the transformation. for ( int i = 0; i < rowsets.size(); i++ ) { RowSet rs = rowsets.get( i ); if ( rs.getOriginStepName().equalsIgnoreCase( from ) && rs.getDestinationStepName().equalsIgnoreCase( to ) && rs .getOriginStepCopy() == fromcopy && rs.getDestinationStepCopy() == tocopy ) { return rs; } } return null; }
/** * Puts a row on to the underlying row set, optionally blocking until the row can be successfully put. * * @return true if the row was successfully added to the rowset and false if this buffer was full. If {@code block} is * true this will always return true. * @see RowSet#putRow(RowMetaInterface, Object[]) */ public boolean putRow( RowMetaInterface rowMeta, Object[] row, boolean block ) { if ( block ) { boolean added = false; while ( !added ) { added = rowSet.putRowWait( rowMeta, row, Long.MAX_VALUE, TimeUnit.DAYS ); } return true; } return rowSet.putRow( rowMeta, row ); }
} else { if ( data.outputRowMeta == null ) { data.outputRowMeta = data.headRowSet.getRowMeta(); data.outputRowMeta = data.tailRowSet.getRowMeta(); checkInputLayoutValid( data.headRowSet.getRowMeta(), data.tailRowSet.getRowMeta() ); } catch ( KettleRowException e ) { throw new KettleException( BaseMessages.getString( PKG, "Append.Exception.InvalidLayoutDetected" ), e );
Mockito.when( rowSet.getOriginStepName() ).thenReturn( name ); Mockito.when( rowSet.getDestinationStepName() ).thenReturn( "Validator" ); Mockito.when( rowSet.getOriginStepCopy() ).thenReturn( 0 ); Mockito.when( rowSet.getDestinationStepCopy() ).thenReturn( 0 ); Mockito.when( rowSet.getRow() ).thenReturn( new String[] { values } ).thenReturn( null ); Mockito.when( rowSet.isDone() ).thenReturn( true ); RowMetaInterface allowedRowMeta = Mockito.mock( RowMetaInterface.class ); Mockito.when( rowSet.getRowMeta() ).thenReturn( allowedRowMeta ); Mockito.when( rowSet.getRowMeta() ).thenReturn( Mockito.mock( RowMetaInterface.class ) ); Mockito.when( allowedRowMeta.indexOfValue( field ) ).thenReturn( 0 ); Mockito.when( allowedRowMeta.getValueMeta( 0 ) ).thenReturn( Mockito.mock( ValueMetaInterface.class ) );
&& !rowSet.isDone() && rowSet.size() <= lowerBufferBoundary && !isStopped() ) { try { Thread.sleep( 0, 1 ); rowData = rowSet.getRow(); while ( rowData == null && !rowSet.isDone() && !stopped.get() ) { rowData = rowSet.getRow(); if ( rowData == null && rowSet.isDone() ) { rowData = rowSet.getRow(); if ( rowData == null && rowSet.isDone() ) { rowData = rowSet.getRow(); if ( rowData == null ) { listener.rowReadEvent( rowSet.getRowMeta(), rowData );
nextInputStream(); inputRowSet = currentInputStream(); row = inputRowSet.getRowImmediate(); && !inputRowSet.isDone() && inputRowSet.size() <= lowerBufferBoundary && !isStopped() ) { try { Thread.sleep( 0, 1 ); row = inputRowSet.getRowWait( 1, TimeUnit.MILLISECONDS ); if ( row != null ) { incrementLinesRead(); if ( inputRowSet.isDone() ) { row = inputRowSet.getRowWait( 1, TimeUnit.MILLISECONDS ); if ( row == null ) { inputRowMeta = inputRowSet.getRowMeta();
private boolean verifyOutput( Object[][] expectedRows, RowSet outputRowSet ) throws KettleValueException { if ( expectedRows.length == outputRowSet.size() ) { for ( Object[] expectedRow : expectedRows ) { Object[] row = outputRowSet.getRow(); if ( expectedRow.length == outputRowSet.getRowMeta().size() ) { for ( int j = 0; j < expectedRow.length; j++ ) { if ( !expectedRow[j].equals( outputRowSet.getRowMeta().getString( row, j ) ) ) { return false; } } return true; } } } return false; }
private RowSet mockLookupRowSet( boolean binary ) { final int storageType = binary ? ValueMetaInterface.STORAGE_TYPE_BINARY_STRING : ValueMetaInterface.STORAGE_TYPE_NORMAL; Object[][] data = { { "Value1", "1" }, { "Value2", "2" } }; if ( binary ) { convertDataToBinary( data ); } RowSet lookupRowSet = smh.getMockInputRowSet( data ); doReturn( "Lookup" ).when( lookupRowSet ).getOriginStepName(); doReturn( "StreamLookup" ).when( lookupRowSet ).getDestinationStepName(); RowMeta lookupRowMeta = new RowMeta(); ValueMetaString valueMeta = new ValueMetaString( "Value" ); valueMeta.setStorageType( storageType ); valueMeta.setStorageMetadata( new ValueMetaString() ); lookupRowMeta.addValueMeta( valueMeta ); ValueMetaString idMeta = new ValueMetaString( "Id" ); idMeta.setStorageType( storageType ); idMeta.setStorageMetadata( new ValueMetaString() ); lookupRowMeta.addValueMeta( idMeta ); doReturn( lookupRowMeta ).when( lookupRowSet ).getRowMeta(); return lookupRowSet; }
private StepMockHelper<DatabaseLookupMeta, DatabaseLookupData> createMockHelper() { StepMockHelper<DatabaseLookupMeta, DatabaseLookupData> mockHelper = new StepMockHelper<DatabaseLookupMeta, DatabaseLookupData>( "test DatabaseLookup", DatabaseLookupMeta.class, DatabaseLookupData.class ); when( mockHelper.logChannelInterfaceFactory.create( any(), any( LoggingObjectInterface.class ) ) ) .thenReturn( mockHelper.logChannelInterface ); when( mockHelper.trans.isRunning() ).thenReturn( true ); RowMeta inputRowMeta = new RowMeta(); RowSet rowSet = mock( RowSet.class ); when( rowSet.getRowWait( anyLong(), any( TimeUnit.class ) ) ).thenReturn( new Object[ 0 ] ).thenReturn( null ); when( rowSet.getRowMeta() ).thenReturn( inputRowMeta ); when( mockHelper.trans.findRowSet( anyString(), anyInt(), anyString(), anyInt() ) ).thenReturn( rowSet ); when( mockHelper.transMeta.findNextSteps( Matchers.any( StepMeta.class ) ) ) .thenReturn( Collections.singletonList( mock( StepMeta.class ) ) ); when( mockHelper.transMeta.findPreviousSteps( any( StepMeta.class ), anyBoolean() ) ) .thenReturn( Collections.singletonList( mock( StepMeta.class ) ) ); return mockHelper; }
data.fileRowMeta[data.filenr] = rowSet.getRowMeta().clone(); PKG, "JoinRows.Log.RowsFound", meta.getCacheSize() + "", data.rs[data.filenr].getOriginStepName() ) );
/** * Compares using the target steps and copy, not the source. * That way, re-partitioning is always done in the same way. */ @Override public int compareTo( RowSet rowSet ) { lock.readLock().lock(); String target; try { target = remoteSlaveServerName + "." + destinationStepName + "." + destinationStepCopy.intValue(); } finally { lock.readLock().unlock(); } String comp = rowSet.getRemoteSlaveServerName() + "." + rowSet.getDestinationStepName() + "." + rowSet.getDestinationStepCopy(); return target.compareTo( comp ); }
public RowSet getMockInputRowSet( final List<Object[]> rows ) { final AtomicInteger index = new AtomicInteger( 0 ); RowSet rowSet = mock( RowSet.class, Mockito.RETURNS_MOCKS ); Answer<Object[]> answer = new Answer<Object[]>() { @Override public Object[] answer( InvocationOnMock invocation ) throws Throwable { int i = index.getAndIncrement(); return i < rows.size() ? rows.get( i ) : null; } }; when( rowSet.getRowWait( anyLong(), any( TimeUnit.class ) ) ).thenAnswer( answer ); when( rowSet.getRow() ).thenAnswer( answer ); when( rowSet.isDone() ).thenAnswer( new Answer<Boolean>() { @Override public Boolean answer( InvocationOnMock invocation ) throws Throwable { return index.get() >= rows.size(); } } ); return rowSet; }
private boolean isRowSetContainsValue( RowSet rowSet, Object[] allowed, Object[] illegal ) { boolean ok = true; Set<Object> yes = new HashSet<Object>(); yes.addAll( Arrays.asList( allowed ) ); Set<Object> no = new HashSet<Object>(); no.addAll( Arrays.asList( illegal ) ); for ( int i = 0; i < rowSet.size(); i++ ) { Object[] row = rowSet.getRow(); Object val = row[0]; ok = yes.contains( val ) && !no.contains( val ); if ( !ok ) { // this is not ok now return false; } } return ok; }
private DatabaseLookup spyLookup( StepMockHelper<DatabaseLookupMeta, DatabaseLookupData> mocks, Database db, DatabaseMeta dbMeta ) { DatabaseLookup lookup = new DatabaseLookup( mocks.stepMeta, mocks.stepDataInterface, 1, mocks.transMeta, mocks.trans ); lookup = Mockito.spy( lookup ); doReturn( db ).when( lookup ).getDatabase( eq( dbMeta ) ); for ( RowSet rowSet : lookup.getOutputRowSets() ) { if ( mockingDetails( rowSet ).isMock() ) { when( rowSet.putRow( any( RowMetaInterface.class ), any( Object[].class ) ) ).thenReturn( true ); } } return lookup; }
/** * Create ExcelWriterStep object and mock some of its required data * * @param fileType * @throws Exception */ private void setupStepMock( String fileType ) throws Exception { step = new ExcelWriterStep( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans ); step.init( stepMockHelper.initStepMetaInterface, stepMockHelper.initStepDataInterface ); List<Object[]> rows = createRowData(); String[] outFields = new String[] { "col 1", "col 2", "col 3", "col 4" }; RowSet inputRowSet = stepMockHelper.getMockInputRowSet( rows ); RowMetaInterface inputRowMeta = createRowMeta(); inputRowSet.setRowMeta( inputRowMeta ); RowMetaInterface mockOutputRowMeta = mock( RowMetaInterface.class ); when( mockOutputRowMeta.size() ).thenReturn( outFields.length ); when( inputRowSet.getRowMeta() ).thenReturn( inputRowMeta ); step.addRowSetToInputRowSets( inputRowSet ); step.setInputRowMeta( inputRowMeta ); step.addRowSetToOutputRowSets( inputRowSet ); }
@Override public boolean canProcessOneRow() { inputRowSetsLock.readLock().lock(); try { switch ( inputRowSets.size() ) { case 0: return false; case 1: RowSet set = inputRowSets.get( 0 ); if ( set.isDone() ) { return false; } return set.size() > 0; default: boolean allDone = true; for ( RowSet rowSet : inputRowSets ) { if ( !rowSet.isDone() ) { allDone = false; } if ( rowSet.size() > 0 ) { return true; } } return !allDone; } } finally { inputRowSetsLock.readLock().unlock(); } }