default void addRowSetToOutputRowSets( RowSet rowSet ) { getOutputRowSets().add( rowSet ); }
/** * This checks transformation initialization when using many copies to one next step * * @throws KettleException */ @Test public void testManyToOneCopies() throws KettleException { prepareStepMetas_x2_1(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 2 rowsets finally", 2, rowsets.size() ); assertEquals( "We have 4 steps: 2 copies of producer and 2 copies of consumer", 3, trans.getSteps().size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( S10 ); assertTrue( "1 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1 step have 1 output rowsets", 1, stepOne0.getOutputRowSets().size() ); StepInterface stepOne1 = getStepByName( S11 ); assertTrue( "1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() ); assertEquals( "1 step have 1 output rowsets", 1, stepOne1.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( S20 ); Assert.assertEquals( "2.0 step have 2 input row sets", 2, stepTwo0.getInputRowSets().size() ); Assert.assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() ); }
/** * This is PDI-12140 case. 2 steps with same partitions ID's count but different partitioner. This is not a swim lines * cases and we need repartitioning here. * * @throws KettleException */ @Test public void testDifferentPartitioningFlow() throws KettleException { prepareStepMetas_cl1_cl2(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 4 rowsets finally since repartitioning happens", 4, rowsets.size() ); assertEquals( "We have 4 steps: 2 producer copies and 2 copies of consumer since they both partitioned", 4, trans .getSteps().size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( SP10 ); assertTrue( "1.0 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1.0 step have 2 output rowsets", 2, stepOne0.getOutputRowSets().size() ); StepInterface stepOne1 = getStepByName( SP11 ); assertTrue( "1.1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() ); assertEquals( "1.1 step have 2 output rowsets", 2, stepOne1.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( SP20 ); assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() ); assertEquals( "2.0 step have 1 input rowsets", 2, stepTwo0.getInputRowSets().size() ); StepInterface stepTwo2 = getStepByName( SP21 ); assertTrue( "2.1 step have no output row sets", stepTwo2.getOutputRowSets().isEmpty() ); assertEquals( "2.2 step have 2 input rowsets", 2, stepTwo2.getInputRowSets().size() ); }
/** * Test 'Swim lines partitioning' * * @throws KettleException */ @Test public void testSwimLanesPartitioning() throws KettleException { prepareStepMetas_cl1_cl1(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 2 rowsets finally", 2, rowsets.size() ); assertEquals( "We have 3 steps: 1 producer and 2 copies of consumer since it is partitioned", 4, trans.getSteps() .size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( SP10 ); assertTrue( "1.0 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1.0 step have 1 output rowsets", 1, stepOne0.getOutputRowSets().size() ); StepInterface stepOne1 = getStepByName( SP11 ); assertTrue( "1.1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() ); assertEquals( "1.1 step have 1 output rowsets", 1, stepOne1.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( SP20 ); assertEquals( "2.0 step have 2 input row sets", 1, stepTwo0.getInputRowSets().size() ); assertTrue( "2.0 step have no output rowsets", stepTwo0.getOutputRowSets().isEmpty() ); StepInterface stepTwo2 = getStepByName( SP21 ); assertTrue( "2.2 step have no output row sets", stepTwo2.getOutputRowSets().isEmpty() ); assertEquals( "2.2 step have 2 output rowsets", 1, stepTwo2.getInputRowSets().size() ); }
/** * This checks transformation initialization when using one to many copies * * @throws KettleException */ @Test public void testOneToManyCopies() throws KettleException { prepareStepMetas_1_x2(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 2 rowsets finally", 2, rowsets.size() ); assertEquals( "We have 3 steps: one producer and 2 copies of consumer", 3, trans.getSteps().size() ); // Ok, examine initialized steps now. StepInterface stepOne = getStepByName( S10 ); assertTrue( "1 step have no input row sets", stepOne.getInputRowSets().isEmpty() ); assertEquals( "1 step have 2 output rowsets", 2, stepOne.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( S20 ); Assert.assertEquals( "2.0 step have 12 input row sets", 1, stepTwo0.getInputRowSets().size() ); Assert.assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() ); StepInterface stepTwo1 = getStepByName( S21 ); Assert.assertEquals( "2.1 step have 1 input row sets", 1, stepTwo1.getInputRowSets().size() ); Assert.assertTrue( "2.1 step have no output row sets", stepTwo1.getOutputRowSets().isEmpty() ); }
/** * This checks transformation initialization when using many to many copies. * * @throws KettleException */ @Test public void testManyToManyCopies() throws KettleException { prepareStepMetas_x2_x2(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 2 rowsets finally", 2, rowsets.size() ); assertEquals( "We have 4 steps: 2 copies of producer and 2 copies of consumer", 4, trans.getSteps().size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( S10 ); assertTrue( "1 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1 step have 1 output rowsets", 1, stepOne0.getOutputRowSets().size() ); StepInterface stepOne1 = getStepByName( S11 ); assertTrue( "1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() ); assertEquals( "1 step have 1 output rowsets", 1, stepOne1.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( S20 ); Assert.assertEquals( "2.0 step have 1 input row sets", 1, stepTwo0.getInputRowSets().size() ); Assert.assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() ); StepInterface stepTwo1 = getStepByName( S21 ); Assert.assertEquals( "2.1 step have 1 input row sets", 1, stepTwo1.getInputRowSets().size() ); Assert.assertTrue( "2.1 step have no output row sets", stepTwo1.getOutputRowSets().isEmpty() ); }
/** * Test one to one partitioning step transformation organization. * * @throws KettleException */ @Test public void testOneToPartitioningSchema() throws KettleException { prepareStepMetas_1_cl1(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 2 rowsets finally", 2, rowsets.size() ); assertEquals( "We have 3 steps: 1 producer and 2 copies of consumer since it is partitioned", 3, trans.getSteps() .size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( S10 ); assertTrue( "1 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1 step have 2 output rowsets", 2, stepOne0.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( SP20 ); assertEquals( "2.0 step have one input row sets", 1, stepTwo0.getInputRowSets().size() ); assertTrue( "2.0 step have no output rowsets", stepTwo0.getOutputRowSets().isEmpty() ); StepInterface stepTwo1 = getStepByName( SP21 ); Assert.assertEquals( "2.1 step have 1 input row sets", 1, stepTwo1.getInputRowSets().size() ); Assert.assertTrue( "2.1 step have no output row sets", stepTwo1.getOutputRowSets().isEmpty() ); }
/** * This is a case when step running in many copies meets partitioning one. * * @throws KettleException */ @Test public void testManyCopiesToPartitioningFlow() throws KettleException { prepareStepMetas_x2_cl1(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 4 rowsets finally since repartitioning happens", 4, rowsets.size() ); assertEquals( "We have 4 steps: 2 producer copies and 2 copies of consumer since consumer is partitioned", 4, trans .getSteps().size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( S10 ); assertTrue( "1.0 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1.0 step have 2 output rowsets", 2, stepOne0.getOutputRowSets().size() ); StepInterface stepOne1 = getStepByName( S11 ); assertTrue( "1.1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() ); assertEquals( "1.1 step have 2 output rowsets", 2, stepOne1.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( SP20 ); assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() ); assertEquals( "2.0 step have 2 input rowsets", 2, stepTwo0.getInputRowSets().size() ); StepInterface stepTwo2 = getStepByName( SP21 ); assertTrue( "2.1 step have no output row sets", stepTwo2.getOutputRowSets().isEmpty() ); assertEquals( "2.2 step have 2 input rowsets", 2, stepTwo2.getInputRowSets().size() ); }
List<RowSet> combiOutputRowSets = combi.step.getOutputRowSets(); if ( inputSize > 0 && inputSize == totalSize && combiOutputRowSets.size() > 1 ) { RowSet outputFull = null;