public boolean isRepartitioning() { if ( !isPartitioned() && isTargetPartitioned() ) { return true; } if ( isPartitioned() && isTargetPartitioned() && !stepPartitioningMeta.equals( targetStepPartitioningMeta ) ) { return true; } return false; }
return step.getCopies(); if ( !step.isPartitioned() ) { return step.getCopies();
/** * Get the number of copies to start of a step. This takes into account the partitioning logic. * * @return the number of step copies to start. */ public int getCopies() { // If the step is partitioned, that's going to determine the number of copies, nothing else... // if ( isPartitioned() && getStepPartitioningMeta().getPartitionSchema() != null ) { List<String> partitionIDs = getStepPartitioningMeta().getPartitionSchema().getPartitionIDs(); if ( partitionIDs != null && partitionIDs.size() > 0 ) { // these are the partitions the step can "reach" return partitionIDs.size(); } } if ( copiesCache != null ) { return copiesCache.intValue(); } if ( parentTransMeta != null ) { // Return -1 to indicate that the variable or string value couldn't be converted to number // copiesCache = Const.toInt( parentTransMeta.environmentSubstitute( copiesString ), -1 ); } else { copiesCache = Const.toInt( copiesString, 1 ); } return copiesCache; }
int dispatchType; boolean repartitioning; if ( prevSteps[ i ].isPartitioned() ) { repartitioning = !prevSteps[ i ].getStepPartitioningMeta() .equals( stepMeta.getStepPartitioningMeta() ); } else { repartitioning = stepMeta.isPartitioned(); int dispatchType; boolean repartitioning; if ( stepMeta.isPartitioned() ) { repartitioning = !stepMeta.getStepPartitioningMeta() .equals( nextSteps[ i ].getStepPartitioningMeta() ); } else { repartitioning = nextSteps[ i ].isPartitioned();
/** * Create a copy of a step from the original transformation for use in the a slave transformation. If the step is * partitioned, the partitioning will be changed to "schemaName (slave)" * * @param stepMeta * The step to copy / clone. * @return a copy of the specified step for use in a slave transformation. */ private StepMeta addSlaveCopy( TransMeta transMeta, StepMeta stepMeta, SlaveServer slaveServer ) { StepMeta copy = (StepMeta) stepMeta.clone(); if ( copy.isPartitioned() ) { StepPartitioningMeta stepPartitioningMeta = copy.getStepPartitioningMeta(); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); String slavePartitionSchemaName = createSlavePartitionSchemaName( partitionSchema.getName() ); PartitionSchema slaveSchema = transMeta.findPartitionSchema( slavePartitionSchemaName ); if ( slaveSchema != null ) { stepPartitioningMeta.setPartitionSchema( slaveSchema ); } // Always just start a single copy on the slave server... // Otherwise the confusion w.r.t. to partitioning & re-partitioning would be complete. // copy.setCopies( 1 ); } // Remove the clustering information on the slave transformation step // We don't need it anymore, it only confuses. // copy.setClusterSchema( null ); transMeta.addStep( copy ); return copy; }
if ( stepMeta.isPartitioned() ) { remoteInputSteps = new ArrayList<RemoteStep>(); if ( ( stepMeta.isPartitioned() && getClusterSize() > 1 ) || stepMeta.getCopies() > 1 ) {
if ( thisStep.isPartitioned() ) { repartitioning = !thisStep.getStepPartitioningMeta().equals( nextStep.getStepPartitioningMeta() ); } else { repartitioning = nextStep.isPartitioned(); if ( stepMeta.isPartitioned() ) { List<String> partitionIDs = stepMeta.getStepPartitioningMeta().getPartitionSchema().getPartitionIDs(); if ( partitionIDs != null && partitionIDs.size() > 0 ) { StepInterface baseStep = sid.step; baseStep.setPartitioned( stepMeta.isPartitioned() ); boolean isThisPartitioned = stepMeta.isPartitioned(); PartitionSchema thisPartitionSchema = null; if ( isThisPartitioned ) { for ( int p = 0; p < nrNext; p++ ) { StepMeta nextStep = nextSteps.get( p ); if ( nextStep.isPartitioned() ) { isNextPartitioned = true; nextStepPartitioningMeta = nextStep.getStepPartitioningMeta();
if ( stepMeta.isPartitioned() ) { gc.setLineWidth( 1 ); gc.setForeground( EColor.RED ); if ( stepMeta.isPartitioned() && meta != null ) { partitioned = true;
if ( slaveStep.isPartitioned() ) { slaveStepCopyPartitionDistribution.addPartition( sourceSlaveServer.getName(), previousPartitionSchema.getName(), sourceCopyNr ); if ( referenceStep.isPartitioned() ) { if ( targetStep.isPartitioned() ) { slaveStepCopyPartitionDistribution.addPartition( targetSlaveServer.getName(), targetPartitionSchema.getName(), targetCopyNr ); if ( previousStep.isPartitioned() && referenceStep.isPartitioned() && sourceStepPartitioningMeta.equals( targetStepPartitioningMeta ) ) { targetStep.setStepPartitioningMeta( stepPartitioningMeta ); slaveTransMeta.addOrReplacePartitionSchema( partitionSchema ); } else if ( ( !previousStep.isPartitioned() && referenceStep.isPartitioned() ) || ( previousStep.isPartitioned() && referenceStep.isPartitioned() && !sourceStepPartitioningMeta .equals( targetStep.getStepPartitioningMeta() ) ) ) { slaveServerPartitionsMap.get( sourceSlaveServer ); int nrOfTargetPartitions = 1; if ( targetStep.isPartitioned() && targetPartitionSchema != null ) { List<String> targetPartitionsList = partitionsMap.get( targetPartitionSchema ); nrOfTargetPartitions = targetPartitionsList.size(); if ( sourceStep.isPartitioned() && sourcePartitionSchema != null ) { List<String> sourcePartitionsList = partitionsMap.get( sourcePartitionSchema ); nrOfSourcePartitions = sourcePartitionsList.size();