public StepPartitioningMeta loadStepPartitioningMeta( ObjectId id_step ) throws KettleException { StepPartitioningMeta stepPartitioningMeta = new StepPartitioningMeta(); stepPartitioningMeta.setPartitionSchemaName( repository .getStepAttributeString( id_step, "PARTITIONING_SCHEMA" ) ); String methodCode = repository.getStepAttributeString( id_step, "PARTITIONING_METHOD" ); stepPartitioningMeta.setMethod( StepPartitioningMeta.getMethod( methodCode ) ); if ( stepPartitioningMeta.getPartitioner() != null ) { stepPartitioningMeta.getPartitioner().loadRep( repository, id_step ); } stepPartitioningMeta.hasChanged( true ); return stepPartitioningMeta; }
/** * Saves partitioning properties in the repository for the given step. * * @param meta * the partitioning metadata to store. * @param id_transformation * the ID of the transformation * @param id_step * the ID of the step * @throws KettleDatabaseException * In case anything goes wrong * */ public void saveStepPartitioningMeta( StepPartitioningMeta meta, ObjectId id_transformation, ObjectId id_step ) throws KettleException { repository.saveStepAttribute( id_transformation, id_step, "PARTITIONING_SCHEMA", meta.getPartitionSchema() != null ? meta .getPartitionSchema().getName() : "" ); // selected schema repository.saveStepAttribute( id_transformation, id_step, "PARTITIONING_METHOD", meta.getMethodCode() ); if ( meta.getPartitioner() != null ) { meta.getPartitioner().saveRep( repository, id_transformation, id_step ); } }
public void rollback( StepMeta before ) throws KettlePluginException { updateMethod( before.getStepPartitioningMeta().getMethod() ); updateMethodType( before.getStepPartitioningMeta().getMethodType() ); updateSchema( before.getStepPartitioningMeta().getPartitionSchema() ); }
public StepPartitioningMeta clone() { try { StepPartitioningMeta stepPartitioningMeta = new StepPartitioningMeta( method, partitionSchema != null ? (PartitionSchema) partitionSchema.clone() : null ); stepPartitioningMeta.partitionSchemaName = partitionSchemaName; stepPartitioningMeta.setMethodType( methodType ); stepPartitioningMeta.setPartitioner( partitioner == null ? null : partitioner.clone() ); return stepPartitioningMeta; } catch ( KettlePluginException e ) { throw new RuntimeException( "Unable to load partitioning plugin", e ); } }
if ( partitioningMeta != null && partitioningMeta.getPartitionSchema() != null && partitioningMeta.isPartitioned() ) { DataNodeRef ref = new DataNodeRef( partitioningMeta.getPartitionSchema().getObjectId().getId() ); stepNode.setProperty( PROP_PARTITIONING_SCHEMA, ref ); stepNode.setProperty( PROP_PARTITIONING_METHOD, partitioningMeta.getMethodCode() ); // method of partitioning if ( partitioningMeta.getPartitioner() != null ) { DataNode partitionerCustomNode = new DataNode( NODE_PARTITIONER_CUSTOM ); proxy = new RepositoryProxy( partitionerCustomNode ); partitioningMeta.getPartitioner().saveRep( proxy, null, null ); stepNode.addNode( partitionerCustomNode );
public void modifyText( ModifyEvent e ) { partitioningMeta.hasChanged( true ); } };
stepMetas.add( mockHelper.stepMeta ); stepMetas.add( mockHelper.stepMeta ); StepPartitioningMeta stepPartitioningMeta = spy( new StepPartitioningMeta() ); BasePartitioner partitioner = mock( BasePartitioner.class ); when( mockHelper.transMeta.findNextSteps( any( StepMeta.class ) ) ).thenReturn( stepMetas ); when( mockHelper.stepMeta.getStepPartitioningMeta() ).thenReturn( stepPartitioningMeta ); when( stepPartitioningMeta.getPartitioner() ).thenReturn( partitioner ); when( partitioner.getNrPartitions() ).thenReturn( 2 ); when( stepPartitioningMeta.getPartition( rowMeta0, objects0 ) ).thenReturn( 0 ); when( stepPartitioningMeta.getPartition( rowMeta1, objects1 ) ).thenReturn( 1 );
partitionNr = nextStepPartitioningMeta.getPartition( rowMeta, row ); } catch ( KettleException e ) { throw new KettleStepException( .getPartitionSchema().getName() ); throw new KettleStepException( "Unable to find partition using rowset data, slave=" + outputRowSet.getRemoteSlaveServerName() + ", partition schema=" + nextStepPartitioningMeta.getPartitionSchema().getName() + ", copy=" + outputRowSet.getDestinationStepCopy() ); int partCount = ( (BasePartitioner) nextStepPartitioningMeta.getPartitioner() ).getNrPartitions();
private static StepPartitioningMeta createStepPartitioningMeta( String method, String schemaName ) throws Exception { StepPartitioningMeta meta = new StepPartitioningMeta( method, new PartitionSchema( schemaName, Collections.<String>emptyList() ) ); meta.setPartitionSchemaName( "schema_name" ); return meta; }
public ModPartitionerDialog( Shell parent, StepMeta stepMeta, StepPartitioningMeta partitioningMeta, TransMeta transMeta ) { super( parent, (BaseStepMeta) stepMeta.getStepMetaInterface(), transMeta, partitioningMeta .getPartitioner().getDescription() ); this.stepMeta = stepMeta; this.partitioningMeta = partitioningMeta; partitioner = (ModPartitioner) partitioningMeta.getPartitioner(); fieldName = partitioner.getFieldName(); }
/** * Inits the. * * @param rowMeta * the row meta * @throws KettleException * the kettle exception */ public void init( RowMetaInterface rowMeta ) throws KettleException { if ( nrPartitions < 0 ) { nrPartitions = meta.getPartitionSchema().getPartitionIDs().size(); } }
public StepPartitioningMeta( Node partitioningMethodNode ) throws KettleException { this(); setMethod( getMethod( XMLHandler.getTagValue( partitioningMethodNode, "method" ) ) ); partitionSchemaName = XMLHandler.getTagValue( partitioningMethodNode, "schema_name" ); hasChanged = false; if ( partitioner != null ) { partitioner.loadXML( partitioningMethodNode ); } }
PartitionSchema previousPartitionSchema = previousStepPartitioningMeta.getPartitionSchema(); StepPartitioningMeta stepPartitioningMeta = previousStepPartitioningMeta.clone(); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createTargetPartitionSchemaName( partitionSchema.getName() ) ); stepPartitioningMeta = previousStepPartitioningMeta.clone(); partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createSlavePartitionSchemaName( partitionSchema.getName() ) ); PartitionSchema targetPartitionSchema = targetStepPartitioningMeta.getPartitionSchema(); if ( targetStepPartitioningMeta.isPartitioned() ) { StepPartitioningMeta stepPartitioningMeta = targetStepPartitioningMeta.clone(); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createTargetPartitionSchemaName( partitionSchema.getName() ) ); stepPartitioningMeta = targetStepPartitioningMeta.clone(); partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createSlavePartitionSchemaName( partitionSchema.getName() ) ); && sourceStepPartitioningMeta.equals( targetStepPartitioningMeta ) ) { StepPartitioningMeta stepPartitioningMeta = sourceStepPartitioningMeta.clone(); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createSlavePartitionSchemaName( partitionSchema.getName() ) ); } else if ( ( !previousStep.isPartitioned() && referenceStep.isPartitioned() )
&& partitioningMeta.getMethodType() != StepPartitioningMeta.PARTITIONING_METHOD_NONE && partitioningMeta.getPartitionSchema() != null ) { PartitionSchema partitionSchema = partitioningMeta.getPartitionSchema(); List<String> partitionsList = schemaPartitionsMap.get( partitionSchema ); if ( partitionsList != null ) {
repartitioning = !thisStep.getStepPartitioningMeta().equals( nextStep.getStepPartitioningMeta() ); } else { repartitioning = nextStep.isPartitioned(); List<String> partitionIDs = stepMeta.getStepPartitioningMeta().getPartitionSchema().getPartitionIDs(); if ( partitionIDs != null && partitionIDs.size() > 0 ) { PartitionSchema thisPartitionSchema = null; if ( isThisPartitioned ) { thisPartitionSchema = stepMeta.getStepPartitioningMeta().getPartitionSchema(); isNextPartitioned = true; nextStepPartitioningMeta = nextStep.getStepPartitioningMeta(); nextPartitionSchema = nextStepPartitioningMeta.getPartitionSchema(); baseStep.setRepartitioning( nextStepPartitioningMeta.getMethodType() ); baseStep.setRepartitioning( targetStepPartitioningMeta.getMethodType() );
/** * @param stepname * The name of the new step * @param stepMetaInterface * The step metadata interface to use (TextFileInputMeta, etc) */ public StepMeta( String stepname, StepMetaInterface stepMetaInterface ) { if ( stepMetaInterface != null ) { PluginRegistry registry = PluginRegistry.getInstance(); this.stepid = registry.getPluginId( StepPluginType.class, stepMetaInterface ); setDeprecationAndSuggestedStep(); } this.name = stepname; setStepMetaInterface( stepMetaInterface ); selected = false; distributes = true; copiesString = "1"; location = new Point( 0, 0 ); drawstep = false; description = null; stepPartitioningMeta = new StepPartitioningMeta(); // targetStepPartitioningMeta = new StepPartitioningMeta(); clusterSchema = null; // non selected by default. remoteInputSteps = new ArrayList<RemoteStep>(); remoteOutputSteps = new ArrayList<RemoteStep>(); attributesMap = new HashMap<String, Map<String, String>>(); }
/** * Create a copy of a step from the original transformation for use in the a slave transformation. If the step is * partitioned, the partitioning will be changed to "schemaName (slave)" * * @param stepMeta * The step to copy / clone. * @return a copy of the specified step for use in a slave transformation. */ private StepMeta addSlaveCopy( TransMeta transMeta, StepMeta stepMeta, SlaveServer slaveServer ) { StepMeta copy = (StepMeta) stepMeta.clone(); if ( copy.isPartitioned() ) { StepPartitioningMeta stepPartitioningMeta = copy.getStepPartitioningMeta(); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); String slavePartitionSchemaName = createSlavePartitionSchemaName( partitionSchema.getName() ); PartitionSchema slaveSchema = transMeta.findPartitionSchema( slavePartitionSchemaName ); if ( slaveSchema != null ) { stepPartitioningMeta.setPartitionSchema( slaveSchema ); } // Always just start a single copy on the slave server... // Otherwise the confusion w.r.t. to partitioning & re-partitioning would be complete. // copy.setCopies( 1 ); } // Remove the clustering information on the slave transformation step // We don't need it anymore, it only confuses. // copy.setClusterSchema( null ); transMeta.addStep( copy ); return copy; }
/** * @param method * @param partitionSchema */ public StepPartitioningMeta( String method, PartitionSchema partitionSchema ) throws KettlePluginException { setMethod( method ); this.partitionSchema = partitionSchema; hasChanged = false; }
public int getDefaultSelectedMethodIndex() { for ( int i = 0; i < codes.length; i++ ) { if ( codes[ i ].equals( stepMeta.getStepPartitioningMeta().getMethod() ) ) { return i; } } return 0; }
public void createPartitioner( String method ) throws KettlePluginException { methodType = getMethodType( method ); switch ( methodType ) { case PARTITIONING_METHOD_SPECIAL: { PluginRegistry registry = PluginRegistry.getInstance(); PluginInterface plugin = registry.findPluginWithId( PartitionerPluginType.class, method ); partitioner = (Partitioner) registry.loadClass( plugin ); partitioner.setId( plugin.getIds()[0] ); break; } case PARTITIONING_METHOD_NONE: default: partitioner = null; } if ( partitioner != null ) { partitioner.setMeta( this ); } }