/** * Inits the. * * @param rowMeta * the row meta * @throws KettleException * the kettle exception */ public void init( RowMetaInterface rowMeta ) throws KettleException { if ( nrPartitions < 0 ) { nrPartitions = meta.getPartitionSchema().getPartitionIDs().size(); } }
if ( partitioningMeta != null && partitioningMeta.getMethodType() != StepPartitioningMeta.PARTITIONING_METHOD_NONE && partitioningMeta.getPartitionSchema() != null ) { PartitionSchema partitionSchema = partitioningMeta.getPartitionSchema(); List<String> partitionsList = schemaPartitionsMap.get( partitionSchema ); if ( partitionsList != null ) {
public int getDefaultSelectedSchemaIndex() { List<String> schemaNames; try { schemaNames = schemasProvider.getPartitionSchemasNames( transMeta ); } catch ( KettleException e ) { schemaNames = Collections.emptyList(); } PartitionSchema partitioningSchema = stepMeta.getStepPartitioningMeta().getPartitionSchema(); int defaultSelectedSchemaIndex = 0; if ( partitioningSchema != null && partitioningSchema.getName() != null && !schemaNames.isEmpty() ) { defaultSelectedSchemaIndex = Const.indexOfString( partitioningSchema.getName(), schemaNames ); } return defaultSelectedSchemaIndex != -1 ? defaultSelectedSchemaIndex : 0; }
/** * Get the number of copies to start of a step. This takes into account the partitioning logic. * * @return the number of step copies to start. */ public int getCopies() { // If the step is partitioned, that's going to determine the number of copies, nothing else... // if ( isPartitioned() && getStepPartitioningMeta().getPartitionSchema() != null ) { List<String> partitionIDs = getStepPartitioningMeta().getPartitionSchema().getPartitionIDs(); if ( partitionIDs != null && partitionIDs.size() > 0 ) { // these are the partitions the step can "reach" return partitionIDs.size(); } } if ( copiesCache != null ) { return copiesCache.intValue(); } if ( parentTransMeta != null ) { // Return -1 to indicate that the variable or string value couldn't be converted to number // copiesCache = Const.toInt( parentTransMeta.environmentSubstitute( copiesString ), -1 ); } else { copiesCache = Const.toInt( copiesString, 1 ); } return copiesCache; }
/** * Saves partitioning properties in the repository for the given step. * * @param meta * the partitioning metadata to store. * @param id_transformation * the ID of the transformation * @param id_step * the ID of the step * @throws KettleDatabaseException * In case anything goes wrong * */ public void saveStepPartitioningMeta( StepPartitioningMeta meta, ObjectId id_transformation, ObjectId id_step ) throws KettleException { repository.saveStepAttribute( id_transformation, id_step, "PARTITIONING_SCHEMA", meta.getPartitionSchema() != null ? meta .getPartitionSchema().getName() : "" ); // selected schema repository.saveStepAttribute( id_transformation, id_step, "PARTITIONING_METHOD", meta.getMethodCode() ); if ( meta.getPartitioner() != null ) { meta.getPartitioner().saveRep( repository, id_transformation, id_step ); } }
/** * Checks if the transformation is using the specified partition schema. * * @param partitionSchema * the partition schema * @return true if the transformation is using the partition schema, false otherwise */ public boolean isUsingPartitionSchema( PartitionSchema partitionSchema ) { // Loop over all steps and see if the partition schema is used. for ( int i = 0; i < nrSteps(); i++ ) { StepPartitioningMeta stepPartitioningMeta = getStep( i ).getStepPartitioningMeta(); if ( stepPartitioningMeta != null ) { PartitionSchema check = stepPartitioningMeta.getPartitionSchema(); if ( check != null && check.equals( partitionSchema ) ) { return true; } } } return false; }
PartitionSchema partitionSchema = (PartitionSchema) stepPartitioningMeta.getPartitionSchema().clone();
PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
public void rollback( StepMeta before ) throws KettlePluginException { updateMethod( before.getStepPartitioningMeta().getMethod() ); updateMethodType( before.getStepPartitioningMeta().getMethodType() ); updateSchema( before.getStepPartitioningMeta().getPartitionSchema() ); }
/** * Create a copy of a step from the original transformation for use in the a slave transformation. If the step is * partitioned, the partitioning will be changed to "schemaName (slave)" * * @param stepMeta * The step to copy / clone. * @return a copy of the specified step for use in a slave transformation. */ private StepMeta addSlaveCopy( TransMeta transMeta, StepMeta stepMeta, SlaveServer slaveServer ) { StepMeta copy = (StepMeta) stepMeta.clone(); if ( copy.isPartitioned() ) { StepPartitioningMeta stepPartitioningMeta = copy.getStepPartitioningMeta(); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); String slavePartitionSchemaName = createSlavePartitionSchemaName( partitionSchema.getName() ); PartitionSchema slaveSchema = transMeta.findPartitionSchema( slavePartitionSchemaName ); if ( slaveSchema != null ) { stepPartitioningMeta.setPartitionSchema( slaveSchema ); } // Always just start a single copy on the slave server... // Otherwise the confusion w.r.t. to partitioning & re-partitioning would be complete. // copy.setCopies( 1 ); } // Remove the clustering information on the slave transformation step // We don't need it anymore, it only confuses. // copy.setClusterSchema( null ); transMeta.addStep( copy ); return copy; }
PartitionSchema partitionSchema = stepMeta.getStepPartitioningMeta().getPartitionSchema(); int partitionNr = partitionDistribution.getPartition( slaveServerName, partitionSchema.getName(), stepCopyNr ); String partitionSchemaName = stepMeta.getStepPartitioningMeta().getPartitionSchema().getName(); String partitionNrString = new DecimalFormat( "000" ).format( partitionNr ); setVariable( Const.INTERNAL_VARIABLE_STEP_PARTITION_NR, partitionNrString ); final List<String> partitionIDList = stepMeta.getStepPartitioningMeta().getPartitionSchema().getPartitionIDs(); } else { logError( BaseMessages.getString( PKG, "BaseStep.Log.UnableToRetrievePartitionId", stepMeta.getStepPartitioningMeta().getPartitionSchema().getName() ) ); return false;
.getPartitionSchema().getName() ); throw new KettleStepException( "Unable to find partition using rowset data, slave=" + outputRowSet.getRemoteSlaveServerName() + ", partition schema=" + nextStepPartitioningMeta.getPartitionSchema().getName() + ", copy=" + outputRowSet.getDestinationStepCopy() );
List<String> partitionIDs = stepMeta.getStepPartitioningMeta().getPartitionSchema().getPartitionIDs(); if ( partitionIDs != null && partitionIDs.size() > 0 ) { PartitionSchema thisPartitionSchema = null; if ( isThisPartitioned ) { thisPartitionSchema = stepMeta.getStepPartitioningMeta().getPartitionSchema(); isNextPartitioned = true; nextStepPartitioningMeta = nextStep.getStepPartitioningMeta(); nextPartitionSchema = nextStepPartitioningMeta.getPartitionSchema();
gc.setFont( EFont.GRAPH ); PartitionSchema partitionSchema = stepMeta.getStepPartitioningMeta().getPartitionSchema(); if ( partitionSchema != null ) {
PartitionSchema previousPartitionSchema = previousStepPartitioningMeta.getPartitionSchema(); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createTargetPartitionSchemaName( partitionSchema.getName() ) ); partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createSlavePartitionSchemaName( partitionSchema.getName() ) ); PartitionSchema targetPartitionSchema = targetStepPartitioningMeta.getPartitionSchema(); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createTargetPartitionSchemaName( partitionSchema.getName() ) ); partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createSlavePartitionSchemaName( partitionSchema.getName() ) ); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createSlavePartitionSchemaName( partitionSchema.getName() ) ); PartitionSchema targetPartitionSchema = targetStepPartitioningMeta.getPartitionSchema(); PartitionSchema sourcePartitionSchema = sourceStepPartitioningMeta.getPartitionSchema(); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createSlavePartitionSchemaName( partitionSchema.getName() ) ); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema(); partitionSchema.setName( createSlavePartitionSchemaName( partitionSchema.getName() ) ); PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
if ( partitioningMeta != null && partitioningMeta.getPartitionSchema() != null && partitioningMeta.isPartitioned() ) { DataNodeRef ref = new DataNodeRef( partitioningMeta.getPartitionSchema().getObjectId().getId() ); stepNode.setProperty( PROP_PARTITIONING_SCHEMA, ref );