case BROADCAST_EDGE: UnorderedKVEdgeConfig et1Conf = UnorderedKVEdgeConfig .newBuilder(keyClass, valClass) .setFromConfiguration(conf) .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null) .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); return et1Conf.createDefaultBroadcastEdgeProperty(); case CUSTOM_EDGE: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); UnorderedPartitionedKVEdgeConfig et2Conf = UnorderedPartitionedKVEdgeConfig .newBuilder(keyClass, valClass, MRPartitioner.class.getName(), partitionerConf) .setFromConfiguration(conf) .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null) .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); EdgeManagerPluginDescriptor edgeDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); case CUSTOM_SIMPLE_EDGE: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); UnorderedPartitionedKVEdgeConfig et3Conf = UnorderedPartitionedKVEdgeConfig .newBuilder(keyClass, valClass, MRPartitioner.class.getName(), partitionerConf) .setFromConfiguration(conf)
@Override public Builder setAdditionalConfiguration(String key, String value) { outputBuilder.setAdditionalConfiguration(key, value); inputBuilder.setAdditionalConfiguration(key, value); return this; }
/** * Build and return an instance of the configuration * @return an instance of the acatual configuration */ public OrderedPartitionedKVEdgeConfig build() { return new OrderedPartitionedKVEdgeConfig(outputBuilder.build(), inputBuilder.build()); }
/** * Build and return an instance of the configuration * @return an instance of the acatual configuration */ public UnorderedKVEdgeConfig build() { return new UnorderedKVEdgeConfig(outputBuilder.build(), inputBuilder.build()); }
/** * Build and return an instance of the configuration * @return an instance of the acatual configuration */ public UnorderedPartitionedKVEdgeConfig build() { return new UnorderedPartitionedKVEdgeConfig(outputBuilder.build(), inputBuilder.build()); }
@Override public Builder setAdditionalConfiguration(String key, String value) { outputBuilder.setAdditionalConfiguration(key, value); inputBuilder.setAdditionalConfiguration(key, value); return this; }
@Override public Builder setAdditionalConfiguration(Map<String, String> confMap) { outputBuilder.setAdditionalConfiguration(confMap); inputBuilder.setAdditionalConfiguration(confMap); return this; }
@Override /** * Edge config options are derived from client-side tez-site.xml (recommended). * Optionally invoke setFromConfiguration to override these config options via commandline arguments. * * @param conf * @return this object for further chained method calls */ public Builder setFromConfiguration(Configuration conf) { outputBuilder.setFromConfiguration(conf); inputBuilder.setFromConfiguration(conf); return this; }
@Override public Builder setFromConfigurationUnfiltered( Configuration conf) { outputBuilder.setFromConfigurationUnfiltered(conf); inputBuilder.setFromConfigurationUnfiltered(conf); return this; }
@Override public Builder setFromConfigurationUnfiltered(Configuration conf) { outputBuilder.setFromConfigurationUnfiltered(conf); inputBuilder.setFromConfigurationUnfiltered(conf); return this; }
@Override public Builder setFromConfigurationUnfiltered(Configuration conf) { outputBuilder.setFromConfigurationUnfiltered(conf); inputBuilder.setFromConfigurationUnfiltered(conf); return this; }
@Override public SpecificBuilder<E> setPostMergeBufferFraction(float postMergeBufferFraction) { builder.setPostMergeBufferFraction(postMergeBufferFraction); return this; }
@Override public SpecificBuilder<E> setSortBufferSize(int sortBufferSize) { builder.setSortBufferSize(sortBufferSize); return this; }
/** * Create the actual configuration instance. * * @return an instance of the Configuration */ public OrderedPartitionedKVOutputConfig build() { return new OrderedPartitionedKVOutputConfig(this.conf); } }
@Override public SpecificBuilder<E> setAdditionalConfiguration(String key, String value) { builder.setAdditionalConfiguration(key, value); return this; }
@Override public SpecificBuilder setFromConfigurationUnfiltered(Configuration conf) { builder.setFromConfigurationUnfiltered(conf); return this; }
case BROADCAST_EDGE: UnorderedKVEdgeConfig et1Conf = UnorderedKVEdgeConfig .newBuilder(keyClass, valClass) .setFromConfiguration(conf) .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null) .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); return et1Conf.createDefaultBroadcastEdgeProperty(); case CUSTOM_EDGE: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); UnorderedPartitionedKVEdgeConfig et2Conf = UnorderedPartitionedKVEdgeConfig .newBuilder(keyClass, valClass, MRPartitioner.class.getName(), partitionerConf) .setFromConfiguration(conf) .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null) .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null) .build(); EdgeManagerPluginDescriptor edgeDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); case CUSTOM_SIMPLE_EDGE: assert partitionerClassName != null; partitionerConf = createPartitionerConf(partitionerClassName, conf); UnorderedPartitionedKVEdgeConfig et3Conf = UnorderedPartitionedKVEdgeConfig .newBuilder(keyClass, valClass, MRPartitioner.class.getName(), partitionerConf) .setFromConfiguration(conf)
@Override public Builder setAdditionalConfiguration(Map<String, String> confMap) { outputBuilder.setAdditionalConfiguration(confMap); inputBuilder.setAdditionalConfiguration(confMap); return this; }
@Override public Builder setAdditionalConfiguration(Map<String, String> confMap) { outputBuilder.setAdditionalConfiguration(confMap); inputBuilder.setAdditionalConfiguration(confMap); return this; }
@Override public Builder setAdditionalConfiguration(String key, String value) { outputBuilder.setAdditionalConfiguration(key, value); inputBuilder.setAdditionalConfiguration(key, value); return this; }