ConfigHelper.setOutputPartitioner(conf, System.getenv(PIG_PARTITIONER)); ConfigHelper.setOutputPartitioner(conf, System.getenv(PIG_OUTPUT_PARTITIONER)); if (System.getenv(PIG_INPUT_FORMAT) != null) inputFormatClass = getFullyQualifiedClassName(System.getenv(PIG_INPUT_FORMAT));
/** * Returns hadoop configuration. * Some example of things that could be set are the output rpc port (cassandra.output.thrift.port) * and other cassandra specific hadoop configuration settings. */ public org.apache.hadoop.conf.Configuration getHadoopConf() { final org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration(); if (this.getUserName().isPresent() && this.getPassword().isPresent()) { ConfigHelper.setOutputKeyspaceUserNameAndPassword( hadoopConf, this.getUserName().get(), this.getPassword().get() ); } if (!this.conf.getProperty(DATACENTER).isPresent()) { // should never happen, but a sanity check throw new MissingPropertyException("The datacenter information is missing"); } // By default use Murmur3 Partitioner for now, we may want this to be changed in future ConfigHelper.setOutputPartitioner(hadoopConf, Murmur3Partitioner.class.getName()); log.info("Setting output local DC only to true"); ConfigHelper.setOutputLocalDCOnly(hadoopConf, true); this.conf.getPropertiesWithPrefix(CassandraSinkConfiguration.HADOOP_COMMON_PREFIX, true).forEach( (key, value) -> { log.info("hadoop-conf-update:key:[{}]:value:[{}]", key, value); hadoopConf.set(key, value); } ); return hadoopConf; }
ConfigHelper.setInputInitialAddress(job.getConfiguration(), getConf().get("cassandraHost")); ConfigHelper.setInputRpcPort(job.getConfiguration(), getConf().get("cassandraPort")); ConfigHelper.setOutputPartitioner(job.getConfiguration(), "org.apache.cassandra.dht.RandomPartitioner"); ConfigHelper.setOutputInitialAddress(job.getConfiguration(), getConf().get("cassandraHost")); ConfigHelper.setOutputRpcPort(job.getConfiguration(), getConf().get("cassandraPort"));
ConfigHelper.setInputSplitSize(conf, splitSize); if (partitionerClass!= null) ConfigHelper.setOutputPartitioner(conf, partitionerClass); if (rpcPort != null)
ConfigHelper.setInputSplitSize(conf, splitSize); if (partitionerClass!= null) ConfigHelper.setOutputPartitioner(conf, partitionerClass); if (rpcPort != null)