public void configure(JobConf job) { super.setConf(job); } }
int numReducers = 10; KeyFieldBasedPartitioner<Text, Text> kfbp = new KeyFieldBasedPartitioner<Text, Text>(); Configuration conf = new Configuration(); conf.setInt("num.key.fields.for.partition", 10); kfbp.setConf(conf); assertEquals("Empty key should map to 0th partition", 0, kfbp.getPartition(new Text(), new Text(), numReducers)); kfbp = new KeyFieldBasedPartitioner<Text, Text>(); conf = new Configuration(); kfbp.setConf(conf); String input = "abc\tdef\txyz"; int hashCode = input.hashCode(); int expectedPartition = kfbp.getPartition(hashCode, numReducers); assertEquals("Partitioner doesnt work as expected", expectedPartition, kfbp.getPartition(new Text(input), new Text(), numReducers)); kfbp = new KeyFieldBasedPartitioner<Text, Text>(); conf = new Configuration(); conf.set(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS, "-k2,2"); kfbp.setConf(conf); String expectedOutput = "def"; byte[] eBytes = expectedOutput.getBytes(); hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0); expectedPartition = kfbp.getPartition(hashCode, numReducers); assertEquals("Partitioner doesnt work as expected", expectedPartition, kfbp.getPartition(new Text(input), new Text(), numReducers));
return getPartition(key.toString().hashCode(), numReduceTasks); currentHash = hashCode(keyBytes, startChar, endChar, currentHash); return getPartition(currentHash, numReduceTasks);
int numReducers = 10; KeyFieldBasedPartitioner<Text, Text> kfbp = new KeyFieldBasedPartitioner<Text, Text>(); Configuration conf = new Configuration(); conf.setInt("num.key.fields.for.partition", 10); kfbp.setConf(conf); assertEquals("Empty key should map to 0th partition", 0, kfbp.getPartition(new Text(), new Text(), numReducers)); kfbp = new KeyFieldBasedPartitioner<Text, Text>(); conf = new Configuration(); kfbp.setConf(conf); String input = "abc\tdef\txyz"; int hashCode = input.hashCode(); int expectedPartition = kfbp.getPartition(hashCode, numReducers); assertEquals("Partitioner doesnt work as expected", expectedPartition, kfbp.getPartition(new Text(input), new Text(), numReducers)); kfbp = new KeyFieldBasedPartitioner<Text, Text>(); conf = new Configuration(); conf.set(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS, "-k2,2"); kfbp.setConf(conf); String expectedOutput = "def"; byte[] eBytes = expectedOutput.getBytes(); hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0); expectedPartition = kfbp.getPartition(hashCode, numReducers); assertEquals("Partitioner doesnt work as expected", expectedPartition, kfbp.getPartition(new Text(input), new Text(), numReducers));
return getPartition(key.toString().hashCode(), numReduceTasks); currentHash = hashCode(keyBytes, startChar, endChar, currentHash); return getPartition(currentHash, numReduceTasks);
return getPartition(key.toString().hashCode(), numReduceTasks); currentHash = hashCode(keyBytes, startChar, endChar, currentHash); return getPartition(currentHash, numReduceTasks);
public void configure(JobConf job) { super.setConf(job); } }
return getPartition(key.toString().hashCode(), numReduceTasks); currentHash = hashCode(keyBytes, startChar, endChar, currentHash); return getPartition(currentHash, numReduceTasks);
public void configure(JobConf job) { super.setConf(job); } }
return getPartition(key.toString().hashCode(), numReduceTasks); currentHash = hashCode(keyBytes, startChar, endChar, currentHash); return getPartition(currentHash, numReduceTasks);
public void configure(JobConf job) { super.setConf(job); } }
public void configure(JobConf job) { super.setConf(job); } }