@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || !(o instanceof BasicPartition)) { return false; } BasicPartition that = (BasicPartition) o; return key.equals(that.key) && relativePath.equals(that.relativePath); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || !(o instanceof BasicPartition)) { return false; } BasicPartition that = (BasicPartition) o; return key.equals(that.key) && relativePath.equals(that.relativePath); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } DefaultConsumablePartition that = (DefaultConsumablePartition) o; return numFailures == that.numFailures && timestamp == that.timestamp && partitionKey.equals(that.partitionKey) && processState == that.processState; }
/** * @return the ConsumablePartition with the given PartitionKey, from the working set, after removing it from * the partitions list */ public ConsumablePartition remove(PartitionKey partitionKey) { for (int i = 0; i < partitions.size(); i++) { if (partitionKey.equals(partitions.get(i).getPartitionKey())) { return partitions.remove(i); } } throw new IllegalArgumentException("PartitionKey not found: " + partitionKey); }
/** * @return the ConsumablePartition with the given PartitionKey, from the working set */ public ConsumablePartition lookup(PartitionKey partitionKey) { for (int i = 0; i < partitions.size(); i++) { if (partitionKey.equals(partitions.get(i).getPartitionKey())) { return partitions.get(i); } } throw new IllegalArgumentException("PartitionKey not found: " + partitionKey); }
@Override public void write(K key, V value) throws IOException, InterruptedException { PartitionKey partitionKey = dynamicPartitioner.getPartitionKey(key, value); if (!partitionKey.equals(currPartitionKey)) { // make sure we haven't written to this partition previously if (closedKeys.contains(partitionKey)) { throw new IllegalStateException( String.format("Encountered a partition key for which the writer has already been closed: '%s'.", partitionKey)); } // currPartitionKey can be null for the first key value pair, in which case there's no writer to close if (currPartitionKey != null) { // close the existing RecordWriter and create a new one for the new PartitionKEy currRecordWriter.close(currContext); closedKeys.add(currPartitionKey); } currPartitionKey = partitionKey; currContext = getKeySpecificContext(currPartitionKey); currRecordWriter = getBaseRecordWriter(currContext); } currRecordWriter.write(key, value); }
@Override public void write(K key, V value) throws IOException, InterruptedException { PartitionKey partitionKey = dynamicPartitioner.getPartitionKey(key, value); if (!partitionKey.equals(currPartitionKey)) { // make sure we haven't written to this partition previously if (closedKeys.contains(partitionKey)) { throw new IllegalStateException( String.format("Encountered a partition key for which the writer has already been closed: '%s'.", partitionKey)); } // currPartitionKey can be null for the first key value pair, in which case there's no writer to close if (currPartitionKey != null) { // close the existing RecordWriter and create a new one for the new PartitionKEy currRecordWriter.close(currContext); closedKeys.add(currPartitionKey); } currPartitionKey = partitionKey; currContext = getKeySpecificContext(currPartitionKey); currRecordWriter = getBaseRecordWriter(currContext); } currRecordWriter.write(key, value); }