@Override public GenericInputSplit[] createInputSplits(int minNumSplits) { GenericInputSplit[] splits = new GenericInputSplit[minNumSplits]; for (int i = 0; i < minNumSplits; i++) { splits[i] = new GenericInputSplit(i, minNumSplits); } return splits; }
@Override public void open(GenericInputSplit split) throws IOException { super.open(split); this.splitIterator = this.source.getSplit(split.getSplitNumber(), split.getTotalNumberOfSplits()); }
@Override public void open(GenericInputSplit split) throws IOException { this.partitionNumber = split.getSplitNumber(); }
@Override public void open(GenericInputSplit split) throws IOException { this.partitionNumber = split.getSplitNumber(); }
@Override public InputSplit[] createInputSplits(int minNumSplits) throws IOException { if (parameterValues == null) { return new GenericInputSplit[]{new GenericInputSplit(0, 1)}; } GenericInputSplit[] ret = new GenericInputSplit[parameterValues.length]; for (int i = 0; i < ret.length; i++) { ret[i] = new GenericInputSplit(i, ret.length); } return ret; }
@Override public void open(GenericInputSplit split) throws IOException { this.partitionId = split.getSplitNumber(); // total number of partitions int numPartitions = split.getTotalNumberOfSplits(); // ensure even distribution of records and keys Preconditions.checkArgument( numRecords % numPartitions == 0, "Records cannot be evenly distributed among partitions"); Preconditions.checkArgument( numKeys % numPartitions == 0, "Keys cannot be evenly distributed among partitions"); this.recordsPerPartition = numRecords / numPartitions; this.keysPerPartition = numKeys / numPartitions; this.recordCnt = 0; }
@Override public void run() { String host = ""; GenericInputSplit split; while ((split = (GenericInputSplit) ia.getNextInputSplit(host, 0)) != null) { splitsRetrieved.incrementAndGet(); sumOfIds.addAndGet(split.getSplitNumber()); } } };
@Override public InputSplit[] createInputSplits(int minNumSplits) { return new GenericInputSplit[]{new GenericInputSplit(0, 1)}; }
@Override public void open(GenericInputSplit split) throws IOException { super.open(split); this.splitIterator = this.source.getSplit(split.getSplitNumber(), split.getTotalNumberOfSplits()); }
@Override public void open(GenericInputSplit split) throws IOException { this.partitionNumber = split.getSplitNumber(); }
@Override public GenericInputSplit[] createInputSplits(int numSplits) throws IOException { if (numSplits < 1) { throw new IllegalArgumentException("Number of input splits has to be at least 1."); } numSplits = (this instanceof NonParallelInput) ? 1 : numSplits; GenericInputSplit[] splits = new GenericInputSplit[numSplits]; for (int i = 0; i < splits.length; i++) { splits[i] = new GenericInputSplit(i, numSplits); } return splits; }
@Override public void open(GenericInputSplit split) throws IOException { super.open(split); this.splitIterator = this.source.getSplit(split.getSplitNumber(), split.getTotalNumberOfSplits()); }
@Override public void open(GenericInputSplit split) throws IOException { this.partitionNumber = split.getSplitNumber(); }
@Override public GenericInputSplit[] createInputSplits(int numSplits) throws IOException { if (numSplits < 1) { throw new IllegalArgumentException("Number of input splits has to be at least 1."); } numSplits = (this instanceof NonParallelInput) ? 1 : numSplits; GenericInputSplit[] splits = new GenericInputSplit[numSplits]; for (int i = 0; i < splits.length; i++) { splits[i] = new GenericInputSplit(i, numSplits); } return splits; }
split.getTotalNumberOfSplits(), tableName, user, options.put(AccumuloTables.KEY_PREDICATE, predicate.getReduceFilter().encode()); if (split.getSplitNumber() + 1 > ranges.size()) { scanner = null; iterator = new ArrayList<Map.Entry<Key, Value>>().iterator(); scanner = conn.createBatchScanner(tableName, auth, batchSize); attachIterator(scanner, iteratorPriority, options); scanner.setRanges(Lists.newArrayList(ranges.get(split.getSplitNumber()))); iterator = scanner.iterator();
@Test public void testSerialSplitAssignment() { try { final int NUM_SPLITS = 50; Set<InputSplit> splits = new HashSet<InputSplit>(); for (int i = 0; i < NUM_SPLITS; i++) { splits.add(new GenericInputSplit(i, NUM_SPLITS)); } DefaultInputSplitAssigner ia = new DefaultInputSplitAssigner(splits); InputSplit is = null; while ((is = ia.getNextInputSplit("", 0)) != null) { assertTrue(splits.remove(is)); } assertTrue(splits.isEmpty()); assertNull(ia.getNextInputSplit("", 0)); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
splits.add(new GenericInputSplit(i, NUM_SPLITS));
CollectionInputFormat<ElementType> result = (CollectionInputFormat<ElementType>) serializationResult; GenericInputSplit inputSplit = new GenericInputSplit(0, 1); inputFormat.open(inputSplit); result.open(inputSplit);
@SuppressWarnings("unchecked") CollectionInputFormat<String> in = (CollectionInputFormat<String>) result; in.open(new GenericInputSplit(0, 1));
@Override public GenericInputSplit[] createInputSplits(int numSplits) { // Always return a single split because only one global "impulse" will ever be sent. return new GenericInputSplit[] {new GenericInputSplit(1, 1)}; }