case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); reduceFacade = table.new ReduceFacade(reducer, output, objectReuseEnabled); break; default:
numRecordsIn.inc(); try { reduceFacade.updateTableEntryWithReduce(value); } catch (EOFException ex) { reduceFacade.emitAndReset(); reduceFacade.updateTableEntryWithReduce(value); numRecordsIn.inc(); try { reduceFacade.updateTableEntryWithReduce(value); } catch (EOFException ex) { reduceFacade.emitAndReset(); reduceFacade.updateTableEntryWithReduce(value); reduceFacade.emit();
numRecordsIn.inc(); try { reduceFacade.updateTableEntryWithReduce(value); } catch (EOFException ex) { reduceFacade.emitAndReset(); reduceFacade.updateTableEntryWithReduce(value); numRecordsIn.inc(); try { reduceFacade.updateTableEntryWithReduce(value); } catch (EOFException ex) { reduceFacade.emitAndReset(); reduceFacade.updateTableEntryWithReduce(value); reduceFacade.emit();
numRecordsIn.inc(); try { reduceFacade.updateTableEntryWithReduce(value); } catch (EOFException ex) { reduceFacade.emitAndReset(); reduceFacade.updateTableEntryWithReduce(value); numRecordsIn.inc(); try { reduceFacade.updateTableEntryWithReduce(value); } catch (EOFException ex) { reduceFacade.emitAndReset(); reduceFacade.updateTableEntryWithReduce(value); reduceFacade.emit();
numRecordsIn.inc(); try { reduceFacade.updateTableEntryWithReduce(value); } catch (EOFException ex) { reduceFacade.emitAndReset(); reduceFacade.updateTableEntryWithReduce(value); numRecordsIn.inc(); try { reduceFacade.updateTableEntryWithReduce(value); } catch (EOFException ex) { reduceFacade.emitAndReset(); reduceFacade.updateTableEntryWithReduce(value); reduceFacade.emit();
case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); reduceFacade = table.new ReduceFacade(reducer, output, objectReuseEnabled); break; default:
case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); reduceFacade = table.new ReduceFacade(reducer, output, objectReuseEnabled); break; default:
case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); reduceFacade = table.new ReduceFacade(reducer, output, objectReuseEnabled); break; default:
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = config.getStubParameters(); BatchTask.openUserCode(reducer, stubConfig); // instantiate the serializer / comparator serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer(); comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator(); MemoryManager memManager = parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver()); memory = memManager.allocatePages(parent, numMemoryPages); LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); switch (strategy) { case SORTED_PARTIAL_REDUCE: // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory); } else { sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory); } break; case HASHED_PARTIAL_REDUCE: table = new InPlaceMutableHashTable<T>(serializer, comparator, memory); table.open(); reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled); break; } }
@Override public void close() { // send the final batch try { switch (strategy) { case SORTED_PARTIAL_REDUCE: sortAndCombine(); break; case HASHED_PARTIAL_REDUCE: reduceFacade.emit(); break; } } catch (Exception ex2) { throw new ExceptionInChainedStubException(taskName, ex2); } outputCollector.close(); }
@Override public void close() { // send the final batch try { switch (strategy) { case SORTED_PARTIAL_REDUCE: sortAndCombine(); break; case HASHED_PARTIAL_REDUCE: reduceFacade.emit(); break; } } catch (Exception ex2) { throw new ExceptionInChainedStubException(taskName, ex2); } outputCollector.close(); }
@Override public void close() { // send the final batch try { switch (strategy) { case SORTED_PARTIAL_REDUCE: sortAndCombine(); break; case HASHED_PARTIAL_REDUCE: reduceFacade.emit(); break; } } catch (Exception ex2) { throw new ExceptionInChainedStubException(taskName, ex2); } outputCollector.close(); }
@Override public void close() { // send the final batch try { switch (strategy) { case SORTED_PARTIAL_REDUCE: sortAndCombine(); break; case HASHED_PARTIAL_REDUCE: reduceFacade.emit(); break; } } catch (Exception ex2) { throw new ExceptionInChainedStubException(taskName, ex2); } outputCollector.close(); }
private void collectHashed(T record) throws Exception { try { reduceFacade.updateTableEntryWithReduce(record); } catch (EOFException ex) { // the table has run out of memory reduceFacade.emitAndReset(); // try again reduceFacade.updateTableEntryWithReduce(record); } }
private void collectHashed(T record) throws Exception { try { reduceFacade.updateTableEntryWithReduce(record); } catch (EOFException ex) { // the table has run out of memory reduceFacade.emitAndReset(); // try again reduceFacade.updateTableEntryWithReduce(record); } }
private void collectHashed(T record) throws Exception { try { reduceFacade.updateTableEntryWithReduce(record); } catch (EOFException ex) { // the table has run out of memory reduceFacade.emitAndReset(); // try again reduceFacade.updateTableEntryWithReduce(record); } }
private void collectHashed(T record) throws Exception { try { reduceFacade.updateTableEntryWithReduce(record); } catch (EOFException ex) { // the table has run out of memory reduceFacade.emitAndReset(); // try again reduceFacade.updateTableEntryWithReduce(record); } }