@Override public void abort() { close(); }
protected void crossMatchingGroup(Iterator<T1> values1, Iterator<T2> values2, FlatJoinFunction<T1, T2, O> joinFunction, Collector<O> collector) throws Exception { final T1 firstV1 = values1.next(); final T2 firstV2 = values2.next(); final boolean v1HasNext = values1.hasNext(); final boolean v2HasNext = values2.hasNext(); // check if one side is already empty // this check could be omitted if we put this in MatchTask. // then we can derive the local strategy (with build side). if (v1HasNext) { if (v2HasNext) { // both sides contain more than one value // TODO: Decide which side to spill and which to block! crossMwithNValues(firstV1, values1, firstV2, values2, joinFunction, collector); } else { crossSecond1withNValues(firstV2, firstV1, values1, joinFunction, collector); } } else { if (v2HasNext) { crossFirst1withNValues(firstV1, firstV2, values2, joinFunction, collector); } else { // both sides contain only one value joinFunction.join(firstV1, firstV2, collector); } } }
/** * Crosses a single value from the second side with N values, all sharing a common key. * Effectively realizes a <i>N:1</i> join. * * @param val1 The value form the <i>1</i> side. * @param firstValN The first of the values from the <i>N</i> side. * @param valsN Iterator over remaining <i>N</i> side values. * @throws Exception Forwards all exceptions thrown by the stub. */ private void crossSecond1withNValues(T2 val1, T1 firstValN, Iterator<T1> valsN, FlatJoinFunction<T1, T2, O> joinFunction, Collector<O> collector) throws Exception { T2 copy2 = createCopy(serializer2, val1, this.copy2); joinFunction.join(firstValN, copy2, collector); // set copy and join first element boolean more = true; do { final T1 nRec = valsN.next(); if (valsN.hasNext()) { copy2 = createCopy(serializer2, val1, this.copy2); joinFunction.join(nRec, copy2, collector); } else { joinFunction.join(nRec, val1, collector); more = false; } } while (more); }
public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { if (numMemoryPages < 2) { throw new IllegalArgumentException("Merger needs at least 2 memory pages."); } this.pairComparator = pairComparator; this.serializer1 = serializer1; this.serializer2 = serializer2; this.memoryManager = memoryManager; this.ioManager = ioManager; this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate()); this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate()); final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1; this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2, (numMemoryPages - numPagesForSpiller), parentTask); this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller); }
/** * Crosses a single value from the second side with N values, all sharing a common key. * Effectively realizes a <i>N:1</i> join. * * @param val1 The value form the <i>1</i> side. * @param firstValN The first of the values from the <i>N</i> side. * @param valsN Iterator over remaining <i>N</i> side values. * @throws Exception Forwards all exceptions thrown by the stub. */ private void crossSecond1withNValues(T2 val1, T1 firstValN, Iterator<T1> valsN, FlatJoinFunction<T1, T2, O> joinFunction, Collector<O> collector) throws Exception { T2 copy2 = createCopy(serializer2, val1, this.copy2); joinFunction.join(firstValN, copy2, collector); // set copy and join first element boolean more = true; do { final T1 nRec = valsN.next(); if (valsN.hasNext()) { copy2 = createCopy(serializer2, val1, this.copy2); joinFunction.join(nRec, copy2, collector); } else { joinFunction.join(nRec, val1, collector); more = false; } } while (more); }
public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { if (numMemoryPages < 2) { throw new IllegalArgumentException("Merger needs at least 2 memory pages."); } this.pairComparator = pairComparator; this.serializer1 = serializer1; this.serializer2 = serializer2; this.memoryManager = memoryManager; this.ioManager = ioManager; this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate()); this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate()); final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1; this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2, (numMemoryPages - numPagesForSpiller), parentTask); this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller); }
protected void crossMatchingGroup(Iterator<T1> values1, Iterator<T2> values2, FlatJoinFunction<T1, T2, O> joinFunction, Collector<O> collector) throws Exception { final T1 firstV1 = values1.next(); final T2 firstV2 = values2.next(); final boolean v1HasNext = values1.hasNext(); final boolean v2HasNext = values2.hasNext(); // check if one side is already empty // this check could be omitted if we put this in MatchTask. // then we can derive the local strategy (with build side). if (v1HasNext) { if (v2HasNext) { // both sides contain more than one value // TODO: Decide which side to spill and which to block! crossMwithNValues(firstV1, values1, firstV2, values2, joinFunction, collector); } else { crossSecond1withNValues(firstV2, firstV1, values1, joinFunction, collector); } } else { if (v2HasNext) { crossFirst1withNValues(firstV1, firstV2, values2, joinFunction, collector); } else { // both sides contain only one value joinFunction.join(firstV1, firstV2, collector); } } }
/** * Crosses a single value from the second side with N values, all sharing a common key. * Effectively realizes a <i>N:1</i> join. * * @param val1 The value form the <i>1</i> side. * @param firstValN The first of the values from the <i>N</i> side. * @param valsN Iterator over remaining <i>N</i> side values. * @throws Exception Forwards all exceptions thrown by the stub. */ private void crossSecond1withNValues(T2 val1, T1 firstValN, Iterator<T1> valsN, FlatJoinFunction<T1, T2, O> joinFunction, Collector<O> collector) throws Exception { T2 copy2 = createCopy(serializer2, val1, this.copy2); joinFunction.join(firstValN, copy2, collector); // set copy and join first element boolean more = true; do { final T1 nRec = valsN.next(); if (valsN.hasNext()) { copy2 = createCopy(serializer2, val1, this.copy2); joinFunction.join(nRec, copy2, collector); } else { joinFunction.join(nRec, val1, collector); more = false; } } while (more); }
@Override public void abort() { close(); }
public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { if (numMemoryPages < 2) { throw new IllegalArgumentException("Merger needs at least 2 memory pages."); } this.pairComparator = pairComparator; this.serializer1 = serializer1; this.serializer2 = serializer2; this.memoryManager = memoryManager; this.ioManager = ioManager; this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate()); this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate()); final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1; this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2, (numMemoryPages - numPagesForSpiller), parentTask); this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller); }
protected void crossMatchingGroup(Iterator<T1> values1, Iterator<T2> values2, FlatJoinFunction<T1, T2, O> joinFunction, Collector<O> collector) throws Exception { final T1 firstV1 = values1.next(); final T2 firstV2 = values2.next(); final boolean v1HasNext = values1.hasNext(); final boolean v2HasNext = values2.hasNext(); // check if one side is already empty // this check could be omitted if we put this in MatchTask. // then we can derive the local strategy (with build side). if (v1HasNext) { if (v2HasNext) { // both sides contain more than one value // TODO: Decide which side to spill and which to block! crossMwithNValues(firstV1, values1, firstV2, values2, joinFunction, collector); } else { crossSecond1withNValues(firstV2, firstV1, values1, joinFunction, collector); } } else { if (v2HasNext) { crossFirst1withNValues(firstV1, firstV2, values2, joinFunction, collector); } else { // both sides contain only one value joinFunction.join(firstV1, firstV2, collector); } } }
/** * Crosses a single value from the second side with N values, all sharing a common key. * Effectively realizes a <i>N:1</i> join. * * @param val1 The value form the <i>1</i> side. * @param firstValN The first of the values from the <i>N</i> side. * @param valsN Iterator over remaining <i>N</i> side values. * @throws Exception Forwards all exceptions thrown by the stub. */ private void crossSecond1withNValues(T2 val1, T1 firstValN, Iterator<T1> valsN, FlatJoinFunction<T1, T2, O> joinFunction, Collector<O> collector) throws Exception { T2 copy2 = createCopy(serializer2, val1, this.copy2); joinFunction.join(firstValN, copy2, collector); // set copy and join first element boolean more = true; do { final T1 nRec = valsN.next(); if (valsN.hasNext()) { copy2 = createCopy(serializer2, val1, this.copy2); joinFunction.join(nRec, copy2, collector); } else { joinFunction.join(nRec, val1, collector); more = false; } } while (more); }
@Override public void abort() { close(); }
public AbstractMergeIterator(MutableObjectIterator<T1> input1, MutableObjectIterator<T2> input2, TypeSerializer<T1> serializer1, TypeComparator<T1> comparator1, TypeSerializer<T2> serializer2, TypeComparator<T2> comparator2, TypePairComparator<T1, T2> pairComparator, MemoryManager memoryManager, IOManager ioManager, int numMemoryPages, AbstractInvokable parentTask) throws MemoryAllocationException { if (numMemoryPages < 2) { throw new IllegalArgumentException("Merger needs at least 2 memory pages."); } this.pairComparator = pairComparator; this.serializer1 = serializer1; this.serializer2 = serializer2; this.memoryManager = memoryManager; this.ioManager = ioManager; this.iterator1 = createKeyGroupedIterator(input1, serializer1, comparator1.duplicate()); this.iterator2 = createKeyGroupedIterator(input2, serializer2, comparator2.duplicate()); final int numPagesForSpiller = numMemoryPages > 20 ? 2 : 1; this.blockIt = new NonReusingBlockResettableIterator<>(this.memoryManager, this.serializer2, (numMemoryPages - numPagesForSpiller), parentTask); this.memoryForSpillingIterator = memoryManager.allocatePages(parentTask, numPagesForSpiller); }
protected void crossMatchingGroup(Iterator<T1> values1, Iterator<T2> values2, FlatJoinFunction<T1, T2, O> joinFunction, Collector<O> collector) throws Exception { final T1 firstV1 = values1.next(); final T2 firstV2 = values2.next(); final boolean v1HasNext = values1.hasNext(); final boolean v2HasNext = values2.hasNext(); // check if one side is already empty // this check could be omitted if we put this in MatchTask. // then we can derive the local strategy (with build side). if (v1HasNext) { if (v2HasNext) { // both sides contain more than one value // TODO: Decide which side to spill and which to block! crossMwithNValues(firstV1, values1, firstV2, values2, joinFunction, collector); } else { crossSecond1withNValues(firstV2, firstV1, values1, joinFunction, collector); } } else { if (v2HasNext) { crossFirst1withNValues(firstV1, firstV2, values2, joinFunction, collector); } else { // both sides contain only one value joinFunction.join(firstV1, firstV2, collector); } } }
/** * Crosses a single value from the first input with N values, all sharing a common key. * Effectively realizes a <i>1:N</i> join. * * @param val1 The value form the <i>1</i> side. * @param firstValN The first of the values from the <i>N</i> side. * @param valsN Iterator over remaining <i>N</i> side values. * @throws Exception Forwards all exceptions thrown by the stub. */ private void crossFirst1withNValues(final T1 val1, final T2 firstValN, final Iterator<T2> valsN, final FlatJoinFunction<T1, T2, O> joinFunction, final Collector<O> collector) throws Exception { T1 copy1 = createCopy(serializer1, val1, this.copy1); joinFunction.join(copy1, firstValN, collector); // set copy and join first element boolean more = true; do { final T2 nRec = valsN.next(); if (valsN.hasNext()) { copy1 = createCopy(serializer1, val1, this.copy1); joinFunction.join(copy1, nRec, collector); } else { joinFunction.join(val1, nRec, collector); more = false; } } while (more); }
@Override public void abort() { close(); }
/** * Crosses a single value from the first input with N values, all sharing a common key. * Effectively realizes a <i>1:N</i> join. * * @param val1 The value form the <i>1</i> side. * @param firstValN The first of the values from the <i>N</i> side. * @param valsN Iterator over remaining <i>N</i> side values. * @throws Exception Forwards all exceptions thrown by the stub. */ private void crossFirst1withNValues(final T1 val1, final T2 firstValN, final Iterator<T2> valsN, final FlatJoinFunction<T1, T2, O> joinFunction, final Collector<O> collector) throws Exception { T1 copy1 = createCopy(serializer1, val1, this.copy1); joinFunction.join(copy1, firstValN, collector); // set copy and join first element boolean more = true; do { final T2 nRec = valsN.next(); if (valsN.hasNext()) { copy1 = createCopy(serializer1, val1, this.copy1); joinFunction.join(copy1, nRec, collector); } else { joinFunction.join(val1, nRec, collector); more = false; } } while (more); }
/** * Crosses a single value from the first input with N values, all sharing a common key. * Effectively realizes a <i>1:N</i> join. * * @param val1 The value form the <i>1</i> side. * @param firstValN The first of the values from the <i>N</i> side. * @param valsN Iterator over remaining <i>N</i> side values. * @throws Exception Forwards all exceptions thrown by the stub. */ private void crossFirst1withNValues(final T1 val1, final T2 firstValN, final Iterator<T2> valsN, final FlatJoinFunction<T1, T2, O> joinFunction, final Collector<O> collector) throws Exception { T1 copy1 = createCopy(serializer1, val1, this.copy1); joinFunction.join(copy1, firstValN, collector); // set copy and join first element boolean more = true; do { final T2 nRec = valsN.next(); if (valsN.hasNext()) { copy1 = createCopy(serializer1, val1, this.copy1); joinFunction.join(copy1, nRec, collector); } else { joinFunction.join(val1, nRec, collector); more = false; } } while (more); }
/** * Crosses a single value from the first input with N values, all sharing a common key. * Effectively realizes a <i>1:N</i> join. * * @param val1 The value form the <i>1</i> side. * @param firstValN The first of the values from the <i>N</i> side. * @param valsN Iterator over remaining <i>N</i> side values. * @throws Exception Forwards all exceptions thrown by the stub. */ private void crossFirst1withNValues(final T1 val1, final T2 firstValN, final Iterator<T2> valsN, final FlatJoinFunction<T1, T2, O> joinFunction, final Collector<O> collector) throws Exception { T1 copy1 = createCopy(serializer1, val1, this.copy1); joinFunction.join(copy1, firstValN, collector); // set copy and join first element boolean more = true; do { final T2 nRec = valsN.next(); if (valsN.hasNext()) { copy1 = createCopy(serializer1, val1, this.copy1); joinFunction.join(copy1, nRec, collector); } else { joinFunction.join(val1, nRec, collector); more = false; } } while (more); }