@Override public LongValue copy(LongValue from, LongValue reuse) { reuse.setValue(from.getValue()); return reuse; }
@Override public LongValue copy(LongValue from, LongValue reuse) { reuse.setValue(from.getValue()); return reuse; }
private final Value convertAvroPrimitiveToValue(Type type, Object avroRecord) { switch (type) { case STRING: sString.setValue((CharSequence) avroRecord); return sString; case INT: sInt.setValue((Integer) avroRecord); return sInt; case BOOLEAN: sBool.setValue((Boolean) avroRecord); return sBool; case DOUBLE: sDouble.setValue((Double) avroRecord); return sDouble; case FLOAT: sFloat.setValue((Float) avroRecord); return sFloat; case LONG: sLong.setValue((Long) avroRecord); return sLong; case NULL: return NullValue.getInstance(); default: throw new RuntimeException( "Type " + type + " for AvroInputFormat is not implemented. Open an issue on GitHub."); } }
@Override public void coGroup(Iterator<Record> candidates, Iterator<Record> current, Collector<Record> out) throws Exception { if (!current.hasNext()) { throw new Exception("Error: Id not encountered before."); } Record old = current.next(); long oldId = old.getField(1, LongValue.class).getValue(); long minimumComponentID = Long.MAX_VALUE; while (candidates.hasNext()) { long candidateComponentID = candidates.next().getField(1, LongValue.class).getValue(); if (candidateComponentID < minimumComponentID) { minimumComponentID = candidateComponentID; } } if (minimumComponentID < oldId) { newComponentId.setValue(minimumComponentID); old.setField(1, newComponentId); out.collect(old); } }
@Override public void coGroup(Iterator<Record> candidates, Iterator<Record> current, Collector<Record> out) throws Exception { if (!current.hasNext()) { throw new Exception("Error: Id not encountered before."); } Record old = current.next(); long oldId = old.getField(1, LongValue.class).getValue(); long minimumComponentID = Long.MAX_VALUE; while (candidates.hasNext()) { long candidateComponentID = candidates.next().getField(1, LongValue.class).getValue(); if (candidateComponentID < minimumComponentID) { minimumComponentID = candidateComponentID; } } if (minimumComponentID < oldId) { newComponentId.setValue(minimumComponentID); old.setField(1, newComponentId); out.collect(old); } } }
@Override public void reduce(Iterator<Record> records, Collector<Record> out) { final Record first = records.next(); final long vertexID = first.getField(0, LongValue.class).getValue(); long minimumComponentID = first.getField(1, LongValue.class).getValue(); while (records.hasNext()) { long candidateComponentID = records.next().getField(1, LongValue.class).getValue(); if (candidateComponentID < minimumComponentID) { minimumComponentID = candidateComponentID; } } this.vertexId.setValue(vertexID); this.minComponentId.setValue(minimumComponentID); this.result.setField(0, this.vertexId); this.result.setField(1, this.minComponentId); out.collect(this.result); } }
@Override public void join(Record pageWithRank, Record adjacencyList, Collector<Record> collector) { rank = pageWithRank.getField(1, rank); adjacentNeighbors = adjacencyList.getField(1, adjacentNeighbors); int numNeighbors = adjacentNeighbors.size(); double rankToDistribute = rank.getValue() / (double) numNeighbors; partialRank.setValue(rankToDistribute); record.setField(1, partialRank); boolean isFailure = currentIteration == failingIteration && failingWorkers.contains(workerIndex); for (int n = 0; n < numNeighbors; n++) { vertexID.setValue(adjacentNeighbors.getQuick(n)); record.setField(0, vertexID); if (isFailure) { if (random.nextDouble() >= messageLoss) { collector.collect(record); } } else { collector.collect(record); } } } }
vertexID.setValue(arrayView.element()); } else { adjacentVertices.setQuick(pos - 1, arrayView.element());
vertexID.setValue(arrayView.element()); } else { adjacentVertices.setQuick(pos - 1, arrayView.element());
@Override public void map(Record record, Collector<Record> out) throws Exception { for (Record model : this.models) { // compute dot product between model and pair long product = 0; for (int i = 1; i <= NUM_FEATURES; i++) { product += model.getField(i, this.lft).getValue() * record.getField(i, this.rgt).getValue(); } this.prd.setValue(product); // construct result this.result.copyFrom(model, new int[] { 0 }, new int[] { 0 }); this.result.copyFrom(record, new int[] { 0 }, new int[] { 1 }); this.result.setField(2, this.prd); // emit result out.collect(this.result); } } }
@Override public void join(Record pageWithRank, Record adjacencyList, Collector<Record> collector) throws Exception { rank = pageWithRank.getField(1, rank); adjacentNeighbors = adjacencyList.getField(1, adjacentNeighbors); int numNeighbors = adjacentNeighbors.size(); double rankToDistribute = rank.getValue() / (double) numNeighbors; partialRank.setValue(rankToDistribute); record.setField(1, partialRank); for (int n = 0; n < numNeighbors; n++) { vertexID.setValue(adjacentNeighbors.getQuick(n)); record.setField(0, vertexID); collector.collect(record); } } }
@Override public void join(Record pageWithRank, Record edges, Collector<Record> out) throws Exception { rank = pageWithRank.getField(1, rank); adjacentNeighbors = edges.getField(1, adjacentNeighbors); int numNeighbors = adjacentNeighbors.size(); double rankToDistribute = rank.getValue() / (double) numNeighbors; partialRank.setValue(rankToDistribute); record.setField(1, partialRank); for (int n = 0; n < numNeighbors; n++) { vertexID.setValue(adjacentNeighbors.getQuick(n)); record.setField(0, vertexID); out.collect(record); } } }
@Override public Record readRecord(Record target, byte[] bytes, int offset, int numBytes) { arrayView.set(bytes, offset, numBytes); try { arrayView.next(); vertexID.setValue(arrayView.element()); if (arrayView.next()) { isDangling.set(arrayView.element() == DANGLING_MARKER); } else { isDangling.set(false); } } catch (NumberFormatException e) { throw new RuntimeException("Error parsing " + arrayView.toString(), e); } target.clear(); target.addField(vertexID); target.addField(initialRank); target.addField(isDangling); return target; } }
@Override public Record readRecord(Record target, byte[] bytes, int offset, int numBytes) { arrayView.set(bytes, offset, numBytes); try { arrayView.next(); vertexID.setValue(arrayView.element()); if (arrayView.next()) { isDangling.set(arrayView.element() == DANGLING_MARKER); } else { isDangling.set(false); } } catch (NumberFormatException e) { throw new RuntimeException("Error parsing " + arrayView.toString(), e); } target.clear(); target.addField(vertexID); target.addField(initialRank); target.addField(isDangling); return target; } }