newData.foreachPartition(p -> {}); pastData.foreachPartition(p -> {});
@Test public void foreachPartition() { LongAccumulator accum = sc.sc().longAccumulator(); JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World")); rdd.foreachPartition(iter -> { while (iter.hasNext()) { iter.next(); accum.add(1); } }); assertEquals(2, accum.value().intValue()); }
@Test public void foreachPartition() { LongAccumulator accum = sc.sc().longAccumulator(); JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World")); rdd.foreachPartition(iter -> { while (iter.hasNext()) { iter.next(); accum.add(1); } }); assertEquals(2, accum.value().intValue()); }
@Test public void foreachPartition() { LongAccumulator accum = sc.sc().longAccumulator(); JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World")); rdd.foreachPartition(iter -> { while (iter.hasNext()) { iter.next(); accum.add(1); } }); assertEquals(2, accum.value().intValue()); }
@Override public void call(JavaRDD<MessageAndMetadata<byte[]>> rdd) throws Exception { rdd.foreachPartition(new VoidFunction<Iterator<MessageAndMetadata<byte[]>>>() {
@Override public Void call(JavaRDD<String> rdd) { rdd.foreachPartition(new VoidFunction<Iterator<String>>() { @Override public void call(Iterator<String> items) throws Exception { FileWriter fw; BufferedWriter bw = null; try { fw = new FileWriter(file.getAbsoluteFile()); bw = new BufferedWriter(fw); while (items.hasNext()) { bw.append(items.next() + System.lineSeparator()); } } catch (IOException ioe) { throw new RuntimeException(ioe); } finally { if (bw != null) { bw.close(); } } } }); return null; } });
private void applyMutations(JavaRDD<Row> planned, Config outputConfig) { planned.foreachPartition(new ApplyMutationsForPartitionFunction(outputConfig, accumulators)); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, SparkExecutor sparkExecutor, OptimizationContext.OperatorContext operatorContext) { RddChannel.Instance input = (RddChannel.Instance) inputs[0]; final JavaRDD<Object> rdd = input.provideRdd(); final JavaRDD<Object> cachedRdd = rdd.cache(); cachedRdd.foreachPartition(iterator -> { }); RddChannel.Instance output = (RddChannel.Instance) outputs[0]; output.accept(cachedRdd, sparkExecutor); return ExecutionOperator.modelQuasiEagerExecution(inputs, outputs, operatorContext); }
newData.foreachPartition(p -> {}); pastData.foreachPartition(p -> {});
static void streamSpansToStorage( JavaDStream<byte[]> stream, ReadSpans readSpans, AdjustAndConsumeSpansSharingTraceId adjustAndConsumeSpansSharingTraceId ) { JavaDStream<Span> spans = stream.flatMap(readSpans); // TODO: plug in some filter to drop spans regardless of trace ID // spans = spans.filter(spanFilter); JavaPairDStream<String, Iterable<Span>> tracesById = spans .mapToPair(s -> new Tuple2<>(Util.toLowerHex(s.traceIdHigh, s.traceId), s)) .groupByKey(); tracesById.foreachRDD(rdd -> { rdd.values().foreachPartition(adjustAndConsumeSpansSharingTraceId); }); }
@Override public void applyBulkMutations(List<Tuple2<MutationType, Dataset<Row>>> planned) { for (Tuple2<MutationType, Dataset<Row>> mutation : planned) { MutationType mutationType = mutation._1(); Dataset<Row> mutationDF = mutation._2(); if (mutationType.equals(MutationType.INSERT)) { mutationDF.javaRDD().foreachPartition(new SendRowToKafkaFunction(config)); } } }
if (isS3) { final String s3FinalEndpointUrl = s3EndpointUrl; fileRDD.foreachPartition(uri -> { S3FileSystem fs = initializeS3FS(s3FinalEndpointUrl); List<URI> inputFiles = new ArrayList<URI>(); fileRDD.foreachPartition(uri -> { processInput( configFile,
new WriteUnsortedDataFunction(store.getTempFilesDir(), store.getSchemaUtils(), groupToSplitPoints); input .foreachPartition(writeUnsortedDataFunction); LOGGER.debug("Finished writing the unsorted Parquet data to {}", tempDataDirString);
data.foreachPartition(new VoidFunction<Iterator<Record>>() { private static final long serialVersionUID = -4641037124928675165L;
printLogTime("Training start..."); dummydata.foreachPartition(new VoidFunction<Iterator<Integer>>() { private static final long serialVersionUID = -4641037124928675165L;
printLogTime("Training start..."); dummydata.foreachPartition(new VoidFunction<Iterator<Integer>>() { private static final long serialVersionUID = -4641037124928675165L;
printLogTime("Training start..."); data.foreachPartition(new VoidFunction<Iterator<Record>>() { private static final long serialVersionUID = -4641037124928675165L;