org.apache.samza.application.descriptors.StreamApplicationDescriptor.getOutputStream java code examples

Add the Codota plugin to your IDE and get smart completions

private void myMethod () {
 FileOutputStream f =File file;new FileOutputStream(file)
String name;new FileOutputStream(name)
File file;new FileOutputStream(file, true)
Smart code suggestions by Codota
}

@Test(expected = IllegalStateException.class)
public void testGetSameOutputStreamTwice() {
 String streamId = "test-stream-1";
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd1 = sd.getOutputDescriptor(streamId, mock(Serde.class));
 GenericOutputDescriptor osd2 = sd.getOutputDescriptor(streamId, mock(Serde.class));
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd1);
   appDesc.getOutputStream(osd2); // should throw exception
  }, getConfig());
}

 private void sendToOutputStream(String queryLogicalId, String logicalOpId, String sinkStream, StreamApplicationDescriptor appDesc, TranslatorContext translatorContext, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(sinkStream);
  MessageStream<SamzaSqlRelMessage> stream = translatorContext.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(queryLogicalId, logicalOpId, sinkStream, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamId(), noOpKVSerde);
   OutputStream stm = outputMsgStreams.computeIfAbsent(sinkConfig.getSource(), v -> appDesc.getOutputStream(osd));
   outputStream.sendTo(stm);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamStreamJoin() {
 /**
  * Creates the following stream-stream join which is invalid due to partition count disagreement
  * between the 2 input streams.
  *
  *   input1 (64) --
  *                 |
  *                join -> output1 (8)
  *                 |
  *   input3 (32) --
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
   MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   messageStream1
     .join(messageStream3,
       mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(2), "j1")
     .sendTo(output1);
  }, config);
}

private StreamApplicationDescriptorImpl createStreamGraphWithJoinAndWindow() {
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor).map(m -> m);
   MessageStream<KV<Object, Object>> messageStream2 =
    appDesc.getInputStream(input2Descriptor)
      .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1")
      .filter(m -> true);
   MessageStream<KV<Object, Object>> messageStream3 =
    appDesc.getInputStream(input3Descriptor)
      .filter(m -> true)
      .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p2")
      .map(m -> m);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   OutputStream<KV<Object, Object>> output2 = appDesc.getOutputStream(output2Descriptor);
   messageStream1.map(m -> m)
     .filter(m -> true)
     .window(Windows.keyedTumblingWindow(m -> m, Duration.ofMillis(8), (Serde<KV<Object, Object>>) mock(Serde.class), (Serde<KV<Object, Object>>) mock(Serde.class)), "w1");
   messageStream2.map(m -> m)
     .filter(m -> true)
     .window(Windows.keyedTumblingWindow(m -> m, Duration.ofMillis(16), (Serde<KV<Object, Object>>) mock(Serde.class), (Serde<KV<Object, Object>>) mock(Serde.class)), "w2");
   messageStream1.join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofMillis(1600), "j1").sendTo(output1);
   messageStream3.join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofMillis(100), "j2").sendTo(output2);
   messageStream3.join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofMillis(252), "j3").sendTo(output2);
  }, config);
}

 private void sendToOutputStream(StreamApplicationDescriptor appDesc, TranslatorContext context, RelNode node, int queryId) {
  SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG);
  MessageStream<SamzaSqlRelMessage> stream = context.getMessageStream(node.getId());
  MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG, queryId));
  Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor();
  if (!tableDescriptor.isPresent()) {
   KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>());
   String systemName = sinkConfig.getSystemName();
   DelegatingSystemDescriptor
     sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new);
   GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamName(), noOpKVSerde);
   if (OutputMapFunction.logOutputStream == null) {
    OutputMapFunction.logOutputStream = appDesc.getOutputStream(osd);
   }
   outputStream.sendTo(OutputMapFunction.logOutputStream);
  } else {
   Table outputTable = appDesc.getTable(tableDescriptor.get());
   if (outputTable == null) {
    String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
    throw new SamzaException(msg);
   }
   outputStream.sendTo(outputTable);
  }
 }
}

@Test(expected = IllegalStateException.class)
public void testSetDefaultSystemDescriptorAfterGettingOutputStream() {
 String streamId = "test-stream-1";
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mock(Serde.class));
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd);
   appDesc.withDefaultSystem(sd); // should throw exception
  }, getConfig());
}

 OutputStream stm = outputMsgStreams.computeIfAbsent(source, v -> streamAppDesc.getOutputStream(osd));
 outputStream.sendTo(stm);
} else {

private StreamApplicationDescriptorImpl createSimpleGraph() {
 /**
  * a simple graph of partitionBy and map
  *
  * input1 -> partitionBy -> map -> output1
  *
  */
 return new StreamApplicationDescriptorImpl(appDesc-> {
   MessageStream<KV<Object, Object>> input1 = appDesc.getInputStream(input1Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   input1
     .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1")
     .map(kv -> kv)
     .sendTo(output1);
  }, config);
}

@Test
public void testGetOutputStreamWithKeyValueSerde() {
 String streamId = "test-stream-1";
 KVSerde mockKVSerde = mock(KVSerde.class);
 Serde mockKeySerde = mock(Serde.class);
 Serde mockValueSerde = mock(Serde.class);
 doReturn(mockKeySerde).when(mockKVSerde).getKeySerde();
 doReturn(mockValueSerde).when(mockKVSerde).getValueSerde();
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockKVSerde);
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd);
  }, getConfig());
 OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId);
 assertEquals(streamId, outputStreamImpl.getStreamId());
 assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId));
 assertEquals(mockKeySerde, outputStreamImpl.getKeySerde());
 assertEquals(mockValueSerde, outputStreamImpl.getValueSerde());
}

@Test(expected = IllegalArgumentException.class)
public void testGetOutputStreamWithNullSerde() {
 String streamId = "test-stream-1";
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, null);
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd);
  }, getConfig());
}

    .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p2")
    .map(m -> m);
OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
OutputStream<KV<Object, Object>> output2 = appDesc.getOutputStream(output2Descriptor);

@Test
public void testMaxPartitionLimit() {
 int partitionLimit = IntermediateStreamManager.MAX_INFERRED_PARTITIONS;
 ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
 StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> input1 = appDesc.getInputStream(input4Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   input1.partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1").map(kv -> kv).sendTo(output1);
  }, config);
 JobGraph jobGraph = (JobGraph) planner.plan(graphSpec);
 // Partitions should be the same as input1
 jobGraph.getIntermediateStreams().forEach(edge -> {
   assertEquals(partitionLimit, edge.getPartitionCount()); // max of input1 and output1
  });
}

GenericOutputDescriptor outputDescriptor = sd.getOutputDescriptor(outputStreamId, mock(Serde.class));
MessageStream<Object> inputStream = appDesc.getInputStream(inputDescriptor);
OutputStream<Object> outputStream = appDesc.getOutputStream(outputDescriptor);

@Test
public void testGetOutputStreamWithValueSerde() {
 String streamId = "test-stream-1";
 Serde mockValueSerde = mock(Serde.class);
 GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockValueSerde);
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getOutputStream(osd);
  }, getConfig());
 OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId);
 assertEquals(streamId, outputStreamImpl.getStreamId());
 assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId));
 assertTrue(outputStreamImpl.getKeySerde() instanceof NoOpSerde);
 assertEquals(mockValueSerde, outputStreamImpl.getValueSerde());
}

StreamApplication getRepartitionJoinStreamApplication() {
 return appDesc -> {
  MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor);
  MessageStream<KV<String, Object>> input2 = appDesc.getInputStream(input2Descriptor);
  OutputStream<KV<String, Object>> output = appDesc.getOutputStream(outputDescriptor);
  JoinFunction<String, Object, Object, KV<String, Object>> mockJoinFn = mock(JoinFunction.class);
  input1
    .partitionBy(KV::getKey, KV::getValue, defaultSerde, "p1")
    .map(kv -> kv.value)
    .join(input2.map(kv -> kv.value), mockJoinFn,
      new StringSerde(), new JsonSerdeV2<>(Object.class), new JsonSerdeV2<>(Object.class),
      Duration.ofHours(1), "j1")
    .sendTo(output);
 };
}

@Test
public void testMultipleSystemDescriptorForSameSystemName() {
 GenericSystemDescriptor sd1 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericSystemDescriptor sd2 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
 GenericInputDescriptor isd1 = sd1.getInputDescriptor("test-stream-1", mock(Serde.class));
 GenericInputDescriptor isd2 = sd2.getInputDescriptor("test-stream-2", mock(Serde.class));
 GenericOutputDescriptor osd1 = sd2.getOutputDescriptor("test-stream-3", mock(Serde.class));
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.getInputStream(isd1);
   try {
    appDesc.getInputStream(isd2);
    fail("Adding input stream with the same system name but different SystemDescriptor should have failed");
   } catch (IllegalStateException e) { }
   try {
    appDesc.getOutputStream(osd1);
    fail("adding output stream with the same system name but different SystemDescriptor should have failed");
   } catch (IllegalStateException e) { }
  }, getConfig());
 new StreamApplicationDescriptorImpl(appDesc -> {
   appDesc.withDefaultSystem(sd2);
   try {
    appDesc.getInputStream(isd1);
    fail("Adding input stream with the same system name as the default system but different SystemDescriptor should have failed");
   } catch (IllegalStateException e) { }
  }, getConfig());
}

MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor);
OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() {
 /**
  * Example stream-table join that is invalid due to disagreement in partition count
  * between the 2 input streams.
  *
  *    input1 (64) -> send-to-table t
  *
  *                   join-table t -> output1 (8)
  *                         |
  *    input2 (16) —————————
  *
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
   MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
    "table-id", new KVSerde(new StringSerde(), new StringSerde()));
   Table table = appDesc.getTable(tableDescriptor);
   messageStream1.sendTo(table);
   messageStream1
     .join(table, mock(StreamTableJoinFunction.class))
     .join(messageStream2,
       mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2")
     .sendTo(output1);
  }, config);
}

private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinWithSideInputs() {
 /**
  * Example stream-table join where table t is configured with input1 (64) as a side-input stream.
  *
  *                                   join-table t -> output1 (8)
  *                                        |
  *    input2 (16) -> partitionBy ("64") __|
  *
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
    "table-id", new KVSerde(new StringSerde(), new StringSerde()))
     .withSideInputs(Arrays.asList("input1"))
     .withSideInputsProcessor(mock(SideInputsProcessor.class));
   Table table = appDesc.getTable(tableDescriptor);
   messageStream2
     .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1")
     .join(table, mock(StreamTableJoinFunction.class))
     .sendTo(output1);
  }, config);
}

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoinWithSideInputs() {
 /**
  * Example stream-table join that is invalid due to disagreement in partition count between the
  * stream behind table t and another joined stream. Table t is configured with input2 (16) as
  * side-input stream.
  *
  *                   join-table t -> output1 (8)
  *                         |
  *    input1 (64) —————————
  *
  */
 return new StreamApplicationDescriptorImpl(appDesc -> {
   MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
   OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
   TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor(
    "table-id", new KVSerde(new StringSerde(), new StringSerde()))
     .withSideInputs(Arrays.asList("input2"))
     .withSideInputsProcessor(mock(SideInputsProcessor.class));
   Table table = appDesc.getTable(tableDescriptor);
   messageStream1
     .join(table, mock(StreamTableJoinFunction.class))
     .sendTo(output1);
  }, config);
}

Javadoc

Gets the OutputStream corresponding to the outputDescriptor.

An OutputStream>, obtained by calling this method with a descriptor with a KVSerde, can send messages of type KV. An OutputStream, obtained using a descriptor with any other Serde, can send messages of type M without a key.

A KVSerde or NoOpSerde may be used for the descriptor if the SystemProducer serializes the outgoing messages itself, and no prior serialization is required from the framework.

When sending messages to an OutputStream>, messages are partitioned using their serialized key. When sending messages to any other OutputStream, messages are partitioned using a null partition key.

Multiple invocations of this method with the same outputDescriptor will throw an IllegalStateException.

Popular methods of StreamApplicationDescriptor

getInputStream
Gets the input MessageStream corresponding to the inputDescriptor. A MessageStream, obtained by call
getTable
Gets the Table corresponding to the TableDescriptor. Multiple invocations of this method with the sa
withApplicationTaskContextFactory
getConfig
withApplicationContainerContextFactory
withDefaultSystem
withProcessorLifecycleListenerFactory

Popular in Java

Updating database using SQL prepared statement
runOnUiThread (Activity)
getSystemService (Context)
getResourceAsStream (ClassLoader)
Returns a stream for the resource with the specified name. See #getResource(String) for a descriptio
FileWriter (java.io)
Convenience class for writing character files. The constructors of this class assume that the defaul
ByteBuffer (java.nio)
A buffer for bytes. A byte buffer can be created in either one of the following ways: * #allocate(i
LinkedList (java.util)
Doubly-linked list implementation of the List and Dequeinterfaces. Implements all optional list oper
SortedMap (java.util)
A map that has its keys ordered. The sorting is according to either the natural ordering of its keys
Callable (java.util.concurrent)
A task that returns a result and may throw an exception. Implementors define a single method with no
JButton (javax.swing)

How to use getOutputStreammethodin org.apache.samza.application.descriptors.StreamApplicationDescriptor

Best Java code snippets using org.apache.samza.application.descriptors.StreamApplicationDescriptor.getOutputStream (Showing top 20 results out of 315)

How to use
getOutputStream
method
in
org.apache.samza.application.descriptors.StreamApplicationDescriptor