- Add the Codota plugin to your IDE and get smart completions
private void myMethod () {FileOutputStream f =
File file;new FileOutputStream(file)
String name;new FileOutputStream(name)
File file;new FileOutputStream(file, true)
- Smart code suggestions by Codota
}
@Test(expected = IllegalStateException.class) public void testGetTableWithBadId() { Config mockConfig = getConfig(); new StreamApplicationDescriptorImpl(appDesc -> { BaseTableDescriptor mockTableDescriptor = mock(BaseTableDescriptor.class); when(mockTableDescriptor.getTableId()).thenReturn("my.table"); appDesc.getTable(mockTableDescriptor); }, mockConfig); }
@Test public void testGetTable() throws Exception { Config mockConfig = getConfig(); String tableId = "t1"; BaseTableDescriptor mockTableDescriptor = mock(BaseTableDescriptor.class); when(mockTableDescriptor.getTableId()).thenReturn(tableId); AtomicReference<TableImpl> table = new AtomicReference<>(); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { table.set((TableImpl) appDesc.getTable(mockTableDescriptor)); }, mockConfig); assertEquals(tableId, table.get().getTableId()); }
private Table loadLocalTable(boolean isTablePosOnRight, List<Integer> tableKeyIds, LogicalJoin join, TranslatorContext context) { RelNode relNode = isTablePosOnRight ? join.getRight() : join.getLeft(); MessageStream<SamzaSqlRelMessage> relOutputStream = context.getMessageStream(relNode.getId()); SqlIOConfig sourceConfig = resolveSourceConfig(relNode); if (!sourceConfig.getTableDescriptor().isPresent()) { String errMsg = "Failed to resolve table source in join operation: node=" + relNode; log.error(errMsg); throw new SamzaException(errMsg); } // Create a table backed by RocksDb store with the fields in the join condition as composite key and relational // message as the value. Send the messages from the input stream denoted as 'table' to the created table store. Table<KV<SamzaSqlCompositeKey, SamzaSqlRelMessage>> table = context.getStreamAppDescriptor().getTable(sourceConfig.getTableDescriptor().get()); relOutputStream .map(m -> new KV(createSamzaSqlCompositeKey(m, tableKeyIds), m)) .sendTo(table); return table; } }
private void sendToOutputStream(String queryLogicalId, String logicalOpId, String sinkStream, StreamApplicationDescriptor appDesc, TranslatorContext translatorContext, RelNode node, int queryId) { SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(sinkStream); MessageStream<SamzaSqlRelMessage> stream = translatorContext.getMessageStream(node.getId()); MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(queryLogicalId, logicalOpId, sinkStream, queryId)); Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor(); if (!tableDescriptor.isPresent()) { KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()); String systemName = sinkConfig.getSystemName(); DelegatingSystemDescriptor sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new); GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamId(), noOpKVSerde); OutputStream stm = outputMsgStreams.computeIfAbsent(sinkConfig.getSource(), v -> appDesc.getOutputStream(osd)); outputStream.sendTo(stm); } else { Table outputTable = appDesc.getTable(tableDescriptor.get()); if (outputTable == null) { String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource(); throw new SamzaException(msg); } outputStream.sendTo(outputTable); } } }
outputStream.sendTo(stm); } else { Table outputTable = streamAppDesc.getTable(tableDescriptor.get()); if (outputTable == null) { String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource();
private void sendToOutputStream(StreamApplicationDescriptor appDesc, TranslatorContext context, RelNode node, int queryId) { SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG); MessageStream<SamzaSqlRelMessage> stream = context.getMessageStream(node.getId()); MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG, queryId)); Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor(); if (!tableDescriptor.isPresent()) { KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()); String systemName = sinkConfig.getSystemName(); DelegatingSystemDescriptor sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new); GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamName(), noOpKVSerde); if (OutputMapFunction.logOutputStream == null) { OutputMapFunction.logOutputStream = appDesc.getOutputStream(osd); } outputStream.sendTo(OutputMapFunction.logOutputStream); } else { Table outputTable = appDesc.getTable(tableDescriptor.get()); if (outputTable == null) { String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource(); throw new SamzaException(msg); } outputStream.sendTo(outputTable); } } }
context.getStreamAppDescriptor().getTable(sourceTableConfig.getTableDescriptor().get());
private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() { /** * Example stream-table join that is invalid due to disagreement in partition count * between the 2 input streams. * * input1 (64) -> send-to-table t * * join-table t -> output1 (8) * | * input2 (16) ————————— * */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())); Table table = appDesc.getTable(tableDescriptor); messageStream1.sendTo(table); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2") .sendTo(output1); }, config); }
Table table = appDesc.getTable(tableDescriptor);
private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinWithSideInputs() { /** * Example stream-table join where table t is configured with input1 (64) as a side-input stream. * * join-table t -> output1 (8) * | * input2 (16) -> partitionBy ("64") __| * */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())) .withSideInputs(Arrays.asList("input1")) .withSideInputsProcessor(mock(SideInputsProcessor.class)); Table table = appDesc.getTable(tableDescriptor); messageStream2 .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1") .join(table, mock(StreamTableJoinFunction.class)) .sendTo(output1); }, config); }
private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoinWithSideInputs() { /** * Example stream-table join that is invalid due to disagreement in partition count between the * stream behind table t and another joined stream. Table t is configured with input2 (16) as * side-input stream. * * join-table t -> output1 (8) * | * input1 (64) ————————— * */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())) .withSideInputs(Arrays.asList("input2")) .withSideInputsProcessor(mock(SideInputsProcessor.class)); Table table = appDesc.getTable(tableDescriptor); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .sendTo(output1); }, config); }
private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinAndSendToSameTable() { /** * A special example of stream-table join where a stream is joined with a table, and the result is * sent to the same table. This example is necessary to ensure {@link ExecutionPlanner} does not * get stuck traversing the virtual cycle between stream-table-join and send-to-table operator specs * indefinitely. * * The reason such virtual cycle is present is to support computing partitions of intermediate * streams participating in stream-table joins. Please, refer to SAMZA SEP-16 for more details. */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())); Table table = appDesc.getTable(tableDescriptor); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .sendTo(table); }, config); }