/** * calculate the mapping from output streams to input streams * @param specGraph the user {@link OperatorSpecGraph} * @return mapping from output streams to input streams */ static Multimap<SystemStream, SystemStream> getIntermediateToInputStreamsMap( OperatorSpecGraph specGraph, StreamConfig streamConfig) { Multimap<SystemStream, SystemStream> outputToInputStreams = HashMultimap.create(); specGraph.getInputOperators().entrySet().stream() .forEach(entry -> { SystemStream systemStream = streamConfig.streamIdToSystemStream(entry.getKey()); computeOutputToInput(systemStream, entry.getValue(), outputToInputStreams, streamConfig); }); return outputToInputStreams; }
/** * calculate the mapping from output streams to input streams * @param specGraph the user {@link OperatorSpecGraph} * @return mapping from output streams to input streams */ static Multimap<SystemStream, SystemStream> getIntermediateToInputStreamsMap( OperatorSpecGraph specGraph, StreamConfig streamConfig) { Multimap<SystemStream, SystemStream> outputToInputStreams = HashMultimap.create(); specGraph.getInputOperators().entrySet().stream() .forEach(entry -> { SystemStream systemStream = streamConfig.streamIdToSystemStream(entry.getKey()); computeOutputToInput(systemStream, entry.getValue(), outputToInputStreams, streamConfig); }); return outputToInputStreams; }
/** * calculate the mapping from output streams to input streams * @param specGraph the user {@link OperatorSpecGraph} * @return mapping from output streams to input streams */ static Multimap<SystemStream, SystemStream> getIntermediateToInputStreamsMap( OperatorSpecGraph specGraph, StreamConfig streamConfig) { Multimap<SystemStream, SystemStream> outputToInputStreams = HashMultimap.create(); specGraph.getInputOperators().entrySet().stream() .forEach(entry -> { SystemStream systemStream = streamConfig.streamIdToSystemStream(entry.getKey()); computeOutputToInput(systemStream, entry.getValue(), outputToInputStreams, streamConfig); }); return outputToInputStreams; }
/** * calculate the mapping from output streams to input streams * @param specGraph the user {@link OperatorSpecGraph} * @return mapping from output streams to input streams */ static Multimap<SystemStream, SystemStream> getIntermediateToInputStreamsMap( OperatorSpecGraph specGraph, StreamConfig streamConfig) { Multimap<SystemStream, SystemStream> outputToInputStreams = HashMultimap.create(); specGraph.getInputOperators().entrySet().stream() .forEach(entry -> { SystemStream systemStream = streamConfig.streamIdToSystemStream(entry.getKey()); computeOutputToInput(systemStream, entry.getValue(), outputToInputStreams, streamConfig); }); return outputToInputStreams; }
/** * calculate the mapping from output streams to input streams * @param specGraph the user {@link OperatorSpecGraph} * @return mapping from output streams to input streams */ static Multimap<SystemStream, SystemStream> getIntermediateToInputStreamsMap( OperatorSpecGraph specGraph, StreamConfig streamConfig) { Multimap<SystemStream, SystemStream> outputToInputStreams = HashMultimap.create(); specGraph.getInputOperators().entrySet().stream() .forEach(entry -> { SystemStream systemStream = streamConfig.streamIdToSystemStream(entry.getKey()); computeOutputToInput(systemStream, entry.getValue(), outputToInputStreams, streamConfig); }); return outputToInputStreams; }
private static void computeOutputToInput(SystemStream input, OperatorSpec opSpec, Multimap<SystemStream, SystemStream> outputToInputStreams, StreamConfig streamConfig) { if (opSpec instanceof PartitionByOperatorSpec) { PartitionByOperatorSpec spec = (PartitionByOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); outputToInputStreams.put(systemStream, input); } else if (opSpec instanceof BroadcastOperatorSpec) { BroadcastOperatorSpec spec = (BroadcastOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); outputToInputStreams.put(systemStream, input); } else { Collection<OperatorSpec> nextOperators = opSpec.getRegisteredOperatorSpecs(); nextOperators.forEach(spec -> computeOutputToInput(input, spec, outputToInputStreams, streamConfig)); } }
private static void computeOutputToInput(SystemStream input, OperatorSpec opSpec, Multimap<SystemStream, SystemStream> outputToInputStreams, StreamConfig streamConfig) { if (opSpec instanceof PartitionByOperatorSpec) { PartitionByOperatorSpec spec = (PartitionByOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); outputToInputStreams.put(systemStream, input); } else if (opSpec instanceof BroadcastOperatorSpec) { BroadcastOperatorSpec spec = (BroadcastOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); outputToInputStreams.put(systemStream, input); } else { Collection<OperatorSpec> nextOperators = opSpec.getRegisteredOperatorSpecs(); nextOperators.forEach(spec -> computeOutputToInput(input, spec, outputToInputStreams, streamConfig)); } }
private static void computeOutputToInput(SystemStream input, OperatorSpec opSpec, Multimap<SystemStream, SystemStream> outputToInputStreams, StreamConfig streamConfig) { if (opSpec instanceof PartitionByOperatorSpec) { PartitionByOperatorSpec spec = (PartitionByOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); outputToInputStreams.put(systemStream, input); } else if (opSpec instanceof BroadcastOperatorSpec) { BroadcastOperatorSpec spec = (BroadcastOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); outputToInputStreams.put(systemStream, input); } else { Collection<OperatorSpec> nextOperators = opSpec.getRegisteredOperatorSpecs(); nextOperators.forEach(spec -> computeOutputToInput(input, spec, outputToInputStreams, streamConfig)); } }
/** * Gets the {@link SystemStream} corresponding to the provided stream, which may be * a streamId, or stream name of the format systemName.streamName. * * @param config the config for the job * @param stream the stream name or id to get the {@link SystemStream} for. * @return the {@link SystemStream} for the stream */ public static SystemStream getSystemStreamFromNameOrId(Config config, String stream) { String[] parts = stream.split("\\."); if (parts.length == 0 || parts.length > 2) { throw new SamzaException( String.format("Invalid stream %s. Expected to be of the format streamId or systemName.streamName", stream)); } if (parts.length == 1) { return new StreamConfig(config).streamIdToSystemStream(stream); } else { return new SystemStream(parts[0], parts[1]); } }
private static void computeOutputToInput(SystemStream input, OperatorSpec opSpec, Multimap<SystemStream, SystemStream> outputToInputStreams, StreamConfig streamConfig) { if (opSpec instanceof PartitionByOperatorSpec) { PartitionByOperatorSpec spec = (PartitionByOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); outputToInputStreams.put(systemStream, input); } else if (opSpec instanceof BroadcastOperatorSpec) { BroadcastOperatorSpec spec = (BroadcastOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); outputToInputStreams.put(systemStream, input); } else { Collection<OperatorSpec> nextOperators = opSpec.getRegisteredOperatorSpecs(); nextOperators.forEach(spec -> computeOutputToInput(input, spec, outputToInputStreams, streamConfig)); } }
private static void computeOutputToInput(SystemStream input, OperatorSpec opSpec, Multimap<SystemStream, SystemStream> outputToInputStreams, StreamConfig streamConfig) { if (opSpec instanceof PartitionByOperatorSpec) { PartitionByOperatorSpec spec = (PartitionByOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); outputToInputStreams.put(systemStream, input); } else if (opSpec instanceof BroadcastOperatorSpec) { BroadcastOperatorSpec spec = (BroadcastOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); outputToInputStreams.put(systemStream, input); } else { Collection<OperatorSpec> nextOperators = opSpec.getRegisteredOperatorSpecs(); nextOperators.forEach(spec -> computeOutputToInput(input, spec, outputToInputStreams, streamConfig)); } }
/** * Gets the {@link SystemStream} corresponding to the provided stream, which may be * a streamId, or stream name of the format systemName.streamName. * * @param config the config for the job * @param stream the stream name or id to get the {@link SystemStream} for. * @return the {@link SystemStream} for the stream */ public static SystemStream getSystemStreamFromNameOrId(Config config, String stream) { String[] parts = stream.split("\\."); if (parts.length == 0 || parts.length > 2) { throw new SamzaException( String.format("Invalid stream %s. Expected to be of the format streamId or systemName.streamName", stream)); } if (parts.length == 1) { return new StreamConfig(config).streamIdToSystemStream(stream); } else { return new SystemStream(parts[0], parts[1]); } }
/** * Gets the {@link SystemStream} corresponding to the provided stream, which may be * a streamId, or stream name of the format systemName.streamName. * * @param config the config for the job * @param stream the stream name or id to get the {@link SystemStream} for. * @return the {@link SystemStream} for the stream */ public static SystemStream getSystemStreamFromNameOrId(Config config, String stream) { String[] parts = stream.split("\\."); if (parts.length == 0 || parts.length > 2) { throw new SamzaException( String.format("Invalid stream %s. Expected to be of the format streamId or systemName.streamName", stream)); } if (parts.length == 1) { return new StreamConfig(config).streamIdToSystemStream(stream); } else { return new SystemStream(parts[0], parts[1]); } }
/** * Gets the {@link SystemStream} corresponding to the provided stream, which may be * a streamId, or stream name of the format systemName.streamName. * * @param config the config for the job * @param stream the stream name or id to get the {@link SystemStream} for. * @return the {@link SystemStream} for the stream */ public static SystemStream getSystemStreamFromNameOrId(Config config, String stream) { String[] parts = stream.split("\\."); if (parts.length == 0 || parts.length > 2) { throw new SamzaException( String.format("Invalid stream %s. Expected to be of the format streamId or systemName.streamName", stream)); } if (parts.length == 1) { return new StreamConfig(config).streamIdToSystemStream(stream); } else { return new SystemStream(parts[0], parts[1]); } }
/** * Gets the {@link SystemStream} corresponding to the provided stream, which may be * a streamId, or stream name of the format systemName.streamName. * * @param config the config for the job * @param stream the stream name or id to get the {@link SystemStream} for. * @return the {@link SystemStream} for the stream */ public static SystemStream getSystemStreamFromNameOrId(Config config, String stream) { String[] parts = stream.split("\\."); if (parts.length == 0 || parts.length > 2) { throw new SamzaException( String.format("Invalid stream %s. Expected to be of the format streamId or systemName.streamName", stream)); } if (parts.length == 1) { return new StreamConfig(config).streamIdToSystemStream(stream); } else { return new SystemStream(parts[0], parts[1]); } }
SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId); InputOperatorImpl inputOperatorImpl = (InputOperatorImpl) createAndRegisterOperatorImpl(null, inputOpSpec, systemStream, context);
SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId); InputOperatorImpl inputOperatorImpl = (InputOperatorImpl) createAndRegisterOperatorImpl(null, inputOpSpec, systemStream, context);
SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId); InputOperatorImpl inputOperatorImpl = (InputOperatorImpl) createAndRegisterOperatorImpl(null, inputOpSpec, systemStream, context);
SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId); InputOperatorImpl inputOperatorImpl = (InputOperatorImpl) createAndRegisterOperatorImpl(null, inputOpSpec, systemStream, context);
SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId); InputOperatorImpl inputOperatorImpl = (InputOperatorImpl) createAndRegisterOperatorImpl(null, inputOpSpec, systemStream, context);