if (streams.containsKey(inputName)) { Input.StreamInput streamInput = streams.get(inputName); FormatSpecification formatSpec = streamInput.getBodyFormatSpec(); if (formatSpec != null) { return (JavaPairRDD<K, V>) sec.fromStream(streamInput.getName(), formatSpec, streamInput.getStartTime(), streamInput.getEndTime(), StructuredRecord.class); String decoderType = streamInput.getDecoderType(); if (decoderType == null) { return (JavaPairRDD<K, V>) sec.fromStream(streamInput.getName(), streamInput.getStartTime(), streamInput.getEndTime(), valueClass); } else { Class<StreamEventDecoder<K, V>> decoderClass = (Class<StreamEventDecoder<K, V>>) Thread.currentThread().getContextClassLoader().loadClass(decoderType); return sec.fromStream(streamInput.getName(), streamInput.getStartTime(), streamInput.getEndTime(), decoderClass, keyClass, valueClass); } catch (Exception e) {
@Override public void addInput(Input input, @Nullable Class<?> mapperCls) { if (input.getNamespace() != null && input.getNamespace().equals(NamespaceId.SYSTEM.getNamespace()) && !getProgram().getNamespaceId().equals(NamespaceId.SYSTEM.getNamespace())) { // trying to access system namespace from a program outside system namespace is not allowed throw new IllegalArgumentException(String.format("Accessing Input %s in system namespace " + "is not allowed from the namespace %s", input.getName(), getProgram().getNamespaceId())); } if (input instanceof Input.DatasetInput) { Input.DatasetInput datasetInput = (Input.DatasetInput) input; Input.InputFormatProviderInput createdInput = createInput(datasetInput); addInput(createdInput.getAlias(), createdInput.getInputFormatProvider(), mapperCls); } else if (input instanceof Input.StreamInput) { Input.StreamInput streamInput = (Input.StreamInput) input; String namespace = streamInput.getNamespace(); if (namespace == null) { namespace = getProgram().getNamespaceId(); } addInput(input.getAlias(), new StreamInputFormatProvider(new NamespaceId(namespace), streamInput, streamAdmin), mapperCls); } else if (input instanceof Input.InputFormatProviderInput) { addInput(input.getAlias(), ((Input.InputFormatProviderInput) input).getInputFormatProvider(), mapperCls); } else { // shouldn't happen unless user defines their own Input class throw new IllegalArgumentException(String.format("Input %s has unknown input class %s", input.getName(), input.getClass().getCanonicalName())); } }
@Override public Map<String, String> getInputFormatConfiguration() { try { StreamConfig streamConfig = streamAdmin.getConfig(streamId); Location streamPath = StreamUtils.createGenerationLocation(streamConfig.getLocation(), StreamUtils.getGeneration(streamConfig)); Configuration hConf = new Configuration(); hConf.clear(); AbstractStreamInputFormat.setStreamId(hConf, streamId); AbstractStreamInputFormat.setTTL(hConf, streamConfig.getTTL()); AbstractStreamInputFormat.setStreamPath(hConf, streamPath.toURI()); AbstractStreamInputFormat.setTimeRange(hConf, streamInput.getStartTime(), streamInput.getEndTime()); FormatSpecification formatSpec = streamInput.getBodyFormatSpec(); if (formatSpec != null) { AbstractStreamInputFormat.setBodyFormatSpecification(hConf, formatSpec); } else { String decoderType = streamInput.getDecoderType(); if (decoderType != null) { AbstractStreamInputFormat.setDecoderClassName(hConf, decoderType); } } return ConfigurationUtil.toMap(hConf); } catch (IOException e) { throw Throwables.propagate(e); } } }
/** * Sets the {@link StreamEventDecoder} to be used by the InputFormat for the given type. If the * {@link Input.StreamInput} already defined a {@link StreamEventDecoder} or {@link FormatSpecification}, * this method is a no-op. * * @param configuration configuration to update * @param type type for {@link StreamEventData} to decode to * @return the same configuration map as in the argument. */ public Map<String, String> setDecoderType(Map<String, String> configuration, Type type) { if (streamInput.getBodyFormatSpec() == null && streamInput.getDecoderType() == null) { Configuration hConf = new Configuration(); hConf.clear(); AbstractStreamInputFormat.inferDecoderClass(hConf, type); configuration.putAll(ConfigurationUtil.toMap(hConf)); } return configuration; }
public void addInput(String stageName, Input input) { if (input instanceof Input.DatasetInput) { // Note if input format provider is trackable then it comes in as DatasetInput Input.DatasetInput datasetInput = (Input.DatasetInput) input; addInput(stageName, datasetInput.getName(), datasetInput.getAlias(), datasetInput.getArguments(), datasetInput.getSplits()); } else if (input instanceof Input.InputFormatProviderInput) { Input.InputFormatProviderInput ifpInput = (Input.InputFormatProviderInput) input; addInput(stageName, ifpInput.getAlias(), new BasicInputFormatProvider(ifpInput.getInputFormatProvider().getInputFormatClassName(), ifpInput.getInputFormatProvider().getInputFormatConfiguration())); } else if (input instanceof Input.StreamInput) { Input.StreamInput streamInput = (Input.StreamInput) input; addInput(stageName, streamInput.getAlias(), streamInput); } }
public StreamInputFormatProvider(NamespaceId namespaceId, Input.StreamInput streamInput, StreamAdmin streamAdmin) { this.streamId = namespaceId.stream(streamInput.getName()); this.streamInput = streamInput; this.streamAdmin = streamAdmin; }