@Override public RecordReader createRecordReader(final FlowFile flowFile, final ComponentLog logger) throws IOException, MalformedRecordException, SchemaNotFoundException { if (recordReader != null) { throw new IllegalStateException("Cannot create RecordReader because already created"); } final InputStream in = socketChannel.socket().getInputStream(); recordReader = readerFactory.createRecordReader(flowFile, in, logger); return recordReader; }
/** * Create a RecordReader instance to read records from specified InputStream. * This method calls {@link #createRecordReader(Map, InputStream, ComponentLog)} with Attributes of the specified FlowFile. * @param flowFile Attributes of this FlowFile are used to resolve Record Schema via Expression Language dynamically. This can be null. * @param in InputStream containing Records. This can be null or empty stream. * @param logger A logger bind to a component * @return Created RecordReader instance */ default RecordReader createRecordReader(FlowFile flowFile, InputStream in, ComponentLog logger) throws MalformedRecordException, IOException, SchemaNotFoundException { return createRecordReader(flowFile == null ? Collections.emptyMap() : flowFile.getAttributes(), in, logger); }
@Override public RecordReader createRecordReader(final FlowFile flowFile, final ComponentLog logger) throws IOException, MalformedRecordException, SchemaNotFoundException { if (recordReader != null) { throw new IllegalStateException("Cannot create RecordReader because already created"); } final InputStream in = new SSLSocketChannelInputStream(sslSocketChannel); recordReader = readerFactory.createRecordReader(flowFile, in, logger); return recordReader; }
@Override public RecordReader createRecordReader(Map<String, String> variables, InputStream in, ComponentLog logger) throws MalformedRecordException, IOException, SchemaNotFoundException { if (recordFactory.get() != null) { try { return recordFactory.get().createRecordReader(variables, in, logger); } catch (UndeclaredThrowableException ute) { throw new IOException(ute.getCause()); } } return null; }
@Override public void reset() { if (rawIn != null) { try { rawIn.close(); } catch (final Exception e) { logger.warn("Could not close FlowFile's input due to " + e, e); } } rawIn = session.read(flowFile); try { recordParser = recordParserFactory.createRecordReader(flowFile, rawIn, logger); } catch (final Exception e) { throw new ProcessException("Failed to reset stream", e); } }
RecordReaderFactory factory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class); final Integer limit = context.getProperty(LIMIT).evaluateAttributeExpressions(flowFile).asInteger(); RecordReader reader = factory.createRecordReader(flowFile, is, getLogger());
return new Tuple<>(null, readerFactory.createRecordReader(recordReaderVariables, in, getLogger())); } catch (Exception e) { return new Tuple<>(e, null);
private Record handleResponse(InputStream is, Map<String, String> context) throws SchemaNotFoundException, MalformedRecordException, IOException { try (RecordReader reader = readerFactory.createRecordReader(context, is, getLogger())) { Record record = reader.nextRecord(); if (recordPath != null) { Optional<FieldValue> fv = recordPath.evaluate(record).getSelectedFields().findFirst(); if (fv.isPresent()) { FieldValue fieldValue = fv.get(); RecordSchema schema = new SimpleRecordSchema(Collections.singletonList(fieldValue.getField())); Record temp; Object value = fieldValue.getValue(); if (value instanceof Record) { temp = (Record) value; } else if (value instanceof Map) { temp = new MapRecord(schema, (Map<String, Object>) value); } else { Map<String, Object> val = new HashMap<>(); val.put(fieldValue.getField().getFieldName(), value); temp = new MapRecord(schema, val); } record = temp; } else { record = null; } } return record; } catch (Exception ex) { is.close(); throw ex; } }
final RecordReader reader = readerFactory.createRecordReader(flowFile, in, getLogger())) {
@Override public RelDataType getRowType(final RelDataTypeFactory typeFactory) { if (relDataType != null) { return relDataType; } RecordSchema schema; try (final InputStream in = session.read(flowFile)) { final RecordReader recordParser = recordParserFactory.createRecordReader(flowFile, in, logger); schema = recordParser.getSchema(); } catch (final Exception e) { throw new ProcessException("Failed to determine schema of data records for " + flowFile, e); } final List<String> names = new ArrayList<>(); final List<RelDataType> types = new ArrayList<>(); final JavaTypeFactory javaTypeFactory = (JavaTypeFactory) typeFactory; for (final RecordField field : schema.getFields()) { names.add(field.getFieldName()); final RelDataType relDataType = getRelDataType(field.getDataType(), javaTypeFactory); types.add(javaTypeFactory.createTypeWithNullability(relDataType, field.isNullable())); } logger.debug("Found Schema: {}", new Object[] {schema}); if (recordSchema == null) { recordSchema = schema; } relDataType = typeFactory.createStructType(Pair.zip(names, types)); return relDataType; }
@Override public void process(final InputStream in, final OutputStream out) throws IOException { try (final RecordReader reader = readerFactory.createRecordReader(originalAttributes, in, getLogger())) { final RecordSchema writeSchema = writerFactory.getSchema(originalAttributes, reader.getSchema()); try (final RecordSetWriter writer = writerFactory.createWriter(getLogger(), writeSchema, out)) { writer.beginRecordSet(); Record record; while ((record = reader.nextRecord()) != null) { final Record processed = AbstractRecordProcessor.this.process(record, writeSchema, original, context); writer.write(processed); } final WriteResult writeResult = writer.finishRecordSet(); attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); attributes.putAll(writeResult.getAttributes()); recordCount.set(writeResult.getRecordCount()); } } catch (final SchemaNotFoundException e) { throw new ProcessException(e.getLocalizedMessage(), e); } catch (final MalformedRecordException e) { throw new ProcessException("Could not parse incoming data", e); } } });
@Override public void process(final InputStream in) throws IOException { try (final RecordReader reader = readerFactory.createRecordReader(originalAttributes, in, getLogger())) { final RecordSchema writeSchema = writerFactory.getSchema(originalAttributes, reader.getSchema()); Record record; while ((record = reader.nextRecord()) != null) { final Set<Relationship> relationships = route(record, writeSchema, original, context, flowFileContext); numRecords.incrementAndGet(); for (final Relationship relationship : relationships) { final RecordSetWriter recordSetWriter; Tuple<FlowFile, RecordSetWriter> tuple = writers.get(relationship); if (tuple == null) { FlowFile outFlowFile = session.create(original); final OutputStream out = session.write(outFlowFile); recordSetWriter = writerFactory.createWriter(getLogger(), writeSchema, out); recordSetWriter.beginRecordSet(); tuple = new Tuple<>(outFlowFile, recordSetWriter); writers.put(relationship, tuple); } else { recordSetWriter = tuple.getValue(); } recordSetWriter.write(record); } } } catch (final SchemaNotFoundException | MalformedRecordException e) { throw new ProcessException("Could not parse incoming data", e); } } });
@Override public void process(final InputStream in) throws IOException { try (final RecordReader reader = readerFactory.createRecordReader(originalAttributes, in, getLogger())) {
for (FlowFile flowFile : flowFiles) { try (final InputStream in = session.read(flowFile); final RecordReader recordReader = recordReaderFactory.createRecordReader(flowFile, in, getLogger())) { final List<String> fieldNames = recordReader.getSchema().getFieldNames(); final RecordSet recordSet = recordReader.createRecordSet();
final RecordReader reader = readerFactory.createRecordReader(flowFile, in, getLogger())) {
final RecordReader reader = readerFactory.createRecordReader(originalAttributes, in, getLogger());
private void binFlowFile(final ProcessContext context, final FlowFile flowFile, final ProcessSession session, final RecordBinManager binManager, final boolean block) { final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class); try (final InputStream in = session.read(flowFile); final RecordReader reader = readerFactory.createRecordReader(flowFile, in, getLogger())) { final RecordSchema schema = reader.getSchema(); final String groupId = getGroupId(context, flowFile, schema, session); getLogger().debug("Got Group ID {} for {}", new Object[] {groupId, flowFile}); binManager.add(groupId, flowFile, reader, session, block); } catch (MalformedRecordException | IOException | SchemaNotFoundException e) { throw new ProcessException(e); } }
reader = readerFactory.createRecordReader(Collections.emptyMap(), in, logger); firstRecord = reader.nextRecord(); } catch (final Exception e) {
recordReader = recordReaderFactory.createRecordReader(flowFileIn, in, getLogger()); } catch (Exception e) { final RecordReaderFactoryException rrfe = new RecordReaderFactoryException("Unable to create RecordReader", e);
final List<Record> records = new ArrayList<>(); try (final InputStream in = new ByteArrayInputStream(event.getData())) { reader = readerFactory.createRecordReader(Collections.emptyMap(), in, getLogger());