@Override public void updateCounters(ProcessSession session) { final WriteResult result = writeResultRef.get(); if (result != null) { session.adjustCounter("Records Written", result.getRecordCount(), false); } }
@Override public Map<String, String> getAttributesToAdd() { Map<String, String> attributesToAdd = new HashMap<>(); attributesToAdd.put(CoreAttributes.MIME_TYPE.key(), mimeType); // Add any attributes from the record writer (if present) final WriteResult result = writeResultRef.get(); if (result != null) { if (result.getAttributes() != null) { attributesToAdd.putAll(result.getAttributes()); } attributesToAdd.put("record.count", String.valueOf(result.getRecordCount())); } return attributesToAdd; }
private void completeFlowFile(final ProcessSession session, final FlowFile flowFile, final RecordSetWriter writer, final Relationship relationship, final String details) throws IOException { final WriteResult writeResult = writer.finishRecordSet(); writer.close(); final Map<String, String> attributes = new HashMap<>(); attributes.putAll(writeResult.getAttributes()); attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); session.putAllAttributes(flowFile, attributes); session.transfer(flowFile, relationship); session.getProvenanceReporter().route(flowFile, relationship, details); }
private boolean processBundle(final BundleTracker bundle) throws IOException { final RecordSetWriter writer = bundle.recordWriter; if (writer != null) { final WriteResult writeResult; try { writeResult = writer.finishRecordSet(); } finally { writer.close(); } if (writeResult.getRecordCount() == 0) { getProcessSession().remove(bundle.flowFile); return false; } final Map<String, String> attributes = new HashMap<>(); attributes.putAll(writeResult.getAttributes()); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); bundle.flowFile = getProcessSession().putAllAttributes(bundle.flowFile, attributes); } populateAttributes(bundle); return true; }
@Override public void process(final InputStream in, final OutputStream out) throws IOException { try (final RecordReader reader = readerFactory.createRecordReader(originalAttributes, in, getLogger())) { final RecordSchema writeSchema = writerFactory.getSchema(originalAttributes, reader.getSchema()); try (final RecordSetWriter writer = writerFactory.createWriter(getLogger(), writeSchema, out)) { writer.beginRecordSet(); Record record; while ((record = reader.nextRecord()) != null) { final Record processed = AbstractRecordProcessor.this.process(record, writeSchema, original, context); writer.write(processed); } final WriteResult writeResult = writer.finishRecordSet(); attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); attributes.putAll(writeResult.getAttributes()); recordCount.set(writeResult.getRecordCount()); } } catch (final SchemaNotFoundException e) { throw new ProcessException(e.getLocalizedMessage(), e); } catch (final MalformedRecordException e) { throw new ProcessException("Could not parse incoming data", e); } } });
private boolean processBundle(final BundleTracker bundle) throws IOException { final RecordSetWriter writer = bundle.recordWriter; if (writer != null) { final WriteResult writeResult; try { writeResult = writer.finishRecordSet(); } finally { writer.close(); } if (writeResult.getRecordCount() == 0) { getProcessSession().remove(bundle.flowFile); return false; } final Map<String, String> attributes = new HashMap<>(); attributes.putAll(writeResult.getAttributes()); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); bundle.flowFile = getProcessSession().putAllAttributes(bundle.flowFile, attributes); } populateAttributes(bundle); return true; }
private boolean processBundle(final BundleTracker bundle) throws IOException { final RecordSetWriter writer = bundle.recordWriter; if (writer != null) { final WriteResult writeResult; try { writeResult = writer.finishRecordSet(); } finally { writer.close(); } if (writeResult.getRecordCount() == 0) { getProcessSession().remove(bundle.flowFile); return false; } final Map<String, String> attributes = new HashMap<>(); attributes.putAll(writeResult.getAttributes()); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); bundle.flowFile = getProcessSession().putAllAttributes(bundle.flowFile, attributes); } populateAttributes(bundle); return true; }
attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), recordWriter.getMimeType()); attributes.put(MERGE_COUNT_ATTRIBUTE, Integer.toString(flowFiles.size())); session.transfer(merged, MergeRecord.REL_MERGED); session.transfer(flowFiles, MergeRecord.REL_ORIGINAL); session.adjustCounter("Records Merged", writeResult.getRecordCount(), false); session.commit(); logger.debug("Completed bin {} with {} records with Merged FlowFile {} using input FlowFiles {}", new Object[] {this, writeResult.getRecordCount(), merged, ids});
private boolean processBundle(final BundleTracker bundle) throws IOException { final RecordSetWriter writer = bundle.recordWriter; if (writer != null) { final WriteResult writeResult; try { writeResult = writer.finishRecordSet(); } finally { writer.close(); } if (writeResult.getRecordCount() == 0) { getProcessSession().remove(bundle.flowFile); return false; } final Map<String, String> attributes = new HashMap<>(); attributes.putAll(writeResult.getAttributes()); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); bundle.flowFile = getProcessSession().putAllAttributes(bundle.flowFile, attributes); } populateAttributes(bundle); return true; }
attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); attributes.put(FRAGMENT_INDEX, String.valueOf(fragmentIndex)); attributes.putAll(writeResult.getAttributes()); session.adjustCounter("Records Split", writeResult.getRecordCount(), false);
@Override public long writeResultSet(ResultSet resultSet, OutputStream outputStream, ComponentLog logger, AbstractQueryDatabaseTable.MaxValueResultSetRowCollector callback) throws Exception { final RecordSet recordSet; try { if (fullRecordSet == null) { final Schema avroSchema = JdbcCommon.createSchema(resultSet, options); final RecordSchema recordAvroSchema = AvroTypeUtil.createSchema(avroSchema); fullRecordSet = new ResultSetRecordSetWithCallback(resultSet, recordAvroSchema, callback); writeSchema = recordSetWriterFactory.getSchema(originalAttributes, fullRecordSet.getSchema()); } recordSet = (maxRowsPerFlowFile > 0) ? fullRecordSet.limit(maxRowsPerFlowFile) : fullRecordSet; } catch (final SQLException | SchemaNotFoundException | IOException e) { throw new ProcessException(e); } try (final RecordSetWriter resultSetWriter = recordSetWriterFactory.createWriter(logger, writeSchema, outputStream)) { writeResultRef.set(resultSetWriter.write(recordSet)); if (mimeType == null) { mimeType = resultSetWriter.getMimeType(); } return writeResultRef.get().getRecordCount(); } catch (final Exception e) { throw new IOException(e); } }
attributes.putAll(valueMap.getAttributes()); attributes.putAll(writeResult.getAttributes()); attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); childFlowFile = session.putAllAttributes(childFlowFile, attributes); session.adjustCounter("Record Processed", writeResult.getRecordCount(), false);
writeResult = recordWriter.write(record); while (record != null && writeResult.getRecordCount() < recordBatchSize) { if (writeResult.getRecordCount() <= 0) { getLogger().debug("Removing flow file, no records were written"); session.remove(flowFile); attributes.put("tcp.sender", sender); attributes.put("tcp.port", String.valueOf(port)); attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); flowFile = session.putAllAttributes(flowFile, attributes);
attributes.put(RECORD_COUNT_ATTR, String.valueOf(writeResult.get().getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), mimeTypeRef.get()); successFlowFile = session.putAllAttributes(successFlowFile, attributes);
attributes.put(CoreAttributes.FILENAME.key(), newFilename); attributes.put(ABSOLUTE_HDFS_PATH_ATTRIBUTE, hdfsPath); attributes.put(RECORD_COUNT_ATTR, String.valueOf(writeResult.get().getRecordCount())); putFlowFile = session.putAllAttributes(putFlowFile, attributes);
if (writeResult.getRecordCount() == 0) { session.remove(flowFile); continue; attributes.putAll(writeResult.getAttributes()); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); attributes.put(RECORD_COUNT_ATTR, String.valueOf(writeResult.getRecordCount()));
writeCount.set(writeResult.getRecordCount()); attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), recordSetWriter.getMimeType()); attributes.putAll(writeResult.getAttributes());
attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); attributes.putAll(writeResult.getAttributes()); session.adjustCounter("Records Processed", writeResult.getRecordCount(), false); session.adjustCounter("Records Routed to " + relationship.getName(), writeResult.getRecordCount(), false);
recordCount += lastWriteResult.getRecordCount();
attributes.put("record.count", String.valueOf(writeResult.getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), writer.getMimeType()); attributes.putAll(writeResult.getAttributes());