private RepositoryRecord createRepositoryRecord(final FlowFileRecord flowFile) { final StandardRepositoryRecord record = new StandardRepositoryRecord(flowFileQueue, flowFile); record.markForDelete(); return record; }
@Override public void swapFlowFilesIn(final String swapLocation, final List<FlowFileRecord> swapRecords, final FlowFileQueue queue) throws IOException { final List<RepositoryRecord> repoRecords = new ArrayList<>(); for (final FlowFileRecord swapRecord : swapRecords) { final StandardRepositoryRecord repoRecord = new StandardRepositoryRecord(queue, swapRecord); repoRecord.setSwapLocation(swapLocation); // set the swap file to indicate that it's being swapped in. repoRecord.setDestination(queue); repoRecords.add(repoRecord); } updateRepository(repoRecords, true); logger.info("Repository updated to reflect that {} FlowFiles were swapped in to {}", new Object[]{swapRecords.size(), queue}); }
/** * Swaps the FlowFiles that live on the given Connection out to disk, using * the specified Swap File and returns the number of FlowFiles that were * persisted. * * @param queue queue to swap out * @param swapLocation location to swap to * @throws IOException ioe */ @Override public void swapFlowFilesOut(final List<FlowFileRecord> swappedOut, final FlowFileQueue queue, final String swapLocation) throws IOException { final List<RepositoryRecord> repoRecords = new ArrayList<>(); if (swappedOut == null || swappedOut.isEmpty()) { return; } for (final FlowFileRecord swapRecord : swappedOut) { final RepositoryRecord repoRecord = new StandardRepositoryRecord(queue, swapRecord, swapLocation); repoRecords.add(repoRecord); } // TODO: We should probably update this to support bulk 'SWAP OUT' records. As-is, we have to write out a // 'SWAP OUT' record for each FlowFile, which includes the Update Type, FlowFile ID, swap file location, and Queue ID. // We could instead have a single record with Update Type of 'SWAP OUT' and just include swap file location, Queue ID, // and all FlowFile ID's. // update WALI to indicate that the records were swapped out. wal.update(repoRecords, true); logger.info("Successfully swapped out {} FlowFiles from {} to Swap File {}", new Object[]{swappedOut.size(), queue, swapLocation}); }
private void updateFlowFileRepository(final List<RemoteFlowFileRecord> flowFiles, final FlowFileQueue flowFileQueue) throws IOException { final List<RepositoryRecord> repoRecords = flowFiles.stream() .map(remoteFlowFile -> { final StandardRepositoryRecord record = new StandardRepositoryRecord(flowFileQueue, remoteFlowFile.getFlowFile()); record.setDestination(flowFileQueue); return record; }) .collect(Collectors.toList()); flowFileRepository.updateRepository(repoRecords); }
private void registerDequeuedRecord(final FlowFileRecord flowFile, final Connection connection) { final StandardRepositoryRecord record = new StandardRepositoryRecord(connection.getFlowFileQueue(), flowFile); records.put(flowFile.getId(), record); flowFilesIn++; contentSizeIn += flowFile.getSize(); Set<FlowFileRecord> set = unacknowledgedFlowFiles.get(connection.getFlowFileQueue()); if (set == null) { set = new HashSet<>(); unacknowledgedFlowFiles.put(connection.getFlowFileQueue(), set); } set.add(flowFile); incrementConnectionOutputCounts(connection, flowFile); }
successfulFlowFiles.remove(flowFile); final StandardRepositoryRecord repoRecord = new StandardRepositoryRecord(flowFileQueue, flowFile); repoRecord.markForAbort();
@Override public FlowFile create() { verifyTaskActive(); final Map<String, String> attrs = new HashMap<>(); final String uuid = UUID.randomUUID().toString(); attrs.put(CoreAttributes.FILENAME.key(), uuid); attrs.put(CoreAttributes.PATH.key(), DEFAULT_FLOWFILE_PATH); attrs.put(CoreAttributes.UUID.key(), uuid); final FlowFileRecord fFile = new StandardFlowFileRecord.Builder().id(context.getNextFlowFileSequence()) .addAttributes(attrs) .build(); final StandardRepositoryRecord record = new StandardRepositoryRecord(null); record.setWorking(fFile, attrs); records.put(fFile.getId(), record); createdFlowFiles.add(fFile.getAttribute(CoreAttributes.UUID.key())); return fFile; }
successfulFlowFiles.remove(flowFile); final StandardRepositoryRecord repoRecord = new StandardRepositoryRecord(flowFileQueue, flowFile); repoRecord.markForAbort();
private RepositoryRecord deleteRecord(final Record record) { final Long recordId = (Long) record.getFieldValue(RepositoryRecordSchema.RECORD_ID_FIELD); final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder().id(recordId); final FlowFileRecord flowFileRecord = ffBuilder.build(); final StandardRepositoryRecord repoRecord = new StandardRepositoryRecord((FlowFileQueue) null, flowFileRecord); repoRecord.markForDelete(); return repoRecord; }
final StandardRepositoryRecord record = new StandardRepositoryRecord(null); record.setWorking(fFile, newAttributes); records.put(fFile.getId(), record);
final StandardRepositoryRecord record = new StandardRepositoryRecord((FlowFileQueue) null, flowFileRecord); record.markForDelete(); return record; record = new StandardRepositoryRecord(queue, flowFile); if (swapLocation != null) { record.setSwapLocation(swapLocation);
final StandardRepositoryRecord record = new StandardRepositoryRecord((FlowFileQueue) null, flowFileRecord); record.markForDelete(); .build(); return new StandardRepositoryRecord(queue, flowFileRecord, location); final StandardRepositoryRecord standardRepoRecord = new StandardRepositoryRecord(queue, flowFile); if (swapLocation != null) { standardRepoRecord.setSwapLocation(swapLocation);
.build(); final StandardRepositoryRecord record = new StandardRepositoryRecord(null); record.setWorking(fFile, newAttributes); records.put(fFile.getId(), record);
final StandardRepositoryRecord record = new StandardRepositoryRecord(this, flowFile); record.markForDelete(); expiredRecords.add(record);
recordIdMap.put(flowFile.getAttribute(CoreAttributes.UUID.key()), flowFile); final StandardRepositoryRecord record = new StandardRepositoryRecord(connection.getFlowFileQueue(), flowFile); record.markForDelete(); expiredRecords.add(record);
private RepositoryRecord swapOutRecord(final Record record) { final Long recordId = (Long) record.getFieldValue(RepositoryRecordSchema.RECORD_ID_FIELD); final String queueId = (String) record.getFieldValue(new SimpleRecordField(RepositoryRecordSchema.QUEUE_IDENTIFIER, FieldType.STRING, Repetition.EXACTLY_ONE)); final String swapLocation = (String) record.getFieldValue(new SimpleRecordField(RepositoryRecordSchema.SWAP_LOCATION, FieldType.STRING, Repetition.EXACTLY_ONE)); final FlowFileQueue queue = getFlowFileQueue(queueId); final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder() .id(recordId) .build(); return new StandardRepositoryRecord(queue, flowFileRecord, swapLocation); }
final StandardRepositoryRecord newRecord = new StandardRepositoryRecord(destination.getFlowFileQueue()); provenanceReporter.clone(currRec, clone, false);
@Override public FlowFile clone(FlowFile example, final long offset, final long size) { verifyTaskActive(); example = validateRecordState(example); final StandardRepositoryRecord exampleRepoRecord = getRecord(example); final FlowFileRecord currRec = exampleRepoRecord.getCurrent(); final ContentClaim claim = exampleRepoRecord.getCurrentClaim(); if (offset + size > example.getSize()) { throw new FlowFileHandlingException("Specified offset of " + offset + " and size " + size + " exceeds size of " + example.toString()); } final StandardFlowFileRecord.Builder builder = new StandardFlowFileRecord.Builder().fromFlowFile(currRec); builder.id(context.getNextFlowFileSequence()); builder.contentClaimOffset(currRec.getContentClaimOffset() + offset); builder.size(size); final String newUuid = UUID.randomUUID().toString(); builder.addAttribute(CoreAttributes.UUID.key(), newUuid); final FlowFileRecord clone = builder.build(); if (claim != null) { context.getContentRepository().incrementClaimaintCount(claim); } final StandardRepositoryRecord record = new StandardRepositoryRecord(null); record.setWorking(clone, clone.getAttributes()); records.put(clone.getId(), record); if (offset == 0L && size == example.getSize()) { provenanceReporter.clone(example, clone); } else { registerForkEvent(example, clone); } return clone; }
final StandardRepositoryRecord record = new StandardRepositoryRecord(queue); record.setWorking(flowFileRecord); record.setDestination(queue);
@SuppressWarnings("unchecked") private StandardRepositoryRecord createRecord(final Record record) { final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder(); ffBuilder.id((Long) record.getFieldValue(RepositoryRecordSchema.RECORD_ID)); ffBuilder.entryDate((Long) record.getFieldValue(FlowFileSchema.ENTRY_DATE)); final Long lastQueueDate = (Long) record.getFieldValue(FlowFileSchema.QUEUE_DATE); final Long queueDateIndex = (Long) record.getFieldValue(FlowFileSchema.QUEUE_DATE_INDEX); ffBuilder.lastQueued(lastQueueDate, queueDateIndex); final Long lineageStartDate = (Long) record.getFieldValue(FlowFileSchema.LINEAGE_START_DATE); final Long lineageStartIndex = (Long) record.getFieldValue(FlowFileSchema.LINEAGE_START_INDEX); ffBuilder.lineageStart(lineageStartDate, lineageStartIndex); populateContentClaim(ffBuilder, record); ffBuilder.size((Long) record.getFieldValue(FlowFileSchema.FLOWFILE_SIZE)); ffBuilder.addAttributes((Map<String, String>) record.getFieldValue(FlowFileSchema.ATTRIBUTES)); final FlowFileRecord flowFileRecord = ffBuilder.build(); final String queueId = (String) record.getFieldValue(RepositoryRecordSchema.QUEUE_IDENTIFIER); final FlowFileQueue queue = getFlowFileQueue(queueId); final StandardRepositoryRecord repoRecord = new StandardRepositoryRecord(queue, flowFileRecord); requireFlowFileQueue(repoRecord, queueId); return repoRecord; }