@Override public long getCurrentClaimOffset() { return flowFile.getContentClaimOffset(); }
@Override public long getCurrentClaimOffset() { return (getCurrent() == null) ? 0L : getCurrent().getContentClaimOffset(); }
out.writeLong(claim.getOffset()); out.writeLong(claim.getLength()); out.writeLong(flowFile.getContentClaimOffset()); out.writeBoolean(resourceClaim.isLossTolerant());
if (flowFile.getContentClaimOffset() > 0) { try { StreamUtils.skip(rawIn, flowFile.getContentClaimOffset()); } catch (final EOFException eof) { throw new ContentNotFoundException(flowFile, flowFile.getContentClaim(), "FlowFile has a Content Claim Offset of " + flowFile.getContentClaimOffset() + " bytes but the Content Claim does not have that many bytes");
public FlowFileRecordFieldMap(final FlowFileRecord flowFile, final RecordSchema schema) { this.flowFile = flowFile; this.schema = schema; final RecordField contentClaimField = schema.getField(FlowFileSchema.CONTENT_CLAIM); contentClaimSchema = new RecordSchema(contentClaimField.getSubFields()); contentClaim = flowFile.getContentClaim() == null ? null : new ContentClaimFieldMap(flowFile.getContentClaim(), flowFile.getContentClaimOffset(), contentClaimSchema); }
final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize());
final int claimOffsetComparison = Long.compare(f1.getContentClaimOffset(), f2.getContentClaimOffset()); if (claimOffsetComparison != 0) { return claimOffsetComparison;
public Builder fromFlowFile(final FlowFileRecord specFlowFile) { if (specFlowFile == null) { return this; } bId = specFlowFile.getId(); bEntryDate = specFlowFile.getEntryDate(); bLineageStartDate = specFlowFile.getLineageStartDate(); bLineageStartIndex = specFlowFile.getLineageStartIndex(); bLineageIdentifiers.clear(); bPenaltyExpirationMs = specFlowFile.getPenaltyExpirationMillis(); bSize = specFlowFile.getSize(); // If this is a StandardFlowFileRecord, access the attributes map directly. Do not use the // getAttributes() method, because that will wrap the original in an UnmodifiableMap. As a result, // a Processor that continually calls session.append() for instance will have a FlowFile whose attributes // Map is wrapped thousands of times until it hits a StackOverflowError. We want the getter to return // UnmodifiableMap, though, so that Processors cannot directly modify that Map. bAttributes = specFlowFile instanceof StandardFlowFileRecord ? ((StandardFlowFileRecord) specFlowFile).attributes : specFlowFile.getAttributes(); bAttributesCopied = false; bClaim = specFlowFile.getContentClaim(); bClaimOffset = specFlowFile.getContentClaimOffset(); bLastQueueDate = specFlowFile.getLastQueueDate(); bQueueDateIndex = specFlowFile.getQueueDateIndex(); return this; }
private void registerReceiveProvenanceEvents(final List<RemoteFlowFileRecord> flowFiles, final String nodeName, final String connectionId, final long startTimestamp) { final long duration = System.currentTimeMillis() - startTimestamp; final List<ProvenanceEventRecord> events = new ArrayList<>(flowFiles.size()); for (final RemoteFlowFileRecord remoteFlowFile : flowFiles) { final FlowFileRecord flowFileRecord = remoteFlowFile.getFlowFile(); final ProvenanceEventBuilder provenanceEventBuilder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFileRecord) .setEventType(ProvenanceEventType.RECEIVE) .setTransitUri("nifi://" + nodeName + "/loadbalance/" + connectionId) .setSourceSystemFlowFileIdentifier(remoteFlowFile.getRemoteUuid()) .setEventDuration(duration) .setComponentId(connectionId) .setComponentType("Load Balanced Connection"); final ContentClaim contentClaim = flowFileRecord.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); provenanceEventBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFileRecord.getContentClaimOffset(), flowFileRecord.getSize()); } final ProvenanceEventRecord provenanceEvent = provenanceEventBuilder.build(); events.add(provenanceEvent); } provenanceRepository.registerEvents(events); }
private void updateEventContentClaims(final ProvenanceEventBuilder builder, final FlowFile flowFile, final StandardRepositoryRecord repoRecord) { final ContentClaim originalClaim = repoRecord.getOriginalClaim(); if (originalClaim == null) { builder.setCurrentContentClaim(null, null, null, null, 0L); } else { final ResourceClaim resourceClaim = originalClaim.getResourceClaim(); builder.setCurrentContentClaim( resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), repoRecord.getOriginal().getContentClaimOffset() + originalClaim.getOffset(), repoRecord.getOriginal().getSize()); builder.setPreviousContentClaim( resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), repoRecord.getOriginal().getContentClaimOffset() + originalClaim.getOffset(), repoRecord.getOriginal().getSize()); } }
resourceClaim = flowFile.getContentClaim().getResourceClaim(); stream = contentRepository.read(flowFile.getContentClaim()); final long contentClaimOffset = flowFile.getContentClaimOffset(); if (contentClaimOffset > 0L) { StreamUtils.skip(stream, contentClaimOffset); contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize());
@Override public FlowFile clone(FlowFile example, final long offset, final long size) { verifyTaskActive(); example = validateRecordState(example); final StandardRepositoryRecord exampleRepoRecord = getRecord(example); final FlowFileRecord currRec = exampleRepoRecord.getCurrent(); final ContentClaim claim = exampleRepoRecord.getCurrentClaim(); if (offset + size > example.getSize()) { throw new FlowFileHandlingException("Specified offset of " + offset + " and size " + size + " exceeds size of " + example.toString()); } final StandardFlowFileRecord.Builder builder = new StandardFlowFileRecord.Builder().fromFlowFile(currRec); builder.id(context.getNextFlowFileSequence()); builder.contentClaimOffset(currRec.getContentClaimOffset() + offset); builder.size(size); final String newUuid = UUID.randomUUID().toString(); builder.addAttribute(CoreAttributes.UUID.key(), newUuid); final FlowFileRecord clone = builder.build(); if (claim != null) { context.getContentRepository().incrementClaimaintCount(claim); } final StandardRepositoryRecord record = new StandardRepositoryRecord(null); record.setWorking(clone, clone.getAttributes()); records.put(clone.getId(), record); if (offset == 0L && size == example.getSize()) { provenanceReporter.clone(example, clone); } else { registerForkEvent(example, clone); } return clone; }
private ProvenanceEventRecord createDropEvent(final FlowFileRecord flowFile, final String details) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFile) .setEventType(ProvenanceEventType.DROP) .setDetails(details) .setComponentId(flowFileQueue.getIdentifier()) .setComponentType("Connection") .setSourceQueueIdentifier(flowFileQueue.getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); } final ProvenanceEventRecord dropEvent = builder.build(); return dropEvent; }
private ProvenanceEventRecord createSendEvent(final FlowFileRecord flowFile, final NodeIdentifier nodeIdentifier) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFile) .setEventType(ProvenanceEventType.SEND) .setDetails("Re-distributed for Load-balanced connection") .setComponentId(flowFileQueue.getIdentifier()) .setComponentType("Connection") .setSourceQueueIdentifier(flowFileQueue.getIdentifier()) .setSourceSystemFlowFileIdentifier(flowFile.getAttribute(CoreAttributes.UUID.key())) .setTransitUri("nifi://" + nodeIdentifier.getApiAddress() + "/loadbalance/" + flowFileQueue.getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); } final ProvenanceEventRecord sendEvent = builder.build(); return sendEvent; }
/** * Creates a FlowFileDTO from the specified FlowFileRecord. * * @param record record * @return dto */ public FlowFileDTO createFlowFileDTO(final FlowFileRecord record) { final Date now = new Date(); final FlowFileDTO dto = new FlowFileDTO(); dto.setUuid(record.getAttribute(CoreAttributes.UUID.key())); dto.setFilename(record.getAttribute(CoreAttributes.FILENAME.key())); dto.setPenalized(record.isPenalized()); dto.setSize(record.getSize()); dto.setAttributes(record.getAttributes()); final long queuedDuration = now.getTime() - record.getLastQueueDate(); dto.setQueuedDuration(queuedDuration); final long age = now.getTime() - record.getLineageStartDate(); dto.setLineageDuration(age); final ContentClaim contentClaim = record.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); dto.setContentClaimSection(resourceClaim.getSection()); dto.setContentClaimContainer(resourceClaim.getContainer()); dto.setContentClaimIdentifier(resourceClaim.getId()); dto.setContentClaimOffset(contentClaim.getOffset() + record.getContentClaimOffset()); dto.setContentClaimFileSizeBytes(record.getSize()); dto.setContentClaimFileSize(FormatUtils.formatDataSize(record.getSize())); } return dto; }
final long originalOffset = repoRecord.getOriginal().getContentClaimOffset(); final long originalSize = repoRecord.getOriginal().getSize();
final long originalOffset = repoRecord.getOriginal().getContentClaimOffset(); final long originalSize = repoRecord.getOriginal().getSize();
@Override public long getCurrentClaimOffset() { return (getCurrent() == null) ? 0L : getCurrent().getContentClaimOffset(); }
public FlowFileRecordFieldMap(final FlowFileRecord flowFile, final RecordSchema schema) { this.flowFile = flowFile; this.schema = schema; final RecordField contentClaimField = schema.getField(FlowFileSchema.CONTENT_CLAIM); contentClaimSchema = new RecordSchema(contentClaimField.getSubFields()); contentClaim = flowFile.getContentClaim() == null ? null : new ContentClaimFieldMap(flowFile.getContentClaim(), flowFile.getContentClaimOffset(), contentClaimSchema); }
public Builder fromFlowFile(final FlowFileRecord specFlowFile) { if (specFlowFile == null) { return this; } bId = specFlowFile.getId(); bEntryDate = specFlowFile.getEntryDate(); bLineageStartDate = specFlowFile.getLineageStartDate(); bLineageStartIndex = specFlowFile.getLineageStartIndex(); bLineageIdentifiers.clear(); bPenaltyExpirationMs = specFlowFile.getPenaltyExpirationMillis(); bSize = specFlowFile.getSize(); // If this is a StandardFlowFileRecord, access the attributes map directly. Do not use the // getAttributes() method, because that will wrap the original in an UnmodifiableMap. As a result, // a Processor that continually calls session.append() for instance will have a FlowFile whose attributes // Map is wrapped thousands of times until it hits a StackOverflowError. We want the getter to return // UnmodifiableMap, though, so that Processors cannot directly modify that Map. bAttributes = specFlowFile instanceof StandardFlowFileRecord ? ((StandardFlowFileRecord) specFlowFile).attributes : specFlowFile.getAttributes(); bAttributesCopied = false; bClaim = specFlowFile.getContentClaim(); bClaimOffset = specFlowFile.getContentClaimOffset(); bLastQueueDate = specFlowFile.getLastQueueDate(); bQueueDateIndex = specFlowFile.getQueueDateIndex(); return this; }