private Path getArchivePath(final ResourceClaim claim) { final String claimId = claim.getId(); final Path containerPath = containers.get(claim.getContainer()); final Path archivePath = containerPath.resolve(claim.getSection()).resolve(ARCHIVE_DIR_NAME).resolve(claimId); return archivePath; }
private Path getPath(final ResourceClaim resourceClaim) { final Path containerPath = containers.get(resourceClaim.getContainer()); if (containerPath == null) { return null; } return containerPath.resolve(resourceClaim.getSection()).resolve(resourceClaim.getId()); }
@Override public boolean equals(final Object other) { if (this == other) { return true; } if (other == null) { return false; } if (hashCode != other.hashCode()) { // We check hash code before instanceof because instanceof is fairly expensive and for // StandardResourceClaim, calling hashCode() simply returns a pre-calculated value. return false; } if (!(other instanceof ResourceClaim)) { return false; } final ResourceClaim otherClaim = (ResourceClaim) other; return id.equals(otherClaim.getId()) && container.equals(otherClaim.getContainer()) && section.equals(otherClaim.getSection()); }
final String container = claim.getContainer(); final BlockingQueue<ResourceClaim> claimQueue = reclaimable.get(container);
/** * Provides the natural ordering for ResourceClaim objects. By default they are sorted by their id, then container, then section * * @param other other claim * @return x such that x <= -1 if this is less than other; * x=0 if this.equals(other); * x >= 1 if this is greater than other */ @Override default int compareTo(final ResourceClaim other) { final int idComparison = getId().compareTo(other.getId()); if (idComparison != 0) { return idComparison; } final int containerComparison = getContainer().compareTo(other.getContainer()); if (containerComparison != 0) { return containerComparison; } return getSection().compareTo(other.getSection()); } }
private void serializeContentClaim(final ContentClaim claim, final long offset, final DataOutputStream out) throws IOException { if (claim == null) { out.write(0); } else { out.write(1); final ResourceClaim resourceClaim = claim.getResourceClaim(); writeString(resourceClaim.getId(), out); writeString(resourceClaim.getContainer(), out); writeString(resourceClaim.getSection(), out); out.writeLong(claim.getOffset()); out.writeLong(claim.getLength()); out.writeLong(offset); out.writeBoolean(resourceClaim.isLossTolerant()); } }
@Override public Object getFieldValue(final String fieldName) { switch (fieldName) { case ContentClaimSchema.CLAIM_CONTAINER: return resourceClaim.getContainer(); case ContentClaimSchema.CLAIM_SECTION: return resourceClaim.getSection(); case ContentClaimSchema.CLAIM_IDENTIFIER: return resourceClaim.getId(); case ContentClaimSchema.LOSS_TOLERANT: return resourceClaim.isLossTolerant(); } return null; }
private Path getPath(final ContentClaim claim, final boolean verifyExists) throws ContentNotFoundException { final ResourceClaim resourceClaim = claim.getResourceClaim(); final Path containerPath = containers.get(resourceClaim.getContainer()); if (containerPath == null) { if (verifyExists) { throw new ContentNotFoundException(claim); } else { return null; } } // Create the Path that points to the data Path resolvedPath = containerPath.resolve(resourceClaim.getSection()).resolve(resourceClaim.getId()); // If the data does not exist, create a Path that points to where the data would exist in the archive directory. if (!Files.exists(resolvedPath)) { resolvedPath = getArchivePath(claim.getResourceClaim()); if (verifyExists && !Files.exists(resolvedPath)) { throw new ContentNotFoundException(claim); } } return resolvedPath; }
final ResourceClaim resourceClaim = claim.getResourceClaim(); out.writeUTF(resourceClaim.getId()); out.writeUTF(resourceClaim.getContainer()); out.writeUTF(resourceClaim.getSection()); out.writeLong(claim.getOffset());
if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize());
private void registerReceiveProvenanceEvents(final List<RemoteFlowFileRecord> flowFiles, final String nodeName, final String connectionId, final long startTimestamp) { final long duration = System.currentTimeMillis() - startTimestamp; final List<ProvenanceEventRecord> events = new ArrayList<>(flowFiles.size()); for (final RemoteFlowFileRecord remoteFlowFile : flowFiles) { final FlowFileRecord flowFileRecord = remoteFlowFile.getFlowFile(); final ProvenanceEventBuilder provenanceEventBuilder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFileRecord) .setEventType(ProvenanceEventType.RECEIVE) .setTransitUri("nifi://" + nodeName + "/loadbalance/" + connectionId) .setSourceSystemFlowFileIdentifier(remoteFlowFile.getRemoteUuid()) .setEventDuration(duration) .setComponentId(connectionId) .setComponentType("Load Balanced Connection"); final ContentClaim contentClaim = flowFileRecord.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); provenanceEventBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFileRecord.getContentClaimOffset(), flowFileRecord.getSize()); } final ProvenanceEventRecord provenanceEvent = provenanceEventBuilder.build(); events.add(provenanceEvent); } provenanceRepository.registerEvents(events); }
private void updateEventContentClaims(final ProvenanceEventBuilder builder, final FlowFile flowFile, final StandardRepositoryRecord repoRecord) { final ContentClaim originalClaim = repoRecord.getOriginalClaim(); if (originalClaim == null) { builder.setCurrentContentClaim(null, null, null, null, 0L); } else { final ResourceClaim resourceClaim = originalClaim.getResourceClaim(); builder.setCurrentContentClaim( resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), repoRecord.getOriginal().getContentClaimOffset() + originalClaim.getOffset(), repoRecord.getOriginal().getSize()); builder.setPreviousContentClaim( resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), repoRecord.getOriginal().getContentClaimOffset() + originalClaim.getOffset(), repoRecord.getOriginal().getSize()); } }
private ProvenanceEventRecord createDropEvent(final FlowFileRecord flowFile, final String details) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFile) .setEventType(ProvenanceEventType.DROP) .setDetails(details) .setComponentId(flowFileQueue.getIdentifier()) .setComponentType("Connection") .setSourceQueueIdentifier(flowFileQueue.getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); } final ProvenanceEventRecord dropEvent = builder.build(); return dropEvent; }
sendEventBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize());
recordBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), currentOffset + currentClaim.getOffset(), size); recordBuilder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), originalOffset + originalClaim.getOffset(), originalSize);
private ProvenanceEventRecord createDropProvenanceEvent(final FlowFileRecord flowFile, final String requestor) { final ProvenanceEventBuilder builder = provRepository.eventBuilder(); builder.fromFlowFile(flowFile); builder.setEventType(ProvenanceEventType.DROP); builder.setLineageStartDate(flowFile.getLineageStartDate()); builder.setComponentId(getIdentifier()); builder.setComponentType("Connection"); builder.setAttributes(flowFile.getAttributes(), Collections.emptyMap()); builder.setDetails("FlowFile Queue emptied by " + requestor); builder.setSourceQueueIdentifier(getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset(), flowFile.getSize()); } return builder.build(); }
.setFlowFileUUID(provEvent.getFlowFileUuid()) .setAttributes(provEvent.getAttributes(), Collections.emptyMap()) .setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), offset, size) .setTransitUri(requestUri) .setEventTime(System.currentTimeMillis())
private ProvenanceEventRecord createSendEvent(final FlowFileRecord flowFile, final NodeIdentifier nodeIdentifier) { final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder() .fromFlowFile(flowFile) .setEventType(ProvenanceEventType.SEND) .setDetails("Re-distributed for Load-balanced connection") .setComponentId(flowFileQueue.getIdentifier()) .setComponentType("Connection") .setSourceQueueIdentifier(flowFileQueue.getIdentifier()) .setSourceSystemFlowFileIdentifier(flowFile.getAttribute(CoreAttributes.UUID.key())) .setTransitUri("nifi://" + nodeIdentifier.getApiAddress() + "/loadbalance/" + flowFileQueue.getIdentifier()); final ContentClaim contentClaim = flowFile.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); builder.setCurrentContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); builder.setPreviousContentClaim(resourceClaim.getContainer(),resourceClaim.getSection() ,resourceClaim.getId(), contentClaim.getOffset() + flowFile.getContentClaimOffset(), flowFile.getSize()); } final ProvenanceEventRecord sendEvent = builder.build(); return sendEvent; }
/** * Creates a FlowFileDTO from the specified FlowFileRecord. * * @param record record * @return dto */ public FlowFileDTO createFlowFileDTO(final FlowFileRecord record) { final Date now = new Date(); final FlowFileDTO dto = new FlowFileDTO(); dto.setUuid(record.getAttribute(CoreAttributes.UUID.key())); dto.setFilename(record.getAttribute(CoreAttributes.FILENAME.key())); dto.setPenalized(record.isPenalized()); dto.setSize(record.getSize()); dto.setAttributes(record.getAttributes()); final long queuedDuration = now.getTime() - record.getLastQueueDate(); dto.setQueuedDuration(queuedDuration); final long age = now.getTime() - record.getLineageStartDate(); dto.setLineageDuration(age); final ContentClaim contentClaim = record.getContentClaim(); if (contentClaim != null) { final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); dto.setContentClaimSection(resourceClaim.getSection()); dto.setContentClaimContainer(resourceClaim.getContainer()); dto.setContentClaimIdentifier(resourceClaim.getId()); dto.setContentClaimOffset(contentClaim.getOffset() + record.getContentClaimOffset()); dto.setContentClaimFileSizeBytes(record.getSize()); dto.setContentClaimFileSize(FormatUtils.formatDataSize(record.getSize())); } return dto; }
recordBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), currentOffset + currentClaim.getOffset(), size); recordBuilder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), originalOffset + originalClaim.getOffset(), originalSize);