switch (fieldName) { case RepositoryRecordSchema.ACTION_TYPE: return record.getType().name(); case RepositoryRecordSchema.RECORD_ID: return record.getCurrent().getId(); case RepositoryRecordSchema.SWAP_LOCATION: return record.getSwapLocation(); case FlowFileSchema.ATTRIBUTES: return flowFile.getAttributes(); return flowFile.getQueueDateIndex(); case FlowFileSchema.CONTENT_CLAIM: final ContentClaimFieldMap contentClaimFieldMap = record.getCurrentClaim() == null ? null : new ContentClaimFieldMap(record.getCurrentClaim(), record.getCurrentClaimOffset(), contentClaimSchema); return contentClaimFieldMap; case RepositoryRecordSchema.QUEUE_IDENTIFIER: final FlowFileQueue queue = record.getDestination() == null ? record.getOriginalQueue() : record.getDestination(); return queue == null ? null : queue.getIdentifier(); default:
private void updateRepository(final Collection<RepositoryRecord> records, final boolean sync) throws IOException { for (final RepositoryRecord record : records) { if (record.getType() != RepositoryRecordType.DELETE && record.getType() != RepositoryRecordType.CONTENTMISSING && record.getType() != RepositoryRecordType.CLEANUP_TRANSIENT_CLAIMS && record.getDestination() == null) { throw new IllegalArgumentException("Record " + record + " has no destination and Type is " + record.getType()); .collect(Collectors.partitioningBy(record -> record.getType() == RepositoryRecordType.CLEANUP_TRANSIENT_CLAIMS)); if (record.getType() == RepositoryRecordType.DELETE) { if (record.getCurrentClaim() != null && isDestructable(record.getCurrentClaim())) { claimsToAdd.add(record.getCurrentClaim().getResourceClaim()); if (record.getOriginalClaim() != null && !record.getOriginalClaim().equals(record.getCurrentClaim()) && isDestructable(record.getOriginalClaim())) { claimsToAdd.add(record.getOriginalClaim().getResourceClaim()); } else if (record.getType() == RepositoryRecordType.UPDATE) { if (record.getOriginalClaim() != null && record.getCurrentClaim() != record.getOriginalClaim() && isDestructable(record.getOriginalClaim())) { claimsToAdd.add(record.getOriginalClaim().getResourceClaim()); final List<ContentClaim> transientClaims = record.getTransientClaims(); if (transientClaims != null) { for (final ContentClaim transientClaim : transientClaims) {
@Override public void updateRepository(final Collection<RepositoryRecord> records) throws IOException { for (final RepositoryRecord record : records) { if (record.getType() == RepositoryRecordType.DELETE) { // For any DELETE record that we have, if current claim's claimant count <= 0, mark it as destructable if (record.getCurrentClaim() != null && getClaimantCount(record.getCurrentClaim()) <= 0) { markDestructable(record.getCurrentClaim()); } // If the original claim is different than the current claim and the original claim has a claimant count <= 0, mark it as destructable. if (record.getOriginalClaim() != null && !record.getOriginalClaim().equals(record.getCurrentClaim()) && getClaimantCount(record.getOriginalClaim()) <= 0) { markDestructable(record.getOriginalClaim()); } } else if (record.getType() == RepositoryRecordType.UPDATE) { // if we have an update, and the original is no longer needed, mark original as destructable if (record.getOriginalClaim() != null && record.getCurrentClaim() != record.getOriginalClaim() && getClaimantCount(record.getOriginalClaim()) <= 0) { markDestructable(record.getOriginalClaim()); } } } }
public void serializeEdit(final RepositoryRecord previousRecordState, final RepositoryRecord record, final DataOutputStream out, final boolean forceAttributesWritten) throws IOException { if (record.isMarkedForAbort()) { logger.warn("Repository Record {} is marked to be aborted; it will be persisted in the FlowFileRepository as a DELETE record", record); out.write(ACTION_DELETE); out.writeLong(getRecordIdentifier(record)); serializeContentClaim(record.getCurrentClaim(), record.getCurrentClaimOffset(), out); return; out.write(ACTION_DELETE); out.writeLong(getRecordIdentifier(record)); serializeContentClaim(record.getCurrentClaim(), record.getCurrentClaimOffset(), out); return; FlowFileQueue associatedQueue = record.getDestination(); if (associatedQueue == null) { associatedQueue = record.getOriginalQueue(); final FlowFile flowFile = record.getCurrent(); final ContentClaim claim = record.getCurrentClaim(); serializeContentClaim(claim, record.getCurrentClaimOffset(), out); if (forceAttributesWritten || record.isAttributesChanged() || updateType == UpdateType.CREATE || updateType == UpdateType.SWAP_IN) { out.write(1); // indicate attributes changed final Map<String, String> attributes = flowFile.getAttributes(); out.writeUTF(record.getSwapLocation());
final ContentClaim claim = record.getCurrentClaim(); if (claim != null) { claimManager.incrementClaimantCount(claim.getResourceClaim()); final FlowFileRecord flowFile = record.getCurrent(); final FlowFileQueue queue = record.getOriginalQueue(); if (queue == null) { numFlowFilesMissingQueue++;
public RepositoryRecordFieldMap(final RepositoryRecord record, final RecordSchema repoRecordSchema, final RecordSchema contentClaimSchema) { this.schema = repoRecordSchema; this.contentClaimSchema = contentClaimSchema; this.record = record; this.flowFile = record.getCurrent(); }
@Override public UpdateType getUpdateType(final RepositoryRecord record) { switch (record.getType()) { case CONTENTMISSING: case DELETE: return UpdateType.DELETE; case CREATE: return UpdateType.CREATE; case UPDATE: return UpdateType.UPDATE; case SWAP_OUT: return UpdateType.SWAP_OUT; case SWAP_IN: return UpdateType.SWAP_IN; } return null; }
contentRepo.decrementClaimantCount(expiredRecord.getCurrentClaim());
@Override public String getLocation(final RepositoryRecord record) { return record.getSwapLocation(); } }
public void serializeEdit(final RepositoryRecord previousRecordState, final RepositoryRecord record, final DataOutputStream out, final boolean forceAttributesWritten) throws IOException { if (record.isMarkedForAbort()) { logger.warn("Repository Record {} is marked to be aborted; it will be persisted in the FlowFileRepository as a DELETE record", record); out.write(ACTION_DELETE); out.writeLong(getRecordIdentifier(record)); serializeContentClaim(record.getCurrentClaim(), record.getCurrentClaimOffset(), out); return; out.write(ACTION_DELETE); out.writeLong(getRecordIdentifier(record)); serializeContentClaim(record.getCurrentClaim(), record.getCurrentClaimOffset(), out); return; FlowFileQueue associatedQueue = record.getDestination(); if (associatedQueue == null) { associatedQueue = record.getOriginalQueue(); final FlowFile flowFile = record.getCurrent(); final ContentClaim claim = record.getCurrentClaim(); serializeContentClaim(claim, record.getCurrentClaimOffset(), out); if (forceAttributesWritten || record.isAttributesChanged() || updateType == UpdateType.CREATE || updateType == UpdateType.SWAP_IN) { out.write(1); // indicate attributes changed final Map<String, String> attributes = flowFile.getAttributes(); out.writeUTF(record.getSwapLocation());
@Override public Long getRecordIdentifier(final RepositoryRecord record) { return record.getCurrent().getId(); }
@Override public UpdateType getUpdateType(final RepositoryRecord record) { switch (record.getType()) { case CONTENTMISSING: case DELETE: return UpdateType.DELETE; case CREATE: return UpdateType.CREATE; case UPDATE: return UpdateType.UPDATE; case SWAP_OUT: return UpdateType.SWAP_OUT; case SWAP_IN: return UpdateType.SWAP_IN; } return null; }
@Override public String getLocation(final RepositoryRecord record) { return record.getSwapLocation(); }
switch (fieldName) { case RepositoryRecordSchema.ACTION_TYPE: return record.getType().name(); case RepositoryRecordSchema.RECORD_ID: return record.getCurrent().getId(); case RepositoryRecordSchema.SWAP_LOCATION: return record.getSwapLocation(); case FlowFileSchema.ATTRIBUTES: return flowFile.getAttributes(); return flowFile.getQueueDateIndex(); case FlowFileSchema.CONTENT_CLAIM: final ContentClaimFieldMap contentClaimFieldMap = record.getCurrentClaim() == null ? null : new ContentClaimFieldMap(record.getCurrentClaim(), record.getCurrentClaimOffset(), contentClaimSchema); return contentClaimFieldMap; case RepositoryRecordSchema.QUEUE_IDENTIFIER: final FlowFileQueue queue = record.getDestination() == null ? record.getOriginalQueue() : record.getDestination(); return queue == null ? null : queue.getIdentifier(); default:
@Override public Long getRecordIdentifier(final RepositoryRecord record) { return record.getCurrent().getId(); }
@Override public void serializeRecord(final RepositoryRecord record, final DataOutputStream out) throws IOException { final RecordSchema schema; switch (record.getType()) { case CREATE: case UPDATE: schema = RepositoryRecordSchema.CREATE_OR_UPDATE_SCHEMA_V2; break; case CONTENTMISSING: case DELETE: schema = RepositoryRecordSchema.DELETE_SCHEMA_V2; break; case SWAP_IN: schema = RepositoryRecordSchema.SWAP_IN_SCHEMA_V2; break; case SWAP_OUT: schema = RepositoryRecordSchema.SWAP_OUT_SCHEMA_V2; break; default: throw new IllegalArgumentException("Received Repository Record with unknown Update Type: " + record.getType()); // won't happen. } serializeRecord(record, out, schema, RepositoryRecordSchema.REPOSITORY_RECORD_SCHEMA_V2); }
@Override public String getLocation(final RepositoryRecord record) { return record.getSwapLocation(); } }
final FlowFileRecord abortedFlowFile = abortedRecord.getCurrent(); provenanceEvents.add(createDropEvent(abortedFlowFile, "Content Not Found"));
@Override public UpdateType getUpdateType(final RepositoryRecord record) { switch (record.getType()) { case CONTENTMISSING: case DELETE: return UpdateType.DELETE; case CREATE: return UpdateType.CREATE; case UPDATE: return UpdateType.UPDATE; case SWAP_OUT: return UpdateType.SWAP_OUT; case SWAP_IN: return UpdateType.SWAP_IN; } return null; }
@Override public String getLocation(final RepositoryRecord record) { return record.getSwapLocation(); }