public static Long getContentClaimOffset(final Record claimRecord) { return (Long) claimRecord.getFieldValue(ContentClaimSchema.CONTENT_CLAIM_OFFSET); } }
public static ResourceClaim getResourceClaim(final Record record, final ResourceClaimManager claimManager) { final String container = (String) record.getFieldValue(ContentClaimSchema.CLAIM_CONTAINER); final String section = (String) record.getFieldValue(ContentClaimSchema.CLAIM_SECTION); final String identifier = (String) record.getFieldValue(ContentClaimSchema.CLAIM_IDENTIFIER); final Boolean lossTolerant = (Boolean) record.getFieldValue(ContentClaimSchema.LOSS_TOLERANT); // Make sure that we preserve the existing ResourceClaim, if there is already one held by the Resource Claim Manager // because we need to honor its determination of whether or not the claim is writable. If the Resource Claim Manager // does not have a copy of this Resource Claim, then we can go ahead and just create one and assume that it is not // writable (because if it were writable, then the Resource Claim Manager would know about it). ResourceClaim resourceClaim = claimManager.getResourceClaim(container, section, identifier); if (resourceClaim == null) { resourceClaim = claimManager.newResourceClaim(container, section, identifier, lossTolerant, false); } return resourceClaim; }
public static ContentClaim getContentClaim(final Record claimRecord, final ResourceClaimManager resourceClaimManager) { final Record resourceClaimRecord = (Record) claimRecord.getFieldValue(ContentClaimSchema.RESOURCE_CLAIM); final String container = (String) resourceClaimRecord.getFieldValue(ContentClaimSchema.CLAIM_CONTAINER); final String section = (String) resourceClaimRecord.getFieldValue(ContentClaimSchema.CLAIM_SECTION); final String identifier = (String) resourceClaimRecord.getFieldValue(ContentClaimSchema.CLAIM_IDENTIFIER); final Boolean lossTolerant = (Boolean) resourceClaimRecord.getFieldValue(ContentClaimSchema.LOSS_TOLERANT); final Long length = (Long) claimRecord.getFieldValue(ContentClaimSchema.CONTENT_CLAIM_LENGTH); final Long resourceOffset = (Long) claimRecord.getFieldValue(ContentClaimSchema.RESOURCE_CLAIM_OFFSET); // Make sure that we preserve the existing ResourceClaim, if there is already one held by the Resource Claim Manager // because we need to honor its determination of whether or not the claim is writable. If the Resource Claim Manager // does not have a copy of this Resource Claim, then we can go ahead and just create one and assume that it is not // writable (because if it were writable, then the Resource Claim Manager would know about it). ResourceClaim resourceClaim = resourceClaimManager.getResourceClaim(container, section, identifier); if (resourceClaim == null) { resourceClaim = resourceClaimManager.newResourceClaim(container, section, identifier, lossTolerant, false); } final StandardContentClaim contentClaim = new StandardContentClaim(resourceClaim, resourceOffset); contentClaim.setLength(length); return contentClaim; }
default Object getFieldValue(RecordField field) { return getFieldValue(field.getFieldName()); }
@SuppressWarnings("unchecked") public static SwapSummary getSwapSummary(final Record record, final ResourceClaimManager claimManager) { final int flowFileCount = (Integer) record.getFieldValue(SwapSchema.FLOWFILE_COUNT); final long flowFileSize = (Long) record.getFieldValue(SwapSchema.FLOWFILE_SIZE); final QueueSize queueSize = new QueueSize(flowFileCount, flowFileSize); final long maxFlowFileId = (Long) record.getFieldValue(SwapSchema.MAX_RECORD_ID); final Map<Record, Integer> resourceClaimRecords = (Map<Record, Integer>) record.getFieldValue(SwapSchema.RESOURCE_CLAIMS); final List<ResourceClaim> resourceClaims = new ArrayList<>(); for (final Map.Entry<Record, Integer> entry : resourceClaimRecords.entrySet()) { final Record resourceClaimRecord = entry.getKey(); final ResourceClaim claim = ResourceClaimFieldMap.getResourceClaim(resourceClaimRecord, claimManager); for (int i = 0; i < entry.getValue(); i++) { resourceClaims.add(claim); } } return new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims); } }
@SuppressWarnings("unchecked") public static FlowFileRecord getFlowFile(final Record record, final ResourceClaimManager claimManager) { final StandardFlowFileRecord.Builder builder = new StandardFlowFileRecord.Builder(); builder.id((Long) record.getFieldValue(FlowFileSchema.RECORD_ID)); builder.entryDate((Long) record.getFieldValue(FlowFileSchema.ENTRY_DATE)); builder.size((Long) record.getFieldValue(FlowFileSchema.FLOWFILE_SIZE)); builder.addAttributes((Map<String, String>) record.getFieldValue(FlowFileSchema.ATTRIBUTES)); builder.lineageStart((Long) record.getFieldValue(FlowFileSchema.LINEAGE_START_DATE), (Long) record.getFieldValue(FlowFileSchema.LINEAGE_START_INDEX)); builder.lastQueued((Long) record.getFieldValue(FlowFileSchema.QUEUE_DATE), (Long) record.getFieldValue(FlowFileSchema.QUEUE_DATE_INDEX)); final Record contentClaimRecord = (Record) record.getFieldValue(FlowFileSchema.CONTENT_CLAIM); if (contentClaimRecord != null) { final ContentClaim claim = ContentClaimFieldMap.getContentClaim(contentClaimRecord, claimManager); builder.contentClaim(claim); final Long offset = ContentClaimFieldMap.getContentClaimOffset(contentClaimRecord); if (offset != null) { builder.contentClaimOffset(offset); } } return builder.build(); } }
private void writeRecordFields(final Record record, final RecordSchema schema, final OutputStream out, final byte[] buffer) throws IOException { final DataOutputStream dos = out instanceof DataOutputStream ? (DataOutputStream) out : new DataOutputStream(out); for (final RecordField field : schema.getFields()) { final Object value = record.getFieldValue(field); try { writeFieldRepetitionAndValue(field, value, dos, buffer); } catch (final Exception e) { throw new IOException("Failed to write field '" + field.getFieldName() + "'", e); } } }
private RepositoryRecord swapInRecord(final Record record) { final StandardRepositoryRecord repoRecord = createRecord(record); final String swapLocation = (String) record.getFieldValue(new SimpleRecordField(RepositoryRecordSchema.SWAP_LOCATION, FieldType.STRING, Repetition.EXACTLY_ONE)); repoRecord.setSwapLocation(swapLocation); final String queueId = (String) record.getFieldValue(RepositoryRecordSchema.QUEUE_IDENTIFIER); requireFlowFileQueue(repoRecord, queueId); return repoRecord; }
@SuppressWarnings("unchecked") private StandardRepositoryRecord createRecord(final Record record) { final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder(); ffBuilder.id((Long) record.getFieldValue(RepositoryRecordSchema.RECORD_ID)); ffBuilder.entryDate((Long) record.getFieldValue(FlowFileSchema.ENTRY_DATE)); final Long lastQueueDate = (Long) record.getFieldValue(FlowFileSchema.QUEUE_DATE); final Long queueDateIndex = (Long) record.getFieldValue(FlowFileSchema.QUEUE_DATE_INDEX); ffBuilder.lastQueued(lastQueueDate, queueDateIndex); final Long lineageStartDate = (Long) record.getFieldValue(FlowFileSchema.LINEAGE_START_DATE); final Long lineageStartIndex = (Long) record.getFieldValue(FlowFileSchema.LINEAGE_START_INDEX); ffBuilder.lineageStart(lineageStartDate, lineageStartIndex); populateContentClaim(ffBuilder, record); ffBuilder.size((Long) record.getFieldValue(FlowFileSchema.FLOWFILE_SIZE)); ffBuilder.addAttributes((Map<String, String>) record.getFieldValue(FlowFileSchema.ATTRIBUTES)); final FlowFileRecord flowFileRecord = ffBuilder.build(); final String queueId = (String) record.getFieldValue(RepositoryRecordSchema.QUEUE_IDENTIFIER); final FlowFileQueue queue = getFlowFileQueue(queueId); final StandardRepositoryRecord repoRecord = new StandardRepositoryRecord(queue, flowFileRecord); requireFlowFileQueue(repoRecord, queueId); return repoRecord; }
@Override @SuppressWarnings("unchecked") protected synchronized void readHeader(final DataInputStream in, final int serializationVersion) throws IOException { verifySerializationVersion(serializationVersion); final int eventSchemaLength = in.readInt(); final byte[] buffer = new byte[eventSchemaLength]; StreamUtils.fillBuffer(in, buffer); try (final ByteArrayInputStream bais = new ByteArrayInputStream(buffer)) { schema = RecordSchema.readFrom(bais); } recordReader = SchemaRecordReader.fromSchema(schema); final int headerSchemaLength = in.readInt(); final byte[] headerSchemaBuffer = new byte[headerSchemaLength]; StreamUtils.fillBuffer(in, headerSchemaBuffer); final RecordSchema headerSchema; try (final ByteArrayInputStream bais = new ByteArrayInputStream(headerSchemaBuffer)) { headerSchema = RecordSchema.readFrom(bais); } final SchemaRecordReader headerReader = SchemaRecordReader.fromSchema(headerSchema); final Record headerRecord = headerReader.readRecord(in); componentIds = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.COMPONENT_IDS); componentTypes = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.COMPONENT_TYPES); queueIds = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.QUEUE_IDS); eventTypes = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.EVENT_TYPES); firstEventId = (Long) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.FIRST_EVENT_ID); systemTimeOffset = (Long) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.TIMESTAMP_OFFSET); }
@SuppressWarnings("unchecked") public static StandardProvenanceEventRecord getEvent(final Record record, final String storageFilename, final long storageByteOffset, final int maxAttributeLength) { final StandardProvenanceEventRecord.Builder builder = new StandardProvenanceEventRecord.Builder(); builder.setAlternateIdentifierUri((String) record.getFieldValue(EventFieldNames.ALTERNATE_IDENTIFIER)); builder.setChildUuids((List<String>) record.getFieldValue(EventFieldNames.CHILD_UUIDS)); builder.setComponentId((String) record.getFieldValue(EventFieldNames.COMPONENT_ID)); builder.setComponentType((String) record.getFieldValue(EventFieldNames.COMPONENT_TYPE)); builder.setDetails((String) record.getFieldValue(EventFieldNames.EVENT_DETAILS)); builder.setEventDuration((Long) record.getFieldValue(EventFieldNames.EVENT_DURATION)); builder.setEventTime((Long) record.getFieldValue(EventFieldNames.EVENT_TIME)); builder.setEventType(ProvenanceEventType.valueOf((String) record.getFieldValue(EventFieldNames.EVENT_TYPE))); builder.setFlowFileEntryDate((Long) record.getFieldValue(EventFieldNames.FLOWFILE_ENTRY_DATE)); builder.setFlowFileUUID((String) record.getFieldValue(EventFieldNames.FLOWFILE_UUID)); builder.setLineageStartDate((Long) record.getFieldValue(EventFieldNames.LINEAGE_START_DATE)); builder.setParentUuids((List<String>) record.getFieldValue(EventFieldNames.PARENT_UUIDS)); builder.setPreviousAttributes(truncateAttributes((Map<String, String>) record.getFieldValue(EventFieldNames.PREVIOUS_ATTRIBUTES), maxAttributeLength)); builder.setRelationship((String) record.getFieldValue(EventFieldNames.RELATIONSHIP)); builder.setSourceQueueIdentifier((String) record.getFieldValue(EventFieldNames.SOURCE_QUEUE_IDENTIFIER)); builder.setSourceSystemFlowFileIdentifier((String) record.getFieldValue(EventFieldNames.SOURCE_SYSTEM_FLOWFILE_IDENTIFIER)); builder.setTransitUri((String) record.getFieldValue(EventFieldNames.TRANSIT_URI)); builder.setUpdatedAttributes(truncateAttributes((Map<String, String>) record.getFieldValue(EventFieldNames.UPDATED_ATTRIBUTES), maxAttributeLength)); final Long eventId = (Long) record.getFieldValue(EventFieldNames.EVENT_IDENTIFIER); if (eventId != null) { builder.setEventId(eventId); final Record currentClaimRecord = (Record) record.getFieldValue(EventFieldNames.CONTENT_CLAIM); if (currentClaimRecord == null) { builder.setCurrentContentClaim(null, null, null, null, 0L); } else {
private RepositoryRecord swapOutRecord(final Record record) { final Long recordId = (Long) record.getFieldValue(RepositoryRecordSchema.RECORD_ID_FIELD); final String queueId = (String) record.getFieldValue(new SimpleRecordField(RepositoryRecordSchema.QUEUE_IDENTIFIER, FieldType.STRING, Repetition.EXACTLY_ONE)); final String swapLocation = (String) record.getFieldValue(new SimpleRecordField(RepositoryRecordSchema.SWAP_LOCATION, FieldType.STRING, Repetition.EXACTLY_ONE)); final FlowFileQueue queue = getFlowFileQueue(queueId); final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder() .id(recordId) .build(); return new StandardRepositoryRecord(queue, flowFileRecord, swapLocation); }
@Override @SuppressWarnings("unchecked") public SwapContents deserializeFlowFiles(final DataInputStream in, final String swapLocation, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException { final RecordSchema schema = RecordSchema.readFrom(in); final SchemaRecordReader reader = SchemaRecordReader.fromSchema(schema); final Record parentRecord = reader.readRecord(in); final List<Record> flowFileRecords = (List<Record>) parentRecord.getFieldValue(SwapSchema.FLOWFILE_CONTENTS); final List<FlowFileRecord> flowFiles = new ArrayList<>(flowFileRecords.size()); for (final Record record : flowFileRecords) { flowFiles.add(FlowFileRecordFieldMap.getFlowFile(record, claimManager)); } final Record summaryRecord = (Record) parentRecord.getFieldValue(SwapSchema.SWAP_SUMMARY); final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager); return new StandardSwapContents(swapSummary, flowFiles); }
final Map<String, String> previousAttributes = truncateAttributes((Map<String, String>) record.getFieldValue(EventFieldNames.PREVIOUS_ATTRIBUTES), maxAttributeLength); final Map<String, String> updatedAttributes = truncateAttributes((Map<String, String>) record.getFieldValue(EventFieldNames.UPDATED_ATTRIBUTES), maxAttributeLength); final List<String> childUuids = (List<String>) record.getFieldValue(EventFieldNames.CHILD_UUIDS); final List<String> parentUuids = (List<String>) record.getFieldValue(EventFieldNames.PARENT_UUIDS); builder.setAlternateIdentifierUri((String) record.getFieldValue(EventFieldNames.ALTERNATE_IDENTIFIER)); builder.setChildUuids(childUuids); builder.setDetails((String) record.getFieldValue(EventFieldNames.EVENT_DETAILS)); builder.setParentUuids(parentUuids); builder.setPreviousAttributes(previousAttributes); builder.setRelationship((String) record.getFieldValue(EventFieldNames.RELATIONSHIP)); builder.setSourceSystemFlowFileIdentifier((String) record.getFieldValue(EventFieldNames.SOURCE_SYSTEM_FLOWFILE_IDENTIFIER)); builder.setTransitUri((String) record.getFieldValue(EventFieldNames.TRANSIT_URI)); builder.setUpdatedAttributes(updatedAttributes); builder.setComponentId(readLookupValue(record.getFieldValue(EventFieldNames.COMPONENT_ID), componentIds)); builder.setComponentType(readLookupValue(record.getFieldValue(EventFieldNames.COMPONENT_TYPE), componentTypes)); builder.setSourceQueueIdentifier(readLookupValue(record.getFieldValue(EventFieldNames.SOURCE_QUEUE_IDENTIFIER), queueIds)); final Integer eventTypeOrdinal = (Integer) record.getFieldValue(EventFieldNames.EVENT_TYPE); ProvenanceEventType eventType; if (eventTypeOrdinal == null || eventTypeOrdinal > eventTypes.size() || eventTypeOrdinal < 0) { builder.setEventDuration((Integer) record.getFieldValue(EventFieldNames.EVENT_DURATION)); builder.setEventTime(addLong((Integer) record.getFieldValue(EventFieldNames.EVENT_TIME), startTimeOffset)); builder.setFlowFileEntryDate(addLong((Integer) record.getFieldValue(EventFieldNames.FLOWFILE_ENTRY_DATE), startTimeOffset)); builder.setLineageStartDate(addLong((Integer) record.getFieldValue(EventFieldNames.LINEAGE_START_DATE), startTimeOffset));
private RepositoryRecord createRepositoryRecord(final Record updateRecord) throws IOException { if (updateRecord == null) { // null may be returned by reader.readRecord() if it encounters end-of-stream return null; } // Top level is always going to be a "Repository Record Update" record because we need a 'Union' type record at the // top level that indicates which type of record we have. final Record record = (Record) updateRecord.getFieldValue(RepositoryRecordSchema.REPOSITORY_RECORD_UPDATE_V2); final String actionType = (String) record.getFieldValue(RepositoryRecordSchema.ACTION_TYPE_FIELD); final RepositoryRecordType recordType = RepositoryRecordType.valueOf(actionType); switch (recordType) { case CREATE: return createRecord(record); case CONTENTMISSING: case DELETE: return deleteRecord(record); case SWAP_IN: return swapInRecord(record); case SWAP_OUT: return swapOutRecord(record); case UPDATE: return updateRecord(record); } throw new IOException("Found unrecognized Update Type '" + actionType + "'"); }
private void populateContentClaim(final StandardFlowFileRecord.Builder ffBuilder, final Record record) { final Object claimMap = record.getFieldValue(FlowFileSchema.CONTENT_CLAIM); if (claimMap == null) { return; } final Record claimRecord = (Record) claimMap; final ContentClaim contentClaim = ContentClaimFieldMap.getContentClaim(claimRecord, resourceClaimManager); final Long offset = ContentClaimFieldMap.getContentClaimOffset(claimRecord); ffBuilder.contentClaim(contentClaim); ffBuilder.contentClaimOffset(offset); }
private RepositoryRecord deleteRecord(final Record record) { final Long recordId = (Long) record.getFieldValue(RepositoryRecordSchema.RECORD_ID_FIELD); final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder().id(recordId); final FlowFileRecord flowFileRecord = ffBuilder.build(); final StandardRepositoryRecord repoRecord = new StandardRepositoryRecord((FlowFileQueue) null, flowFileRecord); repoRecord.markForDelete(); return repoRecord; }
@Override public SwapSummary getSwapSummary(final DataInputStream in, final String swapLocation, final ResourceClaimManager claimManager) throws IOException { final RecordSchema schema = RecordSchema.readFrom(in); final List<RecordField> summaryFields = schema.getField(SwapSchema.SWAP_SUMMARY).getSubFields(); final RecordField summaryRecordField = new ComplexRecordField(SwapSchema.SWAP_SUMMARY, Repetition.EXACTLY_ONE, summaryFields); final RecordSchema summarySchema = new RecordSchema(Collections.singletonList(summaryRecordField)); final Record summaryRecordParent = SchemaRecordReader.fromSchema(summarySchema).readRecord(in); final Record summaryRecord = (Record) summaryRecordParent.getFieldValue(SwapSchema.SWAP_SUMMARY); final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager); return swapSummary; }
private RepositoryRecord swapInRecord(final Record record) { final StandardRepositoryRecord repoRecord = createRecord(record); final String swapLocation = (String) record.getFieldValue(new SimpleRecordField(RepositoryRecordSchema.SWAP_LOCATION, FieldType.STRING, Repetition.EXACTLY_ONE)); repoRecord.setSwapLocation(swapLocation); final String queueId = (String) record.getFieldValue(RepositoryRecordSchema.QUEUE_IDENTIFIER); requireFlowFileQueue(repoRecord, queueId); return repoRecord; }
private RepositoryRecord swapOutRecord(final Record record) { final Long recordId = (Long) record.getFieldValue(RepositoryRecordSchema.RECORD_ID_FIELD); final String queueId = (String) record.getFieldValue(new SimpleRecordField(RepositoryRecordSchema.QUEUE_IDENTIFIER, FieldType.STRING, Repetition.EXACTLY_ONE)); final String swapLocation = (String) record.getFieldValue(new SimpleRecordField(RepositoryRecordSchema.SWAP_LOCATION, FieldType.STRING, Repetition.EXACTLY_ONE)); final FlowFileQueue queue = getFlowFileQueue(queueId); final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder() .id(recordId) .build(); return new StandardRepositoryRecord(queue, flowFileRecord, swapLocation); }