private static Record createContentClaimRecord(final RecordSchema contentClaimSchema, final String container, final String section, final String identifier, final Long offset, final Long size) { if (container == null || section == null || identifier == null) { final Map<RecordField, Object> lookupValues = Collections.singletonMap(LookupTableEventRecordFields.NO_VALUE, EventFieldNames.NO_VALUE); final List<RecordField> noValueFields = Collections.singletonList(contentClaimSchema.getField(EventFieldNames.NO_VALUE)); return new FieldMapRecord(lookupValues, new RecordSchema(noValueFields)); } final Map<RecordField, Object> fieldValues = new HashMap<>(); fieldValues.put(EventRecordFields.CONTENT_CLAIM_CONTAINER, container); fieldValues.put(EventRecordFields.CONTENT_CLAIM_SECTION, section); fieldValues.put(EventRecordFields.CONTENT_CLAIM_IDENTIFIER, identifier); fieldValues.put(EventRecordFields.CONTENT_CLAIM_OFFSET, offset); fieldValues.put(EventRecordFields.CONTENT_CLAIM_SIZE, size); final List<RecordField> explicitClaimFields = contentClaimSchema.getField(EventFieldNames.EXPLICIT_VALUE).getSubFields(); final Record explicitClaimRecord = new FieldMapRecord(fieldValues, new RecordSchema(explicitClaimFields)); return explicitClaimRecord; }
private Record readInlineRecord(final InputStream in) throws IOException { final List<RecordField> schemaFields = schema.getFields(); final Map<RecordField, Object> fields = new HashMap<>(schemaFields.size()); for (final RecordField field : schema.getFields()) { final Object value = readField(in, field); fields.put(field, value); } return new FieldMapRecord(fields, schema); }
@Override public SwapSummary getSwapSummary(final DataInputStream in, final String swapLocation, final ResourceClaimManager claimManager) throws IOException { final RecordSchema schema = RecordSchema.readFrom(in); final List<RecordField> summaryFields = schema.getField(SwapSchema.SWAP_SUMMARY).getSubFields(); final RecordField summaryRecordField = new ComplexRecordField(SwapSchema.SWAP_SUMMARY, Repetition.EXACTLY_ONE, summaryFields); final RecordSchema summarySchema = new RecordSchema(Collections.singletonList(summaryRecordField)); final Record summaryRecordParent = SchemaRecordReader.fromSchema(summarySchema).readRecord(in); final Record summaryRecord = (Record) summaryRecordParent.getFieldValue(SwapSchema.SWAP_SUMMARY); final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager); return swapSummary; }
public static RecordSchema readFrom(final InputStream in) throws IOException { try { final DataInputStream dis = (in instanceof DataInputStream) ? (DataInputStream) in : new DataInputStream(in); final int numFields = dis.readInt(); final List<RecordField> fields = new ArrayList<>(numFields); for (int i = 0; i < numFields; i++) { final RecordField field = readField(dis); fields.add(field); } return new RecordSchema(fields); } catch (final IOException ioe) { throw new IOException("Unable to read Record Schema from stream", ioe); } }
@Override public void serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream out) throws IOException { schema.writeTo(out); final RecordField contentsField = new ComplexRecordField(SwapSchema.FLOWFILE_CONTENTS, Repetition.ZERO_OR_MORE, FlowFileSchema.FLOWFILE_SCHEMA_V2.getFields()); final List<RecordField> fields = new ArrayList<>(2); fields.add(summaryField); swapFileMap.put(summaryField, summaryRecord); swapFileMap.put(contentsField, flowFileRecords); final Record swapFileRecord = new FieldMapRecord(swapFileMap, new RecordSchema(fields));
final RecordSchema schema = new RecordSchema(fields); return schema;
@Override public void readHeader(final DataInputStream in) throws IOException { final RecordSchema recoverySchema = RecordSchema.readFrom(in); reader = SchemaRecordReader.fromSchema(recoverySchema); }
@Override public void writeHeader(final DataOutputStream out) throws IOException { writeSchema.writeTo(out); }
public void writeTo(final OutputStream out) throws IOException { try { final DataOutputStream dos = (out instanceof DataOutputStream) ? (DataOutputStream) out : new DataOutputStream(out); dos.writeInt(fields.size()); for (final RecordField field : fields) { writeField(field, dos); } } catch (final IOException ioe) { throw new IOException("Unable to write Record Schema to stream", ioe); } }
final List<RecordField> subFields = new ArrayList<>(numFields); for (int j = 0; j < numFields; j++) { subFields.add(readField(dis));
final RecordSchema schema = new RecordSchema(fields); return schema;
@Override protected void readHeader(final DataInputStream in, final int serializationVersion) throws IOException { verifySerializationVersion(serializationVersion); final int schemaLength = in.readInt(); final byte[] buffer = new byte[schemaLength]; StreamUtils.fillBuffer(in, buffer); try (final ByteArrayInputStream bais = new ByteArrayInputStream(buffer)) { schema = RecordSchema.readFrom(bais); } recordReader = SchemaRecordReader.fromSchema(schema); }
@Override public void writeHeader(final long firstEventId, final DataOutputStream out) throws IOException { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); eventSchema.writeTo(baos); out.writeInt(baos.size()); baos.writeTo(out); }
private void writeField(final RecordField field, final DataOutputStream dos) throws IOException { dos.writeInt(4); // A field is made up of 4 "elements": Field Name, Field Type, Field Repetition, Sub-Fields. // For each of the elements, we write a String indicating the Element Name, a String indicating the Element Type, and // finally the Element data itself. dos.writeUTF(FIELD_NAME); dos.writeUTF(STRING_TYPE); dos.writeUTF(field.getFieldName()); dos.writeUTF(FIELD_TYPE); dos.writeUTF(STRING_TYPE); dos.writeUTF(field.getFieldType().name()); dos.writeUTF(REPETITION); dos.writeUTF(STRING_TYPE); dos.writeUTF(field.getRepetition().name()); dos.writeUTF(SUBFIELDS); dos.writeUTF(SUBFIELD_TYPE); final List<RecordField> subFields = field.getSubFields(); dos.writeInt(subFields.size()); // SubField is encoded as number of Sub-Fields followed by the fields themselves. for (final RecordField subField : subFields) { writeField(subField, dos); } }
public ContentClaimFieldMap(final ContentClaim contentClaim, final long contentClaimOffset, final RecordSchema schema) { this.contentClaim = contentClaim; this.contentClaimOffset = contentClaimOffset; this.schema = schema; final List<RecordField> resourceClaimFields = schema.getField(ContentClaimSchema.RESOURCE_CLAIM).getSubFields(); final RecordSchema resourceClaimSchema = new RecordSchema(resourceClaimFields); this.resourceClaimFieldMap = new ResourceClaimFieldMap(contentClaim.getResourceClaim(), resourceClaimSchema); }
private static RecordSchema buildSchema() { final List<RecordField> fields = new ArrayList<>(); fields.add(new SimpleRecordField(FieldNames.FIRST_EVENT_ID, FieldType.LONG, Repetition.EXACTLY_ONE)); fields.add(new SimpleRecordField(FieldNames.TIMESTAMP_OFFSET, FieldType.LONG, Repetition.EXACTLY_ONE)); fields.add(new SimpleRecordField(FieldNames.COMPONENT_IDS, FieldType.STRING, Repetition.ZERO_OR_MORE)); fields.add(new SimpleRecordField(FieldNames.COMPONENT_TYPES, FieldType.STRING, Repetition.ZERO_OR_MORE)); fields.add(new SimpleRecordField(FieldNames.QUEUE_IDS, FieldType.STRING, Repetition.ZERO_OR_MORE)); fields.add(new SimpleRecordField(FieldNames.EVENT_TYPES, FieldType.STRING, Repetition.ZERO_OR_MORE)); return new RecordSchema(fields); } }
@Override @SuppressWarnings("unchecked") public SwapContents deserializeFlowFiles(final DataInputStream in, final String swapLocation, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException { final RecordSchema schema = RecordSchema.readFrom(in); final SchemaRecordReader reader = SchemaRecordReader.fromSchema(schema); final Record parentRecord = reader.readRecord(in); final List<Record> flowFileRecords = (List<Record>) parentRecord.getFieldValue(SwapSchema.FLOWFILE_CONTENTS); final List<FlowFileRecord> flowFiles = new ArrayList<>(flowFileRecords.size()); for (final Record record : flowFileRecords) { flowFiles.add(FlowFileRecordFieldMap.getFlowFile(record, claimManager)); } final Record summaryRecord = (Record) parentRecord.getFieldValue(SwapSchema.SWAP_SUMMARY); final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager); return new StandardSwapContents(swapSummary, flowFiles); }
@Override protected synchronized void writeHeader(final long firstEventId, final DataOutputStream out) throws IOException { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); eventSchema.writeTo(baos); out.writeInt(baos.size()); baos.writeTo(out); baos.reset(); headerSchema.writeTo(baos); out.writeInt(baos.size()); baos.writeTo(out); this.firstEventId = firstEventId; this.systemTimeOffset = System.currentTimeMillis(); final Map<String, Object> headerValues = new HashMap<>(); headerValues.put(EventIdFirstHeaderSchema.FieldNames.FIRST_EVENT_ID, firstEventId); headerValues.put(EventIdFirstHeaderSchema.FieldNames.TIMESTAMP_OFFSET, systemTimeOffset); headerValues.put(EventIdFirstHeaderSchema.FieldNames.COMPONENT_IDS, idLookup.getComponentIdentifiers()); headerValues.put(EventIdFirstHeaderSchema.FieldNames.COMPONENT_TYPES, idLookup.getComponentTypes()); headerValues.put(EventIdFirstHeaderSchema.FieldNames.QUEUE_IDS, idLookup.getQueueIdentifiers()); headerValues.put(EventIdFirstHeaderSchema.FieldNames.EVENT_TYPES, eventTypeNames); final FieldMapRecord headerInfo = new FieldMapRecord(headerSchema, headerValues); schemaRecordWriter.writeRecord(headerInfo, out); }
private void writeRecordFields(final Record record, final RecordSchema schema, final OutputStream out, final byte[] buffer) throws IOException { final DataOutputStream dos = out instanceof DataOutputStream ? (DataOutputStream) out : new DataOutputStream(out); for (final RecordField field : schema.getFields()) { final Object value = record.getFieldValue(field); try { writeFieldRepetitionAndValue(field, value, dos, buffer); } catch (final Exception e) { throw new IOException("Failed to write field '" + field.getFieldName() + "'", e); } } }
public SwapSummaryFieldMap(final SwapSummary summary, final String queueIdentifier, final RecordSchema schema) { this.swapSummary = summary; this.queueIdentifier = queueIdentifier; this.schema = schema; final RecordField resourceClaimField = schema.getField(SwapSchema.RESOURCE_CLAIMS).getSubFields().get(0); final RecordSchema resourceClaimSchema = new RecordSchema(resourceClaimField.getSubFields()); final List<ResourceClaim> resourceClaims = summary.getResourceClaims(); claimCounts = new HashMap<>(); for (final ResourceClaim claim : resourceClaims) { final ResourceClaimFieldMap fieldMap = new ResourceClaimFieldMap(claim, resourceClaimSchema); final Integer count = claimCounts.get(fieldMap); if (count == null) { claimCounts.put(fieldMap, 1); } else { claimCounts.put(fieldMap, count + 1); } } }