@Override public void readHeader(final DataInputStream in) throws IOException { final RecordSchema recoverySchema = RecordSchema.readFrom(in); reader = SchemaRecordReader.fromSchema(recoverySchema); }
@Override public Record next() throws IOException { return readRecord(bufferedIn); }
public static SchemaRecordReader fromSchema(final RecordSchema schema) { return new SchemaRecordReader(schema); }
@Override @SuppressWarnings("unchecked") public SwapContents deserializeFlowFiles(final DataInputStream in, final String swapLocation, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException { final RecordSchema schema = RecordSchema.readFrom(in); final SchemaRecordReader reader = SchemaRecordReader.fromSchema(schema); final Record parentRecord = reader.readRecord(in); final List<Record> flowFileRecords = (List<Record>) parentRecord.getFieldValue(SwapSchema.FLOWFILE_CONTENTS); final List<FlowFileRecord> flowFiles = new ArrayList<>(flowFileRecords.size()); for (final Record record : flowFileRecords) { flowFiles.add(FlowFileRecordFieldMap.getFlowFile(record, claimManager)); } final Record summaryRecord = (Record) parentRecord.getFieldValue(SwapSchema.SWAP_SUMMARY); final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager); return new StandardSwapContents(swapSummary, flowFiles); }
return readInt(in); final int length = readInt(in); final byte[] buffer = new byte[length]; fillBuffer(in, buffer); return new String(buffer, StandardCharsets.UTF_8); final int length = readInt(in); final byte[] buffer = new byte[length]; fillBuffer(in, buffer); return buffer; final int numEntries = readInt(in); final RecordField keyField = subFields.get(0); final RecordField valueField = subFields.get(1); final Object key = readField(in, keyField); final Object value = readField(in, valueField); entries.put(key, value); final Map<RecordField, Object> subFieldValues = new HashMap<>(numSubFields); for (int i = 0; i < numSubFields; i++) { final Object subFieldValue = readField(in, subFields.get(i)); subFieldValues.put(subFields.get(i), subFieldValue); return readField(in, matchingField);
case ZERO_OR_MORE: { final int iterations = readInt(in); if (iterations == 0) { return Collections.emptyList(); value.add(readFieldValue(in, field.getFieldType(), field.getFieldName(), field.getSubFields())); return readFieldValue(in, field.getFieldType(), field.getFieldName(), field.getSubFields()); } catch (final EOFException eof) { final EOFException exception = new EOFException("Failed to read field '" + field.getFieldName() + "'");
public Record readRecord(final InputStream in) throws IOException { final int recordIndicator = in.read(); if (recordIndicator < 0) { return null; } if (recordIndicator == SchemaRecordWriter.EXTERNAL_FILE_INDICATOR) { throw new IOException("Expected to read a Sentinel Byte of '1' indicating that the next record is inline but the Sentinel value was '" + SchemaRecordWriter.EXTERNAL_FILE_INDICATOR + ", indicating that data was written to an External File. This data cannot be recovered via calls to #readRecord(InputStream) but must be recovered via #readRecords(InputStream)"); } if (recordIndicator != 1) { throw new IOException("Expected to read a Sentinel Byte of '1' but got a value of '" + recordIndicator + "' instead"); } return readInlineRecord(in); }
@Override public RepositoryRecord deserializeRecord(final DataInputStream in, final int version) throws IOException { if (recordIterator != null) { final RepositoryRecord record = nextRecord(); if (record != null) { return record; } recordIterator.close(); } recordIterator = reader.readRecords(in); if (recordIterator == null) { return null; } return nextRecord(); }
private int readInt(final InputStream in) throws IOException { final byte[] buffer = new byte[4]; fillBuffer(in, buffer); return ByteBuffer.wrap(buffer).getInt(); } }
private Record readInlineRecord(final InputStream in) throws IOException { final List<RecordField> schemaFields = schema.getFields(); final Map<RecordField, Object> fields = new HashMap<>(schemaFields.size()); for (final RecordField field : schema.getFields()) { final Object value = readField(in, field); fields.put(field, value); } return new FieldMapRecord(fields, schema); }
@Override public SwapSummary getSwapSummary(final DataInputStream in, final String swapLocation, final ResourceClaimManager claimManager) throws IOException { final RecordSchema schema = RecordSchema.readFrom(in); final List<RecordField> summaryFields = schema.getField(SwapSchema.SWAP_SUMMARY).getSubFields(); final RecordField summaryRecordField = new ComplexRecordField(SwapSchema.SWAP_SUMMARY, Repetition.EXACTLY_ONE, summaryFields); final RecordSchema summarySchema = new RecordSchema(Collections.singletonList(summaryRecordField)); final Record summaryRecordParent = SchemaRecordReader.fromSchema(summarySchema).readRecord(in); final Record summaryRecord = (Record) summaryRecordParent.getFieldValue(SwapSchema.SWAP_SUMMARY); final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager); return swapSummary; }
final Record nextRecord = readInlineRecord(in); return new SingleRecordIterator(nextRecord);
@Override public RepositoryRecord deserializeRecord(final DataInputStream in, final int version) throws IOException { if (recordIterator != null) { final RepositoryRecord record = nextRecord(); if (record != null) { return record; } recordIterator.close(); } recordIterator = reader.readRecords(in); if (recordIterator == null) { return null; } return nextRecord(); }
@Override @SuppressWarnings("unchecked") protected synchronized void readHeader(final DataInputStream in, final int serializationVersion) throws IOException { verifySerializationVersion(serializationVersion); final int eventSchemaLength = in.readInt(); final byte[] buffer = new byte[eventSchemaLength]; StreamUtils.fillBuffer(in, buffer); try (final ByteArrayInputStream bais = new ByteArrayInputStream(buffer)) { schema = RecordSchema.readFrom(bais); } recordReader = SchemaRecordReader.fromSchema(schema); final int headerSchemaLength = in.readInt(); final byte[] headerSchemaBuffer = new byte[headerSchemaLength]; StreamUtils.fillBuffer(in, headerSchemaBuffer); final RecordSchema headerSchema; try (final ByteArrayInputStream bais = new ByteArrayInputStream(headerSchemaBuffer)) { headerSchema = RecordSchema.readFrom(bais); } final SchemaRecordReader headerReader = SchemaRecordReader.fromSchema(headerSchema); final Record headerRecord = headerReader.readRecord(in); componentIds = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.COMPONENT_IDS); componentTypes = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.COMPONENT_TYPES); queueIds = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.QUEUE_IDS); eventTypes = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.EVENT_TYPES); firstEventId = (Long) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.FIRST_EVENT_ID); systemTimeOffset = (Long) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.TIMESTAMP_OFFSET); }
@Override protected StandardProvenanceEventRecord nextRecord(final DataInputStream in, final int serializationVersion) throws IOException { verifySerializationVersion(serializationVersion); final long byteOffset = getBytesConsumed(); final int recordLength = in.readInt(); final InputStream limitedIn = new LimitingInputStream(in, recordLength); final Record eventRecord = recordReader.readRecord(limitedIn); if (eventRecord == null) { return null; } return EventRecord.getEvent(eventRecord, getFilename(), byteOffset, getMaxAttributeLength()); }
@Override protected void readHeader(final DataInputStream in, final int serializationVersion) throws IOException { verifySerializationVersion(serializationVersion); final int schemaLength = in.readInt(); final byte[] buffer = new byte[schemaLength]; StreamUtils.fillBuffer(in, buffer); try (final ByteArrayInputStream bais = new ByteArrayInputStream(buffer)) { schema = RecordSchema.readFrom(bais); } recordReader = SchemaRecordReader.fromSchema(schema); }
private StandardProvenanceEventRecord readRecord(final DataInputStream in, final long eventId, final long startOffset, final int recordLength) throws IOException { final InputStream limitedIn = new LimitingInputStream(in, recordLength); final Record eventRecord = recordReader.readRecord(limitedIn); if (eventRecord == null) { return null; } final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(), firstEventId, systemTimeOffset, componentIds, componentTypes, queueIds, eventTypes); deserializedEvent.setEventId(eventId); return deserializedEvent; }
@Override public void readHeader(final DataInputStream in) throws IOException { final RecordSchema recoverySchema = RecordSchema.readFrom(in); reader = SchemaRecordReader.fromSchema(recoverySchema); }
private StandardProvenanceEventRecord readRecord(final DataInputStream inputStream, final long eventId, final long startOffset, final int recordLength) throws IOException { try { final InputStream limitedIn = new LimitingInputStream(inputStream, recordLength); byte[] encryptedSerializedBytes = new byte[recordLength]; DataInputStream encryptedInputStream = new DataInputStream(limitedIn); encryptedInputStream.readFully(encryptedSerializedBytes); byte[] plainSerializedBytes = decrypt(encryptedSerializedBytes, Long.toString(eventId)); InputStream plainStream = new ByteArrayInputStream(plainSerializedBytes); final Record eventRecord = getRecordReader().readRecord(plainStream); if (eventRecord == null) { return null; } final StandardProvenanceEventRecord deserializedEvent = LookupTableEventRecord.getEvent(eventRecord, getFilename(), startOffset, getMaxAttributeLength(), getFirstEventId(), getSystemTimeOffset(), getComponentIds(), getComponentTypes(), getQueueIds(), getEventTypes()); deserializedEvent.setEventId(eventId); return deserializedEvent; } catch (EncryptionException e) { logger.error("Encountered an error reading the record: ", e); throw new IOException(e); } }