protected void serializeRecord(final RepositoryRecord record, final DataOutputStream out, RecordSchema schema, RecordSchema repositoryRecordSchema) throws IOException { final RepositoryRecordFieldMap fieldMap = new RepositoryRecordFieldMap(record, schema, contentClaimSchema); final RepositoryRecordUpdate update = new RepositoryRecordUpdate(fieldMap, repositoryRecordSchema); new SchemaRecordWriter().writeRecord(update, out); }
private void writeUTFLimited(final DataOutputStream out, final String utfString, final String fieldName) throws IOException { try { out.writeUTF(utfString); } catch (UTFDataFormatException e) { final String truncated = utfString.substring(0, getCharsInUTF8Limit(utfString, MAX_ALLOWED_UTF_LENGTH)); logger.warn("Truncating repository record value for field '{}'! Attempted to write {} chars that encode to a UTF8 byte length greater than " + "supported maximum ({}), truncating to {} chars.", (fieldName == null) ? "" : fieldName, utfString.length(), MAX_ALLOWED_UTF_LENGTH, truncated.length()); if (logger.isDebugEnabled()) { logger.warn("String value was:\n{}", truncated); } out.writeUTF(truncated); } }
@Override public void writeExternalFileReference(final File externalFile, final DataOutputStream out) throws IOException { new SchemaRecordWriter().writeExternalFileReference(out, externalFile); }
@Override protected void writeRecord(final ProvenanceEventRecord event, final long eventId, final DataOutputStream out) throws IOException { final Record eventRecord = createRecord(event, eventId); schemaRecordWriter.writeRecord(eventRecord, out); }
break; case STRING: writeUTFLimited(out, (String) value, field.getFieldName()); break; case LONG_STRING: final int length = string.length(); if (length <= buffer.length && allSingleByteInUtf8(string)) { out.writeInt(length); writeFieldRepetitionAndValue(keyField, entry.getKey(), out, buffer); writeFieldRepetitionAndValue(valueField, entry.getValue(), out, buffer); writeUTFLimited(out, namedValue.getName(), field.getFieldName()); final Record childRecord = (Record) namedValue.getValue(); writeRecordFields(childRecord, out, buffer); break; case COMPLEX: final Record record = (Record) value; writeRecordFields(record, out, buffer); break;
private void writeRecordFields(final Record record, final OutputStream out, final byte[] buffer) throws IOException { writeRecordFields(record, record.getSchema(), out, buffer); }
private void writeRecordFields(final Record record, final RecordSchema schema, final OutputStream out, final byte[] buffer) throws IOException { final DataOutputStream dos = out instanceof DataOutputStream ? (DataOutputStream) out : new DataOutputStream(out); for (final RecordField field : schema.getFields()) { final Object value = record.getFieldValue(field); try { writeFieldRepetitionAndValue(field, value, dos, buffer); } catch (final Exception e) { throw new IOException("Failed to write field '" + field.getFieldName() + "'", e); } } }
throw new IllegalArgumentException("Record does not have a value for the '" + field.getFieldName() + "' but the field is required"); writeFieldValue(field, value, dos, buffer); break; dos.writeInt(collection.size()); for (final Object fieldValue : collection) { writeFieldValue(field, fieldValue, dos, buffer); writeFieldValue(field, value, dos, buffer); break;
@Override protected void writeRecord(final ProvenanceEventRecord event, final long eventId, final DataOutputStream out) throws IOException { try (final ByteArrayOutputStream baos = new ByteArrayOutputStream(256)) { final Record eventRecord = createRecord(event, eventId); recordWriter.writeRecord(eventRecord, baos); out.writeInt(baos.size()); baos.writeTo(out); } } }
public void writeRecord(final Record record, final OutputStream out) throws IOException { // write sentinel value to indicate that there is a record. This allows the reader to then read one // byte and check if -1. If so, the reader knows there are no more records. If not, then the reader // knows that it should be able to continue reading. out.write(INLINE_RECORD_INDICATOR); final byte[] buffer = byteArrayCache.checkOut(); try { writeRecordFields(record, out, buffer); } finally { byteArrayCache.checkIn(buffer); } }
final Record swapFileRecord = new FieldMapRecord(swapFileMap, new RecordSchema(fields)); final SchemaRecordWriter writer = new SchemaRecordWriter(); writer.writeRecord(swapFileRecord, out); out.flush();
@Override protected synchronized void writeHeader(final long firstEventId, final DataOutputStream out) throws IOException { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); eventSchema.writeTo(baos); out.writeInt(baos.size()); baos.writeTo(out); baos.reset(); headerSchema.writeTo(baos); out.writeInt(baos.size()); baos.writeTo(out); this.firstEventId = firstEventId; this.systemTimeOffset = System.currentTimeMillis(); final Map<String, Object> headerValues = new HashMap<>(); headerValues.put(EventIdFirstHeaderSchema.FieldNames.FIRST_EVENT_ID, firstEventId); headerValues.put(EventIdFirstHeaderSchema.FieldNames.TIMESTAMP_OFFSET, systemTimeOffset); headerValues.put(EventIdFirstHeaderSchema.FieldNames.COMPONENT_IDS, idLookup.getComponentIdentifiers()); headerValues.put(EventIdFirstHeaderSchema.FieldNames.COMPONENT_TYPES, idLookup.getComponentTypes()); headerValues.put(EventIdFirstHeaderSchema.FieldNames.QUEUE_IDS, idLookup.getQueueIdentifiers()); headerValues.put(EventIdFirstHeaderSchema.FieldNames.EVENT_TYPES, eventTypeNames); final FieldMapRecord headerInfo = new FieldMapRecord(headerSchema, headerValues); schemaRecordWriter.writeRecord(headerInfo, out); }
@Override public void writeExternalFileReference(final File externalFile, final DataOutputStream out) throws IOException { new SchemaRecordWriter().writeExternalFileReference(out, externalFile); }
protected void serializeRecord(final RepositoryRecord record, final DataOutputStream out, RecordSchema schema, RecordSchema repositoryRecordSchema) throws IOException { final RepositoryRecordFieldMap fieldMap = new RepositoryRecordFieldMap(record, schema, contentClaimSchema); final RepositoryRecordUpdate update = new RepositoryRecordUpdate(fieldMap, repositoryRecordSchema); new SchemaRecordWriter().writeRecord(update, out); }