long getOffset() { return logRecord.getTransactionId() - logRecord.getPayload().length; } }
private void printEntry(Entry.Reader reader) throws Exception { LogRecordWithDLSN record = reader.nextRecord(); while (null != record) { System.out.println("\t" + record); if (!skipPayload) { if (printHex) { System.out.println(Hex.encodeHexString(record.getPayload())); } else { System.out.println(new String(record.getPayload(), UTF_8)); } } System.out.println(""); record = reader.nextRecord(); } }
@Override public void onSuccess(LogRecordWithDLSN record) { System.out.println("Received record " + record.getDlsn()); System.out.println("\"\"\""); System.out.println(new String(record.getPayload(), UTF_8)); System.out.println("\"\"\""); reader.readNext().addEventListener(this); } };
@Override public void onSuccess(LogRecordWithDLSN record) { System.out.println("Received record " + record.getDlsn()); System.out.println("\"\"\""); System.out.println(new String(record.getPayload(), UTF_8)); System.out.println("\"\"\""); lastDLSN.set(record.getDlsn()); reader.readNext().addEventListener(this); } };
@Override public void onSuccess(LogRecordWithDLSN record) { System.out.println("Received record " + record.getDlsn()); System.out.println("\"\"\""); System.out.println(new String(record.getPayload(), UTF_8)); System.out.println("\"\"\""); long diffInMilliseconds = System.currentTimeMillis() - record.getTransactionId(); if (!caughtup.get() && diffInMilliseconds < 2000) { System.out.println("Reader caught with latest data"); caughtup.set(true); } reader.readNext().addEventListener(this); } };
/** * Poll next record from the readahead queue. * * @return next record from readahead queue. null if no records available in the queue. * @throws IOException */ public LogRecordWithDLSN getNextReadAheadRecord() throws IOException { if (null != lastException.get()) { throw lastException.get(); } LogRecordWithDLSN record = readAheadRecords.poll(); if (null != record) { cacheBytes.addAndGet(-record.getPayload().length); if (!isCacheFull()) { invokeReadAheadCallback(); } } return record; }
@Override public void onSuccess(LogRecordWithDLSN record) { System.out.println("Received record " + record.getDlsn() + " from stream " + reader.getStreamName()); System.out.println("\"\"\""); System.out.println(new String(record.getPayload(), UTF_8)); System.out.println("\"\"\""); reader.readNext().addEventListener(this); } };
LogRecordWithInputStream(LogRecordWithDLSN logRecord) { Preconditions.checkNotNull(logRecord); LOG.debug("Got record dlsn = {}, txid = {}, len = {}", new Object[] {logRecord.getDlsn(), logRecord.getTransactionId(), logRecord.getPayload().length}); this.logRecord = logRecord; this.payloadStream = logRecord.getPayLoadInputStream(); }
@Override public void onSuccess(LogRecordWithDLSN record) { if (record.getDlsn().compareTo(fromDLSN) <= 0) { reader.readNext().addEventListener(this); return; } System.out.println("Received record " + record.getDlsn()); System.out.println("\"\"\""); System.out.println(new String(record.getPayload(), UTF_8)); System.out.println("\"\"\""); try { transform(targetWriter, record, replicationTransformer, keepAliveLatch); } catch (Exception e) { System.err.println("Encountered error on transforming record " + record.getDlsn() + " from stream " + dlm.getStreamName()); e.printStackTrace(System.err); keepAliveLatch.countDown(); } reader.readNext().addEventListener(this); } };
public void processRecord(final LogRecordWithDLSN record) { Message msg; try { msg = Utils.parseMessage(record.getPayload()); } catch (TException e) { invalidRecordsCounter.inc(); LOG.warn("Failed to parse record {} for stream {} : size = {} , ", new Object[] { record, streamIdx, record.getPayload().length, e }); return; } long curTimeMillis = System.currentTimeMillis(); long e2eLatency = curTimeMillis - msg.getPublishTime(); long deliveryLatency = curTimeMillis - record.getTransactionId(); if (e2eLatency >= 0) { e2eStat.registerSuccessfulEvent(e2eLatency); } else { negativeE2EStat.registerSuccessfulEvent(-e2eLatency); } if (deliveryLatency >= 0) { deliveryStat.registerSuccessfulEvent(deliveryLatency); } else { negativeDeliveryStat.registerSuccessfulEvent(-deliveryLatency); } prevDLSN = record.getDlsn(); }
public static Reader of(LogRecordWithDLSN record) throws IOException { Preconditions.checkArgument(record.isRecordSet(), "record is not a recordset"); byte[] data = record.getPayload(); DLSN dlsn = record.getDlsn(); int startPosition = record.getPositionWithinLogSegment(); long startSequenceId = record.getStartSequenceIdOfCurrentSegment(); return new EnvelopedRecordSetReader( dlsn.getLogSegmentSequenceNo(), dlsn.getEntryId(), record.getTransactionId(), dlsn.getSlotId(), startPosition, startSequenceId, new ByteArrayInputStream(data)); }
private static void transform(final AsyncLogWriter writer, LogRecordWithDLSN record, Transformer<byte[], byte[]> replicationTransformer, final CountDownLatch keepAliveLatch) throws Exception { DLSN srcDLSN = record.getDlsn(); byte[] payload = record.getPayload(); byte[] transformedPayload = replicationTransformer.transform(payload); TransformedRecord transformedRecord = new TransformedRecord(ByteBuffer.wrap(transformedPayload)); transformedRecord.setSrcDlsn(srcDLSN.serializeBytes()); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); transformedRecord.write(protocolFactory.getProtocol(new TIOStreamTransport(baos))); byte[] data = baos.toByteArray(); writer.write(new LogRecord(record.getSequenceId(), data)) .addEventListener(new FutureEventListener<DLSN>() { @Override public void onFailure(Throwable cause) { System.err.println("Encountered error on writing records to stream " + writer.getStreamName()); cause.printStackTrace(System.err); keepAliveLatch.countDown(); } @Override public void onSuccess(DLSN dlsn) { System.out.println("Write transformed record " + dlsn); } }); }
try { lastTransformedRecord.read(protocolFactory.getProtocol( new TIOStreamTransport(new ByteArrayInputStream(lastTargetRecord.getPayload())))); srcDlsn = DLSN.deserializeBytes(lastTransformedRecord.getSrcDlsn()); System.out.println("Last transformed record is " + srcDlsn);
cacheBytes.addAndGet(record.getPayload().length);
assertEquals(expectedTxId, record.getTransactionId()); assertNotNull("record " + record + " payload is null", record.getPayload()); assertEquals("record-" + expectedTxId, new String(record.getPayload(), UTF_8)); expectedDLSN = expectedDLSN.getNextDLSN(); ++expectedTxId; assertEquals(txIdOfRecordSet, record.getTransactionId()); assertNotNull("record " + record + " payload is null", record.getPayload()); assertEquals("record-" + expectedTxId, new String(record.getPayload(), UTF_8)); expectedDLSN = expectedDLSN.getNextDLSN(); ++expectedTxId; assertEquals(expectedTxId, record.getTransactionId()); assertNotNull("record " + record + " payload is null", record.getPayload()); assertEquals("record-" + expectedTxId, new String(record.getPayload(), UTF_8)); expectedDLSN = expectedDLSN.getNextDLSN(); ++expectedTxId;
assertArrayEquals(DLMTestUtil.generatePayload(i+1), record.getPayload()); assertEquals(1L + i, record.getTransactionId()); assertEquals(i + 1, record.getPositionWithinLogSegment()); assertArrayEquals(DLMTestUtil.generatePayload(i+1), record.getPayload()); } else if (i >= 6L) { assertEquals(new DLSN(1L, 6L + i - 6, 0L), record.getDlsn()); assertEquals(11L + i - 6, record.getTransactionId()); assertEquals(11 + i - 6, record.getPositionWithinLogSegment()); assertArrayEquals(DLMTestUtil.generatePayload(11L + i - 6), record.getPayload()); } else { assertEquals(new DLSN(1L, 5L, 0), record.getDlsn());
@Override public void onSuccess(LogRecordWithDLSN record) { System.out.println("Received record " + record.getDlsn()); System.out.println("\"\"\""); System.out.println(new String(record.getPayload(), UTF_8)); System.out.println("\"\"\""); lastDLSN.set(record.getDlsn()); reader.readNext().addEventListener(this); } };
@Override public void onSuccess(LogRecordWithDLSN record) { if (record.getDlsn().compareTo(fromDLSN) <= 0) { reader.readNext().addEventListener(this); return; } System.out.println("Received record " + record.getDlsn()); System.out.println("\"\"\""); System.out.println(new String(record.getPayload(), UTF_8)); System.out.println("\"\"\""); try { transform(targetWriter, record, replicationTransformer, keepAliveLatch); } catch (Exception e) { System.err.println("Encountered error on transforming record " + record.getDlsn() + " from stream " + dlm.getStreamName()); e.printStackTrace(System.err); keepAliveLatch.countDown(); } reader.readNext().addEventListener(this); } };
public static Reader of(LogRecordWithDLSN record) throws IOException { Preconditions.checkArgument(record.isRecordSet(), "record is not a recordset"); byte[] data = record.getPayload(); DLSN dlsn = record.getDlsn(); int startPosition = record.getPositionWithinLogSegment(); long startSequenceId = record.getStartSequenceIdOfCurrentSegment(); return new EnvelopedRecordSetReader( dlsn.getLogSegmentSequenceNo(), dlsn.getEntryId(), record.getTransactionId(), dlsn.getSlotId(), startPosition, startSequenceId, new ByteArrayInputStream(data)); }
private static void transform(final AsyncLogWriter writer, LogRecordWithDLSN record, Transformer<byte[], byte[]> replicationTransformer, final CountDownLatch keepAliveLatch) throws Exception { DLSN srcDLSN = record.getDlsn(); byte[] payload = record.getPayload(); byte[] transformedPayload = replicationTransformer.transform(payload); TransformedRecord transformedRecord = new TransformedRecord(ByteBuffer.wrap(transformedPayload)); transformedRecord.setSrcDlsn(srcDLSN.serializeBytes()); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); transformedRecord.write(protocolFactory.getProtocol(new TIOStreamTransport(baos))); byte[] data = baos.toByteArray(); writer.write(new LogRecord(record.getSequenceId(), data)) .addEventListener(new FutureEventListener<DLSN>() { @Override public void onFailure(Throwable cause) { System.err.println("Encountered error on writing records to stream " + writer.getStreamName()); cause.printStackTrace(System.err); keepAliveLatch.countDown(); } @Override public void onSuccess(DLSN dlsn) { System.out.println("Write transformed record " + dlsn); } }); }