private boolean validateSnapshot(final StateSnapshotMetadata metadata) { final boolean wasFound = logStreamReader.seek(metadata.getLastWrittenEventPosition()); boolean isValid = false; if (wasFound && logStreamReader.hasNext()) { final LoggedEvent event = logStreamReader.next(); isValid = event.getRaftTerm() == metadata.getLastWrittenEventTerm(); } return isValid; }
@Override public void onRecovered() { long lowestPosition = -1; for (final ExporterContainer container : containers) { container.exporter.open(container); container.position = state.getPosition(container.getId()); if (lowestPosition == -1 || lowestPosition > container.position) { lowestPosition = container.position; } } // in case the lowest known position is not found, start from the // beginning again if (lowestPosition <= 0 || !logStreamReader.seek(lowestPosition)) { logStreamReader.seekToFirstEvent(); } else { if (logStreamReader.hasNext()) { logStreamReader.seek(lowestPosition + 1); } } }
@Override public void close() { reader.close(); } }
public static void printRecords(final LogStream logStream) { final StringBuilder sb = new StringBuilder(); sb.append("Records on partition "); sb.append(logStream.getPartitionId()); sb.append(":\n"); try (final LogStreamReader streamReader = new BufferedLogStreamReader(logStream)) { streamReader.seekToFirstEvent(); while (streamReader.hasNext()) { final LoggedEvent event = streamReader.next(); writeRecord(event, sb); } } LOGGER.info(sb.toString()); }
private void reprocessNextEvent() { try { if (logStreamReader.hasNext()) { currentEvent = logStreamReader.next(); if (currentEvent.getPosition() > lastSourceEventPosition) { throw new IllegalStateException( String.format( ERROR_MESSAGE_REPROCESSING_NO_SOURCE_EVENT, getName(), lastSourceEventPosition)); } reprocessEvent(currentEvent); } else { throw new IllegalStateException( String.format( ERROR_MESSAGE_REPROCESSING_NO_SOURCE_EVENT, getName(), lastSourceEventPosition)); } } catch (final RuntimeException e) { onFailure(); throw e; } }
@Override @SuppressWarnings({"unchecked"}) public <T extends UnpackedObject> TypedRecord<T> readValue(long position, Class<T> eventClass) { final boolean success = reader.seek(position); if (!success) { throw new RuntimeException("Could not find an event at position " + position); } final LoggedEvent rawEvent = reader.next(); metadata.reset(); rawEvent.readMetadata(metadata); final UnpackedObject value = eventCache.get(eventClass); value.reset(); rawEvent.readValue(value); event.wrap(rawEvent, metadata, value); return event; }
public Stream<LoggedEvent> events(final String logName) { final LogStream logStream = managedLogs.get(logName); final LogStreamReader reader = new BufferedLogStreamReader(logStream); closeables.manage(reader); reader.seekToFirstEvent(); final Iterable<LoggedEvent> iterable = () -> reader; return StreamSupport.stream(iterable.spliterator(), false); }
@Override protected void onActorStarting() { final LogStream logStream = streamProcessorContext.getLogStream(); final MetricsManager metricsManager = actorScheduler.getMetricsManager(); final String partitionId = String.valueOf(logStream.getPartitionId()); final String processorName = getName(); metrics = new StreamProcessorMetrics(metricsManager, processorName, partitionId); logStreamReader.wrap(logStream); logStreamWriter.wrap(logStream); try { snapshotPosition = recoverFromSnapshot(logStream.getCommitPosition(), logStream.getTerm()); lastSourceEventPosition = seekFromSnapshotPositionToLastSourceEvent(); final ZeebeDb zeebeDb = snapshotController.openDb(); streamProcessor = streamProcessorFactory.createProcessor(zeebeDb); streamProcessor.onOpen(streamProcessorContext); } catch (final Exception e) { onFailure(); LangUtil.rethrowUnchecked(e); } }
public static void printRecords(final LogStream logStream) { final StringBuilder sb = new StringBuilder(); sb.append("Records on partition "); sb.append(logStream.getPartitionId()); sb.append(":\n"); try (final LogStreamReader streamReader = new BufferedLogStreamReader(logStream)) { streamReader.seekToFirstEvent(); while (streamReader.hasNext()) { final LoggedEvent event = streamReader.next(); writeRecord(event, sb); } } LOGGER.info(sb.toString()); }
private void reprocessNextEvent() { try { if (logStreamReader.hasNext()) { currentEvent = logStreamReader.next(); if (currentEvent.getPosition() > lastSourceEventPosition) { throw new IllegalStateException( String.format( ERROR_MESSAGE_REPROCESSING_NO_SOURCE_EVENT, getName(), lastSourceEventPosition)); } reprocessEvent(currentEvent); } else { throw new IllegalStateException( String.format( ERROR_MESSAGE_REPROCESSING_NO_SOURCE_EVENT, getName(), lastSourceEventPosition)); } } catch (final RuntimeException e) { onFailure(); throw e; } }
@Override @SuppressWarnings({"unchecked"}) public <T extends UnpackedObject> TypedRecord<T> readValue(long position, Class<T> eventClass) { final boolean success = reader.seek(position); if (!success) { throw new RuntimeException("Could not find an event at position " + position); } final LoggedEvent rawEvent = reader.next(); metadata.reset(); rawEvent.readMetadata(metadata); final UnpackedObject value = eventCache.get(eventClass); value.reset(); rawEvent.readValue(value); event.wrap(rawEvent, metadata, value); return event; }
public Stream<LoggedEvent> events(final String logName) { final LogStream logStream = managedLogs.get(logName); final LogStreamReader reader = new BufferedLogStreamReader(logStream); closeables.manage(reader); reader.seekToFirstEvent(); final Iterable<LoggedEvent> iterable = () -> reader; return StreamSupport.stream(iterable.spliterator(), false); }
@Override protected void onActorStarting() { final LogStream logStream = streamProcessorContext.getLogStream(); final MetricsManager metricsManager = actorScheduler.getMetricsManager(); final String partitionId = String.valueOf(logStream.getPartitionId()); final String processorName = getName(); metrics = new StreamProcessorMetrics(metricsManager, processorName, partitionId); logStreamReader.wrap(logStream); logStreamWriter.wrap(logStream); try { snapshotPosition = recoverFromSnapshot(logStream.getCommitPosition(), logStream.getTerm()); lastSourceEventPosition = seekFromSnapshotPositionToLastSourceEvent(); final ZeebeDb zeebeDb = snapshotController.openDb(); streamProcessor = streamProcessorFactory.createProcessor(zeebeDb); streamProcessor.onOpen(streamProcessorContext); } catch (final Exception e) { onFailure(); LangUtil.rethrowUnchecked(e); } }
private boolean validateSnapshot(final StateSnapshotMetadata metadata) { final boolean wasFound = logStreamReader.seek(metadata.getLastWrittenEventPosition()); boolean isValid = false; if (wasFound && logStreamReader.hasNext()) { final LoggedEvent event = logStreamReader.next(); isValid = event.getRaftTerm() == metadata.getLastWrittenEventTerm(); } return isValid; }
@Override public void onRecovered() { long lowestPosition = -1; for (final ExporterContainer container : containers) { container.exporter.open(container); container.position = state.getPosition(container.getId()); if (lowestPosition == -1 || lowestPosition > container.position) { lowestPosition = container.position; } } // in case the lowest known position is not found, start from the // beginning again if (lowestPosition <= 0 || !logStreamReader.seek(lowestPosition)) { logStreamReader.seekToFirstEvent(); } else { if (logStreamReader.hasNext()) { logStreamReader.seek(lowestPosition + 1); } } }
private void readNextEvent() { if (isOpened() && !isSuspended() && logStreamReader.hasNext() && eventProcessor == null) { currentEvent = logStreamReader.next(); if (eventFilter == null || eventFilter.applies(currentEvent)) { processEvent(currentEvent); } else { // continue with the next event actor.submit(readNextEvent); metrics.incrementEventsSkippedCount(); } } }
@Override public void close() { reader.close(); } }
private long seekFromSnapshotPositionToLastSourceEvent() { long lastSourceEventPosition = -1L; if (!isReadOnlyProcessor && logStreamReader.hasNext()) { lastSourceEventPosition = snapshotPosition; while (logStreamReader.hasNext()) { final LoggedEvent newEvent = logStreamReader.next(); // ignore events from other producers if (newEvent.getProducerId() == streamProcessorContext.getId()) { final long sourceEventPosition = newEvent.getSourceEventPosition(); if (sourceEventPosition > 0 && sourceEventPosition > lastSourceEventPosition) { lastSourceEventPosition = sourceEventPosition; } } } // reset position logStreamReader.seek(snapshotPosition + 1); } return lastSourceEventPosition; }
private long recoverFromSnapshot(final long commitPosition, final int term) throws Exception { final StateSnapshotMetadata recovered = snapshotController.recover(commitPosition, term, this::validateSnapshot); final long snapshotPosition = recovered.getLastSuccessfulProcessedEventPosition(); logStreamReader.seekToFirstEvent(); // reset seek position if (!recovered.isInitial()) { final boolean found = logStreamReader.seek(snapshotPosition); if (found && logStreamReader.hasNext()) { logStreamReader.seek(snapshotPosition + 1); } else { throw new IllegalStateException( String.format(ERROR_MESSAGE_RECOVER_FROM_SNAPSHOT_FAILED, getName())); } snapshotController.purgeAllExcept(recovered); } return snapshotPosition; }
private void readNextEvent() { if (isOpened() && !isSuspended() && logStreamReader.hasNext() && eventProcessor == null) { currentEvent = logStreamReader.next(); if (eventFilter == null || eventFilter.applies(currentEvent)) { processEvent(currentEvent); } else { // continue with the next event actor.submit(readNextEvent); metrics.incrementEventsSkippedCount(); } } }