ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() {
public void processProcedureWALFile(ProcedureWALFile log) throws IOException { log.open(); ProcedureWALHeader header = log.getHeader(); printHeader(header); FSDataInputStream stream = log.getStream(); try { boolean hasMore = true; while (hasMore) { ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); if (entry == null) { out.println("No more entry, exiting with missing EOF"); hasMore = false; break; } switch (entry.getType()) { case PROCEDURE_WAL_EOF: hasMore = false; break; default: printEntry(entry); } } } catch (IOException e) { out.println("got an exception while reading the procedure WAL " + e.getMessage()); } finally { log.close(); } }
public void open() throws IOException { if (stream == null) { stream = fs.open(logFile); } if (header == null) { header = ProcedureWALFormat.readHeader(stream); startPos = stream.getPos(); } else { stream.seek(startPos); } }
public static void writeInsert(ByteSlot slot, Procedure<?> proc) throws IOException { writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null); }
private void delete(long[] procIds) { if (LOG.isTraceEnabled()) { LOG.trace("Delete " + Arrays.toString(procIds)); } final ByteSlot slot = acquireSlot(); try { // Serialize the delete for (int i = 0; i < procIds.length; ++i) { ProcedureWALFormat.writeDelete(slot, procIds[i]); } // Push the transaction data and wait until it is persisted pushData(PushType.DELETE, slot, Procedure.NO_PROC_ID, procIds); } catch (IOException e) { // We are not able to serialize the procedure. // this is a code error, and we are not able to go on. LOG.error("Unable to serialize the procedures: " + Arrays.toString(procIds), e); throw new RuntimeException(e); } finally { releaseSlot(slot); } }
@Override public void insert(Procedure<?>[] procs) { if (LOG.isTraceEnabled()) { LOG.trace("Insert " + Arrays.toString(procs)); } ByteSlot slot = acquireSlot(); try { // Serialize the insert long[] procIds = new long[procs.length]; for (int i = 0; i < procs.length; ++i) { assert !procs[i].hasParent(); procIds[i] = procs[i].getProcId(); ProcedureWALFormat.writeInsert(slot, procs[i]); } // Push the transaction data and wait until it is persisted pushData(PushType.INSERT, slot, Procedure.NO_PROC_ID, procIds); } catch (IOException e) { // We are not able to serialize the procedure. // this is a code error, and we are not able to go on. LOG.error(HBaseMarkers.FATAL, "Unable to serialize one of the procedure: " + Arrays.toString(procs), e); throw new RuntimeException(e); } finally { releaseSlot(slot); } }
public ProcedureWALTrailer readTrailer() throws IOException { try { return ProcedureWALFormat.readTrailer(stream, startPos, logSize); } finally { stream.seek(startPos); } }
@Override public void update(Procedure<?> proc) { if (LOG.isTraceEnabled()) { LOG.trace("Update " + proc); } ByteSlot slot = acquireSlot(); try { // Serialize the update ProcedureWALFormat.writeUpdate(slot, proc); // Push the transaction data and wait until it is persisted pushData(PushType.UPDATE, slot, proc.getProcId(), null); } catch (IOException e) { // We are not able to serialize the procedure. // this is a code error, and we are not able to go on. LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e); throw new RuntimeException(e); } finally { releaseSlot(slot); } }
private void closeCurrentLogStream(boolean abort) { if (stream == null || logs.isEmpty()) { return; } try { ProcedureWALFile log = logs.getLast(); // If the loading flag is true, it usually means that we fail when loading procedures, so we // should not persist the store tracker, as its state may not be correct. if (!loading.get()) { log.setProcIds(storeTracker.getModifiedMinProcId(), storeTracker.getModifiedMaxProcId()); log.updateLocalTracker(storeTracker); if (!abort) { long trailerSize = ProcedureWALFormat.writeTrailer(stream, storeTracker); log.addToSize(trailerSize); } } } catch (IOException e) { LOG.warn("Unable to write the trailer", e); } try { stream.close(); } catch (IOException e) { LOG.error("Unable to close the stream", e); } stream = null; }
ProcedureWALFormat.writeHeader(newStream, header); startPos = newStream.getPos(); } catch (IOException ioe) {
public static void writeInsert(ByteSlot slot, Procedure<?> proc, Procedure<?>[] subprocs) throws IOException { writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs); }
@Override public void delete(long procId) { LOG.trace("Delete {}", procId); ByteSlot slot = acquireSlot(); try { // Serialize the delete ProcedureWALFormat.writeDelete(slot, procId); // Push the transaction data and wait until it is persisted pushData(PushType.DELETE, slot, procId, null); } catch (IOException e) { // We are not able to serialize the procedure. // this is a code error, and we are not able to go on. LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + procId, e); throw new RuntimeException(e); } finally { releaseSlot(slot); } }
ProcedureWALFormat.writeInsert(slot, proc, subprocs); subProcIds = new long[subprocs.length]; for (int i = 0; i < subprocs.length; ++i) { ProcedureWALFormat.writeInsert(slot, proc);
public ProcedureWALTrailer readTrailer() throws IOException { try { return ProcedureWALFormat.readTrailer(stream, startPos, logSize); } finally { stream.seek(startPos); } }
@Override public void update(final Procedure proc) { if (LOG.isTraceEnabled()) { LOG.trace("Update " + proc); } ByteSlot slot = acquireSlot(); try { // Serialize the update ProcedureWALFormat.writeUpdate(slot, proc); // Push the transaction data and wait until it is persisted pushData(PushType.UPDATE, slot, proc.getProcId(), null); } catch (IOException e) { // We are not able to serialize the procedure. // this is a code error, and we are not able to go on. LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e); throw new RuntimeException(e); } finally { releaseSlot(slot); } }
private void closeCurrentLogStream() { if (stream == null || logs.isEmpty()) { return; } try { ProcedureWALFile log = logs.getLast(); log.setProcIds(storeTracker.getUpdatedMinProcId(), storeTracker.getUpdatedMaxProcId()); log.updateLocalTracker(storeTracker); long trailerSize = ProcedureWALFormat.writeTrailer(stream, storeTracker); log.addToSize(trailerSize); } catch (IOException e) { LOG.warn("Unable to write the trailer: " + e.getMessage()); } try { stream.close(); } catch (IOException e) { LOG.error("Unable to close the stream", e); } stream = null; }
ProcedureWALFormat.writeHeader(newStream, header); startPos = newStream.getPos(); } catch (IOException ioe) {
public static void writeUpdate(ByteSlot slot, Procedure<?> proc) throws IOException { writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null); }
public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size) throws IOException { // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset long trailerPos = size - 17; if (trailerPos < startPos) { throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos); } stream.seek(trailerPos); int version = stream.read(); if (version != TRAILER_VERSION) { throw new InvalidWALDataException("Invalid Trailer version. got " + version + " expected " + TRAILER_VERSION); } long magic = StreamUtils.readLong(stream); if (magic != TRAILER_MAGIC) { throw new InvalidWALDataException("Invalid Trailer magic. got " + magic + " expected " + TRAILER_MAGIC); } long trailerOffset = StreamUtils.readLong(stream); stream.seek(trailerOffset); ProcedureWALEntry entry = readEntry(stream); if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) { throw new InvalidWALDataException("Invalid Trailer begin"); } ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder() .setVersion(version) .setTrackerPos(stream.getPos()) .build(); return trailer; }
@Override public void delete(Procedure<?> proc, long[] subProcIds) { assert proc != null : "expected a non-null procedure"; assert subProcIds != null && subProcIds.length > 0 : "expected subProcIds"; if (LOG.isTraceEnabled()) { LOG.trace("Update " + proc + " and Delete " + Arrays.toString(subProcIds)); } ByteSlot slot = acquireSlot(); try { // Serialize the delete ProcedureWALFormat.writeDelete(slot, proc, subProcIds); // Push the transaction data and wait until it is persisted pushData(PushType.DELETE, slot, proc.getProcId(), subProcIds); } catch (IOException e) { // We are not able to serialize the procedure. // this is a code error, and we are not able to go on. LOG.error(HBaseMarkers.FATAL, "Unable to serialize the procedure: " + proc, e); throw new RuntimeException(e); } finally { releaseSlot(slot); } }