public static CommitLogDescriptor fromFileName(String name) { Matcher matcher; if (!(matcher = COMMIT_LOG_FILE_PATTERN.matcher(name)).matches()) throw new RuntimeException("Cannot parse the version of the file: " + name); if (matcher.group(3) == null) throw new UnsupportedOperationException("Commitlog segment is too old to open; upgrade to 1.2.5+ first"); long id = Long.parseLong(matcher.group(3).split(SEPARATOR)[1]); return new CommitLogDescriptor(Integer.parseInt(matcher.group(2)), id); }
public boolean equals(Object that) { return that instanceof CommitLogDescriptor && equals((CommitLogDescriptor) that); }
static boolean shouldReplay(String name) { return CommitLogDescriptor.fromFileName(name).id < replayLimitId; }
CommitLogDescriptor fromHeader = CommitLogDescriptor.fromHeader(fromFile, DatabaseDescriptor.getEncryptionContext()); CommitLogDescriptor fromName = CommitLogDescriptor.isValid(fromFile.getName()) ? CommitLogDescriptor.fromFileName(fromFile.getName()) : null; CommitLogDescriptor descriptor; if (fromHeader == null && fromName == null) throw new IllegalStateException("Cannot safely construct descriptor for segment, either from its name or its header: " + fromFile.getPath()); else if (fromHeader != null && fromName != null && !fromHeader.equalsIgnoringCompression(fromName)) throw new IllegalStateException(String.format("Cannot safely construct descriptor for segment, as name and header descriptors do not match (%s vs %s): %s", fromHeader, fromName, fromFile.getPath())); else if (fromName != null && fromHeader == null && fromName.version >= CommitLogDescriptor.VERSION_21) File toFile = new File(DatabaseDescriptor.getCommitLogLocation(), descriptor.fileName()); if (toFile.exists())
CommitLogDescriptor fromHeader = CommitLogDescriptor.fromHeader(fromFile); CommitLogDescriptor fromName = CommitLogDescriptor.isValid(fromFile.getName()) ? CommitLogDescriptor.fromFileName(fromFile.getName()) : null; CommitLogDescriptor descriptor; if (fromHeader == null && fromName == null) throw new IllegalStateException("Cannot safely construct descriptor for segment, either from its name or its header: " + fromFile.getPath()); else if (fromHeader != null && fromName != null && !fromHeader.equals(fromName)) throw new IllegalStateException(String.format("Cannot safely construct descriptor for segment, as name and header descriptors do not match (%s vs %s): %s", fromHeader, fromName, fromFile.getPath())); else if (fromName != null && fromHeader == null && fromName.version >= CommitLogDescriptor.VERSION_21) throw new IllegalStateException("Unsupported commit log version: " + descriptor.version); File toFile = new File(DatabaseDescriptor.getCommitLogLocation(), descriptor.fileName()); if (toFile.exists())
CommitLogDescriptor desc = CommitLogDescriptor.fromFileName(file.getName()); reader.seek(minPosition.position); ReadStatusTracker statusTracker = new ReadStatusTracker(mutationLimit, tolerateTruncation); statusTracker.errorContext = desc.fileName(); readSection(handler, reader, minPosition, (int) reader.length(), statusTracker, desc); desc = CommitLogDescriptor.readHeader(reader, DatabaseDescriptor.getEncryptionContext()); continue; statusTracker.errorContext = String.format("Next section at %d in %s", syncSegment.fileStartPosition, desc.fileName());
descriptor = new CommitLogDescriptor(id); logFile = new File(DatabaseDescriptor.getCommitLogLocation(), descriptor.fileName()); boolean isCreating = true; CommitLogDescriptor.writeHeader(buffer, descriptor);
/** * Constructs a new segment file. */ CommitLogSegment(CommitLog commitLog, AbstractCommitLogSegmentManager manager) { this.manager = manager; id = getNextId(); descriptor = new CommitLogDescriptor(id, commitLog.configuration.getCompressorClass(), commitLog.configuration.getEncryptionContext()); logFile = new File(manager.storageDirectory, descriptor.fileName()); try { channel = FileChannel.open(logFile.toPath(), StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE); fd = NativeLibrary.getfd(channel); } catch (IOException e) { throw new FSWriteError(e, logFile); } buffer = createBuffer(commitLog); }
private static boolean shouldSkip(File file) throws IOException, ConfigurationException { CommitLogDescriptor desc = CommitLogDescriptor.fromFileName(file.getName()); if (desc.version < CommitLogDescriptor.VERSION_21) { return false; } try(RandomAccessReader reader = RandomAccessReader.open(file)) { CommitLogDescriptor.readHeader(reader, DatabaseDescriptor.getEncryptionContext()); int end = reader.readInt(); long filecrc = reader.readInt() & 0xffffffffL; return end == 0 && filecrc == 0; } }
public static CommitLogDescriptor readHeader(DataInput input, EncryptionContext encryptionContext) throws IOException { CRC32 checkcrc = new CRC32(); int version = input.readInt(); updateChecksumInt(checkcrc, version); long id = input.readLong(); updateChecksumInt(checkcrc, (int) (id & 0xFFFFFFFFL)); updateChecksumInt(checkcrc, (int) (id >>> 32)); int parametersLength = 0; if (version >= VERSION_22) { parametersLength = input.readShort() & 0xFFFF; updateChecksumInt(checkcrc, parametersLength); } // This should always succeed as parametersLength cannot be too long even for a // corrupt segment file. byte[] parametersBytes = new byte[parametersLength]; input.readFully(parametersBytes); checkcrc.update(parametersBytes, 0, parametersBytes.length); int crc = input.readInt(); if (crc == (int) checkcrc.getValue()) { Map<?, ?> map = (Map<?, ?>) JSONValue.parse(new String(parametersBytes, StandardCharsets.UTF_8)); return new CommitLogDescriptor(version, id, parseCompression(map), EncryptionContext.createFromMap(map, encryptionContext)); } return null; }
/** * Differs from the above because it can work on any file instead of just existing * commit log segments managed by this manager. * * @param file segment file that is no longer in use. */ void recycleSegment(final File file) { if (isCapExceeded() || CommitLogDescriptor.fromFileName(file.getName()).getMessagingVersion() != MessagingService.current_version || !DatabaseDescriptor.getCommitLogSegmentRecyclingEnabled()) { // (don't decrease managed size, since this was never a "live" segment) logger.debug("(Unopened) segment {} is no longer needed and will be deleted now", file); FileUtils.deleteWithConfirm(file); return; } logger.debug("Recycling {}", file); // this wasn't previously a live segment, so add it to the managed size when we make it live size.addAndGet(DatabaseDescriptor.getCommitLogSegmentSize()); segmentManagementTasks.add(new Callable<CommitLogSegment>() { public CommitLogSegment call() { return new CommitLogSegment(file.getPath()); } }); }
public static void writeHeader(ByteBuffer out, CommitLogDescriptor descriptor) { writeHeader(out, descriptor, Collections.<String, String>emptyMap()); }
/** * Any segment with id >= minPosition.segmentId is a candidate for read. */ private boolean shouldSkipSegmentId(File file, CommitLogDescriptor desc, CommitLogPosition minPosition) { logger.debug("Reading {} (CL version {}, messaging version {}, compression {})", file.getPath(), desc.version, desc.getMessagingVersion(), desc.compression); if (minPosition.segmentId > desc.id) { logger.trace("Skipping read of fully-flushed {}", file); return true; } return false; }
public boolean accept(File dir, String name) { // we used to try to avoid instantiating commitlog (thus creating an empty segment ready for writes) // until after recover was finished. this turns out to be fragile; it is less error-prone to go // ahead and allow writes before recover(), and just skip active segments when we do. return CommitLogDescriptor.isValid(name) && !instance.allocator.manages(name); } };
public boolean equals(CommitLogDescriptor that) { return equalsIgnoringCompression(that) && Objects.equal(this.compression, that.compression) && Objects.equal(encryptionContext, that.encryptionContext); } }
/** * @param additionalHeaders Allow segments to pass custom header data */ public static void writeHeader(ByteBuffer out, CommitLogDescriptor descriptor, Map<String, String> additionalHeaders) { CRC32 crc = new CRC32(); out.putInt(descriptor.version); updateChecksumInt(crc, descriptor.version); out.putLong(descriptor.id); updateChecksumInt(crc, (int) (descriptor.id & 0xFFFFFFFFL)); updateChecksumInt(crc, (int) (descriptor.id >>> 32)); if (descriptor.version >= VERSION_22) { String parametersString = constructParametersString(descriptor.compression, descriptor.encryptionContext, additionalHeaders); byte[] parametersBytes = parametersString.getBytes(StandardCharsets.UTF_8); if (parametersBytes.length != (((short) parametersBytes.length) & 0xFFFF)) throw new ConfigurationException(String.format("Compression parameters too long, length %d cannot be above 65535.", parametersBytes.length)); out.putShort((short) parametersBytes.length); updateChecksumInt(crc, parametersBytes.length); out.put(parametersBytes); crc.update(parametersBytes, 0, parametersBytes.length); } else assert descriptor.compression == null; out.putInt((int) crc.getValue()); }
CommitLogDescriptor fromHeader = CommitLogDescriptor.fromHeader(fromFile, DatabaseDescriptor.getEncryptionContext()); CommitLogDescriptor fromName = CommitLogDescriptor.isValid(fromFile.getName()) ? CommitLogDescriptor.fromFileName(fromFile.getName()) : null; CommitLogDescriptor descriptor; if (fromHeader == null && fromName == null) throw new IllegalStateException("Cannot safely construct descriptor for segment, either from its name or its header: " + fromFile.getPath()); else if (fromHeader != null && fromName != null && !fromHeader.equalsIgnoringCompression(fromName)) throw new IllegalStateException(String.format("Cannot safely construct descriptor for segment, as name and header descriptors do not match (%s vs %s): %s", fromHeader, fromName, fromFile.getPath())); else if (fromName != null && fromHeader == null && fromName.version >= CommitLogDescriptor.VERSION_21) File toFile = new File(DatabaseDescriptor.getCommitLogLocation(), descriptor.fileName()); if (toFile.exists())
CommitLogDescriptor desc = CommitLogDescriptor.fromFileName(file.getName()); reader.seek(minPosition.position); ReadStatusTracker statusTracker = new ReadStatusTracker(mutationLimit, tolerateTruncation); statusTracker.errorContext = desc.fileName(); readSection(handler, reader, minPosition, (int) reader.length(), statusTracker, desc); desc = CommitLogDescriptor.readHeader(reader, DatabaseDescriptor.getEncryptionContext()); continue; statusTracker.errorContext = String.format("Next section at %d in %s", syncSegment.fileStartPosition, desc.fileName());
/** * Constructs a new segment file. */ CommitLogSegment(CommitLog commitLog, AbstractCommitLogSegmentManager manager) { this.manager = manager; id = getNextId(); descriptor = new CommitLogDescriptor(id, commitLog.configuration.getCompressorClass(), commitLog.configuration.getEncryptionContext()); logFile = new File(manager.storageDirectory, descriptor.fileName()); try { channel = FileChannel.open(logFile.toPath(), StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE); fd = NativeLibrary.getfd(channel); } catch (IOException e) { throw new FSWriteError(e, logFile); } buffer = createBuffer(commitLog); }
private static boolean shouldSkip(File file) throws IOException, ConfigurationException { CommitLogDescriptor desc = CommitLogDescriptor.fromFileName(file.getName()); if (desc.version < CommitLogDescriptor.VERSION_21) { return false; } try(RandomAccessReader reader = RandomAccessReader.open(file)) { CommitLogDescriptor.readHeader(reader, DatabaseDescriptor.getEncryptionContext()); int end = reader.readInt(); long filecrc = reader.readInt() & 0xffffffffL; return end == 0 && filecrc == 0; } }