public Index findIndex(int id) { for (Index i : indexes) { if (i.getId() == id) { return i; } } return null; } }
public final Index addIndex(int id) throws FileNotFoundException { for (Index i : indexes) { if (i.getId() == id) { throw new IllegalArgumentException("index " + id + " already exists"); } } Index index = new Index(id); this.indexes.add(index); return index; }
@Override public void saveArchive(Archive archive, byte[] bytes) throws IOException { data.put((long) archive.getIndex().getId() << 32 | archive.getArchiveId(), bytes); } }
@Override public byte[] loadArchive(Archive archive) throws IOException { return data.get((long) archive.getIndex().getId() << 32 | archive.getArchiveId()); }
@Override public byte[] loadArchive(Archive archive) throws IOException { Index index = archive.getIndex(); IndexFile indexFile = getIndex(index.getId()); assert indexFile.getIndexFileId() == index.getId(); IndexEntry entry = indexFile.read(archive.getArchiveId()); if (entry == null) { logger.debug("can't read archive " + archive.getArchiveId() + " from index " + index.getId()); return null; } assert entry.getId() == archive.getArchiveId(); logger.trace("Loading archive {} for index {} from sector {} length {}", archive.getArchiveId(), index.getId(), entry.getSector(), entry.getLength()); byte[] archiveData = data.read(index.getId(), entry.getId(), entry.getSector(), entry.getLength()); return archiveData; }
for (Index idx : store.getIndexes()) String file = idx.getId() + EXTENSION; try (PrintStream br = new PrintStream(openWriter(file)))
@Override public void saveArchive(Archive a, byte[] archiveData) throws IOException { Index index = a.getIndex(); IndexFile indexFile = getIndex(index.getId()); assert indexFile.getIndexFileId() == index.getId(); DataFileWriteResult res = data.write(index.getId(), a.getArchiveId(), archiveData); indexFile.write(new IndexEntry(indexFile, a.getArchiveId(), res.sector, res.compressedLength)); byte compression = archiveData[0]; int compressedSize = Ints.fromBytes(archiveData[1], archiveData[2], archiveData[3], archiveData[4]); // don't crc the appended revision, if it is there int length = 1 // compression type + 4 // compressed size + compressedSize + (compression != CompressionType.NONE ? 4 : 0); Crc32 crc = new Crc32(); crc.update(archiveData, 0, length); a.setCrc(crc.getHash()); logger.trace("Saved archive {}/{} at sector {}, compressed length {}", index.getId(), a.getArchiveId(), res.sector, res.compressedLength); } }
private void saveIndex(Index index) throws IOException { IndexData indexData = index.toIndexData(); byte[] data = indexData.writeIndexData(); Container container = new Container(index.getCompression(), -1); // index data revision is always -1 container.compress(data, null); byte[] compressedData = container.data; DataFileWriteResult res = this.data.write(index255.getIndexFileId(), index.getId(), compressedData); index255.write(new IndexEntry(index255, index.getId(), res.sector, res.compressedLength)); Crc32 crc = new Crc32(); crc.update(compressedData, 0, compressedData.length); index.setCrc(crc.getHash()); }
logger.warn("Index {} revision is going BACKWARDS! (our revision {}, their revision {})", index.getId(), index.getRevision(), revision); logger.info("Index {} has the wrong revision (our revision {}, their revision {})", index.getId(), index.getRevision(), revision); logger.info("Index {} is up to date", index.getId()); ad.getId(), indexData.getArchives().length, index.getId()); continue; ad.getId(), indexData.getArchives().length, index.getId()); ad.getId(), indexData.getArchives().length, index.getId(), existing.getRevision(), ad.getRevision()); ad.getId(), indexData.getArchives().length, index.getId(), existing.getRevision(), ad.getRevision(), existing.getCrc(), ad.getCrc(), CompletableFuture<FileResult> future = requestFile(index.getId(), ad.getId(), false); future.handle((fr, ex) -> archive.getIndex().getId(), archive.getArchiveId(), hash, archive.getCrc()); throw new RuntimeException("crc mismatch");
private void loadIndex(Index index) throws IOException { logger.trace("Loading index {}", index.getId()); byte[] indexData = readIndex(index.getId()); Container res = Container.decompress(indexData, null); byte[] data = res.data; IndexData id = new IndexData(); id.load(data); index.setProtocol(id.getProtocol()); index.setRevision(id.getRevision()); index.setNamed(id.isNamed()); for (ArchiveData ad : id.getArchives()) { Archive archive = index.addArchive(ad.getId()); archive.setNameHash(ad.getNameHash()); archive.setCrc(ad.getCrc()); archive.setRevision(ad.getRevision()); archive.setFileData(ad.getFiles()); assert ad.getFiles().length > 0; } index.setCrc(res.crc); index.setCompression(res.compression); assert res.revision == -1; }
@Override public void save(Store store) throws IOException { for (Index index : store.getIndexes()) { IndexEntry entry = cacheDao.createIndex(con, cacheEntry, index.getId(), index.getCrc(), index.getRevision()); for (Archive archive : index.getArchives()) { ArchiveEntry archiveEntry = cacheDao.findArchive(con, entry, archive.getArchiveId(), archive.getNameHash(), archive.getCrc(), archive.getRevision()); if (archiveEntry == null) { byte[] hash = archive.getHash(); archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(), archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash); for (FileData file : archive.getFileData()) { cacheDao.associateFileToArchive(con, archiveEntry, file.getId(), file.getNameHash()); } } cacheDao.associateArchiveToIndex(con, archiveEntry, entry); } } }