@Override public void save(Store store) throws IOException { for (Index index : store.getIndexes()) { IndexEntry entry = cacheDao.createIndex(con, cacheEntry, index.getId(), index.getCrc(), index.getRevision()); for (Archive archive : index.getArchives()) { ArchiveEntry archiveEntry = cacheDao.findArchive(con, entry, archive.getArchiveId(), archive.getNameHash(), archive.getCrc(), archive.getRevision()); if (archiveEntry == null) { byte[] hash = archive.getHash(); archiveEntry = cacheDao.createArchive(con, entry, archive.getArchiveId(), archive.getNameHash(), archive.getCrc(), archive.getRevision(), hash); for (FileData file : archive.getFileData()) { cacheDao.associateFileToArchive(con, archiveEntry, file.getId(), file.getNameHash()); } } cacheDao.associateArchiveToIndex(con, archiveEntry, entry); } } }
archive.setFileData(fileData.toArray(new FileData[0])); fileData = null; archive.setNameHash(Integer.parseInt(value)); continue; case "revision": archive.setRevision(Integer.parseInt(value)); continue; case "crc": archive.setCrc(Integer.parseInt(value)); continue; case "hash": archive.setHash(Base64.getDecoder().decode(value)); continue; case "compression": archive.setCompression(Integer.parseInt(value)); continue; case "contents": data.put((long) idx.getId() << 32 | archive.getArchiveId(), Base64.getDecoder().decode(value)); continue; archive.setFileData(fileData.toArray(new FileData[0])); fileData = null;
public byte[] decompress(byte[] data) throws IOException { return decompress(data, null); }
@Override public void saveArchive(Archive archive, byte[] bytes) throws IOException { data.put((long) archive.getIndex().getId() << 32 | archive.getArchiveId(), bytes); } }
logger.warn("crc mismatch for archive {}/{}", index.getId(), this.getArchiveId()); throw new IOException("CRC mismatch for " + index.getId() + "/" + this.getArchiveId()); if (container.revision != -1 && this.getRevision() != container.revision) index.getId(), this.getArchiveId(), this.getRevision(), container.revision); this.setRevision(container.revision); setCompression(container.compression); return decompressedData;
if (existing != null && existing.getRevision() == ad.getRevision() && existing.getCrc() == ad.getCrc() && existing.getNameHash() == ad.getNameHash()) ad.getId(), indexData.getArchives().length, index.getId()); else if (ad.getRevision() < existing.getRevision()) existing.getRevision(), ad.getRevision()); "revision: ours: {} theirs: {}, crc: ours: {} theirs {}, name: ours {} theirs {}", ad.getId(), indexData.getArchives().length, index.getId(), existing.getRevision(), ad.getRevision(), existing.getCrc(), ad.getCrc(), existing.getNameHash(), ad.getNameHash()); archive.setRevision(ad.getRevision()); archive.setCrc(ad.getCrc()); archive.setNameHash(ad.getNameHash()); archive.setFileData(ad.getFiles()); if (hash != archive.getCrc()) archive.getIndex().getId(), archive.getArchiveId(), hash, archive.getCrc()); throw new RuntimeException("crc mismatch");
archive.setFileData(fileData); fileData[0] = new FileData(); archive2.setFileData(fileData2); fileData2[0] = new FileData(); Container container = new Container(archive.getCompression(), -1); container.compress(data, null); byte[] compressedData = container.data; storage.saveArchive(archive, compressedData); container = new Container(archive.getCompression(), 42); container.compress(data, null); compressedData = container.data; archive2.setRevision(42); storage.saveArchive(archive2, compressedData); byte[] data = archive2_1.decompress(comprsesedData); assertArrayEquals("test".getBytes(), data); assertEquals(archive.getCrc(), archive2_1.getCrc()); assertEquals(archive.getRevision(), archive2_1.getRevision()); data = archive2_2.decompress(comprsesedData); assertArrayEquals("test".getBytes(), data); assertEquals(archive2.getCrc(), archive2_2.getCrc()); assertEquals(archive2.getRevision(), archive2_2.getRevision());
private void loadIndex(Index index) throws IOException { logger.trace("Loading index {}", index.getId()); byte[] indexData = readIndex(index.getId()); Container res = Container.decompress(indexData, null); byte[] data = res.data; IndexData id = new IndexData(); id.load(data); index.setProtocol(id.getProtocol()); index.setRevision(id.getRevision()); index.setNamed(id.isNamed()); for (ArchiveData ad : id.getArchives()) { Archive archive = index.addArchive(ad.getId()); archive.setNameHash(ad.getNameHash()); archive.setCrc(ad.getCrc()); archive.setRevision(ad.getRevision()); archive.setFileData(ad.getFiles()); assert ad.getFiles().length > 0; } index.setCrc(res.crc); index.setCompression(res.compression); assert res.revision == -1; }
archive.setNameHash(random.nextInt(Integer.MAX_VALUE)); archive.setFileData(new FileData[NUMBER_OF_FILES]); archive2.setFileData(new FileData[NUMBER_OF_FILES]); archive3.setFileData(new FileData[NUMBER_OF_FILES]); FileData[] fileData = archive.getFileData(); FileData file = fileData[i] = new FileData(); file.setNameHash(random.nextInt(Integer.MAX_VALUE)); FileData[] fileData = archive2.getFileData(); FileData file = fileData[i] = new FileData(); file.setNameHash(random.nextInt(Integer.MAX_VALUE)); FileData[] fileData = archive3.getFileData(); FileData file = fileData[i] = new FileData(); file.setNameHash(random.nextInt(Integer.MAX_VALUE));
private void dumpTrackArchive(File dumpDir, Storage storage, Archive archive) throws IOException { byte[] contents = archive.decompress(storage.loadArchive(archive)); if (contents == null) { return; } TrackLoader loader = new TrackLoader(); TrackDefinition def = loader.load(contents); String name; if (archive.getNameHash() != 0) { name = djb2.getName(archive.getNameHash()); if (name == null) { name = "name-" + archive.getNameHash(); } } else { name = "archive-" + archive.getArchiveId(); } File dest = new File(dumpDir, name + ".midi"); assert !dest.exists(); Files.write(def.midi, dest); }
@Override public void load(Store store) throws IOException { List<IndexEntry> indexes = cacheDao.findIndexesForCache(con, cacheEntry); for (IndexEntry indexEntry : indexes) { Index index = store.addIndex(indexEntry.getIndexId()); index.setCrc(indexEntry.getCrc()); index.setRevision(indexEntry.getRevision()); try (ResultSetIterable<ArchiveEntry> archives = cacheDao.findArchivesForIndex(con, indexEntry)) { for (ArchiveEntry archiveEntry : archives) { if (index.getArchive(archiveEntry.getArchiveId()) != null) { throw new IOException("Duplicate archive " + archiveEntry + " on " + indexEntry); } Archive archive = index.addArchive(archiveEntry.getArchiveId()); archive.setNameHash(archiveEntry.getNameHash()); archive.setCrc(archiveEntry.getCrc()); archive.setRevision(archiveEntry.getRevision()); archive.setHash(archiveEntry.getHash()); // File data is not necessary for cache updating } } } }
public void load() throws IOException { Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.SPRITES); for (Archive a : index.getArchives()) { byte[] contents = a.decompress(storage.loadArchive(a)); SpriteLoader loader = new SpriteLoader(); SpriteDefinition[] defs = loader.load(a.getArchiveId(), contents); for (SpriteDefinition sprite : defs) { sprites.put(sprite.getId(), sprite); } } }
public void load() throws IOException { InterfaceLoader loader = new InterfaceLoader(); Storage storage = store.getStorage(); Index index = store.getIndex(IndexType.INTERFACES); int max = index.getArchives().stream().mapToInt(a -> a.getArchiveId()).max().getAsInt(); interfaces = new InterfaceDefinition[max + 1][]; for (Archive archive : index.getArchives()) { int archiveId = archive.getArchiveId(); byte[] archiveData = storage.loadArchive(archive); ArchiveFiles files = archive.getFiles(archiveData); InterfaceDefinition[] ifaces = interfaces[archiveId]; if (ifaces == null) { ifaces = interfaces[archiveId] = new InterfaceDefinition[archive.getFileData().length]; } for (FSFile file : files.getFiles()) { int fileId = file.getFileId(); int widgetId = (archiveId << 16) + fileId; InterfaceDefinition iface = loader.load(widgetId, file.getContents()); ifaces[fileId] = iface; } } }
@Override public void saveArchive(Archive a, byte[] archiveData) throws IOException { Index index = a.getIndex(); IndexFile indexFile = getIndex(index.getId()); assert indexFile.getIndexFileId() == index.getId(); DataFileWriteResult res = data.write(index.getId(), a.getArchiveId(), archiveData); indexFile.write(new IndexEntry(indexFile, a.getArchiveId(), res.sector, res.compressedLength)); byte compression = archiveData[0]; int compressedSize = Ints.fromBytes(archiveData[1], archiveData[2], archiveData[3], archiveData[4]); // don't crc the appended revision, if it is there int length = 1 // compression type + 4 // compressed size + compressedSize + (compression != CompressionType.NONE ? 4 : 0); Crc32 crc = new Crc32(); crc.update(archiveData, 0, length); a.setCrc(crc.getHash()); logger.trace("Saved archive {}/{} at sector {}, compressed length {}", index.getId(), a.getArchiveId(), res.sector, res.compressedLength); } }
ArchiveFiles archiveFiles = archive.getFiles(archiveData); for (FSFile archiveFile : archiveFiles.getFiles()) byte[] framemapContents = framemapArchive.decompress(archiveData); FramemapDefinition framemap = fmloader.load(framemapArchive.getArchiveId(), framemapContents); Files.write(gson.toJson(frames), new File(outDir, archive.getArchiveId() + ".json"), Charset.defaultCharset()); ++count;
@Test public void testOneFile() throws IOException { try (Store store = new Store(folder.getRoot())) { Index index = store.addIndex(0); Archive archive = index.addArchive(0); archive.setFileData(new FileData[1]); FileData fileData = archive.getFileData()[0] = new FileData(); fileData.setId(42); fileData.setNameHash(7); store.save(); try (Store store2 = new Store(folder.getRoot())) { store2.load(); Assert.assertEquals(store, store2); } } }
public Archive getArchive(int id) { for (Archive a : archives) { if (a.getArchiveId() == id) { return a; } } return null; }
public Archive findArchiveByName(String name) { int hash = Djb2.hash(name); for (Archive a : archives) { if (a.getNameHash() == hash) { return a; } } return null; }
archive.setFileData(fileData); fileData[0] = new FileData(); archive2.setFileData(fileData2); fileData2[0] = new FileData(); Container container = new Container(archive.getCompression(), -1); container.compress(data, null); byte[] compressedData = container.data; storage.saveArchive(archive, compressedData); container = new Container(archive.getCompression(), 42); container.compress(data, null); compressedData = container.data; archive2.setRevision(42); storage.saveArchive(archive2, compressedData); byte[] data = archive2_1.decompress(comprsesedData); assertArrayEquals("test".getBytes(), data); assertEquals(archive.getCrc(), archive2_1.getCrc()); assertEquals(archive.getRevision(), archive2_1.getRevision()); data = archive2_2.decompress(comprsesedData); assertArrayEquals("test".getBytes(), data); assertEquals(archive2.getCrc(), archive2_2.getCrc()); assertEquals(archive2.getRevision(), archive2_2.getRevision());
@Test public void testManyFiles() throws IOException { Random random = new Random(42L); File root = folder.newFolder(); try (Store store = new Store(root)) { Index index = store.addIndex(0); Archive archive = index.addArchive(0); archive.setNameHash(random.nextInt()); archive.setFileData(new FileData[NUMBER_OF_FILES]); for (int i = 0; i < NUMBER_OF_FILES; ++i) { FileData[] fileData = archive.getFileData(); FileData file = fileData[i] = new FileData(); file.setId(i); file.setNameHash(random.nextInt()); } store.save(); try (Store store2 = new Store(root)) { store2.load(); Assert.assertEquals(store, store2); } } }