@Override public void onFileAddChunk(File file, Chunk chunk) { logger.log(Level.INFO, " - Adding chunk " + StringUtil.toHex(chunk.getChecksum()) + " to inputFileToChunkIDs-map for file " + file + " ..."); List<ChunkChecksum> chunkIDsForFile = chunkIndex.inputFileToChunkIDs.get(file); if (chunkIDsForFile == null) { chunkIDsForFile = new ArrayList<ChunkChecksum>(); } chunkIDsForFile.add(new ChunkChecksum(chunk.getChecksum())); chunkIndex.inputFileToChunkIDs.put(file, chunkIDsForFile); }
if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/chunks/chunk")) { String chunkChecksumStr = attributes.getValue("checksum"); ChunkChecksum chunkChecksum = ChunkChecksum.parseChunkChecksum(chunkChecksumStr); int chunkSize = Integer.parseInt(attributes.getValue("size")); String chunkChecksumStr = attributes.getValue("ref"); fileContent.addChunk(ChunkChecksum.parseChunkChecksum(chunkChecksumStr)); String chunkChecksumStr = attributes.getValue("ref"); multiChunk.addChunk(ChunkChecksum.parseChunkChecksum(chunkChecksumStr));
int i = 0; for (ChunkChecksum checksum : chunkChecksumSet) { checksums[i] = checksum.toString(); i++; try (ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { result.put(ChunkChecksum.parseChunkChecksum(resultSet.getString("chunk_checksum")), MultiChunkId.parseMultiChunkId(resultSet.getString("multichunk_id")));
logger.log(Level.INFO, " - Adding chunk " + StringUtil.toHex(chunk.getChecksum()) + " to multichunk " + multiChunk.getId() + " ..."); chunkIndex.chunkIDToMultiChunkID.put(new ChunkChecksum(chunk.getChecksum()), multiChunk.getId()); chunkIDsForFile.add(new ChunkChecksum(chunk.getChecksum())); chunkIndex.inputFileToChunkIDs.put(file, chunkIDsForFile); if (chunkIndex.chunkIDToMultiChunkID.containsKey(new ChunkChecksum(chunk.getChecksum()))) { logger.log(Level.INFO, " + Known chunk " + StringUtil.toHex(chunk.getChecksum())); return false;
InputStream chunkInputStream = multiChunk.getChunkInputStream(chunkChecksum.getBytes());
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (!(obj instanceof ChunkEntry)) { return false; } ChunkEntry other = (ChunkEntry) obj; if (checksum == null) { if (other.checksum != null) { return false; } } else if (!checksum.equals(other.checksum)) { return false; } if (size != other.size) { return false; } return true; }
private FileContent getFileContentWithChunkChecksums(FileChecksum fileChecksum) { try (PreparedStatement preparedStatement = getStatement("filecontent.select.all.getFileContentByChecksumWithChunkChecksums.sql")) { preparedStatement.setString(1, fileChecksum.toString()); try (ResultSet resultSet = preparedStatement.executeQuery()) { FileContent fileContent = null; while (resultSet.next()) { if (fileContent == null) { fileContent = new FileContent(); fileContent.setChecksum(FileChecksum.parseFileChecksum(resultSet.getString("checksum"))); fileContent.setSize(resultSet.getLong("size")); } // Add chunk references ChunkChecksum chunkChecksum = ChunkChecksum.parseChunkChecksum(resultSet.getString("chunk_checksum")); fileContent.addChunk(chunkChecksum); } return fileContent; } } catch (SQLException e) { throw new RuntimeException(e); } }
private Map<FileChecksum, FileContent> createFileContents(ResultSet resultSet) throws SQLException { Map<FileChecksum, FileContent> fileContents = new HashMap<FileChecksum, FileContent>(); FileChecksum currentFileChecksum = null; while (resultSet.next()) { FileChecksum fileChecksum = FileChecksum.parseFileChecksum(resultSet.getString("checksum")); FileContent fileContent = null; if (currentFileChecksum != null && currentFileChecksum.equals(fileChecksum)) { fileContent = fileContents.get(fileChecksum); } else { fileContent = new FileContent(); fileContent.setChecksum(fileChecksum); fileContent.setSize(resultSet.getLong("size")); } ChunkChecksum chunkChecksum = ChunkChecksum.parseChunkChecksum(resultSet.getString("chunk_checksum")); fileContent.addChunk(chunkChecksum); fileContents.put(fileChecksum, fileContent); currentFileChecksum = fileChecksum; } return fileContents; }
/** * Writes a list of {@link ChunkEntry}s to the database using <tt>INSERT</tt>s and the given connection. * * <p><b>Note:</b> This method executes, but <b>does not commit</b> the query. * * @param connection The connection used to execute the statements * @param databaseVersionId * @param chunks List of {@link ChunkEntry}s to be inserted in the database * @throws SQLException If the SQL statement fails */ public void writeChunks(Connection connection, long databaseVersionId, Collection<ChunkEntry> chunks) throws SQLException { if (chunks.size() > 0) { PreparedStatement preparedStatement = getStatement(connection, "chunk.insert.all.writeChunks.sql"); for (ChunkEntry chunk : chunks) { preparedStatement.setString(1, chunk.getChecksum().toString()); preparedStatement.setLong(2, databaseVersionId); preparedStatement.setInt(3, chunk.getSize()); preparedStatement.addBatch(); } preparedStatement.executeBatch(); preparedStatement.close(); } }
/** * Checks if chunk already exists in all database versions * Afterwards checks if chunk exists in new introduced database version. */ @Override public boolean onChunk(Chunk chunk) { ChunkChecksum chunkChecksum = new ChunkChecksum(chunk.getChecksum()); chunkEntry = localDatabase.getChunk(chunkChecksum); if (chunkEntry == null) { chunkEntry = newDatabaseVersion.getChunk(chunkChecksum); if (chunkEntry == null) { logger.log(Level.FINER, "- Chunk new: {0}", chunkChecksum.toString()); chunkEntry = new ChunkEntry(chunkChecksum, chunk.getSize()); newDatabaseVersion.addChunk(chunkEntry); return true; } } logger.log(Level.FINER, "- Chunk exists: {0}", StringUtil.toHex(chunk.getChecksum())); return false; } }
private void writeMultiChunks(IndentXmlStreamWriter xmlOut, Collection<MultiChunkEntry> multiChunks) throws XMLStreamException { if (multiChunks.size() > 0) { xmlOut.writeStartElement("multiChunks"); for (MultiChunkEntry multiChunk : multiChunks) { xmlOut.writeStartElement("multiChunk"); xmlOut.writeAttribute("id", multiChunk.getId().toString()); xmlOut.writeAttribute("size", multiChunk.getSize()); xmlOut.writeStartElement("chunkRefs"); Collection<ChunkChecksum> multiChunkChunks = multiChunk.getChunks(); for (ChunkChecksum chunkChecksum : multiChunkChunks) { xmlOut.writeEmptyElement("chunkRef"); xmlOut.writeAttribute("ref", chunkChecksum.toString()); } xmlOut.writeEndElement(); // </chunkRefs> xmlOut.writeEndElement(); // </multiChunk> } xmlOut.writeEndElement(); // </multiChunks> } }
private Map<MultiChunkId, MultiChunkEntry> createMultiChunkEntriesWithChunks(ResultSet resultSet) throws SQLException { Map<MultiChunkId, MultiChunkEntry> multiChunkEntries = new HashMap<MultiChunkId, MultiChunkEntry>(); MultiChunkId currentMultiChunkId = null; while (resultSet.next()) { MultiChunkId multiChunkId = MultiChunkId.parseMultiChunkId(resultSet.getString("multichunk_id")); long multiChunkSize = resultSet.getLong("size"); MultiChunkEntry multiChunkEntry = null; if (currentMultiChunkId != null && currentMultiChunkId.equals(multiChunkId)) { multiChunkEntry = multiChunkEntries.get(multiChunkId); } else { multiChunkEntry = new MultiChunkEntry(multiChunkId, multiChunkSize); } multiChunkEntry.addChunk(ChunkChecksum.parseChunkChecksum(resultSet.getString("chunk_checksum"))); multiChunkEntries.put(multiChunkId, multiChunkEntry); currentMultiChunkId = multiChunkId; } return multiChunkEntries; } }
private void writeFileContents(IndentXmlStreamWriter xmlOut, Collection<FileContent> fileContents) throws XMLStreamException { if (fileContents.size() > 0) { xmlOut.writeStartElement("fileContents"); for (FileContent fileContent : fileContents) { xmlOut.writeStartElement("fileContent"); xmlOut.writeAttribute("checksum", fileContent.getChecksum().toString()); xmlOut.writeAttribute("size", fileContent.getSize()); xmlOut.writeStartElement("chunkRefs"); Collection<ChunkChecksum> fileContentChunkChunks = fileContent.getChunks(); for (ChunkChecksum chunkChecksum : fileContentChunkChunks) { xmlOut.writeEmptyElement("chunkRef"); xmlOut.writeAttribute("ref", chunkChecksum.toString()); } xmlOut.writeEndElement(); // </chunkRefs> xmlOut.writeEndElement(); // </fileContent> } xmlOut.writeEndElement(); // </fileContents> } }
private Map<ChunkChecksum, File> extractChunksFromMultiChunks(List<File> outputMultiChunkFiles, FrameworkCombination combination) throws IOException { Map<ChunkChecksum, File> extractedChunks = new HashMap<ChunkChecksum, File>(); for (File outputMultiChunkFile : outputMultiChunkFiles) { logger.log(Level.INFO, "- Extracting multichunk " + outputMultiChunkFile + " ..."); MultiChunk outputMultiChunk = combination.multiChunker.createMultiChunk( combination.transformer.createInputStream(new FileInputStream(outputMultiChunkFile))); Chunk outputChunkInMultiChunk = null; while (null != (outputChunkInMultiChunk = outputMultiChunk.read())) { File extractedChunkFile = new File(tempDir + "/chunk-" + StringUtil.toHex((outputChunkInMultiChunk.getChecksum())) + "-from-multichunk-" + outputMultiChunk.getId()); logger.log(Level.INFO, " + Writing chunk " + StringUtil.toHex((outputChunkInMultiChunk.getChecksum())) + " to " + extractedChunkFile + " ..."); TestFileUtil.writeToFile(outputChunkInMultiChunk.getContent(), extractedChunkFile); extractedChunks.put(new ChunkChecksum(outputChunkInMultiChunk.getChecksum()), extractedChunkFile); } } return extractedChunks; }
/** * Note: This method selects also {@link DatabaseVersionStatus#DIRTY DIRTY}. */ public MultiChunkId getMultiChunkId(ChunkChecksum chunkChecksum) { try (PreparedStatement preparedStatement = getStatement("multichunk.select.all.getMultiChunkIdForChunk.sql")) { preparedStatement.setString(1, chunkChecksum.toString()); try (ResultSet resultSet = preparedStatement.executeQuery()) { if (resultSet.next()) { return MultiChunkId.parseMultiChunkId(resultSet.getString("multichunk_id")); } } return null; } catch (SQLException e) { throw new RuntimeException(e); } }
protected ChunkEntry createChunkEntryFromRow(ResultSet resultSet) throws SQLException { ChunkChecksum chunkChecksum = ChunkChecksum.parseChunkChecksum(resultSet.getString("checksum")); return new ChunkEntry(chunkChecksum, resultSet.getInt("size")); }
private void writeFileContentChunkRefs(Connection connection, FileContent fileContent) throws SQLException { PreparedStatement preparedStatement = getStatement(connection, "filecontent.insert.all.writeFileContentChunkRefs.sql"); int order = 0; for (ChunkChecksum chunkChecksum : fileContent.getChunks()) { preparedStatement.setString(1, fileContent.getChecksum().toString()); preparedStatement.setString(2, chunkChecksum.toString()); preparedStatement.setInt(3, order); preparedStatement.addBatch(); order++; } preparedStatement.executeBatch(); preparedStatement.close(); }
public static ChunkChecksum parseChunkChecksum(String s) { return new ChunkChecksum(ObjectId.parseObjectId(s)); } }
@Override public void onMultiChunkWrite(MultiChunk multiChunk, Chunk chunk) { logger.log(Level.INFO, " - Adding chunk " + StringUtil.toHex(chunk.getChecksum()) + " to multichunk " + multiChunk.getId() + " ..."); chunkIndex.chunkIDToMultiChunkID.put(new ChunkChecksum(chunk.getChecksum()), multiChunk.getId()); }
private void writeMultiChunkRefs(Connection connection, MultiChunkEntry multiChunk) throws SQLException { PreparedStatement preparedStatement = getStatement("multichunk.insert.all.writeMultiChunkRefs.sql"); for (ChunkChecksum chunkChecksum : multiChunk.getChunks()) { preparedStatement.setString(1, multiChunk.getId().toString()); preparedStatement.setString(2, chunkChecksum.toString()); preparedStatement.addBatch(); } preparedStatement.executeBatch(); preparedStatement.close(); }