BlobKey remoteKey = BlobKey.readFromInputStream(is); byte[] localHash = md.digest(); if (blobType != remoteKey.getType()) { throw new IOException("Detected data corruption during transfer"); if (!Arrays.equals(localHash, remoteKey.getHash())) { throw new IOException("Detected data corruption during transfer");
/** * Returns the path for the given blob key. * * <p>The returned path can be used with the state backend for recovery purposes. * * <p>This follows the same scheme as {@link #getStorageLocation(File, BlobKey)} * and is used for HA. */ static String getRecoveryPath(String basePath, BlobKey blobKey) { // format: $base/cache/blob_$key return String.format("%s/cache/%s%s", basePath, BLOB_FILE_PREFIX, blobKey.toString()); }
private BlobKey receivePutResponseAndCompare(InputStream is, MessageDigest md) throws IOException { int response = is.read(); if (response < 0) { throw new EOFException("Premature end of response"); } else if (response == RETURN_OKAY) { if (md == null) { // not content addressable return null; } BlobKey remoteKey = BlobKey.readFromInputStream(is); BlobKey localKey = new BlobKey(md.digest()); if (!localKey.equals(remoteKey)) { throw new IOException("Detected data corruption during transfer"); } return localKey; } else if (response == RETURN_ERROR) { Throwable cause = readExceptionFromStream(is); throw new IOException("Server side error: " + cause.getMessage(), cause); } else { throw new IOException("Unrecognized response: " + response + '.'); } }
@Override public int read() throws IOException { if (this.bytesReceived == this.bytesToReceive) { return -1; } final int read = this.wrappedInputStream.read(); if (read < 0) { throwEOFException(); } ++this.bytesReceived; if (this.md != null) { this.md.update((byte) read); if (this.bytesReceived == this.bytesToReceive) { final BlobKey computedKey = new BlobKey(this.md.digest()); if (!computedKey.equals(this.blobKey)) { throw new IOException("Detected data corruption during transfer"); } } } return read; }
blobKey.writeToOutputStream(outputStream);
@Override public int read(byte[] b, int off, int len) throws IOException { final int bytesMissing = this.bytesToReceive - this.bytesReceived; if (bytesMissing == 0) { return -1; } final int maxRecv = Math.min(len, bytesMissing); final int read = this.wrappedInputStream.read(b, off, maxRecv); if (read < 0) { throwEOFException(); } this.bytesReceived += read; if (this.md != null) { this.md.update(b, off, read); if (this.bytesReceived == this.bytesToReceive) { final byte[] computedKey = this.md.digest(); if (!Arrays.equals(computedKey, this.blobKey.getHash())) { this.wrappedOutputStream.write(RETURN_ERROR); throw new IOException("Detected data corruption during transfer"); } this.wrappedOutputStream.write(RETURN_OKAY); } } return read; }
throw new IOException("Unknown type of BLOB addressing: " + mode + '.'); blobKey = BlobKey.readFromInputStream(inputStream);
return createKey(blobType, key, random);
BlobKey blobKey = new BlobKey(md.digest()); File storageFile = blobServer.getStorageLocation(blobKey); blobKey.writeToOutputStream(outputStream);
/** * Auxiliary method to read a BLOB key from an input stream. * * @param inputStream * the input stream to read the BLOB key from * @return the read BLOB key * @throws IOException * throw if an I/O error occurs while reading from the input stream */ static BlobKey readFromInputStream(InputStream inputStream) throws IOException { final byte[] key = new byte[BlobKey.SIZE]; int bytesRead = 0; while (bytesRead < BlobKey.SIZE) { final int read = inputStream.read(key, bytesRead, BlobKey.SIZE - bytesRead); if (read < 0) { throw new EOFException("Read an incomplete BLOB key"); } bytesRead += read; } return new BlobKey(key); }
/** * Constructs and writes the header data for a GET operation to the given output stream. * * @param outputStream * the output stream to write the header data to * @param jobId * ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated) * @param blobKey * blob key associated with the requested file * * @throws IOException * thrown if an I/O error occurs while writing the header data to the output stream */ private static void sendGetHeader( OutputStream outputStream, @Nullable JobID jobId, BlobKey blobKey) throws IOException { checkNotNull(blobKey); checkArgument(jobId != null || blobKey instanceof TransientBlobKey, "permanent BLOBs must be job-related"); // Signal type of operation outputStream.write(GET_OPERATION); // Send job ID and key if (jobId == null) { outputStream.write(JOB_UNRELATED_CONTENT); } else { outputStream.write(JOB_RELATED_CONTENT); outputStream.write(jobId.getBytes()); } blobKey.writeToOutputStream(outputStream); }
@Override public int read() throws IOException { if (this.bytesReceived == this.bytesToReceive) { return -1; } final int read = this.wrappedInputStream.read(); if (read < 0) { throwEOFException(); } ++this.bytesReceived; if (this.md != null) { this.md.update((byte) read); if (this.bytesReceived == this.bytesToReceive) { final byte[] computedKey = this.md.digest(); if (!Arrays.equals(computedKey, this.blobKey.getHash())) { this.wrappedOutputStream.write(RETURN_ERROR); throw new IOException("Detected data corruption during transfer"); } this.wrappedOutputStream.write(RETURN_OKAY); } } return read; }
throw new IOException("Unknown type of BLOB addressing: " + mode + '.'); blobKey = BlobKey.readFromInputStream(inputStream);
return createKey(blobType, key, random);
@Override public int read(byte[] b, int off, int len) throws IOException { final int bytesMissing = this.bytesToReceive - this.bytesReceived; if (bytesMissing == 0) { return -1; } final int maxRecv = Math.min(len, bytesMissing); final int read = this.wrappedInputStream.read(b, off, maxRecv); if (read < 0) { throwEOFException(); } this.bytesReceived += read; if (this.md != null) { this.md.update(b, off, read); if (this.bytesReceived == this.bytesToReceive) { final BlobKey computedKey = new BlobKey(this.md.digest()); if (!computedKey.equals(this.blobKey)) { throw new IOException("Detected data corruption during transfer"); } } } return read; }
BlobKey remoteKey = BlobKey.readFromInputStream(is); byte[] localHash = md.digest(); if (blobType != remoteKey.getType()) { throw new IOException("Detected data corruption during transfer"); if (!Arrays.equals(localHash, remoteKey.getHash())) { throw new IOException("Detected data corruption during transfer");
/** * Constructs and writes the header data for a GET operation to the given output stream. * * @param outputStream * the output stream to write the header data to * @param jobId * ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated) * @param blobKey * blob key associated with the requested file * * @throws IOException * thrown if an I/O error occurs while writing the header data to the output stream */ private static void sendGetHeader( OutputStream outputStream, @Nullable JobID jobId, BlobKey blobKey) throws IOException { checkNotNull(blobKey); checkArgument(jobId != null || blobKey instanceof TransientBlobKey, "permanent BLOBs must be job-related"); // Signal type of operation outputStream.write(GET_OPERATION); // Send job ID and key if (jobId == null) { outputStream.write(JOB_UNRELATED_CONTENT); } else { outputStream.write(JOB_RELATED_CONTENT); outputStream.write(jobId.getBytes()); } blobKey.writeToOutputStream(outputStream); }
@Override public int read() throws IOException { if (this.bytesReceived == this.bytesToReceive) { return -1; } final int read = this.wrappedInputStream.read(); if (read < 0) { throwEOFException(); } ++this.bytesReceived; if (this.md != null) { this.md.update((byte) read); if (this.bytesReceived == this.bytesToReceive) { final byte[] computedKey = this.md.digest(); if (!Arrays.equals(computedKey, this.blobKey.getHash())) { this.wrappedOutputStream.write(RETURN_ERROR); throw new IOException("Detected data corruption during transfer"); } this.wrappedOutputStream.write(RETURN_OKAY); } } return read; }
/** * Returns the (designated) physical storage location of the BLOB with the given key. * * @param key * the key identifying the BLOB * @return the (designated) physical storage location of the BLOB */ static File getStorageLocation(File storageDir, BlobKey key) { return new File(getCacheDirectory(storageDir), BLOB_FILE_PREFIX + key.toString()); }
blobKey = BlobKey.readFromInputStream(inputStream); blobFile = blobServer.getStorageLocation(blobKey);