/** Append a CRC32 to a (short) byte[] */ private byte[] appendChecksum(byte[] data) { return checksumChecker.appendChecksum(data); }
/** Verify a checksum or throw */ public void verifyChecksum(byte[] data, int offset, int length, byte[] checksum) throws ChecksumFailedException { if(!checkChecksum(data, offset, length, checksum)) throw new ChecksumFailedException(); }
public int lengthAndChecksumOverhead() { return 8 + checksumLength(); }
private void writeRequestIdentifier(DataOutput os, RequestIdentifier req) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream oos = checker.checksumWriter(baos); DataOutputStream dos = new DataOutputStream(oos); req.writeTo(dos); dos.close(); byte[] buf = baos.toByteArray(); os.writeShort(buf.length - checker.checksumLength()); os.write(buf); }
void preadChecksummed(long fileOffset, byte[] buf, int offset, int length) throws IOException, ChecksumFailedException { byte[] checksumBuf = new byte[checker.checksumLength()]; RAFLock lock = raf.lockOpen(); try { raf.pread(fileOffset, buf, offset, length); raf.pread(fileOffset+length, checksumBuf, 0, checker.checksumLength()); } finally { lock.unlock(); } if(!checker.checkChecksum(buf, offset, length, checksumBuf)) { Arrays.fill(buf, offset, offset+length, (byte)0); throw new ChecksumFailedException(); } }
this.errors = new FailureCodeTracker(false); this.checksumChecker = checker; this.checksumLength = checker.checksumLength(); this.persistent = persistent; this.completeViaTruncation = (storageFile != null); dos = new DataOutputStream(baos); dos.writeInt(0); // flags dos.writeShort(checksumChecker.getChecksumTypeID()); dos.writeInt(VERSION); byte[] version = baos.toByteArray(); System.arraycopy(version, 0, bufToChecksum, bufToWrite.length, version.length); byte[] checksum = checksumChecker.generateChecksum(bufToChecksum);
private byte[] encodeGeneralProgress() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { OutputStream ccos = checksumChecker.checksumWriterWithLength(baos, new ArrayBucketFactory()); DataOutputStream dos = new DataOutputStream(ccos); long flags = 0; if(hasCheckedDatastore) flags |= HAS_CHECKED_DATASTORE_FLAG; dos.writeLong(flags); errors.writeFixedLengthTo(dos); dos.close(); } catch (IOException e) { throw new Error(e); } byte[] ret = baos.toByteArray(); return ret; }
try { dos.writeLong(MAGIC); dos.writeInt(checker.getChecksumTypeID()); OutputStream os = checker.checksumWriterWithLength(baos, new ArrayBucketFactory()); dos = new DataOutputStream(os); dos.writeInt(VERSION);
int checksumType = dis.readInt(); try { this.checker = ChecksumChecker.create(checksumType); } catch (IllegalArgumentException e) { throw new StorageFormatException("Bad checksum type"); InputStream is = checker.checksumReaderWithLength(ois, new ArrayBucketFactory(), 1024*1024); dis = new DataInputStream(is); int version = dis.readInt(); is = checker.checksumReaderWithLength(ois, new ArrayBucketFactory(), 1024*1024); dis = new DataInputStream(is); if(hasPaddedLastBlock) { is = checker.checksumReaderWithLength(ois, new ArrayBucketFactory(), 1024*1024); dis = new DataInputStream(is); long blocks = 0; throw new StorageFormatException("Total data blocks should be "+totalDataBlocks+" but is "+blocks); if(crossSegments != null) { is = checker.checksumReaderWithLength(ois, new ArrayBucketFactory(), 1024*1024); dis = new DataInputStream(is); for(int i=0;i<crossSegments.length;i++) { dis = new DataInputStream(checker.checksumReaderWithLength(ois, new ArrayBucketFactory(), 1024*1024)); errors = new FailureCodeTracker(true, dis); dis.close();
public OutputStream checksumWriter(OutputStream os) { return checksumWriter(os, 0); }
private Object readChecksummedObject(ObjectInputStream is, long totalLength) throws IOException, ChecksumFailedException, ClassNotFoundException { InputStream ois = checker.checksumReaderWithLength(is, this.tempBucketFactory, totalLength); try { ObjectInputStream oo = new ObjectInputStream(ois); Object ret = oo.readObject(); oo.close(); oo = null; ois = null; return ret; } catch (Throwable t) { Logger.error(this, "Serialization failed: "+t, t); return null; } finally { if(ois != null) ois.close(); } }
public byte[] generateChecksum(byte[] bufToChecksum) { return generateChecksum(bufToChecksum, 0, bufToChecksum.length); }
public InputStream checksumReaderWithLength(InputStream dis, BucketFactory bf, long maxLength) throws IOException, ChecksumFailedException { // IMHO it is better to implement this with copying, because then we don't start // constructing objects from bad data... long length = new DataInputStream(dis).readLong(); if(length < 0 || length > maxLength) throw new IOException("Bad length"); final Bucket bucket = bf.makeBucket(-1); OutputStream os = bucket.getOutputStream(); copyAndStripChecksum(dis, os, length); os.close(); return ReadBucketAndFreeInputStream.create(bucket); }
ClientCHK readKey(int blockNumber) throws IOException, MissingKeyException { byte[] buf = parent.innerReadSegmentKey(segNo, blockNumber); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); dos.writeInt(segNo); dos.writeInt(blockNumber); dos.close(); byte[] prefix = baos.toByteArray(); byte[] checkBuf = new byte[prefix.length + buf.length]; System.arraycopy(prefix, 0, checkBuf, 0, prefix.length); int checksumLength = parent.checker.checksumLength(); System.arraycopy(buf, 0, checkBuf, prefix.length, buf.length - checksumLength); byte[] checksum = Arrays.copyOfRange(buf, buf.length - checksumLength, buf.length); if(parent.checker.checkChecksum(checkBuf, 0, checkBuf.length, checksum)) throw new MissingKeyException(); DataInputStream dis = new DataInputStream(new ByteArrayInputStream(buf)); byte b = dis.readByte(); if(b != 1) throw new MissingKeyException(); ClientCHK key = innerReadKey(dis); setHasKey(blockNumber); if(logDEBUG) Logger.debug(this, "Returning "+key); return key; }
/** Encode the offsets. */ private byte[] encodeOffsets() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { OutputStream os = checker.checksumWriterWithLength(baos, new ArrayBucketFactory()); DataOutputStream dos = new DataOutputStream(os); if(this.hasPaddedLastBlock) dos.writeLong(offsetPaddedLastBlock); dos.writeLong(offsetOverallStatus); dos.writeInt(overallStatusLength); if(crossSegments != null) { for(long l : offsetCrossSegmentBlocks) dos.writeLong(l); } for(long l : offsetSegmentCheckBlocks) dos.writeLong(l); for(long l : offsetSegmentStatus) dos.writeLong(l); if(crossSegments != null) { for(long l : offsetCrossSegmentStatus) dos.writeLong(l); } for(long l : offsetSegmentKeys) dos.writeLong(l); dos.close(); return baos.toByteArray(); } catch (IOException e) { throw new Error(e); // Impossible } }
OutputStream checksumOutputStream(OutputStream os) { return checksumChecker.checksumWriter(os); }
private ClientRequest readRequestFromRecoveryData(ObjectInputStream is, long totalLength, RequestIdentifier reqID) throws IOException, ChecksumFailedException, StorageFormatException { InputStream tmp = checker.checksumReaderWithLength(is, this.tempBucketFactory, totalLength); try { DataInputStream dis = new DataInputStream(tmp); ClientRequest request = ClientRequest.restartFrom(dis, reqID, getClientContext(), checker); dis.close(); dis = null; tmp = null; return request; } catch (Throwable t) { Logger.error(this, "Serialization failed: "+t, t); return null; } finally { if(tmp != null) tmp.close(); } }
public void writeAndChecksum(OutputStream os, byte[] buf, int offset, int length) throws IOException { os.write(buf, offset, length); os.write(generateChecksum(buf, offset, length)); }
byte[] preadChecksummedWithLength(long fileOffset) throws IOException, ChecksumFailedException, StorageFormatException { byte[] checksumBuf = new byte[checker.checksumLength()]; RAFLock lock = raf.lockOpen(); byte[] lengthBuf = new byte[8]; byte[] buf; int length; try { raf.pread(fileOffset, lengthBuf, 0, lengthBuf.length); long len = new DataInputStream(new ByteArrayInputStream(lengthBuf)).readLong(); if(len + fileOffset > rafLength || len > Integer.MAX_VALUE || len < 0) throw new StorageFormatException("Bogus length "+len); length = (int)len; buf = new byte[length]; raf.pread(fileOffset+lengthBuf.length, buf, 0, length); raf.pread(fileOffset+length+lengthBuf.length, checksumBuf, 0, checker.checksumLength()); } finally { lock.unlock(); } if(!checker.checkChecksum(buf, 0, length, checksumBuf)) { Arrays.fill(buf, 0, length, (byte)0); throw new ChecksumFailedException(); } return buf; }
private void skipChecksummedObject(ObjectInputStream is, long totalLength) throws IOException { long length = is.readLong(); if(length > totalLength) throw new IOException("Too long: "+length+" > "+totalLength); FileUtil.skipFully(is, length + checker.checksumLength()); }