private DeltaIndex index(DeltaWindowEntry ent) throws MissingObjectException, IncorrectObjectTypeException, IOException, LargeObjectException { DeltaIndex idx = ent.index; if (idx == null) { checkLoadable(ent, estimateIndexSize(ent)); try { idx = new DeltaIndex(buffer(ent)); } catch (OutOfMemoryError noMemory) { LargeObjectException.OutOfMemory e; e = new LargeObjectException.OutOfMemory(noMemory); e.setObjectId(ent.object); throw e; } if (maxMemory != 0) loaded += idx.getIndexSize() - idx.getSourceSize(); ent.index = idx; } return idx; }
/** * Construct an index from the source file. * * @param sourceBuffer * the source file's raw contents. The buffer will be held by the * index instance to facilitate matching, and therefore must not * be modified by the caller. */ public DeltaIndex(byte[] sourceBuffer) { src = sourceBuffer; DeltaIndexScanner scan = new DeltaIndexScanner(src, src.length); // Reuse the same table the scanner made. We will replace the // values at each position, but we want the same-length array. // table = scan.table; tableMask = scan.tableMask; // Because entry index 0 means there are no entries for the // slot in the table, we have to allocate one extra position. // entries = new long[1 + countEntries(scan)]; copyEntries(scan); }
private TemporaryBuffer.Heap delta(ObjectToPack otp) throws IOException { DeltaIndex index = new DeltaIndex(buffer(otp.getDeltaBaseId())); byte[] res = buffer(otp); // We never would have proposed this pair if the delta would be // larger than the unpacked version of the object. So using it // as our buffer limit is valid: we will never reach it. // TemporaryBuffer.Heap delta = new TemporaryBuffer.Heap(res.length); index.encode(delta, res); return delta; }
throws IOException { final int end = res.length; final DeltaEncoder enc = newEncoder(out, end, deltaSizeLimit); int hash = hashBlock(res, 0); hash = step(hash, res[blkPtr++], res[blkEnd++]); continue; do { long ent = entries[entryIdx++]; if (keyOf(ent) == hash) { int neg = 0; if (resPtr < blkPtr) { neg = negmatch(res, blkPtr, src, valOf(ent), neg); int len = neg + fwdmatch(res, blkPtr, src, valOf(ent)); if (bestLen < len) { bestLen = len; bestPtr = valOf(ent); bestNeg = neg; } else if ((keyOf(ent) & tableMask) != tableIdx) break; } while (bestLen < 4096 && entryIdx < entries.length); hash = step(hash, res[blkPtr++], res[blkEnd++]); continue;
private static long estimateSize(ObjectToPack ent) { return DeltaIndex.estimateIndexSize(ent.getWeight()); }
/** {@inheritDoc} */ @Override @SuppressWarnings("nls") public String toString() { String[] units = { "bytes", "KiB", "MiB", "GiB" }; long sz = getIndexSize(); int u = 0; while (1024 <= sz && u < units.length - 1) { int rem = (int) (sz % 1024); sz /= 1024; if (rem != 0) sz++; u++; } return "DeltaIndex[" + sz + " " + units[u] + "]"; }
/** * Generate a delta sequence to recreate the result buffer. * <p> * There is no limit on the size of the delta sequence created. This is the * same as {@code encode(out, res, 0)}. * * @param out * stream to receive the delta instructions that can transform * this index's source buffer into {@code res}. This stream * should be buffered, as instructions are written directly to it * in small bursts. * @param res * the desired result buffer. The generated instructions will * recreate this buffer when applied to the source buffer stored * within this index. * @throws java.io.IOException * the output stream refused to write the instructions. */ public void encode(OutputStream out, byte[] res) throws IOException { encode(out, res, 0 /* no limit */); }
private DeltaEncoder newEncoder(OutputStream out, long resSize, int limit) throws IOException { return new DeltaEncoder(out, getSourceSize(), resSize, limit); }
private void scan(byte[] raw, int end) { // We scan the input backwards, and always insert onto the // front of the chain. This ensures that chains will have lower // offsets at the front of the chain, allowing us to prefer the // earlier match rather than the later match. // int lastHash = 0; int ptr = end - DeltaIndex.BLKSZ; do { final int key = DeltaIndex.hashBlock(raw, ptr); final int tIdx = key & tableMask; final int head = table[tIdx]; if (head != 0 && lastHash == key) { // Two consecutive blocks have the same content hash, // prefer the earlier block because we want to use the // longest sequence we can during encoding. // entries[head] = (((long) key) << 32) | ptr; } else { final int eIdx = ++entryCnt; entries[eIdx] = (((long) key) << 32) | ptr; next[eIdx] = head; table[tIdx] = eIdx; } lastHash = key; ptr -= DeltaIndex.BLKSZ; } while (0 <= ptr); }
throws IOException { final int end = res.length; final DeltaEncoder enc = newEncoder(out, end, deltaSizeLimit); int hash = hashBlock(res, 0); hash = step(hash, res[blkPtr++], res[blkEnd++]); continue; do { long ent = entries[entryIdx++]; if (keyOf(ent) == hash) { int neg = 0; if (resPtr < blkPtr) { neg = negmatch(res, blkPtr, src, valOf(ent), neg); int len = neg + fwdmatch(res, blkPtr, src, valOf(ent)); if (bestLen < len) { bestLen = len; bestPtr = valOf(ent); bestNeg = neg; } else if ((keyOf(ent) & tableMask) != tableIdx) break; } while (bestLen < 4096 && entryIdx < entries.length); hash = step(hash, res[blkPtr++], res[blkEnd++]); continue;
private static long estimateIndexSize(DeltaWindowEntry ent) { if (ent.buffer == null) return estimateSize(ent.object); int len = ent.buffer.length; return DeltaIndex.estimateIndexSize(len) - len; }
private void clear(DeltaWindowEntry ent) { if (ent.index != null) loaded -= ent.index.getIndexSize(); else if (ent.buffer != null) loaded -= ent.buffer.length; ent.set(null); }
/** * Generate a delta sequence to recreate the result buffer. * <p> * There is no limit on the size of the delta sequence created. This is the * same as {@code encode(out, res, 0)}. * * @param out * stream to receive the delta instructions that can transform * this index's source buffer into {@code res}. This stream * should be buffered, as instructions are written directly to it * in small bursts. * @param res * the desired result buffer. The generated instructions will * recreate this buffer when applied to the source buffer stored * within this index. * @throws IOException * the output stream refused to write the instructions. */ public void encode(OutputStream out, byte[] res) throws IOException { encode(out, res, 0 /* no limit */); }
private DeltaEncoder newEncoder(OutputStream out, long resSize, int limit) throws IOException { return new DeltaEncoder(out, getSourceSize(), resSize, limit); }
private void scan(byte[] raw, final int end) { // We scan the input backwards, and always insert onto the // front of the chain. This ensures that chains will have lower // offsets at the front of the chain, allowing us to prefer the // earlier match rather than the later match. // int lastHash = 0; int ptr = end - DeltaIndex.BLKSZ; do { final int key = DeltaIndex.hashBlock(raw, ptr); final int tIdx = key & tableMask; final int head = table[tIdx]; if (head != 0 && lastHash == key) { // Two consecutive blocks have the same content hash, // prefer the earlier block because we want to use the // longest sequence we can during encoding. // entries[head] = (((long) key) << 32) | ptr; } else { final int eIdx = ++entryCnt; entries[eIdx] = (((long) key) << 32) | ptr; next[eIdx] = head; table[tIdx] = eIdx; } lastHash = key; ptr -= DeltaIndex.BLKSZ; } while (0 <= ptr); }
throws IOException { final int end = res.length; final DeltaEncoder enc = newEncoder(out, end, deltaSizeLimit); int hash = hashBlock(res, 0); hash = step(hash, res[blkPtr++], res[blkEnd++]); continue; do { long ent = entries[entryIdx++]; if (keyOf(ent) == hash) { int neg = 0; if (resPtr < blkPtr) { neg = negmatch(res, blkPtr, src, valOf(ent), neg); int len = neg + fwdmatch(res, blkPtr, src, valOf(ent)); if (bestLen < len) { bestLen = len; bestPtr = valOf(ent); bestNeg = neg; } else if ((keyOf(ent) & tableMask) != tableIdx) break; } while (bestLen < 4096 && entryIdx < entries.length); hash = step(hash, res[blkPtr++], res[blkEnd++]); continue;
private DeltaIndex index(DeltaWindowEntry ent) throws MissingObjectException, IncorrectObjectTypeException, IOException, LargeObjectException { DeltaIndex idx = ent.index; if (idx == null) { checkLoadable(ent, estimateIndexSize(ent)); try { idx = new DeltaIndex(buffer(ent)); } catch (OutOfMemoryError noMemory) { LargeObjectException.OutOfMemory e; e = new LargeObjectException.OutOfMemory(noMemory); e.setObjectId(ent.object); throw e; } if (maxMemory != 0) loaded += idx.getIndexSize() - idx.getSourceSize(); ent.index = idx; } return idx; }
private TemporaryBuffer.Heap delta(final ObjectToPack otp) throws IOException { DeltaIndex index = new DeltaIndex(buffer(otp.getDeltaBaseId())); byte[] res = buffer(otp); // We never would have proposed this pair if the delta would be // larger than the unpacked version of the object. So using it // as our buffer limit is valid: we will never reach it. // TemporaryBuffer.Heap delta = new TemporaryBuffer.Heap(res.length); index.encode(delta, res); return delta; }
/** * Construct an index from the source file. * * @param sourceBuffer * the source file's raw contents. The buffer will be held by the * index instance to facilitate matching, and therefore must not * be modified by the caller. */ public DeltaIndex(byte[] sourceBuffer) { src = sourceBuffer; DeltaIndexScanner scan = new DeltaIndexScanner(src, src.length); // Reuse the same table the scanner made. We will replace the // values at each position, but we want the same-length array. // table = scan.table; tableMask = scan.tableMask; // Because entry index 0 means there are no entries for the // slot in the table, we have to allocate one extra position. // entries = new long[1 + countEntries(scan)]; copyEntries(scan); }
private static long estimateIndexSize(DeltaWindowEntry ent) { if (ent.buffer == null) return estimateSize(ent.object); int len = ent.buffer.length; return DeltaIndex.estimateIndexSize(len) - len; }