public void testReadAndWrite() throws Exception { SortedOplogConfiguration config = new SortedOplogConfiguration("lex") .setComparator(new LexicographicalComparator()); hfile = new HFileSortedOplog(new File("lexi.soplog"), config); SortedOplogWriter wtr = hfile.createWriter(); for (int i = -100; i < 100; i++) { wtr.append(ComparisonTestCase.convert(i), ComparisonTestCase.convert(i)); } wtr.close(null); SortedReader<ByteBuffer> rdr = hfile.createReader(); for (int i = -100; i < 100; i++) { byte[] key = ComparisonTestCase.convert(i); assertTrue(rdr.mightContain(key)); ByteBuffer val = rdr.read(key); assertEquals(i, ComparisonTestCase.recover(val.array(), val.arrayOffset(), val.remaining())); } rdr.close(); }
public AbstractCompactor(SortedOplogFactory factory, Fileset<T> fileset, CompactionTracker<T> tracker, Executor exec) { assert factory != null; assert fileset != null; assert tracker != null; assert exec != null; this.factory = factory; this.fileset = fileset; this.tracker = tracker; compactor = new AbortableTaskService(exec); inactive = new ConcurrentLinkedQueue<TrackedReference<SortedOplogReader>>(); levelLock = new ReentrantReadWriteLock(); levels = new ArrayList<Level>(); logger = ComponentLogWriter.getSoplogLogWriter(factory.getConfiguration().getName(), LogService.logger()); }
@Override public SortedIterator<ByteBuffer> scan( byte[] from, boolean fromInclusive, byte[] to, boolean toInclusive, boolean ascending, MetadataFilter filter) throws IOException { SerializedComparator sc = factory.getConfiguration().getComparator(); sc = ascending ? sc : ReversingSerializedComparator.reverse(sc); List<SortedIterator<ByteBuffer>> scans = new ArrayList<SortedIterator<ByteBuffer>>(); Collection<TrackedReference<SortedOplogReader>> soplogs; rwlock.readLock().lock(); try { scans.add(getCurrent().scan(from, fromInclusive, to, toInclusive, ascending, filter)); for (SortedBuffer<Integer> sb : unflushed) { scans.add(sb.scan(from, fromInclusive, to, toInclusive, ascending, filter)); } soplogs = compactor.getActiveReaders(from, to); } finally { rwlock.readLock().unlock(); } for (TrackedReference<SortedOplogReader> tr : soplogs) { scans.add(tr.get().scan(from, fromInclusive, to, toInclusive, ascending, filter)); } return new MergedIterator(sc, soplogs, scans); }
long start = factory.getConfiguration().getStatistics().getMinorCompaction().begin(); try { compacted |= level.compact(aborted); factory.getConfiguration().getStatistics().getMinorCompaction().end(start); factory.getConfiguration().getStatistics().getMinorCompaction().error(start);
public void testMultiblockReverseScan2() throws Exception { SortedOplogConfiguration config = new SortedOplogConfiguration("backwards2"); config.setBlockSize(4096);
public void testMultiblockReverseScan() throws Exception { int entries = 10000; SortedOplogConfiguration config = new SortedOplogConfiguration("backwards"); config.setBlockSize(4096);
assert handler != null; long start = factory.getConfiguration().getStatistics().getFlush().begin(); if (isClosed()) { handler.complete(); factory.getConfiguration().getStatistics().getFlush().end(0, start);
long start = factory.getConfiguration().getStatistics().getClear().begin(); factory.getConfiguration().getStatistics().getClear().end(start); factory.getConfiguration().getStatistics().getClear().error(start); throw (IOException) e.fillInStackTrace();
}); SortedOplogConfiguration config = new SortedOplogConfiguration("lex") .setComparator(ac);
@Override public ByteBuffer read(byte[] key) throws IOException { assert key != null; if (logger.finestEnabled()) { logger.finest(String.format("Reading key %s from %s", Hex.toHex(key), path)); } long start = sopConfig.getStatistics().getRead().begin(); try { HFileScanner seek = reader.getScanner(true, true); if (seek.seekTo(key) == 0) { ByteBuffer val = seek.getValue(); sopConfig.getStatistics().getRead().end(val.remaining(), start); return val; } sopConfig.getStatistics().getRead().end(start); sopConfig.getStatistics().getBloom().falsePositive(); return null; } catch (IOException e) { sopConfig.getStatistics().getRead().error(start); throw (IOException) e.fillInStackTrace(); } }
protected EnumMap<Metadata, byte[]> mergeMetadata( Collection<TrackedReference<SortedOplogReader>> readers) throws IOException { // merge the metadata into the compacted file EnumMap<Metadata, byte[]> metadata = new EnumMap<Metadata, byte[]>(Metadata.class); for (Metadata meta : Metadata.values()) { byte[] val = null; for (TrackedReference<SortedOplogReader> tr : readers) { byte[] tmp = tr.get().getMetadata(meta); if (val == null) { val = tmp; } else if (tmp != null) { val = factory.getConfiguration().getMetadataCompactor(meta).compact(val, tmp); } } if (val != null) { metadata.put(meta, val); } } return metadata; }
public HFileSortedOplogWriter() throws IOException { HFileContext hcontext = new HFileContextBuilder() .withBlockSize(sopConfig.getBlockSize()) .withBytesPerCheckSum(sopConfig.getBytesPerChecksum()) .withChecksumType(HFileSortedOplogFactory.convertChecksum( sopConfig.getChecksum())) .withCompression(HFileSortedOplogFactory.convertCompression( sopConfig.getCompression())) .withDataBlockEncoding(HFileSortedOplogFactory.convertEncoding( sopConfig.getKeyEncoding()).getDataBlockEncoding()) .build(); writer = HFile.getWriterFactory(hconf, hcache) .withPath(fs, path) .withFileContext(hcontext) .withComparator(sopConfig.getComparator()) .create(); bfw = sopConfig.isBloomFilterEnabled() ? // BloomFilterFactory.createGeneralBloomAtWrite(hconf, hcache, BloomType.ROW, // 0, writer, sopConfig.getComparator()) BloomFilterFactory.createGeneralBloomAtWrite(hconf, hcache, BloomType.ROW, 0, writer) : null; }
@Override public void run() { try { compactor.add(soplog); compactor.compact(false, null); unflushed.removeFirstOccurrence(buffer); // TODO need to invoke this while NOT holding write lock handler.complete(); factory.getConfiguration().getStatistics().getFlush().end(buffer.dataSize(), start); } catch (Exception e) { handleError(e, aborted); return; } } };
public HFileSortedIterator(HFileScanner scan, SerializedComparator comparator, byte[] from, boolean fromInclusive, byte[] to, boolean toInclusive) throws IOException { this.scan = scan; this.comparator = comparator; this.from = from; this.fromInclusive = fromInclusive; this.to = to; this.toInclusive = toInclusive; assert from == null || to == null || comparator.compare(from, 0, from.length, to, 0, to.length) <= 0; start = sopConfig.getStatistics().getScan().begin(); foundNext = evalFrom(); }
@Override public void destroy() throws IOException { if (logger.fineEnabled()) { logger.fine("Destroying soplog set"); } long start = factory.getConfiguration().getStatistics().getDestroy().begin(); try { unsetCurrent(); clear(); close(); factory.getConfiguration().getStatistics().getDestroy().end(start); } catch (IOException e) { factory.getConfiguration().getStatistics().getDestroy().error(start); throw (IOException) e.fillInStackTrace(); } }
@Override public ByteBuffer next() { long startNext = sopConfig.getStatistics().getScan().beginIteration(); if (!hasNext()) { throw new NoSuchElementException(); } foundNext = false; key = scan.getKey(); value = scan.getValue(); int len = key.remaining() + value.remaining(); bytes += len; sopConfig.getStatistics().getScan().endIteration(len, startNext); return key; }
public SortedOplogSetImpl(final SortedOplogFactory factory, Executor exec, Compactor ctor) throws IOException { this.factory = factory; this.flusher = new AbortableTaskService(exec); this.compactor = ctor; rwlock = new ReentrantReadWriteLock(); bufferCount = new AtomicInteger(0); unflushed = new ArrayDeque<SortedBuffer<Integer>>(); current = new AtomicReference<SortedBuffer<Integer>>( new SortedBuffer<Integer>(factory.getConfiguration(), 0)); logger = ComponentLogWriter.getSoplogLogWriter(factory.getConfiguration().getName(), LogService.logger()); if (logger.fineEnabled()) { logger.fine("Creating soplog set"); } }
public NoFileSortedOplogFactory(String name) { config = new SortedOplogConfiguration(name); }
@Override public void close() { sopConfig.getStatistics().getScan().end(bytes, start); }
@Override protected SortedReader<ByteBuffer> createReader(NavigableMap<byte[], byte[]> data) throws IOException { SortedOplogConfiguration config = new SortedOplogConfiguration("test"); hfile = new HFileSortedOplog(new File("test.soplog"), config); SortedOplogWriter wtr = hfile.createWriter(); for (Entry<byte[], byte[]> entry : data.entrySet()) { wtr.append(entry.getKey(), entry.getValue()); } wtr.close(null); return hfile.createReader(); } }