ObjectWriter(DhtReader ctx, Prefetcher prefetch) { this.ctx = ctx; this.prefetch = prefetch; batchSize = ctx.getOptions().getObjectIndexBatchSize(); metaBatches = new Semaphore(batchSize); metaError = new AtomicReference<DhtException>(); allVisits = new LinkedHashMap<ChunkKey, Integer>(); allMeta = new HashMap<ChunkKey, ChunkMeta>(); metaMissing = new HashSet<ChunkKey>(); metaToRead = new HashSet<ChunkKey>(); curVisit = 1; }
ObjectWriter(DhtReader ctx, Prefetcher prefetch) { this.ctx = ctx; this.prefetch = prefetch; batchSize = ctx.getOptions().getObjectIndexBatchSize(); metaBatches = new Semaphore(batchSize); metaError = new AtomicReference<DhtException>(); allVisits = new LinkedHashMap<ChunkKey, Integer>(); allMeta = new HashMap<ChunkKey, ChunkMeta>(); metaMissing = new HashSet<ChunkKey>(); metaToRead = new HashSet<ChunkKey>(); curVisit = 1; }
RecentChunks(DhtReader reader) { this.reader = reader; this.stats = reader.getStatistics(); this.byKey = new HashMap<ChunkKey, Node>(); this.maxBytes = reader.getOptions().getChunkLimit(); }
RecentChunks(DhtReader reader) { this.reader = reader; this.stats = reader.getStatistics(); this.byKey = new HashMap<ChunkKey, Node>(); this.maxBytes = reader.getOptions().getChunkLimit(); }
private void endPrefetch() { recentChunks.setMaxBytes(getOptions().getChunkLimit()); prefetcher = null; }
QueueObjectLookup(DhtReader reader, boolean reportMissing) { this.repo = reader.getRepositoryKey(); this.db = reader.getDatabase(); this.reader = reader; this.options = reader.getOptions(); this.reportMissing = reportMissing; this.tmp = new ArrayList<ObjectInfo>(4); this.context = Context.FAST_MISSING_OK; this.toRetry = new ArrayList<T>(); this.concurrentBatches = options.getObjectIndexConcurrentBatches(); }
QueueObjectLookup(DhtReader reader, boolean reportMissing) { this.repo = reader.getRepositoryKey(); this.db = reader.getDatabase(); this.reader = reader; this.options = reader.getOptions(); this.reportMissing = reportMissing; this.tmp = new ArrayList<ObjectInfo>(4); this.context = Context.FAST_MISSING_OK; this.toRetry = new ArrayList<T>(); this.concurrentBatches = options.getObjectIndexConcurrentBatches(); }
private void endPrefetch() { recentChunks.setMaxBytes(getOptions().getChunkLimit()); prefetcher = null; }
private void awaitPendingBatches() throws InterruptedException, DhtTimeoutException, DhtException { Timeout to = ctx.getOptions().getTimeout(); if (!metaBatches.tryAcquire(batchSize, to.getTime(), to.getUnit())) throw new DhtTimeoutException(DhtText.get().timeoutChunkMeta); if (metaError.get() != null) throw metaError.get(); }
private void awaitPendingBatches() throws InterruptedException, DhtTimeoutException, DhtException { Timeout to = ctx.getOptions().getTimeout(); if (!metaBatches.tryAcquire(batchSize, to.getTime(), to.getUnit())) throw new DhtTimeoutException(DhtText.get().timeoutChunkMeta); if (metaError.get() != null) throw metaError.get(); }
DeltaBaseCache(DhtReader reader) { stats = reader.getStatistics(); DhtReaderOptions options = reader.getOptions(); maxByteCount = options.getDeltaBaseCacheLimit(); table = new Slot[options.getDeltaBaseCacheSize()]; }
Prefetcher(DhtReader reader, int objectType, int prefetchLimitInBytes) { this.db = reader.getDatabase(); this.stats = reader.getStatistics(); this.objectType = objectType; this.ready = new HashMap<ChunkKey, PackChunk>(); this.status = new HashMap<ChunkKey, Status>(); this.queue = new LinkedList<ChunkKey>(); this.followEdgeHints = reader.getOptions().isPrefetchFollowEdgeHints(); this.averageChunkSize = reader.getInserterOptions().getChunkSize(); this.highWaterMark = prefetchLimitInBytes; int lwm = (highWaterMark / averageChunkSize) - 4; if (lwm <= 0) lwm = (highWaterMark / averageChunkSize) / 2; lowWaterMark = lwm * averageChunkSize; }
@Override public void release() { reader.getRecentChunks().setMaxBytes(reader.getOptions().getChunkLimit()); prefetcher = null; currChunk = null; }
DeltaBaseCache(DhtReader reader) { stats = reader.getStatistics(); DhtReaderOptions options = reader.getOptions(); maxByteCount = options.getDeltaBaseCacheLimit(); table = new Slot[options.getDeltaBaseCacheSize()]; }
@Override public void release() { reader.getRecentChunks().setMaxBytes(reader.getOptions().getChunkLimit()); prefetcher = null; currChunk = null; }
private void startBatch(Context context) throws InterruptedException, DhtTimeoutException { Timeout to = ctx.getOptions().getTimeout(); if (!metaBatches.tryAcquire(1, to.getTime(), to.getUnit())) throw new DhtTimeoutException(DhtText.get().timeoutChunkMeta); Set<ChunkKey> keys = metaToRead; ctx.getDatabase().chunk().getMeta( context, keys, new MetaLoader(context, keys)); metaToRead = new HashSet<ChunkKey>(); }
private void startBatch(Context context) throws InterruptedException, DhtTimeoutException { Timeout to = ctx.getOptions().getTimeout(); if (!metaBatches.tryAcquire(1, to.getTime(), to.getUnit())) throw new DhtTimeoutException(DhtText.get().timeoutChunkMeta); Set<ChunkKey> keys = metaToRead; ctx.getDatabase().chunk().getMeta( context, keys, new MetaLoader(context, keys)); metaToRead = new HashSet<ChunkKey>(); }
void copyAsIs(PackOutputStream out, boolean validate, DhtReader ctx) throws IOException { if (keyList == null) init(); // Clear the recent chunks because all of the reader's // chunk limit should be made available for prefetch. int cacheLimit = ctx.getOptions().getChunkLimit(); ctx.getRecentChunks().setMaxBytes(0); try { Prefetcher p = new Prefetcher(ctx, 0, cacheLimit); p.push(Arrays.asList(keyList)); copyPack(out, p, validate); } finally { ctx.getRecentChunks().setMaxBytes(cacheLimit); } }
DhtReader(DhtObjDatabase objdb) { this.repository = objdb.getRepository(); this.repo = objdb.getRepository().getRepositoryKey(); this.db = objdb.getDatabase(); this.readerOptions = objdb.getReaderOptions(); this.inserterOptions = objdb.getInserterOptions(); this.stats = new Statistics(); this.recentInfo = new RecentInfoCache(getOptions()); this.recentChunks = new RecentChunks(this); this.deltaBaseCache = new DeltaBaseCache(this); }
DhtReader(DhtObjDatabase objdb) { this.repository = objdb.getRepository(); this.repo = objdb.getRepository().getRepositoryKey(); this.db = objdb.getDatabase(); this.readerOptions = objdb.getReaderOptions(); this.inserterOptions = objdb.getInserterOptions(); this.stats = new Statistics(); this.recentInfo = new RecentInfoCache(getOptions()); this.recentChunks = new RecentChunks(this); this.deltaBaseCache = new DeltaBaseCache(this); }