public List<EntryList> execute(final BackendTransaction tx) { int total = 0; final List<EntryList> result = new ArrayList<>(Math.min(getLimit(), queries.size())); for (KeySliceQuery ksq : queries) { EntryList next =tx.indexQuery(ksq.updateLimit(getLimit()-total)); result.add(next); total+=next.size(); if (total>=getLimit()) break; } return result; }
public EntryList getSubset(final SliceQuery otherQuery, final EntryList otherResult) { assert otherQuery.subsumes(this); int pos = Collections.binarySearch(otherResult, sliceStart); if (pos < 0) pos = -pos - 1; final List<Entry> result = new ArrayList<>(); for (; pos < otherResult.size() && result.size() < getLimit(); pos++) { Entry e = otherResult.get(pos); if (e.getColumnAs(StaticBuffer.STATIC_FACTORY).compareTo(sliceEnd) < 0) result.add(e); else break; } return StaticArrayEntryList.of(result); }
public ExpirationKCVSCache(final KeyColumnValueStore store, String metricsName, final long cacheTimeMS, final long invalidationGracePeriodMS, final long maximumByteSize) { super(store, metricsName); Preconditions.checkArgument(cacheTimeMS > 0, "Cache expiration must be positive: %s", cacheTimeMS); Preconditions.checkArgument(System.currentTimeMillis()+1000L*3600*24*365*100+cacheTimeMS>0,"Cache expiration time too large, overflow may occur: %s",cacheTimeMS); this.cacheTimeMS = cacheTimeMS; final int concurrencyLevel = Runtime.getRuntime().availableProcessors(); Preconditions.checkArgument(invalidationGracePeriodMS >=0,"Invalid expiration grace period: %s", invalidationGracePeriodMS); this.invalidationGracePeriodMS = invalidationGracePeriodMS; CacheBuilder<KeySliceQuery,EntryList> cachebuilder = CacheBuilder.newBuilder() .maximumWeight(maximumByteSize) .concurrencyLevel(concurrencyLevel) .initialCapacity(1000) .expireAfterWrite(cacheTimeMS, TimeUnit.MILLISECONDS) .weigher((keySliceQuery, entries) -> GUAVA_CACHE_ENTRY_SIZE + KEY_QUERY_SIZE + entries.getByteSize()); cache = cachebuilder.build(); expiredKeys = new ConcurrentHashMap<>(50, 0.75f, concurrencyLevel); penaltyCountdown = new CountDownLatch(PENALTY_THRESHOLD); cleanupThread = new CleanupThread(); cleanupThread.start(); }
public Stream<Object> query(final JointIndexQuery.Subquery query, final BackendTransaction tx) { final IndexType index = query.getIndex(); if (index.isCompositeIndex()) { final MultiKeySliceQuery sq = query.getCompositeQuery(); final List<EntryList> rs = sq.execute(tx); final List<Object> results = new ArrayList<>(rs.get(0).size()); for (final EntryList r : rs) { for (final java.util.Iterator<Entry> iterator = r.reuseIterator(); iterator.hasNext(); ) { final Entry entry = iterator.next(); final ReadBuffer entryValue = entry.asReadBuffer(); entryValue.movePositionTo(entry.getValuePosition()); switch(index.getElement()) { case VERTEX: results.add(VariableLong.readPositive(entryValue)); break; default: results.add(bytebuffer2RelationId(entryValue)); } } } return results.stream(); } else { return tx.indexQuery(index.getBackingIndexName(), query.getMixedQuery()).map(IndexSerializer::string2ElementId); } }
private EntryList findEntriesMatchingQuery(SliceQuery query, EntryList sortedEntries) { int lowestStartMatch = sortedEntries.size(); // Inclusive int highestEndMatch = -1; // Inclusive int high = sortedEntries.size() - 1; Entry midVal = sortedEntries.get(mid); if (sortedEntries.size() == lowestStartMatch) { return EntryList.EMPTY_LIST; high = sortedEntries.size() - 1; Entry midVal = sortedEntries.get(mid); return EntryArrayList.of(sortedEntries.subList(lowestStartMatch, endIndex /* exclusive */)); } else { return EntryList.EMPTY_LIST;
protected boolean isGhostVertex(long vertexId, EntryList firstEntries) { if (idManager.isPartitionedVertex(vertexId) && !idManager.isCanonicalVertexId(vertexId)) return false; RelationCache relCache = tx.getEdgeSerializer().parseRelation( firstEntries.get(0),true,tx); return relCache.typeId != BaseKey.VertexExists.longId(); }
public static boolean containsKey(KeyColumnValueStore store, StaticBuffer key, int maxColumnLength, StoreTransaction txh) throws BackendException { final StaticBuffer end; if (maxColumnLength>32) { end = BufferUtil.oneBuffer(maxColumnLength); } else { end = END; } return !store.getSlice(new KeySliceQuery(key, START, end).setLimit(1),txh).isEmpty(); }
@Override public RecordIterator<Entry> getEntries() { ensureOpen(); if (columnSlice == null) throw new IllegalStateException("getEntries() requires SliceQuery to be set."); final KeySliceQuery keySlice = new KeySliceQuery(currentRow.getKey(), columnSlice); return new RecordIterator<Entry>() { private final Iterator<Entry> items = currentRow.getValue().getSlice(keySlice, transaction).iterator(); @Override public boolean hasNext() { ensureOpen(); return items.hasNext(); } @Override public Entry next() { ensureOpen(); return items.next(); } @Override public void close() { isClosed = true; } @Override public void remove() { throw new UnsupportedOperationException("Column removal not supported"); } }; }
private EntryList findEntriesMatchingQuery(SliceQuery query, EntryList sortedEntries) { int lowestStartMatch = sortedEntries.size(); // Inclusive int highestEndMatch = -1; // Inclusive int high = sortedEntries.size() - 1; Entry midVal = sortedEntries.get(mid); if (sortedEntries.size() == lowestStartMatch) { return EntryList.EMPTY_LIST; high = sortedEntries.size() - 1; Entry midVal = sortedEntries.get(mid); return EntryArrayList.of(sortedEntries.subList(lowestStartMatch, endIndex /* exclusive */)); } else { return EntryList.EMPTY_LIST;
if (entries==null) { entries = retriever.retrieveSchemaRelations(schemaId, type, dir); if (!entries.isEmpty()) { //only cache if type exists schemaRelationsBackup.put(typePlusRelation, entries);
/** * Executes the query by executing its on {@link SliceQuery} sub-query. * * @return */ private Iterator<Entry> getBasicIterator() { final EntryList result = vertex.loadRelations(sliceQuery, query -> QueryProfiler.profile(profiler, query, q -> tx.getGraph().edgeQuery(vertex.longId(), q, tx.getTxHandle()))); return result.iterator(); }
@Override public void process(StaticBuffer key, Map<SliceQuery, EntryList> entries, ScanMetrics metrics) { assertNotNull(key); assertTrue(keyFilter.test(key)); metrics.incrementCustom(KEY_COUNT); assertNotNull(entries); assertTrue(qs.size() >= entries.size()); for (SliceQuery q : qs) { if (!entries.containsKey(q)) { continue; } EntryList result = entries.get(q); metrics.incrementCustom(TOTAL_COUNT,result.size()); } }
public void checkSlice(String[] values, Set<Integer> removed, int start, int end, int limit) throws BackendException { EntryList entries; if (limit <= 0) entries = KVUtil.getSlice(store, KeyValueStoreUtil.getBuffer(start), KeyValueStoreUtil.getBuffer(end), tx); else entries = KVUtil.getSlice(store, KeyValueStoreUtil.getBuffer(start), KeyValueStoreUtil.getBuffer(end), limit, tx); int pos = 0; for (int i = start; i < end; i++) { if (removed.contains(i)) continue; if (pos < limit) { Entry entry = entries.get(pos); int id = KeyValueStoreUtil.getID(entry.getColumn()); String str = KeyValueStoreUtil.getString(entry.getValueAs(StaticBuffer.STATIC_FACTORY)); Assert.assertEquals(i, id); Assert.assertEquals(values[i], str); } pos++; } if (limit > 0 && pos >= limit) Assert.assertEquals(limit, entries.size()); else { Assert.assertNotNull(entries); Assert.assertEquals(pos, entries.size()); } }
List<EntryList> existence = graph.edgeMultiQuery(vertexIds,graph.vertexExistenceQuery,txHandle); for (int i = 0; i < vertexIds.size(); i++) { if (!existence.get(i).isEmpty()) { long id = vertexIds.get(i); result.add(vertexCache.get(id, existingVertexRetriever));
for (int t = 0; t < trials; t++) { int key = r.nextInt(keys)*2; assertEquals(2,store.getSlice(new KeySliceQuery(KeyValueStoreUtil.getBuffer(key), KeyValueStoreUtil.getBuffer(2002), KeyValueStoreUtil.getBuffer(2004)), tx).size());
long canonicalVertexId = idInspector.isPartitionedVertex(vertexId)?idManager.getCanonicalVertexId(vertexId):vertexId; if (verifyExistence) { if (graph.edgeQuery(canonicalVertexId, graph.vertexExistenceQuery, txHandle).isEmpty()) lifecycle = ElementLifeCycle.Removed;
@Override public void process(StaticBuffer key, Map<SliceQuery, EntryList> entries, ScanMetrics metrics) { long vertexId = getVertexId(key); assert entries.size()==1; assert entries.get(everythingQueryLimit)!=null; final EntryList everything = entries.get(everythingQueryLimit); if (!isGhostVertex(vertexId, everything)) { return; } if (everything.size()>=RELATION_COUNT_LIMIT) { metrics.incrementCustom(SKIPPED_GHOST_LIMIT_COUNT); return; } JanusGraphVertex vertex = tx.getInternalVertex(vertexId); Preconditions.checkArgument(vertex instanceof CacheVertex, "The bounding transaction is not configured correctly"); CacheVertex v = (CacheVertex)vertex; v.loadRelations(EVERYTHING_QUERY, input -> everything); int removedRelations = 0; Iterator<JanusGraphRelation> iterator = v.query().noPartitionRestriction().relations().iterator(); while (iterator.hasNext()) { iterator.next(); iterator.remove(); removedRelations++; } //There should be no more system relations to remove metrics.incrementCustom(REMOVED_VERTEX_COUNT); metrics.incrementCustom(REMOVED_RELATION_COUNT,removedRelations); }
@Override public void process(StaticBuffer key, Map<SliceQuery, EntryList> entries, ScanMetrics metrics) { long vertexId = getVertexId(key); assert entries.get(VERTEX_EXISTS_QUERY)!=null; if (isGhostVertex(vertexId, entries.get(VERTEX_EXISTS_QUERY))) { metrics.incrementCustom(GHOST_VERTEX_COUNT); return; } JanusGraphVertex vertex = tx.getInternalVertex(vertexId); Preconditions.checkArgument(vertex instanceof PreloadedVertex, "The bounding transaction is not configured correctly"); PreloadedVertex v = (PreloadedVertex)vertex; v.setAccessCheck(PreloadedVertex.OPENSTAR_CHECK); for (Map.Entry<SliceQuery,EntryList> entry : entries.entrySet()) { SliceQuery sq = entry.getKey(); if (sq.equals(VERTEX_EXISTS_QUERY)) continue; EntryList entryList = entry.getValue(); if (entryList.size()>=sq.getLimit()) metrics.incrementCustom(TRUNCATED_ENTRY_LISTS); v.addToQueryCache(sq.updateLimit(Query.NO_LIMIT),entryList); } job.process(v, metrics); }
assertEquals(10,cache.getSlice(getQuery(i,0,numCols+1).setLimit(10),tx).size()); assertEquals(3,cache.getSlice(getQuery(i,2,5),tx).size()); for (StaticBuffer key : keys) assertTrue(result.containsKey(key)); for (EntryList r : result.values()) { assertEquals(5,r.size()); assertEquals(numCols,cache.getSlice(new KeySliceQuery(key,getQuery(0,numCols+1)),tx).size()); Map<StaticBuffer,EntryList> result = cache.getSlice(keys,getQuery(2,8),tx); assertEquals(keys.size(),result.size()); assertEquals(6,result.get(key).size()); assertEquals(numCols/2,cache.getSlice(new KeySliceQuery(key,getQuery(0,numCols+1)),tx).size()); result = cache.getSlice(keys,getQuery(2,8),tx); assertEquals(keys.size(),result.size()); assertEquals(3,result.get(key).size()); tx.commit(); assertEquals(4,store.getSliceCalls());