@Test public void largeTest() throws Exception { int numDoc = 30000; String store = "vertex"; initialize(store); for (int i = 1; i <= numDoc; i++) { add(store, "doc" + i, getRandomDocument(), true); } clopen(); // List<String> result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(WEIGHT, Cmp.INTERVAL, Interval.of(0.2,0.3))))); // List<String> result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(LOCATION, Geo.WITHIN,Geoshape.circle(48.5,0.5,1000.00))))); long time = System.currentTimeMillis(); List<String> result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN_EQUAL, 0.2), PredicateCondition.of(WEIGHT, Cmp.LESS_THAN, 0.6), PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 1000.00))))); int oldresultSize = result.size(); System.out.println(result.size() + " vs " + (numDoc / 1000 * 2.4622623015)); System.out.println("Query time on " + numDoc + " docs (ms): " + (System.currentTimeMillis() - time)); result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN_EQUAL, 0.2), PredicateCondition.of(WEIGHT, Cmp.LESS_THAN, 0.6), PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 1000.00))), numDoc / 1000)); assertEquals(numDoc / 1000, result.size()); result = tx.query(new IndexQuery(store, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN_EQUAL, 0.2), PredicateCondition.of(WEIGHT, Cmp.LESS_THAN, 0.6), PredicateCondition.of(LOCATION, Geo.WITHIN, Geoshape.circle(48.5, 0.5, 1000.00))), numDoc / 1000 * 100)); assertEquals(oldresultSize, result.size()); }
@Test public void testLastWriteWins() throws Exception { runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, defTextValue, false); tx.add(defStore, defDoc, TEXT, "sugar sugar", false); } }, new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, defTextValue, false); tx.add(defStore, defDoc, TEXT, "honey honey", false); } }); //only last write should be visible checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),null); checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "sugar")),null); checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "honey")),defDoc); }
@Test public void testAddFieldThenDeleteDoc() throws Exception { final String nameValue = "jm keynes"; runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.add(defStore, defDoc, NAME, nameValue, false); } }, new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, ImmutableMap.of(), true); } }); //neither should be visible checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),null); checkResult(new IndexQuery(defStore, PredicateCondition.of(NAME, Cmp.EQUAL, nameValue)),null); }
@Test public void testConflictingAdd() throws Exception { final String doc2 = "docy2"; runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { Multimap<String, Object> initialProps = ImmutableMultimap.<String, Object>of(TEXT, "sugar sugar"); add(defStore, doc2, initialProps, true); } }, new TxJob() { @Override public void run(IndexTransaction tx) { Multimap<String, Object> initialProps = ImmutableMultimap.<String, Object>of(TEXT, "honey honey"); add(defStore, doc2, initialProps, true); } }); //only last write should be visible checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),defDoc); checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "sugar")),null); checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "honey")),doc2); }
@Test public void testDeleteDocumentThenAddField() throws Exception { final String nameValue = "jm keynes"; runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, ImmutableMap.of(), true); } }, new TxJob() { @Override public void run(IndexTransaction tx) { tx.add(defStore, defDoc, NAME, nameValue, false); } }); // TEXT field should have been deleted when document was checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),null); // but name field should be visible checkResult(new IndexQuery(defStore, PredicateCondition.of(NAME, Cmp.EQUAL, nameValue)),defDoc); }
/** * Test overwriting a single existing field on an existing document * (isNew=false). Non-contentious test. * */ @Test public void testUpdateAddition() throws Exception { final String revisedText = "its a sunny day"; runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.add(defStore, defDoc, TEXT, revisedText, false); } }, new TxJob() { @Override public void run(IndexTransaction tx) { //do nothing } }); // Should no longer return old text checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")), null); // but new one checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "sunny")),defDoc); }
@Test public void testTTL() throws Exception { if (!index.getFeatures().supportsDocumentTTL()) return; final String store = "store1"; initialize(store); // add couple of documents with weight > 4.0d add(store, "expiring-doc1", ImmutableMultimap.of(NAME, "first", TIME, 1L, WEIGHT, 10.2d), true, 2); add(store, "expiring-doc2", ImmutableMultimap.of(NAME, "second", TIME, 2L, WEIGHT, 4.7d), true); add(store, "expiring-doc3", ImmutableMultimap.of(NAME, "third", TIME, 3L, WEIGHT, 5.2d), true, 2); add(store, "expiring-doc4", ImmutableMultimap.of(NAME, "fourth", TIME, 3L, WEIGHT, 7.7d), true, 7);// bigger ttl then one recycle interval, should still show up in the results clopen(); // initial query Set<String> results = Sets.newHashSet(tx.query(new IndexQuery(store, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN_EQUAL, 4.0))))); assertEquals(4, results.size()); Thread.sleep(6000); // sleep for elastic search ttl recycle results = Sets.newHashSet(tx.query(new IndexQuery(store, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN_EQUAL, 4.0))))); assertEquals(2, results.size()); assertTrue(results.contains("expiring-doc2")); assertTrue(results.contains("expiring-doc4")); Thread.sleep(5000); // sleep for elastic search ttl recycle results = Sets.newHashSet(tx.query(new IndexQuery(store, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN_EQUAL, 4.0))))); assertEquals(1, results.size()); assertTrue(results.contains("expiring-doc2")); }
@Test public void testDeleteDocumentThenModifyField() throws Exception { runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, ImmutableMap.of(), true); } }, new TxJob() { @Override public void run(IndexTransaction tx) { tx.add(defStore, defDoc, TEXT, "the slow brown fox jumps over the lazy dog", false); } }); //2nd tx should put document back into existence checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),defDoc); }
@Test public void testDeleteDocumentThenDeleteField() throws Exception { runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, ImmutableMap.of(), true); } }, new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, defTextValue, false); } }); // Document must not exist checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),null); }
/** * Test deleting a single field from a single document (deleteAll=false). * Non-contentious test. * */ @Test public void testUpdateDeletion() throws Exception { runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, ImmutableMap.of(), false); } }, new TxJob() { @Override public void run(IndexTransaction tx) { //do nothing } }); // Should no longer return deleted text checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),null); }
Set<String> results = Sets.newHashSet(tx.query(new IndexQuery(store1, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN_EQUAL, 4.0))))); assertEquals(2, results.size()); results = Sets.newHashSet(tx.query(new IndexQuery(store1, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN_EQUAL, 4.0))))); assertEquals(1, results.size()); assertTrue(results.contains("restore-doc3")); results = Sets.newHashSet(tx.query(new IndexQuery(store1, And.of(PredicateCondition.of(NAME, Cmp.EQUAL, "third"), PredicateCondition.of(TIME, Cmp.EQUAL, 3L))))); assertEquals(1, results.size()); assertTrue(results.contains("restore-doc3")); results = Sets.newHashSet(tx.query(new IndexQuery(store1, And.of(PredicateCondition.of(NAME, Cmp.EQUAL, "not-second"), PredicateCondition.of(TIME, Cmp.EQUAL, 0L))))); assertEquals(1, results.size()); assertTrue(results.contains("restore-doc2")); results = Sets.newHashSet(tx.query(new IndexQuery(store1, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN_EQUAL, 4.0))))); assertEquals(2, results.size()); assertTrue(results.contains("restore-doc1")); results = Sets.newHashSet(tx.query(new IndexQuery(store1, And.of(PredicateCondition.of(NAME, Cmp.EQUAL, "first-restored"), PredicateCondition.of(TIME, Cmp.EQUAL, 4L))))); assertEquals(1, results.size()); assertTrue(results.contains("restore-doc1")); results = Sets.newHashSet(tx.query(new IndexQuery(store2, And.of(PredicateCondition.of(WEIGHT, Cmp.GREATER_THAN_EQUAL, 4.0))))); assertEquals(1, results.size()); assertTrue(results.contains("restore-doc1")); results = Sets.newHashSet(tx.query(new IndexQuery(store2, And.of(PredicateCondition.of(NAME, Cmp.EQUAL, "first-in-second-store"), PredicateCondition.of(TIME, Cmp.EQUAL, 5L))))); assertEquals(1, results.size());
private void runConflictingTx(TxJob job1, TxJob job2) throws Exception { initialize(defStore); Multimap<String, Object> initialProps = ImmutableMultimap.of(TEXT, defTextValue); add(defStore, defDoc, initialProps, true); clopen(); // Sanity check checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),defDoc); checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "periwinkle")),null); IndexTransaction tx1 = openTx(), tx2 = openTx(); job1.run(tx1); tx1.commit(); job2.run(tx2); tx2.commit(); clopen(); }
List<String> result = tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "world"))); assertEquals(ImmutableSet.of("doc1", "doc2"), ImmutableSet.copyOf(result)); assertEquals(ImmutableSet.copyOf(result), ImmutableSet.copyOf(tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "wOrLD"))))); assertEquals(1, tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "bob"))).size()); assertEquals(0, tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "worl"))).size()); assertEquals(1, tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "Tomorrow world"))).size()); assertEquals(1, tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "WorLD HELLO"))).size()); result = tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "world"), orderTimeDesc)); assertEquals(ImmutableList.of("doc2", "doc1"), result); result = tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "world"), orderWeightDesc)); assertEquals(ImmutableList.of("doc2", "doc1"), result); result = tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "world"), orderTimeAsc)); assertEquals(ImmutableList.of("doc1", "doc2"), result); result = tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "world"), orderWeightAsc)); assertEquals(ImmutableList.of("doc1", "doc2"), result); result = tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS, "world"), jointOrder)); assertEquals(ImmutableList.of("doc2", "doc1"), result); result = tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS_PREFIX, "w"))); assertEquals(ImmutableSet.of("doc1", "doc2"), ImmutableSet.copyOf(result)); result = tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS_PREFIX, "wOr"))); assertEquals(ImmutableSet.of("doc1", "doc2"), ImmutableSet.copyOf(result)); assertEquals(0,tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS_PREFIX, "bobi"))).size()); result = tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS_REGEX, "he[l]+(.*)"))); assertEquals(ImmutableSet.of("doc1", "doc3"), ImmutableSet.copyOf(result)); result = tx.query(new IndexQuery(store, PredicateCondition.of(TEXT, Text.CONTAINS_REGEX, "[h]+e[l]+(.*)"))); assertEquals(ImmutableSet.of("doc1", "doc3"), ImmutableSet.copyOf(result));
@Test public void testUnescapedDollarInSet() throws Exception { initialize("vertex"); Multimap<String, Object> initialDoc = HashMultimap.create(); initialDoc.put(PHONE_SET, "12345"); add("vertex", "unescaped", initialDoc, true); clopen(); Multimap<String, Object> updateDoc = HashMultimap.create(); updateDoc.put(PHONE_SET, "$123"); add("vertex", "unescaped", updateDoc, false); add("vertex", "other", getRandomDocument(), true); clopen(); assertEquals("unescaped", tx.query(new IndexQuery("vertex", PredicateCondition.of(PHONE_SET, Cmp.EQUAL, "$123"))).get(0)); assertEquals("unescaped", tx.query(new IndexQuery("vertex", PredicateCondition.of(PHONE_SET, Cmp.EQUAL, "12345"))).get(0)); } }
private void simpleWriteAndQuery(IndexProvider idx) throws BackendException, InterruptedException { final Duration maxWrite = Duration.ofMillis(2000L); final String storeName = "jvmlocal_test_store"; final KeyInformation.IndexRetriever indexRetriever = IndexProviderTest.getIndexRetriever(IndexProviderTest.getMapping(idx.getFeatures())); BaseTransactionConfig txConfig = StandardBaseTransactionConfig.of(TimestampProviders.MILLI); IndexTransaction itx = new IndexTransaction(idx, indexRetriever, txConfig, maxWrite); assertEquals(0, itx.query(new IndexQuery(storeName, PredicateCondition.of(IndexProviderTest.NAME, Text.PREFIX, "ali"))).size()); itx.add(storeName, "doc", IndexProviderTest.NAME, "alice", false); itx.commit(); Thread.sleep(1500L); // Slightly longer than default 1s index.refresh_interval itx = new IndexTransaction(idx, indexRetriever, txConfig, maxWrite); assertEquals(0, itx.query(new IndexQuery(storeName, PredicateCondition.of(IndexProviderTest.NAME, Text.PREFIX, "zed"))).size()); assertEquals(1, itx.query(new IndexQuery(storeName, PredicateCondition.of(IndexProviderTest.NAME, Text.PREFIX, "ali"))).size()); itx.rollback(); } }
@Test public void testDeleteDocumentThenAddField() throws Exception { final String nameValue = "jm keynes"; runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, ImmutableMap.of(), true); } }, new TxJob() { @Override public void run(IndexTransaction tx) { tx.add(defStore, defDoc, NAME, nameValue, false); } }); // TEXT field should have been deleted when document was checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),null); // but name field should be visible checkResult(new IndexQuery(defStore, PredicateCondition.of(NAME, Cmp.EQUAL, nameValue)),defDoc); }
@Test public void testAddFieldThenDeleteDoc() throws Exception { final String nameValue = "jm keynes"; runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.add(defStore, defDoc, NAME, nameValue, false); } }, new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, ImmutableMap.of(), true); } }); //neither should be visible checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),null); checkResult(new IndexQuery(defStore, PredicateCondition.of(NAME, Cmp.EQUAL, nameValue)),null); }
@Test public void testDeleteDocumentThenModifyField() throws Exception { runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, ImmutableMap.of(), true); } }, new TxJob() { @Override public void run(IndexTransaction tx) { tx.add(defStore, defDoc, TEXT, "the slow brown fox jumps over the lazy dog", false); } }); //2nd tx should put document back into existence checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),defDoc); }
@Test public void testDeleteDocumentThenDeleteField() throws Exception { runConflictingTx(new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, ImmutableMap.of(), true); } }, new TxJob() { @Override public void run(IndexTransaction tx) { tx.delete(defStore, defDoc, TEXT, defTextValue, false); } }); // Document must not exist checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),null); }
private void runConflictingTx(TxJob job1, TxJob job2) throws Exception { initialize(defStore); Map<String, Object> initialProps = ImmutableMap.<String, Object>of(TEXT, defTextValue); add(defStore, defDoc, initialProps, true); clopen(); // Sanity check checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "brown")),defDoc); checkResult(new IndexQuery(defStore, PredicateCondition.of(TEXT, Text.CONTAINS, "periwinkle")),null); IndexTransaction tx1 = openTx(), tx2 = openTx(); job1.run(tx1); tx1.commit(); job2.run(tx2); tx2.commit(); clopen(); }