public void testPutFinished() throws Exception { op = new ObjectPipe(); op.finish(); try { op.put(new Integer(1)); fail("Expected: IllegalArgumentException"); } catch (IllegalArgumentException e) { } }
public void testFinishFinished() throws Exception { op = new ObjectPipe(); op.finish(); try { op.finish(); fail("Expected: IllegalArgumentException"); } catch (IllegalArgumentException e) { } }
public void test() throws Exception { op = new ObjectPipe(); op.put(new Integer(1)); op.put(new Integer(2)); assertTrue(op.hasNext()); assertEquals(new Integer(1), op.next()); op.finish(); assertTrue(op.hasNext()); assertEquals(new Integer(2), op.next()); assertFalse(op.hasNext()); try { op.next(); fail("Expected: NoSuchElementException"); } catch (NoSuchElementException e) { } }
public void testMultiThreaded() throws Exception { op = new ObjectPipe(2); progress = 0; op.put(new Integer(1)); for (int i = 0; (i < 40) && (progress < 1); i++) { Thread.sleep(50); op.put(new Integer(2)); for (int i = 0; (i < 40) && (progress < 2); i++) { Thread.sleep(50); l.add(new Integer(5)); l.add(new Integer(6)); op.putAll(l); for (int i = 0; (i < 40) && (progress < 6); i++) { Thread.sleep(50); op.finish(); for (int i = 0; (i < 40) && (progress < 2000); i++) { Thread.sleep(50);
public void testMultiThreaded2() throws Exception { op = new ObjectPipe(2); progress = 0; assertTrue(op.hasNext()); assertEquals(new Integer(1), op.next()); Thread.sleep(200); assertEquals(4, progress); assertTrue(op.hasNext()); assertEquals(new Integer(2), op.next()); Thread.sleep(200); assertEquals(4, progress); assertTrue(op.hasNext()); assertEquals(new Integer(3), op.next()); for (int i = 0; (i < 40) && (progress < 5); i++) { Thread.sleep(50); assertTrue(op.hasNext()); assertEquals(new Integer(4), op.next()); for (int i = 0; (i < 40) && (progress < 2000); i++) { Thread.sleep(50); assertTrue(op.hasNext()); assertEquals(new Integer(5), op.next()); assertTrue(op.hasNext()); assertEquals(new Integer(6), op.next()); assertFalse(op.hasNext());
/** * {@inheritDoc} */ @Override public void close(Source source) { jobs.finish(); LOG.info("Parallel Batching equivalent object query summary for source " + source + " :" + getSummary(source).toString() + "\nQueried " + batchQueried + " objects by batch, cache misses: " + cacheMisses); }
private int iterateOverObjects(long time, long objectParseTime, HashSet<Class<? extends InterMineObject>> seenClasses, HashMap<String, InterMineResultsContainer> referenceResults, Results results, ListIterator<ResultsRow<InterMineObject>> it) throws PathException, ObjectStoreException, IllegalAccessException { int i = 0; int size = results.size(); LOG.info("Query returned " + size + " results"); //iterate over objects while (it.hasNext()) { ResultsRow<InterMineObject> row = it.next(); if (i % 10000 == 1) { LOG.info("IMOFetcher: fetched " + i + " of " + size + " in " + (System.currentTimeMillis() - time) + "ms total, " + (objectParseTime) + "ms spent on parsing"); } for (InterMineObject object : row) { SolrInputDocument doc = handleObject(object, seenClasses, referenceResults); // finally add doc to queue indexingQueue.put(doc); } i++; } return i; }
public void run() { try { while (true) { WorkUnit u = jobs.next(); u.fetch(); } } catch (NoSuchElementException e) { // Fetcher has been closed } } }
public void testMaxBufferTooSmall() throws Exception { try { op = new ObjectPipe(0); fail("Expected: IllegalArgumentException"); } catch (IllegalArgumentException e) { } } }
/** * {@inheritDoc} */ @Override public void close(Source source) { jobs.finish(); LOG.info("Parallel Batching equivalent object query summary for source " + source + " :" + getSummary(source).toString() + "\nQueried " + batchQueried + " objects by batch, cache misses: " + cacheMisses); }
private int iterateOverObjects(long time, long objectParseTime, HashSet<Class<? extends InterMineObject>> seenClasses, HashMap<String, InterMineResultsContainer> referenceResults, Results results, ListIterator<ResultsRow<InterMineObject>> it) throws PathException, ObjectStoreException, IllegalAccessException { int i = 0; int size = results.size(); LOG.info("Query returned " + size + " results"); //iterate over objects while (it.hasNext()) { ResultsRow<InterMineObject> row = it.next(); if (i % 10000 == 1) { LOG.info("IMOFetcher: fetched " + i + " of " + size + " in " + (System.currentTimeMillis() - time) + "ms total, " + (objectParseTime) + "ms spent on parsing"); } for (InterMineObject object : row) { SolrInputDocument doc = handleObject(object, seenClasses, referenceResults); // finally add doc to queue indexingQueue.put(doc); } i++; } return i; }
public void run() { try { while (true) { WorkUnit u = jobs.next(); u.fetch(); } } catch (NoSuchElementException e) { // Fetcher has been closed } } }
indexingQueue.finish();
ClassDescriptor cld = pksToDo.get(pk); if (canDoPkNow(pk, cld, pksNotDone)) { jobs.put(new WorkUnit(pk, cld, results, cldToObjectsForCld.get(cld), pksNotDone, exceptions)); pkIter.remove();
indexingQueue.finish();
ClassDescriptor cld = pksToDo.get(pk); if (canDoPkNow(pk, cld, pksNotDone)) { jobs.put(new WorkUnit(pk, cld, results, cldToObjectsForCld.get(cld), pksNotDone, exceptions)); pkIter.remove();