private int queryResultSetSize(String query) { return collector.getRecords().size(); }
@Test public void testSampleWithoutSeed() throws Exception { morphline = createMorphline("test-morphlines/sampleWithoutSeed"); int size = 100; for (int i = 0; i < size; i++) { assertTrue(morphline.process(new Record())); } int actual = collector.getRecords().size(); double delta = 0.2; assertTrue("actual: " + actual, actual >= Math.round(size * (0.9 - delta))); assertTrue("actual: " + actual, actual <= Math.round(size * (0.9 + delta))); }
@Test public void testHead() throws Exception { morphline = createMorphline("test-morphlines/head"); int size = 5; for (int i = 0; i < size; i++) { assertTrue(morphline.process(new Record())); } assertEquals(3, collector.getRecords().size()); }
@Test public void testHeadUnlimited() throws Exception { morphline = createMorphline("test-morphlines/headUnlimited"); int size = 5; for (int i = 0; i < size; i++) { assertTrue(morphline.process(new Record())); } assertEquals(size, collector.getRecords().size()); }
private void processAndVerifySuccess(Record input, Multimap... expectedMaps) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); Iterator<Record> iter = collector.getRecords().iterator(); int i = 0; for (Multimap expected : expectedMaps) { //System.out.println("iter" + i); assertTrue(iter.hasNext()); assertEquals(expected, iter.next().getFields()); i++; } assertFalse(iter.hasNext()); }
private void processAndVerifySuccess(Record input, Multimap... expectedMaps) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); Iterator<Record> iter = collector.getRecords().iterator(); for (Multimap expected : expectedMaps) { assertTrue(iter.hasNext()); Record record = iter.next(); assertEquals(expected, record.getFields()); } assertFalse(iter.hasNext()); }
@Test public void testSampleWithSeed() throws Exception { morphline = createMorphline("test-morphlines/sampleWithSeed"); int size = 10; for (int i = 0; i < size; i++) { assertTrue(morphline.process(new Record())); } assertEquals(9, collector.getRecords().size()); }
@Test public void testSampleWithProbabilityOne() throws Exception { morphline = createMorphline("test-morphlines/sampleWithProbabilityOne"); int size = 10; for (int i = 0; i < size; i++) { assertTrue(morphline.process(new Record())); } assertEquals(size, collector.getRecords().size()); }
private void processAndVerifySuccess(Record input, Multimap... expectedMaps) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); Iterator<Record> iter = collector.getRecords().iterator(); int i = 0; for (Multimap expected : expectedMaps) { //System.out.println("iter" + i); assertTrue(iter.hasNext()); assertEquals(expected, iter.next().getFields()); i++; } assertFalse(iter.hasNext()); }
private void processAndVerifySuccess(Command myMorphline, Collector myCollector, Record input, Multimap... expectedMaps) { myCollector.reset(); Notifications.notifyStartSession(myMorphline); assertEquals(1, myCollector.getNumStartEvents()); assertTrue(myMorphline.process(input)); Iterator<Record> iter = myCollector.getRecords().iterator(); int i = 0; for (Multimap expected : expectedMaps) { //System.out.println("iter" + i); assertTrue(iter.hasNext()); assertEquals(expected, iter.next().getFields()); i++; } assertFalse(iter.hasNext()); }
private void processAndVerifyFailure(Record input) { collector.reset(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertFalse(morphline.process(input)); assertEquals(0, collector.getRecords().size()); }
@Test public void testTryRulesCatchException() throws Exception { morphline = createMorphline("test-morphlines/tryRulesCatchException"); Record record = new Record(); record.put("first_name", "Nadja"); List<Record> expectedList = Lists.newArrayList(); for (int i = 0; i < 2; i++) { Record expected = record.copy(); expected.put("foo2", "bar2"); expected.replaceValues("iter2", i); expectedList.add(expected); } startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertEquals(expectedList, collector.getRecords()); assertNotSame(record, collector.getRecords().get(0)); }
@Test public void testNotWithTrue() throws Exception { morphline = createMorphline("test-morphlines/notWithTrue"); Record record = createBasicRecord(); startSession(); assertEquals(1, collector.getNumStartEvents()); assertFalse(morphline.process(record)); assertEquals(Lists.newArrayList(), collector.getRecords()); }
@Test public void testJavaRuntimeException() throws Exception { morphline = createMorphline("test-morphlines/javaRuntimeException"); Record record = new Record(); startSession(); assertEquals(1, collector.getNumStartEvents()); try { morphline.process(record); fail(); } catch (MorphlineRuntimeException e) { assertTrue(e.getMessage().startsWith("Cannot execute script")); } assertEquals(Lists.newArrayList(), collector.getRecords()); }
/** * Test that Solr queries on a parsed SequenceFile document * return the expected content and fields. */ @Test public void testSequenceFileContentCustomParsers() throws Exception { morphline = createMorphline("test-morphlines/sequenceFileMorphlineSimple"); String path = RESOURCES_DIR; File sequenceFile = new File(path, "testSequenceFileContentCustomParsers.seq"); int numRecords = 10; HashMap<String, Record> expected = createTextSequenceFile(sequenceFile, numRecords); InputStream in = new FileInputStream(sequenceFile.getAbsolutePath()); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, in); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(record)); assertTrue(areFieldsEqual(expected, collector.getRecords())); }
@Test public void testExtractAvroPathsArrayInUnion() throws Exception { List<String> items = Arrays.asList("a", "b", "c"); ArrayInUnionTestRecord avroRecord = new ArrayInUnionTestRecord(items, items); morphline = createMorphline("test-morphlines/extractAvroPathsArrayInUnion"); deleteAllDocuments(); Record record = new Record(); record.put(Fields.ATTACHMENT_BODY, avroRecord); startSession(); assertTrue(morphline.process(record)); assertEquals(1, collector.getRecords().size()); assertEquals(Arrays.asList(items), collector.getFirstRecord().get("/items[]")); assertEquals(Arrays.asList(items), collector.getFirstRecord().get("/itemsInUnion[]")); }
@Test public void testRCFileColumnWise() throws Exception { morphline = createMorphline("test-morphlines/rcFileMorphlineColumn"); String rcFileName = "testRCFileColumnWise.rc"; List<Record> expected = setupRCFile(rcFileName, NUM_RECORDS, NUM_COLUMNS, false); Path inputFile = dfs.makeQualified(new Path(testDirectory, rcFileName)); Record input = new Record(); input.put(Fields.ATTACHMENT_NAME, inputFile.toString()); input.put(Fields.ATTACHMENT_BODY, readPath(inputFile)); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertTrue(areFieldsEqual(expected, collector.getRecords(), NUM_COLUMNS, false)); }
@Test public void testRCFileWithNull() throws Exception { morphline = createMorphline("test-morphlines/rcFileMorphlineRow"); String rcFileName = "testRCFileRowWise.rc"; List<Record> expected = setupRCFile(rcFileName, NUM_RECORDS, NUM_COLUMNS, true, true); Path inputFile = dfs.makeQualified(new Path(testDirectory, rcFileName)); Record input = new Record(); input.put(Fields.ATTACHMENT_NAME, inputFile.toString()); input.put(Fields.ATTACHMENT_BODY, readPath(inputFile)); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertTrue( areFieldsEqual(expected, collector.getRecords(), NUM_COLUMNS, true)); }
@Test public void testRCFileRowWise() throws Exception { morphline = createMorphline("test-morphlines/rcFileMorphlineRow"); String rcFileName = "testRCFileRowWise.rc"; List<Record> expected = setupRCFile(rcFileName, NUM_RECORDS, NUM_COLUMNS, true); Path inputFile = dfs.makeQualified(new Path(testDirectory, rcFileName)); Record input = new Record(); input.put(Fields.ATTACHMENT_NAME, inputFile.toString()); input.put(Fields.ATTACHMENT_BODY, readPath(inputFile)); startSession(); assertEquals(1, collector.getNumStartEvents()); assertTrue(morphline.process(input)); assertTrue( areFieldsEqual(expected, collector.getRecords(), NUM_COLUMNS, true)); }
@Test public void testTryRulesFailTwice() throws Exception { morphline = createMorphline("test-morphlines/tryRulesFailTwice"); Record record = new Record(); record.put("first_name", "Nadja"); List<Record> expectedList = Lists.newArrayList(); startSession(); assertEquals(1, collector.getNumStartEvents()); try { morphline.process(record); fail(); } catch (MorphlineRuntimeException e) { assertTrue(e.getMessage().startsWith("tryRules command found no successful rule for record")); } assertEquals(expectedList, collector.getRecords()); }