@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<RowKeyWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<RowKeyWritable, KeyValue> p1 = result.get(0); Pair<RowKeyWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("usd_amt", new String(p1.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("35.43", new String(p1.getSecond().getValue(), StandardCharsets.UTF_8)); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("item_count", new String(p2.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("2", new String(p2.getSecond().getValue(), StandardCharsets.UTF_8)); } }
@SuppressWarnings("unchecked") @Test public void testMapperWithoutHeader() throws IOException { Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey3 = new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey4 = new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey5 = new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey6 = new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey7 = new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); mapDriver.addInput(inputKey1, new Text("abc")); mapDriver.addInput(inputKey2, new Text("abc")); mapDriver.addInput(inputKey3, new Text("abc")); mapDriver.addInput(inputKey4, new Text("abc")); mapDriver.addInput(inputKey5, new Text("abc")); mapDriver.addInput(inputKey6, new Text("abc")); mapDriver.addInput(inputKey7, new Text("abc")); List<Pair<Text, LongWritable>> result = mapDriver.run(); assertEquals(1, result.size()); byte[] key1 = result.get(0).getFirst().getBytes(); LongWritable value1 = result.get(0).getSecond(); assertArrayEquals(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1); assertEquals(147, value1.get()); }
@SuppressWarnings("unchecked") @Test public void testMapperWithHeader() throws IOException { Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey3 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey4 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey5 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey6 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); Text inputKey7 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); mapDriver.addInput(inputKey1, new Text("abc")); mapDriver.addInput(inputKey2, new Text("abc")); mapDriver.addInput(inputKey3, new Text("abc")); mapDriver.addInput(inputKey4, new Text("abc")); mapDriver.addInput(inputKey5, new Text("abc")); mapDriver.addInput(inputKey6, new Text("abc")); mapDriver.addInput(inputKey7, new Text("abc")); List<Pair<Text, LongWritable>> result = mapDriver.run(); assertEquals(1, result.size()); byte[] key1 = result.get(0).getFirst().getBytes(); LongWritable value1 = result.get(0).getSecond(); assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1); assertEquals(273, value1.get()); }
@SuppressWarnings("unchecked") @Test public void testMapperOnComma() throws IOException { mapDriver.clearInput(); LongWritable inputKey1 = new LongWritable(1); LongWritable inputKey2 = new LongWritable(2); LongWritable inputKey3 = new LongWritable(3); LongWritable inputKey4 = new LongWritable(4); LongWritable inputKey5 = new LongWritable(5); LongWritable inputKey6 = new LongWritable(6); LongWritable inputKey7 = new LongWritable(7); mapDriver.addInput(inputKey1, new Text()); mapDriver.addInput(inputKey2, new Text(strArr)); mapDriver.addInput(inputKey3, new Text(strArr)); mapDriver.addInput(inputKey4, new Text(strArr)); mapDriver.addInput(inputKey5, new Text(strArr)); mapDriver.addInput(inputKey6, new Text(strArr)); mapDriver.addInput(inputKey7, new Text(strArr)); List<Pair<IntWritable, BytesWritable>> result = mapDriver.run(); assertEquals(9, result.size()); int key1 = result.get(0).getFirst().get(); BytesWritable value1 = result.get(0).getSecond(); byte[] bytes = value1.getBytes(); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(); hllc.readRegisters(ByteBuffer.wrap(bytes)); System.out.println("ab\177ab".length()); assertTrue(key1 > 0); assertEquals(1, hllc.getCountEstimate()); }
@Test public void test() throws IOException { List<Text> data = new ArrayList<Text>(); for (int i = 0; i < 1001; i++) { data.add(new Text(String.valueOf(i))); } for (Text t : data) { mapDriver.addInput(t, new Text("abc")); } mapDriver.getConfiguration().set(BatchConstants.MAPPER_SAMPLE_NUMBER, "100"); List<Pair<Text, LongWritable>> result = mapDriver.run(); assertEquals(100, result.size()); for (Pair<Text, LongWritable> p : result) { System.out.println(p.getFirst()); } }
@SuppressWarnings({ "unchecked" }) @Test @Ignore public void testMapperOn177() throws IOException { mapDriver.clearInput(); File file = new File("src/test/resources/data/test_cal_dt/part-r-00000"); FileReader reader = new FileReader(file); BufferedReader breader = new BufferedReader(reader); String s = breader.readLine(); int i = 0; while (s != null) { LongWritable inputKey = new LongWritable(i++); mapDriver.addInput(inputKey, new Text(s)); s = breader.readLine(); } // breader.close(); List<Pair<IntWritable, BytesWritable>> result = mapDriver.run(); breader.close(); assertEquals(9, result.size()); int key1 = result.get(0).getFirst().get(); BytesWritable value1 = result.get(0).getSecond(); byte[] bytes = value1.getBytes(); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(); hllc.readRegisters(ByteBuffer.wrap(bytes)); assertTrue(key1 > 0); assertEquals(8, hllc.getCountEstimate()); }
@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0); Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily())); assertEquals("usd_amt", new String(p1.getSecond().getQualifier())); assertEquals("35.43", new String(p1.getSecond().getValue())); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily())); assertEquals("item_count", new String(p2.getSecond().getQualifier())); assertEquals("2", new String(p2.getSecond().getValue())); } }
/** * Generates data for use in tests * * @param driver * Driver * @param num * Number of tuples to generate */ protected void generateData(MapDriver<LongWritable, T, NodeWritable, T> driver, int num) { for (int i = 0; i < num; i++) { LongWritable inputKey = new LongWritable(i); T value = this.createValue(i); NodeWritable outputKey = this.getOutputKey(value); driver.addInput(inputKey, value); driver.addOutput(outputKey, value); } }
/** * Generates tuples for the tests * * @param driver * Driver * @param num * Number of tuples to generate */ protected void generateData(MapDriver<LongWritable, T, NodeWritable, LongWritable> driver, int num) { LongWritable expectedCount = new LongWritable(1); for (int i = 0; i < num; i++) { LongWritable key = new LongWritable(i); T value = this.createValue(i); NodeWritable[] nodes = this.getNodes(value); driver.addInput(key, value); for (NodeWritable n : nodes) { driver.addOutput(n, expectedCount); } } }
/** * Generates data for use in tests * * @param driver * Driver * @param num * Number of tuples to generate */ protected void generateData(MapDriver<LongWritable, T, T, NodeWritable> driver, int num) { for (int i = 0; i < num; i++) { LongWritable key = new LongWritable(i); T value = this.createValue(i); NodeWritable[] nodes = this.getNodes(value); driver.addInput(key, value); for (NodeWritable n : nodes) { driver.addOutput(value, n); } } }
/** * Generates data for use in tests * * @param driver * Driver * @param num * Number of tuples to generate */ protected void generateData(MapDriver<LongWritable, T, LongWritable, NodeWritable> driver, int num) { for (int i = 0; i < num; i++) { LongWritable key = new LongWritable(i); T value = this.createValue(i); NodeWritable[] nodes = this.getNodes(value); driver.addInput(key, value); for (NodeWritable n : nodes) { driver.addOutput(key, n); } } }
protected final void generateData(MapDriver<LongWritable, T, LongWritable, T> driver, int num) { for (int i = 0; i < num; i++) { LongWritable key = new LongWritable(i); if (i % 2 == 0 && !this.noValidInputs()) { T value = this.createValidValue(i); driver.addInput(key, value); if (!this.isInverted()) driver.addOutput(key, value); } else { T value = this.createInvalidValue(i); driver.addInput(key, value); if (this.isInverted()) driver.addOutput(key, value); } } }
protected void generateData(MapDriver<LongWritable, TripleWritable, LongWritable, QuadWritable> driver, int num) { for (int i = 0; i < num; i++) { Triple t = new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"), NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)); Quad q = new Quad(Quad.defaultGraphNodeGenerated, t); driver.addInput(new LongWritable(i), new TripleWritable(t)); driver.addOutput(new LongWritable(i), new QuadWritable(q)); } }
protected void generateData(MapDriver<LongWritable, TripleWritable, LongWritable, QuadWritable> driver, int num) { for (int i = 0; i < num; i++) { Triple t = new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"), NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)); Quad q = new Quad(t.getSubject(), t); driver.addInput(new LongWritable(i), new TripleWritable(t)); driver.addOutput(new LongWritable(i), new QuadWritable(q)); } }
protected void generateData(MapDriver<LongWritable, QuadWritable, LongWritable, TripleWritable> driver, int num) { for (int i = 0; i < num; i++) { Triple t = new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"), NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)); Quad q = new Quad(Quad.defaultGraphNodeGenerated, t); driver.addInput(new LongWritable(i), new QuadWritable(q)); driver.addOutput(new LongWritable(i), new TripleWritable(t)); } }
@Test public void testHypercubeMapper() throws IOException { MapDriver<Writable, VectorWritable, IntWritable, CentroidWritable> mapDriver = MapDriver.newMapDriver(new StreamingKMeansMapper()); configure(mapDriver.getConfiguration()); System.out.printf("%s mapper test\n", mapDriver.getConfiguration().get(StreamingKMeansDriver.SEARCHER_CLASS_OPTION)); for (Centroid datapoint : syntheticData.getFirst()) { mapDriver.addInput(new IntWritable(0), new VectorWritable(datapoint)); } List<org.apache.hadoop.mrunit.types.Pair<IntWritable,CentroidWritable>> results = mapDriver.run(); BruteSearch resultSearcher = new BruteSearch(new SquaredEuclideanDistanceMeasure()); for (org.apache.hadoop.mrunit.types.Pair<IntWritable, CentroidWritable> result : results) { resultSearcher.add(result.getSecond().getCentroid()); } System.out.printf("Clustered the data into %d clusters\n", results.size()); for (Vector mean : syntheticData.getSecond()) { WeightedThing<Vector> closest = resultSearcher.search(mean, 1).get(0); assertTrue("Weight " + closest.getWeight() + " not less than 0.5", closest.getWeight() < 0.5); } }
mapDriver.addInput(new IntWritable(0), new VectorWritable(datapoint));