@Override public void collect(HiveKey key, BytesWritable value) throws IOException { lastRecordOutput.add(SparkUtilities.copyHiveKey(key), SparkUtilities.copyBytesWritable(value)); }
@Override public void collect(HiveKey key, BytesWritable value) throws IOException { lastRecordOutput.add(SparkUtilities.copyHiveKey(key), SparkUtilities.copyBytesWritable(value)); }
/** Helper method which inserts numRecords and retrieves them from cache and verifies */ private void testSpillingHelper(HiveKVResultCache cache, int numRecords) { for(int i=0; i<numRecords; i++) { String key = "key_" + i; String value = "value_" + i; cache.add(new HiveKey(key.getBytes(), key.hashCode()), new BytesWritable(value.getBytes())); } int recordsSeen = 0; while(cache.hasNext()) { String key = "key_" + recordsSeen; String value = "value_" + recordsSeen; Tuple2<HiveKey, BytesWritable> row = cache.next(); assertTrue("Unexpected key at position: " + recordsSeen, new String(row._1().getBytes()).equals(key)); assertTrue("Unexpected value at position: " + recordsSeen, new String(row._2().getBytes()).equals(value)); recordsSeen++; } assertTrue("Retrieved record count doesn't match inserted record count", numRecords == recordsSeen); cache.clear(); }
@Test public void testSimple() throws Exception { // Create KV result cache object, add one (k,v) pair and retrieve them. HiveKVResultCache cache = new HiveKVResultCache(); HiveKey key = new HiveKey("key".getBytes(), "key".hashCode()); BytesWritable value = new BytesWritable("value".getBytes()); cache.add(key, value); assertTrue("KV result cache should have at least one element", cache.hasNext()); Tuple2<HiveKey, BytesWritable> row = cache.next(); assertTrue("Incorrect key", row._1().equals(key)); assertTrue("Incorrect value", row._2().equals(value)); assertTrue("Cache shouldn't have more records", !cache.hasNext()); }
@Override public void collect(HiveKey key, BytesWritable value) throws IOException { lastRecordOutput.add(SparkUtilities.copyHiveKey(key), SparkUtilities.copyBytesWritable(value)); }