@Override public void write(K key, T value) throws IOException, InterruptedException { try{ store.put(key, (Persistent) value); counter.increment(); if (counter.isModulo()) { LOG.info("Flushing the datastore after " + counter.getRecordsNumber() + " records"); store.flush(); } }catch(Exception e){ LOG.warn("Exception at GoraRecordWriter.class while writing to datastore: {}", e.getMessage()); LOG.warn("Trace: {}", e.getStackTrace()); throw new RuntimeException(e); } } }
/** * <p> * If the {@link org.apache.gora.store.DataStore} instance is not null, we * execute a {@link org.apache.gora.store.DataStore#flush()}. This forces * the write caches to be flushed. DataStore implementations may optimize * their writing by deferring the actual put / delete operations until * this moment. * </p> * <p>Otherwise, we utilize {@link org.apache.gora.store.DataStore#close()} * which closes the DataStore. This should release any resources held by * the implementation, so that the instance is ready for GC. All other * DataStore methods cannot be used after this method was called. * Subsequent calls of this method are ignored. * </p> * @see org.apache.hadoop.chukwa.datacollection.writer.ChukwaWriter#close() */ @Override public void close() throws WriterException { if (chunkStore != null) { chunkStore.flush(); } else { chunkStore.close(); } log.debug("Gora datastore successfully closed."); }
@Override public T load(K key) throws CacheLoaderException { T persistent = null; try { persistent = dataStore.get(key); LOG.info("Loaded data bean from persistent datastore on key {}.", key.toString()); } catch (CacheLoaderException ex) { throw ex; } return persistent; }
/** * Sets the output parameters for the conf that Spark will use * * @param job the job to set the properties for * @param dataStore the datastore as the output * @return a populated output {@link org.apache.hadoop.conf.Configuration} */ public <K, V extends Persistent> Configuration generateOutputConf(Job job, DataStore<K, V> dataStore) { return generateOutputConf(job, dataStore.getClass(), dataStore.getKeyClass(), dataStore.getPersistentClass()); }
public static void testGetEmployee(DataStore<String, Employee> dataStore) throws Exception { dataStore.createSchema(); Employee employee = DataStoreTestUtil.createEmployee(); String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); }
public static void testGetEmployeeRecursive(DataStore<String, Employee> dataStore) throws Exception { Employee employee = DataStoreTestUtil.createEmployee(); Employee boss = DataStoreTestUtil.createBoss(); employee.setBoss(boss); String ssn = employee.getSsn().toString(); dataStore.put(ssn, employee); dataStore.flush(); Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$)); assertEqualEmployeeObjects(employee, after); }
query = this.webPageStore.newQuery(); query.setFields("outlinks" , "parsedContent", "content"); assertNumResults(this.webPageStore.newQuery(), URLS.length); this.webPageStore.deleteByQuery(query); this.webPageStore.flush(); assertNumResults(this.webPageStore.newQuery(), URLS.length); WebPage page = this.webPageStore.get(SORTED_URL); assertNotNull(page); query = this.webPageStore.newQuery(); query.setFields("url"); String startKey = SORTED_URLS[NUM_KEYS]; query.setEndKey(endKey); assertNumResults(this.webPageStore.newQuery(), URLS.length); this.webPageStore.deleteByQuery(query); this.webPageStore.flush();
public static void testDelete(DataStore<String, WebPage> store) throws Exception { WebPageDataCreator.createWebPageData(store); //delete one by one int deletedSoFar = 0; for(String url : URLS) { assertTrue(store.delete(url)); store.flush(); //assert that it is actually deleted assertNull(store.get(url)); //assert that other records are not deleted assertNumResults(store.newQuery(), URLS.length - ++deletedSoFar); } }
public static byte[] testPutBytes(DataStore<String, WebPage> store) throws Exception { store.createSchema(); WebPage page = WebPage.newBuilder().build(); page.setUrl(new Utf8("http://example.com")); byte[] contentBytes = "example content in example.com".getBytes(Charset.defaultCharset()); ByteBuffer buff = ByteBuffer.wrap(contentBytes); page.setContent(buff); store.put("com.example/http", page); store.close(); return contentBytes; }
query = store.newQuery(); assertNumResults(store.newQuery(), URLS.length); store.deleteByQuery(query); store.flush(); assertEmptyResults(store.newQuery()); query = store.newQuery(); query.setFields(AvroUtils.getSchemaFieldNames(WebPage.SCHEMA$)); assertNumResults(store.newQuery(), URLS.length); store.deleteByQuery(query); store.flush(); assertEmptyResults(store.newQuery()); query = store.newQuery(); query.setKeyRange("a", "z"); //all start with "http://" assertNumResults(store.newQuery(), URLS.length); store.deleteByQuery(query); store.flush(); assertEmptyResults(store.newQuery()); query = store.newQuery(); query.setEndKey(SORTED_URLS[NUM_KEYS]); assertNumResults(store.newQuery(), URLS.length); store.deleteByQuery(query); store.flush();
@Test public void testPutGet() throws Exception { String key = "org.apache.gora:http:/"; DataStore<String, WebPage> store = new MemStore<>(); assumeTrue(store.get(key, new String[0]) == null); store.put(key, WebPage.newBuilder().build()); assertNotNull(store.get(key, new String[0])); store.close(); }
/** * Initializes the Mapper, and sets input parameters for the job * * @param job the job to set the properties for * @param dataStore the datastore as the input * @param outKeyClass Map output key class * @param outValueClass Map output value class * @param mapperClass the mapper class extending GoraMapper * @param reuseObjects whether to reuse objects in serialization * @param <K1> Map input key class * @param <V1> Map input value class * @param <K2> Map output key class * @param <V2> Map output value class * @throws IOException if there is an error initializing the Map job */ @SuppressWarnings({ "rawtypes" }) public static <K1, V1 extends Persistent, K2, V2> void initMapperJob( Job job, DataStore<K1,V1> dataStore, Class<K2> outKeyClass, Class<V2> outValueClass, Class<? extends GoraMapper> mapperClass, boolean reuseObjects) throws IOException { initMapperJob(job, dataStore.newQuery(), outKeyClass, outValueClass, mapperClass, reuseObjects); }
protected T getOrCreatePersistent(T persistent) throws Exception { if(persistent != null) { return persistent; } return dataStore.newPersistent(); }
AvroSerializer(CassandraClient cassandraClient, DataStore<K, T> dataStore, CassandraMapping mapping) { super(cassandraClient, dataStore.getKeyClass(), dataStore.getPersistentClass(), mapping); if (PersistentBase.class.isAssignableFrom(dataStore.getPersistentClass())) { persistentSchema = ((PersistentBase) dataStore.getBeanFactory().getCachedPersistent()).getSchema(); } else { persistentSchema = null; } this.cassandraDataStore = dataStore; try { analyzePersistent(); } catch (Exception e) { throw new RuntimeException("Error occurred while analyzing the persistent class, :" + e.getMessage()); } }
/** Stores the pageview object with the given key */ private void storePageview(long key, Pageview pageview) throws Exception { log.info("Storing Pageview in: " + dataStore.toString()); dataStore.put(key, pageview); }
public static void testDeleteSchema(DataStore<String, WebPage> dataStore) throws Exception { dataStore.createSchema(); WebPageDataCreator.createWebPageData(dataStore); dataStore.deleteSchema(); dataStore.createSchema(); assertEmptyResults(dataStore.newQuery()); }
/** This method illustrates delete by query call */ private void deleteByQuery(long startKey, long endKey) throws Exception { //Constructs a query from the dataStore. The matching rows to this query will be deleted Query<Long, Pageview> query = dataStore.newQuery(); //set the properties of query query.setStartKey(startKey); query.setEndKey(endKey); dataStore.deleteByQuery(query); log.info("pageviews with keys between {} and {} are deleted.", startKey, endKey); }