private void createDefaultUriProperties() { config = new LensConf(); config.addProperty(PATH_PREFIX + "hdfs", "://"); config.addProperty(PATH_PREFIX + "s3", "://"); config.addProperty(PATH_PREFIX + "s3n", "://"); }
private void createDefaultUriProperties() { config = new LensConf(); config.addProperty(PATH_PREFIX + "hdfs", "://"); config.addProperty(PATH_PREFIX + "s3", "://"); config.addProperty(PATH_PREFIX + "s3n", "://"); }
public LensConf getSessionConf() { LensConf conf = new LensConf(); Iterator<Map.Entry<String, String>> itr = this.conf.iterator(); while (itr.hasNext()) { Map.Entry<String, String> entry = itr.next(); conf.addProperty(entry.getKey(), entry.getValue()); } for (Map.Entry<String, String> entry : lensConfs.entrySet()) { conf.addProperty(entry.getKey(), entry.getValue()); } for (Map.Entry<String, String> entry : sessionVars.entrySet()) { conf.addProperty(entry.getKey(), entry.getValue()); } for (Map.Entry<String, String> entry : lensVars.entrySet()) { conf.addProperty(entry.getKey(), entry.getValue()); } return conf; }
private LensConf getLensConf(XJob job, SchedulerJobInstanceHandle instanceHandle, DateTime scheduledTime) { List<MapType> jobConf = job.getExecution().getQuery().getConf(); LensConf queryConf = new LensConf(); for (MapType element : jobConf) { queryConf.addProperty(element.getKey(), element.getValue()); } queryConf.addProperty(JOB_INSTANCE_ID_KEY, instanceHandle.getHandleId()); // Current time is used for resolving date. queryConf.addProperty(LensConfConstants.QUERY_CURRENT_TIME_IN_MILLIS, scheduledTime.getMillis()); return queryConf; }
private QueryHandle launchInmemoryQuery(MediaType mt) { LensConf conf = LensServerAPITestUtil.getLensConf(QUERY_METRIC_UNIQUE_ID_CONF_KEY, UUID.randomUUID()); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "false"); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); conf.addProperty(LensConfConstants.INMEMORY_RESULT_SET_TTL_SECS, 600); return RestAPITestUtil.executeAndGetHandle(target(), Optional.of(lensSessionId), Optional.of("select ID from " + TEST_TABLE), Optional.of(conf), mt); }
public static LensConf getLensConf(Object... args) { assert (args.length % 2 == 0); LensConf conf = new LensConf(); for (int i = 0; i < args.length; i += 2) { conf.addProperty(args[i], args[i + 1]); } return conf; } }
/** * Test persistent result with max size. * * @throws InterruptedException the interrupted exception * @throws IOException Signals that an I/O exception has occurred. */ @Test(dataProvider = "mediaTypeData") public void testPersistentResultWithMaxSize(MediaType mt) throws InterruptedException, IOException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true"); conf.addProperty(LensConfConstants.RESULT_FORMAT_SIZE_THRESHOLD, "1"); testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, true, null, mt); }
/** * Test result formatter failure. * * @throws InterruptedException the interrupted exception * @throws IOException Signals that an I/O exception has occurred. */ @Test(dataProvider = "mediaTypeData") public void testResultFormatterFailure(MediaType mt) throws InterruptedException, IOException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); conf.addProperty(LensConfConstants.QUERY_OUTPUT_SERDE, "NonexistentSerde.class"); testResultFormatter(conf, QueryStatus.Status.FAILED, false, null, mt); }
/** * Test result formatter in memory result. * * @throws InterruptedException the interrupted exception * @throws IOException Signals that an I/O exception has occurred. */ @Test(dataProvider = "mediaTypeData") public void testResultFormatterInMemoryResult(MediaType mt) throws InterruptedException, IOException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); conf.addProperty(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName()); testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, null, mt); queryService.conf.set(LensConfConstants.RESULT_FS_READ_URL, "filereadurl://"); testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, "filereadurl://", mt); queryService.conf.unset(LensConfConstants.RESULT_FS_READ_URL); }
/** * Creates the add/delete resource query. * * @param command * @param sessionHandle * @param driver * @return * @throws LensException */ private QueryContext createResourceQuery(String command, LensSessionHandle sessionHandle, LensDriver driver) throws LensException { LensConf qconf = new LensConf(); qconf.addProperty(QUERY_PERSISTENT_RESULT_INDRIVER, "false"); QueryContext addQuery = QueryContext.createContextWithSingleDriver(command, getSession(sessionHandle).getLoggedInUser(), qconf, getLensConf( sessionHandle, qconf), driver, sessionHandle.getPublicId().toString(), true); return addQuery; }
/** * Drop table. * * @param tblName the tbl name * @param parent the parent * @param lensSessionId the lens session id * @throws InterruptedException the interrupted exception */ public static void dropTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); dropTableWithConf(tblName, parent, lensSessionId, conf, mt); }
@Test(dataProvider = "mediaTypeData") public void testNonSelectQueriesWithPersistResult(MediaType mt) throws InterruptedException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true"); String tblName = "testNonSelectQueriesWithPersistResult"; LensServerTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf, mt); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); LensServerTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf, mt); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); LensServerTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf, mt); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "false"); LensServerTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf, mt); }
@Test(enabled = false) public void differentConfSameNameSessionQuery() throws Exception { String query = QueryInventory.getSleepQuery("5"); LensConf lensConf = new LensConf(); lensConf.addProperty(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "false"); QueryHandle qhr1 = (QueryHandle) qHelper.executeQuery(query, "query-name", null, sessionHandleString, lensConf).getData(); lensConf.addProperty(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "true"); QueryHandle qhr2 = (QueryHandle) qHelper.executeQuery(query, "query-name", null, sessionHandleString, lensConf).getData(); Assert.assertFalse(qhr1.getHandleIdString().equals(qhr2.getHandleIdString())); } }
private void checkQuery(LensSessionHandle sessionHandle, XJob job) throws LensException { List<MapType> jobConf = job.getExecution().getQuery().getConf(); LensConf queryConf = new LensConf(); for (MapType element : jobConf) { queryConf.addProperty(element.getKey(), element.getValue()); } queryConf.addProperty(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false); queryService.estimate(LensServices.get().getLogSegregationContext().getLogSegragationId(), sessionHandle, job.getExecution().getQuery().getQuery(), queryConf); }
/** * Test result formatter hdfs persistent result. * * @throws InterruptedException the interrupted exception * @throws IOException Signals that an I/O exception has occurred. */ @Test(dataProvider = "mediaTypeData") public void testResultFormatterHDFSpersistentResult(MediaType mt) throws InterruptedException, IOException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true"); testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, null, mt); queryService.conf.set(LensConfConstants.RESULT_FS_READ_URL, "filereadurl://"); testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, "filereadurl://", mt); queryService.conf.unset(LensConfConstants.RESULT_FS_READ_URL); }
/** * Test result formatter hdfs persistent result location override from driver conf. * * @throws InterruptedException the interrupted exception * @throws IOException Signals that an I/O exception has occurred. */ @Test public void testResultDirOverrideFromDriverConf() throws InterruptedException, IOException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true"); LensQuery lensQuery = testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, null, MediaType.APPLICATION_XML_TYPE); assertEquals(lensQuery.getResultSetPath(), "file:"+System.getProperty("user.dir")+"/target/hive-lens-results/" +lensQuery.getQueryHandle().getHandleIdString()+".csv"); }
@Test public void passPropertyQueryConf() throws Exception { LensConf lensConf = new LensConf(); lensConf.addProperty(LensConfConstants.QUERY_OUTPUT_SERDE, "org.apache.lens.lib.query.CSVSerde1"); QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.QUERY, null, null, sessionHandleString, lensConf).getData(); LensQuery lensQuery = qHelper.waitForCompletion(queryHandle); Assert.assertEquals(lensQuery.getStatus().getStatusMessage().trim(), "Result formatting failed!", "Query did not Fail"); lensConf.addProperty(LensConfConstants.QUERY_OUTPUT_SERDE, "org.apache.lens.lib.query.CSVSerde"); queryHandle = (QueryHandle) qHelper.executeQuery(QueryInventory.QUERY, null, null, sessionHandleString, lensConf).getData(); lensQuery = qHelper.waitForCompletion(queryHandle); Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, "Query did not succeed"); }
@Test public void testQueryTimeoutOnWaitingQuery() throws Exception { String query = "select mock, fail, autocancel from " + TEST_TABLE; LensConf lconf = new LensConf(); lconf.addProperty(LensConfConstants.QUERY_TIMEOUT_MILLIS, 300); QueryHandle handle = executeAndGetHandle(target(), Optional.of(lensSessionId), Optional.of(query), Optional.of(lconf), defaultMT); // query should have been cancelled, as query timeout is 300 millis waitForQueryToFinish(target(), lensSessionId, handle, Status.CANCELED, defaultMT); LensQuery lensQuery = getLensQuery(target(), lensSessionId, handle, defaultMT); assertTrue((lensQuery.getFinishTime() - lensQuery.getLaunchTime()) >= 300, "Query time is " + (lensQuery.getFinishTime() - lensQuery.getLaunchTime())); assertTrue((lensQuery.getFinishTime() - lensQuery.getLaunchTime()) < 400, "Query time is " + (lensQuery.getFinishTime() - lensQuery.getLaunchTime())); }
@Test(enabled=true, groups= "large_purge_interval") public void testQueryRunningAfterTimeout() throws Exception { sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); LensConf lensConf = new LensConf(); lensConf.addProperty(LensConfConstants.CANCEL_QUERY_ON_TIMEOUT, "false"); QueryHandleWithResultSet qhr = (QueryHandleWithResultSet) qHelper.executeQueryTimeout( QueryInventory.getSleepQuery("5"), "10", null, sessionHandleString, lensConf).getData(); InMemoryQueryResult inmemoryResult = (InMemoryQueryResult) qhr.getResult(); Assert.assertNull(inmemoryResult); LensQuery lensQuery = qHelper.waitForCompletion(qhr.getQueryHandle()); Assert.assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); PersistentQueryResult persistResult = (PersistentQueryResult)qHelper.getResultSet(qhr.getQueryHandle()); Assert.assertNotNull(persistResult); Assert.assertEquals(persistResult.getNumRows().intValue(), 8); }
@Test(dataProvider = "persistance_values") public void testDontCancelQueryOnTimeout(String serverPersistent, String driverPersistent) throws Exception { sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, serverPersistent); sHelper.setAndValidateParam(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, driverPersistent); String query = QueryInventory.getSleepQuery("5"); LensConf lensConf = new LensConf(); lensConf.addProperty(LensConfConstants.CANCEL_QUERY_ON_TIMEOUT, "false"); QueryHandleWithResultSet qHWithResultSet = (QueryHandleWithResultSet) qHelper.executeQueryTimeout(query, "5", null, sessionHandleString, lensConf).getData(); LensQuery lq = qHelper.waitForCompletion(qHWithResultSet.getQueryHandle()); Assert.assertEquals(lq.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); }