@Override public <E extends TaggedLogAPIEntity> GenericServiceAPIResponseEntity<String> delete(List<E> entities) throws IOException, EagleServiceClientException { checkNotNull(entities, "entities"); Map<String, List<E>> serviceEntityMap = groupEntitiesByService(entities); if (LOG.isDebugEnabled()) { LOG.debug("Creating entities for " + serviceEntityMap.keySet().size() + " services"); } List<String> deletedKeys = new LinkedList<String>(); for (Map.Entry<String, List<E>> entry : serviceEntityMap.entrySet()) { GenericServiceAPIResponseEntity<String> response = delete(entry.getValue(), entry.getKey()); if (!response.isSuccess()) { LOG.error("Got service exception: " + response.getException()); throw new IOException(response.getException()); } else if (response.getObj() != null) { deletedKeys.addAll(response.getObj()); } } GenericServiceAPIResponseEntity<String> entity = new GenericServiceAPIResponseEntity<String>(); entity.setObj(deletedKeys); entity.setSuccess(true); return entity; }
private void checkResult(GenericServiceAPIResponseEntity result) throws Exception { if (!result.isSuccess()) { logger.error(result.getException()); throw new Exception("Entity creation fails going to EagleService"); } } }
private List<JobExecutionAPIEntity> getRunningJobs(String site, long currentTime, String startTime, String endTime) throws Exception { GenericEntityServiceResource resource = new GenericEntityServiceResource(); String query = String.format("%s[@site=\"%s\" and @startTime<=%s and (@internalState=\"RUNNING\" or @endTime>%s)]{@jobId, @user, @queue, @allocatedMB}", MR_RUNNING_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); GenericServiceAPIResponseEntity<JobExecutionAPIEntity> runningJobResponse = resource.search(query, startTime, endTime, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false); if (!runningJobResponse.isSuccess() || runningJobResponse.getObj() == null) { throw new IOException(runningJobResponse.getException()); } return runningJobResponse.getObj(); }
@Test public void testGenericEntityServiceResourceupdateDatabaseNullDataStorage() throws IllegalDataStorageTypeException { when(DataStorageManager.getDataStorageByEagleConfig()).thenReturn(null); GenericServiceAPIResponseEntity responseEntity = genericEntityServiceResource.updateDatabase(null); Assert.assertFalse(responseEntity.isSuccess()); Assert.assertEquals(null, responseEntity.getMeta()); Assert.assertEquals(null, responseEntity.getType()); Assert.assertEquals(null, responseEntity.getObj()); Assert.assertTrue(responseEntity.getException().startsWith("org.apache.eagle.storage.exception.IllegalDataStorageException: Data storage is null")); }
@Override public GenericServiceAPIResponseEntity deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException { GenericServiceAPIResponseEntity entity = new GenericServiceAPIResponseEntity(); ObjectCodec objectCodec = jp.getCodec(); Map.Entry<String, JsonNode> field = fields.next(); if (META_FIELD.equals(field.getKey()) && field.getValue() != null) { entity.setMeta(objectCodec.readValue(field.getValue().traverse(), Map.class)); } else if (SUCCESS_FIELD.equals(field.getKey()) && field.getValue() != null) { entity.setSuccess(field.getValue().booleanValue()); } else if (EXCEPTION_FIELD.equals(field.getKey()) && field.getValue() != null) { entity.setException(new Exception(field.getValue().textValue())); } else if (TYPE_FIELD.endsWith(field.getKey()) && field.getValue() != null) { Preconditions.checkNotNull(field.getValue().textValue(), "Response type class is null"); try { entity.setType(Class.forName(field.getValue().textValue())); } catch (ClassNotFoundException e) { throw new IOException(e); if (entity.getType() != null) { collectionType = TypeFactory.defaultInstance().constructCollectionType(LinkedList.class, entity.getType()); } else { collectionType = TypeFactory.defaultInstance().constructCollectionType(LinkedList.class, entity.setObj(obj);
private List<HiveResourceSensitivityAPIEntity> load(JobDataMap jobDataMap) throws Exception { Map<String, Object> map = (Map<String,Object>)jobDataMap.get(EagleConfigConstants.EAGLE_SERVICE); String eagleServiceHost = (String)map.get(EagleConfigConstants.HOST); Integer eagleServicePort = Integer.parseInt(map.get(EagleConfigConstants.PORT).toString()); String username = map.containsKey(EagleConfigConstants.USERNAME) ? (String)map.get(EagleConfigConstants.USERNAME) : null; String password = map.containsKey(EagleConfigConstants.PASSWORD) ? (String)map.get(EagleConfigConstants.PASSWORD) : null; // load from eagle database LOG.info("Load hive resource sensitivity information from eagle service " + eagleServiceHost + ":" + eagleServicePort); IEagleServiceClient client = new EagleServiceClientImpl(eagleServiceHost, eagleServicePort, username, password); String query = "HiveResourceSensitivityService[]{*}"; GenericServiceAPIResponseEntity<HiveResourceSensitivityAPIEntity> response = client.search().pageSize(Integer.MAX_VALUE).query(query).send(); client.close(); if (response.getException() != null) throw new IllegalStateException(response.getException()); return response.getObj(); } }
@QueryParam("jobDefId") String jobDefId, @QueryParam("site") String site) { GenericServiceAPIResponseEntity response = new GenericServiceAPIResponseEntity(); if ((jobId == null && jobDefId == null) || site == null) { response.setException(new IllegalArgumentException("Error: (jobId == null && jobDefId == null) || site == null")); response.setSuccess(false); return response; response.setException(new Exception("Search condition is empty")); response.setSuccess(false); return response; String queryString = String.format(queryFormat, Constants.MR_JOB_EXECUTION_SERVICE_NAME, condition); GenericServiceAPIResponseEntity<TaggedLogAPIEntity> res = ResourceUtils.getQueryResult(queryString, null, null); if (res.isSuccess() && res.getObj() != null) { for (TaggedLogAPIEntity o : res.getObj()) { jobs.add(o); jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString())); if (res.isSuccess() && res.getObj() != null) { for (TaggedLogAPIEntity o : res.getObj()) { String key = o.getTags().get(MRJobTagName.JOB_ID.toString()); if (!ResourceUtils.isDuplicate(jobIds, key)) { if (res.isSuccess()) { response.setSuccess(true); } else { response.setSuccess(false); response.setException(new Exception(res.getException()));
public Object getJobMetrics(String site, String timePoint, String metricName, long intervalmin, int top, Function6<String, String, String, Long, Integer, String, Object> metricQueryFunc) { GenericServiceAPIResponseEntity response = new GenericServiceAPIResponseEntity(); if (site == null || timePoint == null || metricName == null) { response.setException(new IllegalArgumentException("Error: site, timePoint, metricName may be unset")); response.setSuccess(false); return response; searchStartTime = helper.moveTimeForwardOneDay(searchStartTime); } catch (ParseException e) { response.setException(e); response.setSuccess(false); return response; Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, timePointsInMills, timePointsInMills); GenericServiceAPIResponseEntity<JobExecutionAPIEntity> historyRes = ResourceUtils.getQueryResult(query, searchStartTime, searchEndTime); if (!historyRes.isSuccess() || historyRes.getObj() == null) { return historyRes; List<String> timeDuration = helper.getSearchTimeDuration(historyRes.getObj()); LOG.info(String.format("new search time range: startTime=%s, endTime=%s", timeDuration.get(0), timeDuration.get(1))); query = String.format("%s[@site=\"%s\"]<@jobId>{sum(value)}.{sum(value) desc}", Constants.GENERIC_METRIC_SERVICE, site);
private long getServiceLatestUpdateTime(String serviceName, IEagleServiceClient client) throws Exception { String query = String.format("%s[@site=\"%s\"]<@site>{max(lastUpdateTime)}", serviceName, topologyCheckAppConfig.dataExtractorConfig.site); GenericServiceAPIResponseEntity response = client .search(query) .pageSize(10) .send(); List<Map<List<String>, List<Double>>> results = response.getObj(); if (results.size() == 0) { return Long.MAX_VALUE; } long currentProcessTimeStamp = results.get(0).get("value").get(0).longValue(); return currentProcessTimeStamp; } }
@Test public void testSerDeserialize() throws IOException { // mock up service side to serialize GenericServiceAPIResponseEntity<Item> entity = new GenericServiceAPIResponseEntity<Item>(Item.class); entity.setObj(Arrays.asList(new Item("a", 1.2), new Item("b", 1.3), new Item("c", 1.4))); entity.setMeta(new HashMap<String, Object>() { { put("tag1", "val1"); put("tag2", "val2"); } }); // entity.setTypeByObj(); entity.setSuccess(true); String json = objectMapper.writeValueAsString(entity); LOG.info(json); // mock up client side to deserialize GenericServiceAPIResponseEntity deserEntity = objectMapper .readValue(json, GenericServiceAPIResponseEntity.class); Assert.assertEquals(json, objectMapper.writeValueAsString(deserEntity)); Assert.assertEquals(3, deserEntity.getObj().size()); Assert.assertEquals(LinkedList.class, deserEntity.getObj().getClass()); Assert.assertEquals(Item.class, deserEntity.getObj().get(0).getClass()); } }
@Override public void execute(Tuple input) { if (input == null) { return; } String serviceName = input.getStringByField(TopologyConstants.SERVICE_NAME_FIELD); TopologyEntityParserResult result = (TopologyEntityParserResult) input.getValueByField(TopologyConstants.TOPOLOGY_DATA_FIELD); Set<String> availableHostNames = new HashSet<String>(); List<TopologyBaseAPIEntity> entitiesForDeletion = new ArrayList<>(); List<TopologyBaseAPIEntity> entitiesToWrite = new ArrayList<>(); filterEntitiesToWrite(result, availableHostNames, entitiesToWrite); String query = String.format("%s[@site=\"%s\"]{*}", serviceName, this.config.dataExtractorConfig.site); try { GenericServiceAPIResponseEntity<TopologyBaseAPIEntity> response = client.search().query(query).pageSize(Integer.MAX_VALUE).send(); if (response.isSuccess() && response.getObj() != null) { for (TopologyBaseAPIEntity entity : response.getObj()) { if (!availableHostNames.isEmpty() && !availableHostNames.contains(generatePersistKey(entity))) { entitiesForDeletion.add(entity); } } } deleteEntities(entitiesForDeletion, serviceName); writeEntities(entitiesToWrite, result.getMetrics(), serviceName); emitToKafkaBolt(result); this.collector.ack(input); } catch (Exception e) { LOG.error(e.getMessage(), e); this.collector.fail(input); } }
public boolean doPersist(List<? extends TaggedLogAPIEntity> list) { if (list.isEmpty()) return false; LOG.info("Going to persist entities, type: " + " " + list.get(0).getClass().getSimpleName() + ", list size: " + list.size()); try { IEagleServiceClient client = new EagleServiceClientImpl(eagleServiceHost, eagleServicePort, username, password); GenericServiceAPIResponseEntity<String> response = client.create(list); client.close(); if (response.isSuccess()) { LOG.info("Successfully create entities " + list.toString()); return true; } else { LOG.error("Fail to create entities"); return false; } } catch (Exception ex) { LOG.error("Got an exception in persisting entities" + ex.getMessage(), ex); return false; } } }
/** * Just compiling passed is ok */ //@Test @SuppressWarnings("unused") public void testCreate() throws IOException, EagleServiceClientException, IllegalAccessException, InstantiationException { EntityDefinition entityDefinition = EntityDefinitionManager.getEntityDefinitionByEntityClass(GenericMetricEntity.class); hbase.createTable(entityDefinition.getTable(), entityDefinition.getColumnFamily()); client = new EagleServiceClientImpl("localhost", EagleConfigFactory.load().getServicePort()); List<GenericMetricEntity> metricEntityList = new ArrayList<GenericMetricEntity>(); GenericServiceAPIResponseEntity<String> unTypedResponse = client.create(metricEntityList); GenericServiceAPIResponseEntity<String> weakTypedResponse = client.create(metricEntityList,GenericMetricEntity.GENERIC_METRIC_SERVICE); GenericServiceAPIResponseEntity<String> strongTypedResponse = client.create(metricEntityList,GenericMetricEntity.class); GenericServiceAPIResponseEntity<GenericMetricEntity> weakTypedSearchResponse = client.search("").send(); if(weakTypedSearchResponse!=null) { Class<GenericMetricEntity> typedClazz = weakTypedSearchResponse.getType(); List<GenericMetricEntity> typedEntities = weakTypedSearchResponse.getObj(); } }
private List<org.apache.eagle.jpm.mr.historyentity.JobExecutionAPIEntity> getJobs(String site, long currentTime, String startTime, String endTime) throws Exception { GenericEntityServiceResource resource = new GenericEntityServiceResource(); String query = String.format("%s[@site=\"%s\" and @startTime<=%s and @endTime>%s]{@jobId}", MR_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); GenericServiceAPIResponseEntity<org.apache.eagle.jpm.mr.historyentity.JobExecutionAPIEntity> response = resource.search(query, startTime, endTime, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false); if (!response.isSuccess() || response.getObj() == null) { throw new IOException(response.getException()); } return response.getObj(); }
@Test public void testGenericEntityServiceResourceSearchNullServiceName() throws IllegalAccessException, InstantiationException, IOException { GenericServiceAPIResponseEntity responseEntity = resources.client().resource("/entities/rowkey").queryParam("value", "test").get(GenericServiceAPIResponseEntity.class); Assert.assertFalse(responseEntity.isSuccess()); Assert.assertEquals(null, responseEntity.getMeta()); Assert.assertEquals(null, responseEntity.getType()); Assert.assertEquals(null, responseEntity.getObj()); Assert.assertTrue(responseEntity.getException().startsWith("java.lang.Exception: java.lang.IllegalArgumentException: serviceName is null")); verify(dataStorage, never()).queryById(any(), any()); }
@POST @Path("attributeresolve") @Consumes( {"application/json"}) @Produces( {"application/json"}) public GenericServiceAPIResponseEntity attributeResolve(InputStream request, @QueryParam("resolver") String resolver) { GenericServiceAPIResponseEntity response = new GenericServiceAPIResponseEntity(); try { if (resolver == null) { throw new AttributeResolveException("resolver is null"); } AttributeResolvable resolvable = AttributeResolverFactory.getAttributeResolver(resolver, entityService, eagleServerConfig); ObjectMapper objectMapper = new ObjectMapper(); Class<?> resolveRequestClass = resolvable.getRequestClass(); if (resolveRequestClass == null) { throw new AttributeResolveException("Request class is null for resolver " + resolver); } GenericAttributeResolveRequest resolveRequest = (GenericAttributeResolveRequest) objectMapper.readValue(request, resolvable.getRequestClass()); resolvable.validateRequest(resolveRequest); List result = resolvable.resolve(resolveRequest); response.setSuccess(true); response.setObj(result); } catch (Exception e) { response.setSuccess(false); response.setException(e); return response; } return response; }
protected <R extends TaggedLogAPIEntity> List<R> load(JobDataMap jobDataMap, String service) throws Exception { Map<String, Object> map = (Map<String, Object>) jobDataMap.get(EagleConfigConstants.EAGLE_SERVICE); String eagleServiceHost = (String) map.get(EagleConfigConstants.HOST); Integer eagleServicePort = Integer.parseInt(map.get(EagleConfigConstants.PORT).toString()); String username = map.containsKey(EagleConfigConstants.USERNAME) ? (String) map.get(EagleConfigConstants.USERNAME) : null; String password = map.containsKey(EagleConfigConstants.PASSWORD) ? (String) map.get(EagleConfigConstants.PASSWORD) : null; // load from eagle database LOG.info("Load resource sensitivity information from eagle service " + eagleServiceHost + ":" + eagleServicePort); IEagleServiceClient client = new EagleServiceClientImpl(eagleServiceHost, eagleServicePort, username, password); String query = service + "[]{*}"; GenericServiceAPIResponseEntity<R> response = client.search().pageSize(Integer.MAX_VALUE).query(query).send(); client.close(); if (response.getException() != null) throw new IllegalStateException(response.getException()); return response.getObj(); } }
public void flush() throws IOException, EagleServiceClientException { if(this.entityBucket.size() == 0 && LOG.isDebugEnabled()){ LOG.debug("No entities to flush"); return; } LOG.info("Writing "+this.entityBucket.size()+" entities"); GenericServiceAPIResponseEntity<String> response = this.client.create(this.entityBucket); if(!response.isSuccess()){ LOG.error("Got service exception: "+response.getException()); throw new IOException("Service exception"+response.getException()); }else{ this.entityBucket.clear(); } }
@QueryParam("metricName") String metricName, @QueryParam("verbose") Boolean verbose) throws ParseException { GenericServiceAPIResponseEntity response = new GenericServiceAPIResponseEntity(); GenericServiceAPIResponseEntity<JobExecutionAPIEntity> res = resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose); if (res.isSuccess() && res.getObj() != null) { long maxFinishedTime = DateTimeUtil.humanDateToSeconds(endTime) * DateTimeUtil.ONESECOND; for (JobExecutionAPIEntity o : res.getObj()) { if (o.getEndTime() <= maxFinishedTime) { finishedJobs.add(o); GenericServiceAPIResponseEntity<org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity> runningRes = resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose); if (runningRes.isSuccess() && runningRes.getObj() != null) { for (org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity o : runningRes.getObj()) { String key = o.getTags().get(MRJobTagName.JOB_ID.toString()); if (!ResourceUtils.isDuplicate(jobIds, key)) { if (res.isSuccess()) { response.setSuccess(true); } else { response.setSuccess(false); response.setException(new Exception(res.getException())); response.setObj(jobs); response.setMeta(meta); return response;