public void start() throws InterruptedException, IOException { Map<String, Object> context = new HashMap<String, Object>(); context.put("qnodes", qNodes); Map<Integer, Long> rowIdsPerPartition = new HashMap<Integer, Long>(); context.put("rowIdsPerPartition", rowIdsPerPartition); // First get Tablespace metadata - nPartitions SploutClient client = new SploutClient(((String) context.get("qnodes")).split(",")); Tablespace tablespace = client.tablespace(TABLESPACE); int nPartitions = tablespace.getPartitionMap().getPartitionEntries().size(); // Then gather number of registers for each partition for (int partition = 0; partition < nPartitions; partition++) { String query = "SELECT MAX(rowid) FROM pagecounts;"; Map<String, Object> obj = (Map<String, Object>) client .query(TABLESPACE, null, query, partition + "").getResult().get(0); rowIdsPerPartition.put(partition, Long.parseLong(obj.get("MAX(rowid)").toString())); } SploutBenchmark benchmark = new SploutBenchmark(); for (int i = 0; i < nIterations; i++) { benchmark.stressTest(nThreads, nQueries, PageCountsStressThreadImpl.class, context); benchmark.printStats(System.out); } }
@Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { long now = System.currentTimeMillis(); if (cachedString != null && (now - cachedFrom) < CACHE_FOR) { resp.getWriter().write(cachedString); log.info("Returning XML content from cache."); return; } HttpRequest trendReq = requestFactory.buildGetRequest(new GenericUrl("http://www.trendingtopics.org/pages.xml?page=1")); cachedString = SploutClient.asString(trendReq.execute().getContent()); log.info("Got XML content from trendingtopics: " + cachedString); cachedFrom = System.currentTimeMillis(); resp.getWriter().write(cachedString); } }
protected PartitionMap retrievePartitionMapfromQNode(String tablespace) throws Exception { log.info("Retrieving partition map for tablespace[" + tablespace + "] from QNode[" + qnodeURL +"] for being reused at indexation. "); SploutClient client = new SploutClient(qnodeURL.split(",")); Tablespace tb = client.tablespace(tablespace); if (tb == null) { throw new Exception("Partition map for tablespace [" + tablespace + "] not found at QNode[" + qnodeURL +"]."); } PartitionMap pmap = tb.getPartitionMap(); log.info("Successfully retrieved partition map for tablespace[" + tablespace + "] from QNode[" + qnodeURL +"]."); return pmap; }
@SuppressWarnings("unchecked") @Override public void init(Map<String, Object> context) throws Exception { this.context = context; client = new SploutClient(20 * 1000, ((String) context.get("qnodes")).split(",")); String query = "SELECT stn, wban FROM stations"; System.out.println("Retrieving stations list"); long statTime = System.currentTimeMillis(); QueryStatus st = client.query(tablespace, "any", query, null); if (st.getResult() == null) { throw new RuntimeException("Impossible to retrieve stations list. " + st); } stations = st.getResult(); System.out.println("Loaded " + stations.size() + " stations in " + (System.currentTimeMillis() - statTime) + " ms."); }
SploutClient client = new SploutClient(qnode); QNodeStatus status = client.overview(); long previousVersion = -1; if (status.getTablespaceMap().get("pagecountsintegration") != null) { status = client.overview(); while (status.getTablespaceMap().get("pagecountsintegration") == null || previousVersion == status.getTablespaceMap().get("pagecountsintegration").getVersion()) { Thread.sleep(2000); waitedSoFar += 2000; status = client.overview(); if (waitedSoFar > 90000) { throw new RuntimeException( QueryStatus qStatus = client.query("pagecountsintegration", "*", "SELECT * FROM pagecounts;", null); System.out.println(qStatus.getResult());
SploutClient client = new SploutClient(qnode); List<String> dnodes = client.dNodeList(); if (dnodes == null || dnodes.size() == 0) { throw new IOException("No available DNodes in Splout cluster."); DeployInfo dInfo = client.deploy(deployRequests);
public SploutState(boolean failFast, String... qNodes) { this.sploutClient = new SploutClient(qNodes); this.failFast = failFast; }
SploutClient client = new SploutClient(qnode); List<String> dnodes = client.dNodeList(); final int nDNodes = dnodes.size();
@Override public int nextQuery() throws Exception { int key = ((int) (Math.random() * (maxKey - minKey))) + minKey; String strKey = String.format(paddingExp, key); return client .query(tablespace, strKey + "", "SELECT * FROM " + tablespace + " WHERE key = " + key + ";", null) .getResult().size(); } }
public DeployInfo deploy(String tablespace, PartitionMap partitionMap, ReplicationMap replicationMap, URI dataUri) throws IOException { DeployRequest deployRequest = new DeployRequest(); deployRequest.setTablespace(tablespace); deployRequest.setData_uri(dataUri.toString()); deployRequest.setPartitionMap(partitionMap.getPartitionEntries()); deployRequest.setReplicationMap(replicationMap.getReplicationEntries()); return deploy(deployRequest); }
String tsName = def.getName(); SploutClient client = new SploutClient(1000 * 60 * 60 * 24, qNodes.split(",")); QNodeStatus overview = client.overview(); String query = "SELECT COUNT(*) FROM " + tblName; QueryStatus status = client.query(tsName, null, query, part + ""); if (status.getError() != null) { throw new Exception("Query error: " + status.getError()); + tblName + " GROUP BY key ORDER by c DESC LIMIT " + topSize; status = client.query(tsName, null, query, part + ""); if (status.getError() != null) { throw new Exception("Query error: " + status.getError());
@Override public void init(Map<String, Object> context) throws Exception { client = new SploutClient(((String) context.get("qnodes")).split(",")); rowIdsPerPartition = (Map<Integer, Long>) context.get("rowIdsPerPartition"); }
@Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { String sql = req.getParameter("sql"); String key = req.getParameter("key"); log.info("query, key[" + key + "] sql[" + sql + "]"); try { resp.getWriter().write(JSONSerDe.ser(client.query("pagecounts", key, sql, null))); } catch (JSONSerDeException e) { throw new IOException(e); } } }
public DeployInfo deploy(String tablespace, PartitionMap partitionMap, ReplicationMap replicationMap, URI dataUri) throws IOException { DeployRequest deployRequest = new DeployRequest(); deployRequest.setTablespace(tablespace); deployRequest.setData_uri(dataUri.toString()); deployRequest.setPartitionMap(partitionMap.getPartitionEntries()); deployRequest.setReplicationMap(replicationMap.getReplicationEntries()); return deploy(deployRequest); }
public QueryStatus query(String tablespace, String key, String query, String partition) throws IOException { URI uri; try { uri = new URI("http", qNodesNoProtocol[(int) (Math.random() * qNodes.length)], "/api/query/" + tablespace, "&sql=" + query + (key != null ? "&key=" + key : "") + (partition != null ? "&partition=" + partition : ""), null); HttpRequest request = requestFactory.buildGetRequest(new GenericUrl(uri.toASCIIString())); HttpResponse resp = request.execute(); try { return JSONSerDe.deSer(asString(resp.getContent()), QueryStatus.class); } catch(JSONSerDeException e) { throw new IOException(e); } } catch(URISyntaxException e) { throw new IllegalArgumentException(e); } }
@Override public void init(Map<String, Object> context) throws Exception { client = new SploutClient(((String) context.get("qnodes")).split(",")); minKey = (Integer) context.get("minKey"); maxKey = (Integer) context.get("maxKey"); paddingExp = (String) context.get("paddingExp"); tablespace = (String) context.get("tablespace"); }
/** * Given a SQL query and a partition key, query Splout. We take advantage of Trident's batch and perform several * queries in a row. This could actually be a lot more efficient if we used a thread-pool here since Splout can handle * parallel queries quite well, but I kept it simple just for the sake of this example. */ public List<Object> querySplout(String tablespace, List<String> sql, List<String> keys) { List<Object> result = new ArrayList<Object>(); for(int i = 0; i < sql.size(); i++) { String partitionKey = keys.get(i); String sqlQuery = sql.get(i); try { result.add(sploutClient.query(tablespace, partitionKey, sqlQuery, null)); } catch(IOException e) { if(!failFast) { result.add(null); } else { throw new RuntimeException(e); } } } return result; } }
public DeployInfo deploy(final DeployRequest... requests) throws IOException { try { String strCont = JSONSerDe.ser(new ArrayList<DeployRequest>(Arrays.asList(requests))); HttpContent content = new StringHttpContent(strCont); HttpRequest request = requestFactory.buildPostRequest(new GenericUrl( qNodes[(int) (Math.random() * qNodes.length)] + "/api/deploy"), content); HttpResponse resp = request.execute(); return JSONSerDe.deSer(asString(resp.getContent()), DeployInfo.class); } catch (JSONSerDeException e) { throw new IOException(e); } }
public void run() throws Exception { Server server = new Server(port); WebAppContext context = new WebAppContext(); context.setContextPath("/"); context.addServlet(new ServletHolder(new DefaultServlet()), "/pagecounts/*"); context.addServlet(new ServletHolder(new PageCountsServlet(new SploutClient(qNodes.split(",")))), "/api"); context.addServlet(new ServletHolder(new PageCountsTrendingServlet()), "/trends"); ResourceCollection resources = new ResourceCollection(new String[]{Resource.newClassPathResource( "pagecounts").toString()}); context.setBaseResource(resources); server.setHandler(context); server.start(); try { while (true) { Thread.sleep(5000); } } catch (InterruptedException e) { e.printStackTrace(); server.stop(); } }
@Override public int nextQuery() throws Exception { int rndIdx = rand.nextInt(stations.size()); int stn = stations.get(rndIdx).get("stn"); int wban = stations.get(rndIdx).get("wban"); String query; if (context.get("month").equals("false")) { query = "select year,month,day,min,max from meteo where stn=" + stn + " and wban = " + wban + " order by year,month,day"; } else { query = "select year,month,min(min) as min,max(max) as max from meteo where stn=" + stn + " and wban=" + wban + " group by year,month order by year,month"; } QueryStatus st = client.query(tablespace, stn + "" + wban, query, null); if (st.getResult() != null) { return st.getResult().size(); } else { return 0; } } }