private static void robustCreateFile(File candidate) throws IOException { int tries = 0; while ( ! candidate.exists()) { if (tries > 30) { throw new IOException("Could not create file: " + candidate); } if (candidate.createNewFile()) { break; } tries++; try { Thread.sleep(1000); } catch (InterruptedException e) { log(e); throw new RuntimeInterruptedException(e); } } }
private static HttpsServer addSSLContext(HttpsServer server) { log("Adding SSL context to server; key=" + StanfordCoreNLPServer.key); try (InputStream is = IOUtils.getInputStreamFromURLOrClasspathOrFileSystem(key)) { KeyStore ks = KeyStore.getInstance("JKS");
log(b.toString());
/** @see #onBlacklist(Inet4Address) */ private boolean onBlacklist(HttpExchange exchange) { if ( ! stanford) { return false; } InetAddress addr = exchange.getRemoteAddress().getAddress(); if (addr instanceof Inet4Address) { return onBlacklist((Inet4Address) addr); } else { log("Not checking IPv6 address against blacklist: " + addr); return false; // TODO(gabor) we should eventually check ipv6 addresses too } }
log("option overrides " + target + " to '" + value + '\''); fillField(class2object.get(target.getDeclaringClass()), target, value); } else {
log("processed " + numExamplesProcessed + " training sentences: " + dataset.size() + " datums"); try { IOUtils.writeObjectToFile(toSave, modelPath.get()); log("SUCCESS: wrote model to " + modelPath.get().getPath()); } catch (IOException e) { log("ERROR: failed to save model to path: " + modelPath.get().getPath()); err(e);
log("[" + httpExchange.getRemoteAddress() + "] API call w/annotators " + props.getProperty("annotators", "<unknown>"));
URL srResource = classLoader.getResource("edu/stanford/nlp/models/srparser/englishSR.ser.gz"); log("setting default constituency parser"); if (srResource != null) { defaultParserPath = "edu/stanford/nlp/models/srparser/englishSR.ser.gz"; log("using SR parser: edu/stanford/nlp/models/srparser/englishSR.ser.gz"); } else { defaultParserPath = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"; log("warning: cannot find edu/stanford/nlp/models/srparser/englishSR.ser.gz"); log("using: edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz instead"); log("to use shift reduce parser download English models jar from:"); log("http://stanfordnlp.github.io/CoreNLP/download.html");
combinedMapping.get(fileToWriteTo).put(entry.getKey(), entry.getValue()); log("[" + new DecimalFormat("0000").format(i) + "/" + constituents.length + "] read " + constituent.cacheDir + " [" + (Runtime.getRuntime().freeMemory() / 1000000) + "MB free memory]"); constituent.clear();
log("Checking cp " + entry);
log("--- " + StanfordCoreNLPServer.class.getSimpleName() + "#main() called ---"); String build = System.getenv("BUILD"); if (build != null) { log(" Build: " + build); Runtime.getRuntime().addShutdownHook(new Thread(() -> log("CoreNLP Server is shutting down."))); server.statusPort = Integer.parseInt(serverProperties.getProperty("port")); log(" Threads: " + ArgumentParser.threads); log("Starting server..."); if (server.ssl) { server.run(credentials, req -> true, res -> {}, homepage, true, live);
server.start(); live.set(true); log("StanfordCoreNLPServer listening at " + server.getAddress()); } catch (IOException e) { e.printStackTrace();
if (lockOrNull == null || !lockOrNull.isValid()) { try { Thread.sleep(1000); } catch (InterruptedException e) { log(e); throw new RuntimeInterruptedException(e);
/** * If we have a separate liveness port, start a server on a separate thread pool whose only * job is to watch for when the CoreNLP server becomes ready. * This will also immediately signal liveness. * * @param live The boolean to track when CoreNLP has initialized and the server is ready * to serve requests. */ private void livenessServer(AtomicBoolean live) { if (this.serverPort != this.statusPort) { try { // Create the server if (this.ssl) { server = addSSLContext(HttpsServer.create(new InetSocketAddress(statusPort), 0)); // 0 is the default 'backlog' } else { server = HttpServer.create(new InetSocketAddress(statusPort), 0); // 0 is the default 'backlog' } // Add the two status endpoints withAuth(server.createContext("/live", new LiveHandler()), Optional.empty()); withAuth(server.createContext("/ready", new ReadyHandler(live)), Optional.empty()); // Start the server server.start(); // Server started log("Liveness server started at " + server.getAddress()); } catch (IOException e) { err("Could not start liveness server. This will probably result in very bad things happening soon.", e); } } }
/** * The main entry point of the code. */ public static void main(String[] args) throws IOException { forceTrack("Processing treebanks"); List<Pair<CoreMap, Collection<Pair<Span, Span>>>> trainingData = new ArrayList<>(); trainingData.addAll(processDirectory("WSJ", new File("/home/gabor/lib/data/penn_treebank/wsj"))); trainingData.addAll(processDirectory("Brown", new File("/home/gabor/lib/data/penn_treebank/brown"))); endTrack("Processing treebanks"); forceTrack("Training"); log("dataset size: " + trainingData.size()); ClauseSplitter.train( trainingData.stream(), new File("/home/gabor/tmp/clauseSearcher.ser.gz"), new File("/home/gabor/tmp/clauseSearcherData.tab.gz")); endTrack("Training"); // Execution.fillOptions(CreateClauseDataset.class, args); // // new CreateClauseDataset().runAndExit(in, System.err, code -> code); } }
/** * A helper function to respond to a request with an error stating that the user is not authorized * to make this request. * * @param httpExchange The exchange to send the error over. * * @throws IOException Thrown if the HttpExchange cannot communicate the error. */ private static void respondUnauthorized(HttpExchange httpExchange) throws IOException { log("Responding unauthorized to " + httpExchange.getRemoteAddress()); httpExchange.getResponseHeaders().add("Content-type", "application/javascript"); byte[] content = "{\"message\": \"Unauthorized API request\"}".getBytes("utf-8"); httpExchange.sendResponseHeaders(HTTP_UNAUTHORIZED, content.length); httpExchange.getResponseBody().write(content); httpExchange.close(); }
/** @see #onBlacklist(Inet4Address) */ private boolean onBlacklist(HttpExchange exchange) { if ( ! stanford) { return false; } InetAddress addr = exchange.getRemoteAddress().getAddress(); if (addr instanceof Inet4Address) { return onBlacklist((Inet4Address) addr); } else { log("Not checking IPv6 address against blacklist: " + addr); return false; // TODO(gabor) we should eventually check ipv6 addresses too } }