/** * Get content source for input server. * @param conf job configuration. * @return ContentSource for input server. * @throws URISyntaxException * @throws XccConfigException * @throws IOException */ public static ContentSource getInputContentSource(Configuration conf) throws URISyntaxException, XccConfigException, IOException { String host = conf.getStrings(INPUT_HOST)[0]; if (host == null || host.isEmpty()) { throw new IllegalArgumentException(INPUT_HOST + " is not specified."); } return getInputContentSource(conf, host); }
@Override public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException { String[] hosts = conf.getStrings(OUTPUT_HOST); if (hosts == null || hosts.length == 0) { throw new IllegalStateException(OUTPUT_HOST + " is not specified."); } for (int i = 0; i < hosts.length; i++) { try { ContentSource cs = InternalUtilities.getOutputContentSource(conf, hosts[i]); checkOutputSpecs(conf, cs); return; } catch (Exception ex) { if (ex.getCause() instanceof ServerConnectionException) { LOG.warn("Unable to connect to " + hosts[i] + " to query destination information"); continue; } else { throw new IOException(ex); } } } // No usable output hostname found at this point throw new IOException("Unable to query destination information," + " no usable hostname found"); }
@SuppressWarnings("unchecked") @Override protected boolean nextResult(ResultItem result) { if (key == null) { key = new LongWritable(getCount()); } else { key.set(getCount()); } if (value == null) { value = (VALUEIN)ReflectionUtils.newInstance(valueClass, getConf()); } InternalUtilities.assignResultValue(valueClass, result, value); return true; }
/** * Get output content source. * * @param conf job configuration * @param hostName host name * @return content source * @throws IOException * @throws XccConfigException * @throws IOException */ public static ContentSource getOutputContentSource(Configuration conf, String hostName) throws XccConfigException, IOException { String user = conf.get(OUTPUT_USERNAME, ""); String password = conf.get(OUTPUT_PASSWORD, ""); String port = conf.get(OUTPUT_PORT,"8000"); String db = conf.get(OUTPUT_DATABASE_NAME); int portInt = Integer.parseInt(port); boolean useSsl = conf.getBoolean(OUTPUT_USE_SSL, false); if (useSsl) { return getSecureContentSource(hostName, portInt, user, password, db, getOutputSslOptions(conf)); } return ContentSourceFactory.newContentSource(hostName, portInt, user, password, db); }
/** * Get input content source. * * @param conf job configuration * @param host host to connect to * @return content source * @throws IOException * @throws XccConfigException */ public static ContentSource getInputContentSource(Configuration conf, String host) throws XccConfigException, IOException { String user = conf.get(INPUT_USERNAME, ""); String password = conf.get(INPUT_PASSWORD, ""); String port = conf.get(INPUT_PORT,"8000"); String db = conf.get(INPUT_DATABASE_NAME); int portInt = Integer.parseInt(port); boolean useSsl = conf.getBoolean(INPUT_USE_SSL, false); if (useSsl) { return getSecureContentSource(host, portInt, user, password, db, getInputSslOptions(conf)); } return ContentSourceFactory.newContentSource(host, portInt, user, password, db); }
curHostName = replicas.get(curForest).getHostName(); ContentSource cs = InternalUtilities.getInputContentSource(conf, curHostName); session = cs.newSession("#"+curForestName); InternalUtilities.sleep(sleepTime); } catch (Exception e2) {
InterruptedException { int fId = 0; String uri = InternalUtilities.getUriWithOutputDir(key, outputDir); if (fastLoad) { if (!countBased) {
while (hostIdx < inputHosts.length) { try { ContentSource cs = InternalUtilities.getInputContentSource(jobConf, inputHosts[hostIdx]); session = cs.newSession(); InternalUtilities.checkQueryLanguage(queryLanguage); options.setQueryLanguage(queryLanguage);
if (++retry < maxRetries) { try { InternalUtilities.sleep(sleepTime); } catch (Exception e2) {
@Override public RecordWriter<NodePath, MarkLogicNode> getRecordWriter( TaskAttemptContext context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); TextArrayWritable hosts = getHosts(conf); String host = InternalUtilities.getHost(hosts); return new NodeWriter(conf, host); }
@Override public void checkOutputSpecs(Configuration conf, ContentSource cs) throws IOException { // check for required configuration if (conf.get(OUTPUT_QUERY) == null) { throw new IllegalArgumentException(OUTPUT_QUERY + " is not specified."); } // warn against unsupported configuration if (conf.get(BATCH_SIZE) != null) { LOG.warn("Config entry for " + "\"mapreduce.marklogic.output.batchsize\" is not " + "supported for " + this.getClass().getName() + " and will be ignored."); } String queryLanguage = conf.get(OUTPUT_QUERY_LANGUAGE); if (queryLanguage != null) { InternalUtilities.checkQueryLanguage(queryLanguage); } // store hosts into config system DefaultStringifier.store(conf, queryHosts(cs), OUTPUT_FOREST_HOST); } }
break; int comp = InternalUtilities.compareUnsignedLong(offset, treeDataSize); if (comp > 0) { && InternalUtilities.compareUnsignedLong(offset, prevOffset) <= 0) { throw new RuntimeException(
@Override public void write(DocumentURI key, VALUEOUT value) throws IOException, InterruptedException { InternalUtilities.getUriWithOutputDir(key, outputDir); int fId = 0; if (fastLoad) {
InternalUtilities.sleep(sleepTime); } catch (Exception e2) {
@Override public RecordWriter<DocumentURI, MarkLogicNode> getRecordWriter( TaskAttemptContext context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); TextArrayWritable hosts = getHosts(conf); String host = InternalUtilities.getHost(hosts); return new PropertyWriter(conf, host); }
/** * Get the session for this writer. One writer only maintains one session. * * @return Session for this writer. * @throws IOException */ protected Session getSession() throws IOException { if (session == null) { // start a session try { ContentSource cs = InternalUtilities.getOutputContentSource( conf, hostName); session = cs.newSession(); if (LOG.isDebugEnabled()) { LOG.debug("Connect to " + session.getConnectionUri().getHost()); } if (txnSize > 1) { session.setTransactionMode(TransactionMode.UPDATE); } } catch (XccConfigException e) { LOG.error("Error creating a new session: ", e); throw new IOException(e); } } return session; }
static void setQueryTimestamp(Configuration conf) throws IOException { try { ContentSource cs = InternalUtilities.getInputContentSource(conf); Session session = cs.newSession(); conf.set(MarkLogicConstants.INPUT_QUERY_TIMESTAMP, session.getCurrentServerPointInTime().toString()); } catch (Exception ex) { throw new IOException("Error getting query timestamp", ex); } }
InterruptedException { int fId = 0; String uri = InternalUtilities.getUriWithOutputDir(key, outputDir); if (fastLoad) { if(!countBased) {
if (retryable && ++retry < maxRetries) { try { InternalUtilities.sleep(sleepTime); } catch (Exception e2) {
@Override public RecordWriter<KEYOUT, VALUEOUT> getRecordWriter( TaskAttemptContext context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); TextArrayWritable hosts = getHosts(conf); String host = InternalUtilities.getHost(hosts); return new KeyValueWriter<KEYOUT, VALUEOUT>(conf, host); }