private void uploadIndexSegments() throws Exception { if (!_conf.isUploadIndexes()) { LOGGER.info("Skipping upload index segments step."); return; } String indexDirectory = _conf.getIndexDirectory(); File[] indexFiles = new File(indexDirectory).listFiles(); Preconditions.checkNotNull(indexFiles); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { URI uploadSegmentHttpURI = FileUploadDownloadClient.getUploadSegmentHttpURI(_controllerHost, _controllerPort); for (File indexFile : indexFiles) { LOGGER.info("Uploading index segment: {}", indexFile.getAbsolutePath()); fileUploadDownloadClient.uploadSegment(uploadSegmentHttpURI, indexFile.getName(), indexFile); } } }
/** * Send segment uri using default settings. * * @param uri URI * @param downloadUri Segment download uri * @return Response * @throws IOException * @throws HttpErrorStatusException */ public SimpleHttpResponse sendSegmentUri(URI uri, String downloadUri) throws IOException, HttpErrorStatusException { return sendSegmentUri(uri, downloadUri, null, null, DEFAULT_SOCKET_TIMEOUT_MS); }
/** * Send segment json using default settings. * * @param uri URI * @param jsonString Segment json string * @return Response * @throws IOException * @throws HttpErrorStatusException */ public SimpleHttpResponse sendSegmentJson(URI uri, String jsonString) throws IOException, HttpErrorStatusException { return sendSegmentJson(uri, jsonString, null, null, DEFAULT_SOCKET_TIMEOUT_MS); }
public void pushOneTarFile(FileSystem fs, Path path) throws Exception { String fileName = path.getName(); if (!fileName.endsWith(JobConfigConstants.TARGZ)) { return; } try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { for (String host : _hosts) { String uri = String.format("%s%s%s", _pushUriPrefix, path.toUri().getRawPath(), _pushUriSuffix); LOGGER .info("******** Uploading file: {} to Host: {} and Port: {} with download uri: {} *******", fileName, host, _port, uri); try { SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(host, _port), uri); LOGGER.info("Response {}: {}", response.getStatusCode(), response.getResponse()); } catch (Exception e) { LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); LOGGER.error("Caught exception during upload", e); throw e; } } } } }
@Override public boolean execute() throws Exception { if (_controllerHost == null) { _controllerHost = NetUtil.getHostAddress(); } if (!_exec) { LOGGER.warn("Dry Running Command: " + toString()); LOGGER.warn("Use the -exec option to actually execute the command."); return true; } File schemaFile = new File(_schemaFile); LOGGER.info("Executing command: " + toString()); if (!schemaFile.exists()) { throw new FileNotFoundException("file does not exist, + " + _schemaFile); } Schema schema = Schema.fromFile(schemaFile); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { fileUploadDownloadClient.addSchema( FileUploadDownloadClient.getUploadSchemaHttpURI(_controllerHost, Integer.parseInt(_controllerPort)), schema.getSchemaName(), schemaFile); } return true; } }
@Test public void testSendFileWithJson() throws Exception { ObjectNode segmentJson = JsonUtils.newObjectNode(); segmentJson.put(CommonConstants.Segment.Offline.DOWNLOAD_URL, TEST_URI); String jsonString = segmentJson.toString(); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentJson(FileUploadDownloadClient.getUploadSegmentHttpURI(TEST_HOST, TEST_PORT), jsonString); Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); Assert.assertEquals(response.getResponse(), "OK"); } }
try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient(sslContext)) { retryPolicy.attempt(() -> { try { SimpleHttpResponse response = fileUploadDownloadClient .uploadSegment(new URI(uploadURL), segmentName, fileToUpload, httpHeaders, parameters, FileUploadDownloadClient.DEFAULT_SOCKET_TIMEOUT_MS); LOGGER.info("Got response {}: {} while uploading table: {}, segment: {} with uploadURL: {}",
protected void initHttpClient(Configuration configs) { _httpClient = new FileUploadDownloadClient(); }
/** * Upload segment with segment file using default settings. * * @param uri URI * @param segmentName Segment name * @param segmentFile Segment file * @return Response * @throws IOException * @throws HttpErrorStatusException */ public SimpleHttpResponse uploadSegment(URI uri, String segmentName, File segmentFile) throws IOException, HttpErrorStatusException { return uploadSegment(uri, segmentName, segmentFile, null, null, DEFAULT_SOCKET_TIMEOUT_MS); }
String[] segmentNames = segmentDir.list(); Assert.assertNotNull(segmentNames); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { final URI uploadSegmentHttpURI = FileUploadDownloadClient.getUploadSegmentHttpURI(LOCAL_HOST, _controllerPort);
public String getSchema() { FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient(); List<URI> schemaURIs = new ArrayList<>(); try { for (PushLocation pushLocation : _pushLocations) { schemaURIs.add(FileUploadDownloadClient .getRetrieveSchemaHttpURI(pushLocation.getHost(), pushLocation.getPort(), _tableName)); } } catch (URISyntaxException e) { LOGGER.error("Could not construct schema URI for table {}", _tableName); throw new RuntimeException(e); } for (URI schemaURI : schemaURIs) { try { SimpleHttpResponse response = fileUploadDownloadClient.getSchema(schemaURI); if (response != null) { return response.getResponse(); } } catch (Exception e) { LOGGER.warn("Caught exception while trying to get schema for " + _tableName + " " + e); } } LOGGER.error("Could not get schema configs for any push locations provided for " + _tableName); throw new RuntimeException("Could not get schema for table " + _tableName); } }
public TableConfig getTableConfig() { FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient(); List<URI> tableConfigURIs = new ArrayList<>(); try { for (PushLocation pushLocation : _pushLocations) { tableConfigURIs.add(FileUploadDownloadClient .getRetrieveTableConfigURI(pushLocation.getHost(), pushLocation.getPort(), _tableName)); } } catch (URISyntaxException e) { LOGGER.error("Could not construct table config URI for table {}", _tableName); throw new RuntimeException(e); } // Return the first table config it can retrieve for (URI uri : tableConfigURIs) { try { SimpleHttpResponse response = fileUploadDownloadClient.getTableConfig(uri); JsonNode offlineTableConfig = JsonUtils.stringToJsonNode(response.getResponse()).get(OFFLINE); if (offlineTableConfig != null) { LOGGER.info("Got table config {}", offlineTableConfig); return TableConfig.fromJSONConfig(offlineTableConfig); } } catch (Exception e) { LOGGER.warn("Caught exception while trying to get table config for " + _tableName + " " + e); } } LOGGER.error("Could not get table configs from any push locations provided for " + _tableName); throw new RuntimeException("Could not get table config for table " + _tableName); }
@Override public Integer call() throws Exception { return fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(LOCAL_HOST, _controllerPort), downloadUri, httpHeaders, null, 60 * 1000).getStatusCode(); } }));
@Test public void testSendFileWithUriAndCrypter() throws Exception { try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { Header crypterClassHeader = new BasicHeader(FileUploadDownloadClient.CustomHeaders.CRYPTER, TEST_CRYPTER); List<Header> headers = Collections.singletonList(crypterClassHeader); List<NameValuePair> params = null; SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(TEST_HOST, TEST_PORT), TEST_URI, headers, params, FileUploadDownloadClient.DEFAULT_SOCKET_TIMEOUT_MS); Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); Assert.assertEquals(response.getResponse(), "OK"); } }
protected void addSchema(File schemaFile, String schemaName) throws Exception { if (!isUsingNewConfigFormat()) { try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { fileUploadDownloadClient .addSchema(FileUploadDownloadClient.getUploadSchemaHttpURI(LOCAL_HOST, _controllerPort), schemaName, schemaFile); } } else { _schema = Schema.fromFile(schemaFile); } }
public ServerSegmentCompletionProtocolHandler(ServerMetrics serverMetrics) { _fileUploadDownloadClient = new FileUploadDownloadClient(_sslContext); _serverMetrics = serverMetrics; }
/** * Upload segment with segment file input stream using default settings. * * @param uri URI * @param segmentName Segment name * @param inputStream Segment file input stream * @return Response * @throws IOException * @throws HttpErrorStatusException */ public SimpleHttpResponse uploadSegment(URI uri, String segmentName, InputStream inputStream) throws IOException, HttpErrorStatusException { return uploadSegment(uri, segmentName, inputStream, null, null, DEFAULT_SOCKET_TIMEOUT_MS); }
public void pushOneTarFile(FileSystem fs, Path path) throws Exception { String fileName = path.getName(); if (!fileName.endsWith(".tar.gz")) { return; } try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { for (String host : _hosts) { try (InputStream inputStream = fs.open(path)) { fileName = fileName.split(JobConfigConstants.TARGZ)[0]; LOGGER.info("******** Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); SimpleHttpResponse response = fileUploadDownloadClient .uploadSegment(FileUploadDownloadClient.getUploadSegmentHttpURI(host, _port), fileName, inputStream); LOGGER.info("Response {}: {}", response.getStatusCode(), response.getResponse()); } catch (Exception e) { LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); LOGGER.error("Caught exception during upload", e); throw e; } } } } }
@Override protected void initHttpClient(Configuration configs) { SSLContext sslContext = new ClientSSLContextGenerator(configs.subset(CommonConstants.PREFIX_OF_SSL_SUBSET)).generate(); _httpClient = new FileUploadDownloadClient(sslContext); }
@Override public Integer call() throws Exception { return fileUploadDownloadClient.uploadSegment(uploadSegmentHttpURI, segmentName, segmentFile) .getStatusCode(); } }));