public void pushOneTarFile(FileSystem fs, Path path) throws Exception { String fileName = path.getName(); if (!fileName.endsWith(".tar.gz")) { return; } try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { for (String host : _hosts) { try (InputStream inputStream = fs.open(path)) { fileName = fileName.split(JobConfigConstants.TARGZ)[0]; LOGGER.info("******** Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); SimpleHttpResponse response = fileUploadDownloadClient .uploadSegment(FileUploadDownloadClient.getUploadSegmentHttpURI(host, _port), fileName, inputStream); LOGGER.info("Response {}: {}", response.getStatusCode(), response.getResponse()); } catch (Exception e) { LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); LOGGER.error("Caught exception during upload", e); throw e; } } } } }
private SimpleHttpResponse sendRequest(HttpUriRequest request) throws IOException, HttpErrorStatusException { try (CloseableHttpResponse response = _httpClient.execute(request)) { String controllerHost = null; String controllerVersion = null; if (response.containsHeader(CommonConstants.Controller.HOST_HTTP_HEADER)) { controllerHost = response.getFirstHeader(CommonConstants.Controller.HOST_HTTP_HEADER).getValue(); controllerVersion = response.getFirstHeader(CommonConstants.Controller.VERSION_HTTP_HEADER).getValue(); } if (controllerHost != null) { LOGGER.info(String .format("Sending request: %s to controller: %s, version: %s", request.getURI(), controllerHost, controllerVersion)); } int statusCode = response.getStatusLine().getStatusCode(); if (statusCode >= 300) { throw new HttpErrorStatusException(getErrorMessage(request, response), statusCode); } return new SimpleHttpResponse(statusCode, EntityUtils.toString(response.getEntity())); } }
public String getSchema() { FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient(); List<URI> schemaURIs = new ArrayList<>(); try { for (PushLocation pushLocation : _pushLocations) { schemaURIs.add(FileUploadDownloadClient .getRetrieveSchemaHttpURI(pushLocation.getHost(), pushLocation.getPort(), _tableName)); } } catch (URISyntaxException e) { LOGGER.error("Could not construct schema URI for table {}", _tableName); throw new RuntimeException(e); } for (URI schemaURI : schemaURIs) { try { SimpleHttpResponse response = fileUploadDownloadClient.getSchema(schemaURI); if (response != null) { return response.getResponse(); } } catch (Exception e) { LOGGER.warn("Caught exception while trying to get schema for " + _tableName + " " + e); } } LOGGER.error("Could not get schema configs for any push locations provided for " + _tableName); throw new RuntimeException("Could not get schema for table " + _tableName); } }
@Override public Integer call() throws Exception { return fileUploadDownloadClient.uploadSegment(uploadSegmentHttpURI, segmentName, segmentFile) .getStatusCode(); } }));
public TableConfig getTableConfig() { FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient(); List<URI> tableConfigURIs = new ArrayList<>(); try { for (PushLocation pushLocation : _pushLocations) { tableConfigURIs.add(FileUploadDownloadClient .getRetrieveTableConfigURI(pushLocation.getHost(), pushLocation.getPort(), _tableName)); } } catch (URISyntaxException e) { LOGGER.error("Could not construct table config URI for table {}", _tableName); throw new RuntimeException(e); } // Return the first table config it can retrieve for (URI uri : tableConfigURIs) { try { SimpleHttpResponse response = fileUploadDownloadClient.getTableConfig(uri); JsonNode offlineTableConfig = JsonUtils.stringToJsonNode(response.getResponse()).get(OFFLINE); if (offlineTableConfig != null) { LOGGER.info("Got table config {}", offlineTableConfig); return TableConfig.fromJSONConfig(offlineTableConfig); } } catch (Exception e) { LOGGER.warn("Caught exception while trying to get table config for " + _tableName + " " + e); } } LOGGER.error("Could not get table configs from any push locations provided for " + _tableName); throw new RuntimeException("Could not get table config for table " + _tableName); }
@Override public Integer call() throws Exception { return fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(LOCAL_HOST, _controllerPort), downloadUri, httpHeaders, null, 60 * 1000).getStatusCode(); } }));
public void pushOneTarFile(FileSystem fs, Path path) throws Exception { String fileName = path.getName(); if (!fileName.endsWith(JobConfigConstants.TARGZ)) { return; } try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { for (String host : _hosts) { String uri = String.format("%s%s%s", _pushUriPrefix, path.toUri().getRawPath(), _pushUriSuffix); LOGGER .info("******** Uploading file: {} to Host: {} and Port: {} with download uri: {} *******", fileName, host, _port, uri); try { SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(host, _port), uri); LOGGER.info("Response {}: {}", response.getStatusCode(), response.getResponse()); } catch (Exception e) { LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); LOGGER.error("Caught exception during upload", e); throw e; } } } } }
private SegmentCompletionProtocol.Response sendRequest(String url) { SegmentCompletionProtocol.Response response; try { String responseStr = _fileUploadDownloadClient.sendSegmentCompletionProtocolRequest(new URI(url), OTHER_REQUESTS_TIMEOUT) .getResponse(); response = SegmentCompletionProtocol.Response.fromJsonString(responseStr); LOGGER.info("Controller response {} for {}", response.toJsonString(), url); if (response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER)) { ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } } catch (Exception e) { // Catch all exceptions, we want the protocol to handle the case assuming the request was never sent. response = SegmentCompletionProtocol.RESP_NOT_SENT; LOGGER.error("Could not send request {}", url, e); // Invalidate controller leader cache, as exception could be because of leader being down (deployment/failure) and hence unable to send {@link SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER} // If cache is not invalidated, we will not recover from exceptions until the controller comes back up ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } raiseSegmentCompletionProtocolResponseMetric(response); return response; }
/** * Uploads the segment tar to the controller. */ public boolean uploadSegment(String segmentName, File segmentDir, File outputDir) { boolean success = true; File segmentTar = new File(outputDir, segmentName + TAR_SUFFIX); try { TarGzCompressionUtils.createTarGzOfDirectory(segmentDir.getAbsolutePath(), segmentTar.getAbsolutePath()); LOGGER.info("Created tar of {} at {}", segmentDir.getAbsolutePath(), segmentTar.getAbsolutePath()); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { SimpleHttpResponse response = fileUploadDownloadClient.uploadSegment( FileUploadDownloadClient.getUploadSegmentHttpURI(_controllerHost, Integer.parseInt(_controllerPort)), segmentName, segmentTar); int statusCode = response.getStatusCode(); if (statusCode != HttpStatus.SC_OK) { success = false; } LOGGER.info("Uploaded segment: {} and got response {}: {}", segmentName, statusCode, response.getResponse()); } } catch (Exception e) { LOGGER.error("Exception in segment upload {}", segmentTar, e); success = false; } return success; } }
private SegmentCompletionProtocol.Response uploadSegment(String url, final String segmentName, final File segmentTarFile) { SegmentCompletionProtocol.Response response; try { String responseStr = _fileUploadDownloadClient .uploadSegment(new URI(url), segmentName, segmentTarFile, null, null, SEGMENT_UPLOAD_REQUEST_TIMEOUT_MS) .getResponse(); response = SegmentCompletionProtocol.Response.fromJsonString(responseStr); LOGGER.info("Controller response {} for {}", response.toJsonString(), url); if (response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER)) { ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } } catch (Exception e) { // Catch all exceptions, we want the protocol to handle the case assuming the request was never sent. response = SegmentCompletionProtocol.RESP_NOT_SENT; LOGGER.error("Could not send request {}", url, e); // Invalidate controller leader cache, as exception could be because of leader being down (deployment/failure) and hence unable to send {@link SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER} // If cache is not invalidated, we will not recover from exceptions until the controller comes back up ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } raiseSegmentCompletionProtocolResponseMetric(response); return response; }
FileUploadDownloadClient.DEFAULT_SOCKET_TIMEOUT_MS); LOGGER.info("Got response {}: {} while uploading table: {}, segment: {} with uploadURL: {}", response.getStatusCode(), response.getResponse(), tableNameWithType, segmentName, uploadURL); return true; } catch (HttpErrorStatusException e) {
@Test public void testSendFileWithUriAndCrypter() throws Exception { try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { Header crypterClassHeader = new BasicHeader(FileUploadDownloadClient.CustomHeaders.CRYPTER, TEST_CRYPTER); List<Header> headers = Collections.singletonList(crypterClassHeader); List<NameValuePair> params = null; SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(TEST_HOST, TEST_PORT), TEST_URI, headers, params, FileUploadDownloadClient.DEFAULT_SOCKET_TIMEOUT_MS); Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); Assert.assertEquals(response.getResponse(), "OK"); } }
@Test public void testSendFileWithJson() throws Exception { ObjectNode segmentJson = JsonUtils.newObjectNode(); segmentJson.put(CommonConstants.Segment.Offline.DOWNLOAD_URL, TEST_URI); String jsonString = segmentJson.toString(); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentJson(FileUploadDownloadClient.getUploadSegmentHttpURI(TEST_HOST, TEST_PORT), jsonString); Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); Assert.assertEquals(response.getResponse(), "OK"); } }