/** * Upload segment with segment file using default settings. * * @param uri URI * @param segmentName Segment name * @param segmentFile Segment file * @return Response * @throws IOException * @throws HttpErrorStatusException */ public SimpleHttpResponse uploadSegment(URI uri, String segmentName, File segmentFile) throws IOException, HttpErrorStatusException { return uploadSegment(uri, segmentName, segmentFile, null, null, DEFAULT_SOCKET_TIMEOUT_MS); }
/** * Upload segment with segment file input stream using default settings. * * @param uri URI * @param segmentName Segment name * @param inputStream Segment file input stream * @return Response * @throws IOException * @throws HttpErrorStatusException */ public SimpleHttpResponse uploadSegment(URI uri, String segmentName, InputStream inputStream) throws IOException, HttpErrorStatusException { return uploadSegment(uri, segmentName, inputStream, null, null, DEFAULT_SOCKET_TIMEOUT_MS); }
private void uploadIndexSegments() throws Exception { if (!_conf.isUploadIndexes()) { LOGGER.info("Skipping upload index segments step."); return; } String indexDirectory = _conf.getIndexDirectory(); File[] indexFiles = new File(indexDirectory).listFiles(); Preconditions.checkNotNull(indexFiles); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { URI uploadSegmentHttpURI = FileUploadDownloadClient.getUploadSegmentHttpURI(_controllerHost, _controllerPort); for (File indexFile : indexFiles) { LOGGER.info("Uploading index segment: {}", indexFile.getAbsolutePath()); fileUploadDownloadClient.uploadSegment(uploadSegmentHttpURI, indexFile.getName(), indexFile); } } }
public void pushOneTarFile(FileSystem fs, Path path) throws Exception { String fileName = path.getName(); if (!fileName.endsWith(".tar.gz")) { return; } try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { for (String host : _hosts) { try (InputStream inputStream = fs.open(path)) { fileName = fileName.split(JobConfigConstants.TARGZ)[0]; LOGGER.info("******** Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); SimpleHttpResponse response = fileUploadDownloadClient .uploadSegment(FileUploadDownloadClient.getUploadSegmentHttpURI(host, _port), fileName, inputStream); LOGGER.info("Response {}: {}", response.getStatusCode(), response.getResponse()); } catch (Exception e) { LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); LOGGER.error("Caught exception during upload", e); throw e; } } } } }
fileUploadDownloadClient.uploadSegment(uploadSegmentHttpURI, tgzFile.getName(), tgzFile);
/** * Uploads the segment tar to the controller. */ public boolean uploadSegment(String segmentName, File segmentDir, File outputDir) { boolean success = true; File segmentTar = new File(outputDir, segmentName + TAR_SUFFIX); try { TarGzCompressionUtils.createTarGzOfDirectory(segmentDir.getAbsolutePath(), segmentTar.getAbsolutePath()); LOGGER.info("Created tar of {} at {}", segmentDir.getAbsolutePath(), segmentTar.getAbsolutePath()); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { SimpleHttpResponse response = fileUploadDownloadClient.uploadSegment( FileUploadDownloadClient.getUploadSegmentHttpURI(_controllerHost, Integer.parseInt(_controllerPort)), segmentName, segmentTar); int statusCode = response.getStatusCode(); if (statusCode != HttpStatus.SC_OK) { success = false; } LOGGER.info("Uploaded segment: {} and got response {}: {}", segmentName, statusCode, response.getResponse()); } } catch (Exception e) { LOGGER.error("Exception in segment upload {}", segmentTar, e); success = false; } return success; } }
@Override public Integer call() throws Exception { return fileUploadDownloadClient.uploadSegment(uploadSegmentHttpURI, segmentName, segmentFile) .getStatusCode(); } }));
try { SimpleHttpResponse response = fileUploadDownloadClient .uploadSegment(new URI(uploadURL), segmentName, fileToUpload, httpHeaders, parameters, FileUploadDownloadClient.DEFAULT_SOCKET_TIMEOUT_MS); LOGGER.info("Got response {}: {} while uploading table: {}, segment: {} with uploadURL: {}",
private SegmentCompletionProtocol.Response uploadSegment(String url, final String segmentName, final File segmentTarFile) { SegmentCompletionProtocol.Response response; try { String responseStr = _fileUploadDownloadClient .uploadSegment(new URI(url), segmentName, segmentTarFile, null, null, SEGMENT_UPLOAD_REQUEST_TIMEOUT_MS) .getResponse(); response = SegmentCompletionProtocol.Response.fromJsonString(responseStr); LOGGER.info("Controller response {} for {}", response.toJsonString(), url); if (response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER)) { ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } } catch (Exception e) { // Catch all exceptions, we want the protocol to handle the case assuming the request was never sent. response = SegmentCompletionProtocol.RESP_NOT_SENT; LOGGER.error("Could not send request {}", url, e); // Invalidate controller leader cache, as exception could be because of leader being down (deployment/failure) and hence unable to send {@link SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER} // If cache is not invalidated, we will not recover from exceptions until the controller comes back up ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } raiseSegmentCompletionProtocolResponseMetric(response); return response; }