private void uploadIndexSegments() throws Exception { if (!_conf.isUploadIndexes()) { LOGGER.info("Skipping upload index segments step."); return; } String indexDirectory = _conf.getIndexDirectory(); File[] indexFiles = new File(indexDirectory).listFiles(); Preconditions.checkNotNull(indexFiles); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { URI uploadSegmentHttpURI = FileUploadDownloadClient.getUploadSegmentHttpURI(_controllerHost, _controllerPort); for (File indexFile : indexFiles) { LOGGER.info("Uploading index segment: {}", indexFile.getAbsolutePath()); fileUploadDownloadClient.uploadSegment(uploadSegmentHttpURI, indexFile.getName(), indexFile); } } }
public void pushOneTarFile(FileSystem fs, Path path) throws Exception { String fileName = path.getName(); if (!fileName.endsWith(".tar.gz")) { return; } try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { for (String host : _hosts) { try (InputStream inputStream = fs.open(path)) { fileName = fileName.split(JobConfigConstants.TARGZ)[0]; LOGGER.info("******** Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); SimpleHttpResponse response = fileUploadDownloadClient .uploadSegment(FileUploadDownloadClient.getUploadSegmentHttpURI(host, _port), fileName, inputStream); LOGGER.info("Response {}: {}", response.getStatusCode(), response.getResponse()); } catch (Exception e) { LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); LOGGER.error("Caught exception during upload", e); throw e; } } } } }
public void pushOneTarFile(FileSystem fs, Path path) throws Exception { String fileName = path.getName(); if (!fileName.endsWith(JobConfigConstants.TARGZ)) { return; } try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { for (String host : _hosts) { String uri = String.format("%s%s%s", _pushUriPrefix, path.toUri().getRawPath(), _pushUriSuffix); LOGGER .info("******** Uploading file: {} to Host: {} and Port: {} with download uri: {} *******", fileName, host, _port, uri); try { SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(host, _port), uri); LOGGER.info("Response {}: {}", response.getStatusCode(), response.getResponse()); } catch (Exception e) { LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); LOGGER.error("Caught exception during upload", e); throw e; } } } } }
FileUploadDownloadClient.getUploadSegmentHttpURI(_controllerHost, Integer.parseInt(_controllerPort)); for (File file : files) { File tgzFile = file;
/** * Uploads the segment tar to the controller. */ public boolean uploadSegment(String segmentName, File segmentDir, File outputDir) { boolean success = true; File segmentTar = new File(outputDir, segmentName + TAR_SUFFIX); try { TarGzCompressionUtils.createTarGzOfDirectory(segmentDir.getAbsolutePath(), segmentTar.getAbsolutePath()); LOGGER.info("Created tar of {} at {}", segmentDir.getAbsolutePath(), segmentTar.getAbsolutePath()); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { SimpleHttpResponse response = fileUploadDownloadClient.uploadSegment( FileUploadDownloadClient.getUploadSegmentHttpURI(_controllerHost, Integer.parseInt(_controllerPort)), segmentName, segmentTar); int statusCode = response.getStatusCode(); if (statusCode != HttpStatus.SC_OK) { success = false; } LOGGER.info("Uploaded segment: {} and got response {}: {}", segmentName, statusCode, response.getResponse()); } } catch (Exception e) { LOGGER.error("Exception in segment upload {}", segmentTar, e); success = false; } return success; } }
Assert.assertNotNull(segmentNames); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { final URI uploadSegmentHttpURI = FileUploadDownloadClient.getUploadSegmentHttpURI(LOCAL_HOST, _controllerPort);
@Override public Integer call() throws Exception { return fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(LOCAL_HOST, _controllerPort), downloadUri, httpHeaders, null, 60 * 1000).getStatusCode(); } }));
@Test public void testSendFileWithUriAndCrypter() throws Exception { try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { Header crypterClassHeader = new BasicHeader(FileUploadDownloadClient.CustomHeaders.CRYPTER, TEST_CRYPTER); List<Header> headers = Collections.singletonList(crypterClassHeader); List<NameValuePair> params = null; SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(TEST_HOST, TEST_PORT), TEST_URI, headers, params, FileUploadDownloadClient.DEFAULT_SOCKET_TIMEOUT_MS); Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); Assert.assertEquals(response.getResponse(), "OK"); } }
@Test public void testSendFileWithJson() throws Exception { ObjectNode segmentJson = JsonUtils.newObjectNode(); segmentJson.put(CommonConstants.Segment.Offline.DOWNLOAD_URL, TEST_URI); String jsonString = segmentJson.toString(); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentJson(FileUploadDownloadClient.getUploadSegmentHttpURI(TEST_HOST, TEST_PORT), jsonString); Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); Assert.assertEquals(response.getResponse(), "OK"); } }