public ServerSegmentCompletionProtocolHandler(ServerMetrics serverMetrics) { _fileUploadDownloadClient = new FileUploadDownloadClient(_sslContext); _serverMetrics = serverMetrics; }
protected void initHttpClient(Configuration configs) { _httpClient = new FileUploadDownloadClient(); }
private void uploadIndexSegments() throws Exception { if (!_conf.isUploadIndexes()) { LOGGER.info("Skipping upload index segments step."); return; } String indexDirectory = _conf.getIndexDirectory(); File[] indexFiles = new File(indexDirectory).listFiles(); Preconditions.checkNotNull(indexFiles); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { URI uploadSegmentHttpURI = FileUploadDownloadClient.getUploadSegmentHttpURI(_controllerHost, _controllerPort); for (File indexFile : indexFiles) { LOGGER.info("Uploading index segment: {}", indexFile.getAbsolutePath()); fileUploadDownloadClient.uploadSegment(uploadSegmentHttpURI, indexFile.getName(), indexFile); } } }
public void pushOneTarFile(FileSystem fs, Path path) throws Exception { String fileName = path.getName(); if (!fileName.endsWith(".tar.gz")) { return; } try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { for (String host : _hosts) { try (InputStream inputStream = fs.open(path)) { fileName = fileName.split(JobConfigConstants.TARGZ)[0]; LOGGER.info("******** Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); SimpleHttpResponse response = fileUploadDownloadClient .uploadSegment(FileUploadDownloadClient.getUploadSegmentHttpURI(host, _port), fileName, inputStream); LOGGER.info("Response {}: {}", response.getStatusCode(), response.getResponse()); } catch (Exception e) { LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); LOGGER.error("Caught exception during upload", e); throw e; } } } } }
public void pushOneTarFile(FileSystem fs, Path path) throws Exception { String fileName = path.getName(); if (!fileName.endsWith(JobConfigConstants.TARGZ)) { return; } try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { for (String host : _hosts) { String uri = String.format("%s%s%s", _pushUriPrefix, path.toUri().getRawPath(), _pushUriSuffix); LOGGER .info("******** Uploading file: {} to Host: {} and Port: {} with download uri: {} *******", fileName, host, _port, uri); try { SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(host, _port), uri); LOGGER.info("Response {}: {}", response.getStatusCode(), response.getResponse()); } catch (Exception e) { LOGGER.error("******** Error Uploading file: {} to Host: {} and Port: {} *******", fileName, host, _port); LOGGER.error("Caught exception during upload", e); throw e; } } } } }
Preconditions.checkNotNull(files); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { URI uploadSegmentHttpURI = FileUploadDownloadClient.getUploadSegmentHttpURI(_controllerHost, Integer.parseInt(_controllerPort));
/** * Uploads the segment tar to the controller. */ public boolean uploadSegment(String segmentName, File segmentDir, File outputDir) { boolean success = true; File segmentTar = new File(outputDir, segmentName + TAR_SUFFIX); try { TarGzCompressionUtils.createTarGzOfDirectory(segmentDir.getAbsolutePath(), segmentTar.getAbsolutePath()); LOGGER.info("Created tar of {} at {}", segmentDir.getAbsolutePath(), segmentTar.getAbsolutePath()); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { SimpleHttpResponse response = fileUploadDownloadClient.uploadSegment( FileUploadDownloadClient.getUploadSegmentHttpURI(_controllerHost, Integer.parseInt(_controllerPort)), segmentName, segmentTar); int statusCode = response.getStatusCode(); if (statusCode != HttpStatus.SC_OK) { success = false; } LOGGER.info("Uploaded segment: {} and got response {}: {}", segmentName, statusCode, response.getResponse()); } } catch (Exception e) { LOGGER.error("Exception in segment upload {}", segmentTar, e); success = false; } return success; } }
public String getSchema() { FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient(); List<URI> schemaURIs = new ArrayList<>(); try { for (PushLocation pushLocation : _pushLocations) { schemaURIs.add(FileUploadDownloadClient .getRetrieveSchemaHttpURI(pushLocation.getHost(), pushLocation.getPort(), _tableName)); } } catch (URISyntaxException e) { LOGGER.error("Could not construct schema URI for table {}", _tableName); throw new RuntimeException(e); } for (URI schemaURI : schemaURIs) { try { SimpleHttpResponse response = fileUploadDownloadClient.getSchema(schemaURI); if (response != null) { return response.getResponse(); } } catch (Exception e) { LOGGER.warn("Caught exception while trying to get schema for " + _tableName + " " + e); } } LOGGER.error("Could not get schema configs for any push locations provided for " + _tableName); throw new RuntimeException("Could not get schema for table " + _tableName); } }
@Override protected void initHttpClient(Configuration configs) { SSLContext sslContext = new ClientSSLContextGenerator(configs.subset(CommonConstants.PREFIX_OF_SSL_SUBSET)).generate(); _httpClient = new FileUploadDownloadClient(sslContext); }
public TableConfig getTableConfig() { FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient(); List<URI> tableConfigURIs = new ArrayList<>(); try { for (PushLocation pushLocation : _pushLocations) { tableConfigURIs.add(FileUploadDownloadClient .getRetrieveTableConfigURI(pushLocation.getHost(), pushLocation.getPort(), _tableName)); } } catch (URISyntaxException e) { LOGGER.error("Could not construct table config URI for table {}", _tableName); throw new RuntimeException(e); } // Return the first table config it can retrieve for (URI uri : tableConfigURIs) { try { SimpleHttpResponse response = fileUploadDownloadClient.getTableConfig(uri); JsonNode offlineTableConfig = JsonUtils.stringToJsonNode(response.getResponse()).get(OFFLINE); if (offlineTableConfig != null) { LOGGER.info("Got table config {}", offlineTableConfig); return TableConfig.fromJSONConfig(offlineTableConfig); } } catch (Exception e) { LOGGER.warn("Caught exception while trying to get table config for " + _tableName + " " + e); } } LOGGER.error("Could not get table configs from any push locations provided for " + _tableName); throw new RuntimeException("Could not get table config for table " + _tableName); }
@Override public boolean execute() throws Exception { if (_controllerHost == null) { _controllerHost = NetUtil.getHostAddress(); } if (!_exec) { LOGGER.warn("Dry Running Command: " + toString()); LOGGER.warn("Use the -exec option to actually execute the command."); return true; } File schemaFile = new File(_schemaFile); LOGGER.info("Executing command: " + toString()); if (!schemaFile.exists()) { throw new FileNotFoundException("file does not exist, + " + _schemaFile); } Schema schema = Schema.fromFile(schemaFile); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { fileUploadDownloadClient.addSchema( FileUploadDownloadClient.getUploadSchemaHttpURI(_controllerHost, Integer.parseInt(_controllerPort)), schema.getSchemaName(), schemaFile); } return true; } }
try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient(sslContext)) { retryPolicy.attempt(() -> { try {
@Test public void testSendFileWithUriAndCrypter() throws Exception { try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { Header crypterClassHeader = new BasicHeader(FileUploadDownloadClient.CustomHeaders.CRYPTER, TEST_CRYPTER); List<Header> headers = Collections.singletonList(crypterClassHeader); List<NameValuePair> params = null; SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentUri(FileUploadDownloadClient.getUploadSegmentHttpURI(TEST_HOST, TEST_PORT), TEST_URI, headers, params, FileUploadDownloadClient.DEFAULT_SOCKET_TIMEOUT_MS); Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); Assert.assertEquals(response.getResponse(), "OK"); } }
String[] segmentNames = segmentDir.list(); Assert.assertNotNull(segmentNames); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { final URI uploadSegmentHttpURI = FileUploadDownloadClient.getUploadSegmentHttpURI(LOCAL_HOST, _controllerPort);
Assert.assertNotNull(segmentNames); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) {
@Test public void testSendFileWithJson() throws Exception { ObjectNode segmentJson = JsonUtils.newObjectNode(); segmentJson.put(CommonConstants.Segment.Offline.DOWNLOAD_URL, TEST_URI); String jsonString = segmentJson.toString(); try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { SimpleHttpResponse response = fileUploadDownloadClient .sendSegmentJson(FileUploadDownloadClient.getUploadSegmentHttpURI(TEST_HOST, TEST_PORT), jsonString); Assert.assertEquals(response.getStatusCode(), HttpStatus.SC_OK); Assert.assertEquals(response.getResponse(), "OK"); } }
protected void addSchema(File schemaFile, String schemaName) throws Exception { if (!isUsingNewConfigFormat()) { try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { fileUploadDownloadClient .addSchema(FileUploadDownloadClient.getUploadSchemaHttpURI(LOCAL_HOST, _controllerPort), schemaName, schemaFile); } } else { _schema = Schema.fromFile(schemaFile); } }