private BasicAWSCredentials getCredentialsForBucket(String bucketName) { if (configuration.getS3BucketCredentials().containsKey(bucketName)) { return configuration.getS3BucketCredentials().get(bucketName).toAWSCredentials(); } return new BasicAWSCredentials(configuration.getS3AccessKey().get(), configuration.getS3SecretKey().get()); }
@Inject public SingularityExecutorArtifactFetcher(@Named(SingularityExecutorModule.LOCAL_DOWNLOAD_HTTP_CLIENT) AsyncHttpClient localDownloadHttpClient, SingularityS3Configuration s3Configuration, SingularityExecutorConfiguration executorConfiguration, ObjectMapper objectMapper, SingularityRunnerExceptionNotifier exceptionNotifier, SingularityRunnerBaseConfiguration runnerBaseConfiguration) { this.localDownloadHttpClient = localDownloadHttpClient; this.executorConfiguration = executorConfiguration; this.s3Configuration = s3Configuration; this.objectMapper = objectMapper; this.exceptionNotifier = exceptionNotifier; this.runnerBaseConfiguration = runnerBaseConfiguration; this.localDownloadUri = String.format(LOCAL_DOWNLOAD_STRING_FORMAT, s3Configuration.getLocalDownloadHttpPort(), s3Configuration.getLocalDownloadPath()); }
public AbstractFileMetricsReporter(MetricRegistry registry, SingularityS3Configuration configuration, ObjectMapper metricsObjectMapper) { this.registry = registry; this.configuration = configuration; this.metricsObjectMapper = metricsObjectMapper; this.fileReporterExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("metrics-file-reporter").build()); if (configuration.getMetricsFilePath().isPresent()) { startFileReporter(); } }
@Override public void startAndWait() { Preconditions.checkState(!server.isPresent()); Preconditions.checkState(s3Configuration.getS3AccessKey().isPresent(), "s3AccessKey not set!"); Preconditions.checkState(s3Configuration.getS3SecretKey().isPresent(), "s3SecretKey not set!"); Server server = new Server(); ServerConnector http = new ServerConnector(server); http.setHost("localhost"); http.setPort(s3Configuration.getLocalDownloadHttpPort()); http.setIdleTimeout(configuration.getHttpServerTimeout()); server.addConnector(http); server.setHandler(handler); try { LOG.info("Starting server on {} (configuration: {})", s3Configuration.getLocalDownloadHttpPort(), configuration); server.start(); this.server = Optional.of(server); server.join(); } catch (Throwable t) { throw Throwables.propagate(t); } }
bucketCreds = Optional.of(configuration.getS3BucketCredentials().get(metadata.getS3Bucket()).toAWSCredentials()); final BasicAWSCredentials defaultCredentials = new BasicAWSCredentials(configuration.getS3AccessKey().or(s3Configuration.getS3AccessKey()).get(), configuration.getS3SecretKey().or(s3Configuration.getS3SecretKey()).get());
@Override public void startAndWait() { if (!configuration.getS3AccessKey().or(s3Configuration.getS3AccessKey()).isPresent()) { throw new MissingConfigException("s3AccessKey not set in any s3 configs!"); if (!configuration.getS3SecretKey().or(s3Configuration.getS3SecretKey()).isPresent()) { throw new MissingConfigException("s3SecretKey not set in any s3 configs!");
.withSocketTimeout(configuration.getS3ChunkDownloadTimeoutMillis()); if (configuration.isS3UseV2Signing()) { clientConfiguration.setSignerOverride("S3SignerType"); if (configuration.getS3Endpoint().isPresent()) { s3Client.setEndpoint(configuration.getS3Endpoint().get()); int numChunks = (int) (length / configuration.getS3ChunkSize()); if (length % configuration.getS3ChunkSize() > 0) { numChunks++; long remainingMillis = configuration.getS3DownloadTimeoutMillis(); boolean failed = false;
public ArtifactManager(SingularityRunnerBaseConfiguration runnerBaseConfiguration, SingularityS3Configuration configuration, Logger log, SingularityRunnerExceptionNotifier exceptionNotifier) { super(log); this.cacheDirectory = Paths.get(configuration.getArtifactCacheDirectory()); this.log = log; this.s3ArtifactDownloader = new S3ArtifactDownloader(configuration, log, exceptionNotifier); this.useCompressProgram = runnerBaseConfiguration.getUseCompressProgram(); }
@Before public void setup() { SingularityRunnerBaseConfiguration baseConfig = new SingularityRunnerBaseConfiguration(); SingularityS3Configuration s3Config = new SingularityS3Configuration(); s3Config.setArtifactCacheDirectory(cacheDir.toString()); artifactManager = new ArtifactManager( baseConfig, s3Config, LoggerFactory.getLogger(ArtifactManagerTest.class), new SingularityRunnerExceptionNotifier(baseConfig) ); }
@Override public Path call() throws Exception { final long start = System.currentTimeMillis(); final ExecutorService chunkExecutorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(true).setNameFormat("S3ArtifactDownloaderChunk-" + chunk + "-Thread-%d").build()); try { while (retryNum <= configuration.getS3ChunkRetries()) { final long timeout = System.currentTimeMillis(); final Future<Path> future = chunkExecutorService.submit(createDownloader(retryNum)); try { return future.get(configuration.getS3ChunkDownloadTimeoutMillis(), TimeUnit.MILLISECONDS); } catch (TimeoutException te) { log.error("Chunk {} (retry {}) for {} timed out after {} - total duration {}", chunk, retryNum, s3Artifact.getFilename(), JavaUtils.duration(timeout), JavaUtils.duration(start)); future.cancel(true); if (retryNum == configuration.getS3ChunkRetries()) { exceptionNotifier.notify("Timeout downloading chunk", te, ImmutableMap.of("filename", s3Artifact.getFilename(), "chunk", Integer.toString(chunk), "retry", Integer.toString(retryNum))); } } catch (InterruptedException ie) { log.warn("Chunk {} (retry {}) for {} interrupted", chunk, retryNum, s3Artifact.getFilename()); exceptionNotifier.notify("Interrupted during download", ie, ImmutableMap.of("filename", s3Artifact.getFilename(), "chunk", Integer.toString(chunk), "retry", Integer.toString(retryNum))); } catch (Throwable t) { log.error("Error while downloading chunk {} (retry {}) for {}", chunk, retryNum, s3Artifact.getFilename(), t); exceptionNotifier.notify(String.format("Error downloading chunk (%s)", t.getMessage()), t, ImmutableMap.of("filename", s3Artifact.getFilename(), "chunk", Integer.toString(chunk), "retry", Integer.toString(retryNum))); } retryNum++; } throw new IllegalStateException(String.format("Chunk %s for %s failed to download after %s tries", chunk, s3Artifact.getFilename(), retryNum + 1)); } finally { chunkExecutorService.shutdownNow(); } }
@Override public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { metrics.getRequestsMeter().mark(); if (!target.equals(s3Configuration.getLocalDownloadPath())) { metrics.getClientErrorsMeter().mark(); response.sendError(404); return; } if (!request.getMethod().equalsIgnoreCase(HttpMethod.POST.name())) { metrics.getClientErrorsMeter().mark(); response.sendError(405); return; } Optional<ArtifactDownloadRequest> artifactOptional = readDownloadRequest(request); if (!artifactOptional.isPresent()) { metrics.getClientErrorsMeter().mark(); response.sendError(400); return; } Continuation continuation = ContinuationSupport.getContinuation(request); continuation.suspend(response); if (artifactOptional.get().getTimeoutMillis().isPresent()) { continuation.setTimeout(artifactOptional.get().getTimeoutMillis().get()); } downloaderCoordinator.register(continuation, artifactOptional.get()); }
private void checkArtifactSignature(S3ArtifactSignature s3ArtifactSignature) { final Path artifactPath = Paths.get(s3Configuration.getArtifactCacheDirectory(), s3ArtifactSignature.getArtifactFilename()); final Path artifactSignaturePath = Paths.get(s3Configuration.getArtifactCacheDirectory(), s3ArtifactSignature.getFilename());
private void startFileReporter() { fileReporterExecutor.scheduleAtFixedRate(() -> { File metricsFile = new File(configuration.getMetricsFilePath().get()); try (Writer metricsFileWriter = new FileWriter(metricsFile, false)) { metricsFileWriter.write(metricsObjectMapper.writeValueAsString(registry.getMetrics())); metricsFileWriter.flush(); } catch (IOException e) { LOG.error("Unable to write metrics to file", e); } }, 10, 30, TimeUnit.SECONDS); }