public static Collection<String> getS3KeyPrefixes(String s3KeyFormat, String requestId, String deployId, Optional<String> tag, long start, long end, String group) { String keyFormat = getS3KeyFormat(s3KeyFormat, requestId, deployId, tag, group); keyFormat = trimTaskId(keyFormat, requestId + "-" + deployId); return getS3KeyPrefixes(keyFormat, DISALLOWED_FOR_DEPLOY, start, end); }
private static Collection<String> getS3KeyPrefixes(String s3KeyFormat, List<String> disallowedKeys, long start, long end) { String trimKeyFormat = trimKeyFormat(s3KeyFormat, disallowedKeys); keyBuilder.replace(indexOfY, indexOfY + 4, getYear(calendar.get(Calendar.YEAR))); keyBuilder.replace(indexOfM, indexOfM + 2, getDayOrMonth(getMonth(calendar))); keyBuilder.replace(indexOfD, indexOfD + 2, getDayOrMonth(calendar.get(Calendar.DAY_OF_MONTH)));
public static Collection<String> getS3KeyPrefixes(String s3KeyFormat, SingularityTaskId taskId, Optional<String> tag, long start, long end, String group) { String keyFormat = getS3KeyFormat(s3KeyFormat, taskId, tag, group); return getS3KeyPrefixes(keyFormat, DISALLOWED_FOR_TASK, start, end); }
s3KeyFormat = s3KeyFormat.replace("%Y", getYear(calendar.get(Calendar.YEAR))); s3KeyFormat = s3KeyFormat.replace("%m", getDayOrMonth(getMonth(calendar))); s3KeyFormat = s3KeyFormat.replace("%d", getDayOrMonth(calendar.get(Calendar.DAY_OF_MONTH)));
private Collection<String> getS3PrefixesForRequest(S3Configuration s3Configuration, String requestId, Optional<Long> startArg, Optional<Long> endArg, String group) { Optional<SingularityRequestHistory> firstHistory = requestHistoryHelper.getFirstHistory(requestId); checkNotFound(firstHistory.isPresent(), "No request history found for %s", requestId); long start = firstHistory.get().getCreatedAt(); if (startArg.isPresent()) { start = Math.max(startArg.get(), start); } Optional<SingularityRequestHistory> lastHistory = requestHistoryHelper.getLastHistory(requestId); long end = System.currentTimeMillis(); if (lastHistory.isPresent() && (lastHistory.get().getEventType() == RequestHistoryType.DELETED || lastHistory.get().getEventType() == RequestHistoryType.PAUSED)) { end = lastHistory.get().getCreatedAt() + TimeUnit.DAYS.toMillis(1); } if (endArg.isPresent()) { end = Math.min(endArg.get(), end); } Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, start, end, group); for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, start, end, group)); } } LOG.trace("Request {} got S3 prefixes {} for start {}, end {}", requestId, prefixes, start, end); return prefixes; }
@Test public void testS3FormatHelper() throws Exception { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); SingularityTaskId taskId = new SingularityTaskId("rid", "did", 1, 1, "host", "rack"); long start = 1414610537117L; // Wed, 29 Oct 2014 19:22:17 GMT long end = 1415724215000L; // Tue, 11 Nov 2014 16:43:35 GMT Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%m/%taskId", taskId, Optional.<String> absent(), start, end, "default"); Assert.assertTrue(prefixes.size() == 2); end = 1447265861000L; // Tue, 11 Nov 2015 16:43:35 GMT prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%taskId", taskId, Optional.<String> absent(), start, end, "default"); Assert.assertTrue(prefixes.size() == 2); start = 1415750399999L; end = 1415771999000L; prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%m/%d/%taskId", taskId, Optional.<String> absent(), start, end, "default"); Assert.assertTrue(prefixes.size() == 2); prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%requestId/%group/%Y/%m", taskId, Optional.<String> absent(), start, end, "groupName"); Assert.assertEquals("rid/groupName/2014/11", prefixes.iterator().next()); final long NOV2014TUES11 = 1415724215000L; Assert.assertEquals("wat-hostname", SingularityS3FormatHelper.getKey("wat-%host", 0, System.currentTimeMillis(), "filename", "hostname")); Assert.assertEquals("file1.txt-2", SingularityS3FormatHelper.getKey("%filename-%index", 2, System.currentTimeMillis(), "file1.txt", "hostname")); Assert.assertEquals("yo-2014-11-" + NOV2014TUES11 + "-.txt", SingularityS3FormatHelper.getKey("yo-%Y-%m-%s-%fileext", 2, NOV2014TUES11, "file1.txt", "hostname")); String guid = SingularityS3FormatHelper.getKey("yo-%guid", 2, NOV2014TUES11, "file1.txt", "hostname"); Assert.assertTrue(guid.startsWith("yo-")); Assert.assertTrue(guid.length() > 10); String guid2 = SingularityS3FormatHelper.getKey("yo-%guid", 2, NOV2014TUES11, "file1.txt", "hostname"); Assert.assertTrue(!guid.equals(guid2)); }
public static String getS3KeyFormat(String s3KeyFormat, String requestId, String deployId, Optional<String> loggingTag, String group) { s3KeyFormat = getS3KeyFormat(s3KeyFormat, requestId, group); s3KeyFormat = s3KeyFormat.replace("%tag", loggingTag.or("")); s3KeyFormat = s3KeyFormat.replace("%deployId", deployId); return s3KeyFormat; }
final long start = System.currentTimeMillis(); final String key = SingularityS3FormatHelper.getKey(uploadMetadata.getS3KeyFormat(), sequence, Files.getLastModifiedTime(file).toMillis(), Objects.toString(file.getFileName()), hostname);
private Collection<String> getS3PrefixesForDeploy(S3Configuration s3Configuration, String requestId, String deployId, Optional<Long> startArg, Optional<Long> endArg, String group, SingularityUser user) { SingularityDeployHistory deployHistory = getDeployHistory(requestId, deployId, user); long start = deployHistory.getDeployMarker().getTimestamp(); if (startArg.isPresent()) { start = Math.max(startArg.get(), start); } long end = System.currentTimeMillis(); if (!isCurrentDeploy(requestId, deployId) && deployHistory.getDeployStatistics().isPresent() && deployHistory.getDeployStatistics().get().getLastFinishAt().isPresent()) { end = deployHistory.getDeployStatistics().get().getLastFinishAt().get() + TimeUnit.DAYS.toMillis(1); } if (endArg.isPresent()) { end = Math.min(endArg.get(), end); } Optional<String> tag = Optional.absent(); if (deployHistory.getDeploy().isPresent() && deployHistory.getDeploy().get().getExecutorData().isPresent()) { tag = deployHistory.getDeploy().get().getExecutorData().get().getLoggingTag(); } Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, deployId, tag, start, end, group); for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, deployId, tag, start, end, group)); } } LOG.trace("Request {}, deploy {} got S3 prefixes {} for start {}, end {}, tag {}", requestId, deployId, prefixes, start, end, tag); return prefixes; }
@Test public void testS3FormatHelper() throws Exception { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); SingularityTaskId taskId = new SingularityTaskId("rid", "did", 1, 1, "host", "rack"); long start = 1414610537117L; // Wed, 29 Oct 2014 19:22:17 GMT long end = 1415724215000L; // Tue, 11 Nov 2014 16:43:35 GMT Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%m/%taskId", taskId, Optional.<String> absent(), start, end, "default"); Assert.assertTrue(prefixes.size() == 2); end = 1447265861000L; // Tue, 11 Nov 2015 16:43:35 GMT prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%taskId", taskId, Optional.<String> absent(), start, end, "default"); Assert.assertTrue(prefixes.size() == 2); start = 1415750399999L; end = 1415771999000L; prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%m/%d/%taskId", taskId, Optional.<String> absent(), start, end, "default"); Assert.assertTrue(prefixes.size() == 2); prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%requestId/%group/%Y/%m", taskId, Optional.<String> absent(), start, end, "groupName"); Assert.assertEquals("rid/groupName/2014/11", prefixes.iterator().next()); final long NOV2014TUES11 = 1415724215000L; Assert.assertEquals("wat-hostname", SingularityS3FormatHelper.getKey("wat-%host", 0, System.currentTimeMillis(), "filename", "hostname")); Assert.assertEquals("file1.txt-2", SingularityS3FormatHelper.getKey("%filename-%index", 2, System.currentTimeMillis(), "file1.txt", "hostname")); Assert.assertEquals("yo-2014-11-" + NOV2014TUES11 + "-.txt", SingularityS3FormatHelper.getKey("yo-%Y-%m-%s-%fileext", 2, NOV2014TUES11, "file1.txt", "hostname")); String guid = SingularityS3FormatHelper.getKey("yo-%guid", 2, NOV2014TUES11, "file1.txt", "hostname"); Assert.assertTrue(guid.startsWith("yo-")); Assert.assertTrue(guid.length() > 10); String guid2 = SingularityS3FormatHelper.getKey("yo-%guid", 2, NOV2014TUES11, "file1.txt", "hostname"); Assert.assertTrue(!guid.equals(guid2)); }
public static String getS3KeyFormat(String s3KeyFormat, SingularityTaskId taskId, Optional<String> loggingTag, String group) { s3KeyFormat = getS3KeyFormat(s3KeyFormat, taskId.getRequestId(), taskId.getDeployId(), loggingTag, group); s3KeyFormat = s3KeyFormat.replace("%host", taskId.getSanitizedHost()); s3KeyFormat = s3KeyFormat.replace("%taskId", taskId.toString()); return s3KeyFormat; }
final long start = System.currentTimeMillis(); final String key = SingularityS3FormatHelper.getKey(uploadMetadata.getS3KeyFormat(), sequence, Files.getLastModifiedTime(file).toMillis(), Objects.toString(file.getFileName()), hostname);
public static Collection<String> getS3KeyPrefixes(String s3KeyFormat, String requestId, long start, long end, String group) { s3KeyFormat = getS3KeyFormat(s3KeyFormat, requestId, group); s3KeyFormat = trimTaskId(s3KeyFormat, requestId); return getS3KeyPrefixes(s3KeyFormat, DISALLOWED_FOR_REQUEST, start, end); }
Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), taskId, tag, start, end, group); for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), taskId, tag, start, end, group));
private String getS3KeyPattern(String s3KeyPattern) { final SingularityTaskId singularityTaskId = getSingularityTaskId(); return SingularityS3FormatHelper.getS3KeyFormat(s3KeyPattern, singularityTaskId, taskDefinition.getExecutorData().getLoggingTag(), taskDefinition.getExecutorData().getRequestGroup().or(SingularityS3FormatHelper.DEFAULT_GROUP_NAME)); }
private Collection<String> getS3PrefixesForRequest(S3Configuration s3Configuration, String requestId, Optional<Long> startArg, Optional<Long> endArg, String group) { Optional<SingularityRequestHistory> firstHistory = requestHistoryHelper.getFirstHistory(requestId); checkNotFound(firstHistory.isPresent(), "No request history found for %s", requestId); long start = firstHistory.get().getCreatedAt(); if (startArg.isPresent()) { start = Math.max(startArg.get(), start); } Optional<SingularityRequestHistory> lastHistory = requestHistoryHelper.getLastHistory(requestId); long end = System.currentTimeMillis(); if (lastHistory.isPresent() && (lastHistory.get().getEventType() == RequestHistoryType.DELETED || lastHistory.get().getEventType() == RequestHistoryType.PAUSED)) { end = lastHistory.get().getCreatedAt() + TimeUnit.DAYS.toMillis(1); } if (endArg.isPresent()) { end = Math.min(endArg.get(), end); } Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, start, end, group); for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, start, end, group)); } } LOG.trace("Request {} got S3 prefixes {} for start {}, end {}", requestId, prefixes, start, end); return prefixes; }
private Collection<String> getS3PrefixesForDeploy(S3Configuration s3Configuration, String requestId, String deployId, Optional<Long> startArg, Optional<Long> endArg, String group, SingularityUser user) { SingularityDeployHistory deployHistory = getDeployHistory(requestId, deployId, user); long start = deployHistory.getDeployMarker().getTimestamp(); if (startArg.isPresent()) { start = Math.max(startArg.get(), start); } long end = System.currentTimeMillis(); if (!isCurrentDeploy(requestId, deployId) && deployHistory.getDeployStatistics().isPresent() && deployHistory.getDeployStatistics().get().getLastFinishAt().isPresent()) { end = deployHistory.getDeployStatistics().get().getLastFinishAt().get() + TimeUnit.DAYS.toMillis(1); } if (endArg.isPresent()) { end = Math.min(endArg.get(), end); } Optional<String> tag = Optional.absent(); if (deployHistory.getDeploy().isPresent() && deployHistory.getDeploy().get().getExecutorData().isPresent()) { tag = deployHistory.getDeploy().get().getExecutorData().get().getLoggingTag(); } Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, deployId, tag, start, end, group); for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, deployId, tag, start, end, group)); } } LOG.trace("Request {}, deploy {} got S3 prefixes {} for start {}, end {}, tag {}", requestId, deployId, prefixes, start, end, tag); return prefixes; }
Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), taskId, tag, start, end, group); for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), taskId, tag, start, end, group));