@Override public ExternalStorageLocation getLocation(Operation operation, PayloadType payloadType, String path) { ExternalStorageLocation location = new ExternalStorageLocation(); location.setUri("http://some/uri"); switch (payloadType) { case TASK_INPUT: location.setPath("task/input"); break; case WORKFLOW_OUTPUT: location.setPath("workflow/output"); break; } return location; }
/** * Uses the {@link PayloadStorage} for storing large payloads. * Gets the uri for storing the payload from the server and then uploads to this location. * * @param payloadType the {@link com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType} to be uploaded * @param payloadBytes the byte array containing the payload * @param payloadSize the size of the payload * @return the path where the payload is stored in external storage */ protected String uploadToExternalPayloadStorage(ExternalPayloadStorage.PayloadType payloadType, byte[] payloadBytes, long payloadSize) { Preconditions.checkArgument(payloadType.equals(ExternalPayloadStorage.PayloadType.WORKFLOW_INPUT) || payloadType.equals(ExternalPayloadStorage.PayloadType.TASK_OUTPUT), "Payload type must be workflow input or task output"); ExternalStorageLocation externalStorageLocation = payloadStorage.getLocation(ExternalPayloadStorage.Operation.WRITE, payloadType, ""); payloadStorage.upload(externalStorageLocation.getUri(), new ByteArrayInputStream(payloadBytes), payloadSize); return externalStorageLocation.getPath(); }
@Before public void setup() { externalPayloadStorage = mock(ExternalPayloadStorage.class); Configuration configuration = new TestConfiguration(); objectMapper = new ObjectMapper(); location = new ExternalStorageLocation(); location.setPath("some/test/path"); externalPayloadStorageUtils = new ExternalPayloadStorageUtils(externalPayloadStorage, configuration); }
@VisibleForTesting String uploadHelper(byte[] payloadBytes, long payloadSize, ExternalPayloadStorage.PayloadType payloadType) { ExternalStorageLocation location = externalPayloadStorage.getLocation(ExternalPayloadStorage.Operation.WRITE, payloadType, ""); externalPayloadStorage.upload(location.getPath(), new ByteArrayInputStream(payloadBytes), payloadSize); return location.getPath(); }
/** * Uses the {@link PayloadStorage} for downloading large payloads to be used by the client. * Gets the uri of the payload fom the server and then downloads from this location. * * @param payloadType the {@link com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType} to be downloaded * @param path the relative of the payload in external storage * @return the payload object that is stored in external storage */ @SuppressWarnings("unchecked") protected Map<String, Object> downloadFromExternalStorage(ExternalPayloadStorage.PayloadType payloadType, String path) { Preconditions.checkArgument(StringUtils.isNotBlank(path), "uri cannot be blank"); ExternalStorageLocation externalStorageLocation = payloadStorage.getLocation(ExternalPayloadStorage.Operation.READ, payloadType, path); try (InputStream inputStream = payloadStorage.download(externalStorageLocation.getUri())) { return objectMapper.readValue(inputStream, Map.class); } catch (IOException e) { String errorMsg = String.format("Unable to download payload frome external storage location: %s", path); logger.error(errorMsg, e); throw new ConductorClientException(errorMsg, e); } }
@Test public void testUploadHelper() { AtomicInteger uploadCount = new AtomicInteger(0); String path = "some/test/path.json"; ExternalStorageLocation location = new ExternalStorageLocation(); location.setPath(path); when(externalPayloadStorage.getLocation(any(), any(), any())).thenReturn(location); doAnswer(invocation -> { uploadCount.incrementAndGet(); return null; }).when(externalPayloadStorage).upload(anyString(), any(), anyLong()); assertEquals(path, externalPayloadStorageUtils.uploadHelper(new byte[]{}, 10L, ExternalPayloadStorage.PayloadType.TASK_OUTPUT)); assertEquals(1, uploadCount.get()); }
@VisibleForTesting String uploadHelper(byte[] payloadBytes, long payloadSize, ExternalPayloadStorage.PayloadType payloadType) { ExternalStorageLocation location = externalPayloadStorage.getLocation(ExternalPayloadStorage.Operation.WRITE, payloadType, ""); externalPayloadStorage.upload(location.getPath(), new ByteArrayInputStream(payloadBytes), payloadSize); return location.getPath(); }
/** * Uses the {@link PayloadStorage} for downloading large payloads to be used by the client. * Gets the uri of the payload fom the server and then downloads from this location. * * @param payloadType the {@link com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType} to be downloaded * @param path the relative of the payload in external storage * @return the payload object that is stored in external storage */ @SuppressWarnings("unchecked") protected Map<String, Object> downloadFromExternalStorage(ExternalPayloadStorage.PayloadType payloadType, String path) { Preconditions.checkArgument(StringUtils.isNotBlank(path), "uri cannot be blank"); ExternalStorageLocation externalStorageLocation = payloadStorage.getLocation(ExternalPayloadStorage.Operation.READ, payloadType, path); try (InputStream inputStream = payloadStorage.download(externalStorageLocation.getUri())) { return objectMapper.readValue(inputStream, Map.class); } catch (IOException e) { String errorMsg = String.format("Unable to download payload frome external storage location: %s", path); logger.error(errorMsg, e); throw new ConductorClientException(errorMsg, e); } }
public ExternalStorageLocation getLocation(Operation operation, PayloadType payloadType, String path) { try { ExternalStorageLocation externalStorageLocation = new ExternalStorageLocation(); objectKey = getObjectKey(payloadType); externalStorageLocation.setPath(objectKey); .withExpiration(expiration); externalStorageLocation.setUri(s3Client.generatePresignedUrl(generatePresignedUrlRequest).toURI().toASCIIString()); return externalStorageLocation; } catch (SdkClientException e) {
/** * Uses the {@link PayloadStorage} for storing large payloads. * Gets the uri for storing the payload from the server and then uploads to this location. * * @param payloadType the {@link com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType} to be uploaded * @param payloadBytes the byte array containing the payload * @param payloadSize the size of the payload * @return the path where the payload is stored in external storage */ protected String uploadToExternalPayloadStorage(ExternalPayloadStorage.PayloadType payloadType, byte[] payloadBytes, long payloadSize) { Preconditions.checkArgument(payloadType.equals(ExternalPayloadStorage.PayloadType.WORKFLOW_INPUT) || payloadType.equals(ExternalPayloadStorage.PayloadType.TASK_OUTPUT), "Payload type must be workflow input or task output"); ExternalStorageLocation externalStorageLocation = payloadStorage.getLocation(ExternalPayloadStorage.Operation.WRITE, payloadType, ""); payloadStorage.upload(externalStorageLocation.getUri(), new ByteArrayInputStream(payloadBytes), payloadSize); return externalStorageLocation.getPath(); }
public ExternalStorageLocation getLocation(Operation operation, PayloadType payloadType, String path) { try { ExternalStorageLocation externalStorageLocation = new ExternalStorageLocation(); objectKey = getObjectKey(payloadType); externalStorageLocation.setPath(objectKey); .withExpiration(expiration); externalStorageLocation.setUri(s3Client.generatePresignedUrl(generatePresignedUrlRequest).toURI().toASCIIString()); return externalStorageLocation; } catch (SdkClientException e) {