public PolicyWorker(String workerIdentifier, CorePolicyApplicationContext applicationContext, LoadingCache<Long, SequenceWorkflow> workflowCache, ApiProvider apiProvider, TaskDataConverter taskDataConverter, TaskStatus taskStatus, final TaskData task, final Codec codec, final byte[] data, final byte[] context, String resultQueue, DataStore dataStore) throws InvalidTaskException { super(task, resultQueue, codec); this.workerIdentifier = workerIdentifier; this.taskDataConverter = taskDataConverter; this.data = data; this.executeTaskData = new ExecuteTaskData(applicationContext, this, new DataStoreSource(dataStore, codec), workflowCache, apiProvider, codec); this.taskStatus = taskStatus; this.context = context; }
public static InputStream retrieveReferencedData(DataStore dataStore, Codec codec, ReferencedData referencedData) throws DataSourceException { DataStoreSource source = new DataStoreSource(dataStore, codec); return referencedData.acquire(source); } }
private void convertReferencedData(Collection<ReferencedData> resultData, BoilerplateResultForTest resultForTest, TestItem<BoilerplateTestInput, BoilerplateTestExpectation> testItem) throws DataSourceException, IOException { Collection testResultData = resultForTest.getData(); if(testResultData== null){ testResultData = new ArrayList<>(); resultForTest.setData(testResultData); } for(ReferencedData data : resultData) { ContentFileTestExpectation expectation = new ContentFileTestExpectation(); expectation.setComparisonType(ContentComparisonType.BINARY); expectation.setExpectedSimilarityPercentage(100); Path savePath = Paths.get(configuration.getTestDataFolder(), testItem.getTag() + ".content"); DataSource dataSource = new DataStoreSource(workerServices.getDataStore(), workerServices.getCodec()); String content = IOUtils.toString(data.acquire(dataSource)); Files.write(savePath, content.getBytes(), StandardOpenOption.CREATE); expectation.setExpectedContentFile(savePath.toString()); testResultData.add(expectation); } }
expectation.setExpectedSimilarityPercentage(100); Path savePath = Paths.get(configuration.getTestDataFolder(), testItem.getTag() + getFileExtension(entry.getKey())); DataSource dataSource = new DataStoreSource(workerServices.getDataStore(), workerServices.getCodec()); String content = IOUtils.toString(entry.getValue().acquire(dataSource)); Files.write(savePath, content.getBytes(), StandardOpenOption.CREATE); DataSource dataSource = new DataStoreSource(workerServices.getDataStore(), workerServices.getCodec()); String content = IOUtils.toString(entry.getValue().acquire(dataSource)); Files.createDirectories(savePath.getParent());
/** * Extracts the signatures from the content of each field in the task. * * @param emailSegregation The email content segregation library. * @param signature The SelectedEmailSignature that specifies the sender of the email. * @return The BoilerplateWorkerResponse with the email signatures extracted and if RedactionType.REMOVE * was set on the task the content with the signatures removed. */ public BoilerplateWorkerResponse extractSignatures(EmailSegregation emailSegregation, SelectedEmailSignature signature) { Multimap<String, ReferencedData> sourceData = task.getSourceData(); DataSource dataSource = new DataStoreSource(dataStore, codec); BoilerplateWorkerResponse boilerplateWorkerResponse = new BoilerplateWorkerResponse(); boilerplateWorkerResponse.setTaskResults(new HashMap<>()); for (Map.Entry<String, ReferencedData> referencedDataEntry : sourceData.entries()) { try { String content = IOUtils.toString(referencedDataEntry.getValue().acquire(dataSource)); boilerplateWorkerResponse.getTaskResults().put(referencedDataEntry.getKey(), extractSignatureForField(content, signature, emailSegregation)); } catch (DataSourceException e) { throw new TaskFailedException("Failed to retrieve content from storage.", e); } catch (IOException e) { throw new TaskFailedException("Failed to read input stream from storage.", e); } catch (InterruptedException | ExecutionException e) { throw new TaskFailedException("Jep Python thread encountered an error.", e); } catch (DataStoreException e) { throw new TaskFailedException("Failed to store content to storage.", e); } } return boilerplateWorkerResponse; }
@Override protected byte[] getOutputContent(TWorkerResult workerResult, TaskMessage message, TestItem<TInput, TExpected> testItem) throws Exception { Path contentFile = null; ReferencedData textData = getContentFunc.apply(workerResult); if (textData != null) { InputStream dataStream = textData.acquire(new DataStoreSource(dataStore, getCodec())); contentFile = saveContentFile(testItem, testItem.getTag(), "result", dataStream); } TExpected expectation = testItem.getExpectedOutputData(); expectation.setExpectedContentFile(contentFile == null ? null : contentFile.toString()); expectation.setExpectedSimilarityPercentage(80); return super.getOutputContent(workerResult, message, testItem); } }
/** * Performs email key content segregation. * * @param selectedEmail Rules defining what the primary, secondary and tertiary content of the email is. * @param jepThreadPool A single thread pool that the Jep/Python code executes on. * @param dataStore The storage service data store. * @param datastoreReference The reference Id to use in the storage service. * @param resultSizeLimit The size limit in bytes that will be encoded into base64. * Content over this limit will be saved to the data store * @return A BoilerplateWorkerResponse with the separated key content. */ private BoilerplateWorkerResponse executeEmailExpressions(SelectedEmail selectedEmail, ExecutorService jepThreadPool, DataStore dataStore, String datastoreReference, int resultSizeLimit) throws TaskRejectedException { EmailSegregation emailSegregation = new EmailSegregation(jepThreadPool, dataStore, datastoreReference, resultSizeLimit); BoilerplateWorkerTask task = getTask(); Multimap<String, ReferencedData> sourceData = task.getSourceData(); DataSource dataSource = new DataStoreSource(dataStore, getCodec()); BoilerplateWorkerResponse boilerplateWorkerResponse = new BoilerplateWorkerResponse(); boilerplateWorkerResponse.setTaskResults(new HashMap<>()); for (Map.Entry<String, ReferencedData> referencedDataEntry : sourceData.entries()) { try { String content = IOUtils.toString(referencedDataEntry.getValue().acquire(dataSource)); boilerplateWorkerResponse.getTaskResults().put(referencedDataEntry.getKey(), emailSegregation.retrieveKeyContent(content, selectedEmail.primaryContent, selectedEmail.secondaryContent, selectedEmail.tertiaryContent)); } catch (DataSourceException e) { throw new TaskRejectedException("Failed to retrieve content from storage", e); } catch (IOException e) { throw new TaskRejectedException("Failed to read input stream from storage", e); } } return boilerplateWorkerResponse; }
InputStream stream = null; try { stream = referencedDataEntry.getValue().acquire(new DataStoreSource(dataStore, codec));
DataSource dataSource = new DataStoreSource(dataStore, getCodec());
DataSource dataSource = new DataStoreSource(dataStore, codec);