/** * Helper method to open an identification request. */ private void openIdentificationRequest(File file) { try { URI resourceUri = file.toURI(); RequestMetaData metadata = new RequestMetaData(file.length(), file.lastModified(), file.getName()); RequestIdentifier requestIdentifier = new RequestIdentifier(resourceUri); identificationRequest = new FileSystemIdentificationRequest(metadata, requestIdentifier); FileInputStream inputStream = new FileInputStream(file); identificationRequest.open(inputStream); } catch (Exception e) { log.error("Could not open identification request.", e); } }
ZipArchiveWalker(RequestIdentifier identifier, ZipFile zipFile) { this.zipFile = zipFile; this.parentId = identifier.getResourceId(); this.parentName = identifier.getUri(); this.originatorNodeId = identifier.getAncestorId(); }
/** * Create instance. * * @param droid async droid. * @param resultHandler result handler(used for directory handling). * @param requestIdentifier ReqIdentifier. */ private FatArchiveWalker(AsynchDroid droid, ResultHandler resultHandler, RequestIdentifier requestIdentifier) { this.droid = droid; this.resultHandler = resultHandler; this.rootParentId = requestIdentifier.getResourceId(); this.fatFileUri = requestIdentifier.getUri(); this.originatorNodeId = requestIdentifier.getNodeId(); }
/** * * @param request the original request. */ public IdentificationResultCollection(IdentificationRequest request) { correlationId = request.getIdentifier().getParentResourceId(); resourceUri = request.getIdentifier().getUri(); }
InputStream in = request.getSourceInputStream(); try { URI parent = request.getIdentifier().getUri(); long correlationId = request.getIdentifier().getNodeId(); final URI uri = ArchiveFileUtils.toGZipUri(parent); final RequestMetaData metaData = new RequestMetaData(null, null, fileName); RequestIdentifier identifier = new RequestIdentifier(uri); identifier.setAncestorId(request.getIdentifier().getAncestorId()); identifier.setParentId(correlationId);
/** * Submits a request for a single file to droid. * @param webArchiveType arc or warc * @param metaData file size, time and name * @param parentName the name of the parent file * @param payload the file input stream * @param correlationId the correlation Id for the request * @param originatorNodeId the Id of the originator node * @throws IOException if the input stream could not be read */ final void submit(String webArchiveType, RequestMetaData metaData, URI parentName, InputStream payload, ResourceId correlationId, long originatorNodeId) throws IOException { RequestIdentifier identifier = new RequestIdentifier(ArchiveFileUtils.toWebArchiveUri(webArchiveType, parentName, metaData.getName())); identifier.setAncestorId(originatorNodeId); identifier.setParentResourceId(correlationId); IdentificationRequest request = factory.newRequest(metaData, identifier); request.open(payload); // get the inputstream back from the record, and write it out to file droidCore.submit(request); }
/** * Submits a request to droid. * @param info the sevenzip entry and stream to submit * @param entryName the name of the entry * @param correlationId the correlation iod for the request * @throws IOException if the input stream could not be read */ final void submit(SevenZEntryInfo info, String entryName, ResourceId correlationId) throws IOException { long size = info.entry.getSize(); Date time = new Date(); RequestMetaData metaData = new RequestMetaData( size == -1 ? null : size, time == null ? null : time.getTime(), entryName); RequestIdentifier identifier = new RequestIdentifier(ArchiveFileUtils.toSevenZUri(parentName, info.entry.getName())); identifier.setAncestorId(originatorNodeId); identifier.setParentResourceId(correlationId); if (identifier.getParentPrefix() != null && identifier.getParentPrefix().isEmpty()) { identifier.setParentPrefix(null); } IdentificationRequest<InputStream> request = factory.newRequest(metaData, identifier); request.open(info.stream); droid.submit(request); }
jobCounter.incrementPostProcess(); RequestIdentifier identifier = request.getIdentifier(); identifier.setResourceId(id); if (identifier.getAncestorId() == null) { identifier.setAncestorId(id.getId()); archiveFormat, request.getIdentifier().getUri().toString(), e.getMessage(), causeMessage); log.warn(message, e); resultHandler.handleError(new IdentificationException( request.getIdentifier().setNodeId(id.getId());
/** * Constructs a new GZip file resource. * @param metaData the name of the tar entry * @param identifier request identification object * @param tempDir the location to write temp files. */ public GZipIdentificationRequest(RequestMetaData metaData, RequestIdentifier identifier, File tempDir) { this.identifier = identifier; String path = identifier.getUri().getSchemeSpecificPart(); extension = ResourceUtils.getExtension(path); fileName = FilenameUtils.getName(path); this.tempDir = tempDir; this.requestMetaData = metaData; }
RequestIdentifier identifier = new RequestIdentifier(nameUri); identifier.setParentId(1L);
final IdentificationRequest request = e.getRequest(); final RequestIdentifier identifier = request.getIdentifier(); final URI uri = identifier.getUri(); final Long nodeId = identifier.getNodeId(); resultHandlerDao.save(node, identifier.getParentResourceId()); progressMonitor.stopJob(node);
.lastModified(), file.getName()); RequestIdentifier identifier = new RequestIdentifier(uri); identifier.setParentResourceId(parentId); identifier.setResourceId(nodeId); IdentificationRequest<File> request = requestFactory.newRequest(metaData, identifier); try {
/** * Handles a directory. * @param dir the directory to handle * @param parentId the directory's parent id * @param depth the depth of the directory in the tree * @param restricted true if access to the directory was restricted, false otherwise * @return the id of the directory */ public ResourceId onEvent(final Path dir, ResourceId parentId, int depth, boolean restricted) { IdentificationResultImpl result = new IdentificationResultImpl(); result.setMethod(IdentificationMethod.NULL); final FileTime lastModified = FileUtil.lastModifiedQuietly(dir); RequestMetaData metaData = new RequestMetaData( FileUtil.sizeQuietly(dir), lastModified == null ? new Date(0).getTime() : new Date(lastModified.toMillis()).getTime(), depth == 0 ? dir.toAbsolutePath().toString() : FileUtil.fileName(dir)); RequestIdentifier identifier = new RequestIdentifier(SubmitterUtils.toURI(dir.toFile(), uriStringBuilder)); identifier.setParentResourceId(parentId); result.setRequestMetaData(metaData); result.setIdentifier(identifier); return resultHandler.handleDirectory(result, parentId, restricted); }
/** * Re-submits all unfinished archival nodes as new Identification requests. * All existing data under those nodes are destroyed. * */ public void replay() { SubmissionQueueData queueData = submissionQueue.list(); if (queueData != null) { // de-duplicate any resources with the same ancestor ID Set<Long> ancestorIds = new HashSet<Long>(); List<RequestIdentifier> identifiers = queueData.getReplayUris(); for (Iterator<RequestIdentifier> it = identifiers.iterator(); it.hasNext();) { RequestIdentifier identifier = it.next(); ancestorIds.add(identifier.getAncestorId()); } // Delete and re-submit all the distinct nodes as new requests for (Long ancestorId : ancestorIds) { ProfileResourceNode node = resultHandlerDao.loadNode(ancestorId); ResourceId parentId = getParentResourceId(node); resultHandlerDao.deleteNode(node.getId()); Path file = Paths.get(node.getUri()); fileEventHandler.onEvent(file, parentId, null); } } }
InputStream in = request.getSourceInputStream(); try { URI parent = request.getIdentifier().getUri(); long correlationId = request.getIdentifier().getNodeId(); final URI uri = ArchiveFileUtils.toBZipUri(parent); final RequestMetaData metaData = new RequestMetaData(null, null, fileName); RequestIdentifier identifier = new RequestIdentifier(uri); identifier.setAncestorId(request.getIdentifier().getAncestorId()); identifier.setParentId(correlationId);
/** * Submits a request for a single file to droid. * @param webArchiveType arc or warc * @param metaData file size, time and name * @param parentName the name of the parent file * @param payload the file input stream * @param correlationId the correlation Id for the request * @param originatorNodeId the Id of the originator node * @throws IOException if the input stream could not be read */ final void submit(String webArchiveType, RequestMetaData metaData, URI parentName, InputStream payload, ResourceId correlationId, long originatorNodeId) throws IOException { RequestIdentifier identifier = new RequestIdentifier(ArchiveFileUtils.toWebArchiveUri(webArchiveType, parentName, metaData.getName())); identifier.setAncestorId(originatorNodeId); identifier.setParentResourceId(correlationId); IdentificationRequest request = factory.newRequest(metaData, identifier); request.open(payload); // get the inputstream back from the record, and write it out to file droidCore.submit(request); }
/** * Submits a request to droid. * @param entry the tar entry to submit * @param entryName the name of the entry * @param correlationId the correlation iod for the request * @throws IOException if the input stream could not be read */ final void submit(SevenZArchiveEntry entry, String entryName, ResourceId correlationId) throws IOException { long size = entry.getSize(); Date time = new Date(); RequestMetaData metaData = new RequestMetaData( size == -1 ? null : size, time == null ? null : time.getTime(), entryName); RequestIdentifier identifier = new RequestIdentifier(ArchiveFileUtils.toSevenZUri(parentName, entry.getName())); identifier.setAncestorId(originatorNodeId); identifier.setParentResourceId(correlationId); if (identifier.getParentPrefix() != null && identifier.getParentPrefix().isEmpty()) { identifier.setParentPrefix(null); } IdentificationRequest<InputStream> request = factory.newRequest(metaData, identifier); BoundedInputStream entryInputStream = new BoundedInputStream(in, entry.getSize()); entryInputStream.setPropagateClose(false); request.open(entryInputStream); if (!entry.isDirectory()) { droid.submit(request); } }
jobCounter.incrementPostProcess(); RequestIdentifier identifier = request.getIdentifier(); identifier.setResourceId(id); if (identifier.getAncestorId() == null) { identifier.setAncestorId(id.getId()); archiveFormat, request.getIdentifier().getUri().toString(), e.getMessage(), causeMessage); log.warn(message, e); resultHandler.handleError(new IdentificationException( request.getIdentifier().setNodeId(id.getId());
/** * Constructs a new BZip file resource. * @param metaData the name of the tar entry * @param identifier request identification object * @param tempDir the location to write temp files. */ public BZipIdentificationRequest(RequestMetaData metaData, RequestIdentifier identifier, File tempDir) { this.identifier = identifier; String path = identifier.getUri().getSchemeSpecificPart(); extension = ResourceUtils.getExtension(path); fileName = FilenameUtils.getName(path); this.tempDir = tempDir; this.requestMetaData = metaData; }
RequestMetaData metaData = new RequestMetaData(file.length(), file.lastModified(), fileName); RequestIdentifier identifier = new RequestIdentifier(uri); identifier.setParentId(1L);