private void removeIncompleteContent(final String containerName, final Path containerPath, final Path fileToRemove) { if (Files.isDirectory(fileToRemove)) { final Path lastPathName = fileToRemove.subpath(1, fileToRemove.getNameCount()); final String fileName = lastPathName.toFile().getName(); if (fileName.equals(ARCHIVE_DIR_NAME)) { return; } final File[] children = fileToRemove.toFile().listFiles(); if (children != null) { for (final File child : children) { removeIncompleteContent(containerName, containerPath, child.toPath()); } } return; } final Path relativePath = containerPath.relativize(fileToRemove); final Path sectionPath = relativePath.subpath(0, 1); if (relativePath.getNameCount() < 2) { return; } final Path idPath = relativePath.subpath(1, relativePath.getNameCount()); final String id = idPath.toFile().getName(); final String sectionName = sectionPath.toFile().getName(); final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(containerName, sectionName, id, false, false); if (resourceClaimManager.getClaimantCount(resourceClaim) == 0) { removeIncompleteContent(fileToRemove); } }
public static ResourceClaim getResourceClaim(final Record record, final ResourceClaimManager claimManager) { final String container = (String) record.getFieldValue(ContentClaimSchema.CLAIM_CONTAINER); final String section = (String) record.getFieldValue(ContentClaimSchema.CLAIM_SECTION); final String identifier = (String) record.getFieldValue(ContentClaimSchema.CLAIM_IDENTIFIER); final Boolean lossTolerant = (Boolean) record.getFieldValue(ContentClaimSchema.LOSS_TOLERANT); // Make sure that we preserve the existing ResourceClaim, if there is already one held by the Resource Claim Manager // because we need to honor its determination of whether or not the claim is writable. If the Resource Claim Manager // does not have a copy of this Resource Claim, then we can go ahead and just create one and assume that it is not // writable (because if it were writable, then the Resource Claim Manager would know about it). ResourceClaim resourceClaim = claimManager.getResourceClaim(container, section, identifier); if (resourceClaim == null) { resourceClaim = claimManager.newResourceClaim(container, section, identifier, lossTolerant, false); } return resourceClaim; }
@Override public int decrementClaimantCount(final ContentClaim claim) { if (claim == null) { return 0; } return resourceClaimManager.decrementClaimantCount(claim.getResourceClaim()); }
private ContentClaim createLossTolerant() { final long id = idGenerator.getAndIncrement(); final ResourceClaim resourceClaim = claimManager.newResourceClaim(CONTAINER_NAME, "section", String.valueOf(id), true, false); final ContentClaim claim = new StandardContentClaim(resourceClaim, 0L); final ContentBlock contentBlock = new ContentBlock(claim, repoSize); claimManager.incrementClaimantCount(resourceClaim, true); claimMap.put(claim, contentBlock); logger.debug("Created {} and mapped to {}", claim, contentBlock); return claim; }
ResourceClaim resourceClaim = resourceClaimManager.getResourceClaim(event.getPreviousContentClaimContainer(), event.getPreviousContentClaimSection(), event.getPreviousContentClaimIdentifier()); if (resourceClaim == null) { resourceClaim = resourceClaimManager.newResourceClaim(event.getPreviousContentClaimContainer(), event.getPreviousContentClaimSection(), event.getPreviousContentClaimIdentifier(), false, false); resourceClaimManager.incrementClaimantCount(resourceClaim); final long claimOffset = event.getPreviousContentClaimOffset() == null ? 0L : event.getPreviousContentClaimOffset().longValue(); final StandardContentClaim contentClaim = new StandardContentClaim(resourceClaim, claimOffset); resourceClaimManager.decrementClaimantCount(resourceClaim); throw new IllegalStateException("Cannot replay data from Provenance Event because the data is no longer available in the Content Repository");
private ContentClaim createClaim(final String container, final String section, final String identifier, final Long offset) { if (container == null || section == null || identifier == null) { return null; } final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(container, section, identifier, false, false); return new StandardContentClaim(resourceClaim, offset == null ? 0L : offset.longValue()); }
@Override public boolean isInUse() { // Note that it is critical here that we always check isWritable() BEFORE checking // the claimant count. This is due to the fact that if the claim is in fact writable, the claimant count // could increase. So if we first check claimant count and that is 0, and then we check isWritable, it may be // that the claimant count has changed to 1 before checking isWritable. // However, if isWritable() is false, then the only way that the claimant count can increase is if a FlowFile referencing // the Resource Claim is cloned. In this case, though, the claimant count has not become 0. // Said another way, if isWritable() == false, then the claimant count can never increase from 0. return isWritable() || claimManager.getClaimantCount(this) > 0; } }
protected int incrementClaimantCount(final ResourceClaim resourceClaim, final boolean newClaim) { if (resourceClaim == null) { return 0; } return resourceClaimManager.incrementClaimantCount(resourceClaim, newClaim); }
private void markDestructable(final ResourceClaim resourceClaim) { if (resourceClaim == null) { return; } claimManager.markDestructable(resourceClaim); }
@Override public void run() { final List<ResourceClaim> destructable = new ArrayList<>(1000); while (true) { destructable.clear(); claimManager.drainDestructableClaims(destructable, 1000, 5, TimeUnit.SECONDS); if (destructable.isEmpty()) { return; } for (final ResourceClaim claim : destructable) { remove(claim); } } } }
@Override public void purge() { // delete all content from repositories for (final Path path : containers.values()) { FileUtils.deleteFilesInDir(path.toFile(), null, LOG, true); } for (final Path path : containers.values()) { if (!Files.exists(path)) { throw new RepositoryPurgeException("File " + path.toFile().getAbsolutePath() + " does not exist"); } // Try up to 10 times to see if the directory is writable, in case another process (like a // virus scanner) has the directory temporarily locked boolean writable = false; for (int i = 0; i < 10; i++) { if (Files.isWritable(path)) { writable = true; break; } else { try { Thread.sleep(100L); } catch (final Exception e) { } } } if (!writable) { throw new RepositoryPurgeException("File " + path.toFile().getAbsolutePath() + " is not writable"); } } resourceClaimManager.purge(); }
final String claimId = System.currentTimeMillis() + "-" + currentIndex; resourceClaim = resourceClaimManager.newResourceClaim(containerName, section, claimId, lossTolerant, true); resourceOffset = 0L; LOG.debug("Creating new Resource Claim {}", resourceClaim);
private int getClaimantCount(final ContentClaim claim) { if (claim == null) { return 0; } final ResourceClaim resourceClaim = claim.getResourceClaim(); if (resourceClaim == null) { return 0; } return claimManager.getClaimantCount(resourceClaim); }
@Override public int incrementClaimaintCount(final ContentClaim claim) { if (claim == null) { return 0; } final ContentClaim backupClaim = getBackupClaim(claim); if (backupClaim == null) { return claimManager.incrementClaimantCount(resolveClaim(claim).getResourceClaim()); } else { return getBackupRepository().incrementClaimaintCount(backupClaim); } }
private void markDestructable(final ContentClaim contentClaim) { if (contentClaim == null) { return; } final ResourceClaim resourceClaim = contentClaim.getResourceClaim(); if (resourceClaim == null) { return; } claimManager.markDestructable(resourceClaim); }
while (true) { toDestroy.clear(); resourceClaimManager.drainDestructableClaims(toDestroy, 10000); if (toDestroy.isEmpty()) { return;
resourceClaim = claimManager.getResourceClaim(container, section, claimId); if (resourceClaim == null) { logger.error("Swap file indicates that FlowFile was referencing Resource Claim at container={}, section={}, claimId={}, " + "but this Resource Claim cannot be found! Will create a temporary Resource Claim, but this may affect the framework's " + "ability to properly clean up this resource", container, section, claimId); resourceClaim = claimManager.newResourceClaim(container, section, claimId, lossTolerant, true);
final ResourceClaim resourceClaim = claimManager.newResourceClaim(container, section, claimId, lossTolerant, false); final StandardContentClaim contentClaim = new StandardContentClaim(resourceClaim, resourceOffset); contentClaim.setLength(resourceLength);
@Override public int getClaimantCount(final ContentClaim claim) { if (claim == null) { return 0; } return resourceClaimManager.getClaimantCount(claim.getResourceClaim()); }