private int getDIDfromResourcePath(String resourcePath) throws HyracksDataException { LocalResource lr = resourceRepository.get(resourcePath); if (lr == null) { return -1; } return ((DatasetLocalResource) lr.getResource()).getDatasetId(); }
public Map<Long, LocalResource> getPartitionResources(int partition) throws HyracksDataException { return getResources(resource -> { DatasetLocalResource dsResource = (DatasetLocalResource) resource.getResource(); return dsResource.getPartition() == partition; }); }
private static void assignIds(LocalResource localResource, DatasetResourceReference lrr) { final DatasetLocalResource dsResource = (DatasetLocalResource) localResource.getResource(); lrr.datasetId = dsResource.getDatasetId(); lrr.partitionId = dsResource.getPartition(); lrr.resourceId = localResource.getId(); }
public List<String> getPartitionReplicatedFiles(int partition, IReplicationStrategy strategy) throws HyracksDataException { final List<String> partitionReplicatedFiles = new ArrayList<>(); final Set<File> replicatedIndexes = new HashSet<>(); final Map<Long, LocalResource> partitionResources = getPartitionResources(partition); for (LocalResource lr : partitionResources.values()) { DatasetLocalResource datasetLocalResource = (DatasetLocalResource) lr.getResource(); if (strategy.isMatch(datasetLocalResource.getDatasetId())) { replicatedIndexes.add(ioManager.resolve(lr.getPath()).getFile()); } } for (File indexDir : replicatedIndexes) { partitionReplicatedFiles.addAll(getIndexFiles(indexDir)); } return partitionReplicatedFiles; }
private LocalResource readIndex() throws HyracksDataException { // Get local resource LocalResource lr = getResource(); if (lr == null) { throw HyracksDataException.create(ErrorCode.INDEX_DOES_NOT_EXIST); } IResource resource = lr.getResource(); index = resource.createInstance(ctx); return lr; }
private long getPartitionsMinLSN(Set<Integer> partitions) throws HyracksDataException { final IIndexCheckpointManagerProvider idxCheckpointMgrProvider = appCtx.getIndexCheckpointManagerProvider(); long minRemoteLSN = Long.MAX_VALUE; for (Integer partition : partitions) { final List<DatasetResourceReference> partitionResources = localResourceRepository.getResources(resource -> { DatasetLocalResource dsResource = (DatasetLocalResource) resource.getResource(); return dsResource.getPartition() == partition; }).values().stream().map(DatasetResourceReference::of).collect(Collectors.toList()); for (DatasetResourceReference indexRef : partitionResources) { try { final IIndexCheckpointManager idxCheckpointMgr = idxCheckpointMgrProvider.get(indexRef); if (idxCheckpointMgr.getCheckpointCount() > 0) { long remoteIndexMaxLSN = idxCheckpointMgrProvider.get(indexRef).getLowWatermark(); minRemoteLSN = Math.min(minRemoteLSN, remoteIndexMaxLSN); } } catch (Exception e) { LOGGER.warn("Failed to get min LSN of resource {}", indexRef, e); // ensure no logs will be deleted in case of unexpected failures return SMALLEST_POSSIBLE_LSN; } } } return minRemoteLSN; }
/** * Gets a set of files for the indexes in partition {@code partition}. Each file points * to where the index's files are stored. * * @param partition * @return The set of indexes files * @throws HyracksDataException */ public Set<File> getPartitionIndexes(int partition) throws HyracksDataException { final Map<Long, LocalResource> partitionResourcesMap = getResources(resource -> { DatasetLocalResource dsResource = (DatasetLocalResource) resource.getResource(); return dsResource.getPartition() == partition; }); Set<File> indexes = new HashSet<>(); for (LocalResource localResource : partitionResourcesMap.values()) { indexes.add(ioManager.resolve(localResource.getPath()).getFile()); } return indexes; }
private void checkpointReplicaIndexes(RemoteLogRecord remoteLogMapping, int datasetId) throws HyracksDataException { final Set<Integer> masterPartitions = appCtx.getReplicaManager().getPartitions(); final Predicate<LocalResource> replicaIndexesPredicate = lr -> { DatasetLocalResource dls = (DatasetLocalResource) lr.getResource(); return dls.getDatasetId() == datasetId && !masterPartitions.contains(dls.getPartition()); }; final Map<Long, LocalResource> resources = localResourceRep.getResources(replicaIndexesPredicate); final List<DatasetResourceReference> replicaIndexesRef = resources.values().stream().map(DatasetResourceReference::of).collect(Collectors.toList()); for (DatasetResourceReference replicaIndexRef : replicaIndexesRef) { final IIndexCheckpointManager indexCheckpointManager = indexCheckpointManagerProvider.get(replicaIndexRef); synchronized (indexCheckpointManager) { indexCheckpointManager.masterFlush(remoteLogMapping.getMasterLsn(), remoteLogMapping.getLSN()); } } } }
public long getReplicatedIndexesMaxComponentId(int partition, IReplicationStrategy strategy) throws HyracksDataException { long maxComponentId = LSMComponentId.MIN_VALID_COMPONENT_ID; final Map<Long, LocalResource> partitionResources = getPartitionResources(partition); for (LocalResource lr : partitionResources.values()) { DatasetLocalResource datasetLocalResource = (DatasetLocalResource) lr.getResource(); if (strategy.isMatch(datasetLocalResource.getDatasetId())) { final IIndexCheckpointManager indexCheckpointManager = indexCheckpointManagerProvider.get(DatasetResourceReference.of(lr)); maxComponentId = Math.max(maxComponentId, indexCheckpointManager.getLatest().getLastComponentId()); } } return maxComponentId; }
DatasetLocalResource lr = (DatasetLocalResource) localResourceRepository.get(indexPath).getResource(); int partition = lr.getPartition(); localResourceRepository.cleanup(partition);
public void register(LocalResource resource, ILSMIndex index) throws HyracksDataException { long resourceID = resource.getId(); if (!datasetInfo.isRegistered()) { synchronized (datasetInfo) { if (!datasetInfo.isRegistered()) { datasetInfo.setExternal(index.getNumberOfAllMemoryComponents() == 0); datasetInfo.setRegistered(true); datasetInfo.setDurable(index.isDurable()); } } } if (datasetInfo.getIndexes().containsKey(resourceID)) { throw new HyracksDataException("Index with resource ID " + resourceID + " already exists."); } if (index == null) { throw new HyracksDataException("Attempt to register a null index"); } datasetInfo.addIndex(resourceID, new IndexInfo(index, datasetInfo.getDatasetID(), resource, ((DatasetLocalResource) resource.getResource()).getPartition())); }
localResourceMetadata = (DatasetLocalResource) localResource.getResource(); index = (ILSMIndex) datasetLifecycleManager.get(localResource.getPath()); if (index == null) {
PersistentLocalResourceRepository localResourceRepository = (PersistentLocalResourceRepository) ncAppCtx.getLocalResourceRepository(); DatasetLocalResource lr = (DatasetLocalResource) localResourceRepository.get(indexPath).getResource();
final long resourceId = ncAppCtx.getLocalResourceRepository().get(indexPath).getId(); final DatasetLocalResource datasetLocalResource = (DatasetLocalResource) ncAppCtx.getLocalResourceRepository().get(indexPath).getResource(); final ITransactionContext txnCtx = beingTransaction(ncAppCtx, index, resourceId); final ILogManager logManager = ncAppCtx.getTransactionSubsystem().getLogManager();
@Override public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); IResourceLifecycleManager<IIndex> indexLifeCycleManager = txnSubsystem.getApplicationContext().getDatasetLifecycleManager(); ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); if (index == null) { throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); } try { IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); ITransactionContext txnCtx = txnSubsystem.getTransactionManager() .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(datasetId)); DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); IModificationOperationCallback modCallback = new PrimaryIndexModificationOperationCallback( new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(), aResource.getPartition(), resourceType, indexOp, operatorNodePushable); txnCtx.register(resource.getId(), aResource.getPartition(), index, modCallback, true); return modCallback; } catch (ACIDException e) { throw HyracksDataException.create(e); } } }
@Override public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); IResourceLifecycleManager indexLifeCycleManager = txnSubsystem.getApplicationContext().getDatasetLifecycleManager(); ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); if (index == null) { throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); } try { IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); ITransactionContext txnCtx = txnSubsystem.getTransactionManager() .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(datasetId)); IModificationOperationCallback modCallback = new UpsertOperationCallback(new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(), aResource.getPartition(), resourceType, indexOp); txnCtx.register(resource.getId(), aResource.getPartition(), index, modCallback, true); return modCallback; } catch (ACIDException e) { throw HyracksDataException.create(e); } } }
@Override public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); IResourceLifecycleManager indexLifeCycleManager = txnSubsystem.getApplicationContext().getDatasetLifecycleManager(); ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); if (index == null) { throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); } try { IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); ITransactionContext txnCtx = txnSubsystem.getTransactionManager() .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(datasetId)); DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); IModificationOperationCallback modCallback = new SecondaryIndexModificationOperationCallback( new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(), aResource.getPartition(), resourceType, indexOp); txnCtx.register(resource.getId(), aResource.getPartition(), index, modCallback, false); return modCallback; } catch (ACIDException e) { throw HyracksDataException.create(e); } } }