@Override public synchronized void insert(LocalResource resource) throws HyracksDataException { long id = resource.getId(); if (id2ResourceMap.containsKey(id)) { throw new HyracksDataException("Duplicate resource"); } id2ResourceMap.put(id, resource); name2ResourceMap.put(resource.getPath(), resource); }
private static DatasetResourceReference parse(LocalResource localResource) { final DatasetResourceReference datasetResourceReference = new DatasetResourceReference(); final String filePath = Paths.get(localResource.getPath(), StorageConstants.METADATA_FILE_NAME).toString(); parse(datasetResourceReference, filePath); assignIds(localResource, datasetResourceReference); return datasetResourceReference; }
private void closePartitionResources(int partition) throws HyracksDataException { final PersistentLocalResourceRepository resourceRepository = (PersistentLocalResourceRepository) appCtx.getLocalResourceRepository(); final Map<Long, LocalResource> partitionResources = resourceRepository.getPartitionResources(partition); final IDatasetLifecycleManager datasetLifecycleManager = appCtx.getDatasetLifecycleManager(); for (LocalResource resource : partitionResources.values()) { datasetLifecycleManager.close(resource.getPath()); } }
IndexInfo value = entry.getValue(); ILSMIndex index = value.getIndex(); String path = value.getLocalResource().getPath(); strBuilder.append('{'); strBuilder.append("\"node\":\"");
public List<String> getPartitionReplicatedFiles(int partition, IReplicationStrategy strategy) throws HyracksDataException { final List<String> partitionReplicatedFiles = new ArrayList<>(); final Set<File> replicatedIndexes = new HashSet<>(); final Map<Long, LocalResource> partitionResources = getPartitionResources(partition); for (LocalResource lr : partitionResources.values()) { DatasetLocalResource datasetLocalResource = (DatasetLocalResource) lr.getResource(); if (strategy.isMatch(datasetLocalResource.getDatasetId())) { replicatedIndexes.add(ioManager.resolve(lr.getPath()).getFile()); } } for (File indexDir : replicatedIndexes) { partitionReplicatedFiles.addAll(getIndexFiles(indexDir)); } return partitionReplicatedFiles; }
@Override protected void performOpOnIndex(IIndexDataflowHelper indexHelper, IHyracksTaskContext ctx) throws HyracksDataException { String path = indexHelper.getResource().getPath(); IIOManager ioManager = ctx.getIoManager(); FileReference file = ioManager.resolve(path); LOGGER.warn("performing the operation on " + file.getFile().getAbsolutePath()); // Get index IIndex index = indexHelper.getIndexInstance(); // commit transaction ((ITwoPCIndex) index).commitTransaction(); LOGGER.warn("operation on " + file.getFile().getAbsolutePath() + " Succeded"); } }
/** * Gets a set of files for the indexes in partition {@code partition}. Each file points * to where the index's files are stored. * * @param partition * @return The set of indexes files * @throws HyracksDataException */ public Set<File> getPartitionIndexes(int partition) throws HyracksDataException { final Map<Long, LocalResource> partitionResourcesMap = getResources(resource -> { DatasetLocalResource dsResource = (DatasetLocalResource) resource.getResource(); return dsResource.getPartition() == partition; }); Set<File> indexes = new HashSet<>(); for (LocalResource localResource : partitionResourcesMap.values()) { indexes.add(ioManager.resolve(localResource.getPath()).getFile()); } return indexes; }
@SuppressWarnings("squid:S1181") @Override public synchronized void insert(LocalResource resource) throws HyracksDataException { String relativePath = getFileName(resource.getPath()); FileReference resourceFile = ioManager.resolve(relativePath); if (resourceFile.getFile().exists()) { ExitUtil.halt(ExitUtil.EC_ERROR_CREATING_RESOURCES); resourceCache.put(resource.getPath(), resource);
@Override protected void performOpOnIndex(IIndexDataflowHelper indexDataflowHelper, IHyracksTaskContext ctx) throws HyracksDataException { String path = indexDataflowHelper.getResource().getPath(); IIOManager ioManager = ctx.getIoManager(); FileReference file = ioManager.resolve(path); AbortRecoverLSMIndexFileManager fileManager = new AbortRecoverLSMIndexFileManager(ctx.getIoManager(), file); fileManager.recoverTransaction(); } }
@Override public void open() throws HyracksDataException { //Get local resource file synchronized (lcManager) { index = lcManager.get(resourceRef.getRelativePath()); if (index == null) { LocalResource lr = readIndex(); lcManager.register(lr.getPath(), index); } lcManager.open(resourceRef.getRelativePath()); } }
@Override protected void performOpOnIndex(IIndexDataflowHelper indexDataflowHelper, IHyracksTaskContext ctx) throws HyracksDataException { String path = indexDataflowHelper.getResource().getPath(); IIOManager ioManager = ctx.getIoManager(); FileReference file = ioManager.resolve(path); AbortRecoverLSMIndexFileManager fileManager = new AbortRecoverLSMIndexFileManager(ctx.getIoManager(), file); fileManager.deleteTransactionFiles(); }
index = (ILSMIndex) datasetLifecycleManager.get(localResource.getPath()); if (index == null) { datasetLifecycleManager.register(localResource.getPath(), index); datasetLifecycleManager.open(localResource.getPath()); try { final DatasetResourceReference resourceReference = indexCheckpointManagerProvider.get(resourceReference).getLowWatermark(); } catch (HyracksDataException e) { datasetLifecycleManager.close(localResource.getPath()); throw e; datasetLifecycleManager.close(resourcesMap.get(r).getPath());
@Override public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); IResourceLifecycleManager<IIndex> indexLifeCycleManager = txnSubsystem.getApplicationContext().getDatasetLifecycleManager(); ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); if (index == null) { throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); } try { IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); ITransactionContext txnCtx = txnSubsystem.getTransactionManager() .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(datasetId)); DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); IModificationOperationCallback modCallback = new PrimaryIndexModificationOperationCallback( new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(), aResource.getPartition(), resourceType, indexOp, operatorNodePushable); txnCtx.register(resource.getId(), aResource.getPartition(), index, modCallback, true); return modCallback; } catch (ACIDException e) { throw HyracksDataException.create(e); } } }
@Override public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); IResourceLifecycleManager indexLifeCycleManager = txnSubsystem.getApplicationContext().getDatasetLifecycleManager(); ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); if (index == null) { throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); } try { IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); ITransactionContext txnCtx = txnSubsystem.getTransactionManager() .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(datasetId)); IModificationOperationCallback modCallback = new UpsertOperationCallback(new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(), aResource.getPartition(), resourceType, indexOp); txnCtx.register(resource.getId(), aResource.getPartition(), index, modCallback, true); return modCallback; } catch (ACIDException e) { throw HyracksDataException.create(e); } } }
@Override public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); IResourceLifecycleManager indexLifeCycleManager = txnSubsystem.getApplicationContext().getDatasetLifecycleManager(); ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); if (index == null) { throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); } try { IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); ITransactionContext txnCtx = txnSubsystem.getTransactionManager() .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(datasetId)); DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); IModificationOperationCallback modCallback = new SecondaryIndexModificationOperationCallback( new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(), aResource.getPartition(), resourceType, indexOp); txnCtx.register(resource.getId(), aResource.getPartition(), index, modCallback, false); return modCallback; } catch (ACIDException e) { throw HyracksDataException.create(e); } } }
LocalResource resource = treeIndexHelper.getResource(); IIOManager ioManager = ctx.getIoManager(); FileReference fileRef = ioManager.resolve(resource.getPath()); TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getPageManager(), fileRef, treeIndex.getRootPageId());
@Before public void createIndex() throws Exception { PrimaryIndexInfo primaryIndexInfo = StorageTestUtils.createPrimaryIndex(nc, PARTITION); IndexDataflowHelperFactory iHelperFactory = new IndexDataflowHelperFactory(nc.getStorageManager(), primaryIndexInfo.getFileSplitProvider()); JobId jobId = nc.newJobId(); ctx = nc.createTestContext(jobId, PARTITION, false); indexDataflowHelper = iHelperFactory.create(ctx.getJobletContext().getServiceContext(), PARTITION); indexDataflowHelper.open(); lsmBtree = (TestLsmBtree) indexDataflowHelper.getIndexInstance(); indexDataflowHelper.close(); txnCtx = nc.getTransactionManager().beginTransaction(nc.getTxnJobId(ctx), new TransactionOptions(ITransactionManager.AtomicityLevel.ENTITY_LEVEL)); insertOp = StorageTestUtils.getInsertPipeline(nc, ctx); indexPath = indexDataflowHelper.getResource().getPath(); }