public IndexInfo(ILSMIndex index, int datasetId, LocalResource localResource, int partition) { this.index = index; this.datasetId = datasetId; this.localResource = localResource; this.resourceId = localResource.getId(); this.partition = partition; }
@Override public synchronized void insert(LocalResource resource) throws HyracksDataException { long id = resource.getId(); if (id2ResourceMap.containsKey(id)) { throw new HyracksDataException("Duplicate resource"); } id2ResourceMap.put(id, resource); name2ResourceMap.put(resource.getPath(), resource); }
private static void assignIds(LocalResource localResource, DatasetResourceReference lrr) { final DatasetLocalResource dsResource = (DatasetLocalResource) localResource.getResource(); lrr.datasetId = dsResource.getDatasetId(); lrr.partitionId = dsResource.getPartition(); lrr.resourceId = localResource.getId(); }
/** * Gets a set of files for the indexes in partition {@code partition}. Each file points * to where the index's files are stored. * * @param partition * @return The set of indexes files * @throws HyracksDataException */ public Set<File> getPartitionIndexes(int partition) throws HyracksDataException { final Map<Long, LocalResource> partitionResourcesMap = getResources(resource -> { DatasetLocalResource dsResource = (DatasetLocalResource) resource.getResource(); return dsResource.getPartition() == partition; }); Set<File> indexes = new HashSet<>(); for (LocalResource localResource : partitionResourcesMap.values()) { indexes.add(ioManager.resolve(localResource.getPath()).getFile()); } return indexes; }
@Override public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); IResourceLifecycleManager<IIndex> indexLifeCycleManager = txnSubsystem.getApplicationContext().getDatasetLifecycleManager(); ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); if (index == null) { throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); } try { IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); ITransactionContext txnCtx = txnSubsystem.getTransactionManager() .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(datasetId)); DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); IModificationOperationCallback modCallback = new PrimaryIndexModificationOperationCallback( new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(), aResource.getPartition(), resourceType, indexOp, operatorNodePushable); txnCtx.register(resource.getId(), aResource.getPartition(), index, modCallback, true); return modCallback; } catch (ACIDException e) { throw HyracksDataException.create(e); } } }
private static DatasetResourceReference parse(LocalResource localResource) { final DatasetResourceReference datasetResourceReference = new DatasetResourceReference(); final String filePath = Paths.get(localResource.getPath(), StorageConstants.METADATA_FILE_NAME).toString(); parse(datasetResourceReference, filePath); assignIds(localResource, datasetResourceReference); return datasetResourceReference; }
private int getDIDfromResourcePath(String resourcePath) throws HyracksDataException { LocalResource lr = resourceRepository.get(resourcePath); if (lr == null) { return -1; } return ((DatasetLocalResource) lr.getResource()).getDatasetId(); }
long resourceId = lr == null ? -1 : lr.getId(); if (resourceId != -1) { localResourceRepository.delete(resourceRef.getRelativePath()); lr = new LocalResource(resourceId, ITreeIndexFrame.Constants.VERSION, durable, resource); IIndex index = lcManager.get(resourceRef.getRelativePath()); if (index != null) {
@SuppressWarnings("squid:S1181") @Override public synchronized void insert(LocalResource resource) throws HyracksDataException { String relativePath = getFileName(resource.getPath()); FileReference resourceFile = ioManager.resolve(relativePath); if (resourceFile.getFile().exists()) { byte[] bytes = OBJECT_MAPPER.writeValueAsBytes(resource.toJson(persistedResourceRegistry)); final Path path = Paths.get(resourceFile.getAbsolutePath()); Files.write(path, bytes); ExitUtil.halt(ExitUtil.EC_ERROR_CREATING_RESOURCES); resourceCache.put(resource.getPath(), resource);
public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json) throws HyracksDataException { final long id = json.get("id").asLong(); final int version = json.get("version").asInt(); final boolean durable = json.get("durable").asBoolean(); final IResource resource = (IResource) registry.deserialize(json.get("resource")); return new LocalResource(id, version, durable, resource); } }
private LocalResource readLocalResource(File file) throws HyracksDataException { final Path path = Paths.get(file.getAbsolutePath()); try { final JsonNode jsonNode = OBJECT_MAPPER.readValue(Files.readAllBytes(path), JsonNode.class); LocalResource resource = (LocalResource) persistedResourceRegistry.deserialize(jsonNode); if (resource.getVersion() == ITreeIndexFrame.Constants.VERSION) { return resource; } else { throw new AsterixException("Storage version mismatch."); } } catch (Exception e) { throw HyracksDataException.create(e); } }
@Override public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); IResourceLifecycleManager indexLifeCycleManager = txnSubsystem.getApplicationContext().getDatasetLifecycleManager(); ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); if (index == null) { throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); } try { IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); ITransactionContext txnCtx = txnSubsystem.getTransactionManager() .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(datasetId)); IModificationOperationCallback modCallback = new UpsertOperationCallback(new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(), aResource.getPartition(), resourceType, indexOp); txnCtx.register(resource.getId(), aResource.getPartition(), index, modCallback, true); return modCallback; } catch (ACIDException e) { throw HyracksDataException.create(e); } } }
public List<String> getPartitionReplicatedFiles(int partition, IReplicationStrategy strategy) throws HyracksDataException { final List<String> partitionReplicatedFiles = new ArrayList<>(); final Set<File> replicatedIndexes = new HashSet<>(); final Map<Long, LocalResource> partitionResources = getPartitionResources(partition); for (LocalResource lr : partitionResources.values()) { DatasetLocalResource datasetLocalResource = (DatasetLocalResource) lr.getResource(); if (strategy.isMatch(datasetLocalResource.getDatasetId())) { replicatedIndexes.add(ioManager.resolve(lr.getPath()).getFile()); } } for (File indexDir : replicatedIndexes) { partitionReplicatedFiles.addAll(getIndexFiles(indexDir)); } return partitionReplicatedFiles; }
private void closePartitionResources(int partition) throws HyracksDataException { final PersistentLocalResourceRepository resourceRepository = (PersistentLocalResourceRepository) appCtx.getLocalResourceRepository(); final Map<Long, LocalResource> partitionResources = resourceRepository.getPartitionResources(partition); final IDatasetLifecycleManager datasetLifecycleManager = appCtx.getDatasetLifecycleManager(); for (LocalResource resource : partitionResources.values()) { datasetLifecycleManager.close(resource.getPath()); } }
public Map<Long, LocalResource> getPartitionResources(int partition) throws HyracksDataException { return getResources(resource -> { DatasetLocalResource dsResource = (DatasetLocalResource) resource.getResource(); return dsResource.getPartition() == partition; }); }
@Override public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); IResourceLifecycleManager indexLifeCycleManager = txnSubsystem.getApplicationContext().getDatasetLifecycleManager(); ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); if (index == null) { throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); } try { IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); ITransactionContext txnCtx = txnSubsystem.getTransactionManager() .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(datasetId)); DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); IModificationOperationCallback modCallback = new SecondaryIndexModificationOperationCallback( new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, resource.getId(), aResource.getPartition(), resourceType, indexOp); txnCtx.register(resource.getId(), aResource.getPartition(), index, modCallback, false); return modCallback; } catch (ACIDException e) { throw HyracksDataException.create(e); } } }
private long getResourceIDfromResourcePath(String resourcePath) throws HyracksDataException { LocalResource lr = resourceRepository.get(resourcePath); if (lr == null) { return -1; } return lr.getId(); }
public void register(LocalResource resource, ILSMIndex index) throws HyracksDataException { long resourceID = resource.getId(); if (!datasetInfo.isRegistered()) { synchronized (datasetInfo) { if (!datasetInfo.isRegistered()) { datasetInfo.setExternal(index.getNumberOfAllMemoryComponents() == 0); datasetInfo.setRegistered(true); datasetInfo.setDurable(index.isDurable()); } } } if (datasetInfo.getIndexes().containsKey(resourceID)) { throw new HyracksDataException("Index with resource ID " + resourceID + " already exists."); } if (index == null) { throw new HyracksDataException("Attempt to register a null index"); } datasetInfo.addIndex(resourceID, new IndexInfo(index, datasetInfo.getDatasetID(), resource, ((DatasetLocalResource) resource.getResource()).getPartition())); }
localResourceMetadata = (DatasetLocalResource) localResource.getResource(); index = (ILSMIndex) datasetLifecycleManager.get(localResource.getPath()); if (index == null) { datasetLifecycleManager.register(localResource.getPath(), index); datasetLifecycleManager.open(localResource.getPath()); try { final DatasetResourceReference resourceReference = indexCheckpointManagerProvider.get(resourceReference).getLowWatermark(); } catch (HyracksDataException e) { datasetLifecycleManager.close(localResource.getPath()); throw e; datasetLifecycleManager.close(resourcesMap.get(r).getPath());
IndexInfo value = entry.getValue(); ILSMIndex index = value.getIndex(); String path = value.getLocalResource().getPath(); strBuilder.append('{'); strBuilder.append("\"node\":\"");