try { mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse); if (dv == null) { return false; Library library = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName); if (library == null) { return false; List<Function> functions = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverse); for (Function function : functions) { MetadataManager.INSTANCE.dropFunction(mdTxnCtx, new FunctionSignature(dataverse, function.getName(), function.getArity())); List<DatasourceAdapter> adapters = MetadataManager.INSTANCE.getDataverseAdapters(mdTxnCtx, dataverse); for (DatasourceAdapter adapter : adapters) { MetadataManager.INSTANCE.dropAdapter(mdTxnCtx, dataverse, adapter.getAdapterIdentifier().getName()); MetadataManager.INSTANCE.dropLibrary(mdTxnCtx, dataverse, libraryName); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); } catch (Exception e) { MetadataManager.INSTANCE.abortTransaction(mdTxnCtx); throw new AsterixException(e);
protected void handleCreateNodeGroupStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception { NodegroupDecl stmtCreateNodegroup = (NodegroupDecl) stmt; SourceLocation sourceLoc = stmtCreateNodegroup.getSourceLocation(); String ngName = stmtCreateNodegroup.getNodegroupName().getValue(); MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); lockManager.acquireNodeGroupWriteLock(metadataProvider.getLocks(), ngName); try { NodeGroup ng = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, ngName); if (ng != null) { if (!stmtCreateNodegroup.getIfNotExists()) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "A nodegroup with this name " + ngName + " already exists."); } } else { List<Identifier> ncIdentifiers = stmtCreateNodegroup.getNodeControllerNames(); List<String> ncNames = new ArrayList<>(ncIdentifiers.size()); for (Identifier id : ncIdentifiers) { ncNames.add(id.getValue()); } MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(ngName, ncNames)); } MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); } catch (Exception e) { abort(e, e, mdTxnCtx); throw e; } finally { metadataProvider.getLocks().unlock(); } }
String dataverseName = stmtDelete.getDataverseName().getValue(); ProgressState progress = ProgressState.NO_PROGRESS; MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); boolean bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); List<JobSpecification> jobsToExecute = new ArrayList<>(); try { Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName); if (dv == null) { if (stmtDelete.getIfExists()) { MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); return false; } else { MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverseName); for (Function function : functionsInDataverse) { if (isFunctionUsed(mdTxnCtx, function.getSignature(), dataverseName)) { MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); doDropFeed(hcc, metadataProvider, feedListener.getFeed(), sourceLoc); MetadataManager.INSTANCE.commitTransaction(metadataProvider.getMetadataTxnContext()); bActiveTxn = false; mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
private List<Dataset> getAllDatasetsForRebalance(String dataverseName) throws Exception { List<Dataset> datasets; MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); try { datasets = getDatasetsInDataverseForRebalance(dataverseName, mdTxnCtx); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); } catch (Exception e) { MetadataManager.INSTANCE.abortTransaction(mdTxnCtx); throw e; } return datasets; }
private List<Dataset> getAllDatasetsForRebalance() throws Exception { List<Dataset> datasets = new ArrayList<>(); MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); try { List<Dataverse> dataverses = MetadataManager.INSTANCE.getDataverses(mdTxnCtx); for (Dataverse dv : dataverses) { datasets.addAll(getDatasetsInDataverseForRebalance(dv.getDataverseName(), mdTxnCtx)); } MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); } catch (Exception e) { MetadataManager.INSTANCE.abortTransaction(mdTxnCtx); throw e; } return datasets; }
protected void recover(ICcApplicationContext appCtx) throws HyracksDataException { try { LOGGER.info("Starting Global Recovery"); MetadataManager.INSTANCE.init(); MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); mdTxnCtx = doRecovery(appCtx, mdTxnCtx); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); recoveryCompleted = true; recovering = false; LOGGER.info("Global Recovery Completed. Refreshing cluster state..."); appCtx.getClusterStateManager().refreshState(); } catch (Exception e) { throw HyracksDataException.create(e); } }
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); boolean bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); rewriteCompileQuery(hcc, metadataProvider, query, null, stmtParams, stmtRewriter); afterCompile(); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; return query.isExplain() || !sessionConfig.isExecuteQuery() ? null : jobSpec;
try { List<Dataset> datasets = MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx, dataverse.getDataverseName()); for (Dataset dataset : datasets) { if (dataset.getDatasetType() == DatasetType.EXTERNAL) { List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName()); if (datasetState == TransactionState.BEGIN) { List<ExternalFile> files = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset); MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file); MetadataManager.INSTANCE.updateDataset(mdTxnCtx, dataset); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); } else if (datasetState == TransactionState.READY_TO_COMMIT) { List<ExternalFile> files = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset); MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file); file.setPendingOp(ExternalFilePendingOp.NO_OP); MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file); } else if (file.getPendingOp() == ExternalFilePendingOp.DROP_OP) { MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file); MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, originalFile);
MetadataTransactionContext mdTxnCtx = null; try { mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); Library libraryInMetadata = MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverse, libraryName); if (libraryInMetadata != null && !wasUninstalled) { MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); return; MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName)); if (LOGGER.isInfoEnabled()) { LOGGER.info("Added library " + libraryName + " to Metadata"); Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse); if (dv == null) { MetadataManager.INSTANCE.addDataverse(mdTxnCtx, new Dataverse(dataverse, NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT, MetadataUtil.PENDING_NO_OP)); Function f = new Function(signature, args, functionReturnType, functionDefinition, functionLanguage, functionType, null); MetadataManager.INSTANCE.addFunction(mdTxnCtx, f); if (LOGGER.isInfoEnabled()) { LOGGER.info("Installed function: " + functionFullName); DatasourceAdapter dsa = new DatasourceAdapter(aid, adapterFactoryClass, IDataSourceAdapter.AdapterType.EXTERNAL); MetadataManager.INSTANCE.addAdapter(mdTxnCtx, dsa); if (LOGGER.isInfoEnabled()) { LOGGER.info("Installed adapter: " + adapterName);
filesIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), index.getDataverseName(), index.getDatasetName(), IndexingConstants.getFilesIndexName(index.getDatasetName())); filesIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), index.getDataverseName(), index.getDatasetName(), IndexingConstants.getFilesIndexName(index.getDatasetName())); ExternalIndexingOperations.FILE_INDEX_FIELD_TYPES, false, false, false, MetadataUtil.PENDING_ADD_OP); MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex); MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file); List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes( metadataProvider.getMetadataTxnContext(), index.getDataverseName(), index.getDatasetName()); for (Index existingIndex : indexes) { MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA; mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName, datasetName); for (int j = 0; j < indexes.size(); j++) { if (indexes.get(j).isSecondaryIndex()) { MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName); MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(), new Dataset(dataverseName, datasetName, getItemTypeDataverseName(), getItemTypeName(), getMetaItemTypeDataverseName(), getMetaItemTypeName(), getNodeGroupName(), MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue()); bActiveTxn.setValue(false); progress.setValue(ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA); mdTxnCtx.setValue(MetadataManager.INSTANCE.beginTransaction()); bActiveTxn.setValue(true); metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue()); MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName, datasetName); for (int j = 0; j < indexes.size(); j++) { if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) { MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName); MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(), new Dataset(dataverseName, datasetName, getItemTypeDataverseName(), getItemTypeName(), getNodeGroupName(), getCompactionPolicy(), getCompactionPolicyProperties(), MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue()); bActiveTxn.setValue(false); progress.setValue(ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA);
String indexName = stmtIndexDrop.getIndexName().getValue(); ProgressState progress = ProgressState.NO_PROGRESS; MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); boolean bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); Index index = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataverseName, datasetName, indexName); if (index == null) { if (stmtIndexDrop.getIfExists()) { MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); return; } else { MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName); MetadataManager.INSTANCE.addIndex(mdTxnCtx, new Index(dataverseName, datasetName, indexName, index.getIndexType(), index.getKeyFieldNames(), index.getKeyFieldSourceIndicators(), index.getKeyFieldTypes(), MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA; mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName); } else {
.getDdlOrDefaultCompressionScheme(dd.getDatasetCompressionScheme()); boolean defaultCompactionPolicy = compactionPolicy == null; MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); boolean bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); if (ds != null) { if (dd.getIfNotExists()) { MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); return; } else { Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), itemTypeDataverseName, itemTypeName); if (dt == null) { DatasetIdFactory.initialize(MetadataManager.INSTANCE.getMostRecentDatasetId()); datasetDetails, dd.getHints(), dsType, DatasetIdFactory.generateDatasetId(), MetadataUtil.PENDING_ADD_OP, compressionScheme); MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(), dataset); if (dd.getDatasetType() == DatasetType.INTERNAL) { JobSpecification jobSpec = DatasetUtil.createDatasetJobSpec(dataset, metadataProvider); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; progress.setValue(ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA); mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
String dataverseName = getActiveDataverse(stmtCreateType.getDataverseName()); String typeName = stmtCreateType.getIdent().getValue(); MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), dataverseName, dataverseName + "." + typeName); try { Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName); if (dv == null) { throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc, dataverseName); Datatype dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, dataverseName, typeName); if (dt != null) { if (!stmtCreateType.getIfNotExists()) { TypeSignature typeSignature = new TypeSignature(dataverseName, typeName); IAType type = typeMap.get(typeSignature); MetadataManager.INSTANCE.addDatatype(mdTxnCtx, new Datatype(dataverseName, typeName, type, false)); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); } catch (Exception e) { abort(e, e, mdTxnCtx);
ctx = MetadataManager.INSTANCE.beginTransaction(); Datatype t = MetadataManager.INSTANCE.getDatatype(ctx, dataverseName, datatypeName); if (t == null || t.getDatatype().getTypeTag() != ATypeTag.OBJECT) { throw new MetadataException(ErrorCode.FEED_METADATA_UTIL_UNEXPECTED_FEED_DATATYPE, datatypeName); MetadataManager.INSTANCE.commitTransaction(ctx); } catch (ACIDException | RemoteException e) { if (ctx != null) { try { MetadataManager.INSTANCE.abortTransaction(ctx); } catch (ACIDException | RemoteException e2) { e.addSuppressed(e2);
protected void handleNodegroupDropStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception { NodeGroupDropStatement stmtDelete = (NodeGroupDropStatement) stmt; SourceLocation sourceLoc = stmtDelete.getSourceLocation(); String nodegroupName = stmtDelete.getNodeGroupName().getValue(); MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); lockManager.acquireNodeGroupWriteLock(metadataProvider.getLocks(), nodegroupName); try { NodeGroup ng = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, nodegroupName); if (ng == null) { if (!stmtDelete.getIfExists()) { throw new CompilationException(ErrorCode.UNKNOWN_NODEGROUP, sourceLoc, nodegroupName); } } else { MetadataManager.INSTANCE.dropNodegroup(mdTxnCtx, nodegroupName, false); } MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); } catch (Exception e) { abort(e, e, mdTxnCtx); throw e; } finally { metadataProvider.getLocks().unlock(); } }
private void addDataset(ICcApplicationContext appCtx, Dataset source, int datasetPostfix, boolean abort) throws Exception { Dataset dataset = new Dataset(source.getDataverseName(), "ds_" + datasetPostfix, source.getDataverseName(), source.getDatasetType().name(), source.getNodeGroupName(), NoMergePolicyFactory.NAME, null, source.getDatasetDetails(), source.getHints(), DatasetConfig.DatasetType.INTERNAL, datasetPostfix, 0); MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); final MetadataTransactionContext writeTxn = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(writeTxn); try { MetadataManager.INSTANCE.addDataset(writeTxn, dataset); if (abort) { MetadataManager.INSTANCE.abortTransaction(writeTxn); } else { MetadataManager.INSTANCE.commitTransaction(writeTxn); } } finally { metadataProvider.getLocks().unlock(); } } }
String policyName = cfs.getPolicy(); String whereClauseBody = cfs.getWhereClauseBody(); MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); List<FunctionSignature> appliedFunctions = cfs.getAppliedFunctions(); for (FunctionSignature func : appliedFunctions) { if (MetadataManager.INSTANCE.getFunction(mdTxnCtx, func) == null) { throw new CompilationException(ErrorCode.FEED_CONNECT_FEED_APPLIED_INVALID_FUNCTION, sourceLoc, func.getName()); fc = MetadataManager.INSTANCE.getFeedConnection(metadataProvider.getMetadataTxnContext(), dataverseName, feedName, datasetName); if (fc != null) { MetadataManager.INSTANCE.addFeedConnection(metadataProvider.getMetadataTxnContext(), fc); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); if (listener != null) { listener.add(dataset);
String indexName = stmtCreateIndex.getIndexName().getValue(); List<Integer> keySourceIndicators = stmtCreateIndex.getFieldSourceIndicators(); MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); String datasetFullyQualifiedName = dataverseName + "." + datasetName; index = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName, indexName); if (index != null) { if (stmtCreateIndex.getIfNotExists()) { MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); return; } else { "Cannot create index on meta fields"); Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), ds.getItemTypeDataverseName(), ds.getItemTypeName()); ARecordType aRecordType = (ARecordType) dt.getDatatype(); ARecordType metaRecordType = null; if (ds.hasMetaPart()) { Datatype metaDt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName()); metaRecordType = (ARecordType) metaDt.getDatatype();
(ICcApplicationContext) integrationUtil.getClusterControllerService().getApplicationContext(); final MetadataProvider metadataProvider = new MetadataProvider(ccAppCtx, null); final MetadataTransactionContext mdTxn = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxn); final String nodeGroupName = "ng"; final List<String> ngNodes = Collections.singletonList("asterix_nc1"); try { MetadataManager.INSTANCE.addNodegroup(mdTxn, new NodeGroup(nodeGroupName, ngNodes)); MetadataManager.INSTANCE.commitTransaction(mdTxn); } finally { metadataProvider.getLocks().unlock(); .getIndex(idx.getDatasetId().getId(), idx.getResourceId()); PrimaryIndexOperationTracker opTracker = (PrimaryIndexOperationTracker) index.getOperationTracker(); final MetadataTransactionContext mdTxn2 = MetadataManager.INSTANCE.beginTransaction(); int mutableComponentBeforeFlush = index.getCurrentMemoryComponentIndex(); int diskComponentsBeforeFlush = index.getDiskComponents().size(); final MetadataTransactionContext committedMdTxn = MetadataManager.INSTANCE.beginTransaction(); MetadataManager.INSTANCE.addNodegroup(committedMdTxn, new NodeGroup(committedNodeGroup, ngNodes)); MetadataManager.INSTANCE.commitTransaction(committedMdTxn); opTracker.setFlushOnExit(true); opTracker.flushIfNeeded(); MetadataManager.INSTANCE.dropNodegroup(mdTxn2, nodeGroupName, false); Assert.assertEquals(1, opTracker.getNumActiveOperations()); MetadataManager.INSTANCE.commitTransaction(mdTxn2);