public RecoveryTask(ICcApplicationContext appCtx, ActiveEntityEventsListener listener, IRetryPolicyFactory retryPolicyFactory) { this.listener = listener; this.retryPolicyFactory = retryPolicyFactory; this.metadataProvider = new MetadataProvider(appCtx, null); this.clusterStateManager = appCtx.getClusterStateManager(); }
TestUserActor newUser(String name, CcApplicationContext appCtx) { MetadataProvider actorMdProvider = new MetadataProvider(appCtx, null); return new TestUserActor("User: " + name, actorMdProvider, clusterController); }
private void rebalanceDataset(String dataverseName, String datasetName, String[] targetNodes) throws Exception { IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR); MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); try { ActiveNotificationHandler activeNotificationHandler = (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); activeNotificationHandler.suspend(metadataProvider); try { IMetadataLockManager lockManager = appCtx.getMetadataLockManager(); lockManager.acquireDatasetExclusiveModificationLock(metadataProvider.getLocks(), dataverseName + '.' + datasetName); RebalanceUtil.rebalance(dataverseName, datasetName, new LinkedHashSet<>(Arrays.asList(targetNodes)), metadataProvider, hcc, NoOpDatasetRebalanceCallback.INSTANCE); } finally { activeNotificationHandler.resume(metadataProvider); } } finally { metadataProvider.getLocks().unlock(); } }
public ConstantFoldingRule(ICcApplicationContext appCtx) { MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); jobGenCtx = new JobGenContext(null, metadataProvider, appCtx, SerializerDeserializerProvider.INSTANCE, BinaryHashFunctionFactoryProvider.INSTANCE, BinaryHashFunctionFamilyProvider.INSTANCE, BinaryComparatorFactoryProvider.INSTANCE, TypeTraitProvider.INSTANCE, BinaryBooleanInspector.FACTORY, BinaryIntegerInspector.FACTORY, ADMPrinterFactoryProvider.INSTANCE, MissingWriterFactory.INSTANCE, null, new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())), ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null); }
public ActiveEntityEventsListener(IStatementExecutor statementExecutor, ICcApplicationContext appCtx, IHyracksClientConnection hcc, EntityId entityId, List<Dataset> datasets, AlgebricksAbsolutePartitionConstraint locations, String runtimeName, IRetryPolicyFactory retryPolicyFactory) throws HyracksDataException { this.statementExecutor = statementExecutor; this.appCtx = appCtx; this.clusterStateManager = appCtx.getClusterStateManager(); this.metadataProvider = new MetadataProvider(appCtx, null); this.hcc = hcc; this.entityId = entityId; this.datasets = datasets; this.retryPolicyFactory = retryPolicyFactory; this.state = ActivityState.STOPPED; this.statsTimestamp = -1; this.isFetchingStats = false; this.statsUpdatedEvent = new ActiveEvent(null, Kind.STATS_UPDATED, entityId, null); this.stats = DEFAULT_ACTIVE_STATS; this.runtimeName = runtimeName; this.locations = locations; this.numRegistered = 0; this.numDeRegistered = 0; this.handler = (ActiveNotificationHandler) metadataProvider.getApplicationContext().getActiveNotificationHandler(); handler.registerListener(this); }
.thenReturn(new FunctionManager(FunctionCollection.createDefaultFunctionCollection())); Mockito.when(appCtx.getExtensionManager()).thenReturn(functionExtensionManager); metadataProvider = new MetadataProvider(appCtx, null); clusterController = new TestClusterControllerActor("CC", handler, allDatasets); nodeControllers = new TestNodeControllerActor[2];
ICcApplicationContext appCtx = (ICcApplicationContext) integrationUtil.getClusterControllerService().getApplicationContext(); final MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); final MetadataTransactionContext mdTxn = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxn);
public IResourceFactory getPrimaryResourceFactory(IHyracksTaskContext ctx, PrimaryIndexInfo primaryIndexInfo, IStorageComponentProvider storageComponentProvider, Dataset dataset) throws AlgebricksException { Dataverse dataverse = new Dataverse(dataset.getDataverseName(), NonTaggedDataFormat.class.getName(), MetadataUtil.PENDING_NO_OP); Index index = primaryIndexInfo.getIndex(); CcApplicationContext appCtx = (CcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(); MetadataProvider mdProvider = new MetadataProvider(appCtx, dataverse); try { return dataset.getResourceFactory(mdProvider, index, primaryIndexInfo.recordType, primaryIndexInfo.metaType, primaryIndexInfo.mergePolicyFactory, primaryIndexInfo.mergePolicyProperties); } finally { mdProvider.getLocks().unlock(); } }
MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); final MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx);
ICcApplicationContext appCtx = (ICcApplicationContext) integrationUtil.getClusterControllerService().getApplicationContext(); final MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); MetadataTransactionContext mdTxn = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxn);
ICcApplicationContext ccAppCtx = (ICcApplicationContext) integrationUtil.getClusterControllerService().getApplicationContext(); MetadataProvider metadataProvider = new MetadataProvider(ccAppCtx, null); try { ActiveNotificationHandler activeNotificationHandler =
@Test public void abortMetadataTxn() throws Exception { ICcApplicationContext appCtx = (ICcApplicationContext) integrationUtil.getClusterControllerService().getApplicationContext(); final MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); final MetadataTransactionContext mdTxn = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxn); final String nodeGroupName = "ng"; try { final List<String> ngNodes = Arrays.asList("asterix_nc1"); MetadataManager.INSTANCE.addNodegroup(mdTxn, new NodeGroup(nodeGroupName, ngNodes)); MetadataManager.INSTANCE.abortTransaction(mdTxn); } finally { metadataProvider.getLocks().unlock(); } // ensure that the node group was not added final MetadataTransactionContext readMdTxn = MetadataManager.INSTANCE.beginTransaction(); try { final NodeGroup nodegroup = MetadataManager.INSTANCE.getNodegroup(readMdTxn, nodeGroupName); if (nodegroup != null) { throw new AssertionError("nodegroup was found after metadata txn was aborted"); } } finally { MetadataManager.INSTANCE.commitTransaction(readMdTxn); } }
/** * Gets the reference of dataset {@code dataset} from metadata * * @param integrationUtil * @param datasetName * @return the dataset reference if found. Otherwise null. * @throws AlgebricksException * @throws RemoteException */ public static Dataset getDataset(AsterixHyracksIntegrationUtil integrationUtil, String datasetName) throws AlgebricksException, RemoteException { final ICcApplicationContext appCtx = (ICcApplicationContext) integrationUtil.getClusterControllerService().getApplicationContext(); final MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); final MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); Dataset dataset; try { dataset = metadataProvider.findDataset(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME, datasetName); } finally { MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); metadataProvider.getLocks().unlock(); } return dataset; }
MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); final MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); metadataProvider = new MetadataProvider(appCtx, null); final MetadataTransactionContext rebalanceTxn = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(rebalanceTxn);
private ARecordType getMetadataRecordType(String dataverseName, String datasetName) throws Exception { MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); // Retrieves file splits of the dataset. MetadataProvider metadataProvider = new MetadataProvider( (ICcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(), null); try { metadataProvider.setMetadataTxnContext(mdTxnCtx); Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName); ARecordType recordType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); // Metadata transaction commits. MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); return recordType; } finally { metadataProvider.getLocks().unlock(); } } }
public SecondaryIndexInfo createSecondaryIndex(PrimaryIndexInfo primaryIndexInfo, Index secondaryIndex, IStorageComponentProvider storageComponentProvider, int partition) throws AlgebricksException, HyracksDataException, RemoteException, ACIDException { MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); org.apache.hyracks.algebricks.common.utils.Pair<ILSMMergePolicyFactory, Map<String, String>> mergePolicy = DatasetUtil.getMergePolicyFactory(primaryIndexInfo.dataset, mdTxnCtx); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); Dataverse dataverse = new Dataverse(primaryIndexInfo.dataset.getDataverseName(), NonTaggedDataFormat.class.getName(), MetadataUtil.PENDING_NO_OP); MetadataProvider mdProvider = new MetadataProvider( (ICcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(), dataverse); SecondaryIndexInfo secondaryIndexInfo = new SecondaryIndexInfo(primaryIndexInfo, secondaryIndex); try { IResourceFactory resourceFactory = primaryIndexInfo.dataset.getResourceFactory(mdProvider, secondaryIndex, primaryIndexInfo.recordType, primaryIndexInfo.metaType, mergePolicy.first, mergePolicy.second); IndexBuilderFactory indexBuilderFactory = new IndexBuilderFactory(storageComponentProvider.getStorageManager(), secondaryIndexInfo.fileSplitProvider, resourceFactory, true); IHyracksTaskContext ctx = createTestContext(newJobId(), partition, false); IIndexBuilder indexBuilder = indexBuilderFactory.create(ctx, partition); indexBuilder.build(); } finally { mdProvider.getLocks().unlock(); } return secondaryIndexInfo; }
public PrimaryIndexInfo createPrimaryIndex(Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType, int[] filterFields, IStorageComponentProvider storageComponentProvider, int[] primaryKeyIndexes, List<Integer> primaryKeyIndicators, int partition) throws AlgebricksException, HyracksDataException, RemoteException, ACIDException { MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); org.apache.hyracks.algebricks.common.utils.Pair<ILSMMergePolicyFactory, Map<String, String>> mergePolicy = DatasetUtil.getMergePolicyFactory(dataset, mdTxnCtx); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType, mergePolicy.first, mergePolicy.second, filterFields, primaryKeyIndexes, primaryKeyIndicators); Dataverse dataverse = new Dataverse(dataset.getDataverseName(), NonTaggedDataFormat.class.getName(), MetadataUtil.PENDING_NO_OP); MetadataProvider mdProvider = new MetadataProvider( (ICcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(), dataverse); try { IResourceFactory resourceFactory = dataset.getResourceFactory(mdProvider, primaryIndexInfo.index, recordType, metaType, mergePolicy.first, mergePolicy.second); IndexBuilderFactory indexBuilderFactory = new IndexBuilderFactory(storageComponentProvider.getStorageManager(), primaryIndexInfo.getFileSplitProvider(), resourceFactory, true); IHyracksTaskContext ctx = createTestContext(newJobId(), partition, false); IIndexBuilder indexBuilder = indexBuilderFactory.create(ctx, partition); indexBuilder.build(); } finally { mdProvider.getLocks().unlock(); } return primaryIndexInfo; }
private void addDataset(ICcApplicationContext appCtx, Dataset source, int datasetPostfix, boolean abort) throws Exception { Dataset dataset = new Dataset(source.getDataverseName(), "ds_" + datasetPostfix, source.getDataverseName(), source.getDatasetType().name(), source.getNodeGroupName(), NoMergePolicyFactory.NAME, null, source.getDatasetDetails(), source.getHints(), DatasetConfig.DatasetType.INTERNAL, datasetPostfix, 0); MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); final MetadataTransactionContext writeTxn = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(writeTxn); try { MetadataManager.INSTANCE.addDataset(writeTxn, dataset); if (abort) { MetadataManager.INSTANCE.abortTransaction(writeTxn); } else { MetadataManager.INSTANCE.commitTransaction(writeTxn); } } finally { metadataProvider.getLocks().unlock(); } } }
ICcApplicationContext appCtx = (ICcApplicationContext) ExecutionTestUtil.integrationUtil .getClusterControllerService().getApplicationContext(); MetadataProvider metadataProver = new MetadataProvider(appCtx, null); metadataProver.setMetadataTxnContext(mdTxn); final String defaultDv = MetadataBuiltinEntities.DEFAULT_DATAVERSE.getDataverseName();
IFrameOperationCallbackFactory frameOpCallbackFactory, boolean hasSecondaries) throws Exception { MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); MetadataProvider mdProvider = new MetadataProvider( (ICcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(), MetadataBuiltinEntities.DEFAULT_DATAVERSE);