public static int[] createFilterFields(Dataset dataset) { if (dataset.getDatasetType() == DatasetType.EXTERNAL) { return null; } List<String> filterField = getFilterField(dataset); if (filterField == null) { return null; } List<List<String>> partitioningKeys = dataset.getPrimaryKeys(); int numKeys = partitioningKeys.size(); int[] filterFields = new int[1]; filterFields[0] = numKeys + 1; return filterFields; }
/*** * Creates a node group that is associated with a new dataset. * * @param dataverseName, * the dataverse name of the dataset. * @param datasetName, * the name of the dataset. * @param ncNames, * the set of node names. * @param metadataProvider, * the metadata provider. * @return the name of the created node group. * @throws Exception */ public static String createNodeGroupForNewDataset(String dataverseName, String datasetName, Set<String> ncNames, MetadataProvider metadataProvider) throws Exception { return createNodeGroupForNewDataset(dataverseName, datasetName, 0L, ncNames, metadataProvider); }
@Override public IResourceFactory getResourceFactory(MetadataProvider mdProvider, Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties) throws AlgebricksException { ITypeTraits[] filterTypeTraits = DatasetUtil.computeFilterTypeTraits(this, recordType); IBinaryComparatorFactory[] filterCmpFactories = DatasetUtil.computeFilterBinaryComparatorFactories(this, recordType, mdProvider.getStorageComponentProvider().getComparatorFactoryProvider()); IResourceFactory resourceFactory = TestLsmBTreeResourceFactoryProvider.INSTANCE.getResourceFactory(mdProvider, this, index, recordType, metaType, mergePolicyFactory, mergePolicyProperties, filterTypeTraits, filterCmpFactories); return new DatasetLocalResourceFactory(getDatasetId(), resourceFactory); }
int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1; int numOfAdditionalFields = additionalNonFilterFields == null ? 0 : additionalNonFilterFields.size(); return DatasetUtil.createPrimaryIndexUpsertOp(spec, this, dataset, recordDesc, fieldPermutation, context.getMissingWriterFactory());
private static void populateDataToRebalanceTarget(Dataset source, Dataset target, MetadataProvider metadataProvider, IHyracksClientConnection hcc) throws Exception { JobSpecification spec = new JobSpecification(); TxnId txnId = metadataProvider.getTxnIdFactory().create(); JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(txnId, true); spec.setJobletEventListenerFactory(jobEventListenerFactory); // The pipeline starter. IOperatorDescriptor starter = DatasetUtil.createDummyKeyProviderOp(spec, source, metadataProvider); // Creates primary index scan op. IOperatorDescriptor primaryScanOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, source); // Creates secondary BTree upsert op. IOperatorDescriptor upsertOp = createPrimaryIndexUpsertOp(spec, metadataProvider, source, target); // The final commit operator. IOperatorDescriptor commitOp = createUpsertCommitOp(spec, metadataProvider, target); // Connects empty-tuple-source and scan. spec.connect(new OneToOneConnectorDescriptor(spec), starter, 0, primaryScanOp, 0); // Connects scan and upsert. int numKeys = target.getPrimaryKeys().size(); int[] keys = IntStream.range(0, numKeys).toArray(); IConnectorDescriptor connectorDescriptor = new MToNPartitioningConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(keys, target.getPrimaryHashFunctionFactories(metadataProvider))); spec.connect(connectorDescriptor, primaryScanOp, 0, upsertOp, 0); // Connects upsert and sink. spec.connect(new OneToOneConnectorDescriptor(spec), upsertOp, 0, commitOp, 0); // Executes the job. JobUtils.runJob(hcc, spec, true); }
numPrimaryKeys = dataset.getPrimaryKeys().size(); if (dataset.getDatasetType() == DatasetType.INTERNAL) { filterFieldName = DatasetUtil.getFilterField(dataset); if (filterFieldName != null) { numFilterFields = 1; numElementsHint = metadataProvider.getCardinalityPerPartitionHint(dataset); Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext()); mergePolicyFactory = compactionInfo.first; mergePolicyProperties = compactionInfo.second;
private static void rebalanceSwitch(Dataset source, Dataset target, MetadataProvider metadataProvider) throws AlgebricksException, RemoteException { MetadataTransactionContext mdTxnCtx = metadataProvider.getMetadataTxnContext(); // upgrade lock ICcApplicationContext appCtx = metadataProvider.getApplicationContext(); ActiveNotificationHandler activeNotificationHandler = (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); IMetadataLockManager lockManager = appCtx.getMetadataLockManager(); lockManager.upgradeDatasetLockToWrite(metadataProvider.getLocks(), DatasetUtil.getFullyQualifiedName(source)); LOGGER.info("Updating dataset {} node group from {} to {}", source.getDatasetName(), source.getNodeGroupName(), target.getNodeGroupName()); try { // Updates the dataset entry in the metadata storage MetadataManager.INSTANCE.updateDataset(mdTxnCtx, target); for (IActiveEntityEventsListener listener : activeNotificationHandler.getEventListeners()) { if (listener instanceof IActiveEntityController) { IActiveEntityController controller = (IActiveEntityController) listener; controller.replace(target); } } MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); LOGGER.info("dataset {} node group updated to {}", target.getDatasetName(), target.getNodeGroupName()); } finally { lockManager.downgradeDatasetLockToExclusiveModify(metadataProvider.getLocks(), DatasetUtil.getFullyQualifiedName(target)); } }
public static List<List<List<String>>> getFunctionDependencies(IQueryRewriter rewriter, Expression expression, MetadataProvider metadataProvider) throws CompilationException { Set<CallExpr> functionCalls = rewriter.getFunctionCalls(expression); //Get the List of used functions and used datasets List<List<String>> datasourceDependencies = new ArrayList<>(); List<List<String>> functionDependencies = new ArrayList<>(); for (CallExpr functionCall : functionCalls) { FunctionSignature signature = functionCall.getFunctionSignature(); FunctionIdentifier fid = new FunctionIdentifier(signature.getNamespace(), signature.getName(), signature.getArity()); if (fid.equals(BuiltinFunctions.DATASET)) { Pair<String, String> path = DatasetUtil.getDatasetInfo(metadataProvider, ((LiteralExpr) functionCall.getExprList().get(0)).getValue().getStringValue()); datasourceDependencies.add(Arrays.asList(path.first, path.second)); } else if (BuiltinFunctions.isBuiltinCompilerFunction(signature, false)) { continue; } else { functionDependencies.add(Arrays.asList(signature.getNamespace(), signature.getName(), Integer.toString(signature.getArity()))); } } List<List<List<String>>> dependencies = new ArrayList<>(); dependencies.add(datasourceDependencies); dependencies.add(functionDependencies); return dependencies; }
IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider);
jobsToExecute.add(DatasetUtil.dropDatasetJobSpec(this, metadataProvider)); .add(IndexUtil.buildDropIndexJobSpec(indexes.get(j), metadataProvider, this, sourceLoc)); } else { jobsToExecute.add(DatasetUtil.buildDropFilesIndexJobSpec(metadataProvider, this));
MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(), dataset); if (dd.getDatasetType() == DatasetType.INTERNAL) { JobSpecification jobSpec = DatasetUtil.createDatasetJobSpec(dataset, metadataProvider); metadataProvider.setMetadataTxnContext(mdTxnCtx); try { JobSpecification jobSpec = DatasetUtil.dropDatasetJobSpec(dataset, metadataProvider); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false;
private static IOperatorDescriptor createPrimaryIndexUpsertOp(JobSpecification spec, MetadataProvider metadataProvider, Dataset source, Dataset target) throws AlgebricksException { int numKeys = source.getPrimaryKeys().size(); int numValues = source.hasMetaPart() ? 2 : 1; int[] fieldPermutation = IntStream.range(0, numKeys + numValues).toArray(); Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> upsertOpAndConstraints = DatasetUtil.createPrimaryIndexUpsertOp(spec, metadataProvider, target, source.getPrimaryRecordDescriptor(metadataProvider), fieldPermutation, MissingWriterFactory.INSTANCE); IOperatorDescriptor upsertOp = upsertOpAndConstraints.first; AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, upsertOp, upsertOpAndConstraints.second); return upsertOp; }
IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider); IOperatorDescriptor primaryScanOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset);
@Override protected void doExecute(MetadataProvider mdProvider) throws Exception { String dataverseName = actionListener.getEntityId().getDataverse(); String entityName = actionListener.getEntityId().getEntityName(); try { lockManager.acquireActiveEntityWriteLock(mdProvider.getLocks(), dataverseName + '.' + entityName); List<Dataset> datasets = actionListener.getDatasets(); for (Dataset dataset : datasets) { lockManager.upgradeDatasetLockToWrite(mdProvider.getLocks(), DatasetUtil.getFullyQualifiedName(dataset)); lockManager.downgradeDatasetLockToExclusiveModify(mdProvider.getLocks(), DatasetUtil.getFullyQualifiedName(dataset)); } actionListener.resume(mdProvider); } finally { mdProvider.getLocks().reset(); } } };
Pair<String, String> argInfo = DatasetUtil.getDatasetInfo(metadata, typeArg); String dataverseName = argInfo.first; String typeName = argInfo.second;
IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider);
private static int[] getPrimaryKeyPermutationForUpsert(Dataset dataset) { // upsertIndicatorVar + prev record int f = 2; // add the previous meta second if (dataset.hasMetaPart()) { f++; } // add the previous filter third int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1; if (numFilterFields > 0) { f++; } int numPrimaryKeys = dataset.getPrimaryKeys().size(); int[] pkIndexes = new int[numPrimaryKeys]; for (int i = 0; i < pkIndexes.length; i++) { pkIndexes[i] = f; f++; } return pkIndexes; }
IOperatorDescriptor sourceOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider); IOperatorDescriptor targetOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset); spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
protected static String configureNodegroupForDataset(ICcApplicationContext appCtx, Map<String, String> hints, String dataverseName, String datasetName, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws Exception { IClusterStateManager csm = appCtx.getClusterStateManager(); Set<String> allNodes = csm.getParticipantNodes(true); Set<String> selectedNodes = new LinkedHashSet<>(); String hintValue = hints.get(DatasetNodegroupCardinalityHint.NAME); if (hintValue == null) { selectedNodes.addAll(allNodes); } else { int nodegroupCardinality; final Pair<Boolean, String> validation = DatasetHints.validate(appCtx, DatasetNodegroupCardinalityHint.NAME, hints.get(DatasetNodegroupCardinalityHint.NAME)); boolean valid = validation.first; if (!valid) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Incorrect use of hint '" + DatasetNodegroupCardinalityHint.NAME + "': " + validation.second); } else { nodegroupCardinality = Integer.parseInt(hints.get(DatasetNodegroupCardinalityHint.NAME)); } List<String> allNodeList = new ArrayList<>(allNodes); Collections.shuffle(allNodeList); selectedNodes.addAll(allNodeList.subList(0, nodegroupCardinality)); } // Creates the associated node group for the dataset. return DatasetUtil.createNodeGroupForNewDataset(dataverseName, datasetName, selectedNodes, metadataProvider); }
@Override protected void doExecute(MetadataProvider mdProvider) throws Exception { String entityDataverseName = actionListener.getEntityId().getDataverse(); String entityName = actionListener.getEntityId().getEntityName(); try { lockManager.acquireActiveEntityReadLock(mdProvider.getLocks(), entityDataverseName + '.' + entityName); lockManager.acquireDatasetWriteLock(mdProvider.getLocks(), DatasetUtil.getFullyQualifiedName(dataset)); List<Dataset> datasets = clusterController.getAllDatasets(); if (datasets.contains(dataset)) { throw new HyracksDataException("Dataset " + dataset + " already exists"); } actionListener.add(dataset); datasets.add(dataset); } finally { mdProvider.getLocks().reset(); } } };