public String getDefaultDataverseName() { return defaultDataverse.getDataverseName(); }
private static void insertInitialDataverses(MetadataTransactionContext mdTxnCtx) throws AlgebricksException { String dataFormat = NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT; MetadataManager.INSTANCE.addDataverse(mdTxnCtx, new Dataverse(MetadataConstants.METADATA_DATAVERSE_NAME, dataFormat, MetadataUtil.PENDING_NO_OP)); MetadataManager.INSTANCE.addDataverse(mdTxnCtx, MetadataBuiltinEntities.DEFAULT_DATAVERSE); }
private static void recoverDataverse(MetadataTransactionContext mdTxnCtx, Dataverse dataverse) throws AlgebricksException { if (dataverse.getPendingOp() != MetadataUtil.PENDING_NO_OP) { // drop pending dataverse MetadataManager.INSTANCE.dropDataverse(mdTxnCtx, dataverse.getDataverseName()); if (LOGGER.isInfoEnabled()) { LOGGER.info("Dropped a pending dataverse: " + dataverse.getDataverseName()); } } else { List<Dataset> datasets = MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx, dataverse.getDataverseName()); for (Dataset dataset : datasets) { recoverDataset(mdTxnCtx, dataset); } } }
new Dataverse(dataverseName, dv.getDataFormat(), MetadataUtil.PENDING_DROP_OP)); if (activeDataverse != null && activeDataverse.getDataverseName() == dataverseName) { activeDataverse = null; if (activeDataverse != null && activeDataverse.getDataverseName() == dataverseName) { activeDataverse = null;
aString.setValue(instance.getDataverseName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(instance.getDataverseName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.DATAVERSE_ARECORD_NAME_FIELD_INDEX, fieldValue); aString.setValue(instance.getDataFormat()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.DATAVERSE_ARECORD_FORMAT_FIELD_INDEX, fieldValue); aInt32.setValue(instance.getPendingOp()); aInt32Serde.serialize(aInt32, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.DATAVERSE_ARECORD_PENDINGOP_FIELD_INDEX, fieldValue);
@Override public boolean equals(Object o) { if (!(o instanceof Dataverse)) { return false; } Dataverse other = (Dataverse) o; return dataverseName.equals(other.getDataverseName()); }
public void dropDataverse(String dataverseName) { Dataverse dataverse = new Dataverse(dataverseName, null, MetadataUtil.PENDING_NO_OP); droppedCache.addDataverseIfNotExists(dataverse); logAndApply(new MetadataLogicalOperation(dataverse, false)); }
@Override public String getActiveDataverseName(String dataverse) { return (dataverse != null && !dataverse.isEmpty()) ? dataverse : activeDataverse.getDataverseName(); }
@Override public Dataverse getMetadataEntityFromTuple(ITupleReference frameTuple) throws HyracksDataException { byte[] serRecord = frameTuple.getFieldData(DATAVERSE_PAYLOAD_TUPLE_FIELD_INDEX); int recordStartOffset = frameTuple.getFieldStart(DATAVERSE_PAYLOAD_TUPLE_FIELD_INDEX); int recordLength = frameTuple.getFieldLength(DATAVERSE_PAYLOAD_TUPLE_FIELD_INDEX); ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength); DataInput in = new DataInputStream(stream); ARecord dataverseRecord = recordSerDes.deserialize(in); return new Dataverse(((AString) dataverseRecord.getValueByPos(0)).getStringValue(), ((AString) dataverseRecord.getValueByPos(1)).getStringValue(), ((AInt32) dataverseRecord.getValueByPos(3)).getIntegerValue()); }
public Dataverse addDataverseIfNotExists(Dataverse dataverse) { synchronized (dataverses) { synchronized (datasets) { synchronized (datatypes) { if (!dataverses.containsKey(dataverse)) { datasets.put(dataverse.getDataverseName(), new HashMap<String, Dataset>()); datatypes.put(dataverse.getDataverseName(), new HashMap<String, Datatype>()); adapters.put(dataverse.getDataverseName(), new HashMap<String, DatasourceAdapter>()); return dataverses.put(dataverse.getDataverseName(), dataverse); } return null; } } } }
MetadataManager.INSTANCE.addDataverse(mdTxnCtx, new Dataverse(dataverse, NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT, MetadataUtil.PENDING_NO_OP));
synchronized (feeds) { synchronized (compactionPolicies) { datasets.remove(dataverse.getDataverseName()); indexes.remove(dataverse.getDataverseName()); datatypes.remove(dataverse.getDataverseName()); adapters.remove(dataverse.getDataverseName()); compactionPolicies.remove(dataverse.getDataverseName()); List<FunctionSignature> markedFunctionsForRemoval = new ArrayList<>(); for (FunctionSignature signature : functions.keySet()) { if (signature.getNamespace().equals(dataverse.getDataverseName())) { markedFunctionsForRemoval.add(signature); functions.remove(signature); libraries.remove(dataverse.getDataverseName()); feeds.remove(dataverse.getDataverseName()); return dataverses.remove(dataverse.getDataverseName());
public IResourceFactory getPrimaryResourceFactory(IHyracksTaskContext ctx, PrimaryIndexInfo primaryIndexInfo, IStorageComponentProvider storageComponentProvider, Dataset dataset) throws AlgebricksException { Dataverse dataverse = new Dataverse(dataset.getDataverseName(), NonTaggedDataFormat.class.getName(), MetadataUtil.PENDING_NO_OP); Index index = primaryIndexInfo.getIndex(); CcApplicationContext appCtx = (CcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(); MetadataProvider mdProvider = new MetadataProvider(appCtx, dataverse); try { return dataset.getResourceFactory(mdProvider, index, primaryIndexInfo.recordType, primaryIndexInfo.metaType, primaryIndexInfo.mergePolicyFactory, primaryIndexInfo.mergePolicyProperties); } finally { mdProvider.getLocks().unlock(); } }
public static Pair<String, String> getDatasetInfo(MetadataProvider metadata, String datasetArg) { String first; String second; int i = datasetArg.indexOf('.'); if (i > 0 && i < datasetArg.length() - 1) { first = datasetArg.substring(0, i); second = datasetArg.substring(i + 1); } else { first = metadata.getDefaultDataverse() == null ? null : metadata.getDefaultDataverse().getDataverseName(); second = datasetArg; } return new Pair<>(first, second); } }
@SuppressWarnings("squid:S00112") protected boolean doCreateDataverseStatement(MetadataTransactionContext mdTxnCtx, MetadataProvider metadataProvider, CreateDataverseStatement stmtCreateDataverse) throws Exception { String dvName = stmtCreateDataverse.getDataverseName().getValue(); Dataverse dv = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), dvName); if (dv != null) { if (stmtCreateDataverse.getIfNotExists()) { MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); return false; } else { throw new CompilationException(ErrorCode.DATAVERSE_EXISTS, stmtCreateDataverse.getSourceLocation(), dvName); } } MetadataManager.INSTANCE.addDataverse(metadataProvider.getMetadataTxnContext(), new Dataverse(dvName, stmtCreateDataverse.getFormat(), MetadataUtil.PENDING_NO_OP)); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); return true; }
protected boolean isFunctionUsed(MetadataTransactionContext ctx, FunctionSignature signature, String currentDataverse) throws AlgebricksException { List<Dataverse> allDataverses = MetadataManager.INSTANCE.getDataverses(ctx); for (Dataverse dataverse : allDataverses) { if (dataverse.getDataverseName().equals(currentDataverse)) { continue; } List<Feed> feeds = MetadataManager.INSTANCE.getFeeds(ctx, dataverse.getDataverseName()); for (Feed feed : feeds) { List<FeedConnection> feedConnections = MetadataManager.INSTANCE.getFeedConections(ctx, dataverse.getDataverseName(), feed.getFeedName()); for (FeedConnection conn : feedConnections) { if (conn.containsFunction(signature)) { return true; } } } } return false; }
public SecondaryIndexInfo createSecondaryIndex(PrimaryIndexInfo primaryIndexInfo, Index secondaryIndex, IStorageComponentProvider storageComponentProvider, int partition) throws AlgebricksException, HyracksDataException, RemoteException, ACIDException { MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); org.apache.hyracks.algebricks.common.utils.Pair<ILSMMergePolicyFactory, Map<String, String>> mergePolicy = DatasetUtil.getMergePolicyFactory(primaryIndexInfo.dataset, mdTxnCtx); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); Dataverse dataverse = new Dataverse(primaryIndexInfo.dataset.getDataverseName(), NonTaggedDataFormat.class.getName(), MetadataUtil.PENDING_NO_OP); MetadataProvider mdProvider = new MetadataProvider( (ICcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(), dataverse); SecondaryIndexInfo secondaryIndexInfo = new SecondaryIndexInfo(primaryIndexInfo, secondaryIndex); try { IResourceFactory resourceFactory = primaryIndexInfo.dataset.getResourceFactory(mdProvider, secondaryIndex, primaryIndexInfo.recordType, primaryIndexInfo.metaType, mergePolicy.first, mergePolicy.second); IndexBuilderFactory indexBuilderFactory = new IndexBuilderFactory(storageComponentProvider.getStorageManager(), secondaryIndexInfo.fileSplitProvider, resourceFactory, true); IHyracksTaskContext ctx = createTestContext(newJobId(), partition, false); IIndexBuilder indexBuilder = indexBuilderFactory.create(ctx, partition); indexBuilder.build(); } finally { mdProvider.getLocks().unlock(); } return secondaryIndexInfo; }
protected MetadataTransactionContext doRecovery(ICcApplicationContext appCtx, MetadataTransactionContext mdTxnCtx) throws Exception { // Loop over datasets for (Dataverse dataverse : MetadataManager.INSTANCE.getDataverses(mdTxnCtx)) { mdTxnCtx = recoverDatasets(appCtx, mdTxnCtx, dataverse); // Fixes ASTERIXDB-2386 by caching the dataverse during recovery MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverse.getDataverseName()); } return mdTxnCtx; }
public PrimaryIndexInfo createPrimaryIndex(Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType, int[] filterFields, IStorageComponentProvider storageComponentProvider, int[] primaryKeyIndexes, List<Integer> primaryKeyIndicators, int partition) throws AlgebricksException, HyracksDataException, RemoteException, ACIDException { MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); org.apache.hyracks.algebricks.common.utils.Pair<ILSMMergePolicyFactory, Map<String, String>> mergePolicy = DatasetUtil.getMergePolicyFactory(dataset, mdTxnCtx); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType, mergePolicy.first, mergePolicy.second, filterFields, primaryKeyIndexes, primaryKeyIndicators); Dataverse dataverse = new Dataverse(dataset.getDataverseName(), NonTaggedDataFormat.class.getName(), MetadataUtil.PENDING_NO_OP); MetadataProvider mdProvider = new MetadataProvider( (ICcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(), dataverse); try { IResourceFactory resourceFactory = dataset.getResourceFactory(mdProvider, primaryIndexInfo.index, recordType, metaType, mergePolicy.first, mergePolicy.second); IndexBuilderFactory indexBuilderFactory = new IndexBuilderFactory(storageComponentProvider.getStorageManager(), primaryIndexInfo.getFileSplitProvider(), resourceFactory, true); IHyracksTaskContext ctx = createTestContext(newJobId(), partition, false); IIndexBuilder indexBuilder = indexBuilderFactory.create(ctx, partition); indexBuilder.build(); } finally { mdProvider.getLocks().unlock(); } return primaryIndexInfo; }
private Pair<String, String> parseDatasetReference(MetadataProvider metadataProvider, String datasetArg) throws AlgebricksException { String[] datasetNameComponents = datasetArg.split("\\."); String dataverseName; String datasetName; if (datasetNameComponents.length == 1) { Dataverse defaultDataverse = metadataProvider.getDefaultDataverse(); if (defaultDataverse == null) { throw new AlgebricksException("Unresolved dataset " + datasetArg + " Dataverse not specified."); } dataverseName = defaultDataverse.getDataverseName(); datasetName = datasetNameComponents[0]; } else { dataverseName = datasetNameComponents[0]; datasetName = datasetNameComponents[1]; } return new Pair<>(dataverseName, datasetName); }