public void dropFeedConnection(String dataverseName, String feedName, String datasetName) { FeedConnection feedConnection = new FeedConnection(dataverseName, feedName, datasetName, null, null, null, null); droppedCache.addFeedConnectionIfNotExists(feedConnection); logAndApply(new MetadataLogicalOperation(feedConnection, false)); }
private static JobSpecification getConnectionJob(MetadataProvider metadataProvider, FeedConnection feedConn, IStatementExecutor statementExecutor, IHyracksClientConnection hcc, Boolean insertFeed) throws AlgebricksException, RemoteException, ACIDException { metadataProvider.getConfig().put(FeedActivityDetails.FEED_POLICY_NAME, feedConn.getPolicyName()); Query feedConnQuery = makeConnectionQuery(feedConn); CompiledStatements.ICompiledDmlStatement clfrqs; if (insertFeed) { InsertStatement stmtUpsert = new InsertStatement(new Identifier(feedConn.getDataverseName()), new Identifier(feedConn.getDatasetName()), feedConnQuery, -1, null, null); clfrqs = new CompiledStatements.CompiledInsertStatement(feedConn.getDataverseName(), feedConn.getDatasetName(), feedConnQuery, stmtUpsert.getVarCounter(), null, null); } else { UpsertStatement stmtUpsert = new UpsertStatement(new Identifier(feedConn.getDataverseName()), new Identifier(feedConn.getDatasetName()), feedConnQuery, -1, null, null); clfrqs = new CompiledStatements.CompiledUpsertStatement(feedConn.getDataverseName(), feedConn.getDatasetName(), feedConnQuery, stmtUpsert.getVarCounter(), null, null); } return statementExecutor.rewriteCompileQuery(hcc, metadataProvider, feedConnQuery, clfrqs, null, null); }
protected void writeOpenPart(FeedConnection fc) throws HyracksDataException { if (fc.getWhereClauseBody() != null && fc.getWhereClauseBody().length() > 0) { fieldName.reset(); aString.setValue(MetadataRecordTypes.FIELD_NAME_WHERE_CLAUSE); stringSerde.serialize(aString, fieldName.getDataOutput()); fieldValue.reset(); aString.setValue(fc.getWhereClauseBody()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(fieldName, fieldValue); } }
public FeedConnection addFeedConnectionIfNotExists(FeedConnection feedConnection) { synchronized (feedConnections) { Map<String, FeedConnection> feedConnsInDataverse = feedConnections.get(feedConnection.getDataverseName()); if (feedConnsInDataverse == null) { feedConnections.put(feedConnection.getDataverseName(), new HashMap<>()); feedConnsInDataverse = feedConnections.get(feedConnection.getDataverseName()); } return feedConnsInDataverse.put(feedConnection.getConnectionId(), feedConnection); } }
@Override public synchronized void remove(Dataset dataset) throws HyracksDataException { super.remove(dataset); feedConnections.removeIf(o -> o.getDataverseName().equals(dataset.getDataverseName()) && o.getDatasetName().equals(dataset.getDatasetName())); }
private static Query makeConnectionQuery(FeedConnection feedConnection) throws AlgebricksException { VarIdentifier fromVarId = SqlppVariableUtil.toInternalVariableIdentifier(feedConnection.getFeedName()); VariableExpr fromTermLeftExpr = new VariableExpr(fromVarId); List<Expression> exprList = addArgs(feedConnection.getDataverseName(), feedConnection.getFeedId().getEntityName(), feedConnection.getFeedId().getEntityName(), FeedRuntimeType.INTAKE.toString(), feedConnection.getDatasetName(), feedConnection.getOutputType()); CallExpr datasrouceCallFunction = new CallExpr(new FunctionSignature(BuiltinFunctions.FEED_COLLECT), exprList); FromTerm fromterm = new FromTerm(datasrouceCallFunction, fromTermLeftExpr, null, null); FromClause fromClause = new FromClause(Arrays.asList(fromterm)); WhereClause whereClause = null; if (feedConnection.getWhereClauseBody().length() != 0) { String whereClauseExpr = feedConnection.getWhereClauseBody() + ";"; IParserFactory sqlppParserFactory = new SqlppParserFactory(); IParser sqlppParser = sqlppParserFactory.createParser(whereClauseExpr); for (FunctionSignature functionSignature : feedConnection.getAppliedFunctions()) { CallExpr functionCallExpr = new CallExpr(functionSignature, addArgs(previousVarExpr)); previousVarExpr = functionCallExpr;
aString.setValue(me.getDataverseName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(me.getFeedName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(me.getDatasetName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(me.getDataverseName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.FEED_CONN_DATAVERSE_NAME_FIELD_INDEX, fieldValue); aString.setValue(me.getFeedName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.FEED_CONN_FEED_NAME_FIELD_INDEX, fieldValue); aString.setValue(me.getDatasetName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.FEED_CONN_DATASET_NAME_FIELD_INDEX, fieldValue); aString.setValue(me.getOutputType()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.FEED_CONN_OUTPUT_TYPE_INDEX, fieldValue); aString.setValue(me.getPolicyName());
private void writeAppliedFunctionsField(IARecordBuilder rb, FeedConnection fc, ArrayBackedValueStorage buffer) throws HyracksDataException { UnorderedListBuilder listBuilder = new UnorderedListBuilder(); ArrayBackedValueStorage listEleBuffer = new ArrayBackedValueStorage(); listBuilder.reset((AUnorderedListType) MetadataRecordTypes.FEED_CONNECTION_RECORDTYPE .getFieldTypes()[MetadataRecordTypes.FEED_CONN_APPLIED_FUNCTIONS_FIELD_INDEX]); if (fc.getAppliedFunctions() != null) { List<FunctionSignature> appliedFunctions = fc.getAppliedFunctions(); for (FunctionSignature af : appliedFunctions) { listEleBuffer.reset(); aString.setValue(af.getNamespace() + "." + af.getName()); stringSerde.serialize(aString, listEleBuffer.getDataOutput()); listBuilder.addItem(listEleBuffer); } } listBuilder.write(buffer.getDataOutput(), true); rb.addField(MetadataRecordTypes.FEED_CONN_APPLIED_FUNCTIONS_FIELD_INDEX, buffer); } }
@Override public void dropFeed(MetadataTransactionContext ctx, String dataverse, String feedName) throws AlgebricksException { Feed feed = null; List<FeedConnection> feedConnections = null; try { feed = metadataNode.getFeed(ctx.getTxnId(), dataverse, feedName); feedConnections = metadataNode.getFeedConnections(ctx.getTxnId(), dataverse, feedName); metadataNode.dropFeed(ctx.getTxnId(), dataverse, feedName); for (FeedConnection feedConnection : feedConnections) { metadataNode.dropFeedConnection(ctx.getTxnId(), dataverse, feedName, feedConnection.getDatasetName()); ctx.dropFeedConnection(dataverse, feedName, feedConnection.getDatasetName()); } } catch (RemoteException e) { throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); } ctx.dropFeed(feed); }
protected boolean isFunctionUsed(MetadataTransactionContext ctx, FunctionSignature signature, String currentDataverse) throws AlgebricksException { List<Dataverse> allDataverses = MetadataManager.INSTANCE.getDataverses(ctx); for (Dataverse dataverse : allDataverses) { if (dataverse.getDataverseName().equals(currentDataverse)) { continue; } List<Feed> feeds = MetadataManager.INSTANCE.getFeeds(ctx, dataverse.getDataverseName()); for (Feed feed : feeds) { List<FeedConnection> feedConnections = MetadataManager.INSTANCE.getFeedConections(ctx, dataverse.getDataverseName(), feed.getFeedName()); for (FeedConnection conn : feedConnections) { if (conn.containsFunction(signature)) { return true; } } } } return false; }
String fqName = feedConnection.getDataverseName() + "." + feedConnection.getDatasetName(); lockManager.acquireDatasetReadLock(metadataProvider.getLocks(), fqName); for (FeedConnection connection : feedConnections) { Dataset ds = metadataProvider.findDataset(connection.getDataverseName(), connection.getDatasetName()); datasets.add(ds);
public FeedConnection dropFeedConnection(FeedConnection feedConnection) { synchronized (feedConnections) { Map<String, FeedConnection> feedConnsInDataverse = feedConnections.get(feedConnection.getDataverseName()); if (feedConnsInDataverse != null) { return feedConnsInDataverse.remove(feedConnection.getConnectionId()); } else { return null; } } }
if (feedConnection.getAppliedFunctions() == null || feedConnection.getAppliedFunctions().size() == 0) { return false;
feedConnections = getFeedConnections(txnId, dataverseName, feed.getFeedName()); for (FeedConnection feedConnection : feedConnections) { dropFeedConnection(txnId, dataverseName, feed.getFeedName(), feedConnection.getDatasetName());
operatorIdMapping.clear(); Map<OperatorDescriptorId, IOperatorDescriptor> operatorsMap = subJob.getOperatorMap(); String datasetName = feedConnections.get(iter1).getDatasetName(); FeedConnectionId feedConnectionId = new FeedConnectionId(ingestionOp.getEntityId(), datasetName); FeedMetadataUtil.validateIfPolicyExists(curFeedConnection.getDataverseName(), curFeedConnection.getPolicyName(), metadataProvider.getMetadataTxnContext()); .findDataset(curFeedConnection.getDataverseName(), curFeedConnection.getDatasetName()) .getDatasetId(); TxnId txnId = ((JobEventListenerFactory) subJob.getJobletEventListenerFactory()).getTxnId(datasetId);
whereClauseIdx >= 0 ? ((AString) feedConnRecord.getValueByPos(whereClauseIdx)).getStringValue() : ""; return new FeedConnection(dataverseName, feedName, datasetName, appliedFunctions, policyName, whereClauseBody, outputType);
"Feed" + feedName + " is already connected dataset " + datasetName); fc = new FeedConnection(dataverseName, feedName, datasetName, appliedFunctions, policyName, whereClauseBody, outputType.getTypeName()); MetadataManager.INSTANCE.addFeedConnection(metadataProvider.getMetadataTxnContext(), fc);