@Override public synchronized void start(MetadataProvider metadataProvider) throws HyracksDataException, InterruptedException { super.start(metadataProvider); // Note: The current implementation of the wait for completion flag is problematic due to locking issues: // Locks obtained during the start of the feed are not released, and so, the feed can't be stopped // and also, read locks over dataverses, datasets, etc, are never released. boolean wait = Boolean.parseBoolean((String) metadataProvider.getConfig().get(StartFeedStatement.WAIT_FOR_COMPLETION)); if (wait) { IActiveEntityEventSubscriber stoppedSubscriber = new WaitForStateSubscriber(this, EnumSet.of(ActivityState.STOPPED)); stoppedSubscriber.sync(); } }
private static int getSortNumFrames(MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { return OptimizationConfUtil.getSortNumFrames(metadataProvider.getApplicationContext().getCompilerProperties(), metadataProvider.getConfig(), sourceLoc); }
public static Pair<JobSpecification, AlgebricksAbsolutePartitionConstraint> buildStartFeedJob( MetadataProvider metadataProvider, Feed feed, List<FeedConnection> feedConnections, IStatementExecutor statementExecutor, IHyracksClientConnection hcc) throws Exception { FeedPolicyAccessor fpa = new FeedPolicyAccessor(new HashMap<>()); Pair<JobSpecification, IAdapterFactory> intakeInfo = buildFeedIntakeJobSpec(feed, metadataProvider, fpa); List<JobSpecification> jobsList = new ArrayList<>(); // TODO: Figure out a better way to handle insert/upsert per conn instead of per feed Boolean insertFeed = ExternalDataUtils.isInsertFeed(feed.getConfiguration()); // Construct the ingestion Job JobSpecification intakeJob = intakeInfo.getLeft(); IAdapterFactory ingestionAdaptorFactory = intakeInfo.getRight(); String[] ingestionLocations = ingestionAdaptorFactory.getPartitionConstraint().getLocations(); // Add metadata configs metadataProvider.getConfig().put(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, Boolean.TRUE.toString()); metadataProvider.getConfig().put(FeedActivityDetails.COLLECT_LOCATIONS, StringUtils.join(ingestionLocations, ',')); // TODO: Once we deprecated AQL, this extra queryTranslator can be removed. IStatementExecutor translator = getSQLPPTranslator(metadataProvider, ((QueryTranslator) statementExecutor).getSessionOutput()); // Add connection job for (FeedConnection feedConnection : feedConnections) { JobSpecification connectionJob = getConnectionJob(metadataProvider, feedConnection, translator, hcc, insertFeed); jobsList.add(connectionJob); } return Pair.of(combineIntakeCollectJobs(metadataProvider, feed, intakeJob, jobsList, feedConnections, ingestionLocations), intakeInfo.getRight().getPartitionConstraint()); } }
return functionDecls; String value = (String) metadataProvider.getConfig().get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS); boolean includePrivateFunctions = (value != null) ? Boolean.valueOf(value.toLowerCase()) : false; Set<CallExpr> functionCalls = functionCollector.getFunctionCalls(expression);
Map<String, Object> config = mock(Map.class); when(metadataProvider.getDefaultDataverseName()).thenReturn(dvName); when(metadataProvider.getConfig()).thenReturn(config); when(config.get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS)).thenReturn("true"); when(metadataProvider.findDataset(anyString(), anyString())).thenReturn(mock(Dataset.class));
MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); DataSourceId asid = new DataSourceId(dataverse, getTargetFeed); String policyName = (String) metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME); FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName); if (policy == null) { String csLocations = (String) metadataProvider.getConfig().get(FeedActivityDetails.COLLECT_LOCATIONS); List<LogicalVariable> pkVars = new ArrayList<>(); FeedDataSource ds = createFeedDataSource(asid, targetDataset, sourceFeedName, subscriptionLocation,
private static JobSpecification getConnectionJob(MetadataProvider metadataProvider, FeedConnection feedConn, IStatementExecutor statementExecutor, IHyracksClientConnection hcc, Boolean insertFeed) throws AlgebricksException, RemoteException, ACIDException { metadataProvider.getConfig().put(FeedActivityDetails.FEED_POLICY_NAME, feedConn.getPolicyName()); Query feedConnQuery = makeConnectionQuery(feedConn); CompiledStatements.ICompiledDmlStatement clfrqs; if (insertFeed) { InsertStatement stmtUpsert = new InsertStatement(new Identifier(feedConn.getDataverseName()), new Identifier(feedConn.getDatasetName()), feedConnQuery, -1, null, null); clfrqs = new CompiledStatements.CompiledInsertStatement(feedConn.getDataverseName(), feedConn.getDatasetName(), feedConnQuery, stmtUpsert.getVarCounter(), null, null); } else { UpsertStatement stmtUpsert = new UpsertStatement(new Identifier(feedConn.getDataverseName()), new Identifier(feedConn.getDatasetName()), feedConnQuery, -1, null, null); clfrqs = new CompiledStatements.CompiledUpsertStatement(feedConn.getDataverseName(), feedConn.getDatasetName(), feedConnQuery, stmtUpsert.getVarCounter(), null, null); } return statementExecutor.rewriteCompileQuery(hcc, metadataProvider, feedConnQuery, clfrqs, null, null); }
metadataProvider.getConfig().putAll(config); metadataProvider.setWriterFactory(writerFactory); metadataProvider.setResultSerializerFactoryProvider(resultSerializerFactoryProvider);
Map<String, Object> queryConfig = metadataProvider.getConfig(); int frameLimit = OptimizationConfUtil.getTextSearchNumFrames(compilerProp, queryConfig, op.getSourceLocation());
Map<String, Object> querySpecificConfig = validateConfig(metadataProvider.getConfig(), sourceLoc); final PhysicalOptimizationConfig physOptConf = OptimizationConfUtil.createPhysicalOptimizationConf(compilerProperties, querySpecificConfig, sourceLoc);