public static JobSpecification createJobSpecification(ICcApplicationContext appCtx) { CompilerProperties compilerProperties = appCtx.getCompilerProperties(); int frameSize = compilerProperties.getFrameSize(); return new JobSpecification(frameSize); } }
@Override public CompilerProperties newCompilerProperties() { return new CompilerProperties(propertiesAccessor); }
public static int getSortNumFrames(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig, SourceLocation sourceLoc) throws AlgebricksException { return getFrameLimit(CompilerProperties.COMPILER_SORTMEMORY_KEY, (String) querySpecificConfig.get(CompilerProperties.COMPILER_SORTMEMORY_KEY), compilerProperties.getSortMemorySize(), compilerProperties.getFrameSize(), MIN_FRAME_LIMIT_FOR_SORT, sourceLoc); }
public static int getTextSearchNumFrames(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig, SourceLocation sourceLoc) throws AlgebricksException { return getFrameLimit(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY, (String) querySpecificConfig.get(CompilerProperties.COMPILER_TEXTSEARCHMEMORY_KEY), compilerProperties.getTextSearchMemorySize(), compilerProperties.getFrameSize(), MIN_FRAME_LIMIT_FOR_TEXT_SEARCH, sourceLoc); }
public static PhysicalOptimizationConfig createPhysicalOptimizationConf(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig, SourceLocation sourceLoc) throws AlgebricksException { int frameSize = compilerProperties.getFrameSize(); int sortFrameLimit = getSortNumFrames(compilerProperties, querySpecificConfig, sourceLoc); int groupFrameLimit = getFrameLimit(CompilerProperties.COMPILER_GROUPMEMORY_KEY, (String) querySpecificConfig.get(CompilerProperties.COMPILER_GROUPMEMORY_KEY), compilerProperties.getGroupMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_GROUP_BY, sourceLoc); int joinFrameLimit = getFrameLimit(CompilerProperties.COMPILER_JOINMEMORY_KEY, (String) querySpecificConfig.get(CompilerProperties.COMPILER_JOINMEMORY_KEY), compilerProperties.getJoinMemorySize(), frameSize, MIN_FRAME_LIMIT_FOR_JOIN, sourceLoc); int textSearchFrameLimit = getTextSearchNumFrames(compilerProperties, querySpecificConfig, sourceLoc); int sortNumSamples = getSortSamples(compilerProperties, querySpecificConfig, sourceLoc); boolean fullParallelSort = getSortParallel(compilerProperties, querySpecificConfig); PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig(); physOptConf.setFrameSize(frameSize); physOptConf.setMaxFramesExternalSort(sortFrameLimit); physOptConf.setMaxFramesExternalGroupBy(groupFrameLimit); physOptConf.setMaxFramesForJoin(joinFrameLimit); physOptConf.setMaxFramesForTextSearch(textSearchFrameLimit); physOptConf.setSortParallel(fullParallelSort); physOptConf.setSortSamples(sortNumSamples); return physOptConf; }
private static boolean getSortParallel(CompilerProperties compilerProperties, Map<String, Object> querySpecificConfig) { String valueInQuery = (String) querySpecificConfig.get(CompilerProperties.COMPILER_SORT_PARALLEL_KEY); if (valueInQuery != null) { return OptionTypes.BOOLEAN.parse(valueInQuery); } return compilerProperties.getSortParallel(); }
compilerProperties.getParallelism()); AlgebricksAbsolutePartitionConstraint computationLocations = chooseLocations(clusterInfoCollector, parallelism, metadataProvider.getClusterLocations());
static IResultSet getResultSet(IHyracksClientConnection hcc, IApplicationContext appCtx, final Map<String, Object> ctx) throws Exception { IResultSet resultSet = (IResultSet) ctx.get(RESULTSET_ATTR); if (resultSet == null) { synchronized (ctx) { resultSet = (IResultSet) ctx.get(RESULTSET_ATTR); if (resultSet == null) { resultSet = new ResultSet(hcc, appCtx.getServiceContext().getControllerService().getNetworkSecurityManager() .getSocketChannelFactory(), appCtx.getCompilerProperties().getFrameSize(), ResultReader.NUM_READERS); ctx.put(RESULTSET_ATTR, resultSet); } } } return resultSet; } }
compilerProperties = new CompilerProperties(propertiesAccessor); externalProperties = new ExternalProperties(propertiesAccessor); metadataProperties = new MetadataProperties(propertiesAccessor);
public ResultPrinter(IApplicationContext appCtx, SessionOutput output, Stats stats, ARecordType recordType) { this.output = output; this.conf = output.config(); this.stats = stats; this.recordType = recordType; this.indentJSON = conf.is(SessionConfig.FORMAT_INDENT_JSON); this.quoteRecord = conf.is(SessionConfig.FORMAT_QUOTE_RECORD); this.resultDisplayFrameMgr = new FrameManager(appCtx.getCompilerProperties().getFrameSize()); if (indentJSON) { this.om = new ObjectMapper(); DefaultPrettyPrinter.Indenter i = new DefaultPrettyPrinter.Indenter() { @Override public void writeIndentation(JsonGenerator jsonGenerator, int i) throws IOException { jsonGenerator.writeRaw('\n'); for (int j = 0; j < i + 1; ++j) { jsonGenerator.writeRaw('\t'); } } @Override public boolean isInline() { return false; } }; PrettyPrinter pp = new DefaultPrettyPrinter().withObjectIndenter(i).withArrayIndenter(i); this.ow = om.writer(pp); } }
final CompilerProperties compilerProperties = Mockito.mock(CompilerProperties.class); Mockito.when(appCtx.getCompilerProperties()).thenReturn(compilerProperties); Mockito.when(compilerProperties.getFrameSize()).thenReturn(StorageUtil.getIntSizeInBytes(32, KILOBYTE)); final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final PrintWriter out = new PrintWriter(baos, true);
private static Pair<JobSpecification, IAdapterFactory> buildFeedIntakeJobSpec(Feed feed, MetadataProvider metadataProvider, FeedPolicyAccessor policyAccessor) throws Exception { JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext()); spec.setFrameSize(metadataProvider.getApplicationContext().getCompilerProperties().getFrameSize()); IAdapterFactory adapterFactory; IOperatorDescriptor feedIngestor; AlgebricksPartitionConstraint ingesterPc; Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IAdapterFactory> t = metadataProvider.buildFeedIntakeRuntime(spec, feed, policyAccessor); feedIngestor = t.first; ingesterPc = t.second; adapterFactory = t.third; AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, feedIngestor, ingesterPc); NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(spec); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, nullSink, ingesterPc); spec.connect(new OneToOneConnectorDescriptor(spec), feedIngestor, 0, nullSink, 0); spec.addRoot(nullSink); return Pair.of(spec, adapterFactory); }
public static void flushDataset(IHyracksClientConnection hcc, MetadataProvider metadataProvider, Dataset dataset) throws Exception { CompilerProperties compilerProperties = metadataProvider.getApplicationContext().getCompilerProperties(); int frameSize = compilerProperties.getFrameSize(); JobSpecification spec = new JobSpecification(frameSize); RecordDescriptor[] rDescs = new RecordDescriptor[] { new RecordDescriptor(new ISerializerDeserializer[] {}) }; AlgebricksMetaOperatorDescriptor emptySource = new AlgebricksMetaOperatorDescriptor(spec, 0, 1, new IPushRuntimeFactory[] { new EmptyTupleSourceRuntimeFactory() }, rDescs); TxnId txnId = metadataProvider.getTxnIdFactory().create(); FlushDatasetOperatorDescriptor flushOperator = new FlushDatasetOperatorDescriptor(spec, txnId, dataset.getDatasetId()); spec.connect(new OneToOneConnectorDescriptor(spec), emptySource, 0, flushOperator, 0); Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset, dataset.getDatasetName()); AlgebricksPartitionConstraint primaryPartitionConstraint = primarySplitsAndConstraint.second; AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, emptySource, primaryPartitionConstraint); JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(txnId, true); spec.setJobletEventListenerFactory(jobEventListenerFactory); JobUtils.runJob(hcc, spec, true); }
isShuttingdown = false; activeManager = new ActiveManager(threadExecutor, getServiceContext().getNodeId(), activeProperties.getMemoryComponentGlobalBudget(), compilerProperties.getFrameSize(), this.ncServiceContext);