protected void calculateNumElementPerPage() { int frameSize = ctx.getInitialFrameSize(); // The count of Minframe, and the count of tuples in a frame should be deducted. frameSize = frameSize - FixedSizeFrameTupleAppender.MINFRAME_COUNT_SIZE - FixedSizeFrameTupleAppender.TUPLE_COUNT_SIZE; numPossibleElementPerPage = (int) Math.floor((double) frameSize / (invListElementSize + ELEMENT_COUNT_SIZE)); }
private int selectPartitionsToReload(long freeSpace, int pid, int inMemTupCount) { for (int i = spilledStatus.nextSetBit(pid); i >= 0 && i < numOfPartitions; i = spilledStatus.nextSetBit(i + 1)) { int spilledTupleCount = buildPSizeInTups[i]; // Expected hash table size increase after reloading this partition long expectedHashTableByteSizeIncrease = SerializableHashTable.calculateByteSizeDeltaForTableSizeChange( inMemTupCount, spilledTupleCount, ctx.getInitialFrameSize()); if (freeSpace >= buildRFWriters[i].getFileSize() + expectedHashTableByteSizeIncrease) { return i; } } return -1; }
public FrameSpiller(IHyracksTaskContext ctx, String fileNamePrefix, long budgetInBytes) throws HyracksDataException { this.frame = new VSizeFrame(ctx); this.fileNamePrefix = fileNamePrefix; this.budget = (int) Math.min(budgetInBytes / ctx.getInitialFrameSize(), Integer.MAX_VALUE); if (budget <= 0) { throw new HyracksDataException("Invalid budget " + budgetInBytes + ". Budget must be larger than 0"); } }
/** * Tries to allocate buffers to accommodate the results in memory. */ protected boolean tryAllocateBuffers(int numExpectedPages) throws HyracksDataException { boolean allBufferAllocated = true; while (buffers.size() < numExpectedPages) { ByteBuffer tmpBuffer = bufferManager.acquireFrame(ctx.getInitialFrameSize()); if (tmpBuffer == null) { // Budget exhausted allBufferAllocated = false; break; } else { clearBuffer(tmpBuffer); } buffers.add(tmpBuffer); } return allBufferAllocated; }
private void prepareFrames(int extraFreeMem, List<GroupVSizeFrame> inFrames, List<GeneratedRunFileReader> partialRuns) throws HyracksDataException { if (extraFreeMem > 0 && partialRuns.size() > 1) { int extraFrames = extraFreeMem / ctx.getInitialFrameSize(); int avg = (extraFrames / partialRuns.size()) * ctx.getInitialFrameSize(); int residue = extraFrames % partialRuns.size(); for (int i = 0; i < residue; i++) { partialRuns.get(i).updateSize(Math.min(FrameConstants.MAX_FRAMESIZE, partialRuns.get(i).getMaxFrameSize() + avg + ctx.getInitialFrameSize())); } for (int i = residue; i < partialRuns.size() && avg > 0; i++) { partialRuns.get(i) .updateSize(Math.min(FrameConstants.MAX_FRAMESIZE, partialRuns.get(i).getMaxFrameSize() + avg)); } } if (inFrames.size() > partialRuns.size()) { inFrames.subList(partialRuns.size(), inFrames.size()).clear(); } int i; for (i = 0; i < inFrames.size(); i++) { inFrames.get(i).resize(partialRuns.get(i).getMaxFrameSize()); } for (; i < partialRuns.size(); i++) { inFrames.add(new GroupVSizeFrame(ctx, partialRuns.get(i).getMaxFrameSize())); } }
public synchronized void registerPartitionRequest(PartitionId partitionId, NetworkOutputChannel writer) { if (failedJobsCache.getIfPresent(partitionId.getJobId()) != null) { writer.abort(AbstractChannelWriteInterface.REMOTE_ERROR_CODE); return; } List<IPartition> pList = availablePartitionMap.get(partitionId); if (pList != null && !pList.isEmpty()) { IPartition partition = pList.get(0); writer.setFrameSize(partition.getTaskContext().getInitialFrameSize()); partition.writeTo(writer); if (!partition.isReusable()) { availablePartitionMap.remove(partitionId); } } else { partitionRequests.put(partitionId, writer); } }
public ResultPartitionWriter(IHyracksTaskContext ctx, IResultPartitionManager manager, JobId jobId, ResultSetId rsId, boolean asyncMode, boolean orderedResult, int partition, int nPartitions, ResultMemoryManager resultMemoryManager, IWorkspaceFileFactory fileFactory, long maxReads) { this.manager = manager; this.jobId = jobId; this.resultSetId = rsId; this.orderedResult = orderedResult; this.partition = partition; this.nPartitions = nPartitions; this.resultMemoryManager = resultMemoryManager; resultSetPartitionId = new ResultSetPartitionId(jobId, rsId, partition); resultState = new ResultState(resultSetPartitionId, asyncMode, ctx.getIoManager(), fileFactory, ctx.getInitialFrameSize(), maxReads); }
public void initBuild() throws HyracksDataException { framePool = new DeallocatableFramePool(ctx, memSizeInFrames * ctx.getInitialFrameSize()); bufferManagerForHashTable = new FramePoolBackedFrameBufferManager(framePool); bufferManager = new VPartitionTupleBufferManager( PreferToSpillFullyOccupiedFramePolicy.createAtMostOneFrameForSpilledPartitionConstrain(spilledStatus), numOfPartitions, framePool); spillPolicy = new PreferToSpillFullyOccupiedFramePolicy(bufferManager, spilledStatus); spilledStatus.clear(); buildPSizeInTups = new int[numOfPartitions]; }
@Override public void open() throws HyracksDataException { IFramePool framePool = new VariableFramePool(ctx, (frameLimit - 1) * ctx.getInitialFrameSize()); IDeletableTupleBufferManager bufferManager = new VariableDeletableTupleMemoryManager(framePool, recordDescriptor); tupleSorter = new TupleSorterHeapSort(ctx, bufferManager, topK, sortFields, nmkFactories, comparatorFactories); super.open(); }
public InvertedIndexSearchResult(ITypeTraits[] invListFields, IHyracksTaskContext ctx, ISimpleFrameBufferManager bufferManager) throws HyracksDataException { initTypeTraits(invListFields); this.ctx = ctx; appender = new FixedSizeFrameTupleAppender(ctx.getInitialFrameSize(), typeTraits); accessor = new FixedSizeFrameTupleAccessor(ctx.getInitialFrameSize(), typeTraits); tuple = new FixedSizeTupleReference(typeTraits); this.bufferManager = bufferManager; this.isInReadMode = false; this.isWriteFinished = false; this.isInMemoryOpMode = false; this.isFileOpened = false; this.ioBufferFrame = null; this.ioBuffer = null; this.buffers = null; this.currentWriterBufIdx = 0; this.currentReaderBufIdx = 0; this.numResults = 0; calculateNumElementPerPage(); // Allocates one frame for read/write operation. prepareIOBuffer(); }
public synchronized void registerPartition(PartitionId pid, CcId ccId, TaskAttemptId taId, IPartition partition, PartitionState state, boolean updateToCC) throws HyracksDataException { try { /* * process pending requests */ NetworkOutputChannel writer = partitionRequests.remove(pid); if (writer != null) { writer.setFrameSize(partition.getTaskContext().getInitialFrameSize()); partition.writeTo(writer); if (!partition.isReusable()) { return; } } /* * put a coming available partition into the available partition map */ List<IPartition> pList = availablePartitionMap.computeIfAbsent(pid, k -> new ArrayList<>()); pList.add(partition); /* * update to CC only when necessary */ if (updateToCC) { updatePartitionState(ccId, pid, taId, partition, state); } } catch (Exception e) { throw HyracksDataException.create(e); } }
@Override public void open() throws HyracksDataException { tupleCount = 0; tupleAccessors = new IFrameTupleAccessor[runCursors.size()]; outFrameAppender = new FrameTupleAppender(); Comparator<ReferenceEntry> comparator = createEntryComparator(comparators); topTuples = new ReferencedPriorityQueue(runCursors.size(), comparator, sortFields, nmkComputer); tupleIndexes = new int[runCursors.size()]; for (int i = 0; i < runCursors.size(); i++) { tupleIndexes[i] = 0; int runIndex = topTuples.peek().getRunid(); runCursors.get(runIndex).open(); if (runCursors.get(runIndex).nextFrame(inFrames.get(runIndex))) { tupleAccessors[runIndex] = new GroupFrameAccessor(ctx.getInitialFrameSize(), recordDesc); tupleAccessors[runIndex].reset(inFrames.get(runIndex).getBuffer()); setNextTopTuple(runIndex, tupleIndexes, runCursors, inFrames, tupleAccessors, topTuples); } else { closeRun(runIndex, runCursors, tupleAccessors); topTuples.pop(); } } }
if (runs[i] != null) { int memoryBudgetInBytes = ctx.getInitialFrameSize() * frameLimit; int groupByColumnsCount = mergeGroupFields.length; int hashTableCardinality = ExternalGroupOperatorDescriptor.calculateGroupByTableCardinality( memoryBudgetInBytes, groupByColumnsCount, ctx.getInitialFrameSize()); hashTableCardinality = Math.min(hashTableCardinality, numOfTuples[i]); ISpillableTable partitionTable = spillableTableFactory.buildSpillableTable(ctx, hashTableCardinality,
public LSMInvertedIndexSearchOperatorNodePushable(IHyracksTaskContext ctx, RecordDescriptor inputRecDesc, int partition, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, IIndexDataflowHelperFactory indexHelperFactory, boolean retainInput, boolean retainMissing, IMissingWriterFactory missingWriterFactory, ISearchOperationCallbackFactory searchCallbackFactory, IInvertedIndexSearchModifier searchModifier, IBinaryTokenizerFactory binaryTokenizerFactory, int queryFieldIndex, boolean isFullTextSearchQuery, int numOfFields, boolean appendIndexFilter, int frameLimit) throws HyracksDataException { super(ctx, inputRecDesc, partition, minFilterFieldIndexes, maxFilterFieldIndexes, indexHelperFactory, retainInput, retainMissing, missingWriterFactory, searchCallbackFactory, appendIndexFilter); this.searchModifier = searchModifier; this.binaryTokenizerFactory = binaryTokenizerFactory; this.queryFieldIndex = queryFieldIndex; this.isFullTextSearchQuery = isFullTextSearchQuery; // If retainInput is true, the frameTuple is created in IndexSearchOperatorNodePushable.open(). if (!retainInput) { this.frameTuple = new FrameTupleReference(); } this.numOfFields = numOfFields; // Intermediate and final search result will use this buffer manager to get frames. this.framePool = new DeallocatableFramePool(ctx, frameLimit * ctx.getInitialFrameSize()); this.bufferManagerForSearch = new FramePoolBackedFrameBufferManager(framePool); // Keep the buffer manager in the hyracks context so that the search process can get it via the context. TaskUtil.put(HyracksConstants.INVERTED_INDEX_SEARCH_FRAME_MANAGER, bufferManagerForSearch, ctx); }
public PreclusteredGroupWriter(IHyracksTaskContext ctx, int[] groupFields, IBinaryComparator[] comparators, IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor inRecordDesc, RecordDescriptor outRecordDesc, IFrameWriter writer, boolean outputPartial, boolean groupAll, int framesLimit) throws HyracksDataException { this.groupFields = groupFields; this.comparators = comparators; if (framesLimit >= 0 && framesLimit <= 2) { throw HyracksDataException.create(ErrorCode.ILLEGAL_MEMORY_BUDGET, "GROUP BY", Long.toString(((long) (framesLimit)) * ctx.getInitialFrameSize()), Long.toString(2L * ctx.getInitialFrameSize())); } // Deducts input/output frames. this.memoryLimit = framesLimit <= 0 ? -1 : ((long) (framesLimit - 2)) * ctx.getInitialFrameSize(); this.aggregator = aggregatorFactory.createAggregator(ctx, inRecordDesc, outRecordDesc, groupFields, groupFields, writer, this.memoryLimit); this.aggregateState = aggregator.createAggregateStates(); copyFrame = new VSizeFrame(ctx); inFrameAccessor = new FrameTupleAccessor(inRecordDesc); copyFrameAccessor = new FrameTupleAccessor(inRecordDesc); copyFrameAccessor.reset(copyFrame.getBuffer()); VSizeFrame outFrame = new VSizeFrame(ctx); FrameTupleAppender appender = new FrameTupleAppender(); appender.reset(outFrame, true); appenderWrapper = new FrameTupleAppenderWrapper(appender, writer); tupleBuilder = new ArrayTupleBuilder(outRecordDesc.getFields().length); this.outputPartial = outputPartial; this.groupAll = groupAll; }
public FeedRuntimeInputHandler(IHyracksTaskContext ctx, FeedConnectionId connectionId, ActiveRuntimeId runtimeId, IFrameWriter writer, FeedPolicyAccessor fpa, FrameTupleAccessor fta, ConcurrentFramePool framePool) throws HyracksDataException { this.writer = writer; this.spiller = fpa.spillToDiskOnCongestion() ? new FrameSpiller(ctx, connectionId.getFeedId() + "_" + connectionId.getDatasetName() + "_" + runtimeId.getPartition(), fpa.getMaxSpillOnDisk()) : null; this.exceptionHandler = new FeedExceptionHandler(ctx, fta); this.fpa = fpa; this.framePool = framePool; this.inbox = new LinkedBlockingQueue<>(); this.consumer = new FrameTransporter(); this.consumerThread = new Thread(consumer, "FeedRuntimeInputHandler-FrameTransporter"); this.initialFrameSize = ctx.getInitialFrameSize(); this.frameAction = new FrameAction(); }
public AbstractExternalSortRunGenerator(IHyracksTaskContext ctx, int[] sortFields, INormalizedKeyComputerFactory[] keyNormalizerFactories, IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDesc, Algorithm alg, EnumFreeSlotPolicy policy, int framesLimit, int outputLimit) throws HyracksDataException { super(); this.ctx = ctx; maxSortFrames = framesLimit - 1; IFrameFreeSlotPolicy freeSlotPolicy = FrameFreeSlotPolicyFactory.createFreeSlotPolicy(policy, maxSortFrames); IFrameBufferManager bufferManager = new VariableFrameMemoryManager( new VariableFramePool(ctx, maxSortFrames * ctx.getInitialFrameSize()), freeSlotPolicy); if (alg == Algorithm.MERGE_SORT) { frameSorter = new FrameSorterMergeSort(ctx, bufferManager, maxSortFrames, sortFields, keyNormalizerFactories, comparatorFactories, recordDesc, outputLimit); } else { frameSorter = new FrameSorterQuickSort(ctx, bufferManager, maxSortFrames, sortFields, keyNormalizerFactories, comparatorFactories, recordDesc, outputLimit); } }
new VariableFrameMemoryManager(new VariableFramePool(ctx, ctx.getInitialFrameSize() * (memSize - 2)), FrameFreeSlotPolicyFactory.createFreeSlotPolicy(EnumFreeSlotPolicy.LAST_FIT, memSize - 2));
public AbstractTOccurrenceSearcher(IInPlaceInvertedIndex invIndex, IHyracksTaskContext ctx) throws HyracksDataException { this.invIndex = invIndex; this.ctx = ctx; if (ctx == null) { throw HyracksDataException.create(ErrorCode.CANNOT_CONTINUE_TEXT_SEARCH_HYRACKS_TASK_IS_NULL); } this.bufferManager = TaskUtil.get(HyracksConstants.INVERTED_INDEX_SEARCH_FRAME_MANAGER, ctx); if (bufferManager == null) { throw HyracksDataException.create(ErrorCode.CANNOT_CONTINUE_TEXT_SEARCH_BUFFER_MANAGER_IS_NULL); } this.finalSearchResult = new InvertedIndexFinalSearchResult(invIndex.getInvListTypeTraits(), ctx, bufferManager); this.invListMerger = new InvertedListMerger(ctx, invIndex, bufferManager); this.invListCmp = MultiComparator.create(invIndex.getInvListCmpFactories()); this.invListCursorFactory = new InvertedListCursorFactory(invIndex, ctx); this.invListCursorCache = new ObjectCache<>(invListCursorFactory, OBJECT_CACHE_INIT_SIZE, OBJECT_CACHE_EXPAND_SIZE); this.queryTokenFrame = new BufferManagerBackedVSizeFrame(ctx, bufferManager); if (queryTokenFrame.getBuffer() == null) { throw HyracksDataException.create(ErrorCode.NOT_ENOUGH_BUDGET_FOR_TEXTSEARCH, this.getClass().getSimpleName()); } this.queryTokenAppender = new FrameTupleAppenderAccessor(QUERY_TOKEN_REC_DESC); this.queryTokenAppender.reset(queryTokenFrame, true); this.isSingleInvertedList = false; this.searchResultTuple = new FixedSizeTupleReference(invIndex.getInvListTypeTraits()); this.searchResultFta = new FixedSizeFrameTupleAccessor(ctx.getInitialFrameSize(), invIndex.getInvListTypeTraits()); }
this.maxSortMemory = Long.MAX_VALUE; } else { this.maxSortMemory = (long) ctx.getInitialFrameSize() * maxSortFrames;