/** * @throws Exception If failed. */ @Test public void testAddUpdateAdd() throws Exception { IgfsFileMap map = new IgfsFileMap(); IgniteUuid affKey = IgniteUuid.randomUuid(); map.addRange(new IgfsFileAffinityRange(0, 9, affKey)); map.updateRangeStatus(new IgfsFileAffinityRange(0, 9, affKey), RANGE_STATUS_MOVING); map.addRange(new IgfsFileAffinityRange(10, 19, affKey)); List<IgfsFileAffinityRange> ranges = map.ranges(); assertEquals(2, ranges.size()); assertEquals(RANGE_STATUS_MOVING, ranges.get(0).status()); assertTrue(ranges.get(0).regionEqual(new IgfsFileAffinityRange(0, 9, affKey))); assertEquals(RANGE_STATUS_INITIAL, ranges.get(1).status()); assertTrue(ranges.get(1).regionEqual(new IgfsFileAffinityRange(10, 19, affKey))); }
/** * Updates range status at given position (will split range into two if necessary). * * @param origIdx Original range index. * @param orig Original range at index. * @param update Range being updated. * @param status New status for range. */ private void updateRangeStatus0(int origIdx, IgfsFileAffinityRange orig, IgfsFileAffinityRange update, int status) { assert F.eq(orig.affinityKey(), update.affinityKey()); assert ranges.get(origIdx) == orig; if (orig.regionEqual(update)) ranges.set(origIdx, new IgfsFileAffinityRange(update, status)); else { // If range was expanded, new one should be larger. assert orig.endOffset() > update.endOffset(); ranges.set(origIdx, new IgfsFileAffinityRange(update, status)); ranges.add(origIdx + 1, new IgfsFileAffinityRange(update.endOffset() + 1, orig.endOffset(), orig.affinityKey())); } }
IgniteUuid affKey = IgniteUuid.randomUuid(); IgfsFileAffinityRange range = new IgfsFileAffinityRange(0, 9999, affKey); Collection<IgfsFileAffinityRange> split = range.split(10000); assertTrue(range.regionEqual(F.first(split))); split = range.split(5000); assertTrue(part.regionEqual(new IgfsFileAffinityRange(0, 4999, affKey))); assertTrue(part.regionEqual(new IgfsFileAffinityRange(5000, 9999, affKey))); split = range.split(3000); assertTrue(part.regionEqual(new IgfsFileAffinityRange(0, 2999, affKey))); assertTrue(part.regionEqual(new IgfsFileAffinityRange(3000, 5999, affKey))); assertTrue(part.regionEqual(new IgfsFileAffinityRange(6000, 8999, affKey))); assertTrue(part.regionEqual(new IgfsFileAffinityRange(9000, 9999, affKey)));
/** * Read IgfsFileAffinityRange. * * @param reader Reader. * @return File affinity range. */ public static IgfsFileAffinityRange readFileAffinityRange(BinaryRawReader reader) { if (reader.readBoolean()) { IgfsFileAffinityRange affRange = new IgfsFileAffinityRange(); affRange.readRawBinary(reader); return affRange; } else return null; }
if (leftRange.less(blockOff)) return null; if (leftRange.belongs(blockOff)) return leftRange.status() != RANGE_STATUS_MOVED ? leftRange.affinityKey() : includeMoved ? leftRange.affinityKey() : null; if (rightRange.greater(blockOff)) return null; if (rightRange.belongs(blockOff)) return rightRange.status() != RANGE_STATUS_MOVED ? rightRange.affinityKey() : includeMoved ? leftRange.affinityKey() : null; if (midRange.belongs(blockOff)) return midRange.status() != RANGE_STATUS_MOVED ? midRange.affinityKey() : includeMoved ? leftRange.affinityKey() : null; if (midRange.less(blockOff)) rightIdx = midIdx; else { assert midRange.greater(blockOff);
if (last.regionEqual(range)) { assert last.status() == RANGE_STATUS_MOVED; if (first.regionEqual(range)) { assert first.status() == RANGE_STATUS_MOVED; if (midRange.regionEqual(range)) { assert midRange.status() == RANGE_STATUS_MOVED; if (midRange.less(range.startOffset())) lastIdx = midIdx; else { assert midRange.greater(range.startOffset());
/** * Tries to remove blocks affected by fragmentizer. If {@code cleanNonColocated} is {@code true}, will remove * non-colocated blocks as well. * * @param fileInfo File info to clean up. * @param range Range to clean up. * @param cleanNonColocated {@code True} if all blocks should be cleaned. */ public void cleanBlocks(IgfsEntryInfo fileInfo, IgfsFileAffinityRange range, boolean cleanNonColocated) { long startIdx = range.startOffset() / fileInfo.blockSize(); long endIdx = range.endOffset() / fileInfo.blockSize(); if (log.isDebugEnabled()) log.debug("Cleaning blocks [fileInfo=" + fileInfo + ", range=" + range + ", cleanNonColocated=" + cleanNonColocated + ", startIdx=" + startIdx + ", endIdx=" + endIdx + ']'); try { try (IgniteDataStreamer<IgfsBlockKey, byte[]> ldr = dataStreamer()) { for (long idx = startIdx; idx <= endIdx; idx++) { ldr.removeData(new IgfsBlockKey(fileInfo.id(), range.affinityKey(), fileInfo.evictExclude(), idx)); if (cleanNonColocated) ldr.removeData(new IgfsBlockKey(fileInfo.id(), null, fileInfo.evictExclude(), idx)); } } } catch (IgniteException e) { log.error("Failed to clean up file range [fileInfo=" + fileInfo + ", range=" + range + ']', e); } }
if (range.less(pos)) { long partEnd = Math.min(end, range.startOffset()); if (range.belongs(pos)) { long partEnd = Math.min(range.endOffset() + 1, end); range.affinityKey());
/** * Tries to concatenate this range with a given one. If ranges are not adjacent, will return {@code null}. * * @param range Range to concatenate with. * @return Concatenation result or {@code null} if ranges are not adjacent. */ @Nullable public IgfsFileAffinityRange concat(IgfsFileAffinityRange range) { if (endOff + 1 != range.startOff || !F.eq(affKey, range.affKey) || status != RANGE_STATUS_INITIAL) return null; return new IgfsFileAffinityRange(startOff, range.endOff, affKey); }
/** * Adds range to the list of already existing ranges. Added range must be located after * the last range in this map. If added range is adjacent to the last range in the map, * added range will be concatenated to the last one. * * @param range Range to add. */ public void addRange(IgfsFileAffinityRange range) { if (range == null || range.empty()) return; // We cannot add range in the middle of the file. if (ranges == null) { ranges = new ArrayList<>(); ranges.add(range); return; } assert !ranges.isEmpty(); IgfsFileAffinityRange last = ranges.get(ranges.size() - 1); // Ensure that range being added is located to the right of last range in list. assert last.greater(range.startOffset()) : "Cannot add range to middle of map [last=" + last + ", range=" + range + ']'; // Try to concat last and new range. IgfsFileAffinityRange concat = last.concat(range); // Simply add range to the end of the list if they are not adjacent. if (concat == null) ranges.add(range); else ranges.set(ranges.size() - 1, concat); }
/** * Creates block key based on block ID, file info and local affinity range. * * @param block Block ID. * @param fileInfo File info being written. * @param locRange Local affinity range to update. * @return Block key. */ private IgfsBlockKey createBlockKey( long block, IgfsEntryInfo fileInfo, IgfsFileAffinityRange locRange ) { // If affinityKey is present, return block key as is. if (fileInfo.affinityKey() != null) return new IgfsBlockKey(fileInfo.id(), fileInfo.affinityKey(), fileInfo.evictExclude(), block); // If range is done, no colocated writes are attempted. if (locRange == null) return new IgfsBlockKey(fileInfo.id(), null, fileInfo.evictExclude(), block); long blockStart = block * fileInfo.blockSize(); // If block does not belong to new range, return old affinity key. if (locRange.less(blockStart)) { IgniteUuid affKey = fileInfo.fileMap().affinityKey(blockStart, false); return new IgfsBlockKey(fileInfo.id(), affKey, fileInfo.evictExclude(), block); } if (!locRange.belongs(blockStart)) locRange.expand(blockStart, fileInfo.blockSize()); return new IgfsBlockKey(fileInfo.id(), locRange.affinityKey(), fileInfo.evictExclude(), block); }
if (last.startOffset() == range.startOffset()) { updateRangeStatus0(lastIdx, last, range, status); if (first.startOffset() == range.startOffset()) { updateRangeStatus0(firstIdx, first, range, status); if (midRange.startOffset() == range.startOffset()) { updateRangeStatus0(midIdx, midRange, range, status); if (midRange.less(range.startOffset())) lastIdx = midIdx; else { assert midRange.greater(range.startOffset());
IgfsUtils.DELETE_LOCK_ID, false, null, t, t); IgfsFileAffinityRange range = new IgfsFileAffinityRange(); assertTrue(range.regionEqual(new IgfsFileAffinityRange(0, writesCnt * chunkSize - 1, null)));
assertTrue(ranges.get(0).startOffset() == 0); assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1); assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE); assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1);
UUID nodeId = igfsCtx.data().affinityNode(range.affinityKey()).id(); nodeRanges.addAll(range.split(igfsCtx.data().groupBlockSize()));
if (!writer.writeHeader(directType(), fieldsCount())) return false;
IgfsEntryInfo updated; switch (range.status()) { case RANGE_STATUS_INITIAL: {
if (last.regionEqual(range)) { assert last.status() == RANGE_STATUS_MOVED; if (first.regionEqual(range)) { assert first.status() == RANGE_STATUS_MOVED; if (midRange.regionEqual(range)) { assert midRange.status() == RANGE_STATUS_MOVED; if (midRange.less(range.startOffset())) lastIdx = midIdx; else { assert midRange.greater(range.startOffset());
long startIdx = range.startOffset() / fileInfo.blockSize(); long endIdx = range.endOffset() / fileInfo.blockSize(); IgfsBlockKey colocatedKey = new IgfsBlockKey(fileInfo.id(), range.affinityKey(), fileInfo.evictExclude(), idx);
if (range.less(pos)) { long partEnd = Math.min(end, range.startOffset()); if (range.belongs(pos)) { long partEnd = Math.min(range.endOffset() + 1, end); range.affinityKey());