public static void persist(ILSMComponentId id, IComponentMetadata metadata) throws HyracksDataException { LSMComponentId componentId = (LSMComponentId) id; metadata.put(COMPONENT_ID_MIN_KEY, LongPointable.FACTORY.createPointable(componentId.getMinId())); metadata.put(COMPONENT_ID_MAX_KEY, LongPointable.FACTORY.createPointable(componentId.getMaxId())); }
public static ILSMComponentId union(ILSMComponentId id1, ILSMComponentId id2) { long minId = Long.min(((LSMComponentId) id1).getMinId(), ((LSMComponentId) id2).getMinId()); long maxId = Long.max(((LSMComponentId) id1).getMaxId(), ((LSMComponentId) id2).getMaxId()); return new LSMComponentId(minId, maxId); }
@Override public IdCompareResult compareTo(ILSMComponentId id) { if (this.missing() || id == null || id.missing()) { return IdCompareResult.UNKNOWN; } LSMComponentId componentId = (LSMComponentId) id; if (this.getMinId() > componentId.getMaxId()) { return IdCompareResult.GREATER_THAN; } else if (this.getMaxId() < componentId.getMinId()) { return IdCompareResult.LESS_THAN; } else if (this.getMinId() <= componentId.getMinId() && this.getMaxId() >= componentId.getMaxId()) { return IdCompareResult.INCLUDE; } else { return IdCompareResult.INTERSECT; } }
private void addComponentToCheckpoint(ILSMIOOperation operation) throws HyracksDataException { // will always update the checkpoint file even if no new component was created FileReference target = operation.getTarget(); Map<String, Object> map = operation.getParameters(); final Long lsn = operation.getIOOpertionType() == LSMIOOperationType.FLUSH ? (Long) map.get(KEY_FLUSH_LOG_LSN) : 0L; final LSMComponentId id = (LSMComponentId) map.get(KEY_FLUSHED_COMPONENT_ID); final ResourceReference ref = ResourceReference.of(target.getAbsolutePath()); final long componentSequence = IndexComponentFileReference.of(ref.getName()).getSequenceEnd(); indexCheckpointManagerProvider.get(ref).flushed(componentSequence, lsn, id.getMaxId()); }
private long getReplicatedComponentId() throws HyracksDataException { final ILSMIndexReplicationJob indexReplJob = (ILSMIndexReplicationJob) job; if (indexReplJob.getLSMOpType() != LSMOperationType.FLUSH) { return -1L; } final ILSMIndexOperationContext ctx = indexReplJob.getLSMIndexOperationContext(); LSMComponentId id = (LSMComponentId) ctx.getComponentsToBeReplicated().get(0).getId(); return id.getMinId(); } }
@Override public synchronized void refresh() { final long nextId = ++lastUsedId; componentId = new LSMComponentId(nextId, nextId); currentComponentIndex = (currentComponentIndex + 1) % numComponents; }
private void initIndexCheckpoint(INcApplicationContext appCtx) throws HyracksDataException { final ResourceReference indexRef = ResourceReference.of(file); final IIndexCheckpointManagerProvider checkpointManagerProvider = appCtx.getIndexCheckpointManagerProvider(); final IIndexCheckpointManager indexCheckpointManager = checkpointManagerProvider.get(indexRef); final long currentLSN = appCtx.getTransactionSubsystem().getLogManager().getAppendLSN(); indexCheckpointManager.delete(); indexCheckpointManager.init(Long.MIN_VALUE, currentLSN, LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID.getMaxId()); LOGGER.info(() -> "Checkpoint index: " + indexRef); }
long min = id.getMinId();
public static ILSMComponentId readFrom(IComponentMetadata metadata, ArrayBackedValueStorage buffer) throws HyracksDataException { long minId = ComponentUtils.getLong(metadata, COMPONENT_ID_MIN_KEY, LSMComponentId.NOT_FOUND, buffer); long maxId = ComponentUtils.getLong(metadata, COMPONENT_ID_MAX_KEY, LSMComponentId.NOT_FOUND, buffer); if (minId == LSMComponentId.NOT_FOUND || maxId == LSMComponentId.NOT_FOUND) { LOGGER.warn("Invalid component id {} was persisted to a component metadata", LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID); return LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID; } else { return new LSMComponentId(minId, maxId); } }
Files.write(path, bytes); indexCheckpointManagerProvider.get(DatasetResourceReference.of(resource)).init(Long.MIN_VALUE, 0, LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID.getMaxId()); deleteResourceFileMask(resourceFile); } catch (Exception e) {
long min = id.getMinId();
@Test public void testSecondaryMissing() { try { List<ILSMComponentId> primaryComponentIDs = Arrays.asList(new LSMComponentId(40, 50), new LSMComponentId(30, 35), new LSMComponentId(25, 29), new LSMComponentId(20, 24), new LSMComponentId(10, 19)); List<ILSMComponentId> resultPrimaryIDs = new ArrayList<>(); IndexInfo primary = mockIndex(true, primaryComponentIDs, resultPrimaryIDs, 0); List<ILSMComponentId> secondaryComponentIDs = Arrays.asList(new LSMComponentId(30, 35), new LSMComponentId(25, 29), new LSMComponentId(20, 24)); List<ILSMComponentId> resultSecondaryIDs = new ArrayList<>(); IndexInfo secondary = mockIndex(false, secondaryComponentIDs, resultSecondaryIDs, 0); ILSMMergePolicy policy = mockMergePolicy(primary, secondary); policy.diskComponentAdded(secondary.getIndex(), false); Assert.assertTrue(resultPrimaryIDs.isEmpty()); Assert.assertTrue(resultSecondaryIDs.isEmpty()); policy.diskComponentAdded(primary.getIndex(), false); Assert.assertEquals(Arrays.asList(new LSMComponentId(30, 35), new LSMComponentId(25, 29), new LSMComponentId(20, 24), new LSMComponentId(10, 19)), resultPrimaryIDs); Assert.assertEquals( Arrays.asList(new LSMComponentId(30, 35), new LSMComponentId(25, 29), new LSMComponentId(20, 24)), resultSecondaryIDs); } catch (HyracksDataException e) { Assert.fail(e.getMessage()); } }
private void deleteComponentsFromCheckpoint(ILSMIOOperation operation) throws HyracksDataException { // component was deleted... if a flush, do nothing.. if a merge, must update the checkpoint file if (operation.getIOOpertionType() == LSMIOOperationType.MERGE) { // Get component id of the last disk component LSMComponentId mostRecentComponentId = getMostRecentComponentId(operation.getAccessor().getOpContext().getComponentsToBeMerged()); // Update the checkpoint file FileReference target = operation.getTarget(); final ResourceReference ref = ResourceReference.of(target.getAbsolutePath()); indexCheckpointManagerProvider.get(ref).setLastComponentId(mostRecentComponentId.getMaxId()); } else if (operation.getIOOpertionType() != LSMIOOperationType.FLUSH) { throw new IllegalStateException("Unexpected IO operation: " + operation.getIOOpertionType()); } }
long min = id.getMinId();
@Test public void testBasic() { try { List<ILSMComponentId> componentIDs = Arrays.asList(new LSMComponentId(5, 5), new LSMComponentId(4, 4), new LSMComponentId(3, 3), new LSMComponentId(2, 2), new LSMComponentId(1, 1)); List<ILSMComponentId> resultPrimaryIDs = new ArrayList<>(); IndexInfo primary = mockIndex(true, componentIDs, resultPrimaryIDs, 0); List<ILSMComponentId> resultSecondaryIDs = new ArrayList<>(); IndexInfo secondary = mockIndex(false, componentIDs, resultSecondaryIDs, 0); ILSMMergePolicy policy = mockMergePolicy(primary, secondary); policy.diskComponentAdded(secondary.getIndex(), false); Assert.assertTrue(resultPrimaryIDs.isEmpty()); Assert.assertTrue(resultSecondaryIDs.isEmpty()); policy.diskComponentAdded(primary.getIndex(), false); Assert.assertEquals(Arrays.asList(new LSMComponentId(4, 4), new LSMComponentId(3, 3), new LSMComponentId(2, 2), new LSMComponentId(1, 1)), resultPrimaryIDs); Assert.assertEquals(Arrays.asList(new LSMComponentId(4, 4), new LSMComponentId(3, 3), new LSMComponentId(2, 2), new LSMComponentId(1, 1)), resultSecondaryIDs); } catch (HyracksDataException e) { Assert.fail(e.getMessage()); } }
@Test public void testIDIntervals() { try { List<ILSMComponentId> componentIDs = Arrays.asList(new LSMComponentId(40, 50), new LSMComponentId(30, 35), new LSMComponentId(25, 29), new LSMComponentId(20, 24), new LSMComponentId(10, 19)); List<ILSMComponentId> resultPrimaryIDs = new ArrayList<>(); IndexInfo primary = mockIndex(true, componentIDs, resultPrimaryIDs, 0); List<ILSMComponentId> resultSecondaryIDs = new ArrayList<>(); IndexInfo secondary = mockIndex(false, componentIDs, resultSecondaryIDs, 0); ILSMMergePolicy policy = mockMergePolicy(primary, secondary); policy.diskComponentAdded(secondary.getIndex(), false); Assert.assertTrue(resultPrimaryIDs.isEmpty()); Assert.assertTrue(resultSecondaryIDs.isEmpty()); policy.diskComponentAdded(primary.getIndex(), false); Assert.assertEquals(Arrays.asList(new LSMComponentId(30, 35), new LSMComponentId(25, 29), new LSMComponentId(20, 24), new LSMComponentId(10, 19)), resultPrimaryIDs); Assert.assertEquals(Arrays.asList(new LSMComponentId(30, 35), new LSMComponentId(25, 29), new LSMComponentId(20, 24), new LSMComponentId(10, 19)), resultSecondaryIDs); } catch (HyracksDataException e) { Assert.fail(e.getMessage()); } }
@Test public void testMultiPartition() { try { List<ILSMComponentId> componentIDs = Arrays.asList(new LSMComponentId(40, 50), new LSMComponentId(30, 35), new LSMComponentId(25, 29), new LSMComponentId(20, 24), new LSMComponentId(10, 19)); List<ILSMComponentId> resultPrimaryIDs = new ArrayList<>(); IndexInfo primary = mockIndex(true, componentIDs, resultPrimaryIDs, 0); List<ILSMComponentId> resultSecondaryIDs = new ArrayList<>(); IndexInfo secondary = mockIndex(false, componentIDs, resultSecondaryIDs, 0); List<ILSMComponentId> resultSecondaryIDs1 = new ArrayList<>(); IndexInfo secondary1 = mockIndex(false, componentIDs, resultSecondaryIDs, 1); ILSMMergePolicy policy = mockMergePolicy(primary, secondary, secondary1); policy.diskComponentAdded(secondary.getIndex(), false); Assert.assertTrue(resultPrimaryIDs.isEmpty()); Assert.assertTrue(resultSecondaryIDs.isEmpty()); policy.diskComponentAdded(primary.getIndex(), false); Assert.assertEquals(Arrays.asList(new LSMComponentId(30, 35), new LSMComponentId(25, 29), new LSMComponentId(20, 24), new LSMComponentId(10, 19)), resultPrimaryIDs); Assert.assertEquals(Arrays.asList(new LSMComponentId(30, 35), new LSMComponentId(25, 29), new LSMComponentId(20, 24), new LSMComponentId(10, 19)), resultSecondaryIDs); Assert.assertTrue(resultSecondaryIDs1.isEmpty()); } catch (HyracksDataException e) { Assert.fail(e.getMessage()); } }