@Override protected int computeSplitsCount() { return getNamespaceService().getSplitCount(splitFilter); }
public static SplitsPointer of(NamespaceService namespaceService, DatasetConfig datasetConfig) { final EntityId datasetId = Preconditions.checkNotNull(datasetConfig.getId()); final ReadDefinition readDefinition = Preconditions.checkNotNull(datasetConfig.getReadDefinition()); final long splitVersion = readDefinition.getSplitVersion(); int splitsCount = namespaceService.getSplitCount(new FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))); return new DatasetSplitsPointer(namespaceService, datasetId, splitVersion, splitsCount); }
@Test public void testKeepCurrentVersion() throws Exception { namespaceService.deleteSplitOrphans(KEEP_CURRENT_VERSION_ONLY); assertThat( namespaceService.getSplitCount(new FindByCondition().setCondition(SearchQueryUtils.newMatchAllQuery())), is(10 + 20 + 100 + 1000)); DatasetConfig dataset1 = namespaceService.getDataset(new NamespaceKey(Arrays.asList("test", "dataset1"))); generateSplits(now, 100).forEach(split -> { DatasetSplitId id = DatasetSplitId.of(dataset1, split, now); assertThat(splitsStore.get(id), is(split)); }); DatasetConfig dataset2 = namespaceService.getDataset(new NamespaceKey(Arrays.asList("test", "dataset1"))); generateSplits(now, 100).forEach(split -> { DatasetSplitId id = DatasetSplitId.of(dataset2, split, now); assertThat(splitsStore.get(id), is(split)); }); }
@Test public void testKeepValidSplits() throws Exception { namespaceService.deleteSplitOrphans(SplitOrphansRetentionPolicy.KEEP_VALID_SPLITS); assertThat( namespaceService.getSplitCount(new FindByCondition().setCondition(SearchQueryUtils.newMatchAllQuery())), is(10 + 20 + 24 * 100 + 1000)); DatasetConfig dataset1 = namespaceService.getDataset(new NamespaceKey(Arrays.asList("test", "dataset1"))); for(int i = 23; i >= 0; i--) { final long splitVersion = now - TimeUnit.HOURS.toMillis(i); generateSplits(splitVersion, 100).forEach(split -> { DatasetSplitId id = DatasetSplitId.of(dataset1, split, splitVersion); assertThat(splitsStore.get(id), is(split)); }); } DatasetConfig dataset2 = namespaceService.getDataset(new NamespaceKey(Arrays.asList("test", "dataset1"))); generateSplits(now, 100).forEach(split -> { DatasetSplitId id = DatasetSplitId.of(dataset2, split, now); assertThat(splitsStore.get(id), is(split)); }); }
ns.addOrUpdateDataset(new NamespaceKey(datasetConfig.getFullPathList()), datasetConfig, splits); assertEquals(10, ns.getSplitCount(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig)))); assertTrue(!NamespaceServiceImpl.compareSplits(datasetConfig, splits, ns.findSplits(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))))); assertEquals(10, ns.getSplitCount(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig)))); assertTrue(!NamespaceServiceImpl.compareSplits(datasetConfig, splits, ns.findSplits(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))))); assertEquals(10, ns.getSplitCount(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig)))); assertTrue(!NamespaceServiceImpl.compareSplits(datasetConfig, splits, ns.findSplits(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))))); assertEquals(9, ns.getSplitCount(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig)))); assertTrue(!NamespaceServiceImpl.compareSplits(datasetConfig, splits, ns.findSplits(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))))); assertEquals(10, ns.getSplitCount(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig)))); assertTrue(!NamespaceServiceImpl.compareSplits(datasetConfig, splits, ns.findSplits(new IndexedStore.FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))))); int count = ns.getSplitCount(new IndexedStore.FindByCondition().setCondition(searchQuery)); int deleted = ns.deleteSplitOrphans(DatasetSplitId.SplitOrphansRetentionPolicy.KEEP_CURRENT_VERSION_ONLY); int newCount = ns.getSplitCount(new IndexedStore.FindByCondition().setCondition(searchQuery));