@Test public void testCustomParts() throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(DefaultStoreEngine.DEFAULT_COMPACTOR_CLASS_KEY, DummyCompactor.class.getName()); conf.set(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, DummyCompactionPolicy.class.getName()); conf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY, DummyStoreFlusher.class.getName()); HStore mockStore = Mockito.mock(HStore.class); StoreEngine<?, ?, ?, ?> se = StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR); Assert.assertTrue(se instanceof DefaultStoreEngine); Assert.assertTrue(se.getCompactionPolicy() instanceof DummyCompactionPolicy); Assert.assertTrue(se.getStoreFlusher() instanceof DummyStoreFlusher); Assert.assertTrue(se.getCompactor() instanceof DummyCompactor); } }
@Override public int getCompactPriority() { int priority = this.storeEngine.getStoreFileManager().getStoreCompactionPriority(); if (priority == PRIORITY_USER) { LOG.warn("Compaction priority is USER despite there being no user compaction"); } return priority; }
private void createComponentsOnce( Configuration conf, HStore store, CellComparator cellComparator) throws IOException { assert compactor == null && compactionPolicy == null && storeFileManager == null && storeFlusher == null; createComponents(conf, store, cellComparator); assert compactor != null && compactionPolicy != null && storeFileManager != null && storeFlusher != null; }
@Override public boolean shouldPerformMajorCompaction() throws IOException { for (HStoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) { // TODO: what are these reader checks all over the place? if (sf.getReader() == null) { LOG.debug("StoreFile {} has null Reader", sf); return false; } } return storeEngine.getCompactionPolicy().shouldPerformMajorCompaction( this.storeEngine.getStoreFileManager().getStorefiles()); }
try { synchronized (filesCompacting) { filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles()); if (!filesCompacting.isEmpty()) { isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount()); filesCompacting.addAll(filesToCompact); Collections.sort(filesCompacting, storeEngine.getStoreFileManager() .getStoreFileComparator()); List<Path> newFiles = ((DefaultCompactor)this.storeEngine.getCompactor()) .compactForTesting(filesToCompact, isMajor); for (Path newFile: newFiles) {
public boolean throttleCompaction(long compactionSize) { return storeEngine.getCompactionPolicy().throttleCompaction(compactionSize); }
this.storeEngine = StoreEngine.create(this, this.conf, this.comparator); this.storeEngine.getStoreFileManager().loadFiles(loadStoreFiles());
/** * getter for CompactionProgress object * @return CompactionProgress object; can be null */ public CompactionProgress getCompactionProgress() { return this.storeEngine.getCompactor().getProgress(); }
/** * Creates the store engine configured for the given Store. * @param store The store. An unfortunate dependency needed due to it * being passed to coprocessors via the compactor. * @param conf Store configuration. * @param kvComparator KVComparator for storeFileManager. * @return StoreEngine to use. */ protected StoreEngine<?, ?, ?, ?> createStoreEngine(HStore store, Configuration conf, CellComparator kvComparator) throws IOException { return StoreEngine.create(store, conf, comparator); }
StoreFlusher flusher = storeEngine.getStoreFlusher(); IOException lastException = null; for (int i = 0; i < flushRetriesNumber; i++) {
@Override public boolean needsCompaction() { List<HStoreFile> filesCompactingClone = null; synchronized (filesCompacting) { filesCompactingClone = Lists.newArrayList(filesCompacting); } return this.storeEngine.needsCompaction(filesCompactingClone); }
/** * Create the StoreEngine configured for the given Store. * @param store The store. An unfortunate dependency needed due to it * being passed to coprocessors via the compactor. * @param conf Store configuration. * @param cellComparator CellComparator for storeFileManager. * @return StoreEngine to use. */ public static StoreEngine<?, ?, ?, ?> create( HStore store, Configuration conf, CellComparator cellComparator) throws IOException { String className = conf.get(STORE_ENGINE_CLASS_KEY, DEFAULT_STORE_ENGINE_CLASS.getName()); try { StoreEngine<?,?,?,?> se = ReflectionUtils.instantiateWithCustomCtor( className, new Class[] { }, new Object[] { }); se.createComponentsOnce(conf, store, cellComparator); return se; } catch (Exception e) { throw new IOException("Unable to load configured store engine '" + className + "'", e); } } }
final CompactionContext compaction = storeEngine.createCompaction(); CompactionRequestImpl request = null; this.lock.readLock().lock();
void compactEquals(List<HStoreFile> candidates, boolean forcemajor, boolean isOffPeak, long... expected) throws IOException { store.forceMajor = forcemajor; // Test Default compactions CompactionRequestImpl result = ((RatioBasedCompactionPolicy) store.storeEngine.getCompactionPolicy()).selectCompaction( candidates, new ArrayList<>(), false, isOffPeak, forcemajor); List<HStoreFile> actual = new ArrayList<>(result.getFiles()); if (isOffPeak && !forcemajor) { Assert.assertTrue(result.isOffPeak()); } Assert.assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual))); store.forceMajor = false; } }
@Override public boolean isMajorCompaction() throws IOException { for (StoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) { // TODO: what are these reader checks all over the place? if (sf.getReader() == null) { LOG.debug("StoreFile " + sf + " has null Reader"); return false; } } return storeEngine.getCompactionPolicy().isMajorCompaction( this.storeEngine.getStoreFileManager().getStorefiles()); }
try { synchronized (filesCompacting) { filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles()); if (!filesCompacting.isEmpty()) { isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount()); filesCompacting.addAll(filesToCompact); Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID); List<Path> newFiles = ((DefaultCompactor)this.storeEngine.getCompactor()) .compactForTesting(filesToCompact, isMajor); for (Path newFile: newFiles) {
@Test public void testStoreUsesSearchEngineOverride() throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName()); init(this.name.getMethodName(), conf); assertEquals(DummyStoreEngine.lastCreatedCompactor, this.store.storeEngine.getCompactor()); }
private static TestStoreEngine createEngine(Configuration conf) throws Exception { HStore store = mock(HStore.class); CellComparatorImpl kvComparator = mock(CellComparatorImpl.class); return (TestStoreEngine)StoreEngine.create(store, conf, kvComparator); }
StoreFlusher flusher = storeEngine.getStoreFlusher(); IOException lastException = null; for (int i = 0; i < flushRetriesNumber; i++) {
@Override public boolean needsCompaction() { return this.storeEngine.needsCompaction(this.filesCompacting); }