.thenReturn(Collections.singletonList(getTestIndexMaintainer())); indexBuilder = new NonTxIndexBuilder(); indexBuilder.setup(env);
addCleanupForCurrentBatch(updateMap, batchTs, state, indexMetaData); addUpdateForGivenTimestamp(batchTs, state, updateMap, indexMetaData);
private long addUpdateForGivenTimestamp(long ts, LocalTableState state, IndexUpdateManager updateMap, IndexMetaData indexMetaData) throws IOException { state.setCurrentTimestamp(ts); ts = addCurrentStateMutationsForBatch(updateMap, state, indexMetaData); return ts; }
@Override public void setup(RegionCoprocessorEnvironment env) throws IOException { super.setup(env); this.indexMetaDataBuilder = new PhoenixIndexMetaDataBuilder(env); }
@Override public Collection<Pair<Mutation, byte[]>> getIndexUpdate(Mutation mutation, IndexMetaData indexMetaData) throws IOException { // create a state manager, so we can manage each batch LocalTableState state = new LocalTableState(localTable, mutation); // build the index updates for each group IndexUpdateManager manager = new IndexUpdateManager(indexMetaData); batchMutationAndAddUpdates(manager, state, mutation, indexMetaData); if (LOG.isTraceEnabled()) { LOG.trace("Found index updates for Mutation: " + mutation + "\n" + manager); } return manager.toMap(); }
private void addCleanupForCurrentBatch(IndexUpdateManager updateMap, long batchTs, LocalTableState state, IndexMetaData indexMetaData) throws IOException { // get the cleanup for the current state state.setCurrentTimestamp(batchTs); addDeleteUpdatesToMap(updateMap, state, batchTs, indexMetaData); // ignore any index tracking from the delete state.resetTrackedColumns(); }
/** * Split the mutation into batches based on the timestamps of each keyvalue. We need to check each key-value in the * update to see if it matches the others. Generally, this will be the case, but you can add kvs to a mutation that * don't all have the timestamp, so we need to manage everything in batches based on timestamp. * <p> * Adds all the updates in the {@link Mutation} to the state, as a side-effect. * @param state * current state of the row for the mutation. * @param m * mutation to batch * @param indexMetaData TODO * @param updateMap * index updates into which to add new updates. Modified as a side-effect. * * @throws IOException */ private void batchMutationAndAddUpdates(IndexUpdateManager manager, LocalTableState state, Mutation m, IndexMetaData indexMetaData) throws IOException { // The cells of a mutation are broken up into time stamp batches prior to this call (in Indexer). long ts = m.getFamilyCellMap().values().iterator().next().iterator().next().getTimestamp(); Batch batch = new Batch(ts); for (List<Cell> family : m.getFamilyCellMap().values()) { for (Cell kv : family) { batch.add(kv); if(ts != kv.getTimestamp()) { throw new IllegalStateException("Time stamps must match for all cells in a batch"); } } } addMutationsForBatch(manager, batch, state, indexMetaData); }
/** * Tests getting an index update for a mutation with 200 versions Before, the issue PHOENIX-3807 * was causing this test to take >90 seconds, so here we set a timeout of 5 seconds */ @Test(timeout = 10000) public void testManyVersions() throws IOException { // when doing a rebuild, we are replaying mutations so we want to ignore newer mutations // see LocalTable#getCurrentRowState() Mockito.when(mockIndexMetaData.getReplayWrite()).thenReturn(ReplayWrite.INDEX_ONLY); MultiMutation mutation = getMultipleVersionMutation(200); currentRowCells = mutation.getFamilyCellMap().get(FAM); Collection<Pair<Mutation, byte[]>> indexUpdates = Lists.newArrayList(); for (Mutation m : IndexManagementUtil.flattenMutationsByTimestamp(Collections.singletonList(mutation))) { indexUpdates.addAll(indexBuilder.getIndexUpdate(m, mockIndexMetaData)); } assertNotEquals(0, indexUpdates.size()); }
@Override public void setup(RegionCoprocessorEnvironment env) throws IOException { super.setup(env); this.indexMetaDataBuilder = new PhoenixIndexMetaDataBuilder(env); }
@Override public Collection<Pair<Mutation, byte[]>> getIndexUpdate(Mutation mutation, IndexMetaData indexMetaData) throws IOException { // create a state manager, so we can manage each batch LocalTableState state = new LocalTableState(localTable, mutation); // build the index updates for each group IndexUpdateManager manager = new IndexUpdateManager(indexMetaData); batchMutationAndAddUpdates(manager, state, mutation, indexMetaData); if (LOG.isTraceEnabled()) { LOG.trace("Found index updates for Mutation: " + mutation + "\n" + manager); } return manager.toMap(); }
private void addCleanupForCurrentBatch(IndexUpdateManager updateMap, long batchTs, LocalTableState state, IndexMetaData indexMetaData) throws IOException { // get the cleanup for the current state state.setCurrentTimestamp(batchTs); addDeleteUpdatesToMap(updateMap, state, batchTs, indexMetaData); // ignore any index tracking from the delete state.resetTrackedColumns(); }
/** * Split the mutation into batches based on the timestamps of each keyvalue. We need to check each key-value in the * update to see if it matches the others. Generally, this will be the case, but you can add kvs to a mutation that * don't all have the timestamp, so we need to manage everything in batches based on timestamp. * <p> * Adds all the updates in the {@link Mutation} to the state, as a side-effect. * @param state * current state of the row for the mutation. * @param m * mutation to batch * @param indexMetaData TODO * @param updateMap * index updates into which to add new updates. Modified as a side-effect. * * @throws IOException */ private void batchMutationAndAddUpdates(IndexUpdateManager manager, LocalTableState state, Mutation m, IndexMetaData indexMetaData) throws IOException { // The cells of a mutation are broken up into time stamp batches prior to this call (in Indexer). long ts = m.getFamilyCellMap().values().iterator().next().iterator().next().getTimestamp(); Batch batch = new Batch(ts); for (List<Cell> family : m.getFamilyCellMap().values()) { for (Cell kv : family) { batch.add(kv); if(ts != kv.getTimestamp()) { throw new IllegalStateException("Time stamps must match for all cells in a batch"); } } } addMutationsForBatch(manager, batch, state, indexMetaData); }
/** * Tests that updating an indexed column results in a DeleteFamily (prior index cell) and a Put * (new index cell) */ @Test public void testGetMutableIndexUpdate() throws IOException { setCurrentRowState(FAM, INDEXED_QUALIFIER, 1, VALUE_1); // update ts and value Put put = new Put(ROW); put.addImmutable(FAM, INDEXED_QUALIFIER, 2, VALUE_2); MultiMutation mutation = new MultiMutation(new ImmutableBytesPtr(ROW)); mutation.addAll(put); Collection<Pair<Mutation, byte[]>> indexUpdates = indexBuilder.getIndexUpdate(mutation, mockIndexMetaData); assertEquals(2, indexUpdates.size()); assertContains(indexUpdates, 2, ROW, KeyValue.Type.DeleteFamily, FAM, new byte[0] /* qual not needed */, 2); assertContains(indexUpdates, ColumnTracker.NO_NEWER_PRIMARY_TABLE_ENTRY_TIMESTAMP, ROW, KeyValue.Type.Put, FAM, QueryConstants.EMPTY_COLUMN_BYTES, 2); }
addCleanupForCurrentBatch(updateMap, batchTs, state, indexMetaData); addUpdateForGivenTimestamp(batchTs, state, updateMap, indexMetaData);
@Override public void setup(RegionCoprocessorEnvironment env) throws IOException { super.setup(env); this.indexMetaDataBuilder = new PhoenixIndexMetaDataBuilder(env); }
@Override public Collection<Pair<Mutation, byte[]>> getIndexUpdate(Mutation mutation, IndexMetaData indexMetaData) throws IOException { // create a state manager, so we can manage each batch LocalTableState state = new LocalTableState(localTable, mutation); // build the index updates for each group IndexUpdateManager manager = new IndexUpdateManager(indexMetaData); batchMutationAndAddUpdates(manager, state, mutation, indexMetaData); if (LOG.isTraceEnabled()) { LOG.trace("Found index updates for Mutation: " + mutation + "\n" + manager); } return manager.toMap(); }
private long addUpdateForGivenTimestamp(long ts, LocalTableState state, IndexUpdateManager updateMap, IndexMetaData indexMetaData) throws IOException { state.setCurrentTimestamp(ts); ts = addCurrentStateMutationsForBatch(updateMap, state, indexMetaData); return ts; }
private void addCleanupForCurrentBatch(IndexUpdateManager updateMap, long batchTs, LocalTableState state, IndexMetaData indexMetaData) throws IOException { // get the cleanup for the current state state.setCurrentTimestamp(batchTs); addDeleteUpdatesToMap(updateMap, state, batchTs, indexMetaData); // ignore any index tracking from the delete state.resetTrackedColumns(); }
/** * Split the mutation into batches based on the timestamps of each keyvalue. We need to check each key-value in the * update to see if it matches the others. Generally, this will be the case, but you can add kvs to a mutation that * don't all have the timestamp, so we need to manage everything in batches based on timestamp. * <p> * Adds all the updates in the {@link Mutation} to the state, as a side-effect. * @param state * current state of the row for the mutation. * @param m * mutation to batch * @param indexMetaData TODO * @param updateMap * index updates into which to add new updates. Modified as a side-effect. * * @throws IOException */ private void batchMutationAndAddUpdates(IndexUpdateManager manager, LocalTableState state, Mutation m, IndexMetaData indexMetaData) throws IOException { // The cells of a mutation are broken up into time stamp batches prior to this call (in Indexer). long ts = m.getFamilyCellMap().values().iterator().next().iterator().next().getTimestamp(); Batch batch = new Batch(ts); for (List<Cell> family : m.getFamilyCellMap().values()) { for (Cell kv : family) { batch.add(kv); if(ts != kv.getTimestamp()) { throw new IllegalStateException("Time stamps must match for all cells in a batch"); } } } addMutationsForBatch(manager, batch, state, indexMetaData); }
indexUpdates.addAll(indexBuilder.getIndexUpdate(m, mockIndexMetaData));