try { bs = ReplicationTable.getBatchScanner(client, 4); bw = ReplicationTable.getBatchWriter(client);
try { bs = ReplicationTable.getBatchScanner(client, 4); replBw = ReplicationTable.getBatchWriter(client); } catch (ReplicationTableOfflineException e) { log.debug("Table is no longer online, will retry");
s = ReplicationTable.getScanner(client); if (writer == null) { setBatchWriter(ReplicationTable.getBatchWriter(client));
replicationWriter = ReplicationTable.getBatchWriter(client); } catch (ReplicationTableOfflineException | AccumuloSecurityException | AccumuloException e) {
m.put(StatusSection.NAME, new Text(tableId), StatusUtil.fileCreatedValue(System.currentTimeMillis())); BatchWriter bw = ReplicationTable.getBatchWriter(conn); bw.addMutation(m); bw.flush();
try { bs = ReplicationTable.getBatchScanner(conn, 4); bw = ReplicationTable.getBatchWriter(conn);
@Test public void doNotCreateWorkForFilesNotNeedingIt() throws Exception { ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1", "1"), target2 = new ReplicationTarget("cluster1", "table2", "2"); Text serializedTarget1 = target1.toText(), serializedTarget2 = target2.toText(); // Create two mutations, both of which need replication work done BatchWriter bw = ReplicationTable.getBatchWriter(conn); String filename1 = UUID.randomUUID().toString(), filename2 = UUID.randomUUID().toString(); String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2; Mutation m = new Mutation(file1); WorkSection.add(m, serializedTarget1, StatusUtil.fileCreatedValue(5)); bw.addMutation(m); m = new Mutation(file2); WorkSection.add(m, serializedTarget2, StatusUtil.fileCreatedValue(10)); bw.addMutation(m); bw.close(); DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class); HashSet<String> queuedWork = new HashSet<>(); assigner.setQueuedWork(queuedWork); assigner.setMaxQueueSize(Integer.MAX_VALUE); replay(workQueue); assigner.createWork(); verify(workQueue); }
@Test public void notYetReplicationRecordsIgnored() throws Exception { BatchWriter bw = ReplicationTable.getBatchWriter(conn); int numRecords = 3; for (int i = 0; i < numRecords; i++) { String file = "/accumulo/wal/tserver+port/" + UUID.randomUUID(); Mutation m = new Mutation(file); StatusSection.add(m, Integer.toString(i), StatusUtil.openWithUnknownLengthValue()); bw.addMutation(m); } bw.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); BatchScanner bs = ReplicationTable.getBatchScanner(conn, 1); bs.setRanges(Collections.singleton(new Range())); IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class); bs.addScanIterator(cfg); bw = EasyMock.createMock(BatchWriter.class); EasyMock.replay(bw); rcrr.removeCompleteRecords(conn, bs, bw); bs.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); }
BatchWriter bw = ReplicationTable.getBatchWriter(conn); String file1 = "/accumulo/wal/tserver+port/wal1"; Mutation m = new Mutation(file1);
@Test public void partiallyReplicatedReferencedWalsAreNotClosed() throws Exception { String file = "file:/accumulo/wal/tserver+port/12345"; Set<String> wals = Collections.singleton(file); BatchWriter bw = ReplicationTable.getBatchWriter(conn); Mutation m = new Mutation(file); StatusSection.add(m, "1", ProtobufUtil.toValue(StatusUtil.ingestedUntil(1000))); bw.addMutation(m); bw.close(); refs.updateReplicationEntries(conn, wals); Scanner s = ReplicationTable.getScanner(conn); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Status status = Status.parseFrom(entry.getValue().get()); Assert.assertFalse(status.getClosed()); }
Mutation m = new Mutation(new Path(file).toString()); m.put(StatusSection.NAME, new Text(tableId), StatusUtil.fileCreatedValue(timeCreated)); BatchWriter bw = ReplicationTable.getBatchWriter(conn); bw.addMutation(m); bw.flush();
@Test public void dontCreateWorkForEntriesWithNothingToReplicate() throws Exception { String table = testName.getMethodName(); conn.tableOperations().create(table); String tableId = conn.tableOperations().tableIdMap().get(table); String file = "hdfs://localhost:8020/accumulo/wal/123456-1234-1234-12345678"; Mutation m = new Mutation(new Path(file).toString()); m.put(StatusSection.NAME, new Text(tableId), StatusUtil.fileCreatedValue(System.currentTimeMillis())); BatchWriter bw = ReplicationTable.getBatchWriter(conn); bw.addMutation(m); bw.flush(); // Assert that we have one record in the status section Scanner s = ReplicationTable.getScanner(conn); StatusSection.limit(s); Assert.assertEquals(1, Iterables.size(s)); MockWorkMaker workMaker = new MockWorkMaker(conn); conn.tableOperations().setProperty(ReplicationTable.NAME, Property.TABLE_REPLICATION_TARGET.getKey() + "remote_cluster_1", "4"); workMaker.setBatchWriter(bw); // If we don't shortcircuit out, we should get an exception because // ServerConfiguration.getTableConfiguration // won't work with MockAccumulo workMaker.run(); s = ReplicationTable.getScanner(conn); WorkSection.limit(s); Assert.assertEquals(0, Iterables.size(s)); }
@Test public void partiallyReplicatedRecordsIgnored() throws Exception { BatchWriter bw = ReplicationTable.getBatchWriter(conn); int numRecords = 3; Status.Builder builder = Status.newBuilder(); builder.setClosed(false); builder.setEnd(10000); builder.setInfiniteEnd(false); for (int i = 0; i < numRecords; i++) { String file = "/accumulo/wal/tserver+port/" + UUID.randomUUID(); Mutation m = new Mutation(file); StatusSection.add(m, Integer.toString(i), ProtobufUtil.toValue(builder.setBegin(1000 * (i + 1)).build())); bw.addMutation(m); } bw.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); BatchScanner bs = ReplicationTable.getBatchScanner(conn, 1); bs.setRanges(Collections.singleton(new Range())); IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class); bs.addScanIterator(cfg); bw = EasyMock.createMock(BatchWriter.class); EasyMock.replay(bw); // We don't remove any records, so we can just pass in a fake BW for both rcrr.removeCompleteRecords(conn, bs, bw); bs.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); }
conn.securityOperations().grantTablePermission(user.getPrincipal(), ReplicationTable.NAME, TablePermission.WRITE); BatchWriter bw = ReplicationTable.getBatchWriter(conn); long createTime = System.currentTimeMillis(); try { Assert.assertEquals(StatusUtil.fileCreatedValue(createTime), entry.getValue()); bw = ReplicationTable.getBatchWriter(conn); try { Mutation m = new Mutation(
@Test public void recordsWithProgressUpdateBothTables() throws Exception { conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.READ); conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.WRITE); ReplicationTable.setOnline(conn); String file = "/accumulo/wals/tserver+port/" + UUID.randomUUID(); Status stat = Status.newBuilder().setBegin(100).setEnd(200).setClosed(true) .setInfiniteEnd(false).build(); ReplicationTarget target = new ReplicationTarget("peer", "table1", "1"); // Create a single work record for a file to some peer BatchWriter bw = ReplicationTable.getBatchWriter(conn); Mutation m = new Mutation(file); WorkSection.add(m, target.toText(), ProtobufUtil.toValue(stat)); bw.addMutation(m); bw.close(); updater.run(); Scanner s = ReplicationTable.getScanner(conn); s.setRange(Range.exact(file)); StatusSection.limit(s); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Assert.assertEquals(entry.getKey().getColumnFamily(), StatusSection.NAME); Assert.assertEquals(entry.getKey().getColumnQualifier().toString(), target.getSourceTableId()); // We should only rely on the correct begin attribute being returned Status actual = Status.parseFrom(entry.getValue().get()); Assert.assertEquals(stat.getBegin(), actual.getBegin()); }
BatchWriter bw = ReplicationTable.getBatchWriter(conn);
@Test public void replicatedClosedWorkRecordsAreNotRemovedWithoutClosedStatusRecords() throws Exception { BatchWriter replBw = ReplicationTable.getBatchWriter(conn); int numRecords = 3;
@Test public void partiallyReplicatedEntriesPrecludeRowDeletion() throws Exception { BatchWriter replBw = ReplicationTable.getBatchWriter(conn); int numRecords = 3;
BatchWriter bw = ReplicationTable.getBatchWriter(conn); Mutation m = new Mutation(file); WorkSection.add(m, target1.toText(), ProtobufUtil.toValue(stat1));
BatchWriter bw = ReplicationTable.getBatchWriter(conn); Mutation m = new Mutation(file); WorkSection.add(m, target1.toText(), ProtobufUtil.toValue(stat1));