@Override public Iterator<Entry<String,Status>> getReplicationNeededIterator() { AccumuloClient client = getClient(); try { Scanner s = ReplicationTable.getScanner(client); StatusSection.limit(s); return Iterators.transform(s.iterator(), input -> { String file = input.getKey().getRow().toString(); Status stat; try { stat = Status.parseFrom(input.getValue().get()); } catch (InvalidProtocolBufferException e) { log.warn("Could not deserialize protobuf for: {}", input.getKey()); stat = null; } return Maps.immutableEntry(file, stat); }); } catch (ReplicationTableOfflineException e) { // No elements that we need to preclude return Collections.emptyIterator(); } } }
public void run() { if (!ReplicationTable.isOnline(client)) { log.debug("Replication table is not yet online"); return; final Scanner s; try { s = ReplicationTable.getScanner(client); if (writer == null) { setBatchWriter(ReplicationTable.getBatchWriter(client));
log.debug("Looking for finished replication work"); if (!ReplicationTable.isOnline(client)) { log.debug("Replication table is not yet online, will retry"); return; BatchWriter replBw; try { bs = ReplicationTable.getBatchScanner(client, 4); replBw = ReplicationTable.getBatchWriter(client); } catch (ReplicationTableOfflineException e) { log.debug("Table is no longer online, will retry");
BatchWriter bw; try { bs = ReplicationTable.getBatchScanner(client, 4); bw = ReplicationTable.getBatchWriter(client);
while (!ReplicationTable.isOnline(conn)) { sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS); Assert.assertTrue("Replication table did not exist", ReplicationTable.isOnline(conn)); int attempts = 5; while (replRows.isEmpty() && attempts > 0) { scanner = ReplicationTable.getScanner(conn); StatusSection.limit(scanner); for (Entry<Key,Value> entry : scanner) {
@Test public void notYetReplicationRecordsIgnored() throws Exception { BatchWriter bw = ReplicationTable.getBatchWriter(conn); int numRecords = 3; for (int i = 0; i < numRecords; i++) { String file = "/accumulo/wal/tserver+port/" + UUID.randomUUID(); Mutation m = new Mutation(file); StatusSection.add(m, Integer.toString(i), StatusUtil.openWithUnknownLengthValue()); bw.addMutation(m); } bw.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); BatchScanner bs = ReplicationTable.getBatchScanner(conn, 1); bs.setRanges(Collections.singleton(new Range())); IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class); bs.addScanIterator(cfg); bw = EasyMock.createMock(BatchWriter.class); EasyMock.replay(bw); rcrr.removeCompleteRecords(conn, bs, bw); bs.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); }
ClusterUser user = getAdminUser(); ReplicationTable.setOnline(conn); conn.securityOperations().grantTablePermission(user.getPrincipal(), ReplicationTable.NAME, TablePermission.WRITE); BatchWriter bw = ReplicationTable.getBatchWriter(conn); long createTime = System.currentTimeMillis(); try { Scanner s = ReplicationTable.getScanner(conn); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Assert.assertEquals(StatusUtil.fileCreatedValue(createTime), entry.getValue()); bw = ReplicationTable.getBatchWriter(conn); try { Mutation m = new Mutation( s = ReplicationTable.getScanner(conn); entry = Iterables.getOnlyElement(s); Status stat = Status.parseFrom(entry.getValue().get());
final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD)); ReplicationTable.setOnline(connMaster); Iterators.size(ReplicationTable.getScanner(connMaster).iterator()); log.info("TabletServer is online"); while (!ReplicationTable.isOnline(connMaster)) { log.info("Replication table still offline, waiting"); Thread.sleep(5000); for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) { log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get()))); for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) { log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
@Test public void partiallyReplicatedReferencedWalsAreNotClosed() throws Exception { String file = "file:/accumulo/wal/tserver+port/12345"; Set<String> wals = Collections.singleton(file); BatchWriter bw = ReplicationTable.getBatchWriter(conn); Mutation m = new Mutation(file); StatusSection.add(m, "1", ProtobufUtil.toValue(StatusUtil.ingestedUntil(1000))); bw.addMutation(m); bw.close(); refs.updateReplicationEntries(conn, wals); Scanner s = ReplicationTable.getScanner(conn); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Status status = Status.parseFrom(entry.getValue().get()); Assert.assertFalse(status.getClosed()); }
final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD)); ReplicationTable.setOnline(connMaster); Iterators.size(ReplicationTable.getScanner(connMaster).iterator()); log.info("TabletServer is online"); for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) { log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get()))); for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) { log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
if (!ReplicationTable.isOnline(context)) { log.debug("Replication table isn't online, not attempting to clean up wals"); return;
@Test public void doNotCreateWorkForFilesNotNeedingIt() throws Exception { ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1", "1"), target2 = new ReplicationTarget("cluster1", "table2", "2"); Text serializedTarget1 = target1.toText(), serializedTarget2 = target2.toText(); // Create two mutations, both of which need replication work done BatchWriter bw = ReplicationTable.getBatchWriter(conn); String filename1 = UUID.randomUUID().toString(), filename2 = UUID.randomUUID().toString(); String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2; Mutation m = new Mutation(file1); WorkSection.add(m, serializedTarget1, StatusUtil.fileCreatedValue(5)); bw.addMutation(m); m = new Mutation(file2); WorkSection.add(m, serializedTarget2, StatusUtil.fileCreatedValue(10)); bw.addMutation(m); bw.close(); DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class); HashSet<String> queuedWork = new HashSet<>(); assigner.setQueuedWork(queuedWork); assigner.setMaxQueueSize(Integer.MAX_VALUE); replay(workQueue); assigner.createWork(); verify(workQueue); }
ReplicationTable.setOnline(conn);
ReplicationTable.setOnline(client); replicationWriter = ReplicationTable.getBatchWriter(client); } catch (ReplicationTableOfflineException | AccumuloSecurityException | AccumuloException e) {
ReplicationTable.isOnline(conn)); Assert.assertFalse(ReplicationTable.isOnline(conn)); while (!ReplicationTable.isOnline(conn)) { sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS); Assert.assertTrue(ReplicationTable.isOnline(conn)); Scanner s = ReplicationTable.getScanner(conn); StatusSection.limit(s); for (int i = 0; i < 5; i++) { s = ReplicationTable.getScanner(conn); StatusSection.limit(s); Iterator<Entry<Key,Value>> iter = s.iterator();
@Test public void partiallyReplicatedRecordsIgnored() throws Exception { BatchWriter bw = ReplicationTable.getBatchWriter(conn); int numRecords = 3; Status.Builder builder = Status.newBuilder(); builder.setClosed(false); builder.setEnd(10000); builder.setInfiniteEnd(false); for (int i = 0; i < numRecords; i++) { String file = "/accumulo/wal/tserver+port/" + UUID.randomUUID(); Mutation m = new Mutation(file); StatusSection.add(m, Integer.toString(i), ProtobufUtil.toValue(builder.setBegin(1000 * (i + 1)).build())); bw.addMutation(m); } bw.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); BatchScanner bs = ReplicationTable.getBatchScanner(conn, 1); bs.setRanges(Collections.singleton(new Range())); IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class); bs.addScanIterator(cfg); bw = EasyMock.createMock(BatchWriter.class); EasyMock.replay(bw); // We don't remove any records, so we can just pass in a fake BW for both rcrr.removeCompleteRecords(conn, bs, bw); bs.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); }
@Test public void recordsWithProgressUpdateBothTables() throws Exception { conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.READ); conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.WRITE); ReplicationTable.setOnline(conn); String file = "/accumulo/wals/tserver+port/" + UUID.randomUUID(); Status stat = Status.newBuilder().setBegin(100).setEnd(200).setClosed(true) .setInfiniteEnd(false).build(); ReplicationTarget target = new ReplicationTarget("peer", "table1", "1"); // Create a single work record for a file to some peer BatchWriter bw = ReplicationTable.getBatchWriter(conn); Mutation m = new Mutation(file); WorkSection.add(m, target.toText(), ProtobufUtil.toValue(stat)); bw.addMutation(m); bw.close(); updater.run(); Scanner s = ReplicationTable.getScanner(conn); s.setRange(Range.exact(file)); StatusSection.limit(s); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Assert.assertEquals(entry.getKey().getColumnFamily(), StatusSection.NAME); Assert.assertEquals(entry.getKey().getColumnQualifier().toString(), target.getSourceTableId()); // We should only rely on the correct begin attribute being returned Status actual = Status.parseFrom(entry.getValue().get()); Assert.assertEquals(stat.getBegin(), actual.getBegin()); }
m.put(StatusSection.NAME, new Text(tableId), StatusUtil.fileCreatedValue(System.currentTimeMillis())); BatchWriter bw = ReplicationTable.getBatchWriter(conn); bw.addMutation(m); bw.flush(); Scanner s = ReplicationTable.getScanner(conn); StatusSection.limit(s); Assert.assertEquals(1, Iterables.size(s)); targetClusters, tableId); s = ReplicationTable.getScanner(conn); WorkSection.limit(s);
ReplicationTable.setOnline(conn); conn.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.WRITE); allClosed = true; Scanner s = ReplicationTable.getScanner(conn); Iterator<Entry<Key,Value>> iter = s.iterator(); Scanner s = ReplicationTable.getScanner(conn); StatusSection.limit(s); for (Entry<Key,Value> entry : s) {