@Override public Iterator<Entry<String,Status>> getReplicationNeededIterator() { AccumuloClient client = getClient(); try { Scanner s = ReplicationTable.getScanner(client); StatusSection.limit(s); return Iterators.transform(s.iterator(), input -> { String file = input.getKey().getRow().toString(); Status stat; try { stat = Status.parseFrom(input.getValue().get()); } catch (InvalidProtocolBufferException e) { log.warn("Could not deserialize protobuf for: {}", input.getKey()); stat = null; } return Maps.immutableEntry(file, stat); }); } catch (ReplicationTableOfflineException e) { // No elements that we need to preclude return Collections.emptyIterator(); } } }
protected int removeReplicationEntries(Map<UUID,TServerInstance> candidates) { try { try { final Scanner s = ReplicationTable.getScanner(context); StatusSection.limit(s); for (Entry<Key,Value> entry : s) { UUID id = path2uuid(new Path(entry.getKey().getRow().toString())); candidates.remove(id); log.info("Ignore closed log " + id + " because it is being replicated"); } } catch (ReplicationTableOfflineException ex) { return candidates.size(); } final Scanner scanner = context.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.fetchColumnFamily(MetadataSchema.ReplicationSection.COLF); scanner.setRange(MetadataSchema.ReplicationSection.getRange()); for (Entry<Key,Value> entry : scanner) { Text file = new Text(); MetadataSchema.ReplicationSection.getFile(entry.getKey(), file); UUID id = path2uuid(new Path(file.toString())); candidates.remove(id); log.info("Ignore closed log " + id + " because it is being replicated"); } return candidates.size(); } catch (TableNotFoundException e) { log.error("Failed to scan metadata table", e); throw new IllegalArgumentException(e); } }
protected Status getStatus(String file, ReplicationTarget target) throws ReplicationTableOfflineException, InvalidProtocolBufferException { Scanner s = ReplicationTable.getScanner(context); s.setRange(Range.exact(file)); s.fetchColumn(WorkSection.NAME, target.toText()); return Status.parseFrom(Iterables.getOnlyElement(s).getValue().get()); } }
s = ReplicationTable.getScanner(client); } catch (ReplicationTableOfflineException e) { workScanner = ReplicationTable.getScanner(client); } catch (ReplicationTableOfflineException e) { log.warn("Replication table is offline. Will retry...");
final Scanner s; try { s = ReplicationTable.getScanner(client); if (writer == null) { setBatchWriter(ReplicationTable.getBatchWriter(client));
private Set<String> getReferencesToFilesToBeReplicated(final Connector conn) throws ReplicationTableOfflineException { Scanner s = ReplicationTable.getScanner(conn); StatusSection.limit(s); Set<String> replFiles = new HashSet<>(); for (Entry<Key,Value> entry : s) { replFiles.add(entry.getKey().getRow().toString()); } return replFiles; }
@Override public Iterator<Entry<String,Status>> getReplicationNeededIterator() throws AccumuloException, AccumuloSecurityException { Connector conn = getConnector(); try { Scanner s = ReplicationTable.getScanner(conn); StatusSection.limit(s); return Iterators.transform(s.iterator(), new Function<Entry<Key,Value>,Entry<String,Status>>() { @Override public Entry<String,Status> apply(Entry<Key,Value> input) { String file = input.getKey().getRow().toString(); Status stat; try { stat = Status.parseFrom(input.getValue().get()); } catch (InvalidProtocolBufferException e) { log.warn("Could not deserialize protobuf for: " + input.getKey()); stat = null; } return Maps.immutableEntry(file, stat); } }); } catch (ReplicationTableOfflineException e) { // No elements that we need to preclude return Collections.emptyIterator(); } }
conn = context.getConnector(); try { final Scanner s = ReplicationTable.getScanner(conn); StatusSection.limit(s); for (Entry<Key,Value> entry : s) {
protected Status getStatus(String file, ReplicationTarget target) throws ReplicationTableOfflineException, AccumuloException, AccumuloSecurityException, InvalidProtocolBufferException { Scanner s = ReplicationTable.getScanner(context.getConnector()); s.setRange(Range.exact(file)); s.fetchColumn(WorkSection.NAME, target.toText()); return Status.parseFrom(Iterables.getOnlyElement(s).getValue().get()); } }
@Test public void notYetReplicationRecordsIgnored() throws Exception { BatchWriter bw = ReplicationTable.getBatchWriter(conn); int numRecords = 3; for (int i = 0; i < numRecords; i++) { String file = "/accumulo/wal/tserver+port/" + UUID.randomUUID(); Mutation m = new Mutation(file); StatusSection.add(m, Integer.toString(i), StatusUtil.openWithUnknownLengthValue()); bw.addMutation(m); } bw.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); BatchScanner bs = ReplicationTable.getBatchScanner(conn, 1); bs.setRanges(Collections.singleton(new Range())); IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class); bs.addScanIterator(cfg); bw = EasyMock.createMock(BatchWriter.class); EasyMock.replay(bw); rcrr.removeCompleteRecords(conn, bs, bw); bs.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); }
Scanner s = ReplicationTable.getScanner(conn); StatusSection.limit(s); Assert.assertEquals(1, Iterables.size(s)); targetClusters, tableId); s = ReplicationTable.getScanner(conn); WorkSection.limit(s);
@Test public void dontCreateWorkForEntriesWithNothingToReplicate() throws Exception { String table = testName.getMethodName(); conn.tableOperations().create(table); String tableId = conn.tableOperations().tableIdMap().get(table); String file = "hdfs://localhost:8020/accumulo/wal/123456-1234-1234-12345678"; Mutation m = new Mutation(new Path(file).toString()); m.put(StatusSection.NAME, new Text(tableId), StatusUtil.fileCreatedValue(System.currentTimeMillis())); BatchWriter bw = ReplicationTable.getBatchWriter(conn); bw.addMutation(m); bw.flush(); // Assert that we have one record in the status section Scanner s = ReplicationTable.getScanner(conn); StatusSection.limit(s); Assert.assertEquals(1, Iterables.size(s)); MockWorkMaker workMaker = new MockWorkMaker(conn); conn.tableOperations().setProperty(ReplicationTable.NAME, Property.TABLE_REPLICATION_TARGET.getKey() + "remote_cluster_1", "4"); workMaker.setBatchWriter(bw); // If we don't shortcircuit out, we should get an exception because // ServerConfiguration.getTableConfiguration // won't work with MockAccumulo workMaker.run(); s = ReplicationTable.getScanner(conn); WorkSection.limit(s); Assert.assertEquals(0, Iterables.size(s)); }
@Test public void partiallyReplicatedReferencedWalsAreNotClosed() throws Exception { String file = "file:/accumulo/wal/tserver+port/12345"; Set<String> wals = Collections.singleton(file); BatchWriter bw = ReplicationTable.getBatchWriter(conn); Mutation m = new Mutation(file); StatusSection.add(m, "1", ProtobufUtil.toValue(StatusUtil.ingestedUntil(1000))); bw.addMutation(m); bw.close(); refs.updateReplicationEntries(conn, wals); Scanner s = ReplicationTable.getScanner(conn); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Status status = Status.parseFrom(entry.getValue().get()); Assert.assertFalse(status.getClosed()); }
Scanner s = ReplicationTable.getScanner(conn); StatusSection.limit(s); Assert.assertEquals(1, Iterables.size(s)); s = ReplicationTable.getScanner(conn); WorkSection.limit(s);
@Test public void partiallyReplicatedRecordsIgnored() throws Exception { BatchWriter bw = ReplicationTable.getBatchWriter(conn); int numRecords = 3; Status.Builder builder = Status.newBuilder(); builder.setClosed(false); builder.setEnd(10000); builder.setInfiniteEnd(false); for (int i = 0; i < numRecords; i++) { String file = "/accumulo/wal/tserver+port/" + UUID.randomUUID(); Mutation m = new Mutation(file); StatusSection.add(m, Integer.toString(i), ProtobufUtil.toValue(builder.setBegin(1000 * (i + 1)).build())); bw.addMutation(m); } bw.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); BatchScanner bs = ReplicationTable.getBatchScanner(conn, 1); bs.setRanges(Collections.singleton(new Range())); IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class); bs.addScanIterator(cfg); bw = EasyMock.createMock(BatchWriter.class); EasyMock.replay(bw); // We don't remove any records, so we can just pass in a fake BW for both rcrr.removeCompleteRecords(conn, bs, bw); bs.close(); Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn))); }
Scanner s = ReplicationTable.getScanner(conn); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Assert.assertEquals(StatusUtil.fileCreatedValue(createTime), entry.getValue()); s = ReplicationTable.getScanner(conn); entry = Iterables.getOnlyElement(s); Status stat = Status.parseFrom(entry.getValue().get());
@Test public void recordsWithProgressUpdateBothTables() throws Exception { conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.READ); conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.WRITE); ReplicationTable.setOnline(conn); String file = "/accumulo/wals/tserver+port/" + UUID.randomUUID(); Status stat = Status.newBuilder().setBegin(100).setEnd(200).setClosed(true) .setInfiniteEnd(false).build(); ReplicationTarget target = new ReplicationTarget("peer", "table1", "1"); // Create a single work record for a file to some peer BatchWriter bw = ReplicationTable.getBatchWriter(conn); Mutation m = new Mutation(file); WorkSection.add(m, target.toText(), ProtobufUtil.toValue(stat)); bw.addMutation(m); bw.close(); updater.run(); Scanner s = ReplicationTable.getScanner(conn); s.setRange(Range.exact(file)); StatusSection.limit(s); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Assert.assertEquals(entry.getKey().getColumnFamily(), StatusSection.NAME); Assert.assertEquals(entry.getKey().getColumnQualifier().toString(), target.getSourceTableId()); // We should only rely on the correct begin attribute being returned Status actual = Status.parseFrom(entry.getValue().get()); Assert.assertEquals(stat.getBegin(), actual.getBegin()); }
Assert.assertEquals(numRecords, Iterables.size(ReplicationTable.getScanner(conn)));
Scanner s = ReplicationTable.getScanner(conn); s.setRange(Range.exact(file)); StatusSection.limit(s);
Scanner s = ReplicationTable.getScanner(conn); s.setRange(Range.exact(file)); StatusSection.limit(s);