public ReplicationProcessor(ServerContext context, AccumuloConfiguration conf, VolumeManager fs) { this.context = context; this.conf = conf; this.fs = fs; this.helper = new ReplicaSystemHelper(context); this.factory = new ReplicaSystemFactory(); }
/** * @param builder * Existing {@link Builder} to use * @param recordsReplicated * Offset of records which have been replicated * @return A {@link Status} tracking data that must be replicated */ public static Status replicated(Status.Builder builder, long recordsReplicated) { return replicatedAndIngested(builder, recordsReplicated, 0); }
/** * @return Should a Work entry be created for this status */ protected boolean shouldCreateWork(Status status) { // Only creating work when there is work to do (regardless of closed status) is safe // as long as the ReplicaSystem implementation is correctly observing // that a file is completely replicated only when the file is closed return StatusUtil.isWorkRequired(status); }
/** * Is the given Status fully replicated and is its file ready for deletion on the source * * @param status * a Status protobuf * @return True if the file this Status references can be deleted. */ public static boolean isSafeForRemoval(Status status) { return status.getClosed() && isFullyReplicated(status); }
/** * @param recordsReplicated * Offset of records which have been replicated * @return A {@link Status} tracking data that must be replicated */ public static Status replicated(long recordsReplicated) { return replicated(Status.newBuilder(), recordsReplicated); }
/** * Creates a {@link Status} for newly-created data that must be replicated * * @param recordsIngested * Offset of records which need to be replicated * @return A {@link Status} tracking data that must be replicated */ public static Status ingestedUntil(long recordsIngested) { return ingestedUntil(Status.newBuilder(), recordsIngested); }
/** * Creates a @{link Status} for a file which has new data and data which has been replicated * * @param recordsReplicated * Offset of records which have been replicated * @param recordsIngested * Offset for records which need to be replicated * @return A {@link Status} for the given parameters */ public static Status replicatedAndIngested(long recordsReplicated, long recordsIngested) { return replicatedAndIngested(Status.newBuilder(), recordsReplicated, recordsIngested); }
protected int getNumFilesPendingReplication() { // The total set of configured targets Set<ReplicationTarget> allConfiguredTargets = replicationUtil.getReplicationTargets(); // Number of files per target we have to replicate Map<ReplicationTarget,Long> targetCounts = replicationUtil.getPendingReplications(); int filesPending = 0; // Sum pending replication over all targets for (ReplicationTarget configuredTarget : allConfiguredTargets) { Long numFiles = targetCounts.get(configuredTarget); if (numFiles != null) { filesPending += numFiles; } } return filesPending; }
public ReplicationUtil(ServerContext context) { this(context, new ReplicaSystemFactory()); }
@Override public int getNumConfiguredPeers() { return replicationUtil.getPeers().size(); }
protected int getMaxReplicationThreads() { return replicationUtil.getMaxReplicationThreads(master.getMasterMonitorInfo()); }
@Override public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException { super.init(source, options, env); setEncoder(new StatusEncoder()); }
@Override public Void run() { try { helper.recordNewStatus(p, copy, target); } catch (Exception e) { exceptionRef.set(e); } return null; } });
/** * @return A {@link Value} for a new file that was just created */ public static Value fileCreatedValue(long timeCreated) { return ProtobufUtil.toValue(fileCreated(timeCreated)); }
protected ReplicaSystem getReplicaSystem(ReplicationTarget target) { // Find the configured replication peer so we know how to replicate to it // Classname,Configuration String peerType = getPeerType(target.getPeerName()); // Get the peer that we're replicating to return factory.get(context, peerType); }
protected StringBuilder appendText(StringBuilder sb, Text t) { return appendBytes(sb, t.getBytes(), 0, t.getLength()); }
protected boolean isWorkRequired(Status status) { return StatusUtil.isWorkRequired(status); }
protected int getNumConfiguredPeers() { return replicationUtil.getPeers().size(); }
@Override public int getMaxReplicationThreads() { return replicationUtil.getMaxReplicationThreads(master.getMasterMonitorInfo()); }
public static Status ingestedUntil(Builder builder, long recordsIngested) { return replicatedAndIngested(builder, 0, recordsIngested); }