/** * Create a tracked topology. * @deprecated use {@link org.apache.storm.testing.TrackedTopology} directly. */ @Deprecated public static TrackedTopology mkTrackedTopology(ILocalCluster cluster, StormTopology topology) { return new TrackedTopology(topology, cluster); }
/** * Simulated time wait for a tracked topology. This is intended for internal testing */ public static void trackedWait(TrackedTopology topo, Integer amt) { topo.trackedWait(amt); }
@Override public LocalTopology submitTopology(String topologyName, Map<String, Object> conf, TrackedTopology topology) throws TException { return submitTopology(topologyName, conf, topology.getTopology()); }
/** * Read a metric from the tracked cluster (NOT JUST THIS TOPOLOGY) * @param key one of "spout-emitted", "processed", or "transferred" * @return the amount of that metric */ public int globalAmt(String key) { return globalAmt(cluster.getTrackedId(), key); } }
public StormTopology getTopology() { return (StormTopology)get(Keyword.intern("topology")); } }
@Override public LocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, TrackedTopology topology, SubmitOptions submitOpts) throws TException { return submitTopologyWithOpts(topologyName, conf, topology.getTopology(), submitOpts); }
/** * Wait for amt tuples to be fully processed timeoutMs happens. */ public void trackedWait(int amt, int timeoutMs) { final int target = amt + lastSpoutCommit.get(); final String id = cluster.getTrackedId(); Random rand = ThreadLocalRandom.current(); whileTimeout(timeoutMs, () -> { int se = globalAmt(id, "spout-emitted"); int transferred = globalAmt(id, "transferred"); int processed = globalAmt(id, "processed"); LOG.info("emitted {} target {} transferred {} processed {}", se, target, transferred, processed); return (target != se) || (transferred != processed); }, () -> { Time.advanceTimeSecs(1); try { Thread.sleep(rand.nextInt(200)); } catch (Exception e) { throw new RuntimeException(e); } }); lastSpoutCommit.set(target); }
/** * Track and capture a topology. * This is intended mostly for internal testing. */ public static CapturedTopology<TrackedTopology> trackAndCaptureTopology(ILocalCluster cluster, StormTopology topology) { CapturedTopology<StormTopology> captured = captureTopology(topology); return new CapturedTopology<>(new TrackedTopology(captured.topology, cluster), captured.capturer); }
/** * Simulated time wait for a tracked topology. This is intended for internal testing */ public static void trackedWait(TrackedTopology topo, Integer amt, Integer timeoutMs) { topo.trackedWait(amt, timeoutMs); }
@Test public void testAckBranching() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withTracked() .build()) { AckTrackingFeeder feeder = new AckTrackingFeeder("num"); Map<String, SpoutDetails> spoutMap = new HashMap<>(); spoutMap.put("1", Thrift.prepareSpoutDetails(feeder.getSpout())); Map<String, BoltDetails> boltMap = new HashMap<>(); boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new IdentityBolt())); boltMap.put("3", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new IdentityBolt())); Map<GlobalStreamId, Grouping> aggregatorInputs = new HashMap<>(); aggregatorInputs.put(Utils.getGlobalStreamId("2", null), Thrift.prepareShuffleGrouping()); aggregatorInputs.put(Utils.getGlobalStreamId("3", null), Thrift.prepareShuffleGrouping()); boltMap.put("4", Thrift.prepareBoltDetails(aggregatorInputs, new AggBolt(4))); TrackedTopology tracked = new TrackedTopology(Thrift.buildTopology(spoutMap, boltMap), cluster);; cluster.submitTopology("test-acking2", Collections.emptyMap(), tracked); cluster.advanceClusterTime(11); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 1); feeder.assertNumAcks(0); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 1); feeder.assertNumAcks(2); } }
/** * Simulated time wait for a tracked topology. This is intended for internal testing */ public static void trackedWait(CapturedTopology<TrackedTopology> topo, Integer amt, Integer timeoutMs) { topo.topology.trackedWait(amt, timeoutMs); }
@Test public void testWithTrackedCluster() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withTracked() .build()) { AckTrackingFeeder feeder = new AckTrackingFeeder("num"); Map<String, Thrift.SpoutDetails> spoutMap = new HashMap<>(); spoutMap.put("1", Thrift.prepareSpoutDetails(feeder.getSpout())); Map<String, Thrift.BoltDetails> boltMap = new HashMap<>(); boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new IdentityBolt())); boltMap.put("3", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new IdentityBolt())); Map<GlobalStreamId, Grouping> aggregatorInputs = new HashMap<>(); aggregatorInputs.put(Utils.getGlobalStreamId("2", null), Thrift.prepareShuffleGrouping()); aggregatorInputs.put(Utils.getGlobalStreamId("3", null), Thrift.prepareShuffleGrouping()); boltMap.put("4", Thrift.prepareBoltDetails(aggregatorInputs, new AggBolt(4))); TrackedTopology tracked = new TrackedTopology(Thrift.buildTopology(spoutMap, boltMap), cluster);; cluster.submitTopology("test-acking2", new Config(), tracked); cluster.advanceClusterTime(11); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 1); feeder.assertNumAcks(0); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 1); feeder.assertNumAcks(2); } }
/** * Simulated time wait for a tracked topology. This is intended for internal testing */ public static void trackedWait(TrackedTopology topo) { topo.trackedWait(); }
@Test public void testAckingSelfAnchor() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withTracked() .build()) { AckTrackingFeeder feeder = new AckTrackingFeeder("num"); Map<String, SpoutDetails> spoutMap = new HashMap<>(); spoutMap.put("1", Thrift.prepareSpoutDetails(feeder.getSpout())); Map<String, BoltDetails> boltMap = new HashMap<>(); boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new DupAnchorBolt())); boltMap.put("3", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("2", null), Thrift.prepareShuffleGrouping()), new AckBolt())); TrackedTopology tracked = new TrackedTopology(Thrift.buildTopology(spoutMap, boltMap), cluster);; cluster.submitTopology("test", Collections.emptyMap(), tracked); cluster.advanceClusterTime(11); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 1); feeder.assertNumAcks(1); feeder.feed(new Values(1)); feeder.feed(new Values(1)); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 3); feeder.assertNumAcks(3); } }
/** * Wait for amt tuples to be fully processed. */ public void trackedWait(int amt) { trackedWait(amt, Testing.TEST_TIMEOUT_MS); }
boltMap.put("9", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("8", null), Thrift.prepareShuffleGrouping()), new AckBolt())); TrackedTopology tracked = new TrackedTopology(Thrift.buildTopology(spoutMap, boltMap), cluster);;
/** * Wait for 1 tuple to be fully processed */ public void trackedWait() { trackedWait(1, Testing.TEST_TIMEOUT_MS); }
/** * Simulated time wait for a tracked topology. This is intended for internal testing */ public static void trackedWait(CapturedTopology<TrackedTopology> topo) { topo.topology.trackedWait(); }
/** * Simulated time wait for a tracked topology. This is intended for internal testing */ public static void trackedWait(CapturedTopology<TrackedTopology> topo, Integer amt) { topo.topology.trackedWait(amt); }