public LocalState nimbusTopoHistoryStateImpl(Map<String, Object> conf) throws IOException { return new LocalState((masterLocalDir(conf) + FILE_SEPARATOR + "history"), true); } }
public void addTopologyHistory(LSTopoHistory lsTopoHistory) { LSTopoHistoryList lsTopoHistoryListWrapper = (LSTopoHistoryList) get(LS_TOPO_HISTORY); List<LSTopoHistory> currentTopoHistoryList = new ArrayList<>(); if (null != lsTopoHistoryListWrapper) { currentTopoHistoryList.addAll(lsTopoHistoryListWrapper.get_topo_history()); } currentTopoHistoryList.add(lsTopoHistory); put(LS_TOPO_HISTORY, new LSTopoHistoryList(currentTopoHistoryList)); }
@Override public void cleanUpForRestart() throws IOException { String origWorkerId = _workerId; super.cleanUpForRestart(); synchronized (_localState) { Map<String, Integer> workersToPort = _localState.getApprovedWorkers(); if (workersToPort != null) { workersToPort.remove(origWorkerId); removeWorkersOn(workersToPort, _port); _localState.setApprovedWorkers(workersToPort); LOG.info("Removed Worker ID {}", origWorkerId); } else { LOG.warn("No approved workers exists"); } } }
@Override public void prepare(Map<String, Object> topoConf, String schedulerLocalDir) { try { LocalState localState = new LocalState(schedulerLocalDir, true); String supervisorId = localState.getSupervisorId(); if (supervisorId == null) { supervisorId = generateSupervisorId(); localState.setSupervisorId(supervisorId); } this.conf = topoConf; this.supervisorId = supervisorId; } catch (IOException e) { throw new RuntimeException(e); } }
public TBase get(String key) { TDeserializer td = new TDeserializer(); Map<String, ThriftSerializedObject> partial = partialSnapshot(td); ThriftSerializedObject tso = partial.get(key); TBase ret = null; if (tso != null) { ret = deserialize(tso, td); } return ret; }
private Map<String, TBase> deserializeLatestVersion() throws IOException { Map<String, TBase> result = new HashMap<>(); TDeserializer td = new TDeserializer(); for (Map.Entry<String, ThriftSerializedObject> ent : partialDeserializeLatestVersion(td).entrySet()) { result.put(ent.getKey(), deserialize(ent.getValue(), td)); } return result; }
@Test public void testLocalState() throws Exception { try (TmpPath dir1_tmp = new TmpPath(); TmpPath dir2_tmp = new TmpPath()) { GlobalStreamId globalStreamId_a = new GlobalStreamId("a", "a"); GlobalStreamId globalStreamId_b = new GlobalStreamId("b", "b"); GlobalStreamId globalStreamId_c = new GlobalStreamId("c", "c"); GlobalStreamId globalStreamId_d = new GlobalStreamId("d", "d"); LocalState ls1 = new LocalState(dir1_tmp.getPath(), true); LocalState ls2 = new LocalState(dir2_tmp.getPath(), true); Assert.assertTrue(ls1.snapshot().isEmpty()); ls1.put("a", globalStreamId_a); ls1.put("b", globalStreamId_b); Map<String, GlobalStreamId> expected = new HashMap<>(); expected.put("a", globalStreamId_a); expected.put("b", globalStreamId_b); Assert.assertEquals(expected, ls1.snapshot()); Assert.assertEquals(expected, new LocalState(dir1_tmp.getPath(), true).snapshot()); Assert.assertTrue(ls2.snapshot().isEmpty()); ls2.put("b", globalStreamId_a); ls2.put("b", globalStreamId_b); ls2.put("b", globalStreamId_c); ls2.put("b", globalStreamId_d); Assert.assertEquals(globalStreamId_d, ls2.get("b")); } }
@Test public void testEmptyState() throws IOException { TmpPath tmp_dir = new TmpPath(); String dir = tmp_dir.getPath(); LocalState ls = new LocalState(dir, true); GlobalStreamId gs_a = new GlobalStreamId("a", "a"); FileOutputStream data = FileUtils.openOutputStream(new File(dir, "12345")); FileOutputStream version = FileUtils.openOutputStream(new File(dir, "12345.version")); Assert.assertNull(ls.get("c")); ls.put("a", gs_a); Assert.assertEquals(gs_a, ls.get("a")); } }
public LSWorkerHeartbeat getWorkerHeartBeat() { return (LSWorkerHeartbeat) get(LS_WORKER_HEARTBEAT); }
public void put(String key, TBase val) { put(key, val, true); }
private void saveNewAssignment(LocalAssignment assignment) { synchronized (staticState.localState) { Map<Integer, LocalAssignment> assignments = staticState.localState.getLocalAssignmentsMap(); if (assignments == null) { assignments = new HashMap<>(); } if (assignment == null) { assignments.remove(staticState.port); } else { assignments.put(staticState.port, assignment); } staticState.localState.setLocalAssignmentsMap(assignments); } Map<Long, LocalAssignment> update = null; Map<Long, LocalAssignment> orig = null; do { Long lport = new Long(staticState.port); orig = cachedCurrentAssignments.get(); update = new HashMap<>(orig); if (assignment == null) { update.remove(lport); } else { update.put(lport, assignment); } } while (!cachedCurrentAssignments.compareAndSet(orig, update)); }
public void remove(String key) { remove(key, true); }
@Test public void testRecoveryMiss() throws Exception { final String topoId = "test_topology"; final int supervisorPort = 6628; final int port = 8080; LocalAssignment la = new LocalAssignment(); la.set_topology_id(topoId); Map<String, Integer> workerState = new HashMap<String, Integer>(); workerState.put("somethingelse", port + 1); LocalState ls = mock(LocalState.class); when(ls.getApprovedWorkers()).thenReturn(workerState); try { new MockBasicContainer(ContainerType.RECOVER_FULL, new HashMap<String, Object>(), "SUPERVISOR", supervisorPort, port, la, null, ls, null, new StormMetricsRegistry(), new HashMap<>(), null, "profile"); fail("Container recovered worker incorrectly"); } catch (ContainerRecoveryException e) { //Expected } }
LocalAssignment newAssignment = null; Map<Integer, LocalAssignment> assignments = localState.getLocalAssignmentsMap(); if (assignments != null) { currentAssignment = assignments.get(port);
public synchronized Map<String, TBase> snapshot() { int attempts = 0; while (true) { try { return deserializeLatestVersion(); } catch (Exception e) { attempts++; if (attempts >= 10) { throw new RuntimeException(e); } } } }
public Map<String, Integer> getApprovedWorkers() { LSApprovedWorkers lsApprovedWorkers = (LSApprovedWorkers) get(LS_APPROVED_WORKERS); if (null != lsApprovedWorkers) { return lsApprovedWorkers.get_approved_workers(); } return null; }
public void setWorkerHeartBeat(LSWorkerHeartbeat workerHeartBeat) { put(LS_WORKER_HEARTBEAT, workerHeartBeat, false); }
@Override public void prepare(Map stormConf, String schedulerLocalDir) { try { LocalState localState = new LocalState(schedulerLocalDir); String supervisorId = localState.getSupervisorId(); if (supervisorId == null) { supervisorId = generateSupervisorId(); localState.setSupervisorId(supervisorId); } this.conf = stormConf; this.supervisorId = supervisorId; } catch (IOException e) { throw new RuntimeException(e); } }
public TBase get(String key) { TDeserializer td = new TDeserializer(); Map<String, ThriftSerializedObject> partial = partialSnapshot(td); ThriftSerializedObject tso = partial.get(key); TBase ret = null; if (tso != null) { ret = deserialize(tso, td); } return ret; }
private Map<String, TBase> deserializeLatestVersion() throws IOException { Map<String, TBase> result = new HashMap<>(); TDeserializer td = new TDeserializer(); for (Map.Entry<String, ThriftSerializedObject> ent: partialDeserializeLatestVersion(td).entrySet()) { result.put(ent.getKey(), deserialize(ent.getValue(), td)); } return result; }