private void compareLoop(JobIdFactory factory, boolean overflow) { Set<Boolean> overflowed = new HashSet<>(Collections.singleton(false)); JobId prevMax = null; for (int i = 0; i < 1000; i++) { final JobId jobId = factory.create(); Assert.assertTrue("max == last", factory.maxJobId().compareTo(jobId) == 0); if (i > 0) { Assert.assertTrue("last > previous max", prevMax.compareTo(jobId) < 0 || overflowed.add(overflow)); } prevMax = factory.maxJobId(); } }
public void setMaxJobId(long maxJobId) { ensureMinimumId(maxJobId + 1); } }
public JobId maxJobId() { return new JobId(maxId()); }
private void testOverflow(int id) throws IllegalAccessException { CcId ccId = CcId.valueOf(id); long expected = (long) id << 48; JobIdFactory factory = new JobIdFactory(ccId); AtomicLong theId = (AtomicLong) idField.get(factory); Assert.assertEquals(expected, theId.get()); theId.set((((long) 1 << 48) - 1) | expected); JobId jobId = factory.create(); Assert.assertEquals(ccId, jobId.getCcId()); Assert.assertEquals(CcIdPartitionedLongFactory.MAX_ID, jobId.getIdOnly()); jobId = factory.create(); Assert.assertEquals(ccId, jobId.getCcId()); Assert.assertEquals(0, jobId.getIdOnly()); }
@Test public void testComparability() throws IllegalAccessException { JobIdFactory factory = new JobIdFactory(CcId.valueOf(0)); compareLoop(factory, false); factory = new JobIdFactory(CcId.valueOf(0xFFFF)); compareLoop(factory, false); AtomicLong theId = (AtomicLong) idField.get(factory); theId.set(0xFFFFFFFFFFFFFFF0L); compareLoop(factory, true); }
@Override protected void doRun() throws Exception { IJobManager jobManager = ccs.getJobManager(); try { final CCServiceContext ccServiceCtx = ccs.getContext(); JobId jobId; JobRun run; jobId = jobIdFactory.create(); if (deployedJobSpecId == null) { //Need to create the ActivityClusterGraph IActivityClusterGraphGeneratorFactory acggf = (IActivityClusterGraphGeneratorFactory) DeploymentUtils .deserialize(acggfBytes, deploymentId, ccServiceCtx); IActivityClusterGraphGenerator acgg = acggf.createActivityClusterGraphGenerator(ccServiceCtx, jobFlags); run = new JobRun(ccs, deploymentId, jobId, acggf, acgg, jobFlags); } else { //ActivityClusterGraph has already been distributed run = new JobRun(ccs, deploymentId, jobId, jobFlags, ccs.getDeployedJobSpecStore().getDeployedJobSpecDescriptor(deployedJobSpecId), jobParameters, deployedJobSpecId); } jobManager.add(run); callback.setValue(jobId); } catch (Exception e) { callback.setException(e); } } }
public JobId create() { return new JobId(nextId()); }
private void updateClusterCounters(String nodeId, NcLocalCounters localCounters) { final IResourceIdManager resourceIdManager = appCtx.getResourceIdManager(); resourceIdManager.report(nodeId, localCounters.getMaxResourceId()); appCtx.getTxnIdFactory().ensureMinimumId(localCounters.getMaxTxnId()); ((ClusterControllerService) appCtx.getServiceContext().getControllerService()).getJobIdFactory() .setMaxJobId(localCounters.getMaxJobId()); }
@Test public void testCcIds() { JobIdFactory factory = new JobIdFactory(CcId.valueOf(0)); for (int i = 0; i < 1000; i++) { final JobId jobId = factory.create(); Assert.assertEquals(0, jobId.getCcId().shortValue()); Assert.assertEquals(i, jobId.getIdOnly()); } }
public ClusterControllerService(final CCConfig config, final ICCApplication application) throws Exception { this.ccConfig = config; this.configManager = ccConfig.getConfigManager(); if (application == null) { throw new IllegalArgumentException("ICCApplication cannot be null"); } this.application = application; File jobLogFolder = new File(ccConfig.getRootDir(), "logs/jobs"); jobLog = new LogFile(jobLogFolder); final INetworkSecurityConfig securityConfig = getNetworkSecurityConfig(); networkSecurityManager = new NetworkSecurityManager(securityConfig); // WorkQueue is in charge of heartbeat as well as other events. workQueue = new WorkQueue("ClusterController", Thread.MAX_PRIORITY); this.timer = new Timer(true); final ClusterTopology topology = computeClusterTopology(ccConfig); ccContext = new ClusterControllerContext(topology); sweeper = new DeadNodeSweeper(); resultDirectoryService = new ResultDirectoryService(ccConfig.getResultTTL(), ccConfig.getResultSweepThreshold()); deploymentRunMap = new HashMap<>(); stateDumpRunMap = new HashMap<>(); threadDumpRunMap = Collections.synchronizedMap(new HashMap<>()); // Node manager is in charge of cluster membership management. nodeManager = new NodeManager(this, ccConfig, resourceManager, application.getGatekeeper()); ccId = ccConfig.getCcId(); jobIdFactory = new JobIdFactory(ccId); deployedJobSpecIdFactory = new DeployedJobSpecIdFactory(); }
JobId jobId = jobIdFactory.create(); Action startJob = clusterController.startActiveJob(jobId, entityId); try {
@Test public void testParse() throws HyracksDataException { for (int ccId : Arrays.asList(0xFFFF, 0, (int) Short.MAX_VALUE)) { JobIdFactory factory = new JobIdFactory(CcId.valueOf(ccId)); for (int i = 0; i < 1000; i++) { final JobId jobId = factory.create(); Assert.assertEquals(jobId.getId(), JobId.parse(jobId.toString()).getId()); Assert.assertEquals(jobId, JobId.parse(jobId.toString())); Assert.assertFalse(jobId.toString(), jobId.toString().contains("-")); System.err.println(jobId.toString()); } } } }
private CcApplicationContext ccAppContext(ClusterStateManager csm) throws HyracksDataException { CcApplicationContext ccApplicationContext = Mockito.mock(CcApplicationContext.class); ConfigManager configManager = new ConfigManager(null); IApplicationConfig applicationConfig = new ConfigManagerApplicationConfig(configManager); ICCServiceContext iccServiceContext = Mockito.mock(CCServiceContext.class); final ClusterControllerService ccs = Mockito.mock(ClusterControllerService.class); JobIdFactory jobIdFactory = new JobIdFactory(CcId.valueOf(0)); Mockito.when(ccs.getJobIdFactory()).thenReturn(jobIdFactory); Mockito.when(iccServiceContext.getAppConfig()).thenReturn(applicationConfig); Mockito.when(iccServiceContext.getControllerService()).thenReturn(ccs); Mockito.when(ccApplicationContext.getServiceContext()).thenReturn(iccServiceContext); NcLifecycleCoordinator coordinator = new NcLifecycleCoordinator(ccApplicationContext.getServiceContext(), false); coordinator.bindTo(csm); Mockito.when(ccApplicationContext.getNcLifecycleCoordinator()).thenReturn(coordinator); MetadataProperties metadataProperties = mockMetadataProperties(); Mockito.when(ccApplicationContext.getMetadataProperties()).thenReturn(metadataProperties); ResourceIdManager resourceIdManager = new ResourceIdManager(csm); Mockito.when(ccApplicationContext.getResourceIdManager()).thenReturn(resourceIdManager); IMetadataBootstrap metadataBootstrap = Mockito.mock(IMetadataBootstrap.class); Mockito.doNothing().when(metadataBootstrap).init(); Mockito.when(ccApplicationContext.getMetadataBootstrap()).thenReturn(metadataBootstrap); IGlobalRecoveryManager globalRecoveryManager = Mockito.mock(IGlobalRecoveryManager.class); Mockito.when(globalRecoveryManager.isRecoveryCompleted()).thenReturn(true); Mockito.when(ccApplicationContext.getGlobalRecoveryManager()).thenReturn(globalRecoveryManager); BulkTxnIdFactory bulkTxnIdFactory = new BulkTxnIdFactory(); Mockito.when(ccApplicationContext.getTxnIdFactory()).thenReturn(bulkTxnIdFactory); return ccApplicationContext; }
@Test public void testNegativeCcId() { JobIdFactory factory = new JobIdFactory(CcId.valueOf(0xFFFF)); for (int i = 0; i < 1000; i++) { final JobId jobId = factory.create(); Assert.assertEquals((short) 0xFFFF, jobId.getCcId().shortValue()); Assert.assertEquals(i, jobId.getIdOnly()); Assert.assertTrue("JID not negative", jobId.getId() < 0); Assert.assertEquals(0xFFFF000000000000L + i, jobId.getId()); } }
@Before public void setUp() throws Exception { jobIdFactory = new JobIdFactory(CcId.valueOf((short) 0)); handler = new ActiveNotificationHandler(); allDatasets = new ArrayList<>();