private Map<String, ResourceConfig> refreshResourceConfigs(HelixDataAccessor accessor) { Map<String, ResourceConfig> refreshedResourceConfigs = Maps.newHashMap(); long startTime = System.currentTimeMillis(); PropertyKey.Builder keyBuilder = accessor.keyBuilder(); Set<PropertyKey> currentResourceConfigKeys = new HashSet<>(); for (String resourceConfig : accessor.getChildNames(keyBuilder.resourceConfigs())) { currentResourceConfigKeys.add(keyBuilder.resourceConfig(resourceConfig)); } Set<PropertyKey> cachedKeys = new HashSet<>(); Map<PropertyKey, ResourceConfig> cachedResourceConfigMap = Maps.newHashMap(); for (String resourceConfig : _resourceConfigMap.keySet()) { cachedKeys.add(keyBuilder.resourceConfig(resourceConfig)); cachedResourceConfigMap.put(keyBuilder.resourceConfig(resourceConfig), _resourceConfigMap.get(resourceConfig)); } cachedKeys.retainAll(currentResourceConfigKeys); Set<PropertyKey> reloadKeys = new HashSet<>(currentResourceConfigKeys); reloadKeys.removeAll(cachedKeys); Map<PropertyKey, ResourceConfig> updatedMap = refreshProperties(accessor, new LinkedList<>(reloadKeys), new ArrayList<>(cachedKeys), cachedResourceConfigMap); for (ResourceConfig resourceConfig : updatedMap.values()) { refreshedResourceConfigs.put(resourceConfig.getResourceName(), resourceConfig); } long endTime = System.currentTimeMillis(); LogUtil.logInfo(LOG, getEventId(), "Refresh " + refreshedResourceConfigs.size() + " resource configs for cluster " + _clusterName + ", took " + (endTime - startTime) + " ms"); return refreshedResourceConfigs; }
/** * Trigger the controller to perform rebalance for a given resource. * @param accessor Helix data accessor * @param resource the name of the resource changed to triggering the execution */ public static void invokeRebalanceForResourceConfig(HelixDataAccessor accessor, String resource) { LOG.info("invoke rebalance for " + resource); PropertyKey key = accessor.keyBuilder().resourceConfig(resource); ResourceConfig cfg = accessor.getProperty(key); if (cfg != null) { if (!accessor.updateProperty(key, cfg)) { LOG.warn("Failed to invoke rebalance on resource config {}", resource); } } else { LOG.warn("Can't find resource config for {}", resource); } } }
@Deprecated public static PropertyKey getWorkflowConfigKey(final HelixDataAccessor accessor, String workflow) { return accessor.keyBuilder().resourceConfig(workflow); }
StringRepresentation getHostedEntitiesRepresentation(String clusterName, String jobQueueName) throws Exception { ZkClient zkClient = ResourceUtil.getAttributeFromCtx(getContext(), ResourceUtil.ContextKey.ZKCLIENT); HelixDataAccessor accessor = ClusterRepresentationUtil.getClusterDataAccessor(zkClient, clusterName); PropertyKey.Builder keyBuilder = accessor.keyBuilder(); TaskDriver taskDriver = new TaskDriver(zkClient, clusterName); // Get job queue config // TODO: fix this to use workflowConfig. ResourceConfig jobQueueConfig = accessor.getProperty(keyBuilder.resourceConfig(jobQueueName)); // Get job queue context WorkflowContext ctx = taskDriver.getWorkflowContext(jobQueueName); // Create the result ZNRecord hostedEntitiesRecord = new ZNRecord(jobQueueName); if (jobQueueConfig != null) { hostedEntitiesRecord.merge(jobQueueConfig.getRecord()); } if (ctx != null) { hostedEntitiesRecord.merge(ctx.getRecord()); } StringRepresentation representation = new StringRepresentation(ClusterRepresentationUtil.ZNRecordToJson(hostedEntitiesRecord), MediaType.APPLICATION_JSON); return representation; }
@Test public void testExpiry() throws Exception { String jobName = "Expiry"; long expiry = 1000; Map<String, String> commandConfig = ImmutableMap.of(MockTask.JOB_DELAY, String.valueOf(100)); JobConfig.Builder jobBuilder = JobConfig.Builder.fromMap(WorkflowGenerator.DEFAULT_JOB_CONFIG); jobBuilder.setJobCommandConfigMap(commandConfig); Workflow flow = WorkflowGenerator .generateSingleJobWorkflowBuilder(jobName, jobBuilder) .setExpiry(expiry).build(); _driver.start(flow); _driver.pollForWorkflowState(jobName, TaskState.IN_PROGRESS); // Running workflow should have config and context viewable through accessor HelixDataAccessor accessor = _manager.getHelixDataAccessor(); PropertyKey workflowCfgKey = accessor.keyBuilder().resourceConfig(jobName); String workflowPropStoreKey = Joiner.on("/").join(TaskConstants.REBALANCER_CONTEXT_ROOT, jobName); // Ensure context and config exist Assert.assertTrue(_manager.getHelixPropertyStore().exists(workflowPropStoreKey, AccessOption.PERSISTENT)); Assert.assertNotSame(accessor.getProperty(workflowCfgKey), null); // Wait for job to finish and expire _driver.pollForWorkflowState(jobName, TaskState.COMPLETED); Thread.sleep(expiry + 100); // Ensure workflow config and context were cleaned up by now Assert.assertFalse(_manager.getHelixPropertyStore().exists(workflowPropStoreKey, AccessOption.PERSISTENT)); Assert.assertEquals(accessor.getProperty(workflowCfgKey), null); }
/** * Remove workflow or job config. * @param accessor * @param workflowJobResource the workflow or job name */ private static boolean removeWorkflowJobConfig(HelixDataAccessor accessor, String workflowJobResource) { PropertyKey cfgKey = accessor.keyBuilder().resourceConfig(workflowJobResource); if (accessor.getPropertyStat(cfgKey) != null) { if (!accessor.removeProperty(cfgKey)) { LOG.warn(String.format( "Error occurred while trying to remove config for %s. Failed to remove node %s.", workflowJobResource, cfgKey)); return false; } } return true; }
/** * Remove workflow or job config. * @param accessor * @param workflowJobResource the workflow or job name */ private static boolean removeWorkflowJobConfig(HelixDataAccessor accessor, String workflowJobResource) { PropertyKey cfgKey = accessor.keyBuilder().resourceConfig(workflowJobResource); if (accessor.getPropertyStat(cfgKey) != null) { if (!accessor.removeProperty(cfgKey)) { LOG.warn(String.format( "Error occurred while trying to remove config for %s. Failed to remove node %s.", workflowJobResource, cfgKey)); return false; } } return true; }
private static PropertyKey getConfigPropertyKey(HelixDataAccessor accessor, String resource) { return accessor.keyBuilder().resourceConfig(resource); }
/** * Create the resource config. Fails if it already exists in ZK. * @param accessor * @param resource * @param resourceConfig * @return */ private static boolean createResourceConfig(HelixDataAccessor accessor, String resource, ResourceConfig resourceConfig) { PropertyKey.Builder keyBuilder = accessor.keyBuilder(); return accessor.getBaseDataAccessor().create(keyBuilder.resourceConfig(resource).getPath(), resourceConfig.getRecord(), AccessOption.PERSISTENT); }
private static HelixProperty getResourceConfig(HelixDataAccessor accessor, String resource) { PropertyKey.Builder keyBuilder = accessor.keyBuilder(); return accessor.getProperty(keyBuilder.resourceConfig(resource)); }
private static PropertyKey getConfigPropertyKey(HelixDataAccessor accessor, String resource) { return accessor.keyBuilder().resourceConfig(resource); }
@Deprecated public static PropertyKey getWorkflowConfigKey(final HelixDataAccessor accessor, String workflow) { return accessor.keyBuilder().resourceConfig(workflow); }
private static HelixProperty getResourceConfig(HelixDataAccessor accessor, String resource) { PropertyKey.Builder keyBuilder = accessor.keyBuilder(); return accessor.getProperty(keyBuilder.resourceConfig(resource)); }
/** * Set the resource config * @param accessor Accessor to Helix configs * @param resource The resource name * @param resourceConfig The resource config to be set * @return True if set successfully, otherwise false */ private static boolean setResourceConfig(HelixDataAccessor accessor, String resource, ResourceConfig resourceConfig) { PropertyKey.Builder keyBuilder = accessor.keyBuilder(); return accessor.setProperty(keyBuilder.resourceConfig(resource), resourceConfig); }
private void deleteJobConfigs(String workflowName, String jobName) { String oldPath = _manager.getHelixDataAccessor().keyBuilder().resourceConfig(jobName).getPath(); String newPath = _manager.getHelixDataAccessor().keyBuilder() .jobConfigZNode(workflowName, jobName).getPath(); _baseAccessor.remove(oldPath, AccessOption.PERSISTENT); _baseAccessor.remove(newPath, AccessOption.PERSISTENT); }
@Override public void dropResource(String clusterName, String resourceName) { logger.info("Drop resource {} from cluster {}", resourceName, clusterName); HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); accessor.removeProperty(keyBuilder.idealStates(resourceName)); accessor.removeProperty(keyBuilder.resourceConfig(resourceName)); }
@Override public void dropResource(String clusterName, String resourceName) { logger.info("Drop resource {} from cluster {}", resourceName, clusterName); HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); accessor.removeProperty(keyBuilder.idealStates(resourceName)); accessor.removeProperty(keyBuilder.resourceConfig(resourceName)); }
/** * Set the resource config * @param accessor Accessor to Helix configs * @param resource The resource name * @param resourceConfig The resource config to be set * @return True if set successfully, otherwise false */ private static boolean setResourceConfig(HelixDataAccessor accessor, String resource, ResourceConfig resourceConfig) { PropertyKey.Builder keyBuilder = accessor.keyBuilder(); return accessor.setProperty(keyBuilder.resourceConfig(resource), resourceConfig); }
private void verifyJobDeleted(String queueName, String jobName) throws Exception { HelixDataAccessor accessor = _manager.getHelixDataAccessor(); PropertyKey.Builder keyBuilder = accessor.keyBuilder(); Assert.assertNull(accessor.getProperty(keyBuilder.idealStates(jobName))); Assert.assertNull(accessor.getProperty(keyBuilder.resourceConfig(jobName))); TaskTestUtil.pollForEmptyJobState(_driver, queueName, jobName); }
private void verifyJobDeleted(String queueName, String jobName) throws Exception { HelixDataAccessor accessor = _manager.getHelixDataAccessor(); PropertyKey.Builder keyBuilder = accessor.keyBuilder(); Assert.assertNull(accessor.getProperty(keyBuilder.idealStates(jobName)), jobName + "'s idealstate has not been deleted!"); Assert.assertNull(accessor.getProperty(keyBuilder.resourceConfig(jobName)), jobName + "'s resourceConfig has not been deleted!"); TaskTestUtil.pollForEmptyJobState(_driver, queueName, jobName); }