/** * Build the image dir. This path is relative and only valid at the far end * @param instance instance options * @param bindir bin subdir * @param script script in bin subdir * @return the path to the script * @throws FileNotFoundException if a file is not found, or it is not a directory* */ public String buildPathToScript(AggregateConf instance, String bindir, String script) throws FileNotFoundException { return buildPathToScript(instance.getInternalOperations(), bindir, script); } /**
/** * Build the image dir. This path is relative and only valid at the far end * @param instance instance options * @param bindir bin subdir * @param script script in bin subdir * @return the path to the script * @throws FileNotFoundException if a file is not found, or it is not a directory* */ public String buildPathToScript(AggregateConf instance, String bindir, String script) throws FileNotFoundException { return buildPathToScript(instance.getInternalOperations(), bindir, script); } /**
/** * Get the global internal options for the AM * @return a map to access the internals */ public MapOperations getGlobalInternalOptions() { return getInstanceDefinition() .getInternalOperations(). getGlobalOptions(); }
/** * probe to see if accumulo has already been installed. * @param cd cluster description * @return true if the relevant data directory looks inited * @throws IOException IO problems */ private boolean isInited(AggregateConf cd) throws IOException, BadConfigException { String dataDir = cd.getInternalOperations() .getGlobalOptions() .getMandatoryOption( InternalKeys.INTERNAL_DATA_DIR_PATH); Path accumuloInited = new Path(dataDir, INSTANCE_ID); FileSystem fs2 = FileSystem.get(accumuloInited.toUri(), getConf()); return fs2.exists(accumuloInited); }
/** * Get the global internal options for the AM * @return a map to access the internals */ public MapOperations getGlobalInternalOptions() { return getInstanceDefinition() .getInternalOperations(). getGlobalOptions(); }
@Override public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem, Configuration serviceConf, AbstractLauncher launcher, AggregateConf instanceDefinition, Path snapshotConfDirPath, Path generatedConfDirPath, Configuration clientConfExtras, String libdir, Path tempPath, boolean miniClusterTestRun) throws IOException, SliderException { String agentImage = instanceDefinition.getInternalOperations(). get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); if (SliderUtils.isUnset(agentImage)) { Path agentPath = new Path(tempPath.getParent(), AgentKeys.PROVIDER_AGENT); log.info("Automatically uploading the agent tarball at {}", agentPath); fileSystem.getFileSystem().mkdirs(agentPath); if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) { instanceDefinition.getInternalOperations().set( InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, new Path(agentPath, AGENT_TAR).toUri()); } } }
@Override public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem, Configuration serviceConf, AbstractLauncher launcher, AggregateConf instanceDefinition, Path snapshotConfDirPath, Path generatedConfDirPath, Configuration clientConfExtras, String libdir, Path tempPath, boolean miniClusterTestRun) throws IOException, SliderException { String agentImage = instanceDefinition.getInternalOperations(). get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); if (SliderUtils.isUnset(agentImage)) { Path agentPath = new Path(tempPath.getParent(), AgentKeys.PROVIDER_AGENT); log.info("Automatically uploading the agent tarball at {}", agentPath); fileSystem.getFileSystem().mkdirs(agentPath); if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) { instanceDefinition.getInternalOperations().set( InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, new Path(agentPath, AGENT_TAR).toUri()); } } }
/** * Set up the image/app home path * @param appImage path in the DFS to the tar file * @param appHomeDir other strategy: home dir * @throws BadConfigException if both are found */ public void setImageDetailsIfAvailable( Path appImage, String appHomeDir) throws BadConfigException { boolean appHomeUnset = SliderUtils.isUnset(appHomeDir); // App home or image if (appImage != null) { if (!appHomeUnset) { // both args have been set throw new BadConfigException( ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED); } instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_IMAGE_PATH, appImage.toUri()); } else { // the alternative is app home, which now MUST be set if (!appHomeUnset) { instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_HOME, appHomeDir); } } }
/** * Set up the image/app home path * @param appImage path in the DFS to the tar file * @param appHomeDir other strategy: home dir * @throws BadConfigException if both are found */ public void setImageDetailsIfAvailable( Path appImage, String appHomeDir) throws BadConfigException { boolean appHomeUnset = SliderUtils.isUnset(appHomeDir); // App home or image if (appImage != null) { if (!appHomeUnset) { // both args have been set throw new BadConfigException( ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED); } instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_IMAGE_PATH, appImage.toUri()); } else { // the alternative is app home, which now MUST be set if (!appHomeUnset) { instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_HOME, appHomeDir); } } }
/** * Set the queue used to start the application * @param queue * @throws BadConfigException */ public void setQueue(String queue) throws BadConfigException { if(queue != null) { if(SliderUtils.isUnset(queue)) { throw new BadConfigException("Queue value cannot be empty."); } instanceDescription.getInternalOperations().set(INTERNAL_QUEUE, queue); } }
/** * Set the queue used to start the application * @param queue * @throws BadConfigException */ public void setQueue(String queue) throws BadConfigException { if(queue != null) { if(SliderUtils.isUnset(queue)) { throw new BadConfigException("Queue value cannot be empty."); } instanceDescription.getInternalOperations().set(INTERNAL_QUEUE, queue); } }
@Override //Client public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem, String clustername, Configuration configuration, AggregateConf instanceDefinition, Path clusterDirPath, Path generatedConfDirPath, boolean secure) throws SliderException, IOException { super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure); //add a check for the directory being writeable by the current user String dataPath = instanceDefinition.getInternalOperations() .getGlobalOptions() .getMandatoryOption( InternalKeys.INTERNAL_DATA_DIR_PATH); Path path = new Path(dataPath); sliderFileSystem.verifyDirectoryWriteAccess(path); Path historyPath = new Path(clusterDirPath, SliderKeys.HISTORY_DIR_NAME); sliderFileSystem.verifyDirectoryWriteAccess(historyPath); }
@Override //Client public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem, String clustername, Configuration configuration, AggregateConf instanceDefinition, Path clusterDirPath, Path generatedConfDirPath, boolean secure) throws SliderException, IOException { super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure); //add a check for the directory being writeable by the current user String dataPath = instanceDefinition.getInternalOperations() .getGlobalOptions() .getMandatoryOption( InternalKeys.INTERNAL_DATA_DIR_PATH); Path path = new Path(dataPath); sliderFileSystem.verifyDirectoryWriteAccess(path); Path historyPath = new Path(clusterDirPath, SliderKeys.HISTORY_DIR_NAME); sliderFileSystem.verifyDirectoryWriteAccess(historyPath); }
/** * Build the image dir. This path is relative and only valid at the far end * @param instanceDefinition instance definition * @param bindir bin subdir * @param script script in bin subdir * @return the path to the script * @throws FileNotFoundException if a file is not found, or it is not a directory* */ public String buildPathToHomeDir(AggregateConf instanceDefinition, String bindir, String script) throws FileNotFoundException, BadConfigException { MapOperations globalOptions = instanceDefinition.getInternalOperations().getGlobalOptions(); String applicationHome = globalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME); String imagePath = globalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); return buildPathToHomeDir(imagePath, applicationHome, bindir, script); }
private void actionDiagnosticSlider(ActionDiagnosticArgs diagnosticArgs) throws YarnException, IOException { // not using member variable clustername because we want to place // application name after --application option and member variable // cluster name has to be put behind action String clusterName = diagnosticArgs.name; if(isUnset(clusterName)){ throw new BadCommandArgumentsException("application name must be provided with --name option"); } AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName); String imagePath = instanceDefinition.getInternalOperations().get( INTERNAL_APPLICATION_IMAGE_PATH); // if null, it will be uploaded by Slider and thus at slider's path if (imagePath == null) { ApplicationReport appReport = findInstance(clusterName); if (appReport != null) { Path path1 = sliderFileSystem.getTempPathForCluster(clusterName); Path subPath = new Path(path1, appReport.getApplicationId().toString() + "/agent"); imagePath = subPath.toString(); } } log.info("The path of slider agent tarball on HDFS is: " + imagePath); }
private void actionDiagnosticSlider(ActionDiagnosticArgs diagnosticArgs) throws YarnException, IOException { // not using member variable clustername because we want to place // application name after --application option and member variable // cluster name has to be put behind action String clusterName = diagnosticArgs.name; if(isUnset(clusterName)){ throw new BadCommandArgumentsException("application name must be provided with --name option"); } AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName); String imagePath = instanceDefinition.getInternalOperations().get( INTERNAL_APPLICATION_IMAGE_PATH); // if null, it will be uploaded by Slider and thus at slider's path if (imagePath == null) { ApplicationReport appReport = findInstance(clusterName); if (appReport != null) { Path path1 = sliderFileSystem.getTempPathForCluster(clusterName); Path subPath = new Path(path1, appReport.getApplicationId().toString() + "/agent"); imagePath = subPath.toString(); } } log.info("The path of slider agent tarball on HDFS is: " + imagePath); }
/** * Build the image dir. This path is relative and only valid at the far end * @param instanceDefinition instance definition * @param bindir bin subdir * @param script script in bin subdir * @return the path to the script * @throws FileNotFoundException if a file is not found, or it is not a directory* */ public String buildPathToHomeDir(AggregateConf instanceDefinition, String bindir, String script) throws FileNotFoundException, BadConfigException { MapOperations globalOptions = instanceDefinition.getInternalOperations().getGlobalOptions(); String applicationHome = globalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME); String imagePath = globalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); return buildPathToHomeDir(imagePath, applicationHome, bindir, script); }
/** * Build the accumulo-site.xml file * This the configuration used by Accumulo directly * @param instanceDescription this is the cluster specification used to define this * @return a map of the dynamic bindings for this Slider instance */ public Map<String, String> buildSiteConfFromInstance( AggregateConf instanceDescription) throws BadConfigException { ConfTreeOperations appconf = instanceDescription.getAppConfOperations(); MapOperations globalAppOptions = appconf.getGlobalOptions(); MapOperations globalInstanceOptions = instanceDescription.getInternalOperations().getGlobalOptions(); Map<String, String> sitexml = new HashMap<String, String>(); providerUtils.propagateSiteOptions(globalAppOptions, sitexml); propagateClientFSBinding(sitexml); setDatabasePath(sitexml, globalInstanceOptions.getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH)); String quorum = globalAppOptions.getMandatoryOption(OptionKeys.ZOOKEEPER_QUORUM); sitexml.put(AccumuloConfigFileOptions.ZOOKEEPER_HOST, quorum); return sitexml; }
ConfTreeOperations template = ConfTreeOperations.fromResource(internalTemplate); instanceConf.getInternalOperations() .mergeWithoutOverwrite(template.confTree);
aggregateConf.getInternalOperations().getGlobalOptions(); MapOperations appOptions = aggregateConf.getAppConfOperations().getGlobalOptions();