public static String resourceId(String url, JobContext job) { return String.format("%s-%s", job.getId(), url); }
@Override public void init(JobContext context, String pluginId) throws SparklerException { super.init(context, pluginId); SparklerConfiguration conf = context.getConfiguration(); if (conf.containsKey(Constants.key.FETCHER_USER_AGENTS)) { Object agents = conf.get(Constants.key.FETCHER_USER_AGENTS);
@Override public void init(JobContext context, String pluginId) throws SparklerException { super.init(context, pluginId); SparklerConfiguration config = jobContext.getConfiguration(); //TODO should change everywhere pluginConfig = config.getPluginConfiguration(pluginId); driver = createBrowserInstance(); }
@Override public void init(JobContext context, String pluginId) throws SparklerException { super.init(context, pluginId); try { SparklerConfiguration config = jobContext.getConfiguration(); LinkedHashMap pluginConfig = config.getPluginConfiguration(pluginId); Reader reader = getRulesReader(pluginConfig); this.rules = readRules(reader); } catch (IOException e) { throw new RuntimeException(e); } }
public static String resourceId(String url, JobContext job, Date timestamp) { return StringUtil.sha256hash(String.format("%s-%s-%s", job.getId(), url, timestamp.getTime())); }