@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
this.rmResourceFetcher = new RMResourceFetcher(crawlConfig.endPointConfig.RMBasePaths);
this.partitionId = calculatePartitionId(context);
if (partitionId < 0 || partitionId > crawlConfig.controlConfig.numTotalPartitions) {
throw new IllegalStateException("partitionId should be less than numTotalPartitions with partitionId "
+ partitionId + " and numTotalPartitions " + crawlConfig.controlConfig.numTotalPartitions);
}
Class<? extends JobIdPartitioner> partitionerCls = crawlConfig.controlConfig.partitionerCls;
try {
this.jobFilter = new JobIdFilterByPartition(partitionerCls.newInstance(),
crawlConfig.controlConfig.numTotalPartitions, partitionId);
} catch (Exception e) {
LOG.error("failing instantiating job partitioner class " + partitionerCls.getCanonicalName());
throw new IllegalStateException(e);
}
this.collector = collector;
this.runningJobManager = new RunningJobManager(crawlConfig.zkStateConfig.zkQuorum,
crawlConfig.zkStateConfig.zkSessionTimeoutMs,
crawlConfig.zkStateConfig.zkRetryTimes,
crawlConfig.zkStateConfig.zkRetryInterval,
crawlConfig.zkStateConfig.zkRoot,
crawlConfig.zkStateConfig.zkLockPath);
this.lastFinishAppTime = this.runningJobManager.recoverLastFinishedTime(partitionId);
if (this.lastFinishAppTime == 0L) {
this.lastFinishAppTime = Calendar.getInstance().getTimeInMillis() - 24 * 60 * 60000L;
this.runningJobManager.updateLastFinishTime(partitionId, this.lastFinishAppTime);
}
}