@Override public void close() { // The executor is a single thread that is tied to this worker. Once the worker shuts down // the executor will stop. worker.shutdown(); }
/** * Start consuming data from the stream, and pass it to the application record processors. */ public void run() { if (shutdown) { return; } try { initialize(); LOG.info("Initialization complete. Starting worker loop."); } catch (RuntimeException e1) { LOG.error("Unable to initialize after " + MAX_INITIALIZATION_ATTEMPTS + " attempts. Shutting down.", e1); shutdown(); } while (!shouldShutdown()) { runProcessLoop(); } finalShutdown(); LOG.info("Worker loop is complete. Exiting from worker."); }
@Override public Integer call() throws Exception { int exitCode = 0; try { worker.run(); } catch (Throwable t) { LOG.error("Caught throwable while processing data.", t); exitCode = 1; } return exitCode; }
boolean foundCompletedShard = false; Set<ShardInfo> assignedShards = new HashSet<>(); for (ShardInfo shardInfo : getShardInfoForAssignments()) { ShardConsumer shardConsumer = createOrGetShardConsumer(shardInfo, recordProcessorFactory); if (shardConsumer.isShutdown() && shardConsumer.getShutdownReason().equals(ShutdownReason.TERMINATE)) { foundCompletedShard = true; cleanupShardConsumers(assignedShards);
/** * NOTE: This method is internal/private to the Worker class. It has package access solely for testing. * * @param shardInfo * Kinesis shard info * @param processorFactory * RecordProcessor factory * @return ShardConsumer for the shard */ ShardConsumer createOrGetShardConsumer(ShardInfo shardInfo, IRecordProcessorFactory processorFactory) { ShardConsumer consumer = shardInfoShardConsumerMap.get(shardInfo); // Instantiate a new consumer if we don't have one, or the one we // had was from an earlier // lease instance (and was shutdown). Don't need to create another // one if the shard has been // completely processed (shutdown reason terminate). if ((consumer == null) || (consumer.isShutdown() && consumer.getShutdownReason().equals(ShutdownReason.ZOMBIE))) { consumer = buildConsumer(shardInfo, processorFactory); shardInfoShardConsumerMap.put(shardInfo, consumer); wlog.infoForce("Created new shardConsumer for : " + shardInfo); } return consumer; }
@Override public void run() { if (worker != null) { // Start Amazon Kinesis Client Library worker to process records LOG.info("Starting worker in " + getClass().getSimpleName()); try { worker.run(); } catch (Throwable t) { LOG.error(t); throw t; } finally { LOG.error("Worker " + getClass().getSimpleName() + " is not running."); } } else { throw new RuntimeException("Initialize must be called before run."); } }
public int run() throws Exception { configure(); System.out.println(String.format("Starting %s", appName)); LOG.info(String.format("Running %s to process stream %s", appName, streamName)); IRecordProcessorFactory recordProcessorFactory = new AggregatorProcessorFactory( aggGroup); worker = new Worker(recordProcessorFactory, this.config); int exitCode = 0; int failures = 0; // run the worker, tolerating as many failures as is configured while (failures < failuresToTolerate || failuresToTolerate == -1) { try { worker.run(); } catch (Throwable t) { LOG.error("Caught throwable while processing data.", t); failures++; if (failures < failuresToTolerate) { LOG.error("Restarting..."); } else { shutdown(); } exitCode = 1; } } return exitCode; }
public void stop() { worker.shutdown(); }
@Override public void run() { if (worker != null) { // Start Amazon Kinesis Client Library worker to process records LOG.info("Starting worker in " + getClass().getSimpleName()); try { worker.run(); } catch (Throwable t) { LOG.error(t); throw t; } finally { LOG.error("Worker " + getClass().getSimpleName() + " is not running."); } } else { throw new RuntimeException("Initialize must be called before run."); } }
return new Worker(factory, kclConfig, streamsAdapterClient, kclDynamoDBClient, kclCloudWatchClient);
@Override public void run() { try { String workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID(); KinesisClientLibConfiguration kinesisClientLibConfiguration = new KinesisClientLibConfiguration(APPLICATION_NAME, kdsStreamName, credentialsProvider, workerId); kinesisClientLibConfiguration.withInitialPositionInStream(SAMPLE_APPLICATION_INITIAL_POSITION_IN_STREAM).withRegionName(region.getName()); final IRecordProcessorFactory recordProcessorFactory = () -> new KinesisRecordProcessor(rekognizedFragmentsIndex, credentialsProvider); final Worker worker = new Worker(recordProcessorFactory, kinesisClientLibConfiguration); System.out.printf("Running %s to process stream %s as worker %s...", APPLICATION_NAME, kdsStreamName, workerId); int exitCode = 0; try { worker.run(); } catch (Throwable t) { System.err.println("Caught throwable while processing data."); t.printStackTrace(); exitCode = 1; } System.out.println("Exit code : " + exitCode); } catch (Exception e) { e.printStackTrace(); } }
public void stop() { worker.shutdown(); }
/** * Command line main method entry point * * @param args * command line arguments */ public static void main(String[] args) { try { final Optional<Worker> workerOption = mainUnsafe(args); if (!workerOption.isPresent()) { return; } System.out.println("Starting replication now, check logs for more details."); workerOption.get().run(); } catch (ParameterException e) { log.error(e); JCommander.getConsole().println(e.toString()); System.exit(StatusCodes.EINVAL); } catch (Exception e) { log.fatal(e); JCommander.getConsole().println(e.toString()); System.exit(StatusCodes.EINVAL); } }
return new Worker(config.getApplicationName(), recordProcessorFactory, config,
@Override protected void doStop() throws FlumeException { worker.shutdown(); executor.shutdownNow(); }
public void start() { int mb = 1024 * 1024; LOG.info("Max memory: {} mb", Runtime.getRuntime().maxMemory() / mb); LOG.info("Starting up Kinesis Consumer... (may take a few seconds)"); AmazonKinesisClient kinesisClient = new AmazonKinesisClient(kinesisCfg.getKinesisCredentialsProvider(), kinesisCfg.getKinesisClientConfiguration()); AmazonDynamoDBClient dynamoDBClient = new AmazonDynamoDBClient(kinesisCfg.getDynamoDBCredentialsProvider(), kinesisCfg.getDynamoDBClientConfiguration()); AmazonCloudWatch cloudWatchClient = new AmazonCloudWatchClient(kinesisCfg.getCloudWatchCredentialsProvider(), kinesisCfg.getCloudWatchClientConfiguration()); Worker worker = new Worker.Builder() .recordProcessorFactory(() -> new RecordProcessor(unitOfWorkListener, exceptionStrategy, metricsCallback, dry)) .config(kinesisCfg) .kinesisClient(kinesisClient) .dynamoDBClient(dynamoDBClient) .cloudWatchClient(cloudWatchClient) .build(); worker.run(); }