private Properties initKafkaProperties(StreamingConfig conf) throws StreamingException { Properties kafkaProperties = new Properties(); kafkaProperties.put(KafkaConfig.KAFKA_CON_ZK_CONNECT, conf.getStringValue(StreamingConfig.OPERATOR_KAFKA_ZOOKEEPERS)); kafkaProperties.put(KafkaConfig.KAFKA_GROUP_ID, conf.getStringValue(StreamingConfig.OPERATOR_KAFKA_GROUPID)); kafkaProperties.put(KafkaConfig.KAFKA_SERIAL_CLASS, conf.getStringValue(StreamingConfig.OPERATOR_KAFKA_MESSAGESERIALIZERCLASS)); kafkaProperties.put(KafkaConfig.KAFKA_SESSION_TIME, conf.getStringValue(StreamingConfig.OPERATOR_KAFKA_ZKSESSIONTIMEOUT)); kafkaProperties.put(KafkaConfig.KAFKA_SYNC_TIME, conf.getStringValue(StreamingConfig.OPERATOR_KAFKA_ZKSYNCTIME)); if (conf.getBooleanValue(StreamingConfig.OPERATOR_KAFKA_READ_FROMBEGINNING)) { kafkaProperties.put(KafkaConfig.KAFKA_OFFSET_RESET, OffsetRequest.SmallestTimeString()); } else { kafkaProperties.put(KafkaConfig.KAFKA_OFFSET_RESET, OffsetRequest.LargestTimeString()); } return kafkaProperties; }
public void initOffset(boolean isNew) { if (LOGGER.isInfoEnabled()) LOGGER.info( "Init offset for " + m_clientId); if (isNew) { m_readOffset = fetchResetOffset(SmallestTimeString()); // commit the init offset first, useful to revert commitOffset(); } else { if (offsetExists()) m_readOffset = readZkOffset(); if (m_readOffset < 0) { m_readOffset = fetchResetOffset(m_config.getAutoOffsetReset()); // commit the init offset first, useful to revert commitOffset(); } } if (m_readOffset < 0) throw new RuntimeException( "Fatal errors in getting init offset for " + m_clientId); if (LOGGER.isInfoEnabled()) LOGGER.info( m_clientId + ": init offset is set to " + m_readOffset); }
private long fetchResetOffset(String reset) { if (reset != null && reset.equals(SmallestTimeString())) time = EarliestTime();
long smallest = -1L, largest = -1L; try { smallest = fetchResetOffset(SmallestTimeString()); largest = fetchResetOffset(LargestTimeString()); } catch (Exception e) {