Before this commit offset consumer was able to fetch offset just from one partition.
This commit solve this.
Change-Id: I2c2c300219e43ab422b237094ad775ca8795169e
Issue-ID: DCAEGEN2-1783
Signed-off-by: kjaniak <kornel.janiak@nokia.com>
private const val defaultUpdateInterval = 500L
private val defaultPollTimeoutMs: Duration = Duration.ofMillis(10L)
}
-}
\ No newline at end of file
+}
private val topics: Set<String>,
private val metrics: Metrics,
private val dispatcher: CoroutineDispatcher = Dispatchers.IO)
- : MetricsKafkaConsumer{
+ : MetricsKafkaConsumer {
- override suspend fun start(updateInterval: Long, pollTimeout: Duration):Job =
+ override suspend fun start(updateInterval: Long, pollTimeout: Duration): Job =
GlobalScope.launch(dispatcher) {
- kafkaConsumer.assign(topics.map { TopicPartition(it, 0) })
- while (isActive) {
- val topicPartitions = kafkaConsumer.assignment()
+ val topicPartitions = topics.flatMap {
+ listOf(TopicPartition(it, 0), TopicPartition(it, 1), TopicPartition(it, 2))
+ }
+ kafkaConsumer.assign(topicPartitions)
- kafkaConsumer.endOffsets(topicPartitions)
- .forEach { (topicPartition, offset) ->
- update(topicPartition, offset)
- }
- kafkaConsumer.commitSync()
- delay(updateInterval)
- }
+ while (isActive) {
+ kafkaConsumer.endOffsets(kafkaConsumer.assignment())
+ .forEach { (topicPartition, offset) ->
+ update(topicPartition, offset)
+ }
+ kafkaConsumer.commitSync()
+ delay(updateInterval)
+ }
}
private fun update(topicPartition: TopicPartition, offset: Long) {
private val registry: PrometheusMeterRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
) : Metrics {
- private val currentOffsetByTopicPartition = { topic: String ->
- registry.gauge(name(CONSUMER, OFFSET, TOPIC), listOf(Tag.of(TOPIC, topic)), AtomicLong(0))
+ private val currentOffsetByTopicPartition = { topicPartition: String ->
+ registry.gauge(name(OFFSET, PARTITION, topicPartition.toLowerCase()),
+ listOf(Tag.of(PARTITION, topicPartition)),
+ AtomicLong(0))
}.memoize<String, AtomicLong>()
private val travelTime = Timer.builder(name(TRAVEL,TIME))
companion object {
val INSTANCE by lazy { MicrometerMetrics() }
- private const val CONSUMER = "consumer"
private const val OFFSET = "offset"
- private const val TOPIC = "topic"
+ private const val PARTITION = "partition"
private const val TRAVEL = "travel"
private const val TIME = "time"
private const val PREFIX = "hv-kafka-consumer"
}
}
- given("two topics with partition") {
+ given("two topics with one partition each") {
val topics = setOf(topicName1, topicName2)
val kafkaSource = OffsetKafkaConsumer(mockedKafkaConsumer, topics, mockedMetrics, testDispatcher)
val offsetKafkaConsumer = OffsetKafkaConsumer(mockedKafkaConsumer, emptyTopics, mockedMetrics, testDispatcher)
on("call of function start") {
- val emptyTopicPartitions = setOf(null)
+ val emptyTopicPartitions = emptySet<TopicPartition>()
whenever(mockedKafkaConsumer.assignment()).thenReturn(emptyTopicPartitions)
whenever(mockedKafkaConsumer.endOffsets(emptyTopicPartitions))
.thenReturn(emptyMap())
}
describe("Gauges") {
- val gaugeName = "$PREFIX.consumer.offset.topic"
+ val gaugeName1 = "$PREFIX.offset.partition.sample_topic-0"
+ val gaugeName2 = "$PREFIX.offset.partition.sample_topic-1"
+ val offset1 = 966L
+ val offset2 = 967L
+ val topicPartition1 = TopicPartition("sample_topic", 0)
+ val topicPartition2 = TopicPartition("sample_topic", 1)
on("notifyOffsetChanged") {
- val offset = 966L
- val topicPartition = TopicPartition("sample_topic", 1)
+ it("should update $gaugeName1") {
+ cut.notifyOffsetChanged(offset1, topicPartition1)
- it("should update $gaugeName") {
- cut.notifyOffsetChanged(offset, topicPartition)
+ registry.verifyGauge(gaugeName1) {
+ assertThat(it.value()).isCloseTo(offset1.toDouble(), doublePrecision)
+ }
+ }
+ }
+
+ on("two partition update") {
+ it("should update $gaugeName1") {
+ cut.notifyOffsetChanged(offset1, topicPartition1)
+ cut.notifyOffsetChanged(offset2, topicPartition2)
+
+ registry.verifyGauge(gaugeName1) {
+ assertThat(it.value()).isCloseTo(offset1.toDouble(), doublePrecision)
+ }
- registry.verifyGauge(gaugeName) {
- assertThat(it.value()).isCloseTo(offset.toDouble(), doublePrecision)
+ registry.verifyGauge(gaugeName2) {
+ assertThat(it.value()).isCloseTo(offset2.toDouble(), doublePrecision)
}
}
}
echo "./cloud-based-performance-test.sh setup"
echo "./cloud-based-performance-test.sh start"
echo "./cloud-based-performance-test.sh start --containers 10"
- echo "./cloud-based-performance-test.sh start --containers 10"
+ echo "./cloud-based-performance-test.sh start --properties-file ~/other_test.properties"
echo "./cloud-based-performance-test.sh clean"
exit 1
}
# CONSUMER CONFIGURATION
# Addresses of Kafka services to consume from
-consumer.kafka.bootstrapServers=message-router-kafka-0:9093,message-router-kafka-1:9093,message-router-kafka-2:9093
+consumer.kafka.bootstrapServers=message-router-kafka:9092
# Kafka topics to subscribe to
consumer.kafka.topics=HV_VES_PERF3GPP
"tableColumn": "",
"targets": [
{
- "expr": "hv_kafka_consumer_consumer_offset_topic",
+ "expr": "hv_kafka_consumer_offset_partition_hv_ves_perf3gpp_0",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
"thresholds": "",
"timeFrom": null,
"timeShift": null,
- "title": "Current offset",
+ "title": "Current offset on partition 0",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
Code Review / dcaegen2 / collectors / hv-ves.git / commitdiff