gerrit.onap Code Review - ccsdk/cds.git/blob

   1 /*
   2  * Copyright © 2018-2019 AT&T Intellectual Property.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 package org.onap.ccsdk.cds.blueprintsprocessor.functions.message.prioritization.topology
  18
  19 import org.apache.kafka.streams.processor.Cancellable
  20 import org.apache.kafka.streams.processor.ProcessorContext
  21 import org.apache.kafka.streams.processor.PunctuationType
  22 import org.apache.kafka.streams.processor.To
  23 import org.onap.ccsdk.cds.blueprintsprocessor.functions.message.prioritization.AbstractMessagePrioritizeProcessor
  24 import org.onap.ccsdk.cds.blueprintsprocessor.functions.message.prioritization.MessagePrioritizationConstants
  25 import org.onap.ccsdk.cds.blueprintsprocessor.functions.message.prioritization.MessageState
  26 import org.onap.ccsdk.cds.blueprintsprocessor.functions.message.prioritization.db.MessagePrioritization
  27 import org.onap.ccsdk.cds.blueprintsprocessor.functions.message.prioritization.utils.MessageCorrelationUtils
  28 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
  29 import org.onap.ccsdk.cds.controllerblueprints.core.logger
  30 import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
  31 import java.time.Duration
  32 import java.util.*
  33
  34
  35 open class MessagePrioritizeProcessor : AbstractMessagePrioritizeProcessor<ByteArray, ByteArray>() {
  36
  37     private val log = logger(MessagePrioritizeProcessor::class)
  38
  39     lateinit var expiryCancellable: Cancellable
  40     lateinit var cleanCancellable: Cancellable
  41
  42     override suspend fun processNB(key: ByteArray, value: ByteArray) {
  43         log.info("***** received in prioritize processor key(${String(key)})")
  44         val messagePrioritize = JacksonUtils.readValue(String(value), MessagePrioritization::class.java)
  45                 ?: throw BluePrintProcessorException("failed to convert")
  46         try {
  47             // Save the Message
  48             messagePrioritizationStateService.saveMessage(messagePrioritize)
  49             handleCorrelationAndNextStep(messagePrioritize)
  50         } catch (e: Exception) {
  51             messagePrioritize.error = "failed in Prioritize message(${messagePrioritize.id}) : ${e.message}"
  52             log.error(messagePrioritize.error)
  53             /** Update the data store */
  54             messagePrioritizationStateService.setMessageStateANdError(messagePrioritize.id, MessageState.ERROR.name,
  55                     messagePrioritize.error!!)
  56             /** Publish to Output topic */
  57             this.processorContext.forward(messagePrioritize.id, messagePrioritize,
  58                     To.child(MessagePrioritizationConstants.SINK_OUTPUT))
  59         }
  60     }
  61
  62     override fun init(context: ProcessorContext) {
  63         super.init(context)
  64         /** set up expiry marking cron */
  65         initializeExpiryPunctuator()
  66         /** Set up cleaning records cron */
  67         initializeCleanPunctuator()
  68     }
  69
  70     override fun close() {
  71         log.info("closing prioritization processor applicationId(${processorContext.applicationId()}), " +
  72                 "taskId(${processorContext.taskId()})")
  73         expiryCancellable.cancel()
  74         cleanCancellable.cancel()
  75     }
  76
  77     open fun initializeExpiryPunctuator() {
  78         val expiryPunctuator = MessagePriorityExpiryPunctuator(messagePrioritizationStateService)
  79         expiryPunctuator.processorContext = processorContext
  80         expiryPunctuator.configuration = prioritizationConfiguration
  81         val expiryConfiguration = prioritizationConfiguration.expiryConfiguration
  82         expiryCancellable = processorContext.schedule(Duration.ofMillis(expiryConfiguration.frequencyMilli),
  83                 PunctuationType.WALL_CLOCK_TIME, expiryPunctuator)
  84         log.info("Expiry punctuator setup complete with frequency(${expiryConfiguration.frequencyMilli})mSec")
  85     }
  86
  87     open fun initializeCleanPunctuator() {
  88         val cleanPunctuator = MessagePriorityCleanPunctuator(messagePrioritizationStateService)
  89         cleanPunctuator.processorContext = processorContext
  90         cleanPunctuator.configuration = prioritizationConfiguration
  91         val cleanConfiguration = prioritizationConfiguration.cleanConfiguration
  92         cleanCancellable = processorContext.schedule(Duration.ofDays(cleanConfiguration.expiredRecordsHoldDays.toLong()),
  93                 PunctuationType.WALL_CLOCK_TIME, cleanPunctuator)
  94         log.info("Clean punctuator setup complete with expiry " +
  95                 "hold(${cleanConfiguration.expiredRecordsHoldDays})days")
  96     }
  97
  98     open suspend fun handleCorrelationAndNextStep(messagePrioritization: MessagePrioritization) {
  99         /** Check correlation enabled and correlation field has populated */
 100         if (!messagePrioritization.correlationId.isNullOrBlank()) {
 101             val id = messagePrioritization.id
 102             val group = messagePrioritization.group
 103             val correlationId = messagePrioritization.correlationId!!
 104             val types = getGroupCorrelationTypes(messagePrioritization)
 105             log.info("checking correlation for message($id), group($group), types($types), " +
 106                     "correlation id($correlationId)")
 107
 108             /** Get all previously received messages from database for group and optional types and correlation Id */
 109             val waitingCorrelatedStoreMessages = messagePrioritizationStateService.getCorrelatedMessages(group,
 110                     arrayListOf(MessageState.NEW.name, MessageState.WAIT.name), types, correlationId)
 111
 112             /** If multiple records found, then check correlation */
 113             if (!waitingCorrelatedStoreMessages.isNullOrEmpty() && waitingCorrelatedStoreMessages.size > 1) {
 114                 /** Check all correlation satisfies */
 115                 val correlationResults = MessageCorrelationUtils
 116                         .correlatedMessagesWithTypes(waitingCorrelatedStoreMessages, types)
 117
 118                 if (correlationResults.correlated) {
 119                     /** Correlation  satisfied */
 120                     val correlatedIds = waitingCorrelatedStoreMessages.map { it.id }.joinToString(",")
 121                     /**  Send only correlated ids to next processor */
 122                     this.processorContext.forward(UUID.randomUUID().toString(), correlatedIds,
 123                             To.child(MessagePrioritizationConstants.PROCESSOR_AGGREGATE))
 124                 } else {
 125                     /** Correlation not satisfied */
 126                     log.trace("correlation not matched : ${correlationResults.message}")
 127                     val waitMessageIds = waitingCorrelatedStoreMessages.map { it.id }
 128                     // Update the Message state to Wait
 129                     messagePrioritizationStateService.setMessagesState(waitMessageIds, MessageState.WAIT.name)
 130                 }
 131             } else {
 132                 /** received first message of group and correlation Id, update the message with wait state */
 133                 messagePrioritizationStateService.setMessageState(messagePrioritization.id, MessageState.WAIT.name)
 134             }
 135         } else {
 136             // No Correlation check needed, simply forward to next processor.
 137             messagePrioritizationStateService.setMessageState(messagePrioritization.id, MessageState.PRIORITIZED.name)
 138             this.processorContext.forward(messagePrioritization.id, messagePrioritization.id,
 139                     To.child(MessagePrioritizationConstants.PROCESSOR_AGGREGATE))
 140         }
 141     }
 142
 143     /** If consumer wants specific correlation with respect to group and types, then populate the specific types,
 144      * otherwise correlation happens with group and correlationId */
 145     open fun getGroupCorrelationTypes(messagePrioritization: MessagePrioritization): List<String>? {
 146         return null
 147     }
 148 }