From 9202b5b16450c05bbe28a97b1699bddf3f28b258 Mon Sep 17 00:00:00 2001 From: ToineSiebelink Date: Mon, 28 Apr 2025 09:15:06 +0100 Subject: [PATCH] Control start-up delay: set host-names explicitly in docker compose - Use sequenced hostnames for cps-ncmp in docker compose based on a common template - Use explicit port definitions for each instance instead of range (instrumentation/Grafana) - Calculated startup delay as sequence number times 1 second when correct pattern used - Fall back on hash code modulus 10 seconds otherwise - clear logging which algorithm is used, why and outcome - update kpi and endurance environment files to use correct instance names Issue-ID:CPS-2752 Change-Id: Ie788aff60d6e9c470136b2a2051d0c5ccc173e9a Signed-off-by: ToineSiebelink --- .../src/main/java/org/onap/cps/Application.java | 2 +- .../cps/startup/InstanceStartupDelayManager.java | 38 ++++++++++------- .../startup/InstanceStartupDelayManagerSpec.groovy | 48 ++++++++++++++++------ docker-compose/config/nginx/nginx.conf | 5 ++- docker-compose/docker-compose.yml | 32 ++++++++++++--- docker-compose/env/endurance.env | 7 +++- docker-compose/env/kpi.env | 7 +++- 7 files changed, 102 insertions(+), 37 deletions(-) diff --git a/cps-application/src/main/java/org/onap/cps/Application.java b/cps-application/src/main/java/org/onap/cps/Application.java index d1c99bcc31..3c7750d528 100644 --- a/cps-application/src/main/java/org/onap/cps/Application.java +++ b/cps-application/src/main/java/org/onap/cps/Application.java @@ -40,7 +40,7 @@ public class Application { * @param args Command-line arguments passed to the application (not used in this implementation). */ public static void main(final String[] args) { - instanceStartupDelayManager.applyHostnameBasedStartupDelay(); + instanceStartupDelayManager.applyHostNameBasedStartupDelay(); log.info("Initializing Spring Application context..."); SpringApplication.run(Application.class, args); log.info("🚀 APPLICATION STARTED"); diff --git a/cps-application/src/main/java/org/onap/cps/startup/InstanceStartupDelayManager.java b/cps-application/src/main/java/org/onap/cps/startup/InstanceStartupDelayManager.java index 927c59f75d..d7baec7c08 100644 --- a/cps-application/src/main/java/org/onap/cps/startup/InstanceStartupDelayManager.java +++ b/cps-application/src/main/java/org/onap/cps/startup/InstanceStartupDelayManager.java @@ -23,27 +23,37 @@ package org.onap.cps.startup; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; @Slf4j public class InstanceStartupDelayManager { + private static final Pattern HOST_NAME_WITH_SEQUENCE_PATTERN = Pattern.compile(".*-([\\d][\\d]?)$"); /** - * Applies a consistent hash-based startup delay based on the host's name - * to avoid race conditions during schema migration. - * This method is useful in environments with multiple instances - * (e.g., Docker Compose, Kubernetes), where simultaneous Liquibase executions - * might result in conflicts. - * Delay logic: - * - A hash of the hostname is calculated. - * - The result is used to derive a delay up to 5000 milliseconds. - * - This provides a reasonably distributed delay across instances. + * Applies a consistent startup delay based on the host's name to avoid race conditions during liquibase set steps. + * This method is useful in environments with multiple instances. + * (e.g., Docker Compose, Kubernetes), where simultaneous Liquibase executions might result in conflicts. + * Delay calculation: + * - For host names that match {host-name}-{sequence-number} the delay wil be 1 second times the sequence number. + * - please note, the sequence number can be 2 digits at most. + * - For other names the delay is calculated as the hash code of that name modulus 10,000 ms i.e. up to 10,000 ms. */ - public void applyHostnameBasedStartupDelay() { + public void applyHostNameBasedStartupDelay() { try { - final String hostname = getHostName(); - final long startupDelayInMillis = Math.abs(hostname.hashCode() % 5_000L); - log.info("Startup delay applied for Hostname: {} | Delay: {} ms", hostname, startupDelayInMillis); + final String hostName = getHostName(); + log.info("Host name: {}", hostName); + final Matcher matcher = HOST_NAME_WITH_SEQUENCE_PATTERN.matcher(hostName); + final long startupDelayInMillis; + if (matcher.matches()) { + startupDelayInMillis = Integer.valueOf(matcher.group(1)) * 1_000L; + log.info("Sequenced host name detected, calculated delay = {} ms", startupDelayInMillis); + } else { + startupDelayInMillis = Math.abs(hostName.hashCode() % 10_000L); + log.warn("No Sequenced host name detected (-), hash-based delay = {} ms", + startupDelayInMillis); + } haveALittleSleepInMs(startupDelayInMillis); } catch (final InterruptedException e) { log.warn("Sleep interrupted, re-interrupting the thread"); @@ -60,4 +70,4 @@ public class InstanceStartupDelayManager { protected void haveALittleSleepInMs(final long timeInMs) throws InterruptedException { TimeUnit.MILLISECONDS.sleep(timeInMs); } -} \ No newline at end of file +} diff --git a/cps-application/src/test/groovy/org/onap/cps/startup/InstanceStartupDelayManagerSpec.groovy b/cps-application/src/test/groovy/org/onap/cps/startup/InstanceStartupDelayManagerSpec.groovy index 0ad02f4551..ad92322eb1 100644 --- a/cps-application/src/test/groovy/org/onap/cps/startup/InstanceStartupDelayManagerSpec.groovy +++ b/cps-application/src/test/groovy/org/onap/cps/startup/InstanceStartupDelayManagerSpec.groovy @@ -26,22 +26,45 @@ class InstanceStartupDelayManagerSpec extends Specification { def objectUnderTest = Spy(InstanceStartupDelayManager) - def 'Startup delay with real hostname.'() { - given: 'a hostname is resolved' - objectUnderTest.getHostName() >> 'hostX' - and: 'the expected delay is based on hash code with max of 5,000 ms' - def expectedDelay = Math.abs('hostX'.hashCode() % 5_000) + def 'Startup delay with sequenced host name with #scenario'() { + given: 'a sequenced host name' + objectUnderTest.getHostName() >> hostName + and: 'the expected delay is based on the sequence number' + def expectedDelay = expectedDelayInSeconds * 1_000; when: 'startup delay is called' - objectUnderTest.applyHostnameBasedStartupDelay() - then: 'the system will sleep for expected time' - 1 * objectUnderTest.haveALittleSleepInMs(expectedDelay) + objectUnderTest.applyHostNameBasedStartupDelay() + then: 'the system will sleep for expected number of seconds as defined by sequence number' + 1 * objectUnderTest.haveALittleSleepInMs(expectedDelay) >> { /* don't sleep for testing purposes */ } + where: ' following sequenced host names are used' + scenario | hostName || expectedDelayInSeconds + 'our usual host-name' | 'cps-and-ncmp-0' || 0 + 'dash and 1 digit at end' | 'host-1' || 1 + 'dash and 2 digits at end' | 'host-23' || 23 + 'digits in name' | 'host-2-34' || 34 + 'weird name ending in digits' | 't@st : - { " -56' || 56 } - def 'Startup delay when hostname cannot be resolved.'() { - given: 'an exception is thrown while getting the hostname' + def 'Startup delay with un-sequenced host name.'() { + given: 'a un-sequenced host name: #hostName' + objectUnderTest.getHostName() >> hostName + when: 'startup delay is called' + objectUnderTest.applyHostNameBasedStartupDelay() + then: 'the system will sleep for an expected time based on the hash' + 1 * objectUnderTest.haveALittleSleepInMs(expectedDelayBasedOnHashInMs) >> { /* don't sleep for testing purposes */ } + where: ' following un-sequenced host names are used' + hostName || expectedDelayBasedOnHashInMs + 'no_digits_at_all' || 784 + 'digits-12-in-the-middle' || 1484 + 'non-digit-after-digit-1a' || 753 + 'dash-after-digit-1-' || 7256 + 'three-digits-at-end-is-not-accepted-123' || 9941 + } + + def 'Startup delay when host name cannot be resolved.'() { + given: 'an exception is thrown while getting the host name' objectUnderTest.getHostName() >> { throw new Exception('some message') } when: 'startup delay is called' - objectUnderTest.applyHostnameBasedStartupDelay() + objectUnderTest.applyHostNameBasedStartupDelay() then: 'system will not sleep' 0 * objectUnderTest.haveALittleSleepInMs(_) } @@ -50,9 +73,10 @@ class InstanceStartupDelayManagerSpec extends Specification { given: 'sleep method throws InterruptedException' objectUnderTest.haveALittleSleepInMs(_) >> { throw new InterruptedException('some message') } when: 'startup delay is called' - objectUnderTest.applyHostnameBasedStartupDelay() + objectUnderTest.applyHostNameBasedStartupDelay() then: 'interrupt exception is ignored' noExceptionThrown() } } + diff --git a/docker-compose/config/nginx/nginx.conf b/docker-compose/config/nginx/nginx.conf index 435b860821..e6eab378c9 100644 --- a/docker-compose/config/nginx/nginx.conf +++ b/docker-compose/config/nginx/nginx.conf @@ -1,5 +1,5 @@ # ============LICENSE_START=============================================== -# Copyright (C) 2024 Nordix Foundation. All rights reserved. +# Copyright (C) 2024-2025 OpenInfra Foundation Europe. All rights reserved # ======================================================================== # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,8 @@ http { # Add more server entries here for scaling or load balancing upstream cps-and-ncmp { least_conn; - server cps-and-ncmp:8080; + server cps-and-ncmp-0:8080; + server cps-and-ncmp-1:8080; } # Set the max allowed size of the incoming request diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 126b9590fc..38dcb5d237 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -54,10 +54,8 @@ services: start_period: 30s # Ignore failed health checks for first 30 seconds, to give system time to start # Full start up time allowed = 30 seconds start period + 3 tries * 10 seconds interval = 60 seconds - cps-and-ncmp: + cps-and-ncmp-template: image: ${DOCKER_REPO:-nexus3.onap.org:10003}/onap/cps-and-ncmp:${CPS_VERSION:-latest} - ports: - - ${CPS_PORT_RANGE:-8698-8699}:8080 ### DEBUG: Uncomment next line to enable java debugging (ensure 'ports' aligns with 'deploy') ### - ${CPS_CORE_DEBUG_PORT:-5005}:5005- environment: @@ -81,8 +79,7 @@ services: depends_on: - dbpostgresql deploy: - ### DEBUG: For easier debugging use just 1 instance (also update docker-compose/config/nginx/nginx.conf !) - replicas: 2 + replicas: 0 resources: limits: cpus: '3' @@ -95,13 +92,36 @@ services: retries: 10 start_period: 60s + cps-and-ncmp-0: + extends: + service: cps-and-ncmp-template + container_name: ${CPS_INSTANCE_0_CONTAINER_NAME:-cps-and-ncmp-0} + deploy: + replicas: 1 + hostname: cps-ncmp-0 + ports: + - ${CPS_INSTANCE_0_REST_PORT:-8698}:8080 + + ### DEBUG: For easier debugging use just 1 instance and comment out below + cps-and-ncmp-1: + extends: + service: cps-and-ncmp-template + container_name: ${CPS_INSTANCE_1_CONTAINER_NAME:-cps-and-ncmp-1} + deploy: + replicas: 1 + hostname: cps-ncmp-1 + ports: + - ${CPS_INSTANCE_1_REST_PORT:-8699}:8080 + nginx: container_name: ${NGINX_CONTAINER_NAME:-nginx-loadbalancer} image: nginx:latest ports: - ${CPS_CORE_PORT:-8883}:80 depends_on: - - cps-and-ncmp + - cps-and-ncmp-0 + ### DEBUG: For easier debugging use just 1 instance and comment out below + - cps-and-ncmp-1 volumes: - ./config/nginx/nginx.conf:/etc/nginx/nginx.conf - ./config/nginx/proxy_params:/etc/nginx/proxy_params diff --git a/docker-compose/env/endurance.env b/docker-compose/env/endurance.env index 907c63a6ae..176fc22998 100644 --- a/docker-compose/env/endurance.env +++ b/docker-compose/env/endurance.env @@ -6,7 +6,12 @@ POSTGRES_EXPORTER_PORT=9188 NGINX_CONTAINER_NAME=endurance-nginx-loadbalancer CPS_CORE_PORT=8884 -CPS_PORT_RANGE=8798-8799 + +CPS_INSTANCE_0_CONTAINER_NAME=endurance-cps-and-ncmp-0 +CPS_INSTANCE_1_CONTAINER_NAME=endurance-cps-and-ncmp-1 + +CPS_INSTANCE_0_REST_PORT=8798 +CPS_INSTANCE_1_REST_PORT=8799 ZOOKEEPER_CONTAINER_NAME=endurance-zookeeper ZOOKEEPER_PORT=2182 diff --git a/docker-compose/env/kpi.env b/docker-compose/env/kpi.env index 0fd8ef2ef8..2a779c2f06 100644 --- a/docker-compose/env/kpi.env +++ b/docker-compose/env/kpi.env @@ -6,7 +6,12 @@ POSTGRES_EXPORTER_PORT=9187 NGINX_CONTAINER_NAME=kpi-nginx-loadbalancer CPS_CORE_PORT=8883 -CPS_PORT_RANGE=8698-8699 + +CPS_INSTANCE_0_CONTAINER_NAME=kpi-cps-and-ncmp-0 +CPS_INSTANCE_1_CONTAINER_NAME=kpi-cps-and-ncmp-1 + +CPS_INSTANCE_0_REST_PORT=8698 +CPS_INSTANCE_1_REST_PORT=8699 ZOOKEEPER_CONTAINER_NAME=kpi-zookeeper ZOOKEEPER_PORT=2181 -- 2.16.6