From: emaclee Date: Tue, 16 Dec 2025 19:06:09 +0000 (+0000) Subject: Bug: k6 logs not saving X-Git-Url: https://gerrit.onap.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F78%2F142778%2F2;p=cps.git Bug: k6 logs not saving - renamed make-logs.sh to archive-logs.sh - refactored archive-logs.sh for better readability - implemented log file numbering with newest files first - modified so only one zip folder for each test run; zip folder contains all services logs - improve timestamp format for better readability - moved log collection action from teardown to main script exit handler - ensure logs are captured regardless of test outcome Issue-ID:CPS-3079 Change-Id: I6d90c9be95ea61c29a1fa2ecdc4ea4d022420e85 Signed-off-by: emaclee --- diff --git a/k6-tests/archive-logs.sh b/k6-tests/archive-logs.sh new file mode 100644 index 0000000000..942ce4d0e2 --- /dev/null +++ b/k6-tests/archive-logs.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# +# Copyright 2025 OpenInfra Foundation Europe. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +# ------------------------ +# Constants & Configuration +# ------------------------ +readonly LOG_DIR="${WORKSPACE:-.}/logs" +readonly LOG_RETENTION_DAYS=14 +readonly TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") +readonly DOCKER_SERVICES=("cps-and-ncmp" "ncmp-dmi-plugin-demo-and-csit-stub" "dbpostgresql") +deploymentType=${1:-dockerHosts} + +mkdir -p "$LOG_DIR" + +# ------------------------ +# Helper Functions +# ------------------------ + +# Fetch logs from a Docker container +fetch_container_logs() { + local container_id="$1" dest_dir="$2" + local name + name=$(docker inspect --format="{{.Name}}" "$container_id" | sed 's/\///g') + docker logs "$container_id" > "$dest_dir/${name}_logs_$TIMESTAMP.log" +} + +# Fetch logs from Kubernetes pods +fetch_pod_logs() { + local service="$1" dest_dir="$2" + local pods + pods=$(kubectl get pods -o custom-columns=":metadata.name" | grep "^$service") + + [ -z "$pods" ] && { echo "No pods found for $service"; return 1; } + + for pod in $pods; do + echo " Fetching logs for pod: $pod" + kubectl logs "$pod" > "$dest_dir/${pod}_logs.log" + done +} + +# Zip collected logs +create_log_archive() { + local service="$1" src_dir="$2" zip_file="$3" + if [ -n "$(ls -A "$src_dir")" ]; then + echo " Zipping logs to $zip_file" + zip -r -j "$zip_file" "$src_dir" + echo " Logs saved for $service" + else + echo " No logs fetched for $service" + fi +} + +# Archive logs for a service (Docker or K8s) +archive_service_logs() { + local service="$1" type="$2" + local temp_dir="$LOG_DIR/temp_${type}_${service}_$TIMESTAMP" + local zip_file="$LOG_DIR/logs_${type}_${service}_$TIMESTAMP.zip" + local fetched=false + + echo "Processing $service..." + mkdir -p "$temp_dir" + + if [ "$type" = "docker" ]; then + local containers + containers=$(docker ps --filter "name=$service" --format "{{.ID}}") + if [ -z "$containers" ]; then + echo "No Docker containers found for $service" + else + for c in $containers; do fetch_container_logs "$c" "$temp_dir"; done + fetched=true + fi + + elif [ "$type" = "k8s" ]; then + fetch_pod_logs "$service" "$temp_dir" && fetched=true + fi + + $fetched && create_log_archive "$service" "$temp_dir" "$zip_file" + rm -r "$temp_dir" +} + +# Remove old log files +cleanup_old_logs() { + local pattern="$1" + echo "Cleaning up logs older than $LOG_RETENTION_DAYS days..." + find "$LOG_DIR" -name "$pattern" -mtime +$LOG_RETENTION_DAYS -delete +} + +# Renumber log files newest first +renumber_logs_latest_first() { + local pattern="$1" width=3 index=1 + mapfile -t files < <(ls -1t "$LOG_DIR"/$pattern 2>/dev/null) || return + for f in "${files[@]}"; do + local base=$(basename "$f" | sed -E 's/^[0-9]+_//') + printf -v prefix "%0*d" "$width" "$index" + mv "$f" "$LOG_DIR/${prefix}_${base}" + ((index++)) + done +} + +# ------------------------ +# Main Process +# ------------------------ +case "$deploymentType" in + dockerHosts) + echo "Processing Docker Compose logs..." + for service in "${DOCKER_SERVICES[@]}"; do + archive_service_logs "$service" "docker" + done + cleanup_old_logs "logs_docker_*.zip" + renumber_logs_latest_first "logs_docker_*.zip" + ls -la "$LOG_DIR"/logs_docker_*.zip 2>/dev/null || echo "No Docker logs found." + ;; + + k8sHosts) + echo "Processing Kubernetes logs..." + archive_service_logs "cps-ncmp" "k8s" + cleanup_old_logs "*logs_*.zip" + renumber_logs_latest_first "*logs_k8s_*.zip" + ls -la "$LOG_DIR"/logs_k8s_*.zip 2>/dev/null || echo "No Kubernetes logs found." + ;; + + *) + echo "Error: Unknown deployment type '$deploymentType'. Supported: dockerHosts, k8sHosts" + exit 1 + ;; +esac diff --git a/k6-tests/k6-main.sh b/k6-tests/k6-main.sh index c842d6ae73..06bcf84d94 100755 --- a/k6-tests/k6-main.sh +++ b/k6-tests/k6-main.sh @@ -26,10 +26,18 @@ testProfile=${1:-kpi} # The default deployment type is dockerCompose deploymentType=${2:-dockerHosts} +# Function to create and store logs +make_logs() { + echo "Creating logs for deployment type: $deploymentType" + chmod +x archive-logs.sh + ./archive-logs.sh "$deploymentType" +} + # Cleanup handler: capture exit status, run teardown, # and restore directory, report failures, and exit with original code. on_exit() { rc=$? + make_logs chmod +x teardown.sh ./teardown.sh "$testProfile" "$deploymentType" popd diff --git a/k6-tests/make-logs.sh b/k6-tests/make-logs.sh deleted file mode 100755 index 05b4b3133b..0000000000 --- a/k6-tests/make-logs.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash -# -# Copyright 2025 OpenInfra Foundation Europe. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -euo pipefail - -# Constants -readonly LOG_DIR="${WORKSPACE:-.}/logs" -readonly LOG_RETENTION_DAYS=14 -readonly TIMESTAMP=$(date +"%Y%m%d%H%M%S") - -# Docker services -readonly DOCKER_SERVICES_TO_BE_LOGGED=("cps-and-ncmp" "ncmp-dmi-plugin-demo-and-csit-stub" "dbpostgresql") - -# Kubernetes services -readonly K8S_SERVICES_TO_BE_LOGGED=("cps-ncmp" "cps-ncmp-dmi-stub" "cps-ncmp-postgresql") -readonly K8S_APP_LABEL="app=cps-and-ncmp" - -# The default deployment type is dockerHosts -deploymentType=${1:-dockerHosts} - -# Ensure log directory exists -mkdir -p "$LOG_DIR" - -# Function to fetch logs from a Docker container -fetch_container_logs() { - local container_id="$1" - local container_name - container_name=$(docker inspect --format="{{.Name}}" "$container_id" | sed 's/\///g') - local log_file="$2/${container_name}_logs_$TIMESTAMP.log" - docker logs "$container_id" > "$log_file" -} - -# Function to fetch logs from Kubernetes pods -fetch_pod_logs() { - local service_name="$1" - local temp_dir="$2" - - # Get pod names for the current service, filtering by the app label and then grepping by service name. - local pod_names - pod_names=$(kubectl get pods -l "$K8S_APP_LABEL" --no-headers -o custom-columns=":metadata.name" | grep "^${service_name}" || echo "") - - if [ -z "$pod_names" ]; then - echo "No running pods found for service: $service_name" - return 1 - fi - - for pod_name in $pod_names; do - echo " Fetching logs for pod: $pod_name" - local log_file="$temp_dir/${pod_name}_logs.log" - kubectl logs "$pod_name" > "$log_file" - done - - return 0 -} - -# Generic function to create zip archive from collected logs -create_log_archive() { - local service_name="$1" - local temp_dir="$2" - local zip_file="$3" - - # Only create a zip file if logs were collected - if [ -n "$(ls -A "$temp_dir")" ]; then - echo " Zipping logs to $zip_file" - # The -j option flattens the directory structure. Logs will be at the root of the zip. - zip -r -j "$zip_file" "$temp_dir" - echo " Logs for service '$service_name' saved to $zip_file" - else - echo " No logs were fetched for service '$service_name'" - fi -} - -# Generic function to archive logs for a service (works for both Docker and Kubernetes) -archive_service_logs() { - local service_name="$1" - local deployment_type="$2" - local temp_dir="$LOG_DIR/temp_${deployment_type}_${service_name}_$TIMESTAMP" - local zip_file="$LOG_DIR/logs_${deployment_type}_${service_name}_$TIMESTAMP.zip" - local logs_fetched=false - - echo "Processing service: $service_name" - mkdir -p "$temp_dir" - - case "$deployment_type" in - "docker") - local container_ids - container_ids=$(docker ps --filter "name=$service_name" --format "{{.ID}}") - - if [ -z "$container_ids" ]; then - echo "No running containers found for service: $service_name" - else - for container_id in $container_ids; do - fetch_container_logs "$container_id" "$temp_dir" - done - logs_fetched=true - fi - ;; - "k8s") - if fetch_pod_logs "$service_name" "$temp_dir"; then - logs_fetched=true - fi - ;; - esac - - if [ "$logs_fetched" = true ]; then - create_log_archive "$service_name" "$temp_dir" "$zip_file" - fi - - # Clean up the temporary directory - rm -r "$temp_dir" -} - -# Function to clean up old logs -cleanup_old_logs() { - local pattern="$1" - echo "Cleaning up logs older than $LOG_RETENTION_DAYS days..." - find "$LOG_DIR" -name "$pattern" -mtime +$LOG_RETENTION_DAYS -delete -} - -# Main process - handle different deployment types -case "$deploymentType" in - "dockerHosts") - echo "Processing Docker Compose deployment logs..." - for service_name in "${DOCKER_SERVICES_TO_BE_LOGGED[@]}"; do - archive_service_logs "$service_name" "docker" - done - cleanup_old_logs "logs_docker_*.zip" - ls -la "$LOG_DIR"/logs_docker_*.zip 2>/dev/null || echo "No Docker log zip files found." - ;; - "k8sHosts") - echo "Processing Kubernetes deployment logs..." - for service_name in "${K8S_SERVICES_TO_BE_LOGGED[@]}"; do - archive_service_logs "$service_name" "k8s" - done - cleanup_old_logs "logs_k8s_*.zip" - ls -la "$LOG_DIR"/logs_k8s_*.zip 2>/dev/null || echo "No Kubernetes log zip files found." - ;; - *) - echo "Error: Unknown deployment type '$deploymentType'. Supported types: 'dockerHosts', 'k8sHosts'" - exit 1 - ;; -esac \ No newline at end of file diff --git a/k6-tests/teardown.sh b/k6-tests/teardown.sh index 254c91fa40..88e79ca2ad 100755 --- a/k6-tests/teardown.sh +++ b/k6-tests/teardown.sh @@ -19,14 +19,6 @@ testProfile=${1:-kpi} deploymentType=${2:-dockerHosts} -# Function to create and store logs -make_logs() { - local deployment_type=$1 - echo "Creating logs for deployment type: $deployment_type" - chmod +x make-logs.sh - ./make-logs.sh "$deployment_type" -} - # Function to clean Docker images based on CLEAN_DOCKER_IMAGES environment variable clean_docker_images_if_needed() { if [[ "${CLEAN_DOCKER_IMAGES:-0}" -eq 1 ]]; then @@ -77,10 +69,7 @@ teardown_docker_deployment() { # Function to teardown kubernetes deployment teardown_k8s_deployment() { echo '================================== k8s info ==========================' - kubectl get all -l app=cps-and-ncmp - - # Zip and store logs for the containers - make_logs "k8sHosts" + kubectl get all -l app=ncmp echo '================================== uninstalling cps... ==========================' helm uninstall cps