X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=shell%2Fmaven-coverity.sh;h=eed10c4ea2782258f0e80f962e843d80a56d9749;hb=642bef62c839a1194487eec69049d7bb223f0e9c;hp=e319b06d93544ff38513531e156f46b8e8aea124;hpb=fa0fb85381038891aa3247ede745b3a263d3eae2;p=ci-management.git

diff --git a/shell/maven-coverity.sh b/shell/maven-coverity.sh
index e319b06d9..eed10c4ea 100644
--- a/shell/maven-coverity.sh
+++ b/shell/maven-coverity.sh
@@ -19,29 +19,86 @@ PS4='+['$(readlink -f "$0")' ${FUNCNAME[0]%main}#$LINENO] '
 
 echo '---> maven-coverity.sh'
 
+SUBMISSION_ATTEMPTS=5
+SUBMISSION_INITIAL_REST_INTERVAL=30 # seconds, will be doubled after each attempt
+
+#-----------------------------------------------------------------------------
+# Check for git repo changes within the last $MAX_GIT_REPO_AGE_HOURS hours
+#
+# It makes sense to set the value twice the 'cron' interval for the job (e.g.
+# if 'cron: @daily', then MAX_GIT_REPO_AGE_HOURS=48)
+
+if ! [[ "${MAX_GIT_REPO_AGE_HOURS:=0}" =~ ^[0-9]+$ ]]; then
+  echo '[ERROR] MAX_GIT_REPO_AGE_HOURS must be non-negative integer.' \
+    >&2
+  exit 1
+fi
+
+if [ ${MAX_GIT_REPO_AGE_HOURS:=0} -ne 0 ]; then
+  LAST_COMMIT_AGE=$(( $(date +%s) - $(git log -1 --pretty=format:%ct) ))
+
+  if [ $LAST_COMMIT_AGE -gt $(( MAX_GIT_REPO_AGE_HOURS *60*60 )) ]; then
+    echo '[NOTICE] Git repository did not have any commits last' \
+      "${MAX_GIT_REPO_AGE_HOURS} hours - no need to re-analyse it." \
+      >&2
+    exit 0
+  fi
+fi
+
+#-----------------------------------------------------------------------------
+# Process parameters for JS/TS/Python/Ruby/PHP files analysis
+
+if [ -n "${SEARCH_PATHS:=}" ]; then
+  for SEARCH_PATH in ${SEARCH_PATHS}; do
+    if [ -d "${SEARCH_PATH}" ]; then
+      FS_CAPTURE_SEARCH_PARAMS="${FS_CAPTURE_SEARCH_PARAMS:=} --fs-capture-search '${SEARCH_PATH}'"
+    else
+      echo "[ERROR] '${SEARCH_PATH}' from \$SEARCH_PATHS is not an" \
+        "existing directory." \
+        >&2
+      exit 1
+    fi
+  done
+
+  for EXCLUDE_REGEX in ${SEARCH_EXCLUDE_REGEXS:=}; do
+    EXCLUDE_REGEX=${EXCLUDE_REGEX//\'/\'\\\'\'} # escape single quote "'"
+    FS_CAPTURE_SEARCH_PARAMS="${FS_CAPTURE_SEARCH_PARAMS} --fs-capture-search-exclude-regex '${EXCLUDE_REGEX}'"
+
+    # FIXME: a hack to deal with temporary(?) non-functional filter to ignore
+    # specific source code parts by Coverity Scan ("--fs-capture-search-exclude-regex"
+    # CLI parameter for "cov-build" tool). The hack can be removed when this CLI
+    # parameter is fixed on Coverity side.
+    FS_CAPTURE_SEARCH_EXCLUDE_HACK_PARAMS="${FS_CAPTURE_SEARCH_EXCLUDE_HACK_PARAMS:=} --tu-pattern 'file('\\''${EXCLUDE_REGEX}'\\'')'"
+  done
+fi
+
 #-----------------------------------------------------------------------------
 # Check if we are allowed to submit results to Coverity Scan service
 # and have not exceeded our upload quota limits
 # See also: https://scan.coverity.com/faq#frequency
 
-CURL_OUTPUT=$(
-  curl \
-    --verbose \
-    --silent \
-    --show-error \
-    --fail \
-    --form "project=${COVERITY_PROJECT_NAME}" \
-    --form "token=${COVERITY_TOKEN}" \
-    'https://scan.coverity.com/api/upload_permitted'
-)
+if [ "${DRY_RUN}" != 'true' ]; then
+  CURL_OUTPUT=$(
+    curl \
+      --verbose \
+      --silent \
+      --show-error \
+      --fail \
+      --form "project=${COVERITY_PROJECT_NAME}" \
+      --form "token=${COVERITY_TOKEN}" \
+      'https://scan.coverity.com/api/upload_permitted'
+  )
 
-IS_COVERITY_UPLOAD_PERMITTED=$(
-  echo "${CURL_OUTPUT}" \
-  | jq '.upload_permitted'
-)
-if [ x"${IS_COVERITY_UPLOAD_PERMITTED}" != x'true' ]; then
-  echo "Upload quota reached. Next upload permitted at "$(echo "${CURL_OUTPUT}" | jq '.next_upload_permitted_at') >&2
-  exit 1
+  IS_COVERITY_UPLOAD_PERMITTED=$(
+    echo "${CURL_OUTPUT}" \
+    | jq '.upload_permitted'
+  )
+  if [ x"${IS_COVERITY_UPLOAD_PERMITTED}" != x'true' ]; then
+    echo "[WARNING] Upload quota reached. Next upload permitted at" \
+      $(echo "${CURL_OUTPUT}" | jq '.next_upload_permitted_at') \
+      >&2
+    exit 1
+  fi
 fi
 
 #-----------------------------------------------------------------------------
@@ -54,7 +111,7 @@ curl \
   --fail \
   --form "project=${COVERITY_PROJECT_NAME}" \
   --form "token=${COVERITY_TOKEN}" \
-  --output 'coverity_tool.tgz' \
+  --output '/tmp/coverity_tool.tgz' \
   'https://scan.coverity.com/download/linux64'
 
 curl \
@@ -65,23 +122,24 @@ curl \
   --form "project=${COVERITY_PROJECT_NAME}" \
   --form "token=${COVERITY_TOKEN}" \
   --form 'md5=1' \
-  --output 'coverity_tool.md5' \
+  --output '/tmp/coverity_tool.md5' \
   'https://scan.coverity.com/download/linux64'
 
-echo -n ' coverity_tool.tgz' >> 'coverity_tool.md5'
-md5sum --check 'coverity_tool.md5'
+echo -n ' /tmp/coverity_tool.tgz' >> '/tmp/coverity_tool.md5'
+md5sum --check '/tmp/coverity_tool.md5'
 
 tar \
   --extract \
   --gunzip \
-  --file='coverity_tool.tgz'
+  --file='/tmp/coverity_tool.tgz' \
+  --directory='/tmp'
 
-COVERITY_BUILD_TOOL_DIRECTORY=$(
+COVERITY_BUILD_TOOL_DIRECTORY='/tmp/'$(
   head -1 <( \
     tar \
       --list \
       --gunzip \
-      --file='coverity_tool.tgz'
+      --file='/tmp/coverity_tool.tgz'
   )
 )
 COVERITY_BINARY_DIRECTORY="${COVERITY_BUILD_TOOL_DIRECTORY}bin"
@@ -89,15 +147,17 @@ test -d "${COVERITY_BINARY_DIRECTORY}" \
   || exit 1
 export PATH="${PATH}:${COVERITY_BINARY_DIRECTORY}"
 
-rm 'coverity_tool.tgz'
+rm '/tmp/coverity_tool.tgz'
 
 #-----------------------------------------------------------------------------
 # Build
 
 export MAVEN_OPTS
 
-cov-build \
+eval cov-build \
   --dir 'cov-int' \
+  --append-log \
+  ${FS_CAPTURE_SEARCH_PARAMS:=} \
   "${MVN}" clean install \
     --errors \
     --global-settings "${GLOBAL_SETTINGS_FILE}" \
@@ -105,32 +165,97 @@ cov-build \
     ${MAVEN_OPTIONS:=} \
     ${MAVEN_PARAMS:=}
 
+# FIXME: a hack to deal with temporary(?) non-functional filter to ignore
+# specific source code parts by Coverity Scan ("--fs-capture-search-exclude-regex"
+# CLI parameter for "cov-build" tool). The hack can be removed when this CLI
+# parameter is fixed on Coverity side.
+if [ -n "${FS_CAPTURE_SEARCH_EXCLUDE_HACK_PARAMS:=}" ]; then
+  eval cov-manage-emit \
+    --dir 'cov-int' \
+    ${FS_CAPTURE_SEARCH_EXCLUDE_HACK_PARAMS} \
+    delete
+fi
+
+# Extract git data for analysed files
 cov-import-scm \
   --dir 'cov-int' \
   --scm 'git'
 
+# List all analysed files from the project
+cov-manage-emit \
+  --dir cov-int \
+  list \
+| grep \
+  --invert-match \
+  '^Translation unit:$' \
+| sed \
+  --regexp-extended \
+  's!^[[:digit:]]+ -> !!' \
+| sort \
+> 'cov-int/coverity-scan-analysed-files.txt'
+
+# List all analyzed files that are not tracked by SCM repository
+cov-manage-emit \
+  --dir cov-int \
+  list-scm-unknown \
+| sed \
+  --regexp-extended \
+  's!^[^ ]+ !!' \
+| sort \
+> 'cov-int/scm-untracked-files.txt'
+
+if [ -s 'cov-int/scm-untracked-files.txt' ]; then
+  echo '[WARNING] There are some files analysed but not tracked by SCM repository.' \
+    'There might be 3rd-party or auto-generated sources. See details in' \
+    '"cov-int/scm-untracked-files.txt" file.' \
+    >&2
+fi
+
 #-----------------------------------------------------------------------------
 # Submit results to Coverity service
 
-tar \
-  --create \
-  --gzip \
-  --file='results.tgz' \
-  'cov-int'
+if [ "${DRY_RUN}" != 'true' ]; then
+  tar \
+    --create \
+    --gzip \
+    --file='results.tgz' \
+    'cov-int'
 
-curl \
-  --verbose \
-  --silent \
-  --show-error \
-  --fail \
-  --form "project=${COVERITY_PROJECT_NAME}" \
-  --form "email=${COVERITY_USER_EMAIL}" \
-  --form "token=${COVERITY_TOKEN}" \
-  --form 'file=@results.tgz' \
-  --form "version=${GIT_COMMIT:0:7}" \
-  --form "description=${GIT_BRANCH}" \
-  'https://scan.coverity.com/builds'
+  for (( ATTEMPT=1; ATTEMPT<=SUBMISSION_ATTEMPTS; ATTEMPT++ )); do
+    CURL_OUTPUT=$(
+      curl \
+        --verbose \
+        --silent \
+        --show-error \
+        --fail \
+        --write-out '\n%{http_code}' \
+        --form "project=${COVERITY_PROJECT_NAME}" \
+        --form "email=${COVERITY_USER_EMAIL}" \
+        --form "token=${COVERITY_TOKEN}" \
+        --form 'file=@results.tgz' \
+        --form "version=${GIT_COMMIT:0:7}" \
+        --form "description=${GIT_BRANCH}" \
+        'https://scan.coverity.com/builds'
+    )
+    HTTP_RESPONSE_CODE=$(echo -n "${CURL_OUTPUT}" | tail -1)
+    test x"${HTTP_RESPONSE_CODE}" = x"200" \
+      && break
 
-#-----------------------------------------------------------------------------
+    sleep "${SUBMISSION_REST_INTERVAL:-$SUBMISSION_INITIAL_REST_INTERVAL}"
 
+    SUBMISSION_REST_INTERVAL=$(( ${SUBMISSION_REST_INTERVAL:-$SUBMISSION_INITIAL_REST_INTERVAL} * 2 ))
+  done
+
+  HTTP_RESPONSE=$(echo -n "${CURL_OUTPUT}" | head -n -1 | tr -d '\n')
+  if [ x"${HTTP_RESPONSE}" != x"Build successfully submitted." ]; then
+    echo "[ERROR] Coverity Scan service responded with '${HTTP_RESPONSE}'" \
+      "while 'Build successfully submitted.' expected." \
+      >&2
+    exit 1
+  fi
+
+  echo "[INFO] Build successfully submitted to Coverity Scan server." >&2
+fi
+
+#-----------------------------------------------------------------------------
 exit 0