2 #set -x # uncomment for bash script debugging
4 ### ============================================================================
5 ### Licensed under the Apache License, Version 2.0 (the "License");
6 ### you may not use this file except in compliance with the License.
7 ### You may obtain a copy of the License at
9 ### http://www.apache.org/licenses/LICENSE-2.0
11 ### Unless required by applicable law or agreed to in writing, software
12 ### distributed under the License is distributed on an "AS IS" BASIS,
13 ### WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 ### See the License for the specific language governing permissions and
15 ### limitations under the License.
16 ### ============LICENSE_END=====================================================
22 ### Thomas Kulik, Deutsche Telekom AG, 2020
25 ### Retrieves a full list of ONAP repos from gerrit inluding their state.
26 ### Clones all active repos of the ONAP master branch plus other requested ONAP
27 ### branches. Then the script does some docs related analyses depending on the
28 ### clone results. It creates logfiles containing filtered results. In addition
29 ### a table.csv is created which can be used to import it in a spreadsheed.
30 ### Also a zip-file is created which contains all the results.
33 ### - in the output, repo names are shown in square brackets for readability
34 ### e.g [aai/aai-common]/docs/release-notes.rst
35 ### - in the table.csv file you see data for the requested branch if available.
36 ### if not available, data is retrieved from the master branch. it will be
37 ### denoted in round brackets, e.g. (3) (tox.ini)
45 ### SOME HELPING COMMANDS TO PROCESS LOG FILES:
47 ### curl -s https://git.onap.org/ | grep "^<tr><td class='toplevel-repo'><a title='" | sed -r "s:^<tr><td class='toplevel-repo'><a title='::" | sed -r "s:'.*::"
49 ### remove branchname from the line
50 ### cat frankfurt_gerritclone.log | sed 's:frankfurt|::'
52 ### list only image names
53 ### cat master_dockerimagesfull.log | grep image | sed -r 's:image\:::' | sed -r 's:^ +::' | sed '/^[[:space:]]*$/d'
55 ### more interesting stuff ...
56 ### curl https://gerrit.onap.org/r/projects/?d
57 ### LONG: curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g' | sed -r 's:["{}]::g' | sed -r 's:id\:::' | sed -r 's:,state\::|:' | sed '/All-Projects/d' | sed '/All-Users/d'
58 ### SHORT: curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d'
61 script_version="1.2 (2020-11-18)"
63 # save command for the restart with logging enabled
66 fullcommand="${command} ${arguments}"
75 echo " checkdocs.sh Version ${script_version}"
78 echo " ./checkdocs.sh <arguments> "
81 echo " -u|--user username "
82 echo " linux foundation username used to clone ONAP Gerrit repos"
84 echo " -b|--branches branch1,branch2,branch3 "
85 echo " list of branches to be cloned. master is automatically "
86 echo " added to the list. do not add manually! "
89 echo " development-mode - limits number of repos to be cloned "
95 echo "*******************************************************************************"
98 # remove lockfile in case script is interrupted
99 trap InterruptedScript SIGINT SIGTERM SIGHUP SIGKILL SIGSTOP
100 function InterruptedScript {
102 echo "Script was interrupted."
103 if [ -f $lockfile ] ; then
110 ### arguments handling
122 if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then
126 echo "Error: Argument for $1 is missing" >&2
132 if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then
136 echo "Error: Argument for $1 is missing" >&2
141 -*|--*=) # unsupported flags
142 echo "Error: Unsupported argument $1" >&2
146 *) # preserve positional arguments
153 # set positional arguments in their proper place
154 eval set -- "$PARAMS"
156 # old: declare -a branches=("master" "frankfurt" "guilin")
157 if [[ $branches_csv == "" || $lfusername == "" ]]; then
162 # master branch is automatically added and must not part of the user arguments
163 if [[ $branches_csv == *"master"* ]]; then
167 # clone master first, the the other branches
168 branches_csv="master,${branches_csv}"
170 # create the branches array by readinging in the values from the variable
171 IFS=',' read -r -a branches <<< "${branches_csv}"
173 #echo "DBUG: devmode = \"${devmode}\""
174 #echo "DBUG: branches_csv = \"${branches_csv}\""
175 #echo "DBUG: lfusername = \"${lfusername}\""
176 #echo "DBUG: branches = \"${branches[@]}\""
178 # restart script with logging enabled
179 lockfile="checkdocs-runtime-lockfile"
180 if [ ! -f $lockfile ] ; then
182 echo "Restarting script with logging enabled."
183 ${fullcommand} 2>&1 | tee checkdocs.log
189 echo "checkdocs.sh Version ${script_version}"
192 # curl must be installed
193 if ! command -v curl &> /dev/null
195 echo "ERROR: curl command could not be found"
199 today=$(date '+%Y-%m-%d');
200 repolist="gerrit-repos-master-"$today".txt";
203 echo "Retrieving a full list of ONAP repositories (master) from gerrit.onap.org."
205 # retrieve the full repolist from gerrit
206 # workaround because of the (wrong?) response of gerrit.onap.org which makes jq command fail
207 # "| awk '{if(NR>1)print}'" filters the first line of the response so that jq will work again (thx marek)
208 curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d' >./$repolist
210 # process the created repolist
211 # only active projects will be cloned in case the requested branch of the project exists
212 echo "Accessing gerrit.onap.org with username \"${lfusername}\"."
213 echo "Start cloning of repositories."
215 for branch in "${branches[@]}"
224 branch_upper=$(echo "${branch}" | tr '[:lower:]' '[:upper:]')
236 if [[ $devmode == "TRUE" ]]; then
237 devcounter=$((devcounter+1))
240 if [[ $devcounter -lt "11" ]]; then
242 if [[ $devmode == "TRUE" ]]; then
243 echo "INFO: devmode! counter=${devcounter}"
247 reponame=$(echo $line | awk -F "|" '{print $1}');
248 repostate=$(echo $line | awk -F "|" '{print $2}');
252 if [[ $repostate == "ACTIVE" ]]; then
253 echo "Cloning \"${branch}\" branch of ACTIVE project ${reponame}..."
255 git clone --branch ${branch} --recurse-submodules ssh://${lfusername}@gerrit.onap.org:29418/$reponame ./$reponame
258 if [[ ! ${gitexitcode} == "0" ]]; then
259 errormsg=$(tail -1 ../checkdocs.log)
264 # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
265 echo "${gitexitcode}|${reponame}|${repostate}|${errormsg}" | tee -a ${branch}_gerritclone.log
267 elif [[ $repostate == "READ_ONLY" ]]; then
268 echo "-|${reponame}|${repostate}|ignored" | tee -a ${branch}_gerritclone.log
270 echo "-|${reponame}|unknown repo state \"${repostate}\"|-" | tee -a ${branch}_gerritclone.log
274 if [[ ${gitexitcode} == "0" ]]; then
276 printf "\ndocs directories:\n"
277 find ./$reponame -type d -name docs | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_docs.log
279 printf "\nrst files:\n"
280 find ./$reponame -type f -name *.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_rstfiles.log
282 printf "\nrelease notes rst:\n"
283 find ./$reponame -type f | grep 'release.*note.*.rst' | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_releasenotes.log
285 printf "\ntox.ini files:\n"
286 find ./$reponame -type f -name tox.ini | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_toxini.log
288 printf "\nconf.py files:\n"
289 find ./$reponame -type f -name conf.py | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_confpy.log
291 printf "\nindex.rst files:\n"
292 find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst.log
296 # end defcounter loop
305 find . -type f -name values.yaml -print -exec grep "image:" {} \; | sed -r 's:^ +::' | tee ${branch}_dockerimagesfull.log
307 ls --format single-column -d */ | sed 's:/$::' | tee ${branch}_directories.log
309 cat ${branch}_dockerimagesfull.log | grep image | sed -r 's:image\:::' | sed -r 's:^ +::' | sed '/^[[:space:]]*$/d' >${branch}_dockerimages.log
311 ls --format single-column -d oom/kubernetes/*/ | tee ${branch}_oomkubernetes.log
315 readarray -t docs_array < ./${branch}_docs.log;
317 for line in "${docs_array[@]}"
320 echo $line | tee -a ${branch}_docsconfig.log
322 # remove [ and ] which are distinguish the project name in the output
323 line=$(echo $line | sed -r 's:\[:: ; s:\]::')
325 if [ -f ./${line}/conf.py ] ; then
326 echo " conf.py ..... found" | tee -a ${branch}_docsconfig.log
328 echo " conf.py ..... NOT FOUND" | tee -a ${branch}_docsconfig.log
331 if [ -f ./${line}/index.rst ] ; then
332 echo " index.rst ... found" | tee -a ${branch}_docsconfig.log
334 echo " index.rst ... NOT FOUND" | tee -a ${branch}_docsconfig.log
337 if [ -f ./${line}/tox.ini ] ; then
338 echo " tox.ini ..... found" | tee -a ${branch}_docsconfig.log
340 echo " tox.ini ..... NOT FOUND" | tee -a ${branch}_docsconfig.log
343 echo " " | tee -a ${branch}_docsconfig.log
351 ### build a csv table that combines results
355 # csv column #1: project name
358 readarray -t array < ./${repolist};
362 for line in "${array[@]}"
364 reponame=$(echo $line | awk -F "|" '{print $1}');
365 project=$(echo $reponame | sed 's:/.*$::')
366 #echo "DBUG: reponame=${reponame}"
367 #echo "DBUG: project=${project}"
378 # csv column #2: repo name
381 readarray -t array < ./${repolist};
383 csv[i]="${csv[i]},MASTER repo name"
385 for line in "${array[@]}"
387 reponame=$(echo $line | awk -F "|" '{print $1}');
388 csv[i]="${csv[i]},${reponame}"
396 # csv column #3: repo state
399 readarray -t array < ./${repolist};
401 csv[i]="${csv[i]},MASTER repo state"
403 for line in "${array[@]}"
405 repostate=$(echo $line | awk -F "|" '{print $2}');
406 csv[i]="${csv[i]},${repostate}"
414 # csv column #4: clone message
417 readarray -t array < ./${branch}_gerritclone.log;
419 csv[i]="${csv[i]},${branch_upper} clone message"
421 for line in "${array[@]}"
423 # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
424 errormsg=$(echo $line | awk -F "|" '{print $4}');
425 csv[i]="${csv[i]},${errormsg}"
433 # csv column #5: RELEASE component (yes|no|maybe)
434 # to be filled with values of the planned release config file maintained by
435 # the onap release manager
438 # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
439 readarray -t array < ./${branch}_gerritclone.log;
441 csv[i]="${csv[i]},${branch_upper} component"
443 for line in "${array[@]}"
446 # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
447 gitexitcode=$(echo $line | awk -F "|" '{print $1}');
448 reponame=$(echo $line | awk -F "|" '{print $2}');
449 repostate=$(echo $line | awk -F "|" '{print $3}');
450 errormsg=$(echo $line | awk -F "|" '{print $4}');
452 if [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "0" ]]; then
453 releasecomponent="yes"
454 elif [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "128" ]]; then
455 releasecomponent="maybe"
456 elif [ ${repostate} == "READ_ONLY" ]; then
457 releasecomponent="no"
459 releasecomponent="unknown"
462 csv[i]="${csv[i]},${releasecomponent}"
471 unset releasecomponent
474 # csv column #6: docs (at repo root directory only; no recursive search!)
475 # csv column #7: conf.py
476 # csv column #8: tox.ini
477 # csv column #9: index.rst
479 # columns are filled with values from requested branch.
480 # if data is not available values from master branch are used.
481 # to identify master branch values, data is put into brackets "(...)"
484 readarray -t array < ./${repolist};
486 csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst"
488 for line in "${array[@]}"
490 line=$(echo $line | sed 's:|.*$::')
491 #echo "DBUG: line=${line}"
495 if [ -d ./${line}/docs ] ; then
497 elif [ -d ../master/${line}/docs ] ; then
504 if [ -f ./${line}/docs/conf.py ] ; then
505 docs="${docs},conf.py"
506 elif [ -f ../master/${line}/docs/conf.py ] ; then
507 docs="${docs},(conf.py)"
513 if [ -f ./${line}/docs/tox.ini ] ; then
514 docs="${docs},tox.ini"
515 elif [ -f ../master/${line}/docs/tox.ini ] ; then
516 docs="${docs},(tox.ini)"
522 if [ -f ./${line}/docs/index.rst ] ; then
523 docs="${docs},index.rst"
524 elif [ -f ../master/${line}/docs/index.rst ] ; then
525 docs="${docs},(index.rst)"
530 #echo "DBUG: docs=${docs}"
531 line="${csv[i]},${docs}"
540 # csv column #10: index.html@RTD accessibility check
541 # csv column #11: index.html url
544 readarray -t array < ./${branch}_gerritclone.log;
546 csv[i]="${csv[i]},index.html@RTD,index.html url"
548 for line in "${array[@]}"
550 # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
551 gitexitcode=$(echo $line | awk -F "|" '{print $1}');
552 reponame=$(echo $line | awk -F "|" '{print $2}');
553 repostate=$(echo $line | awk -F "|" '{print $3}');
554 errormsg=$(echo $line | awk -F "|" '{print $4}');
559 # this routine works only with release "frankfurt" and later because
560 # earlier releases are using submodule structure for documentation files
561 if echo "$branch" | grep -q '^[abcde]'; then
562 curl_result="unsupported release"
566 # we are working on "frankfurt" branch or later ...
567 # only if repostate IS ACTIVE a curl test is required
568 if [[ ${repostate} == "ACTIVE" ]]; then
570 # OPTIONAL: USE ALSO GITEXITCODE AS A FILTER CRITERIA ???
573 # important! only doc project needs a different url base
574 if [[ ${reponame} == "doc" ]]; then
575 url_start="https://docs.onap.org"
577 url_start="https://docs.onap.org/projects/onap"
582 # "master" branch documentation is available as "latest" in RTD
583 if [[ ${url_branch} == "master" ]]; then
587 # replace all / characters in repo name with - charachter
588 url_repo=$(echo ${reponame} | sed -r 's/\//-/g')
589 url_file="index.html"
592 if [[ ${reponame} == "doc" ]]; then
593 # build the full url for the doc project
594 url="${url_start}/${url_lang}/${url_branch}/${url_file}"
596 # build the full url for the other projects
597 url="${url_start}-${url_repo}/${url_lang}/${url_branch}/${url_file}"
599 #echo "DBUG: url=$url"
601 # test accessibility of url
602 curl --head --silent --fail "${url}?${unique}" >/dev/null
605 # convert numeric results to text
606 if [ "${curl_result}" = "0" ]; then
607 curl_result="accessible"
608 elif [ "${curl_result}" = "22" ]; then
609 curl_result="does not exist"
611 curl_result="ERROR:${curl_result}"
614 # url does not exist for this branch.
615 # in case the requested url is not already for "master" branch,
616 # we try to access the url of the master branch and denote the
617 # result by using round brackets (result)
618 if [[ ${curl_result} == "does not exist" && ! $branch == "master" ]]; then
620 # build the full (master/latest) url
621 url="${url_start}-${url_repo}/${url_lang}/latest/${url_file}"
622 #echo "DBUG: url=$url"
624 # test accessibility of url in "master branch" (latest)
625 curl --head --silent --fail "${url}?${unique}" >/dev/null
627 # denote result as a value from "master" branch (latest)
630 # convert numeric results to text
631 if [ "${curl_result}" = "0" ]; then
632 curl_result="(accessible)"
633 elif [ "${curl_result}" = "22" ]; then
634 curl_result="(does not exist)"
636 curl_result="(ERROR:${curl_result})"
641 # repostate IS NOT ACTIVE - no curl test required
647 echo "$url ... $curl_result"
648 csv[i]="${csv[i]},${curl_result},${url}"
649 #echo "DBUG: csv line=${csv[i]}"
655 # csv column #12: release notes
658 readarray -t array < ../${repolist};
660 csv[i]="${csv[i]},release notes"
662 for line in "${array[@]}"
664 line=$(echo $line | sed 's:|.*$::')
665 #echo "DBUG: line=\"${line}\""
669 # put repo name in square brackets for increased grep hit rate
670 # escape minus and bracket characters to avoid problems with the grep command
671 #repo_grepable=$(echo ${line} | sed -r s:${line}:[${line}]: | sed -r 's/-/\\-/g' | sed -r 's/\[/\\[/g' | sed -r 's/\]/\\]/g')
672 #echo "DBUG: repo_grepable=\"${repo_grepable}\""
674 # check if repo dir exists in this branch
675 if [ -d ./${line} ] ; then
676 # if yes, check if repo name appears in the branch releasenotes.log
677 relnote=$(find "./${line}" -type f | grep 'release.*note.*.rst' | wc -l);
678 # repo dir DOES NOT exist in this branch - so check if repo dir exists in MASTER branch
679 elif [ -d ../master/${line} ] ; then
680 # if yes, check if repo name appears in the MASTER releasenotes.log
681 # count release notes files in MASTER branch (in repo root and its subdirectories)
682 relnote=$(find "../master/${line}" -type f | grep 'release.*note.*.rst' | wc -l);
683 # put results in round brackets to show that this is MASTER data
684 relnote=$(echo ${relnote} | sed -r s:${relnote}:\(${relnote}\):)
689 line="${csv[i]},${relnote}"
700 # build the table.csv file
705 echo "$i" | tee -a ./${branch}_table.csv
709 # create data package for this branch and zip it
712 datadir=${branch}_data
714 cp $repolist $datadir
715 cp ${branch}_table.csv $datadir
716 cp ${branch}_*.log $datadir
717 zip -r ${datadir}.zip $datadir
719 # return from the branch directory
722 # return and work on the next requested branch ... or exit