2 #set -x # uncomment for bash script debugging
4 ### ============================================================================
5 ### Licensed under the Apache License, Version 2.0 (the "License");
6 ### you may not use this file except in compliance with the License.
7 ### You may obtain a copy of the License at
9 ### http://www.apache.org/licenses/LICENSE-2.0
11 ### Unless required by applicable law or agreed to in writing, software
12 ### distributed under the License is distributed on an "AS IS" BASIS,
13 ### WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 ### See the License for the specific language governing permissions and
15 ### limitations under the License.
16 ### ============LICENSE_END=====================================================
22 ### Thomas Kulik, Deutsche Telekom AG, 2020 - 2021
25 ### Retrieves a full list of ONAP repos from gerrit inluding their state.
26 ### Clones all repos of the ONAP master branch plus other requested ONAP
27 ### branches. Then the script does some docs related analyses depending on the
28 ### clone results. It creates logfiles containing filtered results. In addition
29 ### a table.csv is created which can be used to import it in a spreadsheed.
30 ### Also a zip-file is created which contains all the results.
33 ### - in the output, repo names are shown in square brackets for readability
34 ### e.g [aai/aai-common]/docs/release-notes.rst
35 ### - in the table.csv file you see data for the requested branch if available.
36 ### if not available, data is retrieved from the master branch. it will be
37 ### denoted in round brackets, e.g. (3) (tox.ini)
45 ### SOME HELPING COMMANDS TO PROCESS LOG FILES:
47 ### curl -s https://git.onap.org/ | grep "^<tr><td class='toplevel-repo'><a title='" | sed -r "s:^<tr><td class='toplevel-repo'><a title='::" | sed -r "s:'.*::"
49 ### remove branchname from the line:
50 ### cat frankfurt_repoclone.log | sed 's:frankfurt|::'
52 ### list only image names
53 ### cat master_dockerimagesfull.log | grep image | sed -r 's:image\:::' | sed -r 's:^ +::' | sed '/^[[:space:]]*$/d'
55 ### more interesting stuff ...
56 ### curl https://gerrit.onap.org/r/projects/?d
57 ### LONG: curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g' | sed -r 's:["{}]::g' | sed -r 's:id\:::' | sed -r 's:,state\::|:' | sed '/All-Projects/d' | sed '/All-Users/d'
58 ### SHORT: curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d'
61 script_version="1.12 (2021-11-12)"
63 # save command for the restart with logging enabled
66 fullcommand="${command} ${arguments}"
75 echo " checkdocs.sh Version ${script_version}"
78 echo " ./checkdocs.sh <arguments> "
81 echo " -u|--user username "
82 echo " linux foundation username used to clone ONAP repositories"
84 echo " -b|--branches branch1,branch2,branch3 "
85 echo " list of branches to be cloned. master is automatically "
86 echo " added to the list. do not add manually! "
89 echo " development-mode - limits number of repos to be cloned "
95 echo "*******************************************************************************"
98 # remove lockfile in case script is interrupted
99 trap InterruptedScript SIGINT SIGTERM SIGHUP SIGKILL SIGSTOP
100 function InterruptedScript {
102 echo "Script was interrupted."
103 if [ -f $lockfile ] ; then
109 # function to parse wiki (project) lifecycle state information
110 # call: getwikilifecyclestate "projectname"
111 # result: $return_from_getwikilifecyclestate
112 # because bash supports only returning numeric values a variable $return_from_getwikilifecyclestate is used
114 function getwikilifecyclestate {
121 return_from_getwikilifecyclestate=""
123 for wikiline in "${wikiplsarray[@]}"
126 wikirepo=$(echo $wikiline | awk -F ";" '{print $1}');
127 wikistate=$(echo $wikiline | awk -F ";" '{print $2}');
129 #echo "DBUG: getwikilifecyclestate wikiline = \"${wikiline}\"";
130 #echo "DBUG: getwikilifecyclestate wikirepo = \"${wikirepo}\""
131 #echo "DBUG: getwikilifecyclestate wikistate = \"${wikistate}\""
133 if [[ ${wikirepo} == ${requested} ]]; then
134 return_from_getwikilifecyclestate=${wikistate}
135 #echo "DBUG: getwikilifecyclestate wikirepo = \"${wikirepo}\""
136 #echo "DBUG: getwikilifecyclestate requested = \"${requested}\""
137 #echo "DBUG: return_from_getwikilifecyclestate = \"${return_from_getwikilifecyclestate}\"";
143 #echo "DBUG: getwikilifecyclestate requested \"${requested}\" NOT FOUND in list"
144 return_from_getwikilifecyclestate=""
148 # function to parse release partizipation information
149 # call: getrpinfo "projectname"
150 # result: $return_from_getrpinfo
151 # because bash supports only returning numeric values a variable $return_from_getrpinfo is used
162 local current_branch_starting_letter=""
163 return_from_getrpinfo=""
165 # finds first matching line in the array using grep (currently every line shows the same partizipation for the project (NOT repository!) )
166 # this is much faster then looping line by line
167 rpline=$(IFS=$'\n'; echo "${rparray[*]}" | grep -m 1 ";${requested};");
168 rpline=$(echo ${rpline} | tr -d '^M')
169 rprepo=$(echo ${rpline} | awk -F ";" '{print $1}');
170 rpproject=$(echo ${rpline} | awk -F ";" '{print $2}');
171 # concatenate details to do an easy grep later on to find out if or if not the project/repo has partizipated to a release
172 rpdetails=$(echo ${rpline} | awk -F ";" '{print "-" $3 "-" $4 "-" $5 "-" $6 "-" $7 "-" $8 "-" $9 "-" $10 "-" $11 "-" $12 "-"}');
174 # result will be e.g. "-g" and this avoids false positives with the "m" release
175 # (because "m" is also used to indicate the maintenance release, e.g. "gm")
176 current_branch_starting_letter="-${branch:0:1}"
178 #echo "DBUG: getrpinfo ****************************";
179 #echo "DBUG: getrpinfo requested = \"${requested}\"";
180 #echo "DBUG: getrpinfo rpproject = \"${rpproject}\"";
181 #echo "DBUG: getrpinfo rpdetails = \"${rpdetails}\"";
182 #echo "DBUG: current branch = \"${branch}\"";
183 #echo "DBUG: starting_letter = \"${current_branch_starting_letter}\"";
185 ## check if PROJECT has partizipated to INITIAL release
186 #if [[ ${rpproject} = ${requested} ]] && [[ "${rpdetails}" == *"${current_branch_starting_letter}-"* ]]; then
187 # return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}"
188 # # check ADDITIONALLY if PROJECT has ALSO partizipated to MAINTENANCE release
189 # if [[ "${rpdetails}" == *"${current_branch_starting_letter}m-"* ]]; then
190 # return_from_getrpinfo="${return_from_getrpinfo} | ${current_branch_starting_letter:1:1}m"
191 # #echo "DBUG: getrpinfo return = \"${return_from_getrpinfo}\"";
194 ## check if PROJECT has ONLY partizipated to MAINTENANCE release
195 #elif [[ ${rpproject} = ${requested} ]] && [[ "${rpdetails}" == *"${current_branch_starting_letter:1:1}m-"* ]]; then
196 # return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}m"
197 # #echo "DBUG: getrpinfo return = \"${return_from_getrpinfo}\"";
201 # check if requested PROJECT was found in the array of partizipating projects
202 if [[ ${rpproject} = ${requested} ]]; then
203 # check if PROJECT has partizipated to INITIAL release
204 if [[ "${rpdetails}" == *"${current_branch_starting_letter}-"* ]]; then
205 return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}"
206 # check ADDITIONALLY if PROJECT has ALSO partizipated to MAINTENANCE release
207 if [[ "${rpdetails}" == *"${current_branch_starting_letter}m-"* ]]; then
208 return_from_getrpinfo="${return_from_getrpinfo} ${current_branch_starting_letter:1:1}m"
209 #echo "DBUG: getrpinfo return = \"${return_from_getrpinfo}\"";
212 elif [[ "${rpdetails}" == *"${current_branch_starting_letter:1:1}m-"* ]]; then
213 return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}m"
214 #echo "DBUG: getrpinfo return = \"${return_from_getrpinfo}\"";
218 #echo "DBUG: getrpinfo requested \"${requested}\" NOT FOUND in list"
219 return_from_getrpinfo=""
222 function find_repo_in_confpy {
225 local search_term_line_number=""
226 local confpy_branch_entries=""
227 local confpy_line_number=""
228 local confpy_branch_name=""
231 return_from_find_repo_in_confpy=""
232 search_term="'${search_term}'"
234 search_term_line_number=$(cat ./doc/docs/conf.py | grep -n '^intersphinx_mapping\[' | grep -m 1 ${search_term} | sed 's/:.*//')
235 #echo "DBUG: search_term is ............... ${search_term}"
236 #echo "DBUG: search_term_line_number is ... ${search_term_line_number}"
238 # nothing (or multiple entries) found - return
239 if [[ ${search_term_line_number} == "" ]]; then
240 #echo "DBUG: search_term_line_number is empty - returning"
241 return_from_find_repo_in_confpy=""
245 readarray -t confpy_branch_entries <<< "$(cat ./doc/docs/conf.py | grep -n '^branch = ' | sed 's/branch = //' | sed s/\'//g)"
247 #echo "DBUG: confpy_branch_entries"
248 #printf -- "%s\n" "${confpy_branch_entries[@]}"
249 #for confpy_branch_entry in ${confpy_branch_entries[@]}
251 # confpy_line_number=$(echo $confpy_branch_entry | awk -F ":" '{print $1}');
252 # confpy_branch_name=$(echo $confpy_branch_entry | awk -F ":" '{print $2}');
253 # echo "DBUG: ${confpy_branch_name} entries are below line ${confpy_line_number}"
256 # search in the list of branches in reverse order
257 for (( idx=${#confpy_branch_entries[@]}-1 ; idx>=0 ; idx-- ))
259 #echo "DBUG: working entry is ${confpy_branch_entries[idx]}"
260 confpy_line_number=$(echo ${confpy_branch_entries[idx]} | awk -F ":" '{print $1}');
261 confpy_branch_name=$(echo ${confpy_branch_entries[idx]} | awk -F ":" '{print $2}');
262 #echo "DBUG: ${confpy_branch_name} entries are below line ${confpy_line_number}"
264 if (( ${search_term_line_number} > ${confpy_line_number} )); then
265 #echo "DBUG: search_term_line_number is greater than confpy_line_number"
266 #echo "DBUG: ${search_term} found in ${confpy_branch_name} section"
267 return_from_find_repo_in_confpy=${confpy_branch_name}
274 ### arguments handling
286 if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then
290 echo "Error: Argument for $1 is missing" >&2
296 if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then
300 echo "Error: Argument for $1 is missing" >&2
305 -*|--*=) # unsupported flags
306 echo "Error: Unsupported argument $1" >&2
310 *) # preserve positional arguments
317 # set positional arguments in their proper place
318 eval set -- "$PARAMS"
320 # old: declare -a branches=("master" "frankfurt" "guilin")
321 if [[ $branches_csv == "" || $lfusername == "" ]]; then
326 # master branch is automatically added and must not part of the user arguments
327 if [[ $branches_csv == *"master"* ]]; then
331 # clone master first, then the other branches
332 branches_csv="master,${branches_csv}"
334 # create the branches array by readinging in the values from the variable
335 IFS=',' read -r -a branches <<< "${branches_csv}"
337 #echo "DBUG: devmode = \"${devmode}\""
338 #echo "DBUG: branches_csv = \"${branches_csv}\""
339 #echo "DBUG: lfusername = \"${lfusername}\""
340 #echo "DBUG: branches = \"${branches[@]}\""
342 # restart script with logging enabled
343 lockfile="checkdocs-runtime-lockfile"
344 if [ ! -f $lockfile ] ; then
346 echo "Restarting script with logging enabled."
347 ${fullcommand} 2>&1 | tee checkdocs.log
353 echo "checkdocs.sh Version ${script_version}"
357 # read in wiki (project) lifecycle state
358 # always use the lastest available file (derived from date in filename e.g. wiki_lifecycle_state_210409.txt)
359 # format is <reponame abbrev>;<state>;<reponame full>
362 wikiplsfile=$(ls | sed -nr '/wiki_lifecycle_state_[0-9]{6}.txt/Ip' | tail -1);
363 if [[ $wikiplsfile == "" ]]; then
364 echo "ERROR: wiki_lifecycle_state_yymmdd.txt missing"
367 echo "Using \"${wikiplsfile}\" as the source for wiki (project) lifecycle state information."
368 readarray -t wikiplsarray < ./${wikiplsfile};
371 # read in release_partizipation_YYMMDD.csv file
372 # always use the latest available file (derived from date in filename e.g. release_partizipation_210409.csv)
373 # format is: $1=repository;$2=project;$3=g;$4=gm;$5=h;$6=hm;$7=i;$8=im;$9=j;$10=jm;$11=k;$12=km;;;;
374 # example: "g" = project partizipated to the (g)uilin release
375 # "gm" = project partizipated to the (g)uilin (m)aintenance release
376 # file may contain windows control charaters at end of line (^M)
379 rpfile=$(ls | sed -nr '/release_partizipation_[0-9]{6}.csv/Ip' | tail -1);
380 if [[ $rpfile == "" ]]; then
381 echo "ERROR: release_partizipation_yymmdd.csv missing"
384 echo "Using \"${rpfile}\" as the source for release partizipation information."
385 readarray -t rparray < ./${rpfile};
387 rparray=("${rparray[@]:1}")
388 #printf '%s\n' "${rparray[@]}" #DBUG ONLY
391 # curl must be installed
394 if ! command -v curl &> /dev/null
396 echo "ERROR: curl command could not be found"
400 today=$(date '+%Y-%m-%d');
401 repolist="gerrit-repos-master-"$today".txt";
404 echo "Retrieving a full list of ONAP repositories (master) from gerrit.onap.org."
407 # retrieve the full repolist from gerrit
408 # workaround because of the (wrong?) response of gerrit.onap.org which makes jq command fail
409 # "| awk '{if(NR>1)print}'" filters the first line of the response so that jq will work again (thx marek)
412 curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d' >./$repolist
414 # process the created repolist and try to clone the projects from the mirror
416 source="git://cloud.onap.org/mirror"
417 echo "Using \"${source}\" as the source and username \"${lfusername}\" for cloning the repositories."
418 echo "Start cloning of repositories ..."
420 for branch in "${branches[@]}"
429 branch_upper=$(echo "${branch}" | tr '[:lower:]' '[:upper:]')
441 if [[ $devmode == "TRUE" ]]; then
442 devcounter=$((devcounter+1))
445 if [[ $devcounter -lt "50" ]]; then
447 if [[ $devmode == "TRUE" ]]; then
448 echo "INFO: devmode! counter=${devcounter}"
452 reponame=$(echo $line | awk -F "|" '{print $1}');
453 repostate=$(echo $line | awk -F "|" '{print $2}');
457 if [[ $repostate == "ACTIVE" ]] || [[ $repostate == "READ_ONLY" ]]; then
458 echo "Cloning \"${branch}\" branch of \"${repostate}\" project ${reponame}..."
460 # previously used: git clone --branch ${branch} --recurse-submodules ssh://${lfusername}@gerrit.onap.org:29418/$reponame ./$reponame
461 # clone script Jess: git clone "git://cloud.onap.org/mirror/${i}" "${LOCALNAME}"
462 git clone --branch ${branch} --recurse-submodules ${source}/${reponame} ./${reponame}
465 if [[ ! ${gitexitcode} == "0" ]]; then
466 errormsg=$(tail -1 ../checkdocs.log)
471 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
472 echo "${gitexitcode}|${reponame}|${repostate}|${errormsg}" | tee -a ${branch}_repoclone.log
474 #elif [[ $repostate == "READ_ONLY" ]]; then
475 #echo "-|${reponame}|${repostate}|ignored" | tee -a ${branch}_repoclone.log
477 echo "-|${reponame}|unknown repo state \"${repostate}\"|-" | tee -a ${branch}_repoclone.log
481 if [[ ${gitexitcode} == "0" ]]; then
483 printf "\ndocs directories:\n"
484 find ./$reponame -type d -name docs | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_docs.log
486 printf "\nrst files:\n"
487 find ./$reponame -type f -name *.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_rstfiles.log
489 printf "\nrelease notes rst:\n"
490 find ./$reponame -type f | grep '.*release.*note.*.rst' | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_releasenotes.log
492 printf "\ntox.ini files:\n"
493 find ./$reponame -type f -name tox.ini | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_toxini.log
495 printf "\nconf.py files:\n"
496 find ./$reponame -type f -name conf.py | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_confpy.log
498 printf "\nindex.rst files (all):\n"
499 find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst_all.log
501 printf "\nindex.rst files (docs root directory):\n"
502 find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | grep ']/docs/index.rst' | tee -a ${branch}_indexrst_docs_root.log
504 printf "\nINFO.yaml files:\n"
505 find ./$reponame -type f -name INFO.yaml | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_infoyaml.log
509 # end defcounter loop
516 # get (first) title for a rst file
518 python3 ../getrsttitle.py ${branch}_rstfiles.log | tee ${branch}_rstfiles_titles.log
520 python3 ../getrsttitle.py ${branch}_indexrst_docs_root.log | tee ${branch}_indexrst_docs_root_titles.log
524 find . -type f -name values.yaml -print -exec grep "image:" {} \; | sed -r 's:^ +::' | tee ${branch}_dockerimagesfull.log
526 ls --format single-column -d */ | sed 's:/$::' | tee ${branch}_directories.log
528 cat ${branch}_dockerimagesfull.log | grep image | sed -r 's:image\:::' | sed -r 's:^ +::' | sed '/^[[:space:]]*$/d' >${branch}_dockerimages.log
530 ls --format single-column -d oom/kubernetes/*/ | tee ${branch}_oomkubernetes.log
534 readarray -t docs_array < ./${branch}_docs.log;
536 for line in "${docs_array[@]}"
539 echo $line | tee -a ${branch}_docsconfig.log
541 # remove [ and ] which are distinguish the project name in the output
542 line=$(echo $line | sed -r 's:\[:: ; s:\]::')
544 if [ -f ./${line}/conf.py ] ; then
545 echo " conf.py ..... found" | tee -a ${branch}_docsconfig.log
547 echo " conf.py ..... NOT FOUND" | tee -a ${branch}_docsconfig.log
550 if [ -f ./${line}/index.rst ] ; then
551 echo " index.rst ... found" | tee -a ${branch}_docsconfig.log
553 echo " index.rst ... NOT FOUND" | tee -a ${branch}_docsconfig.log
556 if [ -f ./${line}/tox.ini ] ; then
557 echo " tox.ini ..... found" | tee -a ${branch}_docsconfig.log
559 echo " tox.ini ..... NOT FOUND" | tee -a ${branch}_docsconfig.log
562 echo " " | tee -a ${branch}_docsconfig.log
570 ### build a csv table that combines results
574 # csv column #1: project name
577 readarray -t array < ./${repolist};
581 for line in "${array[@]}"
583 reponame=$(echo $line | awk -F "|" '{print $1}');
584 project=$(echo $reponame | sed 's:/.*$::')
585 #echo "DBUG: reponame=${reponame}"
586 #echo "DBUG: project=${project}"
597 # csv column #2: repo name
600 readarray -t array < ./${repolist};
602 csv[i]="${csv[i]},MASTER repo name"
604 for line in "${array[@]}"
606 reponame=$(echo $line | awk -F "|" '{print $1}');
607 csv[i]="${csv[i]},${reponame}"
615 # csv column #3: repo state
618 readarray -t array < ./${repolist};
620 csv[i]="${csv[i]},MASTER repo state"
622 for line in "${array[@]}"
624 repostate=$(echo $line | awk -F "|" '{print $2}');
625 csv[i]="${csv[i]},${repostate}"
633 # csv column #4: clone message
636 readarray -t array < ./${branch}_repoclone.log;
638 csv[i]="${csv[i]},${branch_upper} clone message"
640 for line in "${array[@]}"
642 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
643 errormsg=$(echo $line | awk -F "|" '{print $4}');
644 csv[i]="${csv[i]},${errormsg}"
652 # csv column #5: latest branch
655 readarray -t array < ./${repolist};
657 csv[i]="${csv[i]},latest branch"
659 for line in "${array[@]}"
661 reponame=$(echo $line | awk -F "|" '{print $1}');
662 latestbranch=$(git ls-remote -q --heads "${source}/${reponame}" | sed 's/^.*heads\///' | sed -nr '/^master$|^amsterdam$|^beijing$|^casablanca$|^dublin$|^elalto$|^frankfurt$|^guilin$|^honolulu$|^istanbul$/Ip' | tail -2 | head -1);
663 #echo "DBUG: reponame=${reponame}"
664 #echo "DBUG: latestbranch=${latestbranch}"
665 echo "latest available branch for repo \"${reponame}\" is \"${latestbranch}\""
666 csv[i]="${csv[i]},${latestbranch}"
675 # csv column #6: INFO.yaml LC state (project lifecycle state based on INFO.yaml / per repo)
676 # csv column #7: WIKI LC state (project lifecycle state based on ONAP Dev Wiki / per project)
677 # csv column #8: LC state match shows a "match" if both LC states match
680 readarray -t array < ./${repolist};
682 csv[i]="${csv[i]},INFO.yaml LC state,WIKI LC state,LC state match"
684 for line in "${array[@]}"
686 reponame=$(echo $line | awk -F "|" '{print $1}');
687 project=$(echo $reponame | sed 's:/.*$::')
689 if [ -f ./${reponame}/INFO.yaml ] ; then
690 # check if repo/branch has a INFO.yaml
691 lifecycleproject=$(grep '^project: ' ./${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
692 lifecyclestate=$(grep '^lifecycle_state: ' ./${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
693 elif [ ${branch} != "master" ] && [ -f ../master/${reponame}/INFO.yaml ] ; then
694 # IF current branch is not master AND if info.yaml not found in the current repo/branch THAN use INFO.yaml of repo/master if available
695 #echo "DBUG: branch=${branch} - checking master for INFO.yaml"
696 lifecycleproject=$(grep '^project: ' ../master/${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
697 lifecyclestate=$(grep '^lifecycle_state: ' ../master/${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
698 lifecyclestate="(${lifecyclestate})"
700 lifecyclestate="INFO.yaml not found"
703 getwikilifecyclestate ${project}
704 # returns value in ${return_from_getwikilifecyclestate}
706 #echo "DBUG: working dir is ...";pwd
707 #echo "DBUG: lifecycleproject=${lifecycleproject}"
708 #echo "DBUG: lifecyclestate=${lifecyclestate}"
709 #echo "DBUG: wikilifecyclestate=${return_from_getwikilifecyclestate}"
711 #check if YAML.info LC state is not empty _AND_ if WIKI LC state is not empty _AND_ if YAML.info LC state contains WIKI LC state
712 if [[ ${lifecyclestate} != "" ]] && [[ ${return_from_getwikilifecyclestate} != "" ]] && [[ ${lifecyclestate} == *"${return_from_getwikilifecyclestate}"* ]]; then
713 lcstatesmatch="match"
718 csv[i]="${csv[i]},${lifecyclestate},${return_from_getwikilifecyclestate},${lcstatesmatch}"
725 unset lifecycleproject
730 # csv column #9: intersphinx
731 # intersphinx mappings in conf.py
732 # provided is the branch used for linking the repository
735 readarray -t array < ./${repolist};
737 csv[i]="${csv[i]},intersphinx"
739 for line in "${array[@]}"
741 reponame=$(echo $line | awk -F "|" '{print $1}');
742 project=$(echo $reponame | sed 's:/.*$::')
743 #echo "DBUG: reponame=${reponame}"
744 #echo "DBUG: project=${project}"
746 reponame=$(echo ${reponame} | sed -r 's/\//-/g')
747 search_repo="onap-${reponame}"
748 #echo "DBUG: search_repo=${search_repo}"
749 find_repo_in_confpy ${search_repo}
750 csv[i]="${csv[i]},${return_from_find_repo_in_confpy}"
757 unset return_from_find_repo_in_confpy
760 # csv column #10: RELEASE component (yes|maybe|unknown)
761 # to be filled with values of the planned release config file maintained by
762 # the onap release manager
766 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
767 readarray -t array < ./${branch}_repoclone.log;
769 csv[i]="${csv[i]},${branch_upper} component"
771 for line in "${array[@]}"
774 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
775 gitexitcode=$(echo $line | awk -F "|" '{print $1}');
776 reponame=$(echo $line | awk -F "|" '{print $2}');
777 repostate=$(echo $line | awk -F "|" '{print $3}');
778 errormsg=$(echo $line | awk -F "|" '{print $4}');
780 #if [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "0" ]]; then
781 # releasecomponent="yes"
782 #elif [ ${repostate} == "ACTIVE" ]; then
783 ##elif [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "128" ]]; then
784 # releasecomponent="maybe"
785 #elif [[ ${repostate} == "READ_ONLY" && ${gitexitcode} == "0" ]]; then
786 # releasecomponent="yes"
787 #elif [ ${repostate} == "READ_ONLY" ]; then
788 # releasecomponent="maybe"
790 # releasecomponent="unknown"
793 # not functional yet!
796 csv[i]="${csv[i]},${releasecomponent}"
805 unset releasecomponent
808 # csv column #11: RELEASE partizipation
811 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
812 readarray -t array < ./${branch}_repoclone.log;
814 csv[i]="${csv[i]},${branch_upper} partizipation"
816 echo "INFO: determine release partizipation for project ..."
817 for line in "${array[@]}"
820 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
821 gitexitcode=$(echo $line | awk -F "|" '{print $1}');
822 reponame=$(echo $line | awk -F "|" '{print $2}');
823 repostate=$(echo $line | awk -F "|" '{print $3}');
824 errormsg=$(echo $line | awk -F "|" '{print $4}');
825 projectname=$(echo $reponame | sed 's:/.*$::')
827 if [[ $branch == "master" ]]; then
828 return_from_getrpinfo="";
830 #echo "DBUG: calling getrpinfo for projectname ${projectname}"
831 getrpinfo ${projectname}
834 csv[i]="${csv[i]},${return_from_getrpinfo}"
846 unset return_from_getrpinfo
849 # csv column #12: docs (at repo root directory only; no recursive search!)
850 # csv column #13: conf.py
851 # csv column #14: tox.ini
852 # csv column #15: index.rst
853 # csv column #16: first title in index.rst
855 # columns are filled with values from requested branch.
856 # if data is not available values from master branch are used.
857 # to identify master branch values, data is put into round brackets "(...)"
860 readarray -t array < ./${repolist};
862 csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst,first title in index.rst"
864 for line in "${array[@]}"
866 line=$(echo $line | sed 's:|.*$::')
867 #echo "DBUG: line=${line}"
871 if [ -d ./${line}/docs ] ; then
873 elif [ -d ../master/${line}/docs ] ; then
880 if [ -f ./${line}/docs/conf.py ] ; then
881 docs="${docs},conf.py"
882 elif [ -f ../master/${line}/docs/conf.py ] ; then
883 docs="${docs},(conf.py)"
888 # tox.ini (check docs dir and also check project root dir)
889 if [ -f ./${line}/docs/tox.ini ] || [ -f ./${line}/tox.ini ]; then
890 docs="${docs},tox.ini"
891 # tox.ini @ branch/docs dir
892 if [ -f ./${line}/docs/tox.ini ] ; then
895 # tox.ini @ branch/project root dir
896 if [ -f ./${line}/tox.ini ] ; then
899 elif [ -f ../master/${line}/docs/tox.ini ] || [ -f ../master/${line}/tox.ini ]; then
900 docs="${docs},(tox.ini"
901 # tox.ini @ master/docs dir
902 if [ -f ../master/${line}/docs/tox.ini ] ; then
905 # tox.ini @ master/project root dir
906 if [ -f ../master/${line}/tox.ini ] ; then
909 # just add a round bracket at the end of the value
912 # no tox.ini found in docs or root dir
916 # index.rst, first title in index.rst
918 if [ -f ./${line}/docs/index.rst ] ; then
919 indexrsttitle=$(cat ${branch}_indexrst_docs_root_titles.log | grep -F '['${line}']/docs/index.rst,' | awk -F "," '{print $4}');
920 docs="${docs},index.rst,${indexrsttitle}"
921 elif [ -f ../master/${line}/docs/index.rst ] ; then
922 indexrsttitle=$(cat ../master/master_indexrst_docs_root_titles.log | grep -F '['${line}']/docs/index.rst,' | awk -F "," '{print $4}');
923 docs="${docs},(index.rst),(${indexrsttitle})"
928 #echo "DBUG: docs=${docs}"
929 line="${csv[i]},${docs}"
938 # csv column #17: index.html@RTD accessibility check
939 # csv column #18: index.html url
942 readarray -t array < ./${branch}_repoclone.log;
944 csv[i]="${csv[i]},index.html@RTD,index.html url"
946 for line in "${array[@]}"
948 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
949 gitexitcode=$(echo $line | awk -F "|" '{print $1}');
950 reponame=$(echo $line | awk -F "|" '{print $2}');
951 repostate=$(echo $line | awk -F "|" '{print $3}');
952 errormsg=$(echo $line | awk -F "|" '{print $4}');
957 # this script works only with release "frankfurt" and later because
958 # earlier releases are using submodule structure for documentation files
959 if echo "$branch" | grep -q '^[abcde]'; then
960 curl_result="unsupported release"
964 # we are working on "frankfurt" branch or later ...
965 if [[ ${repostate} == "ACTIVE" ]] || [[ ${repostate} == "READ_ONLY" ]]; then
967 # OPTIONAL: USE ALSO GITEXITCODE AS A FILTER CRITERIA ???
970 # important! only doc project needs a different url base
971 if [[ ${reponame} == "doc" ]]; then
972 url_start="https://docs.onap.org"
974 url_start="https://docs.onap.org/projects/onap"
979 # "master" branch documentation is available as "latest" in RTD
980 if [[ ${url_branch} == "master" ]]; then
984 # replace all / characters in repo name with - charachter
985 url_repo=$(echo ${reponame} | sed -r 's/\//-/g')
986 url_file="index.html"
989 if [[ ${reponame} == "doc" ]]; then
990 # build the full url for the doc project
991 url="${url_start}/${url_lang}/${url_branch}/${url_file}"
993 # build the full url for the other projects
994 url="${url_start}-${url_repo}/${url_lang}/${url_branch}/${url_file}"
996 #echo "DBUG: url=$url"
998 # test accessibility of url
999 curl --head --silent --fail "${url}?${unique}" >/dev/null
1002 # convert numeric results to text
1003 if [ "${curl_result}" = "0" ]; then
1004 curl_result="accessible"
1005 elif [ "${curl_result}" = "22" ]; then
1006 curl_result="does not exist"
1008 curl_result="ERROR:${curl_result}"
1011 # url does not exist for this branch.
1012 # in case the requested url is not already for "master" branch,
1013 # we try to access the url of the master branch and denote the
1014 # result by using round brackets (result)
1015 if [[ ${curl_result} == "does not exist" && ! $branch == "master" ]]; then
1017 # build the full (master/latest) url
1018 url="${url_start}-${url_repo}/${url_lang}/latest/${url_file}"
1019 #echo "DBUG: url=$url"
1021 # test accessibility of url in "master branch" (latest)
1022 curl --head --silent --fail "${url}?${unique}" >/dev/null
1024 # denote result as a value from "master" branch (latest)
1027 # convert numeric results to text
1028 if [ "${curl_result}" = "0" ]; then
1029 curl_result="(accessible)"
1030 elif [ "${curl_result}" = "22" ]; then
1031 curl_result="(does not exist)"
1033 curl_result="(ERROR:${curl_result})"
1038 # repostate IS NOT ACTIVE OR READ_ONLY - no curl test required
1044 echo "$url ... $curl_result"
1045 csv[i]="${csv[i]},${curl_result},${url}"
1046 #echo "DBUG: csv line=${csv[i]}"
1052 # csv column #19: release notes
1055 readarray -t array < ../${repolist};
1057 csv[i]="${csv[i]},release notes"
1059 for line in "${array[@]}"
1061 line=$(echo $line | sed 's:|.*$::')
1062 #echo "DBUG: line=\"${line}\""
1063 #echo "DBUG: i=${i}"
1066 # put repo name in square brackets for increased grep hit rate
1067 # escape minus and bracket characters to avoid problems with the grep command
1068 #repo_grepable=$(echo ${line} | sed -r s:${line}:[${line}]: | sed -r 's/-/\\-/g' | sed -r 's/\[/\\[/g' | sed -r 's/\]/\\]/g')
1069 #echo "DBUG: repo_grepable=\"${repo_grepable}\""
1071 # check if repo dir exists in this branch
1072 if [ -d ./${line} ] ; then
1073 # if yes, check if repo name appears in the branch releasenotes.log
1074 relnote=$(find "./${line}" -type f | grep '.*release.*note.*.rst' | wc -l);
1075 #echo "DBUG: relnote=${relnote}"
1076 # repo dir DOES NOT exist in this branch - so check if repo dir exists in MASTER branch
1077 elif [ -d ../master/${line} ] ; then
1078 # if yes, check if repo name appears in the MASTER releasenotes.log
1079 # count release notes files in MASTER branch (in repo root and its subdirectories)
1080 relnote=$(find "../master/${line}" -type f | grep 'release.*note.*.rst' | wc -l);
1081 #echo "DBUG: relnote=${relnote}"
1082 # put results in round brackets to show that this is MASTER data
1083 relnote=$(echo ${relnote} | sed -r s:${relnote}:\(${relnote}\):)
1087 #echo "DBUG: relnote=${relnote}"
1089 line="${csv[i]},${relnote}"
1100 # build the table.csv file
1103 for i in "${csv[@]}"
1105 echo "$i" | tee -a ./${branch}_table.csv
1109 # create data package for this branch and zip it
1112 datadir=${branch}_data
1114 cp $repolist $datadir
1115 cp ../$wikiplsfile $datadir
1116 cp ../$rpfile $datadir
1117 cp ${branch}_table.csv $datadir
1118 cp ${branch}_*.log $datadir
1119 zip -r ${datadir}.zip $datadir
1121 # return from the branch directory
1124 # return and work on the next requested branch ... or exit