2 #set -x # uncomment for bash script debugging
4 ### ============================================================================
5 ### Licensed under the Apache License, Version 2.0 (the "License");
6 ### you may not use this file except in compliance with the License.
7 ### You may obtain a copy of the License at
9 ### http://www.apache.org/licenses/LICENSE-2.0
11 ### Unless required by applicable law or agreed to in writing, software
12 ### distributed under the License is distributed on an "AS IS" BASIS,
13 ### WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 ### See the License for the specific language governing permissions and
15 ### limitations under the License.
16 ### ============LICENSE_END=====================================================
22 ### Thomas Kulik, Deutsche Telekom AG, 2020 - 2021
25 ### Retrieves a full list of ONAP repos from gerrit inluding their state.
26 ### Clones all active repos of the ONAP master branch plus other requested ONAP
27 ### branches. Then the script does some docs related analyses depending on the
28 ### clone results. It creates logfiles containing filtered results. In addition
29 ### a table.csv is created which can be used to import it in a spreadsheed.
30 ### Also a zip-file is created which contains all the results.
33 ### - in the output, repo names are shown in square brackets for readability
34 ### e.g [aai/aai-common]/docs/release-notes.rst
35 ### - in the table.csv file you see data for the requested branch if available.
36 ### if not available, data is retrieved from the master branch. it will be
37 ### denoted in round brackets, e.g. (3) (tox.ini)
45 ### SOME HELPING COMMANDS TO PROCESS LOG FILES:
47 ### curl -s https://git.onap.org/ | grep "^<tr><td class='toplevel-repo'><a title='" | sed -r "s:^<tr><td class='toplevel-repo'><a title='::" | sed -r "s:'.*::"
49 ### remove branchname from the line:
50 ### cat frankfurt_repoclone.log | sed 's:frankfurt|::'
52 ### list only image names
53 ### cat master_dockerimagesfull.log | grep image | sed -r 's:image\:::' | sed -r 's:^ +::' | sed '/^[[:space:]]*$/d'
55 ### more interesting stuff ...
56 ### curl https://gerrit.onap.org/r/projects/?d
57 ### LONG: curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g' | sed -r 's:["{}]::g' | sed -r 's:id\:::' | sed -r 's:,state\::|:' | sed '/All-Projects/d' | sed '/All-Users/d'
58 ### SHORT: curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d'
61 script_version="1.7 (2021-04-12)"
63 # save command for the restart with logging enabled
66 fullcommand="${command} ${arguments}"
75 echo " checkdocs.sh Version ${script_version}"
78 echo " ./checkdocs.sh <arguments> "
81 echo " -u|--user username "
82 echo " linux foundation username used to clone ONAP repositories"
84 echo " -b|--branches branch1,branch2,branch3 "
85 echo " list of branches to be cloned. master is automatically "
86 echo " added to the list. do not add manually! "
89 echo " development-mode - limits number of repos to be cloned "
95 echo "*******************************************************************************"
98 # remove lockfile in case script is interrupted
99 trap InterruptedScript SIGINT SIGTERM SIGHUP SIGKILL SIGSTOP
100 function InterruptedScript {
102 echo "Script was interrupted."
103 if [ -f $lockfile ] ; then
109 # function to parse wiki (project) lifecycle state information
110 # call: getwikilifecyclestate "projectname"
111 # result: $return_from_getwikilifecyclestate
112 # because bash supports only returning numeric values a variable $return_from_getwikilifecyclestate is used
114 function getwikilifecyclestate {
121 return_from_getwikilifecyclestate=""
123 for wikiline in "${wikiplsarray[@]}"
126 wikirepo=$(echo $wikiline | awk -F ";" '{print $1}');
127 wikistate=$(echo $wikiline | awk -F ";" '{print $2}');
129 #echo "DBUG: getwikilifecyclestate wikiline = \"${wikiline}\"";
130 #echo "DBUG: getwikilifecyclestate wikirepo = \"${wikirepo}\""
131 #echo "DBUG: getwikilifecyclestate wikistate = \"${wikistate}\""
133 if [[ ${wikirepo} == ${requested} ]]; then
134 return_from_getwikilifecyclestate=${wikistate}
135 #echo "DBUG: getwikilifecyclestate wikirepo = \"${wikirepo}\""
136 #echo "DBUG: getwikilifecyclestate requested = \"${requested}\""
137 #echo "DBUG: return_from_getwikilifecyclestate = \"${return_from_getwikilifecyclestate}\"";
143 #echo "DBUG: getwikilifecyclestate requested \"${requested}\" NOT FOUND in list"
144 return_from_getwikilifecyclestate=""
149 ### arguments handling
161 if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then
165 echo "Error: Argument for $1 is missing" >&2
171 if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then
175 echo "Error: Argument for $1 is missing" >&2
180 -*|--*=) # unsupported flags
181 echo "Error: Unsupported argument $1" >&2
185 *) # preserve positional arguments
192 # set positional arguments in their proper place
193 eval set -- "$PARAMS"
195 # old: declare -a branches=("master" "frankfurt" "guilin")
196 if [[ $branches_csv == "" || $lfusername == "" ]]; then
201 # master branch is automatically added and must not part of the user arguments
202 if [[ $branches_csv == *"master"* ]]; then
206 # clone master first, then the other branches
207 branches_csv="master,${branches_csv}"
209 # create the branches array by readinging in the values from the variable
210 IFS=',' read -r -a branches <<< "${branches_csv}"
212 #echo "DBUG: devmode = \"${devmode}\""
213 #echo "DBUG: branches_csv = \"${branches_csv}\""
214 #echo "DBUG: lfusername = \"${lfusername}\""
215 #echo "DBUG: branches = \"${branches[@]}\""
217 # restart script with logging enabled
218 lockfile="checkdocs-runtime-lockfile"
219 if [ ! -f $lockfile ] ; then
221 echo "Restarting script with logging enabled."
222 ${fullcommand} 2>&1 | tee checkdocs.log
228 echo "checkdocs.sh Version ${script_version}"
232 # read in wiki (project) lifecycle state
233 # always use the lastest available file (derived from date in filename e.g. wiki_lifecycle_state_210409.txt)
234 # format is <reponame abbrev>;<state>;<reponame full>
237 wikiplsfile=$(ls | sed -nr '/wiki_lifecycle_state_[0-9]{6}.txt/Ip' | tail -1);
239 if [[ $wikiplsfile == "" ]]; then
240 echo "ERROR: wiki_lifecycle_state_yymmdd.txt missing"
244 echo "Using \"${wikiplsfile}\" as the source for wiki (project) lifecycle state information."
246 readarray -t wikiplsarray < ./${wikiplsfile};
249 for line in "${wikiplsarray[@]}"
251 wikiplsrepo=$(echo $line | awk -F ";" '{print $1}');
252 wikiplsstate=$(echo $line | awk -F ";" '{print $2}');
253 #echo "DBUG: wikipls line=\"${line}\"";
254 #echo "DBUG: wikipls ${wikiplsrepo}=${wikiplsstate}"
262 # curl must be installed
265 if ! command -v curl &> /dev/null
267 echo "ERROR: curl command could not be found"
271 today=$(date '+%Y-%m-%d');
272 repolist="gerrit-repos-master-"$today".txt";
275 echo "Retrieving a full list of ONAP repositories (master) from gerrit.onap.org."
278 # retrieve the full repolist from gerrit
279 # workaround because of the (wrong?) response of gerrit.onap.org which makes jq command fail
280 # "| awk '{if(NR>1)print}'" filters the first line of the response so that jq will work again (thx marek)
283 curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d' >./$repolist
285 # process the created repolist and try to clone the projects from the mirror
287 source="git://cloud.onap.org/mirror"
288 echo "Using \"${source}\" as the source and username \"${lfusername}\" for cloning the repositories."
289 echo "Start cloning of repositories ..."
291 for branch in "${branches[@]}"
300 branch_upper=$(echo "${branch}" | tr '[:lower:]' '[:upper:]')
312 if [[ $devmode == "TRUE" ]]; then
313 devcounter=$((devcounter+1))
316 if [[ $devcounter -lt "50" ]]; then
318 if [[ $devmode == "TRUE" ]]; then
319 echo "INFO: devmode! counter=${devcounter}"
323 reponame=$(echo $line | awk -F "|" '{print $1}');
324 repostate=$(echo $line | awk -F "|" '{print $2}');
328 if [[ $repostate == "ACTIVE" ]] || [[ $repostate == "READ_ONLY" ]]; then
329 echo "Cloning \"${branch}\" branch of \"${repostate}\" project ${reponame}..."
331 # previously used: git clone --branch ${branch} --recurse-submodules ssh://${lfusername}@gerrit.onap.org:29418/$reponame ./$reponame
332 # clone script Jess: git clone "git://cloud.onap.org/mirror/${i}" "${LOCALNAME}"
333 git clone --branch ${branch} --recurse-submodules ${source}/${reponame} ./${reponame}
336 if [[ ! ${gitexitcode} == "0" ]]; then
337 errormsg=$(tail -1 ../checkdocs.log)
342 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
343 echo "${gitexitcode}|${reponame}|${repostate}|${errormsg}" | tee -a ${branch}_repoclone.log
345 #elif [[ $repostate == "READ_ONLY" ]]; then
346 #echo "-|${reponame}|${repostate}|ignored" | tee -a ${branch}_repoclone.log
348 echo "-|${reponame}|unknown repo state \"${repostate}\"|-" | tee -a ${branch}_repoclone.log
352 if [[ ${gitexitcode} == "0" ]]; then
354 printf "\ndocs directories:\n"
355 find ./$reponame -type d -name docs | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_docs.log
357 printf "\nrst files:\n"
358 find ./$reponame -type f -name *.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_rstfiles.log
360 printf "\nrelease notes rst:\n"
361 find ./$reponame -type f | grep 'release.*note.*.rst' | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_releasenotes.log
363 printf "\ntox.ini files:\n"
364 find ./$reponame -type f -name tox.ini | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_toxini.log
366 printf "\nconf.py files:\n"
367 find ./$reponame -type f -name conf.py | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_confpy.log
369 printf "\nindex.rst files:\n"
370 find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst.log
372 printf "\nINFO.yaml files:\n"
373 find ./$reponame -type f -name INFO.yaml | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_infoyaml.log
377 # end defcounter loop
386 find . -type f -name values.yaml -print -exec grep "image:" {} \; | sed -r 's:^ +::' | tee ${branch}_dockerimagesfull.log
388 ls --format single-column -d */ | sed 's:/$::' | tee ${branch}_directories.log
390 cat ${branch}_dockerimagesfull.log | grep image | sed -r 's:image\:::' | sed -r 's:^ +::' | sed '/^[[:space:]]*$/d' >${branch}_dockerimages.log
392 ls --format single-column -d oom/kubernetes/*/ | tee ${branch}_oomkubernetes.log
396 readarray -t docs_array < ./${branch}_docs.log;
398 for line in "${docs_array[@]}"
401 echo $line | tee -a ${branch}_docsconfig.log
403 # remove [ and ] which are distinguish the project name in the output
404 line=$(echo $line | sed -r 's:\[:: ; s:\]::')
406 if [ -f ./${line}/conf.py ] ; then
407 echo " conf.py ..... found" | tee -a ${branch}_docsconfig.log
409 echo " conf.py ..... NOT FOUND" | tee -a ${branch}_docsconfig.log
412 if [ -f ./${line}/index.rst ] ; then
413 echo " index.rst ... found" | tee -a ${branch}_docsconfig.log
415 echo " index.rst ... NOT FOUND" | tee -a ${branch}_docsconfig.log
418 if [ -f ./${line}/tox.ini ] ; then
419 echo " tox.ini ..... found" | tee -a ${branch}_docsconfig.log
421 echo " tox.ini ..... NOT FOUND" | tee -a ${branch}_docsconfig.log
424 echo " " | tee -a ${branch}_docsconfig.log
432 ### build a csv table that combines results
436 # csv column #1: project name
439 readarray -t array < ./${repolist};
443 for line in "${array[@]}"
445 reponame=$(echo $line | awk -F "|" '{print $1}');
446 project=$(echo $reponame | sed 's:/.*$::')
447 #echo "DBUG: reponame=${reponame}"
448 #echo "DBUG: project=${project}"
459 # csv column #2: repo name
462 readarray -t array < ./${repolist};
464 csv[i]="${csv[i]},MASTER repo name"
466 for line in "${array[@]}"
468 reponame=$(echo $line | awk -F "|" '{print $1}');
469 csv[i]="${csv[i]},${reponame}"
477 # csv column #3: repo state
480 readarray -t array < ./${repolist};
482 csv[i]="${csv[i]},MASTER repo state"
484 for line in "${array[@]}"
486 repostate=$(echo $line | awk -F "|" '{print $2}');
487 csv[i]="${csv[i]},${repostate}"
495 # csv column #4: clone message
498 readarray -t array < ./${branch}_repoclone.log;
500 csv[i]="${csv[i]},${branch_upper} clone message"
502 for line in "${array[@]}"
504 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
505 errormsg=$(echo $line | awk -F "|" '{print $4}');
506 csv[i]="${csv[i]},${errormsg}"
514 # csv column #5: latest branch
517 readarray -t array < ./${repolist};
519 csv[i]="${csv[i]},latest branch"
521 for line in "${array[@]}"
523 reponame=$(echo $line | awk -F "|" '{print $1}');
524 latestbranch=$(git ls-remote -q --heads "${source}/${reponame}" | sed 's/^.*heads\///' | sed -nr '/^master$|^amsterdam$|^beijing$|^casablanca$|^dublin$|^elalto$|^frankfurt$|^guilin$|^honolulu$|^istanbul$/Ip' | tail -2 | head -1);
525 #echo "DBUG: reponame=${reponame}"
526 #echo "DBUG: latestbranch=${latestbranch}"
527 echo "latest available branch for repo \"${reponame}\" is \"${latestbranch}\""
528 csv[i]="${csv[i]},${latestbranch}"
537 # csv column #6: INFO.yaml LC state (project lifecycle state based on INFO.yaml / per repo)
538 # csv column #7: WIKI LC state (project lifecycle state based on ONAP Dev Wiki / per project)
539 # csv column #8: LC state match shows a "match" if both LC states match
542 readarray -t array < ./${repolist};
544 csv[i]="${csv[i]},INFO.yaml LC state,WIKI LC state,LC state match"
546 for line in "${array[@]}"
548 reponame=$(echo $line | awk -F "|" '{print $1}');
549 project=$(echo $reponame | sed 's:/.*$::')
551 if [ -f ./${reponame}/INFO.yaml ] ; then
552 # check if repo/branch has a INFO.yaml
553 lifecycleproject=$(grep '^project: ' ./${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
554 lifecyclestate=$(grep '^lifecycle_state: ' ./${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
555 elif [ ${branch} != "master" ] && [ -f ../master/${reponame}/INFO.yaml ] ; then
556 # if current branch is not master AND if info.yaml not found in the current repo/branch THAN use INFO.yaml of repo/master if available
557 #echo "DBUG: branch=${branch} - checking master for INFO.yaml"
558 lifecycleproject=$(grep '^project: ' ../master/${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
559 lifecyclestate=$(grep '^lifecycle_state: ' ../master/${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
560 lifecyclestate="(${lifecyclestate})"
562 lifecyclestate="INFO.yaml not found"
565 getwikilifecyclestate ${project}
566 # returns value in ${return_from_getwikilifecyclestate}
568 #echo "DBUG: working dir is ...";pwd
569 #echo "DBUG: lifecycleproject=${lifecycleproject}"
570 #echo "DBUG: lifecyclestate=${lifecyclestate}"
571 #echo "DBUG: wikilifecyclestate=${return_from_getwikilifecyclestate}"
573 #check if YAML.info LC state is not empty _AND_ if WIKI LC state is not empty _AND_ if YAML.info LC state contains WIKI LC state
574 if [[ ${lifecyclestate} != "" ]] && [[ ${return_from_getwikilifecyclestate} != "" ]] && [[ ${lifecyclestate} == *"${return_from_getwikilifecyclestate}"* ]]; then
575 lcstatesmatch="match"
580 csv[i]="${csv[i]},${lifecyclestate},${return_from_getwikilifecyclestate},${lcstatesmatch}"
587 unset lifecycleproject
592 # csv column #9: RELEASE component (yes|maybe|unknown)
593 # to be filled with values of the planned release config file maintained by
594 # the onap release manager
597 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
598 readarray -t array < ./${branch}_repoclone.log;
600 csv[i]="${csv[i]},${branch_upper} component"
602 for line in "${array[@]}"
605 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
606 gitexitcode=$(echo $line | awk -F "|" '{print $1}');
607 reponame=$(echo $line | awk -F "|" '{print $2}');
608 repostate=$(echo $line | awk -F "|" '{print $3}');
609 errormsg=$(echo $line | awk -F "|" '{print $4}');
611 if [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "0" ]]; then
612 releasecomponent="yes"
613 elif [ ${repostate} == "ACTIVE" ]; then
614 #elif [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "128" ]]; then
615 releasecomponent="maybe"
616 elif [[ ${repostate} == "READ_ONLY" && ${gitexitcode} == "0" ]]; then
617 releasecomponent="yes"
618 elif [ ${repostate} == "READ_ONLY" ]; then
619 releasecomponent="maybe"
621 releasecomponent="unknown"
624 csv[i]="${csv[i]},${releasecomponent}"
633 unset releasecomponent
636 # csv column #10: docs (at repo root directory only; no recursive search!)
637 # csv column #11: conf.py
638 # csv column #12: tox.ini
639 # csv column #13: index.rst
641 # columns are filled with values from requested branch.
642 # if data is not available values from master branch are used.
643 # to identify master branch values, data is put into round brackets "(...)"
646 readarray -t array < ./${repolist};
648 csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst"
650 for line in "${array[@]}"
652 line=$(echo $line | sed 's:|.*$::')
653 #echo "DBUG: line=${line}"
657 if [ -d ./${line}/docs ] ; then
659 elif [ -d ../master/${line}/docs ] ; then
666 if [ -f ./${line}/docs/conf.py ] ; then
667 docs="${docs},conf.py"
668 elif [ -f ../master/${line}/docs/conf.py ] ; then
669 docs="${docs},(conf.py)"
674 # tox.ini (check docs dir and also check project root dir)
675 if [ -f ./${line}/docs/tox.ini ] || [ -f ./${line}/tox.ini ]; then
676 docs="${docs},tox.ini"
677 # tox.ini @ branch/docs dir
678 if [ -f ./${line}/docs/tox.ini ] ; then
681 # tox.ini @ branch/project root dir
682 if [ -f ./${line}/tox.ini ] ; then
685 elif [ -f ../master/${line}/docs/tox.ini ] || [ -f ../master/${line}/tox.ini ]; then
686 docs="${docs},(tox.ini"
687 # tox.ini @ master/docs dir
688 if [ -f ../master/${line}/docs/tox.ini ] ; then
691 # tox.ini @ master/project root dir
692 if [ -f ../master/${line}/tox.ini ] ; then
695 # just add a round bracket at the end of the value
698 # no tox.ini found in docs or root dir
703 if [ -f ./${line}/docs/index.rst ] ; then
704 docs="${docs},index.rst"
705 elif [ -f ../master/${line}/docs/index.rst ] ; then
706 docs="${docs},(index.rst)"
711 #echo "DBUG: docs=${docs}"
712 line="${csv[i]},${docs}"
721 # csv column #14: index.html@RTD accessibility check
722 # csv column #15: index.html url
725 readarray -t array < ./${branch}_repoclone.log;
727 csv[i]="${csv[i]},index.html@RTD,index.html url"
729 for line in "${array[@]}"
731 # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
732 gitexitcode=$(echo $line | awk -F "|" '{print $1}');
733 reponame=$(echo $line | awk -F "|" '{print $2}');
734 repostate=$(echo $line | awk -F "|" '{print $3}');
735 errormsg=$(echo $line | awk -F "|" '{print $4}');
740 # this script works only with release "frankfurt" and later because
741 # earlier releases are using submodule structure for documentation files
742 if echo "$branch" | grep -q '^[abcde]'; then
743 curl_result="unsupported release"
747 # we are working on "frankfurt" branch or later ...
748 if [[ ${repostate} == "ACTIVE" ]] || [[ ${repostate} == "READ_ONLY" ]]; then
750 # OPTIONAL: USE ALSO GITEXITCODE AS A FILTER CRITERIA ???
753 # important! only doc project needs a different url base
754 if [[ ${reponame} == "doc" ]]; then
755 url_start="https://docs.onap.org"
757 url_start="https://docs.onap.org/projects/onap"
762 # "master" branch documentation is available as "latest" in RTD
763 if [[ ${url_branch} == "master" ]]; then
767 # replace all / characters in repo name with - charachter
768 url_repo=$(echo ${reponame} | sed -r 's/\//-/g')
769 url_file="index.html"
772 if [[ ${reponame} == "doc" ]]; then
773 # build the full url for the doc project
774 url="${url_start}/${url_lang}/${url_branch}/${url_file}"
776 # build the full url for the other projects
777 url="${url_start}-${url_repo}/${url_lang}/${url_branch}/${url_file}"
779 #echo "DBUG: url=$url"
781 # test accessibility of url
782 curl --head --silent --fail "${url}?${unique}" >/dev/null
785 # convert numeric results to text
786 if [ "${curl_result}" = "0" ]; then
787 curl_result="accessible"
788 elif [ "${curl_result}" = "22" ]; then
789 curl_result="does not exist"
791 curl_result="ERROR:${curl_result}"
794 # url does not exist for this branch.
795 # in case the requested url is not already for "master" branch,
796 # we try to access the url of the master branch and denote the
797 # result by using round brackets (result)
798 if [[ ${curl_result} == "does not exist" && ! $branch == "master" ]]; then
800 # build the full (master/latest) url
801 url="${url_start}-${url_repo}/${url_lang}/latest/${url_file}"
802 #echo "DBUG: url=$url"
804 # test accessibility of url in "master branch" (latest)
805 curl --head --silent --fail "${url}?${unique}" >/dev/null
807 # denote result as a value from "master" branch (latest)
810 # convert numeric results to text
811 if [ "${curl_result}" = "0" ]; then
812 curl_result="(accessible)"
813 elif [ "${curl_result}" = "22" ]; then
814 curl_result="(does not exist)"
816 curl_result="(ERROR:${curl_result})"
821 # repostate IS NOT ACTIVE OR READ_ONLY - no curl test required
827 echo "$url ... $curl_result"
828 csv[i]="${csv[i]},${curl_result},${url}"
829 #echo "DBUG: csv line=${csv[i]}"
835 # csv column #16: release notes
838 readarray -t array < ../${repolist};
840 csv[i]="${csv[i]},release notes"
842 for line in "${array[@]}"
844 line=$(echo $line | sed 's:|.*$::')
845 #echo "DBUG: line=\"${line}\""
849 # put repo name in square brackets for increased grep hit rate
850 # escape minus and bracket characters to avoid problems with the grep command
851 #repo_grepable=$(echo ${line} | sed -r s:${line}:[${line}]: | sed -r 's/-/\\-/g' | sed -r 's/\[/\\[/g' | sed -r 's/\]/\\]/g')
852 #echo "DBUG: repo_grepable=\"${repo_grepable}\""
854 # check if repo dir exists in this branch
855 if [ -d ./${line} ] ; then
856 # if yes, check if repo name appears in the branch releasenotes.log
857 relnote=$(find "./${line}" -type f | grep 'release.*note.*.rst' | wc -l);
858 #echo "DBUG: relnote=${relnote}"
859 # repo dir DOES NOT exist in this branch - so check if repo dir exists in MASTER branch
860 elif [ -d ../master/${line} ] ; then
861 # if yes, check if repo name appears in the MASTER releasenotes.log
862 # count release notes files in MASTER branch (in repo root and its subdirectories)
863 relnote=$(find "../master/${line}" -type f | grep 'release.*note.*.rst' | wc -l);
864 #echo "DBUG: relnote=${relnote}"
865 # put results in round brackets to show that this is MASTER data
866 relnote=$(echo ${relnote} | sed -r s:${relnote}:\(${relnote}\):)
870 #echo "DBUG: relnote=${relnote}"
872 line="${csv[i]},${relnote}"
883 # build the table.csv file
888 echo "$i" | tee -a ./${branch}_table.csv
892 # create data package for this branch and zip it
895 datadir=${branch}_data
897 cp $repolist $datadir
898 cp $wikiplsfile $datadir
899 cp ${branch}_table.csv $datadir
900 cp ${branch}_*.log $datadir
901 zip -r ${datadir}.zip $datadir
903 # return from the branch directory
906 # return and work on the next requested branch ... or exit