X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=tools%2Fcheckdocs.sh;h=641553fb8752822ba0d177b70e02348af6505a89;hb=HEAD;hp=1f7f10be5467e367cbaf3d697bef3dc5b9c10d21;hpb=d4599fd2bec9c850d80301b32a8e394020a53392;p=doc.git diff --git a/tools/checkdocs.sh b/tools/checkdocs.sh index 1f7f10be5..641553fb8 100755 --- a/tools/checkdocs.sh +++ b/tools/checkdocs.sh @@ -19,11 +19,11 @@ ### checkdocs.sh ### ### AUTHOR(S): -### Thomas Kulik, Deutsche Telekom AG, 2020 +### Thomas Kulik, Deutsche Telekom AG, 2020 - 2021 ### ### DESCRIPTION: ### Retrieves a full list of ONAP repos from gerrit inluding their state. -### Clones all active repos of the ONAP master branch plus other requested ONAP +### Clones all repos of the ONAP master branch plus other requested ONAP ### branches. Then the script does some docs related analyses depending on the ### clone results. It creates logfiles containing filtered results. In addition ### a table.csv is created which can be used to import it in a spreadsheed. @@ -46,8 +46,8 @@ ### create repo list ### curl -s https://git.onap.org/ | grep "^1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d' ### -script_version="1.2 (2020-11-18)" +script_version="1.12 (2021-11-12)" # save command for the restart with logging enabled command=$0 @@ -79,7 +79,7 @@ function usage() { echo " " echo " ARGUMENTS: " echo " -u|--user username " - echo " linux foundation username used to clone ONAP Gerrit repos" + echo " linux foundation username used to clone ONAP repositories" echo " " echo " -b|--branches branch1,branch2,branch3 " echo " list of branches to be cloned. master is automatically " @@ -106,6 +106,170 @@ function InterruptedScript { exit 0 } +# function to parse wiki (project) lifecycle state information +# call: getwikilifecyclestate "projectname" +# result: $return_from_getwikilifecyclestate +# because bash supports only returning numeric values a variable $return_from_getwikilifecyclestate is used + +function getwikilifecyclestate { + + local requested=$1 + local wikiline="" + local wikirepo="" + local wikistate="" + + return_from_getwikilifecyclestate="" + + for wikiline in "${wikiplsarray[@]}" + do + + wikirepo=$(echo $wikiline | awk -F ";" '{print $1}'); + wikistate=$(echo $wikiline | awk -F ";" '{print $2}'); + + #echo "DBUG: getwikilifecyclestate wikiline = \"${wikiline}\""; + #echo "DBUG: getwikilifecyclestate wikirepo = \"${wikirepo}\"" + #echo "DBUG: getwikilifecyclestate wikistate = \"${wikistate}\"" + + if [[ ${wikirepo} == ${requested} ]]; then + return_from_getwikilifecyclestate=${wikistate} + #echo "DBUG: getwikilifecyclestate wikirepo = \"${wikirepo}\"" + #echo "DBUG: getwikilifecyclestate requested = \"${requested}\"" + #echo "DBUG: return_from_getwikilifecyclestate = \"${return_from_getwikilifecyclestate}\""; + return 0; + fi + + done + + #echo "DBUG: getwikilifecyclestate requested \"${requested}\" NOT FOUND in list" + return_from_getwikilifecyclestate="" + +} + +# function to parse release partizipation information +# call: getrpinfo "projectname" +# result: $return_from_getrpinfo +# because bash supports only returning numeric values a variable $return_from_getrpinfo is used + +function getrpinfo { + + local requested=$1 + + # clean up first + local rpdetails="" + local rpline="" + local rprepo="" + local rpproject="" + local current_branch_starting_letter="" + return_from_getrpinfo="" + + # finds first matching line in the array using grep (currently every line shows the same partizipation for the project (NOT repository!) ) + # this is much faster then looping line by line + rpline=$(IFS=$'\n'; echo "${rparray[*]}" | grep -m 1 ";${requested};"); + rpline=$(echo ${rpline} | tr -d '^M') + rprepo=$(echo ${rpline} | awk -F ";" '{print $1}'); + rpproject=$(echo ${rpline} | awk -F ";" '{print $2}'); + # concatenate details to do an easy grep later on to find out if or if not the project/repo has partizipated to a release + rpdetails=$(echo ${rpline} | awk -F ";" '{print "-" $3 "-" $4 "-" $5 "-" $6 "-" $7 "-" $8 "-" $9 "-" $10 "-" $11 "-" $12 "-"}'); + + # result will be e.g. "-g" and this avoids false positives with the "m" release + # (because "m" is also used to indicate the maintenance release, e.g. "gm") + current_branch_starting_letter="-${branch:0:1}" + + #echo "DBUG: getrpinfo ****************************"; + #echo "DBUG: getrpinfo requested = \"${requested}\""; + #echo "DBUG: getrpinfo rpproject = \"${rpproject}\""; + #echo "DBUG: getrpinfo rpdetails = \"${rpdetails}\""; + #echo "DBUG: current branch = \"${branch}\""; + #echo "DBUG: starting_letter = \"${current_branch_starting_letter}\""; + + ## check if PROJECT has partizipated to INITIAL release + #if [[ ${rpproject} = ${requested} ]] && [[ "${rpdetails}" == *"${current_branch_starting_letter}-"* ]]; then + # return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}" + # # check ADDITIONALLY if PROJECT has ALSO partizipated to MAINTENANCE release + # if [[ "${rpdetails}" == *"${current_branch_starting_letter}m-"* ]]; then + # return_from_getrpinfo="${return_from_getrpinfo} | ${current_branch_starting_letter:1:1}m" + # #echo "DBUG: getrpinfo return = \"${return_from_getrpinfo}\""; + # fi + # return 0; + ## check if PROJECT has ONLY partizipated to MAINTENANCE release + #elif [[ ${rpproject} = ${requested} ]] && [[ "${rpdetails}" == *"${current_branch_starting_letter:1:1}m-"* ]]; then + # return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}m" + # #echo "DBUG: getrpinfo return = \"${return_from_getrpinfo}\""; + # return 0; + #fi + + # check if requested PROJECT was found in the array of partizipating projects + if [[ ${rpproject} = ${requested} ]]; then + # check if PROJECT has partizipated to INITIAL release + if [[ "${rpdetails}" == *"${current_branch_starting_letter}-"* ]]; then + return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}" + # check ADDITIONALLY if PROJECT has ALSO partizipated to MAINTENANCE release + if [[ "${rpdetails}" == *"${current_branch_starting_letter}m-"* ]]; then + return_from_getrpinfo="${return_from_getrpinfo} ${current_branch_starting_letter:1:1}m" + #echo "DBUG: getrpinfo return = \"${return_from_getrpinfo}\""; + fi + return 0; + elif [[ "${rpdetails}" == *"${current_branch_starting_letter:1:1}m-"* ]]; then + return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}m" + #echo "DBUG: getrpinfo return = \"${return_from_getrpinfo}\""; + return 0; + fi + fi + #echo "DBUG: getrpinfo requested \"${requested}\" NOT FOUND in list" + return_from_getrpinfo="" +} + +function find_repo_in_confpy { + + local search_term=$1 + local search_term_line_number="" + local confpy_branch_entries="" + local confpy_line_number="" + local confpy_branch_name="" + local idx="" + + return_from_find_repo_in_confpy="" + search_term="'${search_term}'" + + search_term_line_number=$(cat ./doc/docs/conf.py | grep -n '^intersphinx_mapping\[' | grep -m 1 ${search_term} | sed 's/:.*//') + #echo "DBUG: search_term is ............... ${search_term}" + #echo "DBUG: search_term_line_number is ... ${search_term_line_number}" + + # nothing (or multiple entries) found - return + if [[ ${search_term_line_number} == "" ]]; then + #echo "DBUG: search_term_line_number is empty - returning" + return_from_find_repo_in_confpy="" + return 0; + fi + + readarray -t confpy_branch_entries <<< "$(cat ./doc/docs/conf.py | grep -n '^branch = ' | sed 's/branch = //' | sed s/\'//g)" + + #echo "DBUG: confpy_branch_entries" + #printf -- "%s\n" "${confpy_branch_entries[@]}" + #for confpy_branch_entry in ${confpy_branch_entries[@]} + #do + # confpy_line_number=$(echo $confpy_branch_entry | awk -F ":" '{print $1}'); + # confpy_branch_name=$(echo $confpy_branch_entry | awk -F ":" '{print $2}'); + # echo "DBUG: ${confpy_branch_name} entries are below line ${confpy_line_number}" + #done + + # search in the list of branches in reverse order + for (( idx=${#confpy_branch_entries[@]}-1 ; idx>=0 ; idx-- )) + do + #echo "DBUG: working entry is ${confpy_branch_entries[idx]}" + confpy_line_number=$(echo ${confpy_branch_entries[idx]} | awk -F ":" '{print $1}'); + confpy_branch_name=$(echo ${confpy_branch_entries[idx]} | awk -F ":" '{print $2}'); + #echo "DBUG: ${confpy_branch_name} entries are below line ${confpy_line_number}" + + if (( ${search_term_line_number} > ${confpy_line_number} )); then + #echo "DBUG: search_term_line_number is greater than confpy_line_number" + #echo "DBUG: ${search_term} found in ${confpy_branch_name} section" + return_from_find_repo_in_confpy=${confpy_branch_name} + return 0; + fi + done +} + ### ### arguments handling ### @@ -164,7 +328,7 @@ if [[ $branches_csv == *"master"* ]]; then usage exit -1 fi -# clone master first, the the other branches +# clone master first, then the other branches branches_csv="master,${branches_csv}" # create the branches array by readinging in the values from the variable @@ -189,7 +353,44 @@ echo " " echo "checkdocs.sh Version ${script_version}" echo " " +# +# read in wiki (project) lifecycle state +# always use the lastest available file (derived from date in filename e.g. wiki_lifecycle_state_210409.txt) +# format is ;; +# + +wikiplsfile=$(ls | sed -nr '/wiki_lifecycle_state_[0-9]{6}.txt/Ip' | tail -1); +if [[ $wikiplsfile == "" ]]; then + echo "ERROR: wiki_lifecycle_state_yymmdd.txt missing" + exit -1 +fi +echo "Using \"${wikiplsfile}\" as the source for wiki (project) lifecycle state information." +readarray -t wikiplsarray < ./${wikiplsfile}; + +# +# read in release_partizipation_YYMMDD.csv file +# always use the latest available file (derived from date in filename e.g. release_partizipation_210409.csv) +# format is: $1=repository;$2=project;$3=g;$4=gm;$5=h;$6=hm;$7=i;$8=im;$9=j;$10=jm;$11=k;$12=km;;;; +# example: "g" = project partizipated to the (g)uilin release +# "gm" = project partizipated to the (g)uilin (m)aintenance release +# file may contain windows control charaters at end of line (^M) +# + +rpfile=$(ls | sed -nr '/release_partizipation_[0-9]{6}.csv/Ip' | tail -1); +if [[ $rpfile == "" ]]; then + echo "ERROR: release_partizipation_yymmdd.csv missing" + exit -1 +fi +echo "Using \"${rpfile}\" as the source for release partizipation information." +readarray -t rparray < ./${rpfile}; +# remove first line +rparray=("${rparray[@]:1}") +#printf '%s\n' "${rparray[@]}" #DBUG ONLY + +# # curl must be installed +# + if ! command -v curl &> /dev/null then echo "ERROR: curl command could not be found" @@ -202,15 +403,19 @@ unique=$(date +%s) echo "Retrieving a full list of ONAP repositories (master) from gerrit.onap.org." +# # retrieve the full repolist from gerrit # workaround because of the (wrong?) response of gerrit.onap.org which makes jq command fail # "| awk '{if(NR>1)print}'" filters the first line of the response so that jq will work again (thx marek) +# + curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d' >./$repolist -# process the created repolist -# only active projects will be cloned in case the requested branch of the project exists -echo "Accessing gerrit.onap.org with username \"${lfusername}\"." -echo "Start cloning of repositories." +# process the created repolist and try to clone the projects from the mirror + +source="git://cloud.onap.org/mirror" +echo "Using \"${source}\" as the source and username \"${lfusername}\" for cloning the repositories." +echo "Start cloning of repositories ..." for branch in "${branches[@]}" do @@ -237,7 +442,7 @@ do devcounter=$((devcounter+1)) fi - if [[ $devcounter -lt "11" ]]; then + if [[ $devcounter -lt "50" ]]; then if [[ $devmode == "TRUE" ]]; then echo "INFO: devmode! counter=${devcounter}" @@ -249,10 +454,12 @@ do echo $reponame echo $repostate - if [[ $repostate == "ACTIVE" ]]; then - echo "Cloning \"${branch}\" branch of ACTIVE project ${reponame}..." + if [[ $repostate == "ACTIVE" ]] || [[ $repostate == "READ_ONLY" ]]; then + echo "Cloning \"${branch}\" branch of \"${repostate}\" project ${reponame}..." - git clone --branch ${branch} --recurse-submodules ssh://${lfusername}@gerrit.onap.org:29418/$reponame ./$reponame + # previously used: git clone --branch ${branch} --recurse-submodules ssh://${lfusername}@gerrit.onap.org:29418/$reponame ./$reponame + # clone script Jess: git clone "git://cloud.onap.org/mirror/${i}" "${LOCALNAME}" + git clone --branch ${branch} --recurse-submodules ${source}/${reponame} ./${reponame} gitexitcode=$? if [[ ! ${gitexitcode} == "0" ]]; then @@ -261,13 +468,13 @@ do errormsg="cloned" fi - # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg - echo "${gitexitcode}|${reponame}|${repostate}|${errormsg}" | tee -a ${branch}_gerritclone.log + # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg + echo "${gitexitcode}|${reponame}|${repostate}|${errormsg}" | tee -a ${branch}_repoclone.log - elif [[ $repostate == "READ_ONLY" ]]; then - echo "-|${reponame}|${repostate}|ignored" | tee -a ${branch}_gerritclone.log + #elif [[ $repostate == "READ_ONLY" ]]; then + #echo "-|${reponame}|${repostate}|ignored" | tee -a ${branch}_repoclone.log else - echo "-|${reponame}|unknown repo state \"${repostate}\"|-" | tee -a ${branch}_gerritclone.log + echo "-|${reponame}|unknown repo state \"${repostate}\"|-" | tee -a ${branch}_repoclone.log fi # examine repo @@ -280,7 +487,7 @@ do find ./$reponame -type f -name *.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_rstfiles.log printf "\nrelease notes rst:\n" - find ./$reponame -type f | grep 'release.*note.*.rst' | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_releasenotes.log + find ./$reponame -type f | grep '.*release.*note.*.rst' | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_releasenotes.log printf "\ntox.ini files:\n" find ./$reponame -type f -name tox.ini | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_toxini.log @@ -288,8 +495,14 @@ do printf "\nconf.py files:\n" find ./$reponame -type f -name conf.py | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_confpy.log - printf "\nindex.rst files:\n" - find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst.log + printf "\nindex.rst files (all):\n" + find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst_all.log + + printf "\nindex.rst files (docs root directory):\n" + find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | grep ']/docs/index.rst' | tee -a ${branch}_indexrst_docs_root.log + + printf "\nINFO.yaml files:\n" + find ./$reponame -type f -name INFO.yaml | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_infoyaml.log fi @@ -300,6 +513,12 @@ do done <${repolist} + # get (first) title for a rst file + drawline + python3 ../getrsttitle.py ${branch}_rstfiles.log | tee ${branch}_rstfiles_titles.log + drawline + python3 ../getrsttitle.py ${branch}_indexrst_docs_root.log | tee ${branch}_indexrst_docs_root_titles.log + # examine repos drawline find . -type f -name values.yaml -print -exec grep "image:" {} \; | sed -r 's:^ +::' | tee ${branch}_dockerimagesfull.log @@ -414,13 +633,13 @@ do # csv column #4: clone message # - readarray -t array < ./${branch}_gerritclone.log; + readarray -t array < ./${branch}_repoclone.log; i=0 csv[i]="${csv[i]},${branch_upper} clone message" ((i++)) for line in "${array[@]}" do - # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg + # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg errormsg=$(echo $line | awk -F "|" '{print $4}'); csv[i]="${csv[i]},${errormsg}" ((i++)) @@ -430,34 +649,149 @@ do unset errormsg # - # csv column #5: RELEASE component (yes|no|maybe) + # csv column #5: latest branch + # + + readarray -t array < ./${repolist}; + i=0 + csv[i]="${csv[i]},latest branch" + ((i++)) + for line in "${array[@]}" + do + reponame=$(echo $line | awk -F "|" '{print $1}'); + latestbranch=$(git ls-remote -q --heads "${source}/${reponame}" | sed 's/^.*heads\///' | sed -nr '/^master$|^amsterdam$|^beijing$|^casablanca$|^dublin$|^elalto$|^frankfurt$|^guilin$|^honolulu$|^istanbul$/Ip' | tail -2 | head -1); + #echo "DBUG: reponame=${reponame}" + #echo "DBUG: latestbranch=${latestbranch}" + echo "latest available branch for repo \"${reponame}\" is \"${latestbranch}\"" + csv[i]="${csv[i]},${latestbranch}" + ((i++)) + done + unset array + unset i + unset reponame + unset latestbranch + + # + # csv column #6: INFO.yaml LC state (project lifecycle state based on INFO.yaml / per repo) + # csv column #7: WIKI LC state (project lifecycle state based on ONAP Dev Wiki / per project) + # csv column #8: LC state match shows a "match" if both LC states match + # + + readarray -t array < ./${repolist}; + i=0 + csv[i]="${csv[i]},INFO.yaml LC state,WIKI LC state,LC state match" + ((i++)) + for line in "${array[@]}" + do + reponame=$(echo $line | awk -F "|" '{print $1}'); + project=$(echo $reponame | sed 's:/.*$::') + + if [ -f ./${reponame}/INFO.yaml ] ; then + # check if repo/branch has a INFO.yaml + lifecycleproject=$(grep '^project: ' ./${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//') + lifecyclestate=$(grep '^lifecycle_state: ' ./${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//') + elif [ ${branch} != "master" ] && [ -f ../master/${reponame}/INFO.yaml ] ; then + # IF current branch is not master AND if info.yaml not found in the current repo/branch THAN use INFO.yaml of repo/master if available + #echo "DBUG: branch=${branch} - checking master for INFO.yaml" + lifecycleproject=$(grep '^project: ' ../master/${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//') + lifecyclestate=$(grep '^lifecycle_state: ' ../master/${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//') + lifecyclestate="(${lifecyclestate})" + else + lifecyclestate="INFO.yaml not found" + fi + + getwikilifecyclestate ${project} + # returns value in ${return_from_getwikilifecyclestate} + + #echo "DBUG: working dir is ...";pwd + #echo "DBUG: lifecycleproject=${lifecycleproject}" + #echo "DBUG: lifecyclestate=${lifecyclestate}" + #echo "DBUG: wikilifecyclestate=${return_from_getwikilifecyclestate}" + + #check if YAML.info LC state is not empty _AND_ if WIKI LC state is not empty _AND_ if YAML.info LC state contains WIKI LC state + if [[ ${lifecyclestate} != "" ]] && [[ ${return_from_getwikilifecyclestate} != "" ]] && [[ ${lifecyclestate} == *"${return_from_getwikilifecyclestate}"* ]]; then + lcstatesmatch="match" + else + lcstatesmatch="" + fi + + csv[i]="${csv[i]},${lifecyclestate},${return_from_getwikilifecyclestate},${lcstatesmatch}" + ((i++)) + done + unset array + unset i + unset reponame + unset project + unset lifecycleproject + unset lifecyclestate + unset lcstatesmatch + + # + # csv column #9: intersphinx + # intersphinx mappings in conf.py + # provided is the branch used for linking the repository + # + + readarray -t array < ./${repolist}; + i=0 + csv[i]="${csv[i]},intersphinx" + ((i++)) + for line in "${array[@]}" + do + reponame=$(echo $line | awk -F "|" '{print $1}'); + project=$(echo $reponame | sed 's:/.*$::') + #echo "DBUG: reponame=${reponame}" + #echo "DBUG: project=${project}" + #echo "DBUG: i=${i}" + reponame=$(echo ${reponame} | sed -r 's/\//-/g') + search_repo="onap-${reponame}" + #echo "DBUG: search_repo=${search_repo}" + find_repo_in_confpy ${search_repo} + csv[i]="${csv[i]},${return_from_find_repo_in_confpy}" + ((i++)) + done + unset array + unset i + unset reponame + unset project + unset return_from_find_repo_in_confpy + + # + # csv column #10: RELEASE component (yes|maybe|unknown) # to be filled with values of the planned release config file maintained by # the onap release manager + # NOT FUNCTIONAL YET # - # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg - readarray -t array < ./${branch}_gerritclone.log; + # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg + readarray -t array < ./${branch}_repoclone.log; i=0 csv[i]="${csv[i]},${branch_upper} component" ((i++)) for line in "${array[@]}" do - # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg + # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg gitexitcode=$(echo $line | awk -F "|" '{print $1}'); reponame=$(echo $line | awk -F "|" '{print $2}'); repostate=$(echo $line | awk -F "|" '{print $3}'); errormsg=$(echo $line | awk -F "|" '{print $4}'); - if [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "0" ]]; then - releasecomponent="yes" - elif [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "128" ]]; then - releasecomponent="maybe" - elif [ ${repostate} == "READ_ONLY" ]; then - releasecomponent="no" - else - releasecomponent="unknown" - fi + #if [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "0" ]]; then + # releasecomponent="yes" + #elif [ ${repostate} == "ACTIVE" ]; then + ##elif [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "128" ]]; then + # releasecomponent="maybe" + #elif [[ ${repostate} == "READ_ONLY" && ${gitexitcode} == "0" ]]; then + # releasecomponent="yes" + #elif [ ${repostate} == "READ_ONLY" ]; then + # releasecomponent="maybe" + #else + # releasecomponent="unknown" + #fi + + # not functional yet! + releasecomponent="" csv[i]="${csv[i]},${releasecomponent}" ((i++)) @@ -471,19 +805,61 @@ do unset releasecomponent # - # csv column #6: docs (at repo root directory only; no recursive search!) - # csv column #7: conf.py - # csv column #8: tox.ini - # csv column #9: index.rst + # csv column #11: RELEASE partizipation + # + + # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg + readarray -t array < ./${branch}_repoclone.log; + i=0 + csv[i]="${csv[i]},${branch_upper} partizipation" + ((i++)) + echo "INFO: determine release partizipation for project ..." + for line in "${array[@]}" + do + + # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg + gitexitcode=$(echo $line | awk -F "|" '{print $1}'); + reponame=$(echo $line | awk -F "|" '{print $2}'); + repostate=$(echo $line | awk -F "|" '{print $3}'); + errormsg=$(echo $line | awk -F "|" '{print $4}'); + projectname=$(echo $reponame | sed 's:/.*$::') + + if [[ $branch == "master" ]]; then + return_from_getrpinfo=""; + else + #echo "DBUG: calling getrpinfo for projectname ${projectname}" + getrpinfo ${projectname} + fi + + csv[i]="${csv[i]},${return_from_getrpinfo}" + ((i++)) + + done + + unset array + unset i + unset gitexitcode + unset reponame + unset repostate + unset errormsg + unset projectname + unset return_from_getrpinfo + + # + # csv column #12: docs (at repo root directory only; no recursive search!) + # csv column #13: conf.py + # csv column #14: tox.ini + # csv column #15: index.rst + # csv column #16: first title in index.rst # # columns are filled with values from requested branch. # if data is not available values from master branch are used. - # to identify master branch values, data is put into brackets "(...)" + # to identify master branch values, data is put into round brackets "(...)" # readarray -t array < ./${repolist}; i=0 - csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst" + csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst,first title in index.rst" ((i++)) for line in "${array[@]}" do @@ -509,22 +885,44 @@ do docs="${docs},-" fi - # tox.ini - if [ -f ./${line}/docs/tox.ini ] ; then + # tox.ini (check docs dir and also check project root dir) + if [ -f ./${line}/docs/tox.ini ] || [ -f ./${line}/tox.ini ]; then docs="${docs},tox.ini" - elif [ -f ../master/${line}/docs/tox.ini ] ; then - docs="${docs},(tox.ini)" + # tox.ini @ branch/docs dir + if [ -f ./${line}/docs/tox.ini ] ; then + docs="${docs} @docs" + fi + # tox.ini @ branch/project root dir + if [ -f ./${line}/tox.ini ] ; then + docs="${docs} @root" + fi + elif [ -f ../master/${line}/docs/tox.ini ] || [ -f ../master/${line}/tox.ini ]; then + docs="${docs},(tox.ini" + # tox.ini @ master/docs dir + if [ -f ../master/${line}/docs/tox.ini ] ; then + docs="${docs} @docs" + fi + # tox.ini @ master/project root dir + if [ -f ../master/${line}/tox.ini ] ; then + docs="${docs} @root" + fi + # just add a round bracket at the end of the value + docs="${docs})" else + # no tox.ini found in docs or root dir docs="${docs},-" fi - # index.rst + # index.rst, first title in index.rst + indexrsttitle="" if [ -f ./${line}/docs/index.rst ] ; then - docs="${docs},index.rst" + indexrsttitle=$(cat ${branch}_indexrst_docs_root_titles.log | grep -F '['${line}']/docs/index.rst,' | awk -F "," '{print $4}'); + docs="${docs},index.rst,${indexrsttitle}" elif [ -f ../master/${line}/docs/index.rst ] ; then - docs="${docs},(index.rst)" + indexrsttitle=$(cat ../master/master_indexrst_docs_root_titles.log | grep -F '['${line}']/docs/index.rst,' | awk -F "," '{print $4}'); + docs="${docs},(index.rst),(${indexrsttitle})" else - docs="${docs},-" + docs="${docs},-,-" fi #echo "DBUG: docs=${docs}" @@ -537,17 +935,17 @@ do unset docs # - # csv column #10: index.html@RTD accessibility check - # csv column #11: index.html url + # csv column #17: index.html@RTD accessibility check + # csv column #18: index.html url # - readarray -t array < ./${branch}_gerritclone.log; + readarray -t array < ./${branch}_repoclone.log; i=0 csv[i]="${csv[i]},index.html@RTD,index.html url" ((i++)) for line in "${array[@]}" do - # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg + # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg gitexitcode=$(echo $line | awk -F "|" '{print $1}'); reponame=$(echo $line | awk -F "|" '{print $2}'); repostate=$(echo $line | awk -F "|" '{print $3}'); @@ -556,7 +954,7 @@ do url="" curl_result="" - # this routine works only with release "frankfurt" and later because + # this script works only with release "frankfurt" and later because # earlier releases are using submodule structure for documentation files if echo "$branch" | grep -q '^[abcde]'; then curl_result="unsupported release" @@ -564,8 +962,7 @@ do else # we are working on "frankfurt" branch or later ... - # only if repostate IS ACTIVE a curl test is required - if [[ ${repostate} == "ACTIVE" ]]; then + if [[ ${repostate} == "ACTIVE" ]] || [[ ${repostate} == "READ_ONLY" ]]; then # OPTIONAL: USE ALSO GITEXITCODE AS A FILTER CRITERIA ??? @@ -638,7 +1035,7 @@ do fi else - # repostate IS NOT ACTIVE - no curl test required + # repostate IS NOT ACTIVE OR READ_ONLY - no curl test required curl_result="-" url="-" fi @@ -652,7 +1049,7 @@ do done # - # csv column #12: release notes + # csv column #19: release notes # readarray -t array < ../${repolist}; @@ -674,17 +1071,20 @@ do # check if repo dir exists in this branch if [ -d ./${line} ] ; then # if yes, check if repo name appears in the branch releasenotes.log - relnote=$(find "./${line}" -type f | grep 'release.*note.*.rst' | wc -l); + relnote=$(find "./${line}" -type f | grep '.*release.*note.*.rst' | wc -l); + #echo "DBUG: relnote=${relnote}" # repo dir DOES NOT exist in this branch - so check if repo dir exists in MASTER branch elif [ -d ../master/${line} ] ; then # if yes, check if repo name appears in the MASTER releasenotes.log # count release notes files in MASTER branch (in repo root and its subdirectories) relnote=$(find "../master/${line}" -type f | grep 'release.*note.*.rst' | wc -l); + #echo "DBUG: relnote=${relnote}" # put results in round brackets to show that this is MASTER data relnote=$(echo ${relnote} | sed -r s:${relnote}:\(${relnote}\):) else relnote="-" fi + #echo "DBUG: relnote=${relnote}" line="${csv[i]},${relnote}" csv[i]=${line} @@ -712,6 +1112,8 @@ do datadir=${branch}_data mkdir $datadir cp $repolist $datadir + cp ../$wikiplsfile $datadir + cp ../$rpfile $datadir cp ${branch}_table.csv $datadir cp ${branch}_*.log $datadir zip -r ${datadir}.zip $datadir