X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=tools%2Fcheckdocs.sh;h=641553fb8752822ba0d177b70e02348af6505a89;hb=HEAD;hp=4a798b378c073b50beef634b1efa8b195197cb4e;hpb=56180a5ab013ca96c0931e321ddad87a2689b33d;p=doc.git

diff --git a/tools/checkdocs.sh b/tools/checkdocs.sh
index 4a798b378..641553fb8 100755
--- a/tools/checkdocs.sh
+++ b/tools/checkdocs.sh
@@ -23,7 +23,7 @@
 ###
 ### DESCRIPTION:
 ### Retrieves a full list of ONAP repos from gerrit inluding their state.
-### Clones all active repos of the ONAP master branch plus other requested ONAP
+### Clones all repos of the ONAP master branch plus other requested ONAP
 ### branches. Then the script does some docs related analyses depending on the
 ### clone results. It creates logfiles containing filtered results. In addition
 ### a table.csv is created which can be used to import it in a spreadsheed.
@@ -58,7 +58,7 @@
 ### SHORT: curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d'
 ###
 
-script_version="1.6 (2021/03/30)"
+script_version="1.12 (2021-11-12)"
 
 # save command for the restart with logging enabled
 command=$0
@@ -106,6 +106,170 @@ function InterruptedScript {
   exit 0
 }
 
+# function to parse wiki (project) lifecycle state information
+# call:   getwikilifecyclestate "projectname"
+# result: $return_from_getwikilifecyclestate
+# because bash supports only returning numeric values a variable $return_from_getwikilifecyclestate is used
+
+function getwikilifecyclestate {
+
+  local requested=$1
+  local wikiline=""
+  local wikirepo=""
+  local wikistate=""
+
+  return_from_getwikilifecyclestate=""
+
+  for wikiline in "${wikiplsarray[@]}"
+  do
+
+     wikirepo=$(echo $wikiline | awk -F ";" '{print $1}');
+    wikistate=$(echo $wikiline | awk -F ";" '{print $2}');
+
+    #echo "DBUG: getwikilifecyclestate  wikiline = \"${wikiline}\"";
+    #echo "DBUG: getwikilifecyclestate  wikirepo = \"${wikirepo}\""
+    #echo "DBUG: getwikilifecyclestate wikistate = \"${wikistate}\""
+
+    if [[ ${wikirepo} == ${requested} ]]; then
+      return_from_getwikilifecyclestate=${wikistate}
+      #echo "DBUG: getwikilifecyclestate     wikirepo = \"${wikirepo}\""
+      #echo "DBUG: getwikilifecyclestate    requested = \"${requested}\""
+      #echo "DBUG: return_from_getwikilifecyclestate  = \"${return_from_getwikilifecyclestate}\"";
+      return 0;
+    fi
+
+  done
+
+  #echo "DBUG: getwikilifecyclestate requested \"${requested}\" NOT FOUND in list"
+  return_from_getwikilifecyclestate=""
+
+}
+
+# function to parse release partizipation information
+# call:   getrpinfo "projectname"
+# result: $return_from_getrpinfo
+# because bash supports only returning numeric values a variable $return_from_getrpinfo is used
+
+function getrpinfo {
+
+  local requested=$1
+
+  # clean up first
+  local rpdetails=""
+  local rpline=""
+  local rprepo=""
+  local rpproject=""
+  local current_branch_starting_letter=""
+        return_from_getrpinfo=""
+
+  # finds first matching line in the array using grep (currently every line shows the same partizipation for the project (NOT repository!) )
+  # this is much faster then looping line by line
+     rpline=$(IFS=$'\n'; echo "${rparray[*]}" | grep -m 1 ";${requested};");
+     rpline=$(echo ${rpline} | tr -d '^M')
+     rprepo=$(echo ${rpline} | awk -F ";" '{print $1}');
+  rpproject=$(echo ${rpline} | awk -F ";" '{print $2}');
+  # concatenate details to do an easy grep later on to find out if or if not the project/repo has partizipated to a release
+  rpdetails=$(echo ${rpline} | awk -F ";" '{print "-" $3 "-" $4 "-" $5 "-" $6 "-" $7 "-" $8 "-" $9 "-" $10 "-" $11 "-" $12 "-"}');
+
+  # result will be e.g. "-g" and this avoids false positives with the "m" release
+  # (because "m" is also used to indicate the maintenance release, e.g. "gm")
+  current_branch_starting_letter="-${branch:0:1}"
+
+  #echo "DBUG: getrpinfo ****************************";
+  #echo "DBUG: getrpinfo requested = \"${requested}\"";
+  #echo "DBUG: getrpinfo rpproject = \"${rpproject}\"";
+  #echo "DBUG: getrpinfo rpdetails = \"${rpdetails}\"";
+  #echo "DBUG:      current branch = \"${branch}\"";
+  #echo "DBUG:     starting_letter = \"${current_branch_starting_letter}\"";
+
+  ## check if PROJECT has partizipated to INITIAL release
+  #if [[ ${rpproject} = ${requested} ]] && [[ "${rpdetails}" == *"${current_branch_starting_letter}-"* ]]; then
+  #  return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}"
+  #  # check ADDITIONALLY if PROJECT has ALSO partizipated to MAINTENANCE release
+  #  if [[ "${rpdetails}" == *"${current_branch_starting_letter}m-"* ]]; then
+  #    return_from_getrpinfo="${return_from_getrpinfo} | ${current_branch_starting_letter:1:1}m"
+  #    #echo "DBUG:  getrpinfo return = \"${return_from_getrpinfo}\"";
+  #  fi
+  #  return 0;
+  ## check if PROJECT has ONLY partizipated to MAINTENANCE release
+  #elif [[ ${rpproject} = ${requested} ]] && [[ "${rpdetails}" == *"${current_branch_starting_letter:1:1}m-"* ]]; then
+  #  return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}m"
+  #  #echo "DBUG:  getrpinfo return = \"${return_from_getrpinfo}\"";
+  #  return 0;
+  #fi
+
+  # check if requested PROJECT was found in the array of partizipating projects
+  if [[ ${rpproject} = ${requested} ]]; then
+    # check if PROJECT has partizipated to INITIAL release
+    if [[ "${rpdetails}" == *"${current_branch_starting_letter}-"* ]]; then
+      return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}"
+      # check ADDITIONALLY if PROJECT has ALSO partizipated to MAINTENANCE release
+      if [[ "${rpdetails}" == *"${current_branch_starting_letter}m-"* ]]; then
+      return_from_getrpinfo="${return_from_getrpinfo} ${current_branch_starting_letter:1:1}m"
+      #echo "DBUG:  getrpinfo return = \"${return_from_getrpinfo}\"";
+      fi
+      return 0;
+    elif [[ "${rpdetails}" == *"${current_branch_starting_letter:1:1}m-"* ]]; then
+      return_from_getrpinfo="project | ${current_branch_starting_letter:1:1}m"
+      #echo "DBUG:  getrpinfo return = \"${return_from_getrpinfo}\"";
+      return 0;
+    fi
+  fi
+  #echo "DBUG: getrpinfo requested \"${requested}\" NOT FOUND in list"
+  return_from_getrpinfo=""
+}
+
+function find_repo_in_confpy {
+
+  local search_term=$1
+  local search_term_line_number=""
+  local confpy_branch_entries=""
+  local confpy_line_number=""
+  local confpy_branch_name=""
+  local idx=""
+  
+  return_from_find_repo_in_confpy=""
+  search_term="'${search_term}'"
+
+  search_term_line_number=$(cat ./doc/docs/conf.py | grep -n '^intersphinx_mapping\[' | grep -m 1 ${search_term} | sed 's/:.*//')
+  #echo "DBUG: search_term is ............... ${search_term}"
+  #echo "DBUG: search_term_line_number is ... ${search_term_line_number}"
+  
+  # nothing (or multiple entries) found - return
+  if [[ ${search_term_line_number} == "" ]]; then
+    #echo "DBUG: search_term_line_number is empty - returning"
+    return_from_find_repo_in_confpy=""
+    return 0;
+  fi
+  
+  readarray -t confpy_branch_entries <<< "$(cat ./doc/docs/conf.py | grep -n '^branch = ' | sed 's/branch = //' | sed s/\'//g)"
+  
+  #echo "DBUG: confpy_branch_entries"
+  #printf -- "%s\n" "${confpy_branch_entries[@]}"
+  #for confpy_branch_entry in ${confpy_branch_entries[@]}
+  #do
+  #    confpy_line_number=$(echo $confpy_branch_entry | awk -F ":" '{print $1}');
+  #    confpy_branch_name=$(echo $confpy_branch_entry | awk -F ":" '{print $2}');
+  #    echo "DBUG: ${confpy_branch_name} entries are below line ${confpy_line_number}"
+  #done
+  
+  # search in the list of branches in reverse order
+  for (( idx=${#confpy_branch_entries[@]}-1 ; idx>=0 ; idx-- ))
+  do
+      #echo "DBUG: working entry is ${confpy_branch_entries[idx]}"
+      confpy_line_number=$(echo ${confpy_branch_entries[idx]} | awk -F ":" '{print $1}');
+      confpy_branch_name=$(echo ${confpy_branch_entries[idx]} | awk -F ":" '{print $2}');
+      #echo "DBUG: ${confpy_branch_name} entries are below line ${confpy_line_number}"
+  
+      if (( ${search_term_line_number} > ${confpy_line_number} )); then
+        #echo "DBUG: search_term_line_number is greater than confpy_line_number"
+        #echo "DBUG: ${search_term} found in ${confpy_branch_name} section"
+        return_from_find_repo_in_confpy=${confpy_branch_name}
+        return 0;
+      fi
+  done
+}
+
 ###
 ### arguments handling
 ###
@@ -189,7 +353,44 @@ echo " "
 echo "checkdocs.sh Version ${script_version}"
 echo " "
 
+#
+# read in wiki (project) lifecycle state
+# always use the lastest available file (derived from date in filename e.g. wiki_lifecycle_state_210409.txt)
+# format is <reponame abbrev>;<state>;<reponame full>
+#
+
+wikiplsfile=$(ls | sed -nr '/wiki_lifecycle_state_[0-9]{6}.txt/Ip' | tail -1);
+if [[ $wikiplsfile == "" ]]; then
+  echo "ERROR: wiki_lifecycle_state_yymmdd.txt missing"
+  exit -1
+fi
+echo "Using \"${wikiplsfile}\" as the source for wiki (project) lifecycle state information."
+readarray -t wikiplsarray < ./${wikiplsfile};
+
+#
+# read in release_partizipation_YYMMDD.csv file
+# always use the latest available file (derived from date in filename e.g. release_partizipation_210409.csv)
+# format is: $1=repository;$2=project;$3=g;$4=gm;$5=h;$6=hm;$7=i;$8=im;$9=j;$10=jm;$11=k;$12=km;;;;
+# example: "g"  = project partizipated to the (g)uilin release
+#          "gm" = project partizipated to the (g)uilin (m)aintenance release
+# file may contain windows control charaters at end of line (^M)
+#
+
+rpfile=$(ls | sed -nr '/release_partizipation_[0-9]{6}.csv/Ip' | tail -1);
+if [[ $rpfile == "" ]]; then
+  echo "ERROR: release_partizipation_yymmdd.csv missing"
+  exit -1
+fi
+echo "Using \"${rpfile}\" as the source for release partizipation information."
+readarray -t rparray < ./${rpfile};
+# remove first line
+rparray=("${rparray[@]:1}")
+#printf '%s\n' "${rparray[@]}" #DBUG ONLY
+
+#
 # curl must be installed
+#
+
 if ! command -v curl &> /dev/null
 then
   echo "ERROR: curl command could not be found"
@@ -202,9 +403,12 @@ unique=$(date +%s)
 
 echo "Retrieving a full list of ONAP repositories (master) from gerrit.onap.org."
 
+#
 # retrieve the full repolist from gerrit
 # workaround because of the (wrong?) response of gerrit.onap.org which makes jq command fail
 # "| awk '{if(NR>1)print}'" filters the first line of the response so that jq will work again (thx marek)
+#
+
 curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d' >./$repolist
 
 # process the created repolist and try to clone the projects from the mirror
@@ -283,7 +487,7 @@ do
         find ./$reponame -type f -name *.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_rstfiles.log
 
         printf "\nrelease notes rst:\n"
-        find ./$reponame -type f | grep 'release.*note.*.rst' | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_releasenotes.log
+        find ./$reponame -type f | grep '.*release.*note.*.rst' | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_releasenotes.log
 
         printf "\ntox.ini files:\n"
         find ./$reponame -type f -name tox.ini | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_toxini.log
@@ -291,8 +495,11 @@ do
         printf "\nconf.py files:\n"
         find ./$reponame -type f -name conf.py | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_confpy.log
 
-        printf "\nindex.rst files:\n"
-        find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst.log
+        printf "\nindex.rst files (all):\n"
+        find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst_all.log
+
+        printf "\nindex.rst files (docs root directory):\n"
+        find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | grep ']/docs/index.rst' | tee -a ${branch}_indexrst_docs_root.log
 
         printf "\nINFO.yaml files:\n"
         find ./$reponame -type f -name INFO.yaml | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_infoyaml.log
@@ -306,6 +513,12 @@ do
 
   done <${repolist}
 
+  # get (first) title for a rst file
+  drawline
+  python3 ../getrsttitle.py ${branch}_rstfiles.log | tee ${branch}_rstfiles_titles.log
+  drawline
+  python3 ../getrsttitle.py ${branch}_indexrst_docs_root.log | tee ${branch}_indexrst_docs_root_titles.log
+
   # examine repos
   drawline
   find . -type f -name values.yaml -print -exec grep "image:" {} \; | sed -r 's:^ +::' | tee ${branch}_dockerimagesfull.log
@@ -436,23 +649,49 @@ do
   unset errormsg
 
   #
-  # csv column #5: lifecycle state
-  # extracted from the INFO.yaml
+  # csv column #5: latest branch
   #
 
   readarray -t array < ./${repolist};
   i=0
-  csv[i]="${csv[i]},project lifecycle state"
+  csv[i]="${csv[i]},latest branch"
   ((i++))
   for line in "${array[@]}"
   do
     reponame=$(echo $line | awk -F "|" '{print $1}');
+    latestbranch=$(git ls-remote -q --heads "${source}/${reponame}" | sed 's/^.*heads\///' | sed -nr '/^master$|^amsterdam$|^beijing$|^casablanca$|^dublin$|^elalto$|^frankfurt$|^guilin$|^honolulu$|^istanbul$/Ip' | tail -2 | head -1);
+    #echo "DBUG:     reponame=${reponame}"
+    #echo "DBUG: latestbranch=${latestbranch}"
+    echo "latest available branch for repo \"${reponame}\" is \"${latestbranch}\""
+    csv[i]="${csv[i]},${latestbranch}"
+    ((i++))
+  done
+  unset array
+  unset i
+  unset reponame
+  unset latestbranch
+
+  #
+  # csv column #6: INFO.yaml LC state (project lifecycle state based on INFO.yaml / per repo)
+  # csv column #7: WIKI LC state (project lifecycle state based on ONAP Dev Wiki / per project)
+  # csv column #8: LC state match shows a "match" if both LC states match
+  #
+
+  readarray -t array < ./${repolist};
+  i=0
+  csv[i]="${csv[i]},INFO.yaml LC state,WIKI LC state,LC state match"
+  ((i++))
+  for line in "${array[@]}"
+  do
+    reponame=$(echo $line | awk -F "|" '{print $1}');
+     project=$(echo $reponame | sed 's:/.*$::')
+
     if [ -f ./${reponame}/INFO.yaml ] ; then
       # check if repo/branch has a INFO.yaml
       lifecycleproject=$(grep '^project: ' ./${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
       lifecyclestate=$(grep '^lifecycle_state: ' ./${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
     elif [ ${branch} != "master" ] && [ -f ../master/${reponame}/INFO.yaml ] ; then
-      # if current branch is not master AND if info.yaml not found in the current repo/branch THAN use INFO.yaml of repo/master if available
+      # IF current branch is not master AND if info.yaml not found in the current repo/branch THAN use INFO.yaml of repo/master if available
       #echo "DBUG: branch=${branch} - checking master for INFO.yaml"
       lifecycleproject=$(grep '^project: ' ../master/${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
       lifecyclestate=$(grep '^lifecycle_state: ' ../master/${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
@@ -460,21 +699,68 @@ do
     else
       lifecyclestate="INFO.yaml not found"
     fi
+
+    getwikilifecyclestate ${project}
+    # returns value in ${return_from_getwikilifecyclestate}
+
     #echo "DBUG: working dir is ...";pwd
-    #echo "DBUG: lifecycleproject=${lifecycleproject}"
-    #echo "DBUG:   lifecyclestate=${lifecyclestate}"
-    csv[i]="${csv[i]},${lifecyclestate}"
+    #echo "DBUG:   lifecycleproject=${lifecycleproject}"
+    #echo "DBUG:     lifecyclestate=${lifecyclestate}"
+    #echo "DBUG: wikilifecyclestate=${return_from_getwikilifecyclestate}"
+
+    #check if YAML.info LC state is not empty _AND_ if WIKI LC state is not empty _AND_ if YAML.info LC state contains WIKI LC state
+    if [[ ${lifecyclestate} != "" ]] && [[ ${return_from_getwikilifecyclestate} != "" ]] && [[ ${lifecyclestate} == *"${return_from_getwikilifecyclestate}"* ]]; then
+      lcstatesmatch="match"
+    else
+      lcstatesmatch=""
+    fi
+
+    csv[i]="${csv[i]},${lifecyclestate},${return_from_getwikilifecyclestate},${lcstatesmatch}"
     ((i++))
   done
   unset array
   unset i
+  unset reponame
+  unset project
   unset lifecycleproject
   unset lifecyclestate
+  unset lcstatesmatch
 
   #
-  # csv column #6: RELEASE component (yes|maybe|unknown)
+  # csv column #9: intersphinx
+  # intersphinx mappings in conf.py
+  # provided is the branch used for linking the repository
+  #
+
+  readarray -t array < ./${repolist};
+  i=0
+  csv[i]="${csv[i]},intersphinx"
+  ((i++))
+  for line in "${array[@]}"
+  do
+    reponame=$(echo $line | awk -F "|" '{print $1}');
+    project=$(echo $reponame | sed 's:/.*$::')
+    #echo "DBUG: reponame=${reponame}"
+    #echo "DBUG:  project=${project}"
+    #echo "DBUG:        i=${i}"
+    reponame=$(echo ${reponame} | sed -r 's/\//-/g')
+    search_repo="onap-${reponame}"
+    #echo "DBUG: search_repo=${search_repo}"
+    find_repo_in_confpy ${search_repo}
+    csv[i]="${csv[i]},${return_from_find_repo_in_confpy}"
+    ((i++))
+  done
+  unset array
+  unset i
+  unset reponame
+  unset project
+  unset return_from_find_repo_in_confpy
+
+  #
+  # csv column #10: RELEASE component (yes|maybe|unknown)
   # to be filled with values of the planned release config file maintained by
   # the onap release manager
+  # NOT FUNCTIONAL YET
   #
 
   # repoclone.log format:  $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
@@ -491,18 +777,21 @@ do
       repostate=$(echo $line | awk -F "|" '{print $3}');
        errormsg=$(echo $line | awk -F "|" '{print $4}');
 
-    if [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "0" ]]; then
-      releasecomponent="yes"
-    elif [ ${repostate} == "ACTIVE" ]; then
-    #elif [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "128" ]]; then
-      releasecomponent="maybe"
-    elif [[ ${repostate} == "READ_ONLY" && ${gitexitcode} == "0" ]]; then
-      releasecomponent="yes"
-    elif [ ${repostate} == "READ_ONLY" ]; then
-      releasecomponent="maybe"
-    else
-      releasecomponent="unknown"
-    fi
+    #if [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "0" ]]; then
+    #  releasecomponent="yes"
+    #elif [ ${repostate} == "ACTIVE" ]; then
+    ##elif [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "128" ]]; then
+    #  releasecomponent="maybe"
+    #elif [[ ${repostate} == "READ_ONLY" && ${gitexitcode} == "0" ]]; then
+    #  releasecomponent="yes"
+    #elif [ ${repostate} == "READ_ONLY" ]; then
+    #  releasecomponent="maybe"
+    #else
+    #  releasecomponent="unknown"
+    #fi
+
+    # not functional yet!
+    releasecomponent=""
 
     csv[i]="${csv[i]},${releasecomponent}"
     ((i++))
@@ -516,10 +805,52 @@ do
   unset releasecomponent
 
   #
-  # csv column #7:  docs (at repo root directory only; no recursive search!)
-  # csv column #8:  conf.py
-  # csv column #9:  tox.ini
-  # csv column #10: index.rst
+  # csv column #11: RELEASE partizipation
+  #
+
+  # repoclone.log format:  $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
+  readarray -t array < ./${branch}_repoclone.log;
+  i=0
+  csv[i]="${csv[i]},${branch_upper} partizipation"
+  ((i++))
+  echo "INFO: determine release partizipation for project ..."
+  for line in "${array[@]}"
+  do
+
+    # repoclone.log format:  $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
+    gitexitcode=$(echo $line | awk -F "|" '{print $1}');
+       reponame=$(echo $line | awk -F "|" '{print $2}');
+      repostate=$(echo $line | awk -F "|" '{print $3}');
+       errormsg=$(echo $line | awk -F "|" '{print $4}');
+    projectname=$(echo $reponame | sed 's:/.*$::')
+
+    if [[ $branch == "master" ]]; then
+      return_from_getrpinfo="";
+    else
+      #echo "DBUG: calling getrpinfo for projectname ${projectname}"
+      getrpinfo ${projectname}
+    fi
+
+    csv[i]="${csv[i]},${return_from_getrpinfo}"
+    ((i++))
+
+  done
+
+  unset array
+  unset i
+  unset gitexitcode
+  unset reponame
+  unset repostate
+  unset errormsg
+  unset projectname
+  unset return_from_getrpinfo
+
+  #
+  # csv column #12: docs (at repo root directory only; no recursive search!)
+  # csv column #13: conf.py
+  # csv column #14: tox.ini
+  # csv column #15: index.rst
+  # csv column #16: first title in index.rst
   #
   # columns are filled with values from requested branch.
   # if data is not available values from master branch are used.
@@ -528,7 +859,7 @@ do
 
   readarray -t array < ./${repolist};
   i=0
-  csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst"
+  csv[$i]="${csv[i]},docs,conf.py,tox.ini,index.rst,first title in index.rst"
   ((i++))
   for line in "${array[@]}"
   do
@@ -574,7 +905,7 @@ do
       # tox.ini @ master/project root dir
       if [ -f ../master/${line}/tox.ini ] ; then
         docs="${docs} @root"
-      fi   
+      fi
       # just add a round bracket at the end of the value
       docs="${docs})"
     else
@@ -582,13 +913,16 @@ do
       docs="${docs},-"
     fi
 
-    # index.rst
+    # index.rst, first title in index.rst
+    indexrsttitle=""
     if [ -f ./${line}/docs/index.rst ] ; then
-      docs="${docs},index.rst"
+      indexrsttitle=$(cat ${branch}_indexrst_docs_root_titles.log | grep -F '['${line}']/docs/index.rst,' | awk -F "," '{print $4}');
+      docs="${docs},index.rst,${indexrsttitle}"
     elif [ -f ../master/${line}/docs/index.rst ] ; then
-      docs="${docs},(index.rst)"
+      indexrsttitle=$(cat ../master/master_indexrst_docs_root_titles.log | grep -F '['${line}']/docs/index.rst,' | awk -F "," '{print $4}');
+      docs="${docs},(index.rst),(${indexrsttitle})"
     else
-      docs="${docs},-"
+      docs="${docs},-,-"
     fi
 
     #echo "DBUG: docs=${docs}"
@@ -601,8 +935,8 @@ do
   unset docs
 
   #
-  # csv column #11: index.html@RTD accessibility check
-  # csv column #12: index.html url
+  # csv column #17: index.html@RTD accessibility check
+  # csv column #18: index.html url
   #
 
   readarray -t array < ./${branch}_repoclone.log;
@@ -715,7 +1049,7 @@ do
   done
 
   #
-  # csv column #13: release notes
+  # csv column #19: release notes
   #
 
   readarray -t array < ../${repolist};
@@ -737,7 +1071,7 @@ do
     # check if repo dir exists in this branch
     if [ -d ./${line} ] ; then
       # if yes, check if repo name appears in the branch releasenotes.log
-      relnote=$(find "./${line}" -type f | grep 'release.*note.*.rst' | wc -l);
+      relnote=$(find "./${line}" -type f | grep '.*release.*note.*.rst' | wc -l);
       #echo "DBUG: relnote=${relnote}"
       # repo dir DOES NOT exist in this branch - so check if repo dir exists in MASTER branch
     elif [ -d ../master/${line} ] ; then
@@ -778,6 +1112,8 @@ do
   datadir=${branch}_data
   mkdir $datadir
   cp $repolist $datadir
+  cp ../$wikiplsfile $datadir
+  cp ../$rpfile $datadir
   cp ${branch}_table.csv $datadir
   cp ${branch}_*.log $datadir
   zip -r ${datadir}.zip $datadir