X-Git-Url: https://gerrit.onap.org/r/gitweb?a=blobdiff_plain;f=tools%2Fwarnstats.sh;h=2535ee62334950895e6aca334b65216373cd6461;hb=61444645522ad9d03be1f8baec736be002a5bb79;hp=96bbfb8bce6067ce32351e075816525ce3e1d248;hpb=fb8a0eea26492371b52eb08c0433db12ef96cf49;p=doc.git

diff --git a/tools/warnstats.sh b/tools/warnstats.sh
index 96bbfb8bc..2535ee623 100755
--- a/tools/warnstats.sh
+++ b/tools/warnstats.sh
@@ -24,35 +24,62 @@
 ###
 ### DESCRIPTION:
 ### warnstat helps to find the onap modules (projects) and rst-files which are
-### responsible for the most warnings during the documentation build process
-### it requires a tox build logfile, parses it line by line and prints out some
-### statistics
+### responsible for the most warnings during the documentation build process.
+### it requires a tox build logfile, parses it line by line, prints out some
+### statistics and provides links to the local rst file, its html version, the
+### related link to readthedocs and as well the doc8 test result for the rst.
 ###
 
 ###
 ### CHANGELOG (LATEST ON TOP)
 ###
-### 1.3.1 (2020-03-10) fixed minor typo in usage message
-### 1.3.0 (2020-03-09) initial release
+### 1.6.2 (2020-05-14) - fixed a major problem with rst files within one module
+###                      which have the same name (but reside in different
+###                      subdirectories). they were not shown in the result
+###                      list. introduced a crc-number for every file for proper
+###                      identification and to build the index.
+###                    - fixed a problem where the results are showing the link
+###                      to a wrong file due to a regex problem in the script.
+### 1.6.1 (2020-04-21) - fixed a problem with duplicates in rst filenames
+### 1.6.0 (2020-04-03) - extended detection of docs pathes in case they are not
+###                      below the submodules directory
+### 1.5.0 (2020-03-23) - doc8 test now executed for every rst file. result is
+###                      provided in the output as "doc8_(nnnnn)" where nnnnn
+###                      is the total number of accumulated doc8 errors.
+###                    - improved readability of output
+### 1.4.0 (2020-03-18) - the link to the local html and rst file is provided in
+###                      the output. this may help to ease the debug process.
+###                      use mouse-over/context menu functionality of bash to
+###                      easily open files with your browser or rst editor.
+###                    - improved handling for module names (in case they are
+###                      no real onap projects/modules but directories which
+###                      contain additional documentation in rst format).
+### 1.3.1 (2020-03-10) - fixed minor typo in usage message
+### 1.3.0 (2020-03-09) - initially released to the community
 ###
 
-script_version="1.3.1 (2020-03-10)"
+script_version="1.6.2 (2020-05-14)"
+doc8_dir=$(pwd)/doc8_results
+logfile=$1;
+doc8_command="doc8 --verbose"; #add options if required
+web_base_url="https://docs.onap.org/en/latest";
 
 echo " ";
-echo "warnstats - Version ${script_version}";
-echo " ";
+echo " warnstats version ${script_version}";
 
 declare -A module_array
 declare -A message_short_array
 declare -A message_long_array
 declare -A rstfile_array
+declare -A rstfilepath_array
+declare -A htmlfilepath_array
+declare -A webpath_array
+declare -A doc8_result_array
 
 ###
 ### simple script argument handling
 ###
 
-logfile=$1;
-
 # check if there is an argument at all
 if [[ "$logfile" == "" ]] ; then
     echo 'Usage: warnstats [tox-logfile]'
@@ -65,37 +92,180 @@ if [ ! -f $logfile ] ; then
     exit 1
 fi
 
+# create and clean doc8 directory
+if [ ! -d "$doc8_dir" ]; then
+  mkdir $doc8_dir;
+else
+  rm ${doc8_dir}/*.txt 2>/dev/null;
+fi
+
+# get local html build directory
+html_build_dir=$(grep "Generated docs available in" $logfile);
+html_build_dir=$(echo "$html_build_dir" | grep -oP " /.*/doc/docs/_build/html$");
+html_build_dir=$(echo "$html_build_dir" | sed -r 's:^ ::');
+echo " html build directory ..... $html_build_dir"
+echo " web base url ............. $web_base_url";
+echo " doc8 command ............. $doc8_command";
+echo " doc8 results directory ... $doc8_dir";
+echo " tox logfile .............. $logfile";
+
 # read in the tox build logfile - use only lines which contain a warning
 readarray -t logfile_array < <(grep ": WARNING:" $logfile);
 
 # process filtered logfile line by line
 for line in "${logfile_array[@]}"
 do
+
     # count warning lines
     (( counter++ ));
-    echo -n -e "lines processed: $counter\r";
-
-    # extract module name from line
-    module=$(echo "$line" | sed -r 's:^/.+/doc/docs/(submodules|guides)/::' | cut -f1 -d\/);
+    echo -n -e " lines processed .......... $counter (doc8 check may take a while ...)\r";
+
+    #
+    # extract path to local rst file
+    #
+
+    # remove problematic text in the original line that causes regex to fail
+    line=$(echo "$line" | sed -r 's:, other instance in.*::');
+
+    path_rst=$line;
+    path_rst_debug=$line;
+    #echo "DBUG      line: $line"
+    # remove problematic text in line that causes regex to fail
+    path_rst=$(echo "$path_rst" | sed -r 's:, other instance in.*::');
+    #echo "DBUG path_rst: $path_rst"
+    # grep the rst file path
+    path_rst=$(echo "$path_rst" | grep -oP "^(/|docs).*\.rst");
+    #echo "DBUG path_rst: $path_rst"
+    # create an unique identifier for the rst file for the case that the rst file name is used multiple times (but in different subdirectories) within one module
+    rst_crc=$(crc32 "$path_rst" 2>/dev/null);
+    #echo "DBUG  rst_crc: $rst_crc"
+    if [[ "$rst_crc" == "" ]] ; then
+      rst_crc="rst_crc_missing"
+    fi
 
-    # in case the extraction has no valid name fill the missing field
+    if [[ "$path_rst" == "" ]] ; then
+      path_rst="path_to_rst_missing"
+      #echo "DBUG       path_rst: $path_rst"
+      #echo "DBUG path_rst_debug: $path_rst_debug"
+    fi
+    # finally embed the full rst path in a message to use mouse-over/context menu of bash to open file
+    path_rst_link='\e]8;;file:'$path_rst'\arst\e]8;;\a';
+    #echo -e "DBUG path_rst: "$path_rst;
+
+    #
+    # extract path to the html version of the local rst file
+    #
+
+    path_html=$line;
+    #echo "DBUG      line: $line"
+    # remove problematic text in line that causes regex to fail
+    path_html=$(echo "$path_html" | sed -r 's:, other instance in.*::');
+    #echo "DBUG path_html: $path_html"
+    # grep the rst file path and modify it so we get the local html build path; grep a little bit more to be save
+    path_html=$(echo "$path_html" | grep -oP "(^|/)docs(/.*|)/[\w -]*\.rst");
+    #echo "DBUG path_html: $path_html"
+    path_html=$(echo "$path_html" | sed -r 's:^/docs::');
+    #echo "DBUG path_html: $path_html"
+    path_html=$(echo "$path_html" | sed -r 's:.rst:.html:');
+    #echo "DBUG path_html: $path_html"
+    # create also the path to the web version
+    path_web_link='\e]8;;'${web_base_url}${path_html}'\aweb\e]8;;\a';
+    #echo "DBUG path_web_link: $path_web_link"
+    # finally embed the full html path in a message to use mouse-over/context menu of bash to open file
+    path_html_link='\e]8;;file:'${html_build_dir}${path_html}'\ahtml\e]8;;\a';
+    #echo -e "DBUG path_html_link: "$path_html_link;
+
+    # extract module name from line (remove all text before module name; then cut out module name)
+    module=$(echo "$line" | sed -r 's:(^.*/doc/docs/submodules/|^docs/submodules/|checking consistency... )::' | cut -f1 -d\/);
+    #echo "DBUG   line: $line"
+    #echo "DBUG module: $module"
+
+    # in case the extraction has not lead to a valid module name do some additional investigation
     if [[ "$module" == "" ]] ; then
-        module="<missing_module_name>";
+
+      if [[ $line =~ doc/docs/release ]] ; then
+          module="docs_release"
+          #echo "DBUG   line: $line"
+          #echo "DBUG module: $module"
+      elif [[ $line =~ doc/docs/use-cases ]] ; then
+          module="docs_use-cases"
+          #echo "DBUG   line: $line"
+          #echo "DBUG module: $module"
+      elif [[ $line =~ doc/docs/guides/onap-developer ]] ; then
+          module="docs_guides_onap-developer"
+          #echo "DBUG   line: $line"
+          #echo "DBUG module: $module"
+      elif [[ $line =~ doc/docs/guides/onap-operator ]] ; then
+          module="docs_guides_onap-operator"
+          #echo "DBUG   line: $line"
+          #echo "DBUG module: $module"
+      elif [[ $line =~ doc/docs/guides/onap-provider ]] ; then
+          module="docs_guides_onap-provider"
+          #echo "DBUG   line: $line"
+          #echo "DBUG module: $module"
+      elif [[ $line =~ doc/docs/guides/onap-user ]] ; then
+          module="docs_guides_onap-user"
+          #echo "DBUG   line: $line"
+          #echo "DBUG module: $module"
+      elif [[ $line =~ doc/docs/guides/overview ]] ; then
+          module="docs_guides_overview"
+          #echo "DBUG   line: $line"
+          #echo "DBUG module: $module"
+      elif [[ $line =~ doc/docs/templates ]] ; then
+          module="docs_templates"
+          #echo "DBUG   line: $line"
+          #echo "DBUG module: $module"
+      elif [[ $line =~ doc/docs/guides ]] ; then
+          module="docs_guides"
+          #echo "DBUG   line: $line"
+          #echo "DBUG module: $module"
+      else
+          module="docs"
+          #echo "DBUG   line: $line"
+          #echo "DBUG module: $module"
+      fi
+
+    fi
+    #echo "DBUG   line: $line";
+    #echo "DBUG module: $module";
+
+    # get the maximum length of the variable entries to adjust table width later on
+    if [[ ${#module} -gt "$maxlength_module" ]]; then
+      maxlength_module=${#module};
     fi
+    #echo "DBUG maxlength_module=$maxlength_module";
 
     # extract rst file name from line and do some formatting to use it later as an array name
     #echo "DBUG line: $line";
-    rstfile=$(echo "$line" | grep -oP "[\w -]*\.rst");
+    rstfile=$(echo "$line" | sed -r 's:, other instance in.*::');
+    rstfile=$(echo -e "${rstfile}" | grep -oP "[\w -]*\.rst");
     rstfile=$(echo -e ${rstfile} | tr '[:blank:]' '_');
-    #echo "DBUG rst-file: $rstfile";
+    #echo "DBUG rstfile: '$rstfile'";
+
+    # get the maximum length of the variable entries to adjust table width later on
+    if [[ ${#rstfile} -gt "$maxlength_rstfile" ]]; then
+      maxlength_rstfile=${#rstfile};
+    fi
+    #echo "DBUG maxlength_rstfile=$maxlength_rstfile";
 
     # count the number of warnings for the module/rstfile combination
-    (( rstfile_array[$module | $rstfile]++ ));
+    (( rstfile_array[$module | $rstfile | $rst_crc]++ ));
 
     # count the number of warnings for the single module
     #echo "DBUG $module | $rstfile | $message";
     (( module_array[$module]++ ));
 
+    # now we have all the information to fill the html/rst/web (file) path arrays
+    htmlfilepath_array[$module | $rstfile | $rst_crc]=$path_html_link;
+     rstfilepath_array[$module | $rstfile | $rst_crc]=$path_rst_link;
+         webpath_array[$module | $rstfile | $rst_crc]=$path_web_link;
+
+    #echo "DBUG -------------------------------------------------------------------";
+    #echo "DBUG               line: $line";
+    #echo "DBUG htmlfilepath_array: $module | $rstfile | $rst_crc = $path_html_link";
+    #echo "DBUG  rstfilepath_array: $module | $rstfile | $rst_crc = $path_rst_link";
+    #echo "DBUG      webpath_array: $module | $rstfile | $rst_crc = $path_web_link";
+
     # extract the warning message and do some formatting
     #message=$(echo "$line" | sed -r 's:^/.+WARNING\:\ ::');
     message=$(echo "$line" | sed -r 's:^.+WARNING\:\ ::');
@@ -113,14 +283,33 @@ do
 
     # reduce length of message to group them more easily and then ...
     # count the number of warnings for the single message (short version)
-    message_short="$(echo -e "${message}" | cut -c -20)";
+    message_short="$(echo -e "${message}" | cut -c -16)";
     (( message_short_array[$message_short]++ ))
 
+    # check rst files with doc8 and store results
+    doc8_result_path="${doc8_dir}/${module}-${rstfile}-${rst_crc}.txt";
+    #echo "DBUG ---------------------------------------------"
+    #echo "DBUG doc8_result_path=\"$doc8_result_path\""
+    # doc8 check only if result file does not exists yet AND if rst file is valid (exists)
+    if [[ ! -f "$doc8_result_path" && -f "$path_rst" ]] ; then
+        echo "FILE:$path_rst" >$doc8_result_path;
+        $doc8_command "$path_rst" >>$doc8_result_path;
+        total_acc_err=$(grep "Total accumulated errors = " $doc8_result_path);
+        #echo "DBUG total_acc_err=$total_acc_err";
+        total_acc_err=$(echo $total_acc_err | sed 's:Total accumulated errors = ::');
+        #echo "DBUG total_acc_err=$total_acc_err";
+        total_acc_err=$(printf "%05d" $total_acc_err);
+        #echo "DBUG command:doc8 ${path_rst} >>${doc8_result_path}";
+        #echo "DBUG total_acc_err=$total_acc_err";
+    fi
+    doc8_result='\e]8;;file:'${doc8_result_path}'\adoc8_('$total_acc_err')\e]8;;\a';
+    doc8_result_array[$module | $rstfile | $rst_crc]=$doc8_result;
+
 done
 
 #format counter to have always x digits
 counter=$(printf "%05d" $counter);
-echo "                              ";
+echo "                                                                      ";
 echo " $counter LINES WITH WARNING IN FILE '$logfile'";
 
 echo " ";
@@ -153,7 +342,7 @@ nc=0
 
 echo " ";
 echo "################################################################################";
-echo "~~~ MESSAGES SHORTENED (FOR BETTER GROUPING) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
+echo "~~~ MESSAGES SHORTENED (FOR SIMPLE GROUPING) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
 echo "################################################################################";
 echo " ";
 
@@ -215,10 +404,61 @@ for i in "${!rstfile_array[@]}"
 do
   m=$i;
   n=${rstfile_array[$i]};
+  p=${htmlfilepath_array[$i]}
+  r=${rstfilepath_array[$i]}
+  w=${webpath_array[$i]}
+  d=${doc8_result_array[$i]};
+  #echo "DBUG -------------------------------"
+  #echo "DBUG i = '$i'"
+  #echo "DBUG m = '$m'"
+  #echo "DBUG n = '$n'"
+  #echo "DBUG p = '$p'"
+  #echo -e "DBUG p = '$p'"
+  #echo "DBUG w = '$w'"
+  #echo "DBUG d = '$d'"
   ((nc += n))
   #format counter to have always x digits
   n=$(printf "%05d" $n);
-  echo " $m | $n" >>tempoutfile;
+
+  # extend module name to the max for better readability
+  tmp_mod=$(echo "$m" | sed -r 's: \|.+$::');
+  #echo "DBUG tmp_mod=$tmp_mod"
+  len_tmp_mod=${#tmp_mod}
+  to_add="$(($maxlength_module-$len_tmp_mod))"
+  #echo "DBUG to_add=$to_add"
+  while [ $to_add -gt 0 ]; do
+      tmp_mod="${tmp_mod} ";
+      ((to_add--));
+      #echo "DBUG  to_add=$to_add"
+      #echo "DBUG tmp_mod=\"$tmp_mod\""
+  done
+
+  # remove crc and extend rst name to the max for better readability
+  #echo "DBUG ******************************************************"
+  #echo "DBUG m = '$m'"
+  tmp_rst=$(echo "$m" | sed -r 's:\| [[:alnum:]_]+$::');
+  #echo "DBUG tmp_rst = '$tmp_rst'"
+  tmp_rst=$(echo "$tmp_rst" | sed -r 's:[[:space:]]$::');
+  #echo "DBUG tmp_rst = '$tmp_rst'"
+  tmp_rst=$(echo "$tmp_rst" | sed -r 's:^.+ \| ::');
+  #echo "DBUG tmp_rst = '$tmp_rst'"
+  len_tmp_rst=${#tmp_rst}
+  #echo "DBUG len_tmp_rst = '$len_tmp_rst'"
+  to_add="$(($maxlength_rstfile-$len_tmp_rst))"
+  #echo "DBUG to_add = '$to_add'"
+  while [ $to_add -gt 0 ]; do
+      tmp_rst="${tmp_rst} ";
+      ((to_add--));
+      #echo "DBUG  to_add = '$to_add'"
+      #echo "DBUG tmp_rst = '$tmp_rst'"
+  done
+
+  # recombine module and rst names
+  m="${tmp_mod} | ${tmp_rst}";
+  #echo "DBUG m = '$m'"
+
+  # print out to temp file
+  echo -e " $m | $r  $p  $w  $d | $n" >>tempoutfile;
 done
 
 #format counter to have always x digits
@@ -244,10 +484,61 @@ for i in "${!rstfile_array[@]}"
 do
   m=$i;
   n=${rstfile_array[$i]};
+  p=${htmlfilepath_array[$i]}
+  r=${rstfilepath_array[$i]}
+  w=${webpath_array[$i]}
+  d=${doc8_result_array[$i]};
+  #echo "DBUG -------------------------------"
+  #echo "DBUG i=$i"
+  #echo "DBUG m=$m"
+  #echo "DBUG n=$n"
+  #echo "DBUG p=$p"
+  #echo -e "DBUG p=$p"
+  #echo "DBUG m=$r"
+  #echo "DBUG n=$w"
+  #echo "DBUG p=$d"
   ((nc += n))
   #format counter to have always x digits
   n=$(printf "%05d" $n);
-  echo " $n | $m" >>tempoutfile;
+
+  # extend module name to the max for better readability
+  tmp_mod=$(echo "$m" | sed -r 's: \|.+$::');
+  #echo "DBUG tmp_mod=$tmp_mod"
+  len_tmp_mod=${#tmp_mod}
+  to_add="$(($maxlength_module-$len_tmp_mod))"
+  #echo "DBUG to_add=$to_add"
+  while [ $to_add -gt 0 ]; do
+      tmp_mod="${tmp_mod} ";
+      ((to_add--));
+      #echo "DBUG  to_add=$to_add"
+      #echo "DBUG tmp_mod=\"$tmp_mod\""
+  done
+
+  # remove crc and extend rst name to the max for better readability
+  #echo "DBUG ******************************************************"
+  #echo "DBUG m = '$m'"
+  tmp_rst=$(echo "$m" | sed -r 's:\| [[:alnum:]_]+$::');
+  #echo "DBUG tmp_rst = '$tmp_rst'"
+  tmp_rst=$(echo "$tmp_rst" | sed -r 's:[[:space:]]$::');
+  #echo "DBUG tmp_rst = '$tmp_rst'"
+  tmp_rst=$(echo "$tmp_rst" | sed -r 's:^.+ \| ::');
+  #echo "DBUG tmp_rst = '$tmp_rst'"
+  len_tmp_rst=${#tmp_rst}
+  #echo "DBUG len_tmp_rst = '$len_tmp_rst'"
+  to_add="$(($maxlength_rstfile-$len_tmp_rst))"
+  #echo "DBUG to_add = '$to_add'"
+  while [ $to_add -gt 0 ]; do
+      tmp_rst="${tmp_rst} ";
+      ((to_add--));
+      #echo "DBUG  to_add = '$to_add'"
+      #echo "DBUG tmp_rst = '$tmp_rst'"
+  done
+
+  # recombine module and rst names
+  m="${tmp_mod} | ${tmp_rst}";
+
+  # print out to temp file
+  echo -e " $n | $m | $r  $p  $w  $d" >>tempoutfile;
 done
 
 #format counter to have always x digits