3 #set -x # uncomment for bash script debugging
5 # ============================================================================
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 # ============LICENSE_END=====================================================
23 ### Thomas Kulik, Deutsche Telekom AG, 2020
26 ### warnstat helps to find the onap modules (projects) and rst-files which are
27 ### responsible for the most warnings during the documentation build process.
28 ### it requires a tox build logfile, parses it line by line, prints out some
29 ### statistics and provides links to the local rst file, its html version, the
30 ### related link to readthedocs and as well the doc8 test result for the rst.
34 ### CHANGELOG (LATEST ON TOP)
36 ### 1.5.0 (2020-03-23) - doc8 test now executed for every rst file. result is
37 ### provided in the output as "doc8_(nnnnn)" where nnnnn
38 ### is the total number of accumulated doc8 errors.
39 ### - improved readability of output
40 ### 1.4.0 (2020-03-18) - the link to the local html and rst file is provided in
41 ### the output. this may help to ease the debug process.
42 ### use mouse-over/context menu functionality of bash to
43 ### easily open files with your browser or rst editor.
44 ### - improved handling for module names (in case they are
45 ### no real onap projects/modules but directories which
46 ### contain additional documentation in rst format).
47 ### 1.3.1 (2020-03-10) - fixed minor typo in usage message
48 ### 1.3.0 (2020-03-09) - initial release
51 script_version="1.5.0 (2020-03-23)"
52 doc8_dir=$(pwd)/doc8_results
54 doc8_command="doc8 --verbose"; #add options if required
55 web_base_url="https://docs.onap.org/en/latest";
58 echo " warnstats version ${script_version}";
60 declare -A module_array
61 declare -A message_short_array
62 declare -A message_long_array
63 declare -A rstfile_array
64 declare -A rstfilepath_array
65 declare -A htmlfilepath_array
66 declare -A webpath_array
67 declare -A doc8_result_array
70 ### simple script argument handling
73 # check if there is an argument at all
74 if [[ "$logfile" == "" ]] ; then
75 echo 'Usage: warnstats [tox-logfile]'
79 # check if argument is a file
80 if [ ! -f $logfile ] ; then
81 echo "Error: can't find tox-logfile \"$logfile\""
85 # create and clean doc8 directory
86 if [ ! -d "$doc8_dir" ]; then
89 rm ${doc8_dir}/*.txt 2>/dev/null;
92 # get local html build directory
93 html_build_dir=$(grep "Generated docs available in" $logfile);
94 html_build_dir=$(echo "$html_build_dir" | grep -oP " /.*/doc/docs/_build/html$");
95 html_build_dir=$(echo "$html_build_dir" | sed -r 's:^ ::');
96 echo " html build directory ..... $html_build_dir"
97 echo " web base url ............. $web_base_url";
98 echo " doc8 command ............. $doc8_command";
99 echo " doc8 results directory ... $doc8_dir";
100 echo " tox logfile .............. $logfile";
102 # read in the tox build logfile - use only lines which contain a warning
103 readarray -t logfile_array < <(grep ": WARNING:" $logfile);
105 # process filtered logfile line by line
106 for line in "${logfile_array[@]}"
109 # count warning lines
111 echo -n -e " lines processed .......... $counter (doc8 check may take a while ...)\r";
114 # extract path to local rst file
117 #echo "DBUG line: $line"
118 # remove problematic text in line that causes regex to fail
119 path_rst=$(echo "$path_rst" | sed -r 's:, other instance in.*::');
120 #echo "DBUG path_rst: $path_rst"
121 # grep the rst file path
122 path_rst=$(echo "$path_rst" | grep -oP "^/.*\.rst");
123 #echo "DBUG path_rst: $path_rst"
124 if [[ "$path_rst" == "" ]] ; then
125 path_rst="path_to_rst_missing"
126 #echo "DBUG path_rst: $path_rst"
128 # finally embed the full rst path in a message to use mouse-over/context menu of bash to open file
129 path_rst_link='\e]8;;file:'$path_rst'\arst\e]8;;\a';
130 #echo -e "DBUG path_rst: "$path_rst;
133 # extract path to the html version of the local rst file
136 #echo "DBUG line: $line"
137 # remove problematic text in line that causes regex to fail
138 path_html=$(echo "$path_html" | sed -r 's:, other instance in.*::');
139 #echo "DBUG path_html: $path_html"
140 # grep the rst file path and modify it so we get the local html build path; grep a little bit more to be save
141 path_html=$(echo "$path_html" | grep -oP "(^|/)docs(/.*|)/[\w -]*\.rst");
142 #echo "DBUG path_html: $path_html"
143 path_html=$(echo "$path_html" | sed -r 's:^/docs::');
144 #echo "DBUG path_html: $path_html"
145 path_html=$(echo "$path_html" | sed -r 's:.rst:.html:');
146 #echo "DBUG path_html: $path_html"
147 # create also the path to the web version
148 path_web_link='\e]8;;'${web_base_url}${path_html}'\aweb\e]8;;\a';
149 #echo "DBUG path_web_link: $path_web_link"
150 # finally embed the full html path in a message to use mouse-over/context menu of bash to open file
151 path_html_link='\e]8;;file:'${html_build_dir}${path_html}'\ahtml\e]8;;\a';
152 #echo -e "DBUG path_html_link: "$path_html_link;
154 # extract module name from line (remove all text before module name; then cut out module name)
155 module=$(echo "$line" | sed -r 's:(^.*/doc/docs/submodules/|^docs/submodules/|checking consistency... )::' | cut -f1 -d\/);
156 #echo "DBUG line: $line"
157 #echo "DBUG module: $module"
159 # in case the extraction has not lead to a valid module name do some additional investigation
160 if [[ "$module" == "" ]] ; then
162 if [[ $line =~ doc/docs/release ]] ; then
163 module="docs_release"
164 #echo "DBUG line: $line"
165 #echo "DBUG module: $module"
166 elif [[ $line =~ doc/docs/use-cases ]] ; then
167 module="docs_use-cases"
168 #echo "DBUG line: $line"
169 #echo "DBUG module: $module"
170 elif [[ $line =~ doc/docs/guides ]] ; then
172 #echo "DBUG line: $line"
173 #echo "DBUG module: $module"
176 #echo "DBUG line: $line"
177 #echo "DBUG module: $module"
181 #echo "DBUG line: $line";
182 #echo "DBUG module: $module";
184 # get the maximum length of the variable entries to adjust table width later on
185 if [[ ${#module} -gt "$maxlength_module" ]]; then
186 maxlength_module=${#module};
188 #echo "DBUG maxlength_module=$maxlength_module";
190 # extract rst file name from line and do some formatting to use it later as an array name
191 #echo "DBUG line: $line";
192 rstfile=$(echo "$line" | grep -oP "[\w -]*\.rst");
193 rstfile=$(echo -e ${rstfile} | tr '[:blank:]' '_');
194 #echo "DBUG rst-file: $rstfile";
196 # get the maximum length of the variable entries to adjust table width later on
197 if [[ ${#rstfile} -gt "$maxlength_rstfile" ]]; then
198 maxlength_rstfile=${#rstfile};
200 #echo "DBUG maxlength_rstfile=$maxlength_rstfile";
202 # count the number of warnings for the module/rstfile combination
203 (( rstfile_array[$module | $rstfile]++ ));
205 # count the number of warnings for the single module
206 #echo "DBUG $module | $rstfile | $message";
207 (( module_array[$module]++ ));
209 # now we have all the information to fill the html/rst/web (file) path arrays
210 htmlfilepath_array[$module | $rstfile]=$path_html_link;
211 rstfilepath_array[$module | $rstfile]=$path_rst_link;
212 webpath_array[$module | $rstfile]=$path_web_link;
214 # extract the warning message and do some formatting
215 #message=$(echo "$line" | sed -r 's:^/.+WARNING\:\ ::');
216 message=$(echo "$line" | sed -r 's:^.+WARNING\:\ ::');
217 message=$(echo -e ${message} | tr '[:blank:]' '_');
218 message=$(echo -e ${message} | tr '/' '_');
219 message=$(echo -e ${message} | tr '.' '_');
221 # remove all characters from message which may cause problems in the shell
222 message="$(echo -e "${message}" | sed -e 's/[^A-Za-z0-9_-]//g')";
223 #echo "DBUG message=\"$message\""
225 # count the number of warnings for the single message (long version)
226 message_long="$(echo -e "${message}")";
227 (( message_long_array[$message_long]++ ))
229 # reduce length of message to group them more easily and then ...
230 # count the number of warnings for the single message (short version)
231 message_short="$(echo -e "${message}" | cut -c -16)";
232 (( message_short_array[$message_short]++ ))
234 # check rst files with doc8 and store results
235 doc8_result_path="${doc8_dir}/${module}-${rstfile}.txt";
236 #echo "DBUG ---------------------------------------------"
237 #echo "DBUG doc8_result_path=\"$doc8_result_path\""
238 # doc8 check only if result file does not exists yet AND if rst file is valid (exists)
239 if [[ ! -f "$doc8_result_path" && -f "$path_rst" ]] ; then
240 echo "FILE:$path_rst" >$doc8_result_path;
241 $doc8_command "$path_rst" >>$doc8_result_path;
242 total_acc_err=$(grep "Total accumulated errors = " $doc8_result_path);
243 #echo "DBUG total_acc_err=$total_acc_err";
244 total_acc_err=$(echo $total_acc_err | sed 's:Total accumulated errors = ::');
245 #echo "DBUG total_acc_err=$total_acc_err";
246 total_acc_err=$(printf "%05d" $total_acc_err);
247 #echo "DBUG command:doc8 ${path_rst} >>${doc8_result_path}";
248 #echo "DBUG total_acc_err=$total_acc_err";
250 doc8_result='\e]8;;file:'${doc8_result_path}'\adoc8_('$total_acc_err')\e]8;;\a';
251 doc8_result_array[$module | $rstfile]=$doc8_result;
255 #format counter to have always x digits
256 counter=$(printf "%05d" $counter);
258 echo " $counter LINES WITH WARNING IN FILE '$logfile'";
261 echo "################################################################################";
262 echo "~~~ MESSAGES LONG ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
263 echo "################################################################################";
266 #print array content and append to temporary outfile
267 for i in "${!message_long_array[@]}"
270 n=${message_long_array[$i]};
272 #format counter to have always x digits
273 n=$(printf "%05d" $n);
274 echo " $n | $m" >>tempoutfile;
277 #format counter to have always x digits
278 nc=$(printf "%05d" $nc);
279 echo " $nc WARNINGS IN TOTAL WITH ${#message_long_array[@]} UNIQUE MESSAGES" >>tempoutfile;
281 #print a sorted version of the temporary outfile
289 echo "################################################################################";
290 echo "~~~ MESSAGES SHORTENED (FOR SIMPLE GROUPING) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
291 echo "################################################################################";
294 #print array content and append to temporary outfile
295 for i in "${!message_short_array[@]}"
298 n=${message_short_array[$i]};
300 #format counter to have always x digits
301 n=$(printf "%05d" $n);
302 echo " $n | $m" >>tempoutfile;
305 #format counter to have always x digits
306 nc=$(printf "%05d" $nc);
307 echo " $nc WARNINGS IN TOTAL WITH ${#message_short_array[@]} UNIQUE MESSAGES" >>tempoutfile;
309 #print a sorted version of the temporary outfile
317 echo "################################################################################";
318 echo "~~~ MODULES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
319 echo "################################################################################";
322 #create temporary outfile
323 for i in "${!module_array[@]}"
326 n=${module_array[$i]};
328 n=$(printf "%05d" $n);
329 echo " $n | $m" >>tempoutfile;
332 #format counter to have always x digits
333 nc=$(printf "%05d" $nc);
334 echo " $nc WARNINGS IN TOTAL IN ${#module_array[@]} MODULES" >>tempoutfile;
336 #print a sorted version of the temporary outfile
342 echo "################################################################################";
343 echo "~~~ MODULES WITH RSTFILES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
344 echo "################################################################################";
347 #print array content and append to temporary outfile
348 for i in "${!rstfile_array[@]}"
351 n=${rstfile_array[$i]};
352 p=${htmlfilepath_array[$i]}
353 r=${rstfilepath_array[$i]}
354 w=${webpath_array[$i]}
355 d=${doc8_result_array[$i]};
356 #echo "DBUG -------------------------------"
363 #format counter to have always x digits
364 n=$(printf "%05d" $n);
366 # extend module name to the max for better readability
367 tmp_mod=$(echo "$m" | sed -r 's: \|.+$::');
368 #echo "DBUG tmp_mod=$tmp_mod"
369 len_tmp_mod=${#tmp_mod}
370 to_add="$(($maxlength_module-$len_tmp_mod))"
371 #echo "DBUG to_add=$to_add"
372 while [ $to_add -gt 0 ]; do
373 tmp_mod="${tmp_mod} ";
375 #echo "DBUG to_add=$to_add"
376 #echo "DBUG tmp_mod=\"$tmp_mod\""
379 # extend rst name to the max for better readability
380 tmp_rst=$(echo "$m" | sed -r 's:^.+ \| ::');
381 #echo "DBUG tmp_rst=$tmp_rst"
382 len_tmp_rst=${#tmp_rst}
383 to_add="$(($maxlength_rstfile-$len_tmp_rst))"
384 #echo "DBUG to_add=$to_add"
385 while [ $to_add -gt 0 ]; do
386 tmp_rst="${tmp_rst} ";
388 #echo "DBUG to_add=$to_add"
389 #echo "DBUG tmp_rst=\"$tmp_rst\""
392 # recombine module and rst names
393 m="${tmp_mod} | ${tmp_rst}";
395 # print out to temp file
396 echo -e " $m | $r $p $w $d | $n" >>tempoutfile;
399 #format counter to have always x digits
400 nc=$(printf "%05d" $nc);
401 #in case the name (e.g) index.rst is used multiple times in the same module warnings are combined
402 echo " $nc WARNINGS IN TOTAL IN APPROX. ${#rstfile_array[@]} RST FILES" >>tempoutfile;
404 #print a sorted version of the temporary outfile
412 echo "################################################################################";
413 echo "~~~ RSTFILES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
414 echo "################################################################################";
417 #print array content and append to temporary outfile
418 for i in "${!rstfile_array[@]}"
421 n=${rstfile_array[$i]};
422 p=${htmlfilepath_array[$i]}
423 r=${rstfilepath_array[$i]}
424 w=${webpath_array[$i]}
425 d=${doc8_result_array[$i]};
427 #format counter to have always x digits
428 n=$(printf "%05d" $n);
430 # extend module name to the max for better readability
431 tmp_mod=$(echo "$m" | sed -r 's: \|.+$::');
432 #echo "DBUG tmp_mod=$tmp_mod"
433 len_tmp_mod=${#tmp_mod}
434 to_add="$(($maxlength_module-$len_tmp_mod))"
435 #echo "DBUG to_add=$to_add"
436 while [ $to_add -gt 0 ]; do
437 tmp_mod="${tmp_mod} ";
439 #echo "DBUG to_add=$to_add"
440 #echo "DBUG tmp_mod=\"$tmp_mod\""
443 # extend rst name to the max for better readability
444 tmp_rst=$(echo "$m" | sed -r 's:^.+ \| ::');
445 #echo "DBUG tmp_rst=$tmp_rst"
446 len_tmp_rst=${#tmp_rst}
447 to_add="$(($maxlength_rstfile-$len_tmp_rst))"
448 #echo "DBUG to_add=$to_add"
449 while [ $to_add -gt 0 ]; do
450 tmp_rst="${tmp_rst} ";
452 #echo "DBUG to_add=$to_add"
453 #echo "DBUG tmp_rst=\"$tmp_rst\""
456 # recombine module and rst names
457 m="${tmp_mod} | ${tmp_rst}";
459 # print out to temp file
460 echo -e " $n | $m | $r $p $w $d" >>tempoutfile;
463 #format counter to have always x digits
464 nc=$(printf "%05d" $nc);
465 #in case the name (e.g) index.rst is used multiple times in the same module warnings are combined
466 echo " $nc WARNINGS IN TOTAL IN APPROX. ${#rstfile_array[@]} RST FILES" >>tempoutfile;
468 #print a sorted version of the temporary outfile
479 ### backup code for future extensions
483 # Block_quote_ends_without_a_blank_line_unexpected_unindent
484 # Bullet_list_ends_without_a_blank_line_unexpected_unindent
485 # Citation_[\w-]_is_not_referenced
486 # Citation_unit_test_is_not_referenced
487 # Content_block_expected_for_the_code_directive_none_found
488 # Content_block_expected_for_the_container_directive_none_found
489 # Could_not_lex_literal_block_as_bash__Highlighting_skipped
490 # Could_not_lex_literal_block_as_console__Highlighting_skipped
491 # Could_not_lex_literal_block_as_guess__Highlighting_skipped
492 # Could_not_lex_literal_block_as_json__Highlighting_skipped
493 # Could_not_lex_literal_block_as_yaml__Highlighting_skipped
494 # Definition_list_ends_without_a_blank_line_unexpected_unindent
495 # document_isnt_included_in_any_toctree
496 # download_file_not_readable
497 # Duplicate_explicit_target_name
499 # Enumerated_list_ends_without_a_blank_line_unexpected_unindent
500 # Error_in_code_directive
501 # Error_in_code-block_directive
502 # Error_in_image_directive
503 # Explicit_markup_ends_without_a_blank_line_unexpected_unindent
504 # Field_list_ends_without_a_blank_line_unexpected_unindent
505 # Footnote_[0-9.*]_is_not_referenced
506 # image_file_not_readable
508 # Inconsistent_literal_block_quoting
509 # Inline_emphasis_start-string_without_end-string
510 # Inline_interpreted_text_or_phrase_reference_start-string_without_end-string
511 # Inline_strong_start-string_without_end-string
512 # Inline_substitution_reference_start-string_without_end-string
513 # Literal_block_ends_without_a_blank_line_unexpected_unindent
514 # Literal_block_expected_none_found
516 # Pygments_lexer_name_asn_is_not_known
517 # Title_level_inconsistent
518 # Title_overline__underline_mismatch
519 # Title_overline_too_short
520 # Title_underline_too_short
521 # toctree_contains_reference_to_nonexisting_document
522 # Too_many_autonumbered_footnote_references_only_0_corresponding_footnotes_available
523 # undecodable_source_characters_replacing_with
525 # Unexpected_indentation
526 # Unknown_directive_type_clode-block
528 # Unknown_target_name