3 #set -x # uncomment for bash script debugging
5 # ============================================================================
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 # ============LICENSE_END=====================================================
23 ### Thomas Kulik, Deutsche Telekom AG, 2020
26 ### warnstat helps to find the onap modules (projects) and rst-files which are
27 ### responsible for the most warnings during the documentation build process
28 ### it requires a tox build logfile, parses it line by line and prints out some
33 ### CHANGELOG (LATEST ON TOP)
35 ### 1.3.1 (2020-03-10) fixed minor typo in usage message
36 ### 1.3.0 (2020-03-09) initial release
39 script_version="1.3.1 (2020-03-10)"
42 echo "warnstats - Version ${script_version}";
45 declare -A module_array
46 declare -A message_short_array
47 declare -A message_long_array
48 declare -A rstfile_array
51 ### simple script argument handling
56 # check if there is an argument at all
57 if [[ "$logfile" == "" ]] ; then
58 echo 'Usage: warnstats [tox-logfile]'
62 # check if argument is a file
63 if [ ! -f $logfile ] ; then
64 echo "Error: can't find tox-logfile \"$logfile\""
68 # read in the tox build logfile - use only lines which contain a warning
69 readarray -t logfile_array < <(grep ": WARNING:" $logfile);
71 # process filtered logfile line by line
72 for line in "${logfile_array[@]}"
76 echo -n -e "lines processed: $counter\r";
78 # extract module name from line
79 module=$(echo "$line" | sed -r 's:^/.+/doc/docs/(submodules|guides)/::' | cut -f1 -d\/);
81 # in case the extraction has no valid name fill the missing field
82 if [[ "$module" == "" ]] ; then
83 module="<missing_module_name>";
86 # extract rst file name from line and do some formatting to use it later as an array name
87 #echo "DBUG line: $line";
88 rstfile=$(echo "$line" | grep -oP "[\w -]*\.rst");
89 rstfile=$(echo -e ${rstfile} | tr '[:blank:]' '_');
90 #echo "DBUG rst-file: $rstfile";
92 # count the number of warnings for the module/rstfile combination
93 (( rstfile_array[$module | $rstfile]++ ));
95 # count the number of warnings for the single module
96 #echo "DBUG $module | $rstfile | $message";
97 (( module_array[$module]++ ));
99 # extract the warning message and do some formatting
100 #message=$(echo "$line" | sed -r 's:^/.+WARNING\:\ ::');
101 message=$(echo "$line" | sed -r 's:^.+WARNING\:\ ::');
102 message=$(echo -e ${message} | tr '[:blank:]' '_');
103 message=$(echo -e ${message} | tr '/' '_');
104 message=$(echo -e ${message} | tr '.' '_');
106 # remove all characters from message which may cause problems in the shell
107 message="$(echo -e "${message}" | sed -e 's/[^A-Za-z0-9_-]//g')";
108 #echo "DBUG message=\"$message\""
110 # count the number of warnings for the single message (long version)
111 message_long="$(echo -e "${message}")";
112 (( message_long_array[$message_long]++ ))
114 # reduce length of message to group them more easily and then ...
115 # count the number of warnings for the single message (short version)
116 message_short="$(echo -e "${message}" | cut -c -20)";
117 (( message_short_array[$message_short]++ ))
121 #format counter to have always x digits
122 counter=$(printf "%05d" $counter);
124 echo " $counter LINES WITH WARNING IN FILE '$logfile'";
127 echo "################################################################################";
128 echo "~~~ MESSAGES LONG ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
129 echo "################################################################################";
132 #print array content and append to temporary outfile
133 for i in "${!message_long_array[@]}"
136 n=${message_long_array[$i]};
138 #format counter to have always x digits
139 n=$(printf "%05d" $n);
140 echo " $n | $m" >>tempoutfile;
143 #format counter to have always x digits
144 nc=$(printf "%05d" $nc);
145 echo " $nc WARNINGS IN TOTAL WITH ${#message_long_array[@]} UNIQUE MESSAGES" >>tempoutfile;
147 #print a sorted version of the temporary outfile
155 echo "################################################################################";
156 echo "~~~ MESSAGES SHORTENED (FOR BETTER GROUPING) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
157 echo "################################################################################";
160 #print array content and append to temporary outfile
161 for i in "${!message_short_array[@]}"
164 n=${message_short_array[$i]};
166 #format counter to have always x digits
167 n=$(printf "%05d" $n);
168 echo " $n | $m" >>tempoutfile;
171 #format counter to have always x digits
172 nc=$(printf "%05d" $nc);
173 echo " $nc WARNINGS IN TOTAL WITH ${#message_short_array[@]} UNIQUE MESSAGES" >>tempoutfile;
175 #print a sorted version of the temporary outfile
183 echo "################################################################################";
184 echo "~~~ MODULES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
185 echo "################################################################################";
188 #create temporary outfile
189 for i in "${!module_array[@]}"
192 n=${module_array[$i]};
194 n=$(printf "%05d" $n);
195 echo " $n | $m" >>tempoutfile;
198 #format counter to have always x digits
199 nc=$(printf "%05d" $nc);
200 echo " $nc WARNINGS IN TOTAL IN ${#module_array[@]} MODULES" >>tempoutfile;
202 #print a sorted version of the temporary outfile
208 echo "################################################################################";
209 echo "~~~ MODULES WITH RSTFILES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
210 echo "################################################################################";
213 #print array content and append to temporary outfile
214 for i in "${!rstfile_array[@]}"
217 n=${rstfile_array[$i]};
219 #format counter to have always x digits
220 n=$(printf "%05d" $n);
221 echo " $m | $n" >>tempoutfile;
224 #format counter to have always x digits
225 nc=$(printf "%05d" $nc);
226 #in case the name (e.g) index.rst is used multiple times in the same module warnings are combined
227 echo " $nc WARNINGS IN TOTAL IN APPROX. ${#rstfile_array[@]} RST FILES" >>tempoutfile;
229 #print a sorted version of the temporary outfile
237 echo "################################################################################";
238 echo "~~~ RSTFILES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~";
239 echo "################################################################################";
242 #print array content and append to temporary outfile
243 for i in "${!rstfile_array[@]}"
246 n=${rstfile_array[$i]};
248 #format counter to have always x digits
249 n=$(printf "%05d" $n);
250 echo " $n | $m" >>tempoutfile;
253 #format counter to have always x digits
254 nc=$(printf "%05d" $nc);
255 #in case the name (e.g) index.rst is used multiple times in the same module warnings are combined
256 echo " $nc WARNINGS IN TOTAL IN APPROX. ${#rstfile_array[@]} RST FILES" >>tempoutfile;
258 #print a sorted version of the temporary outfile
269 ### backup code for future extensions
273 # Block_quote_ends_without_a_blank_line_unexpected_unindent
274 # Bullet_list_ends_without_a_blank_line_unexpected_unindent
275 # Citation_[\w-]_is_not_referenced
276 # Citation_unit_test_is_not_referenced
277 # Content_block_expected_for_the_code_directive_none_found
278 # Content_block_expected_for_the_container_directive_none_found
279 # Could_not_lex_literal_block_as_bash__Highlighting_skipped
280 # Could_not_lex_literal_block_as_console__Highlighting_skipped
281 # Could_not_lex_literal_block_as_guess__Highlighting_skipped
282 # Could_not_lex_literal_block_as_json__Highlighting_skipped
283 # Could_not_lex_literal_block_as_yaml__Highlighting_skipped
284 # Definition_list_ends_without_a_blank_line_unexpected_unindent
285 # document_isnt_included_in_any_toctree
286 # download_file_not_readable
287 # Duplicate_explicit_target_name
289 # Enumerated_list_ends_without_a_blank_line_unexpected_unindent
290 # Error_in_code_directive
291 # Error_in_code-block_directive
292 # Error_in_image_directive
293 # Explicit_markup_ends_without_a_blank_line_unexpected_unindent
294 # Field_list_ends_without_a_blank_line_unexpected_unindent
295 # Footnote_[0-9.*]_is_not_referenced
296 # image_file_not_readable
298 # Inconsistent_literal_block_quoting
299 # Inline_emphasis_start-string_without_end-string
300 # Inline_interpreted_text_or_phrase_reference_start-string_without_end-string
301 # Inline_strong_start-string_without_end-string
302 # Inline_substitution_reference_start-string_without_end-string
303 # Literal_block_ends_without_a_blank_line_unexpected_unindent
304 # Literal_block_expected_none_found
306 # Pygments_lexer_name_asn_is_not_known
307 # Title_level_inconsistent
308 # Title_overline__underline_mismatch
309 # Title_overline_too_short
310 # Title_underline_too_short
311 # toctree_contains_reference_to_nonexisting_document
312 # Too_many_autonumbered_footnote_references_only_0_corresponding_footnotes_available
313 # undecodable_source_characters_replacing_with
315 # Unexpected_indentation
316 # Unknown_directive_type_clode-block
318 # Unknown_target_name