3 #set -x # uncomment for bash script debugging
5 ### ============================================================================
6 ### Licensed under the Apache License, Version 2.0 (the "License");
7 ### you may not use this file except in compliance with the License.
8 ### You may obtain a copy of the License at
10 ### http://www.apache.org/licenses/LICENSE-2.0
12 ### Unless required by applicable law or agreed to in writing, software
13 ### distributed under the License is distributed on an "AS IS" BASIS,
14 ### WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 ### See the License for the specific language governing permissions and
16 ### limitations under the License.
17 ### ============LICENSE_END=====================================================
24 ### Thomas Kulik, Deutsche Telekom AG, 2020
27 ### c2m automates additional tasks required in case you want to export and
28 ### convert a set of wiki pages. the export and first conversion to markdown is
29 ### done by confluence2md, provided by viaboxx.
30 ### c2m processes a list of (to be exported) wiki pages, creates corresponding
31 ### export directories, exports and converts pages (in various formats if
32 ### required), opens an editor and cleans up afterwards.
33 ### c2m checks also for problematic content in the export and creates a warning
34 ### in case of detection.
37 ### - markdown (md) output of confluence2md contains sometimes tags that are
38 ### somehow "merged" with the topic headline; manual edit is required here
41 ### - confluence2md does not support all of the currently used confluence page
42 ### types (structured-macros) - result for unsupported pages is
43 ### "not satisfying"; enhancements (java) are required
44 ### - opt: toc creation in root document in case you export a tree of documents
46 ### - opt: remove wiki credentials from script
49 ### - pandoc, retext, confluence2md, java (older version for confluence2md),
50 ### login for the confluence wiki
53 ### - https://www.viaboxx.de/code/confluence2md/
54 ### - https://github.com/viaboxxsystems/confluence2md
59 ### CHANGELOG (LATEST ON TOP)
61 ### 1.2.0 (2021-08-02) Corrections to http/https proxy handling and support to
62 ### get Confluence credentials from env variables instead of
63 ### directly from the code.
64 ### 1.1.0 (2020-03-10) added support for http/https proxy and anonymous wiki
65 ### access. thx to eric, nicolas and sylvain (orange, france)
66 ### confluence2md jar file now has to be in the same path as
68 ### 1.0.0 (2020-03-09) initial release
73 ### c2m example pagelist
75 ### example pagelist (field descriptions below); it uses the delimiter "|" for
76 ### the four fields per line.
77 ### copy/paste page id and title from wiki; to get the wiki page_id you have to
78 ### login to the wiki, open the page and choose e.g. the history.
79 ### depth: use depth to follow down the child-pages hierarchy if required:
80 ### -1=infinte, 0=no children, #=number of child-pages to follow.
81 ### every hierarchy "0" entry will lead into the creation of a dedicated working
82 ### directory where the page and child-pages are stored.
83 ### for better readability you can add spaces to the list, but use "|" as a
84 ### delimiter. lines starting with a # are filtered by c2m.
86 ### hierarchy | page_id | page_title | depth
88 ### 0 | 1018748 | ONAP Portal | 0
89 ### 1.1 | 1018759 | ONAP Portal for users | 0
90 ### 1.2 | 1018762 | ONAP Portal for administrators | 0
91 ### 1.2.1 | 1018764 | Admins | 0
92 ### 1.2.2 | 1018811 | Users | 0
93 ### 1.2.3 | 1018821 | Portal Admins | 0
94 ### 1.2.4 | 1018826 | Application Onboarding | 0
95 ### 1.2.5 | 1018832 | Widget Onboarding | 0
96 ### 1.2.6 | 1018835 | Edit Functional Menu | 0
97 ### 1.2.7 | 16004953 | Portal Microservices Onboarding | 0
99 ### in case you want to export to only one single output page (that contains all
100 ### child-pages of the above example) use:
102 ### 0 | 1018748 | ONAP Portal | -1
107 ### some initial variables
110 script_version="1.2.0 (2021-08-02)"
112 if [[ -z "$CONFLUENCE_USERNAME" || -z "$CONFLUENCE_PASSWORD" ]]
114 echo "Mandatory environment variables:"
115 echo " CONFLUENCE_USERNAME: Confluence username"
116 echo " CONFLUENCE_PASSWORD: Confluence password."
117 echo "Be aware! Setting bash debuging on will print credentials."
121 user="${CONFLUENCE_USERNAME}";
122 passwd="${CONFLUENCE_PASSWORD}";
123 credentials="${user}":"${passwd}";
124 server="https://wiki.onap.org";
125 rst_editor="retext --preview";
127 # remove credentials for those using anonymous access
128 test "${credentials}" = "*****:*****" && credentials=""
130 # explicit script dir to locate jar file
131 basedir="$(cd "$(dirname "$0")"; pwd)"
134 ### some inital tasks after script has been started
138 ### print script version, date and time
141 echo "INFO ***************************************************************************"
142 echo "INFO c2m Version ${script_version}, started $(date)";
145 ### simple script argument handling
150 # check if there is an argument at all
151 if [[ "$page_list" == "" ]] ; then
152 echo 'Usage: c2m [PAGELIST]'
156 # check if argument is a file
157 if [ ! -f $page_list ] ; then
158 echo "Error: can't find pagelist \"$page_list\""
163 ### declare the functions of this script
167 ### function: create working directory; save (only the last) existing one; remove older versions; do some error handling
170 function create_working_dir {
172 # compose name for working directory
173 #working_dir="${page_id}-${page_title}";
174 #working_dir="${page_title}-id${page_id}";
175 working_dir="${page_title}";
176 echo "INFO ***************************************************************************"
177 echo "INFO working directory \"$working_dir\" will be created"
179 # check if current working directory is already in the list
180 if [[ " ${existing_working_dirs[@]} " =~ " ${working_dir} " ]]; then
181 echo "ERRR ***************************************************************************"
182 echo "ERRR working directory \"${working_dir}\" already exists - check entries in page_list for duplicates"
183 echo "ERRR exiting ..."
186 # store working_dir name for error handling
187 existing_working_dirs+=(${working_dir})
191 #if [[ ! " ${array[@]} " =~ " ${value} " ]]; then
192 # # whatever you want to do when arr doesn't contain value
195 # check existence of working directory
196 if [ -d "$working_dir" ]; then
197 # check existence of old saved working directory
198 if [ -d "${working_dir}.old" ]; then
199 # remove the old saved working directory
200 rm -r "${working_dir}.old";
202 # save (only) the latest working directory
203 mv $working_dir "$working_dir.old";
205 # finally create the working directory and cd into it
211 ### function: pull pages from wiki - currently we are testing some export variations
214 function pull_pages_from_wiki {
216 # define outfile name
217 #out_file="${page_title}-id${page_id}";
218 out_file="${page_title}";
220 # set proxy if needed
221 if [[ -v http_proxy && ! -z "$http_proxy" ]]; then
222 proxy_to_parse="${http_proxy/http:\/\//""}";
223 echo "http_proxy is set to \"${proxy_to_parse}\"";
224 elif [[ -v https_proxy && ! -z "$https_proxy" ]]; then
225 proxy_to_parse="${https_proxy/https:\/\//""}";
226 echo "https_proxy is set to \"${proxy_to_parse}\"";
229 if [[ $proxy_to_parse =~ ^([\.0-9]+) ]]; then
230 java_options=" -Dhttp.proxyHost=${BASH_REMATCH[1]}"
231 echo "${java_options}"
233 if [[ $proxy_to_parse =~ .*:([0-9]+) ]]; then
234 java_options="${java_options} -Dhttps.proxyPort=${BASH_REMATCH[1]}"
235 echo "${java_options}"
239 # pull pages from wiki and convert to markdown (as a source for conversion by pandoc)
240 java $java_options -jar "${basedir}"/confluence2md-2.1-fat.jar +H true +T false +RootPageTitle false +FootNotes true -maxHeaderDepth 7 -depth $depth -v true -o ${out_file}.md -u "${credentials}" -server $server $page_id
244 ### function: simple search and (red colored) warning if special terms are detected in the md output file
247 function detect_unwanted_content_in_md_outfile {
248 for search_term in "ecomp" "wiki.onap.com" "10.53.199.7" "at&t"
250 if grep $search_term ${out_file}.md; then
251 echo -e "\e[31mWARN ***************************************************************************\e[39m";
252 echo -e "\e[31mWARN term \"${search_term}\" detected in ${out_file}.md\e[39m";
258 ### function: pandoc conversion from md (variants) to rst - currenty testing some conversion formats
261 function convert_md_outfile_to_rst {
262 #depending on the given source format (--from) the results may vary
263 #pandoc -s --toc --toc-depth=5 --from markdown_mmd --to rst "${out_file}.md" -o "${out_file}-markdown_mmd.rst"
264 #pandoc -s --toc --toc-depth=5 --from markdown_strict --to rst "${out_file}.md" -o "${out_file}-markdown_strict.rst"
265 #pandoc -s --toc --toc-depth=5 --from markdown_phpextra --to rst "${out_file}.md" -o "${out_file}-markdown_phpextra.rst"
266 #pandoc -s --toc-depth=5 --from markdown_phpextra --to rst "${out_file}.md" -o "${out_file}-markdown_phpextra.rst"
267 pandoc -s --toc-depth=5 --from markdown_phpextra --to rst "${out_file}.md" -o "${out_file}.rst"
271 ### function: check results in rst editor
274 function open_rst_editor {
275 #echo "DBUG ***************************************************************************"
276 #echo "DBUG open \"${out_file}\*.rst\" with rst editor"
277 $rst_editor ${out_file}*.rst &
281 ### function: clean up export directories from files no longer needed
286 rm attachments/*.json 2>/dev/null
287 rm attachments/.*.json 2>/dev/null
291 ### main: let's start the work ...
294 # read in pagelist file, filter lines starting with a comment and create an array that contains all (uncommented) lines of the file
297 # IFS=',' read -r -a page_array <<< "$page_list" # in case $page_list was defined as a varable in this script; use "," as the delimiter
298 #readarray -t page_array < $page_list; # old version
300 readarray -t page_array < <(grep -v "^#" $page_list); # new version which skips line with comments
302 # INFO: show list of pages by printing every line of the array
303 echo "INFO ***************************************************************************"
304 for line in "${page_array[@]}"
309 # the main loop reads the page_array line by line and processes the content
310 for line in "${page_array[@]}"
312 echo "INFO - bupp $line"
313 # cut out values from the current line (delimiter is now the "|") and assign them to the correct variables
314 hierarchy=$(echo $line | cut -f1 -d\|)
315 page_id=$(echo $line | cut -f2 -d\|)
316 page_title=$(echo $line | cut -f3 -d\|)
317 depth=$(echo $line | cut -f4 -d\|)
319 # remove leading and trailing spaces from variables
320 hierarchy="$(echo -e "${hierarchy}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')";
321 page_id="$(echo -e "${page_id}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')";
322 page_title="$(echo -e "${page_title}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')";
323 depth="$(echo -e "${depth}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')";
325 # substitude all blanks in page_title with a minus sign
326 page_title=$(echo -e ${page_title} | tr '[:blank:]' '-');
327 echo "DBUG page_title=\"$page_title\""
329 # convert page_title to lowercase
330 page_title=$(echo -e ${page_title} | tr '[:upper:]' '[:lower:]');
331 #echo "DBUG page_title=\"$page_title\""
333 # remove all characters from page_title which may cause problems in the shell ... or are reserved by conventions of this script
334 #page_title="$(echo -e "${page_title}" | sed -e 's/[^A-Za-z0-9._-]//g')"; # a less strict version
335 page_title="$(echo -e "${page_title}" | sed -e 's/[^A-Za-z0-9-]//g')";
336 echo "DBUG page_title=\"$page_title\""
338 # INFO: print variables to check content
339 echo "INFO ***************************************************************************"
340 echo "INFO hierarchy = \"$hierarchy\""
341 echo "INFO page_id = \"$page_id\""
342 echo "INFO page_title = \"$page_title\""
343 echo "INFO depth = \"$depth\""
345 # create working directory - done for every! "hierarchy 0" entry of page_list
346 if [ "$hierarchy" == "0" ]
351 # call functions to process page
353 detect_unwanted_content_in_md_outfile
354 convert_md_outfile_to_rst
362 echo "INFO ***************************************************************************"
363 echo "INFO c2m Version ${script_version}, ended $(date)"