### checkdocs.sh
###
### AUTHOR(S):
-### Thomas Kulik, Deutsche Telekom AG, 2020
+### Thomas Kulik, Deutsche Telekom AG, 2020 - 2021
###
### DESCRIPTION:
### Retrieves a full list of ONAP repos from gerrit inluding their state.
### create repo list
### curl -s https://git.onap.org/ | grep "^<tr><td class='toplevel-repo'><a title='" | sed -r "s:^<tr><td class='toplevel-repo'><a title='::" | sed -r "s:'.*::"
###
-### remove branchname from the line
-### cat frankfurt_gerritclone.log | sed 's:frankfurt|::'
+### remove branchname from the line:
+### cat frankfurt_repoclone.log | sed 's:frankfurt|::'
###
### list only image names
### cat master_dockerimagesfull.log | grep image | sed -r 's:image\:::' | sed -r 's:^ +::' | sed '/^[[:space:]]*$/d'
### SHORT: curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d'
###
-script_version="1.1 (2020-11-17)"
+script_version="1.6 (2021/03/30)"
# save command for the restart with logging enabled
command=$0
# print usage
function usage() {
+ echo " "
+ echo " checkdocs.sh Version ${script_version}"
echo " "
echo " USAGE: "
- echo " ./checkdocs.sh "
+ echo " ./checkdocs.sh <arguments> "
echo " "
echo " ARGUMENTS: "
echo " -u|--user username "
- echo " linux foundation username used to clone ONAP Gerrit repos"
+ echo " linux foundation username used to clone ONAP repositories"
echo " "
echo " -b|--branches branch1,branch2,branch3 "
echo " list of branches to be cloned. master is automatically "
usage
exit -1
fi
-# clone master first, the the other branches
+# clone master first, then the other branches
branches_csv="master,${branches_csv}"
# create the branches array by readinging in the values from the variable
# "| awk '{if(NR>1)print}'" filters the first line of the response so that jq will work again (thx marek)
curl -s 'https://gerrit.onap.org/r/projects/?d' | awk '{if(NR>1)print}' | jq -c '.[] | {id, state}' | sed -r 's:%2F:/:g; s:["{}]::g; s:id\:::; s:,state\::|:; /All-Projects/d; /All-Users/d' >./$repolist
-# process the created repolist
-# only active projects will be cloned in case the requested branch of the project exists
-echo "Accessing gerrit.onap.org with username \"${lfusername}\"."
-echo "Start cloning of repositories."
+# process the created repolist and try to clone the projects from the mirror
+
+source="git://cloud.onap.org/mirror"
+echo "Using \"${source}\" as the source and username \"${lfusername}\" for cloning the repositories."
+echo "Start cloning of repositories ..."
for branch in "${branches[@]}"
do
devcounter=$((devcounter+1))
fi
- if [[ $devcounter -lt "11" ]]; then
+ if [[ $devcounter -lt "50" ]]; then
if [[ $devmode == "TRUE" ]]; then
echo "INFO: devmode! counter=${devcounter}"
echo $reponame
echo $repostate
- if [[ $repostate == "ACTIVE" ]]; then
- echo "Cloning \"${branch}\" branch of ACTIVE project ${reponame}..."
+ if [[ $repostate == "ACTIVE" ]] || [[ $repostate == "READ_ONLY" ]]; then
+ echo "Cloning \"${branch}\" branch of \"${repostate}\" project ${reponame}..."
- git clone --branch ${branch} --recurse-submodules ssh://${lfusername}@gerrit.onap.org:29418/$reponame ./$reponame
+ # previously used: git clone --branch ${branch} --recurse-submodules ssh://${lfusername}@gerrit.onap.org:29418/$reponame ./$reponame
+ # clone script Jess: git clone "git://cloud.onap.org/mirror/${i}" "${LOCALNAME}"
+ git clone --branch ${branch} --recurse-submodules ${source}/${reponame} ./${reponame}
gitexitcode=$?
if [[ ! ${gitexitcode} == "0" ]]; then
errormsg="cloned"
fi
- # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
- echo "${gitexitcode}|${reponame}|${repostate}|${errormsg}" | tee -a ${branch}_gerritclone.log
+ # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
+ echo "${gitexitcode}|${reponame}|${repostate}|${errormsg}" | tee -a ${branch}_repoclone.log
- elif [[ $repostate == "READ_ONLY" ]]; then
- echo "-|${reponame}|${repostate}|ignored" | tee -a ${branch}_gerritclone.log
+ #elif [[ $repostate == "READ_ONLY" ]]; then
+ #echo "-|${reponame}|${repostate}|ignored" | tee -a ${branch}_repoclone.log
else
- echo "-|${reponame}|unknown repo state \"${repostate}\"|-" | tee -a ${branch}_gerritclone.log
+ echo "-|${reponame}|unknown repo state \"${repostate}\"|-" | tee -a ${branch}_repoclone.log
fi
# examine repo
printf "\nindex.rst files:\n"
find ./$reponame -type f -name index.rst | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_indexrst.log
+ printf "\nINFO.yaml files:\n"
+ find ./$reponame -type f -name INFO.yaml | sed -r 's:./::' | sed -r s:${reponame}:[${reponame}]: | tee -a ${branch}_infoyaml.log
+
fi
# end defcounter loop
# csv column #4: clone message
#
- readarray -t array < ./${branch}_gerritclone.log;
+ readarray -t array < ./${branch}_repoclone.log;
i=0
csv[i]="${csv[i]},${branch_upper} clone message"
((i++))
for line in "${array[@]}"
do
- # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
+ # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
errormsg=$(echo $line | awk -F "|" '{print $4}');
csv[i]="${csv[i]},${errormsg}"
((i++))
unset errormsg
#
- # csv column #5: RELEASE component (yes|no|maybe)
+ # csv column #5: lifecycle state
+ # extracted from the INFO.yaml
+ #
+
+ readarray -t array < ./${repolist};
+ i=0
+ csv[i]="${csv[i]},project lifecycle state"
+ ((i++))
+ for line in "${array[@]}"
+ do
+ reponame=$(echo $line | awk -F "|" '{print $1}');
+ if [ -f ./${reponame}/INFO.yaml ] ; then
+ # check if repo/branch has a INFO.yaml
+ lifecycleproject=$(grep '^project: ' ./${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
+ lifecyclestate=$(grep '^lifecycle_state: ' ./${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
+ elif [ ${branch} != "master" ] && [ -f ../master/${reponame}/INFO.yaml ] ; then
+ # if current branch is not master AND if info.yaml not found in the current repo/branch THAN use INFO.yaml of repo/master if available
+ #echo "DBUG: branch=${branch} - checking master for INFO.yaml"
+ lifecycleproject=$(grep '^project: ' ../master/${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
+ lifecyclestate=$(grep '^lifecycle_state: ' ../master/${reponame}/INFO.yaml | awk -F ":" '{print $2}' | sed 's:^ ::' | sed "s:'::g" | tr '[:upper:]' '[:lower:]' | sed 's/\r$//')
+ lifecyclestate="(${lifecyclestate})"
+ else
+ lifecyclestate="INFO.yaml not found"
+ fi
+ #echo "DBUG: working dir is ...";pwd
+ #echo "DBUG: lifecycleproject=${lifecycleproject}"
+ #echo "DBUG: lifecyclestate=${lifecyclestate}"
+ csv[i]="${csv[i]},${lifecyclestate}"
+ ((i++))
+ done
+ unset array
+ unset i
+ unset lifecycleproject
+ unset lifecyclestate
+
+ #
+ # csv column #6: RELEASE component (yes|maybe|unknown)
# to be filled with values of the planned release config file maintained by
# the onap release manager
#
- # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
- readarray -t array < ./${branch}_gerritclone.log;
+ # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
+ readarray -t array < ./${branch}_repoclone.log;
i=0
csv[i]="${csv[i]},${branch_upper} component"
((i++))
for line in "${array[@]}"
do
- # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
+ # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
gitexitcode=$(echo $line | awk -F "|" '{print $1}');
reponame=$(echo $line | awk -F "|" '{print $2}');
repostate=$(echo $line | awk -F "|" '{print $3}');
if [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "0" ]]; then
releasecomponent="yes"
- elif [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "128" ]]; then
+ elif [ ${repostate} == "ACTIVE" ]; then
+ #elif [[ ${repostate} == "ACTIVE" && ${gitexitcode} == "128" ]]; then
releasecomponent="maybe"
+ elif [[ ${repostate} == "READ_ONLY" && ${gitexitcode} == "0" ]]; then
+ releasecomponent="yes"
elif [ ${repostate} == "READ_ONLY" ]; then
- releasecomponent="no"
+ releasecomponent="maybe"
else
releasecomponent="unknown"
fi
unset releasecomponent
#
- # csv column #6: docs (at repo root directory only; no recursive search!)
- # csv column #7: conf.py
- # csv column #8: tox.ini
- # csv column #9: index.rst
+ # csv column #7: docs (at repo root directory only; no recursive search!)
+ # csv column #8: conf.py
+ # csv column #9: tox.ini
+ # csv column #10: index.rst
#
# columns are filled with values from requested branch.
# if data is not available values from master branch are used.
- # to identify master branch values, data is put into brackets "(...)"
+ # to identify master branch values, data is put into round brackets "(...)"
#
readarray -t array < ./${repolist};
docs="${docs},-"
fi
- # tox.ini
- if [ -f ./${line}/docs/tox.ini ] ; then
+ # tox.ini (check docs dir and also check project root dir)
+ if [ -f ./${line}/docs/tox.ini ] || [ -f ./${line}/tox.ini ]; then
docs="${docs},tox.ini"
- elif [ -f ../master/${line}/docs/tox.ini ] ; then
- docs="${docs},(tox.ini)"
+ # tox.ini @ branch/docs dir
+ if [ -f ./${line}/docs/tox.ini ] ; then
+ docs="${docs} @docs"
+ fi
+ # tox.ini @ branch/project root dir
+ if [ -f ./${line}/tox.ini ] ; then
+ docs="${docs} @root"
+ fi
+ elif [ -f ../master/${line}/docs/tox.ini ] || [ -f ../master/${line}/tox.ini ]; then
+ docs="${docs},(tox.ini"
+ # tox.ini @ master/docs dir
+ if [ -f ../master/${line}/docs/tox.ini ] ; then
+ docs="${docs} @docs"
+ fi
+ # tox.ini @ master/project root dir
+ if [ -f ../master/${line}/tox.ini ] ; then
+ docs="${docs} @root"
+ fi
+ # just add a round bracket at the end of the value
+ docs="${docs})"
else
+ # no tox.ini found in docs or root dir
docs="${docs},-"
fi
unset docs
#
- # csv column #10: index.html@RTD accessibility check
- # csv column #11: index.html url
+ # csv column #11: index.html@RTD accessibility check
+ # csv column #12: index.html url
#
- readarray -t array < ./${branch}_gerritclone.log;
+ readarray -t array < ./${branch}_repoclone.log;
i=0
csv[i]="${csv[i]},index.html@RTD,index.html url"
((i++))
for line in "${array[@]}"
do
- # gerritclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
+ # repoclone.log format: $1=gitexitcode|$2=reponame|$3=repostate|$4=errormsg
gitexitcode=$(echo $line | awk -F "|" '{print $1}');
reponame=$(echo $line | awk -F "|" '{print $2}');
repostate=$(echo $line | awk -F "|" '{print $3}');
url=""
curl_result=""
- # this routine works only with release "frankfurt" and later because
+ # this script works only with release "frankfurt" and later because
# earlier releases are using submodule structure for documentation files
if echo "$branch" | grep -q '^[abcde]'; then
curl_result="unsupported release"
else
# we are working on "frankfurt" branch or later ...
- # only if repostate IS ACTIVE a curl test is required
- if [[ ${repostate} == "ACTIVE" ]]; then
+ if [[ ${repostate} == "ACTIVE" ]] || [[ ${repostate} == "READ_ONLY" ]]; then
# OPTIONAL: USE ALSO GITEXITCODE AS A FILTER CRITERIA ???
fi
else
- # repostate IS NOT ACTIVE - no curl test required
+ # repostate IS NOT ACTIVE OR READ_ONLY - no curl test required
curl_result="-"
url="-"
fi
done
#
- # csv column #12: release notes
+ # csv column #13: release notes
#
readarray -t array < ../${repolist};
if [ -d ./${line} ] ; then
# if yes, check if repo name appears in the branch releasenotes.log
relnote=$(find "./${line}" -type f | grep 'release.*note.*.rst' | wc -l);
+ #echo "DBUG: relnote=${relnote}"
# repo dir DOES NOT exist in this branch - so check if repo dir exists in MASTER branch
elif [ -d ../master/${line} ] ; then
# if yes, check if repo name appears in the MASTER releasenotes.log
# count release notes files in MASTER branch (in repo root and its subdirectories)
relnote=$(find "../master/${line}" -type f | grep 'release.*note.*.rst' | wc -l);
+ #echo "DBUG: relnote=${relnote}"
# put results in round brackets to show that this is MASTER data
relnote=$(echo ${relnote} | sed -r s:${relnote}:\(${relnote}\):)
else
relnote="-"
fi
+ #echo "DBUG: relnote=${relnote}"
line="${csv[i]},${relnote}"
csv[i]=${line}
Code Review / doc.git / blobdiff