external-schema-repo-generator/generator/generate.sh

   1 #!/bin/sh
   2
   3 # ============LICENSE_START=======================================================
   4 # OOM
   5 # ================================================================================
   6 # Copyright (C) 2020-2021 Nokia. All rights reserved.
   7 # ================================================================================
   8 # Licensed under the Apache License, Version 2.0 (the "License");
   9 # you may not use this file except in compliance with the License.
  10 # You may obtain a copy of the License at
  11 #      http://www.apache.org/licenses/LICENSE-2.0
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17 # ============LICENSE_END=========================================================
  18
  19 . ./utils.sh
  20
  21 # Arguments renaming
  22 arguments_number=$#
  23 repo_url=$1
  24 branches=$2
  25 schemas_location=$3
  26 vendor=$4
  27 configmap_filename=$5
  28 configmap_name=$6
  29 snippet_filename=$7
  30 specs_directory=$8
  31 generation_directory=$9
  32 _5g_api_repo_url=${10}
  33
  34 # Constants
  35 SCHEMA_MAP_FILENAME="schema-map.json"
  36 SCHEMA_MAP_NAME="schema-map"
  37 SUCCESS_CODE=0
  38 TREE=blob
  39 INDENTATION_LEVEL_1=1
  40 INDENTATION_LEVEL_2=2
  41 INDENTATION_LEVEL_3=3
  42 INDENTATION_LEVEL_4=4
  43 INDENTATION_LEVEL_5=5
  44
  45 # Variables
  46 tmp_location=$(mktemp -d)
  47 valid_branches=""
  48
  49 # Create and move to directory for storing generated files
  50 move_to_generation_directory() {
  51   mkdir "$generation_directory"
  52   cd ./"$generation_directory"
  53 }
  54
  55 # Indents each line of string by adding indent_size*indent_string spaces on the beginning
  56 # Optional argument is indent_string level, default: 1
  57 # correct usage example:
  58 # echo "Sample Text" | indent_string 2
  59 indent_string() {
  60   indent_size=2
  61   indent_string=1
  62   if [ -n "$1" ]; then indent_string=$1; fi
  63   pr -to $(expr "$indent_string" \* "$indent_size")
  64 }
  65
  66 # Clones all branches selected in $BRANCH from $repo_url
  67 clone_repo() {
  68   for actual_branch in $branches; do
  69     clone_branch "$actual_branch"
  70   done
  71 }
  72
  73 # Clones single branch $1 from $repo_url.
  74 # $1 - branch name
  75 clone_branch() {
  76   check_arguments $# $EXPECTED_1_ARG
  77   configmapSize=800000
  78   branch=""
  79   # check if OPEN API branch value provided by user contains 5G API branch after colon for example: Rel-16-SA-91:TSG91-Rel16
  80   if [ -z "${1##*":"*}" ]; then
  81     OpenApiBranch=${1%:*}
  82     _5GApiBranch=${1#"$OpenApiBranch"}
  83     _5GApiBranch=${_5GApiBranch#:}
  84     if [ -d $tmp_location/"$OpenApiBranch" ]; then
  85       echo "Skipping cloning repository."
  86       echo "Branch $OpenApiBranch has already been cloned in the directory ./$tmp_location/$OpenApiBranch"
  87       echo "To redownload branch remove ./$tmp_location/$OpenApiBranch."
  88     else
  89       echo "Cloning repository from branch $OpenApiBranch"
  90       git clone --quiet --single-branch --branch "$OpenApiBranch" "$repo_url" "$tmp_location/$OpenApiBranch" 2>/dev/null
  91       OpenApiBranchResult=$?
  92       # clones 5G API branch moves schemas to OPEN API, splits folder to folder less than 1mb
  93       echo "Cloning repository from branch $_5GApiBranch"
  94       git clone --quiet --single-branch --branch "$_5GApiBranch" "$_5g_api_repo_url" "$tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch" 2>/dev/null
  95       _5GApiBranchResult=$?
  96       mv $tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch/*.yaml $tmp_location/$OpenApiBranch/$schemas_location/
  97       dirSize=$(du -bs $tmp_location/$OpenApiBranch/$schemas_location/ | cut -f1)
  98       if [ "$configmapSize" -le "$dirSize"  ]; then
  99         find $tmp_location/$OpenApiBranch/$schemas_location/ -type f -iname '*yaml' -printf "%s %p\n" | awk -v c=$configmapSize -v tmpLocation=$tmp_location -v schema=$OpenApiBranch -f bin_packing.awk
 100         # adds each folder as valid branch to generate separate configmap (less then 1 mb) for each folder
 101         for i in $(ls -d -p $tmp_location/${OpenApiBranch}-* |  grep -Eo '[^/]+/?$' | cut -d / -f1); do branch="${branch} $i"; done
 102       else
 103         branch=$OpenApiBranch
 104       fi
 105       # fix schema file references both remote and local against folder structure for whole schema
 106       fix_schema_references $tmp_location $OpenApiBranch $schemas_location
 107       if  [ $OpenApiBranchResult -ne $SUCCESS_CODE ] && [ $_5GApiBranchResult -ne $SUCCESS_CODE ] ; then
 108         echo "Problem with cloning branch $OpenApiBranch."
 109         echo "Branch $OpenApiBranch= will not be added to spec."
 110       else
 111         valid_branches="${valid_branches} $branch"
 112         echo "valid_branches: $valid_branches"
 113       fi
 114     fi
 115   else
 116     OpenApiBranch=$1
 117     if [ -d $tmp_location/"$OpenApiBranch" ]; then
 118       echo "Skipping cloning repository."
 119       echo "Branch $OpenApiBranch has already been cloned in the directory ./$tmp_location/$OpenApiBranch"
 120       echo "To redownload branch remove ./$tmp_location/$OpenApiBranch."
 121     else
 122       echo "Cloning repository from branch $OpenApiBranch"
 123       git clone --quiet --single-branch --branch "$OpenApiBranch" "$repo_url" "$tmp_location/$OpenApiBranch" 2>/dev/null
 124       OpenApiBranchResult=$?
 125       branch=""
 126       # script checks schema files if they don’t contain remote refs to 5G API even user didn't define it after colon for example: Rel-16-SA-91:TSG91-Rel16
 127       # if contain then script parse repo tag from reference checkouts it, split to folders, resolve refs and cerates configmap for each folder
 128       _5GApiBranch=$(check5GApiBranchExistenceInRefs "$tmp_location/$OpenApiBranch/$schemas_location")
 129       if [ ! -z "$_5GApiBranch" ]; then
 130         echo "Cloning repository from branch $_5GApiBranch"
 131         git clone --quiet --single-branch --branch "$_5GApiBranch" "$_5g_api_repo_url" "$tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch" 2>/dev/null
 132         _5GApiBranchResult=$?
 133         mv $tmp_location/$OpenApiBranch/$schemas_location/$_5GApiBranch/*.yaml $tmp_location/$OpenApiBranch/$schemas_location
 134         dirSize=$(du -bs $tmp_location/$OpenApiBranch/$schemas_location/ | cut -f1)
 135         if [ "$configmapSize" -le "$dirSize"  ]; then
 136           find $tmp_location/$OpenApiBranch/$schemas_location/ -type f -iname '*yaml' -printf "%s %p\n" | awk -v c=$configmapSize -v tmpLocation=$tmp_location -v schema=$OpenApiBranch -f bin_packing.awk
 137           for i in $(ls -d -p $tmp_location/${OpenApiBranch}-* |  grep -Eo '[^/]+/?$' | cut -d / -f1); do branch="${branch} $i"; done
 138         else
 139           branch=$OpenApiBranch
 140         fi
 141         fix_schema_references $tmp_location $OpenApiBranch $schemas_location
 142         if  [ ! -z "$_5GApiBranch" ] && [ $_5GApiBranchResult -ne $SUCCESS_CODE ] && [ $OpenApiBranchResult -ne $SUCCESS_CODE ]  ; then
 143           echo "Problem with cloning branch $OpenApiBranch."
 144           echo "Branch $OpenApiBranch= will not be added to spec."
 145         else
 146           valid_branches="${valid_branches} $branch"
 147           echo "valid_branches: $valid_branches"
 148         fi
 149       #if schema files don't contain remote refs to 5G API script goes with old path and doesn't checkout 5G API branch
 150       else
 151         if  [ $OpenApiBranchResult -ne $SUCCESS_CODE ] ; then
 152           echo "Problem with cloning branch $OpenApiBranch."
 153           echo "Branch $OpenApiBranch= will not be added to spec."
 154         else
 155           valid_branches="${valid_branches} $OpenApiBranch"
 156           echo "valid_branches: $valid_branches"
 157         fi
 158       fi
 159     fi
 160   fi
 161 }
 162
 163 # Creates file with name $configmap_filename
 164 # Inserts ConfigMap metadata and sets name as $configmap_name
 165 # $1 - branch name
 166 add_config_map_metadata() {
 167   branch_configmap_filename="${configmap_filename}-$1.yaml"
 168   branch_configmap_name=$( echo "$configmap_name-$1" | tr '[:upper:]' '[:lower:]' )
 169   echo "Creating ConfigMap spec file: $branch_configmap_filename"
 170   cat << EOF > "$branch_configmap_filename"
 171 apiVersion: v1
 172 kind: ConfigMap
 173 metadata:
 174   name: $branch_configmap_name
 175   labels:
 176     name: $branch_configmap_name
 177   namespace: onap
 178 data:
 179 EOF
 180 }
 181
 182 # For each selected branch:
 183 #   clones the branch from repository,
 184 #   adds schemas from branch to ConfigMap spec
 185 # $1 - branch name
 186 add_schemas() {
 187   echo "Adding schemas from branch $1 to spec"
 188   add_schemas_from_branch "$1"
 189 }
 190
 191 # Adds schemas from single branch to spec
 192 # $1 - branch name
 193 add_schemas_from_branch() {
 194   branch_configmap_filename="${configmap_filename}-$1.yaml"
 195   check_arguments $# $EXPECTED_1_ARG
 196   schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}')
 197   for schema in $schemas; do
 198     echo "$(basename $schema): |-" | indent_string $INDENTATION_LEVEL_1
 199     cat "$schema" | indent_string $INDENTATION_LEVEL_2
 200   done
 201 } >> "$branch_configmap_filename"
 202
 203 move_to_spec_directory() {
 204   mkdir "$specs_directory"
 205   cd ./"$specs_directory"
 206 }
 207
 208 # Generates mapping file for collected schemas directly in spec
 209 # $1 - schema map name
 210 generate_mapping_file() {
 211   schema_map_filename="${configmap_filename}-$1.yaml"
 212   echo "Generating mapping file in spec"
 213   echo "$SCHEMA_MAP_FILENAME"": |-" | indent_string $INDENTATION_LEVEL_1 >> "$schema_map_filename"
 214   echo "[" | indent_string $INDENTATION_LEVEL_2 >> "$schema_map_filename"
 215
 216   for actual_branch in $valid_branches; do
 217     echo "Adding mappings from branch: $actual_branch"
 218     add_mappings_from_branch $actual_branch $schema_map_filename
 219   done
 220
 221   truncate -s-2 "$schema_map_filename"
 222   echo "" >> "$schema_map_filename"
 223   echo "]" | indent_string $INDENTATION_LEVEL_2 >> "$schema_map_filename"
 224 }
 225
 226 # Adds mappings from single branch directly to spec
 227 # $1 - branch name
 228 # $2 - schema map file name
 229 add_mappings_from_branch() {
 230   check_arguments $# $EXPECTED_2_ARGS
 231   schema_map_filename="$2"
 232   schemas=$(ls -g $tmp_location/$1/$schemas_location/*.yaml | awk '{print $NF}' )
 233
 234   for schema in $schemas; do
 235     repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev)
 236     schema_repo_path=$(echo "$schema" | cut -d/ -f4-)
 237     schema_repo_path_fixed=$(echo "$schema_repo_path" | sed -r "s%"-subdir-[0-9]"%%" )
 238     public_url_schemas_location=${repo_url%.*}
 239     public_url=$public_url_schemas_location/$TREE/$schema_repo_path_fixed
 240     local_url=$vendor/$repo_endpoint/$TREE/$schema_repo_path
 241
 242     echo "{" | indent_string $INDENTATION_LEVEL_3 >> "$schema_map_filename"
 243     echo "\"publicURL\": \"$public_url\"," | indent_string $INDENTATION_LEVEL_4 >> "$schema_map_filename"
 244     echo "\"localURL\": \"$local_url\"" | indent_string $INDENTATION_LEVEL_4 >> "$schema_map_filename"
 245     echo "}," | indent_string $INDENTATION_LEVEL_3 >> "$schema_map_filename"
 246   done
 247 }
 248
 249 # Create snippet file to describe how to connect mount ConfigMaps in VES
 250 create_snippet() {
 251   echo "Generating snippets in file: $snippet_filename"
 252   generate_entries
 253   base_mounts_path="/opt/app/VESCollector/etc/externalRepo"
 254   base_mounts_name="external-repo-$vendor-schemas"
 255   mounts_paths="
 256         - mountPath: $base_mounts_path
 257           name: $base_mounts_name"
 258
 259   config_maps="
 260       - configMap:
 261           defaultMode: 420
 262           name: $configmap_name-$SCHEMA_MAP_NAME
 263         name: $base_mounts_name"
 264
 265   for actual_branch in $valid_branches; do
 266
 267     actual_branch_name=$( echo "$actual_branch" | tr '[:upper:]' '[:lower:]' )
 268
 269     repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev)
 270     local_url=$vendor/$repo_endpoint/$TREE/$actual_branch/$schemas_location
 271     mounts_paths="$mounts_paths
 272         - mountPath: $base_mounts_path/$local_url
 273           name: $base_mounts_name-$actual_branch_name"
 274
 275     config_maps="$config_maps
 276       - configMap:
 277           defaultMode: 420
 278           name: $configmap_name-$actual_branch_name
 279         name: $base_mounts_name-$actual_branch_name"
 280   done
 281
 282
 283   cat << EOF > "$snippet_filename"
 284 Snippets for mounting ConfigMap in DCAE VESCollector Deployment
 285 =========================================================================
 286
 287 ## Description
 288 These snippets will override existing in VESCollector schemas and mapping file.
 289
 290 No extra configuration in VESCollector is needed with these snippets.
 291
 292 ## Snippets
 293 #### spec.template.spec.containers[0].volumeMounts
 294 \`\`\`$mounts_paths
 295 \`\`\`
 296
 297 #### spec.template.spec.volumes
 298 \`\`\`$config_maps
 299 \`\`\`
 300 EOF
 301 }
 302
 303 generate_entries() {
 304   for actual_branch in $valid_branches; do
 305     schemas=$(ls -g $tmp_location/$actual_branch/$schemas_location/*.yaml | awk '{print $NF}')
 306     for schema in $schemas; do
 307       repo_endpoint=$(echo "$repo_url" | cut -d/ -f4- | rev | cut -d. -f2- | rev)
 308       schema_repo_path=$(echo "$schema" | cut -d/ -f4-)
 309
 310       key="$actual_branch-$(basename "$schema")"
 311       path=$vendor/$repo_endpoint/$TREE/$schema_repo_path
 312       schemas_entries="$schemas_entries- key: $key\n  path: $path\n"
 313     done
 314   done
 315   schemas_entries=$(echo "$schemas_entries" | indent_string $INDENTATION_LEVEL_5)
 316 }
 317
 318 # Generate specs for branch
 319 # $1 - branch name
 320 generate_specs_for_branch() {
 321   check_arguments $# $EXPECTED_1_ARG
 322   add_config_map_metadata $1
 323   add_schemas $1
 324 }
 325
 326 # Generate specs for schema map
 327 generate_specs_for_schema_map() {
 328   add_config_map_metadata "$SCHEMA_MAP_NAME"
 329   generate_mapping_file "$SCHEMA_MAP_NAME"
 330 }
 331
 332 # Generate specs for all releases and for schema map
 333 generate_specs() {
 334   move_to_spec_directory
 335   for actual_branch in $valid_branches; do
 336     generate_specs_for_branch $actual_branch
 337   done
 338   generate_specs_for_schema_map
 339   cd ..
 340 }
 341
 342
 343 # todo add check of global env whether script should be ran
 344 main() {
 345   check_arguments $arguments_number $EXPECTED_10_ARGS
 346   clone_repo
 347   move_to_generation_directory
 348   generate_specs
 349   create_snippet
 350   move_to_starting_directory
 351 }
 352
 353 main