boot/dcae2_vm_init.sh

   1 #!/bin/bash
   2 #############################################################################
   3 #
   4 # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #        http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 #############################################################################
  18
  19
  20 # prepare a curl command
  21 # parameters: URL METHOD CURLOPTIONS EXTRA_HEADERS_AS_A_STRING AUTH_AS_USER:PASS DATA
  22 assemble_curl_command()
  23 {
  24     export http_proxy=""
  25     export https_proxy=""
  26
  27     local URL="$1"
  28     local METHOD="$2"
  29     local CURLOPTIONS="$3"
  30     local EXTRA_HEADERS="$4"
  31     local AUTH="$5"
  32     local DATA="$6"
  33     local CMD=''
  34     if [ ! -z "$METHOD" ]; then
  35         CMD="curl $CURLOPTIONS $METHOD"
  36     else
  37         CMD="curl $CURLOPTIONS -X GET"
  38     fi
  39     if [ ! -z "$EXTRA_HEADERS" ]; then
  40         CMD="$CMD $EXTRA_HEADERS"
  41     fi
  42     if [ ! -z "$AUTH" ]; then
  43         CMD="$CMD $AUTH"
  44     fi
  45     if [ ! -z "$DATA" ]; then
  46         CMD="$CMD $DATA"
  47     fi
  48     CMD="$CMD $URL"
  49     echo "$CMD"
  50 }
  51
  52
  53 # Make a rest API call
  54 # parameters: URL METHOD expected_response_code EXTRA_HEADERS_AS_A_STRING AUTH_AS_USER:PASS DATA
  55 call_api_for_response_code()
  56 {
  57     export http_proxy=""
  58     export https_proxy=""
  59
  60     local CURLOPTIONS='-kIso /dev/null -w "%{http_code}"'
  61     read -r CMDF <<-END
  62 $(assemble_curl_command "$1" "$2" "$CURLOPTIONS" "$4" "$5" "$6")
  63 END
  64     eval "$CMDF";
  65 }
  66 call_api_for_response_body()
  67 {
  68     export http_proxy=""
  69     export https_proxy=""
  70
  71     local CURLOPTIONS='-ksb'
  72     read -r CMDF <<-END
  73 $(assemble_curl_command "$1" "$2" "$CURLOPTIONS" "$4" "$5" "$6")
  74 END
  75     eval "$CMDF"
  76 }
  77 call_api_for_response_header()
  78 {
  79     export http_proxy=""
  80     export https_proxy=""
  81
  82     local CURLOPTIONS='-ks -o /dev/null -D -'
  83     read -r CMDF <<-END
  84 $(assemble_curl_command "$1" "$2" "$CURLOPTIONS" "$4" "$5" "$6")
  85 END
  86     eval "$CMDF"
  87 }
  88 call_api_for_verbose()
  89 {
  90     export http_proxy=""
  91     export https_proxy=""
  92
  93     local CURLOPTIONS='-kIv'
  94     read -r CMDF <<-END
  95 $(assemble_curl_command "$1" "$2" "$CURLOPTIONS" "$4" "$5" "$6")
  96 END
  97     eval "$CMDF"
  98     #local TFILE=$(mktemp /tmp/curlcmd.XXXXXXXXX)
  99     #echo $CMD > $TFILE
 100     #eval $(cat $TFILE)
 101     #rm -f $TFILE
 102 }
 103
 104
 105 # Wait till a web service API return specified response code
 106 # parameters: URL METHOD EXPECTED_RESP_CODE EXTRA_HEADERS_AS_A_STRING AUTH_AS_USER:PASS DATA
 107 wait_for_api()
 108 {
 109     export http_proxy=""
 110     export https_proxy=""
 111
 112     local RESP="$3"
 113     local ACTUALRESP
 114     ACTUALRESP=$(call_api_for_response_code "$1" "$2" "$3" "$4" "$5" "$6")
 115     while [ "$ACTUALRESP" != "$RESP" ]; do
 116         echo "RESP CODE $ACTUALRESP, not as expected RESP CODE $RESP @ $(date)."
 117         sleep 30
 118         ACTUALRESP=$(call_api_for_response_code "$1" "$2" "$3" "$4" "$5" "$6")
 119     done
 120     echo "RESP CODE $ACTUALRESP, matches with expected RESP CODE $RESP."
 121 }
 122
 123 # Wait till a TCP port is open
 124 # parameters: HOST PORT
 125 wait_for_tcp_port()
 126 {
 127     export http_proxy=""
 128     export https_proxy=""
 129
 130     local DEST="$1"
 131     local PORT="$2"
 132     while ! nc -z -w 1 "$DEST" "$PORT"; do
 133         sleep 4
 134         echo '.'
 135     done
 136 }
 137
 138
 139
 140
 141 wait_for_aai_ready()
 142 {
 143     export http_proxy=""
 144     export https_proxy=""
 145
 146     # wait till A&AI up and ready
 147     local AAIHOST
 148     AAIHOST=$(cat /opt/config/aai1_ip_addr.txt)
 149     local AAIURL="https://$AAIHOST:8443/aai/v11/examples/cloud-regions"
 150     local AAIMETHOD='-X GET'
 151     local AAIRESP='200'
 152     local AAIHEADERS='-H "X-FromAppId: AAI-Temp-Tool" -H "X-TransactionId: AAI-Temp-Tool" -H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
 153     local AAIAUTH='-u AAI:AAI'
 154     local AAIDATA=''
 155     echo "===> Waiting for A&AI to get ready for getting $AAIRESP from $AAIURL @ $(date)"
 156     wait_for_api "$AAIURL" "$AAIMETHOD" "$AAIRESP" "$AAIHEADERS" "$AAIAUTH" "$AAIDATA"
 157     echo "===> A&AI ready @ $(date)"
 158 }
 159
 160
 161
 162 wait_for_multicloud_ready()
 163 {
 164     export http_proxy=""
 165     export https_proxy=""
 166
 167     # wait till MultiCloud up and ready
 168     local MCHOST
 169     MCHOST=$(cat /opt/config/multiserv_ip_addr.txt)
 170     local MCURL="http://$MCHOST:9005/api/multicloud-titaniumcloud/v0/swagger.json"
 171     local MCMETHOD='-X GET'
 172     local MCRESP='200'
 173     local MCHEADERS='-H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
 174     local MCAUTH=''
 175     local MCDATA=''
 176     echo "===> Waiting for MultiCloud to get ready for getting $MCRESP from $MCURL @ $(date)"
 177     wait_for_api "$MCURL" "$MCMETHOD" "$MCRESP" "$MCHEADERS" "$MCAUTH" "$MCDATA"
 178     echo "===> MultiCloud ready @ $(date)"
 179 }
 180
 181 register_multicloud_pod25dns_with_aai()
 182 {
 183     export http_proxy=""
 184     export https_proxy=""
 185
 186     # Register MultiCloud with A&AI
 187     local CLOUD_OWNER='pod25dns'
 188     local CLOUD_VERSION='titanium_cloud'
 189     local CLOUD_REGION
 190     local CLOUD_ENV
 191     local CLOUD_IDENTITY_URL
 192     local DNSAAS_SERVICE_URL
 193     local DNSAAS_USERNAME='demo'
 194     local DNSAAS_PASSWORD='onapdemo'
 195
 196     CLOUD_REGION="$(cat /opt/config/dnsaas_region.txt)"
 197     CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
 198     MCIP="$(cat /opt/config/multiserv_ip_addr.txt)"
 199     CLOUD_IDENTITY_URL="http://${MCIP}/api/multicloud-titaniumcloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}/identity/v2.0"
 200
 201     local RESPCODE
 202     DNSAAS_SERVICE_URL="$(cat /opt/config/dnsaas_keystone_url.txt)"
 203     # a tenant of the same name must be set up on the Deisgnate providing OpenStack
 204     DNSAAS_TENANT_NAME="$(cat /opt/config/dnsaas_tenant_name.txt)"
 205     cat >"/tmp/${CLOUD_OWNER}_${CLOUD_REGION}.json" <<EOL
 206 {
 207     "cloud-owner" : "$CLOUD_OWNER",
 208     "cloud-region-id" : "$CLOUD_REGION",
 209     "cloud-region-version" : "$CLOUD_VERSION",
 210     "cloud-type" : "$CLOUD_ENV",
 211     "cloud-zone" : "cloud zone",
 212     "complex-name" : "complex name2",
 213     "identity-url": "$CLOUD_IDENTITY_URL",
 214     "owner-defined-type" : "owner-defined-type",
 215     "sriov-automation" : false,
 216     "esr-system-info-list" : {
 217         "esr-system-info" : [
 218             {
 219                 "esr-system-info-id": "532ac032-e996-41f2-84ed-9c7a1766eb30",
 220                 "cloud-domain": "Default",
 221                 "default-tenant" : "$DNSAAS_TENANT_NAME",
 222                 "user-name" : "$DNSAAS_USERNAME",
 223                 "password" : "$DNSAAS_PASSWORD",
 224                 "service-url" : "$DNSAAS_SERVICE_URL",
 225                 "ssl-cacert": "example-ssl-cacert-val-75021",
 226                 "ssl-insecure": true,
 227                 "system-name": "example-system-name-val-29071",
 228                 "system-type": "VIM",
 229                 "ip-address": "example-ip-address-val-44432",
 230                 "port": "example-port-val-93235",
 231                 "type": "example-type-val-85255",
 232                 "protocal": "example-protocal-val-52954",
 233                 "vendor": "example-vendor-val-94515",
 234                 "version": "example-version-val-71880"
 235             }
 236         ]
 237     }
 238 }
 239 EOL
 240
 241
 242     local REGHOST
 243     local REGURL
 244     local REGMETHOD='-X PUT'
 245     local REGHEADERS='-H "X-FromAppId: AAI-Temp-Tool" -H "X-TransactionId: AAI-Temp-Tool" -H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
 246     local REGRESP='201'
 247     local REGAUTH='-u AAI:AAI'
 248     local REGDATA
 249
 250     REGHOST="$(cat /opt/config/aai1_ip_addr.txt)"
 251     REGURL="https://$REGHOST:8443/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/$CLOUD_OWNER/$CLOUD_REGION"
 252     REGDATA="-T /tmp/${CLOUD_OWNER}_${CLOUD_REGION}.json"
 253     echo "Register MultiCloud with A&AI owner $CLOUD_OWNER"
 254     RESP=$(call_api_for_response_code "$REGURL" "$REGMETHOD" "$REGRESP" "$REGHEADERS" "$REGAUTH" "$REGDATA")
 255     echo "RESP CODE: $RESP"
 256 }
 257
 258
 259 register_multicloud_pod25_with_aai()
 260 {
 261     export http_proxy=""
 262     export https_proxy=""
 263
 264     # Register MultiCloud with A&AI
 265     local CLOUD_OWNER='pod25'
 266     local CLOUD_VERSION='titanium_cloud'
 267     local CLOUD_REGION
 268     local DNSAAS_CLOUD_REGION
 269     local CLOUD_ENV
 270     local MCIP
 271     local CLOUD_IDENTITY_URL
 272     local KEYSTONE_URL
 273     local USERNAME
 274     local PASSWORD
 275     local TENANT_NAME
 276
 277     CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
 278     DNSAAS_CLOUD_REGION="$(cat /opt/config/dnsaas_region.txt)"
 279     CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
 280     MCIP="$(cat /opt/config/multiserv_ip_addr.txt)"
 281     CLOUD_IDENTITY_URL="http://${MCIP}/api/multicloud-titaniumcloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}/identity/v2.0"
 282     KEYSTONE_URL="$(cat /opt/config/openstack_keystone_url.txt)"
 283     if [[ "$KEYSTONE_URL" == */v3 ]]; then
 284         echo "$KEYSTONE_URL"
 285     elif [[ "$KEYSTONE_URL" == */v2.0 ]]; then
 286         echo "$KEYSTONE_URL"
 287     else
 288         KEYSTONE_URL="${KEYSTONE_URL}/v3"
 289         echo "$KEYSTONE_URL"
 290     fi
 291     USERNAME="$(cat /opt/config/openstack_user.txt)"
 292     PASSWORD="$(cat /opt/config/openstack_password.txt)"
 293     TENANT_NAME="$(cat /opt/config/tenant_name.txt)"
 294     cat >"/tmp/${CLOUD_OWNER}_${CLOUD_REGION}.json" <<EOL
 295 {
 296     "cloud-owner" : "$CLOUD_OWNER",
 297     "cloud-region-id" : "$CLOUD_REGION",
 298     "cloud-region-version" : "$CLOUD_VERSION",
 299     "cloud-type" : "$CLOUD_ENV",
 300     "cloud-zone" : "cloud zone",
 301     "complex-name" : "complex name",
 302     "identity-url": "$CLOUD_IDENTITY_URL",
 303     "owner-defined-type" : "owner-defined-type",
 304     "sriov-automation" : false,
 305     "cloud-extra-info" : "{\"epa-caps\":{\"huge_page\":\"true\",\"cpu_pinning\":\"true\",\"cpu_thread_policy\":\"true\",\"numa_aware\":\"true\",\"sriov\":\"true\",\"dpdk_vswitch\":\"true\",\"rdt\":\"false\",\"numa_locality_pci\":\"true\"},\"dns-delegate\":{\"cloud-owner\":\"pod25dns\",\"cloud-region-id\":\"${DNSAAS_CLOUD_REGION}\"}}",
 306     "esr-system-info-list" : {
 307         "esr-system-info" : [
 308             {
 309                 "esr-system-info-id": "432ac032-e996-41f2-84ed-9c7a1766eb29",
 310                 "cloud-domain": "Default",
 311                 "default-tenant" : "$TENANT_NAME",
 312                 "user-name" : "$USERNAME",
 313                 "password" : "$PASSWORD",
 314                 "service-url" : "$KEYSTONE_URL",
 315                 "ssl-cacert": "example-ssl-cacert-val-75021",
 316                 "ssl-insecure": true,
 317                 "system-name": "example-system-name-val-29070",
 318                 "system-type": "VIM",
 319                 "ip-address": "example-ip-address-val-44431",
 320                 "port": "example-port-val-93234",
 321                 "type": "example-type-val-85254",
 322                 "protocal": "example-protocal-val-52954",
 323                 "vendor": "example-vendor-val-94515",
 324                 "version": "example-version-val-71880"
 325             }
 326         ]
 327     }
 328 }
 329 EOL
 330
 331     local REGHOST
 332     local REGURL
 333     local REGMETHOD='-X PUT'
 334     local REGHEADERS='-H "X-FromAppId: AAI-Temp-Tool" -H "X-TransactionId: AAI-Temp-Tool" -H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
 335     local REGRESP='201'
 336     local REGAUTH='-u AAI:AAI'
 337     local REGDATA
 338
 339     REGHOST="$(cat /opt/config/aai1_ip_addr.txt)"
 340     REGURL="https://$REGHOST:8443/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/$CLOUD_OWNER/$CLOUD_REGION"
 341     REGDATA="-T /tmp/${CLOUD_OWNER}_${CLOUD_REGION}.json"
 342     echo "Register MultiCloud with A&AI owner $CLOUD_OWNER"
 343     RESP=$(call_api_for_response_code "$REGURL" "$REGMETHOD" "$REGRESP" "$REGHEADERS" "$REGAUTH" "$REGDATA")
 344     echo "RESP CODE: $RESP"
 345 }
 346
 347
 348
 349 verify_multicloud_registration()
 350 {
 351     export http_proxy=""
 352     export https_proxy=""
 353
 354     local CLOUD_OWNER='pod25'
 355     local CLOUD_REGION
 356     local CLOUD_VERSION='titanium_cloud'
 357     local CLOUD_ENV
 358     local REGHOST
 359     local REGURL
 360     local REGMETHOD='-X GET'
 361     local REGHEADERS='-H "X-FromAppId: AAI-Temp-Tool" -H "X-TransactionId: AAI-Temp-Tool" -H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
 362     local REGRESP='200'
 363     local REGAUTH='-u AAI:AAI'
 364     local REGDATA=''
 365     local RESPCODE
 366
 367     CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
 368     CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
 369     REGHOST="$(cat /opt/config/aai1_ip_addr.txt)"
 370
 371     REGURL="https://$REGHOST:8443/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/${CLOUD_OWNER}/${CLOUD_REGION}?depth=all"
 372     # Verify MultiCloud with A&AI
 373     RESPCODE=$(call_api_for_response_code "$REGURL" "$REGMETHOD" "$REGRESP" "$REGHEADERS" "$REGAUTH" "$REGDATA")
 374     echo "Register MultiCloud with A&AI owner $CLOUD_OWNER verify response code: $RESPCODE"
 375
 376     CLOUD_OWNER='pod25dns'
 377     REGURL="https://$REGHOST:8443/aai/v11/cloud-infrastructure/cloud-regions/cloud-region/${CLOUD_OWNER}/${CLOUD_REGION}?depth=all"
 378     RESPCODE=$(call_api_for_response_code "$REGURL" "$REGMETHOD" "$REGRESP" "$REGHEADERS" "$REGAUTH" "$REGDATA")
 379     echo "Register MultiCloud with A&AI owner $CLOUD_OWNER verify response code: $RESPCODE"
 380 }
 381
 382
 383 register_dns_zone_proxied_designate()
 384 {
 385     export http_proxy=""
 386     export https_proxy=""
 387
 388     local CLOUD_OWNER='pod25'
 389     local CLOUD_REGION
 390     local CLOUD_VERSION='titanium_cloud'
 391     local CLOUD_ENV
 392     local DNSAAS_TENANT_NAME
 393     local MCHOST
 394     local MCURL
 395     local MCMETHOD='-X POST'
 396     local MCRESP='200'
 397     local MCHEADERS='-H "Content-Type: application/json" -H "Accept: application/json"'
 398     local MCAUTH=''
 399     local MCDATA=''
 400
 401     ## export endpoint prefix
 402     local MULTICLOUD_PLUGIN_ENDPOINT
 403
 404     CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
 405     CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
 406     if [ -z "$1" ]; then DCAE_ZONE="$(cat /opt/config/dcae_zone.txt)"; else DCAE_ZONE="$1"; fi
 407     DNSAAS_TENANT_NAME="$(cat /opt/config/dnsaas_tenant_name.txt)"
 408     MCHOST=$(cat /opt/config/multiserv_ip_addr.txt)
 409     MCURL="http://$MCHOST:9005/api/multicloud-titaniumcloud/v0/swagger.json"
 410
 411     MCDATA='-d "{\"auth\":{\"tenantName\": \"'${DNSAAS_TENANT_NAME}'\"}}"'
 412     MULTICLOUD_PLUGIN_ENDPOINT=http://${MCHOST}/api/multicloud-titaniumcloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}
 413
 414      ### zone operations
 415      # because all VM's use 10.0.100.1 as their first DNS server, the designate DNS server as seocnd, we need to use a
 416      # domain outside of the first DNS server's domain
 417     local DCAE_DOMAIN
 418     local ZONENAME
 419     DCAE_DOMAIN="$(cat /opt/config/dcae_domain.txt)"
 420     ZONENAME="${DCAE_ZONE}.${DCAE_DOMAIN}."
 421
 422     echo "===> Register DNS zone $ZONENAME under $DNSAAS_TENANT_NAME"
 423
 424
 425     ### Get Token
 426     local TOKEN
 427
 428     MCURL="${MULTICLOUD_PLUGIN_ENDPOINT}/identity/v3/auth/tokens"
 429     echo "=====> Getting token from $MCURL"
 430     #TOKEN=$(call_api_for_response_header "$MCURL" "$MCMETHOD" "$MCRESP" "$MCHEADERS" "$MCAUTH" "$MCDATA" | grep 'X-Subject-Token' | sed "s/^.*: //")
 431     TOKEN=$(curl -v -s -H "Content-Type: application/json" -X POST -d "{\"auth\":{\"tenantName\": \"${DNSAAS_TENANT_NAME}\"}}" "${MCURL}" 2>&1 | grep X-Subject-Token | sed "s/^.*: //")
 432     echo "Received Keystone token $TOKEN from $MCURL"
 433     if [ -z "$TOKEN" ]; then
 434         echo "Faile to acquire token for creating DNS zone.  Exit"
 435         exit 1
 436     fi
 437
 438     local PROJECTID
 439     PROJECTID=$(curl -v -s  -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones?name=${ZONENAME}" |grep 'project_id' |sed 's/^.*"project_id":"\([a-zA-Z0-9-]*\)",.*$/\1/')
 440     if [ ! -z "$PROJECTID" ]; then
 441         ### query the zone with zone id
 442         echo "!!!!!!> zone $ZONENAME already registered by project $PROJECTID"
 443     else
 444         ### create a zone
 445         echo "=====> No zone of same name $ZONENAME found, creating new zone "
 446         curl -sv -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X POST -d "{ \"name\": \"$ZONENAME\", \"email\": \"lji@research.att.com\"}" "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones"
 447     fi
 448
 449     ### list zones
 450     echo "=====> Zone listing"
 451     curl -sv -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones" | python -m json.tool
 452
 453     ### query the zone with zone name
 454     #echo "=====> Querying zone $ZONENAME"
 455     #curl -s -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones?name=${ZONENAME}"
 456
 457     ### export ZONE id
 458     local ZONEID
 459     ZONEID=$(curl -v -sb  -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones?name=${ZONENAME}" |grep 'id' |sed 's/^.*"id":"\([a-zA-Z0-9-]*\)",.*$/\1/')
 460     echo "=====> After creation, zone $ZONENAME ID is $ZONEID"
 461
 462     ### query the zone with zone id
 463     #echo "=====> Querying zone $ZONENAME by ID $ZONEID"
 464     #curl -sv -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones/${ZONEID}"
 465 }
 466
 467
 468 register_dns_zone_designate()
 469 {
 470     export http_proxy=""
 471     export https_proxy=""
 472
 473     local HEADER_CONTENT_TYPE_JSON="Content-Type: application/json"
 474     local HEADER_ACCEPT_JSON="Accept: application/json"
 475     local HEADER_TOKEN
 476     local DCAE_ZONE
 477     local DCAE_DOMAIN
 478     local ZONE_NAME
 479     local ZONE_ID
 480     local KEYSTONE_URL
 481     local API_ENDPOINT
 482     local API_DATA
 483     local TENANT_NAME
 484     local TENANT_ID
 485     local ZONE_PROJECT_ID
 486
 487     if [ -z "$1" ]; then DCAE_ZONE="$(cat /opt/config/dcae_zone.txt)"; else DCAE_ZONE="$1"; fi
 488     DCAE_DOMAIN="$(cat /opt/config/dcae_domain.txt)"
 489     ZONE_NAME="${DCAE_ZONE}.${DCAE_DOMAIN}."
 490
 491     TENANT_NAME="$(cat /opt/config/tenant_name.txt)"
 492     TENANT_ID="$(cat /opt/config/tenant_id.txt)"
 493
 494     KEYSTONE_URL="$(cat /opt/config/openstack_keystone_url.txt)"
 495     if [[ "$KEYSTONE_URL" == */v3 ]]; then
 496         echo "$KEYSTONE_URL"
 497     elif [[ "$KEYSTONE_URL" == */v2.0 ]]; then
 498         echo "$KEYSTONE_URL"
 499     else
 500         KEYSTONE_URL="${KEYSTONE_URL}/v2.0"
 501         echo "$KEYSTONE_URL"
 502     fi
 503
 504     USERNAME="$(cat /opt/config/openstack_user.txt)"
 505     PASSWORD="$(cat /opt/config/openstack_password.txt)"
 506
 507
 508     API_ENDPOINT="${KEYSTONE_URL}/tokens"
 509     API_DATA="{\"auth\": {\"project\": \"${TENANT_NAME}\", \"tenantId\": \"${TENANT_ID}\", \"passwordCredentials\": {\"username\": \"${USERNAME}\", \"password\": \"${PASSWORD}\"}}}"
 510
 511     echo "===> Getting token from ${API_ENDPOINT}"
 512     RESP=$(curl -s -v -H "${HEADER_CONTENT_TYPE_JSON}" -d "${API_DATA}" "${API_ENDPOINT}")
 513
 514     TOKEN="$(echo ${RESP} | jq -r .access.token.id)"
 515     if [ -z "$TOKEN" ]; then
 516         echo "Faile to acquire token for creating DNS zone.  Exit"
 517         exit 1
 518     fi
 519     HEADER_TOKEN="X-Auth-Token: ${TOKEN}"
 520
 521
 522     DESIGNATE_URL=$(echo ${RESP} | jq -r '.access.serviceCatalog[] | select(.name=="designate") | .endpoints[0].publicURL')
 523     if [ -z ${DESIGNATE_URL} ]; then
 524         echo "Fail to find Designate API endpoint.  Exit"
 525         exit 1
 526     fi
 527
 528
 529     API_ENDPOINT="${DESIGNATE_URL}/v2/zones"
 530     echo "===> Register DNS zone $ZONE_NAME at Designate API endpoint ${API_ENDPOINT}"
 531
 532     RESP=$(curl -v -s -H "$HEADER_TOKEN" "$API_ENDPOINT")
 533     ZONE_ID=$(echo $RESP |jq -r --arg ZONE_NAME "$ZONE_NAME" '.zones[] |select(.name==$ZONE_NAME) |.id')
 534     if [ -z "$ZONE_ID" ]; then
 535         echo "======> Zone ${ZONE_NAME} does not exist.  Create"
 536         API_DATA="{\"name\": \"${ZONE_NAME}\", \"email\": \"dcae@onap.org\", \"type\": \"PRIMARY\", \"ttl\": 7200, \"description\": \"DCAE DNS zoen created for ONAP deployment $DCAE_ZONE\"}"
 537         RESP=$(curl -s -v -X POST -H "$HEADER_ACCEPT_JSON" -H "$HEADER_CONTENT_TYPE_JSON" -H "$HEADER_TOKEN" -d "$API_DATA" "$API_ENDPOINT")
 538         ZONE_ID=$(echo $RESP | jq .id)
 539
 540         if [ -z "$ZONE_ID" ]; then
 541             echo "Fail to create DNS zone ${ZONE_NAME}.  Exit"
 542             exit 1
 543         fi
 544     else
 545         echo "======> Zone ${ZONE_NAME} already exists."
 546         API_ENDPOINT="${DESIGNATE_URL}/v2/zones/${ZONE_ID}"
 547         RESP=$(curl -s -v -H "$HEADER_ACCEPT_JSON" -H "$HEADER_TOKEN" "$API_ENDPOINT")
 548         ZONE_PROJECT_ID=$(echo $RESP | jq -r .project_id)
 549         if [ "$ZONE_PROJECT_ID" != "noauth-project" ] && [ "$ZONE_PROJECT_ID" != "$TENANT_ID" ]; then
 550             echo "======> Zone ${ZONE_NAME} owned by other projects, may have problem creating records"
 551         else
 552             echo "======> Zone ${ZONE_NAME} okay to create new records"
 553         fi
 554     fi
 555 }
 556
 557 delete_dns_zone()
 558 {
 559     export http_proxy=""
 560     export https_proxy=""
 561
 562     local CLOUD_OWNER='pod25'
 563     local CLOUD_REGION
 564     local CLOUD_VERSION='titanium_cloud'
 565     local CLOUD_ENV
 566     local DCAE_ZONE
 567     local DNSAAS_TENANT_NAME
 568     local MCHOST
 569     local MCURL
 570     local MCMETHOD='-X GET'
 571     local MCRESP='200'
 572     local MCHEADERS='-H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
 573     local MCAUTH=''
 574     local MCDATA=''
 575     local MULTICLOUD_PLUGIN_ENDPOINT
 576
 577     CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
 578     CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
 579     DCAE_ZONE="$(cat /opt/config/dcae_zone.txt)"
 580     DNSAAS_TENANT_NAME="$(cat /opt/config/dnsaas_tenant_name.txt)"
 581     MCHOST=$(cat /opt/config/multiserv_ip_addr.txt)
 582     MCURL="http://$MCHOST:9005/api/multicloud-titaniumcloud/v0/swagger.json"
 583
 584     local DCAE_DOMAIN
 585     local ZONENAME
 586     DCAE_DOMAIN="$(cat /opt/config/dcae_domain.txt)"
 587     ZONENAME="${DCAE_ZONE}.${DCAE_DOMAIN}."
 588
 589     MCDATA='"{\"auth\":{\"tenantName\": \"'${DNSAAS_TENANT_NAME}'\"}}"'
 590     MULTICLOUD_PLUGIN_ENDPOINT=http://${MCHOST}/api/multicloud-titaniumcloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}
 591
 592     ### Get Token
 593     local TOKEN
 594     TOKEN=$(curl -v -s -H "Content-Type: application/json" -X POST -d "{\"auth\":{\"tenantName\": \"${DNSAAS_TENANT_NAME}\"}}" "${MULTICLOUD_PLUGIN_ENDPOINT}/identity/v3/auth/tokens"  2>&1 | grep X-Subject-Token | sed "s/^.*: //")
 595
 596     local ZONEID
 597     ZONEID=$(curl -v -s  -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones?name=${ZONENAME}" |sed 's/^.*"id":"\([a-zA-Z0-9-]*\)",.*$/\1/')
 598
 599     curl -s -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X DELETE "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones/${ZONEID}"
 600 }
 601
 602 list_dns_zone()
 603 {
 604     export http_proxy=""
 605     export https_proxy=""
 606
 607     local CLOUD_OWNER='pod25'
 608     local CLOUD_REGION
 609     local CLOUD_VERSION='titanium_cloud'
 610     local CLOUD_ENV
 611     local DCAE_ZONE
 612     local DNSAAS_TENANT_NAME
 613     local MCHOST
 614     local MCURL
 615     local MCMETHOD='-X GET'
 616     local MCRESP='200'
 617     local MCHEADERS='-H "Real-Time: true" -H "Content-Type: application/json" -H "Accept: application/json"'
 618     local MCAUTH=''
 619     local MCDATA=''
 620     local MULTICLOUD_PLUGIN_ENDPOINT
 621
 622     CLOUD_REGION="$(cat /opt/config/openstack_region.txt)"
 623     CLOUD_ENV="$(cat /opt/config/cloud_env.txt)"
 624     DCAE_ZONE="$(cat /opt/config/dcae_zone.txt)"
 625     DNSAAS_TENANT_NAME="$(cat /opt/config/dnsaas_tenant_name.txt)"
 626     MCHOST=$(cat /opt/config/multiserv_ip_addr.txt)
 627     MCURL="http://$MCHOST:9005/api/multicloud-titaniumcloud/v0/swagger.json"
 628
 629     MCDATA='"{\"auth\":{\"tenantName\": \"'${DNSAAS_TENANT_NAME}'\"}}"'
 630     MULTICLOUD_PLUGIN_ENDPOINT=http://${MCHOST}/api/multicloud-titaniumcloud/v0/${CLOUD_OWNER}_${CLOUD_REGION}
 631
 632     ### Get Token
 633     local TOKEN
 634     TOKEN=$(curl -v -s -H "Content-Type: application/json" -X POST -d "{\"auth\":{\"tenantName\": \"${DNSAAS_TENANT_NAME}\"}}" "${MULTICLOUD_PLUGIN_ENDPOINT}/identity/v3/auth/tokens"  2>&1 | grep X-Subject-Token | sed "s/^.*: //")
 635
 636     local DCAE_DOMAIN
 637     local ZONENAME
 638     DCAE_DOMAIN="$(cat /opt/config/dcae_domain.txt)"
 639     ZONENAME="${DCAE_ZONE}.${DCAE_DOMAIN}."
 640     local ZONEID
 641     ZONEID=$(curl -v -s  -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones?name=${ZONENAME}" |sed 's/^.*"id":"\([a-zA-Z0-9-]*\)",.*$/\1/')
 642
 643     curl -v -s  -H "Content-Type: application/json" -H "X-Auth-Token: $TOKEN" -X GET "${MULTICLOUD_PLUGIN_ENDPOINT}/dns-delegate/v2/zones/${ZONEID}/recordsets"
 644 }
 645
 646 ################################## start of vm_init #####################################
 647
 648 # prepare the configurations needed by DCAEGEN2 installer
 649 rm -rf /opt/app/config
 650 mkdir -p /opt/app/config
 651
 652
 653 # private key
 654 sed -e 's/\\n/\n/g' /opt/config/priv_key | sed -e 's/^[ \t]*//g; s/[ \t]*$//g' > /opt/app/config/key
 655 chmod 777 /opt/app/config/key
 656
 657 # move keystone url file
 658 #cp /opt/config/keystone_url.txt /opt/app/config/keystone_url.txt
 659
 660
 661 URL_ROOT='nexus.onap.org/service/local/repositories/raw/content'
 662 REPO_BLUEPRINTS='org.onap.dcaegen2.platform.blueprints'
 663 REPO_DEPLOYMENTS='org.onap.dcaegen2.deployments'
 664 if [ -e /opt/config/dcae_deployment_profile.txt ]; then
 665   DEPLOYMENT_PROFILE=$(cat /opt/config/dcae_deployment_profile.txt)
 666 fi
 667 DEPLOYMENT_PROFILE=${DEPLOYMENT_PROFILE:-R1}
 668
 669 NEXUS_USER=$(cat /opt/config/nexus_username.txt)
 670 NEXUS_PASSWORD=$(cat /opt/config/nexus_password.txt)
 671 NEXUS_DOCKER_REPO=$(cat /opt/config/nexus_docker_repo.txt)
 672 DOCKER_VERSION=$(cat /opt/config/docker_version.txt)
 673 # use rand_str as zone
 674 ZONE=$(cat /opt/config/rand_str.txt)
 675 MYFLOATIP=$(cat /opt/config/dcae_float_ip.txt)
 676 MYLOCALIP=$(cat /opt/config/dcae_ip_addr.txt)
 677 HTTP_PROXY=$(cat /opt/config/http_proxy.txt)
 678 HTTPS_PROXY=$(cat /opt/config/https_proxy.txt)
 679
 680 if [ $HTTP_PROXY != "no_proxy" ]
 681 then
 682     export http_proxy=$HTTP_PROXY
 683     export https_proxy=$HTTPS_PROXY
 684 fi
 685
 686 # start docker image pulling while we are waiting for A&AI to come online
 687 docker login -u "$NEXUS_USER" -p "$NEXUS_PASSWORD" "$NEXUS_DOCKER_REPO"
 688
 689
 690 if [ "$DEPLOYMENT_PROFILE" == "R1" ]; then
 691   RELEASE_TAG='releases'
 692   # download blueprint input template files
 693   rm -rf /opt/app/inputs-templates
 694   mkdir -p /opt/app/inputs-templates
 695   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_BLUEPRINTS}/${RELEASE_TAG}/input-templates/inputs.yaml
 696   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_BLUEPRINTS}/${RELEASE_TAG}/input-templates/cdapinputs.yaml
 697   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_BLUEPRINTS}/${RELEASE_TAG}/input-templates/phinputs.yaml
 698   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_BLUEPRINTS}/${RELEASE_TAG}/input-templates/dhinputs.yaml
 699   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_BLUEPRINTS}/${RELEASE_TAG}/input-templates/invinputs.yaml
 700   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_BLUEPRINTS}/${RELEASE_TAG}/input-templates/vesinput.yaml
 701   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_BLUEPRINTS}/${RELEASE_TAG}/input-templates/tcainputs.yaml
 702   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_BLUEPRINTS}/${RELEASE_TAG}/input-templates/he-ip.yaml
 703   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_BLUEPRINTS}/${RELEASE_TAG}/input-templates/hr-ip.yaml
 704
 705   # generate blueprint input files
 706   pip install --upgrade jinja2
 707   wget https://${URL_ROOT}/${REPO_DEPLOYMENTS}/${RELEASE_TAG}/scripts/detemplate-bpinputs.py \
 708     && \
 709     (python detemplate-bpinputs.py /opt/config /opt/app/inputs-templates /opt/app/config; \
 710      rm detemplate-bpinputs.py)
 711
 712   # Run docker containers
 713   cd /opt
 714
 715
 716   docker pull "$NEXUS_DOCKER_REPO/onap/org.onap.dcaegen2.deployments.bootstrap:$DOCKER_VERSION" \
 717     && docker pull nginx &
 718
 719   #########################################
 720   # Wait for then register with A&AI
 721   ########################################
 722
 723   DNSAAS_PROXYED=$(tr '[:upper:]' '[:lower:]' < /opt/config/dnsaas_config_enabled.txt)
 724   if [ "$DNSAAS_PROXYED" == 'true' ]; then
 725     echo "Using proxyed DNSaaS service, performing additional registration and configuration"
 726     wait_for_aai_ready
 727
 728     register_multicloud_pod25_with_aai
 729     register_multicloud_pod25dns_with_aai
 730
 731     verify_multicloud_registration
 732
 733     wait_for_multicloud_ready
 734     register_dns_zone_proxied_designate "$ZONE"
 735     echo "Registration and configuration for proxying DNSaaS completed."
 736   else
 737     echo "Using Designate DNSaaS service, performing additional registration and configuration"
 738     register_dns_zone_designate "$ZONE"
 739   fi
 740
 741   #########################################
 742   # Start DCAE Bootstrap container
 743   #########################################
 744
 745   chmod 777 /opt/app/config
 746   rm -f /opt/config/runtime.ip.consul
 747   rm -f /opt/config/runtime.ip.cm
 748
 749
 750   #docker login -u "$NEXUS_USER" -p "$NEXUS_PASSWORD" "$NEXUS_DOCKER_REPO"
 751   #docker pull "$NEXUS_DOCKER_REPO/onap/org.onap.dcaegen2.deployments.bootstrap:$DOCKER_VERSION"
 752   docker run -d --name boot -v /opt/app/config:/opt/app/installer/config -e "LOCATION=$ZONE" "$NEXUS_DOCKER_REPO/onap/org.onap.dcaegen2.deployments.bootstrap:$DOCKER_VERSION"
 753
 754
 755   # waiting for bootstrap to complete then starting nginx for proxying healthcheck calls
 756   echo "Waiting for Consul to become accessible"
 757   while [ ! -f /opt/app/config/runtime.ip.consul ]; do echo "."; sleep 30; done
 758
 759
 760   # start proxy for consul's health check
 761   CONSULIP=$(head -1 /opt/app/config/runtime.ip.consul | sed 's/[[:space:]]//g')
 762   echo "Consul is available at $CONSULIP"
 763 fi
 764
 765 if [[ $DEPLOYMENT_PROFILE == R2* ]]; then
 766   RELEASE_TAG='R2'
 767   set +e
 768   rm -rf /opt/app/inputs-templates
 769   mkdir -p /opt/app/inputs-templates
 770   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_DEPLOYMENTS}/${RELEASE_TAG}/heat/docker-compose-1.yaml
 771   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_DEPLOYMENTS}/${RELEASE_TAG}/heat/docker-compose-2.yaml
 772   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_DEPLOYMENTS}/${RELEASE_TAG}/heat/docker-compose-3.yaml
 773   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_DEPLOYMENTS}/${RELEASE_TAG}/heat/docker-compose-4.yaml
 774   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_DEPLOYMENTS}/${RELEASE_TAG}/heat/register.sh
 775   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_DEPLOYMENTS}/${RELEASE_TAG}/heat/setup.sh
 776   wget -P /opt/app/inputs-templates https://${URL_ROOT}/${REPO_DEPLOYMENTS}/${RELEASE_TAG}/heat/teardown.sh
 777
 778   pip install --upgrade jinja2
 779   wget https://${URL_ROOT}/${REPO_DEPLOYMENTS}/${RELEASE_TAG}/scripts/detemplate-bpinputs.py \
 780     && \
 781     (python detemplate-bpinputs.py /opt/config /opt/app/inputs-templates /opt/app/config; \
 782      rm detemplate-bpinputs.py)
 783
 784   if [ -e /opt/app/config/register.sh ]; then
 785     chmod +x /opt/app/config/register.sh
 786   fi
 787   if [ -e /opt/app/config/setup.sh ]; then
 788     chmod +x /opt/app/config/setup.sh
 789   fi
 790   if [ -e /opt/app/config/build-plugins.sh ]; then
 791     chmod +x /opt/app/config/build-plugins.sh
 792   fi
 793   set -e
 794
 795   cd /opt/app/config
 796   # deploy essentials
 797   /opt/docker/docker-compose -f docker-compose-1.yaml up -d
 798   echo "Waiting for Consul to come up ready"
 799   while ! nc -z localhost 8500; do sleep 1; done
 800   echo "Waiting for DB to come up ready"
 801   while ! nc -z localhost 5432; do sleep 1; done
 802   echo "Waiting for CBS to come up ready"
 803   while ! nc -z localhost 10000; do sleep 1; done
 804   echo "All dependencies are up, proceed to the next phase"
 805   sleep 30
 806
 807   echo "Setup CloudifyManager and Registrator"
 808   ./setup.sh
 809   sleep 10
 810
 811   export http_proxy=""
 812   export https_proxy=""
 813
 814
 815   ./register.sh
 816
 817   echo "Bring up DCAE MIN service components for R2 use cases"
 818   /opt/docker/docker-compose -f docker-compose-2.yaml up -d
 819
 820   if [[ "$DEPLOYMENT_PROFILE" == "R2" || "$DEPLOYMENT_PROFILE" == "R2PLUS" ]]; then
 821     echo "Bring up DCAE platform components"
 822     /opt/docker/docker-compose -f docker-compose-3.yaml up -d
 823
 824     if [ "$DEPLOYMENT_PROFILE" == "R2PLUS" ]; then
 825       echo "Bring up additional (plus) DCAE service components"
 826       /opt/docker/docker-compose -f docker-compose-4.yaml up -d
 827     fi
 828   fi
 829
 830   # start proxy for consul's health check
 831   CONSULIP=$(cat /opt/config/dcae_ip_addr.txt)
 832   echo "Consul is available at $CONSULIP"
 833 fi
 834
 835 cat >./nginx.conf <<EOL
 836 server {
 837     listen 80;
 838     server_name dcae.simpledemo.onap.org;
 839     root /www/healthcheck;
 840
 841     location /healthcheck {
 842         try_files /services.yaml =404;
 843     }
 844     location /R1 {
 845         proxy_pass http://${CONSULIP}:8500/v1/health/state/passing;
 846     }
 847     location /R2MIN{
 848         try_files /r2mvp_healthy.yaml =404;
 849     }
 850     location /R2 {
 851         try_files /r2_healthy.yaml =404;
 852     }
 853     location /R2PLUS {
 854         try_files /r2plus_healthy.yaml =404;
 855     }
 856 }
 857 EOL
 858
 859 HEALTHPORT=8000
 860 docker run -d \
 861 --name dcae-health \
 862 -p ${HEALTHPORT}:80 \
 863 -v "$(pwd)/nginx.conf:/etc/nginx/conf.d/default.conf" \
 864 -v "/tmp/healthcheck:/www/healthcheck" \
 865 --label "SERVICE_80_NAME=dcae-health" \
 866 --label "SERVICE_80_CHECK_HTTP=/healthcheck" \
 867 --label "SERVICE_80_CHECK_INTERVAL=15s" \
 868 --label "SERVICE_80_CHECK_INITIAL_STATUS=passing" \
 869  nginx
 870
 871 echo "Healthcheck API available at http://${MYFLOATIP}:${HEALTHPORT}/healthcheck"
 872 echo "                             http://${MYFLOATIP}:${HEALTHPORT}/R1"
 873 echo "                             http://${MYFLOATIP}:${HEALTHPORT}/R2MIN"
 874 echo "                             http://${MYFLOATIP}:${HEALTHPORT}/R2PLUS"
 875
 876 # run forever for updating health status based on consul
 877 set +e
 878 while :
 879 do
 880   rm -rf /tmp/healthcheck/*
 881   # all registered services
 882   SERVICES=$(curl -s http://consul:8500/v1/agent/services |jq '. | to_entries[] | .value.Service')
 883   # passing services
 884   SERVICES=$(curl -s http://consul:8500/v1/health/state/passing | jq '.[] | .ServiceName')
 885
 886   # remove empty lines/entries
 887   SERVICES=$(echo "$SERVICES" | sed '/^\s*\"\"\s*$/d' |sed '/^\s*$/d')
 888
 889   SERVICES_JSON=$(echo "$SERVICES" | sed 's/\"$/\",/g' | sed '$ s/.$//')
 890
 891   echo "$(date): running healthy services:"
 892   echo ">>> " $SERVICES
 893   PLT_CONSUL=$(echo "$SERVICES" |grep consul)
 894   PLT_CBS=$(echo "$SERVICES" |grep "config_binding_service")
 895   MVP_PG_HOLMES=$(echo "$SERVICES" |grep "pgHolmes")
 896   MVP_VES=$(echo "$SERVICES" |grep "mvp.*ves")
 897   MVP_TCA=$(echo "$SERVICES" |grep "mvp.*tca")
 898   MVP_HR=$(echo "$SERVICES" |grep "mvp.*holmes-rule")
 899   MVP_HE=$(echo "$SERVICES" |grep "mvp.*holmes-engine")
 900
 901   PLT_CM=$(echo "$SERVICES" |grep "cloudify.*manager")
 902   PLT_DH=$(echo "$SERVICES" |grep "deployment.*handler")
 903   PLT_PH=$(echo "$SERVICES" |grep "policy.*handler")
 904   PLT_SCH=$(echo "$SERVICES" |grep "service.*change.*handler")
 905   PLT_INV=$(echo "$SERVICES" |grep "inventory")
 906   PLT_PG_INVENTORY=$(echo "$SERVICES" |grep "pgInventory")
 907
 908   PLUS_MHB=$(echo "$SERVICES" |grep "heartbeat")
 909   PLUS_PRH=$(echo "$SERVICES" |grep "prh")
 910   PLUS_MPR=$(echo "$SERVICES" |grep "mapper")
 911   PLUS_TRAP=$(echo "$SERVICES" |grep "snmptrap")
 912
 913   DATA="{\"healthy\" : \"$(date)\", \"healthy_services\": [${SERVICES_JSON}]}"
 914   if [[ -n "$PLT_CONSUL" && -n "$PLT_CBS" && -n "$MVP_PG_HOLMES" && -n "$MVP_VES" && \
 915         -n "$MVP_TCA" && -n "$MVP_HR" && -n "$MVP_HE" ]]; then
 916     echo "${DATA}" > /tmp/healthcheck/r2mvp_healthy.yaml
 917     echo "${DATA}" > /tmp/healthcheck/services.yaml
 918     echo ">>>>>> enough services satisfying R2MIN service deployment"
 919   else
 920     echo ">>>>>> not enough services satisfying R2MIN service deployment"
 921   fi
 922
 923   if [[ -n "$PLT_CONSUL" && -n "$PLT_CBS" && -n "$PLT_CM" && -n "$PLT_DH" && \
 924         -n "$PLT_PH" && -n "$PLT_SCH" && -n "$PLT_INV" && -n "$PLT_PG_INVENTORY" ]]; then
 925     echo ">>>>>> enough services satisfying R2 platform deployment"
 926     echo "${DATA}" > /tmp/healthcheck/r2_healthy.yaml
 927
 928     if [[ -n "$PLUS_MHB" && -n "$PLUS_PRH" && -n "$PLUS_MPR" && -n "$PLUS_TRAP" ]]; then
 929       echo ">>>>>> enough services satisfying R2PLUS deployment"
 930       echo "${DATA}" > /tmp/healthcheck/r2plus_healthy.yaml
 931     else
 932       echo ">>>>>> not enough services satisfying R2PLUS service deployment"
 933     fi
 934   else
 935     echo ">>>>>> not enough services satisfying R2 platform or R2PLUS service deployment"
 936   fi
 937   sleep 60
 938 done
 939