deployment/aks/cloud.sh

   1 #!/bin/bash
   2 # Copyright 2019 AT&T Intellectual Property. All rights reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #         http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
  17
  18 NO_PROMPT=0
  19 NO_INSTALL=0
  20 NO_VALIDATE=0
  21 POST_INSTALL=0
  22 OVERRIDE=0
  23
  24 OPENSTACK_CLI_POD="os-cli-0"
  25
  26 if [ ! -f $DIR/cloud.conf ]; then
  27   echo "cloud.conf not found, exiting..."
  28   exit 1
  29 fi
  30
  31 . $DIR/cloud.conf
  32
  33 while test $# -gt 0; do
  34   case "$1" in
  35     -h|--help)
  36       echo "./cloud.sh [options]"
  37       echo " "
  38       echo " "
  39       echo "options:"
  40       echo "-f, --no-prompt           executes with no prompt for confirmation"
  41       echo "-n, --no-install          don't install ONAP"
  42       echo "-o, --override            create integration override for robot configuration"
  43       echo "-d, --no-validate         dont validate pre-reqs before executing deployment"
  44       echo "-p, --post-install        execute post-install scripts"
  45       echo "-h, --help                provide brief overview of script"
  46       echo " "
  47       echo "This script deploys a cloud environment in Azure."
  48       echo "It: "
  49       echo "- Uses Azure Kubernetes Service (AKS) to bootstrap a kubernetes cluster."
  50       echo "- Creates a VM with an external disk to be used as NFS storage."
  51       echo "- Creates a VM and installs DevStack, to be used with ONAP."
  52       echo "- Launches ONAP onto the AKS Cluster via OOM."
  53       echo "- Configures Networking, SSH Access, and Security Group Rules"
  54       echo ""
  55       exit 0
  56       ;;
  57     -f|--no-prompt)
  58       shift
  59       NO_PROMPT=1
  60       ;;
  61     -n|--no-install)
  62       shift
  63       NO_INSTALL=1
  64       ;;
  65     -o|--override)
  66       shift
  67       OVERRIDE=1
  68       ;;
  69     -d|--no-validate)
  70       shift
  71       NO_VALIDATE=1
  72       ;;
  73     -p|--post-install)
  74       shift
  75       POST_INSTALL=1
  76       ;;
  77     *)
  78       echo "Unknown Argument. Try running with --help ."
  79       exit 0
  80       ;;
  81   esac
  82 done
  83
  84 if [ $NO_VALIDATE = 0 ]; then
  85   $DIR/pre_install.sh "$AKS_K8_VERSION" "$LOCATION"
  86   if [ $? -ne 0 ]; then
  87     exit 1
  88   fi
  89 fi
  90
  91 cat <<EOF
  92
  93 Here are the parameters to be used in this build:
  94
  95 # GLOBAL PARAMS
  96 LOCATION                = "$LOCATION"
  97 USER_PUBLIC_IP_PREFIX   = "$USER_PUBLIC_IP_PREFIX"
  98 BUILD_DIR               = "$BUILD_DIR"
  99
 100 # AKS PARAMS
 101 AKS_RESOURCE_GROUP_NAME = "$AKS_RESOURCE_GROUP_NAME"
 102 AKS_NAME                = "$AKS_NAME"
 103 AKS_K8_VERSION          = "$AKS_K8_VERSION"
 104 AKS_NODE_COUNT          = "$AKS_NODE_COUNT"
 105 AKS_NODE_SIZE           = "$AKS_NODE_SIZE"
 106 AKS_DNS_PREFIX          = "$AKS_DNS_PREFIX"
 107 AKS_POD_CIDR            = "$AKS_POD_CIDR"
 108 AKS_NODE_CIDR           = "$AKS_NODE_CIDR"
 109 AKS_SERVICE_CIDR        = "$AKS_SERVICE_CIDR"
 110 AKS_DNS_IP              = "$AKS_DNS_IP"
 111 AKS_VNET_NAME           = "$AKS_VNET_NAME"
 112 AKS_ADMIN_USER=         = "$AKS_ADMIN_USER"
 113
 114 # NFS PARAMS
 115 NFS_NAME                = "$NFS_NAME"
 116 NFS_RG                  = "$NFS_RG"
 117 NFS_VM_SIZE             = "$NFS_VM_SIZE"
 118 NFS_LOCATION            = "$NFS_LOCATION"
 119 NFS_CIDR                = "$NFS_CIDR"
 120 NFS_ADMIN_USER          = "$NFS_ADMIN_USER"
 121 NFS_VNET_NAME           = "$NFS_VNET_NAME"
 122 NFS_SUBNET_NAME         = "$NFS_SUBNET_NAME"
 123 NFS_DISK_SIZE           = "$NFS_DISK_SIZE"
 124
 125 # DEVSTACK PARAMS
 126 DEVSTACK_NAME           = "$DEVSTACK_NAME"
 127 DEVSTACK_RG             = "$DEVSTACK_RG"
 128 DEVSTACK_VM_SIZE        = "$DEVSTACK_VM_SIZE"
 129 DEVSTACK_LOCATION       = "$DEVSTACK_LOCATION"
 130 DEVSTACK_CIDR           = "$DEVSTACK_CIDR"
 131 DEVSTACK_PRIVATE_IP     = "$DEVSTACK_PRIVATE_IP"
 132 DEVSTACK_ADMIN_USER     = "$DEVSTACK_ADMIN_USER"
 133 DEVSTACK_VNET_NAME      = "$DEVSTACK_VNET_NAME"
 134 DEVSTACK_SUBNET_NAME    = "$DEVSTACK_SUBNET_NAME"
 135 DEVSTACK_DISK_SIZE      = "$DEVSTACK_DISK_SIZE"
 136 OPENSTACK_USER          = "$OPENSTACK_USER"
 137 OPENSTACK_PASS          = "$OPENSTACK_PASS"
 138 OPENSTACK_TENANT        = "$OPENSTACK_TENANT"
 139 IMAGE_LIST              = "$IMAGE_LIST"
 140
 141 # ONAP PARAMS
 142 CLLI                    = "$CLLI"
 143 CLOUD_OWNER             = "$CLOUD_OWNER"
 144 CLOUD_REGION            = "$CLOUD_REGION"
 145 CUSTOMER                = "$CUSTOMER"
 146 SUBSCRIBER              = "$SUBSCRIBER"
 147 SERVICE_TYPE            = "$SERVICE_TYPE"
 148 AZ                      = "$AZ"
 149 OE                      = "$OE"
 150 LOB                     = "$LOB"
 151 PLATFORM                = "$PLATFORM"
 152 OS_ID                   = "$OS_ID"
 153 OS_TENANT_ROLE          = "$OS_TENANT_ROLE"
 154 OS_KEYSTONE             = "$OS_KEYSTONE"
 155 OPENSTACK_REGION        = "$OPENSTACK_REGION"
 156 PROJECT                 = "$PROJECT"
 157 OOM_BRANCH              = "$OOM_BRANCH"
 158 CHART_VERSION           = "$CHART_VERSION"
 159 OOM_OVERRIDES           = "$OOM_OVERRIDES"
 160 DOCKER_REPOSITORY       = "$DOCKER_REPOSITORY"
 161
 162 EOF
 163
 164 if [ $NO_PROMPT = 0 ]; then
 165   read -p "Would you like to proceed? [y/n]" -n 1 -r
 166   echo " "
 167   if [[ ! $REPLY =~ ^[Yy]$ ]]
 168   then
 169       exit 0
 170   fi
 171 fi
 172
 173 echo "Starting instantiation. This will take a little while..."
 174 sleep 3
 175
 176 set -x
 177 set -e
 178
 179 mkdir -p $BUILD_DIR
 180
 181 echo "#!/bin/bash" > $BUILD_DIR/clean.sh
 182 echo "" >> $BUILD_DIR/clean.sh
 183 chmod 755 $BUILD_DIR/clean.sh
 184
 185 ssh-keygen -t rsa -N "" -f $BUILD_DIR/id_rsa
 186
 187 PUBLIC_KEY=$BUILD_DIR/id_rsa.pub
 188 PRIVATE_KEY=$BUILD_DIR/id_rsa
 189
 190
 191 echo "az group delete --resource-group $DEVSTACK_RG --yes" >> $BUILD_DIR/clean.sh
 192 echo "" >> $BUILD_DIR/clean.sh
 193
 194 echo "Creating DEVSTACK Server $DEVSTACK_NAME in $LOCATION"
 195 $DIR/create_devstack.sh --name "$DEVSTACK_NAME" \
 196                         --resource-group "$DEVSTACK_RG" \
 197                         --size "$DEVSTACK_VM_SIZE" \
 198                         --location "$DEVSTACK_LOCATION" \
 199                         --cidr "$DEVSTACK_CIDR" \
 200                         --admin-user "$DEVSTACK_ADMIN_USER" \
 201                         --directory "$BUILD_DIR" \
 202                         --vnet-name "$DEVSTACK_VNET_NAME" \
 203                         --public-key "$PUBLIC_KEY" \
 204                         --user-public-ip "$USER_PUBLIC_IP_PREFIX" \
 205                         --devstack-private-ip "$DEVSTACK_PRIVATE_IP" \
 206                         --devstack-subnet-name "$DEVSTACK_SUBNET_NAME" \
 207                         --devstack-disk-size "$DEVSTACK_DISK_SIZE" \
 208                         --openstack-username "$OPENSTACK_USER" \
 209                         --openstack-password "$OPENSTACK_PASS" \
 210                         --openstack-tenant "$OPENSTACK_TENANT" \
 211                         --image-list "$IMAGE_LIST" \
 212                         --no-prompt
 213
 214
 215 echo "az group delete --resource-group $NFS_RG --yes" >> $BUILD_DIR/clean.sh
 216 echo "" >> $BUILD_DIR/clean.sh
 217
 218 echo "Creating NFS Server $NFS_NAME in $LOCATION"
 219 $DIR/create_nfs.sh --name "$NFS_NAME" \
 220                    --resource-group "$NFS_RG" \
 221                    --size "$NFS_VM_SIZE" \
 222                    --location "$NFS_LOCATION" \
 223                    --cidr "$NFS_CIDR" \
 224                    --admin-user "$NFS_ADMIN_USER" \
 225                    --directory "$BUILD_DIR" \
 226                    --vnet-name "$NFS_VNET_NAME" \
 227                    --public-key "$PUBLIC_KEY" \
 228                    --user-public-ip "$USER_PUBLIC_IP_PREFIX" \
 229                    --nfs-subnet-name "$NFS_SUBNET_NAME" \
 230                    --aks-node-cidr "$AKS_NODE_CIDR" \
 231                    --nfs-disk-size "$NFS_DISK_SIZE" \
 232                    --no-prompt
 233
 234
 235 echo "az group delete --resource-group $AKS_RESOURCE_GROUP_NAME --yes" >> $BUILD_DIR/clean.sh
 236 echo "" >> $BUILD_DIR/clean.sh
 237
 238 echo "Creating AKS $AKS_NAME in $LOCATION"
 239 $DIR/create_aks.sh --name "$AKS_NAME" \
 240                    --resource-group "$AKS_RESOURCE_GROUP_NAME" \
 241                    --kube-version "$AKS_K8_VERSION" \
 242                    --location "$LOCATION" \
 243                    --node-count "$AKS_NODE_COUNT" \
 244                    --size "$AKS_NODE_SIZE" \
 245                    --service-cidr "$AKS_SERVICE_CIDR" \
 246                    --pod-cidr "$AKS_POD_CIDR" \
 247                    --dns-ip "$AKS_DNS_IP" \
 248                    --node-cidr "$AKS_NODE_CIDR" \
 249                    --vnet-name "$AKS_VNET_NAME" \
 250                    --user-public-ip "$USER_PUBLIC_IP_PREFIX" \
 251                    --public-key "$PUBLIC_KEY" \
 252                    --admin-user "$AKS_ADMIN_USER" \
 253                    --no-prompt
 254
 255
 256 AKS_MANAGEMENT_RESOURCE_GROUP_NAME=`az group list --query "[?starts_with(name, 'MC_${AKS_RESOURCE_GROUP_NAME}')].name | [0]" --output tsv`
 257 AKS_VNET_ID=`az network vnet show --resource-group ${AKS_RESOURCE_GROUP_NAME} --name ${AKS_VNET_NAME} --query 'id' --output tsv`
 258 NFS_VNET_ID=`az network vnet show --name ${NFS_VNET_NAME}  --resource-group ${NFS_RG} --query "id" --output tsv`
 259 DEVSTACK_VNET_ID=`az network vnet show --name ${DEVSTACK_VNET_NAME}  --resource-group ${DEVSTACK_RG} --query "id" --output tsv`
 260 AKS_ROUTE_TABLE_NAME=`az resource list --resource-group ${AKS_MANAGEMENT_RESOURCE_GROUP_NAME} --resource-type "Microsoft.Network/routeTables" --query "[0] | name" --output tsv`
 261 DEVSTACK_PRIVATE_IP=`az vm show --name ${DEVSTACK_NAME} --resource-group ${DEVSTACK_RG} -d  --query "privateIps" --output tsv`
 262 NFS_PRIVATE_IP=`az vm show --name ${NFS_NAME} --resource-group ${NFS_RG} -d  --query "privateIps" --output tsv`
 263 NFS_PUBLIC_IP=`az vm show --name ${NFS_NAME} --resource-group ${NFS_RG} -d  --query "publicIps" --output tsv`
 264 DEVSTACK_PUBLIC_IP=`az vm show --name ${DEVSTACK_NAME} --resource-group ${DEVSTACK_RG} -d  --query "publicIps" --output tsv`
 265
 266 # adding public ip to  aks
 267 NIC_NAME0=`az resource list --resource-group ${AKS_MANAGEMENT_RESOURCE_GROUP_NAME} --resource-type "Microsoft.Network/networkInterfaces" --query "[0] | name" --output tsv`
 268 AKS_NSG_NAME=`az resource list --resource-group ${AKS_MANAGEMENT_RESOURCE_GROUP_NAME} --resource-type "Microsoft.Network/networkSecurityGroups" --query "[0] | name" --output tsv`
 269 $DIR/create_public_ip.sh "AKSPUBLICIP1" "$AKS_MANAGEMENT_RESOURCE_GROUP_NAME" "$NIC_NAME0"
 270 $DIR/create_sg_rule.sh "$AKS_MANAGEMENT_RESOURCE_GROUP_NAME" "$AKS_NSG_NAME" '*' "30000-32000" "$USER_PUBLIC_IP_PREFIX" '*' '*' "ONAP" "120"
 271
 272 AKS_PUBLIC_IP_ADDRESS=`az network public-ip show --resource-group ${AKS_MANAGEMENT_RESOURCE_GROUP_NAME} -n AKSPUBLICIP1 --query "ipAddress" --output tsv`
 273
 274 ### Peering networks ###
 275 # peering requires source = VNet NAME, destination = VNet ID
 276
 277 echo "creating peering from AKS Vnet to NFS Vnet..."
 278 $DIR/create_peering.sh "$AKS_VNET_NAME" \
 279                        "$AKS_RESOURCE_GROUP_NAME" \
 280                        "$NFS_VNET_ID" \
 281                        "kube-to-nfs"
 282
 283 echo "creating peering from AKS Vnet to Devstack Vnet..."
 284 $DIR/create_peering.sh "$AKS_VNET_NAME" \
 285                        "$AKS_RESOURCE_GROUP_NAME" \
 286                        "$DEVSTACK_VNET_ID" \
 287                        "kube-to-devstack"
 288
 289 echo "creating peering from NFS Vnet to AKS Vnet..."
 290 $DIR/create_peering.sh "$NFS_VNET_NAME" \
 291                        "$NFS_RG" \
 292                        "$AKS_VNET_ID" \
 293                        "nfs-to-kube"
 294
 295 echo "creating peering from NFS Vnet to AKS Vnet..."
 296 $DIR/create_peering.sh "$DEVSTACK_VNET_NAME" \
 297                        "$DEVSTACK_RG" \
 298                        "$AKS_VNET_ID" \
 299                        "devstack-to-kube"
 300
 301
 302 ### Adding next hop to kubernetes for devstack ###
 303 echo "creating route from AKS Vnet to Devstack Vnet..."
 304 $DIR/create_route.sh "$DEVSTACK_CIDR" \
 305                      "guestvms" \
 306                      "$AKS_MANAGEMENT_RESOURCE_GROUP_NAME" \
 307                      "$AKS_ROUTE_TABLE_NAME" \
 308                      "$DEVSTACK_PRIVATE_IP"
 309
 310
 311
 312 # TODO
 313 # Lets find a better place for this
 314 az aks get-credentials --resource-group "$AKS_RESOURCE_GROUP_NAME" \
 315                        --name "$AKS_NAME" \
 316                        --file "$BUILD_DIR"/"kubeconfig"
 317
 318 $DIR/configure_nfs_pod.sh "$PRIVATE_KEY" \
 319                           "$BUILD_DIR"/"kubeconfig" \
 320                           "$NFS_PRIVATE_IP" \
 321                           "$AKS_ADMIN_USER"
 322
 323 # TODO
 324 # add this to post-install or post-configure phase
 325 # to support adding multiple devstacks to same ONAP
 326 cat > "$BUILD_DIR/openstack_rc" <<EOF
 327 export OS_USERNAME="$OPENSTACK_USER"
 328 export OS_PROJECT_NAME="$OPENSTACK_TENANT"
 329 export OS_AUTH_URL="http://$DEVSTACK_PRIVATE_IP/identity"
 330 export OS_PASSWORD="$OPENSTACK_PASS"
 331 export OS_USER_DOMAIN_NAME=default
 332 export OS_PROJECT_DOMAIN_ID=default
 333 EOF
 334
 335 $DIR/util/create_openstack_cli.sh "$BUILD_DIR/kubeconfig" \
 336                                   "$BUILD_DIR/openstack_rc" \
 337                                   "$OPENSTACK_CLI_POD"
 338
 339
 340 if [ $OVERRIDE = 1 ]; then
 341
 342 $DIR/util/create_integration_override.sh "$BUILD_DIR" \
 343                                          "$OPENSTACK_CLI_POD" \
 344                                          "$BUILD_DIR/openstack_rc" \
 345                                          "$DOCKER_REPOSITORY" \
 346                                          "$NFS_PRIVATE_IP" \
 347                                          "$AKS_PUBLIC_IP_ADDRESS" \
 348                                          "$BUILD_DIR/kubeconfig"
 349
 350 fi
 351
 352
 353 if [ $NO_INSTALL = 0 ]; then
 354
 355 ### Starting OOM install ###
 356 echo "Installing ONAP..."
 357 $DIR/create_onap.sh "$BUILD" \
 358                     "$BUILD_DIR/kubeconfig" \
 359                     "$OOM_BRANCH" \
 360                     "$BUILD_DIR" \
 361                     "$CHART_VERSION" \
 362                     "$OOM_OVERRIDES"
 363
 364 fi
 365
 366
 367 set +x
 368
 369 cat > "$BUILD_DIR/deployment.notes" <<EOF
 370 ==================================================================
 371 Phew, all done (yay!). ONAP and DevStack might still be installing
 372 but here are the access details...
 373
 374 --------DEVSTACK ACCESS--------
 375 ssh -i ${PRIVATE_KEY} ${DEVSTACK_ADMIN_USER}@${DEVSTACK_PUBLIC_IP}
 376 horizon: http://${DEVSTACK_PUBLIC_IP}
 377 cli: kubectl exec $OPENSTACK_CLI_POD -- sh -lc "<openstack command>"
 378
 379 --------NFS ACCESS--------
 380 ssh -i ${PRIVATE_KEY} ${NFS_ADMIN_USER}@${NFS_PUBLIC_IP}
 381
 382 --------KUBERNETES ACCESS--------
 383 kubeconfig: export KUBECONFIG=$BUILD_DIR/kubeconfig
 384 dashboard: az aks browse --resource-group ${AKS_RESOURCE_GROUP_NAME} --name ${AKS_NAME}
 385
 386 --------BUILD DETAILS--------
 387 Build directory: $BUILD_DIR
 388 Integration repo: $BUILD_DIR/integration
 389 OOM repo: $BUILD_DIR/oom
 390
 391 --------ADD TO /etc/hosts--------
 392 $AKS_PUBLIC_IP_ADDRESS portal.api.simpledemo.onap.org
 393 $AKS_PUBLIC_IP_ADDRESS sdc.api.simpledemo.onap.org
 394 $AKS_PUBLIC_IP_ADDRESS sdc.api.fe.simpledemo.onap.org
 395 $AKS_PUBLIC_IP_ADDRESS sdc.api.be.simpledemo.onap.org
 396 $AKS_PUBLIC_IP_ADDRESS vid.api.simpledemo.onap.org
 397 $AKS_PUBLIC_IP_ADDRESS policy.api.simpledemo.onap.org
 398 $AKS_PUBLIC_IP_ADDRESS aai.api.simpledemo.onap.org
 399 $AKS_PUBLIC_IP_ADDRESS cli.api.simpledemo.onap.org
 400 $AKS_PUBLIC_IP_ADDRESS so.api.simpledemo.onap.org
 401 $AKS_PUBLIC_IP_ADDRESS so.monitoring.api.simpledemo.onap.org
 402 $AKS_PUBLIC_IP_ADDRESS so-monitoring
 403 $AKS_PUBLIC_IP_ADDRESS sdnc.api.simpledemo.onap.org
 404 $AKS_PUBLIC_IP_ADDRESS clamp.api.simpledemo.onap.org
 405 $AKS_PUBLIC_IP_ADDRESS dcae.api.simpledemo.onap.org
 406 $AKS_PUBLIC_IP_ADDRESS appc.api.simpledemo.onap.org
 407 $AKS_PUBLIC_IP_ADDRESS aaf.api.simpledemo.onap.org
 408 $AKS_PUBLIC_IP_ADDRESS portal-sdk.simpledemo.onap.org
 409 $AKS_PUBLIC_IP_ADDRESS robot.api.simpledemo.onap.org
 410 $AKS_PUBLIC_IP_ADDRESS msb.api.discovery.simpledemo.onap.org
 411 $AKS_PUBLIC_IP_ADDRESS msb.api.simpledemo.onap.org
 412 $AKS_PUBLIC_IP_ADDRESS aai.ui.simpledemo.onap.org
 413 $AKS_PUBLIC_IP_ADDRESS policy.api.simpledemo.onap.org
 414
 415 EOF
 416
 417 cat "$BUILD_DIR/deployment.notes"
 418
 419
 420 if [ $POST_INSTALL = 1 ]; then
 421
 422 echo "Executing post installation scripts..."
 423 sleep 3
 424
 425 cat > "$BUILD_DIR/onap.conf" <<EOF
 426 export CLLI=$CLLI
 427 export CLOUD_OWNER=$CLOUD_OWNER
 428 export CLOUD_REGION=$CLOUD_REGION
 429 export OPENSTACK_IP=$DEVSTACK_PRIVATE_IP
 430 export OPENSTACK_USER=$OPENSTACK_USER
 431 export OPENSTACK_PASS=$OPENSTACK_PASS
 432 export OPENSTACK_TENANT=$OPENSTACK_TENANT
 433 export OPENSTACK_REGION=$OPENSTACK_REGION
 434 export CUSTOMER=$CUSTOMER
 435 export SUBSCRIBER=$SUBSCRIBER
 436 export SERVICE_TYPE=$SERVICE_TYPE
 437 export AZ=$AZ
 438 export OE=$OE
 439 export LOB=$LOB
 440 export PLATFORM=$PLATFORM
 441 export PROJECT=$PROJECT
 442 export OS_ID=$OS_ID
 443 export OS_TENANT_ROLE=$OS_TENANT_ROLE
 444 export OS_KEYSTONE=$OS_KEYSTONE
 445 export KUBECONFIG=$BUILD_DIR/kubeconfig
 446 export NFS_PRIVATE_IP=$NFS_PRIVATE_IP
 447 export DEVSTACK_PRIVATE_IP=$DEVSTACK_PRIVATE_IP
 448 export PRIVATE_KEY=$PRIVATE_KEY
 449 EOF
 450
 451 $DIR/post_install.sh "$BUILD_DIR/onap.conf" "$DIR/cloud.conf"
 452
 453 fi
 454