gerrit.onap Code Review - appc.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * APPC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * Copyright (C) 2017 Amdocs
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  21  */
  22
  23 package org.openecomp.appc.adapter.iaas.provider.operation.impl.base;
  24
  25 import org.openecomp.appc.Constants;
  26 import org.openecomp.appc.adapter.iaas.impl.*;
  27 import org.openecomp.appc.i18n.Msg;
  28 import com.att.cdp.exceptions.ContextConnectionException;
  29 import com.att.cdp.exceptions.NotLoggedInException;
  30 import com.att.cdp.exceptions.TimeoutException;
  31 import com.att.cdp.exceptions.ZoneException;
  32 import com.att.cdp.pal.util.StringHelper;
  33 import com.att.cdp.zones.ComputeService;
  34 import com.att.cdp.zones.Context;
  35 import com.att.cdp.zones.ImageService;
  36 import com.att.cdp.zones.NetworkService;
  37 import com.att.cdp.zones.Provider;
  38 import com.att.cdp.zones.model.Hypervisor;
  39 import com.att.cdp.zones.model.Image;
  40 import com.att.cdp.zones.model.Network;
  41 import com.att.cdp.zones.model.Port;
  42 import com.att.cdp.zones.model.Server;
  43 import com.att.eelf.configuration.EELFLogger;
  44 import com.att.eelf.configuration.EELFManager;
  45 import com.att.eelf.i18n.EELFResourceManager;
  46 import org.glassfish.grizzly.http.util.HttpStatus;
  47 import java.util.ArrayList;
  48 import java.util.List;
  49
  50 /**
  51  * @since September 29, 2016
  52  */
  53 public abstract class ProviderServerOperation extends ProviderOperation{
  54
  55     private static final EELFLogger logger = EELFManager.getInstance().getLogger(ProviderServerOperation.class);
  56
  57     /**
  58      * Looks up the indicated server using the provided context and returns the server to the caller
  59      *
  60      * @param rc
  61      *            The request context
  62      * @param context
  63      *            The provider context
  64      * @param id
  65      *            The id of the server
  66      * @return The server, or null if there is a problem
  67      * @throws ZoneException
  68      *             If the server cannot be found
  69      * @throws RequestFailedException
  70      *             If the server cannot be found because we cant connect to the provider
  71      */
  72     @SuppressWarnings("nls")
  73     protected Server lookupServer(RequestContext rc, Context context, String id)
  74             throws ZoneException, RequestFailedException {
  75         ComputeService service = context.getComputeService();
  76         Server server = null;
  77         String msg;
  78         Provider provider = context.getProvider();
  79
  80         while (rc.attempt()) {
  81             try {
  82                 server = service.getServer(id);
  83                 break;
  84             } catch (ContextConnectionException e) {
  85                 msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
  86                         context.getTenant().getName(), context.getTenant().getId(), e.getMessage(),
  87                         Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
  88                         Integer.toString(rc.getRetryLimit()));
  89                 logger.error(msg, e);
  90                 rc.delay();
  91             }
  92         }
  93         if (rc.isFailed()) {
  94             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
  95             logger.error(msg);
  96             doFailure(rc, HttpStatus.BAD_GATEWAY_502, msg);
  97             throw new RequestFailedException("Lookup Server", msg, HttpStatus.BAD_GATEWAY_502, server);
  98         }
  99         return server;
 100     }
 101
 102
 103
 104     /**
 105      * Resume a suspended server and wait for it to enter a running state
 106      *
 107      * @param rc
 108      *            The request context that manages the state and recovery of the request for the life of its processing.
 109      * @param server
 110      *            The server to be resumed
 111      * @throws ZoneException
 112      * @throws RequestFailedException
 113      */
 114     @SuppressWarnings("nls")
 115     protected void resumeServer(RequestContext rc, Server server) throws ZoneException, RequestFailedException {
 116         logger.debug(Msg.RESUME_SERVER, server.getId());
 117
 118         Context context = server.getContext();
 119         String msg;
 120         Provider provider = context.getProvider();
 121         ComputeService service = context.getComputeService();
 122         while (rc.attempt()) {
 123             try {
 124                 server.resume();
 125                 break;
 126             } catch (ContextConnectionException e) {
 127                 msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
 128                         context.getTenant().getName(), context.getTenant().getId(), e.getMessage(),
 129                         Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
 130                         Integer.toString(rc.getRetryLimit()));
 131                 logger.error(msg, e);
 132                 rc.delay();
 133             }
 134         }
 135         if (rc.isFailed()) {
 136             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
 137             logger.error(msg);
 138             throw new RequestFailedException("Resume Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 139         }
 140         rc.reset();
 141         waitForStateChange(rc, server, Server.Status.RUNNING);
 142     }
 143
 144
 145     protected boolean hasImageAccess(@SuppressWarnings("unused") RequestContext rc, Context context) {
 146         logger.info("Checking permissions for image service.");
 147         try {
 148             ImageService service = context.getImageService();
 149             service.getImageByName("CHECK_IMAGE_ACCESS");
 150             logger.info("Image service is accessible.");
 151             return true;
 152         } catch (ZoneException e) {
 153             logger.warn("Image service could not be accessed. Some operations may fail.", e);
 154             return false;
 155         }
 156     }
 157
 158
 159     /**
 160      * Enter a pool-wait loop checking the server state to see if it has entered one of the desired states or not.
 161      * <p>
 162      * This method checks the state of the server periodically for one of the desired states. When the server enters one
 163      * of the desired states, the method returns a successful indication (true). If the server never enters one of the
 164      * desired states within the allocated timeout period, then the method returns a failed response (false). No
 165      * exceptions are thrown from this method.
 166      * </p>
 167      *
 168      * @param rc
 169      *            The request context that manages the state and recovery of the request for the life of its processing.
 170      * @param image
 171      *            The server to wait on
 172      * @param desiredStates
 173      *            A variable list of desired states, any one of which is allowed.
 174      * @throws RequestFailedException
 175      *             If the request times out or fails for some reason
 176      * @throws NotLoggedInException
 177      */
 178     @SuppressWarnings("nls")
 179     protected void waitForStateChange(RequestContext rc, Image image, Image.Status... desiredStates)
 180             throws RequestFailedException, NotLoggedInException {
 181         int pollInterval = configuration.getIntegerProperty(Constants.PROPERTY_OPENSTACK_POLL_INTERVAL);
 182         int timeout = configuration.getIntegerProperty(Constants.PROPERTY_SERVER_STATE_CHANGE_TIMEOUT);
 183         Context context = image.getContext();
 184         Provider provider = context.getProvider();
 185         ImageService service = context.getImageService();
 186         String msg;
 187
 188         long endTime = System.currentTimeMillis() + (timeout * 1000); //
 189
 190         while (rc.attempt()) {
 191             try {
 192                 try {
 193                     image.waitForStateChange(pollInterval, timeout, desiredStates);
 194                     break;
 195                 } catch (TimeoutException e) {
 196                     @SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
 197                     List<String> list = new ArrayList<>();
 198                     for (Image.Status desiredState : desiredStates) {
 199                         list.add(desiredState.name());
 200                     }
 201                     msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
 202                             context.getTenant().getName(), context.getTenant().getId(), e.getMessage(),
 203                             Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
 204                             Integer.toString(rc.getRetryLimit()));
 205                     logger.error(msg, e);
 206                     rc.delay();
 207                 }
 208             } catch (ZoneException e) {
 209                 List<String> list = new ArrayList<>();
 210                 for (Image.Status desiredState : desiredStates) {
 211                     list.add(desiredState.name());
 212                 }
 213                 String reason = EELFResourceManager.format(Msg.STATE_CHANGE_EXCEPTION, e.getClass().getSimpleName(),
 214                         "server", image.getName(), image.getId(), StringHelper.asList(list), image.getStatus().name(),
 215                         e.getMessage());
 216                 logger.error(reason);
 217                 logger.error(EELFResourceManager.format(e));
 218
 219                 // Instead of failing we are going to wait and try again.
 220                 // Timeout is reduced by delay time
 221                 logger.info(String.format("Retrying in %ds", rc.getRetryDelay()));
 222                 rc.delay();
 223                 timeout = (int) (endTime - System.currentTimeMillis()) / 1000;
 224                 // throw new RequestFailedException(e, operation, reason,
 225                 // HttpStatus.BAD_GATEWAY_502, server);
 226             }
 227         }
 228
 229         if (rc.isFailed()) {
 230             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
 231             logger.error(msg);
 232             throw new RequestFailedException("Waiting for State Change", msg, HttpStatus.BAD_GATEWAY_502, new Server());
 233         }
 234         rc.reset();
 235     }
 236
 237
 238     /**
 239      * Enter a pool-wait loop checking the server state to see if it has entered one of the desired states or not.
 240      * <p>
 241      * This method checks the state of the server periodically for one of the desired states. When the server enters one
 242      * of the desired states, the method returns a successful indication (true). If the server never enters one of the
 243      * desired states within the allocated timeout period, then the method returns a failed response (false). No
 244      * exceptions are thrown from this method.
 245      * </p>
 246      *
 247      * @param rc
 248      *            The request context that manages the state and recovery of the request for the life of its processing.
 249      * @param server
 250      *            The server to wait on
 251      * @param desiredStates
 252      *            A variable list of desired states, any one of which is allowed.
 253      * @throws RequestFailedException
 254      *             If the request times out or fails for some reason
 255      */
 256     @SuppressWarnings("nls")
 257     protected void waitForStateChange(RequestContext rc, Server server, Server.Status... desiredStates)
 258             throws RequestFailedException {
 259         int pollInterval = configuration.getIntegerProperty(Constants.PROPERTY_OPENSTACK_POLL_INTERVAL);
 260         int timeout = configuration.getIntegerProperty(Constants.PROPERTY_SERVER_STATE_CHANGE_TIMEOUT);
 261         Context context = server.getContext();
 262         Provider provider = context.getProvider();
 263         ComputeService service = context.getComputeService();
 264         String msg;
 265
 266         long endTime = System.currentTimeMillis() + (timeout * 1000); //
 267
 268         while (rc.attempt()) {
 269             try {
 270                 try {
 271                     server.waitForStateChange(pollInterval, timeout, desiredStates);
 272                     break;
 273                 } catch (TimeoutException e) {
 274                     @SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
 275                     List<String> list = new ArrayList<>();
 276                     for (Server.Status desiredState : desiredStates) {
 277                         list.add(desiredState.name());
 278                     }
 279                     msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
 280                             context.getTenant().getName(), context.getTenant().getId(), e.getMessage(),
 281                             Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
 282                             Integer.toString(rc.getRetryLimit()));
 283                     logger.error(msg, e);
 284                     rc.delay();
 285                 }
 286             } catch (ZoneException e) {
 287                 List<String> list = new ArrayList<>();
 288                 for (Server.Status desiredState : desiredStates) {
 289                     list.add(desiredState.name());
 290                 }
 291                 String reason = EELFResourceManager.format(Msg.STATE_CHANGE_EXCEPTION, e.getClass().getSimpleName(),
 292                         "server", server.getName(), server.getId(), StringHelper.asList(list), server.getStatus().name(),
 293                         e.getMessage());
 294                 logger.error(reason);
 295                 logger.error(EELFResourceManager.format(e));
 296
 297                 // Instead of failing we are going to wait and try again.
 298                 // Timeout is reduced by delay time
 299                 logger.info(String.format("Retrying in %ds", rc.getRetryDelay()));
 300                 rc.delay();
 301                 timeout = (int) (endTime - System.currentTimeMillis()) / 1000;
 302                 // throw new RequestFailedException(e, operation, reason,
 303                 // HttpStatus.BAD_GATEWAY_502, server);
 304             }
 305         }
 306
 307         if (rc.isFailed()) {
 308             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
 309             logger.error(msg);
 310             throw new RequestFailedException("Waiting for State Change", msg, HttpStatus.BAD_GATEWAY_502, server);
 311         }
 312         rc.reset();
 313     }
 314
 315     /**
 316      * Stop the specified server and wait for it to stop
 317      *
 318      * @param rc
 319      *            The request context that manages the state and recovery of the request for the life of its processing.
 320      * @param server
 321      *            The server to be stopped
 322      * @throws ZoneException
 323      * @throws RequestFailedException
 324      */
 325     @SuppressWarnings("nls")
 326     protected void stopServer(RequestContext rc, Server server) throws ZoneException, RequestFailedException {
 327         logger.debug(Msg.STOP_SERVER, server.getId());
 328
 329         String msg;
 330         Context context = server.getContext();
 331         Provider provider = context.getProvider();
 332         ComputeService service = context.getComputeService();
 333         while (rc.attempt()) {
 334             try {
 335                 server.stop();
 336                 break;
 337             } catch (ContextConnectionException e) {
 338                 msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
 339                         context.getTenant().getName(), context.getTenant().getId(), e.getMessage(),
 340                         Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
 341                         Integer.toString(rc.getRetryLimit()));
 342                 logger.error(msg, e);
 343                 rc.delay();
 344             }
 345         }
 346         if (rc.isFailed()) {
 347             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
 348             logger.error(msg);
 349             throw new RequestFailedException("Stop Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 350         }
 351         rc.reset();
 352         waitForStateChange(rc, server, Server.Status.READY, Server.Status.ERROR);
 353     }
 354
 355     /**
 356      * Start the server and wait for it to enter a running state
 357      *
 358      * @param rc
 359      *            The request context that manages the state and recovery of the request for the life of its processing.
 360      * @param server
 361      *            The server to be started
 362      * @throws ZoneException
 363      * @throws RequestFailedException
 364      */
 365     @SuppressWarnings("nls")
 366     protected void startServer(RequestContext rc, Server server) throws ZoneException, RequestFailedException {
 367         logger.debug(Msg.START_SERVER, server.getId());
 368         String msg;
 369         Context context = server.getContext();
 370         Provider provider = context.getProvider();
 371         ComputeService service = context.getComputeService();
 372         while (rc.attempt()) {
 373             try {
 374                 server.start();
 375                 break;
 376             } catch (ContextConnectionException e) {
 377                 msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
 378                         context.getTenant().getName(), context.getTenant().getId(), e.getMessage(),
 379                         Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
 380                         Integer.toString(rc.getRetryLimit()));
 381                 logger.error(msg, e);
 382                 rc.delay();
 383             }
 384         }
 385         if (rc.isFailed()) {
 386             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
 387             logger.error(msg);
 388             throw new RequestFailedException("Start Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 389         }
 390         rc.reset();
 391         waitForStateChange(rc, server, Server.Status.RUNNING);
 392     }
 393
 394
 395     /**
 396      * Un-Pause a paused server and wait for it to enter a running state
 397      *
 398      * @param rc
 399      *            The request context that manages the state and recovery of the request for the life of its processing.
 400      * @param server
 401      *            The server to be un-paused
 402      * @throws ZoneException
 403      * @throws RequestFailedException
 404      */
 405     @SuppressWarnings("nls")
 406     protected void unpauseServer(RequestContext rc, Server server) throws ZoneException, RequestFailedException {
 407         logger.debug(Msg.UNPAUSE_SERVER, server.getId());
 408
 409         String msg;
 410         Context context = server.getContext();
 411         Provider provider = context.getProvider();
 412         ComputeService service = context.getComputeService();
 413         while (rc.attempt()) {
 414             try {
 415                 server.unpause();
 416                 break;
 417             } catch (ContextConnectionException e) {
 418                 msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
 419                         context.getTenant().getName(), context.getTenant().getId(), e.getMessage(),
 420                         Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
 421                         Integer.toString(rc.getRetryLimit()));
 422                 logger.error(msg, e);
 423                 rc.delay();
 424             }
 425         }
 426         if (rc.isFailed()) {
 427             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
 428             logger.error(msg);
 429             throw new RequestFailedException("Unpause Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 430         }
 431         rc.reset();
 432         waitForStateChange(rc, server, Server.Status.RUNNING, Server.Status.READY);
 433     }
 434
 435
 436     /**
 437      * Generates the event indicating what happened
 438      *
 439      * @param rc
 440      *            The request context that manages the state and recovery of the request for the life of its processing.
 441      * @param success
 442      *            True if the event represents a successful outcome
 443      * @param msg
 444      *            The detailed message
 445      */
 446     protected void generateEvent(@SuppressWarnings("unused") RequestContext rc, @SuppressWarnings("unused") boolean success, @SuppressWarnings("unused") String msg) {
 447         // indication to the DG to generate the event?
 448     }
 449
 450     /**
 451      * Checks if the VM is connected to the Virtual Network and reachable
 452      *
 453      * @param rc
 454      *            The request context that manages the state and recovery of the request for the life of its processing.
 455      * @param server
 456      *             The server object representing the server we want to operate on
 457      * @param context
 458      *            The interface cloud service provider to access services or the object model, or both
 459
 460      */
 461         protected void checkVirtualMachineNetworkStatus(RequestContext rc, Server server, Context context)
 462                         throws ZoneException, RequestFailedException {
 463
 464                 logger.info("Performing the VM Server networking status checks...");
 465                 List<Port> ports = server.getPorts();
 466
 467                 NetworkService netSvc = context.getNetworkService();
 468
 469                 String msg;
 470                 for (Port port : ports) {
 471
 472                         switch (port.getPortState().toString().toUpperCase()) {
 473                         /**
 474                          * The port is connected, configured, and usable for communication
 475                          */
 476                         case "ONLINE":
 477                                 Network network = netSvc.getNetworkById(port.getNetwork());
 478                                 // Subnet subnet = netSvc.getSubnetById(port.getSubnetId());
 479                                 if (!network.getStatus().equals(Network.Status.ACTIVE.toString())) {
 480                                         msg = EELFResourceManager.format(Msg.SERVER_NETWORK_ERROR, server.getName(), port.getId());
 481                                         logger.error(msg);
 482                                         doFailure(rc, HttpStatus.PRECONDITION_FAILED_412, msg);
 483                                         throw new RequestFailedException("VM Server Network is DOWN", msg.toString(), HttpStatus.PRECONDITION_FAILED_412,
 484                                                         server);
 485                                 }
 486                                 break;
 487
 488                         /**
 489                          * The port is disconnected or powered-off and cannot be used for
 490                          * communication
 491                          */
 492                         case "OFFLINE":
 493                                 msg = EELFResourceManager.format(Msg.SERVER_NETWORK_ERROR, server.getName(), port.getId());
 494                                 logger.error(msg);
 495                                 doFailure(rc, HttpStatus.PRECONDITION_FAILED_412, msg);
 496                                 throw new RequestFailedException("VM Server Port status is OFFLINE", msg.toString(), HttpStatus.PRECONDITION_FAILED_412,
 497                                                 server);
 498
 499                         /**
 500                          * The port's status is changing because of some event or operation.
 501                          * The final state is yet to be determined.
 502                          */
 503                         case "PENDING":
 504                                 msg = EELFResourceManager.format(Msg.SERVER_NETWORK_ERROR, server.getName(), port.getId());
 505                                 logger.error(msg);
 506                                 doFailure(rc, HttpStatus.PRECONDITION_FAILED_412, msg);
 507                                 throw new RequestFailedException("VM Server Port status is PENDING", msg.toString(), HttpStatus.PRECONDITION_FAILED_412,
 508                                                 server);
 509
 510                         /**
 511                          * The port is in an unknown state and cannot be used.
 512                          */
 513                         case "UNKNOWN":
 514                                 msg = EELFResourceManager.format(Msg.SERVER_NETWORK_ERROR, server.getName(), port.getId());
 515                                 logger.error(msg);
 516                                 doFailure(rc, HttpStatus.PRECONDITION_FAILED_412, msg);
 517                                 throw new RequestFailedException("VM Server Port status is UNKNOWN", msg.toString(), HttpStatus.PRECONDITION_FAILED_412,
 518                                                 server);
 519                         }
 520
 521                 }
 522                 logger.info("Passed the VM Server the Hypervisor status checks..");
 523
 524         }
 525
 526         /**
 527      * Checks if the VM is connected to the Virtual Network and reachable
 528      *
 529      * @param server
 530      *             The server object representing the server we want to operate on
 531      */
 532         protected void checkHypervisor(Server server)
 533                         throws ZoneException, RequestFailedException {
 534
 535                 logger.info("Performing the Hypervisor status checks..");
 536                 String status = null, state = null, msg = null;
 537
 538                 status = server.getHypervisor().getStatus().toString();
 539                 state = server.getHypervisor().getState().toString();
 540
 541                 if (!status.equals(Hypervisor.Status.ENABLED.toString()) || !state.equals(Hypervisor.State.UP.toString())) {
 542                         msg = EELFResourceManager.format(Msg.HYPERVISOR_DOWN_ERROR, server.getHypervisor().getHostName(), server.getName());
 543                         logger.error(msg.toString());
 544
 545                         //doFailure(rc, HttpStatus.PRECONDITION_FAILED_412, msg);
 546                         throw new RequestFailedException("Hypervisor status DOWN or NOT ENABLED", msg.toString(), HttpStatus.PRECONDITION_FAILED_412,
 547                                         server);
 548
 549                 }
 550
 551                 logger.info("Passed the Hypervisor status checks..");
 552
 553         }
 554
 555         /**
 556      * Checks if a Host machine is reachable
 557      *
 558      * @param ipAddress
 559      *            IP Address of the Host Machine.
 560      * @param server
 561      *             The server object representing the Virtual Machine server
 562      * @return boolean
 563      *
 564      */
 565         /*private boolean isHostReachable(String ipAddress) throws IOException {
 566
 567                 InetAddress address = InetAddress.getByName(ipAddress);
 568
 569                 return address.isReachable(15000);
 570
 571
 572         }*/
 573
 574 }