# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
-from .celery import app as celery_app
-
-__all__ = ['celery_app']
+# from .celery import app as celery_app
+#
+# __all__ = ['celery_app']
+++ /dev/null
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import, unicode_literals
-import os
-from celery import Celery
-import logging
-
-# set the default Django settings module for the 'celery' program.
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ocata.settings')
-
-app = Celery('ocata')
-
-# Using a string here means the worker doesn't have to serialize
-# the configuration object to child processes.
-# - namespace='CELERY' means all celery-related configuration keys
-# should have a `CELERY_` prefix.
-app.config_from_object('django.conf:settings', namespace='CELERY')
-
-# Load task modules from all registered Django app configs.
-app.autodiscover_tasks()
-
-logger = logging.getLogger(__name__)
-
-
-@app.task(bind=True)
-def debug_task(self):
- logger.debug("self.request")
from newton_base.openoapi import tenants
from ocata.resource.views import capacity
from ocata.resource.views import events
-from ocata.vesagent import vesagent_ctrl
from ocata.resource.views import infra_workload
urlpatterns = [
# events
url(r'^api/multicloud-ocata/v0/(?P<vimid>[0-9a-zA-Z_-]+)/events_check/?$',
events.EventsCheck.as_view()),
- url(r'^api/multicloud-ocata/v0/(?P<vimid>[0-9a-zA-Z_-]+)/vesagent/?$',
- vesagent_ctrl.VesAgentCtrl.as_view()),
# API upgrading
url(r'^api/multicloud-ocata/v1/(?P<cloud_owner>[0-9a-zA-Z_-]+)/(?P<cloud_region_id>[0-9a-zA-Z_-]+)/registry$',
'(?P<tenantid>[0-9a-zA-Z_-]{20,})/', include('ocata.requests.urlsV1')),
url(r'^api/multicloud-ocata/v1/(?P<cloud_owner>[0-9a-zA-Z_-]+)/(?P<cloud_region_id>[0-9a-zA-Z_-]+)/capacity_check/?$',
capacity.APIv1CapacityCheck.as_view()),
- url(r'^api/multicloud-ocata/v1/(?P<cloud_owner>[0-9a-zA-Z_-]+)/(?P<cloud_region_id>[0-9a-zA-Z_-]+)/vesagent/?$',
- vesagent_ctrl.APIv1VesAgentCtrl.as_view()),
url(r'^api/multicloud-ocata/v1/(?P<cloud_owner>[0-9a-zA-Z_-]+)/(?P<cloud_region_id>[0-9a-zA-Z_-]+)/infra_workload/?$',
infra_workload.APIv1InfraWorkload.as_view()),
url(r'^api/multicloud-ocata/v1/(?P<cloud_owner>[0-9a-zA-Z_-]+)/(?P<cloud_region_id>[0-9a-zA-Z_-]+)/infra_workload/(?P<requri>[0-9a-zA-Z_-]*)/?$',
+++ /dev/null
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+++ /dev/null
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+++ /dev/null
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-import json
-import uuid
-import time
-import datetime
-
-from django.conf import settings
-from ocata.vesagent.vespublish import publishAnyEventToVES
-from common.utils import restcall
-
-logger = logging.getLogger(__name__)
-
-
-def get_epoch_now_usecond():
- '''
- get epoch timestamp of this moment in usecond
- :return:
- '''
- now_time = datetime.datetime.now()
- epoch_time_sec = time.mktime(now_time.timetuple())
- return int(epoch_time_sec * 1e6 + now_time.microsecond)
-
-
-# build backlog with domain:"fault", type:"vm"
-def buildBacklog_fault_vm(vimid, backlog_input):
-
- logger.info("vimid: %s" % vimid)
- logger.debug("with input: %s" % backlog_input)
-
- try:
-
- # must resolve the tenant id and server id while building the backlog
- tenant_id = backlog_input.get("tenantid", None)
- server_id = backlog_input.get("sourceid", None)
- server_name = backlog_input.get("source", None)
-
- # should resolve the name to id later
- if tenant_id is None:
- tenant_name = backlog_input["tenant"]
-
- # get token
- # resolve tenant_name to tenant_id
- auth_api_url_format = "/{f_vim_id}/identity/v2.0/tokens"
- auth_api_url = auth_api_url_format.format(f_vim_id=vimid)
- auth_api_data = {"auth": {"tenantName": tenant_name}}
- base_url = settings.MULTICLOUD_PREFIX
- extra_headers = ''
- ret = restcall._call_req(base_url, "", "", 0, auth_api_url, "POST", extra_headers, json.dumps(auth_api_data))
- if ret[0] > 0 or ret[1] is None:
- logger.critical("call url %s failed with status %s" % (auth_api_url, ret[0]))
- return None
-
- token_resp = json.JSONDecoder().decode(ret[1])
- token = token_resp["access"]["token"]["id"]
- tenant_id = token_resp["access"]["token"]["tenant"]["id"]
-
- if server_id is None and server_name:
- # resolve server_name to server_id in case no wildcast in server_name
- vserver_api_url_format \
- = "/{f_vim_id}/compute/v2.1/{f_tenant_id}/servers?name={f_server_name}"
- vserver_api_url = vserver_api_url_format.format(f_vim_id=vimid,
- f_tenant_id=tenant_id,
- f_server_name=server_name)
- base_url = settings.MULTICLOUD_PREFIX
- extra_headers = {'X-Auth-Token': token}
- ret = restcall._call_req(base_url, "", "", 0, vserver_api_url, "GET", extra_headers, "")
- if ret[0] > 0 or ret[1] is None:
- logger.critical("call url %s failed with status %s" % (vserver_api_url, ret[0]))
- return None
-
- server_resp = json.JSONDecoder().decode(ret[1])
- # find out the server wanted
- for s in server_resp.get("servers", []):
- if s["name"] == server_name:
- server_id = s["id"]
- break
- if server_id is None:
- logger.warn("source %s cannot be found under tenant id %s "
- % (server_name, tenant_id))
- return None
-
- # m.c. proxied OpenStack API
- if server_id is None and server_name is None:
- # monitor all VMs of the specified VIMs since no server_id can be resolved
- api_url_fmt = "/{f_vim_id}/compute/v2.1/{f_tenant_id}/servers/detail"
- api_url = api_url_fmt.format(
- f_vim_id=vimid, f_tenant_id=tenant_id)
- else:
- api_url_fmt = "/{f_vim_id}/compute/v2.1/{f_tenant_id}/servers/{f_server_id}"
- api_url = api_url_fmt.format(f_vim_id=vimid, f_tenant_id=tenant_id, f_server_id=server_id)
-
- backlog = {
- "backlog_uuid": str(uuid.uuid3(uuid.NAMESPACE_URL,
- str("%s-%s-%s" % (vimid, tenant_id, server_id)))),
- "tenant_id": tenant_id,
- "server_id": server_id,
- "api_method": "GET",
- "api_link": api_url,
- }
- backlog.update(backlog_input)
- except Exception as e:
- logger.error("exception:%s" % str(e))
- return None
-
- logger.info("return")
- logger.debug("with backlog: %s" % backlog)
- return backlog
-
-
-# process backlog with domain:"fault", type:"vm"
-def processBacklog_fault_vm(vesAgentConfig, vesAgentState, oneBacklog):
- logger.debug("vesAgentConfig:%s, vesAgentState:%s, oneBacklog: %s"
- % (vesAgentConfig, vesAgentState, oneBacklog))
-
- try:
- vimid = vesAgentConfig["vimid"]
- tenant_name = oneBacklog["tenant"]
-
- # get token
- auth_api_url_format = "/{f_vim_id}/identity/v2.0/tokens"
- auth_api_url = auth_api_url_format.format(f_vim_id=vimid)
- auth_api_data = {"auth": {"tenantName": tenant_name}}
- base_url = settings.MULTICLOUD_PREFIX
- extra_headers = ''
- logger.debug("authenticate with url:%s" % auth_api_url)
- ret = restcall._call_req(base_url, "", "", 0, auth_api_url, "POST", extra_headers, json.dumps(auth_api_data))
- if ret[0] > 0 or ret[1] is None:
- logger.critical("call url %s failed with status %s" % (auth_api_url, ret[0]))
-
- token_resp = json.JSONDecoder().decode(ret[1])
- logger.debug("authenticate resp: %s" % token_resp)
- token = token_resp["access"]["token"]["id"]
-
- # collect data by issue API
- api_link = oneBacklog["api_link"]
- method = oneBacklog["api_method"]
- base_url = settings.MULTICLOUD_PREFIX
- data = ''
- extra_headers = {'X-Auth-Token': token}
- # which one is correct? extra_headers = {'HTTP_X_AUTH_TOKEN': token}
- logger.debug("authenticate with url:%s, header:%s" % (auth_api_url, extra_headers))
- ret = restcall._call_req(base_url, "", "", 0, api_link, method, extra_headers, data)
- if ret[0] > 0 or ret[1] is None:
- logger.critical("call url %s failed with status %s" % (api_link, ret[0]))
-
- server_resp = json.JSONDecoder().decode(ret[1])
- logger.debug("collected data: %s" % server_resp)
-
- # encode data
- backlog_uuid = oneBacklog.get("backlog_uuid", None)
- backlogState = vesAgentState.get("%s" % (backlog_uuid), None)
-
- # iterate all VMs
- all_events = []
- server_1 = server_resp.get("server", None) # in case querying single server
- for s in server_resp.get("servers", [server_1] if server_1 else []):
- server_id = s.get("id", None)
- server_name = s.get("name", None)
- if not server_id:
- continue
-
- last_event = backlogState.get("last_event_%s" % (server_id), None)
- logger.debug("last event for server name %s: %s" % (server_name, last_event))
-
- this_event = data2event_fault_vm(vimid, oneBacklog, last_event, s)
- if this_event is not None:
- logger.debug("this event: %s" % this_event)
- all_events.append(this_event.get("event", None))
- backlogState["last_event_%s" % (server_id)] = this_event
-
- # report data to VES
- if len(all_events) > 0:
- ves_subscription = vesAgentConfig.get("subscription", None)
- publishAnyEventToVES(ves_subscription, all_events)
- # store the latest data into cache, never expire
-
- except Exception as e:
- logger.error("exception:%s" % str(e))
- return
-
- logger.info("return")
- return
-
-
-def data2event_fault_vm(vimid, oneBacklog, last_event, vm_data):
-
- VES_EVENT_VERSION = 3.0
- VES_EVENT_FAULT_VERSION = 2.0
- VES_EVENT_FAULT_DOMAIN = "fault"
-
- try:
-
- if vm_status_is_fault(vm_data["status"]):
- if last_event is not None \
- and last_event['event']['commonEventHeader']['eventName'] == 'Fault_MultiCloud_VMFailure':
- # asserted alarm already, so no need to assert it again
- return None
-
- eventName = "Fault_MultiCloud_VMFailure"
- priority = "High"
- eventSeverity = "CRITICAL"
- alarmCondition = "Guest_Os_Failure"
- specificProblem = "Fault_MultiCloud_VMFailure"
- eventType = ''
- reportingEntityId = vimid
- reportingEntityName = vimid
- sequence = 0
-
- startEpochMicrosec = get_epoch_now_usecond()
- lastEpochMicrosec = startEpochMicrosec
-
- eventId = str(uuid.uuid4())
- pass
- else:
- if last_event is None \
- or last_event['event']['commonEventHeader']['eventName'] != 'Fault_MultiCloud_VMFailure':
- # not assert alarm yet, so no need to clear it
- return None
-
- eventName = "Fault_MultiCloud_VMFailureCleared"
- priority = "Normal"
- eventSeverity = "NORMAL"
- alarmCondition = "Vm_Restart"
- specificProblem = "Fault_MultiCloud_VMFailure"
- eventType = ''
- reportingEntityId = vimid
- reportingEntityName = vimid
- sequence = 1 # last_event['event']['commonEventHeader']['sequence'] + 1
-
- startEpochMicrosec = last_event['event']['commonEventHeader']['startEpochMicrosec']
- lastEpochMicrosec = get_epoch_now_usecond()
- eventId = last_event['event']['commonEventHeader']['eventId']
-
- pass
-
- # now populate the event structure
- this_event = {
- 'event': {
- 'commonEventHeader': {
- 'version': VES_EVENT_VERSION,
- 'eventName': eventName,
- 'domain': VES_EVENT_FAULT_DOMAIN,
- 'eventId': eventId,
- 'eventType': eventType,
- 'sourceId': vm_data['id'],
- 'sourceName': vm_data['name'],
- 'reportingEntityId': reportingEntityId,
- 'reportingEntityName': reportingEntityName,
- 'priority': priority,
- 'startEpochMicrosec': startEpochMicrosec,
- 'lastEpochMicrosec': lastEpochMicrosec,
- 'sequence': sequence
- },
- 'faultFields': {
- 'faultFieldsVersion': VES_EVENT_FAULT_VERSION,
- 'eventSeverity': eventSeverity,
- 'eventSourceType': 'virtualMachine',
- 'alarmCondition': alarmCondition,
- 'specificProblem': specificProblem,
- 'vfStatus': 'Active',
- "alarmInterfaceA": "aaaa",
- "alarmAdditionalInformation": [
- {
- "name": "objectType",
- "value": "VIM"
- },
- {
- "name": "eventTime",
- "value": str(datetime.datetime.now())
- }
- ],
- }
-
- }
-
- }
-
- return this_event
-
- except Exception as e:
- logger.error("exception:%s" % str(e))
- return None
-
-
-def vm_status_is_fault(status):
- '''
- report VM fault when status falls into one of following state
- ['ERROR', 'DELETED', 'PAUSED', 'REBUILD', 'RESCUE',
- 'RESIZE','REVERT_RESIZE', 'SHELVED', 'SHELVED_OFFLOADED',
- 'SHUTOFF', 'SOFT_DELETED','SUSPENDED', 'UNKNOWN', 'VERIFY_RESIZE']
- :param status:
- :return:
- '''
- if status in ['BUILD', 'ACTIVE', 'HARD_REBOOT', 'REBOOT', 'MIGRATING', 'PASSWORD']:
- return False
- else:
- return True
+++ /dev/null
-# Copyright (c) Intel Corporation, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import mock
-
-import unittest
-import json
-
-from ocata.vesagent import vespublish
-from common.utils import restcall
-from ocata.vesagent.event_domain import fault_vm
-
-MOCK_TOKEN_RESPONSE = {
- "access": {
- "token": {
- "issued_at": "2018-05-10T16:56:56.000000Z",
- "expires": "2018-05-10T17:56:56.000000Z",
- "id": "4a832860dd744306b3f66452933f939e",
- "tenant": {
- "domain": {"id": "default", "name": "Default"},
- "enabled": "true",
- "id": "0e148b76ee8c42f78d37013bf6b7b1ae",
- "name": "VIM"
- }
- },
- "serviceCatalog": [],
- "user": {
- "domain": {"id": "default", "name": "Default"},
- "id": "ba76c94eb5e94bb7bec6980e5507aae2",
- "name": "demo"
- }
- }
-}
-MOCK_SERVERS_GET_RESPONSE = {
- "servers": [
- {
- "id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "links": [
- {
- "href": "http://10.12.25.2:8774/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "rel": "self"
- },
- {
- "href": "http://10.12.25.2:8774/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "rel": "bookmark"
- }
- ],
- "name": "onap-aaf"
- }
- ]
-}
-MOCK_BACKLOG_INPUT = {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae", "api_method": "GET",
- "source": "onap-aaf",
- "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "domain": "fault", "type": "vm", "tenant": "VIM"
-}
-MOCK_BACKLOG_INPUT_wo_tenant_id = {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "source": "onap-aaf",
- "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "domain": "fault", "type": "vm", "tenant": "VIM"
-}
-MOCK_BACKLOG_INPUT_wo_tenant = {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "source": "onap-aaf",
- "domain": "fault", "type": "vm"
-}
-MOCK_BACKLOG_INPUT_wo_server_id = {
- "source": "onap-aaf",
- "domain": "fault",
- "type": "vm",
- "tenant": "VIM"
-}
-MOCK_BACKLOG_INPUT_wo_server = {"domain": "fault", "type": "vm", "tenant": "VIM"}
-
-MOCK_SERVER_GET_RESPONSE = {
- "server": {
- "wrs-res:topology": "node:0, 4096MB, pgsize:2M, vcpus:0,1, pol:sha",
- "OS-EXT-STS:task_state": None,
- "addresses": {
- "oam_onap_BTHY": [
- {
- "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:6c:0d:6b",
- "version": 4,
- "addr": "10.0.13.1",
- "OS-EXT-IPS:type": "fixed"
- },
- {
- "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:6c:0d:6b",
- "version": 4,
- "addr": "10.12.5.185",
- "OS-EXT-IPS:type": "floating"
- }
- ]
- },
- "links": [],
- "image": {"id": "6e219e86-cd94-4989-9119-def29aa10b12", "links": []},
- "wrs-if:nics": [],
- "wrs-sg:server_group": "",
- "OS-EXT-STS:vm_state": "active",
- "OS-SRV-USG:launched_at": "2018-04-26T08:01:28.000000",
- "flavor": {},
- "id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "security_groups": [{"name": "onap_sg_BTHY"}],
- "user_id": "ba76c94eb5e94bb7bec6980e5507aae2",
- "OS-DCF:diskConfig": "MANUAL",
- "accessIPv4": "",
- "accessIPv6": "",
- "progress": 0,
- "OS-EXT-STS:power_state": 1,
- "OS-EXT-AZ:availability_zone": "nova",
- "metadata": {},
- "status": "ACTIVE",
- "updated": "2018-04-26T08:01:28Z",
- "hostId": "17acc9f2ae4f618c314e4cdf0c206585b895bc72a9ec57e57b254133",
- "OS-SRV-USG:terminated_at": None,
- "wrs-res:pci_devices": "",
- "wrs-res:vcpus": [2, 2, 2],
- "key_name": "onap_key_BTHY",
- "name": "onap-aaf",
- "created": "2018-04-26T08:01:20Z",
- "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae",
- "os-extended-volumes:volumes_attached": [],
- "config_drive": ""
- }
-}
-
-MOCK_SERVER_GET_RESPONSE_empty = {}
-
-MOCK_vesAgentConfig = {
- "backlogs": [
- {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae", "api_method": "GET",
- "source": "onap-aaf",
- "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "domain": "fault",
- "type": "vm",
- "tenant": "VIM"
- }
- ],
- "poll_interval_default": 10, "vimid": "windriver-hudson-dc_RegionOne",
- "ves_subscription": {
- "username": "user", "password": "password",
- "endpoint": "http://127.0.0.1:9005/sample"
- }
-}
-
-MOCK_vesAgentState = {
- "ce2d7597-22e1-4239-890f-bc303bd67076": {"timestamp": 1525975400}
-}
-MOCK_oneBacklog = {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae",
- "api_method": "GET",
- "source": "onap-aaf",
- "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "domain": "fault",
- "type": "vm",
- "tenant": "VIM"
-}
-
-
-class FaultVMTest(unittest.TestCase):
- def setUp(self):
- pass
-
- def tearDown(self):
- pass
-
- def test_get_epoch_now_usecond(self):
- epoch = fault_vm.get_epoch_now_usecond()
- self.assertGreater(epoch, 1)
- pass
-
- @mock.patch.object(restcall, '_call_req')
- def test_buildBacklog_fault_vm(self, mock_call_req):
-
- mock_call_req.side_effect = [
- (0, json.dumps(MOCK_TOKEN_RESPONSE), "MOCKED response body"),
- (0, json.dumps(MOCK_SERVERS_GET_RESPONSE), "MOCKED response body")
- ]
- backlog = fault_vm.buildBacklog_fault_vm(vimid="windriver-hudson-dc_RegionOne",
- backlog_input=MOCK_BACKLOG_INPUT)
- self.assertIsNotNone(backlog)
- pass
-
- @mock.patch.object(restcall, '_call_req')
- def test_buildBacklog_fault_vm_wo_tenant_id(self, mock_call_req):
-
- mock_call_req.side_effect = [
- (0, json.dumps(MOCK_TOKEN_RESPONSE), "MOCKED response body"),
- (0, json.dumps(MOCK_SERVERS_GET_RESPONSE), "MOCKED response body")
- ]
- backlog = fault_vm.buildBacklog_fault_vm(vimid="windriver-hudson-dc_RegionOne",
- backlog_input=MOCK_BACKLOG_INPUT_wo_tenant_id)
- self.assertIsNotNone(backlog)
- pass
-
- @mock.patch.object(restcall, '_call_req')
- def test_buildBacklog_fault_vm_wo_tenant(self, mock_call_req):
-
- mock_call_req.side_effect = [
- (1, json.dumps(MOCK_TOKEN_RESPONSE), "MOCKED response body: failed"),
- (0, json.dumps(MOCK_SERVERS_GET_RESPONSE), "MOCKED response body")
- ]
- backlog = fault_vm.buildBacklog_fault_vm(vimid="windriver-hudson-dc_RegionOne",
- backlog_input=MOCK_BACKLOG_INPUT_wo_tenant)
- self.assertIsNone(backlog)
- pass
-
- @mock.patch.object(restcall, '_call_req')
- def test_buildBacklog_fault_vm_wo_server_id(self, mock_call_req):
-
- mock_call_req.side_effect = [
- (0, json.dumps(MOCK_TOKEN_RESPONSE), "MOCKED response body"),
- (0, json.dumps(MOCK_SERVERS_GET_RESPONSE), "MOCKED response body")
- ]
- backlog = fault_vm.buildBacklog_fault_vm(vimid="windriver-hudson-dc_RegionOne",
- backlog_input=MOCK_BACKLOG_INPUT_wo_server_id)
- self.assertIsNotNone(backlog)
- pass
-
- @mock.patch.object(restcall, '_call_req')
- def test_buildBacklog_fault_vm_wo_server(self, mock_call_req):
-
- mock_call_req.side_effect = [
- (0, json.dumps(MOCK_TOKEN_RESPONSE), "MOCKED response body"),
- (0, json.dumps(MOCK_SERVERS_GET_RESPONSE), "MOCKED response body")
- ]
- backlog = fault_vm.buildBacklog_fault_vm(vimid="windriver-hudson-dc_RegionOne",
- backlog_input=MOCK_BACKLOG_INPUT_wo_server)
- self.assertIsNotNone(backlog)
- pass
-
- @mock.patch.object(vespublish, 'publishAnyEventToVES')
- @mock.patch.object(restcall, '_call_req')
- def test_processBacklog_fault_vm(self, mock_call_req, mock_publishAnyEventToVES):
-
- mock_call_req.side_effect = [
- (0, json.dumps(MOCK_TOKEN_RESPONSE), "MOCKED response body"),
- (0, json.dumps(MOCK_SERVER_GET_RESPONSE), "MOCKED response body")
- ]
- mock_publishAnyEventToVES.return_value = "mocked return value"
-
- result = fault_vm.processBacklog_fault_vm(vesAgentConfig=MOCK_vesAgentConfig,
- vesAgentState=MOCK_vesAgentState,
- oneBacklog=MOCK_oneBacklog)
- self.assertIsNone(result)
- pass
-
- @mock.patch.object(vespublish, 'publishAnyEventToVES')
- @mock.patch.object(restcall, '_call_req')
- def test_processBacklog_fault_vm_wo_server(self, mock_call_req, mock_publishAnyEventToVES):
-
- mock_call_req.side_effect = [
- (0, json.dumps(MOCK_TOKEN_RESPONSE), "MOCKED response body"),
- (0, json.dumps(MOCK_SERVER_GET_RESPONSE_empty), "MOCKED response body")
- ]
- mock_publishAnyEventToVES.return_value = "mocked return value"
-
- result = fault_vm.processBacklog_fault_vm(vesAgentConfig=MOCK_vesAgentConfig,
- vesAgentState=MOCK_vesAgentState,
- oneBacklog=MOCK_oneBacklog)
- self.assertIsNone(result)
- pass
+++ /dev/null
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# VES agent workers
-from __future__ import absolute_import, unicode_literals
-from ocata.celery import app
-import logging
-import json
-import time
-
-from django.core.cache import cache
-
-from ocata.vesagent.event_domain.fault_vm import processBacklog_fault_vm
-
-logger = logging.getLogger(__name__)
-
-
-@app.task(bind=True)
-def scheduleBacklogs(self, vimid):
- # make sure only one task runs here
- # cannot get vimid ? logger.info("schedule with vimid:%" % (vimid))
-
- logger.debug("scheduleBacklogs starts")
- backlog_count, next_time_slot = processBacklogs()
- logger.debug("processBacklogs return with %s, %s" % (backlog_count, next_time_slot))
-
- # sleep for next_time_slot
- while backlog_count > 0:
- time.sleep(next_time_slot)
- backlog_count, next_time_slot = processBacklogs()
-
- logger.debug("scheduleBacklogs stops")
-
-
-def processBacklogs():
- # find out count of valid backlog and the next time slot
- backlog_count = 0
- next_time_slot = 10
- try:
- # get the whole list of backlog
- VesAgentBacklogsVimListStr = cache.get("VesAgentBacklogs.vimlist")
- if VesAgentBacklogsVimListStr is None:
- logger.warn("VesAgentBacklogs.vimlist cannot be found in cache")
- return 0, next_time_slot
-
- logger.debug("VesAgentBacklogs.vimlist: %s" % (VesAgentBacklogsVimListStr))
-
- backlogsAllVims = json.loads(VesAgentBacklogsVimListStr)
- if backlogsAllVims is None:
- logger.warn("VesAgentBacklogs.vimlist is empty")
- return 0, next_time_slot
-
- for vimid in backlogsAllVims:
- # iterate each backlogs
- backlog_count_tmp, next_time_slot_tmp = processBacklogsOfOneVIM(vimid)
- logger.debug("vimid:%s, backlog_count,next_time_slot:%s,%s" % (vimid, backlog_count_tmp, next_time_slot_tmp))
- backlog_count += backlog_count_tmp
- next_time_slot = next_time_slot_tmp if next_time_slot > next_time_slot_tmp else next_time_slot
- pass
-
- except Exception as e:
- logger.error("exception:%s" % str(e))
-
- return backlog_count, next_time_slot
-
- pass
-
-
-def processBacklogsOfOneVIM(vimid):
- '''
- process all backlogs for a VIM, return count of valid backlogs
- :param vimid:
- :return:
- '''
- backlog_count = 0
- next_time_slot = 10
-
- try:
- vesAgentConfigStr = cache.get("VesAgentBacklogs.config.%s" % (vimid))
- if vesAgentConfigStr is None:
- logger.warn("VesAgentBacklogs.config.%s cannot be found in cache" % (vimid))
- return 0, next_time_slot
-
- logger.debug("VesAgentBacklogs.config.%s: %s" % (vimid, vesAgentConfigStr))
-
- vesAgentConfig = json.loads(vesAgentConfigStr)
- if vesAgentConfig is None:
- logger.warn("VesAgentBacklogs.config.%s corrupts" % (vimid))
- return 0, next_time_slot
-
- vesAgentStateStr = cache.get("VesAgentBacklogs.state.%s" % (vimid))
- vesAgentState = json.loads(vesAgentStateStr) if vesAgentStateStr is not None else {}
-
- ves_info = vesAgentConfig.get("subscription", None)
- if ves_info is None:
- logger.warn("VesAgentBacklogs.config.%s: ves subscription corrupts:%s" % (vimid, vesAgentConfigStr))
- return 0, next_time_slot
-
- poll_interval_default = vesAgentConfig.get("poll_interval_default", None)
- if poll_interval_default is None:
- logger.warn("VesAgentBacklogs.config.%s: poll_interval_default corrupts:%s" % (vimid, vesAgentConfigStr))
- return 0, next_time_slot
-
- if poll_interval_default == 0:
- # invalid interval value
- logger.warn("VesAgentBacklogs.config.%s: poll_interval_default invalid:%s" % (vimid, vesAgentConfigStr))
- return 0, next_time_slot
-
- backlogs_list = vesAgentConfig.get("backlogs", None)
- if backlogs_list is None:
- logger.warn("VesAgentBacklogs.config.%s: backlogs corrupts:%s" % (vimid, vesAgentConfigStr))
- return 0, next_time_slot
-
- for backlog in backlogs_list:
- backlog_count_tmp, next_time_slot_tmp = processOneBacklog(vesAgentConfig,
- vesAgentState,
- poll_interval_default,
- backlog)
- logger.debug("processOneBacklog return with %s,%s" % (backlog_count_tmp, next_time_slot_tmp))
- backlog_count += backlog_count_tmp
- next_time_slot = next_time_slot_tmp if next_time_slot > next_time_slot_tmp else next_time_slot
-
- pass
-
- # save back the updated backlogs state
- vesAgentStateStr = json.dumps(vesAgentState)
- cache.set("VesAgentBacklogs.state.%s" % vimid, vesAgentStateStr, None)
-
- except Exception as e:
- logger.error("exception:%s" % str(e))
-
- return backlog_count, next_time_slot
-
-
-def processOneBacklog(vesAgentConfig, vesAgentState, poll_interval_default, oneBacklog):
- logger.info("Process one backlog")
- # logger.debug("vesAgentConfig:%s, vesAgentState:%s, poll_interval_default:%s, oneBacklog: %s"
- # % (vesAgentConfig, vesAgentState, poll_interval_default, oneBacklog))
-
- backlog_count = 1
- next_time_slot = 10
- try:
- timestamp_now = int(time.time())
- backlog_uuid = oneBacklog.get("backlog_uuid", None)
- if backlog_uuid is None:
- # warning: uuid is None, omit this backlog
- logger.warn("backlog without uuid: %s" % oneBacklog)
- return 0, next_time_slot
-
- backlogState = vesAgentState.get("%s" % (backlog_uuid), None)
- if backlogState is None:
- initialBacklogState = {
- "timestamp": timestamp_now
- }
- vesAgentState["%s" % (backlog_uuid)] = initialBacklogState
- backlogState = initialBacklogState
-
- time_expiration = backlogState["timestamp"] + oneBacklog.get("poll_interval", poll_interval_default)
- # check if poll interval expires
- if timestamp_now < time_expiration:
- # not expired yet
- logger.info("return without dispatching, not expired yet")
- return backlog_count, next_time_slot
-
- logger.info("Dispatching backlog")
-
- # collect data in case of expiration
- if oneBacklog["domain"] == "fault" and oneBacklog["type"] == "vm":
- processBacklog_fault_vm(vesAgentConfig, vesAgentState, oneBacklog)
- else:
- logger.warn("Dispatching backlog fails due to unsupported backlog domain %s,type:%s"
- % (oneBacklog["domain"], oneBacklog["type"]))
- backlog_count = 0
- pass
-
- # update timestamp and internal state
- backlogState["timestamp"] = timestamp_now
- except Exception as e:
- logger.error("exception:%s" % str(e))
-
- logger.info("return")
- return backlog_count, next_time_slot
+++ /dev/null
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import mock
-
-import unittest
-import json
-from django.test import Client
-from rest_framework import status
-
-from django.core.cache import cache
-from common.msapi import extsys
-from ocata.vesagent import vesagent_ctrl
-from ocata.vesagent.tasks import scheduleBacklogs
-
-
-MOCK_VIM_INFO = {
- "createTime": "2017-04-01 02:22:27",
- "domain": "Default",
- "name": "TiS_R4",
- "password": "admin",
- "tenant": "admin",
- "type": "openstack",
- "url": "http://128.224.180.14:5000/v3",
- "userName": "admin",
- "vendor": "WindRiver",
- "version": "newton",
- "vimId": "windriver-hudson-dc_RegionOne",
- 'cloud_owner': 'windriver-hudson-dc',
- 'cloud_region_id': 'RegionOne',
- 'cloud_extra_info': '{"vesagent_config":{"backlogs":[{"source":"onap-aaf","domain":"fault","type":"vm","tenant":"VIM"}],"poll_interval_default":10,"ves_subscription":{"username":"user","password":"password","endpoint":"http://127.0.0.1:9005/sample"}}}',
- 'insecure': 'True',
-}
-
-
-class VesAgentCtrlTest(unittest.TestCase):
- def setUp(self):
- self.client = Client()
- self.view = vesagent_ctrl.VesAgentCtrl()
-
- def tearDown(self):
- pass
-
- @mock.patch.object(cache, 'get')
- @mock.patch.object(extsys, 'get_vim_by_id')
- def test_get(self, mock_get_vim_by_id, mock_get):
- mock_get_vim_by_id.return_value = MOCK_VIM_INFO
- mock_get.return_value = '{"backlogs": [{"backlog_uuid": "2b8f6ff8-bc64-339b-a714-155909db937f", "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721", "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae", "api_method": "GET", "source": "onap-aaf", "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721", "domain": "fault", "type": "vm", "tenant": "VIM"}], "poll_interval_default": 10, "vimid": "onaplab_RegionOne", "subscription": {"username": "user", "password": "password", "endpoint": "http://127.0.0.1:9005/sample"}}'
-
- response = self.client.get("/api/multicloud-ocata/v0/windriver-hudson-dc_RegionOne/vesagent")
- self.assertEqual(status.HTTP_200_OK, response.status_code, response.content)
-
- @mock.patch.object(vesagent_ctrl.VesAgentCtrl, 'buildBacklogsOneVIM')
- @mock.patch.object(extsys, 'get_vim_by_id')
- def test_post(self, mock_get_vim_by_id, mock_buildBacklogsOneVIM):
- mock_get_vim_by_id.return_value = MOCK_VIM_INFO
- mock_buildBacklogsOneVIM.return_value = "mocked vesagent_backlogs"
- mock_request = mock.Mock()
- mock_request.META = {"testkey": "testvalue"}
- mock_request.data = {"testdatakey": "testdatavalue"}
-
- response = self.view.post(request=mock_request, vimid="windriver-hudson-dc_RegionOne")
- self.assertEquals(status.HTTP_201_CREATED, response.status_code)
-
- pass
-
- @mock.patch.object(vesagent_ctrl.VesAgentCtrl, 'clearBacklogsOneVIM')
- @mock.patch.object(extsys, 'get_vim_by_id')
- def test_delete(self, mock_get_vim_by_id, mock_clearBacklogsOneVIM):
- mock_get_vim_by_id.return_value = MOCK_VIM_INFO
- mock_clearBacklogsOneVIM.return_value = "mocked vesagent_backlogs"
- mock_request = mock.Mock()
- mock_request.META = {"testkey": "testvalue"}
-
- response = self.view.delete(request=mock_request, vimid="windriver-hudson-dc_RegionOne")
- self.assertEquals(status.HTTP_200_OK, response.status_code)
-
- pass
-
- @mock.patch.object(cache, 'get')
- def test_getBacklogsOneVIM(self, mock_get):
- mock_vesagent_config = {
- "backlogs": [
- {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae",
- "api_method": "GET",
- "source": "onap-aaf",
- "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "domain": "fault", "type": "vm", "tenant": "VIM"
- }
- ],
- "poll_interval_default": 10,
- "vimid": "onaplab_RegionOne",
- "subscription": {
- "username": "user",
- "password": "password",
- "endpoint": "http://127.0.0.1:9005/sample"
- }
- }
- mock_get.return_value = json.dumps(mock_vesagent_config)
-
- vesAgentConfig = self.view.getBacklogsOneVIM(vimid="windriver-hudson-dc_RegionOne")
- self.assertEquals(vesAgentConfig, mock_vesagent_config)
-
- pass
-
- @mock.patch.object(cache, 'set')
- @mock.patch.object(cache, 'get')
- def test_clearBacklogsOneVIM(self, mock_get, mock_set):
- mock_VesAgentBacklogs_vimlist = ["windriver-hudson-dc_RegionOne"]
- mock_vesagent_config = {
- "backlogs": [
- {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae",
- "api_method": "GET",
- "source": "onap-aaf",
- "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "domain": "fault", "type": "vm", "tenant": "VIM"
- }
- ],
- "poll_interval_default": 10, "vimid": "onaplab_RegionOne",
- "subscription": {
- "username": "user",
- "password": "password",
- "endpoint": "http://127.0.0.1:9005/sample"
- }
- }
-
- mock_get.side_effect = [
- json.dumps(mock_VesAgentBacklogs_vimlist),
- json.dumps(mock_vesagent_config)
- ]
-
- mock_set.return_value = "mocked cache set"
-
- result = self.view.clearBacklogsOneVIM(vimid="windriver-hudson-dc_RegionOne")
- self.assertEquals(0, result)
-
- pass
-
- @mock.patch.object(scheduleBacklogs, 'delay')
- @mock.patch.object(cache, 'set')
- @mock.patch.object(cache, 'get')
- def test_buildBacklogsOneVIM(self, mock_get, mock_set, mock_scheduleBacklogs_delay):
- mock_VesAgentBacklogs_vimlist = ["windriver-hudson-dc_RegionOne"]
- mock_vesagent_config = {
- "backlogs": [
- {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae",
- "api_method": "GET",
- "source": "onap-aaf",
- "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "domain": "fault", "type": "vm", "tenant": "VIM"
- }
- ],
- "poll_interval_default": 10,
- "vimid": "windriver-hudson-dc_RegionOne",
- "ves_subscription": {
- "username": "user",
- "password": "password",
- "endpoint": "http://127.0.0.1:9005/sample"
- }
- }
-
- mock_get.side_effect = [
- json.dumps(mock_VesAgentBacklogs_vimlist),
- ]
-
- mock_set.return_value = "mocked cache set"
- mock_scheduleBacklogs_delay.return_value = "mocked delay"
-
- VesAgentBacklogsConfig = self.view.buildBacklogsOneVIM(vimid="windriver-hudson-dc_RegionOne",
- vesagent_config=mock_vesagent_config)
- self.assertIsNotNone(VesAgentBacklogsConfig)
-
- pass
+++ /dev/null
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import mock
-
-import unittest
-import json
-from django.test import Client
-from django.core.cache import cache
-
-from common.msapi import extsys
-from ocata.vesagent import tasks
-from ocata.vesagent.event_domain import fault_vm
-
-
-MOCK_VIM_INFO = {
- "createTime": "2017-04-01 02:22:27",
- "domain": "Default",
- "name": "TiS_R4",
- "password": "admin",
- "tenant": "admin",
- "type": "openstack",
- "url": "http://128.224.180.14:5000/v3",
- "userName": "admin",
- "vendor": "WindRiver",
- "version": "newton",
- "vimId": "windriver-hudson-dc_RegionOne",
- 'cloud_owner': 'windriver-hudson-dc',
- 'cloud_region_id': 'RegionOne',
- 'cloud_extra_info': '{"vesagent_config":{"backlogs":[{"source":"onap-aaf","domain":"fault","type":"vm","tenant":"VIM"}],"poll_interval_default":10,"ves_subscription":{"username":"user","password":"password","endpoint":"http://127.0.0.1:9005/sample"}}}',
- 'insecure': 'True',
-}
-
-COUNT_TIME_SLOT1 = (1, 1)
-COUNT_TIME_SLOT2 = (0, 1)
-
-
-class VesTaskTest(unittest.TestCase):
- def setUp(self):
- self.client = Client()
-
- def tearDown(self):
- pass
-
- @mock.patch.object(tasks, 'processBacklogs')
- @mock.patch.object(extsys, 'get_vim_by_id')
- def test_tasks_scheduleBacklogs(self, mock_get_vim_by_id, mock_processBacklogs):
- mock_get_vim_by_id.return_value = MOCK_VIM_INFO
- mock_processBacklogs.side_effect = [
- COUNT_TIME_SLOT1,
- COUNT_TIME_SLOT2
- ]
- result = tasks.scheduleBacklogs(vimid="windriver-hudson-dc_RegionOne")
- self.assertEquals(None, result)
- pass
-
- @mock.patch.object(tasks, 'processBacklogsOfOneVIM')
- @mock.patch.object(cache, 'get')
- def test_tasks_processBacklogs(self, mock_cache_get, mock_tasks_processBacklogsOfOneVIM):
- mock_VesAgentBacklogs_vimlist = ["windriver-hudson-dc_RegionOne"]
- COUNT_TIME_SLOT_ONE_VIM = (1, 1)
- mock_tasks_processBacklogsOfOneVIM.return_value = COUNT_TIME_SLOT_ONE_VIM
- mock_cache_get.side_effect = [
- json.dumps(mock_VesAgentBacklogs_vimlist),
- ]
- result = tasks.processBacklogs()
- self.assertEquals(COUNT_TIME_SLOT_ONE_VIM, result)
- pass
-
- @mock.patch.object(tasks, 'processOneBacklog')
- @mock.patch.object(cache, 'set')
- @mock.patch.object(cache, 'get')
- def test_tasks_processBacklogsOfOneVIM(self, mock_cache_get, mock_cache_set, mock_tasks_processOneBacklog):
- mock_vesagent_config = {
- "backlogs": [
- {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae",
- "api_method": "GET",
- "source": "onap-aaf",
- "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "domain": "fault", "type": "vm", "tenant": "VIM"
- }
- ],
- "poll_interval_default": 10, "vimid": "onaplab_RegionOne",
- "subscription": {
- "username": "user",
- "password": "password",
- "endpoint": "http://127.0.0.1:9005/sample"
- }
- }
- mock_cache_get.side_effect = [
- json.dumps(mock_vesagent_config),
- json.dumps({})
- ]
- mock_tasks_processOneBacklog.return_value = (1, 11)
- mock_cache_set.return_value = "mocked cache set"
- result = tasks.processBacklogsOfOneVIM(vimid="windriver-hudson-dc_RegionOne")
- COUNT_TIME_SLOT = (1, 10)
- self.assertEquals(COUNT_TIME_SLOT, result)
- pass
-
- @mock.patch.object(fault_vm, 'processBacklog_fault_vm')
- def test_tasks_processOneBacklog(self, mock_fault_vm_processBacklog_fault_vm):
- mock_fault_vm_processBacklog_fault_vm.return_value = None
- vesagent_config = {
- "backlogs": [
- {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae",
- "api_method": "GET",
- "source": "onap-aaf",
- "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "domain": "fault", "type": "vm", "tenant": "VIM"
- }
- ],
- "poll_interval_default": 10, "vimid": "onaplab_RegionOne",
- "subscription": {
- "username": "user",
- "password": "password",
- "endpoint": "http://127.0.0.1:9005/sample"
- }
- }
- vesagent_onebacklog = {
- "backlog_uuid": "ce2d7597-22e1-4239-890f-bc303bd67076",
- "poll_interval": 10,
- "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721",
- "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae", "api_method": "GET",
- "source": "onap-aaf",
- "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721",
- "domain": "fault", "type": "vm", "tenant": "VIM"
- }
-
- result = tasks.processOneBacklog(vesAgentConfig=vesagent_config,
- vesAgentState={},
- poll_interval_default=10,
- oneBacklog=vesagent_onebacklog)
- COUNT_TIME_SLOT = (1, 10)
- self.assertEquals(COUNT_TIME_SLOT, result)
- pass
+++ /dev/null
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import mock
-
-import unittest
-# import json
-import urllib2
-
-from ocata.vesagent import vespublish
-
-MOCK_VESENDPOINT = {
- "endpoint": "MOCKED_VES_COLLECTOR_EP1",
- "username": "MOCKED_VES_COLLECTOR_USER1",
- "password": "MOCKED_VES_COLLECTOR_PASSWD1",
-}
-
-MOCK_VESPUBLISH_EVENT1 = [{"name": "event1"}]
-
-
-class VespublishTest(unittest.TestCase):
- def setUp(self):
- pass
-
- def tearDown(self):
- pass
-
- @mock.patch.object(urllib2, 'urlopen')
- @mock.patch.object(urllib2, 'Request')
- def test_publishAnyEventToVES(self, mock_Request, mock_urlopen):
- mock_request = mock.Mock()
-
- mock_Request.side_effect = [
- mock_request
- ]
-
- mock_response = mock.Mock(["read"])
- mock_response.read.return_value = "MOCKED_VESPUBLISH_RESPONSE_MESSAGE"
- mock_urlopen.side_effect = [
- mock_response
- ]
-
- vespublish.publishAnyEventToVES(MOCK_VESENDPOINT, MOCK_VESPUBLISH_EVENT1)
+++ /dev/null
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-import json
-
-from rest_framework import status
-from rest_framework.response import Response
-from rest_framework.views import APIView
-
-from django.conf import settings
-from common.msapi import extsys
-from ocata.vesagent.tasks import scheduleBacklogs
-from ocata.vesagent.event_domain.fault_vm import buildBacklog_fault_vm
-
-from django.core.cache import cache
-
-logger = logging.getLogger(__name__)
-
-
-class VesAgentCtrl(APIView):
- '''
- control plane of VesAgent
- Design tips:
- 1, vesagent are multiple processing workers
- 2, the runtime logic is simple: a vesagent worker polls the data source (vm/hypervisor/host/vim/etc.)
- and then feeds the encoded data to VES.
- 3, the vesagent workers can be distributed to different clouds while latency/throughput is concerned,
- this distributed deployment usually comes along with the distributed VES deployment.
- So it is very likely that the collected data from different VIM/Cloud instance will be fed into
- different VES endpoint, however, assuming that there will be at most one VES endpoint serving
- any single VIM/Cloud instance.
- 4, According to VES specs, the collected data can be cataloged by domain:
- domain : fault, heartbeat, measurementsForVfScaling, other, stateChange, syslog, thresholdCrossingAlert
- As far as VIM/Cloud concerned, fault, heartbeat, measurementsForVfScaling, TCAalert are relevant.
- 5, the source of the collected data can be cataloged by eventSourceType:
- eventSourceType: VNF/VNFC/VM
- As far as VIM/Cloud concerned, only VM is relevant. This eventSourceType should be extended to cover
- the data source of hypervisor, VIM, Host,Controller, PIM, etc.
-
- 6, the source of collected data should be specified explicitly,so is the domain of the collected data.
- To specify the source: eventSourceType, uuid or name of the source
- To specify the domain: domain
- the specifications above will be provisioned as a vesagent backlog entry to a VIM/Cloud instance
- to tell a vesagent worker that :
- with regarding to that VIM/Cloud instance, what kind of data to be collected from which source .
-
- 7,the VES endpoint will be also specified for a VIM/Cloud instance, so that all collected data
- will be fed into this VES endpoint
-
- 8, the vesagent backlog are stored into the respective cloud_region's property "cloud-extra-info",
- which implies that those specifications can be CRUD either by ESR portal or the RestAPIs in this view, e.g.
- "cloud-extra-info": {
- ...,
- "vesagent_config":
- {
- "ves_subscription":{
- "endpoint":"http://{VES IP}:{VES port}/{URI}",
- "username":"{VES username}",
- "password":"{VES password}",
- },
- "poll_interval_default" : "{default interval for polling}",
- "backlogs":[
- {
- "domain":"fault"
- "type":"vm",
- "tenant":"{tenant name1}",
- "source":"{VM name1}",
- "poll_interval" : "{optional, interval for polling}",
- },
- {
- "domain":"fault"
- "type":"vm",
- "tenant":"{tenant name2}",
- "source":"{VM name2}",
- "poll_interval" : "{optional, interval for polling}",
- }
- ]
- }
- }
-
- Idea: API dispatching to distributed M.C. service can be determined by Complex Object in AAI:
- cloud-region has been assoicated to a Complex Object
- M.C. plugin service instance should refer to the same Complex Object (by physical_locaton_id ?)
- So the M.C. broker/API distributor/other approach will correlate the cloud-region with
- corresponding M.C. plugin service instance.
-
-
- Backlog built in cache:
-
- maintain backlog in cache and VES agent workers
- cache objects:
- "VesAgentBacklogs.vimlist": [ list of vimid] ### will not expire forever
- "VesAgentBacklogs.state.{vimdid}":
- ### will expire eventually to eliminate the garbage, expiration duration: 1hour?
- {
- "{backlog_uuid}": {
- "timestamp": "{timestamp for last time of data collecting}",
- "api_data": [list of data to populate the format string of the API link]
- "last_event": {object, event reported to ves last time}"
- }
- }
- "VesAgentBacklogs.config.{vimdid}": ### will not expire forever
- {
- "vimid": "{vim id}",
- "subscription": {
- "endpoint": "{ves endpoint, e.g. http://ves_ip:ves_port/eventListener/v5}",
- "username": "{username}",
- "password": "{password}"
- }
- "poll_interval_default" : "{default interval for polling}",
- "backlogs":[
- {
- "backlog_uuid": "{uuid to identify the backlog}"
- "domain":"fault"
- "type":"vm",
- "tenant":"{tenant name1}",
- "source":"{VM name1}",
- "poll_interval" : "{optional, interval in second for polling}",
- "api_method": "{GET/POST/PUT/etc.}",
- "api_link":"{API link to collect data, could be format string}",
- "tenant_id": tenant_id,
- "server_id": server_id,
- },
- {
- "domain":"fault"
- "type":"vm",
- "tenant":"{tenant name2}",
- "source":"{VM name2}",
- "poll_interval" : "{optional, interval in second for polling}",
- "api_method": "{GET/POST/PUT/etc.}",
- "api_link":"{API link to collect data, could be format string}",
- "tenant_id": tenant_id,
- "server_id": server_id,
- }
- ]
- }
- '''
-
- def __init__(self):
- self._logger = logger
- self.proxy_prefix = settings.MULTICLOUD_PREFIX
-
- def get(self, request, vimid=""):
- '''
- get blob of vesagent-config
- :param request:
- :param vimid:
- :return:
- '''
- self._logger.info("vimid: %s" % vimid)
- self._logger.debug("with META: %s" % request.META)
- try:
- # get vesagent_config from cloud region
- try:
- viminfo = extsys.get_vim_by_id(vimid)
- cloud_extra_info_str = viminfo.get('cloud_extra_info', '')
- cloud_extra_info = json.loads(cloud_extra_info_str) if cloud_extra_info_str != '' else None
- vesagent_config = cloud_extra_info.get("vesagent_config", None) if cloud_extra_info is not None else None
- except Exception as e:
- # ignore this error
- self._logger.warn("cloud extra info is provided with data in bad format: %s" % str(e))
- pass
-
- vesagent_backlogs = self.getBacklogsOneVIM(vimid)
-
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- return Response(data={'error': str(e)},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-
- self._logger.info("return with %s" % status.HTTP_200_OK)
- return Response(data={"vesagent_config": vesagent_config,
- "vesagent_backlogs": vesagent_backlogs},
- status=status.HTTP_200_OK)
-
- def post(self, request, vimid=""):
- '''
- update the blob of vesagent-config, rebuild the backlog for the vesagent workers,
- and start the vesagent workers if not started yet
- Implication: the request to this API endpoint will build the backlog locally, hence only local VES agent workers
- will process these backlogs, which conforms to distributed deployment of M.C. services which includes VES agents
- :param request:{"vesagent_config":
- {"ves_subscription":
- {"endpoint":"http://127.0.0.1:9005/sample",
- "username":"user","password":"password"},
- "poll_interval_default":10,
- "backlogs":
- [
- {"domain":"fault","type":"vm","tenant":"VIM","source":"onap-aaf"}
- ]
- }
- }
- :param vimid:
- :return:
- '''
- self._logger.info("vimid: %s" % vimid)
- self._logger.debug("with META: %s, with data: %s" % (request.META, request.data))
- try:
- vesagent_config = None
- if request.data is None or request.data.get("vesagent_config", None) is None:
- # Try to load the vesagent_config out of cloud_region["cloud_extra_info"]
- viminfo = extsys.get_vim_by_id(vimid)
- cloud_extra_info_str = viminfo.get('cloud_extra_info', None)
- cloud_extra_info = json.loads(cloud_extra_info_str) if cloud_extra_info_str is not None else None
- vesagent_config = cloud_extra_info.get("vesagent_config", None) if cloud_extra_info is not None else None
- else:
- vesagent_config = request.data.get("vesagent_config", None)
-
- if vesagent_config is None:
- return Response(data={'vesagent_config is not provided'},
- status=status.HTTP_400_BAD_REQUEST)
-
- vesagent_backlogs = self.buildBacklogsOneVIM(vimid, vesagent_config)
-
- # store back to cloud_extra_info
- # tbd
-
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- return Response(data={'error': str(e)},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-
- self._logger.info("return with %s" % status.HTTP_201_CREATED)
- return Response(data={"vesagent_config": vesagent_config,
- "vesagent_backlogs": vesagent_backlogs},
- status=status.HTTP_201_CREATED)
-
- def delete(self, request, vimid=""):
- '''
- delete the blob of vesagent-config, remove it from backlog and stop the vesagent worker if no backlog
- :param request:
- :param vimid:
- :return:
- '''
- self._logger.info("vimid: %s" % vimid)
- self._logger.debug("with META: %s" % request.META)
- try:
- # tbd
- self.clearBacklogsOneVIM(vimid)
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- return Response(data={'error': str(e)},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-
- self._logger.info("return with %s" % status.HTTP_200_OK)
- return Response(status=status.HTTP_200_OK)
-
- def getBacklogsOneVIM(self, vimid):
- '''
- remove the specified backlogs for a VIM
- :param vimid:
- :return:
- '''
- self._logger.debug("vimid: %s" % vimid)
-
- vesAgentConfig = None
- try:
- # retrive the backlogs
- vesAgentConfigStr = cache.get("VesAgentBacklogs.config.%s" % (vimid))
- if vesAgentConfigStr is None:
- logger.warn("VesAgentBacklogs.config.%s cannot be found in cache" % (vimid))
- return None
-
- logger.debug("VesAgentBacklogs.config.%s: %s" % (vimid, vesAgentConfigStr))
-
- vesAgentConfig = json.loads(vesAgentConfigStr)
- if vesAgentConfig is None:
- logger.warn("VesAgentBacklogs.config.%s corrupts" % (vimid))
- return None
-
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- vesAgentConfig = {"error": "exception occurs"}
-
- self._logger.debug("return")
- return vesAgentConfig
-
- def clearBacklogsOneVIM(self, vimid):
- '''
- remove the specified backlogs for a VIM
- :param vimid:
- :param vesagent_config:
- :return:
- '''
- self._logger.debug("vimid: %s" % vimid)
-
- try:
- # remove vimid from "VesAgentBacklogs.vimlist"
- VesAgentBacklogsVimListStr = cache.get("VesAgentBacklogs.vimlist")
- VesAgentBacklogsVimList = []
- if VesAgentBacklogsVimListStr is not None:
- VesAgentBacklogsVimList = json.loads(VesAgentBacklogsVimListStr)
- VesAgentBacklogsVimList = [v for v in VesAgentBacklogsVimList if v != vimid]
-
- logger.debug("VesAgentBacklogs.vimlist is %s" % VesAgentBacklogsVimList)
-
- # cache forever
- cache.set("VesAgentBacklogs.vimlist", json.dumps(VesAgentBacklogsVimList), None)
-
- # retrieve the backlogs
- vesAgentConfigStr = cache.get("VesAgentBacklogs.config.%s" % (vimid))
- if vesAgentConfigStr is None:
- logger.warn("VesAgentBacklogs.config.%s cannot be found in cache" % (vimid))
- return 0
-
- logger.debug("VesAgentBacklogs.config.%s: %s" % (vimid, vesAgentConfigStr))
-
- vesAgentConfig = json.loads(vesAgentConfigStr)
- if vesAgentConfig is None:
- logger.warn("VesAgentBacklogs.config.%s corrupts" % (vimid))
- return 0
-
- # iterate all backlog and remove the associate state!
- # tbd
-
- # clear the whole backlogs for a VIM
- cache.set("VesAgentBacklogs.config.%s" % vimid, "deleting the backlogs", 1)
-
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
-
- self._logger.debug("return")
- return 0
-
- def buildBacklogsOneVIM(self, vimid, vesagent_config=None):
- '''
- build and cache backlog for specific cloud region,spawn vesagent workers if needed
- :param vimid:
- :param vesagent_config: vesagent_config data in json object
- :return:
- '''
- self._logger.debug("vimid: %s" % vimid)
- self._logger.debug("config data: %s" % vesagent_config)
-
- VesAgentBacklogsConfig = None
- try:
- if vesagent_config:
- # now rebuild the backlog
- VesAgentBacklogsConfig = {
- "vimid": vimid,
- "poll_interval_default": vesagent_config.get("poll_interval_default", 0),
- "subscription": vesagent_config.get("ves_subscription", None),
- "backlogs": [self.buildBacklog(vimid, b) for b in vesagent_config.get("backlogs", [])]
- }
-
- # add/update the backlog into cache
- VesAgentBacklogsConfigStr = json.dumps(VesAgentBacklogsConfig)
- # cache forever
- cache.set("VesAgentBacklogs.config.%s" % vimid, VesAgentBacklogsConfigStr, None)
-
- # update list of vimid for vesagent
- # get the whole list of backlog
- VesAgentBacklogsVimListStr = cache.get("VesAgentBacklogs.vimlist")
- VesAgentBacklogsVimList = [vimid]
- if VesAgentBacklogsVimListStr is not None:
- VesAgentBacklogsVimList = json.loads(VesAgentBacklogsVimListStr)
- VesAgentBacklogsVimList = [v for v in VesAgentBacklogsVimList if v != vimid]
- VesAgentBacklogsVimList.append(vimid)
-
- logger.debug("VesAgentBacklogs.vimlist is %s" % VesAgentBacklogsVimList)
-
- # cache forever
- cache.set("VesAgentBacklogs.vimlist", json.dumps(VesAgentBacklogsVimList), None)
-
- # notify schduler
- scheduleBacklogs.delay(vimid)
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- VesAgentBacklogsConfig = {"error": "exception occurs during build backlogs"}
-
- self._logger.debug("return")
- return VesAgentBacklogsConfig
-
- def buildBacklog(self, vimid, backlog_input):
- self._logger.debug("build backlog for: %s" % vimid)
- self._logger.debug("with input: %s" % backlog_input)
-
- try:
- if backlog_input["domain"] == "fault" and backlog_input["type"] == "vm":
- return buildBacklog_fault_vm(vimid, backlog_input)
- else:
- self._logger.warn("return with failure: unsupported backlog domain:%s, type:%s"
- % (backlog_input["domain"], backlog_input["type"] == "vm"))
- return None
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- return None
-
- self._logger.debug("return without backlog")
- return None
-
-
-class APIv1VesAgentCtrl(VesAgentCtrl):
-
- def __init__(self):
- super(APIv1VesAgentCtrl, self).__init__()
- # self._logger = logger
- self.proxy_prefix = settings.MULTICLOUD_API_V1_PREFIX
-
- def get(self, request, cloud_owner="", cloud_region_id=""):
- '''
- :param request:
- :param cloud_owner:
- :param cloud_region_id:
- :return:
- '''
- self._logger.info("cloud_owner,cloud_region_id: %s,%s" % (cloud_owner, cloud_region_id))
-
- vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id)
- return super(APIv1VesAgentCtrl, self).get(request, vimid)
-
- def post(self, request, cloud_owner="", cloud_region_id=""):
- '''
- wrapper for inherited API with VIM ID
- :param request:
- :param cloud_owner:
- :param cloud_region_id:
- :return:
- '''
- self._logger.info("cloud_owner,cloud_region_id: %s,%s" % (cloud_owner, cloud_region_id))
-
- vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id)
- return super(APIv1VesAgentCtrl, self).post(request, vimid)
-
- def delete(self, request, cloud_owner="", cloud_region_id=""):
- '''
- wrapper of inherited API with VIM ID
- :param request:
- :param cloud_owner:
- :param cloud_region_id:
- :return:
- '''
- self._logger.info("cloud_owner,cloud_region_id: %s,%s" % (cloud_owner, cloud_region_id))
-
- vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id)
- return super(APIv1VesAgentCtrl, self).delete(request, vimid)
+++ /dev/null
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import, unicode_literals
-
-import time
-import logging
-import json
-import urllib2
-
-logger = logging.getLogger(__name__)
-
-
-def publishAnyEventToVES(ves_subscription, events):
- if not events or len(events) == 0:
- return
-
- logger.info("Start to send single event to VES collector.")
- endpoint = ves_subscription.get("endpoint", None)
-
- if endpoint:
- try:
- if len(events) > 1:
- endpoint = "%s/eventBatch" % endpoint
- events = {"eventList": events}
- elif len(events) == 1:
- events = {"event": events[0]}
-
- logger.info("publish event to VES: %s" % endpoint)
- headers = {'Content-Type': 'application/json'}
- request = urllib2.Request(url=endpoint, headers=headers, data=json.dumps(events))
- time.sleep(1)
- response = urllib2.urlopen(request)
- logger.info("VES response is: %s" % response.read())
- except urllib2.URLError as e:
- logger.critical("Failed to publish to %s: %s" % (endpoint, e.reason))
- except Exception as e:
- logger.error("exception:%s" % str(e))
- else:
- logger.info("Missing VES info.")
onappylog>=1.0.8
# for background tasks
-celery >= 4.0
-
memcached -d -m 2048 -u root -c 1024 -p 11211 -P /tmp/memcached1.pid
export PYTHONPATH=lib/share
-#service rabbitmq-server restart
-# make sure only 1 worker due to missing the synchronization between workers now
-#nohup celery -A ocata worker --concurrency=1 --loglevel=info &
-
uwsgi --http :9006 --module ocata.wsgi --master --processes 4
Code Review / multicloud / openstack.git / commitdiff