migrate ansible server python scripts to python3
[ccsdk/distribution.git] / ansible-server / src / main / ansible-server / UsersRestServer.py
1 '''
2 /*-
3 * ============LICENSE_START=======================================================
4 * ONAP : APPC
5 * ================================================================================
6 * Copyright (C) 2019 AT&T Intellectual Property.  All rights reserved.
7 * ================================================================================
8 * Copyright (C) 2019 Amdocs
9 * =============================================================================
10 * Licensed under the Apache License, Version 2.0 (the "License");
11 * you may not use this file except in compliance with the License.
12 * You may obtain a copy of the License at
13 *
14 *      http://www.apache.org/licenses/LICENSE-2.0
15 *
16 * Unless required by applicable law or agreed to in writing, software
17 * distributed under the License is distributed on an "AS IS" BASIS,
18 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19 * See the License for the specific language governing permissions and
20 * limitations under the License.
21
22 * ============LICENSE_END=========================================================
23 */
24 '''
25
26 import time, datetime, json, os, sys, subprocess, re
27 import uuid
28 import tarfile
29 import shutil
30 import glob
31 import crypt
32
33 import requests
34
35 import cherrypy
36 from cherrypy.lib.httputil import parse_query_string
37 from cherrypy.lib import auth_basic
38
39 from multiprocessing import Process, Manager
40
41 from AnsibleModule import ansibleSysCall
42 from BuildHostFile import buildHostsSysCall
43
44 from os import listdir
45 from os.path import isfile, join
46
47 TestRecord = Manager().dict()
48 ActiveProcess = {}
49
50 def validate_password(realm, username, password):
51     comp = crypt.crypt(password, salt)
52     if username in userpassdict and userpassdict[username] == comp:
53        return True
54     return False
55
56 def sys_call (cmd):
57     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
58     output = p.stdout.readlines()
59     retval = p.wait()
60     if len (output) > 0:
61         for i in range(len(output)):
62             output[i] = output[i].strip()
63     return retval, output
64
65 def callback (Id, Result, Output, Log, returncode):
66     
67     print("***> in RestServer.callback")
68
69     if Id in TestRecord:
70         time_now = datetime.datetime.utcnow()
71         delta_time = (time_now - TestRecord[Id]['Time']).total_seconds()
72         Result['PlaybookName'] = TestRecord[Id]['PlaybookName']
73         Result['Version'] = TestRecord[Id]['Version']
74         if returncode == 137:
75             Result['StatusCode'] = 500
76             Result['StatusMessage'] = "TERMINATED"
77         else:
78             Result['StatusCode'] = 200
79             Result['StatusMessage'] = "FINISHED"
80
81         # Need to update the whole data structure for key=Id otherwise Manager is not updated
82         TestRecord[Id] = {'PlaybookName': TestRecord[Id]['PlaybookName'],
83                           'Version': TestRecord[Id]['Version'],
84                           'NodeList': TestRecord[Id]['NodeList'],
85                           'HostGroupList': TestRecord[Id]['HostGroupList'],
86                           'HostNameList': TestRecord[Id]['HostNameList'],
87                           'Time': TestRecord[Id]['Time'],
88                           'Timeout': TestRecord[Id]['Timeout'],
89                           'Duration': str(delta_time),
90                           'EnvParameters': TestRecord[Id]['EnvParameters'],
91                           'LocalParameters': TestRecord[Id]['LocalParameters'],
92                           'FileParameters': TestRecord[Id]['FileParameters'],
93                           'CallBack': TestRecord[Id]['CallBack'],
94                           'Result': Result,
95                           'Log': Log, 
96                           'Output': Output, 
97                           'Path': TestRecord[Id]['Path'],
98                           'Mandatory': TestRecord[Id]['Path']}
99
100         if not TestRecord[Id]['CallBack'] == None:
101             
102             # Posting results to callback server
103
104             data = {"StatusCode": 200,
105                     "StatusMessage": "FINISHED",
106                     "PlaybookName": TestRecord[Id]["PlaybookName"],
107                     "Version": TestRecord[Id]["Version"],
108                     "Duration": TestRecord[Id]["Duration"],
109                     "Results": TestRecord[Id]['Result']['Results']}
110
111             cherrypy.log("CALLBACK: TestRecord[Id]['Output']['Output']:", str(TestRecord[Id]['Output']['Output']))
112             cherrypy.log("CALLBACK: Results:", str(data["Results"]))
113
114             if not TestRecord[Id]['Output']['Output'] == {}:
115                 for key in data["Results"]:
116                     if key in TestRecord[Id]['Output']['Output']:
117                         data["Results"][key]["Output"] = TestRecord[Id]['Output']['Output'][key]
118
119             print("     Posting to", TestRecord[Id]['CallBack'])
120             
121             s = requests.Session()
122             r = s.post(TestRecord[Id]['CallBack'], data = json.dumps(data),
123                        headers = {'content-type': 'application/json'})
124             print("     Response", r.status_code, r.text)
125
126 def RunAnsible_Playbook (callback, Id, Inventory, Playbook, NodeList, TestRecord,
127                          Path, ArchiveFlag):
128
129     print("***> in RestServer.RunAnsible_Playbook")
130
131     # Run test in playbook for given target
132     Result = ''
133
134     retval, log, returncode = ansibleSysCall (Inventory, Playbook, NodeList,
135                                               TestRecord[Id]['Mandatory'],
136                                               TestRecord[Id]['EnvParameters'],
137                                               TestRecord[Id]['LocalParameters'],
138                                               TestRecord[Id]['Timeout'],
139                                               Path)
140
141
142     cherrypy.log("Return code:" + str(returncode))
143     cherrypy.log("Return val:" +  str(retval))
144     
145     Log = ''.join(log)
146     #Output = {'Output': {}}
147     Output = {}
148     
149     onlyfiles = [f for f in listdir(Path)
150                  if isfile(join(Path, f))]
151
152     cherrypy.log("Checking for results.txt files: ")
153     for file in onlyfiles:
154         if "results.txt" in file:
155 #       if file.endswith("results.txt"):
156             cherrypy.log("results file: " + file)
157             f = open(Path + "/" + file, "r")
158             resultsData =  f.read()  # Not to pass vnf instance name
159             OutputP = json.loads(resultsData)
160             Output['Output'] =  OutputP 
161             cherrypy.log("Output = " + str(Output['Output']))
162             #Output['Output'][key] = f.read() # To pass vnf instance name
163             f.close()
164
165     if Output == {}:
166       Output = {'Output': {}}
167
168     Result = {'Results': {}}
169     if 'could not be found' in Log:
170         Result['Results'] = {"StatusCode": 101,
171                              "StatusMessage": "PLAYBOOK NOT FOUND"}
172     if returncode == 137:
173         Result['Results'] = {"StatusCode": 500,
174                              "StatusMessage": "TERMINATED"}
175
176     elif TestRecord[Id]['NodeList'] == []:
177         
178         host_index = None
179         
180         if 'TargetNode' in TestRecord[Id]['EnvParameters']:
181             targetlist = TestRecord[Id]['EnvParameters']['TargetNode'].split(' ')
182         else:
183             targetlist = ["localhost"]
184             
185         for key in retval:
186             for i in range (len(targetlist)):
187                 if key in targetlist[i]:
188                     host_index = i
189     
190             if int(retval[key][0]) > 0 and int(retval[key][2]) == 0 and \
191                    int(retval[key][3]) == 0:
192
193                 if host_index:
194                     Result['Results'][targetlist[host_index]] = \
195                              {"GroupName": 'na', "StatusCode": 200, \
196                               "StatusMessage": "SUCCESS"}
197                 else:
198                     Result['Results'][key] = \
199                              {"GroupName": 'na', "StatusCode": 200, \
200                               "StatusMessage": "SUCCESS"}                    
201             elif int(retval[key][2]) > 0:
202                 if host_index:
203                     Result['Results'][targetlist[host_index]] = \
204                        {"GroupName": 'na', "StatusCode": 400, \
205                         "StatusMessage": "NOT REACHABLE"}
206                 else:
207                     Result['Results'][key] = \
208                        {"GroupName": 'na', "StatusCode": 400, \
209                         "StatusMessage": "NOT REACHABLE"}                    
210             elif int(retval[key][3]) > 0:
211                 if host_index:                
212                     Result['Results'][targetlist[host_index]] = \
213                        {"GroupName": 'na', "StatusCode": 400, \
214                         "StatusMessage": "FAILURE"}
215                 else:
216                     Result['Results'][key] = \
217                        {"GroupName": 'na', "StatusCode": 400, \
218                         "StatusMessage": "FAILURE"}                    
219     else:
220         
221         for key in retval:
222
223             if len(TestRecord[Id]['HostNameList']) > 0:
224
225                 host_index = []
226                 for i in range (len(TestRecord[Id]['HostNameList'])):
227                     if key in TestRecord[Id]['HostNameList'][i]:
228                         host_index.append(i)
229
230                 if int(retval[key][0]) > 0 and int(retval[key][2]) == 0 and \
231                        int(retval[key][3]) == 0:
232
233                     if len(host_index) > 0:
234                         Result['Results'][TestRecord[Id]['HostNameList'][host_index[0]]] = \
235                           {"GroupName": TestRecord[Id]['HostGroupList'][host_index[0]],
236                            "StatusCode": 200, "StatusMessage": "SUCCESS"}
237                     
238                         for i in range (1, len(host_index)):
239                             Result['Results'][TestRecord[Id]['HostNameList'][host_index[i]]]["GroupName"]+=\
240                              "," + TestRecord[Id]['HostGroupList'][host_index[i]]
241                     else:
242                        Result['Results'][key] = \
243                           {"GroupName": key,
244                            "StatusCode": 200, "StatusMessage": "SUCCESS"}  
245
246                 elif int(retval[key][2]) > 0:
247
248                     if len(host_index) > 0:
249                         Result['Results'][TestRecord[Id]['HostNameList'][host_index[0]]] = \
250                           {"GroupName": TestRecord[Id]['HostGroupList'][host_index[0]],
251                            "StatusCode": 400, "StatusMessage": "NOT REACHABLE"}
252                     
253                         for i in range (1, len(host_index)):
254                             Result['Results'][TestRecord[Id]['HostNameList'][host_index[i]]]["GroupName"]+=\
255                              "," + TestRecord[Id]['HostGroupList'][host_index[i]]
256                     else:
257                        Result['Results'][key] = \
258                           {"GroupName": key,
259                            "StatusCode": 200, "StatusMessage": "NOT REACHABLE"}  
260                     
261                 elif int(retval[key][3]) > 0:
262
263                     if len(host_index) > 0:
264                         Result['Results'][TestRecord[Id]['HostNameList'][host_index[0]]] = \
265                           {"GroupName": TestRecord[Id]['HostGroupList'][host_index[0]],
266                            "StatusCode": 400, "StatusMessage": "FAILURE"}
267                     
268                         for i in range (1, len(host_index)):
269                             Result['Results'][TestRecord[Id]['HostNameList'][host_index[i]]]["GroupName"]+=\
270                              "," + TestRecord[Id]['HostGroupList'][host_index[i]]
271                     else:
272                        Result['Results'][key] = \
273                           {"GroupName": key,
274                            "StatusCode": 200, "StatusMessage": "FAILURE"}                          
275             else:
276                 host_index = None
277                 for i in range (len(TestRecord[Id]['NodeList'])):
278                     if key in TestRecord[Id]['NodeList'][i]:
279                         host_index = i
280     
281                 if int(retval[key][0]) > 0 and int(retval[key][2]) == 0 and \
282                        int(retval[key][3]) == 0:
283                     Result['Results'][TestRecord[Id]['NodeList'][host_index]] = \
284                              {"GroupName": 'na', "StatusCode": 200, \
285                              "StatusMessage": "SUCCESS"}
286                 elif int(retval[key][2]) > 0:
287                     Result['Results'][TestRecord[Id]['NodeList'][host_index]] = \
288                        {"GroupName": 'na', "StatusCode": 400, "StatusMessage": "NOT REACHABLE"}
289                 elif int(retval[key][3]) > 0:
290                     Result['Results'][TestRecord[Id]['NodeList'][host_index]] = \
291                        {"GroupName": 'na', "StatusCode": 400, "StatusMessage": "FAILURE"}
292     
293     cherrypy.log("TESTRECORD: " + str(TestRecord[Id]))
294     cherrypy.log("Output: " + str(Output))
295     callback (Id, Result, Output, Log, returncode)
296
297 class TestManager (object):
298
299     @cherrypy.expose
300     @cherrypy.tools.json_out()
301     @cherrypy.tools.json_in()
302     @cherrypy.tools.allow(methods=['POST', 'GET', 'DELETE'])
303
304     def Dispatch(self, **kwargs):
305
306         # Let cherrypy error handler deal with malformed requests
307         # No need for explicit error handler, we use default ones
308
309         time_now = datetime.datetime.utcnow()
310
311         # Erase old test results (2x timeout)
312         # Do cleanup too of ActiveProcess list and old Records - PAP
313         if TestRecord:
314             for key in TestRecord.copy():
315                 cherrypy.log( "LOOKING AT ALL TestRecords: " + str(key))
316                 if key in ActiveProcess:
317                    if not ActiveProcess[key].is_alive(): # Just to cleanup defunct processes
318                       cherrypy.log( "Not ActiveProcess for ID: " + str(key))
319                 delta_time = (time_now - TestRecord[key]['Time']).seconds
320                 if delta_time > 2*TestRecord[key]['Timeout']:
321                     cherrypy.log( "DELETED HISTORY for ID: " + str(key))
322                     if key in ActiveProcess:
323                       if not ActiveProcess[key].is_alive():
324                           ActiveProcess.pop (key)
325                           cherrypy.log( "DELETED ActiveProcess for ID: " + str(key))
326                     #if os.path.exists(TestRecord[key]['Path']):
327                         # don't remove run dirrectory
328                         #shutil.rmtree (TestRecord[key]['Path'])
329                     del TestRecord[key]
330                     
331         cherrypy.log("RestServer.Dispatch: " + cherrypy.request.method)
332
333
334         if 'POST' in cherrypy.request.method:
335             
336             input_json = cherrypy.request.json
337             cherrypy.log("Payload: " + str(input_json))
338
339             if 'Id' in input_json and 'PlaybookName' in input_json and 'EnvParameters' in input_json:
340
341                 if True:
342
343                     if not input_json['Id'] in TestRecord:
344                         # check if Id exists in previous run dirctory
345                         # if so retun error
346                         s_cmd = 'ls ' + ansible_temp + '/*_' + input_json['Id']
347                         #if subprocess.check_output([s_cmd, ]):
348                         Id = input_json['Id']
349                         if glob.glob(  ansible_temp + '/*_' + input_json['Id']):
350                             cherrypy.log("Old directory found for ID: " + Id)
351                             return {"StatusCode": 101, "StatusMessage": "TEST ID FILE ALREADY DEFINED"}
352                     
353                         PlaybookName = input_json['PlaybookName']
354                         # if required it should be passed as an argument
355                         EnvParameters = input_json['EnvParameters']
356
357                         # The lines below are to test multiple EnvParameters being passed
358                         #for i in EnvParameters:
359                         #  cherrypy.log("EnvParameter object: " + i)
360                         #  cherrypy.log("  EnvParameter Value: " + EnvParameters[ i ])
361
362                         # Now get things out of EnvParameters
363                         VNF_instance   = None
364                         VNF_instance = EnvParameters.get('vnf_instance')
365
366                         # Get Version if present
367                         version = None
368                         if 'Version' in input_json:
369                             version = input_json['Version']
370
371                         # GetInventoryNames
372                         HaveNodeList         = False
373                         HaveInventoryNames   = False 
374                         inventory_names = None
375                         if 'InventoryNames' in input_json:
376                            inventory_names = input_json['InventoryNames']
377                            HaveInventoryNames   = True 
378                     
379                         #AnsibleInvFail = True
380                         AnsiblePlaybookFail = True
381
382                         LocalNodeList = None
383
384                         str_uuid = str (uuid.uuid4())
385                         
386
387                         VnfType= PlaybookName.split("/")[0] 
388                         cherrypy.log( "Request USER  :                  " + cherrypy.request.login)
389                         cherrypy.log( "Request Decode: ID               " + Id)
390                         cherrypy.log( "Request Decode: VnfType          " + VnfType)
391                         cherrypy.log( "Request Decode: EnvParameters    " + json.dumps(EnvParameters))
392                         
393                         # Verify VNF_instance was passed in EnvParameters
394                         if VNF_instance != None:
395                           cherrypy.log( "Request Decode: VnfInstance      " + VNF_instance)
396                         else:
397                           cherrypy.log( "StatusCode: 107, StatusMessage: VNF_instance NOT PROVIDED" )
398                           return {"StatusCode": 107,
399                                     "StatusMessage": "VNF_instance NOT PROVIDED"}
400
401                         if inventory_names != None:
402                           cherrypy.log( "Request Decode: Inventory Names  " + inventory_names)
403                         else:
404                           cherrypy.log( "Request Decode: Inventory Names  " + "Not provided")
405
406                         cherrypy.log( "Request Decode: PlaybookName     " + PlaybookName)
407                         PlayBookFunction = PlaybookName.rsplit("/",2)[1]
408                         PlayBookFile = PlayBookFunction + "/site.yml"
409                         cherrypy.log( "Request Decode: PlaybookFunction " + PlayBookFunction)
410                         cherrypy.log( "Request Decode: Playbook file    " + PlayBookFile)
411                         
412                         BaseDir = ansible_path + "/" + PlaybookName.rsplit("/",1)[0]
413                         CopyDir = ansible_path + "/" + PlaybookName.rsplit("/",2)[0]
414                         cherrypy.log( "Request Decode: Basedir          " + BaseDir)
415                         cherrypy.log( "Request Decode: Copydir          " + CopyDir)
416                         
417
418                         PlaybookDir = ansible_temp + "/" + \
419                                       VNF_instance + "_" + str_uuid + "_" + str(Id)
420
421                         # AnsibleInv is the directory where the host file to be run exsists
422                         AnsibleInv = ansible_path + "/" + VnfType + "/latest/ansible/inventory/" + VNF_instance
423                         ArchiveFlag = False
424
425                         # Create base run directory if it doesn't exist
426                         if not os.path.exists(ansible_temp):
427                             cherrypy.log( "Creating Base Run Directory: "  + ansible_temp)
428                             os.makedirs(ansible_temp)
429
430                         if not os.path.exists( CopyDir ):
431                             cherrypy.log("Playbook Not Found")
432                             return {"StatusCode": 101,
433                                     "StatusMessage": "PLAYBOOK NOT FOUND"}
434
435                         # copy static playbook dir to run dir
436                         cherrypy.log("Copying from " + CopyDir + " to " + PlaybookDir)
437                         shutil.copytree(CopyDir, PlaybookDir)
438                         cmd="/usr/bin/find " + PlaybookDir + " -exec /usr/bin/touch {} \;"
439                         cmd="/usr/bin/find " + PlaybookDir + " -exec chmod +rx  {} \;"
440                         sys_call(cmd)
441                         cherrypy.log(cmd)
442
443                         cherrypy.log( "PlaybookDir:    " + PlaybookDir)
444                         cherrypy.log( "AnsibleInv:     " + AnsibleInv)
445
446                         #location of host file
447                         #HostFile = PlaybookDir + "/inventory/" + VNF_instance + "hosts"
448                         #cherrypy.log("HostFile: " +  HostFile)
449
450                         # Process inventory file for target
451                     
452                         hostgrouplist = []
453                         hostnamelist = []
454
455                         NodeList = []
456                         if 'NodeList' in input_json:
457                             NodeList = input_json['NodeList']
458
459                         cherrypy.log("NodeList: " + str(NodeList));
460
461                         # if NodeList empty 
462                         if NodeList == []:
463                                 cherrypy.log( "*** NodeList - Empty ***")
464                                 #AnsibleInvFail = False
465
466                         else:
467                                 #AnsibleInvFail = False # ???
468                                 HaveNodeList = True
469
470                         ###############################################################################
471                         ##### Host file processing                          ###########################
472                         ##### 1. Use file delivered with playbook           ###########################
473                         ##### 2. If HostNames + NodeList generate and use   ###########################
474                         ##### 3. If HostNames = VM or NVF copy and use.     ###########################
475                         ###############################################################################
476
477                         #location of host file - Default
478                         HostFile = PlaybookDir + "/inventory/" + VNF_instance + "hosts"
479                         cherrypy.log("HostFile: " +  HostFile)
480
481                         # if NodeList and InventoryNames need to build host file
482                         if HaveInventoryNames & HaveNodeList:
483                            cherrypy.log("Build host file from NodeList")
484                            ret = buildHostsSysCall (input_json, PlaybookDir, inventory_names)
485                            if (ret < 0):
486                               cherrypy.log("Returning Error: Not running Playbook")
487                               return {"StatusCode": 105,
488                                     "StatusMessage": "NodeList: Missing vnfc-type field"}
489
490                            # Having been built now copy new file to correct file
491                            shutil.copy(PlaybookDir + "/host_file.txt", HostFile)
492                            cherrypy.log("Copying Generated host file to: " + HostFile)
493                         elif HaveInventoryNames & (not HaveNodeList):
494                            ### Copy Instar based Hostfile
495                            if inventory_names == "VNFC":
496                               #test if file
497                               host_file_path = "/storage/inventory/VNFC/" + VNF_instance + "hosts"
498                               if os.path.exists(host_file_path):
499                                 #Copy file
500                                 cherrypy.log("Copying Instar hostfile: " + host_file_path + " -> " + HostFile)
501                                 shutil.copy(host_file_path, HostFile)
502                               else:
503                                 cherrypy.log("Inventory file not found: " + host_file_path)
504                            elif inventory_names == "None":
505                               #test if file
506                               host_file_path = "/storage/inventory/None/" + VNF_instance + "hosts"
507                               if os.path.exists(host_file_path):
508                                 #Copy file
509                                 cherrypy.log("Copying Instar hostfile: " + host_file_path + " -> " + HostFile)
510                                 shutil.copy(host_file_path, HostFile)
511                               else:
512                                 cherrypy.log("Inventory file not found: " + host_file_path)
513                            elif inventory_names == "VM":
514                               #test if file
515                               host_file_path = "/storage/inventory/VM/" + VNF_instance + "hosts"
516                               if os.path.exists(host_file_path):
517                                 #Copy file
518                                 cherrypy.log("Copying Instar hostfile: " + host_file_path + " -> " + HostFile)
519                                 shutil.copy(host_file_path, HostFile)
520                               else:
521                                 cherrypy.log("Inventory file not found: " + host_file_path)
522
523
524                         timeout = timeout_seconds
525                         if 'Timeout' in input_json:
526                             timeout = int (input_json['Timeout'])
527                             cherrypy.log("Timeout from API: " + str(timeout))
528
529                         else:
530                             cherrypy.log("Timeout not passed from API using default: " + str(timeout))
531
532                         EnvParam = {}
533                         if 'EnvParameters' in input_json:
534                             EnvParam = input_json['EnvParameters']
535
536                         LocalParam = {}
537                         if 'LocalParameters' in input_json:
538                             LocalParam = input_json['LocalParameters']
539
540                         FileParam = {}
541                         if 'FileParameters' in input_json:
542                             FileParam = input_json['FileParameters']
543                     
544                         callback_flag = None
545                         if 'CallBack' in input_json:
546                             callback_flag = input_json['CallBack']
547
548                         # if AnsibleServer is not set to 'na'  don't send AnsibleServer in PENDING responce.
549                         if AnsibleServer != 'na':
550                                 TestRecord[Id] = {'PlaybookName': PlaybookName,
551                                           'Version': version,
552                                           'NodeList': NodeList,
553                                           'HostGroupList': hostgrouplist,
554                                           'HostNameList': hostnamelist,
555                                           'Time': time_now,
556                                           'Duration': timeout,
557                                           'Timeout': timeout,
558                                           'EnvParameters': EnvParam,
559                                           'LocalParameters': LocalParam,
560                                           'FileParameters': FileParam,
561                                           'CallBack': callback_flag,
562                                           'Result': {"StatusCode": 100,
563                                                      "StatusMessage": 'PENDING',
564                                                      "AnsibleServer": str(AnsibleServer),
565                                                      "ExpectedDuration": str(timeout) + "sec"},
566                                           'Log': '',
567                                           'Output': {},
568                                           'Path': PlaybookDir,
569                                           'Mandatory': None}
570                         else:
571                                 TestRecord[Id] = {'PlaybookName': PlaybookName,
572                                           'Version': version,
573                                           'NodeList': NodeList,
574                                           'HostGroupList': hostgrouplist,
575                                           'HostNameList': hostnamelist,
576                                           'Time': time_now,
577                                           'Duration': timeout,
578                                           'Timeout': timeout,
579                                           'EnvParameters': EnvParam,
580                                           'LocalParameters': LocalParam,
581                                           'FileParameters': FileParam,
582                                           'CallBack': callback_flag,
583                                           'Result': {"StatusCode": 100,
584                                                      "StatusMessage": 'PENDING',
585                                                      "ExpectedDuration": str(timeout) + "sec"},
586                                           'Log': '',
587                                           'Output': {},
588                                           'Path': PlaybookDir,
589                                           'Mandatory': None}
590
591                         cherrypy.log("Test_Record: " +  str(TestRecord[Id]))
592                         # Write files
593                         
594                         if not TestRecord[Id]['FileParameters'] == {}:
595                             for key in TestRecord[Id]['FileParameters']:
596                                 filename = key
597                                 filecontent = TestRecord[Id]['FileParameters'][key]
598                                 f = open(PlaybookDir + "/" + filename, "w")
599                                 f.write(filecontent)
600                                 f.close()
601                                 
602                         
603                         # Process playbook
604                         if os.path.exists( ansible_path + '/' + PlaybookName):
605                             AnsiblePlaybookFail = False
606                                     
607                         if AnsiblePlaybookFail:
608                             #if os.path.exists(PlaybookDir):
609                                 #shutil.rmtree (PlaybookDir)
610                             del TestRecord[Id]
611                             return {"StatusCode": 101,
612                                     "StatusMessage": "PLAYBOOK NOT FOUND"}
613                         else:
614
615                             # Test EnvParameters
616                             playbook_path = PlaybookDir
617
618                             # Store local vars
619                             if not os.path.exists(playbook_path + "/vars"):
620                                 os.mkdir(playbook_path + "/vars")
621                             if not os.path.isfile(playbook_path + "/vars/defaults.yml"):
622                                 os.mknod(playbook_path + "/vars/defaults.yml")
623
624                             ###################################################
625                             # PAP
626                             #write local parameters passed into defaults.yml
627                             # PAP
628                             f = open(playbook_path + "/vars/defaults.yml","a")
629                             #for id, record in TestRecord.items():
630                             print(TestRecord[Id]['LocalParameters'])
631                             local_parms = TestRecord[Id]['LocalParameters']
632                             for key, value in list(local_parms.items()):
633                                 f.write(key +"=" + value + "\n");
634                             f.close()
635                             ###################################################
636                                  
637                             for key in TestRecord[Id]['LocalParameters']:
638                                 host_index = []
639                                 for i in range(len(TestRecord[Id]['HostNameList'])):
640                                     if key in TestRecord[Id]['HostNameList'][i]:
641                                         host_index.append(i)
642                                 if len(host_index) == 0:
643                                     for i in range(len(TestRecord[Id]['HostGroupList'])):
644                                         if key in TestRecord[Id]['HostGroupList'][i]:
645                                             host_index.append(i)
646                                 if len(host_index) > 0:
647                                     for i in range(len(host_index)):
648                                         f = open(playbook_path + "/vars/" +
649                                                  TestRecord[Id]['HostNameList'][host_index[i]] +
650                                                  ".yml", "a")
651                                         for param in TestRecord[Id]['LocalParameters'][key]:
652                                             f.write(param + ": " +
653                                              str (TestRecord[Id]['LocalParameters'][key][param]) +
654                                                   "\n")
655                                         f.close()
656
657     
658                             # write some info out to files before running
659                             f = open(playbook_path + "/PlaybookName.txt", "a")
660                             f.write(PlaybookName)
661                             f.close()
662                             f = open(playbook_path + "/PlaybookExDir.txt", "a")
663                             f.write(PlaybookDir + "/" + PlayBookFunction)
664                             f.close()
665                             f = open(playbook_path + "/JsonRequest.txt", "w")
666                             #f.write(str(input_json))
667                             print(( json.dumps(input_json, indent=4, sort_keys=True)))
668                             f.write( json.dumps(input_json, indent=4, sort_keys=True))
669                             f.close()
670
671
672                             # Check that HostFile exists
673                             if not os.path.isfile(HostFile):
674                               cherrypy.log("Inventory file Not Found: " + HostFile)
675                               return {"StatusCode": 101,
676                                    "StatusMessage": "PLAYBOOK INVENTORY FILE NOT FOUND"}
677    
678                             # Cannot use thread because ansible module uses
679                             # signals which are only supported in main thread.
680                             # So use multiprocess with shared object
681                                        # args = (callback, Id,  PlaybookDir + "/" + AnsibleInv,
682
683                             p = Process(target = RunAnsible_Playbook,
684                                         args = (callback, Id,  HostFile,
685                                                 PlaybookDir + '/' + PlayBookFile,
686                                                 NodeList, TestRecord, PlaybookDir + "/" + PlayBookFunction,
687                                                 ArchiveFlag))
688                             p.start()
689                             ActiveProcess[Id] = p
690                             return TestRecord[Id]['Result']
691                     else:
692                         cherrypy.log("TEST ID ALREADY DEFINED")
693                         return {"StatusCode": 101, "StatusMessage": "TEST ID ALREADY DEFINED"}
694
695                 else:
696                     return {"StatusCode": 500, "StatusMessage": "REQUEST MUST INCLUDE: NODELIST"}
697                 
698             else:
699                 return {"StatusCode": 500, "StatusMessage": "JSON OBJECT MUST INCLUDE: ID, PLAYBOOKNAME, EnvParameters"}
700
701         elif 'GET' in cherrypy.request.method:
702             
703             # Lets pause for a second just incase the resquest was just kicked off
704             time.sleep(1)
705
706             input_data = parse_query_string(cherrypy.request.query_string)
707              
708             # Verify we have a Type passed in GET request
709             if not ( 'Type' in input_data):
710                 return {"StatusCode": 500, "StatusMessage": "RESULTS TYPE UNDEFINED"}
711
712             cherrypy.log( "Request USER:             " + cherrypy.request.login)
713             cherrypy.log("Payload: " + str(input_data) + " Type " + input_data['Type'])
714
715             if 'LogRest' in input_data['Type']:
716                 sys.stdout.close()
717                 sys.stdout = open("/var/log/RestServer.log", "w")
718
719             # Just a debug to dump any records
720             if 'GetStatus' in input_data['Type']:
721                 cherrypy.log( "******** Dump Records **********")
722                 if list(TestRecord.items()):
723                   for id, record in list(TestRecord.items()):
724                     cherrypy.log( "    Id: " + id)
725                     cherrypy.log( "Record: " + str(record))
726                 else:
727                   cherrypy.log(" No Records to dump")
728             
729             if 'Id' in input_data and 'Type' in input_data:
730                 if not ('GetResult' in input_data['Type'] or 'GetOutputLog' in input_data['Type'] or'GetOutput' in input_data['Type'] or 'GetLog' in input_data['Type']):
731                     return {"StatusCode": 500, "StatusMessage": "RESULTS TYPE UNDEFINED"}
732                 if input_data['Id'] in TestRecord:
733                     
734                     if 'GetResult' in input_data['Type']:
735                         
736                         cherrypy.log( " ** GetResult for: " + str (input_data['Id']))
737
738                         if 'StatusMessage' in TestRecord[input_data['Id']]['Result'] and getresults_block:
739
740
741                             #check if playbook is still running
742                             while ActiveProcess[input_data['Id']].is_alive():
743                                 cherrypy.log( "*** Playbook running returning PENDING for " + str(input_data['Id']))
744                                 ##
745                                 ## If still running return PENDING response
746                                 ##
747                                 if AnsibleServer != 'na':
748                                    return {"StatusCode": 100,
749                                                      "StatusMessage": 'PENDING',
750                                                      "AnsibleServer": str(AnsibleServer)}
751                                 else: 
752                                    return {"StatusCode": 100,
753                                                      "StatusMessage": 'PENDING'}
754                                 #time.sleep(5)
755
756                             #cherrypy.log( "*** Request released " + input_data['Id'])
757
758                         cherrypy.log(str( TestRecord[input_data['Id']]['Result']))
759                         cherrypy.log("Output: " + str( TestRecord[input_data['Id']]['Output']))
760                         cherrypy.log("StatusCode: " + str( TestRecord[input_data['Id']]['Result']['StatusCode']))
761                         cherrypy.log("StatusMessage: " + str( TestRecord[input_data['Id']]['Result']['StatusMessage']))
762
763                         #out_obj gets returned to GET request
764                         if TestRecord[input_data['Id']]['Result']['StatusCode'] == 500:
765                             out_obj = TestRecord[input_data['Id']]['Result']['Results']
766                         else:
767                             out_obj = {"StatusCode": 200,
768                                    "StatusMessage": "FINISHED",
769                                    "PlaybookName": TestRecord[input_data['Id']]["PlaybookName"],
770                                    "Version": TestRecord[input_data['Id']]["Version"],
771                                    "Duration": TestRecord[input_data['Id']]["Duration"],
772                                    "Output": TestRecord[input_data['Id']]["Output"]["Output"],
773                                    "Results": TestRecord[input_data['Id']]['Result']['Results']}
774                         if not TestRecord[input_data['Id']]['Output']['Output'] == {}:
775                             cherrypy.log("TestRecord has Output:" + str(TestRecord[input_data['Id']]['Output']['Output']))
776                             # PAP                
777                             for key in out_obj["Results"]:
778                                 cherrypy.log("Output key: " + str(key))
779                                 if key in TestRecord[input_data['Id']]['Output']['Output']:
780                                     out_obj["Results"][key]["Output"] = TestRecord[input_data['Id']]['Output']['Output'][key]
781
782                         cherrypy.log("***** GET RETURNING RESULTS Back ****")
783                         cherrypy.log(str(out_obj))
784                         return out_obj
785
786                     elif 'GetStatus' in input_data['Type']:
787                         print(" Dump Records")
788                         for id, record in TestRecord,items():
789                            print(" id: " + id)
790                            print("   Record:" + str(reecord))
791
792                     elif 'GetOutput' in input_data['Type']:
793
794                         if TestRecord[input_data['Id']]['Output'] == {} and \
795                                getresults_block:
796
797                             cherrypy.log( "*** Request blocked " + input_data['Id'])
798                             
799                             while TestRecord[input_data['Id']]['Output'] == {} \
800                                       or 'StatusMessage' in TestRecord[input_data['Id']]['Result']:
801                                 time.sleep(5)
802
803                             cherrypy.log( "*** Request released " + input_data['Id'])
804                         
805                         cherrypy.log( "Output: " + str(TestRecord[input_data['Id']]['Output']))
806                         return {"Output": TestRecord[input_data['Id']]['Output']['Output']}
807                     elif 'GetOutputLog' in input_data['Type']:
808 #XXXXXXXXXXX
809                        if glob.glob(  ansible_temp + '/*_' + input_data['Id']):
810                           id = input_data['Id']
811                           cherrypy.log("Old directory found for ID: " + id)
812                           run_dir = glob.glob(  ansible_temp + '/*_' + input_data['Id'])
813                           for dir in run_dir:
814                               rdir=dir
815                           if os.path.exists (rdir + "/PlaybookExDir.txt"):
816                                cherrypy.log("Found PlaybookExDir.txt file")
817                                f = open( rdir + '/PlaybookExDir.txt', 'r')
818                                playbookexdir =  f.readline()
819                                rdir = playbookexdir
820                                f.close()
821                           cherrypy.log("Id:     " + id)
822                           cherrypy.log("RunDir: " + rdir)
823                           if os.path.exists( rdir + "/output.log"):
824                                cherrypy.log("Found output.log file")
825                                f = open( rdir + '/output.log', 'r')
826                                output_log =  f.readline()
827                                f.close()
828                                return output_log
829                        else:
830                          return
831
832 #XXXXXXXXXXX
833                     else:
834                         # GetLog
835
836                         if TestRecord[input_data['Id']]['Log'] == '' and \
837                                getresults_block:
838
839                             cherrypy.log( "*** Request blocked " + input_data['Id'])
840                             
841                             while TestRecord[input_data['Id']]['Log'] == '' \
842                                       or 'StatusMessage' in TestRecord[input_data['Id']]['Result']:
843                                 time.sleep(5)
844
845                             cherrypy.log( "*** Request released " + input_data['Id'])
846                             
847                         cherrypy.log( "Log:" + str(TestRecord[input_data['Id']]['Log']))
848                         return {"Log": TestRecord[input_data['Id']]['Log']}
849                 else:
850                    # Not in memory check for a file 
851                    if glob.glob(  ansible_temp + '/*_' + input_data['Id']):
852                        id = input_data['Id']
853                        cherrypy.log("Old directory found for ID: " + id)
854                        run_dir = glob.glob(  ansible_temp + '/*_' + input_data['Id'])
855                        for dir in run_dir:
856                            rdir=dir
857                        if os.path.exists (rdir + "/PlaybookExDir.txt"):
858                             cherrypy.log("Found PlaybookExDir.txt file")
859                             f = open( rdir + '/PlaybookExDir.txt', 'r')
860                             playbookexdir =  f.readline()
861                             rdir = playbookexdir
862                             f.close()
863                        cherrypy.log("Id:     " + id)
864                        cherrypy.log("RunDir: " + rdir)
865                        if 'GetLog' in input_data['Type']:
866                          if os.path.exists( rdir + "/output.log"):
867                             cherrypy.log("Found output.log file")
868                             f = open( rdir + '/output.log', 'r')
869                             output_log =  f.readline()
870                             f.close()
871                             return output_log
872                        elif 'GetOutputLog' in input_data['Type']:
873                             if os.path.exists( rdir + "/output.log"):
874                                cherrypy.log("Found output.log file")
875                                f = open( rdir + '/output.log', 'r')
876                                output_log =  f.readline()
877                                f.close()
878                                return output_log
879                        elif 'GetResult' in input_data['Type']:
880                             if os.path.exists (rdir + "/PlaybookName.txt"):
881                                cherrypy.log("Found PlaybookName.txt file")
882                                f = open( rdir + '/PlaybookName.txt', 'r')
883                                playbooknametxt =  f.readline()
884                                f.close()
885                             else:
886                                playbooknametxt = "NA"
887
888                             # Add code to get other items not just output.log from files
889                             if os.path.exists( rdir + "/log.file"):
890                                cherrypy.log("Found log.file")
891                                out_results = "NA:"
892                                f = open( rdir + '/log.file', 'r')
893                                
894                                line =  f.readline()
895                                while line :
896                                  if "fatal" in line:
897                                    out_results = out_results +  line
898                                  elif "RECAP" in line:
899                                    out_results = out_results +  line
900                                    recap_line =  f.readline()
901                                    while recap_line :
902                                      out_results = out_results +  recap_line
903                                      recap_line =  f.readline()
904                                  line = f.readline()
905                                f.close()
906                             out_obj = {"StatusCode": 200,
907                                      "StatusMessage": "FINISHED",
908                                      "PlaybookName": playbooknametxt,
909                                      "Version": "Version",
910                                      "Duration": 200,
911                                      "Results": out_results}
912                             return out_obj
913                        else:
914                           return {"StatusCode": 500, "StatusMessage": "PLAYBOOK FAILED "}
915                            
916
917                    return {"StatusCode": 500, "StatusMessage": "TEST ID UNDEFINED"}
918             else:
919                 return {"StatusCode": 500, "StatusMessage": "MALFORMED REQUEST"}
920         elif 'DELETE' in cherrypy.request.method:
921             input_data = parse_query_string(cherrypy.request.query_string)
922             
923             cherrypy.log( "***> in RestServer.DELETE")
924             cherrypy.log("Payload: " + str(input_data))
925             
926             if input_data['Id'] in TestRecord:
927                 if not 'PENDING' in TestRecord[input_data['Id']]['Result']:
928                     cherrypy.log(" Path: " + str(TestRecord[input_data['Id']]['Path']))
929                     TestRecord.pop (input_data['Id'])
930                     if input_data['Id'] in ActiveProcess:
931                         ActiveProcess.pop (input_data['Id'])
932
933                     return {"StatusCode": 200, "StatusMessage": "PLAYBOOK EXECUTION RECORDS DELETED"}
934                 else:
935                     return {"StatusCode": 200, "StatusMessage": "PENDING"}
936             else:
937                 return {"StatusCode": 500, "StatusMessage": "TEST ID UNDEFINED"}
938
939
940 if __name__ == '__main__':
941
942     # Read configuration
943
944     config_file_path = "RestServer_config"
945
946     if not os.path.exists(config_file_path):
947         print('[INFO] The config file does not exist')
948         sys.exit(0)
949
950     ip = 'na'
951     AnsibleServer = 'na'
952     port = 'na'
953     tls = False
954     auth = False
955     pub = 'na'
956     priv = 'na'
957     timeout_seconds = 'na'
958     ansible_path = 'na'
959     ansible_temp = 'na'    
960     host = 'na'
961     users= 'na'
962     getresults_block = False
963     from_files = False
964     
965     file = open(config_file_path, 'r')
966     for line in file.readlines():
967         if '#' not in line:
968             if 'ip:' in line:
969                 ip = line.split(':')[1].strip()
970             elif 'AnsibleServer:' in line:
971                 AnsibleServer = line.split(':')[1].strip()
972             elif 'port:' in line:
973                 port = line.split(':')[1].strip()
974             elif 'ksalt:' in line:
975                 salt = line.split(':')[1].strip()
976             elif 'tls:' in line:
977                 tls = 'YES' in line.split(':')[1].strip().upper()
978             elif 'auth:' in line:
979                 auth = 'YES' in line.split(':')[1].strip().upper()
980             if tls and 'priv:' in line:
981                 priv = line.split(':')[1].strip()
982             if tls and 'pub:' in line:
983                 pub = line.split(':')[1].strip()
984             if tls and 'inter_cert:' in line:
985                 intermediate = line.split(':')[1].strip()
986             if 'timeout_seconds' in line:
987                 timeout_seconds = int (line.split(':')[1].strip())
988             if 'ansible_path' in line:
989                 ansible_path = line.split(':')[1].strip()
990             if 'ansible_temp' in line:
991                 ansible_temp = line.split(':')[1].strip()
992             if 'host' in line:
993                 host = line.split(':')[1].strip()
994             if 'users' in line:
995                 users = line.split(':')[1].strip()
996             if 'getresults_block' in line:
997                 getresults_block = 'YES' in line.split(':')[1].strip().upper()
998             if 'from_files' in line:
999                 from_files = 'YES' in line.split(':')[1].strip().upper()
1000     file.close()
1001
1002     # Initialization
1003     
1004     global_conf = {
1005         'global': {
1006             'log.screen': True,
1007             'response.timeout': 5400,
1008             'server.socket_host': ip,
1009             'server.socket_port': int(port),
1010             'server.protocol_version': 'HTTP/1.1'
1011             }
1012         }
1013
1014     if tls:
1015         # Use pythons built-in SSL
1016         cherrypy.server.ssl_module = 'builtin'
1017
1018         # Point to certificate files
1019
1020         if not os.path.exists(pub):
1021             print('[INFO] The public certificate does not exist')
1022             sys.exit(0)
1023
1024         if not os.path.exists(priv):
1025             print('[INFO] The private key does not exist')
1026             sys.exit(0)            
1027
1028         if not os.path.exists(intermediate):
1029             print('[INFO] The intermediate certificate does not exist')
1030             sys.exit(0)
1031
1032         
1033         cherrypy.server.ssl_certificate = pub
1034         cherrypy.server.ssl_certificate_chain = intermediate
1035         cherrypy.server.ssl_private_key = priv
1036
1037     if auth:
1038         # Read in and build user dictionary
1039         if not os.path.exists(users):
1040           print('[INFO] The users file does not exist: ' + users)
1041           sys.exit(0)
1042         userpassdict = {}
1043         user_file = open(users, 'r')
1044         for line in user_file.readlines():
1045           if '#' not in line:
1046             id = line.split(':')[0].strip()
1047             pw = line.split(':')[1].strip()
1048             userpassdict[id] = pw
1049             #print str(userpassdict)
1050
1051         app_conf = {'/':
1052                     {'tools.auth_basic.on': True,
1053                      'tools.auth_basic.realm': 'earth',
1054                      'tools.auth_basic.checkpassword': validate_password,
1055                      }
1056                     }
1057
1058         application = cherrypy.tree.mount(TestManager(), '/', app_conf)
1059     else:
1060         application = cherrypy.tree.mount(TestManager(), '/')
1061         
1062     cherrypy.config.update({
1063         'log.access_file': "/var/log/RestServer.access"
1064     })
1065     accessLogName = "/var/log/RestServer.access"
1066     applicationLogName = "/var/log/RestServer.log"
1067     cherrypy.config.update(global_conf)
1068
1069     log = application.log
1070     log.error_file = ""
1071     log.access_file = ""
1072     from logging import handlers
1073     applicationLogFileHandler = handlers.RotatingFileHandler(applicationLogName, 'a', 1000000, 5000)
1074     accessLogFileHandler = handlers.RotatingFileHandler(accessLogName, 'a', 1000000, 5000)
1075     import logging
1076     applicationLogFileHandler.setLevel(logging.DEBUG)
1077     log.error_log.addHandler(applicationLogFileHandler)
1078     log.access_log.addHandler(accessLogFileHandler)
1079
1080     # Start server
1081     
1082     cherrypy.engine.start()
1083     cherrypy.engine.block()

© 2017 ONAP. Copyright © The Linux Foundation ®. All Rights Reserved.
The Linux Foundation has registered trademarks and uses trademarks.
For a list of trademarks of The Linux Foundation, please see our Trademark Usage page.
Linux is a registered trademark of Linus Torvalds.
Privacy Policy and Terms of Use