gerrit.onap Code Review - dcaegen2/platform/cdapbroker.git/blob

1 % ============LICENSE_START=======================================================

2 % org.onap.dcae

3 % ================================================================================

5 % ================================================================================

6 % Licensed under the Apache License, Version 2.0 (the "License");

7 % you may not use this file except in compliance with the License.

8 % You may obtain a copy of the License at

9 %

10 % http://www.apache.org/licenses/LICENSE-2.0

11 %

12 % Unless required by applicable law or agreed to in writing, software

13 % distributed under the License is distributed on an "AS IS" BASIS,

14 % WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

15 % See the License for the specific language governing permissions and

16 % limitations under the License.

17 % ============LICENSE_END=========================================================

18 %

19 % ECOMP is a trademark and service mark of AT&T Intellectual Property.

20 -module(apitest_SUITE).

21 -include_lib("common_test/include/ct.hrl").

22 -include("../../src/application.hrl").

23 -export([all/0, groups/0, init_per_suite/1, end_per_suite/1]).

24 -export([server_health_test/1, app_deploy/1, hydrator_deploy/1, app_teardown/1, app_test/1, app_reconfigure/1, test_failures/1, app_botch_flows/1, app_botch_delete/1, app_botch_consul_delete/1, delete_all/1,

25 hydrator_app_teardown/1, hydrator_test/1,

26 hydrator_wdeps_deploy/1,

27 hydrator_wdeps_test/1,

28 hydrator_wdeps_teardown/1

29 ]).

30

31 %lazy shorthands (yay C style macros! miss these in python)

32 -define(SC(L), util:concat(L)).

33 -define(PLG(K, PL), proplists:get_value(K, PL)).

34 -define(XER, "testing-XER").

35 -define(D(X), erlang:display(X)).

36

37 all() -> [

38 {group, progapi},

39 {group, hydratorapi},

40 {group, apibotchedflows},

41 {group, apibotcheddeleted},

42 {group, apibotchedconsuldeleted},

43 {group, apideleteall}

44 ].

45 groups() -> [

46 {progapi, %prog-flow test

47 [],

48 [

49 server_health_test,

50 test_failures,

51 app_deploy,

52 app_test,

53 app_reconfigure,

54 app_test,

55 app_teardown

56 ]},

57 {hydratorapi, %hydrator test

58 [],

59 [

60 server_health_test,

61 test_failures,

62 hydrator_deploy,

63 hydrator_test,

64 hydrator_app_teardown,

65 hydrator_wdeps_deploy,

66 hydrator_wdeps_test,

67 hydrator_wdeps_teardown

68 ]},

69 {apibotchedflows, %deploy, manually stop flows, then try to delete

70 [],

71 [

72 server_health_test,

73 app_deploy,

74 app_botch_flows,

75 app_teardown

76 ]},

77 {apibotcheddeleted, %deploy, manually stop flows, delete it manually from cdap, then try to delete

78 [],

79 [

80 server_health_test,

81 app_deploy,

82 app_botch_delete,

83 app_teardown

84 ]},

85 {apibotchedconsuldeleted, %deploy, manually stop flows, delete it manually from cdap, then try to delete

86 [],

87 [

88 server_health_test,

89 app_deploy,

90 app_botch_consul_delete,

91 app_teardown

92 ]},

93 {apideleteall,

94 [],

95 [

96 server_health_test,

97 app_deploy,

98 delete_all

99 ]}

100 ].

101

102 %HELPER FUNCTIONS

103 setup_rels(Config, D) ->

104 %deploy/delete the testing keys into Consul. This would normally be done by the Cloudify plugin

105 %

106 %#NOTE: This is weird. The sequence of steps is:

107 % 1 Cloudify populates rels key

108 % 2 Cloudify sends broker the unbound config

109 % 3 Broker pushes unbound config to consul

110 % 4 Broker binds config

111 % 5 Broker pushes bound config to CDAP

112 % Between state 1 and 3 consul is in an inconsistent state where it has only the rels key but not the config key. Not so sure about this. They seem to be a pair. Maybe the rels key should be pushed to the source node to be dealt with.

113 % #Here, we are mocking step 1

114 URL = ?SC([?PLG(consul_url, Config), "/v1/kv/", ?PLG(appname, Config), ":rel"]),

115 case D of

116 setup -> {200,"true"} = httpabs:put(?XER, URL, "application/json", jiffy:encode([<<"666_fake_testing_service">>]));

117 teardown -> {200,"true"} = httpabs:delete(?XER, URL)

118 end,

119 httpabs:get(?XER, URL).

120

121 setup_fake_testing_service(Config, D) ->

122 %register a fake testing service to test that the CDAP app recieved it's bound configuration properly

123 Name = <<"666_fake_testing_service">>,

124 SrvURL = ?SC([?PLG(consul_url, Config), "/v1/catalog/service/", Name]),

125 case D of

126 setup ->

127 URL = ?SC([?PLG(consul_url, Config), "/v1/agent/service/register"]),

128 Body = {[{<<"name">>, Name},

129 {<<"Address">>, <<"666.666.666.666">>},

130 {<<"Port">>, 13}

131 ]},

132 {200, []} = httpabs:put(?XER, URL, "application/json", jiffy:encode(Body)),

133 httpabs:get(?XER, SrvURL);

134 teardown ->

135 %total failure on Consul's part for this not to be a delete on the same endpoint

136 URL = ?SC([?PLG(consul_url, Config), "/v1/agent/service/deregister/", Name]),

137 {200, []} = httpabs:put(?XER, URL, "application/json", ""),

138 httpabs:get(?XER, SrvURL)

139 end.

140

141 get_config_consul(C) ->

142 %get config from consul. returns the code too for tests testing for a 404

143 {RC, RB} = consul_interface:consul_get_configuration(?XER, ?PLG(appname, C), ?PLG(consul_url, C)),

144 case RC of

145 200 -> {RC, util:ejson_to_map(RB)};

146 _ -> {RC, RB}

147 end.

148

149 get_config_cdap(C) ->

150 {RC, RB} = cdap_interface:get_app_config(?XER, ?PLG(appname, C), ?PLG(namespace, C),?PLG(cdap_url, C)),

151 case RC of

152 200 ->

153 %I think CDAP is DOUBLY encoding JSON!!

154 {RC, jiffy:decode(jiffy:decode(jiffy:encode(RB)), [return_maps])};

155 _ -> {RC, RB}

156 end.

157

158 get_preferences_cdap(C) ->

159 {RC, RB} = cdap_interface:get_app_preferences(?XER, ?PLG(appname, C), ?PLG(namespace, C),?PLG(cdap_url, C)),

160 case RC of

161 200 -> {RC, util:ejson_to_map(RB)};

162 _ -> {RC, RB}

163 end.

164

165 get_preferences_consul(C) ->

166 %get preferences from consul. returns the code too for tests testing for a 404

167 {RC, RB} = consul_interface:consul_get_preferences(?XER, ?PLG(appname, C), ?PLG(consul_url, C)),

168 case RC of

169 200 -> {RC, util:ejson_to_map(RB)};

170 _ -> {RC, RB}

171 end.

172

173 valid_deploy_body(C) ->

174 {[

175 {<<"cdap_application_type">>, <<"program-flowlet">>},

176 {<<"namespace">>, ?PLG(namespace, C)},

177 {<<"streamname">>, ?PLG(streamname, C)},

178 {<<"jar_url">>, ?PLG(jar_url, C)},

179 {<<"artifact_name">>, ?PLG(art_name, C)},

180 {<<"artifact_version">>, ?PLG(art_ver, C)},

181 {<<"app_config">>, ?PLG(init_config, C)},

182 {<<"app_preferences">>, ?PLG(init_preferences, C)},

183 {<<"services">>, [{[{<<"service_name">>, <<"Greeting">>},

184 {<<"service_endpoint">>, <<"greet">>},

185 {<<"endpoint_method">>, <<"GET">>}]}]},

186 {<<"programs">>, [

187 {[{<<"program_type">>, <<"flows">>},

188 {<<"program_id">>, <<"WhoFlow">>}]},

189 {[{<<"program_type">>, <<"services">>},

190 {<<"program_id">>, <<"Greeting">>}]}]},

191 {<<"program_preferences">>, [

192 {[{<<"program_type">>,<<"flows">>},

193 {<<"program_id">>, <<"WhoFlow">>},

194 {<<"program_pref">>, ?PLG(whoflowpref, C)}]}

195 ]}

196 ]}.

197

198 %%%%%%%%%%%%%%%

199 %TEST FUNCTIONS

200 init_per_suite(_C) ->

201 %get platform ENVs

202 [MyName, ConsulURL, _, _] = util:get_platform_envs_and_config(),

203

204 BrokerUrl = case os:getenv("BROKER_TEST_TYPE") of

205 false -> %no env variable means start the broker on localhost

206 %start a local broker

207 {ok,[syntax_tools,compiler,goldrush,lager,jiffy,mnesia,ranch,cowlib,cowboy,leptus,uuid,iso8601,cdapbroker]} = application:ensure_all_started(cdapbroker),

208 "http://localhost:7777";

209 "DOCKER" ->

210 "http://localhost:7777";

211 "REMOTE" ->

212 %Using MyName, fetch from Consul the broker info

213 {MyIP, MyPort} = consul_interface:consul_get_service_ip_port(MyName, ConsulURL),

214 ?SC(["http://", MyIP, ":", integer_to_binary(MyPort)])

215 end,

216

217 %get NEXUS_ROOT for testing purposes.

218 Nexus = os:getenv("NEXUS_RAW_ROOT"),

219 true = (Nexus /= false), %blow if this wasn't set, we need it.

220

221 {200, RB} = httpabs:get(?XER, BrokerUrl),

222 CDAPUrl = maps:get(<<"managed cdap url">>, jiffy:decode(RB, [return_maps])),

223

224 %set properties that are shared between program-flowlet and hydrator

225 Namespace = <<"testns">>,

226 CDAPUrlNS = ?SC([CDAPUrl, "/v3/namespaces/", Namespace]),

227

228 %set up config for program-flowlet app

229 Appname = <<"hwtest">>,

230 Streamname = <<"who">>,

231

232 %setup config for hydrator pipeline

233 HydratorAppname = <<"hydratortest">>,

234 HydratorAppURL = ?SC([CDAPUrlNS, "/apps/", HydratorAppname]),

235 HydratorStreamname = <<"s1">>, %horrible name but not made by me

236

237 HydratorWDepsAppname = <<"hydratorwdepstest">>,

238 HydratorWDepsAppURL = ?SC([CDAPUrlNS, "/apps/", HydratorWDepsAppname]),

239 HydratorWDepsStreamname = <<"t1">>, %horrible name but not made by me

240

241 %Set up this test suites configuration

242 [{broker_url, BrokerUrl},

243 {cdap_url, CDAPUrl},

244 {cdap_ns_url, CDAPUrlNS},

245 {jar_url, ?SC([Nexus, "/jar_files/HelloWorld-3.4.3.jar"])},

246 {consul_url, ConsulURL},

247 {consul_app_url, ?SC([ConsulURL, "/v1/catalog/service/", Appname])},

248 {app_url, ?SC([CDAPUrlNS, "/apps/", Appname])},

249 {namespace, Namespace},

250 {appname, Appname},

251 {broker_app_url, ?SC([BrokerUrl, "/application/", Appname])},

252 {stream_url, ?SC([CDAPUrlNS, "/streams/", Streamname])},

253 {art_ver, <<"3.4.3">>},

254 {art_name, <<"HelloWorld">>},

255 {streamname, Streamname},

256 {init_config, {[

257 {<<"streams_produces">>, <<"\{\{fake_testing_service\}\}">>},

258 {<<"services_calls">>, <<"\{\{fake_testing_service\}\}">>},

259 {<<"donotresolveme">>, <<"donotabsolveme">>}

260 ]}},

261 {whoflowpref, {[{<<"progfoo">>, <<"progbar">>}]}},

262 {init_preferences, {[{<<"preffoo">>, <<"prefbar">>}]}},

263 {reconfig, {[{<<"foo">>, <<"bar">>}]}},

264

265 %hydrator test properties

266 {hydrator_appname, HydratorAppname},

267 {broker_hydrator_app_url, ?SC([BrokerUrl, "/application/", HydratorAppname])},

268 {hydrator_app_url, HydratorAppURL},

269 {hydrator_json_url, ?SC([Nexus, "/json_files/t1-4.1.2.json "])},

270 {hydrator_pipeline_status_url, ?SC([HydratorAppURL, "/schedules/dataPipelineSchedule/status"])},

271 {hydrator_stream_url, ?SC([CDAPUrlNS, "/streams/", HydratorStreamname])},

272 {hydrator_streamname, HydratorStreamname},

273 {consul_hydrator_app_url, ?SC([ConsulURL, "/v1/catalog/service/", HydratorAppname])},

274

275 %hydrator with deps test properties

276 {hydrator_wdeps_appname, HydratorWDepsAppname},

277 {hydrator_wdeps_artname, <<"demoTCA">>},

278 {hydrator_wdeps_artver, <<"1.0.0-SNAPSHOT">>},

279 {hydrator_wdeps_app_url, HydratorWDepsAppURL},

280 {broker_hydrator_wdeps_app_url, ?SC([BrokerUrl, "/application/", HydratorWDepsAppname])},

281 {hydrator_wdeps_streamname, HydratorWDepsStreamname},

282 {hydrator_wdeps_stream_url, ?SC([CDAPUrlNS, "/streams/", HydratorWDepsStreamname])},

283 {hydrator_wdeps_json_url, ?SC([Nexus, "/json_files/t1-4.1.2.json"])},

284 {hydrator_wdeps_properties_json_url, ?SC([Nexus, "/json_files/demoTCA-1.0.0-SNAPSHOT-properties.json"])},

285 {hydrator_wdeps_jar_url, ?SC([Nexus, "/json_files/demoTCA-1.0.0-SNAPSHOT.jar"])},

286 {hydrator_wdeps_test_data_url, ?SC([Nexus, "/txt_files/tcaDemoData100k.txt"])},

287 {consul_hydrator_wdeps_app_url, ?SC([ConsulURL, "/v1/catalog/service/", HydratorWDepsAppname])},

288 {hydrator_pipeline_wdeps_status_url, ?SC([HydratorWDepsAppURL, "/schedules/dataPipelineSchedule/status"])}

289 ]

290 .

291

292 end_per_suite(_C) ->

293 _ = application:stop(cdapbroker).

294

295 server_health_test(C) ->

296 {200, RB} = httpabs:get(?XER, ?PLG(broker_url,C)),

297 M = jiffy:decode(RB, [return_maps]),

298 true = maps:is_key(<<"managed cdap url">>, M),

299 true = maps:is_key(<<"number of applications registered">>, M),

300 true = maps:is_key(<<"uptime (s)">>, M),

301 true = maps:is_key(<<"cdap cluster version">>, M),

302 true = maps:is_key(<<"cdap GUI port">>, M),

303 true = maps:is_key(<<"broker API version">>, M)

304 .

305

306 app_deploy(C) -> %C == Config

307 %Deploy the test application

308

309 %Deploy the rel key

310 {200, _} = setup_rels(C, setup),

311

312 %Register the fake testing service to test config binding. Make sure it's not empty

313 {200, F} = setup_fake_testing_service(C, setup),

314 true = F /= [],

315

316 ExpectedBoundConfg = maps:from_list([

317 {<<"services_calls">> , [<<"666.666.666.666:13">>]},

318 {<<"streams_produces">>, [<<"666.666.666.666:13">>]},

319 {<<"donotresolveme">> , <<"donotabsolveme">>}

320 ]),

321

322 %Maps can be used safely with the == operator but appears proplists cannot be

323 Expected = maps:from_list([

324 {<<"appname">>, ?PLG(appname, C)},

325 {<<"apptype">>, <<"program-flowlet">>},

326 {<<"namespace">>, ?PLG(namespace, C)},

327 {<<"healthcheckurl">>, list_to_binary(?SC([?PLG(broker_app_url, C), "/healthcheck"]))},

328 {<<"metricsurl">>, list_to_binary(?SC([?PLG(broker_app_url, C), "/metrics"]))},

329 {<<"url">>, list_to_binary(?PLG(broker_app_url, C))},

330 {<<"connectionurl">>, list_to_binary(?PLG(stream_url, C))},

331 {<<"serviceendpoints">>, [#{<<"url">> => list_to_binary(?SC([?PLG(app_url, C), "/services/Greeting/methods/greet"])),

332 <<"method">> => <<"GET">>}]},

333 {<<"unbound_config">>, #{<<"streams_produces">> => <<"\{\{fake_testing_service\}\}">>, <<"services_calls">> => <<"\{\{fake_testing_service\}\}">>, <<"donotresolveme">> => <<"donotabsolveme">>}},

334 {<<"bound_config">>, ExpectedBoundConfg}

335 ]),

336

337 %assert the current appliccation list does not contain our test app

338 {200,RB0} = httpabs:get(?XER, ?SC([?PLG(broker_url, C), "/application"])),

339 true = lists:all(fun(X) -> X /= ?PLG(appname, C) end, jiffy:decode(RB0)),

340

341 %deploy the app

342 Body = valid_deploy_body(C),

343 {200, RB} = httpabs:put(?XER, ?PLG(broker_app_url, C), "application/json", jiffy:encode(Body)),

344

345 %The CDAP APIs return the config as a JSON dumped to a string, so we need to get that back into a real JSON to have key-order-independent equality testing

346 Fix = fun(X) ->

347 RBMap = jiffy:decode(X, [return_maps]),

348 maps:update(<<"bound_config">>, jiffy:decode(maps:get(<<"bound_config">>, RBMap), [return_maps]), RBMap)

349 end,

350

351 %assert that the return and get matches what we put in

352 true = Fix(RB) == Expected,

353

354 %assert hitting the get application endpoint works

355 {200, RB2} = httpabs:get(?XER, ?PLG(broker_app_url, C)),

356 true = Fix(RB2) == Expected,

357

358 %assert the current application list now includes our new app

359 {200, RB3} = httpabs:get(?XER, ?SC([?PLG(broker_url, C), "/application"])),

360 true = lists:any(fun(X) -> X == ?PLG(appname, C) end, jiffy:decode(RB3)),

361

362 %make sure it is in CDAP

363 {200, _} = httpabs:get(?XER, ?PLG(app_url, C)),

364

365 %check metrics

366 {200, _} = httpabs:get(?XER, ?SC([?PLG(broker_app_url, C), "/metrics"])),

367

368 %check healthcheck

369 {200, _} = httpabs:get(?XER,?SC([?PLG(broker_app_url, C), "/healthcheck"])),

370

371 %make sure that the service is registered. TODO! Could get more fancy by manually checking a healthcheck

372 {200, RBHC} = httpabs:get(?XER,?PLG(consul_app_url, C)),

373 true = jiffy:decode(RBHC) /= [],

374

375 %check that the UNbound config is correct

376 true = {200, util:ejson_to_map(?PLG(init_config, C))} == get_config_consul(C),

377

378 %check that the preferences in Consul is correct

379 InitPrefMap = util:ejson_to_map(?PLG(init_preferences, C)),

380 true = {200, InitPrefMap} == get_preferences_consul(C),

381

382 %make sure CDAP has right preferences

383 true = {200, InitPrefMap} == get_preferences_cdap(C),

384

385 %make sure the config binding service and pulling config out of CDAP all match

386 %> get it strait from CBS

387 CBSUrl = util:resolve_cbs(?XER, ?PLG(consul_url, C)),

388 {200, RB4} = httpabs:get(?XER, ?SC([CBSUrl, "/service_component/", ?PLG(appname, C)])),

389 %get it from cdap

390 {200, CDAPConfig} = get_config_cdap(C),

391 %make sure everythng is as expected

392 true = ExpectedBoundConfg == jiffy:decode(RB4, [return_maps]),

393 true = ExpectedBoundConfg == CDAPConfig,

394

395 %try to put the same app again and assert you get a 400

396 {400,"State: Bad Request. Return Body: Put recieved on /application/:appname but appname is already registered. Call /application/:appname/reconfigure if trying to reconfigure or delete first"} =

397 httpabs:put(?XER, ?PLG(broker_app_url, C), "application/json", jiffy:encode(Body)).

398

399 hydrator_deploy(C) ->

400 Body = {[

401 {<<"cdap_application_type">>, <<"hydrator-pipeline">>},

402 {<<"namespace">>, ?PLG(namespace, C)},

403 {<<"streamname">>, ?PLG(hydrator_streamname, C)},

404 {<<"pipeline_config_json_url">>, ?PLG(hydrator_json_url, C)}

405 ]},

406 Expected = maps:from_list([

407 {<<"appname">>, ?PLG(hydrator_appname, C)},

408 {<<"apptype">>, <<"hydrator-pipeline">>},

409 {<<"namespace">>, ?PLG(namespace, C)},

410 {<<"healthcheckurl">>, list_to_binary(?SC([?PLG(broker_hydrator_app_url, C), "/healthcheck"]))},

411 {<<"metricsurl">>, list_to_binary(?SC([?PLG(broker_hydrator_app_url, C), "/metrics"]))},

412 {<<"url">>, list_to_binary(?PLG(broker_hydrator_app_url, C))},

413 {<<"connectionurl">>, list_to_binary(?PLG(hydrator_stream_url, C))},

414 {<<"serviceendpoints">>, []}

415 ]),

416

417 %assert the current appliccation list does not contain our test app

418 {200,RB0} = httpabs:get(?XER, ?SC([?PLG(broker_url, C), "/application"])),

419 true = lists:all(fun(X) -> X /= ?PLG(hydrator_appname, C) end, jiffy:decode(RB0)),

420

421 %try the deploy

422 {200, RB1} = httpabs:put(?XER, ?PLG(broker_hydrator_app_url, C), "application/json", jiffy:encode(Body)),

423 true = jiffy:decode(RB1, [return_maps]) == Expected,

424

425 %make sure the Execution resume worked

426 {200, RB2} = httpabs:get(?XER, ?PLG(hydrator_pipeline_status_url, C)),

427 true = jiffy:decode(RB2) == {[{<<"status">>, <<"SCHEDULED">>}]},

428

429 %make sure it is in CDAP

430 {200, _} = httpabs:get(?XER, ?PLG(hydrator_app_url, C)),

431

432 %assert the current application list now includes our new app

433 {200, RB3} = httpabs:get(?XER, ?SC([?PLG(broker_url, C), "/application"])),

434 true = lists:any(fun(X) -> X == ?PLG(hydrator_appname, C) end, jiffy:decode(RB3)),

435

436 %check healthcheck

437 {200, _} = httpabs:get(?XER,?SC([?PLG(broker_hydrator_app_url, C), "/healthcheck"])),

438

439 %check metrics

440 {200, _} = httpabs:get(?XER,?SC([?PLG(broker_hydrator_app_url, C), "/metrics"])),

441

442 %make sure that the service is registered. TODO! Could get more fancy by manually checking a healthcheck

443 {200, RBHC} = httpabs:get(?XER,?PLG(consul_hydrator_app_url, C)),

444 true = jiffy:decode(RBHC) /= []

445 .

446

447 hydrator_wdeps_deploy(C) ->

448 Body = {[

449 {<<"cdap_application_type">>, <<"hydrator-pipeline">>},

450 {<<"namespace">>, ?PLG(namespace, C)},

451 {<<"streamname">>, ?PLG(hydrator_wdeps_streamname, C)},

452 {<<"pipeline_config_json_url">>, ?PLG(hydrator_wdeps_json_url, C)},

453 {<<"dependencies">>, [

454 {[

455 {<<"artifact_extends_header">>, <<"system:cdap-data-pipeline[4.1.0,5.0.0)">>},

456 {<<"artifact_name">>, ?PLG(hydrator_wdeps_artname, C)},

457 {<<"artifact_version_header">>, ?PLG(hydrator_wdeps_artver, C)},

458 {<<"artifact_url">>, ?PLG(hydrator_wdeps_jar_url, C)},

459 {<<"ui_properties_url">>, ?PLG(hydrator_wdeps_properties_json_url, C)}

460 ]}

461 ]}

462 ]},

463 Expected = maps:from_list([

464 {<<"appname">>, ?PLG(hydrator_wdeps_appname, C)},

465 {<<"apptype">>, <<"hydrator-pipeline">>},

466 {<<"namespace">>, ?PLG(namespace, C)},

467 {<<"healthcheckurl">>, list_to_binary(?SC([?PLG(broker_hydrator_wdeps_app_url, C), "/healthcheck"]))},

468 {<<"metricsurl">>, list_to_binary(?SC([?PLG(broker_hydrator_wdeps_app_url, C), "/metrics"]))},

469 {<<"url">>, list_to_binary(?PLG(broker_hydrator_wdeps_app_url, C))},

470 {<<"connectionurl">>, list_to_binary(?PLG(hydrator_wdeps_stream_url, C))},

471 {<<"serviceendpoints">>, []}

472 ]),

473 %assert the current appliccation list does not contain our test app

474 {200,RB0} = httpabs:get(?XER,?SC([?PLG(broker_url, C), "/application"])),

475 true = lists:all(fun(X) -> X /= ?PLG(hydrator_wdeps_appname, C) end, jiffy:decode(RB0)),

476

477 %try the deploy

478 {200, RB1} = httpabs:put(?XER, ?PLG(broker_hydrator_wdeps_app_url, C), "application/json", jiffy:encode(Body)),

479 true = jiffy:decode(RB1, [return_maps]) == Expected,

480

481 %make sure properties are loaded, test artifact

482 {200, _} = httpabs:get(?XER,?SC([?PLG(cdap_ns_url, C), "/artifacts/", ?PLG(hydrator_wdeps_artname, C), "/versions/", ?PLG(hydrator_wdeps_artver, C), "/properties"])),

483

484 %make sure the Execution resume worked

485 {200, RB2} = httpabs:get(?XER,?PLG(hydrator_pipeline_wdeps_status_url, C)),

486 true = jiffy:decode(RB2) == {[{<<"status">>, <<"SCHEDULED">>}]},

487

488 %make sure it is in CDAP

489 {200, _} = httpabs:get(?XER,?PLG(hydrator_wdeps_app_url, C)),

490

491 %assert the current application list now includes our new app

492 {200, RB3} = httpabs:get(?XER,?SC([?PLG(broker_url, C), "/application"])),

493 true = lists:any(fun(X) -> X == ?PLG(hydrator_wdeps_appname, C) end, jiffy:decode(RB3)),

494

495 %check healthcheck

496 {200, _} = httpabs:get(?XER,?SC([?PLG(broker_hydrator_wdeps_app_url, C), "/healthcheck"])),

497

498 %check metrics

499 {200, _} = httpabs:get(?XER,?SC([?PLG(broker_hydrator_wdeps_app_url, C), "/metrics"])),

500

501 %make sure that the service is registered. TODO! Could get more fancy by manually checking a healthcheck

502 {200, RBHC} = httpabs:get(?XER,?PLG(consul_hydrator_wdeps_app_url, C)),

503 true = jiffy:decode(RBHC) /= []

504 .

505

506 hydrator_test(C) ->

507 %test te app by injecting some data into the stream and getting it out

508 %Sleeping since HTTP services may still be booting up: see https://issues.cask.co/browse/CDAP-812

509 ok = timer:sleep(60000), %60s

510 %curl into stream

511 {200, _} = httpabs:post(?XER, ?PLG(hydrator_stream_url, C), "text/plain", "beer, vodka, gin"),

512 %query data out

513 PB = jiffy:encode({[{<<"query">>, <<"select v1, v2, v3 from dataset_pf1">>}]}),

514 {200, RB} = httpabs:post(?XER, ?SC([?PLG(cdap_ns_url, C), "/data/explore/queries"]), "text/plain", PB),

515 {[{<<"handle">>, Handle}]} = jiffy:decode(RB),

516 %results can take time, sleep again

517 ok = timer:sleep(30000),

518 Expected = {[

519 {<<"status">>,<<"FINISHED">>},

520 {<<"hasResults">>,true}

521 ]},

522 {200, RB2} = httpabs:get(?XER, ?SC([?PLG(cdap_url, C), "/v3/data/explore/queries", "/", Handle, "/status"])),

523 true = Expected == jiffy:decode(RB2),

524 {200, _} = httpabs:post(?XER, ?SC([?PLG(cdap_url, C), "/v3/data/explore/queries", "/", Handle, "/next"]), "text/plain", "")

525 .

526

527 app_test(C) ->

528 %Sleeping since HTTP services may still be booting up: see https://issues.cask.co/browse/CDAP-812

529 ok = timer:sleep(60000), %60s

530 {200, _} = httpabs:post(?XER, ?PLG(stream_url, C), "text/plain", "'Prince of Darkness'"),

531 {200, "Hello 'Prince of Darkness'!"} = httpabs:get(?XER,?SC([?PLG(app_url, C), "/services/Greeting/methods/greet"])).

532

533 app_reconfigure(C) ->

534 %Test app reconfiguration

535 %test new config right in Consul

536 true = {200, util:ejson_to_map(?PLG(init_config, C))} == get_config_consul(C),

537

538 %do the reconfig

539 ReconfigMap = util:ejson_to_map({[{<<"foo REDUX EDITION">>, <<"bar">>}, {<<"LEAVE ME ALONE">>, <<"CONFIG EDITION">>}]}),

540 %do it properly

541 {200, _} = httpabs:put(?XER, ?SC([?PLG(broker_app_url, C), "/reconfigure"]), "application/json", jiffy:encode({[{<<"reconfiguration_type">>, <<"program-flowlet-app-config">>},{<<"config">>, ReconfigMap}]})),

542 %test new config right in consul

543 true = {200, ReconfigMap} == get_config_consul(C),

544 %test new config right in cdap

545 true = {200, ReconfigMap} == get_config_cdap(C),

546

547 %Test preferences reconfiguration

548 %check that the preferences in Consul is correct

549 InitMap = util:ejson_to_map(?PLG(init_preferences, C)),

550 true = {200, InitMap} == get_preferences_consul(C),

551 %check that the preferences in CDAP are correct

552 true = {200, InitMap} == get_preferences_cdap(C),

553 %reconfigure the preferences

554 PreferencesReconfigMap = util:ejson_to_map({[{<<"preffoo REDUX EDITION">>, <<"prefbar REMIXXX">>}, {<<"LEAVE ME ALONE">>, <<"PREFERENCES EDITION">>}]}),

555 {200, _} = httpabs:put(?XER, ?SC([?PLG(broker_app_url, C), "/reconfigure"]), "application/json", jiffy:encode({[{<<"reconfiguration_type">>, <<"program-flowlet-app-preferences">>},{<<"config">>, PreferencesReconfigMap}]})),

556 %make sure consul has right preferences

557 true = {200, PreferencesReconfigMap} == get_preferences_consul(C),

558 %make sure CDAP has right preferences

559 true = {200, PreferencesReconfigMap} == get_preferences_cdap(C),

560

561 %test the smart reconfiguration call

562 %try to give it a smart where there are keys in just preferences

563 SmartReconfigPrefMap = util:ejson_to_map({[{<<"preffoo REDUX EDITION">>, <<"BAR'D AGAIN">>}]}),

564 {200, _} = httpabs:put(?XER, ?SC([?PLG(broker_app_url, C), "/reconfigure"]), "application/json", jiffy:encode({[{<<"reconfiguration_type">>, <<"program-flowlet-smart">>},{<<"config">>, SmartReconfigPrefMap}]})),

565 ExpectedNewPreferences = #{<<"LEAVE ME ALONE">>=><<"PREFERENCES EDITION">>,<<"preffoo REDUX EDITION">>=><<"BAR'D AGAIN">>},

566 true = {200, ExpectedNewPreferences} == get_preferences_consul(C),

567 true = {200, ExpectedNewPreferences} == get_preferences_cdap(C),

568

569 %try to give it a smart where there are keys in just config

570 SmartReconfigConfig = {[{<<"foo REDUX EDITION">>, <<"FOO'D AGAIN">>}]},

571 {200, _} = httpabs:put(?XER, ?SC([?PLG(broker_app_url, C), "/reconfigure"]), "application/json", jiffy:encode({[{<<"reconfiguration_type">>, <<"program-flowlet-smart">>},{<<"config">>, SmartReconfigConfig}]})),

572 %make sure CDAP and Consul agree and are equal to what we expected

573 ExpectedNewConfig = #{<<"LEAVE ME ALONE">>=><<"CONFIG EDITION">>,<<"foo REDUX EDITION">>=><<"FOO'D AGAIN">>},

574 true = {200, ExpectedNewConfig} == get_config_consul(C),

575 true = {200, ExpectedNewConfig} == get_config_cdap(C),

576

577 %try to give it a smart where there are keys in both preferences and config

578 SmartReconfigBoth = {[{<<"foo REDUX EDITION">>, <<"FOO'D AGAIN AGAIN">>}, {<<"preffoo REDUX EDITION">>, <<"BAR'D AGAIN AGAIN">>}]},

579 {200, _} = httpabs:put(?XER, ?SC([?PLG(broker_app_url, C), "/reconfigure"]), "application/json", jiffy:encode({[{<<"reconfiguration_type">>, <<"program-flowlet-smart">>},{<<"config">>, SmartReconfigBoth}]})),

580 ExpectedNewPreferencesBoth = #{<<"LEAVE ME ALONE">>=><<"PREFERENCES EDITION">>,<<"preffoo REDUX EDITION">>=><<"BAR'D AGAIN AGAIN">>},

581 ExpectedNewConfigBoth = #{<<"LEAVE ME ALONE">>=><<"CONFIG EDITION">>,<<"foo REDUX EDITION">>=><<"FOO'D AGAIN AGAIN">>},

582 true = {200, ExpectedNewPreferencesBoth} == get_preferences_consul(C),

583 true = {200, ExpectedNewPreferencesBoth} == get_preferences_cdap(C),

584 true = {200, ExpectedNewConfigBoth} == get_config_consul(C),

585 true = {200, ExpectedNewConfigBoth} == get_config_cdap(C),

586

587 %try to give it a smart where there are no overlaps

588 SmartReconfigNone = {[{<<"EMPTY">>, <<"LIKE YOUR SOUL">>}]},

589 {400, _} = httpabs:put(?XER, ?SC([?PLG(broker_app_url, C), "/reconfigure"]), "application/json", jiffy:encode({[{<<"reconfiguration_type">>, <<"program-flowlet-smart">>},{<<"config">>, SmartReconfigNone}]})),

590 true = {200, ExpectedNewPreferencesBoth} == get_preferences_consul(C),

591 true = {200, ExpectedNewPreferencesBoth} == get_preferences_cdap(C),

592 true = {200, ExpectedNewConfigBoth} == get_config_consul(C),

593 true = {200, ExpectedNewConfigBoth} == get_config_cdap(C)

594 .

595

596 app_botch_flows(C) ->

597 %check healthcheck

598 {200, _} = httpabs:get(?XER,?SC([?PLG(broker_app_url, C), "/healthcheck"])),

599

600 %purposely shut down a flow "manually" to test that undeploy works with a "partially deployed" app

601 {200, []} = cdap_interface:exec_programs(?XER, ?PLG(appname, C), ?PLG(namespace, C), ?PLG(cdap_url, C),

602 [#program{type = <<"flows">>, id = <<"WhoFlow">>}, #program{type = <<"services">>, id = <<"Greeting">>}], "stop"),

603 %make sure healthcheck now fails

604 {400, _} = httpabs:get(?XER,?SC([?PLG(broker_app_url, C), "/healthcheck"]))

605 .

606

607 app_botch_delete(C) ->

608 %purposely shut down flows and then delete the app from the CDAP api to test undeploy works with a [gone] app

609 {200, []} = cdap_interface:exec_programs(?XER, ?PLG(appname, C), ?PLG(namespace, C), ?PLG(cdap_url, C),

610 [#program{type = <<"flows">>, id = <<"WhoFlow">>}, #program{type = <<"services">>, id = <<"Greeting">>}], "stop"),

611 {200, []} = cdap_interface:delete_app(?XER, ?PLG(appname, C), ?PLG(namespace, C), ?PLG(cdap_url, C)),

612

613 %make sure healthcheck now fails

614 {400, _} = httpabs:get(?XER,?SC([?PLG(broker_app_url, C), "/healthcheck"]))

615 .

616

617 app_botch_consul_delete(C) ->

618 %purposefully delete the config in consul to make sure delete doesnt blow up

619 {200, "true"} = consul_interface:consul_delete_config(?XER, ?PLG(appname, C),?PLG(consul_url, C)).

620

621 app_teardown(C) ->

622 %Test app teardown and delete

623 %app is there for now in broker

624 {200,_ } = httpabs:get(?XER,?PLG(broker_app_url, C)),

625

626 %teardown the test application

627 {200, []} = httpabs:delete(?XER, ?PLG(broker_app_url, C)),

628

629 %make sure the broker deleted the config from Consul

630 {404, _} = get_config_consul(C),

631

632 %make sure broker deleted the preferences

633 {404, _} = get_preferences_consul(C),

634

635 %make sure the broker app url no longer exists

636 {404, _ } = httpabs:get(?XER,?PLG(broker_app_url, C)),

637

638 %teardown the testing rels

639 {404, _} = setup_rels(C, teardown),

640

641 %teardown the fake service and make sure it is gone

642 {200, Srv} = setup_fake_testing_service(C, teardown),

643 true = Srv == "[]",

644

645 %cdap app gone

646 {404,"State: Not Found. Return Body: 'application:testns.hwtest.-SNAPSHOT' was not found."} = httpabs:get(?XER,?PLG(app_url, C)),

647

648 %make sure that the service is not registered. TODO! Could get more fancy by manually checking a healthcheck

649 {200, RBHC} = httpabs:get(?XER,?PLG(consul_app_url, C)),

650 true = jiffy:decode(RBHC) == [].

651

652 hydrator_app_teardown(C) ->

653 %Test app teardown and delete

654 %app is there for now in cdap

655 {200, _} = httpabs:get(?XER,?PLG(hydrator_app_url, C)),

656 %app is in broker

657 {200,_ } = httpabs:get(?XER,?PLG(broker_hydrator_app_url, C)),

658 %teardown the test application

659 {200, []} = httpabs:delete(?XER, ?PLG(broker_hydrator_app_url, C)),

660 %make sure the broker deleted the config from Consul

661 ?D(<<"todo! put this back:">>),

662 %{404, _} = get_config_consul(C),

663 %make sure the broker app url no longer exists

664 {404, _ } = httpabs:get(?XER,?PLG(broker_hydrator_app_url, C)),

665 %make sure gone from CDAP

666 {404,"State: Not Found. Return Body: 'application:testns.hydratortest.-SNAPSHOT' was not found."} = httpabs:get(?XER,?PLG(hydrator_app_url, C)),

667 %make sure that the service is not registered. TODO! Could get more fancy by manually checking a healthcheck

668 {200, RBHC} = httpabs:get(?XER,?PLG(consul_hydrator_app_url, C)),

669 true = jiffy:decode(RBHC) == []

670 .

671

672 hydrator_wdeps_test(C) ->

673 %test te app by injecting some data into the stream and getting it out

674 %Sleeping since HTTP services may still be booting up: see https://issues.cask.co/browse/CDAP-812

675 ok = timer:sleep(30000), %30s

676 %curl into stream

677 %grab the test data

678 {200, TestData} = httpabs:get(?XER,?PLG(hydrator_wdeps_test_data_url, C)),

679 %push it in

680 {200, _} = httpabs:post(?XER, ?SC([?PLG(hydrator_wdeps_stream_url, C), "/batch"]), "text/plain", TestData),

681 %query data out

682 PB = jiffy:encode({[{<<"query">>, <<"select ts from dataset_t1file">>}]}),

683 {200, RB} = httpabs:post(?XER, ?SC([?PLG(cdap_ns_url, C), "/data/explore/queries"]), "text/plain", PB),

684 {[{<<"handle">>, Handle}]} = jiffy:decode(RB),

685 %results can take time, sleep again

686 ok = timer:sleep(30000),

687 Expected = {[

688 {<<"status">>,<<"FINISHED">>},

689 {<<"hasResults">>,true}

690 ]},

691 {200, RB2} = httpabs:get(?XER,?SC([?PLG(cdap_url, C), "/v3/data/explore/queries", "/", Handle, "/status"])),

692 true = Expected == jiffy:decode(RB2),

693 {200, _} = httpabs:post(?XER, ?SC([?PLG(cdap_url, C), "/v3/data/explore/queries", "/", Handle, "/next"]), "text/plain", "")

694 .

695

696 hydrator_wdeps_teardown(C) ->

697 %Test app teardown and delete

698 %app is there for now in cdap

699 {200, _} = httpabs:get(?XER,?PLG(hydrator_wdeps_app_url, C)),

700 %app is in broker

701 {200,_ } = httpabs:get(?XER,?PLG(broker_hydrator_wdeps_app_url, C)),

702 %teardown the test application

703 {200, []} = httpabs:delete(?XER, ?PLG(broker_hydrator_wdeps_app_url, C)),

704 %make sure the broker deleted the config from Consul

705 ?D(<<"todo! put this back:">>),

706 %{404, _} = get_config_consul(C),

707 %make sure the broker app url no longer exists

708 {404, _ } = httpabs:get(?XER,?PLG(broker_hydrator_wdeps_app_url, C)),

709 %make sure gone from CDAP

710 {404,"State: Not Found. Return Body: 'application:testns.hydratortest.-SNAPSHOT' was not found."} = httpabs:get(?XER,?PLG(hydrator_app_url, C)),

711 %make sure that the service is not registered. TODO! Could get more fancy by manually checking a healthcheck

712 {200, RBHC} = httpabs:get(?XER,?PLG(consul_hydrator_wdeps_app_url, C)),

713 true = jiffy:decode(RBHC) == []

714 .

715

716 test_failures(C) ->

717 %test things that should fail

718 %delete a non-existent app

719 {404, "State: Not Found. Return Body: Tried to delete an application that was not registered"} =

720 httpabs:delete(?XER, ?SC([?PLG(broker_app_url, C), "MYFRIENDOFMISERY"])),

721

722 %malformed Broker put

723 URL = ?SC([?PLG(broker_app_url, C), "FAILURETEST"]),

724

725 %deploy a bad CDAP app with a bad program_id

726 Body2 = {[

727 {<<"cdap_application_type">>, <<"program-flowlet">>},

728 {<<"namespace">>, ?PLG(namespace, C)},

729 {<<"streamname">>, ?PLG(streamname, C)},

730 {<<"jar_url">>, ?PLG(jar_url, C)},

731 {<<"artifact_name">>, ?PLG(art_name, C)},

732 {<<"artifact_version">>, ?PLG(art_ver, C)},

733 {<<"app_config">>, ?PLG(init_config, C)},

734 {<<"app_preferences">>, ?PLG(init_preferences, C)},

735 {<<"services">>, [{[{<<"service_name">>, <<"Greeting">>},

736 {<<"service_endpoint">>, <<"greet">>},

737 {<<"endpoint_method">>, <<"GET">>}]}]},

738 {<<"programs">>, [

739 {[{<<"program_type">>, <<"flows">>},

740 {<<"program_id">>, <<"DISSAPOINTMENT">>}]}

741 ]},

742 {<<"program_preferences">>, []}

743 ]},

744 %WORKS IN CDAP 3:

745 %{404,"State: Not Found. Return Body: State: Not Found. Return Body: 'program:testns.hwtestFAILURETEST.flow.DISSAPOINTMENT' was not found."} = httpabs:put(?XER, URL, "application/json", jiffy:encode(Body2)),

746 %WORKS IN CDAP 4 (looks like they are doing more intrispection on the jar name)

747 {404,_} = httpabs:put(?XER, URL, "application/json", jiffy:encode(Body2)),

748 %make sure the rollback happened

749 {200, "[]"} = httpabs:get(?XER,?SC([?PLG(broker_url, C), "/application"])),

750

751 %try to deploy with a bad URL where bad means nonexistent (504)

752 Body3 = {[

753 {<<"cdap_application_type">>, <<"program-flowlet">>},

754 {<<"namespace">>, ?PLG(namespace, C)},

755 {<<"streamname">>, ?PLG(streamname, C)},

756 {<<"jar_url">>, ?SC([?PLG(jar_url, C), "DOESNOTEXISTMOSTLIKELY"])},

757 {<<"artifact_name">>, ?PLG(art_name, C)},

758 {<<"artifact_version">>, ?PLG(art_ver, C)},

759 {<<"app_config">>, ?PLG(init_config, C)},

760 {<<"app_preferences">>, ?PLG(init_preferences, C)},

761 {<<"services">>, [{[{<<"service_name">>, <<"Greeting">>}, {<<"service_endpoint">>, <<"greet">>}, {<<"endpoint_method">>, <<"GET">>}]}]},

762 {<<"programs">>, [{[{<<"program_type">>, <<"flows">>},{<<"program_id">>, <<"WhoFlow">>}]},{[{<<"program_type">>, <<"services">>},{<<"program_id">>, <<"Greeting">>}]}]},

763 {<<"program_preferences">>, [{[{<<"program_type">>,<<"flows">>}, {<<"program_id">>, <<"WhoFlow">>}, {<<"program_pref">>, ?PLG(whoflowpref, C)}]}]}

764 ]},

765 {404, _} = httpabs:put(?XER, URL, "application/json", jiffy:encode(Body3)).

766

767 delete_all(C) ->

768 %test invalid key

769 Body1 = jiffy:encode({[{<<"ids">>, [<<"hwtest">>]}]}),

770 {400,"State: Bad Request. Return Body: Invalid PUT Body"} = httpabs:post(?XER, ?SC([?PLG(broker_url, C), "/application/delete"]), "application/json", Body1),

771 %test invalid: not a list

772 Body2 = jiffy:encode({[{<<"appnames">>, <<"hwtest">>}]}),

773 {400,"State: Bad Request. Return Body: Invalid PUT Body"} = httpabs:post(?XER, ?SC([?PLG(broker_url, C), "/application/delete"]), "application/json", Body2),

774 %test undeploy a real app and also an app that is not deployed

775 Body3 = jiffy:encode({[{<<"appnames">>, [<<"hwtest">>, <<"dissapointment">>]}]}),

776 {200, "[200,404]"} = httpabs:post(?XER, ?SC([?PLG(broker_url, C), "/application/delete"]), "application/json", Body3),

777 %teardown the fake service and make sure it is gone

778 {200, Srv} = setup_fake_testing_service(C, teardown),

779 true = Srv == "[]".

780