2 * ============LICENSE_START=======================================================
3 * ONAP : ccsdk features
4 * ================================================================================
5 * Copyright (C) 2019 highstreet technologies GmbH Intellectual Property.
7 * ================================================================================
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 * ============LICENSE_END=========================================================
22 package org.onap.ccsdk.features.sdnr.wt.dataprovider.database.elasticsearch.data.entity;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.Date;
27 import java.util.HashMap;
28 import java.util.List;
30 import java.util.Optional;
31 import javax.annotation.Nonnull;
32 import org.eclipse.jdt.annotation.NonNull;
33 import org.eclipse.jdt.annotation.Nullable;
34 import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient;
35 import org.onap.ccsdk.features.sdnr.wt.common.database.queries.BoolQueryBuilder;
36 import org.onap.ccsdk.features.sdnr.wt.common.database.queries.QueryBuilder;
37 import org.onap.ccsdk.features.sdnr.wt.common.database.queries.QueryBuilders;
38 import org.onap.ccsdk.features.sdnr.wt.common.database.queries.RangeQueryBuilder;
39 import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.DatabaseDataProvider;
40 import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.elasticsearch.EsDataObjectReaderWriter2;
41 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.ArchiveCleanProvider;
42 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.DataInconsistencyException;
43 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.DataProvider;
44 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.NetconfTimeStamp;
45 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.types.NetconfTimeStampImpl;
46 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.DateAndTime;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.ConnectionLogStatus;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.ConnectionlogBuilder;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.ConnectionlogEntity;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Entity;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.EventlogBuilder;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.EventlogEntity;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultcurrentBuilder;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultcurrentEntity;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultlogBuilder;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultlogEntity;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.GranularityPeriodType;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Guicutthrough;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.GuicutthroughBuilder;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.GuicutthroughEntity;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Inventory;
62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.InventoryBuilder;
63 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.InventoryEntity;
64 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.NetworkElementConnectionBuilder;
65 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.NetworkElementConnectionEntity;
66 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.PmdataEntity;
67 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.PmdataEntityBuilder;
68 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.UpdateNetworkElementConnectionInputBuilder;
69 import org.opendaylight.yangtools.yang.common.Uint32;
70 import org.slf4j.Logger;
71 import org.slf4j.LoggerFactory;
74 * Event service, writing all events into the database into the appropriate index.
78 public class HtDatabaseEventsService implements ArchiveCleanProvider, DataProvider {
79 private static final Logger LOG = LoggerFactory.getLogger(HtDatabaseEventsService.class);
81 private static final NetconfTimeStamp NETCONFTIME_CONVERTER = NetconfTimeStampImpl.getConverter();
83 private static final int ROOT_TREE_LEVEL = 0;
85 private HtDatabaseClient client;
86 private EsDataObjectReaderWriter2<EventlogEntity> eventRWEventLogDevicemanager;
87 private EsDataObjectReaderWriter2<InventoryEntity> eventRWEquipment;
88 private EsDataObjectReaderWriter2<FaultcurrentEntity> eventRWFaultCurrentDB;
89 private EsDataObjectReaderWriter2<FaultlogEntity> eventRWFaultLogDB;
90 private EsDataObjectReaderWriter2<ConnectionlogEntity> eventRWConnectionLogDB;
91 private final EsDataObjectReaderWriter2<NetworkElementConnectionEntity> networkelementConnectionDB;
92 private final EsDataObjectReaderWriter2<GuicutthroughEntity> guiCutThroughDB;
93 private final EsDataObjectReaderWriter2<PmdataEntity> pmData15mDB;
94 private final EsDataObjectReaderWriter2<PmdataEntity> pmData24hDB;
96 @SuppressWarnings("unused")
97 private final DatabaseDataProvider dataProvider;
98 // --- Construct and initialize
100 public HtDatabaseEventsService(HtDatabaseClient client, DatabaseDataProvider elasticSearchDataProvider)
103 LOG.info("Create {} start", HtDatabaseEventsService.class);
104 this.dataProvider = elasticSearchDataProvider;
107 // Create control structure
108 this.client = client;
110 eventRWEventLogDevicemanager = new EsDataObjectReaderWriter2<>(client, Entity.Eventlog,
111 EventlogEntity.class, EventlogBuilder.class);
113 eventRWEquipment = new EsDataObjectReaderWriter2<>(client, Entity.Inventoryequipment, InventoryEntity.class,
114 InventoryBuilder.class);
116 eventRWFaultCurrentDB = new EsDataObjectReaderWriter2<>(client, Entity.Faultcurrent,
117 FaultcurrentEntity.class, FaultcurrentBuilder.class);
119 eventRWFaultLogDB = new EsDataObjectReaderWriter2<>(client, Entity.Faultlog, FaultlogEntity.class,
120 FaultlogBuilder.class);
122 eventRWConnectionLogDB = new EsDataObjectReaderWriter2<>(client, Entity.Connectionlog,
123 ConnectionlogEntity.class, ConnectionlogBuilder.class);
125 networkelementConnectionDB = new EsDataObjectReaderWriter2<>(client, Entity.NetworkelementConnection,
126 NetworkElementConnectionEntity.class, NetworkElementConnectionBuilder.class, true)
127 .setEsIdAttributeName("_id");
129 guiCutThroughDB = new EsDataObjectReaderWriter2<>(client, Entity.Guicutthrough, GuicutthroughEntity.class,
130 GuicutthroughBuilder.class);
132 pmData15mDB = new EsDataObjectReaderWriter2<>(client, Entity.Historicalperformance15min, PmdataEntity.class,
133 PmdataEntityBuilder.class);
135 pmData24hDB = new EsDataObjectReaderWriter2<>(client, Entity.Historicalperformance24h, PmdataEntity.class,
136 PmdataEntityBuilder.class);
138 } catch (Exception e) {
139 LOG.error("Can not start database client. Exception: {}", e);
140 throw new Exception("Can not start database client. Exception: {}", e);
142 LOG.info("Create {} finished. DB Service {} started.", HtDatabaseEventsService.class,
143 client != null ? "sucessfully" : "not");
150 public void writeConnectionLog(ConnectionlogEntity event) {
151 if (assertIfClientNull(event)) {
154 LOG.debug("Write event: {}", event);
155 eventRWConnectionLogDB.write(event, null);
161 public void writeEventLog(EventlogEntity event) {
162 if (assertIfClientNull("No client to write {}", event)) {
166 LOG.debug("Write event: {}", event.toString());
167 eventRWEventLogDevicemanager.write(event, null);
173 public void writeFaultLog(FaultlogEntity fault) {
174 if (assertIfClientNull(fault)) {
178 LOG.debug("Write fault to faultlog: {}", fault.toString());
179 eventRWFaultLogDB.write(fault, null);
185 public void updateFaultCurrent(FaultcurrentEntity fault) {
186 if (assertIfClientNull(fault)) {
190 if (FaultEntityManager.isManagedAsCurrentProblem(fault)) {
191 if (FaultEntityManager.isNoAlarmIndication(fault)) {
192 LOG.debug("Remove from currentFaults: {}", fault.toString());
193 eventRWFaultCurrentDB.remove(FaultEntityManager.genSpecificEsId(fault));
195 LOG.debug("Write to currentFaults: {}", fault.toString());
196 eventRWFaultCurrentDB.write(fault, FaultEntityManager.genSpecificEsId(fault));
199 LOG.debug("Ingnore for currentFaults: {}", fault.toString());
204 * Remove all entries for one node
206 * @param nodeName contains the mountpointname
207 * @return number of deleted entries
210 public int clearFaultsCurrentOfNode(String nodeName) {
211 if (assertIfClientNullForNodeName(nodeName)) {
215 LOG.debug("Remove from currentFaults all faults for node: {}", nodeName);
216 return eventRWFaultCurrentDB.remove(EsFaultCurrent.getQueryForOneNode(nodeName));
220 * Remove all entries for one node
222 * @param nodeName contains the mountpointname
223 * @param objectId of element to be deleted
224 * @return number of deleted entries
227 public int clearFaultsCurrentOfNodeWithObjectId(String nodeName, String objectId) {
228 if (assertIfClientNullForNodeName(nodeName)) {
232 LOG.debug("Remove from currentFaults all faults for node/objectId: {}/{}", nodeName, objectId);
233 return eventRWFaultCurrentDB.remove(EsFaultCurrent.getQueryForOneNodeAndObjectId(nodeName, objectId));
237 * Deliver list with all mountpoint/node-names in the database.
239 * @return List of all mountpoint/node-names the had active alarms.
242 public @Nonnull List<String> getAllNodesWithCurrentAlarms() {
243 if (assertIfClientNull("No DB, can not delete for all nodes", null)) {
244 return new ArrayList<>();
247 LOG.debug("Remove from currentFaults faults for all node");
248 List<String> nodeNames = new ArrayList<>();
250 for (FaultcurrentEntity fault : eventRWFaultCurrentDB.doReadAll().getHits()) {
251 String nodeName = fault.getNodeId();
252 if (!nodeNames.contains(nodeName)) {
253 // this.clearFaultsCurrentOfNode(nodeName); -> Function shifted
254 nodeNames.add(nodeName);
260 // -- Inventory and equipment current
263 * write internal equipment to database
265 * @param internalEquipment with mandatory fields.
268 private void writeInventory(Inventory internalEquipment) {
270 if (internalEquipment.getManufacturerIdentifier() == null) {
271 internalEquipment = new InventoryBuilder(internalEquipment).setManufacturerIdentifier("").build();
273 if (internalEquipment.getDate() == null) {
274 internalEquipment = new InventoryBuilder(internalEquipment).setDate("").build();
276 eventRWEquipment.write(internalEquipment, internalEquipment.getNodeId() + "/" + internalEquipment.getUuid());
280 * write internal equipment to database
286 public void writeInventory(String nodeId, List<Inventory> list) {
289 checkConsistency(nodeId, list);
290 } catch (DataInconsistencyException e) {
291 LOG.warn("inventory list for node {} is not consistent", nodeId, e);
292 list = e.getRepairedList();
295 for (Inventory internalEquipment : list) {
296 this.writeInventory(internalEquipment);
300 private static void checkConsistency(String nodeId, List<Inventory> list) throws DataInconsistencyException {
301 final String UNBOUND_INVENTORY_UUID = "unbound";
302 List<String> failures = new ArrayList<>();
305 Map<String, Inventory> repairList = new HashMap<>();
306 InventoryBuilder repairedItem;
307 InventoryBuilder unboundItem = new InventoryBuilder().setNodeId(nodeId).setUuid(UNBOUND_INVENTORY_UUID)
308 .setTreeLevel(Uint32.valueOf(0));;
309 for (Inventory item : list) {
310 repairedItem = new InventoryBuilder(item);
311 // check for bad node-id
312 if (!nodeId.equals(item.getNodeId())) {
313 failures.add(String.format("missing node-id for equipment(uuid=%s)", item.getUuid()));
314 repairedItem.setNodeId(nodeId);
317 // check missing tree-level
318 if (item.getTreeLevel() == null) {
319 failures.add(String.format("missing tree-level for equipment(uuid=%s)", item.getUuid()));
320 repairedItem.setTreeLevel(Uint32.valueOf(ROOT_TREE_LEVEL));
324 treeLevel = item.getTreeLevel().longValue();
325 if (treeLevel > ROOT_TREE_LEVEL) {
326 // check non root elem and missing parent
327 if (item.getParentUuid() == null) {
328 failures.add(String.format("Non root level element (uuid=%s) has to have a parent element",
331 repairedItem.setParentUuid(UNBOUND_INVENTORY_UUID);
332 repairList.put(unboundItem.getUuid(), unboundItem.build());
334 // check that parent exists in list and is tree-level -1
336 Optional<Inventory> parent =
337 list.stream().filter(e -> item.getParentUuid().equals(e.getUuid())).findFirst();
338 if (parent.isEmpty()) {
339 failures.add(String.format("no parent found for uuid=%s with parent-uuid=%s",
340 item.getUuid(), item.getParentUuid()));
341 repairedItem.setParentUuid(UNBOUND_INVENTORY_UUID);
346 // check for duplicated uui
347 Optional<Inventory> duplicate = list
348 .stream().filter(e -> !item.equals(e) && item.getUuid() != null
349 && item.getUuid().equals(e.getUuid()) && repairList.containsKey(e.getUuid()))
351 if (duplicate.isPresent()) {
352 failures.add(String.format("found duplicate uuid=%s", item.getUuid()));
357 if (failCounter > 0) {
358 repairList.put(repairedItem.getUuid(), repairedItem.build());
360 repairList.put(item.getUuid(), item);
365 if (failures.size() > 0) {
366 throw new DataInconsistencyException(new ArrayList<>(repairList.values()),
367 "inventory list is not consistent;\n" + String.join("\n", failures));
374 * join base with parameters of toJoin (only non null values)
376 * @param base base object
377 * @param toJoin object with new property values
378 * @return new joined object
380 @SuppressWarnings("unused")
381 private NetworkElementConnectionEntity joinNe(NetworkElementConnectionEntity base,
382 NetworkElementConnectionEntity toJoin) {
386 NetworkElementConnectionBuilder builder = new NetworkElementConnectionBuilder(base);
387 if (toJoin != null) {
388 if (toJoin.isIsRequired() != null) {
389 builder.setIsRequired(toJoin.isIsRequired());
391 if (toJoin.getCoreModelCapability() != null) {
392 builder.setCoreModelCapability(toJoin.getCoreModelCapability());
394 if (toJoin.getDeviceType() != null) {
395 builder.setDeviceType(toJoin.getDeviceType());
397 if (toJoin.getHost() != null) {
398 builder.setHost(toJoin.getHost());
400 if (toJoin.getNodeDetails() != null) {
401 builder.setNodeDetails(toJoin.getNodeDetails());
403 if (toJoin.getPassword() != null) {
404 builder.setPassword(toJoin.getPassword());
406 if (toJoin.getPort() != null) {
407 builder.setPort(toJoin.getPort());
409 if (toJoin.getStatus() != null) {
410 builder.setStatus(toJoin.getStatus());
412 if (toJoin.getUsername() != null) {
413 builder.setUsername(toJoin.getUsername());
416 return builder.build();
421 * @param networkElementConnectionEntitiy to wirte to DB
422 * @param nodeId Id for this DB element
425 public boolean updateNetworkConnectionDeviceType(NetworkElementConnectionEntity networkElementConnectionEntitiy,
427 return this.networkelementConnectionDB.update(networkElementConnectionEntitiy, nodeId) != null;
428 // NetworkElementConnectionEntity e =
429 // this.networkelementConnectionDB.read(nodeId);
430 // this.networkelementConnectionDB.write(this.joinNe(e,
431 // networkElementConnectionEntitiy), nodeId);
435 * Update after new mountpoint registration
437 * @param networkElementConnectionEntitiy data
438 * @param nodeId of device (mountpoint name)
441 public boolean updateNetworkConnection22(NetworkElementConnectionEntity networkElementConnectionEntitiy,
443 LOG.info("update networkelement-connection for {} with data {}", nodeId, networkElementConnectionEntitiy);
444 return this.networkelementConnectionDB.updateOrCreate(networkElementConnectionEntitiy, nodeId,
445 Arrays.asList("is-required", "username", "password")) != null;
446 // NetworkElementConnectionEntity e =
447 // this.networkelementConnectionDB.read(nodeId);
448 // this.networkelementConnectionDB.write(this.joinNe(e,
449 // networkElementConnectionEntitiy), nodeId);
453 /* please do not remove */
454 // public void cleanNetworkElementConnections() {
455 // this.networkelementConnectionDB.remove(QueryBuilders.matchQuery("is-required",
457 // CreateNetworkElementConnectionInput x = new
458 // CreateNetworkElementConnectionInputBuilder().setStatus(ConnectionLogStatus.Disconnected).build();
459 // this.networkelementConnectionDB.update(x,QueryBuilders.matchAllQuery());
463 public void removeNetworkConnection(String nodeId) {
465 NetworkElementConnectionEntity e = this.networkelementConnectionDB.read(nodeId);
466 if (e != null && (isRequired = e.isIsRequired()) != null) {
468 LOG.debug("updating connection status for {} of required ne to disconnected", nodeId);
469 this.networkelementConnectionDB.update(new UpdateNetworkElementConnectionInputBuilder()
470 .setStatus(ConnectionLogStatus.Disconnected).build(), nodeId);
472 LOG.debug("remove networkelement-connection for {} entry because of non-required", nodeId);
473 this.networkelementConnectionDB.remove(nodeId);
476 LOG.warn("Unable to update connection-status. dbentry for {} not found in networkelement-connection",
484 public int doIndexClean(Date olderAreOutdated) {
486 String netconfTimeStamp = NETCONFTIME_CONVERTER.getTimeStampAsNetconfString(olderAreOutdated);
489 QueryBuilder queryEventBase = EsEventBase.getQueryForTimeStamp(netconfTimeStamp);
490 removed += eventRWEventLogDevicemanager.remove(queryEventBase);
492 QueryBuilder queryFaultLog = EsFaultLogDevicemanager.getQueryForTimeStamp(netconfTimeStamp);
493 removed += eventRWFaultLogDB.remove(queryFaultLog);
498 public long getNumberOfOldObjects(Date olderAreOutdated) {
500 String netconfTimeStamp = NETCONFTIME_CONVERTER.getTimeStampAsNetconfString(olderAreOutdated);
501 int numberOfElements = 0;
503 QueryBuilder queryEventBase = EsEventBase.getQueryForTimeStamp(netconfTimeStamp);
504 numberOfElements += eventRWEventLogDevicemanager.doReadAll(queryEventBase).getTotal();
506 QueryBuilder queryFaultLog = EsFaultLogDevicemanager.getQueryForTimeStamp(netconfTimeStamp);
507 numberOfElements += eventRWFaultLogDB.doReadAll(queryFaultLog).getTotal();
509 return numberOfElements;
515 * Verify status of client
517 * @param event that is printed with message
518 * @return true if client is null
520 private boolean assertIfClientNull(Object event) {
521 return assertIfClientNull("No DB, can not write: {}", event);
524 private boolean assertIfClientNullForNodeName(Object object) {
525 return assertIfClientNull("No DB, can not handle node: {}", object);
529 * Verify status of client
531 * @param message to print including {} for object printout.
532 * @return true if client is null
534 private boolean assertIfClientNull(String message, Object object) {
535 if (client == null) {
536 LOG.debug(message, object);
544 private static class EsEventBase {
546 * Query to get older Elements
548 * @param netconfTimeStamp to identify older Elements
549 * @return QueryBuilder for older elements related to timestamp
551 private static QueryBuilder getQueryForTimeStamp(String netconfTimeStamp) {
552 return new RangeQueryBuilder("timestamp").lte(netconfTimeStamp);
556 private static class EsFaultLogDevicemanager {
560 * @param netconfTimeStamp to identify query elements older than this timestamp.
561 * @return QueryBuilder for related elements
563 public static QueryBuilder getQueryForTimeStamp(String netconfTimeStamp) {
564 return new RangeQueryBuilder("timestamp").lte(netconfTimeStamp);
568 public static class EsFaultCurrent {
570 * @param nodeName name of the node
571 * @return query builder
573 public static QueryBuilder getQueryForOneNode(String nodeName) {
574 return QueryBuilders.matchQuery("node-id", nodeName);
577 public static QueryBuilder getQueryForOneNodeAndObjectId(String nodeName, String objectId) {
578 BoolQueryBuilder bq = QueryBuilders.boolQuery();
579 bq.must(QueryBuilders.matchQuery("node-id", nodeName));
580 bq.must(QueryBuilders.matchQuery("object-id", objectId));
586 public List<NetworkElementConnectionEntity> getNetworkElementConnections() {
587 return this.networkelementConnectionDB.doReadAll().getHits();
591 public void doWritePerformanceData(List<PmdataEntity> list) {
593 list.forEach(elem -> {
594 GranularityPeriodType granularityPeriod = nnGetGranularityPeriodType(elem.getGranularityPeriod());
595 // _id": "Sim12600/LP-MWPS-TTP-01/2017-07-04T15:15:00.0+00:00"
596 StringBuffer id = new StringBuffer();
597 DateAndTime date = elem.getTimeStamp();
598 id.append(elem.getNodeName());
600 id.append(elem.getUuidInterface());
602 id.append(date != null ? date.getValue() : "null");
604 switch (granularityPeriod) {
606 pmData15mDB.write(elem, id.toString());
609 pmData24hDB.write(elem, id.toString());
613 LOG.debug("Unknown granularity {} id {}", granularityPeriod, id);
621 GranularityPeriodType nnGetGranularityPeriodType(@Nullable GranularityPeriodType granularityPeriod) {
622 return granularityPeriod != null ? granularityPeriod : GranularityPeriodType.Unknown;
626 public HtDatabaseClient getRawClient() {
631 public void writeGuiCutThroughData(Guicutthrough gcData, String nodeId) {
632 guiCutThroughDB.write(gcData, nodeId);
636 public int clearGuiCutThroughEntriesOfNode(String nodeName) {
637 guiCutThroughDB.remove(nodeName);