2 * ============LICENSE_START=======================================================
3 * ONAP : ccsdk features
4 * ================================================================================
5 * Copyright (C) 2019 highstreet technologies GmbH Intellectual Property.
7 * ================================================================================
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 * ============LICENSE_END=========================================================
22 package org.onap.ccsdk.features.sdnr.wt.dataprovider.database.elasticsearch.data.entity;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.Date;
27 import java.util.HashMap;
28 import java.util.List;
30 import java.util.Optional;
31 import javax.annotation.Nonnull;
32 import org.eclipse.jdt.annotation.NonNull;
33 import org.eclipse.jdt.annotation.Nullable;
34 import org.onap.ccsdk.features.sdnr.wt.common.database.HtDatabaseClient;
35 import org.onap.ccsdk.features.sdnr.wt.common.database.queries.BoolQueryBuilder;
36 import org.onap.ccsdk.features.sdnr.wt.common.database.queries.QueryBuilder;
37 import org.onap.ccsdk.features.sdnr.wt.common.database.queries.QueryBuilders;
38 import org.onap.ccsdk.features.sdnr.wt.common.database.queries.RangeQueryBuilder;
39 import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.DatabaseDataProvider;
40 import org.onap.ccsdk.features.sdnr.wt.dataprovider.database.elasticsearch.EsDataObjectReaderWriter2;
41 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.ArchiveCleanProvider;
42 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.DataInconsistencyException;
43 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.DataProvider;
44 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.NetconfTimeStamp;
45 import org.onap.ccsdk.features.sdnr.wt.dataprovider.model.types.NetconfTimeStampImpl;
46 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.DateAndTime;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.ConnectionLogStatus;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.ConnectionlogBuilder;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.ConnectionlogEntity;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Entity;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.EventlogBuilder;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.EventlogEntity;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultcurrentBuilder;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultcurrentEntity;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultlogBuilder;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.FaultlogEntity;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.GranularityPeriodType;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Guicutthrough;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.GuicutthroughBuilder;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.GuicutthroughEntity;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.Inventory;
62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.InventoryBuilder;
63 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.InventoryEntity;
64 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.NetworkElementConnectionBuilder;
65 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.NetworkElementConnectionEntity;
66 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.PmdataEntity;
67 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.PmdataEntityBuilder;
68 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.data.provider.rev201110.UpdateNetworkElementConnectionInputBuilder;
69 import org.opendaylight.yangtools.yang.common.Uint32;
70 import org.slf4j.Logger;
71 import org.slf4j.LoggerFactory;
74 * Event service, writing all events into the database into the appropriate index.
78 public class HtDatabaseEventsService implements ArchiveCleanProvider, DataProvider {
79 private static final Logger LOG = LoggerFactory.getLogger(HtDatabaseEventsService.class);
81 private static final NetconfTimeStamp NETCONFTIME_CONVERTER = NetconfTimeStampImpl.getConverter();
83 private HtDatabaseClient client;
84 private EsDataObjectReaderWriter2<EventlogEntity> eventRWEventLogDevicemanager;
85 private EsDataObjectReaderWriter2<InventoryEntity> eventRWEquipment;
86 private EsDataObjectReaderWriter2<FaultcurrentEntity> eventRWFaultCurrentDB;
87 private EsDataObjectReaderWriter2<FaultlogEntity> eventRWFaultLogDB;
88 private EsDataObjectReaderWriter2<ConnectionlogEntity> eventRWConnectionLogDB;
89 private final EsDataObjectReaderWriter2<NetworkElementConnectionEntity> networkelementConnectionDB;
90 private final EsDataObjectReaderWriter2<GuicutthroughEntity> guiCutThroughDB;
91 private final EsDataObjectReaderWriter2<PmdataEntity> pmData15mDB;
92 private final EsDataObjectReaderWriter2<PmdataEntity> pmData24hDB;
94 @SuppressWarnings("unused")
95 private final DatabaseDataProvider dataProvider;
96 // --- Construct and initialize
98 public HtDatabaseEventsService(HtDatabaseClient client, DatabaseDataProvider elasticSearchDataProvider)
101 LOG.info("Create {} start", HtDatabaseEventsService.class);
102 this.dataProvider = elasticSearchDataProvider;
105 // Create control structure
106 this.client = client;
108 eventRWEventLogDevicemanager = new EsDataObjectReaderWriter2<>(client, Entity.Eventlog,
109 EventlogEntity.class, EventlogBuilder.class);
111 eventRWEquipment = new EsDataObjectReaderWriter2<>(client, Entity.Inventoryequipment, InventoryEntity.class,
112 InventoryBuilder.class);
114 eventRWFaultCurrentDB = new EsDataObjectReaderWriter2<>(client, Entity.Faultcurrent,
115 FaultcurrentEntity.class, FaultcurrentBuilder.class);
117 eventRWFaultLogDB = new EsDataObjectReaderWriter2<>(client, Entity.Faultlog, FaultlogEntity.class,
118 FaultlogBuilder.class);
120 eventRWConnectionLogDB = new EsDataObjectReaderWriter2<>(client, Entity.Connectionlog,
121 ConnectionlogEntity.class, ConnectionlogBuilder.class);
123 networkelementConnectionDB = new EsDataObjectReaderWriter2<>(client, Entity.NetworkelementConnection,
124 NetworkElementConnectionEntity.class, NetworkElementConnectionBuilder.class, true)
125 .setEsIdAttributeName("_id");
127 guiCutThroughDB = new EsDataObjectReaderWriter2<>(client, Entity.Guicutthrough, GuicutthroughEntity.class,
128 GuicutthroughBuilder.class);
130 pmData15mDB = new EsDataObjectReaderWriter2<>(client, Entity.Historicalperformance15min, PmdataEntity.class,
131 PmdataEntityBuilder.class);
133 pmData24hDB = new EsDataObjectReaderWriter2<>(client, Entity.Historicalperformance24h, PmdataEntity.class,
134 PmdataEntityBuilder.class);
136 } catch (Exception e) {
137 LOG.error("Can not start database client. Exception: {}", e);
138 throw new Exception("Can not start database client. Exception: {}", e);
140 LOG.info("Create {} finished. DB Service {} started.", HtDatabaseEventsService.class,
141 client != null ? "sucessfully" : "not");
148 public void writeConnectionLog(ConnectionlogEntity event) {
149 if (assertIfClientNull(event)) {
152 LOG.debug("Write event: {}", event);
153 eventRWConnectionLogDB.write(event, null);
159 public void writeEventLog(EventlogEntity event) {
160 if (assertIfClientNull("No client to write {}", event)) {
164 LOG.debug("Write event: {}", event.toString());
165 eventRWEventLogDevicemanager.write(event, null);
171 public void writeFaultLog(FaultlogEntity fault) {
172 if (assertIfClientNull(fault)) {
176 LOG.debug("Write fault to faultlog: {}", fault.toString());
177 eventRWFaultLogDB.write(fault, null);
183 public void updateFaultCurrent(FaultcurrentEntity fault) {
184 if (assertIfClientNull(fault)) {
188 if (FaultEntityManager.isManagedAsCurrentProblem(fault)) {
189 if (FaultEntityManager.isNoAlarmIndication(fault)) {
190 LOG.debug("Remove from currentFaults: {}", fault.toString());
191 eventRWFaultCurrentDB.remove(FaultEntityManager.genSpecificEsId(fault));
193 LOG.debug("Write to currentFaults: {}", fault.toString());
194 eventRWFaultCurrentDB.write(fault, FaultEntityManager.genSpecificEsId(fault));
197 LOG.debug("Ingnore for currentFaults: {}", fault.toString());
202 * Remove all entries for one node
204 * @param nodeName contains the mountpointname
205 * @return number of deleted entries
208 public int clearFaultsCurrentOfNode(String nodeName) {
209 if (assertIfClientNullForNodeName(nodeName)) {
213 LOG.debug("Remove from currentFaults all faults for node: {}", nodeName);
214 return eventRWFaultCurrentDB.remove(EsFaultCurrent.getQueryForOneNode(nodeName));
218 * Remove all entries for one node
220 * @param nodeName contains the mountpointname
221 * @param objectId of element to be deleted
222 * @return number of deleted entries
225 public int clearFaultsCurrentOfNodeWithObjectId(String nodeName, String objectId) {
226 if (assertIfClientNullForNodeName(nodeName)) {
230 LOG.debug("Remove from currentFaults all faults for node/objectId: {}/{}", nodeName, objectId);
231 return eventRWFaultCurrentDB.remove(EsFaultCurrent.getQueryForOneNodeAndObjectId(nodeName, objectId));
235 * Deliver list with all mountpoint/node-names in the database.
237 * @return List of all mountpoint/node-names the had active alarms.
240 public @Nonnull List<String> getAllNodesWithCurrentAlarms() {
241 if (assertIfClientNull("No DB, can not delete for all nodes", null)) {
242 return new ArrayList<>();
245 LOG.debug("Remove from currentFaults faults for all node");
246 List<String> nodeNames = new ArrayList<>();
248 for (FaultcurrentEntity fault : eventRWFaultCurrentDB.doReadAll().getHits()) {
249 String nodeName = fault.getNodeId();
250 if (!nodeNames.contains(nodeName)) {
251 // this.clearFaultsCurrentOfNode(nodeName); -> Function shifted
252 nodeNames.add(nodeName);
258 // -- Inventory and equipment current
261 * write internal equipment to database
263 * @param internalEquipment with mandatory fields.
266 private void writeInventory(Inventory internalEquipment) {
268 if (internalEquipment.getManufacturerIdentifier() == null) {
269 internalEquipment = new InventoryBuilder(internalEquipment).setManufacturerIdentifier("").build();
271 if (internalEquipment.getDate() == null) {
272 internalEquipment = new InventoryBuilder(internalEquipment).setDate("").build();
274 eventRWEquipment.write(internalEquipment, internalEquipment.getNodeId() + "/" + internalEquipment.getUuid());
278 * write internal equipment to database
284 public void writeInventory(String nodeId, List<Inventory> list) {
287 checkConsistency(nodeId, list);
288 } catch (DataInconsistencyException e) {
289 LOG.warn("inventory list for node {} is not consistent", nodeId, e);
290 list = e.getRepairedList();
293 for (Inventory internalEquipment : list) {
294 this.writeInventory(internalEquipment);
298 private static void checkConsistency(String nodeId, List<Inventory> list) throws DataInconsistencyException {
299 final String UNBOUND_INVENTORY_UUID = "unbound";
300 List<String> failures = new ArrayList<>();
303 Map<String, Inventory> repairList = new HashMap<>();
304 InventoryBuilder repairedItem;
305 InventoryBuilder unboundItem = new InventoryBuilder().setNodeId(nodeId).setUuid(UNBOUND_INVENTORY_UUID)
306 .setTreeLevel(Uint32.valueOf(0));;
307 for (Inventory item : list) {
308 repairedItem = new InventoryBuilder(item);
309 // check missing tree-level
310 if (!nodeId.equals(item.getNodeId())) {
311 failures.add(String.format("missing node-id for equipment(uuid=%s)", item.getUuid()));
312 repairedItem.setNodeId(nodeId);
315 if (item.getTreeLevel() == null) {
316 failures.add(String.format("missing tree-level for equipment(uuid=%s)", item.getUuid()));
317 repairedItem.setTreeLevel(Uint32.valueOf(1));
321 treeLevel = item.getTreeLevel().longValue();
323 // check non root elem and missing parent
324 if (item.getParentUuid() == null) {
325 failures.add(String.format("Non root level element (uuid=%s) has to have a parent element",
328 repairedItem.setParentUuid(UNBOUND_INVENTORY_UUID);
329 repairList.put(unboundItem.getUuid(), unboundItem.build());
331 // check that parent exists in list and is tree-level -1
333 Optional<Inventory> parent =
334 list.stream().filter(e -> item.getParentUuid().equals(e.getUuid())).findFirst();
335 if (parent.isEmpty()) {
336 failures.add(String.format("no parent found for uuid=%s with parent-uuid=%s",
337 item.getUuid(), item.getParentUuid()));
338 repairedItem.setParentUuid(UNBOUND_INVENTORY_UUID);
343 // check for duplicated uui
344 Optional<Inventory> duplicate = list
345 .stream().filter(e -> !item.equals(e) && item.getUuid() != null
346 && item.getUuid().equals(e.getUuid()) && repairList.containsKey(e.getUuid()))
348 if (duplicate.isPresent()) {
349 failures.add(String.format("found duplicate uuid=%s", item.getUuid()));
354 if (failCounter > 0) {
355 repairList.put(repairedItem.getUuid(), repairedItem.build());
357 repairList.put(item.getUuid(), item);
362 if (failures.size() > 0) {
363 throw new DataInconsistencyException(new ArrayList<>(repairList.values()),
364 "inventory list is not consistent;\n" + String.join("\n", failures));
371 * join base with parameters of toJoin (only non null values)
373 * @param base base object
374 * @param toJoin object with new property values
375 * @return new joined object
377 @SuppressWarnings("unused")
378 private NetworkElementConnectionEntity joinNe(NetworkElementConnectionEntity base,
379 NetworkElementConnectionEntity toJoin) {
383 NetworkElementConnectionBuilder builder = new NetworkElementConnectionBuilder(base);
384 if (toJoin != null) {
385 if (toJoin.isIsRequired() != null) {
386 builder.setIsRequired(toJoin.isIsRequired());
388 if (toJoin.getCoreModelCapability() != null) {
389 builder.setCoreModelCapability(toJoin.getCoreModelCapability());
391 if (toJoin.getDeviceType() != null) {
392 builder.setDeviceType(toJoin.getDeviceType());
394 if (toJoin.getHost() != null) {
395 builder.setHost(toJoin.getHost());
397 if (toJoin.getNodeDetails() != null) {
398 builder.setNodeDetails(toJoin.getNodeDetails());
400 if (toJoin.getPassword() != null) {
401 builder.setPassword(toJoin.getPassword());
403 if (toJoin.getPort() != null) {
404 builder.setPort(toJoin.getPort());
406 if (toJoin.getStatus() != null) {
407 builder.setStatus(toJoin.getStatus());
409 if (toJoin.getUsername() != null) {
410 builder.setUsername(toJoin.getUsername());
413 return builder.build();
418 * @param networkElementConnectionEntitiy to wirte to DB
419 * @param nodeId Id for this DB element
422 public boolean updateNetworkConnectionDeviceType(NetworkElementConnectionEntity networkElementConnectionEntitiy,
424 return this.networkelementConnectionDB.update(networkElementConnectionEntitiy, nodeId) != null;
425 // NetworkElementConnectionEntity e =
426 // this.networkelementConnectionDB.read(nodeId);
427 // this.networkelementConnectionDB.write(this.joinNe(e,
428 // networkElementConnectionEntitiy), nodeId);
432 * Update after new mountpoint registration
434 * @param networkElementConnectionEntitiy data
435 * @param nodeId of device (mountpoint name)
438 public boolean updateNetworkConnection22(NetworkElementConnectionEntity networkElementConnectionEntitiy,
440 LOG.info("update networkelement-connection for {} with data {}", nodeId, networkElementConnectionEntitiy);
441 return this.networkelementConnectionDB.updateOrCreate(networkElementConnectionEntitiy, nodeId,
442 Arrays.asList("is-required", "username", "password")) != null;
443 // NetworkElementConnectionEntity e =
444 // this.networkelementConnectionDB.read(nodeId);
445 // this.networkelementConnectionDB.write(this.joinNe(e,
446 // networkElementConnectionEntitiy), nodeId);
450 /* please do not remove */
451 // public void cleanNetworkElementConnections() {
452 // this.networkelementConnectionDB.remove(QueryBuilders.matchQuery("is-required",
454 // CreateNetworkElementConnectionInput x = new
455 // CreateNetworkElementConnectionInputBuilder().setStatus(ConnectionLogStatus.Disconnected).build();
456 // this.networkelementConnectionDB.update(x,QueryBuilders.matchAllQuery());
460 public void removeNetworkConnection(String nodeId) {
462 NetworkElementConnectionEntity e = this.networkelementConnectionDB.read(nodeId);
463 if (e != null && (isRequired = e.isIsRequired()) != null) {
465 LOG.debug("updating connection status for {} of required ne to disconnected", nodeId);
466 this.networkelementConnectionDB.update(new UpdateNetworkElementConnectionInputBuilder()
467 .setStatus(ConnectionLogStatus.Disconnected).build(), nodeId);
469 LOG.debug("remove networkelement-connection for {} entry because of non-required", nodeId);
470 this.networkelementConnectionDB.remove(nodeId);
473 LOG.warn("Unable to update connection-status. dbentry for {} not found in networkelement-connection",
481 public int doIndexClean(Date olderAreOutdated) {
483 String netconfTimeStamp = NETCONFTIME_CONVERTER.getTimeStampAsNetconfString(olderAreOutdated);
486 QueryBuilder queryEventBase = EsEventBase.getQueryForTimeStamp(netconfTimeStamp);
487 removed += eventRWEventLogDevicemanager.remove(queryEventBase);
489 QueryBuilder queryFaultLog = EsFaultLogDevicemanager.getQueryForTimeStamp(netconfTimeStamp);
490 removed += eventRWFaultLogDB.remove(queryFaultLog);
495 public long getNumberOfOldObjects(Date olderAreOutdated) {
497 String netconfTimeStamp = NETCONFTIME_CONVERTER.getTimeStampAsNetconfString(olderAreOutdated);
498 int numberOfElements = 0;
500 QueryBuilder queryEventBase = EsEventBase.getQueryForTimeStamp(netconfTimeStamp);
501 numberOfElements += eventRWEventLogDevicemanager.doReadAll(queryEventBase).getTotal();
503 QueryBuilder queryFaultLog = EsFaultLogDevicemanager.getQueryForTimeStamp(netconfTimeStamp);
504 numberOfElements += eventRWFaultLogDB.doReadAll(queryFaultLog).getTotal();
506 return numberOfElements;
512 * Verify status of client
514 * @param event that is printed with message
515 * @return true if client is null
517 private boolean assertIfClientNull(Object event) {
518 return assertIfClientNull("No DB, can not write: {}", event);
521 private boolean assertIfClientNullForNodeName(Object object) {
522 return assertIfClientNull("No DB, can not handle node: {}", object);
526 * Verify status of client
528 * @param message to print including {} for object printout.
529 * @return true if client is null
531 private boolean assertIfClientNull(String message, Object object) {
532 if (client == null) {
533 LOG.debug(message, object);
541 private static class EsEventBase {
543 * Query to get older Elements
545 * @param netconfTimeStamp to identify older Elements
546 * @return QueryBuilder for older elements related to timestamp
548 private static QueryBuilder getQueryForTimeStamp(String netconfTimeStamp) {
549 return new RangeQueryBuilder("timestamp").lte(netconfTimeStamp);
553 private static class EsFaultLogDevicemanager {
557 * @param netconfTimeStamp to identify query elements older than this timestamp.
558 * @return QueryBuilder for related elements
560 public static QueryBuilder getQueryForTimeStamp(String netconfTimeStamp) {
561 return new RangeQueryBuilder("timestamp").lte(netconfTimeStamp);
565 public static class EsFaultCurrent {
567 * @param nodeName name of the node
568 * @return query builder
570 public static QueryBuilder getQueryForOneNode(String nodeName) {
571 return QueryBuilders.matchQuery("node-id", nodeName);
574 public static QueryBuilder getQueryForOneNodeAndObjectId(String nodeName, String objectId) {
575 BoolQueryBuilder bq = QueryBuilders.boolQuery();
576 bq.must(QueryBuilders.matchQuery("node-id", nodeName));
577 bq.must(QueryBuilders.matchQuery("object-id", objectId));
583 public List<NetworkElementConnectionEntity> getNetworkElementConnections() {
584 return this.networkelementConnectionDB.doReadAll().getHits();
588 public void doWritePerformanceData(List<PmdataEntity> list) {
590 list.forEach(elem -> {
591 GranularityPeriodType granularityPeriod = nnGetGranularityPeriodType(elem.getGranularityPeriod());
592 // _id": "Sim12600/LP-MWPS-TTP-01/2017-07-04T15:15:00.0+00:00"
593 StringBuffer id = new StringBuffer();
594 DateAndTime date = elem.getTimeStamp();
595 id.append(elem.getNodeName());
597 id.append(elem.getUuidInterface());
599 id.append(date != null ? date.getValue() : "null");
601 switch (granularityPeriod) {
603 pmData15mDB.write(elem, id.toString());
606 pmData24hDB.write(elem, id.toString());
610 LOG.debug("Unknown granularity {} id {}", granularityPeriod, id);
618 GranularityPeriodType nnGetGranularityPeriodType(@Nullable GranularityPeriodType granularityPeriod) {
619 return granularityPeriod != null ? granularityPeriod : GranularityPeriodType.Unknown;
623 public HtDatabaseClient getRawClient() {
628 public void writeGuiCutThroughData(Guicutthrough gcData, String nodeId) {
629 guiCutThroughDB.write(gcData, nodeId);
633 public int clearGuiCutThroughEntriesOfNode(String nodeName) {
634 guiCutThroughDB.remove(nodeName);