From 48b02dfdc78a6412d568a14b015ccb9439a1dbb5 Mon Sep 17 00:00:00 2001 From: "Thomas Nelson (arthurdent3)" Date: Wed, 2 Oct 2019 18:09:23 -0400 Subject: [PATCH] Update locking to use Threadsafe set Had to create whole new method for Atomic Lock Creation. Added thread safe set to contol lock creation per key. Updated the deadlock to use local_quorum Removed some uneeded import Updated some logging. Issue-ID: MUSIC-512 Signed-off-by: Thomas Nelson (arthurdent3) Change-Id: I7e1a4c34de5dc9a0e90adf30d1f4d1bac698ceae Signed-off-by: Thomas Nelson (arthurdent3) --- distribution/music/Dockerfile | 3 +- distribution/music/startup.sh | 56 ++++++++--- .../org/onap/music/datastore/MusicDataStore.java | 25 +++-- .../lockingservice/cassandra/CassaLockStore.java | 2 +- .../main/java/org/onap/music/main/MusicCore.java | 8 +- .../org/onap/music/service/MusicCoreService.java | 4 +- .../onap/music/service/impl/MusicCassaCore.java | 112 +++++++++++++++++---- .../conductor/conditionals/MusicConditional.java | 6 +- .../onap/music/unittests/TstRestMusicLockAPI.java | 8 +- 9 files changed, 165 insertions(+), 59 deletions(-) diff --git a/distribution/music/Dockerfile b/distribution/music/Dockerfile index 98e8ea41..9868ceee 100644 --- a/distribution/music/Dockerfile +++ b/distribution/music/Dockerfile @@ -23,10 +23,11 @@ FROM openjdk:8 LABEL purpose="Springboot for MUSIC" RUN apt-get update && apt-get install -y netcat telnet vim vim-common curl RUN groupadd --gid 1000 music && useradd --gid 1000 --uid 1000 music -RUN mkdir -p /opt/app/music/logs/MUSIC +RUN mkdir -p /opt/app/music/logs/MUSIC COPY MUSIC-SB.jar /opt/app/music COPY startup.sh /opt/app/music RUN mkdir -p /opt/app/music/logs && \ + mkdir -p /opt/app/music/etc && \ chown -R music:music /opt/app/music/ && \ chmod 755 /opt/app/music/startup.sh USER music:music diff --git a/distribution/music/startup.sh b/distribution/music/startup.sh index 61536832..2e1165d1 100644 --- a/distribution/music/startup.sh +++ b/distribution/music/startup.sh @@ -8,35 +8,61 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# # ============LICENSE_END============================================= # ==================================================================== echo "Running startup script to get password from certman" PWFILE=/opt/app/aafcertman/.password LOGFILE=/opt/app/music/logs/MUSIC/music-sb.log -echo "PWFILE=${PWFILE}" >> $LOGFILE +PROPS=/opt/app/music/etc/music-sb.properties +LOGBACK=/opt/app/music/etc/logback.xml +LOGGING= +DEBUG_PROP= +# Debug Setup. Uses env variables +# DEBUG and DEBUG_PORT +# DEBUG=true/false | DEBUG_PORT= +if [ "${DEBUG}" == "true" ]; then + if [ "${DEBUG_PORT}" == "" ]; then + DEBUG_PORT=8000 + fi + echo "Debug mode on" + DEBUG_PROP="-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=${DEBUG_PORT},suspend=n" +fi + +# LOGBACK file: if /opt/app/music/etc/logback.xml exists thenuse that. +if [ -f $LOGBACK ]; then + LOGGING="--logging.config=file:${LOGBACK}" +fi + +# Get Passwords from /opt/app/aafcertman if [ -f $PWFILE ]; then -echo "Found ${PWFILE}" >> $LOGFILE -PASSWORD=$(cat ${PWFILE}) -echo "#### Using Password from ${PWFILE} for Certs" >> ${LOGFILE} + echo "Found ${PWFILE}" >> $LOGFILE + PASSWORD=$(cat ${PWFILE}) + echo "#### Using Password from ${PWFILE} for Certs" >> ${LOGFILE} else -PASSWORD=changeit -echo "#### Using Default Password for Certs" >> ${LOGFILE} + PASSWORD=changeit + echo "#### Using Default Password for Certs" >> ${LOGFILE} fi -if [ ${EXTAP} ]; then - if [ "${EXTAP_FILE}" != "" ]; then - EXTAP_OPTION="--spring.config.location=file:${EXTAP_FILE}" - fi + +# If music-sb.properties exists in /opt/app/music/etc then use that to override the application.properties +if [ -f $PROPS ]; then + # Run with different Property file + #echo "java ${DEBUG_PROP} -jar MUSIC.jar --spring.config.location=file:${PROPS} ${LOGGING} 2>&1 | tee ${LOGFILE}" + java ${DEBUG_PROP} -jar MUSIC-SB.jar --spring.config.location=file:${PROPS} ${LOGGING} 2>&1 | tee ${LOGFILE} +else + #echo "java ${DEBUG_PROP} -jar MUSIC.jar --server.ssl.key-store-password=${PASSWORD} ${LOGGING} 2>&1 | tee ${LOGFILE}" + java ${DEBUG_PROP} -jar MUSIC-SB.jar --server.ssl.key-store-password="${PASSWORD}" ${LOGGING} 2>&1 | tee ${LOGFILE} fi -echo "java -jar MUSIC-SB.jar ${EXTAP_OPTION} --server.ssl.key-store-password=${PASSWORD} --aaf_password=enc:${PASSWORD} 2>&1 | tee ${LOGFILE}" -java -jar MUSIC-SB.jar ${EXTAP_OPTION} --server.ssl.key-store-password="${PASSWORD}" --aaf_password="enc:${PASSWORD}" 2>&1 | tee ${LOGFILE} + + + diff --git a/music-core/src/main/java/org/onap/music/datastore/MusicDataStore.java b/music-core/src/main/java/org/onap/music/datastore/MusicDataStore.java index 97fc1d33..9ccff828 100755 --- a/music-core/src/main/java/org/onap/music/datastore/MusicDataStore.java +++ b/music-core/src/main/java/org/onap/music/datastore/MusicDataStore.java @@ -25,14 +25,8 @@ package org.onap.music.datastore; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.SocketException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Enumeration; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import org.onap.music.eelf.logging.EELFLoggerDelegate; @@ -58,12 +52,9 @@ import com.datastax.driver.core.Row; import com.datastax.driver.core.Session; import com.datastax.driver.core.SimpleStatement; import com.datastax.driver.core.TableMetadata; -import com.datastax.driver.core.TypeCodec; import com.datastax.driver.core.exceptions.AlreadyExistsException; import com.datastax.driver.core.exceptions.InvalidQueryException; -import com.datastax.driver.core.exceptions.NoHostAvailableException; import com.datastax.driver.extras.codecs.enums.EnumNameCodec; -import com.datastax.driver.extras.codecs.enums.EnumOrdinalCodec; /** * @author nelson24 @@ -73,6 +64,7 @@ public class MusicDataStore { public static final String CONSISTENCY_LEVEL_ONE = "ONE"; public static final String CONSISTENCY_LEVEL_QUORUM = "QUORUM"; + public static final String CONSISTENCY_LEVEL_LOCAL_QUORUM = "LOCAL_QUORUM"; private Session session; private Cluster cluster; @@ -511,7 +503,18 @@ public class MusicDataStore { throws MusicServiceException, MusicQueryException { return executeGet(queryObject, CONSISTENCY_LEVEL_ONE); } - + + /** + * + * This method performs DDL operation on Cassandra using consistency level LOCAL_QUORUM. + * + * @param queryObject Object containing cassandra prepared query and values. + */ + public ResultSet executeLocalQuorumConsistencyGet(PreparedQueryObject queryObject) + throws MusicServiceException, MusicQueryException { + return executeGet(queryObject, CONSISTENCY_LEVEL_LOCAL_QUORUM); + } + /** * * This method performs DDL operation on Cassandra using consistency level QUORUM. @@ -522,5 +525,5 @@ public class MusicDataStore { throws MusicServiceException, MusicQueryException { return executeGet(queryObject, CONSISTENCY_LEVEL_QUORUM); } - + } diff --git a/music-core/src/main/java/org/onap/music/lockingservice/cassandra/CassaLockStore.java b/music-core/src/main/java/org/onap/music/lockingservice/cassandra/CassaLockStore.java index 10898476..edce3fff 100644 --- a/music-core/src/main/java/org/onap/music/lockingservice/cassandra/CassaLockStore.java +++ b/music-core/src/main/java/org/onap/music/lockingservice/cassandra/CassaLockStore.java @@ -480,7 +480,7 @@ public class CassaLockStore { DeadlockDetectionUtil ddu = new DeadlockDetectionUtil(); - ResultSet rs = dsHandle.executeQuorumConsistencyGet(queryObject); + ResultSet rs = dsHandle.executeLocalQuorumConsistencyGet(queryObject); logger.debug("rs has " + rs.getAvailableWithoutFetching() + (rs.isFullyFetched()?"":" (or more!)") ); Iterator it = rs.iterator(); while (it.hasNext()) { diff --git a/music-core/src/main/java/org/onap/music/main/MusicCore.java b/music-core/src/main/java/org/onap/music/main/MusicCore.java index 658f2124..226dfb07 100644 --- a/music-core/src/main/java/org/onap/music/main/MusicCore.java +++ b/music-core/src/main/java/org/onap/music/main/MusicCore.java @@ -81,12 +81,12 @@ public class MusicCore { return musicCore.acquireLockWithLease(key, lockId, leasePeriod); } - public static String createLockReference(String fullyQualifiedKey) throws MusicLockingException { - return musicCore.createLockReference(fullyQualifiedKey); + public static String createLockReferenceAtomic(String fullyQualifiedKey) throws MusicLockingException { + return musicCore.createLockReferenceAtomic(fullyQualifiedKey); } - public static String createLockReference(String fullyQualifiedKey, LockType locktype) throws MusicLockingException { - return musicCore.createLockReference(fullyQualifiedKey, locktype); + public static String createLockReferenceAtomic(String fullyQualifiedKey, LockType locktype) throws MusicLockingException { + return musicCore.createLockReferenceAtomic(fullyQualifiedKey, locktype); } public static String createLockReference(String fullyQualifiedKey, LockType locktype, String owner) throws MusicLockingException { diff --git a/music-core/src/main/java/org/onap/music/service/MusicCoreService.java b/music-core/src/main/java/org/onap/music/service/MusicCoreService.java index b3226906..7629eae2 100644 --- a/music-core/src/main/java/org/onap/music/service/MusicCoreService.java +++ b/music-core/src/main/java/org/onap/music/service/MusicCoreService.java @@ -89,14 +89,14 @@ public interface MusicCoreService { * @param fullyQualifiedKey the key to create a lock on * @see {@link #creatLockReference(String, LockType)} */ - public String createLockReference(String fullyQualifiedKey) throws MusicLockingException; // lock name + public String createLockReferenceAtomic(String fullyQualifiedKey) throws MusicLockingException; // lock name /** * Create a lock ref in the music lock store * @param fullyQualifiedKey the key to create a lock on * @param locktype the type of lock create, see {@link LockType} */ - public String createLockReference(String fullyQualifiedKey, LockType locktype) throws MusicLockingException; + public String createLockReferenceAtomic(String fullyQualifiedKey, LockType locktype) throws MusicLockingException; /** * Create a lock ref in the music lock store diff --git a/music-core/src/main/java/org/onap/music/service/impl/MusicCassaCore.java b/music-core/src/main/java/org/onap/music/service/impl/MusicCassaCore.java index 0d2e3f0a..c7c7cddc 100644 --- a/music-core/src/main/java/org/onap/music/service/impl/MusicCassaCore.java +++ b/music-core/src/main/java/org/onap/music/service/impl/MusicCassaCore.java @@ -26,8 +26,11 @@ package org.onap.music.service.impl; import java.io.StringWriter; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.StringTokenizer; import javax.ws.rs.core.MultivaluedMap; @@ -71,6 +74,7 @@ public class MusicCassaCore implements MusicCoreService { private static CassaLockStore mLockHandle = null; private static EELFLoggerDelegate logger = EELFLoggerDelegate.getLogger(MusicCassaCore.class); private static MusicCassaCore musicCassaCoreInstance = null; + private static Set set = Collections.synchronizedSet(new HashSet()); private MusicCassaCore() { // not going to happen @@ -92,9 +96,6 @@ public class MusicCassaCore implements MusicCoreService { return musicCassaCoreInstance; } - - - public static CassaLockStore getLockingServiceHandle() throws MusicLockingException { logger.info(EELFLoggerDelegate.applicationLogger,"Acquiring lock store handle"); long start = System.currentTimeMillis(); @@ -112,24 +113,98 @@ public class MusicCassaCore implements MusicCoreService { return mLockHandle; } - public String createLockReference(String fullyQualifiedKey) throws MusicLockingException { - return createLockReference(fullyQualifiedKey, LockType.WRITE); + public String createLockReferenceAtomic(String fullyQualifiedKey) throws MusicLockingException { + return createLockReferenceAtomic(fullyQualifiedKey, LockType.WRITE); } - public String createLockReference(String fullyQualifiedKey, LockType locktype) throws MusicLockingException { - return createLockReference(fullyQualifiedKey, locktype, null); + /** + * This will be called for Atomic calls + * + */ + public String createLockReferenceAtomic(String fullyQualifiedKey, LockType locktype) throws MusicLockingException { + String[] splitString = fullyQualifiedKey.split("\\."); + if (splitString.length < 3) { + throw new MusicLockingException("Missing or incorrect lock details. Check table or key name."); + } + String keyspace = splitString[0]; + String table = splitString[1]; + String lockName = splitString[2]; + + logger.info(EELFLoggerDelegate.applicationLogger,"Creating lock reference for lock name:" + lockName); + long start = 0L; + long end = 0L; + String lockReference = null; + LockObject peek = null; + + /** Lets check for an existing lock. + * This will allow us to limit the amount of requests going forward. + */ + start = System.currentTimeMillis(); + try { + peek = getLockingServiceHandle().peekLockQueue(keyspace, table, lockName); + } catch (MusicServiceException | MusicQueryException e) { + //logger.error(EELFLoggerDelegate.errorLogger,e.getMessage(),e); + throw new MusicLockingException("Error getting lockholder info for key [" + lockName +"]:" + e.getMessage()); + } + + if(peek!=null && (peek.getLocktype()!=null && peek.getLocktype().equals(LockType.WRITE)) && peek.getAcquireTime()!=null && peek.getLockRef()!=null) { + long currentTime = System.currentTimeMillis(); + if((currentTime-Long.parseLong(peek.getAcquireTime()))