From b36107389e624b9e5733c806e02a3b37d9650088 Mon Sep 17 00:00:00 2001 From: Fiete Ostkamp Date: Wed, 4 Mar 2026 10:36:48 +0100 Subject: [PATCH] Improve python dependency management - migrate from distutils to setuptools - replace docker + local dependency files with requirements.txt that is used both locally and in docker - remove separate `pip install`s from Dockerfiles and make sure that these deps are in the requirements file this eliminates the possibility for mismatches (between what is run in the tests and what is used in the build) Issue-ID: CCSDK-4153 Change-Id: I9db055d05e8c66c9a83a71123492680259549542 Signed-off-by: Fiete Ostkamp --- .../requirements-test.txt} | 1 - .../{requirements/shared.txt => requirements.txt} | 0 ms/artifact-manager/requirements/docker.txt | 2 -- ms/artifact-manager/requirements/local.txt | 2 -- ms/artifact-manager/setup.py | 8 +++++- ms/artifact-manager/tox.ini | 10 +++++-- ms/command-executor/requirements.txt | 8 ++++++ ms/command-executor/src/main/docker/Dockerfile | 3 +- .../src/main/docker/distribution.xml | 9 +++++- ms/py-executor/docker/Dockerfile | 5 +++- ms/py-executor/docker/distribution.xml | 2 +- .../test.txt => py-executor/requirements-test.txt} | 1 - .../{requirements/shared.txt => requirements.txt} | 0 ms/py-executor/requirements/docker.txt | 4 --- ms/py-executor/requirements/local.txt | 4 --- ms/py-executor/setup.py | 33 ++++++++++++++++++++++ ms/py-executor/tox.ini | 14 +++++++-- py-modules/common/setup.py | 4 +-- 18 files changed, 82 insertions(+), 28 deletions(-) rename ms/{py-executor/requirements/test.txt => artifact-manager/requirements-test.txt} (73%) rename ms/artifact-manager/{requirements/shared.txt => requirements.txt} (100%) delete mode 100644 ms/artifact-manager/requirements/docker.txt delete mode 100644 ms/artifact-manager/requirements/local.txt create mode 100644 ms/command-executor/requirements.txt rename ms/{artifact-manager/requirements/test.txt => py-executor/requirements-test.txt} (73%) rename ms/py-executor/{requirements/shared.txt => requirements.txt} (100%) delete mode 100644 ms/py-executor/requirements/docker.txt delete mode 100644 ms/py-executor/requirements/local.txt create mode 100644 ms/py-executor/setup.py diff --git a/ms/py-executor/requirements/test.txt b/ms/artifact-manager/requirements-test.txt similarity index 73% rename from ms/py-executor/requirements/test.txt rename to ms/artifact-manager/requirements-test.txt index 5304eac63..0c1d9e2dd 100644 --- a/ms/py-executor/requirements/test.txt +++ b/ms/artifact-manager/requirements-test.txt @@ -1,3 +1,2 @@ pytest==5.3.1 pytest-grpc==0.7.0 --r local.txt \ No newline at end of file diff --git a/ms/artifact-manager/requirements/shared.txt b/ms/artifact-manager/requirements.txt similarity index 100% rename from ms/artifact-manager/requirements/shared.txt rename to ms/artifact-manager/requirements.txt diff --git a/ms/artifact-manager/requirements/docker.txt b/ms/artifact-manager/requirements/docker.txt deleted file mode 100644 index 67f68dd22..000000000 --- a/ms/artifact-manager/requirements/docker.txt +++ /dev/null @@ -1,2 +0,0 @@ --r shared.txt -/opt/app/onap/dependencies/common \ No newline at end of file diff --git a/ms/artifact-manager/requirements/local.txt b/ms/artifact-manager/requirements/local.txt deleted file mode 100644 index 6a06e16f3..000000000 --- a/ms/artifact-manager/requirements/local.txt +++ /dev/null @@ -1,2 +0,0 @@ --r shared.txt -../../py-modules/common \ No newline at end of file diff --git a/ms/artifact-manager/setup.py b/ms/artifact-manager/setup.py index b5b4487de..6a96e278d 100644 --- a/ms/artifact-manager/setup.py +++ b/ms/artifact-manager/setup.py @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. """ -from distutils.core import setup +from setuptools import setup setup( @@ -22,4 +22,10 @@ setup( version="0.1", description="CDS Artifact Manager", packages=["manager"], + install_requires=[ + "grpcio-tools>=1.25.0", + "protobuf>=3.20.1", + "onappylog>=1.0.9", + "click>=7.0", + ], ) diff --git a/ms/artifact-manager/tox.ini b/ms/artifact-manager/tox.ini index a95267823..fe7d744b1 100644 --- a/ms/artifact-manager/tox.ini +++ b/ms/artifact-manager/tox.ini @@ -6,7 +6,9 @@ setenv = PYTHONPATH = {toxinidir} CONFIGURATION = {toxinidir}/../configuration-local.ini deps = - -rrequirements/test.txt + -r{toxinidir}/requirements.txt + -r{toxinidir}/requirements-test.txt + {toxinidir}/../../py-modules/common commands = pytest [testenv:codelint] deps = @@ -18,10 +20,12 @@ deps = commands = flake8 --doctest --docstring-convention google --max-line-length 120 --exclude .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg,*test.py --select=D {posargs:.} [testenv:coverage] basepython = python3.7 -setenv = +setenv = PYTHONPATH = {toxinidir} CONFIGURATION = {toxinidir}/../configuration-local.ini deps = - -rrequirements/test.txt + -r{toxinidir}/requirements.txt + -r{toxinidir}/requirements-test.txt + {toxinidir}/../../py-modules/common pytest-cov commands = pytest --cov=manager --cov-fail-under=60 --cov-config={toxinidir}/.coveragerc . diff --git a/ms/command-executor/requirements.txt b/ms/command-executor/requirements.txt new file mode 100644 index 000000000..b4331f499 --- /dev/null +++ b/ms/command-executor/requirements.txt @@ -0,0 +1,8 @@ +requests==2.26.0 +grpcio==1.20.0 +grpcio-tools==1.20.0 +virtualenv==16.7.9 +prometheus-client==0.11.0 +protobuf==3.20.1 +opentelemetry-distro==0.40b0 +opentelemetry-exporter-otlp-proto-http==1.19.0 diff --git a/ms/command-executor/src/main/docker/Dockerfile b/ms/command-executor/src/main/docker/Dockerfile index 8405fe8d5..7c02cb462 100644 --- a/ms/command-executor/src/main/docker/Dockerfile +++ b/ms/command-executor/src/main/docker/Dockerfile @@ -18,8 +18,7 @@ RUN tar -xzf /source.tar.gz -C /tmp \ && chmod -R 755 /opt \ && apt-get update && apt-get install -y procps iputils-ping curl telnet && rm -rf /var/lib/apt/lists/* \ && python -m pip install --no-cache-dir --upgrade pip setuptools \ - && pip install --no-cache-dir requests==2.26.0 grpcio==1.20.0 grpcio-tools==1.20.0 virtualenv==16.7.9 prometheus-client==0.11.0 protobuf==3.20.1 \ - && pip install --no-cache-dir opentelemetry-distro==0.40b0 opentelemetry-exporter-otlp-proto-http==1.19.0 + && pip install --no-cache-dir -r /opt/app/onap/python/requirements.txt USER onap ENTRYPOINT /opt/app/onap/command-executor/start.sh diff --git a/ms/command-executor/src/main/docker/distribution.xml b/ms/command-executor/src/main/docker/distribution.xml index 7a16062cb..3384e52f6 100755 --- a/ms/command-executor/src/main/docker/distribution.xml +++ b/ms/command-executor/src/main/docker/distribution.xml @@ -27,5 +27,12 @@ ${project.basedir}/src/main/python opt/app/onap/python + + ${project.basedir} + opt/app/onap/python + + requirements.txt + + - \ No newline at end of file + diff --git a/ms/py-executor/docker/Dockerfile b/ms/py-executor/docker/Dockerfile index a2c43b83d..4271160ac 100644 --- a/ms/py-executor/docker/Dockerfile +++ b/ms/py-executor/docker/Dockerfile @@ -18,7 +18,10 @@ RUN tar -xzf /source.tar.gz -C /tmp \ && chmod -R 755 /opt RUN python -m pip install --no-cache-dir --upgrade pip setuptools -RUN pip install --no-cache-dir -r /opt/app/onap/python/requirements/docker.txt +RUN pip install --no-cache-dir -r /opt/app/onap/python/requirements.txt +RUN pip install --no-cache-dir /opt/app/onap/dependencies/common +RUN pip install --no-cache-dir -r /opt/app/onap/dependencies/artifact-manager/requirements.txt +RUN pip install --no-cache-dir /opt/app/onap/dependencies/artifact-manager RUN pip install --no-cache-dir opentelemetry-distro==0.40b0 opentelemetry-exporter-otlp-proto-http==1.19.0 diff --git a/ms/py-executor/docker/distribution.xml b/ms/py-executor/docker/distribution.xml index a2739bdfa..af2bc2a33 100755 --- a/ms/py-executor/docker/distribution.xml +++ b/ms/py-executor/docker/distribution.xml @@ -62,7 +62,7 @@ ${project.basedir} opt/app/onap/python - requirements/*.txt + requirements.txt true 0666 diff --git a/ms/artifact-manager/requirements/test.txt b/ms/py-executor/requirements-test.txt similarity index 73% rename from ms/artifact-manager/requirements/test.txt rename to ms/py-executor/requirements-test.txt index 5304eac63..0c1d9e2dd 100644 --- a/ms/artifact-manager/requirements/test.txt +++ b/ms/py-executor/requirements-test.txt @@ -1,3 +1,2 @@ pytest==5.3.1 pytest-grpc==0.7.0 --r local.txt \ No newline at end of file diff --git a/ms/py-executor/requirements/shared.txt b/ms/py-executor/requirements.txt similarity index 100% rename from ms/py-executor/requirements/shared.txt rename to ms/py-executor/requirements.txt diff --git a/ms/py-executor/requirements/docker.txt b/ms/py-executor/requirements/docker.txt deleted file mode 100644 index 58469c9d7..000000000 --- a/ms/py-executor/requirements/docker.txt +++ /dev/null @@ -1,4 +0,0 @@ --r shared.txt -/opt/app/onap/dependencies/common --r /opt/app/onap/dependencies/artifact-manager/requirements/docker.txt -/opt/app/onap/dependencies/artifact-manager \ No newline at end of file diff --git a/ms/py-executor/requirements/local.txt b/ms/py-executor/requirements/local.txt deleted file mode 100644 index 74cc76efa..000000000 --- a/ms/py-executor/requirements/local.txt +++ /dev/null @@ -1,4 +0,0 @@ --r shared.txt -../../py-modules/common --r ../../artifact-manager/requirements/local.txt -../artifact-manager \ No newline at end of file diff --git a/ms/py-executor/setup.py b/ms/py-executor/setup.py new file mode 100644 index 000000000..7aba007a5 --- /dev/null +++ b/ms/py-executor/setup.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +"""Copyright 2019 AT&T Intellectual Property. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from setuptools import setup, find_packages + +setup( + name="py-executor", + version="0.1", + description="CDS Python Executor", + packages=find_packages(exclude=["test", "test.*"]), + install_requires=[ + "grpcio>=1.25.0", + "grpcio-tools>=1.25.0", + "protobuf>=3.20.1", + "configparser>=4.0.2", + "requests>=2.22.0", + "ncclient>=0.6.6", + "ansible>=2.8.5", + ], +) diff --git a/ms/py-executor/tox.ini b/ms/py-executor/tox.ini index d722ed45d..9af22c28d 100644 --- a/ms/py-executor/tox.ini +++ b/ms/py-executor/tox.ini @@ -5,7 +5,11 @@ skipsdist=True setenv = CONFIGURATION = {toxinidir}/../configuration-local.ini deps = - -rrequirements/test.txt + -r{toxinidir}/requirements.txt + -r{toxinidir}/requirements-test.txt + {toxinidir}/../../py-modules/common + -r{toxinidir}/../artifact-manager/requirements.txt + {toxinidir}/../artifact-manager commands = pytest resource_resolution/ [testenv:codelint] deps = @@ -17,9 +21,13 @@ deps = commands = flake8 --doctest --docstring-convention google --max-line-length 120 --exclude .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg,*test.py --select=D {posargs:.} [testenv:coverage] basepython = python3.7 -setenv = +setenv = CONFIGURATION = {toxinidir}/../configuration-local.ini deps = - -rrequirements/test.txt + -r{toxinidir}/requirements.txt + -r{toxinidir}/requirements-test.txt + {toxinidir}/../../py-modules/common + -r{toxinidir}/../artifact-manager/requirements.txt + {toxinidir}/../artifact-manager pytest-cov commands = pytest --cov=manager --cov=resource_resolution --cov-fail-under=60 --cov-config={toxinidir}/.coveragerc resource_resolution/ diff --git a/py-modules/common/setup.py b/py-modules/common/setup.py index 1ff59a6de..f25d3e680 100644 --- a/py-modules/common/setup.py +++ b/py-modules/common/setup.py @@ -14,12 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. """ -from distutils.core import setup +from setuptools import setup setup( name="cdscommon", version="0.1", description="CDS Common Python Modules", packages=["proto"], - install_requires=["grpcio-tools"], + install_requires=["grpcio-tools>=1.25.0"], ) -- 2.16.6