diff --git a/.github/workflows/pypi_build_and_images.yaml b/.github/workflows/pypi_build_and_images.yaml index 7e5f595fc9..4883c17cd6 100644 --- a/.github/workflows/pypi_build_and_images.yaml +++ b/.github/workflows/pypi_build_and_images.yaml @@ -31,8 +31,8 @@ jobs: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing strategy: matrix: - target: [wmagent, wmagent-devtools, wmcore, reqmon, reqmgr2, global-workqueue, acdcserver, reqmgr2ms-unmerged, - reqmgr2ms-output, reqmgr2ms-pileup, reqmgr2ms-rulecleaner, reqmgr2ms-transferor, reqmgr2ms-monitor] + target: [wmagent, wmagent_devtools, wmcore, reqmon, reqmgr2, global_workqueue, acdcserver, reqmgr2ms_unmerged, + reqmgr2ms_output, reqmgr2ms_pileup, reqmgr2ms_rulecleaner, reqmgr2ms_transferor, reqmgr2ms_monitor] steps: - uses: actions/checkout@v4 @@ -54,6 +54,9 @@ jobs: run: python3 setup.py clean sdist - name: Upload package distribution to PyPi uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ # second job, depends on build_and_publish_services, builds and upload # docker images to CERN registry @@ -62,8 +65,8 @@ jobs: needs: [build_and_publish_services] strategy: matrix: - target: [wmagent, reqmon, t0_reqmon, reqmgr2, reqmgr2ms-unmerged, global-workqueue, - reqmgr2ms-output, reqmgr2ms-pileup, reqmgr2ms-rulecleaner, reqmgr2ms-transferor, reqmgr2ms-monitor] + target: [wmagent, reqmon, t0_reqmon, reqmgr2, reqmgr2ms_unmerged, global_workqueue, + reqmgr2ms_output, reqmgr2ms_pileup, reqmgr2ms_rulecleaner, reqmgr2ms_transferor, reqmgr2ms_monitor] uses: ./.github/workflows/docker_images_template.yaml with: wmcore_component: ${{ matrix.target }} diff --git a/requirements.txt b/requirements.txt index d41c260431..ffb0333fc6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,32 +13,32 @@ # https://www.python.org/dev/peps/pep-0440/#version-specifiers Cheetah3~=3.2.6.post1 # wmcore,wmagent,reqmgr2,reqmon -CherryPy~=18.8.0 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner +CherryPy~=18.8.0 # wmcore,wmagent,wmagent_devtools,reqmgr2,reqmon,global_workqueue,reqmgr2ms_unmerged,reqmgr2ms_output,reqmgr2ms_pileup,reqmgr2ms_monitor,reqmgr2ms_transferor,reqmgr2ms_rulecleaner CMSCouchapp~=1.3.4 # wmcore,wmagent -CMSMonitoring~=0.6.10 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner -coverage~=5.4 # wmcore,wmagent,wmagent-devtools +CMSMonitoring~=0.6.10 # wmcore,wmagent,reqmgr2,reqmon,global_workqueue,reqmgr2ms_unmerged,reqmgr2ms_output,reqmgr2ms_pileup,reqmgr2ms_monitor,reqmgr2ms_transferor,reqmgr2ms_rulecleaner +coverage~=5.4 # wmcore,wmagent,wmagent_devtools cx-Oracle~=8.3.0 # wmcore,wmagent -dbs3-client==4.0.19 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms-output,reqmgr2ms-transferor -future~=0.18.2 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner -gfal2-python~=1.11.0.post3 # wmcore,reqmgr2ms-unmerged -httplib2~=0.20.4 # wmcore,wmagent,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner +dbs3-client==4.0.19 # wmcore,wmagent,wmagent_devtools,reqmgr2,reqmon,global_workqueue,reqmgr2ms_output,reqmgr2ms_transferor +future~=0.18.2 # wmcore,wmagent,wmagent_devtools,reqmgr2,reqmon,acdcserver,global_workqueue,reqmgr2ms_unmerged,reqmgr2ms_output,reqmgr2ms_pileup,reqmgr2ms_monitor,reqmgr2ms_transferor,reqmgr2ms_rulecleaner +gfal2-python~=1.11.0.post3 # wmcore,reqmgr2ms_unmerged +httplib2~=0.20.4 # wmcore,wmagent,reqmgr2,reqmon,acdcserver,global_workqueue,reqmgr2ms_unmerged,reqmgr2ms_output,reqmgr2ms_pileup,reqmgr2ms_monitor,reqmgr2ms_transferor,reqmgr2ms_rulecleaner htcondor~=24.3.0 # wmcore,wmagent Jinja2~=3.1.2 # wmcore,wmagent -memory-profiler~=0.60.0 # wmcore,wmagent-devtools -mock~=4.0.3 # wmcore,wmagent,wmagent-devtools -mox3~=1.1.0 # wmcore,wmagent-devtools +memory-profiler~=0.60.0 # wmcore,wmagent_devtools +mock~=4.0.3 # wmcore,wmagent,wmagent_devtools +mox3~=1.1.0 # wmcore,wmagent_devtools mysqlclient~=2.1.1 # wmcore,wmagent -nose~=1.3.7 # wmcore,wmagent-devtools -nose2~=0.12.0 # wmcore,wmagent-devtools -pycodestyle~=2.8.0 # wmcore,wmagent-devtools -psutil~=5.9.1 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue -pycurl~=7.45.1 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner -pylint~=2.14.5 # wmcore,wmagent-devtools -pymongo~=4.2.0 # wmcore,wmagent-devtools,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup +nose~=1.3.7 # wmcore,wmagent_devtools +nose2~=0.12.0 # wmcore,wmagent_devtools +pycodestyle~=2.8.0 # wmcore,wmagent_devtools +psutil~=5.9.1 # wmcore,wmagent,wmagent_devtools,reqmgr2,reqmon,global_workqueue +pycurl~=7.45.1 # wmcore,wmagent,reqmgr2,reqmon,global_workqueue,reqmgr2ms_unmerged,reqmgr2ms_output,reqmgr2ms_pileup,reqmgr2ms_monitor,reqmgr2ms_transferor,reqmgr2ms_rulecleaner +pylint~=2.14.5 # wmcore,wmagent_devtools +pymongo~=4.2.0 # wmcore,wmagent_devtools,reqmgr2ms_unmerged,reqmgr2ms_output,reqmgr2ms_pileup pyOpenSSL~=18.0.0 # wmcore,wmagent pyzmq~=23.2.1 # wmcore,wmagent -retry~=0.9.2 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner -rucio-clients~=1.29.10 # wmcore,wmagent,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner -Sphinx~=5.1.1 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue +retry~=0.9.2 # wmcore,wmagent,wmagent_devtools,reqmgr2,reqmon,global_workqueue,reqmgr2ms_unmerged,reqmgr2ms_output,reqmgr2ms_pileup,reqmgr2ms_monitor,reqmgr2ms_transferor,reqmgr2ms_rulecleaner +rucio-clients~=1.29.10 # wmcore,wmagent,global_workqueue,reqmgr2ms_unmerged,reqmgr2ms_output,reqmgr2ms_pileup,reqmgr2ms_monitor,reqmgr2ms_transferor,reqmgr2ms_rulecleaner +Sphinx~=5.1.1 # wmcore,wmagent,wmagent_devtools,reqmgr2,reqmon,acdcserver,global_workqueue SQLAlchemy~=1.4.40 # wmcore,wmagent -PyJWT~=2.4.0 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner +PyJWT~=2.4.0 # wmcore,wmagent,wmagent_devtools,reqmgr2,reqmon,acdcserver,global_workqueue,reqmgr2ms_unmerged,reqmgr2ms_output,reqmgr2ms_pileup,reqmgr2ms_monitor,reqmgr2ms_transferor,reqmgr2ms_rulecleaner diff --git a/setup_dependencies.py b/setup_dependencies.py index 0e7263b9c6..222b40e8a9 100644 --- a/setup_dependencies.py +++ b/setup_dependencies.py @@ -17,39 +17,39 @@ be used by multiple WMCore systems). """ dependencies = { - 'wmc-rest': { + 'wmc_rest': { 'bin': ['wmc-dist-patch', 'wmc-dist-unpatch', 'wmc-httpd'], 'packages': ['WMCore.REST'], 'modules': ['WMCore.Configuration'], - 'systems': ['wmc-base'] + 'systems': ['wmc_base'] }, - 'wmc-base': { + 'wmc_base': { 'bin': ['wmc-dist-patch', 'wmc-dist-unpatch'], 'packages': ['Utils', 'WMCore.DataStructs', 'WMCore.Cache'], 'modules': ['WMCore.WMFactory', 'WMCore.WMException', 'WMCore.Configuration', 'WMCore.WMExceptions', 'WMCore.WMFactory', 'WMCore.Lexicon', 'WMCore.WMBase', 'WMCore.WMLogging', 'WMCore.Algorithms.Permissions'], }, - 'wmc-component': { + 'wmc_component': { 'packages': ['WMCore.MsgService', 'WMCore.WorkerThreads', 'WMCore.ThreadPool'], 'modules': ['WMComponent.__init__'], - 'systems': ['wmc-base'] + 'systems': ['wmc_base'] }, - 'wmc-database': { + 'wmc_database': { 'packages': ['WMCore.Wrappers+', 'WMCore.GroupUser', 'WMCore.DataStructs+', 'WMCore.Database+', 'WMCore.Algorithms', 'WMCore.Services'], 'modules': ['WMCore.WMConnectionBase', 'WMCore.DAOFactory', 'WMCore.WMInit'], - 'systems': ['wmc-base'] + 'systems': ['wmc_base'] }, - 'wmc-runtime': { + 'wmc_runtime': { 'packages': ['WMCore.WMRuntime+', 'WMCore.WMSpec+', 'PSetTweaks', 'WMCore.FwkJobReport', 'WMCore.Storage+', 'WMCore.Services.HTTPS'], 'modules': ['WMCore.Algorithms.ParseXMLFile'], - 'systems': ['wmc-base'] + 'systems': ['wmc_base'] }, - 'wmc-web': { + 'wmc_web': { 'packages': ['WMCore.WebTools', 'WMCore.Agent+', 'WMCore.WorkerThreads'], - 'systems': ['wmc-database', 'wmc-base'], + 'systems': ['wmc_database', 'wmc_base'], 'statics': ['src/javascript/WMCore/WebTools', 'src/javascript/external/yui', 'src/css/WMCore/WebTools', @@ -75,7 +75,7 @@ 'bin+' ], }, - 'wmagent-devtools': { + 'wmagent_devtools': { 'packages': ['WMCore+', 'WMComponent+', 'WMQuality+', @@ -105,7 +105,7 @@ 'modules': ['WMCore.WorkQueue.__init__', 'WMCore.WorkQueue.DataStructs.__init__', 'WMCore.WorkQueue.DataStructs.WorkQueueElement'], - 'systems': ['wmc-rest', 'wmc-runtime', 'wmc-database'], + 'systems': ['wmc_rest', 'wmc_runtime', 'wmc_database'], 'statics': ['src/couchapps/ReqMgr+', 'src/couchapps/ReqMgrAux+', 'src/couchapps/ConfigCache+', @@ -122,10 +122,10 @@ 'WMCore.ReqMgr.DataStructs.RequestStatus', 'WMCore.ReqMgr.DataStructs.RequestType' ], - 'systems': ['wmc-rest', 'wmc-database'], + 'systems': ['wmc_rest', 'wmc_database'], 'statics': [], }, - 'reqmgr2ms-core': { + 'reqmgr2ms_core': { 'packages': ['WMCore.MicroService.MSCore', 'WMCore.MicroService.DataStructs', 'WMCore.MicroService.Tools', 'WMCore.MicroService.CherryPyThreads', 'WMCore.MicroService.Service', 'WMCore.MicroService.WebGui', @@ -137,34 +137,34 @@ 'WMCore.ReqMgr.DataStructs.RequestStatus', 'WMCore.ReqMgr.DataStructs.RequestType' ], - 'systems': ['wmc-rest', 'wmc-database'], + 'systems': ['wmc_rest', 'wmc_database'], 'statics': [], }, - 'reqmgr2ms-unmerged': { + 'reqmgr2ms_unmerged': { 'packages': ['WMCore.MicroService.MSUnmerged+'], - 'systems': ['reqmgr2ms-core'], + 'systems': ['reqmgr2ms_core'], }, - 'reqmgr2ms-output': { + 'reqmgr2ms_output': { 'packages': ['WMCore.MicroService.MSOutput+'], - 'systems': ['reqmgr2ms-core'], + 'systems': ['reqmgr2ms_core'], }, - 'reqmgr2ms-pileup': { + 'reqmgr2ms_pileup': { 'packages': ['WMCore.MicroService.MSPileup+'], - 'systems': ['reqmgr2ms-core'], + 'systems': ['reqmgr2ms_core'], }, - 'reqmgr2ms-transferor': { + 'reqmgr2ms_transferor': { 'packages': ['WMCore.MicroService.MSTransferor+'], - 'systems': ['reqmgr2ms-core'], + 'systems': ['reqmgr2ms_core'], }, - 'reqmgr2ms-monitor': { + 'reqmgr2ms_monitor': { 'packages': ['WMCore.MicroService.MSMonitor+'], - 'systems': ['reqmgr2ms-core'], + 'systems': ['reqmgr2ms_core'], }, - 'reqmgr2ms-rulecleaner': { + 'reqmgr2ms_rulecleaner': { 'packages': ['WMCore.MicroService.MSRuleCleaner+'], - 'systems': ['reqmgr2ms-core'], + 'systems': ['reqmgr2ms_core'], }, - 'global-workqueue': { + 'global_workqueue': { 'packages': ['WMCore.GlobalWorkQueue+', 'WMCore.WorkQueue+', 'WMCore.Wrappers+', 'WMCore.Services+', 'WMCore.WMSpec', 'WMCore.WMSpec.Steps', 'WMCore.WMSpec.Steps.Templates', @@ -176,7 +176,7 @@ 'WMCore.ReqMgr.__init__', 'WMCore.ReqMgr.DataStructs.__init__', 'WMCore.ReqMgr.DataStructs.RequestStatus', 'WMCore.ReqMgr.DataStructs.RequestType'], - 'systems': ['wmc-rest', 'wmc-database'], + 'systems': ['wmc_rest', 'wmc_database'], 'statics': ['src/couchapps/WorkQueue+'], }, 'wmagent': { @@ -188,7 +188,7 @@ 'WMCore.Services+', 'WMCore.WMSpec+', 'WMCore.ReqMgr.DataStructs+', 'WMCore.ReqMgr.Tools+', 'WMCore.WMBS+', 'WMCore.ResourceControl+'], - 'systems': ['wmc-web', 'wmc-database', 'global-workqueue', 'wmc-runtime'], + 'systems': ['wmc_web', 'wmc_database', 'global_workqueue', 'wmc_runtime'], 'statics': ['bin+', 'deploy+', 'etc+', @@ -208,31 +208,31 @@ }, 'crabcache': { 'packages': ['WMCore.Wrappers+', 'WMCore.Services.UserFileCache+'], - 'systems': ['wmc-rest'], + 'systems': ['wmc_rest'], 'modules': ['WMCore.Services.Requests', 'WMCore.Services.Service', 'WMCore.Services.pycurl_manager', ], }, 'crabserver': { 'packages': ['WMCore.Credential', 'WMCore.Services+', 'WMCore.WMSpec+'], 'modules': ['WMCore.DataStructs.LumiList'], - 'systems': ['wmc-rest', 'wmc-database'], + 'systems': ['wmc_rest', 'wmc_database'], }, 'crabclient': { 'packages': ['WMCore.Wrappers+', 'WMCore.Credential', 'PSetTweaks', 'WMCore.Services.UserFileCache+', 'WMCore.Services.DBS+'], - 'systems': ['wmc-base'], + 'systems': ['wmc_base'], 'modules': ['WMCore.FwkJobReport.FileInfo', 'WMCore.Services.Requests', 'WMCore.DataStructs.LumiList', 'WMCore.Services.Service', 'WMCore.Services.pycurl_manager', ], }, 'crabtaskworker': { 'packages': ['WMCore.Credential', 'WMCore.Algorithms+', 'WMCore.WMSpec+', 'WMCore.JobSplitting', 'WMCore.Services+', 'Utils+'], - 'systems': ['wmc-database', 'wmc-runtime'], + 'systems': ['wmc_database', 'wmc_runtime'], 'modules': ['WMCore.WMBS.File', 'WMCore.WMBS.WMBSBase', 'WMCore.WMBS.__init__', 'WMCore.BossAir.Plugins.BasePlugin', 'WMCore.BossAir.Plugins.__init__'], }, 'wmclient': { - 'systems': ['wmc-runtime', 'wmc-database'] + 'systems': ['wmc_runtime', 'wmc_database'] }, 'reqmon': { 'packages': ['WMCore.WMStats+', 'WMCore.Services+', 'WMCore.Wrappers+', @@ -240,7 +240,7 @@ ], 'modules': ['WMCore.Database.__init__', 'WMCore.Database.CMSCouch', 'WMCore.Database.CouchUtils', 'WMCore.ReqMgr.__init__'], - 'systems': ['wmc-base', 'wmc-rest'], + 'systems': ['wmc_base', 'wmc_rest'], 'statics': ['src/couchapps/WMStats+', 'src/couchapps/WMStatsErl+', 'src/couchapps/WMStatsErl1+', @@ -265,7 +265,7 @@ 'statics': ['src/couchapps/ACDC+', 'src/couchapps/GroupUser+'] }, - 't0-agent': { + 't0_agent': { 'packages': ['WMCore.Agent+', 'WMCore.Algorithms+', 'WMCore.JobStateMachine', 'WMComponent+', 'WMCore.ThreadPool', 'WMCore.WorkerThreads', @@ -281,7 +281,7 @@ 'WMCore.WMException', 'WMCore.Lexicon', 'WMCore.WMBS.File'], - 'systems': ['wmc-web', 'wmc-database', 'wmc-runtime', 'global-workqueue'], + 'systems': ['wmc_web', 'wmc_database', 'wmc_runtime', 'global_workqueue'], 'statics': ['src/javascript/external/graphael', 'src/couchapps/FWJRDump+', 'src/couchapps/T0Request+', diff --git a/tools/build_pypi_packages.sh b/tools/build_pypi_packages.sh index 9b642e4055..13ae5bb681 100644 --- a/tools/build_pypi_packages.sh +++ b/tools/build_pypi_packages.sh @@ -14,8 +14,8 @@ set -x # package passed as parameter, can be one of PACKAGES or "all" TOBUILD=$1 # list of packages that can be built and uploaded to pypi -PACKAGES="wmagent wmagent-devtools wmcore reqmon reqmgr2 global-workqueue acdcserver - reqmgr2ms-unmerged reqmgr2ms-output reqmgr2ms-rulecleaner reqmgr2ms-transferor reqmgr2ms-monitor" +PACKAGES="wmagent wmagent_devtools wmcore reqmon reqmgr2 global_workqueue acdcserver + reqmgr2ms_unmerged reqmgr2ms_output reqmgr2ms_rulecleaner reqmgr2ms_transferor reqmgr2ms_monitor" PACKAGE_REGEX="^($(echo $PACKAGES | sed 's/\ /|/g')|all)$"