From c20cf50a609d9c386656925938e9ce7401273ee0 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 3 Dec 2024 14:25:54 +0100 Subject: [PATCH] fix: even more setup removals --- .github/workflows/cvmfs.yml | 2 +- .../Configuration/ConfigurationStructure/index.rst | 2 +- .../Systems/WorkloadManagement/Pilots/index.rst | 7 ------- .../CheckYourInstallation/index.rst | 8 ++++---- .../interactingWithProductionSetups.rst | 1 - .../DeveloperGuide/Internals/Core/ClientServer.rst | 4 ++-- src/DIRAC/Core/Utilities/ElasticSearchDB.py | 11 ++--------- src/DIRAC/Core/scripts/dirac_configure.py | 2 -- .../MonitoringSystem/Client/MonitoringReporter.py | 9 +++------ src/DIRAC/MonitoringSystem/DB/MonitoringDB.py | 13 +++++++------ .../MonitoringSystem/Service/MonitoringHandler.py | 4 ---- tests/CI/install_client.sh | 7 +++++-- .../Integration/Monitoring/Test_MonitoringSystem.py | 8 +++----- .../TornadoServices/Test_TornadoAndDISETmixed.py | 2 -- tests/Jenkins/install.cfg | 1 - 15 files changed, 28 insertions(+), 53 deletions(-) diff --git a/.github/workflows/cvmfs.yml b/.github/workflows/cvmfs.yml index 0ff7b1239c2..17a38ace821 100644 --- a/.github/workflows/cvmfs.yml +++ b/.github/workflows/cvmfs.yml @@ -23,7 +23,7 @@ jobs: echo "$CVMFS_PROXY_BASE64" | base64 --decode > cvmfs.proxy export X509_USER_PROXY=cvmfs.proxy chmod 600 cvmfs.proxy - ls -l + ls -l cat cvmfs.proxy export PATH=/usr/share/miniconda3/bin:/opt/conda/bin/:/opt/conda/condabin:$PATH type -a openssl diff --git a/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst b/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst index b4bc997436f..8162acd6fbe 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst @@ -52,7 +52,7 @@ in the order of preference of the option resolution: For all the DIRAC commands there is option '-o' defined which takes one configuration option setting. For example:: - dirac-wms-job-submit job.jdl -o /DIRAC/Setup=Dirac-Production + dirac-wms-job-submit job.jdl *Command line argument specifying a CFG file* A config file can be passed to any dirac command with the ``--cfg`` flag:: diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst index 7d1e80c3c09..0fa83304ae2 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst @@ -215,10 +215,6 @@ A simple example using the LHCbPilot extension follows:: DIRAC_SITE="${i#*=}" shift ;; - --lhcb-setup=*) - LHCBDIRAC_SETUP="${i#*=}" - shift - ;; --ce-name=*) CE_NAME="${i#*=}" shift @@ -237,8 +233,6 @@ A simple example using the LHCbPilot extension follows:: esac done - # Default if not given explicitly - LHCBDIRAC_SETUP=${LHCBDIRAC_SETUP:-LHCb-Production} # JOB_ID is used by when reporting LocalJobID by DIRAC watchdog #export JOB_ID="$VMTYPE:$VM_UUID" @@ -270,7 +264,6 @@ A simple example using the LHCbPilot extension follows:: #run the dirac-pilot script python dirac-pilot.py \ - --setup $LHCBDIRAC_SETUP \ --project LHCb \ --Name "$CE_NAME" \ --name "$1" \ diff --git a/docs/source/DeveloperGuide/AddingNewComponents/CheckYourInstallation/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/CheckYourInstallation/index.rst index a3dc7c49205..050eba9014e 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/CheckYourInstallation/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/CheckYourInstallation/index.rst @@ -102,15 +102,15 @@ We will now play with a **dirac.cfg** file. For these exercises you can use the Try this:: >>> from DIRAC import gConfig - >>> gConfig.getValue('/DIRAC/Setup') - 'DeveloperSetup' + >>> gConfig.getValue('/DIRAC/DefaultGroup') + 'dirac_user' -Where does 'DeveloperSetup' come from? Open that dirac.cfg and search for it. Got it? it's in:: +Where does 'dirac_user' come from? Open that dirac.cfg and search for it. Got it? it's in:: DIRAC { ... - Setup = DeveloperSetup + DefaultGroup = dirac_user ... } diff --git a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/interactingWithProductionSetups.rst b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/interactingWithProductionSetups.rst index 9f2cb144b20..e474859578f 100644 --- a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/interactingWithProductionSetups.rst +++ b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/interactingWithProductionSetups.rst @@ -11,7 +11,6 @@ So, the only real thing that you need to have is: - a DIRAC developer installation - a (real) certificate, that is recognized by your server installation -- a dirac.cfg that include the (real) setup of the production environment that you want to connect to (in DIRAC/Setup section) - a dirac.cfg that include the (real) URL of the production Configuration server. The last 2 bullets can be achieved with the following command:: diff --git a/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst b/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst index 944289afe6c..a02da426b28 100644 --- a/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst +++ b/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst @@ -166,10 +166,10 @@ Complete path of packages are not on the diagram for readability: - requestHandler: :py:class:`DIRAC.Core.DISET.RequestHandler` -You can see that the client sends a proposalTuple, proposalTuple contain (service, setup, ClientVO) then (typeOfCall, method) and finaly extra-credentials. +You can see that the client sends a proposalTuple, proposalTuple contain (service, ClientVO) then (typeOfCall, method) and finaly extra-credentials. e.g.:: - (('Framework/serviceName', 'DeveloperSetup', 'unknown'), ('RPC', 'methodName'), '') + (('Framework/serviceName', 'unknown'), ('RPC', 'methodName'), '') diff --git a/src/DIRAC/Core/Utilities/ElasticSearchDB.py b/src/DIRAC/Core/Utilities/ElasticSearchDB.py index f65b4bb29c7..0147e1c6fc0 100644 --- a/src/DIRAC/Core/Utilities/ElasticSearchDB.py +++ b/src/DIRAC/Core/Utilities/ElasticSearchDB.py @@ -142,7 +142,6 @@ def __init__( :param str client_cert: Client certificate. """ - self.__indexPrefix = indexPrefix self._connected = False if user and password: sLog.debug("Specified username and password") @@ -210,12 +209,6 @@ def __init__( except ElasticConnectionError as e: sLog.error(repr(e)) - def getIndexPrefix(self): - """ - It returns the DIRAC setup. - """ - return self.__indexPrefix - @ifConnected def addIndexTemplate( self, name: str, index_patterns: list, mapping: dict, priority: int = 1, settings: dict = None @@ -387,7 +380,7 @@ def getIndexes(self, indexName=None): It returns the available indexes... """ if not indexName: - indexName = self.__indexPrefix + indexName = "" sLog.debug(f"Getting indices alias of {indexName}") # we only return indexes which belong to a specific prefix for example 'lhcb-production' or 'dirac-production etc. return list(self.client.indices.get_alias(f"{indexName}*")) @@ -522,7 +515,7 @@ def bulk_index(self, indexPrefix, data=None, mapping=None, period="day", withTim :returns: S_OK/S_ERROR """ - sLog.verbose("Bulk indexing", f"{len(data)} records will be inserted") + sLog.verbose("Bulk indexing", f"{len(data)} records will be inserted in {indexPrefix}") if mapping is None: mapping = {} diff --git a/src/DIRAC/Core/scripts/dirac_configure.py b/src/DIRAC/Core/scripts/dirac_configure.py index e381adf429b..ad2fba345d5 100755 --- a/src/DIRAC/Core/scripts/dirac_configure.py +++ b/src/DIRAC/Core/scripts/dirac_configure.py @@ -493,8 +493,6 @@ def runDiracConfigure(params): else: DIRAC.gLogger.notice(f"Will update {DIRAC.gConfig.diracConfigFilePath}") - if params.setup: - DIRAC.gLogger.verbose("/DIRAC/Setup =", params.setup) if params.vo: DIRAC.gLogger.verbose("/DIRAC/VirtualOrganization =", params.vo) if params.configurationServer: diff --git a/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py b/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py index 4146f2b94d0..ae413cb2886 100644 --- a/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py +++ b/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py @@ -16,15 +16,12 @@ """ -import threading import json +import threading -from DIRAC import S_OK, S_ERROR, gLogger - -from DIRAC.Resources.MessageQueue.MQCommunication import createConsumer -from DIRAC.Resources.MessageQueue.MQCommunication import createProducer +from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.MonitoringSystem.Client.ServerUtils import getMonitoringDB -from DIRAC.ConfigurationSystem.Client.Config import gConfig +from DIRAC.Resources.MessageQueue.MQCommunication import createConsumer, createProducer class MonitoringReporter: diff --git a/src/DIRAC/MonitoringSystem/DB/MonitoringDB.py b/src/DIRAC/MonitoringSystem/DB/MonitoringDB.py index 0f919b72456..c86c985cf30 100644 --- a/src/DIRAC/MonitoringSystem/DB/MonitoringDB.py +++ b/src/DIRAC/MonitoringSystem/DB/MonitoringDB.py @@ -28,15 +28,15 @@ """ -import time import calendar +import time -from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Base.ElasticDB import ElasticDB -from DIRAC.Core.Utilities.Plotting.TypeLoader import TypeLoader -from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals +from DIRAC import S_ERROR, S_OK from DIRAC.ConfigurationSystem.Client.Config import gConfig +from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals from DIRAC.ConfigurationSystem.Client.PathFinder import getDatabaseSection +from DIRAC.Core.Base.ElasticDB import ElasticDB +from DIRAC.Core.Utilities.Plotting.TypeLoader import TypeLoader ######################################################################## @@ -63,7 +63,7 @@ def __init__(self, name="Monitoring/MonitoringDB"): # Load the files for pythonClassName in sorted(objectsLoaded): typeClass = objectsLoaded[pythonClassName] - indexName = f"{self.getIndexPrefix()}_{typeClass()._getIndex()}" + indexName = f"{typeClass()._getIndex()}" monitoringType = typeClass().__class__.__name__ mapping = typeClass().mapping monfields = typeClass().monitoringFields @@ -85,6 +85,7 @@ def getIndexName(self, typeName): indexName = self.documentTypes.get(typeName).get("indexName", None) if indexName: + self.log.debug("Index name", indexName) return S_OK(indexName) return S_ERROR(f"Monitoring type {typeName} is not defined") diff --git a/src/DIRAC/MonitoringSystem/Service/MonitoringHandler.py b/src/DIRAC/MonitoringSystem/Service/MonitoringHandler.py index 84b7cc4f488..18ee70a9f2a 100644 --- a/src/DIRAC/MonitoringSystem/Service/MonitoringHandler.py +++ b/src/DIRAC/MonitoringSystem/Service/MonitoringHandler.py @@ -71,8 +71,6 @@ def initializeHandler(cls, serviceInfo): return S_ERROR(f"Data location is not writable: {repr(err)}") gDataCache.setGraphsLocation(dataPath) - cls.diracSetup = CSGlobals.getSetup().lower() - return S_OK() types_listUniqueKeyValues = [str] @@ -277,7 +275,6 @@ def export_addRecords(self, indexname, monitoringType, data): :param list data: data to insert :returns: S_OK or S_ERROR """ - indexname = f"{self.diracSetup.lower()}_{indexname}" gLogger.debug("Bulk index:", indexname) mapping = self.__db.getMapping(monitoringType) gLogger.debug("Mapping:", mapping) @@ -292,7 +289,6 @@ def export_deleteIndex(self, indexName): :param str indexName: name of the index """ - indexName = f"{self.diracSetup.lower()}_{indexName}" gLogger.debug("delete index:", indexName) return self.__db.deleteIndex(indexName) diff --git a/tests/CI/install_client.sh b/tests/CI/install_client.sh index 85db87fb450..11c52883e6d 100755 --- a/tests/CI/install_client.sh +++ b/tests/CI/install_client.sh @@ -50,8 +50,11 @@ cd .. echo -e "*** $(date -u) **** Got the DIRAC tests ****\n" source "${DIRAC_CI_SETUP_SCRIPT}" -# shellcheck disable=SC2034 -DIRACSETUP=$(< "${INSTALL_CFG_FILE}" grep "Setup = " | cut -f5 -d " ") + + if [[ -n "${INSTALLATION_BRANCH}" ]]; then + # shellcheck disable=SC2034 + DIRACSETUP=$(< "${INSTALL_CFG_FILE}" grep "Setup = " | cut -f5 -d " ") + fi echo -e "*** $(date -u) **** Client INSTALLATION START ****\n" diff --git a/tests/Integration/Monitoring/Test_MonitoringSystem.py b/tests/Integration/Monitoring/Test_MonitoringSystem.py index bd1302224bd..76c0d4cd5e4 100644 --- a/tests/Integration/Monitoring/Test_MonitoringSystem.py +++ b/tests/Integration/Monitoring/Test_MonitoringSystem.py @@ -5,22 +5,20 @@ """ # pylint: disable=invalid-name,wrong-import-position -import time import json +import time from datetime import datetime import pytest -from DIRAC.tests.Utilities.utils import find_all - import DIRAC +from DIRAC.tests.Utilities.utils import find_all DIRAC.initialize() # Initialize configuration from DIRAC import gLogger -from DIRAC.MonitoringSystem.Client.MonitoringClient import MonitoringClient from DIRAC.Core.Utilities.JEncode import strToIntDict - +from DIRAC.MonitoringSystem.Client.MonitoringClient import MonitoringClient ############################################# diff --git a/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py b/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py index 06faa9740e5..d1b94791ad1 100644 --- a/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py +++ b/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py @@ -56,7 +56,6 @@ ``` DIRAC { - Setup = DeveloperSetup Configuration { Servers = dips://localhost:9135/Configuration/Server # Case 1 @@ -95,7 +94,6 @@ ``` DIRAC { - Setup = DeveloperSetup Configuration { Servers = dips://localhost:9135/Configuration/Server diff --git a/tests/Jenkins/install.cfg b/tests/Jenkins/install.cfg index 7d8eef8177a..080bdf3ef16 100644 --- a/tests/Jenkins/install.cfg +++ b/tests/Jenkins/install.cfg @@ -4,7 +4,6 @@ LocalInstallation Release = VAR_Release TargetPath = VAR_TargetPath SiteName = DIRAC.Jenkins.ch - Setup = dirac-JenkinsSetup VirtualOrganization = vo SkipCADownload = yes UseServerCertificate = yes