diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 11c91afa598..296b89a2447 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -11,7 +11,6 @@ import os from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Utilities.List import intListToString, stringListToString from DIRAC.DataManagementSystem.DB.FileCatalogComponents.DirectoryManager.DirectoryTreeBase import DirectoryTreeBase @@ -37,15 +36,24 @@ def findDir(self, path, connection=False): """ dpath = os.path.normpath(path) - result = self.db.executeStoredProcedure("ps_find_dir", (dpath, "ret1", "ret2"), outputIds=[1, 2]) + # TODO: Deprecated stored procedure ps_find_dir, replace with direct query + req = "SELECT DirID FROM FC_DirectoryList WHERE Name = %s" + result = self.db._query(req, args=(dpath,), conn=connection) if not result["OK"]: return result if not result["Value"]: return S_OK(0) - res = S_OK(result["Value"][0]) - res["Level"] = result["Value"][1] + dir_id = result["Value"][0][0] + req = "SELECT max(Depth) FROM FC_DirectoryClosure WHERE ChildID = %s" + result = self.db._query(req, args=(dir_id,), conn=connection) + if not result["OK"]: + return result + + depth = result["Value"][0][0] if result["Value"] else 0 + res = S_OK(dir_id) + res["Level"] = depth return res def findDirs(self, paths, connection=False): @@ -59,8 +67,12 @@ def findDirs(self, paths, connection=False): dirDict = {} if not paths: return S_OK(dirDict) - dpaths = stringListToString([os.path.normpath(path) for path in paths]) - result = self.db.executeStoredProcedureWithCursor("ps_find_dirs", (dpaths,)) + # TODO: Deprecated stored procedure ps_find_dirs, replace with direct query + normPaths = [os.path.normpath(path) for path in paths] + req = "SELECT Name, DirID FROM FC_DirectoryList WHERE Name IN (" + req += ",".join(["%s"] * len(normPaths)) + req += ")" + result = self.db._query(req, args=normPaths, conn=connection) if not result["OK"]: return result for dirName, dirID in result["Value"]: @@ -90,7 +102,9 @@ def removeDir(self, path): return res dirId = result["Value"] - result = self.db.executeStoredProcedure("ps_remove_dir", (dirId,), outputIds=[]) + # TODO: Deprecated stored procedure ps_remove_dir, replace with direct query + req = "DELETE FROM FC_DirectoryList WHERE DirID = %s" + result = self.db._update(req, args=(dirId,)) if not result["OK"]: return result @@ -123,15 +137,16 @@ def getDirectoryPath(self, dirID): """ - result = self.db.executeStoredProcedure("ps_get_dirName_from_id", (dirID, "out"), outputIds=[1]) + # TODO: Deprecated stored procedure ps_get_dirName_from_id, replace with direct query + req = "SELECT Name FROM FC_DirectoryList WHERE DirID = %s" + result = self.db._query(req, args=(dirID,)) if not result["OK"]: return result - dirName = result["Value"][0] - - if not dirName: + if not result["Value"]: return S_ERROR("Directory with id %d not found" % int(dirID)) + dirName = result["Value"][0][0] return S_OK(dirName) def getDirectoryPaths(self, dirIDList): @@ -147,9 +162,11 @@ def getDirectoryPaths(self, dirIDList): dirDict = {} - # Format the list - dIds = intListToString(dirs) - result = self.db.executeStoredProcedureWithCursor("ps_get_dirNames_from_ids", (dIds,)) + # TODO: Deprecated stored procedure ps_get_dirNames_from_ids, replace with direct query + req = "SELECT DirID, Name FROM FC_DirectoryList WHERE DirID IN (" + req += ",".join(["%s"] * len(dirs)) + req += ")" + result = self.db._query(req, args=dirs) if not result["OK"]: return result @@ -187,7 +204,9 @@ def getPathIDsByID(self, dirID): """ - result = self.db.executeStoredProcedureWithCursor("ps_get_parentIds_from_id", (dirID,)) + # TODO: Deprecated stored procedure ps_get_parentIds_from_id, replace with direct query + req = "SELECT ParentID FROM FC_DirectoryClosure WHERE ChildID = %s ORDER BY Depth DESC" + result = self.db._query(req, args=(dirID,)) if not result["OK"]: return result @@ -206,7 +225,9 @@ def getChildren(self, path, connection=False): else: dirID = path - result = self.db.executeStoredProcedureWithCursor("ps_get_direct_children", (dirID,)) + # TODO: Deprecated stored procedure ps_get_direct_children, replace with direct query + req = "SELECT ChildID FROM FC_DirectoryClosure WHERE ParentID = %s AND Depth = 1" + result = self.db._query(req, args=(dirID,), conn=connection) if not result["OK"]: return result if not result["Value"]: @@ -233,7 +254,16 @@ def getSubdirectoriesByID(self, dirID, requestString=False, includeParent=False) reqStr += " AND Depth != 0" return S_OK(reqStr) - result = self.db.executeStoredProcedureWithCursor("ps_get_sub_directories", (dirID, includeParent)) + # TODO: Deprecated stored procedure ps_get_sub_directories, replace with direct query + req = """SELECT c1.ChildID, max(c1.Depth) AS lvl + FROM FC_DirectoryClosure c1 + JOIN FC_DirectoryClosure c2 ON c1.ChildID = c2.ChildID + WHERE c2.ParentID = %s""" + args = [dirID] + if not includeParent: + req += " AND c2.Depth != 0" + req += " GROUP BY c1.ChildID" + result = self.db._query(req, args=args) if not result["OK"]: return result if not result["Value"]: @@ -252,8 +282,11 @@ def getAllSubdirectoriesByID(self, dirIdList): if not isinstance(dirIdList, list): dirs = [dirIdList] - dIds = intListToString(dirs) - result = self.db.executeStoredProcedureWithCursor("ps_get_multiple_sub_directories", (dIds,)) + # TODO: Deprecated stored procedure ps_get_multiple_sub_directories, replace with direct query + req = "SELECT DISTINCT ChildID FROM FC_DirectoryClosure WHERE ParentID IN (" + req += ",".join(["%s"] * len(dirs)) + req += ")" + result = self.db._query(req, args=dirs) if not result["OK"]: return result @@ -288,13 +321,17 @@ def countSubdirectories(self, dirId, includeParent=True): :returns: S_OK(value) """ - result = self.db.executeStoredProcedure( - "ps_count_sub_directories", (dirId, includeParent, "ret1"), outputIds=[2] - ) + # TODO: Deprecated stored procedure ps_count_sub_directories, replace with direct query + req = "SELECT count(ChildID) FROM FC_DirectoryClosure WHERE ParentID = %s" + result = self.db._query(req, args=(dirId,)) if not result["OK"]: return result - res = S_OK(result["Value"][0]) + count = result["Value"][0][0] if result["Value"] else 0 + # The stored procedure subtracts 1 if not includeParent + if not includeParent: + count = max(0, count - 1) + res = S_OK(count) return res ######################################################################################################## @@ -427,21 +464,7 @@ def getDirectoryParameters(self, pathOrDirId): :returns: S_OK(dict), where dict has the following keys: "DirID", "UID", "Owner", "GID", "OwnerGroup", "Status", "Mode", "CreationDate", "ModificationDate" """ - # Which procedure to use - psName = None - # it is a path ... - if isinstance(pathOrDirId, str): - psName = "ps_get_all_directory_info" - # it is the dirId - elif isinstance(pathOrDirId, ((list,) + (int,))): - psName = "ps_get_all_directory_info_from_id" - else: - return S_ERROR(f"Unknown type of pathOrDirId {type(pathOrDirId)}") - - result = self.db.executeStoredProcedureWithCursor(psName, (pathOrDirId,)) - if not result["OK"]: - return result - + # TODO: Deprecated stored procedures ps_get_all_directory_info and ps_get_all_directory_info_from_id, replace with direct query # All the fields returned fieldNames = [ "DirID", @@ -455,6 +478,27 @@ def getDirectoryParameters(self, pathOrDirId): "ModificationDate", ] + # Build the query based on input type + req = """SELECT d.DirID, d.UID, u.UserName, d.GID, g.GroupName, d.Status, d.Mode, d.CreationDate, d.ModificationDate + FROM FC_DirectoryList d + JOIN FC_Users u ON d.UID = u.UID + JOIN FC_Groups g ON d.GID = g.GID + WHERE """ + if isinstance(pathOrDirId, str): + # TODO: Deprecated stored procedure ps_get_all_directory_info, replace with direct query + req += "d.Name = %s" + args = (pathOrDirId,) + elif isinstance(pathOrDirId, ((list,) + (int,))): + # TODO: Deprecated stored procedure ps_get_all_directory_info_from_id, replace with direct query + req += "d.DirID = %s" + args = (pathOrDirId,) + else: + return S_ERROR(f"Unknown type of pathOrDirId {type(pathOrDirId)}") + + result = self.db._query(req, args=args) + if not result["OK"]: + return result + if not result["Value"]: return S_ERROR(f"Directory does not exist {pathOrDirId}") @@ -480,6 +524,9 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): S_ERROR if the directory does not exist """ + # TODO: Deprecated stored procedures ps_set_dir_uid, ps_set_dir_gid, ps_set_dir_status, ps_set_dir_mode + # and their recursive versions, replace with direct queries + # The PS associated with a given parameter psNames = { "UID": "ps_set_dir_uid", @@ -490,21 +537,76 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): psName = psNames.get(pname, None) - # If we have a recursive procedure and it is wanted, call it - if recursive and pname in ["UID", "GID", "Mode"] and psName: - psName += "_recursive" - # If there is an associated procedure, we go for it if psName: - result = self.db.executeStoredProcedureWithCursor(psName, (path, pvalue)) + # Build the query based on whether it's recursive and which parameter + if recursive and pname in ["UID", "GID", "Mode"]: + # Recursive update: update directory and all its children + # First, get the directory ID + dirResult = self.findDir(path) + if not dirResult["OK"]: + return dirResult + dirId = dirResult["Value"] + if not dirId: + return S_ERROR(errno.ENOENT, f"Directory does not exist: {path}") + + # Update directories recursively + if pname == "UID": + req = """UPDATE FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + SET d.UID = %s, d.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + result = self.db._update(req, args=(pvalue, dirId)) + # Also update files + req_files = """UPDATE FC_Files f + JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID + SET f.UID = %s, f.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + resultFiles = self.db._update(req_files, args=(pvalue, dirId)) + elif pname == "GID": + req = """UPDATE FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + SET d.GID = %s, d.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + result = self.db._update(req, args=(pvalue, dirId)) + # Also update files + req_files = """UPDATE FC_Files f + JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID + SET f.GID = %s, f.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + resultFiles = self.db._update(req_files, args=(pvalue, dirId)) + else: # Mode + req = """UPDATE FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + SET d.Mode = %s, d.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + result = self.db._update(req, args=(pvalue, dirId)) + # Also update files + req_files = """UPDATE FC_Files f + JOIN FC_DirectoryClosure c ON f.DirID = c.ChildID + SET f.Mode = %s, f.ModificationDate = UTC_TIMESTAMP() + WHERE c.ParentID = %s""" + resultFiles = self.db._update(req_files, args=(pvalue, dirId)) + else: + # Non-recursive update + if pname == "UID": + req = "UPDATE FC_DirectoryList SET UID = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" + elif pname == "GID": + req = "UPDATE FC_DirectoryList SET GID = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" + elif pname == "Status": + req = "UPDATE FC_DirectoryList SET Status = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" + elif pname == "Mode": + req = "UPDATE FC_DirectoryList SET Mode = %s, ModificationDate = UTC_TIMESTAMP() WHERE Name = %s" + result = self.db._update(req, args=(pvalue, path)) + resultFiles = S_OK(0) # Non-recursive doesn't update files if not result["OK"]: return result + if not resultFiles["OK"]: + return resultFiles - errno, affected, errMsg = result["Value"][0] - if errno: - return S_ERROR(errMsg) - + affected = result["Value"] + if not affected: # Either there were no changes, or the directory does not exist exists = self.existsDir(path).get("Value", {}).get("Exists") @@ -677,7 +779,18 @@ def _getDirectoryDump(self, path): if not dirID: return S_ERROR(errno.ENOENT, f"{path} does not exist") - result = self.db.executeStoredProcedureWithCursor("ps_get_directory_dump", (dirID,)) + # TODO: Deprecated stored procedure ps_get_directory_dump, replace with direct query + req = """(SELECT d.Name, NULL, d.CreationDate + FROM FC_DirectoryList d + JOIN FC_DirectoryClosure c ON d.DirID = c.ChildID + WHERE c.ParentID = %s AND Depth != 0) + UNION ALL + (SELECT CONCAT(d.Name, '/', f.FileName), Size, f.CreationDate + FROM FC_Files f + JOIN FC_DirectoryList d ON f.DirID = d.DirID + JOIN FC_DirectoryClosure c ON c.ChildID = f.DirID + WHERE ParentID = %s)""" + result = self.db._query(req, args=(dirID, dirID)) if not result["OK"]: return result diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py index da95306b0dc..ea70145e8d2 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py @@ -5,7 +5,7 @@ from DIRAC import S_OK, S_ERROR from DIRAC.DataManagementSystem.DB.FileCatalogComponents.FileManager.FileManagerBase import FileManagerBase -from DIRAC.Core.Utilities.List import stringListToString, intListToString, breakListIntoChunks +from DIRAC.Core.Utilities.List import intListToString, breakListIntoChunks # The logic of some methods is basically a copy/paste from the FileManager class, # so I could have inherited from it. However, I did not want to depend on it @@ -67,13 +67,17 @@ def _findFileIDs(self, lfns, connection=False): if len(lfns) == 1: lfn = list(lfns)[0] # if lfns is a dict, list(lfns) returns lfns.keys() pathPart, filePart = os.path.split(lfn) - result = self.db.executeStoredProcedure( - "ps_get_file_id_from_lfn", (pathPart, filePart, "ret1"), outputIds=[2] + req = ( + "SELECT f.FileID " + "FROM FC_Files f " + "JOIN FC_DirectoryList d ON d.DirID = f.DirID " + "WHERE d.Name = %s AND f.FileName = %s" ) + result = self.db._query(req, args=(pathPart, filePart), conn=connection) if not result["OK"]: return result - fileId = result["Value"][0] + fileId = result["Value"][0][0] if result["Value"] else 0 if not fileId: failed[lfn] = "No such file or directory" @@ -95,11 +99,10 @@ def _findFileIDs(self, lfns, connection=False): fileNames = filesInDirDict[dirPath] dirID = directoryPathToIds[dirPath] - formatedFileNames = stringListToString(fileNames) - - result = self.db.executeStoredProcedureWithCursor( - "ps_get_file_ids_from_dir_id", (dirID, formatedFileNames) - ) + req = "SELECT FileID, FileName FROM FC_Files WHERE DirID = %s AND FileName IN (" + req += ",".join(["%s"] * len(fileNames)) + req += ")" + result = self.db._query(req, args=(dirID, *fileNames), conn=connection) if not result["OK"]: return result for fileID, fileName in result["Value"]: @@ -135,14 +138,26 @@ def _getDirectoryFiles(self, dirID, fileNames, metadata_input, allStatus=False, if "FileID" not in metadata: metadata.append("FileID") - # Format the filenames and status to be used in a IN clause in the sotred procedure - formatedFileNames = stringListToString(fileNames) - fStatus = stringListToString(self.db.visibleFileStatus) + req = """SELECT FileName, DirID, f.FileID, Size, f.uid, UserName, f.gid, GroupName, s.Status, + GUID, Checksum, ChecksumType, Type, CreationDate, ModificationDate, Mode + FROM FC_Files f + JOIN FC_Users u ON f.UID = u.UID + JOIN FC_Groups g ON f.GID = g.GID + JOIN FC_Statuses s ON f.Status = s.StatusID + WHERE DirID = %s""" + args = [dirID] + if not allStatus: + req += " AND s.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleFileStatus)) + req += ")" + args.extend(self.db.visibleFileStatus) + if fileNames: + req += " AND f.FileName IN (" + req += ",".join(["%s"] * len(fileNames)) + req += ")" + args.extend(fileNames) - specificFiles = True if len(fileNames) else False - result = self.db.executeStoredProcedureWithCursor( - "ps_get_all_info_for_files_in_dir", (dirID, specificFiles, formatedFileNames, allStatus, fStatus) - ) + result = self.db._query(req, args=args, conn=connection) if not result["OK"]: return result @@ -187,9 +202,13 @@ def _getFileMetadataByID(self, fileIDs, connection=False): ["FileID", "Size", "UID", "GID", "s.Status", "GUID", "CreationDate"] """ - # Format the filenames and status to be used in a IN clause in the sotred procedure - formatedFileIds = intListToString(fileIDs) - result = self.db.executeStoredProcedureWithCursor("ps_get_all_info_for_file_ids", (formatedFileIds,)) + req = """SELECT f.FileID, Size, UID, GID, s.Status, GUID, CreationDate + FROM FC_Files f + JOIN FC_Statuses s ON f.Status = s.StatusID + WHERE f.FileID IN (""" + req += ",".join(["%s"] * len(fileIDs)) + req += ")" + result = self.db._query(req, args=fileIDs, conn=connection) if not result["OK"]: return result @@ -346,9 +365,10 @@ def _getFileIDFromGUID(self, guids, connection=False): if not isinstance(guids, (list, tuple)): guids = [guids] - # formatedGuids = ','.join( [ '"%s"' % guid for guid in guids ] ) - formatedGuids = stringListToString(guids) - result = self.db.executeStoredProcedureWithCursor("ps_get_file_ids_from_guids", (formatedGuids,)) + req = "SELECT GUID, FileID FROM FC_Files WHERE GUID IN (" + req += ",".join(["%s"] * len(guids)) + req += ")" + result = self.db._query(req, args=guids, conn=connection) if not result["OK"]: return result @@ -366,8 +386,15 @@ def getLFNForGUID(self, guids, connection=False): if not isinstance(guids, (list, tuple)): guids = [guids] - formatedGuids = stringListToString(guids) - result = self.db.executeStoredProcedureWithCursor("ps_get_lfns_from_guids", (formatedGuids,)) + req = ( + 'SELECT GUID, CONCAT(d.Name, "/", f.FileName) ' + "FROM FC_Files f " + "JOIN FC_DirectoryList d on f.DirID = d.DirID " + "WHERE GUID IN (" + ) + req += ",".join(["%s"] * len(guids)) + req += ")" + result = self.db._query(req, args=guids, conn=connection) if not result["OK"]: return result @@ -578,16 +605,19 @@ def _getRepIDsForReplica(self, replicaTuples, connection=False): replicaDict = {} - for fileID, seID in replicaTuples: - result = self.db.executeStoredProcedure("ps_get_replica_id", (fileID, seID, "repIdOut"), outputIds=[2]) - if not result["OK"]: - return result + if not replicaTuples: + return S_OK(replicaDict) - repID = result["Value"][0] + req = "SELECT RepID, FileID, SEID FROM FC_Replicas WHERE (FileID, SEID) IN (" + req += ",".join(["(%s,%s)"] * len(replicaTuples)) + req += ")" + args = tuple(value for replicaTuple in replicaTuples for value in replicaTuple) + result = self.db._query(req, args=args, conn=connection) + if not result["OK"]: + return result - # if the replica exists, we add it to the dict - if repID: - replicaDict.setdefault(fileID, {}).setdefault(seID, repID) + for repID, fileID, seID in result["Value"]: + replicaDict.setdefault(fileID, {}).setdefault(seID, repID) return S_OK(replicaDict) @@ -674,11 +704,12 @@ def _setReplicaStatus(self, fileID, se, status, connection=False): return res seID = res["Value"] - result = self.db.executeStoredProcedureWithCursor("ps_set_replica_status", (fileID, seID, statusID)) + req = "UPDATE FC_Replicas SET Status = %s WHERE FileID = %s AND SEID = %s" + result = self.db._update(req, args=(statusID, fileID, seID), conn=connection) if not result["OK"]: return result - affected = result["Value"][0][0] # Affected is the number of raws updated + affected = result["Value"] if not affected: return S_ERROR("Replica does not exist") @@ -734,36 +765,18 @@ def _setFileParameter(self, fileID, paramName, paramValue, connection=False): """ connection = self._getConnection(connection) - # The PS associated with a given parameter - psNames = { - "UID": "ps_set_file_uid", - "GID": "ps_set_file_gid", - "Status": "ps_set_file_status", - "Mode": "ps_set_file_mode", - } - - psName = psNames.get(paramName, None) - - # If there is an associated procedure, we go for it - if psName: - result = self.db.executeStoredProcedureWithCursor(psName, (fileID, paramValue)) - if not result["OK"]: - return result - - _affected = result["Value"][0][0] - # If affected = 0, the file does not exist, but who cares... - - # In case this is a 'new' parameter, we have a failback solution, but we - # should add a specific ps for it - else: - if not self.db._checkIdentifier(paramName)["OK"]: - return S_ERROR(f"ParamName is invalid: {paramName}") - req = "UPDATE FC_Files SET " - req += f"{paramName}=%s, ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (" - req += ",".join(["%s"] * len(fileID)) - req += ")" - args = [paramValue] + fileID - return self.db._update(req, args=args, conn=connection) + if not isinstance(fileID, (list, tuple)): + fileID = [fileID] + if not self.db._checkIdentifier(paramName)["OK"]: + return S_ERROR(f"ParamName is invalid: {paramName}") + req = "UPDATE FC_Files SET " + req += f"{paramName}=%s, ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (" + req += ",".join(["%s"] * len(fileID)) + req += ")" + args = [paramValue] + fileID + result = self.db._update(req, args=args, conn=connection) + if not result["OK"]: + return result return S_OK() @@ -796,17 +809,23 @@ def _getFileReplicas(self, fileIDs, fields_input=None, allStatus=False, connecti # non existing replicas replicas = {fileID: {} for fileID in fileIDs} - # Format the status to be used in a IN clause in the stored procedure - fStatus = stringListToString(self.db.visibleReplicaStatus) - fieldNames = ["FileID", "SE", "Status", "RepType", "CreationDate", "ModificationDate", "PFN"] for chunks in breakListIntoChunks(fileIDs, 1000): - # Format the FileIDs to be used in a IN clause in the stored procedure - formatedFileIds = intListToString(chunks) - result = self.db.executeStoredProcedureWithCursor( - "ps_get_all_info_of_replicas_bulk", (formatedFileIds, allStatus, fStatus) - ) + req = """SELECT FileID, se.SEName, st.Status, RepType, CreationDate, ModificationDate, PFN + FROM FC_Replicas r + JOIN FC_StorageElements se on r.SEID = se.SEID + JOIN FC_Statuses st on r.Status = st.StatusID + WHERE FileID IN (""" + req += ",".join(["%s"] * len(chunks)) + req += ")" + args = list(chunks) + if not allStatus: + req += " AND st.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleReplicaStatus)) + req += ")" + args.extend(self.db.visibleReplicaStatus) + result = self.db._query(req, args=args, conn=connection) if not result["OK"]: return result @@ -829,11 +848,12 @@ def countFilesInDir(self, dirId): :returns: S_OK(value) or S_ERROR """ - result = self.db.executeStoredProcedure("ps_count_files_in_dir", (dirId, "ret1"), outputIds=[1]) + req = "SELECT count(FileID) FROM FC_Files WHERE DirID = %s" + result = self.db._query(req, args=(dirId,)) if not result["OK"]: return result - res = S_OK(result["Value"][0]) + res = S_OK(result["Value"][0][0] if result["Value"] else 0) return res ########################################################################## @@ -880,16 +900,24 @@ def getDirectoryReplicas(self, dirID, path, allStatus=False, connection=False): values from the configuration """ - # We format the visible file/replica satus so we can give it as argument to the ps - # It is used in an IN clause, so it looks like --'"AprioriGood","Trash"'-- - # fStatus = ','.join( [ '"%s"' % status for status in self.db.visibleFileStatus ] ) - # rStatus = ','.join( [ '"%s"' % status for status in self.db.visibleReplicaStatus ] ) - fStatus = stringListToString(self.db.visibleFileStatus) - rStatus = stringListToString(self.db.visibleReplicaStatus) + req = """SELECT f.FileName, f.FileID, s.SEName, r.PFN + FROM FC_Replicas r + JOIN FC_Files f on f.FileID = r.FileID + JOIN FC_StorageElements s on s.SEID = r.SEID + JOIN FC_Statuses fst on f.Status = fst.StatusID + JOIN FC_Statuses rst on r.Status = rst.StatusID + WHERE DirID = %s""" + args = [dirID] + if not allStatus: + req += " AND fst.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleFileStatus)) + req += ") AND rst.Status IN (" + req += ",".join(["%s"] * len(self.db.visibleReplicaStatus)) + req += ")" + args.extend(self.db.visibleFileStatus) + args.extend(self.db.visibleReplicaStatus) - result = self.db.executeStoredProcedureWithCursor( - "ps_get_replicas_for_files_in_dir", (dirID, allStatus, fStatus, rStatus) - ) + result = self.db._query(req, args=args, conn=connection) if not result["OK"]: return result @@ -906,9 +934,13 @@ def _getFileLFNs(self, fileIDs): successful = {} for chunks in breakListIntoChunks(fileIDs, 1000): - # Format the filenames and status to be used in a IN clause in the sotred procedure - formatedFileIds = intListToString(chunks) - result = self.db.executeStoredProcedureWithCursor("ps_get_full_lfn_for_file_ids", (formatedFileIds,)) + req = """SELECT f.FileID, CONCAT(d.Name, "/", f.FileName) + FROM FC_Files f + JOIN FC_DirectoryList d ON f.DirID = d.DirID + WHERE f.FileID IN (""" + req += ",".join(["%s"] * len(chunks)) + req += ")" + result = self.db._query(req, args=chunks) if not result["OK"]: return result @@ -938,6 +970,13 @@ def getSEDump(self, seNames): return res seIDs.append(res["Value"]) - formatedSEIds = intListToString(seIDs) + req = """SELECT SEName, CONCAT(d.Name, "/", f.FileName), f.Checksum, f.Size + FROM FC_Files f + JOIN FC_Replicas r on f.FileID = r.FileID + JOIN FC_DirectoryList d on d.DirID = f.DirID + JOIN FC_StorageElements s on r.SEID = s.SEID + WHERE s.SEID IN (""" + req += ",".join(["%s"] * len(seIDs)) + req += ")" - return self.db.executeStoredProcedureWithCursor("ps_get_se_dump", (formatedSEIds,)) + return self.db._query(req, args=seIDs) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql b/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql index 0c00503540d..e0524bab338 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql @@ -565,6 +565,7 @@ DELIMITER ; -- ps_count_files_in_dir : counts how many files are in a given directory -- dir_id : directory id -- countFile (out value): amount of files +-- DEPRECATED: FileManagerPs.countFilesInDir now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_count_files_in_dir; DELIMITER // @@ -634,6 +635,7 @@ DELIMITER ; -- visibleFileStatus : status of files to be considered -- visibleReplicaStatus : status of replicas to be considered -- outputs : FileName, FileID, SEName, PFN +-- DEPRECATED: FileManagerPs.getDirectoryReplicas now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_replicas_for_files_in_dir; DELIMITER // @@ -661,6 +663,7 @@ DELIMITER ; -- dirName : name of the directory -- fileName : name of the file -- file_id (out) : id of the file +-- DEPRECATED: FileManagerPs._findFileIDs now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_file_id_from_lfn; DELIMITER // @@ -683,6 +686,7 @@ DELIMITER ; -- dir_id : directory id -- file_names : names of the files we are interested in -- output : FileID, FileName +-- DEPRECATED: FileManagerPs._findFileIDs now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_file_ids_from_dir_id; DELIMITER // @@ -710,6 +714,7 @@ DELIMITER ; -- visibleFileStatus : list of status we are interested in -- output : FileName, DirID, f.FileID, Size, f.uid, UserName, f.gid, GroupName, s.Status, -- GUID, Checksum, ChecksumType, Type, CreationDate,ModificationDate, Mode +-- DEPRECATED: FileManagerPs._getDirectoryFiles now uses a parametrized direct query. drop procedure if exists ps_get_all_info_for_files_in_dir; DELIMITER // @@ -745,6 +750,7 @@ DELIMITER ; -- ps_get_all_info_for_file_ids : get all the info for given file ids -- file_ids : list of file ids -- output : FileID, Size, UID, GID, s.Status, GUID, CreationDate +-- DEPRECATED: FileManagerPs._getFileMetadataByID now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_all_info_for_file_ids; DELIMITER // @@ -856,6 +862,7 @@ DELIMITER ; -- ps_get_file_ids_from_guids : return list of file ids for given guids -- guids : list of guids -- output : GUID, FileID +-- DEPRECATED: FileManagerPs._getFileIDFromGUID now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_file_ids_from_guids; DELIMITER // @@ -875,6 +882,7 @@ DELIMITER ; -- ps_get_lfns_from_guids : return list of file lfns for given guids -- guids : list of guids -- output : GUID, LFN +-- DEPRECATED: FileManagerPs.getLFNForGUID now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_lfns_from_guids; DELIMITER // @@ -1101,6 +1109,7 @@ DELIMITER ; -- file_id : file id -- se id : storage element id -- rep_id (out) : replica id +-- DEPRECATED: FileManagerPs._getRepIDsForReplica now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_replica_id; DELIMITER // @@ -1161,6 +1170,7 @@ DELIMITER ; -- status_id : new status id -- -- output : 0, number of column affected (should be 1 or 0), 'OK' +-- DEPRECATED: FileManagerPs._setReplicaStatus now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_replica_status; DELIMITER // @@ -1219,6 +1229,7 @@ DELIMITER ; -- in_uid : new uid -- -- output : errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_uid; DELIMITER // @@ -1240,6 +1251,7 @@ DELIMITER ; -- in_gid : new gid -- -- output : errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_gid; DELIMITER // @@ -1262,6 +1274,7 @@ DELIMITER ; -- status_id : id of the new status -- -- output errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_status; DELIMITER // @@ -1284,6 +1297,7 @@ DELIMITER ; -- in_mode : new mode -- -- output errno, msg. If errno ==0, all good +-- DEPRECATED: FileManagerPs._setFileParameter now uses a parametrized direct update. DROP PROCEDURE IF EXISTS ps_set_file_mode; DELIMITER // @@ -1350,6 +1364,7 @@ DELIMITER ; -- visibleReplicaStatus : list of status we are interested in -- -- output : FileID, se.SEName, st.Status, RepType, CreationDate, ModificationDate, PFN +-- DEPRECATED: FileManagerPs._getFileReplicas now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_all_info_of_replicas_bulk; DELIMITER // @@ -1903,6 +1918,7 @@ DELIMITER ; -- ps_get_full_lfn_for_file_ids : get the full lfn for given file ids -- file_ids : list of file ids -- output : FileID, lfn +-- DEPRECATED: FileManagerPs._getFileLFNs now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_full_lfn_for_file_ids; DELIMITER // @@ -1927,6 +1943,7 @@ DELIMITER ; -- ps_get_se_dump : dump all the lfns in list of SEs, with checksum and size -- se_id : storageElement IDs -- output : SEName, LFN, Checksum, Size +-- DEPRECATED: FileManagerPs.getSEDump now uses a parametrized direct query. DROP PROCEDURE IF EXISTS ps_get_se_dump; DELIMITER //