Skip to content

Improvements to RucioFileCatalogClient, compatibility of older client with Rucio 41 server #8744

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fstagni merged 5 commits into from
Aug 28, 2026
+37 −33
Merged

Improvements to RucioFileCatalogClient, compatibility of older client with Rucio 41 server #8744

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
563a095
refactor: add helper method to obtain rucio did type
maxnoe Aug 21, 2026
d60c3fd
refactor: Improve readability in RucioFileCatalogClient
maxnoe Aug 21, 2026
d6b55b4
fix: compatibility of older rucio clients with rucio 41 server
maxnoe Aug 21, 2026
48bf019
fix: Error handling of chunked requests in RucioFileCatalogClient
maxnoe Aug 21, 2026
9d67c7a
Fix: Let RucioFileCatalogClient return error for files in listDirectory
maxnoe Aug 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 37 additions & 33 deletions src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@

sLog = gLogger.getSubLogger(__name__)

RUCIO_COLLECTION_TYPES = {"DATASET", "CONTAINER"}


def get_scope(lfn, scopes=None, diracAlgorithm="dirac"):
"""
Expand Down Expand Up @@ -248,6 +250,10 @@ def getReplicas(self, lfns, allStatus=False):
return S_ERROR(str(err))
return result

def _get_did_type(self, scope, name):
"""Return type of rucio DID (CONTAINER, DATASET or FILE)."""
return self.client.get_did(scope, name)["type"]

@checkCatalogArguments
def listDirectory(self, lfns, verbose=False):
"""
Expand All @@ -263,8 +269,8 @@ def listDirectory(self, lfns, verbose=False):
try:
did = self.__getDidsFromLfn(lfn)
# First need to check if it's a dataset or container
meta = self.client.get_metadata(did["scope"], did["name"])
if meta["did_type"] == "CONTAINER":
did_type = self._get_did_type(did["scope"], did["name"])
if did_type == "CONTAINER":
if lfn not in result["Value"]["Successful"]:
result["Value"]["Successful"][lfn] = {"Files": {}, "Links": {}, "SubDirs": {}}
for child in self.client.list_content(scope=did["scope"], name=did["name"]):
Expand Down Expand Up @@ -292,7 +298,7 @@ def listDirectory(self, lfns, verbose=False):
result["Value"]["Successful"][lfn]["Files"][childName] = {"Mode": 509}
if verbose:
pass
elif meta["did_type"] == "DATASET":
elif did_type == "DATASET":
file_dict = {}
for file_did in self.client.list_files(scope=did["scope"], name=did["name"]):
guid = file_did["guid"]
Expand All @@ -301,11 +307,7 @@ def listDirectory(self, lfns, verbose=False):
file_dict[file_did["name"]] = str(uuid.UUID(guid))
if lfn not in result["Value"]["Successful"]:
result["Value"]["Successful"][lfn] = {"Files": {}, "Links": {}, "SubDirs": {}}
for rep in self.client.list_replicas(
[
did,
]
):
for rep in self.client.list_replicas([did]):
if rep:
name = rep["name"]
if self.convertUnicode:
Expand Down Expand Up @@ -339,7 +341,9 @@ def listDirectory(self, lfns, verbose=False):
"PFN": pfn,
"Status": "U",
}
except DataIdentifierNotFound as err:
else:
result["Value"]["Failed"][lfn] = "Not a directory"
except DataIdentifierNotFound:
result["Value"]["Failed"][lfn] = "No such file or directory"
except Exception as err:
return S_ERROR(str(err))
Expand All @@ -354,16 +358,16 @@ def getFileMetadata(self, lfns, ownership=False):
for chunk in lfnChunks:
try:
dids = [self.__getDidsFromLfn(lfn) for lfn in chunk]
for meta in self.client.get_metadata_bulk(dids):
for meta in self.client.get_metadata_bulk(dids, plugin="DID_COLUMN"):
lfn = str(meta["name"])
if meta["did_type"] in ["DATASET", "CONTAINER"]:
if meta["did_type"] in RUCIO_COLLECTION_TYPES:
nlinks = len([child for child in self.client.list_content(meta["scope"], meta["name"])])
successful[lfn] = {
"Checksum": "",
"ChecksumType": "",
"CreationDate": meta["created_at"],
"GUID": "",
"Mode": 509,
"Mode": 0o775,
"ModificationDate": meta["updated_at"],
"NumberOfLinks": nlinks,
"Size": 0,
Expand All @@ -382,7 +386,7 @@ def getFileMetadata(self, lfns, ownership=False):
"ChecksumType": "AD",
"CreationDate": meta["created_at"],
"GUID": guid,
"Mode": 436,
"Mode": 0o664,
"ModificationDate": meta["updated_at"],
"NumberOfLinks": 1,
"Size": meta["bytes"],
Expand All @@ -393,7 +397,8 @@ def getFileMetadata(self, lfns, ownership=False):
except ValueError:
pass
except DataIdentifierNotFound as err:
failed[lfn] = str(err)
for lfn in chunk:
failed[lfn] = str(err)
except Exception as err:
return S_ERROR(str(err))
for lfn in listFiles:
Expand All @@ -408,8 +413,8 @@ def exists(self, lfns):
for lfn in lfns:
try:
did = self.__getDidsFromLfn(lfn)
self.client.get_did(did["scope"], did["name"])
exists = True
self.client.get_metadata(did["scope"], did["name"])
except DataIdentifierNotFound:
exists = False
except Exception as err:
Expand All @@ -424,8 +429,8 @@ def getFileSize(self, lfns):
for lfn in lfns:
try:
did = self.__getDidsFromLfn(lfn)
meta = self.client.get_metadata(did["scope"], did["name"])
if meta["did_type"] == "FILE":
meta = self.client.get_did(did["scope"], did["name"])
if meta["type"] == "FILE":
result["Value"]["Successful"][lfn] = meta["bytes"]
else:
result["Value"]["Successful"][lfn] = 0
Expand All @@ -441,9 +446,9 @@ def isDirectory(self, lfns):
result = {"Successful": {}, "Failed": {}}
dids = [self.__getDidsFromLfn(lfn) for lfn in lfns]
try:
for meta in self.client.get_metadata_bulk(dids):
for meta in self.client.get_metadata_bulk(dids, plugin="DID_COLUMN"):
lfn = str(meta["name"])
result["Successful"][lfn] = meta["did_type"] in ["DATASET", "CONTAINER"]
result["Successful"][lfn] = meta["did_type"] in RUCIO_COLLECTION_TYPES
for lfn in lfns:
if lfn not in result["Successful"] and lfn not in result["Failed"]:
result["Failed"][lfn] = "No such file or directory"
Expand All @@ -459,9 +464,9 @@ def isFile(self, lfns):
for lfn in lfns:
dids.append(self.__getDidsFromLfn(lfn))
try:
for meta in self.client.get_metadata_bulk(dids):
for meta in self.client.get_metadata_bulk(dids, plugin="DID_COLUMN"):
lfn = str(meta["name"])
result["Successful"][lfn] = meta["did_type"] in ["FILE"]
result["Successful"][lfn] = meta["did_type"] == "FILE"
for lfn in lfns:
if lfn not in result["Successful"] and lfn not in result["Failed"]:
result["Failed"][lfn] = "No such file or directory"
Expand Down Expand Up @@ -533,7 +538,7 @@ def addReplica(self, lfns):
try:
did = self.__getDidsFromLfn(lfn)
if not size or not checksum:
meta = self.client.get_metadata(did["scope"], did["name"])
meta = self.client.get_did(did["scope"], did["name"])
size = meta["bytes"]
checksum = meta["adler32"]
rep = {"scope": did["scope"], "name": did["name"], "bytes": size, "adler32": checksum}
Expand Down Expand Up @@ -575,14 +580,14 @@ def removeReplica(self, lfns):
se = info["SE"]
try:
did = self.__getDidsFromLfn(lfn)
meta = self.client.get_metadata(did["scope"], did["name"])
if meta["did_type"] == "FILE":
did_type = self._get_did_type(did["scope"], did["name"])
if did_type == "FILE":
# For file cannot use dataset_locks to identify the rule
for rule in self.client.list_did_rules(did["scope"], did["name"]):
rid = rule["id"]
self.client.update_replication_rule(rid, options={"lifetime": -86400})
successful[lfn] = True
elif meta["did_type"] == "DATASET":
elif did_type == "DATASET":
rules = {}
for lock in self.client.get_dataset_locks(did["scope"], did["name"]):
rule_id = lock["rule_id"]
Expand Down Expand Up @@ -612,8 +617,8 @@ def removeFile(self, lfns):
for lfn in lfns:
try:
did = self.__getDidsFromLfn(lfn)
meta = self.client.get_metadata(did["scope"], did["name"])
if meta["did_type"] == "FILE":
did_type = self._get_did_type(did["scope"], did["name"])
if did_type == "FILE":
parentLfn = "/".join(lfn.split("/")[:-1])
parentDid = self.__getDidsFromLfn(parentLfn)
dsnScope, dsnName = parentDid["scope"], parentDid["name"]
Expand All @@ -637,8 +642,8 @@ def removeDirectory(self, lfns):
for lfn in lfns:
try:
did = self.__getDidsFromLfn(lfn)
meta = self.client.get_metadata(did["scope"], did["name"])
if meta["did_type"] == "DATASET":
did_type = self._get_did_type(did["scope"], did["name"])
if did_type == "DATASET":
try:
self.client.set_metadata(scope=did["scope"], name=did["name"], key="lifetime", value=1)
resDict["Successful"][lfn] = True
Expand All @@ -659,11 +664,11 @@ def getDirectorySize(self, lfns, longOutput=False, rawFiles=False):
for lfn in lfns:
try:
did = self.__getDidsFromLfn(lfn)
meta = self.client.get_metadata(did["scope"], did["name"])
if meta["did_type"] == "FILE":
did_type = self._get_did_type(did["scope"], did["name"])
if did_type == "FILE":
resDict["Failed"][lfn] = "Not a directory"

elif meta["did_type"] == "CONTAINER":
elif did_type == "CONTAINER":
resDict["Successful"][lfn] = {
"ClosedDirs": [],
"Files": 0,
Expand Down Expand Up @@ -728,7 +733,6 @@ def getFileUserMetadataBulk(self, lfns):
all their parents
"""
resDict = {"Successful": {}, "Failed": {}}
dids = []
lfnChunks = breakListIntoChunks(lfns, 1000)
for lfnList in lfnChunks:
try:
Expand Down
Loading