diff --git a/src/python/dbs/apis/dbsClient.py b/src/python/dbs/apis/dbsClient.py index d1ff8b8..3fa0789 100644 --- a/src/python/dbs/apis/dbsClient.py +++ b/src/python/dbs/apis/dbsClient.py @@ -1,14 +1,18 @@ -from dbs.exceptions.dbsClientException import dbsClientException -from RestClient.ErrorHandling.RestClientExceptions import HTTPError -from RestClient.RestApi import RestApi -from RestClient.AuthHandling.X509Auth import X509Auth -from RestClient.ProxyPlugins.Socks5Proxy import Socks5Proxy - import json import os import socket import sys -import urllib.request, urllib.parse, urllib.error +import urllib.error +import urllib.parse +import urllib.request + +from RestClient.AuthHandling.X509Auth import X509Auth +from RestClient.ErrorHandling.RestClientExceptions import HTTPError +from RestClient.ProxyPlugins.Socks5Proxy import Socks5Proxy +from RestClient.RestApi import RestApi + +from dbs.exceptions.dbsClientException import dbsClientException + def slicedIterator(sourceList, sliceSize): """ @@ -26,6 +30,7 @@ def slicedIterator(sourceList, sliceSize): yield sourceList[start: end] start = end + def checkInputParameter(method, parameters, validParameters, requiredParameters=None): """ Helper function to check input by using before sending to the server @@ -48,7 +53,7 @@ def checkInputParameter(method, parameters, validParameters, requiredParameters= if 'multiple' in requiredParameters: match = False for requiredParameter in requiredParameters['multiple']: - if requiredParameter!='detail' and requiredParameter in parameters: + if requiredParameter != 'detail' and requiredParameter in parameters: match = True break if not match: @@ -68,11 +73,12 @@ def checkInputParameter(method, parameters, validParameters, requiredParameters= for requiredParameter in requiredParameters['standalone']: if requiredParameter in parameters: overlap.append(requiredParameter) - if len(overlap) != 1: + if len(overlap) != 1: raise dbsClientException("Invalid input", "API %s does requires only *one* of the parameters %s." \ % (method, requiredParameters['standalone'])) + def list_parameter_splitting(data, key, size_limit=8000, method='GET'): """ Helper function split list used as input parameter for requests, @@ -92,7 +98,7 @@ def list_parameter_splitting(data, key, size_limit=8000, method='GET'): for element in values: data[key].append(element) - if method =='GET': + if method == 'GET': size = len(urllib.parse.urlencode(data)) else: size = len(data) @@ -103,11 +109,13 @@ def list_parameter_splitting(data, key, size_limit=8000, method='GET'): yield data + def split_calls(func): """ Decorator to split up server calls for methods using url parameters, due to the lenght limitation of the URI in Apache. By default 8190 bytes """ + def wrapper(*args, **kwargs): """ The size limit is 8190 bytes minus url and api to call @@ -121,24 +129,28 @@ def wrapper(*args, **kwargs): ###currently only file lists are supported if key in ('logical_file_name', 'block_name', 'lumi_list', 'run_num') and isinstance(value, list): ret_val = [] - for splitted_param in list_parameter_splitting(data=dict(kwargs), #make a copy, since it is manipulated + # make a copy of kwargs, since it is manipulated + for splitted_param in list_parameter_splitting(data=dict(kwargs), key=key, size_limit=size_limit): try: ret_val.extend(func(*args, **splitted_param)) - except (TypeError, AttributeError):#update function call do not return lists - ret_val= [] + except (TypeError, AttributeError): # update function call do not return lists + ret_val = [] return ret_val raise dbsClientException("Invalid input", "The lenght of the urlencoded parameters to API %s \ is exceeding %s bytes and cannot be splitted." % (func.__name__, size_limit)) else: return func(*args, **kwargs) + return wrapper + class DbsApi(object): - #added CAINFO and userAgent (see github issue #431 & #432) - def __init__(self, url="", proxy=None, key=None, cert=None, verifypeer=True, debug=0, ca_info=None, userAgent="", port=8443): + # added CAINFO and userAgent (see github issue #431 & #432) + def __init__(self, url="", proxy=None, key=None, cert=None, verifypeer=True, + debug=0, ca_info=None, userAgent="", port=8443): """ DbsApi Constructor @@ -151,7 +163,7 @@ def __init__(self, url="", proxy=None, key=None, cert=None, verifypeer=True, deb :param cert: full path to the certificate to use :type cert: str :param port: server port - :type port int + :type port int .. note:: By default the DbsApi is trying to lookup the private key and the certificate in the common locations @@ -185,12 +197,12 @@ def __callServer(self, method="", params={}, data={}, callmethod='GET', content= :type data: dict """ - UserID = os.environ['USER']+'@'+socket.gethostname() + UserID = os.environ['USER'] + '@' + socket.gethostname() try: - UserAgent = "DBSClient/"+os.environ['DBS3_CLIENT_VERSION']+"/"+ self.userAgent + UserAgent = "DBSClient/" + os.environ['DBS3_CLIENT_VERSION'] + "/" + self.userAgent except: - UserAgent = "DBSClient/Unknown"+"/"+ self.userAgent - request_headers = {"Content-Type": content, "Accept": content, "UserID": UserID, "User-Agent":UserAgent } + UserAgent = "DBSClient/Unknown" + "/" + self.userAgent + request_headers = {"Content-Type": content, "Accept": content, "UserID": UserID, "User-Agent": UserAgent} method_func = getattr(self.rest_api, callmethod.lower()) @@ -205,10 +217,11 @@ def __callServer(self, method="", params={}, data={}, callmethod='GET', content= return self.http_response.body try: - json_ret=json.loads(self.http_response.body) + json_ret = json.loads(self.http_response.body) except ValueError as ex: - print("The server output is not a valid json, most probably you have a typo in the url.\n%s.\n" % self.url, file=sys.stderr) - raise dbsClientException("Invalid url", "Possible urls are %s" %self.http_response.body) + print("The server output is not a valid json, most probably you have a typo in the url.\n%s.\n" % self.url, + file=sys.stderr) + raise dbsClientException("Invalid url", "Possible urls are %s" % self.http_response.body) return json_ret @@ -220,12 +233,11 @@ def __parseForException(self, http_error): """ data = http_error.body try: - if isinstance(data, str): - data = json.loads(data) + data = json.loads(data) except: raise http_error - if isinstance(data, dict) and 'exception' in data:# re-raise with more details + if isinstance(data, dict) and 'exception' in data: # re-raise with more details raise HTTPError(http_error.url, data['exception'], data['message'], http_error.header, http_error.body) raise http_error @@ -255,7 +267,7 @@ def requestContentLength(self): except AttributeError: return None - def blockDump(self,**kwargs): + def blockDump(self, **kwargs): """ API the list all information related with the block_name @@ -265,10 +277,10 @@ def blockDump(self,**kwargs): """ validParameters = ['block_name'] - requiredParameters = {'forced':validParameters} + requiredParameters = {'forced': validParameters} - checkInputParameter(method="blockDump", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="blockDump", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("blockdump", params=kwargs) @@ -300,7 +312,7 @@ def insertAcquisitionEra(self, acqEraObj): :key end_date: end data of the acquisition era (unixtime, int) (Optional) """ - return self.__callServer("acquisitioneras", data=acqEraObj, callmethod='POST' ) + return self.__callServer("acquisitioneras", data=acqEraObj, callmethod='POST') def insertBlock(self, blockObj): """ @@ -315,7 +327,7 @@ def insertBlock(self, blockObj): :key origin_site_name: Origin Site Name (Required) """ - return self.__callServer("blocks", data=blockObj, callmethod='POST' ) + return self.__callServer("blocks", data=blockObj, callmethod='POST') def insertBulkBlock(self, blockDump): """ @@ -325,7 +337,7 @@ def insertBulkBlock(self, blockDump): :type blockDump: dict """ - #We first check if the first lumi section has event_count or not + # We first check if the first lumi section has event_count or not frst = True if (blockDump['files'][0]['file_lumi_list'][0]).get('event_count') == None: frst = False # when frst == True, we are looking for event_count == None in the data, if we did not find None (redFlg = False), @@ -333,19 +345,19 @@ def insertBulkBlock(self, blockDump): # when frst == False, weare looking for event_count != None in the data, if we did not find Not None (redFlg = False), # everything is good. Otherwise, we have to remove all even_count in lumis and raise exception. redFlag = False if frst: - eventCT = (fl.get('event_count') is None for f in blockDump['files'] for fl in f['file_lumi_list']) - else: - eventCT = (fl.get('event_count') is not None for f in blockDump['files'] for fl in f['file_lumi_list']) + eventCT = (fl.get('event_count') is None for f in blockDump['files'] for fl in f['file_lumi_list']) + else: + eventCT = (fl.get('event_count') is not None for f in blockDump['files'] for fl in f['file_lumi_list']) redFlag = any(eventCT) if redFlag: for f in blockDump['files']: for fl in f['file_lumi_list']: if 'event_count' in fl: del fl['event_count'] - result = self.__callServer("bulkblocks", data=blockDump, callmethod='POST' ) + result = self.__callServer("bulkblocks", data=blockDump, callmethod='POST') if redFlag: - raise dbsClientException("Mixed event_count per lumi in the block: %s" %blockDump['block']['block_name'], - "The block was inserted into DBS, but you need to check if the data is valid.") + raise dbsClientException("Mixed event_count per lumi in the block: %s" % blockDump['block']['block_name'], + "The block was inserted into DBS, but you need to check if the data is valid.") else: return result @@ -368,7 +380,7 @@ def insertDataset(self, datasetObj): :key output_configs: List(dict) with keys release_version, pset_hash, app_name, output_module_label and global tag """ - return self.__callServer("datasets", data = datasetObj, callmethod='POST' ) + return self.__callServer("datasets", data=datasetObj, callmethod='POST') def insertDataTier(self, dataTierObj): """ @@ -379,7 +391,7 @@ def insertDataTier(self, dataTierObj): :key data_tier_name: Data Tier that needs to be inserted """ - return self.__callServer("datatiers", data = dataTierObj, callmethod='POST' ) + return self.__callServer("datatiers", data=dataTierObj, callmethod='POST') def insertFiles(self, filesList, qInserts=False): """ @@ -407,9 +419,9 @@ def insertFiles(self, filesList, qInserts=False): """ - if not qInserts: #turn off qInserts - return self.__callServer("files", params={'qInserts': qInserts}, data=filesList, callmethod='POST' ) - return self.__callServer("files", data=filesList, callmethod='POST' ) + if not qInserts: # turn off qInserts + return self.__callServer("files", params={'qInserts': qInserts}, data=filesList, callmethod='POST') + return self.__callServer("files", data=filesList, callmethod='POST') def insertOutputConfig(self, outputConfigObj): """ @@ -426,7 +438,7 @@ def insertOutputConfig(self, outputConfigObj): :key pset_name: Pset Name (Optional, default is None) """ - return self.__callServer("outputconfigs", data=outputConfigObj, callmethod='POST' ) + return self.__callServer("outputconfigs", data=outputConfigObj, callmethod='POST') def insertPrimaryDataset(self, primaryDSObj): """ @@ -438,7 +450,7 @@ def insertPrimaryDataset(self, primaryDSObj): :key primary_ds_name: Name of the primary dataset (Required) """ - return self.__callServer("primarydatasets", data=primaryDSObj, callmethod='POST' ) + return self.__callServer("primarydatasets", data=primaryDSObj, callmethod='POST') def insertProcessingEra(self, procEraObj): """ @@ -450,7 +462,7 @@ def insertProcessingEra(self, procEraObj): :key description: Description (Optional) """ - return self.__callServer("processingeras", data=procEraObj, callmethod='POST' ) + return self.__callServer("processingeras", data=procEraObj, callmethod='POST') def insertFileParents(self, fileParentObj): """ @@ -463,7 +475,7 @@ def insertFileParents(self, fileParentObj): :key child_parent_id_list: a list of [child_file_id, parent_file_id] pairs (required) """ - return self.__callServer("fileparents", data=fileParentObj, callmethod='POST' ) + return self.__callServer("fileparents", data=fileParentObj, callmethod='POST') def listParentDSTrio(self, **kwargs): """ @@ -478,8 +490,8 @@ def listParentDSTrio(self, **kwargs): validParameters = ['dataset'] requiredParameters = {'forced': ['dataset']} - checkInputParameter(method="listParentDSTrio", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listParentDSTrio", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("parentDSTrio", params=kwargs, callmethod='GET') def listBlockTrio(self, **kwargs): @@ -497,8 +509,8 @@ def listBlockTrio(self, **kwargs): validParameters = ['block_name'] requiredParameters = {'forced': ['block_name']} - checkInputParameter(method="listBlockTrio", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listBlockTrio", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("blockTrio", params=kwargs, callmethod='GET') def listFileParentsByLumi(self, **kwargs): @@ -516,11 +528,10 @@ def listFileParentsByLumi(self, **kwargs): validParameters = ['block_name', 'logical_file_name'] requiredParameters = {'forced': ['block_name']} - checkInputParameter(method="listFileParentsByLumi", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listFileParentsByLumi", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("fileparentsbylumi", data=kwargs, callmethod='POST') - def listApiDocumentation(self): """ API to retrieve the auto-generated documentation page from server @@ -540,11 +551,11 @@ def listAcquisitionEras(self, **kwargs): """ validParameters = ['acquisition_era_name'] - checkInputParameter(method="listAcquisitionEras", parameters=list(kwargs.keys()), validParameters=validParameters) + checkInputParameter(method="listAcquisitionEras", parameters=list(kwargs.keys()), + validParameters=validParameters) return self.__callServer("acquisitioneras", params=kwargs) - def listAcquisitionEras_ci(self, **kwargs): """ API to list all Acquisition Eras (case insensitive) in DBS. @@ -557,7 +568,8 @@ def listAcquisitionEras_ci(self, **kwargs): """ validParameters = ['acquisition_era_name'] - checkInputParameter(method="listAcquisitionEras", parameters=list(kwargs.keys()), validParameters=validParameters) + checkInputParameter(method="listAcquisitionEras", parameters=list(kwargs.keys()), + validParameters=validParameters) return self.__callServer("acquisitioneras_ci", params=kwargs) @@ -575,8 +587,8 @@ def listBlockChildren(self, **kwargs): requiredParameters = {'forced': validParameters} - checkInputParameter(method="listBlockChildren", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listBlockChildren", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("blockchildren", params=kwargs) @@ -593,8 +605,8 @@ def listBlockParents(self, **kwargs): validParameters = ['block_name'] requiredParameters = {'forced': validParameters} - checkInputParameter(method="listBlockParents", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listBlockParents", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) if isinstance(kwargs["block_name"], list): return self.__callServer("blockparents", data=kwargs, callmethod='POST') else: @@ -641,15 +653,15 @@ def listBlocks(self, **kwargs): 'max_cdate', 'min_ldate', 'max_ldate', 'cdate', 'ldate', 'detail'] - #requiredParameters = {'multiple': validParameters} + # requiredParameters = {'multiple': validParameters} requiredParameters = {'multiple': ['dataset', 'block_name', 'data_tier_name', 'logical_file_name']} - #set defaults + # set defaults if 'detail' not in list(kwargs.keys()): kwargs['detail'] = False - checkInputParameter(method="listBlocks", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listBlocks", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("blocks", params=kwargs) @@ -666,9 +678,7 @@ def listBlockSummaries_doc(self, **kwargs): :returns: list of dicts containing total block_sizes, file_counts and event_counts of dataset or blocks provided """ - pass - - + pass @split_calls def listBlockSummaries(self, **kwargs): @@ -688,8 +698,8 @@ def listBlockSummaries(self, **kwargs): requiredParameters = {'standalone': ['block_name', 'dataset']} - checkInputParameter(method="listBlockSummaries", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listBlockSummaries", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer('blocksummaries', params=kwargs) @@ -711,8 +721,8 @@ def listBlockOrigin(self, **kwargs): requiredParameters = {'multiple': ['dataset', 'block_name']} - checkInputParameter(method="listBlockOrigin", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listBlockOrigin", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer('blockorigin', params=kwargs) def listDatasets(self, **kwargs): @@ -788,7 +798,7 @@ def listDatasets(self, **kwargs): 'min_cdate', 'max_cdate', 'min_ldate', 'max_ldate', 'cdate', 'ldate', 'detail', 'dataset_id'] - #set defaults + # set defaults if 'detail' not in list(kwargs.keys()): kwargs['detail'] = False @@ -808,7 +818,8 @@ def listDatasetAccessTypes(self, **kwargs): """ validParameters = ['dataset_access_type'] - checkInputParameter(method="listDatasetAccessTypes", parameters=list(kwargs.keys()), validParameters=validParameters) + checkInputParameter(method="listDatasetAccessTypes", parameters=list(kwargs.keys()), + validParameters=validParameters) return self.__callServer("datasetaccesstypes", params=kwargs) @@ -831,10 +842,10 @@ def listDatasetArray(self, **kwargs): validParameters = ['dataset', 'dataset_access_type', 'detail', 'dataset_id'] requiredParameters = {'multiple': ['dataset', 'dataset_id']} - checkInputParameter(method="listDatasetArray", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listDatasetArray", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) - #set defaults + # set defaults if 'detail' not in list(kwargs.keys()): kwargs['detail'] = False @@ -853,8 +864,8 @@ def listDatasetChildren(self, **kwargs): validParameters = ['dataset'] requiredParameters = {'forced': validParameters} - checkInputParameter(method="listDatasetChildren", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listDatasetChildren", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("datasetchildren", params=kwargs) @@ -871,8 +882,8 @@ def listDatasetParents(self, **kwargs): validParameters = ['dataset'] requiredParameters = {'forced': validParameters} - checkInputParameter(method="listDatasetParents", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listDatasetParents", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("datasetparents", params=kwargs) @@ -945,8 +956,8 @@ def listFileChildren(self, **kwargs): requiredParameters = {'standalone': validParameters} - checkInputParameter(method="listFileChildren", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listFileChildren", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("filechildren", params=kwargs) @@ -971,8 +982,8 @@ def listFileLumis(self, **kwargs): requiredParameters = {'standalone': ['logical_file_name', 'block_name']} - checkInputParameter(method="listFileLumis", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listFileLumis", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("filelumis", params=kwargs) @@ -994,8 +1005,8 @@ def listFileLumiArray(self, **kwargs): validParameters = ['logical_file_name', 'run_num', 'validFileOnly'] requiredParameters = {'forced': ['logical_file_name']} - checkInputParameter(method="listFileLumiArray", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listFileLumiArray", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("filelumis", data=kwargs, callmethod="POST") @@ -1034,8 +1045,8 @@ def listFileParents(self, **kwargs): requiredParameters = {'standalone': validParameters} - checkInputParameter(method="listFileParents", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listFileParents", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("fileparents", params=kwargs) @@ -1094,7 +1105,7 @@ def listFiles_doc(self, **kwargs): """ pass - @split_calls + @split_calls def listFiles(self, **kwargs): """ listFiles(**kwargs) @@ -1147,22 +1158,21 @@ def listFiles(self, **kwargs): """ validParameters = ['dataset', 'block_name', 'logical_file_name', - 'release_version', 'pset_hash', 'app_name', - 'output_module_label', 'run_num', - 'origin_site_name', 'lumi_list', 'detail', 'validFileOnly', 'sumOverLumi'] + 'release_version', 'pset_hash', 'app_name', + 'output_module_label', 'run_num', + 'origin_site_name', 'lumi_list', 'detail', 'validFileOnly', 'sumOverLumi'] requiredParameters = {'multiple': validParameters} - #set defaults + # set defaults if 'detail' not in list(kwargs.keys()): kwargs['detail'] = False - checkInputParameter(method="listFiles", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listFiles", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("files", params=kwargs) - def listFileArray(self, **kwargs): """ API to list files in DBS. Non-wildcarded logical_file_name, non-wildcarded dataset, non-wildcarded block_name or non-wildcarded lfn list is required. @@ -1208,33 +1218,33 @@ def listFileArray(self, **kwargs): """ validParameters = ['dataset', 'block_name', 'logical_file_name', - 'release_version', 'pset_hash', 'app_name', - 'output_module_label', 'run_num', - 'origin_site_name', 'lumi_list', 'detail', 'validFileOnly', 'sumOverLumi'] + 'release_version', 'pset_hash', 'app_name', + 'output_module_label', 'run_num', + 'origin_site_name', 'lumi_list', 'detail', 'validFileOnly', 'sumOverLumi'] requiredParameters = {'multiple': ['dataset', 'block_name', 'logical_file_name']} - #set defaults + # set defaults if 'detail' not in list(kwargs.keys()): kwargs['detail'] = False - checkInputParameter(method="listFileArray", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listFileArray", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) # In order to protect DB and make sure the query can be return in 300 seconds, we limit the length of # logical file names, lumi and run num to 1000. These number may be adjusted later if # needed. YG May-20-2015. # CMS has all MC data with run_num=1. It almost is a full table scan if run_num=1 without lfn. So we will request lfn # to be present when run_num=1. YG Jan 14, 2016 - if 'logical_file_name' in list(kwargs.keys()) and isinstance(kwargs['logical_file_name'], list)\ - and len(kwargs['logical_file_name']) > 1: - if 'run_num' in list(kwargs.keys()) and isinstance(kwargs['run_num'],list) and len(kwargs['run_num']) > 1 : + if 'logical_file_name' in list(kwargs.keys()) and isinstance(kwargs['logical_file_name'], list) \ + and len(kwargs['logical_file_name']) > 1: + if 'run_num' in list(kwargs.keys()) and isinstance(kwargs['run_num'], list) and len(kwargs['run_num']) > 1: raise dbsClientException('Invalid input', 'files API does not supprt two lists: run_num and lfn. ') - elif 'lumi_list' in list(kwargs.keys()) and kwargs['lumi_list'] and len(kwargs['lumi_list']) > 1 : + elif 'lumi_list' in list(kwargs.keys()) and kwargs['lumi_list'] and len(kwargs['lumi_list']) > 1: raise dbsClientException('Invalid input', 'files API does not supprt two lists: lumi_lis and lfn. ') - + elif 'lumi_list' in list(kwargs.keys()) and kwargs['lumi_list']: - if 'run_num' not in list(kwargs.keys()) or not kwargs['run_num'] or kwargs['run_num'] ==-1 : + if 'run_num' not in list(kwargs.keys()) or not kwargs['run_num'] or kwargs['run_num'] == -1: raise dbsClientException('Invalid input', 'When Lumi section is present, a single run is required. ') else: if 'run_num' in list(kwargs.keys()): @@ -1242,51 +1252,55 @@ def listFileArray(self, **kwargs): if 1 in kwargs['run_num'] or '1' in kwargs['run_num']: raise dbsClientException('Invalid input', 'files API does not supprt run_num=1 when no lumi.') else: - if kwargs['run_num']==1 or kwargs['run_num']=='1': + if kwargs['run_num'] == 1 or kwargs['run_num'] == '1': raise dbsClientException('Invalid input', 'files API does not supprt run_num=1 when no lumi.') - #check if no lfn is given, but run_num=1 is used for searching - if ('logical_file_name' not in list(kwargs.keys()) or not kwargs['logical_file_name']) and 'run_num' in list(kwargs.keys()): + # check if no lfn is given, but run_num=1 is used for searching + if ('logical_file_name' not in list(kwargs.keys()) or not kwargs['logical_file_name']) and 'run_num' in list( + kwargs.keys()): if isinstance(kwargs['run_num'], list): if 1 in kwargs['run_num'] or '1' in kwargs['run_num']: - raise dbsClientException('Invalid input', 'files API does not supprt run_num=1 without logical_file_name.') + raise dbsClientException('Invalid input', + 'files API does not supprt run_num=1 without logical_file_name.') else: if kwargs['run_num'] == 1 or kwargs['run_num'] == '1': - raise dbsClientException('Invalid input', 'files API does not supprt run_num=1 without logical_file_name.') - + raise dbsClientException('Invalid input', + 'files API does not supprt run_num=1 without logical_file_name.') + results = [] mykey = None total_lumi_len = 0 split_lumi_list = [] - max_list_len = 1000 #this number is defined in DBS server + max_list_len = 1000 # this number is defined in DBS server for key, value in kwargs.items(): - if key == 'lumi_list' and isinstance(kwargs['lumi_list'], list)\ - and kwargs['lumi_list'] and isinstance(kwargs['lumi_list'][0], list): + if key == 'lumi_list' and isinstance(kwargs['lumi_list'], list) \ + and kwargs['lumi_list'] and isinstance(kwargs['lumi_list'][0], list): lapp = 0 l = 0 sm = [] for i in kwargs['lumi_list']: - while i[0]+max_list_len < i[1]: - split_lumi_list.append([[i[0], i[0]+max_list_len-1]]) + while i[0] + max_list_len < i[1]: + split_lumi_list.append([[i[0], i[0] + max_list_len - 1]]) i[0] = i[0] + max_list_len else: - l += (i[1]-i[0]+1) - if l <= max_list_len: + l += (i[1] - i[0] + 1) + if l <= max_list_len: sm.append([i[0], i[1]]) - lapp = l #number lumis in sm + lapp = l # number lumis in sm else: split_lumi_list.append(sm) - sm=[] + sm = [] sm.append([i[0], i[1]]) - lapp = i[1]-i[0]+1 + lapp = i[1] - i[0] + 1 if sm: split_lumi_list.append(sm) - elif key in ('logical_file_name', 'run_num', 'lumi_list') and isinstance(value, list) and len(value)>max_list_len: - mykey =key -# - if mykey: + elif key in ('logical_file_name', 'run_num', 'lumi_list') and isinstance(value, list) and len( + value) > max_list_len: + mykey = key + # + if mykey: sourcelist = [] - #create a new list to slice + # create a new list to slice sourcelist = kwargs[mykey][:] for slice in slicedIterator(sourcelist, max_list_len): kwargs[mykey] = slice @@ -1297,10 +1311,10 @@ def listFileArray(self, **kwargs): results.extend(self.__callServer("fileArray", data=kwargs, callmethod="POST")) else: return self.__callServer("fileArray", data=kwargs, callmethod="POST") - - #make sure only one dictionary per lfn. - #Make sure this changes when we move to 2.7 or 3.0 - #http://stackoverflow.com/questions/11092511/python-list-of-unique-dictionaries + + # make sure only one dictionary per lfn. + # Make sure this changes when we move to 2.7 or 3.0 + # http://stackoverflow.com/questions/11092511/python-list-of-unique-dictionaries # YG May-26-2015 return list(dict((v['logical_file_name'], v) for v in results).values()) @@ -1338,8 +1352,8 @@ def listFileSummaries(self, **kwargs): requiredParameters = {'standalone': ['block_name', 'dataset']} - checkInputParameter(method="listFileSummaries", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listFileSummaries", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("filesummaries", params=kwargs) @@ -1410,7 +1424,8 @@ def listPrimaryDatasets(self, **kwargs): """ validParameters = ['primary_ds_name', 'primary_ds_type'] - checkInputParameter(method="listPrimaryDatasets", parameters=list(kwargs.keys()), validParameters=validParameters) + checkInputParameter(method="listPrimaryDatasets", parameters=list(kwargs.keys()), + validParameters=validParameters) return self.__callServer("primarydatasets", params=kwargs) @@ -1428,7 +1443,8 @@ def listPrimaryDSTypes(self, **kwargs): """ validParameters = ['primary_ds_type', 'dataset'] - checkInputParameter(method="listPrimaryDSTypes", parameters=list(kwargs.keys()), validParameters=validParameters) + checkInputParameter(method="listPrimaryDSTypes", parameters=list(kwargs.keys()), + validParameters=validParameters) return self.__callServer("primarydstypes", params=kwargs) @@ -1443,9 +1459,10 @@ def listProcessingEras(self, **kwargs): """ validParameters = ['processing_version'] - - checkInputParameter(method="listProcessingEras", parameters=list(kwargs.keys()), validParameters=validParameters) - + + checkInputParameter(method="listProcessingEras", parameters=list(kwargs.keys()), + validParameters=validParameters) + return self.__callServer("processingeras", params=kwargs) def listReleaseVersions(self, **kwargs): @@ -1464,7 +1481,8 @@ def listReleaseVersions(self, **kwargs): """ validParameters = ['dataset', 'release_version', 'logical_file_name'] - checkInputParameter(method="listReleaseVersions", parameters=list(kwargs.keys()), validParameters=validParameters) + checkInputParameter(method="listReleaseVersions", parameters=list(kwargs.keys()), + validParameters=validParameters) return self.__callServer("releaseversions", params=kwargs) @@ -1487,8 +1505,8 @@ def listRuns(self, **kwargs): requiredParameters = {'multiple': validParameters} - checkInputParameter(method="listRuns", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listRuns", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("runs", params=kwargs) @@ -1508,8 +1526,8 @@ def listRunSummaries(self, **kwargs): requiredParameters = {'forced': ['run_num']} - checkInputParameter(method="listRunSummaries", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="listRunSummaries", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("runsummaries", params=kwargs) @@ -1543,7 +1561,7 @@ def statusMigration(self, **kwargs): validParameters = ['migration_rqst_id', 'block_name', 'dataset', 'user'] checkInputParameter(method='statusMigration', parameters=list(kwargs.keys()), validParameters=validParameters) - + return self.__callServer("status", params=kwargs) def removeMigration(self, migrationObj): @@ -1584,8 +1602,8 @@ def updateAcqEraEndDate(self, **kwargs): requiredParameters = {'forced': validParameters} - checkInputParameter(method="updateAcqEraEndDate", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="updateAcqEraEndDate", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("acquisitioneras", params=kwargs, callmethod='PUT') @@ -1603,8 +1621,8 @@ def updateBlockStatus(self, **kwargs): requiredParameters = {'forced': validParameters} - checkInputParameter(method="updateBlockStatus", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="updateBlockStatus", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("blocks", params=kwargs, callmethod='PUT') @@ -1622,8 +1640,8 @@ def updateBlockSiteName(self, **kwargs): requiredParameters = {'forced': validParameters} - checkInputParameter(method="updateBlockSiteName", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="updateBlockSiteName", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("blocks", params=kwargs, callmethod='PUT') @@ -1641,8 +1659,8 @@ def updateDatasetType(self, **kwargs): requiredParameters = {'forced': validParameters} - checkInputParameter(method="updateDatasetType", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="updateDatasetType", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("datasets", params=kwargs, callmethod='PUT') @@ -1666,20 +1684,20 @@ def updateFileStatus(self, **kwargs): requiredParameters = {'forced': ['is_file_valid'], 'multiple': ['logical_file_name', 'dataset']} - - checkInputParameter(method="updateFileStatus", parameters=list(kwargs.keys()), validParameters=validParameters, - requiredParameters=requiredParameters) + checkInputParameter(method="updateFileStatus", parameters=list(kwargs.keys()), + validParameters=validParameters, requiredParameters=requiredParameters) return self.__callServer("files", params=kwargs, callmethod='PUT') + if __name__ == "__main__": # DBS Service URL - url="http://cmssrv18.fnal.gov:8585/dbs3" - #read_proxy="http://cmst0frontier1.cern.ch:3128" - #read_proxy="http://cmsfrontier1.fnal.gov:3128" - read_proxy="" + url = "http://cmssrv18.fnal.gov:8585/dbs3" + # read_proxy="http://cmst0frontier1.cern.ch:3128" + # read_proxy="http://cmsfrontier1.fnal.gov:3128" + read_proxy = "" api = DbsApi(url=url, proxy=read_proxy) print(api.serverinfo()) - #print api.listPrimaryDatasets() - #print api.listAcquisitionEras() - #print api.listProcessingEras() + # print api.listPrimaryDatasets() + # print api.listAcquisitionEras() + # print api.listProcessingEras()