Merge pull request #164 from CVEProject/CNA_match_order

converted to IDR service query, and rate limit handling

Author: jwhitmore-mitre

schema/v5.0/support/CVE_4_to_5_converter/cve4to5up.py

@@ -8,6 +8,7 @@
 import requests
 import settings
 import sys
+import time
 import traceback
 import urllib.parse
 import csv
@@ -104,7 +105,7 @@ def main(argv):
                     except:
                         problemfiles[filepath] = "" + str(sys.exc_info()[0]) + " -- " + str(sys.exc_info()[1]) + " -- "
                 CVECount += 1    
-                if CVECount % 250 == 0: spinner.next()
+                if CVECount % 100 == 0: spinner.next()
 
         print('FINISHED processed directory', inputdir)
         print('')
@@ -343,7 +344,8 @@ def CVE_Convert(inputfile, outputpath):
     global keys_used
     global extra_keys
     global states_processed
-    global all_users
+    # global all_users
+    global all_orgs
     global scoring_other
     global invalid_impact_versions
     global requester_map
@@ -357,13 +359,15 @@ def CVE_Convert(inputfile, outputpath):
     if len(requester_map) < 1:
         getRequesterMap()
 
+    ''' Not needed if querying IDR by CVE ID
     if len(all_users) < 1:
         getAllUsers()
         # get min and max length of org shortname
         for org in all_orgs:
             minShortName = min(minShortName, len(all_orgs[org]["short_name"]))
             maxShortName = max(maxShortName, len(all_orgs[org]["short_name"]))
-
+    '''
+    
     with open(inputfile) as json_file:
         writeout = False
         data = json.load(json_file)
@@ -384,6 +388,7 @@ def CVE_Convert(inputfile, outputpath):
                 if i_meta["STATE"] not in keys_used: keys_used[i_meta["STATE"]] = {}
                 keys_used[i_meta["STATE"]]["CVE_data_meta"] = {}
 
+                '''
                 if "ASSIGNER" in i_meta:
                     # v4 assigner email converted to orgId before v5 upconvert
                     # get org info
@@ -461,8 +466,9 @@ def CVE_Convert(inputfile, outputpath):
                     else:
                         o_meta["assignerOrgId"] = all_users[username]["org_UUID"]
                         o_meta["assignerShortName"] = all_users[username]["org_short_name"]
-
+                
                     # print("in = " + i_meta["ASSIGNER"] + " out = " +o_meta["assignerShortName"])
+                '''
 
                 if "STATE" in i_meta:
                     if i_meta["STATE"] == 'RESERVED':
@@ -478,6 +484,19 @@ def CVE_Convert(inputfile, outputpath):
                 if "ID" in i_meta: 
                     o_meta["cveId"] = i_meta["ID"]
 
+                o_meta["assignerOrgId"] = "Not found"
+                o_meta["assignerShortName"] = "Not found"
+                if i_meta["STATE"] != 'RESERVED':
+                    recData = getIDRInfo( o_meta["cveId"] )   
+                    if recData and "owning_cna" in recData:
+                        org_short_name = recData["owning_cna"]
+                        org_uuid = getOrgUUID(org_short_name)
+                        o_meta["assignerOrgId"] = org_uuid
+                        o_meta["assignerShortName"] = org_short_name
+                    else:
+                        print("Record with data issue: " + o_meta["cveId"])
+                        raise Exception("ERROR - no CNA for record ID - " + o_meta["cveId"])
+                    
                 if "DATE_PUBLIC" in i_meta and i_meta["DATE_PUBLIC"] != "": 
                     o_meta["datePublished"] = i_meta["DATE_PUBLIC"]
                     try:
@@ -514,6 +533,7 @@ def CVE_Convert(inputfile, outputpath):
             else:
                 raise MissingRequiredPropertyValue(inputfile, "CVE_data_meta no STATE")
         except Exception as e:
+            # print("test 5")
             print( str(e) )
             if type(e) is not MissingRequiredPropertyValue:
                 raise MissingRequiredPropertyValue(inputfile, "CVE_data_meta structure error")
@@ -1404,6 +1424,26 @@ def __init__(self, cveid, propertyname, message="Required property missing from
     def __str__(self):
         return self.cveid + " - " + self.propertyname + " - " + self.message 
 
+
+def getOrgUUID( short_name ):
+    global all_orgs
+    
+    if not all_orgs or len(all_orgs) < 1: getOrgData()
+
+    # try/except block to catch integrity error in case the org doesn't exist
+    uuid = None
+    try:
+        for org in all_orgs:
+            # print( json.dumps(all_orgs, indent=2))
+            orgShortName = all_orgs[org]["short_name"]
+            if orgShortName == short_name:
+                uuid = all_orgs[org]["UUID"]
+                break
+    except:
+        pass
+    return uuid
+
+
 def getAllUsers():
     global all_orgs
     global all_users
@@ -1447,27 +1487,61 @@ def getAllUsers():
 
 
 
-def getIDRInfo(cveId):
-    IDR_URL = settings.AWG_IDR_SERVICE_URL + '/cve/' + cveId
+def getIDRInfo(cveId, delay=20, retry=0):
+    IDR_URL = settings.AWG_IDR_SERVICE_URL + '/cve-id/' + cveId
     idr_params = {}
     data = None
+    
     # try/except block to catch integrity error in case the org doesn't exist
     try:
         # Attempt to get org from RSUS
         idr_result = call_idr_service('get', BASE_HEADERS, IDR_URL, idr_params)
-        data = json.loads(idr_result)
-        print( json.dumps(data, indent=4) )
+        if idr_result and idr_result.startswith("{"):
+            data = json.loads(idr_result)
+        else:
+            if retry < 10:
+                # print("delay for: "+ str(delay))
+                time.sleep(delay)
+                data = getIDRInfo(cveId, delay*2, retry+1)    
+            else:
+                print("Record Issue - URL - " + IDR_URL)
+                # print(str(idr_result))
+    except Exception as e:
+        if retry < 10:
+            if delay > 179:
+                print("exception delay for: " + str(delay))
+            time.sleep(delay)
+            data = getIDRInfo(cveId, delay*2, retry+1)    
+        else:
+            # print(str(idr_result))
+            print("Exception -- URL - " + IDR_URL)
+            print(str(e))
+            raise e
+    return data
+
+
+def getRecordMetaData(recordId):
+    ORG_URL = settings.AWG_IDR_SERVICE_URL + '/cve-id/' + str(recordId)
+    org_params = {}
+
+    # try/except block to catch integrity error in case the ID doesn't exist
+    try:
+        # Attempt to get org from RSUS
+        record_result = call_idr_service('get', BASE_HEADERS, ORG_URL, org_params)
+        data = json.loads(record_result)
+        if "owning_cna" in data:
+            return data
+        else:
+            raise Exception(str(recordId) + " did not find an owning_cna.")
     except Exception as e:
         print(str(e))
         raise e
-    return data
-    
+    return None
 
 
 def getOrgData():
     global all_orgs
 
-    # ORG_URL = settings.AWG_IDR_SERVICE_URL + '/org/' + orgId
     ORG_URL = settings.AWG_IDR_SERVICE_URL + '/org'
     org_params = {}