Skip to content

Commit d922e38

Browse files
RuskonertRuskonert
committed
UPDATED: NVD-NIST
1 parent d9f7fc4 commit d922e38

File tree

1 file changed

+60
-45
lines changed

1 file changed

+60
-45
lines changed

‎nvdnist.py

+60-45
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ def add_thread(self, thread):
8585
def workstation(sql, tablename, jsonElement):
8686
try:
8787
# PyMySQL has thread-safety while a per, So It needs to new connect.
88+
# The variable name is very important.
89+
# Because the python interpreter checks name and matchs each of the table columns.
8890
new_db = initialize(sql.user, sql.password, sql.host, sql.port)
8991
base = Nvdnist(new_db)
9092
base.use_database(sql.database)
@@ -98,17 +100,17 @@ def workstation(sql, tablename, jsonElement):
98100
data_format = str(jsonElement['data_format'])
99101
data_version = str(jsonElement['data_version'])
100102
cve_data_version = str(jsonElement['CVE_data_version'])
101-
reference = ""
102-
description = ""
103+
reference_url = ""
104+
description_value = ""
103105
problemtype = ""
104106

105107
for r in base.find_key(jsonElement, r"reference_.+$"):
106-
reference += jsonElement[r] + " | "
107-
reference = reference[:-3]
108+
reference_url += jsonElement[r] + " | "
109+
reference_url = reference_url[:-3]
108110

109111
for d in base.find_key(jsonElement, r"desc_(.+)"):
110-
description += jsonElement[d] + " | "
111-
description = description[:-3]
112+
description_value += jsonElement[d] + " | "
113+
description_value = description_value[:-3]
112114

113115
for d in base.find_key(jsonElement, r"cwe_\d"):
114116
problemtype += jsonElement[d] + " | "
@@ -137,7 +139,7 @@ def workstation(sql, tablename, jsonElement):
137139
baseMetricV3_vectorString = str(jsonElement.get('baseMetricV3.vectorString', ''))
138140
baseMetricV3_attackVector = str(jsonElement.get('baseMetricV3.attackVector', ''))
139141
baseMetricV3_attackComplexity = str(jsonElement.get('baseMetricV3.attackComplexity', ''))
140-
baseMetricV3_privillegesRequired = str(jsonElement.get('baseMetricV3.privillegesRequired', ''))
142+
baseMetricV3_privilegesRequired = str(jsonElement.get('baseMetricV3.privilegesRequired', ''))
141143
baseMetricV3_userInteraction = str(jsonElement.get('baseMetricV3.userInteraction', ''))
142144

143145
baseMetricV3_scope = str(jsonElement.get('baseMetricV3.scope', ''))
@@ -153,35 +155,40 @@ def workstation(sql, tablename, jsonElement):
153155
lastModifiedDate = str(jsonElement.get('lastModifiedDate', ''))
154156

155157
cpe_list = []
156-
r = base.find_key(jsonElement, r"cpe_\d_vulnderable")
158+
r = base.find_key(jsonElement, r"cpe_[0-9]+_vulnderable")
157159
if len(r) != 0:
158160
for index, c in enumerate(r):
159161
cpe_list.append(str(jsonElement[c]) + "||")
160162

161-
for index, c in enumerate(base.find_key(jsonElement, r"cpe_\d_cpe22uri")):
163+
for index, c in enumerate(base.find_key(jsonElement, r"cpe_[0-9]+_cpe22uri")):
162164
cpe_list[index] += (str(jsonElement[c])) + "||"
163165

164166

165-
for index, c in enumerate(base.find_key(jsonElement, r"cpe_\d_cpe23uri")):
167+
for index, c in enumerate(base.find_key(jsonElement, r"cpe_[0-9]+_cpe23uri")):
168+
cpe_list[index] += (str(jsonElement[c])) + "||"
169+
170+
for index, c in enumerate(base.find_key(jsonElement, r"cpe_[0-9]+_number")):
166171
cpe_list[index] += (str(jsonElement[c]))
167172

168173
# each of affect data.
169-
for index, c in enumerate(base.find_key(jsonElement, r"affect_\d")):
170-
r = jsonElement[c].split('|')
171-
affect_vendor_name = r[0]
172-
affect_product_name = r[1]
173-
affect_version_value = r[2]
174+
for index, c in enumerate(base.find_key(jsonElement, r"affect_[0-9]+")):
175+
r = jsonElement[c].split(' || ')
176+
affects_vendor_name = r[0]
177+
affects_product_name = r[1]
178+
affects_version_value = r[2]
174179

175180
nodes_cpe_vulnerable = None
176181
nodes_cpe_cpe22uri = None
177182
nodes_cpe_cpe23uri = None
183+
nodes_cpe = None
178184

179185
if len(cpe_list) != 0:
180186
for c in cpe_list:
181187
r = c.split('||')
182188
nodes_cpe_vulnerable = r[0]
183189
nodes_cpe_cpe22uri = r[1]
184190
nodes_cpe_cpe23uri = r[2]
191+
nodes_cpe = r[3]
185192
a = None
186193
r = None
187194
d = None
@@ -197,9 +204,11 @@ def workstation(sql, tablename, jsonElement):
197204
execute(base, sql, tablename, v)
198205
base.disconnect()
199206
except KeyboardInterrupt:
207+
sql._threadList.kill = True
200208
sys.exit(0)
201209

202210
except SystemExit:
211+
sql._threadList.kill = True
203212
sys.exit(0)
204213

205214
def execute(base, sql, tablename, v):
@@ -226,9 +235,10 @@ def execute(base, sql, tablename, v):
226235
class Nvdnist(CVEDatabase):
227236
def __init__(self, mysql_base):
228237
super().__init__(mysql_base)
238+
self._threadList = None
229239

230240

231-
def export(self, export_path_name, export_type, tablename, option="ORDER BY ID"):
241+
def export(self, export_path_name, export_type, tablename, option="ORDER BY ID, nodes_cpe"):
232242
return super().export(export_path_name, export_type, tablename, option)
233243

234244
def load(self, obj):
@@ -358,13 +368,13 @@ def default_table_column(self):
358368
a6 = TableColumn("data_format", "VARCHAR(10)")
359369
a7 = TableColumn("data_version", "VARCHAR(5)")
360370

361-
a8 = TableColumn("affect_vendor_name", "TEXT")
362-
aa8 = TableColumn("affect_product_name", "TEXT")
363-
ab8 = TableColumn("affect_version_value", "TEXT")
371+
a8 = TableColumn("affects_vendor_name", "TEXT")
372+
aa8 = TableColumn("affects_product_name", "TEXT")
373+
ab8 = TableColumn("affects_version_value", "TEXT")
364374

365375
ac8 = TableColumn("problemtype", "VARCHAR(25)")
366-
a9 = TableColumn("description", "TEXT")
367-
a10 = TableColumn("reference", "TEXT")
376+
a9 = TableColumn("description_value", "TEXT")
377+
a10 = TableColumn("reference_url", "TEXT")
368378
a11 = TableColumn("cve_data_version", "VARCHAR(5)")
369379

370380
cpe1 = TableColumn("nodes_operator", "VARCHAR(5)")
@@ -400,7 +410,7 @@ def default_table_column(self):
400410
a35 = TableColumn('baseMetricV3_impactScore', "VARCHAR(10)")
401411
a36 = TableColumn('baseMetricV3_attackVector', "VARCHAR(36)")
402412
a37 = TableColumn('baseMetricV3_attackComplexity', "VARCHAR(10)")
403-
a38 = TableColumn('baseMetricV3_privillegesRequired', "VARCHAR(10)")
413+
a38 = TableColumn('baseMetricV3_privilegesRequired', "VARCHAR(10)")
404414
aa38 = TableColumn('baseMetricV3_userInteraction', "VARCHAR(10)")
405415
a39 = TableColumn('baseMetricV3_baseSeverity', "VARCHAR(10)")
406416
a40 = TableColumn('baseMetricV3_scope', "VARCHAR(10)")
@@ -421,6 +431,7 @@ def find_key(self, element, pattern):
421431

422432
def table_insert(self, tablename, jsonCollection):
423433
threadlist = ThreadList(thread_count)
434+
self._threadList = threadlist
424435
print("Inserting value ...")
425436

426437
# Singlethreading-based
@@ -558,7 +569,8 @@ def metadata(self, cve):
558569
cve_assigner = cve['CVE_data_meta']['ASSIGNER']
559570
return [cve_id, cve_assigner, cve_data_type, cve_data_format, cve_data_version]
560571

561-
def cve_configuration(self, nodes, ref=None):
572+
def cve_configuration(self, nodes, ref=None, number=0, isChildren=False):
573+
n = number
562574
node_list = None
563575
if ref is not None:
564576
node_list = ref
@@ -573,9 +585,9 @@ def cve_configuration(self, nodes, ref=None):
573585

574586
if operator == 'AND':
575587
try:
576-
self.cve_configuration(node['children'], node_list)
588+
self.cve_configuration(node['children'], node_list, n, True)
577589
except KeyError:
578-
self.cve_configuration(node['cpe'], node_list)
590+
self.cve_configuration(node['cpe'], node_list, n, True)
579591

580592
elif operator == 'OR':
581593
keys = node.get('cpe', None)
@@ -584,36 +596,39 @@ def cve_configuration(self, nodes, ref=None):
584596
if keys is None:
585597
return
586598
for cpe in keys:
599+
587600
chk = cpe.get('cpe', None)
588601
if chk is not None:
589602
cpe = chk[0]
603+
590604
vulnerable = cpe['vulnerable']
591605
cpe22uri = cpe['cpe22Uri']
592606
cpe23uri = cpe['cpe23Uri']
593-
node_list.append(CPENode(vulnerable, cpe22uri, cpe23uri))
607+
node_list.append(CPENode(vulnerable, cpe22uri, cpe23uri, n))
594608
else:
595609
for node in nodes:
596610
vulnerable = node['vulnerable']
597611
cpe22uri = node['cpe22Uri']
598612
cpe23uri = node['cpe23Uri']
599-
node_list.append(CPENode(vulnerable, cpe22uri, cpe23uri))
613+
node_list.append(CPENode(vulnerable, cpe22uri, cpe23uri, n))
614+
n = n + 1
600615
return node_list
601616

602617
def reference_data(self, references):
603618
reference_list = []
604619
for data in references['reference_data']:
605620
url = data['url']
606-
name = data['name']
607-
refsource = data['refsource']
608-
reference_list.append(Reference(url, name, refsource))
621+
#name = data['name']
622+
#refsource = data['refsource']
623+
reference_list.append(Reference(url))
609624
return reference_list
610625

611626
def description_data(self, descriptions):
612627
description_list = []
613628
for data in descriptions['description_data']:
614-
lang = data['lang']
629+
# lang = data['lang']
615630
value = data['value']
616-
description_list.append(DescriptionType(lang, value))
631+
description_list.append(DescriptionType(value))
617632
return description_list
618633

619634
def problemtype_data(self, cve_problemtype):
@@ -636,7 +651,7 @@ def affect_vendor_data(self, cve_affects):
636651
product_name = product_info['product_name']
637652
for version in product_info['version']['version_data']:
638653
version_value = version
639-
product_flag.append(product_name + "|" + version_value.get('version_value', 'invaild'))
654+
product_flag.append(product_name + " || " + version_value.get('version_value', 'invaild'))
640655
affect = Affect(vendor_name, product_flag)
641656
vendor_list.append(affect)
642657
except:
@@ -700,12 +715,12 @@ def serialize(self, jsonDefault, o):
700715
value.serialize(jsonDefault, o)
701716

702717
class Metricv3(AbstractMetric):
703-
def __init__(self, version, vectorString, attackVector, attackComplexity, privillegesRequired, userInteraction, scope, confidentialityImpact, integrityImpact, availabilityImpact, baseScore, baseSeverity, cvss):
718+
def __init__(self, version, vectorString, attackVector, attackComplexity, privilegesRequired, userInteraction, scope, confidentialityImpact, integrityImpact, availabilityImpact, baseScore, baseSeverity, cvss):
704719
super().__init__(version, vectorString, confidentialityImpact, integrityImpact, availabilityImpact, baseScore, cvss)
705720
self.attackVector = attackVector
706721
self.attackComplexity = attackComplexity
707722
self.scope = scope
708-
self.privillegesRequired = privillegesRequired
723+
self.privilegesRequired = privilegesRequired
709724
self.userInteraction = userInteraction
710725
self.baseSeverity = baseSeverity
711726

@@ -721,15 +736,17 @@ def serialize(self, jsonDefault, o):
721736

722737
class CPENode:
723738
__dbcolumn__ = "cpe"
724-
def __init__(self, vulnerable, cpe22uri, cpe23uri):
739+
def __init__(self, vulnerable, cpe22uri, cpe23uri, cpe=-1):
725740
self.vulnerable =vulnerable
726741
self.cpe22uri = cpe22uri
727742
self.cpe23uri = cpe23uri
743+
self.cpe = cpe
728744

729745
def serialize(self, jsonDefault, o):
730746
jsonDefault["cpe_" + str(o) + "_vulnderable"] = self.vulnerable
731747
jsonDefault["cpe_" + str(o) + "_cpe22uri"] = self.cpe22uri
732748
jsonDefault["cpe_" + str(o) + "_cpe23uri"] = self.cpe23uri
749+
jsonDefault["cpe_" + str(o) + "_number"] = self.cpe
733750

734751

735752
class Affect:
@@ -739,29 +756,27 @@ def __init__(self, vendor_name, product_flag):
739756

740757
def serialize(self, jsonDefault, o):
741758
for index, flag in enumerate(self.product_flag):
742-
jsonDefault['affect_{n}'.format(n=index)] = str(self.vendor_name + "|" + flag)
759+
jsonDefault['affect_{n}'.format(n=index)] = str(self.vendor_name + " || " + flag)
743760

744761
class DescriptionType:
745762
__dbcolumn__ = "Description"
746-
def __init__(self, lang, value):
747-
self.lang = lang
763+
def __init__(self, value):
748764
self.value = value
749765

750766
def serialize(self, jsonDefault, o):
751-
jsonDefault["desc_" + str(id(self))] = self.lang + ", " + self.value
767+
jsonDefault["desc_" + str(id(self))] = self.value
752768

753769
class ProblemType(DescriptionType):
754770
def __init__(self, lang, value):
755-
super().__init__(lang, value)
771+
super().__init__(value)
772+
self.lang = lang
756773

757774
def serialize(self, jsonDefault, o):
758775
jsonDefault["cwe_" + str(id(self))] = self.lang + ", " + self.value
759776

760777
class Reference:
761-
def __init__(self, url, name, refsource):
778+
def __init__(self, url):
762779
self.url = url
763-
self.name = name
764-
self.refsource = refsource
765780

766781
def serialize(self, jsonDefault, o):
767-
jsonDefault["reference_" + self.name] = self.url + ", " + self.refsource
782+
jsonDefault["reference_" + id(self)] = self.url

0 commit comments

Comments
 (0)