Skip to content

Commit 75ffd49

Browse files
lanmaster53lanmaster53
committed
Moved several module methods into mixins and utility modules.
1 parent 03f4e4c commit 75ffd49

File tree

7 files changed

+294
-269
lines changed

7 files changed

+294
-269
lines changed

‎recon/core/module.py‎

Lines changed: 0 additions & 268 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,11 @@
1-
import hashlib
2-
import hmac
31
import html.parser
42
import http.cookiejar
53
import io
64
import os
75
import re
8-
import socket
96
import sqlite3
10-
import struct
117
import sys
128
import textwrap
13-
import time
14-
import urllib.parse
15-
import webbrowser
169
import yaml
1710
# framework libs
1811
from recon.core import framework
@@ -117,29 +110,6 @@ def cidr_to_list(self, string):
117110
import ipaddress
118111
return [str(ip) for ip in ipaddress.ip_network(string)]
119112

120-
def parse_name(self, name):
121-
elements = [self.html_unescape(x) for x in name.strip().split()]
122-
# remove prefixes and suffixes
123-
names = []
124-
for i in range(0,len(elements)):
125-
# preserve initials
126-
if re.search(r'^\w\.$', elements[i]):
127-
elements[i] = elements[i][:-1]
128-
# remove unecessary prefixes and suffixes
129-
elif re.search(r'(?:\.|^the$|^jr$|^sr$|^I{2,3}$)', elements[i], re.IGNORECASE):
130-
continue
131-
names.append(elements[i])
132-
# make sense of the remaining elements
133-
if len(names) > 3:
134-
names[2:] = [' '.join(names[2:])]
135-
# clean up any remaining garbage characters
136-
names = [re.sub(r"[,']", '', x) for x in names]
137-
# set values and return names
138-
fname = names[0] if len(names) >= 1 else None
139-
mname = names[1] if len(names) >= 3 else None
140-
lname = names[-1] if len(names) >= 2 else None
141-
return fname, mname, lname
142-
143113
def hosts_to_domains(self, hosts, exclusions=[]):
144114
domains = []
145115
for host in hosts:
@@ -184,244 +154,6 @@ def _get_source(self, params, query=None):
184154
raise framework.FrameworkException('Source contains no input.')
185155
return sources
186156

187-
#==================================================
188-
# 3RD PARTY API METHODS
189-
#==================================================
190-
191-
def get_explicit_oauth_token(self, resource, scope, authorize_url, access_url):
192-
token_name = resource+'_token'
193-
token = self.get_key(token_name)
194-
if token:
195-
return token
196-
client_id = self.get_key(resource+'_api')
197-
client_secret = self.get_key(resource+'_secret')
198-
port = 31337
199-
redirect_uri = f"http://localhost:{port}"
200-
payload = {'response_type': 'code', 'client_id': client_id, 'scope': scope, 'state': self.get_random_str(40), 'redirect_uri': redirect_uri}
201-
authorize_url = f"{authorize_url}?{urllib.parse.urlencode(payload)}"
202-
w = webbrowser.get()
203-
w.open(authorize_url)
204-
# open a socket to receive the access token callback
205-
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
206-
sock.bind(('127.0.0.1', port))
207-
sock.listen(1)
208-
conn, addr = sock.accept()
209-
data = conn.recv(1024)
210-
conn.sendall('HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Recon-ng</title></head><body>Response received. Return to Recon-ng.</body></html>')
211-
conn.close()
212-
# process the received data
213-
if 'error_description' in data:
214-
self.error(urllib.parse.unquote_plus(re.search(r'error_description=([^\s&]*)', data).group(1)))
215-
return None
216-
authorization_code = re.search(r'code=([^\s&]*)', data).group(1)
217-
payload = {'grant_type': 'authorization_code', 'code': authorization_code, 'redirect_uri': redirect_uri, 'client_id': client_id, 'client_secret': client_secret}
218-
resp = self.request('POST', access_url, data=payload)
219-
if 'error' in resp.json():
220-
self.error(resp.json()['error_description'])
221-
return None
222-
access_token = resp.json()['access_token']
223-
self.add_key(token_name, access_token)
224-
return access_token
225-
226-
def get_twitter_oauth_token(self):
227-
token_name = 'twitter_token'
228-
token = self.get_key(token_name)
229-
if token:
230-
return token
231-
twitter_key = self.get_key('twitter_api')
232-
twitter_secret = self.get_key('twitter_secret')
233-
url = 'https://api.twitter.com/oauth2/token'
234-
auth = (twitter_key, twitter_secret)
235-
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'}
236-
payload = {'grant_type': 'client_credentials'}
237-
resp = self.request('POST', url, auth=auth, headers=headers, data=payload)
238-
if 'errors' in resp.json():
239-
raise framework.FrameworkException(f"{resp.json()['errors'][0]['message']}, {resp.json()['errors'][0]['label']}")
240-
access_token = resp.json()['access_token']
241-
self.add_key(token_name, access_token)
242-
return access_token
243-
244-
def build_pwnedlist_payload(self, payload, method, key, secret):
245-
timestamp = int(time.time())
246-
payload['ts'] = timestamp
247-
payload['key'] = key
248-
msg = f"{key}{timestamp}{method}{secret}"
249-
encoding = sys.getdefaultencoding()
250-
hm = hmac.new(bytes(secret, encoding), bytes(msg, encoding), hashlib.sha1)
251-
payload['hmac'] = hm.hexdigest()
252-
return payload
253-
254-
def get_pwnedlist_leak(self, leak_id):
255-
# check if the leak has already been retrieved
256-
leak = self.query('SELECT * FROM leaks WHERE leak_id=?', (leak_id,))
257-
if leak:
258-
leak = dict(zip([x[0] for x in self.get_columns('leaks')], leak[0]))
259-
del leak['module']
260-
return leak
261-
# set up the API call
262-
key = self.get_key('pwnedlist_api')
263-
secret = self.get_key('pwnedlist_secret')
264-
url = 'https://api.pwnedlist.com/api/1/leaks/info'
265-
base_payload = {'leakId': leak_id}
266-
payload = self.build_pwnedlist_payload(base_payload, 'leaks.info', key, secret)
267-
# make the request
268-
resp = self.request('GET', url, params=payload)
269-
if resp.status_code != 200:
270-
self.error(f"Error retrieving leak data.{os.linesep}{resp.text}")
271-
return
272-
leak = resp.json()['leaks'][0]
273-
# normalize the leak for storage
274-
normalized_leak = {}
275-
for item in leak:
276-
value = leak[item]
277-
if type(value) == list:
278-
value = ', '.join(value)
279-
normalized_leak[item] = value
280-
return normalized_leak
281-
282-
def search_twitter_api(self, payload, limit=False):
283-
headers = {'Authorization': f"Bearer {self.get_twitter_oauth_token()}"}
284-
url = 'https://api.twitter.com/1.1/search/tweets.json'
285-
results = []
286-
while True:
287-
resp = self.request('GET', url, params=payload, headers=headers)
288-
if limit:
289-
# app auth rate limit for search/tweets is 450/15min
290-
time.sleep(2)
291-
jsonobj = resp.json()
292-
for item in ['error', 'errors']:
293-
if item in jsonobj:
294-
raise framework.FrameworkException(jsonobj[item])
295-
results += jsonobj['statuses']
296-
if 'next_results' in jsonobj['search_metadata']:
297-
max_id = urllib.parse.parse_qs(jsonobj['search_metadata']['next_results'][1:])['max_id'][0]
298-
payload['max_id'] = max_id
299-
continue
300-
break
301-
return results
302-
303-
def search_shodan_api(self, query, limit=0):
304-
api_key = self.get_key('shodan_api')
305-
url = 'https://api.shodan.io/shodan/host/search'
306-
payload = {'query': query, 'key': api_key}
307-
results = []
308-
cnt = 0
309-
page = 1
310-
self.verbose(f"Searching Shodan API for: {query}")
311-
while True:
312-
time.sleep(1)
313-
resp = self.request('GET', url, params=payload)
314-
if resp.json() == None:
315-
raise framework.FrameworkException(f"Invalid JSON response.{os.linesep}{resp.text}")
316-
if 'error' in resp.json():
317-
raise framework.FrameworkException(resp.json()['error'])
318-
if not resp.json()['matches']:
319-
break
320-
# add new results
321-
results.extend(resp.json()['matches'])
322-
# increment and check the limit
323-
cnt += 1
324-
if limit == cnt:
325-
break
326-
# next page
327-
page += 1
328-
payload['page'] = page
329-
return results
330-
331-
def search_bing_api(self, query, limit=0):
332-
url = 'https://api.cognitive.microsoft.com/bing/v7.0/search'
333-
payload = {'q': query, 'count': 50, 'offset': 0, 'responseFilter': 'WebPages'}
334-
headers = {'Ocp-Apim-Subscription-Key': self.get_key('bing_api')}
335-
results = []
336-
cnt = 0
337-
self.verbose(f"Searching Bing API for: {query}")
338-
while True:
339-
resp = self.request('GET', url, params=payload, headers=headers)
340-
if resp.json() == None:
341-
raise framework.FrameworkException(f"Invalid JSON response.{os.linesep}{resp.text}")
342-
#elif 'error' in resp.json():
343-
elif resp.status_code == 401:
344-
raise framework.FrameworkException(f"{resp.json()['statusCode']}: {resp.json()['message']}")
345-
# add new results, or if there's no more, return what we have...
346-
if 'webPages' in resp.json():
347-
results.extend(resp.json()['webPages']['value'])
348-
else:
349-
return results
350-
# increment and check the limit
351-
cnt += 1
352-
if limit == cnt:
353-
break
354-
# check for more pages
355-
# https://msdn.microsoft.com/en-us/library/dn760787.aspx
356-
if payload['offset'] > (resp.json()['webPages']['totalEstimatedMatches'] - payload['count']):
357-
break
358-
# set the payload for the next request
359-
payload['offset'] += payload['count']
360-
return results
361-
362-
def search_google_api(self, query, limit=0):
363-
api_key = self.get_key('google_api')
364-
cse_id = self.get_key('google_cse')
365-
url = 'https://www.googleapis.com/customsearch/v1'
366-
payload = {'alt': 'json', 'prettyPrint': 'false', 'key': api_key, 'cx': cse_id, 'q': query}
367-
results = []
368-
cnt = 0
369-
self.verbose(f"Searching Google API for: {query}")
370-
while True:
371-
resp = self.request('GET', url, params=payload)
372-
if resp.json() == None:
373-
raise framework.FrameworkException(f"Invalid JSON response.{os.linesep}{resp.text}")
374-
# add new results
375-
if 'items' in resp.json():
376-
results.extend(resp.json()['items'])
377-
# increment and check the limit
378-
cnt += 1
379-
if limit == cnt:
380-
break
381-
# check for more pages
382-
if not 'nextPage' in resp.json()['queries']:
383-
break
384-
payload['start'] = resp.json()['queries']['nextPage'][0]['startIndex']
385-
return results
386-
387-
def search_github_api(self, query):
388-
self.verbose(f"Searching Github for: {query}")
389-
results = self.query_github_api(endpoint='/search/code', payload={'q': query})
390-
# reduce the nested lists of search results and return
391-
results = [result['items'] for result in results]
392-
return [x for sublist in results for x in sublist]
393-
394-
def query_github_api(self, endpoint, payload={}, options={}):
395-
opts = {'max_pages': None}
396-
opts.update(options)
397-
headers = {'Authorization': f"token {self.get_key('github_api')}"}
398-
base_url = 'https://api.github.com'
399-
url = base_url + endpoint
400-
results = []
401-
page = 1
402-
while True:
403-
# Github rate limit is 30 requests per minute
404-
time.sleep(2) # 60s / 30r = 2s/r
405-
payload['page'] = page
406-
resp = self.request('GET', url, headers=headers, params=payload)
407-
# check for errors
408-
if resp.status_code != 200:
409-
# skip 404s returned for no results
410-
if resp.status_code != 404:
411-
self.error(f"Message from Github: {resp.json()['message']}")
412-
break
413-
# some APIs return lists, and others a single dictionary
414-
method = 'extend'
415-
if type(resp.json()) == dict:
416-
method = 'append'
417-
getattr(results, method)(resp.json())
418-
# paginate
419-
if 'link' in resp.headers and 'rel="next"' in resp.headers['link'] and (opts['max_pages'] is None or page < opts['max_pages']):
420-
page += 1
421-
continue
422-
break
423-
return results
424-
425157
#==================================================
426158
# REQUEST METHODS
427159
#==================================================

‎recon/mixins/github.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import time
2+
3+
4+
class GithubMixin(object):
5+
6+
def query_github_api(self, endpoint, payload={}, options={}):
7+
opts = {'max_pages': None}
8+
opts.update(options)
9+
headers = {'Authorization': f"token {self.get_key('github_api')}"}
10+
base_url = 'https://api.github.com'
11+
url = base_url + endpoint
12+
results = []
13+
page = 1
14+
while True:
15+
# Github rate limit is 30 requests per minute
16+
time.sleep(2) # 60s / 30r = 2s/r
17+
payload['page'] = page
18+
resp = self.request('GET', url, headers=headers, params=payload)
19+
# check for errors
20+
if resp.status_code != 200:
21+
# skip 404s returned for no results
22+
if resp.status_code != 404:
23+
self.error(f"Message from Github: {resp.json()['message']}")
24+
break
25+
# some APIs return lists, and others a single dictionary
26+
method = 'extend'
27+
if type(resp.json()) == dict:
28+
method = 'append'
29+
getattr(results, method)(resp.json())
30+
# paginate
31+
if 'link' in resp.headers and 'rel="next"' in resp.headers['link'] and (opts['max_pages'] is None or page < opts['max_pages']):
32+
page += 1
33+
continue
34+
break
35+
return results
36+
37+
def search_github_api(self, query):
38+
self.verbose(f"Searching Github for: {query}")
39+
results = self.query_github_api(endpoint='/search/code', payload={'q': query})
40+
# reduce the nested lists of search results and return
41+
results = [result['items'] for result in results]
42+
return [x for sublist in results for x in sublist]

‎recon/mixins/oauth.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import re
2+
import socket
3+
import urllib.parse
4+
import webbrowser
5+
6+
class ExplicitOauthMixin(object):
7+
8+
def get_explicit_oauth_token(self, resource, scope, authorize_url, access_url):
9+
token_name = resource+'_token'
10+
token = self.get_key(token_name)
11+
if token:
12+
return token
13+
client_id = self.get_key(resource+'_api')
14+
client_secret = self.get_key(resource+'_secret')
15+
port = 31337
16+
redirect_uri = f"http://localhost:{port}"
17+
payload = {'response_type': 'code', 'client_id': client_id, 'scope': scope, 'state': self.get_random_str(40), 'redirect_uri': redirect_uri}
18+
authorize_url = f"{authorize_url}?{urllib.parse.urlencode(payload)}"
19+
w = webbrowser.get()
20+
w.open(authorize_url)
21+
# open a socket to receive the access token callback
22+
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
23+
sock.bind(('127.0.0.1', port))
24+
sock.listen(1)
25+
conn, addr = sock.accept()
26+
data = conn.recv(1024)
27+
conn.sendall('HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Recon-ng</title></head><body>Response received. Return to Recon-ng.</body></html>')
28+
conn.close()
29+
# process the received data
30+
if 'error_description' in data:
31+
self.error(urllib.parse.unquote_plus(re.search(r'error_description=([^\s&]*)', data).group(1)))
32+
return None
33+
authorization_code = re.search(r'code=([^\s&]*)', data).group(1)
34+
payload = {'grant_type': 'authorization_code', 'code': authorization_code, 'redirect_uri': redirect_uri, 'client_id': client_id, 'client_secret': client_secret}
35+
resp = self.request('POST', access_url, data=payload)
36+
if 'error' in resp.json():
37+
self.error(resp.json()['error_description'])
38+
return None
39+
access_token = resp.json()['access_token']
40+
self.add_key(token_name, access_token)
41+
return access_token

0 commit comments

Comments
 (0)