Fincer
/
url-analyzer
1
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
URL data analyzer and extractor. Detect malicious signs and other useful data associated with URLs.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
19 Commits
1 Branch
0 B
 
 
Branch: master
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'master'
${ noResults }
url-analyzer/code/url-analyzer.py

861 lines
22 KiB
Raw Permalink Blame History

#!/bin/env python
"""
URL data extractor
Pekka Helenius <pekka [dot] helenius [at] fjordtek [dot] com>
Requirements:
Python 3
Python 3 BeautifulSoup4 (python-beautifulsoup4)
Python 3 whois (python-whois; PyPI)
Python 3 JSON Schema (python-jsonschema)
Python 3 Numpy (python-numpy)
Python 3 matplotlib (python-matplotlib)
TODO: URL domain part length comparison analysis
TODO: URL non-TLD part length comparison analysis
- in phishing webpages, URL tends to be much longer than legitimate webpages
however, domains themselves tend to be much shorter (without TLD)
- phishing URLs often contain more number of dots and subdomains than legitimate URLs
- legitimate: robots.txt redirects bots to a legitimate domain rather than to the original phishing domain
TODO: Website visual similarity analysis
TODO: consistency of RDN usage in HTML data
"""
######################################
#%matplotlib inline
import matplotlib.pyplot as plt
from bs4 import BeautifulSoup as bs
from collections import Counter
from datetime import date, datetime
import json
import os
import re
import requests
from time import sleep
import urllib
from whois import whois
# Target URLs
urls = [
"https://hoxhunt.com/",
"https://hs.fi",
"https://ts.fi",
"https://facebook.com"
]
# Some web servers may block our request unless we set a widely used, well-known user agent string
request_headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36'
}
# Date format for domain timestamps
dateformat = "%Y/%m/%d"
# All webpages may not like fetching data too fast
# Sleep time in seconds
sleep_interval_between_requests = 0.5
# Write JSON results to a file?
use_file = True
# Full file path + name
filename = os.getcwd() + "/" + "url_info.json"
# Generate plot from existing JSON data?
plot_only = False
# Save generated plot images?
save_plot_images = True
# DPI of plot images
plot_images_dpi = 150
# Common link attribute references in various HTML elements
link_refs = {
'a': 'href',
'img': 'src',
'script': 'src'
}
############################################################################
############################################################################
class json_url_data(object):
# def __init__(self):
######################################
"""
Set a new HTTP session and get response.
Returns a requests.models.Response object.
"""
def set_session(self, url, method='get', redirects=True):
# HTTP response status codes 1XX, 2XX and 3XX are OK
# Treat other codes as errors
sc = re.compile(r"^[123]{1}[0-9]{2}")
sleep(sleep_interval_between_requests)
try:
session = requests.Session()
response = session.request(method, url, headers=request_headers, allow_redirects=redirects)
if not sc.match(str(response.status_code)):
raise Exception("Error: got invalid response status from the web server")
return response
except:
raise Exception("Error: HTTP session could not be established. URL: '" + url + "' (method: " + method + ")") from None
######################################
"""
Fetch HTML data.
Returns a bs4.BeautifulSoup object.
"""
def get_html_data(self, url):
try:
data = bs(self.set_session(url).content, 'html.parser')
return data
except:
raise Exception("Error: HTML data could not be retrieved")
######################################
"""
Get URL redirects and related HTTP status codes.
Returns a list object.
"""
def get_url_redirects(self, url):
response = self.set_session(url)
list_data = []
if response.history:
for r in response.history:
list_data.append({'redirect_url': r.url, 'status': r.status_code})
return list_data
######################################
"""
Extract title HTML element contents from given HTML data.
Returns a string object.
"""
def get_webpage_title(self, url):
html_data = self.get_html_data(url)
title = html_data.title.string
return title
######################################
"""
Get WHOIS domain data.
Returns a dict object.
"""
def get_whois_data(self, url):
dict_data = whois(url)
return dict_data
######################################
"""
Get domain name based on WHOIS domain data.
"""
def get_domain_name(self, url):
domain_name = self.get_whois_data(url).domain_name
if type(domain_name) is list:
return domain_name[0].lower()
else:
return domain_name.lower()
######################################
"""
Get initial and final URLs
Compare whether the final (destination) URL
matches with the initial URL in a request.
Returns a dict object.
"""
def get_startfinal_urls(self, url):
response = self.set_session(url)
end_url = response.url
start_match = False
final_match = False
# dr = re.compile(r"^([a-z]+://)?([^/]+)")
# dr_group_lastindex = dr.match(url).lastindex
# domain_name = dr.match(url).group(dr_group_lastindex)
domain_name = self.get_domain_name(url)
if re.search(domain_name, end_url):
final_match = True
dict_data = {
'startfinal_urls': {
'start_url': {
'url': url
},
'final_url': {
'url': end_url, 'domain_match': final_match
}
}
}
return dict_data
######################################
"""
Get domain registrar
Returns a dict object.
"""
def get_domain_registrar(self, url):
dict_data = {'domain_registrar': self.get_whois_data(url).registrar }
return dict_data
######################################
"""
Do comparison between the domain name, extracted
from WHOIS domain data and contents of a title HTML
element, extracted from HTML data based on a given URL.
Returns a dict object.
"""
def get_domain_title_match(self, url):
domain_name = self.get_domain_name(url)
title = self.get_webpage_title(url)
# If is string:
if type(domain_name) is str:
if re.search(domain_name, title, re.IGNORECASE):
match = True
else:
match = False
# If is list:
elif type(domain_name) is list:
for d in domain_name:
if re.search(d, title, re.IGNORECASE):
match = True
break
else:
match = False
else:
match = False
dict_data = {
'webpage_title': title,
'domain_in_webpage_title': match
}
return dict_data
######################################
"""
Get a single timestamp from given data
Two scenarios are considered: dates argument is either
a list or a string. If it is a list, then we need
to decide which date value to extract.
Returns a date object.
"""
def get_single_date(self, dates, newest=False):
dates_epoch = []
if type(dates) is list:
for d in dates:
dates_epoch.append(d.timestamp())
else:
dates_epoch.append(dates.timestamp())
return datetime.fromtimestamp(sorted(dates_epoch, reverse=newest)[0])
######################################
"""
Get domain time information based on WHOIS domain data.
Returns a dict object.
"""
def get_domain_timeinfo(self, url):
whois_data = self.get_whois_data(url)
domain_creation_date = self.get_single_date(whois_data.creation_date, newest = False)
domain_updated_date = self.get_single_date(whois_data.updated_date, newest = False)
domain_expiration_date = self.get_single_date(whois_data.expiration_date, newest = False)
dict_data = {
'domain_timestamps':
{
'created': domain_creation_date.strftime(dateformat),
'updated': domain_updated_date.strftime(dateformat),
'expires': domain_expiration_date.strftime(dateformat)
}
}
return dict_data
######################################
"""
Get domain time information based on WHOIS domain data,
relative to the current date (UTC time).
Returns a dict object.
"""
def get_domain_timeinfo_relative(self, url):
date_now = datetime.utcnow()
whois_data = self.get_whois_data(url)
domain_creation_date = self.get_single_date(whois_data.creation_date, newest = False)
domain_updated_date = self.get_single_date(whois_data.updated_date, newest = False)
domain_expiration_date = self.get_single_date(whois_data.expiration_date, newest = False)
dict_data = {
'domain_timestamps_relative':
{
'current_date': (date_now.strftime(dateformat)),
'created_days_ago': (date_now - domain_creation_date).days,
'updated_days_ago': (date_now - domain_updated_date).days,
'expires_days_left': (domain_expiration_date - date_now).days
}
}
return dict_data
######################################
"""
Determine whether URL matches syntaxes such as
'../foo/bar/'
'/foo/../../bar/,
'https://foo.bar/foo/../'
etc.
Returns a boolean object.
"""
def is_multidot_url(self, url):
multidot = re.compile(r".*[.]{2}/.*")
if multidot.match(url):
return True
return False
######################################
"""
Get HTML element data from HTML data contents.
Two fetching methods are supported:
- A) use only HTML element/tag name and extract raw contents of
these tags
- B) use both HTML element/tag name and more fine-grained
inner attribute name to determine which HTML elements are extracted
Special case - URL link references:
- attributes 'href' or 'src' are considered as link referrals and
they are handled in a special way
- A) link referrals to directly to domain are placed in 'self_refs' list
(patterns: '/', '#', '../' and '/<anything>')
- B) link referrals to external domains are placed in 'ext_refs' list
(patterns such as 'https://foo.bar.dot/fancysite' etc.)
- Both A) and B) link categories have 'normal' and 'multidot' subcategories
- normal links do not contain pattern '../'
- multidot links contain '../' pattern
Returns a dict object.
"""
def get_tag_data(self, url, tag, attribute=None):
html_data = self.get_html_data(url)
domain_name = self.get_domain_name(url)
data = []
if attribute != None:
for d in html_data.find_all(tag):
# Ignore the HTML tag if it does not contain our attribute
if d.get(attribute) != None:
data.append(d.get(attribute))
if attribute == 'href' or attribute == 'src':
self_refs = { 'normal': [], 'multidot': []}
ext_refs = { 'normal': [], 'multidot': []}
# Syntax: '#<anything>', '/<anything>', '../<anything>'
rs = re.compile(r"^[/#]|^[.]{2}/.*")
# Syntax: '<text>:<text>/'
rd = re.compile(r"^[a-z]+:[a-z]+/")
# Syntax examples:
# 'http://foo.bar/', 'https://foo.bar/, 'foo.bar/', 'https://virus.foo.bar/'
rl = re.compile(r"^([a-z]+://)?([^/]*" + domain_name + "/)")
for s in data:
# Ignore mailto links
if re.match("^mailto:", s): continue
if rs.match(s) or rl.match(s) or rd.match(s):
if self.is_multidot_url(s):
self_refs['multidot'].append(s)
else:
self_refs['normal'].append(s)
else:
if self.is_multidot_url(s):
try:
ext_refs['multidot'].append({'url': s, 'registrar': self.get_whois_data(s).registrar })
except:
# Fallback if WHOIS query fails
ext_refs['normal'].append({'url': s, 'registrar': None })
pass
else:
try:
ext_refs['normal'].append({'url': s, 'registrar': self.get_whois_data(s).registrar })
except:
ext_refs['normal'].append({'url': s, 'registrar': None })
pass
data = None
dict_data = {
tag: {
attribute + '_ext': (ext_refs),
attribute + '_self': (self_refs)
}
}
else:
dict_data = {
tag: {
attribute: (data)
}
}
else:
for d in html_data.find_all(tag):
data.append(d.prettify())
dict_data = {
tag: (data)
}
return dict_data
######################################
"""
How many external URL links have same registrar than
the webpage itself?
"""
def get_registrar_count(self, registrar, urls):
i = 0
for u in urls:
for k,v in u.items():
if k == 'registrar' and v == registrar:
i += 1
o = len(urls) - i
dict_data = {
'same_registrar_count': i,
'other_registrar_count': o
}
return dict_data
######################################
"""
Get values existing in a dict object,
based on a known key string.
Returns a list object.
TODO: Major re-work for the fetch function
TODO: Support for more sophisticated JSON key string filtering
(possibility to use multiple keys for filtering)
"""
class json_fetcher(object):
def __init__(self, dict_data, json_key):
self.json_dict = json.loads(json.dumps(dict_data))
self.json_key = json_key
##########
# Ref: https://www.codespeedy.com/how-to-loop-through-json-with-subkeys-in-python/
def fetch(self, jdata):
if isinstance(jdata, dict):
for k,v in jdata.items():
if k == self.json_key:
yield v
elif isinstance(v, dict):
for val in self.fetch(v):
yield val
elif isinstance(v, list):
for l in v:
if isinstance(l, dict):
for ka,va in l.items():
if ka == self.json_key:
yield va
elif isinstance(jdata, list):
for l in jdata:
if isinstance(l, dict):
for k,v in l.items():
if k == self.json_key:
yield v
elif isinstance(l, list):
for lb in v:
for ka,va in lb.items():
if ka == self.json_key:
yield va
##########
def get_data(self, flatten=True):
data_extract = []
flat_data = []
for i in self.fetch(self.json_dict):
data_extract.append(i)
# Flatten possible nested lists
# (i.e. JSON data contains multiple keys in
# different nested sections)
def get_data_extract(ld):
for l in ld:
if isinstance(l, list):
for la in get_data_extract(l):
yield la
else:
yield l
if flatten == True:
for u in get_data_extract(data_extract):
flat_data.append(u)
return flat_data
else:
return data_extract
######################################
"""
Compile URL related data.
"""
def get_url_data(self, url):
# Dict object for simple, non-nested data
data_simple = {}
# Pre-defined dict object for specific data sets
webpage_data = {}
startfinal_url = self.get_startfinal_urls(url)
redirect_url = self.get_url_redirects(url)
domain_registrar = self.get_domain_registrar(url)
domaintitle_match = self.get_domain_title_match(url)
domain_time_relative = self.get_domain_timeinfo_relative(url)
domain_time = self.get_domain_timeinfo(url)
html_element_iframe = self.get_tag_data(url, 'iframe')
html_element_a_href = self.get_tag_data(url, 'a', link_refs['a'])
html_element_img_src = self.get_tag_data(url, 'img', link_refs['img'])
html_element_script_src = self.get_tag_data(url, 'script', link_refs['script'])
iframes_count = {
'iframes_count':
len(self.json_fetcher(html_element_iframe, 'iframe').get_data())
}
multidot_urls_count = {
'multidot_url_count':
len(self.json_fetcher(html_element_a_href, 'multidot').get_data()) + len(self.json_fetcher(html_element_img_src, 'multidot').get_data()) + len(self.json_fetcher(html_element_script_src, 'multidot').get_data())
}
###################
def get_total_registrars():
same_registrar_counts = 0
other_registrar_counts = 0
for k,v in link_refs.items():
html_element = self.get_tag_data(url, k, v)
same_registrar_counts += self.get_registrar_count(
domain_registrar['domain_registrar'],
html_element[k][v + '_ext']['normal']
)['same_registrar_count']
other_registrar_counts += self.get_registrar_count(
domain_registrar['domain_registrar'],
html_element[k][v + '_ext']['normal']
)['other_registrar_count']
registrar_counts = {
'same_registrar_count': same_registrar_counts,
'other_registrar_count': other_registrar_counts
}
return registrar_counts
# Avoid unnecessary nesting of the following data
data_simple.update(domain_registrar)
data_simple.update(domaintitle_match)
data_simple.update(iframes_count)
data_simple.update(multidot_urls_count)
data_simple.update(get_total_registrars())
url_data = dict({
url: [
data_simple,
startfinal_url,
{'redirects': redirect_url},
domain_time_relative,
domain_time,
{'webpage_data': [
html_element_iframe,
html_element_a_href,
html_element_img_src,
html_element_script_src
]
}
]
})
return url_data
class write_operations(object):
def __init__(self):
self.filename = filename
######################################
"""
Set JSON file name, append number suffix
# if file exists already.
Returns file name path.
"""
def set_filename(self):
c = 0
while True:
if os.path.exists(self.filename):
if c == 0:
self.filename = self.filename + "." + str(c)
else:
self.filename = re.sub("[0-9]+$", str(c), self.filename)
else:
break
c += 1
return self.filename
######################################
"""
Append to a JSON file.
"""
def write_to_file(self, data):
try:
json_file = open(self.filename, "a")
json_file.write(data)
json_file.close()
return 0
except:
return 1
######################################
"""
Fetch all pre-defined URLs.
"""
def fetch_and_store_url_data(self, urls, use_file):
data_parts = {}
fetch_json_data = json_url_data()
for u in urls:
print("URL data: %s" % u)
try:
data_parts.update(fetch_json_data.get_url_data(u))
except:
print("Failed: %s" % u)
pass
json_data = json.dumps(data_parts)
if use_file == True:
self.write_to_file(json_data)
return json_data
######################################
"""
Visualize & summarize data.
"""
class data_visualization(object):
def __init__(self, url, json_data):
self.url = url
self.json_data = json_data
self.data = json.loads(json.dumps(self.json_data)).get(self.url)
self.json_url_obj = json_url_data()
self.domain_registrar = self.json_url_obj.get_domain_registrar(self.url)['domain_registrar']
self.webpage_data = self.json_url_obj.json_fetcher(self.data, 'webpage_data').get_data()
def get_urls_count_summary(self):
unique_refs = []
for k,v in link_refs.items():
if v in unique_refs: continue
unique_refs.append(v)
def link_count(refs, suffix):
urls_cnt = 0
for u in self.webpage_data:
for l in refs:
urls = self.json_url_obj.json_fetcher(u, l + suffix).get_data()
for n in urls:
urls_cnt += len(n['normal'])
urls_cnt += len(n['multidot'])
return urls_cnt
data = {
'local_urls': link_count(unique_refs, '_self'),
'external_urls': link_count(unique_refs, '_ext')
}
return data
def get_registrars(self):
registrars = []
#registrars.append(self.domain_registrar)
for w in self.webpage_data:
webpage_registrars = self.json_url_obj.json_fetcher(w, 'registrar').get_data()
for wa in webpage_registrars:
if wa != None:
registrars.append(wa)
return registrars
def get_registrar_count_summary(self):
domain_counter = dict(Counter(self.get_registrars()))
data = {'fetched_domains': domain_counter, 'url_domain_registrar': self.domain_registrar }
return data
######################################
"""
Execute the main program code.
TODO: this code must figure out the correct JSON file
if multiple generated files are present.
"""
if __name__ == '__main__':
if plot_only == False:
write_obj = write_operations()
write_obj.set_filename()
data = write_obj.fetch_and_store_url_data(urls, use_file)
url_str_pattern = re.compile(r"(^[a-z]+://)?([^/]*)")
if os.path.exists(filename):
with open(filename, "r") as json_file:
json_data = json.load(json_file)
else:
json_data = data
# Get URLs from an available JSON data
for key_url in json_data.keys():
print("Generate statistics: %s" % key_url)
fig = plt.figure()
fig_params = {
'xtick.labelsize': 8,
'figure.figsize': [9,8]
# 'figure.constrained_layout.use': True
}
plt.rcParams.update(fig_params)
domain_string = url_str_pattern.split(key_url)[2].replace('.','')
summary = data_visualization(key_url, json_data)
summary_registrars = summary.get_registrar_count_summary()['fetched_domains']
x_r = list(summary_registrars.keys())
y_r = list(summary_registrars.values())
# Show bar values
for index,data in enumerate(y_r):
plt.text(x=index, y=data+0.5, s=data, fontdict=dict(fontsize=8))
title_r = "Domains associated with HTML URL data (" + key_url + ")"
xlabel_r = "Fetched domains"
ylabel_r = "Domain count"
plt.bar(x_r, y_r, color="green", edgecolor="black")
plt.title(title_r)
plt.xlabel(xlabel_r)
plt.ylabel(ylabel_r)
plt.xticks(rotation=45, horizontalalignment="right")
if save_plot_images == True:
plt.savefig(os.getcwd() + "/" + "domain_figure_" + domain_string + ".png", dpi=plot_images_dpi)
plt.show()
#fig_u = plt.figure()
#summary_urls = summary.get_urls_count_summary()
#x_u = list(summary_urls.keys())
#y_u = list(summary_urls.values())
#title_u = "Local and external URL references (" + key_url + ")"
#xlabel_u = "Fetched URLs"
#ylabel_u = "URL count"
#plt.bar(x_u, y_u, color="blue", edgecolor='black')
#plt.title(title_u)
#plt.xlabel(xlabel_u)
#plt.ylabel(ylabel_u)
#plt.show()