|
|
#!/usr/bin/env python3
|
|
|
|
|
|
import time
|
|
|
import asyncio
|
|
|
from aiohttp import ClientSession
|
|
|
from parsel import Selector
|
|
|
from urllib.parse import urlparse, urlunparse
|
|
|
|
|
|
|
|
|
class Crawler(object):
|
|
|
def __init__(self, url, sleep_time=.5):
|
|
|
|
|
|
self.urls = [url]
|
|
|
scheme, netloc, path, params, query, fragment = urlparse(url)
|
|
|
if not netloc:
|
|
|
netloc, path = path, netloc
|
|
|
url = urlunparse((scheme, netloc, "", params, "", fragment))
|
|
|
self.base_url = url
|
|
|
self.sleep_time = float(sleep_time)
|
|
|
|
|
|
async def fetch(self, url, session):
|
|
|
async with session.get(url) as response:
|
|
|
await asyncio.sleep(self.sleep_time)
|
|
|
return response.content
|
|
|
|
|
|
async def bound_fetch(self, sem, url, session):
|
|
|
# Getter function with semaphore.
|
|
|
async with sem:
|
|
|
await self.fetch(url, session)
|
|
|
|
|
|
def norm_link(self, url:str):
|
|
|
if url.startswith(self.base_url):
|
|
|
return url
|
|
|
elif url.startswith('//'):
|
|
|
return "{scheme}{url}".format(
|
|
|
scheme=self.base_url[:self.base_url.find(":")],
|
|
|
url=url
|
|
|
)
|
|
|
elif url.startswith("/"):
|
|
|
return self.base_url + url
|
|
|
return None
|
|
|
|
|
|
async def parse(self, content):
|
|
|
sel = Selector(content)
|
|
|
links = sel.xpath('//a/@href').getall()
|
|
|
normalized_links = []
|
|
|
for link in links:
|
|
|
link = self.norm_link(link)
|
|
|
if link:
|
|
|
normalized_links.append(link)
|
|
|
self.urls.extend(normalized_links)
|
|
|
|
|
|
async def run(self):
|
|
|
tasks = []
|
|
|
# create instance of Semaphore
|
|
|
sem = asyncio.Semaphore(20)
|
|
|
|
|
|
# Create client session that will ensure we dont open new connection
|
|
|
# per each request.
|
|
|
async with ClientSession() as session:
|
|
|
for url in self.urls:
|
|
|
# pass Semaphore and session to every GET request
|
|
|
task = asyncio.ensure_future(self.bound_fetch(sem, url, session))
|
|
|
tasks.append(task)
|
|
|
|
|
|
responses = asyncio.gather(*tasks)
|
|
|
await responses
|
|
|
|