diff --git a/README.md b/README.md deleted file mode 100644 index 17fa079..0000000 --- a/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# pysitemap -Sitemap generator - -## installing - - pip install sitemap-generator - -## requirements - - asyncio - aiofile - aiohttp - -## example - - import sys - import logging - from pysitemap import crawler - - if __name__ == '__main__': - if '--iocp' in sys.argv: - from asyncio import events, windows_events - sys.argv.remove('--iocp') - logging.info('using iocp') - el = windows_events.ProactorEventLoop() - events.set_event_loop(el) - - # root_url = sys.argv[1] - root_url = 'https://www.haikson.com' - crawler(root_url, out_file='sitemap.xml') - diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..146f689 --- /dev/null +++ b/README.rst @@ -0,0 +1,70 @@ +pysitemap +========= + +Sitemap generator + +installing +---------- + +:: + + pip install sitemap-generator + +requirements +------------ + +:: + + asyncio + aiofile + aiohttp + +example +------- + +:: + + import sys + import logging + from pysitemap import crawler + + if __name__ == '__main__': + if '--iocp' in sys.argv: + from asyncio import events, windows_events + sys.argv.remove('--iocp') + logging.info('using iocp') + el = windows_events.ProactorEventLoop() + events.set_event_loop(el) + + # root_url = sys.argv[1] + root_url = 'https://www.haikson.com' + crawler(root_url, out_file='sitemap.xml') + +TODO +----- + +- big sites with count of pages more then 100K will use more then 100MB + memory. Move queue and done lists into database. Write Queue and Done + backend classes based on +- Lists +- SQLite database +- Redis +- Write api for extending by user backends + +changelog +--------- + +v. 0.9.1 +'''''''' + +- extended readme +- docstrings and code commentaries + +v. 0.9.0 +'''''''' + +- since this version package supports only python version >=3.7 +- all functions recreated but api saved. If You use this package, then + just update it, install requirements and run process +- all requests works asynchronously + diff --git a/setup.py b/setup.py index 6457309..e2e74e9 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ from setuptools import find_packages, setup +from version import VERSION EXCLUDE_FROM_PACKAGES = ['tests',] @@ -8,17 +9,10 @@ def get_requirements(): requirements = df.readlines() return [requirement.strip() for requirement in requirements] -def get_version(major=0, minor=0, build=0): - return '%s.%s.%s' % (major, minor, build) - setup( name='sitemap-generator', - version=get_version( - major=0, - minor=9, - build=0, - ), + version=VERSION, packages=find_packages(exclude=EXCLUDE_FROM_PACKAGES), include_package_data=True, url='https://github.com/Haikson/sitemap-generator', diff --git a/version.py b/version.py new file mode 100644 index 0000000..7e91176 --- /dev/null +++ b/version.py @@ -0,0 +1 @@ +VERSION = '0.9.1' \ No newline at end of file