pysitemap

Sitemap generator

installing

pip install sitemap-generator

Gevent

Sitemap-generator uses gevent to implement multiprocessing. Install gevent:

pip install gevent

example

import pysitemap
if __name__ == '__main__':
    url = 'http://www.example.com/'  # url from to crawl
    logfile = 'errlog.log'  # path to logfile
    oformat = 'xml'  # output format
    crawl = pysitemap.Crawler(url=url, logfile=logfile, oformat=oformat)
    crawl.crawl()

multiprocessing example

import pysitemap
if __name__ == '__main__':
    url = 'http://www.example.com/'  # url from to crawl
    logfile = 'errlog.log'  # path to logfile
    oformat = 'xml'  # output format
    crawl = pysitemap.Crawler(url=url, logfile=logfile, oformat=oformat)
    crawl.crawl(pool_size=10)  # 10 parsing processes