crawl_site() with itertools

 

 

code: 

 

import itertools

def crawl_site(url):
    for page in itertools.count(1):
        print(page)
        pg_url = '{}{}'.format(url,page)
        html = download(pg_url)
        if html is None:
            break
        print(pg_url)