diff --git a/crawler.py b/crawler.py index 24db9da..755a9c5 100644 --- a/crawler.py +++ b/crawler.py @@ -83,7 +83,7 @@ Biographie: {self.cv} # Funktionen ## Ämter im Bundestag {funcs_to_str(self.functions)} -## Sonstige Gremien: {funcs_to_str(self.additional_functions)} +## Sonstige Gremien {funcs_to_str(self.additional_functions)} # Mandat {self.mandate[0], self.mandate[1]} @@ -133,7 +133,11 @@ def main(): if args.debug: links = links[:5] names = names[:5] - bios = [get_bio(link, name) for link, name in zip(links, names)] + sleep_for = 1 + else: + sleep_for = 5 + + bios = [get_bio(link, name, sleep_for) for link, name in zip(links, names)] save_raw(bios, args.out) save_individuals(bios, args.out) @@ -173,10 +177,8 @@ def get_links_and_names(): return (links, names) -def get_bio(url, name): - print(name) +def get_bio(url, name, sleep_for): name, party = name - print(name) name = name.split(", ") print(f"Getting {url} for {name[1]} {name[0]}") response = requests.get(url) @@ -221,8 +223,7 @@ def get_bio(url, name): disclosures, ) - print(bio) - sleep(1) + sleep(sleep_for) return bio