Marco/Bundescrawler
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix formatting

Browse source
This commit is contained in:
Marco Lents 2025-11-14 12:46:11 +01:00
parent 1abf37d175
commit 62b1443798
1 changed files with 8 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

15
crawler.py
View file

@ -83,7 +83,7 @@ Biographie: {self.cv}
# Funktionen # Funktionen
## Ämter im Bundestag {funcs_to_str(self.functions)} ## Ämter im Bundestag {funcs_to_str(self.functions)}
## Sonstige Gremien: {funcs_to_str(self.additional_functions)} ## Sonstige Gremien {funcs_to_str(self.additional_functions)}
# Mandat # Mandat
{self.mandate[0], self.mandate[1]} {self.mandate[0], self.mandate[1]}
@ -133,7 +133,11 @@ def main():
if args.debug: if args.debug:
links = links[:5] links = links[:5]
names = names[:5] names = names[:5]
bios = [get_bio(link, name) for link, name in zip(links, names)] sleep_for = 1
else:
sleep_for = 5
bios = [get_bio(link, name, sleep_for) for link, name in zip(links, names)]
save_raw(bios, args.out) save_raw(bios, args.out)
save_individuals(bios, args.out) save_individuals(bios, args.out)
@ -173,10 +177,8 @@ def get_links_and_names():
return (links, names) return (links, names)
def get_bio(url, name): def get_bio(url, name, sleep_for):
print(name)
name, party = name name, party = name
print(name)
name = name.split(", ") name = name.split(", ")
print(f"Getting {url} for {name[1]} {name[0]}") print(f"Getting {url} for {name[1]} {name[0]}")
response = requests.get(url) response = requests.get(url)
@ -221,8 +223,7 @@ def get_bio(url, name):
disclosures, disclosures,
) )
print(bio) sleep(sleep_for)
sleep(1)
return bio return bio