Marco/Bundescrawler
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix handling of empty speeches

Browse source
This commit is contained in:
Marco Lents 2025-11-15 14:43:31 +01:00
parent 384bd83b20
commit 93310a8030
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crawler.py
View file

@ -133,9 +133,9 @@ def main():
if args.debug: if args.debug:
links = links[:5] links = links[:5]
names = names[:5] names = names[:5]
sleep_for = 1 sleep_for = 0
else: else:
sleep_for = 5 sleep_for = 10
bios = [get_bio(link, name, sleep_for) for link, name in zip(links, names)] bios = [get_bio(link, name, sleep_for) for link, name in zip(links, names)]
@ -252,7 +252,7 @@ def get_functions(elem):
def parse_speech(page): def parse_speech(page):
if not page: if not page:
return (None, None) return None
soup = BeautifulSoup(page.content) soup = BeautifulSoup(page.content)
infos = [s.text.strip() for s in soup.find_all(class_="m-biography__speechTitle")] infos = [s.text.strip() for s in soup.find_all(class_="m-biography__speechTitle")]
titles = [ titles = [