Marco/Bundescrawler
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improve handling of missing jobs

Browse source
This commit is contained in:
Marco Lents 2025-11-18 20:00:07 +01:00
parent 801c38d985
commit b8eda046dd
1 changed files with 11 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

13
crawler.py
View file

@ -70,13 +70,18 @@ disclosures: {self.disclosures}
else: else:
votes_str = "" votes_str = ""
if self.job:
job_str = self.job
else:
job_str = ""
txt = f""" txt = f"""
# Persönliche Angaben # Persönliche Angaben
Name: {self.name[1]} {self.name[0]} Name: {self.name[1]} {self.name[0]}
Partei: {self.party} Partei: {self.party}
Beruf: {self.job} Beruf: {job_str}
Biographie: {self.cv} Biographie: {self.cv}
@ -234,7 +239,11 @@ def get_bio(url, name, sleep_for):
print(f"Getting {url} for {name[1]} {name[0]}") print(f"Getting {url} for {name[1]} {name[0]}")
response = request_handle_rate_limit(url) response = request_handle_rate_limit(url)
soup = BeautifulSoup(response.content, features="html.parser") soup = BeautifulSoup(response.content, features="html.parser")
job = soup.find(class_="m-biography__introInfo").find("span").text job_elem = soup.find(class_="m-biography__introInfo").find("span")
if job_elem:
job = job_elem.text
else:
job = None
cv = soup.find(class_="m-biography__biography").text.strip() cv = soup.find(class_="m-biography__biography").text.strip()
ajax_divs = soup.find_all(class_="m-ajaxLoadedContent") ajax_divs = soup.find_all(class_="m-ajaxLoadedContent")
speech_div = None speech_div = None