improve handling of missing jobs
This commit is contained in:
parent
801c38d985
commit
b8eda046dd
1 changed files with 11 additions and 2 deletions
13
crawler.py
13
crawler.py
|
|
@ -70,13 +70,18 @@ disclosures: {self.disclosures}
|
|||
else:
|
||||
votes_str = ""
|
||||
|
||||
if self.job:
|
||||
job_str = self.job
|
||||
else:
|
||||
job_str = ""
|
||||
|
||||
txt = f"""
|
||||
# Persönliche Angaben
|
||||
Name: {self.name[1]} {self.name[0]}
|
||||
|
||||
Partei: {self.party}
|
||||
|
||||
Beruf: {self.job}
|
||||
Beruf: {job_str}
|
||||
|
||||
Biographie: {self.cv}
|
||||
|
||||
|
|
@ -234,7 +239,11 @@ def get_bio(url, name, sleep_for):
|
|||
print(f"Getting {url} for {name[1]} {name[0]}")
|
||||
response = request_handle_rate_limit(url)
|
||||
soup = BeautifulSoup(response.content, features="html.parser")
|
||||
job = soup.find(class_="m-biography__introInfo").find("span").text
|
||||
job_elem = soup.find(class_="m-biography__introInfo").find("span")
|
||||
if job_elem:
|
||||
job = job_elem.text
|
||||
else:
|
||||
job = None
|
||||
cv = soup.find(class_="m-biography__biography").text.strip()
|
||||
ajax_divs = soup.find_all(class_="m-ajaxLoadedContent")
|
||||
speech_div = None
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue