Marco/Bundescrawler
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add job

Browse source
This commit is contained in:
Marco Lents 2025-11-17 17:26:10 +01:00
parent 3787f756f3
commit 0fcd725192
1 changed files with 6 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crawler.py
View file

@ -21,6 +21,7 @@ class Biography:
self, self,
name, name,
party, party,
job,
cv, cv,
speeches, speeches,
votes, votes,
@ -31,6 +32,7 @@ class Biography:
): ):
self.name = name self.name = name
self.party = party self.party = party
self.job = job
self.cv = cv self.cv = cv
self.speeches = speeches self.speeches = speeches
self.votes = votes self.votes = votes
@ -74,6 +76,8 @@ Name: {self.name[1]} {self.name[0]}
Partei: {self.party} Partei: {self.party}
Beruf: {self.job}
Biographie: {self.cv} Biographie: {self.cv}
# Reden {speeches_str} # Reden {speeches_str}
@ -229,6 +233,7 @@ def get_bio(url, name, sleep_for):
print(f"Getting {url} for {name[1]} {name[0]}") print(f"Getting {url} for {name[1]} {name[0]}")
response = request_handle_rate_limit(url) response = request_handle_rate_limit(url)
soup = BeautifulSoup(response.content) soup = BeautifulSoup(response.content)
job = soup.find(class_="m-biography__introInfo").find("span").text
cv = soup.find(class_="m-biography__biography").text.strip() cv = soup.find(class_="m-biography__biography").text.strip()
ajax_divs = soup.find_all(class_="m-ajaxLoadedContent") ajax_divs = soup.find_all(class_="m-ajaxLoadedContent")
speech_div = None speech_div = None
@ -260,6 +265,7 @@ def get_bio(url, name, sleep_for):
bio = Biography( bio = Biography(
name, name,
party, party,
job,
cv, cv,
speeches, speeches,
votes, votes,