add collected votes directory with persistence
Creates Abstimmungen/ with one markdown file per vote topic, sorted by party. Uses a JSON backing store so votes are preserved even after they are removed from the Bundestag website. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
14670538f6
commit
e5a43977c1
1 changed files with 40 additions and 0 deletions
40
crawler.py
40
crawler.py
|
|
@ -147,6 +147,7 @@ def main():
|
|||
if not args.debug:
|
||||
save_raw(bios, args.out)
|
||||
save_individuals(bios, args.out)
|
||||
save_votes(bios, args.out)
|
||||
save_disclosures(bios, args.out)
|
||||
|
||||
if args.no_git:
|
||||
|
|
@ -203,6 +204,45 @@ def save_disclosures(bios, out):
|
|||
party_file.write(party_str.strip())
|
||||
|
||||
|
||||
def save_votes(bios, out):
|
||||
dir = f"{out}/Abstimmungen"
|
||||
makedirs(dir, exist_ok=True)
|
||||
|
||||
# Load existing votes to preserve those deleted from the website
|
||||
json_path = f"{dir}/votes.json"
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
all_votes = json.load(f)
|
||||
except FileNotFoundError:
|
||||
all_votes = {}
|
||||
|
||||
# Merge new votes
|
||||
for bio in bios:
|
||||
if not bio.votes:
|
||||
continue
|
||||
rep_name = f"{bio.name[1]} {bio.name[0]}"
|
||||
for vote in bio.votes:
|
||||
topic, date, result = vote[0], vote[1], vote[2]
|
||||
key = f"{topic} ({date})"
|
||||
if key not in all_votes:
|
||||
all_votes[key] = {}
|
||||
all_votes[key][rep_name] = {"party": bio.party, "vote": result}
|
||||
|
||||
# Save JSON backing store
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(all_votes, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Generate one markdown file per vote topic
|
||||
for key, votes in sorted(all_votes.items()):
|
||||
md = f"# {key}\n\n"
|
||||
for name in sorted(votes, key=lambda n: votes[n]["party"]):
|
||||
info = votes[name]
|
||||
md += f"- {name} ({info['party']}): {info['vote']}\n"
|
||||
safe_name = re.sub(r'[/<>:"|?*]', "_", key)[:200]
|
||||
with open(f"{dir}/{safe_name}.md", "w", encoding="utf-8") as f:
|
||||
f.write(md)
|
||||
|
||||
|
||||
def group_by_party(bios):
|
||||
grouped = {}
|
||||
for bio in bios:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue