diff --git a/crawler.py b/crawler.py index 28938f6..0d0fcac 100644 --- a/crawler.py +++ b/crawler.py @@ -147,6 +147,7 @@ def main(): if not args.debug: save_raw(bios, args.out) save_individuals(bios, args.out) + save_votes(bios, args.out) save_disclosures(bios, args.out) if args.no_git: @@ -203,6 +204,45 @@ def save_disclosures(bios, out): party_file.write(party_str.strip()) +def save_votes(bios, out): + dir = f"{out}/Abstimmungen" + makedirs(dir, exist_ok=True) + + # Load existing votes to preserve those deleted from the website + json_path = f"{dir}/votes.json" + try: + with open(json_path, "r", encoding="utf-8") as f: + all_votes = json.load(f) + except FileNotFoundError: + all_votes = {} + + # Merge new votes + for bio in bios: + if not bio.votes: + continue + rep_name = f"{bio.name[1]} {bio.name[0]}" + for vote in bio.votes: + topic, date, result = vote[0], vote[1], vote[2] + key = f"{topic} ({date})" + if key not in all_votes: + all_votes[key] = {} + all_votes[key][rep_name] = {"party": bio.party, "vote": result} + + # Save JSON backing store + with open(json_path, "w", encoding="utf-8") as f: + json.dump(all_votes, f, indent=2, ensure_ascii=False) + + # Generate one markdown file per vote topic + for key, votes in sorted(all_votes.items()): + md = f"# {key}\n\n" + for name in sorted(votes, key=lambda n: votes[n]["party"]): + info = votes[name] + md += f"- {name} ({info['party']}): {info['vote']}\n" + safe_name = re.sub(r'[/<>:"|?*]', "_", key)[:200] + with open(f"{dir}/{safe_name}.md", "w", encoding="utf-8") as f: + f.write(md) + + def group_by_party(bios): grouped = {} for bio in bios: