Marco/Bundescrawler
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

git functionality

Browse source
This commit is contained in:
Marco Lents 2025-11-14 11:40:07 +01:00
parent 19cdfb486d
commit aaa372fe21
1 changed files with 7 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

13
crawler.py
View file

@ -4,8 +4,9 @@ import json
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from time import sleep from time import sleep
from os.path import commonprefix from os.path import commonprefix
from os import makedirs from git import Repo
import argparse import argparse
from datetime import datetime
BUNDESTAG_URL = "https://www.bundestag.de/ajax/filterlist/de/abgeordnete/biografien/1040594-1040594?limit=9999&view=BTBiographyList" BUNDESTAG_URL = "https://www.bundestag.de/ajax/filterlist/de/abgeordnete/biografien/1040594-1040594?limit=9999&view=BTBiographyList"
BUNDESTAG_BASE_URL = "https://www.bundestag.de" BUNDESTAG_BASE_URL = "https://www.bundestag.de"
@ -72,11 +73,7 @@ def main():
args = parser.parse_args() args = parser.parse_args()
if not args.out: if not args.out:
raise ValueError("must supply out directory") raise ValueError("must supply out directory")
try: repo = Repo(args.out)
makedirs(args.out)
except FileExistsError:
print("Path already exists")
pass
links, names = get_links_and_names() links, names = get_links_and_names()
if args.debug: if args.debug:
links = links[:5] links = links[:5]
@ -84,6 +81,10 @@ def main():
bios = [get_bio(link, name) for link, name in zip(links, names)] bios = [get_bio(link, name) for link, name in zip(links, names)]
save_info(bios, args.out) save_info(bios, args.out)
repo.git.add("*")
repo.index.commit(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
origin = repo.remote(name="origin")
origin.push()
def save_info(bios, out): def save_info(bios, out):