od/data/szif/main.py at dc71e1eca42214f928de7b8065c6bc63110809b9 · kokes/od · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
import csv
import os
from urllib.request import urlopen

from lxml.etree import iterparse

BASE_URL = (
    "https://www.szif.cz/cs/CmDocument?rid=%2Fapa_anon%2Fcs%2F"
    "dokumenty_ke_stazeni%2Fpkp%2Fspd%2Fopendata%2Fspd{year}.xml"
)
years = [2017, 2018, 2019, 2020, 2021]


def main(outdir: str, partial: bool = False):
    id_prijemce = 1

    with open(os.path.join(outdir, "zadatele.csv"), "w", encoding="utf8") as fz, open(
        os.path.join(outdir, "platby.csv"), "w", encoding="utf8"
    ) as fp:
        cz = csv.DictWriter(
            fz,
            ["id_prijemce", "rok", "jmeno_nazev", "obec", "okres", "castka_bez_pvp"],
            lineterminator="\n",
        )
        cp = csv.DictWriter(
            fp,
            [
                "id_prijemce",
                "rok",
                "fond_typ_podpory",
                "opatreni",
                "zdroje_cr",
                "zdroje_eu",
                "celkem_czk",
            ],
            lineterminator="\n",
        )

        cz.writeheader()
        cp.writeheader()

        for year in years:
            url = BASE_URL.format(year=year)

            with urlopen(url) as r:
                et = iterparse(r)

                elyear = None
                for num, (action, element) in enumerate(et):
                    if partial and num > 1e3:
                        break
                    assert action == "end"
                    if element.tag == "rok":
                        elyear = int(element.text)
                        assert elyear == year, "Necekany rok v datech"

                    if element.tag != "zadatel":
                        continue

                    zadatel = {"id_prijemce": id_prijemce, "rok": elyear}

                    for key in ["jmeno_nazev", "obec", "okres", "castka_bez_pvp"]:
                        zadatel[key] = element.find(key).text

                    for elplatba in element.findall("platby/platba") + element.findall(
                        "platby_pvp/platba_pvp"
                    ):
                        platba = {"id_prijemce": id_prijemce, "rok": elyear}
                        for key in [
                            "fond_typ_podpory",
                            "opatreni",
                            "zdroje_cr",
                            "zdroje_eu",
                            "celkem_czk",
                        ]:
                            platba[key] = getattr(elplatba.find(key), "text", None)

                        cp.writerow(platba)

                    cz.writerow(zadatel)
                    id_prijemce += 1

                    element.clear()