article-matching/getArticleText.py at master · newsgac/article-matching · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#!/usr/bin/python3 -W all
"""
    getArticleText.py: retrieve text of all newspaper articles on a page
    usage: getArticleText.py < file
    20180416 erikt(at)xs4all.nl
"""

import csv
import os
import re
import sys
import time
from urllib.request import urlopen
import xml.etree.ElementTree as ET

ARTICLEURLPATH = "./{http://www.loc.gov/zing/srw/}records/{http://www.loc.gov/zing/srw/}record/{http://www.loc.gov/zing/srw/}recordData/{http://purl.org/dc/elements/1.1/}identifier"
COMMAND = sys.argv.pop(0)
MAXYEAR = 1995
SEPARATOR = "\t"
URLPREFIX = "http://jsru.kb.nl/sru/sru?query=type=artikel+and+page="
URLINFIX1 = "+and+date="
URLINFIX2 = "+and+ppn="
URLPOSTFIX = r"&x-collection=DDD_artikel"

ppns = { "00Algemeen Handelsblad":"400374129",
         "05NRC Handelsblad":"400367629",
         "06De Telegraaf":"832675288",
         "07De Maasbode":"842126635",
         "08De Volkskrant":"412869594" }

def makeDateId(newspaper,date,pageNbr):
    return(newspaper+SEPARATOR+date+SEPARATOR+pageNbr)

def splitDateId(date):
    return(date.split(SEPARATOR))

def checkDate(dateId):
    newspaper,date,pageNbr = splitDateId(dateId)
    try: day,month,year = date.split("-")
    except: sys.exit(COMMAND+": error processing date "+date)
    return(int(year) <= MAXYEAR)

def readDBFile():
    dateIds = {}
    csvReader = csv.DictReader(sys.stdin,delimiter=SEPARATOR)
    lineNbr = 0
    for row in csvReader:
        lineNbr += 1
        try: dateId = makeDateId(row["Titel krant"],row["Datum"],row["Paginanummer"])
        except: sys.exit(COMMAND+": missing data on line "+str(lineNbr))
        if checkDate(dateId): dateIds[dateId] = True
    return(dateIds)

def convertDate(date):
    try: day,month,year = date.split("-")
    except: sys.exit(COMMAND+": error processing date "+date)
    if len(day) == 1: day = "0"+day
    if len(month) == 1: month = "0"+month
    return(year+month+day)

def makeUrl(date):
    newspaper,date,pageNbr = splitDateId(date)
    url = URLPREFIX+str(pageNbr)+URLINFIX1+convertDate(date)+URLINFIX2+ppns[newspaper]+URLPOSTFIX
    return(url)

def getUrlData(url):
    time.sleep(1)
    return(str(urlopen(url,data=None).read(),encoding="utf-8"))

def getArticleUrls(dateId):
    url = makeUrl(dateId)
    htmlData = getUrlData(url)
    root = ET.fromstring(htmlData)
    articleUrls = []
    for articleUrl in root.findall(ARTICLEURLPATH):
        articleUrls.append(articleUrl.text)
    return(articleUrls)

def getArticleTexts(articleUrls):
    articleTexts = []
    for articleUrl in articleUrls:
        articleTexts.append(getUrlData(articleUrl))
    return(articleTexts)

def makeFileName(dateId):
    newspaper,date,pageNbr = splitDateId(dateId)
    dateId = makeDateId(newspaper,convertDate(date),pageNbr)
    return(re.sub(r"\s",r"-",dateId)+".xml")

def storeArticleTexts(dateId,articleTexts):
    fileName = makeFileName(dateId)
    with open(fileName,"w") as outFile:
        outFile.write("<container>")
        for articleText in articleTexts:
            outFile.write(articleText)
        outFile.write("</container>")
        outFile.close()
    return()

def main(argv):
    dateIds = readDBFile()
    for dateId in dateIds:
        fileName = makeFileName(dateId)
        if not os.path.isfile(fileName):
            articleUrls = getArticleUrls(dateId)
            articleTexts = getArticleTexts(articleUrls)
            storeArticleTexts(dateId,articleTexts)
    return(0)

if __name__ == "__main__":
    sys.exit(main(sys.argv))