-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathscrape2.py
More file actions
58 lines (46 loc) · 6.65 KB
/
Copy pathscrape2.py
File metadata and controls
58 lines (46 loc) · 6.65 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
import re
import json
import requests
from bs4 import BeautifulSoup
URL_PARLIAMENT = "http://keputusan.pru13.gov.my/5_KeputusanDR.php?kod="
KODS_PARLIAMENT = map("{0:03d}00".format, range(1, 223))
URL_STATE = "http://keputusan.pru13.gov.my/5_KeputusanDUN.php?kod="
KODS_STATE = ("00101", "00102", "00103", "00104", "00105", "00206", "00207", "00208", "00209", "00210", "00311", "00312", "00313", "00314", "00315", "00401", "00402", "00503", "00504", "00605", "00606", "00707", "00708", "00809", "00810", "00811", "00912", "00913", "00914", "01015", "01016", "01017", "01118", "01119", "01220", "01221", "01222", "01323", "01324", "01425", "01426", "01527", "01528", "01529", "01630", "01631", "01632", "01733", "01734", "01835", "01836", "01901", "01902", "01903", "01904", "02005", "02006", "02007", "02108", "02109", "02110", "02211", "02212", "02213", "02314", "02315", "02316", "02417", "02418", "02419", "02520", "02521", "02522", "02623", "02624", "02625", "02726", "02727", "02728", "02829", "02830", "02831", "02832", "02933", "02934", "02935", "03036", "03037", "03038", "03139", "03140", "03141", "03142", "03243", "03244", "03245", "03301", "03302", "03303", "03304", "03405", "03406", "03407", "03408", "03509", "03510", "03511", "03512", "03613", "03614", "03615", "03616", "03717", "03718", "03719", "03720", "03821", "03822", "03823", "03824", "03925", "03926", "03927", "03928", "04029", "04030", "04031", "04032", "04101", "04102", "04103", "04204", "04205", "04206", "04307", "04308", "04309", "04410", "04411", "04412", "04513", "04514", "04515", "04616", "04617", "04618", "04719", "04720", "04721", "04822", "04823", "04824", "04825", "04926", "04927", "04928", "05029", "05030", "05031", "05132", "05133", "05134", "05235", "05236", "05237", "05338", "05339", "05340", "05401", "05402", "05503", "05504", "05605", "05606", "05607", "05708", "05709", "05810", "05811", "05812", "05913", "05914", "05915", "06016", "06017", "06018", "06119", "06120", "06221", "06222", "06323", "06324", "06425", "06426", "06427", "06528", "06529", "06530", "06631", "06632", "06633", "06734", "06735", "06836", "06837", "06938", "06939", "07040", "07041", "07042", "07143", "07144", "07145", "07246", "07247", "07348", "07349", "07450", "07451", "07452", "07553", "07554", "07655", "07656", "07757", "07758", "07759", "07801", "07802", "07903", "07904", "07905", "08006", "08007", "08008", "08109", "08110", "08111", "08212", "08213", "08314", "08315", "08316", "08417", "08418", "08419", "08520", "08521", "08522", "08523", "08624", "08625", "08626", "08727", "08728", "08729", "08830", "08831", "08832", "08933", "08934", "08935", "08936", "09037", "09038", "09039", "09140", "09141", "09142", "09201", "09202", "09303", "09304", "09405", "09406", "09407", "09508", "09509", "09610", "09611", "09612", "09713", "09714", "09715", "09816", "09817", "09818", "09919", "09920", "10021", "10022", "10123", "10124", "10125", "10226", "10227", "10228", "10329", "10330", "10431", "10432", "10533", "10534", "10635", "10636", "10737", "10738", "10739", "10840", "10841", "10942", "10943", "10944", "10945", "11046", "11047", "11048", "11149", "11150", "11251", "11252", "11253", "11354", "11355", "11356", "12601", "12602", "12603", "12604", "12705", "12706", "12707", "12708", "12809", "12810", "12811", "12812", "12813", "12814", "12915", "12916", "12917", "12918", "12919", "13020", "13021", "13022", "13023", "13024", "13125", "13126", "13127", "13128", "13229", "13230", "13231", "13232", "13233", "13334", "13335", "13336", "13401", "13402", "13403", "13404", "13405", "13506", "13507", "13508", "13509", "13510", "13611", "13612", "13613", "13614", "13715", "13716", "13717", "13718", "13819", "13820", "13821", "13822", "13823", "13924", "13925", "13926", "13927", "13928", "14001", "14002", "14103", "14104", "14205", "14206", "14307", "14308", "14409", "14410", "14411", "14512", "14513", "14514", "14615", "14616", "14717", "14718", "14819", "14820", "14921", "14922", "15023", "15024", "15025", "15126", "15127", "15228", "15229", "15330", "15331", "15432", "15433", "15534", "15535", "15636", "15637", "15738", "15739", "15840", "15841", "15942", "15943", "16044", "16045", "16146", "16147", "16248", "16249", "16350", "16351", "16352", "16453", "16454", "16555", "16556", "16701", "16702", "16703", "16804", "16805", "16906", "16907", "16908", "17009", "17010", "17011", "17112", "17113", "17214", "17215", "17216", "17317", "17318", "17419", "17420", "17521", "17522", "17623", "17624", "17725", "17726", "17827", "17828", "17929", "17930", "17931", "18032", "18033", "18034", "18135", "18136", "18237", "18238", "18339", "18340", "18441", "18442", "18543", "18544", "18645", "18646", "18747", "18748", "18849", "18850", "18851", "18952", "18953", "18954", "19055", "19056", "19057", "19158", "19159", "19160")
CENTRE_TEXT_RE = re.compile(r"\s*NEGERI : (.+) BAHAGIAN PILIHAN RAYA : (.\.)0*(.+) - (.+)\s*")
def scrape_candidate(candidate):
return {
"name": re.split("\.\s*", candidate.contents[0].text)[1],
"party": candidate.contents[1].text.strip(),
"votes": candidate.contents[2].text.replace(',', ''),
"status": candidate.contents[3].text or u"Kalah"
}
def scrape(url, kods):
records = []
for kod in kods:
r = requests.get(url + kod)
soup = BeautifulSoup(r.text)
center_text = soup.select("td[align=center] > b")[0].text
data = CENTRE_TEXT_RE.match(center_text).groups()
record = {"state": data[0], "constituency_code": data[1] + data[2], "constituency_name": data[3]}
candidates = soup.select("tr[valign=center]")
record["candidates"] = map(scrape_candidate, candidates)
winner = (c for c in record["candidates"] if c["status"] == "MENANG").next()
record["winner_name"] = winner["name"]
record["winner_party"] = winner["party"]
lower_table = soup.select("tr[align=right]")
record["total_registered_voters"] = lower_table[0].contents[3].text.replace(',', '')
record["spoilt_votes"] = lower_table[1].contents[3].text.replace(',', '')
record["ballot_sheets_issued"] = lower_table[3].contents[3].text.replace(',', '')
record["unreturned_ballot_sheets"] = lower_table[4].contents[3].text.replace(',', '')
record["voters_turnout"] = lower_table[5].contents[3].text.replace('%', '')
record["majority"] = lower_table[6].contents[3].text.replace(',', '')
print record
records.append(record)
return records
if __name__ == "__main__":
records = scrape(URL_PARLIAMENT, KODS_PARLIAMENT)
with open("pru13_parliament.json", 'w') as f:
json.dump(records, f)
records = scrape(URL_STATE, KODS_STATE)
with open("pru13_state.json", 'w') as f:
json.dump(records, f)