DebateStat/append.py at master · tanavc1/DebateStat · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
from urllib.request import urlopen
from bs4 import BeautifulSoup
import csv
def append():
    x=0
    thisdict = {}
    while(x<1000000):
        x+=1
        if(x in '/home/tanavc/mysite/debatedata.csv'):
            continue
        else:
            url = "https://www.tabroom.com/index/results/team_lifetime_record.mhtml?id1=" + "&" + "id2=" + str(x)
            res = urlopen(url)
            thepage = res.read()
            soup = BeautifulSoup(thepage, 'html.parser')
            try:
                teamname = soup.find_all("h4")[5].text
            except:
                continue
            if (soup.find_all("td")[10].text == "/"):
                continue
            else:
                adjustedteamname = teamname.replace('                    ', '')
                thisdict[x] = adjustedteamname.strip()
                print(thisdict[x])
                print(x)
                writeCSV = [x,thisdict[x]]
                f = open("debatedata.csv", "a")
                w = csv.writer(f)
                w.writerow(writeCSV)
                f.close()
append()