-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathapp.py
86 lines (59 loc) · 1.96 KB
/
app.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
import requests
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
import re
import sys
from datetime import datetime
url = sys.argv[1]
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36"
}
page = requests.get(url, headers=headers)
soup = BeautifulSoup(page.content, "html.parser")
#print(f"SOUP: {soup}")
table = soup.find("table")
columns = []
data = []
rows = table.find_all("tr")
for index, row in enumerate(rows):
#cols = row.find_all("p")
data_row = []
for _index, col in enumerate(row):
# print(col, type(col))
col_text = col.get_text().replace("\n", "").strip()
# skip any column that's not a td or th
if not re.search(r'td|th', str(col)):
continue
if index == 0:
#print(f"column: {col_text}")
columns.append(col_text)
else:
data_row.append(col_text)
if index != 0:
data.append(data_row)
#Let's clear columns and rows
#first add the columns array on top of the data matrix
data_copy = list()
data.insert(0, columns)
data_copy = data
data_copy = np.array(data_copy)
#check for empty columns and rows
empty_columns = np.all(data_copy == '', axis=0)
empty_rows = np.all(data_copy == '', axis=1)
#filtered out the empty columns and rows from the matrix
data_copy = data_copy[:, ~empty_columns]
data_copy = data_copy[~empty_rows]
#now that the matrix empty columns where removed, reset the columns and data in seperate varibles
columns = data_copy[0, :]
data = data_copy[1:, :]
#print(f"Columns: {columns}")
#print(f"Data:{data}")
df = pd.DataFrame(data, columns=columns)
now = datetime.now()
formatted_datetime = now.strftime("%Y%m%d%H%M%S")
df.to_csv(f'downloads/download{formatted_datetime}.csv', index=False)
if __name__ == "__main__":
if len(sys.argv) < 2:
print(f"Usage: python test.py {sys.argv[1]}")
sys.exit(0)