-
Notifications
You must be signed in to change notification settings - Fork 0
/
update_from_github.py
141 lines (88 loc) · 4.06 KB
/
update_from_github.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
# This code has been adapted from https://github.com/RangerDigital/senko, full credit goes to
# Jakub Bednarski for the original functionality. I've merely hacked at it for my purposes!
import os
import gc
from lib import mrequests
from logger import publish_log_message
class Updater:
def __init__(self, username, repository, client, ref="main", api_token=None):
self.api_repository_contents_url = 'https://api.github.com/repos/{}/{}/contents/src?ref={}'.format(username, repository, ref)
self.api_commits_url = 'https://api.github.com/repos/{}/{}/commits?per_page=1&sha={}'.format(username, repository, ref)
self.headers = {
b"User-Agent": '{}'.format(username)
}
if api_token is not None:
self.headers[b'Authorization'] = 'token {}'.format(api_token)
self.client = client
async def _get_repository_contents(self, api_repository_contents_url):
await publish_log_message(message={'message': 'Getting repository contents from {}'.format(api_repository_contents_url)}, client=self.client)
gc.collect()
response = mrequests.get(api_repository_contents_url, headers=self.headers)
if response.status_code == 200:
json = response.json()
response.close()
gc.collect()
for file in json:
await self._process_item(file)
else:
publish_log_message(message={'error': 'Failed to get repository contents, status code was {}'.format(response.status_code)}, client=self.client)
response.close()
gc.collect()
async def _process_item(self, file):
if file['type'] == 'file':
await self._get_file(file['download_url'], file['name'])
if file['type'] == 'dir':
await self._get_dir(file['url'], file['name'])
async def _get_file(self, url, filename):
gc.collect()
await publish_log_message(message={
'message': 'Fetching {}'.format(url),
'mem_free': gc.mem_free(),
}, client=self.client)
gc.collect()
buf = bytearray(1024)
response = mrequests.get(url, headers=self.headers)
gc.collect()
if response.status_code == 200:
gc.collect()
response.save(filename, buf=buf)
gc.collect()
await publish_log_message(message={
'message': 'Sucessfully saved {}'.format(filename),
'mem_free': gc.mem_free(),
}, client=self.client)
gc.collect()
else:
await publish_log_message(message={'error': 'Failed to get {}, status code was {}'.format(filename, response.status_code)}, client=self.client)
response.close()
gc.collect()
async def _get_dir(self, url, dir_name):
await publish_log_message(message={'message': 'Getting directory {}'.format(dir_name)}, client=self.client)
gc.collect()
response = mrequests.get(url, headers=self.headers)
json = response.json()
response.close()
gc.collect()
try:
os.mkdir(dir_name)
except:
pass
os.chdir(dir_name)
for file in json:
await self._process_item(file)
os.chdir('..')
async def _write_version_file(self, api_commits_url):
await publish_log_message(message={'message': 'Getting latest commit hash...'}, client=self.client)
gc.collect()
response = mrequests.get(api_commits_url, headers=self.headers)
json = response.json()
response.close()
gc.collect()
commit_hash = json[0]['sha'][0:7]
gc.collect()
await publish_log_message(message={'message': 'Latest commit hash is {}, writing to .version file...'.format(commit_hash)}, client=self.client)
with open('.version', 'w') as file:
file.write(commit_hash)
async def update(self):
await self._get_repository_contents(self.api_repository_contents_url)
await self._write_version_file(self.api_commits_url)