1
2
3 import json
4 import pprint
5 import sys
6 import os
7 import logging
8 import requests
9 import re
10 import munch
11 import subprocess
12
13 sys.path.append(
14 os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
15 )
16
17 from coprs import db, app, models
18 from coprs.logic.coprs_logic import CoprDirsLogic
19 from coprs.logic.builds_logic import BuildsLogic
20 from coprs.logic.complex_logic import ComplexLogic
21 from coprs.logic.packages_logic import PackagesLogic
22 from coprs import helpers
23
24 from urllib.parse import urlparse
25
26 SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm")
27
28 log = logging.getLogger(__name__)
29 if os.getenv('PAGURE_EVENTS_TESTONLY'):
30 ENDPOINT = 'tcp://stg.pagure.io:9940'
31 else:
32 ENDPOINT = 'tcp://hub.fedoraproject.org:9940'
33
34 log.setLevel(logging.DEBUG)
35 log.info("ENDPOINT = {}".format(ENDPOINT))
36
37 TOPICS = {}
38 for topic, url in app.config["PAGURE_EVENTS"].items():
39 TOPICS['{0}'.format(topic)] = url
42 log.info("getting url {}".format(url))
43 for attempt in range(1, 4):
44 r = requests.get(url)
45 if r.status_code == requests.codes.ok:
46 return r.text
47 else:
48 log.error('Bad http status {0} from url {1}, attempt {2}'.format(
49 r.status_code, url, attempt))
50
51 return ""
52
62
63 - def build(self, source_dict_update, copr_dir, update_callback,
64 scm_object_type, scm_object_id, scm_object_url, agent_url):
75
76 @classmethod
78 if db.engine.url.drivername == 'sqlite':
79 placeholder = '?'
80 true = '1'
81 else:
82 placeholder = '%s'
83 true = 'true'
84
85 rows = db.engine.execute(
86 """
87 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id
88 FROM package JOIN copr_dir ON package.copr_dir_id = copr_dir.id
89 WHERE package.source_type = {0} AND
90 package.webhook_rebuild = {1} AND
91 copr_dir.main = {2} AND
92 package.source_json ILIKE {placeholder}
93 """.format(SCM_SOURCE_TYPE, true, true, placeholder=placeholder), '%'+clone_url+'%'
94 )
95 return [ScmPackage(row) for row in rows]
96
97
99 if not changed_files:
100 return False
101
102 sm = helpers.SubdirMatch(self.subdirectory)
103 for filename in changed_files:
104 if sm.match(filename):
105 return True
106
107 return False
108
147
150 """
151 Message handler for new pull-request opened in pagure.
152 Topic: ``*.pagure.pull-request.new``
153 """
154 return munch.Munch({
155 'object_id': data['msg']['pullrequest']['id'],
156 'object_type': 'pull-request',
157 'base_project_url_path': data['msg']['pullrequest']['project']['url_path'],
158 'base_clone_url_path': data['msg']['pullrequest']['project']['fullname'],
159 'base_clone_url': base_url + data['msg']['pullrequest']['project']['fullname'],
160 'project_url_path': data['msg']['pullrequest']['repo_from']['url_path'],
161 'clone_url_path': data['msg']['pullrequest']['repo_from']['fullname'],
162 'clone_url': base_url + data['msg']['pullrequest']['repo_from']['fullname'],
163 'branch_from': data['msg']['pullrequest']['branch_from'],
164 'branch_to': data['msg']['pullrequest']['branch'],
165 'start_commit': data['msg']['pullrequest']['commit_start'],
166 'end_commit': data['msg']['pullrequest']['commit_stop'],
167 'agent': data['msg']['agent'],
168 })
169
172 """
173 Message handler for push event in pagure.
174 Topic: ``*.pagure.git.receive``
175 """
176 return munch.Munch({
177 'object_id': data['msg']['end_commit'],
178 'object_type': 'commit',
179 'base_project_url_path': data['msg']['repo']['url_path'],
180 'base_clone_url_path': data['msg']['repo']['fullname'],
181 'base_clone_url': base_url + data['msg']['repo']['fullname'],
182 'project_url_path': data['msg']['repo']['url_path'],
183 'clone_url_path': data['msg']['repo']['fullname'],
184 'clone_url': base_url + data['msg']['repo']['fullname'],
185 'branch_from': data['msg']['branch'],
186 'branch_to': data['msg']['branch'],
187 'start_commit': data['msg']['start_commit'],
188 'end_commit': data['msg']['end_commit'],
189 'agent': data['msg']['agent'],
190 })
191
194 url1 = re.sub(r'(\.git)?/*$', '', str(url1))
195 url2 = re.sub(r'(\.git)?/*$', '', str(url2))
196 o1 = urlparse(url1)
197 o2 = urlparse(url2)
198 return (o1.netloc == o2.netloc and o1.path == o2.path)
199
202
204 pp = pprint.PrettyPrinter(width=120)
205
206 log.debug('Parsing...')
207 data = {
208 'topic': message.topic,
209 'msg': message.body
210 }
211
212 log.info('Got topic: {}'.format(data['topic']))
213 base_url = TOPICS.get(data['topic'])
214 if not base_url:
215 log.error('Unknown topic {} received. Continuing.')
216 return
217
218 if re.match(r'^.*.pull-request.(new|rebased|updated)$', data['topic']):
219 event_info = event_info_from_pr(data, base_url)
220 elif re.match(r'^.*.pull-request.comment.added$', data['topic']):
221 event_info = event_info_from_pr_comment(data, base_url)
222 else:
223 event_info = event_info_from_push(data, base_url)
224
225 log.info('event_info = {}'.format(pp.pformat(event_info)))
226
227 if not event_info:
228 log.info('Received event was discarded. Continuing.')
229 return
230
231 candidates = ScmPackage.get_candidates_for_rebuild(event_info.base_clone_url)
232 changed_files = set()
233
234 if candidates:
235 raw_commit_url = base_url + event_info.project_url_path + '/raw/' + event_info.start_commit
236 raw_commit_text = get_repeatedly(raw_commit_url)
237 changed_files |= helpers.raw_commit_changes(raw_commit_text)
238
239 if event_info.start_commit != event_info.end_commit:
240
241
242 change_html_url = '{base_url}{project}/c/{start}..{end}'.format(
243 base_url=base_url,
244 project=event_info.project_url_path,
245 start=event_info.start_commit,
246 end=event_info.end_commit)
247
248 change_html_text = get_repeatedly(change_html_url)
249 changed_files |= helpers.pagure_html_diff_changed(change_html_text)
250
251 log.info("changed files: {}".format(", ".join(changed_files)))
252
253 for pkg in candidates:
254 package = '{}/{}(id={})'.format(
255 pkg.package.copr.full_name,
256 pkg.package.name,
257 pkg.package.id
258 )
259 log.info('Considering pkg package: {}, source_json: {}'
260 .format(package, pkg.source_json_dict))
261
262 if (git_compare_urls(pkg.clone_url, event_info.base_clone_url)
263 and (not pkg.committish or event_info.branch_to.endswith(pkg.committish))
264 and pkg.is_dir_in_commit(changed_files)):
265
266 log.info('\t -> accepted.')
267
268 if event_info.object_type == 'pull-request':
269 dirname = pkg.copr.name + ':pr:' + str(event_info.object_id)
270 copr_dir = CoprDirsLogic.get_or_create(pkg.copr, dirname)
271 update_callback = 'pagure_flag_pull_request'
272 scm_object_url = os.path.join(base_url, event_info.project_url_path,
273 'c', str(event_info.end_commit))
274 else:
275 copr_dir = pkg.copr.main_dir
276 update_callback = 'pagure_flag_commit'
277 scm_object_url = os.path.join(base_url, event_info.base_project_url_path,
278 'c', str(event_info.object_id))
279
280 if not git_compare_urls(pkg.copr.scm_repo_url, event_info.base_clone_url):
281 update_callback = ''
282
283 source_dict_update = {
284 'clone_url': event_info.clone_url,
285 'committish': event_info.end_commit,
286 }
287
288 try:
289 build = pkg.build(
290 source_dict_update,
291 copr_dir,
292 update_callback,
293 event_info.object_type,
294 event_info.object_id,
295 scm_object_url,
296 "{}user/{}".format(base_url, event_info.agent),
297 )
298 if build:
299 log.info('\t -> {}'.format(build.to_dict()))
300 except Exception as e:
301 log.error(str(e))
302 db.session.rollback()
303 else:
304 db.session.commit()
305 else:
306 log.info('\t -> skipping.')
307