Package coprs :: Package logic :: Module builds_logic
[hide private]
[frames] | no frames]

Source Code for Module coprs.logic.builds_logic

   1  import tempfile 
   2  import shutil 
   3  import json 
   4  import os 
   5  import pprint 
   6  import time 
   7  import requests 
   8   
   9  from sqlalchemy.sql import text 
  10  from sqlalchemy.sql.expression import not_ 
  11  from sqlalchemy.orm import joinedload 
  12  from sqlalchemy import or_ 
  13  from sqlalchemy import and_ 
  14  from sqlalchemy import func, desc 
  15  from sqlalchemy.sql import false,true 
  16  from werkzeug.utils import secure_filename 
  17  from sqlalchemy import bindparam, Integer, String 
  18  from sqlalchemy.exc import IntegrityError 
  19   
  20  from copr_common.enums import FailTypeEnum, StatusEnum 
  21  from coprs import app 
  22  from coprs import cache 
  23  from coprs import db 
  24  from coprs import models 
  25  from coprs import helpers 
  26  from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT 
  27  from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, \ 
  28                               UnrepeatableBuildException, RequestCannotBeExecuted, DuplicateException 
  29   
  30  from coprs.logic import coprs_logic 
  31  from coprs.logic import users_logic 
  32  from coprs.logic.actions_logic import ActionsLogic 
  33  from coprs.models import BuildChroot 
  34  from .coprs_logic import MockChrootsLogic 
  35  from coprs.logic.packages_logic import PackagesLogic 
  36   
  37  log = app.logger 
38 39 40 -class BuildsLogic(object):
41 @classmethod
42 - def get(cls, build_id):
43 return models.Build.query.filter(models.Build.id == build_id)
44 45 @classmethod
46 - def get_build_tasks(cls, status, background=None):
47 """ Returns tasks with given status. If background is specified then 48 returns normal jobs (false) or background jobs (true) 49 """ 50 result = models.BuildChroot.query.join(models.Build)\ 51 .filter(models.BuildChroot.status == status)\ 52 .order_by(models.Build.id.asc()) 53 if background is not None: 54 result = result.filter(models.Build.is_background == (true() if background else false())) 55 return result
56 57 @classmethod
58 - def get_srpm_build_tasks(cls, status, background=None):
59 """ Returns srpm build tasks with given status. If background is 60 specified then returns normal jobs (false) or background jobs (true) 61 """ 62 result = models.Build.query\ 63 .filter(models.Build.source_status == status)\ 64 .order_by(models.Build.id.asc()) 65 if background is not None: 66 result = result.filter(models.Build.is_background == (true() if background else false())) 67 return result
68 69 @classmethod 70 @cache.memoize(timeout=2*60)
71 - def get_recent_task_ids(cls, user=None, limit=100, period_days=2):
72 query_args = ( 73 models.BuildChroot.build_id, 74 func.max(models.BuildChroot.ended_on).label('max_ended_on'), 75 models.Build.submitted_on, 76 ) 77 group_by_args = ( 78 models.BuildChroot.build_id, 79 models.Build.submitted_on, 80 ) 81 82 83 if user: 84 query_args += (models.Build.user_id,) 85 group_by_args += (models.Build.user_id,) 86 87 subquery = (db.session.query(*query_args) 88 .join(models.Build) 89 .group_by(*group_by_args) 90 .having(func.count() == func.count(models.BuildChroot.ended_on)) 91 .having(models.Build.submitted_on > time.time() - 3600*24*period_days) 92 ) 93 if user: 94 subquery = subquery.having(models.Build.user_id == user.id) 95 96 subquery = subquery.order_by(desc('max_ended_on')).limit(limit).subquery() 97 98 query = models.Build.query.join(subquery, subquery.c.build_id == models.Build.id) 99 return [i.id for i in query.all()]
100 101 @classmethod
102 - def get_recent_tasks(cls, *args, **kwargs):
103 task_ids = cls.get_recent_task_ids(*args, **kwargs) 104 query = models.Build.query.filter(models.Build.id.in_(task_ids)) 105 return list(query.all())
106 107 @classmethod
108 - def get_running_tasks_by_time(cls, start, end):
109 result = models.BuildChroot.query\ 110 .filter(models.BuildChroot.ended_on > start)\ 111 .filter(models.BuildChroot.started_on < end)\ 112 .order_by(models.BuildChroot.started_on.asc()) 113 114 return result
115 116 @classmethod
117 - def get_chroot_histogram(cls, start, end):
118 chroots = [] 119 chroot_query = BuildChroot.query\ 120 .filter(models.BuildChroot.started_on < end)\ 121 .filter(models.BuildChroot.ended_on > start)\ 122 .with_entities(BuildChroot.mock_chroot_id, 123 func.count(BuildChroot.mock_chroot_id))\ 124 .group_by(BuildChroot.mock_chroot_id)\ 125 .order_by(BuildChroot.mock_chroot_id) 126 127 for chroot in chroot_query: 128 chroots.append([chroot[0], chroot[1]]) 129 130 mock_chroots = coprs_logic.MockChrootsLogic.get_multiple() 131 for mock_chroot in mock_chroots: 132 for l in chroots: 133 if l[0] == mock_chroot.id: 134 l[0] = mock_chroot.name 135 136 return chroots
137 138 @classmethod
139 - def get_pending_jobs_bucket(cls, start, end):
140 query = text(""" 141 SELECT COUNT(*) as result 142 FROM build_chroot JOIN build on build.id = build_chroot.build_id 143 WHERE 144 build.submitted_on < :end 145 AND ( 146 build_chroot.started_on > :start 147 OR (build_chroot.started_on is NULL AND build_chroot.status = :status) 148 -- for currently pending builds we need to filter on status=pending because there might be 149 -- failed builds that have started_on=NULL 150 ) 151 AND NOT build.canceled 152 """) 153 154 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("pending")) 155 return res.first().result
156 157 @classmethod
158 - def get_running_jobs_bucket(cls, start, end):
159 query = text(""" 160 SELECT COUNT(*) as result 161 FROM build_chroot 162 WHERE 163 started_on < :end 164 AND (ended_on > :start OR (ended_on is NULL AND status = :status)) 165 -- for currently running builds we need to filter on status=running because there might be failed 166 -- builds that have ended_on=NULL 167 """) 168 169 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("running")) 170 return res.first().result
171 172 @classmethod
173 - def get_cached_graph_data(cls, params):
174 data = { 175 "pending": [], 176 "running": [], 177 } 178 result = models.BuildsStatistics.query\ 179 .filter(models.BuildsStatistics.stat_type == params["type"])\ 180 .filter(models.BuildsStatistics.time >= params["start"])\ 181 .filter(models.BuildsStatistics.time <= params["end"])\ 182 .order_by(models.BuildsStatistics.time) 183 184 for row in result: 185 data["pending"].append(row.pending) 186 data["running"].append(row.running) 187 188 return data
189 190 @classmethod
191 - def get_task_graph_data(cls, type):
192 data = [["pending"], ["running"], ["avg running"], ["time"]] 193 params = cls.get_graph_parameters(type) 194 cached_data = cls.get_cached_graph_data(params) 195 data[0].extend(cached_data["pending"]) 196 data[1].extend(cached_data["running"]) 197 198 for i in range(len(data[0]) - 1, params["steps"]): 199 step_start = params["start"] + i * params["step"] 200 step_end = step_start + params["step"] 201 pending = cls.get_pending_jobs_bucket(step_start, step_end) 202 running = cls.get_running_jobs_bucket(step_start, step_end) 203 data[0].append(pending) 204 data[1].append(running) 205 cls.cache_graph_data(type, time=step_start, pending=pending, running=running) 206 207 running_total = 0 208 for i in range(1, params["steps"] + 1): 209 running_total += data[1][i] 210 211 data[2].extend([running_total * 1.0 / params["steps"]] * (len(data[0]) - 1)) 212 213 for i in range(params["start"], params["end"], params["step"]): 214 data[3].append(time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(i))) 215 216 return data
217 218 @classmethod
219 - def get_small_graph_data(cls, type):
220 data = [[""]] 221 params = cls.get_graph_parameters(type) 222 cached_data = cls.get_cached_graph_data(params) 223 data[0].extend(cached_data["running"]) 224 225 for i in range(len(data[0]) - 1, params["steps"]): 226 step_start = params["start"] + i * params["step"] 227 step_end = step_start + params["step"] 228 running = cls.get_running_jobs_bucket(step_start, step_end) 229 data[0].append(running) 230 cls.cache_graph_data(type, time=step_start, running=running) 231 232 return data
233 234 @classmethod
235 - def cache_graph_data(cls, type, time, pending=0, running=0):
236 result = models.BuildsStatistics.query\ 237 .filter(models.BuildsStatistics.stat_type == type)\ 238 .filter(models.BuildsStatistics.time == time).first() 239 if result: 240 return 241 242 try: 243 cached_data = models.BuildsStatistics( 244 time = time, 245 stat_type = type, 246 running = running, 247 pending = pending 248 ) 249 db.session.add(cached_data) 250 db.session.commit() 251 except IntegrityError: # other process already calculated the graph data and cached it 252 db.session.rollback()
253 254 @classmethod
255 - def get_graph_parameters(cls, type):
256 if type is "10min": 257 # 24 hours with 10 minute intervals 258 step = 600 259 steps = 144 260 elif type is "30min": 261 # 24 hours with 30 minute intervals 262 step = 1800 263 steps = 48 264 elif type is "24h": 265 # 90 days with 24 hour intervals 266 step = 86400 267 steps = 90 268 269 end = int(time.time()) 270 end = end - (end % step) # align graph interval to a multiple of step 271 start = end - (steps * step) 272 273 return { 274 "type": type, 275 "step": step, 276 "steps": steps, 277 "start": start, 278 "end": end, 279 }
280 281 @classmethod
282 - def get_build_importing_queue(cls, background=None):
283 """ 284 Returns Builds which are waiting to be uploaded to dist git 285 """ 286 query = (models.Build.query 287 .filter(models.Build.canceled == false()) 288 .filter(models.Build.source_status == StatusEnum("importing")) 289 .order_by(models.Build.id.asc())) 290 if background is not None: 291 query = query.filter(models.Build.is_background == (true() if background else false())) 292 return query
293 294 @classmethod
295 - def get_pending_srpm_build_tasks(cls, background=None):
296 query = (models.Build.query 297 .filter(models.Build.canceled == false()) 298 .filter(models.Build.source_status == StatusEnum("pending")) 299 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 300 if background is not None: 301 query = query.filter(models.Build.is_background == (true() if background else false())) 302 return query
303 304 @classmethod
305 - def get_pending_build_tasks(cls, background=None):
306 query = (models.BuildChroot.query 307 .outerjoin(models.Build) 308 .outerjoin(models.CoprDir) 309 .outerjoin(models.Package, models.Package.id == models.Build.package_id) 310 .options(joinedload('build').joinedload('copr_dir'), 311 joinedload('build').joinedload('package')) 312 .filter(models.Build.canceled == false()) 313 .filter(or_( 314 models.BuildChroot.status == StatusEnum("pending"), 315 and_( 316 models.BuildChroot.status == StatusEnum("running"), 317 models.BuildChroot.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT), 318 models.BuildChroot.ended_on.is_(None) 319 ) 320 )) 321 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 322 if background is not None: 323 query = query.filter(models.Build.is_background == (true() if background else false())) 324 return query
325 326 @classmethod
327 - def get_build_task(cls, task_id):
328 try: 329 build_id, chroot_name = task_id.split("-", 1) 330 except ValueError: 331 raise MalformedArgumentException("Invalid task_id {}".format(task_id)) 332 333 build_chroot = BuildChrootsLogic.get_by_build_id_and_name(build_id, chroot_name) 334 return build_chroot.join(models.Build).first()
335 336 @classmethod
337 - def get_srpm_build_task(cls, build_id):
338 return BuildsLogic.get_by_id(build_id).first()
339 340 @classmethod
341 - def get_multiple(cls):
342 return models.Build.query.order_by(models.Build.id.desc())
343 344 @classmethod
345 - def get_multiple_by_copr(cls, copr):
346 """ Get collection of builds in copr sorted by build_id descending 347 """ 348 return cls.get_multiple().filter(models.Build.copr == copr)
349 350 @classmethod
351 - def get_multiple_by_user(cls, user):
352 """ Get collection of builds in copr sorted by build_id descending 353 form the copr belonging to `user` 354 """ 355 return cls.get_multiple().join(models.Build.copr).filter( 356 models.Copr.user == user)
357 358 @classmethod
359 - def init_db(cls):
360 if db.engine.url.drivername == "sqlite": 361 return 362 363 status_to_order = """ 364 CREATE OR REPLACE FUNCTION status_to_order (x integer) 365 RETURNS integer AS $$ BEGIN 366 RETURN CASE WHEN x = 3 THEN 1 367 WHEN x = 6 THEN 2 368 WHEN x = 7 THEN 3 369 WHEN x = 4 THEN 4 370 WHEN x = 0 THEN 5 371 WHEN x = 1 THEN 6 372 WHEN x = 5 THEN 7 373 WHEN x = 2 THEN 8 374 WHEN x = 8 THEN 9 375 WHEN x = 9 THEN 10 376 ELSE x 377 END; END; 378 $$ LANGUAGE plpgsql; 379 """ 380 381 order_to_status = """ 382 CREATE OR REPLACE FUNCTION order_to_status (x integer) 383 RETURNS integer AS $$ BEGIN 384 RETURN CASE WHEN x = 1 THEN 3 385 WHEN x = 2 THEN 6 386 WHEN x = 3 THEN 7 387 WHEN x = 4 THEN 4 388 WHEN x = 5 THEN 0 389 WHEN x = 6 THEN 1 390 WHEN x = 7 THEN 5 391 WHEN x = 8 THEN 2 392 WHEN x = 9 THEN 8 393 WHEN x = 10 THEN 9 394 ELSE x 395 END; END; 396 $$ LANGUAGE plpgsql; 397 """ 398 399 db.engine.connect() 400 db.engine.execute(status_to_order) 401 db.engine.execute(order_to_status)
402 403 @classmethod
404 - def get_copr_builds_list(cls, copr, dirname=''):
405 query_select = """ 406 SELECT build.id, build.source_status, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on, 407 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status, 408 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name, build.copr_id 409 FROM build 410 LEFT OUTER JOIN package 411 ON build.package_id = package.id 412 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses 413 ON statuses.build_id=build.id 414 LEFT OUTER JOIN copr 415 ON copr.id = build.copr_id 416 LEFT OUTER JOIN copr_dir 417 ON build.copr_dir_id = copr_dir.id 418 LEFT OUTER JOIN "user" 419 ON copr.user_id = "user".id 420 LEFT OUTER JOIN "group" 421 ON copr.group_id = "group".id 422 WHERE build.copr_id = :copr_id 423 AND (:dirname = '' OR :dirname = copr_dir.name) 424 GROUP BY 425 build.id 426 ORDER BY 427 build.id DESC; 428 """ 429 430 if db.engine.url.drivername == "sqlite": 431 def sqlite_status_to_order(x): 432 if x == 3: 433 return 1 434 elif x == 6: 435 return 2 436 elif x == 7: 437 return 3 438 elif x == 4: 439 return 4 440 elif x == 0: 441 return 5 442 elif x == 1: 443 return 6 444 elif x == 5: 445 return 7 446 elif x == 2: 447 return 8 448 elif x == 8: 449 return 9 450 elif x == 9: 451 return 10 452 return 1000
453 454 def sqlite_order_to_status(x): 455 if x == 1: 456 return 3 457 elif x == 2: 458 return 6 459 elif x == 3: 460 return 7 461 elif x == 4: 462 return 4 463 elif x == 5: 464 return 0 465 elif x == 6: 466 return 1 467 elif x == 7: 468 return 5 469 elif x == 8: 470 return 2 471 elif x == 9: 472 return 8 473 elif x == 10: 474 return 9 475 return 1000
476 477 conn = db.engine.connect() 478 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order) 479 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status) 480 statement = text(query_select) 481 statement.bindparams(bindparam("copr_id", Integer)) 482 statement.bindparams(bindparam("dirname", String)) 483 result = conn.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 484 else: 485 statement = text(query_select) 486 statement.bindparams(bindparam("copr_id", Integer)) 487 statement.bindparams(bindparam("dirname", String)) 488 result = db.engine.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 489 490 return result 491 492 @classmethod
493 - def join_group(cls, query):
494 return query.join(models.Copr).outerjoin(models.Group)
495 496 @classmethod
497 - def get_multiple_by_name(cls, username, coprname):
498 query = cls.get_multiple() 499 return (query.join(models.Build.copr) 500 .options(db.contains_eager(models.Build.copr)) 501 .join(models.Copr.user) 502 .filter(models.Copr.name == coprname) 503 .filter(models.User.username == username))
504 505 @classmethod
506 - def get_by_ids(cls, ids):
507 return models.Build.query.filter(models.Build.id.in_(ids))
508 509 @classmethod
510 - def get_by_id(cls, build_id):
511 return models.Build.query.filter(models.Build.id == build_id)
512 513 @classmethod
514 - def create_new_from_other_build(cls, user, copr, source_build, 515 chroot_names=None, **build_options):
516 skip_import = False 517 git_hashes = {} 518 519 if source_build.source_type == helpers.BuildSourceEnum('upload'): 520 if source_build.repeatable: 521 skip_import = True 522 for chroot in source_build.build_chroots: 523 git_hashes[chroot.name] = chroot.git_hash 524 else: 525 raise UnrepeatableBuildException("Build sources were not fully imported into CoprDistGit.") 526 527 build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, 528 pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, 529 srpm_url=source_build.srpm_url, copr_dirname=source_build.copr_dir.name, **build_options) 530 build.package_id = source_build.package_id 531 build.pkg_version = source_build.pkg_version 532 build.resubmitted_from_id = source_build.id 533 534 return build
535 536 @classmethod
537 - def create_new_from_url(cls, user, copr, url, chroot_names=None, 538 copr_dirname=None, **build_options):
539 """ 540 :type user: models.User 541 :type copr: models.Copr 542 543 :type chroot_names: List[str] 544 545 :rtype: models.Build 546 """ 547 source_type = helpers.BuildSourceEnum("link") 548 source_json = json.dumps({"url": url}) 549 srpm_url = None if url.endswith('.spec') else url 550 return cls.create_new(user, copr, source_type, source_json, chroot_names, 551 pkgs=url, srpm_url=srpm_url, copr_dirname=copr_dirname, **build_options)
552 553 @classmethod
554 - def create_new_from_scm(cls, user, copr, scm_type, clone_url, 555 committish='', subdirectory='', spec='', srpm_build_method='rpkg', 556 chroot_names=None, copr_dirname=None, **build_options):
557 """ 558 :type user: models.User 559 :type copr: models.Copr 560 561 :type chroot_names: List[str] 562 563 :rtype: models.Build 564 """ 565 source_type = helpers.BuildSourceEnum("scm") 566 source_json = json.dumps({"type": scm_type, 567 "clone_url": clone_url, 568 "committish": committish, 569 "subdirectory": subdirectory, 570 "spec": spec, 571 "srpm_build_method": srpm_build_method}) 572 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
573 574 @classmethod
575 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, spec_template, 576 python_versions, chroot_names=None, copr_dirname=None, **build_options):
577 """ 578 :type user: models.User 579 :type copr: models.Copr 580 :type package_name: str 581 :type version: str 582 :type python_versions: List[str] 583 584 :type chroot_names: List[str] 585 586 :rtype: models.Build 587 """ 588 source_type = helpers.BuildSourceEnum("pypi") 589 source_json = json.dumps({"pypi_package_name": pypi_package_name, 590 "pypi_package_version": pypi_package_version, 591 "spec_template": spec_template, 592 "python_versions": python_versions}) 593 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
594 595 @classmethod
596 - def create_new_from_rubygems(cls, user, copr, gem_name, chroot_names=None, 597 copr_dirname=None, **build_options):
598 """ 599 :type user: models.User 600 :type copr: models.Copr 601 :type gem_name: str 602 :type chroot_names: List[str] 603 :rtype: models.Build 604 """ 605 source_type = helpers.BuildSourceEnum("rubygems") 606 source_json = json.dumps({"gem_name": gem_name}) 607 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
608 609 @classmethod
610 - def create_new_from_custom(cls, user, copr, script, script_chroot=None, script_builddeps=None, 611 script_resultdir=None, chroot_names=None, copr_dirname=None, **kwargs):
612 """ 613 :type user: models.User 614 :type copr: models.Copr 615 :type script: str 616 :type script_chroot: str 617 :type script_builddeps: str 618 :type script_resultdir: str 619 :type chroot_names: List[str] 620 :rtype: models.Build 621 """ 622 source_type = helpers.BuildSourceEnum("custom") 623 source_dict = { 624 'script': script, 625 'chroot': script_chroot, 626 'builddeps': script_builddeps, 627 'resultdir': script_resultdir, 628 } 629 630 return cls.create_new(user, copr, source_type, json.dumps(source_dict), 631 chroot_names, copr_dirname=copr_dirname, **kwargs)
632 633 @classmethod
634 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename, 635 chroot_names=None, copr_dirname=None, **build_options):
636 """ 637 :type user: models.User 638 :type copr: models.Copr 639 :param f_uploader(file_path): function which stores data at the given `file_path` 640 :return: 641 """ 642 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"]) 643 tmp_name = os.path.basename(tmp) 644 filename = secure_filename(orig_filename) 645 file_path = os.path.join(tmp, filename) 646 f_uploader(file_path) 647 648 # make the pkg public 649 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format( 650 baseurl=app.config["PUBLIC_COPR_BASE_URL"], 651 tmp_dir=tmp_name, 652 filename=filename) 653 654 # create json describing the build source 655 source_type = helpers.BuildSourceEnum("upload") 656 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name}) 657 srpm_url = None if pkg_url.endswith('.spec') else pkg_url 658 659 try: 660 build = cls.create_new(user, copr, source_type, source_json, 661 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, 662 copr_dirname=copr_dirname, **build_options) 663 except Exception: 664 shutil.rmtree(tmp) # todo: maybe we should delete in some cleanup procedure? 665 raise 666 667 return build
668 669 @classmethod
670 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="", 671 git_hashes=None, skip_import=False, background=False, batch=None, 672 srpm_url=None, copr_dirname=None, **build_options):
673 """ 674 :type user: models.User 675 :type copr: models.Copr 676 :type chroot_names: List[str] 677 :type source_type: int value from helpers.BuildSourceEnum 678 :type source_json: str in json format 679 :type pkgs: str 680 :type git_hashes: dict 681 :type skip_import: bool 682 :type background: bool 683 :type batch: models.Batch 684 :rtype: models.Build 685 """ 686 chroots = None 687 if chroot_names: 688 chroots = [] 689 for chroot in copr.active_chroots: 690 if chroot.name in chroot_names: 691 chroots.append(chroot) 692 693 build = cls.add( 694 user=user, 695 pkgs=pkgs, 696 copr=copr, 697 chroots=chroots, 698 source_type=source_type, 699 source_json=source_json, 700 enable_net=build_options.get("enable_net", copr.build_enable_net), 701 background=background, 702 git_hashes=git_hashes, 703 skip_import=skip_import, 704 batch=batch, 705 srpm_url=srpm_url, 706 copr_dirname=copr_dirname, 707 ) 708 709 if user.proven: 710 if "timeout" in build_options: 711 build.timeout = build_options["timeout"] 712 713 return build
714 715 @classmethod
716 - def add(cls, user, pkgs, copr, source_type=None, source_json=None, 717 repos=None, chroots=None, timeout=None, enable_net=True, 718 git_hashes=None, skip_import=False, background=False, batch=None, 719 srpm_url=None, copr_dirname=None):
720 721 if chroots is None: 722 chroots = [] 723 724 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action( 725 copr, "Can't build while there is an operation in progress: {action}") 726 users_logic.UsersLogic.raise_if_cant_build_in_copr( 727 user, copr, 728 "You don't have permissions to build in this copr.") 729 730 if not repos: 731 repos = copr.repos 732 733 # todo: eliminate pkgs and this check 734 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs): 735 raise MalformedArgumentException("Trying to create a build using src_pkg " 736 "with bad characters. Forgot to split?") 737 738 # just temporary to keep compatibility 739 if not source_type or not source_json: 740 source_type = helpers.BuildSourceEnum("link") 741 source_json = json.dumps({"url":pkgs}) 742 743 if skip_import and srpm_url: 744 chroot_status = StatusEnum("pending") 745 source_status = StatusEnum("succeeded") 746 else: 747 chroot_status = StatusEnum("waiting") 748 source_status = StatusEnum("pending") 749 750 copr_dir = None 751 if copr_dirname: 752 if not copr_dirname.startswith(copr.name+':') and copr_dirname != copr.name: 753 raise MalformedArgumentException("Copr dirname not starting with copr name.") 754 copr_dir = coprs_logic.CoprDirsLogic.get_or_create(copr, copr_dirname) 755 756 build = models.Build( 757 user=user, 758 pkgs=pkgs, 759 copr=copr, 760 repos=repos, 761 source_type=source_type, 762 source_json=source_json, 763 source_status=source_status, 764 submitted_on=int(time.time()), 765 enable_net=bool(enable_net), 766 is_background=bool(background), 767 batch=batch, 768 srpm_url=srpm_url, 769 copr_dir=copr_dir, 770 ) 771 772 if timeout: 773 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT 774 775 db.session.add(build) 776 777 for chroot in chroots: 778 # Chroots were explicitly set per-build. 779 git_hash = None 780 if git_hashes: 781 git_hash = git_hashes.get(chroot.name) 782 buildchroot = models.BuildChroot( 783 build=build, 784 status=chroot_status, 785 mock_chroot=chroot, 786 git_hash=git_hash, 787 ) 788 db.session.add(buildchroot) 789 790 return build
791 792 @classmethod
793 - def rebuild_package(cls, package, source_dict_update={}, copr_dir=None, update_callback=None, 794 scm_object_type=None, scm_object_id=None, 795 scm_object_url=None, submitted_by=None):
796 797 source_dict = package.source_json_dict 798 source_dict.update(source_dict_update) 799 source_json = json.dumps(source_dict) 800 801 if not copr_dir: 802 copr_dir = package.copr.main_dir 803 804 build = models.Build( 805 user=None, 806 pkgs=None, 807 package=package, 808 copr=package.copr, 809 repos=package.copr.repos, 810 source_status=StatusEnum("pending"), 811 source_type=package.source_type, 812 source_json=source_json, 813 submitted_on=int(time.time()), 814 enable_net=package.copr.build_enable_net, 815 timeout=DEFAULT_BUILD_TIMEOUT, 816 copr_dir=copr_dir, 817 update_callback=update_callback, 818 scm_object_type=scm_object_type, 819 scm_object_id=scm_object_id, 820 scm_object_url=scm_object_url, 821 submitted_by=submitted_by, 822 ) 823 db.session.add(build) 824 825 status = StatusEnum("waiting") 826 for chroot in package.chroots: 827 buildchroot = models.BuildChroot( 828 build=build, 829 status=status, 830 mock_chroot=chroot, 831 git_hash=None 832 ) 833 db.session.add(buildchroot) 834 835 cls.process_update_callback(build) 836 return build
837 838 839 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")} 840 841 @classmethod
842 - def get_buildchroots_by_build_id_and_branch(cls, build_id, branch):
843 """ 844 Returns a list of BuildChroots identified by build_id and dist-git 845 branch name. 846 """ 847 return ( 848 models.BuildChroot.query 849 .join(models.MockChroot) 850 .filter(models.BuildChroot.build_id==build_id) 851 .filter(models.MockChroot.distgit_branch_name==branch) 852 ).all()
853 854 855 @classmethod
856 - def delete_local_source(cls, build):
857 """ 858 Deletes the locally stored data for build purposes. This is typically 859 uploaded srpm file, uploaded spec file or webhook POST content. 860 """ 861 # is it hosted on the copr frontend? 862 data = json.loads(build.source_json) 863 if 'tmp' in data: 864 tmp = data["tmp"] 865 storage_path = app.config["STORAGE_DIR"] 866 try: 867 shutil.rmtree(os.path.join(storage_path, tmp)) 868 except: 869 pass
870 871 872 @classmethod
873 - def update_state_from_dict(cls, build, upd_dict):
874 """ 875 :param build: 876 :param upd_dict: 877 example: 878 { 879 "builds":[ 880 { 881 "id": 1, 882 "copr_id": 2, 883 "started_on": 1390866440 884 }, 885 { 886 "id": 2, 887 "copr_id": 1, 888 "status": 0, 889 "chroot": "fedora-18-x86_64", 890 "result_dir": "baz", 891 "ended_on": 1390866440 892 }] 893 } 894 """ 895 log.info("Updating build {} by: {}".format(build.id, upd_dict)) 896 897 # create the package if it doesn't exist 898 pkg_name = upd_dict.get('pkg_name', None) 899 if pkg_name: 900 if not PackagesLogic.get(build.copr_dir.id, pkg_name).first(): 901 try: 902 package = PackagesLogic.add( 903 build.copr.user, build.copr_dir, 904 pkg_name, build.source_type, build.source_json) 905 db.session.add(package) 906 db.session.commit() 907 except (IntegrityError, DuplicateException) as e: 908 app.logger.exception(e) 909 db.session.rollback() 910 return 911 build.package = PackagesLogic.get(build.copr_dir.id, pkg_name).first() 912 913 for attr in ["built_packages", "srpm_url", "pkg_version"]: 914 value = upd_dict.get(attr, None) 915 if value: 916 setattr(build, attr, value) 917 918 # update source build status 919 if str(upd_dict.get("task_id")) == str(build.task_id): 920 build.result_dir = upd_dict.get("result_dir", "") 921 922 new_status = upd_dict.get("status") 923 if new_status == StatusEnum("succeeded"): 924 new_status = StatusEnum("importing") 925 chroot_status=StatusEnum("waiting") 926 if not build.build_chroots: 927 # create the BuildChroots from Package setting, if not 928 # already set explicitly for concrete build 929 for chroot in build.package.chroots: 930 buildchroot = models.BuildChroot( 931 build=build, 932 status=chroot_status, 933 mock_chroot=chroot, 934 git_hash=None, 935 ) 936 db.session.add(buildchroot) 937 else: 938 for buildchroot in build.build_chroots: 939 buildchroot.status = chroot_status 940 db.session.add(buildchroot) 941 942 build.source_status = new_status 943 if new_status == StatusEnum("failed") or \ 944 new_status == StatusEnum("skipped"): 945 for ch in build.build_chroots: 946 ch.status = new_status 947 ch.ended_on = upd_dict.get("ended_on") or time.time() 948 ch.started_on = upd_dict.get("started_on", ch.ended_on) 949 db.session.add(ch) 950 951 if new_status == StatusEnum("failed"): 952 build.fail_type = FailTypeEnum("srpm_build_error") 953 954 cls.process_update_callback(build) 955 db.session.add(build) 956 return 957 958 if "chroot" in upd_dict: 959 # update respective chroot status 960 for build_chroot in build.build_chroots: 961 if build_chroot.name == upd_dict["chroot"]: 962 build_chroot.result_dir = upd_dict.get("result_dir", "") 963 964 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states: 965 build_chroot.status = upd_dict["status"] 966 967 if upd_dict.get("status") in BuildsLogic.terminal_states: 968 build_chroot.ended_on = upd_dict.get("ended_on") or time.time() 969 970 if upd_dict.get("status") == StatusEnum("starting"): 971 build_chroot.started_on = upd_dict.get("started_on") or time.time() 972 973 db.session.add(build_chroot) 974 975 # If the last package of a module was successfully built, 976 # then send an action to create module repodata on backend 977 if (build.module 978 and upd_dict.get("status") == StatusEnum("succeeded") 979 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)): 980 ActionsLogic.send_build_module(build.copr, build.module) 981 982 cls.process_update_callback(build) 983 db.session.add(build)
984 985 @classmethod
986 - def process_update_callback(cls, build):
987 parsed_git_url = helpers.get_parsed_git_url(build.copr.scm_repo_url) 988 if not parsed_git_url: 989 return 990 991 if build.update_callback == 'pagure_flag_pull_request': 992 api_url = 'https://{0}/api/0/{1}/pull-request/{2}/flag'.format( 993 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 994 return cls.pagure_flag(build, api_url) 995 996 elif build.update_callback == 'pagure_flag_commit': 997 api_url = 'https://{0}/api/0/{1}/c/{2}/flag'.format( 998 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 999 return cls.pagure_flag(build, api_url)
1000 1001 @classmethod
1002 - def pagure_flag(cls, build, api_url):
1003 headers = { 1004 'Authorization': 'token {}'.format(build.copr.scm_api_auth.get('api_key')) 1005 } 1006 1007 if build.srpm_url: 1008 progress = 50 1009 else: 1010 progress = 10 1011 1012 state_table = { 1013 'failed': ('failure', 0), 1014 'succeeded': ('success', 100), 1015 'canceled': ('canceled', 0), 1016 'running': ('pending', progress), 1017 'pending': ('pending', progress), 1018 'skipped': ('error', 0), 1019 'starting': ('pending', progress), 1020 'importing': ('pending', progress), 1021 'forked': ('error', 0), 1022 'waiting': ('pending', progress), 1023 'unknown': ('error', 0), 1024 } 1025 1026 build_url = os.path.join( 1027 app.config['PUBLIC_COPR_BASE_URL'], 1028 'coprs', build.copr.full_name.replace('@', 'g/'), 1029 'build', str(build.id) 1030 ) 1031 1032 data = { 1033 'username': 'Copr build', 1034 'comment': '#{}'.format(build.id), 1035 'url': build_url, 1036 'status': state_table[build.state][0], 1037 'percent': state_table[build.state][1], 1038 'uid': str(build.id), 1039 } 1040 1041 log.debug('Sending data to Pagure API: %s', pprint.pformat(data)) 1042 response = requests.post(api_url, data=data, headers=headers) 1043 log.debug('Pagure API response: %s', response.text)
1044 1045 @classmethod
1046 - def cancel_build(cls, user, build):
1047 if not user.can_build_in(build.copr): 1048 raise InsufficientRightsException( 1049 "You are not allowed to cancel this build.") 1050 if not build.cancelable: 1051 if build.status == StatusEnum("starting"): 1052 # this is not intuitive, that's why we provide more specific message 1053 err_msg = "Cannot cancel build {} in state 'starting'".format(build.id) 1054 else: 1055 err_msg = "Cannot cancel build {}".format(build.id) 1056 raise RequestCannotBeExecuted(err_msg) 1057 1058 if build.status == StatusEnum("running"): # otherwise the build is just in frontend 1059 ActionsLogic.send_cancel_build(build) 1060 1061 build.canceled = True 1062 cls.process_update_callback(build) 1063 1064 for chroot in build.build_chroots: 1065 chroot.status = 2 # canceled 1066 if chroot.ended_on is not None: 1067 chroot.ended_on = time.time()
1068 1069 @classmethod
1070 - def check_build_to_delete(cls, user, build):
1071 """ 1072 :type user: models.User 1073 :type build: models.Build 1074 """ 1075 if not user.can_edit(build.copr) or build.persistent: 1076 raise InsufficientRightsException( 1077 "You are not allowed to delete build `{}`.".format(build.id)) 1078 1079 if not build.finished: 1080 raise ActionInProgressException( 1081 "You can not delete build `{}` which is not finished.".format(build.id), 1082 "Unfinished build")
1083 1084 @classmethod
1085 - def delete_build(cls, user, build, send_delete_action=True):
1086 """ 1087 :type user: models.User 1088 :type build: models.Build 1089 """ 1090 cls.check_build_to_delete(user, build) 1091 1092 if send_delete_action: 1093 ActionsLogic.send_delete_build(build) 1094 1095 db.session.delete(build)
1096 1097 @classmethod
1098 - def delete_multiple_builds(cls, user, builds):
1099 """ 1100 :type user: models.User 1101 :type builds: list of models.Build 1102 """ 1103 to_delete = [] 1104 for build in builds: 1105 cls.check_build_to_delete(user, build) 1106 to_delete.append(build) 1107 1108 if to_delete: 1109 ActionsLogic.send_delete_multiple_builds(to_delete) 1110 1111 for build in to_delete: 1112 for build_chroot in build.build_chroots: 1113 db.session.delete(build_chroot) 1114 1115 db.session.delete(build)
1116 1117 @classmethod
1118 - def mark_as_failed(cls, build_id):
1119 """ 1120 Marks build as failed on all its non-finished chroots 1121 """ 1122 build = cls.get(build_id).one() 1123 chroots = filter(lambda x: x.status != StatusEnum("succeeded"), build.build_chroots) 1124 for chroot in chroots: 1125 chroot.status = StatusEnum("failed") 1126 if build.source_status != StatusEnum("succeeded"): 1127 build.source_status = StatusEnum("failed") 1128 cls.process_update_callback(build) 1129 return build
1130 1131 @classmethod
1132 - def last_modified(cls, copr):
1133 """ Get build datetime (as epoch) of last successful build 1134 1135 :arg copr: object of copr 1136 """ 1137 builds = cls.get_multiple_by_copr(copr) 1138 1139 last_build = ( 1140 builds.join(models.BuildChroot) 1141 .filter((models.BuildChroot.status == StatusEnum("succeeded")) 1142 | (models.BuildChroot.status == StatusEnum("skipped"))) 1143 .filter(models.BuildChroot.ended_on.isnot(None)) 1144 .order_by(models.BuildChroot.ended_on.desc()) 1145 ).first() 1146 if last_build: 1147 return last_build.ended_on 1148 else: 1149 return None
1150 1151 @classmethod
1152 - def filter_is_finished(cls, query, is_finished):
1153 # todo: check that ended_on is set correctly for all cases 1154 # e.g.: failed dist-git import, cancellation 1155 if is_finished: 1156 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.isnot(None)) 1157 else: 1158 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.is_(None))
1159 1160 @classmethod
1161 - def filter_by_group_name(cls, query, group_name):
1162 return query.filter(models.Group.name == group_name)
1163 1164 @classmethod
1165 - def filter_by_package_name(cls, query, package_name):
1166 return query.join(models.Package).filter(models.Package.name == package_name)
1167 1168 @classmethod
1169 - def clean_old_builds(cls):
1170 dirs = ( 1171 db.session.query( 1172 models.CoprDir.id, 1173 models.Package.id, 1174 models.Package.max_builds) 1175 .join(models.Build, models.Build.copr_dir_id==models.CoprDir.id) 1176 .join(models.Package) 1177 .filter(models.Package.max_builds > 0) 1178 .group_by( 1179 models.CoprDir.id, 1180 models.Package.max_builds, 1181 models.Package.id) 1182 .having(func.count(models.Build.id) > models.Package.max_builds) 1183 ) 1184 1185 for dir_id, package_id, limit in dirs.all(): 1186 delete_builds = ( 1187 models.Build.query.filter( 1188 models.Build.copr_dir_id==dir_id, 1189 models.Build.package_id==package_id) 1190 .order_by(desc(models.Build.id)) 1191 .offset(limit) 1192 .all() 1193 ) 1194 1195 for build in delete_builds: 1196 try: 1197 cls.delete_build(build.copr.user, build) 1198 except ActionInProgressException: 1199 # postpone this one to next day run 1200 log.error("Build(id={}) delete failed, unfinished action.".format(build.id))
1201 1202 @classmethod
1203 - def delete_orphaned_builds(cls):
1204 builds_to_delete = models.Build.query\ 1205 .join(models.Copr, models.Build.copr_id == models.Copr.id)\ 1206 .filter(models.Copr.deleted == True) 1207 1208 counter = 0 1209 for build in builds_to_delete: 1210 cls.delete_build(build.copr.user, build) 1211 counter += 1 1212 if counter >= 100: 1213 db.session.commit() 1214 counter = 0 1215 1216 if counter > 0: 1217 db.session.commit()
1218
1219 1220 -class BuildChrootsLogic(object):
1221 @classmethod
1222 - def get_by_build_id_and_name(cls, build_id, name):
1223 mc = MockChrootsLogic.get_from_name(name).one() 1224 1225 return ( 1226 BuildChroot.query 1227 .filter(BuildChroot.build_id == build_id) 1228 .filter(BuildChroot.mock_chroot_id == mc.id) 1229 )
1230 1231 @classmethod
1232 - def get_multiply(cls):
1233 query = ( 1234 models.BuildChroot.query 1235 .join(models.BuildChroot.build) 1236 .join(models.BuildChroot.mock_chroot) 1237 .join(models.Build.copr) 1238 .join(models.Copr.user) 1239 .outerjoin(models.Group) 1240 ) 1241 return query
1242 1243 @classmethod
1244 - def filter_by_build_id(cls, query, build_id):
1245 return query.filter(models.Build.id == build_id)
1246 1247 @classmethod
1248 - def filter_by_project_id(cls, query, project_id):
1249 return query.filter(models.Copr.id == project_id)
1250 1251 @classmethod
1252 - def filter_by_project_user_name(cls, query, username):
1253 return query.filter(models.User.username == username)
1254 1255 @classmethod
1256 - def filter_by_state(cls, query, state):
1257 return query.filter(models.BuildChroot.status == StatusEnum(state))
1258 1259 @classmethod
1260 - def filter_by_group_name(cls, query, group_name):
1261 return query.filter(models.Group.name == group_name)
1262
1263 1264 -class BuildsMonitorLogic(object):
1265 @classmethod
1266 - def get_monitor_data(cls, copr):
1267 query = """ 1268 SELECT 1269 package.id as package_id, 1270 package.name AS package_name, 1271 build.id AS build_id, 1272 build_chroot.status AS build_chroot_status, 1273 build.pkg_version AS build_pkg_version, 1274 mock_chroot.id AS mock_chroot_id, 1275 mock_chroot.os_release AS mock_chroot_os_release, 1276 mock_chroot.os_version AS mock_chroot_os_version, 1277 mock_chroot.arch AS mock_chroot_arch 1278 FROM package 1279 JOIN (SELECT 1280 MAX(build.id) AS max_build_id_for_chroot, 1281 build.package_id AS package_id, 1282 build_chroot.mock_chroot_id AS mock_chroot_id 1283 FROM build 1284 JOIN build_chroot 1285 ON build.id = build_chroot.build_id 1286 WHERE build.copr_id = {copr_id} 1287 AND build_chroot.status != 2 1288 GROUP BY build.package_id, 1289 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot 1290 ON package.id = max_build_ids_for_a_chroot.package_id 1291 JOIN build 1292 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1293 JOIN build_chroot 1294 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id 1295 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1296 JOIN mock_chroot 1297 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id 1298 JOIN copr_dir ON build.copr_dir_id=copr_dir.id WHERE copr_dir.main IS TRUE 1299 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC 1300 """.format(copr_id=copr.id) 1301 rows = db.session.execute(query) 1302 return rows
1303