1 import tempfile
2 import shutil
3 import json
4 import os
5 import pprint
6 import time
7 import flask
8 import sqlite3
9 from sqlalchemy.sql import text
10 from sqlalchemy import or_
11 from sqlalchemy import and_
12 from sqlalchemy.orm import joinedload
13 from sqlalchemy.orm.exc import NoResultFound
14 from sqlalchemy.sql import false,true
15 from werkzeug.utils import secure_filename
16 from sqlalchemy import desc,asc, bindparam, Integer
17 from collections import defaultdict
18
19 from coprs import app
20 from coprs import db
21 from coprs import exceptions
22 from coprs import models
23 from coprs import helpers
24 from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT, DEFER_BUILD_SECONDS
25 from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException
26 from coprs.helpers import StatusEnum
27
28 from coprs.logic import coprs_logic
29 from coprs.logic import users_logic
30 from coprs.logic.actions_logic import ActionsLogic
31 from coprs.models import BuildChroot,Build,Package,MockChroot
32 from .coprs_logic import MockChrootsLogic
33
34 log = app.logger
115
116 @classmethod
134
135 @classmethod
138
139 @classmethod
144
145 @classmethod
152
153 @classmethod
155 query_select = """
156 SELECT build.id, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on,
157 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status,
158 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name
159 FROM build
160 LEFT OUTER JOIN package
161 ON build.package_id = package.id
162 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses
163 ON statuses.build_id=build.id
164 LEFT OUTER JOIN copr
165 ON copr.id = build.copr_id
166 LEFT OUTER JOIN "user"
167 ON copr.user_id = "user".id
168 LEFT OUTER JOIN "group"
169 ON copr.group_id = "group".id
170 WHERE build.copr_id = :copr_id
171 GROUP BY
172 build.id;
173 """
174
175 if db.engine.url.drivername == "sqlite":
176 def sqlite_status_to_order(x):
177 if x == 0:
178 return 0
179 elif x == 3:
180 return 1
181 elif x == 6:
182 return 2
183 elif x == 7:
184 return 3
185 elif x == 4:
186 return 4
187 elif x == 1:
188 return 5
189 elif x == 5:
190 return 6
191 return 1000
192
193 def sqlite_order_to_status(x):
194 if x == 0:
195 return 0
196 elif x == 1:
197 return 3
198 elif x == 2:
199 return 6
200 elif x == 3:
201 return 7
202 elif x == 4:
203 return 4
204 elif x == 5:
205 return 1
206 elif x == 6:
207 return 5
208 return 1000
209
210 conn = db.engine.connect()
211 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order)
212 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status)
213 statement = text(query_select)
214 statement.bindparams(bindparam("copr_id", Integer))
215 result = conn.execute(statement, {"copr_id": copr.id})
216 else:
217 statement = text(query_select)
218 statement.bindparams(bindparam("copr_id", Integer))
219 result = db.engine.execute(statement, {"copr_id": copr.id})
220
221 return result
222
223 @classmethod
226
227 @classmethod
235
236 @classmethod
252
253 @classmethod
272
273 @classmethod
276
277 @classmethod
280
281 @classmethod
307
308 @classmethod
322
323 @classmethod
324 - def create_new_from_tito(cls, user, copr, git_url, git_dir, git_branch, tito_test,
325 chroot_names=None, **build_options):
340
341 @classmethod
342 - def create_new_from_mock(cls, user, copr, scm_type, scm_url, scm_branch, spec,
343 chroot_names=None, **build_options):
358
359 @classmethod
360 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, python_versions,
361 chroot_names=None, **build_options):
378
379 @classmethod
392
393 @classmethod
394 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename,
395 chroot_names=None, **build_options):
396 """
397 :type user: models.User
398 :type copr: models.Copr
399 :param f_uploader(file_path): function which stores data at the given `file_path`
400 :return:
401 """
402 tmp = tempfile.mkdtemp(dir=app.config["SRPM_STORAGE_DIR"])
403 tmp_name = os.path.basename(tmp)
404 filename = secure_filename(orig_filename)
405 file_path = os.path.join(tmp, filename)
406 f_uploader(file_path)
407
408
409 pkg_url = "https://{hostname}/tmp/{tmp_dir}/{srpm}".format(
410 hostname=app.config["PUBLIC_COPR_HOSTNAME"],
411 tmp_dir=tmp_name,
412 srpm=filename)
413
414
415 source_type = helpers.BuildSourceEnum("srpm_upload")
416 source_json = json.dumps({"tmp": tmp_name, "pkg": filename})
417 try:
418 build = cls.create_new(user, copr, source_type, source_json,
419 chroot_names, pkgs=pkg_url, **build_options)
420 except Exception:
421 shutil.rmtree(tmp)
422 raise
423
424 return build
425
426 @classmethod
427 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None,
428 pkgs="", git_hashes=None, skip_import=False, background=False, **build_options):
429 """
430 :type user: models.User
431 :type copr: models.Copr
432 :type chroot_names: List[str]
433 :type source_type: int value from helpers.BuildSourceEnum
434 :type source_json: str in json format
435 :type pkgs: str
436 :type git_hashes: dict
437 :type skip_import: bool
438 :type background: bool
439 :rtype: models.Build
440 """
441 if chroot_names is None:
442 chroots = [c for c in copr.active_chroots]
443 else:
444 chroots = []
445 for chroot in copr.active_chroots:
446 if chroot.name in chroot_names:
447 chroots.append(chroot)
448
449 build = cls.add(
450 user=user,
451 pkgs=pkgs,
452 copr=copr,
453 chroots=chroots,
454 source_type=source_type,
455 source_json=source_json,
456 enable_net=build_options.get("enable_net", copr.build_enable_net),
457 background=background,
458 git_hashes=git_hashes,
459 skip_import=skip_import)
460
461 if user.proven:
462 if "timeout" in build_options:
463 build.timeout = build_options["timeout"]
464
465 return build
466
467 @classmethod
468 - def add(cls, user, pkgs, copr, source_type=None, source_json=None,
469 repos=None, chroots=None, timeout=None, enable_net=True,
470 git_hashes=None, skip_import=False, background=False):
471 if chroots is None:
472 chroots = []
473
474 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(
475 copr, "Can't build while there is an operation in progress: {action}")
476 users_logic.UsersLogic.raise_if_cant_build_in_copr(
477 user, copr,
478 "You don't have permissions to build in this copr.")
479
480 if not repos:
481 repos = copr.repos
482
483
484 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs):
485 raise exceptions.MalformedArgumentException("Trying to create a build using src_pkg "
486 "with bad characters. Forgot to split?")
487
488
489 if not source_type or not source_json:
490 source_type = helpers.BuildSourceEnum("srpm_link")
491 source_json = json.dumps({"url":pkgs})
492
493 build = models.Build(
494 user=user,
495 pkgs=pkgs,
496 copr=copr,
497 repos=repos,
498 source_type=source_type,
499 source_json=source_json,
500 submitted_on=int(time.time()),
501 enable_net=bool(enable_net),
502 is_background=bool(background),
503 )
504
505 if timeout:
506 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT
507
508 db.session.add(build)
509
510
511
512 if not chroots:
513 chroots = copr.active_chroots
514
515 status = helpers.StatusEnum("importing")
516
517 if skip_import:
518 status = StatusEnum("pending")
519
520 for chroot in chroots:
521 git_hash = None
522 if git_hashes:
523 git_hash = git_hashes.get(chroot.name)
524 buildchroot = models.BuildChroot(
525 build=build,
526 status=status,
527 mock_chroot=chroot,
528 git_hash=git_hash)
529
530 db.session.add(buildchroot)
531
532 return build
533
534 @classmethod
536 build = models.Build(
537 user=None,
538 pkgs=None,
539 package_id=package.id,
540 copr=package.copr,
541 repos=package.copr.repos,
542 source_type=package.source_type,
543 source_json=package.source_json,
544 submitted_on=int(time.time()),
545 enable_net=package.enable_net,
546 timeout=DEFAULT_BUILD_TIMEOUT
547 )
548
549 db.session.add(build)
550
551 chroots = package.copr.active_chroots
552
553 status = helpers.StatusEnum("importing")
554
555 for chroot in chroots:
556 buildchroot = models.BuildChroot(
557 build=build,
558 status=status,
559 mock_chroot=chroot,
560 git_hash=None
561 )
562
563 db.session.add(buildchroot)
564
565 return build
566
567
568 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")}
569
570 @classmethod
572 """
573 Returns a list of BuildChroots identified with task_id
574 task_id consists of a name of git branch + build id
575 Example: 42-f22 -> build id 42, chroots fedora-22-*
576 """
577 build_id, branch = task_id.split("-")
578 build = cls.get_by_id(build_id).one()
579 build_chroots = build.build_chroots
580 os, version = helpers.branch_to_os_version(branch)
581 chroot_halfname = "{}-{}".format(os, version)
582 matching = [ch for ch in build_chroots if chroot_halfname in ch.name]
583 return matching
584
585
586 @classmethod
600
601
602 @classmethod
604 """
605 :param build:
606 :param upd_dict:
607 example:
608 {
609 "builds":[
610 {
611 "id": 1,
612 "copr_id": 2,
613 "started_on": 139086644000
614 },
615 {
616 "id": 2,
617 "copr_id": 1,
618 "status": 0,
619 "chroot": "fedora-18-x86_64",
620 "results": "http://server/results/foo/bar/",
621 "ended_on": 139086644000
622 }]
623 }
624 """
625 log.info("Updating build: {} by: {}".format(build.id, upd_dict))
626 if "chroot" in upd_dict:
627
628 for build_chroot in build.build_chroots:
629 if build_chroot.name == upd_dict["chroot"]:
630
631 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states:
632 build_chroot.status = upd_dict["status"]
633
634 if upd_dict.get("status") in BuildsLogic.terminal_states:
635 build_chroot.ended_on = upd_dict.get("ended_on") or time.time()
636
637 if upd_dict.get("status") == StatusEnum("starting"):
638 build_chroot.started_on = upd_dict.get("started_on") or time.time()
639
640 if "last_deferred" in upd_dict:
641 build_chroot.last_deferred = upd_dict["last_deferred"]
642
643 db.session.add(build_chroot)
644
645 for attr in ["results", "built_packages"]:
646 value = upd_dict.get(attr, None)
647 if value:
648 setattr(build, attr, value)
649
650 db.session.add(build)
651
652 @classmethod
665
666 @classmethod
667 - def delete_build(cls, user, build, send_delete_action=True):
688
689 @classmethod
699
700 @classmethod
719
720 @classmethod
728
729 @classmethod
732
735 @classmethod
744
745 @classmethod
756
757 @classmethod
760
761 @classmethod
764
765 @classmethod
768
769 @classmethod
772
773 @classmethod
776
779 @classmethod
781 query = """
782 SELECT
783 package.id as package_id,
784 package.name AS package_name,
785 build.id AS build_id,
786 build_chroot.status AS build_chroot_status,
787 mock_chroot.id AS mock_chroot_id
788 FROM package
789 JOIN (SELECT
790 MAX(build.id) AS max_build_id_for_chroot,
791 build.package_id AS package_id,
792 build_chroot.mock_chroot_id AS mock_chroot_id
793 FROM build
794 JOIN build_chroot
795 ON build.id = build_chroot.build_id
796 WHERE build.copr_id = {copr_id}
797 AND build_chroot.status != 2
798 GROUP BY build.package_id,
799 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot
800 ON package.id = max_build_ids_for_a_chroot.package_id
801 JOIN build
802 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot
803 JOIN build_chroot
804 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id
805 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot
806 JOIN mock_chroot
807 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id
808 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC
809 """.format(copr_id=copr.id)
810 rows = db.session.execute(query)
811 return rows
812