From 8471e2b4e361ab15c6a199df6e73241249da8c6c Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Aug 16 2019 11:15:51 +0000 Subject: Add Python 3 support, move to AMQP, refactor I can break this out into a bunch of commits if folks care, but this adds support for Python 3 (and drops Python 2 support), moves from fedmsg to fedora-messaging, and adds a license. Signed-off-by: Jeremy Cline --- diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/README b/README index 09fe934..b7a467c 100644 --- a/README +++ b/README @@ -9,6 +9,17 @@ https://git.kernel.org/cgit/linux/kernel/git/jwboyer/fedora.git/ Below are some details on how it works, some of the assumptions made, and a big list of things that need to be improved. +## Quick Start + +The script depends on fedpkg, fedora-messaging, python3-koji, and +python3-GitPython. To run the message consumer, run: + +$ fedora-messaging --conf fedkernel.toml.example consume --callback-file kernel-git.py:callback + +The script supports providing a Koji build ID to manually create the tree: + +$ ./kernel-git.py 1349425 + ## How it works. Fedora uses git for its package SCM. This means that for every build in diff --git a/exp_tree.py b/exp_tree.py deleted file mode 100644 index 1e749d5..0000000 --- a/exp_tree.py +++ /dev/null @@ -1,131 +0,0 @@ -#! /usr/bin/env/python - -# Copyright 2015 Red Hat Inc. -# Author(s): Josh Boyer -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; version 2 of the License. -# See http://www.gnu.org/copyleft/gpl.html for the full text of the license. - -import os -import string -import sys -import shutil - -from git import Repo - -from parse_spec import * - -def read_gitrev(dr): - f = None - sha = None - - try: - f = open(dr + '/gitrev', "r") - sha = f.readline() - # Make sure to get rid of the newline. Sigh, so tedious. - sha = sha.strip() - finally: - f and f.close() - - return sha - -def get_base_tag(specv): - if specv['released_kernel'] == '0': - if specv['rcrev'] == '0': - # This should be a merge window kernel, which means gitrev - # shouldn't be 0. Since we only call this if gitrev is 0, we have - # a problem - sys.exit(1) - rc = specv['rcrev'] - # Trivia time: If we're doing a major new release, we use RC - # *tarballs*, not patches. So here, we have an unreleased - # kernel that has special rules. We look to see if parse_spec - # returned a tar_suffix value. If it did, we don't do the - # bump-by-one dance like we would if we're using a prior - # release tarball and then applying an RC patch on top of it. - # - # This is all a house of cards. - if specv['tar_suffix']: - base = '%s' % specv['base_sublevel'] - else: - base = '%s' % (int(specv['base_sublevel']) + 1) - major = specv['major_version'] - tag = 'v%s.%s-rc%s' % (major, base, rc) - else: - if specv['stable_update'] != '0': - tag = 'v%s.%s.%s' % (specv['major_version'], specv['base_sublevel'], specv['stable_update']) - else: - tag = 'v%s.%s' % (specv['major_version'], specv['base_sublevel']) - return tag - -def get_base_commit(pkgdir, specv): - - commit = None - - if specv['gitrev'] == '0': - commit = get_base_tag(specv) - else: - if specv['released_kernel'] != '0': -# commit = get_base_tag(specv) -# return commit - sys.exit(1) - - # read the file that contains the gitrev commit sha - commit = read_gitrev(pkgdir) - - if commit is None: - sys.exit(1) - - return commit - -def get_work_dir(specv, tag): - - dist = re.split('(fc\d+)', tag)[1] - maindir = "kernel-%s.%s" % (specv['major_version'], specv['base_sublevel']) - # If we have a tar_suffix (e.g. kernel-5.0-rc1) we need to use that for - # the main working dir. This is a result of using RC tarballs instead - # of prior release tarballs+rc patches. - if specv['tar_suffix']: - maindir = maindir + specv['tar_suffix'] - maindir = maindir + ".%s" % dist - - # Using uname here is probably hacky, particularly since we tell fedpkg - # that arch is noarch, but that isn't how the kernel works. It always - # preps for the local arch, so if this is ever different then something - # weird happened. - lindir = re.sub('kernel-', 'linux-', tag) + ".%s" % os.uname()[4] - - return maindir + "/" + lindir + "/" - -def prep_exp_tree(pkgdir, lindr, branch, specv): - - lin = Repo(lindr) - lingit = lin.git - - lingit.remote('update') - - if branch == "master": - branch = "rawhide" - - lingit.checkout(branch) - - sha = get_base_commit(pkgdir, specv) - print sha - lingit.reset('--hard', sha) - - return lingit - -def build_exp_tree(lingit, patch, tag, configdir, lindir): - - #Apply the patches - lingit.am(patch.name) - - # Now add the Fedora config files - shutil.copytree(configdir, lindir + '/fedora/configs') - lingit.add('fedora/configs') - log = ('%s configs' % tag) - lingit.commit('-a', '-m', log) - - lingit.tag(tag) diff --git a/fedkernel.toml.example b/fedkernel.toml.example new file mode 100644 index 0000000..d848bd3 --- /dev/null +++ b/fedkernel.toml.example @@ -0,0 +1,83 @@ +# A basic configuration for Fedora's message broker, using the example callback +# which simply prints messages to standard output. +# +# This file is in the TOML format. +amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" + +[tls] +ca_cert = "/etc/fedora-messaging/cacert.pem" +keyfile = "/etc/fedora-messaging/fedora-key.pem" +certfile = "/etc/fedora-messaging/fedora-cert.pem" + +[client_properties] +app = "Fedora Kernel Exploded Source Tree" +app_url = "https://pagure.io/fedkernel/" +app_contacts_email = ["jwboyer@fedoraproject.org"] + +[exchanges."amq.topic"] +type = "topic" +durable = true +auto_delete = false +arguments = {} + +# Queue names *must* be in the normal UUID format: run "uuidgen" and use the +# output as your queue name. If your queue is not exclusive, anyone can connect +# and consume from it, causing you to miss messages, so do not share your queue +# name. Any queues that are not auto-deleted on disconnect are garbage-collected +# after approximately one hour. +# +# If you require a stronger guarantee about delivery, please talk to Fedora's +# Infrastructure team. +[queues.e4398d65-fb11-4480-9a82-5cba83cfc9b1] +durable = true +auto_delete = false +exclusive = false +arguments = {} + +[[bindings]] +queue = "e4398d65-fb11-4480-9a82-5cba83cfc9b1" +exchange = "amq.topic" +routing_keys = ["org.fedoraproject.*.buildsys.build.state.change"] + +[consumer_config] +example_key = "for my consumer" + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stderr" + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.fedkernel] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.twisted] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console"] + +[log_config.root] +level = "WARNING" +handlers = ["console"] diff --git a/kernel-git.py b/kernel-git.py index e14cefd..2e7d6ea 100755 --- a/kernel-git.py +++ b/kernel-git.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2015 Red Hat Inc. # Author(s): Josh Boyer @@ -8,103 +8,266 @@ # Free Software Foundation; version 2 of the License. # See http://www.gnu.org/copyleft/gpl.html for the full text of the license. +from urllib.parse import urlparse +import argparse +import configparser +import logging import os -import string -import time -import thread +import re +import shutil import sys import tempfile -from koji_cli_local import * - -import logging -import fedmsg -import fedmsg.config -import fedmsg.meta - -import pyrpkg -import fedpkg - from git import Repo +import fedpkg +import koji -from parse_spec import * -from exp_tree import * +_log = logging.getLogger("fedkernel") pkg_git_dir = '/home/jwboyer/tmp/kernel' linux_git_dir = '/home/jwboyer/tmp/linux' -def check_pkg(msg): - buildinfo = None - if msg['instance'] == "primary": - if msg['new'] == 1: - if msg['name'] == "kernel": - print(msg['name']) - print(msg['build_id']) - buildinfo = get_build_info(msg['build_id']) - - return buildinfo - - -def lookup_branch(branch): - branches = { - 'f21': 'f21', - 'f22': 'f22', - 'f23': 'f23', - 'f24': 'f24', - 'f25': 'f25', - 'f26': 'f26', - 'f27': 'f27', - 'f28': 'f28', - 'f29': 'f29', - 'f30': 'f30', - 'f31': 'master' - } +KOJI_OPTIONS = { + "krb_rdns": False, + "max_retries": 10, + "retry_internval": 10, + "offline_retry": True, + "offline_retry_interval": 10, + "anon_retry": True, +} +KERBEROS_OPTIONS = { + "ccache": None, + "principal": None, + "keytab": None, +} +HUB = "https://koji.fedoraproject.org/kojihub" + +BRANCHES = { + 'f21': 'f21', + 'f22': 'f22', + 'f23': 'f23', + 'f24': 'f24', + 'f25': 'f25', + 'f26': 'f26', + 'f27': 'f27', + 'f28': 'f28', + 'f29': 'f29', + 'f30': 'f30', + 'f31': 'f31', + 'f32': 'master', +} + + +def get_base_tag(specv): + if specv['released_kernel'] == '0': + if specv['rcrev'] == '0': + # This should be a merge window kernel, which means gitrev + # shouldn't be 0. Since we only call this if gitrev is 0, we have + # a problem + sys.exit(1) + rc = specv['rcrev'] + # Trivia time: If we're doing a major new release, we use RC + # *tarballs*, not patches. So here, we have an unreleased + # kernel that has special rules. We look to see if parse_spec + # returned a tar_suffix value. If it did, we don't do the + # bump-by-one dance like we would if we're using a prior + # release tarball and then applying an RC patch on top of it. + # + # This is all a house of cards. + if specv['tar_suffix']: + base = '%s' % specv['base_sublevel'] + else: + base = '%s' % (int(specv['base_sublevel']) + 1) + major = specv['major_version'] + tag = 'v%s.%s-rc%s' % (major, base, rc) + else: + if specv['stable_update'] != '0': + tag = 'v{}.{}.{}'.format( + specv['major_version'], specv['base_sublevel'], specv['stable_update']) + else: + tag = 'v%s.%s' % (specv['major_version'], specv['base_sublevel']) + return tag - try: - b = branches[branch] - except: - b = None - return b +def get_base_commit(pkgdir, specv): -def prep_pkg_git(): + commit = None - fedcfg = ConfigParser.SafeConfigParser() - fedcfg.read('/etc/rpkg/fedpkg.conf') + if specv['gitrev'] == '0': + commit = get_base_tag(specv) + else: + if specv['released_kernel'] != '0': + sys.exit(1) - fedcli = fedpkg.cli.fedpkgClient(fedcfg, name='fedpkg') - fedcli.do_imports(site='fedpkg') + # read the file that contains the gitrev commit sha + with open(os.path.join(pkgdir, "gitrev"), "r") as fd: + commit = fd.readline().strip() - fedcli.args = fedcli.parser.parse_args(['prep']) - fedcli.args.path = pkg_git_dir + if commit is None: + sys.exit(1) - fedcli.args.arch = None - fedcli.args.builddir = None + return commit - fedcli.args.command() -def create_tree(info): +def get_work_dir(specv, tag): + + dist = re.split(r'(fc\d+)', tag)[1] + maindir = "kernel-%s.%s" % (specv['major_version'], specv['base_sublevel']) + # If we have a tar_suffix (e.g. kernel-5.0-rc1) we need to use that for + # the main working dir. This is a result of using RC tarballs instead + # of prior release tarballs+rc patches. + if specv['tar_suffix']: + maindir = maindir + specv['tar_suffix'] + maindir = maindir + ".%s" % dist + + # Using uname here is probably hacky, particularly since we tell fedpkg + # that arch is noarch, but that isn't how the kernel works. It always + # preps for the local arch, so if this is ever different then something + # weird happened. + lindir = re.sub('kernel-', 'linux-', tag) + ".%s" % os.uname()[4] + + return maindir + "/" + lindir + "/" + + +def prep_exp_tree(pkgdir, lindr, branch, specv): + + lin = Repo(lindr) + lingit = lin.git + + lingit.remote('update') + + if branch == "master": + branch = "rawhide" + + lingit.checkout(branch) + + sha = get_base_commit(pkgdir, specv) + print(sha) + lingit.reset('--hard', sha) + + return lingit + + +def build_exp_tree(lingit, patch, tag, configdir, lindir): + # Apply the patches + lingit.am(patch.name) + + # Now add the Fedora config files + shutil.copytree(configdir, lindir + '/fedora/configs') + lingit.add('fedora/configs') + log = ('%s configs' % tag) + lingit.commit('-a', '-m', log) + + lingit.tag(tag) + + +def parse_spec(specf): + + with open(specf, "r") as fd: + spec = fd.readlines() + specv = {} + + if spec is None: + sys.exit(0) + + for i in range(len(spec)): + match = re.match( + r"^%(global|define)\s+(?P\w+)\s+(?P\d+.*)", spec[i]) + if match is None: + continue + + var = match.group('var') + val = match.group('val') + val = val.strip() + if var == "released_kernel": + specv['released_kernel'] = val + continue + elif var == "base_sublevel": + specv['base_sublevel'] = val + continue + elif var == "rcrev": + specv['rcrev'] = val + continue + elif var == "gitrev": + specv['gitrev'] = val + continue + elif var == "stable_update": + specv['stable_update'] = val + continue + elif var == "kversion": + specv['kversion'] = val + specv['major_version'] = val.split(".")[0] + bar = val.split("-rc") + if len(bar) > 1: + specv['tar_suffix'] = "-rc%s" % specv['rcrev'] + else: + specv['tar_suffix'] = None + continue + else: + continue + + print(specv['released_kernel']) + print(specv['major_version']) + print(specv['base_sublevel']) + print(specv['rcrev']) + print(specv['gitrev']) + print(specv['stable_update']) + + return specv + + +def get_build_info(build_id, nvr=False): + """Get the dist-git commit and NVR when given a Koji build ID or NVR.""" + session = koji.ClientSession(HUB, KOJI_OPTIONS) + if not session.krb_login(**KERBEROS_OPTIONS): + raise Exception("Failed to log into Koji") + + + if nvr is True: + info = session.getBuild(build_id) + else: + info = session.getBuild(int(build_id)) + + if info is None: + raise ValueError("No such build: {}".format(build_id)) + + task = None + if info['task_id']: + task = session.getTaskInfo(info['task_id'], request=True) - print("Creating tree for pkg-git commit %s with tag %s" % info) - sha = info[0] - tag = info[1] + nvrtag = info["nvr"] + print(nvrtag) + + if task is None: + return None + + tasklabel = koji.taskLabel(task) + print(tasklabel) + + sha = urlparse(info["source"]).fragment + print(sha) + return (sha, nvrtag) + + +def create_tree(sha, tag): + _log.info("Creating tree for pkg-git commit %s with tag %s", sha, tag) dist = re.split(r'.*fc(\d+)', tag) - if len(dist) is 1: - print('Not a fedora build. Skipping') + if len(dist) == 1: + _log.info('Not a fedora build. Skipping') return b = 'f' + re.split(r'.*fc(\d+)', tag)[1] - branch = lookup_branch(b) - if branch is None: - print('Could not create tree for branch %s' % b) + try: + branch = BRANCHES[b] + except KeyError: + _log.error('Could not create tree for branch %s', b) + return + if branch == 'f22': + _log.warning('f22 build. f22 is old and special. Skipping.') return else: - if branch is 'f22': - print('f22 build. f22 is old and special. Skipping.') - return - else: - print('sha %s tag %s branch %s' % (sha, tag, branch)) + _log.info('sha %s tag %s branch %s', sha, tag, branch) # Get the package git repo and prep the tree from the commit that # corresponds to this build @@ -115,9 +278,21 @@ def create_tree(info): pkg_git.checkout(branch) pkg_git.reset('--hard', '%s' % sha) - prep_pkg_git() + fedcfg = configparser.SafeConfigParser() + fedcfg.read('/etc/rpkg/fedpkg.conf') - specv = parse_spec("%s/kernel.spec" % pkg_git_dir) + fedcli = fedpkg.cli.fedpkgClient(fedcfg, name='fedpkg') + fedcli.do_imports(site='fedpkg') + + fedcli.args = fedcli.parser.parse_args(['prep']) + fedcli.args.path = pkg_git_dir + + fedcli.args.arch = None + fedcli.args.builddir = None + + fedcli.args.command() + + specv = parse_spec(os.path.join(pkg_git_dir, "kernel.spec")) # Get the working directory for this pkg git tree and the git repo # that it contains @@ -160,39 +335,77 @@ def create_tree(info): for line in patch: temp.write(line.encode("utf-8")) - print(temp.name) + _log.info(temp.name) temp.flush() lingit = prep_exp_tree(pkg_git_dir, linux_git_dir, branch, specv) build_exp_tree(lingit, temp, tag, wdir + 'configs', linux_git_dir) - print("Created exploded tree for %s" % tag) - -if __name__ == '__main__': - - if len(sys.argv) != 1: - info = get_build_info(sys.argv[1], True) - else: - logging.basicConfig() - - config = fedmsg.config.load_config([], None) - fedmsg.meta.make_processors(**config) + _log.info("Created exploded tree for %s", tag) + + +def callback(message): + """ + The message callback for Koji builds. + + This can be run with:: + + $ fedora-messaging consume --callback=fedkernel.kernel_git:callback + + The fedora-messaging configuration should set up a queue with bindings + to the "org.fedoraproject.*.buildsys.build.state.change" topic. + + Message bodies are expected to look like: + + { + "attribute": "state", + "build_id": 1355787, + "epoch": null, + "instance": "primary", + "name": "kernel", + "new": 1, + "old": 0, + "owner": "labbott", + "release": "0.rc4.git1.1.fc32", + "request": [ + "git+https://src.fedoraproject.org/rpms/kernel.git#9a56544597fb8266578104c842002dec3a5fd483", + "rawhide", + {} + ], + "task_id": 37038078, + "version": "5.3.0" + } + + Args: + message (fedora_messaging.api.Message): The AMQP message. + """ + # This is completed undocumented in Koji and when it breaks it's not my + # fault. There's a "new" field, and if it's 0 it apparently means the build + # started, and if it's 1 that means the build completed. We're only + # interested in completed kernel builds. + if message.body.get("new", 0) != 1 or message.body.get("name") != "kernel": + return - for name, endpoint, topic, msg in fedmsg.tail_messages(**config): - info = None - if "buildsys.build.state.change" in topic: - info = check_pkg(msg['msg']) + try: + request_url = message.body["request"][0] + except KeyError: + _log.error("Koji message format has changed (no 'request' key found); " + "dropping %r", message) + return - if info is None: - continue - else: - create_tree(info) + dist_git_sha = urlparse(request_url).fragment + nvrtag = "{}-{}-{}".format( + message.body["name"], message.body["version"], message.body["release"]) + create_tree(dist_git_sha, nvrtag) - create_tree(info) -# if "kernel" in msg['msg']['name']: -# if msg['msg']['instance'] == "primary": -# if msg['msg']['new'] == 1: -# print msg['msg']['name'] -# print msg['msg']['build_id'] +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Produce a Fedora source tree from builds in Koji') + parser.add_argument( + "buildid", help="The Koji build ID or NVR to produce a tree for.") + args = parser.parse_args() + + sha, nvrtag = get_build_info(args.buildid, True) + create_tree(sha, nvrtag) diff --git a/koji_cli_local.py b/koji_cli_local.py deleted file mode 100644 index 9d15cc4..0000000 --- a/koji_cli_local.py +++ /dev/null @@ -1,214 +0,0 @@ -#! /usr/bin/env python - -# Code taken and modified from koji - -# Copyright (c) 2005-2014 Red Hat, Inc. -# -# Koji is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; -# version 2.1 of the License. -# -# This software is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this software; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: -# Dennis Gregorovic -# Mike McLean -# Mike Bonnet -# Cristian Balint - -import os -import sys -import socket -import ConfigParser -import krbV -import koji - -class Options: - debug = False - server = None - weburl = None - pkgurl = None - topdir = None - cert = None - ca = None - serverca = None - authtype = None - noauth = None - user = None - runas = None - - -def get_options(): - global options - # load local config - defaults = { - 'server': 'http://localhost/kojihub', - 'weburl': 'http://localhost/koji', - 'pkgurl': 'http://localhost/packages', - 'topdir': '/mnt/koji', - 'max_retries': None, - 'retry_interval': None, - 'anon_retry': None, - 'offline_retry': None, - 'offline_retry_interval': None, - 'poll_interval': 5, - 'cert': '~/.koji/client.crt', - 'ca': '~/.koji/clientca.crt', - 'serverca': '~/.koji/serverca.crt', - 'keytab': None, - 'principal': None, - 'authtype': None - } - # grab settings from /etc/koji.conf first, and allow them to be - # overridden by user config - progname = 'koji' - for configFile in ('/etc/koji.conf',): - if os.access(configFile, os.F_OK): - f = open(configFile) - config = ConfigParser.ConfigParser() - config.readfp(f) - f.close() - if config.has_section(progname): - for name, value in config.items(progname): - # note the defaults dictionary also serves to indicate which - # options *can* be set via the config file. Such options should - # not have a default value set in the option parser. - if defaults.has_key(name): - if name in ('anon_retry', 'offline_retry'): - defaults[name] = config.getboolean(progname, name) - elif name in ('max_retries', 'retry_interval', - 'offline_retry_interval', 'poll_interval'): - try: - defaults[name] = int(value) - except ValueError: - parser.error( - "value for %s config option must be a valid integer" % name) - assert False - else: - defaults[name] = value - for name, value in defaults.iteritems(): - if getattr(options, name, None) is None: - # print '%s' % getattr(options, name, None) - setattr(options, name, value) - # print '%s' % getattr(options, name, None) - dir_opts = ('topdir', 'cert', 'ca', 'serverca') - for name in dir_opts: - # expand paths here, so we don't have to worry about it later - value = os.path.expanduser(getattr(options, name)) - setattr(options, name, value) - - # honor topdir - if options.topdir: - koji.BASEDIR = options.topdir - koji.pathinfo.topdir = options.topdir - - return options - - -def ensure_connection(session): - try: - ret = session.getAPIVersion() - except xmlrpclib.ProtocolError: - error(_("Error: Unable to connect to server")) - if ret != koji.API_VERSION: - warn(_("WARNING: The server is at API version %d and the client is at %d" % ( - ret, koji.API_VERSION))) - -def has_krb_creds(): - if not sys.modules.has_key('krbV'): - return False - try: - ctx = krbV.default_context() - ccache = ctx.default_ccache() - princ = ccache.principal() - return True - except krbV.Krb5Error: - return False - -def activate_session(session): - """Test and login the session is applicable""" - global options - if options.authtype == "noauth" or options.noauth: - # skip authentication - pass - elif options.authtype == "ssl" or os.path.isfile(options.cert) and options.authtype is None: - # authenticate using SSL client cert - session.ssl_login( - options.cert, options.ca, options.serverca, proxyuser=options.runas) - elif options.authtype == "password" or options.user and options.authtype is None: - # authenticate using user/password - session.login() - elif options.authtype == "kerberos" or has_krb_creds() and options.authtype is None: - try: - if options.keytab and options.principal: - session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas) - else: - session.krb_login(proxyuser=options.runas) - except socket.error as e: - warn(_("Could not connect to Kerberos authentication service: %s") % e.args[1]) - except Exception as e: - if krbV is not None and isinstance(e, krbV.Krb5Error): - print("Kerberos authentication failed: %s (%s)" % (e.args[1], e.args[0])) - else: - raise - - if not options.noauth and options.authtype != "noauth" and not session.logged_in: - error(_("Unable to log in, no authentication methods available")) - ensure_connection(session) - if options.debug: - print("successfully connected to hub") - -### -# Non-koji code below -### -def get_sha(tasklabel): - - sha = tasklabel.split(':')[1] - sha = sha.strip(")") - return sha - - -def get_build_info(build_id, nvr=False): - - global options - options = Options() - options = get_options() - - session = koji.ClientSession(options.server) - activate_session(session) - - if nvr is True: - info = session.getBuild(build_id) - else: - info = session.getBuild(int(build_id)) - - if info is None: - print("No such build: %s" % build) - return None - - task = None - if info['task_id']: - task = session.getTaskInfo(info['task_id'], request=True) - - nvrtag = "%(name)s-%(version)s-%(release)s" % info - print(nvrtag) - - if task is None: - return None - - tasklabel = koji.taskLabel(task) - print(tasklabel) - - sha = get_sha(tasklabel) - print(sha) - return (sha, nvrtag) - - diff --git a/parse_spec.py b/parse_spec.py deleted file mode 100755 index b832e13..0000000 --- a/parse_spec.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/python -# This Python file uses the following encoding: utf-8 -# Copyright 2015 Red Hat Inc. -# Author(s): Josh Boyer -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; version 2 of the License. -# See http://www.gnu.org/copyleft/gpl.html for the full text of the license. - -# Ideas taken from rpmdev-bumpspec script by Michael Schwendt, Ville Skyttä, -# Ralph Bean, et. al - -import re -import sys -import os - -spec_filename = "/home/jwboyer/kernel/kernel.spec" - - -def read_spec(filename): - f = None - lines = None - - try: - f = open(filename, "r") - lines = f.readlines() - finally: - f and f.close() - - return lines - - -def parse_spec(specf): - - spec = read_spec(specf) - specv = {} - - if spec is None: - sys.exit(0) - - for i in range(len(spec)): - match = re.match( - r"^%(global|define)\s+(?P\w+)\s+(?P\d+.*)", spec[i]) - if match is None: - continue - - var = match.group('var') - val = match.group('val') - val = val.strip() - if var == "released_kernel": - specv['released_kernel'] = val - continue - elif var == "base_sublevel": - specv['base_sublevel'] = val - continue - elif var == "rcrev": - specv['rcrev'] = val - continue - elif var == "gitrev": - specv['gitrev'] = val - continue - elif var == "stable_update": - specv['stable_update'] = val - continue - elif var == "kversion": - specv['kversion'] = val - specv['major_version'] = val.split(".")[0] - bar = val.split("-rc") - if len(bar) > 1: - specv['tar_suffix'] = "-rc%s" % specv['rcrev'] - else: - specv['tar_suffix'] = None - continue - else: - continue - - print specv['released_kernel'] - print specv['major_version'] - print specv['base_sublevel'] - print specv['rcrev'] - print specv['gitrev'] - print specv['stable_update'] - - return specv - -if __name__ == "__main__": - parse_spec(spec_filename)