From c99be34c2418d1bae1af7954546f7ae2d57bfedc Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Tue, 31 Aug 2021 18:34:12 +0200 Subject: [PATCH 1/3] Fix source_system parameter error handling Previously this resulted in a KeyError. --- obal/data/modules/srpm.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/obal/data/modules/srpm.py b/obal/data/modules/srpm.py index ed649c24..7027913b 100644 --- a/obal/data/modules/srpm.py +++ b/obal/data/modules/srpm.py @@ -15,6 +15,11 @@ from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.obal import run_command, get_specfile_sources # pylint:disable=import-error,no-name-in-module +SOURCE_SYSTEM_URLS = { + 'jenkins': '{}/lastSuccessfulBuild/artifact/*zip*/archive.zip', +} +VALID_SOURCE_SYSTEMS = list(SOURCE_SYSTEM_URLS.keys()) + @contextmanager def chdir(directory): @@ -63,11 +68,8 @@ def fetch_remote_sources(source_location, source_system, sources_dir): """ Copy RPM sources from a remote source like Jenkins to rpmbuild environment """ - source_system_urls = { - 'jenkins': '{}/lastSuccessfulBuild/artifact/*zip*/archive.zip', - } - url = source_system_urls[source_system].format(source_location) + url = SOURCE_SYSTEM_URLS[source_system].format(source_location) request = urlopen(url) with TemporaryFile() as archive: @@ -118,7 +120,8 @@ def main(): except HTTPError as error: module.fail_json(msg="HTTP %s: %s. Check %s exists." % (error.code, error.reason, source_location)) except KeyError as error: - module.fail_json(msg="Unknown source_system specified.", output=error) + module.fail_json(msg="Unknown source_system specified.", + source_system=source_system, valid_choices=VALID_SOURCE_SYSTEMS) copy_sources(spec_file, package, sources_dir) shutil.copy(spec_file, base_dir) From 69f4f790bf9e7f1291c7763049e464f185b30a86 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Tue, 31 Aug 2021 18:11:16 +0200 Subject: [PATCH 2/3] Correct handling of remote sources in build_srpm The build_srpm module handles this already so there's no need to duplicate it. It also moves the error handling to be local so it's easier to see where that would fail. --- obal/data/modules/srpm.py | 8 +++++--- obal/data/roles/build_srpm/meta/main.yaml | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/obal/data/modules/srpm.py b/obal/data/modules/srpm.py index 7027913b..2ac55d46 100644 --- a/obal/data/modules/srpm.py +++ b/obal/data/modules/srpm.py @@ -123,7 +123,11 @@ def main(): module.fail_json(msg="Unknown source_system specified.", source_system=source_system, valid_choices=VALID_SOURCE_SYSTEMS) - copy_sources(spec_file, package, sources_dir) + try: + copy_sources(spec_file, package, sources_dir) + except subprocess.CalledProcessError as error: + module.fail_json(msg='Failed to build srpm', output=error.output) + shutil.copy(spec_file, base_dir) command = ['rpmbuild', '-bs'] @@ -153,8 +157,6 @@ def main(): path = os.path.join(output, os.path.basename(result)) module.exit_json(changed=True, path=path) - except subprocess.CalledProcessError as error: - module.fail_json(msg='Failed to build srpm', output=error.output) finally: shutil.rmtree(base_dir) diff --git a/obal/data/roles/build_srpm/meta/main.yaml b/obal/data/roles/build_srpm/meta/main.yaml index 7b3819bd..1e1c5b9d 100644 --- a/obal/data/roles/build_srpm/meta/main.yaml +++ b/obal/data/roles/build_srpm/meta/main.yaml @@ -2,4 +2,3 @@ dependencies: - role: setup_workspace - role: ensure_package - - role: setup_sources From f54db22f104b05399f5f2ec682441151bb01dfa2 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Tue, 31 Aug 2021 18:15:01 +0200 Subject: [PATCH 3/3] Implement local source fetching This adds the ability to fetch sources by using: (bundle exec) rake pkg:generate_source This is implemented in the Foreman repositories and makes it easier to run local scratch builds. --- obal/data/modules/srpm.py | 47 ++++++++++++++++----- obal/data/playbooks/srpm/metadata.obal.yaml | 18 +++++++- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/obal/data/modules/srpm.py b/obal/data/modules/srpm.py index 2ac55d46..8d4cf939 100644 --- a/obal/data/modules/srpm.py +++ b/obal/data/modules/srpm.py @@ -64,6 +64,22 @@ def copy_sources(spec_file, package_dir, sources_dir): run_command(["git-annex", "lock", "--force"]) +def fetch_local_sources(source_location, source_system, sources_dir): + if source_system == 'local-bundle-rake': + command = ['bundle', 'exec', 'rake', 'pkg:generate_source'] + elif source_system == 'local-rake': + command = ['rake', 'pkg:generate_source'] + else: + raise ValueError("Unknown source system %s" % source_system) + + with chdir(source_location): + output = run_command(command) + + for line in output.splitlines(): + if os.path.isfile(line): + shutil.copy(line, sources_dir) + + def fetch_remote_sources(source_location, source_system, sources_dir): """ Copy RPM sources from a remote source like Jenkins to rpmbuild environment @@ -115,18 +131,27 @@ def main(): os.mkdir(build_dir) if source_location: + if source_system.startswith('local-'): + try: + fetch_local_sources(source_location, source_system, sources_dir) + except ValueError: + module.fail_json(msg="Unknown source_system specified.", + source_system=source_system, valid_choices=VALID_SOURCE_SYSTEMS) + except subprocess.CalledProcessError as error: + module.fail_json(msg='Failed fetch local sources', output=error.output) + else: + try: + fetch_remote_sources(source_location, source_system, sources_dir) + except HTTPError as error: + module.fail_json(msg="HTTP %s: %s. Check %s exists." % (error.code, error.reason, source_location)) + except KeyError as error: + module.fail_json(msg="Unknown source_system specified.", + source_system=source_system, valid_choices=VALID_SOURCE_SYSTEMS) + else: try: - fetch_remote_sources(source_location, source_system, sources_dir) - except HTTPError as error: - module.fail_json(msg="HTTP %s: %s. Check %s exists." % (error.code, error.reason, source_location)) - except KeyError as error: - module.fail_json(msg="Unknown source_system specified.", - source_system=source_system, valid_choices=VALID_SOURCE_SYSTEMS) - - try: - copy_sources(spec_file, package, sources_dir) - except subprocess.CalledProcessError as error: - module.fail_json(msg='Failed to build srpm', output=error.output) + copy_sources(spec_file, package, sources_dir) + except subprocess.CalledProcessError as error: + module.fail_json(msg='Failed to build srpm', output=error.output) shutil.copy(spec_file, base_dir) diff --git a/obal/data/playbooks/srpm/metadata.obal.yaml b/obal/data/playbooks/srpm/metadata.obal.yaml index 3d7815d4..1de46adb 100644 --- a/obal/data/playbooks/srpm/metadata.obal.yaml +++ b/obal/data/playbooks/srpm/metadata.obal.yaml @@ -2,7 +2,19 @@ help: | Build a SRPM for packages - Building source RPMs can be useful when locally building packages. + Building source RPMs can be useful when locally building packages. For + example, when you've made local changes. + + --source-system local-bundle-rake --source-location $HOME/foreman-installer + + Not all projects need bundler so it's possible to run without. + + --source-system local-bundle-rake --source-location $HOME/smart-proxy + + It is also possible to retrieve the source from a Jenkins job. + + --source-system jenkins --source-location https://ci.theforeman.org/job/foreman-installer-develop-source-release + variables: build_srpm_dist: parameter: --dist @@ -13,3 +25,7 @@ variables: build_srpm_output_dir: parameter: --dir help: Absolute path to output dir for SRPM. Defaults to /SRPMs + source_system: + help: The system to be used when passing source_location. Either jenkins, local-bundle-rake or local-rake. + source_location: + help: Source location. Format is dependent on the source system