From eaad06096d76c10994ba7782a2065d44d8cbc895 Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Sat, 21 Feb 2026 20:27:46 +0100 Subject: [PATCH 1/4] Start implementation of canary release --- .gitignore | 3 +- .gitlab-ci.yml | 4 + .../release-coordinator.canary.gitlab-ci.yml | 67 +++++++++++++ Gemfile | 2 +- lib/pyxis/cli.rb | 8 +- lib/pyxis/commands/components.rb | 2 +- lib/pyxis/commands/internal.rb | 97 +++++++++++++++++++ lib/pyxis/commands/release.rb | 47 +++++++++ lib/pyxis/environment.rb | 4 + lib/pyxis/github_client.rb | 2 + lib/pyxis/gitlab_client.rb | 38 ++++++++ .../managed_versioning/component_info.rb | 67 +++++++++---- lib/pyxis/permission_helper.rb | 13 +++ lib/pyxis/project/base.rb | 6 +- lib/pyxis/project/pyxis.rb | 16 +++ .../create_reticulum_build_service.rb | 6 +- tmp/.gitkeep | 0 17 files changed, 355 insertions(+), 27 deletions(-) create mode 100644 .gitlab/ci/release-coordinator.canary.gitlab-ci.yml create mode 100644 lib/pyxis/commands/internal.rb create mode 100644 lib/pyxis/commands/release.rb create mode 100644 lib/pyxis/project/pyxis.rb create mode 100644 tmp/.gitkeep diff --git a/.gitignore b/.gitignore index ad21aa4..26c448e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,8 @@ /doc/ /log/*.log /pkg/ -/tmp/ +/tmp/* +!/tmp/.gitkeep /private/ # rspec failure tracking diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index caf07b5..f1b907a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,10 @@ +include: + - local: .gitlab/ci/release-coordinator.canary.gitlab-ci.yml + stages: - build - components + - !reference [.release-coordinator:canary:stages] default: image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA diff --git a/.gitlab/ci/release-coordinator.canary.gitlab-ci.yml b/.gitlab/ci/release-coordinator.canary.gitlab-ci.yml new file mode 100644 index 0000000..0f2ab7c --- /dev/null +++ b/.gitlab/ci/release-coordinator.canary.gitlab-ci.yml @@ -0,0 +1,67 @@ +.release-coordinator:canary:stages: + - release-coordinator:canary:build + - release-coordinator:canary:publish + +.release-coordinator:canary: + rules: + - if: $RELEASE_COORDINATOR == "canary" + +release-coordinator:canary:tmp-branch: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:build + script: + - bin/pyxis internal release_canary_tmp_branch --build-id-to-promote $BUILD_ID_TO_PROMOTE + variables: + DRY_RUN: "false" + artifacts: + reports: + dotenv: tmp/reticulum_variables.env + +release-coordinator:canary:build: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:build + needs: + - release-coordinator:canary:tmp-branch + trigger: + project: code0-tech/development/reticulum + branch: pyxis/canary-build/$BUILD_ID_TO_PROMOTE + forward: + pipeline_variables: true + strategy: depend + variables: + RETICULUM_BUILD_TYPE: canary + +release-coordinator:canary:tmp-branch-cleanup: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:build + needs: + - release-coordinator:canary:build + script: + - bin/pyxis internal release_canary_tmp_branch_cleanup --build-id-to-promote $BUILD_ID_TO_PROMOTE + variables: + DRY_RUN: "false" + when: always + +release-coordinator:canary:publish: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:publish + needs: + - release-coordinator:canary:build + script: + - echo "Publishing approved" + when: manual + +release-coordinator:canary:publish-containers: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:publish + needs: + - release-coordinator:canary:publish + script: + - bin/pyxis internal release_canary_publish_tags --coordinator-pipeline-id $CI_PIPELINE_ID + variables: + DRY_RUN: "false" diff --git a/Gemfile b/Gemfile index 03dec3d..fb63e83 100644 --- a/Gemfile +++ b/Gemfile @@ -13,7 +13,7 @@ gem 'jwt', '~> 2.10' gem 'octokit', '~> 10.0' gem 'openssl', '~> 3.3' -gem 'semantic_logger', '~> 4.16' +gem 'semantic_logger', '~> 4.16', require: 'semantic_logger/sync' gem 'json', '~> 2.12' diff --git a/lib/pyxis/cli.rb b/lib/pyxis/cli.rb index d80f81a..faa6e54 100644 --- a/lib/pyxis/cli.rb +++ b/lib/pyxis/cli.rb @@ -5,7 +5,13 @@ class Cli < Thor desc 'components', 'Commands managing projects under managed versioning' subcommand 'components', Pyxis::Commands::Components - def self.exit_on_failure? + desc 'release', 'Commands managing the release process' + subcommand 'release', Pyxis::Commands::Release + + desc 'internal', 'Internal commands for usage by the pipeline', hide: true + subcommand 'internal', Pyxis::Commands::Internal + + def Thor.exit_on_failure? true end end diff --git a/lib/pyxis/commands/components.rb b/lib/pyxis/commands/components.rb index 61feb33..adfcdfb 100644 --- a/lib/pyxis/commands/components.rb +++ b/lib/pyxis/commands/components.rb @@ -71,7 +71,7 @@ def update def list result = 'Available components:' Pyxis::Project.components.each do |project| - result += "\n- #{project.downcase}" + result += "\n- #{project}" end result end diff --git a/lib/pyxis/commands/internal.rb b/lib/pyxis/commands/internal.rb new file mode 100644 index 0000000..c5d5da0 --- /dev/null +++ b/lib/pyxis/commands/internal.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +module Pyxis + module Commands + class Internal < Thor + include Thor::Actions + + RETICULUM_CI_BUILDS_PREFIX = 'ghcr.io/code0-tech/reticulum/ci-builds/' + CONTAINER_RELEASE_PREFIX = 'registry.gitlab.com/code0-tech/packages/' + + desc 'release_canary_tmp_branch', '' + method_option :build_id_to_promote, required: true, type: :numeric + def release_canary_tmp_branch + component_information = Pyxis::ManagedVersioning::ComponentInfo.new( + build_id: options[:build_id_to_promote] + ).execute + + raise 'Build not found' if component_information.nil? + + GitlabClient.client.create_branch( + Project::Reticulum.api_gitlab_path, + "pyxis/canary-build/#{options[:build_id_to_promote]}", + component_information[:reticulum] + ) + + version_variables = component_information.map do |component, version| + next nil unless Project.components.include?(component) + + ["OVERRIDE_#{component}_VERSION", version] + end.compact + + create_env_file( + 'reticulum_variables', + version_variables + [['C0_GH_TOKEN', Pyxis::Environment.github_reticulum_publish_token]] + ) + end + + desc 'release_canary_tmp_branch_cleanup', '' + method_option :build_id_to_promote, required: true, type: :numeric + def release_canary_tmp_branch_cleanup + GitlabClient.client.delete_branch( + Project::Reticulum.api_gitlab_path, + "pyxis/canary-build/#{options[:build_id_to_promote]}" + ) + end + + desc 'release_canary_publish_tags', '' + method_option :coordinator_pipeline_id, required: true, type: :numeric + def release_canary_publish_tags + build_id = GitlabClient.client + .list_pipeline_bridges(Project::Pyxis.api_gitlab_path, options[:coordinator_pipeline_id]) + .find { |bridge| bridge['name'] == 'release-coordinator:canary:build' } + .dig('downstream_pipeline', 'id') + + info = ManagedVersioning::ComponentInfo.new(build_id: build_id) + container_tag = info.find_container_tag_for_build_id + container_tags = info.find_manifests.map do |manifest| + next nil unless Project.components.include?(manifest.first.to_sym) + + next "#{manifest.first}:#{container_tag}" if manifest.length == 1 + + "#{manifest.first}:#{container_tag}-#{manifest.last}" + end.compact + + File.write('tmp/gitlab_token', Pyxis::Environment.gitlab_release_tools_token) + run 'crane auth login -u code0-release-tools --password-stdin registry.gitlab.com < tmp/gitlab_token' + + overall_success = true + + original_pretend = options[:pretend] + options[:pretend] = Pyxis::GlobalStatus.dry_run? + container_tags.each do |tag| + success = run "crane copy #{RETICULUM_CI_BUILDS_PREFIX}#{tag} #{CONTAINER_RELEASE_PREFIX}#{tag}", + abort_on_failure: false + overall_success &&= success + + logger.error('Failed to copy container image to release registry', image: tag) unless success + end + options[:pretend] = original_pretend + + run 'crane auth logout registry.gitlab.com' + File.delete('tmp/gitlab_token') + + abort unless overall_success || Pyxis::GlobalStatus.dry_run? + end + + no_commands do + include SemanticLogger::Loggable + + def create_env_file(name, variables) + path = File.absolute_path(File.join(__FILE__, "../../../../tmp/#{name}.env")) + File.write(path, variables.map { |k, v| "#{k}=#{v}" }.join("\n")) + end + end + end + end +end diff --git a/lib/pyxis/commands/release.rb b/lib/pyxis/commands/release.rb new file mode 100644 index 0000000..9f118a0 --- /dev/null +++ b/lib/pyxis/commands/release.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Pyxis + module Commands + class Release < Thor + include PermissionHelper + + desc 'create_canary', 'Promote an experimental build to canary' + exclusive do + at_least_one do + method_option :build, + aliases: '-b', + desc: 'The build ID', + required: false, + type: :numeric + method_option :container_tag, + aliases: '-c', + desc: 'The container tag excluding variant modifiers', + required: false, + type: :string + end + end + def create_canary + assert_executed_by_delivery_team_member! + + build_id = options[:build] || ManagedVersioning::ComponentInfo.new( + container_tag: options[:container_tag] + ).find_build_id_for_container_tag + + raise Pyxis::MessageError, 'This build does not exist' if build_id.nil? + + pipeline = GitlabClient.client.create_pipeline( + Project::Pyxis.api_gitlab_path, + Project::Pyxis.default_branch, + variables: { + RELEASE_COORDINATOR: 'canary', + BUILD_ID_TO_PROMOTE: build_id.to_s, + } + ) + + raise Pyxis::MessageError, 'Failed to create pipeline' if pipeline.response.status != 201 + + "Created coordinator pipeline at #{pipeline.body.web_url}" + end + end + end +end diff --git a/lib/pyxis/environment.rb b/lib/pyxis/environment.rb index e934f6a..75492f5 100644 --- a/lib/pyxis/environment.rb +++ b/lib/pyxis/environment.rb @@ -16,6 +16,10 @@ def github_release_tools_approver_private_key File.read(ENV.fetch('PYXIS_GH_RELEASE_TOOLS_APPROVER_PRIVATE_KEY')) end + def github_reticulum_publish_token + File.read(ENV.fetch('PYXIS_GH_RETICULUM_PUBLISH_TOKEN')) + end + def gitlab_release_tools_token File.read(ENV.fetch('PYXIS_GL_RELEASE_TOOLS_PRIVATE_TOKEN')) end diff --git a/lib/pyxis/github_client.rb b/lib/pyxis/github_client.rb index 050aee6..e0f85ac 100644 --- a/lib/pyxis/github_client.rb +++ b/lib/pyxis/github_client.rb @@ -4,6 +4,8 @@ module Pyxis class GithubClient include SemanticLogger::Loggable + ORGANIZATION_NAME = 'code0-tech' + CLIENT_CONFIGS = { release_tools: { app_id: 857194, diff --git a/lib/pyxis/gitlab_client.rb b/lib/pyxis/gitlab_client.rb index 250d498..2c99bc3 100644 --- a/lib/pyxis/gitlab_client.rb +++ b/lib/pyxis/gitlab_client.rb @@ -59,7 +59,33 @@ def initialize(faraday) @faraday = faraday end + # @param project_path_or_id Project path or id to create the branch in + # @param branch The name of the branch to create + # @param ref The branch name or commit sha to create the branch from + def create_branch(project_path_or_id, branch, ref) + post_json( + "/api/v4/projects/#{project_path_or_id}/repository/branches", + { + branch: branch, + ref: ref, + } + ) + end + + def delete_branch(project_path_or_id, branch) + delete("/api/v4/projects/#{project_path_or_id}/repository/branches/#{path_encode branch}") + end + def create_pipeline(project_path_or_id, ref, variables: nil) + if variables.is_a?(Hash) + variables = variables.map do |key, value| + { + key: key, + value: value, + } + end + end + post_json( "/api/v4/projects/#{project_path_or_id}/pipeline", { @@ -68,6 +94,18 @@ def create_pipeline(project_path_or_id, ref, variables: nil) } ) end + + def list_pipeline_bridges(project_path_or_id, pipeline_id) + paginate_json("/api/v4/projects/#{project_path_or_id}/pipelines/#{pipeline_id}/bridges") + end + + def path_encode(content) + content.gsub('/', '%2F') + end + + def paginate_json(url, options = {}) + GitlabClient.paginate_json(faraday, url, options) + end end class PageLinks diff --git a/lib/pyxis/managed_versioning/component_info.rb b/lib/pyxis/managed_versioning/component_info.rb index 94ebc5b..2f5ca15 100644 --- a/lib/pyxis/managed_versioning/component_info.rb +++ b/lib/pyxis/managed_versioning/component_info.rb @@ -12,30 +12,20 @@ def initialize(build_id: nil, container_tag: nil) @container_tag = container_tag end + # @return [Hash] The versions of each component in the build + # @return [nil] If the build does not exist def execute - unless container_tag.nil? - @build_id = annotation_for( - 'code0-tech/reticulum/ci-builds/mise', - container_tag, - 'tech.code0.reticulum.pipeline.id' - ) - end + @build_id = find_build_id_for_container_tag unless container_tag.nil? return nil if build_id.nil? - pipeline = GitlabClient.client.get_json( - "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipelines/#{build_id}" - ) - return nil if pipeline.response.status == 404 + pipeline, jobs = load_pipeline(build_id) - jobs = GitlabClient.paginate_json( - GitlabClient.client, - "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipelines/#{build_id}/jobs" - ) + return nil if jobs.nil? container_version = find_container_version(jobs) - manifests = find_manifests(jobs) + manifests = find_manifests_from_jobs(jobs) components = { reticulum: pipeline.body.sha, @@ -57,6 +47,28 @@ def execute components.compact end + def find_build_id_for_container_tag + annotation_for( + 'code0-tech/reticulum/ci-builds/mise', + container_tag, + 'tech.code0.reticulum.pipeline.id' + ) + end + + def find_container_tag_for_build_id + _, jobs = load_pipeline(build_id) + return nil if jobs.nil? + + find_container_version(jobs) + end + + def find_manifests + _, jobs = load_pipeline(build_id) + return nil if jobs.nil? + + find_manifests_from_jobs(jobs) + end + private def ghcr_client @@ -94,7 +106,7 @@ def annotation_for(image, tag, annotation) response.body.annotations&.[](annotation) end - def find_manifests(jobs) + def find_manifests_from_jobs(jobs) jobs.map { |job| job['name'] } .select { |job| job.start_with?('manifest:') } .map { |job| job.delete_prefix('manifest:') } @@ -116,10 +128,27 @@ def find_container_version(jobs) ) trace.body .lines - .drop_while { |line| !(line.include?('section_start') && line.include?('glpa_summary')) } - .find { |line| line =~ /RETICULUM_CONTAINER_VERSION=[0-9a-zA-Z-.]+$/ } + .drop_while { |line| line !~ /\e\[0Ksection_start:\d+:glpa_summary/ } + .drop(1) + .take_while { |line| line !~ /\e\[0Ksection_end:\d+:glpa_summary/ } + .find { |line| line =~ /RETICULUM_CONTAINER_VERSION=[0-9a-zA-Z\-.]+$/ } .split('=')[1].chomp end + + def load_pipeline(pipeline_id) + pipeline = GitlabClient.client.get_json( + "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipelines/#{pipeline_id}" + ) + return nil if pipeline.response.status == 404 + + [ + pipeline, + GitlabClient.paginate_json( + GitlabClient.client, + "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipelines/#{pipeline_id}/jobs" + ) + ] + end end end end diff --git a/lib/pyxis/permission_helper.rb b/lib/pyxis/permission_helper.rb index 5d7efb5..ad6bf5d 100644 --- a/lib/pyxis/permission_helper.rb +++ b/lib/pyxis/permission_helper.rb @@ -21,6 +21,19 @@ def assert_executed_by_known_team_member! raise PermissionError, 'This operation can only be run by a known team member' end + def assert_executed_by_delivery_team_member! + return unless checks_active? + + assert_executed_by_known_team_member! + + team = GithubClient.octokit.team_by_name(GithubClient::ORGANIZATION_NAME, 'delivery') + user = find_current_user + + return if GithubClient.octokit.team_member?(team.id, user['github']) + + raise PermissionError, 'This operation can only be run by a delivery team member' + end + def find_current_user if ENV['GITLAB_USER_LOGIN'] users.find { |user| user['gitlab'] == ENV['GITLAB_USER_LOGIN'] } diff --git a/lib/pyxis/project/base.rb b/lib/pyxis/project/base.rb index d45d609..5e8ef65 100644 --- a/lib/pyxis/project/base.rb +++ b/lib/pyxis/project/base.rb @@ -32,7 +32,11 @@ def component_name end def self.components - constants.reject { |c| %i[Base Reticulum].include?(c) } + %i[aquila draco sagittarius sculptor taurus] + end + + def self.get_project(project) + const_get(project.capitalize) end end end diff --git a/lib/pyxis/project/pyxis.rb b/lib/pyxis/project/pyxis.rb new file mode 100644 index 0000000..4a67d2d --- /dev/null +++ b/lib/pyxis/project/pyxis.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Pyxis + module Project + class Pyxis < Base + class << self + def paths + { + github: 'code0-tech/pyxis', + gitlab: 'code0-tech/infrastructure/pyxis', + } + end + end + end + end +end diff --git a/lib/pyxis/services/create_reticulum_build_service.rb b/lib/pyxis/services/create_reticulum_build_service.rb index 10f5968..f110628 100644 --- a/lib/pyxis/services/create_reticulum_build_service.rb +++ b/lib/pyxis/services/create_reticulum_build_service.rb @@ -24,7 +24,7 @@ def execute pipeline = GitlabClient.client.create_pipeline( Project::Reticulum.api_gitlab_path, ref, - variables: version_override_variables + token_variable, + variables: version_override_variables + token_variable ) pipeline.body if pipeline.response.status == 201 @@ -33,7 +33,7 @@ def execute private def validate_override!(component, version) - project = Pyxis::Project.const_get(component.capitalize) + project = Pyxis::Project.get_project(component.capitalize) begin GithubClient.octokit.tag(project.github_path, version) @@ -63,7 +63,7 @@ def token_variable [ { key: 'C0_GH_TOKEN', - value: File.read(ENV.fetch('PYXIS_GH_RETICULUM_PUBLISH_TOKEN')), + value: Pyxis::Environment.github_reticulum_publish_token, } ] end diff --git a/tmp/.gitkeep b/tmp/.gitkeep new file mode 100644 index 0000000..e69de29 From 2737a04bc305b2a2b94b76db55805f5f76114afa Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Fri, 27 Feb 2026 19:45:05 +0100 Subject: [PATCH 2/4] Refactor canary release implementation --- lib/pyxis/commands/internal.rb | 76 ++--------------------------- lib/pyxis/release/canary.rb | 55 +++++++++++++++++++++ lib/pyxis/release/common.rb | 68 ++++++++++++++++++++++++++ lib/pyxis/utils/pipeline_helpers.rb | 14 ++++++ 4 files changed, 140 insertions(+), 73 deletions(-) create mode 100644 lib/pyxis/release/canary.rb create mode 100644 lib/pyxis/release/common.rb create mode 100644 lib/pyxis/utils/pipeline_helpers.rb diff --git a/lib/pyxis/commands/internal.rb b/lib/pyxis/commands/internal.rb index c5d5da0..214f87f 100644 --- a/lib/pyxis/commands/internal.rb +++ b/lib/pyxis/commands/internal.rb @@ -5,92 +5,22 @@ module Commands class Internal < Thor include Thor::Actions - RETICULUM_CI_BUILDS_PREFIX = 'ghcr.io/code0-tech/reticulum/ci-builds/' - CONTAINER_RELEASE_PREFIX = 'registry.gitlab.com/code0-tech/packages/' - desc 'release_canary_tmp_branch', '' method_option :build_id_to_promote, required: true, type: :numeric def release_canary_tmp_branch - component_information = Pyxis::ManagedVersioning::ComponentInfo.new( - build_id: options[:build_id_to_promote] - ).execute - - raise 'Build not found' if component_information.nil? - - GitlabClient.client.create_branch( - Project::Reticulum.api_gitlab_path, - "pyxis/canary-build/#{options[:build_id_to_promote]}", - component_information[:reticulum] - ) - - version_variables = component_information.map do |component, version| - next nil unless Project.components.include?(component) - - ["OVERRIDE_#{component}_VERSION", version] - end.compact - - create_env_file( - 'reticulum_variables', - version_variables + [['C0_GH_TOKEN', Pyxis::Environment.github_reticulum_publish_token]] - ) + Release::Canary.new.create_build_branch(options[:build_id_to_promote]) end desc 'release_canary_tmp_branch_cleanup', '' method_option :build_id_to_promote, required: true, type: :numeric def release_canary_tmp_branch_cleanup - GitlabClient.client.delete_branch( - Project::Reticulum.api_gitlab_path, - "pyxis/canary-build/#{options[:build_id_to_promote]}" - ) + Release::Canary.new.remove_build_branch(options[:build_id_to_promote]) end desc 'release_canary_publish_tags', '' method_option :coordinator_pipeline_id, required: true, type: :numeric def release_canary_publish_tags - build_id = GitlabClient.client - .list_pipeline_bridges(Project::Pyxis.api_gitlab_path, options[:coordinator_pipeline_id]) - .find { |bridge| bridge['name'] == 'release-coordinator:canary:build' } - .dig('downstream_pipeline', 'id') - - info = ManagedVersioning::ComponentInfo.new(build_id: build_id) - container_tag = info.find_container_tag_for_build_id - container_tags = info.find_manifests.map do |manifest| - next nil unless Project.components.include?(manifest.first.to_sym) - - next "#{manifest.first}:#{container_tag}" if manifest.length == 1 - - "#{manifest.first}:#{container_tag}-#{manifest.last}" - end.compact - - File.write('tmp/gitlab_token', Pyxis::Environment.gitlab_release_tools_token) - run 'crane auth login -u code0-release-tools --password-stdin registry.gitlab.com < tmp/gitlab_token' - - overall_success = true - - original_pretend = options[:pretend] - options[:pretend] = Pyxis::GlobalStatus.dry_run? - container_tags.each do |tag| - success = run "crane copy #{RETICULUM_CI_BUILDS_PREFIX}#{tag} #{CONTAINER_RELEASE_PREFIX}#{tag}", - abort_on_failure: false - overall_success &&= success - - logger.error('Failed to copy container image to release registry', image: tag) unless success - end - options[:pretend] = original_pretend - - run 'crane auth logout registry.gitlab.com' - File.delete('tmp/gitlab_token') - - abort unless overall_success || Pyxis::GlobalStatus.dry_run? - end - - no_commands do - include SemanticLogger::Loggable - - def create_env_file(name, variables) - path = File.absolute_path(File.join(__FILE__, "../../../../tmp/#{name}.env")) - File.write(path, variables.map { |k, v| "#{k}=#{v}" }.join("\n")) - end + Release::Canary.new.publish_tags(options[:coordinator_pipeline_id]) end end end diff --git a/lib/pyxis/release/canary.rb b/lib/pyxis/release/canary.rb new file mode 100644 index 0000000..da7fd44 --- /dev/null +++ b/lib/pyxis/release/canary.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Pyxis + module Release + class Canary + include SemanticLogger::Loggable + + def create_build_branch(build_to_promote) + component_information = Pyxis::ManagedVersioning::ComponentInfo.new( + build_id: build_to_promote + ).execute + + raise 'Build not found' if component_information.nil? + + GitlabClient.client.create_branch( + Project::Reticulum.api_gitlab_path, + "pyxis/canary-build/#{build_to_promote}", + component_information[:reticulum] + ) + + version_variables = component_information.map do |component, version| + next nil unless Project.components.include?(component) + + ["OVERRIDE_#{component}_VERSION", version] + end.compact + + Utils::PipelineHelpers.create_env_file( + 'reticulum_variables', + version_variables + [['C0_GH_TOKEN', Pyxis::Environment.github_reticulum_publish_token]] + ) + end + + def remove_build_branch(build_to_promote) + GitlabClient.client.delete_branch( + Project::Reticulum.api_gitlab_path, + "pyxis/canary-build/#{build_to_promote}" + ) + end + + def publish_tags(coordinator_pipeline_id) + build_id = GitlabClient.client + .list_pipeline_bridges(Project::Pyxis.api_gitlab_path, coordinator_pipeline_id) + .find { |bridge| bridge['name'] == 'release-coordinator:canary:build' } + .dig('downstream_pipeline', 'id') + + info = ManagedVersioning::ComponentInfo.new(build_id: build_id) + common = Common.new + + success = common.copy_container_images_to_release_registry(info) + + raise Pyxis::MessageError, 'Failed to copy all container images' unless success + end + end + end +end diff --git a/lib/pyxis/release/common.rb b/lib/pyxis/release/common.rb new file mode 100644 index 0000000..afc75ff --- /dev/null +++ b/lib/pyxis/release/common.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Pyxis + module Release + class Common + include SemanticLogger::Loggable + + CI_BUILDS_PREFIX = 'ghcr.io/code0-tech/reticulum/ci-builds/' + CONTAINER_RELEASE_REGISTRY = 'registry.gitlab.com' + CONTAINER_RELEASE_PREFIX = "#{CONTAINER_RELEASE_REGISTRY}/code0-tech/packages/".freeze + + CONTAINER_RELEASE_PUBLISH_USER = 'code0-release-tools' + + def copy_container_images_to_release_registry(component_info) + container_tag = info.find_container_tag_for_build_id + container_tags = component_info.find_manifests.map do |manifest| + next nil unless Project.components.include?(manifest.first.to_sym) + + next "#{manifest.first}:#{container_tag}" if manifest.length == 1 + + "#{manifest.first}:#{container_tag}-#{manifest.last}" + end.compact + + success = true + container_tags.each do |tag| + success &&= copy_container_image_to_release_registry(tag) + end + + success + end + + def copy_container_image_to_release_registry(tag) + with_release_registry_auth do + logger.info('Copying container image to release registry', tag: tag) + return if Pyxis::GlobalStatus.dry_run? + + success = system("crane copy #{CI_BUILDS_PREFIX}#{tag} #{CONTAINER_RELEASE_PREFIX}#{tag}") + logger.error('Failed to copy container image to release registry', tag: tag) + success + end + end + + def with_release_registry_auth + return yield if File.exist?('tmp/gitlab_token') + + logger.info('Authentication with release registry') + File.write('tmp/gitlab_token', Pyxis::Environment.gitlab_release_tools_token) + success = system( + "crane auth login + -u #{CONTAINER_RELEASE_PUBLISH_USER} + --password-stdin #{CONTAINER_RELEASE_REGISTRY} + < tmp/gitlab_token" + ) + + unless success + logger.error('Failed to authenticate with release registry') + raise Pyxis::Error, 'Failed to authenticate with release registry' + end + + yield + ensure + system("crane auth logout #{CONTAINER_RELEASE_REGISTRY}") + File.delete('tmp/gitlab_token') + logger.info('Unauthenticated from release registry') + end + end + end +end diff --git a/lib/pyxis/utils/pipeline_helpers.rb b/lib/pyxis/utils/pipeline_helpers.rb new file mode 100644 index 0000000..07a2094 --- /dev/null +++ b/lib/pyxis/utils/pipeline_helpers.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Pyxis + module Utils + module PipelineHelpers + module_function + + def create_env_file(name, variables) + path = File.absolute_path(File.join(__FILE__, "../../../../tmp/#{name}.env")) + File.write(path, variables.map { |k, v| "#{k}=#{v}" }.join("\n")) + end + end + end +end From dea2927727b2c8172794e80445316fe6ac6c204d Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Fri, 27 Feb 2026 19:56:39 +0100 Subject: [PATCH 3/4] Include config-generator in release --- lib/pyxis/release/common.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pyxis/release/common.rb b/lib/pyxis/release/common.rb index afc75ff..310be1a 100644 --- a/lib/pyxis/release/common.rb +++ b/lib/pyxis/release/common.rb @@ -11,10 +11,12 @@ class Common CONTAINER_RELEASE_PUBLISH_USER = 'code0-release-tools' + CONTAINER_IMAGES_TO_RELEASE = Project.components + %i[config-generator] + def copy_container_images_to_release_registry(component_info) container_tag = info.find_container_tag_for_build_id container_tags = component_info.find_manifests.map do |manifest| - next nil unless Project.components.include?(manifest.first.to_sym) + next nil unless CONTAINER_IMAGES_TO_RELEASE.include?(manifest.first.to_sym) next "#{manifest.first}:#{container_tag}" if manifest.length == 1 From 1057e5ddbd1028ab008b78a16bea07125d92fa57 Mon Sep 17 00:00:00 2001 From: Niklas van Schrick Date: Sat, 28 Feb 2026 15:13:57 +0100 Subject: [PATCH 4/4] Add github release creation to canary release --- .../release-coordinator.canary.gitlab-ci.yml | 11 +++ lib/pyxis/commands/internal.rb | 12 ++- .../managed_versioning/component_info.rb | 15 ++-- lib/pyxis/project/codezero.rb | 15 ++++ lib/pyxis/release/canary.rb | 23 +++++- lib/pyxis/release/common.rb | 82 ++++++++++++++++--- 6 files changed, 136 insertions(+), 22 deletions(-) create mode 100644 lib/pyxis/project/codezero.rb diff --git a/.gitlab/ci/release-coordinator.canary.gitlab-ci.yml b/.gitlab/ci/release-coordinator.canary.gitlab-ci.yml index 0f2ab7c..72a3a20 100644 --- a/.gitlab/ci/release-coordinator.canary.gitlab-ci.yml +++ b/.gitlab/ci/release-coordinator.canary.gitlab-ci.yml @@ -65,3 +65,14 @@ release-coordinator:canary:publish-containers: - bin/pyxis internal release_canary_publish_tags --coordinator-pipeline-id $CI_PIPELINE_ID variables: DRY_RUN: "false" + +release-coordinator:canary:publish-release: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:publish + needs: + - release-coordinator:canary:publish-containers + script: + - bin/pyxis internal release_canary_publish_release --coordinator-pipeline-id $CI_PIPELINE_ID + variables: + DRY_RUN: "false" diff --git a/lib/pyxis/commands/internal.rb b/lib/pyxis/commands/internal.rb index 214f87f..c98b94d 100644 --- a/lib/pyxis/commands/internal.rb +++ b/lib/pyxis/commands/internal.rb @@ -8,19 +8,25 @@ class Internal < Thor desc 'release_canary_tmp_branch', '' method_option :build_id_to_promote, required: true, type: :numeric def release_canary_tmp_branch - Release::Canary.new.create_build_branch(options[:build_id_to_promote]) + Pyxis::Release::Canary.new.create_build_branch(options[:build_id_to_promote]) end desc 'release_canary_tmp_branch_cleanup', '' method_option :build_id_to_promote, required: true, type: :numeric def release_canary_tmp_branch_cleanup - Release::Canary.new.remove_build_branch(options[:build_id_to_promote]) + Pyxis::Release::Canary.new.remove_build_branch(options[:build_id_to_promote]) end desc 'release_canary_publish_tags', '' method_option :coordinator_pipeline_id, required: true, type: :numeric def release_canary_publish_tags - Release::Canary.new.publish_tags(options[:coordinator_pipeline_id]) + Pyxis::Release::Canary.new.publish_tags(options[:coordinator_pipeline_id]) + end + + desc 'release_canary_publish_release', '' + method_option :coordinator_pipeline_id, required: true, type: :numeric + def release_canary_publish_release + Pyxis::Release::Canary.new.publish_release(options[:coordinator_pipeline_id]) end end end diff --git a/lib/pyxis/managed_versioning/component_info.rb b/lib/pyxis/managed_versioning/component_info.rb index 2f5ca15..fac3900 100644 --- a/lib/pyxis/managed_versioning/component_info.rb +++ b/lib/pyxis/managed_versioning/component_info.rb @@ -14,8 +14,8 @@ def initialize(build_id: nil, container_tag: nil) # @return [Hash] The versions of each component in the build # @return [nil] If the build does not exist - def execute - @build_id = find_build_id_for_container_tag unless container_tag.nil? + def execute(filter_components: nil) + find_build_id_for_container_tag unless container_tag.nil? return nil if build_id.nil? @@ -28,12 +28,13 @@ def execute manifests = find_manifests_from_jobs(jobs) components = { - reticulum: pipeline.body.sha, + reticulum: (pipeline.body.sha if filter_components.nil? || filter_components.include?(:reticulum)), } manifests.each do |image| component = image.first.to_sym next if components.key?(component) + next if filter_components && !filter_components.include?(component) image_tag = image.length == 1 ? container_version : "#{container_version}-#{image.last}" @@ -48,7 +49,9 @@ def execute end def find_build_id_for_container_tag - annotation_for( + return build_id unless build_id.nil? + + @build_id = annotation_for( 'code0-tech/reticulum/ci-builds/mise', container_tag, 'tech.code0.reticulum.pipeline.id' @@ -56,10 +59,12 @@ def find_build_id_for_container_tag end def find_container_tag_for_build_id + return container_tag unless container_tag.nil? + _, jobs = load_pipeline(build_id) return nil if jobs.nil? - find_container_version(jobs) + @container_tag = find_container_version(jobs) end def find_manifests diff --git a/lib/pyxis/project/codezero.rb b/lib/pyxis/project/codezero.rb new file mode 100644 index 0000000..8b4f3c5 --- /dev/null +++ b/lib/pyxis/project/codezero.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Pyxis + module Project + class Codezero < Base + class << self + def paths + { + github: 'code0-tech/codezero', + } + end + end + end + end +end diff --git a/lib/pyxis/release/canary.rb b/lib/pyxis/release/canary.rb index da7fd44..b7d09bc 100644 --- a/lib/pyxis/release/canary.rb +++ b/lib/pyxis/release/canary.rb @@ -38,10 +38,7 @@ def remove_build_branch(build_to_promote) end def publish_tags(coordinator_pipeline_id) - build_id = GitlabClient.client - .list_pipeline_bridges(Project::Pyxis.api_gitlab_path, coordinator_pipeline_id) - .find { |bridge| bridge['name'] == 'release-coordinator:canary:build' } - .dig('downstream_pipeline', 'id') + build_id = find_build_id(coordinator_pipeline_id) info = ManagedVersioning::ComponentInfo.new(build_id: build_id) common = Common.new @@ -50,6 +47,24 @@ def publish_tags(coordinator_pipeline_id) raise Pyxis::MessageError, 'Failed to copy all container images' unless success end + + def publish_release(coordinator_pipeline_id) + build_id = find_build_id(coordinator_pipeline_id) + + info = ManagedVersioning::ComponentInfo.new(build_id: build_id) + common = Common.new + + common.publish_github_release(info, prerelease: true) + end + + private + + def find_build_id(coordinator_pipeline_id) + GitlabClient.client + .list_pipeline_bridges(Project::Pyxis.api_gitlab_path, coordinator_pipeline_id) + .find { |bridge| bridge['name'] == 'release-coordinator:canary:build' } + .dig('downstream_pipeline', 'id') + end end end end diff --git a/lib/pyxis/release/common.rb b/lib/pyxis/release/common.rb index 310be1a..41f405f 100644 --- a/lib/pyxis/release/common.rb +++ b/lib/pyxis/release/common.rb @@ -14,7 +14,7 @@ class Common CONTAINER_IMAGES_TO_RELEASE = Project.components + %i[config-generator] def copy_container_images_to_release_registry(component_info) - container_tag = info.find_container_tag_for_build_id + container_tag = component_info.find_container_tag_for_build_id container_tags = component_info.find_manifests.map do |manifest| next nil unless CONTAINER_IMAGES_TO_RELEASE.include?(manifest.first.to_sym) @@ -37,21 +37,22 @@ def copy_container_image_to_release_registry(tag) return if Pyxis::GlobalStatus.dry_run? success = system("crane copy #{CI_BUILDS_PREFIX}#{tag} #{CONTAINER_RELEASE_PREFIX}#{tag}") - logger.error('Failed to copy container image to release registry', tag: tag) + logger.error('Failed to copy container image to release registry', tag: tag) unless success success end end def with_release_registry_auth - return yield if File.exist?('tmp/gitlab_token') + token_exists = File.exist?('tmp/gitlab_token') + return yield if token_exists logger.info('Authentication with release registry') File.write('tmp/gitlab_token', Pyxis::Environment.gitlab_release_tools_token) success = system( - "crane auth login - -u #{CONTAINER_RELEASE_PUBLISH_USER} - --password-stdin #{CONTAINER_RELEASE_REGISTRY} - < tmp/gitlab_token" + 'crane auth login ' \ + "-u #{CONTAINER_RELEASE_PUBLISH_USER} " \ + "--password-stdin #{CONTAINER_RELEASE_REGISTRY} " \ + '< tmp/gitlab_token' ) unless success @@ -61,9 +62,70 @@ def with_release_registry_auth yield ensure - system("crane auth logout #{CONTAINER_RELEASE_REGISTRY}") - File.delete('tmp/gitlab_token') - logger.info('Unauthenticated from release registry') + unless token_exists + system("crane auth logout #{CONTAINER_RELEASE_REGISTRY}") + File.delete('tmp/gitlab_token') + logger.info('Unauthenticated from release registry') + end + end + + def publish_github_release(component_info, prerelease:) + logger.info('Starting release to codezero repository', tag: component_info.find_container_tag_for_build_id) + + release_version = component_info.find_container_tag_for_build_id + reticulum_sha = component_info.execute(filter_components: [:reticulum])[:reticulum] + + compose_path = 'docker-compose/docker-compose.yml' + env_path = 'docker-compose/.env' + + codezero_compose_content = GithubClient.octokit.contents( + Project::Codezero.github_path, + path: compose_path + ) + codezero_env_content = GithubClient.octokit.contents( + Project::Codezero.github_path, + path: env_path + ) + + reticulum_compose_content = GithubClient.octokit.contents( + Project::Reticulum.github_path, + path: compose_path, + ref: reticulum_sha + ) + reticulum_env_content = GithubClient.octokit.contents( + Project::Reticulum.github_path, + path: env_path, + ref: reticulum_sha + ) + + reticulum_env = Base64.decode64 reticulum_env_content.content + reticulum_compose = Base64.decode64 reticulum_compose_content.content + + codezero_env = reticulum_env.sub('IMAGE_TAG=', "IMAGE_TAG=#{release_version}") + + GithubClient.octokit.update_contents( + Project::Codezero.github_path, + env_path, + "Update compose env for #{release_version}", + codezero_env_content.sha, + codezero_env, + Project::Codezero.default_branch + ) + GithubClient.octokit.update_contents( + Project::Codezero.github_path, + compose_path, + "Update compose file for #{release_version}", + codezero_compose_content.sha, + reticulum_compose, + Project::Codezero.default_branch + ) + + GithubClient.octokit.create_release( + Project::Codezero.github_path, + release_version, + name: release_version, + prerelease: prerelease + ) end end end