diff --git a/.gitignore b/.gitignore index ad21aa4..26c448e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,8 @@ /doc/ /log/*.log /pkg/ -/tmp/ +/tmp/* +!/tmp/.gitkeep /private/ # rspec failure tracking diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index caf07b5..f1b907a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,10 @@ +include: + - local: .gitlab/ci/release-coordinator.canary.gitlab-ci.yml + stages: - build - components + - !reference [.release-coordinator:canary:stages] default: image: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA diff --git a/.gitlab/ci/release-coordinator.canary.gitlab-ci.yml b/.gitlab/ci/release-coordinator.canary.gitlab-ci.yml new file mode 100644 index 0000000..72a3a20 --- /dev/null +++ b/.gitlab/ci/release-coordinator.canary.gitlab-ci.yml @@ -0,0 +1,78 @@ +.release-coordinator:canary:stages: + - release-coordinator:canary:build + - release-coordinator:canary:publish + +.release-coordinator:canary: + rules: + - if: $RELEASE_COORDINATOR == "canary" + +release-coordinator:canary:tmp-branch: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:build + script: + - bin/pyxis internal release_canary_tmp_branch --build-id-to-promote $BUILD_ID_TO_PROMOTE + variables: + DRY_RUN: "false" + artifacts: + reports: + dotenv: tmp/reticulum_variables.env + +release-coordinator:canary:build: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:build + needs: + - release-coordinator:canary:tmp-branch + trigger: + project: code0-tech/development/reticulum + branch: pyxis/canary-build/$BUILD_ID_TO_PROMOTE + forward: + pipeline_variables: true + strategy: depend + variables: + RETICULUM_BUILD_TYPE: canary + +release-coordinator:canary:tmp-branch-cleanup: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:build + needs: + - release-coordinator:canary:build + script: + - bin/pyxis internal release_canary_tmp_branch_cleanup --build-id-to-promote $BUILD_ID_TO_PROMOTE + variables: + DRY_RUN: "false" + when: always + +release-coordinator:canary:publish: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:publish + needs: + - release-coordinator:canary:build + script: + - echo "Publishing approved" + when: manual + +release-coordinator:canary:publish-containers: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:publish + needs: + - release-coordinator:canary:publish + script: + - bin/pyxis internal release_canary_publish_tags --coordinator-pipeline-id $CI_PIPELINE_ID + variables: + DRY_RUN: "false" + +release-coordinator:canary:publish-release: + extends: + - .release-coordinator:canary + stage: release-coordinator:canary:publish + needs: + - release-coordinator:canary:publish-containers + script: + - bin/pyxis internal release_canary_publish_release --coordinator-pipeline-id $CI_PIPELINE_ID + variables: + DRY_RUN: "false" diff --git a/Gemfile b/Gemfile index 03dec3d..fb63e83 100644 --- a/Gemfile +++ b/Gemfile @@ -13,7 +13,7 @@ gem 'jwt', '~> 2.10' gem 'octokit', '~> 10.0' gem 'openssl', '~> 3.3' -gem 'semantic_logger', '~> 4.16' +gem 'semantic_logger', '~> 4.16', require: 'semantic_logger/sync' gem 'json', '~> 2.12' diff --git a/lib/pyxis/cli.rb b/lib/pyxis/cli.rb index d80f81a..faa6e54 100644 --- a/lib/pyxis/cli.rb +++ b/lib/pyxis/cli.rb @@ -5,7 +5,13 @@ class Cli < Thor desc 'components', 'Commands managing projects under managed versioning' subcommand 'components', Pyxis::Commands::Components - def self.exit_on_failure? + desc 'release', 'Commands managing the release process' + subcommand 'release', Pyxis::Commands::Release + + desc 'internal', 'Internal commands for usage by the pipeline', hide: true + subcommand 'internal', Pyxis::Commands::Internal + + def Thor.exit_on_failure? true end end diff --git a/lib/pyxis/commands/components.rb b/lib/pyxis/commands/components.rb index 61feb33..adfcdfb 100644 --- a/lib/pyxis/commands/components.rb +++ b/lib/pyxis/commands/components.rb @@ -71,7 +71,7 @@ def update def list result = 'Available components:' Pyxis::Project.components.each do |project| - result += "\n- #{project.downcase}" + result += "\n- #{project}" end result end diff --git a/lib/pyxis/commands/internal.rb b/lib/pyxis/commands/internal.rb new file mode 100644 index 0000000..c98b94d --- /dev/null +++ b/lib/pyxis/commands/internal.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Pyxis + module Commands + class Internal < Thor + include Thor::Actions + + desc 'release_canary_tmp_branch', '' + method_option :build_id_to_promote, required: true, type: :numeric + def release_canary_tmp_branch + Pyxis::Release::Canary.new.create_build_branch(options[:build_id_to_promote]) + end + + desc 'release_canary_tmp_branch_cleanup', '' + method_option :build_id_to_promote, required: true, type: :numeric + def release_canary_tmp_branch_cleanup + Pyxis::Release::Canary.new.remove_build_branch(options[:build_id_to_promote]) + end + + desc 'release_canary_publish_tags', '' + method_option :coordinator_pipeline_id, required: true, type: :numeric + def release_canary_publish_tags + Pyxis::Release::Canary.new.publish_tags(options[:coordinator_pipeline_id]) + end + + desc 'release_canary_publish_release', '' + method_option :coordinator_pipeline_id, required: true, type: :numeric + def release_canary_publish_release + Pyxis::Release::Canary.new.publish_release(options[:coordinator_pipeline_id]) + end + end + end +end diff --git a/lib/pyxis/commands/release.rb b/lib/pyxis/commands/release.rb new file mode 100644 index 0000000..9f118a0 --- /dev/null +++ b/lib/pyxis/commands/release.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Pyxis + module Commands + class Release < Thor + include PermissionHelper + + desc 'create_canary', 'Promote an experimental build to canary' + exclusive do + at_least_one do + method_option :build, + aliases: '-b', + desc: 'The build ID', + required: false, + type: :numeric + method_option :container_tag, + aliases: '-c', + desc: 'The container tag excluding variant modifiers', + required: false, + type: :string + end + end + def create_canary + assert_executed_by_delivery_team_member! + + build_id = options[:build] || ManagedVersioning::ComponentInfo.new( + container_tag: options[:container_tag] + ).find_build_id_for_container_tag + + raise Pyxis::MessageError, 'This build does not exist' if build_id.nil? + + pipeline = GitlabClient.client.create_pipeline( + Project::Pyxis.api_gitlab_path, + Project::Pyxis.default_branch, + variables: { + RELEASE_COORDINATOR: 'canary', + BUILD_ID_TO_PROMOTE: build_id.to_s, + } + ) + + raise Pyxis::MessageError, 'Failed to create pipeline' if pipeline.response.status != 201 + + "Created coordinator pipeline at #{pipeline.body.web_url}" + end + end + end +end diff --git a/lib/pyxis/environment.rb b/lib/pyxis/environment.rb index e934f6a..75492f5 100644 --- a/lib/pyxis/environment.rb +++ b/lib/pyxis/environment.rb @@ -16,6 +16,10 @@ def github_release_tools_approver_private_key File.read(ENV.fetch('PYXIS_GH_RELEASE_TOOLS_APPROVER_PRIVATE_KEY')) end + def github_reticulum_publish_token + File.read(ENV.fetch('PYXIS_GH_RETICULUM_PUBLISH_TOKEN')) + end + def gitlab_release_tools_token File.read(ENV.fetch('PYXIS_GL_RELEASE_TOOLS_PRIVATE_TOKEN')) end diff --git a/lib/pyxis/github_client.rb b/lib/pyxis/github_client.rb index 050aee6..e0f85ac 100644 --- a/lib/pyxis/github_client.rb +++ b/lib/pyxis/github_client.rb @@ -4,6 +4,8 @@ module Pyxis class GithubClient include SemanticLogger::Loggable + ORGANIZATION_NAME = 'code0-tech' + CLIENT_CONFIGS = { release_tools: { app_id: 857194, diff --git a/lib/pyxis/gitlab_client.rb b/lib/pyxis/gitlab_client.rb index 250d498..2c99bc3 100644 --- a/lib/pyxis/gitlab_client.rb +++ b/lib/pyxis/gitlab_client.rb @@ -59,7 +59,33 @@ def initialize(faraday) @faraday = faraday end + # @param project_path_or_id Project path or id to create the branch in + # @param branch The name of the branch to create + # @param ref The branch name or commit sha to create the branch from + def create_branch(project_path_or_id, branch, ref) + post_json( + "/api/v4/projects/#{project_path_or_id}/repository/branches", + { + branch: branch, + ref: ref, + } + ) + end + + def delete_branch(project_path_or_id, branch) + delete("/api/v4/projects/#{project_path_or_id}/repository/branches/#{path_encode branch}") + end + def create_pipeline(project_path_or_id, ref, variables: nil) + if variables.is_a?(Hash) + variables = variables.map do |key, value| + { + key: key, + value: value, + } + end + end + post_json( "/api/v4/projects/#{project_path_or_id}/pipeline", { @@ -68,6 +94,18 @@ def create_pipeline(project_path_or_id, ref, variables: nil) } ) end + + def list_pipeline_bridges(project_path_or_id, pipeline_id) + paginate_json("/api/v4/projects/#{project_path_or_id}/pipelines/#{pipeline_id}/bridges") + end + + def path_encode(content) + content.gsub('/', '%2F') + end + + def paginate_json(url, options = {}) + GitlabClient.paginate_json(faraday, url, options) + end end class PageLinks diff --git a/lib/pyxis/managed_versioning/component_info.rb b/lib/pyxis/managed_versioning/component_info.rb index 94ebc5b..fac3900 100644 --- a/lib/pyxis/managed_versioning/component_info.rb +++ b/lib/pyxis/managed_versioning/component_info.rb @@ -12,38 +12,29 @@ def initialize(build_id: nil, container_tag: nil) @container_tag = container_tag end - def execute - unless container_tag.nil? - @build_id = annotation_for( - 'code0-tech/reticulum/ci-builds/mise', - container_tag, - 'tech.code0.reticulum.pipeline.id' - ) - end + # @return [Hash] The versions of each component in the build + # @return [nil] If the build does not exist + def execute(filter_components: nil) + find_build_id_for_container_tag unless container_tag.nil? return nil if build_id.nil? - pipeline = GitlabClient.client.get_json( - "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipelines/#{build_id}" - ) - return nil if pipeline.response.status == 404 + pipeline, jobs = load_pipeline(build_id) - jobs = GitlabClient.paginate_json( - GitlabClient.client, - "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipelines/#{build_id}/jobs" - ) + return nil if jobs.nil? container_version = find_container_version(jobs) - manifests = find_manifests(jobs) + manifests = find_manifests_from_jobs(jobs) components = { - reticulum: pipeline.body.sha, + reticulum: (pipeline.body.sha if filter_components.nil? || filter_components.include?(:reticulum)), } manifests.each do |image| component = image.first.to_sym next if components.key?(component) + next if filter_components && !filter_components.include?(component) image_tag = image.length == 1 ? container_version : "#{container_version}-#{image.last}" @@ -57,6 +48,32 @@ def execute components.compact end + def find_build_id_for_container_tag + return build_id unless build_id.nil? + + @build_id = annotation_for( + 'code0-tech/reticulum/ci-builds/mise', + container_tag, + 'tech.code0.reticulum.pipeline.id' + ) + end + + def find_container_tag_for_build_id + return container_tag unless container_tag.nil? + + _, jobs = load_pipeline(build_id) + return nil if jobs.nil? + + @container_tag = find_container_version(jobs) + end + + def find_manifests + _, jobs = load_pipeline(build_id) + return nil if jobs.nil? + + find_manifests_from_jobs(jobs) + end + private def ghcr_client @@ -94,7 +111,7 @@ def annotation_for(image, tag, annotation) response.body.annotations&.[](annotation) end - def find_manifests(jobs) + def find_manifests_from_jobs(jobs) jobs.map { |job| job['name'] } .select { |job| job.start_with?('manifest:') } .map { |job| job.delete_prefix('manifest:') } @@ -116,10 +133,27 @@ def find_container_version(jobs) ) trace.body .lines - .drop_while { |line| !(line.include?('section_start') && line.include?('glpa_summary')) } - .find { |line| line =~ /RETICULUM_CONTAINER_VERSION=[0-9a-zA-Z-.]+$/ } + .drop_while { |line| line !~ /\e\[0Ksection_start:\d+:glpa_summary/ } + .drop(1) + .take_while { |line| line !~ /\e\[0Ksection_end:\d+:glpa_summary/ } + .find { |line| line =~ /RETICULUM_CONTAINER_VERSION=[0-9a-zA-Z\-.]+$/ } .split('=')[1].chomp end + + def load_pipeline(pipeline_id) + pipeline = GitlabClient.client.get_json( + "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipelines/#{pipeline_id}" + ) + return nil if pipeline.response.status == 404 + + [ + pipeline, + GitlabClient.paginate_json( + GitlabClient.client, + "/api/v4/projects/#{Project::Reticulum.api_gitlab_path}/pipelines/#{pipeline_id}/jobs" + ) + ] + end end end end diff --git a/lib/pyxis/permission_helper.rb b/lib/pyxis/permission_helper.rb index 5d7efb5..ad6bf5d 100644 --- a/lib/pyxis/permission_helper.rb +++ b/lib/pyxis/permission_helper.rb @@ -21,6 +21,19 @@ def assert_executed_by_known_team_member! raise PermissionError, 'This operation can only be run by a known team member' end + def assert_executed_by_delivery_team_member! + return unless checks_active? + + assert_executed_by_known_team_member! + + team = GithubClient.octokit.team_by_name(GithubClient::ORGANIZATION_NAME, 'delivery') + user = find_current_user + + return if GithubClient.octokit.team_member?(team.id, user['github']) + + raise PermissionError, 'This operation can only be run by a delivery team member' + end + def find_current_user if ENV['GITLAB_USER_LOGIN'] users.find { |user| user['gitlab'] == ENV['GITLAB_USER_LOGIN'] } diff --git a/lib/pyxis/project/base.rb b/lib/pyxis/project/base.rb index d45d609..5e8ef65 100644 --- a/lib/pyxis/project/base.rb +++ b/lib/pyxis/project/base.rb @@ -32,7 +32,11 @@ def component_name end def self.components - constants.reject { |c| %i[Base Reticulum].include?(c) } + %i[aquila draco sagittarius sculptor taurus] + end + + def self.get_project(project) + const_get(project.capitalize) end end end diff --git a/lib/pyxis/project/codezero.rb b/lib/pyxis/project/codezero.rb new file mode 100644 index 0000000..8b4f3c5 --- /dev/null +++ b/lib/pyxis/project/codezero.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Pyxis + module Project + class Codezero < Base + class << self + def paths + { + github: 'code0-tech/codezero', + } + end + end + end + end +end diff --git a/lib/pyxis/project/pyxis.rb b/lib/pyxis/project/pyxis.rb new file mode 100644 index 0000000..4a67d2d --- /dev/null +++ b/lib/pyxis/project/pyxis.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Pyxis + module Project + class Pyxis < Base + class << self + def paths + { + github: 'code0-tech/pyxis', + gitlab: 'code0-tech/infrastructure/pyxis', + } + end + end + end + end +end diff --git a/lib/pyxis/release/canary.rb b/lib/pyxis/release/canary.rb new file mode 100644 index 0000000..b7d09bc --- /dev/null +++ b/lib/pyxis/release/canary.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Pyxis + module Release + class Canary + include SemanticLogger::Loggable + + def create_build_branch(build_to_promote) + component_information = Pyxis::ManagedVersioning::ComponentInfo.new( + build_id: build_to_promote + ).execute + + raise 'Build not found' if component_information.nil? + + GitlabClient.client.create_branch( + Project::Reticulum.api_gitlab_path, + "pyxis/canary-build/#{build_to_promote}", + component_information[:reticulum] + ) + + version_variables = component_information.map do |component, version| + next nil unless Project.components.include?(component) + + ["OVERRIDE_#{component}_VERSION", version] + end.compact + + Utils::PipelineHelpers.create_env_file( + 'reticulum_variables', + version_variables + [['C0_GH_TOKEN', Pyxis::Environment.github_reticulum_publish_token]] + ) + end + + def remove_build_branch(build_to_promote) + GitlabClient.client.delete_branch( + Project::Reticulum.api_gitlab_path, + "pyxis/canary-build/#{build_to_promote}" + ) + end + + def publish_tags(coordinator_pipeline_id) + build_id = find_build_id(coordinator_pipeline_id) + + info = ManagedVersioning::ComponentInfo.new(build_id: build_id) + common = Common.new + + success = common.copy_container_images_to_release_registry(info) + + raise Pyxis::MessageError, 'Failed to copy all container images' unless success + end + + def publish_release(coordinator_pipeline_id) + build_id = find_build_id(coordinator_pipeline_id) + + info = ManagedVersioning::ComponentInfo.new(build_id: build_id) + common = Common.new + + common.publish_github_release(info, prerelease: true) + end + + private + + def find_build_id(coordinator_pipeline_id) + GitlabClient.client + .list_pipeline_bridges(Project::Pyxis.api_gitlab_path, coordinator_pipeline_id) + .find { |bridge| bridge['name'] == 'release-coordinator:canary:build' } + .dig('downstream_pipeline', 'id') + end + end + end +end diff --git a/lib/pyxis/release/common.rb b/lib/pyxis/release/common.rb new file mode 100644 index 0000000..41f405f --- /dev/null +++ b/lib/pyxis/release/common.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +module Pyxis + module Release + class Common + include SemanticLogger::Loggable + + CI_BUILDS_PREFIX = 'ghcr.io/code0-tech/reticulum/ci-builds/' + CONTAINER_RELEASE_REGISTRY = 'registry.gitlab.com' + CONTAINER_RELEASE_PREFIX = "#{CONTAINER_RELEASE_REGISTRY}/code0-tech/packages/".freeze + + CONTAINER_RELEASE_PUBLISH_USER = 'code0-release-tools' + + CONTAINER_IMAGES_TO_RELEASE = Project.components + %i[config-generator] + + def copy_container_images_to_release_registry(component_info) + container_tag = component_info.find_container_tag_for_build_id + container_tags = component_info.find_manifests.map do |manifest| + next nil unless CONTAINER_IMAGES_TO_RELEASE.include?(manifest.first.to_sym) + + next "#{manifest.first}:#{container_tag}" if manifest.length == 1 + + "#{manifest.first}:#{container_tag}-#{manifest.last}" + end.compact + + success = true + container_tags.each do |tag| + success &&= copy_container_image_to_release_registry(tag) + end + + success + end + + def copy_container_image_to_release_registry(tag) + with_release_registry_auth do + logger.info('Copying container image to release registry', tag: tag) + return if Pyxis::GlobalStatus.dry_run? + + success = system("crane copy #{CI_BUILDS_PREFIX}#{tag} #{CONTAINER_RELEASE_PREFIX}#{tag}") + logger.error('Failed to copy container image to release registry', tag: tag) unless success + success + end + end + + def with_release_registry_auth + token_exists = File.exist?('tmp/gitlab_token') + return yield if token_exists + + logger.info('Authentication with release registry') + File.write('tmp/gitlab_token', Pyxis::Environment.gitlab_release_tools_token) + success = system( + 'crane auth login ' \ + "-u #{CONTAINER_RELEASE_PUBLISH_USER} " \ + "--password-stdin #{CONTAINER_RELEASE_REGISTRY} " \ + '< tmp/gitlab_token' + ) + + unless success + logger.error('Failed to authenticate with release registry') + raise Pyxis::Error, 'Failed to authenticate with release registry' + end + + yield + ensure + unless token_exists + system("crane auth logout #{CONTAINER_RELEASE_REGISTRY}") + File.delete('tmp/gitlab_token') + logger.info('Unauthenticated from release registry') + end + end + + def publish_github_release(component_info, prerelease:) + logger.info('Starting release to codezero repository', tag: component_info.find_container_tag_for_build_id) + + release_version = component_info.find_container_tag_for_build_id + reticulum_sha = component_info.execute(filter_components: [:reticulum])[:reticulum] + + compose_path = 'docker-compose/docker-compose.yml' + env_path = 'docker-compose/.env' + + codezero_compose_content = GithubClient.octokit.contents( + Project::Codezero.github_path, + path: compose_path + ) + codezero_env_content = GithubClient.octokit.contents( + Project::Codezero.github_path, + path: env_path + ) + + reticulum_compose_content = GithubClient.octokit.contents( + Project::Reticulum.github_path, + path: compose_path, + ref: reticulum_sha + ) + reticulum_env_content = GithubClient.octokit.contents( + Project::Reticulum.github_path, + path: env_path, + ref: reticulum_sha + ) + + reticulum_env = Base64.decode64 reticulum_env_content.content + reticulum_compose = Base64.decode64 reticulum_compose_content.content + + codezero_env = reticulum_env.sub('IMAGE_TAG=', "IMAGE_TAG=#{release_version}") + + GithubClient.octokit.update_contents( + Project::Codezero.github_path, + env_path, + "Update compose env for #{release_version}", + codezero_env_content.sha, + codezero_env, + Project::Codezero.default_branch + ) + GithubClient.octokit.update_contents( + Project::Codezero.github_path, + compose_path, + "Update compose file for #{release_version}", + codezero_compose_content.sha, + reticulum_compose, + Project::Codezero.default_branch + ) + + GithubClient.octokit.create_release( + Project::Codezero.github_path, + release_version, + name: release_version, + prerelease: prerelease + ) + end + end + end +end diff --git a/lib/pyxis/services/create_reticulum_build_service.rb b/lib/pyxis/services/create_reticulum_build_service.rb index 10f5968..f110628 100644 --- a/lib/pyxis/services/create_reticulum_build_service.rb +++ b/lib/pyxis/services/create_reticulum_build_service.rb @@ -24,7 +24,7 @@ def execute pipeline = GitlabClient.client.create_pipeline( Project::Reticulum.api_gitlab_path, ref, - variables: version_override_variables + token_variable, + variables: version_override_variables + token_variable ) pipeline.body if pipeline.response.status == 201 @@ -33,7 +33,7 @@ def execute private def validate_override!(component, version) - project = Pyxis::Project.const_get(component.capitalize) + project = Pyxis::Project.get_project(component.capitalize) begin GithubClient.octokit.tag(project.github_path, version) @@ -63,7 +63,7 @@ def token_variable [ { key: 'C0_GH_TOKEN', - value: File.read(ENV.fetch('PYXIS_GH_RETICULUM_PUBLISH_TOKEN')), + value: Pyxis::Environment.github_reticulum_publish_token, } ] end diff --git a/lib/pyxis/utils/pipeline_helpers.rb b/lib/pyxis/utils/pipeline_helpers.rb new file mode 100644 index 0000000..07a2094 --- /dev/null +++ b/lib/pyxis/utils/pipeline_helpers.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Pyxis + module Utils + module PipelineHelpers + module_function + + def create_env_file(name, variables) + path = File.absolute_path(File.join(__FILE__, "../../../../tmp/#{name}.env")) + File.write(path, variables.map { |k, v| "#{k}=#{v}" }.join("\n")) + end + end + end +end diff --git a/tmp/.gitkeep b/tmp/.gitkeep new file mode 100644 index 0000000..e69de29