Add support to accept incomplete release files for S3 artifact upload

.github/actions/setup/action.yml

@@ -4,7 +4,7 @@ description: Installs the given GardenLinux Python library
 inputs:
   version:
     description: GardenLinux Python library version
-    default: "0.10.13"
+    default: "0.10.14"
   python_version:
     description: Python version to setup
     default: "3.13"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gardenlinux"
-version = "0.10.13"
+version = "0.10.14"
 description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames"
 authors = ["Garden Linux Maintainers <contact@gardenlinux.io>"]
 license = "Apache-2.0"

src/gardenlinux/constants.py

@@ -148,6 +148,7 @@
 GL_DEB_REPO_BASE_URL = "https://packages.gardenlinux.io/gardenlinux"
 GL_DISTRIBUTION_NAME = "Garden Linux"
 GL_HOME_URL = "https://gardenlinux.io"
+GL_PLATFORM_FRANKENSTEIN = "frankenstein"
 GL_RELEASE_ID = "gardenlinux"
 GL_REPOSITORY_URL = "https://github.com/gardenlinux/gardenlinux"
 GL_SUPPORT_URL = "https://github.com/gardenlinux/gardenlinux"

src/gardenlinux/features/cname.py

@@ -8,13 +8,14 @@
 from configparser import UNNAMED_SECTION, ConfigParser
 from os import PathLike, environ
 from pathlib import Path
-from typing import List, Optional, Self
+from typing import Any, Dict, List, Optional, Self
 
 from ..constants import (
     ARCHS,
     GL_BUG_REPORT_URL,
     GL_DISTRIBUTION_NAME,
     GL_HOME_URL,
+    GL_PLATFORM_FRANKENSTEIN,
     GL_RELEASE_ID,
     GL_SUPPORT_URL,
 )
@@ -59,14 +60,21 @@ def __init__(
         self._feature_flags_cached: Optional[List[str]] = None
         self._feature_platforms_cached: Optional[List[str]] = None
         self._feature_set_cached: Optional[str] = None
+        self._features_cached: Optional[Dict[str, Any]] = None
+        self._platform_cached: Optional[str] = None
         self._platform_variant_cached: Optional[str] = None
         self._flavor = ""
         self._version = None
 
+        self._flag_frankenstein = bool(environ.get("GL_ALLOW_FRANKENSTEIN", False))
+
         self._flag_multiple_platforms = bool(
-            environ.get("GL_ALLOW_FRANKENSTEIN", False)
+            environ.get("GL_ALLOW_MULTIPLE_PLATFORMS", False)
         )
 
+        if self._flag_frankenstein:
+            self._flag_multiple_platforms = True
+
         commit_id_or_hash = None
 
         if version is not None:
@@ -213,6 +221,20 @@ def flavor(self) -> str:
 
         return self._flavor
 
+    @property
+    def features(self) -> Dict[str, Any]:
+        """
+        Returns the features for the cname parsed.
+
+        :return: (dict) Features of the cname
+        :since:  0.10.14
+        """
+
+        if self._features_cached is None:
+            self._features_cached = Parser().filter_as_dict(self.flavor)
+
+        return self._features_cached
+
     @property
     def feature_set(self) -> str:
         """
@@ -239,7 +261,7 @@ def feature_set_element(self) -> str:
         if self._feature_elements_cached is not None:
             return ",".join(self._feature_elements_cached)
 
-        return ",".join(Parser().filter_as_dict(self.flavor)["element"])
+        return ",".join(self.features["element"])
 
     @property
     def feature_set_flag(self) -> str:
@@ -253,7 +275,7 @@ def feature_set_flag(self) -> str:
         if self._feature_flags_cached is not None:
             return ",".join(self._feature_flags_cached)
 
-        return ",".join(Parser().filter_as_dict(self.flavor)["flag"])
+        return ",".join(self.features["flag"])
 
     @property
     def feature_set_platform(self) -> str:
@@ -265,7 +287,7 @@ def feature_set_platform(self) -> str:
         """
 
         if self._feature_platforms_cached is None:
-            platforms = Parser().filter_as_dict(self.flavor)["platform"]
+            platforms = self.features["platform"]
         else:
             platforms = self._feature_platforms_cached
 
@@ -274,7 +296,7 @@ def feature_set_platform(self) -> str:
 
         assert len(platforms) < 2
         "Only one platform is supported"
-        return platforms[0]
+        return platforms[0]  # type: ignore[no-any-return]
 
     @property
     def feature_set_list(self) -> List[str]:
@@ -293,19 +315,25 @@ def feature_set_list(self) -> List[str]:
     @property
     def platform(self) -> str:
         """
-        Returns the feature set of type "platform" for the cname parsed.
+        Returns the platform for the cname parsed.
 
-        :return: (str) Feature set platforms
+        :return: (str) Platform
         :since:  0.7.0
         """
 
-        if self._feature_platforms_cached is None:
-            platforms = Parser().filter_as_dict(self.flavor)["platform"]
-        else:
+        if self._platform_cached is not None:
+            platforms = [self._platform_cached]
+        elif self._feature_platforms_cached is not None:
             platforms = self._feature_platforms_cached
+        else:
+            platforms = self.features["platform"]
+
+        if self._flag_frankenstein and len(platforms) > 1:
+            return GL_PLATFORM_FRANKENSTEIN
 
         if not self._flag_multiple_platforms:
             assert len(platforms) < 2
+            "Only one platform is supported"
 
         return platforms[0]
 
@@ -345,18 +373,8 @@ def release_metadata_string(self) -> str:
         :since:  1.0.0
         """
 
-        features = Parser().filter_as_dict(self.flavor)
-
-        if not self._flag_multiple_platforms:
-            assert len(features["platform"]) < 2
-            "Only one platform is supported"
-
         commit_hash = self.commit_hash
         commit_id = self.commit_id
-        elements = ",".join(features["element"])
-        flags = ",".join(features["flag"])
-        platform = features["platform"][0]
-        platforms = ",".join(features["platform"])
         platform_variant = self.platform_variant
         version = self.version
 
@@ -387,10 +405,10 @@ def release_metadata_string(self) -> str:
 BUG_REPORT_URL="{GL_BUG_REPORT_URL}"
 GARDENLINUX_CNAME="{self.cname}"
 GARDENLINUX_FEATURES="{self.feature_set}"
-GARDENLINUX_FEATURES_PLATFORMS="{platforms}"
-GARDENLINUX_FEATURES_ELEMENTS="{elements}"
-GARDENLINUX_FEATURES_FLAGS="{flags}"
-GARDENLINUX_PLATFORM="{platform}"
+GARDENLINUX_FEATURES_PLATFORMS="{self.feature_set_platform}"
+GARDENLINUX_FEATURES_ELEMENTS="{self.feature_set_element}"
+GARDENLINUX_FEATURES_FLAGS="{self.feature_set_flag}"
+GARDENLINUX_PLATFORM="{self.platform}"
 GARDENLINUX_PLATFORM_VARIANT="{platform_variant}"
 GARDENLINUX_VERSION="{version}"
 GARDENLINUX_COMMIT_ID="{commit_id}"
@@ -456,6 +474,7 @@ def _copy_from_cname_object(self, cname_object: Self) -> None:
         self._feature_elements_cached = cname_object.feature_set_element.split(",")
         self._feature_flags_cached = cname_object.feature_set_flag.split(",")
         self._feature_platforms_cached = cname_object.feature_set_platform.split(",")
+        self._platform_cached = cname_object.platform
         self._platform_variant_cached = cname_object.platform_variant
         self._version = cname_object.version
 
@@ -477,6 +496,10 @@ def load_from_release_file(self, release_file: PathLike[str] | str) -> None:
                 and self._commit_id != cname_object.commit_id
             )
             or (self._version is not None and self._version != cname_object.version)
+            or (
+                not self._flag_frankenstein
+                and cname_object.platform not in cname_object.feature_set_platform
+            )
         ):
             raise RuntimeError(
                 f"Release metadata file given is invalid: {release_file} failed consistency check - {self.cname} != {cname_object.cname}"
@@ -531,9 +554,7 @@ def new_from_release_file(release_file: PathLike[str] | str) -> "CName":
             "GARDENLINUX_CNAME",
             "GARDENLINUX_COMMIT_ID_LONG",
             "GARDENLINUX_FEATURES",
-            "GARDENLINUX_FEATURES_ELEMENTS",
-            "GARDENLINUX_FEATURES_FLAGS",
-            "GARDENLINUX_FEATURES_PLATFORMS",
+            "GARDENLINUX_PLATFORM",
             "GARDENLINUX_VERSION",
         ):
             if not release_config.has_option(UNNAMED_SECTION, release_field):
@@ -559,23 +580,30 @@ def new_from_release_file(release_file: PathLike[str] | str) -> "CName":
             UNNAMED_SECTION, "GARDENLINUX_FEATURES"
         ).strip("\"'")
 
-        cname_object._feature_elements_cached = (
-            release_config.get(UNNAMED_SECTION, "GARDENLINUX_FEATURES_ELEMENTS")
-            .strip("\"'")
-            .split(",")
-        )
+        if release_config.has_option(UNNAMED_SECTION, "GARDENLINUX_FEATURES_ELEMENTS"):
+            cname_object._feature_elements_cached = (
+                release_config.get(UNNAMED_SECTION, "GARDENLINUX_FEATURES_ELEMENTS")
+                .strip("\"'")
+                .split(",")
+            )
 
-        cname_object._feature_flags_cached = (
-            release_config.get(UNNAMED_SECTION, "GARDENLINUX_FEATURES_FLAGS")
-            .strip("\"'")
-            .split(",")
-        )
+        if release_config.has_option(UNNAMED_SECTION, "GARDENLINUX_FEATURES_FLAGS"):
+            cname_object._feature_flags_cached = (
+                release_config.get(UNNAMED_SECTION, "GARDENLINUX_FEATURES_FLAGS")
+                .strip("\"'")
+                .split(",")
+            )
 
-        cname_object._feature_platforms_cached = (
-            release_config.get(UNNAMED_SECTION, "GARDENLINUX_FEATURES_PLATFORMS")
-            .strip("\"'")
-            .split(",")
-        )
+        if release_config.has_option(UNNAMED_SECTION, "GARDENLINUX_FEATURES_PLATFORMS"):
+            cname_object._feature_platforms_cached = (
+                release_config.get(UNNAMED_SECTION, "GARDENLINUX_FEATURES_PLATFORMS")
+                .strip("\"'")
+                .split(",")
+            )
+
+        cname_object._platform_cached = release_config.get(
+            UNNAMED_SECTION, "GARDENLINUX_PLATFORM"
+        ).strip("\"'")
 
         if release_config.has_option(UNNAMED_SECTION, "GARDENLINUX_PLATFORM_VARIANT"):
             cname_object._platform_variant_cached = release_config.get(

src/gardenlinux/s3/s3_artifacts.py

@@ -120,7 +120,26 @@ def upload_from_directory(
 
         release_file = artifacts_dir.joinpath(f"{base_name}.release")
 
-        cname_object = CName.new_from_release_file(release_file)
+        try:
+            cname_object = CName.new_from_release_file(release_file)
+        except RuntimeError:
+            if not release_file.exists():
+                raise RuntimeError(
+                    f"Release metadata file given is invalid: {release_file}"
+                )
+
+            release_config = ConfigParser(allow_unnamed_section=True)
+            release_config.read(release_file)
+
+            cname_object = CName(
+                release_config.get(UNNAMED_SECTION, "GARDENLINUX_CNAME").strip("\"'"),
+                commit_hash=release_config.get(
+                    UNNAMED_SECTION, "GARDENLINUX_COMMIT_ID_LONG"
+                ).strip("\"'"),
+                version=release_config.get(
+                    UNNAMED_SECTION, "GARDENLINUX_VERSION"
+                ).strip("\"'"),
+            )
 
         if cname_object.version_and_commit_id is None:
             raise RuntimeError(
@@ -133,6 +152,7 @@ def upload_from_directory(
         requirements_file = artifacts_dir.joinpath(f"{base_name}.requirements")
         require_uefi = None
         secureboot = None
+        tpm2 = None
 
         if requirements_file.exists():
             requirements_config = ConfigParser(allow_unnamed_section=True)
@@ -149,6 +169,9 @@ def upload_from_directory(
                     UNNAMED_SECTION, "secureboot"
                 )
 
+            if requirements_config.has_option(UNNAMED_SECTION, "tpm2"):
+                tpm2 = requirements_config.getboolean(UNNAMED_SECTION, "tpm2")
+
         if arch is None:
             raise RuntimeError(
                 "Architecture could not be determined from release or requirements file"
@@ -160,6 +183,9 @@ def upload_from_directory(
         if secureboot is None:
             secureboot = "_trustedboot" in feature_set_list
 
+        if tpm2 is None:
+            tpm2 = "_tpm2" in feature_set_list
+
         # RegEx for S3 supported characters
         re_object = re.compile("[^a-zA-Z0-9\\s+\\-=.\\_:/@]")
 
@@ -170,16 +196,15 @@ def upload_from_directory(
             commit_id_or_hash = cname_object.commit_id
 
         metadata = {
-            "platform": cname_object.feature_set_platform,
+            "platform": cname_object.platform,
             "architecture": arch,
-            "base_image": None,
             "build_committish": commit_id_or_hash,
             "build_timestamp": datetime.fromtimestamp(release_timestamp),
             "logs": None,
             "modifiers": feature_set_list,
             "require_uefi": require_uefi,
             "secureboot": secureboot,
-            "published_image_metadata": None,
+            "tpm2": tpm2,
             "s3_bucket": self._bucket.name,
             "s3_key": f"meta/singles/{base_name}",
             "test_result": None,

tests/constants.py

@@ -14,7 +14,7 @@
 CONTAINER_NAME_ZOT_EXAMPLE = f"{REGISTRY}/{REPO_NAME}"
 GARDENLINUX_ROOT_DIR_EXAMPLE = f"{TEST_DATA_DIR}/gardenlinux/.build"
 
-TEST_PLATFORMS = ["aws", "azure", "gcp", "openstack", "openstackbaremetal", "metal"]
+TEST_PLATFORMS = ["aws", "azure", "baremetal", "gcp", "openstack"]
 TEST_ARCHITECTURES = ["arm64", "amd64"]
 TEST_FEATURE_STRINGS_SHORT = ["gardener_prod"]
 TEST_FEATURE_SET = "_slim,base,container"

tests/features/test_parser.py

@@ -87,9 +87,9 @@
             },
         ),
         (
-            "metal-khost_dev",
+            "openstack-metal-khost_dev",
             {
-                "platform": ["metal"],
+                "platform": ["openstack"],
                 "element": [
                     "firewall",
                     "log",
@@ -99,6 +99,7 @@
                     "server",
                     "chost",
                     "khost",
+                    "metal",
                 ],
                 "flag": ["_dev", "_fwcfg", "_legacy", "_selinux", "_slim"],
             },

tests/s3/conftest.py

@@ -24,7 +24,7 @@ class S3Env:
 
 
 def make_cname(
-    flavor: str = "container",
+    flavor: str = "container_trustedboot_usi",
     arch: str = "amd64",
     version: str = "1234.1",
     commit: str = "abc123long",