From 78a6a3748b05067920c6c1b8a8679370d316463e Mon Sep 17 00:00:00 2001 From: Kaushalt2004 Date: Tue, 10 Feb 2026 10:05:43 +0530 Subject: [PATCH 1/2] Android testbed: ensure managed emulator has 4GB RAM --- Android/README.md | 8 +- Android/android.py | 38 +-- Android/testbed/app/build.gradle.kts | 416 ++++++++++++++++++++------- 3 files changed, 336 insertions(+), 126 deletions(-) diff --git a/Android/README.md b/Android/README.md index 9f71aeb934f386..0b3608406801c1 100644 --- a/Android/README.md +++ b/Android/README.md @@ -103,13 +103,7 @@ require adding your user to a group, or changing your udev rules. On GitHub Actions, the test script will do this automatically using the commands shown [here](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). -The test suite can usually be run on a device with 2 GB of RAM, but this is -borderline, so you may need to increase it to 4 GB. As of Android -Studio Koala, 2 GB is the default for all emulators, although the user interface -may indicate otherwise. Locate the emulator's directory under `~/.android/avd`, -and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these -manually to the same value, or use the Android Studio Device Manager, which will -update both files. +The Android testbed's Gradle-managed emulator is configured to use 4 GB of RAM. You can run the test suite either: diff --git a/Android/android.py b/Android/android.py index 629696be3db300..0681004905b737 100755 --- a/Android/android.py +++ b/Android/android.py @@ -38,16 +38,19 @@ APP_ID = "org.python.testbed" DECODE_ARGS = ("UTF-8", "backslashreplace") +def get_android_home() -> Path: + try: + return Path(os.environ["ANDROID_HOME"]) + except KeyError: + raise SystemExit("The ANDROID_HOME environment variable is required.") -try: - android_home = Path(os.environ['ANDROID_HOME']) -except KeyError: - sys.exit("The ANDROID_HOME environment variable is required.") -adb = Path( - f"{android_home}/platform-tools/adb" - + (".exe" if os.name == "nt" else "") -) +def get_adb() -> Path: + android_home = get_android_home() + return Path( + f"{android_home}/platform-tools/adb" + + (".exe" if os.name == "nt" else "") + ) gradlew = Path( f"{TESTBED_DIR}/gradlew" @@ -316,17 +319,20 @@ def setup_ci(): else: raise ValueError(f"Failed to find NDK version in {ENV_SCRIPT.name}") - for item in (android_home / "ndk").iterdir(): + for item in (get_android_home() / "ndk").iterdir(): if item.name[0].isdigit() and item.name != ndk_version: delete_glob(item) def setup_sdk(): + android_home = get_android_home() sdkmanager = android_home / ( "cmdline-tools/latest/bin/sdkmanager" + (".bat" if os.name == "nt" else "") ) + adb = get_adb() + # Gradle will fail if it needs to install an SDK package whose license # hasn't been accepted, so pre-accept all licenses. if not all((android_home / "licenses" / path).exists() for path in [ @@ -437,7 +443,7 @@ async def list_devices(): serials = [] header_found = False - lines = (await async_check_output(adb, "devices")).splitlines() + lines = (await async_check_output(get_adb(), "devices")).splitlines() for line in lines: # Ignore blank lines, and all lines before the header. line = line.strip() @@ -487,9 +493,9 @@ async def find_pid(serial): try: # `pidof` requires API level 24 or higher. The level 23 emulator # includes it, but it doesn't work (it returns all processes). - pid = (await async_check_output( - adb, "-s", serial, "shell", "pidof", "-s", APP_ID - )).strip() + pid = (await async_check_output( + get_adb(), "-s", serial, "shell", "pidof", "-s", APP_ID + )).strip() except CalledProcessError as e: # If the app isn't running yet, pidof gives no output. So if there # is output, there must have been some other error. However, this @@ -525,7 +531,7 @@ async def logcat_task(context, initial_devices): # long`). For example, every time pytest runs a test, it prints a "." and # flushes the stream. Each "." becomes a separate log message, but we should # show them all on the same line. - args = [adb, "-s", serial, "logcat", "--pid", pid, "--binary"] + args = [get_adb(), "-s", serial, "logcat", "--pid", pid, "--binary"] logcat_started = False async with async_process( *args, stdout=subprocess.PIPE, stderr=None @@ -599,7 +605,7 @@ async def read_int(size): payload_fields = (await read_bytes(payload_len - 1)).split(b"\0") if len(payload_fields) < 2: raise ValueError( - f"payload {payload!r} does not contain at least 2 " + f"payload {payload_fields!r} does not contain at least 2 " f"null-separated fields" ) tag, message, *_ = [ @@ -609,7 +615,7 @@ async def read_int(size): def stop_app(serial): - run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False) + run([get_adb(), "-s", serial, "shell", "am", "force-stop", APP_ID], log=False) async def gradle_task(context): diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index 53cdc591fa35fd..c87f3b91e4e544 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -1,11 +1,22 @@ import com.android.build.api.variant.* import kotlin.math.max +import java.io.File plugins { id("com.android.application") id("org.jetbrains.kotlin.android") } +val requestedTaskNames = gradle.startParameter.taskNames + .filterNot { it.startsWith("-") } + .map { it.substringAfterLast(":") } +val isEmulatorSetupOnly = requestedTaskNames.isNotEmpty() && requestedTaskNames.all { + it == "help" || + it == "tasks" || + it.endsWith("Setup") || + it.endsWith("PatchAvdRam") +} + val ANDROID_DIR = file("../..") val PYTHON_DIR = ANDROID_DIR.parentFile!! val PYTHON_CROSS_DIR = file("$PYTHON_DIR/cross-build") @@ -20,58 +31,60 @@ val KNOWN_ABIS = mapOf( // Discover prefixes. val prefixes = ArrayList() -if (inSourceTree) { - for ((triplet, _) in KNOWN_ABIS.entries) { - val prefix = file("$PYTHON_CROSS_DIR/$triplet/prefix") - if (prefix.exists()) { - prefixes.add(prefix) - } - } -} else { - // Testbed is inside a release package. - val prefix = file("$ANDROID_DIR/prefix") - if (prefix.exists()) { - prefixes.add(prefix) - } -} -if (prefixes.isEmpty()) { - throw GradleException( - "No Android prefixes found: see README.md for testing instructions" - ) -} - -// Detect Python versions and ABIs. lateinit var pythonVersion: String var abis = HashMap() -for ((i, prefix) in prefixes.withIndex()) { - val libDir = file("$prefix/lib") - val version = run { - for (filename in libDir.list()!!) { - """python(\d+\.\d+[a-z]*)""".toRegex().matchEntire(filename)?.let { - return@run it.groupValues[1] +if (!isEmulatorSetupOnly) { + if (inSourceTree) { + for ((triplet, _) in KNOWN_ABIS.entries) { + val prefix = file("$PYTHON_CROSS_DIR/$triplet/prefix") + if (prefix.exists()) { + prefixes.add(prefix) } } - throw GradleException("Failed to find Python version in $libDir") + } else { + // Testbed is inside a release package. + val prefix = file("$ANDROID_DIR/prefix") + if (prefix.exists()) { + prefixes.add(prefix) + } } - if (i == 0) { - pythonVersion = version - } else if (pythonVersion != version) { + if (prefixes.isEmpty()) { throw GradleException( - "${prefixes[0]} is Python $pythonVersion, but $prefix is Python $version" + "No Android prefixes found: see README.md for testing instructions" ) } - val libPythonDir = file("$libDir/python$pythonVersion") - val triplet = run { - for (filename in libPythonDir.list()!!) { - """_sysconfigdata_[a-z]*_android_(.+).py""".toRegex() - .matchEntire(filename)?.let { + // Detect Python versions and ABIs. + for ((i, prefix) in prefixes.withIndex()) { + val libDir = file("$prefix/lib") + val version = run { + for (filename in libDir.list()!!) { + """python(\d+\.\d+[a-z]*)""".toRegex().matchEntire(filename)?.let { return@run it.groupValues[1] } + } + throw GradleException("Failed to find Python version in $libDir") + } + if (i == 0) { + pythonVersion = version + } else if (pythonVersion != version) { + throw GradleException( + "${prefixes[0]} is Python $pythonVersion, but $prefix is Python $version" + ) } - throw GradleException("Failed to find Python triplet in $libPythonDir") + + val libPythonDir = file("$libDir/python$pythonVersion") + val triplet = run { + for (filename in libPythonDir.list()!!) { + """_sysconfigdata_[a-z]*_android_(.+).py""".toRegex() + .matchEntire(filename)?.let { + return@run it.groupValues[1] + } + } + throw GradleException("Failed to find Python triplet in $libPythonDir") + } + abis[prefix] = KNOWN_ABIS[triplet]!! } - abis[prefix] = KNOWN_ABIS[triplet]!! } @@ -105,18 +118,20 @@ android { versionCode = 1 versionName = "1.0" - ndk.abiFilters.addAll(abis.values) - externalNativeBuild.cmake.arguments( - "-DPYTHON_PREFIX_DIR=" + if (inSourceTree) { - // AGP uses the ${} syntax for its own purposes, so use a Jinja style - // placeholder. - "$PYTHON_CROSS_DIR/{{triplet}}/prefix" - } else { - prefixes[0] - }, - "-DPYTHON_VERSION=$pythonVersion", - "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", - ) + if (!isEmulatorSetupOnly) { + ndk.abiFilters.addAll(abis.values) + externalNativeBuild.cmake.arguments( + "-DPYTHON_PREFIX_DIR=" + if (inSourceTree) { + // AGP uses the ${} syntax for its own purposes, so use a Jinja style + // placeholder. + "$PYTHON_CROSS_DIR/{{triplet}}/prefix" + } else { + prefixes[0] + }, + "-DPYTHON_VERSION=$pythonVersion", + "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", + ) + } testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" } @@ -129,8 +144,10 @@ android { } throw GradleException("Failed to find NDK version in $androidEnvFile") } - externalNativeBuild.cmake { - path("src/main/c/CMakeLists.txt") + if (!isEmulatorSetupOnly) { + externalNativeBuild.cmake { + path("src/main/c/CMakeLists.txt") + } } // Set this property to something nonexistent but non-empty. Otherwise it'll use the @@ -170,7 +187,70 @@ android { // If the previous test run succeeded and nothing has changed, // Gradle thinks there's no need to run it again. Override that. afterEvaluate { - (localDevices.names + listOf("connected")).forEach { + val managedDeviceNames = localDevices.names + + // Ensure the emulator has enough RAM to run the CPython test suite. + // As of Android Studio Koala / AGP 8.x, Gradle Managed Devices may + // create an AVD with 2 GB even when the device profile specifies + // more. Patch the AVD config after the *Setup task creates it. + managedDeviceNames.forEach { deviceName -> + val patchTaskName = "${deviceName}PatchAvdRam" + val patchTask = tasks.register(patchTaskName) { + group = "verification" + description = "Patch Gradle managed device AVD RAM to 4 GB for '$deviceName'." + + // Ensure the AVD exists before we try to patch it. + dependsOn("${deviceName}Setup") + + doLast { + val ramMb = 4096 + val avdHome = resolveAvdHome() + if (!avdHome.exists()) { + throw GradleException( + "AVD home directory not found: $avdHome (set ANDROID_AVD_HOME or ANDROID_USER_HOME)" + ) + } + + val avdDir = findManagedDeviceAvdDirectory(avdHome, deviceName) + ?: throw GradleException( + "Failed to locate AVD directory for '$deviceName' under $avdHome (including gradle-managed)" + ) + + logger.lifecycle( + "Patching AVD RAM for '$deviceName': avdHome=$avdHome, avdDir=$avdDir, ramMb=$ramMb" + ) + + val configIni = File(avdDir, "config.ini") + val hardwareQemuIni = File(avdDir, "hardware-qemu.ini") + + val changedConfig = setIniProperty( + configIni, "hw.ramSize", ramMb.toString() + ) + val changedHardware = setIniProperty( + hardwareQemuIni, "hw.ramSize", ramMb.toString() + ) + + logger.lifecycle( + "AVD RAM patch for '$deviceName': config.ini=" + + (if (changedConfig) "updated" else "unchanged") + + ", hardware-qemu.ini=" + + (if (changedHardware) "updated" else "unchanged") + ) + } + } + + // Ensure running *Setup also patches RAM (and running tests + // indirectly via *DebugAndroidTest will get this too). + tasks.matching { it.name == "${deviceName}Setup" } + .configureEach { finalizedBy(patchTask) } + + // Also run the patch whenever tests run, even if *Setup is + // up-to-date from a previous invocation. + tasks.matching { it.name == "${deviceName}DebugAndroidTest" } + .configureEach { dependsOn(patchTask) } + } + + (managedDeviceNames + listOf("connected")).forEach { tasks.named("${it}DebugAndroidTest") { outputs.upToDateWhen { false } } @@ -180,6 +260,134 @@ android { } } + +fun resolveAvdHome(): File { + // ANDROID_AVD_HOME overrides the whole AVD directory. + System.getenv("ANDROID_AVD_HOME")?.let { return File(it) } + + // ANDROID_USER_HOME is the parent of the .android directory (see Android docs). + val androidUserHome = System.getenv("ANDROID_USER_HOME") + ?: System.getProperty("user.home") + return File(File(androidUserHome, ".android"), "avd") +} + + +fun findManagedDeviceAvdDirectory(avdHome: File, deviceName: String): File? { + // Gradle Managed Devices create AVDs under /gradle-managed. + // The AVD name is generated from device parameters and doesn't necessarily + // match the managed device name (e.g. "maxVersion"). + val homes = listOf(avdHome, File(avdHome, "gradle-managed")) + .filter { it.exists() } + + for (home in homes) { + findAvdDirectory(home, deviceName)?.let { return it } + } + + // As a fallback, pick the most recently modified AVD under gradle-managed. + // This runs immediately after *Setup, so the newest one should correspond + // to the device that was just created. + for (home in homes) { + findNewestAvdDirectory(home)?.let { return it } + } + + return null +} + + +fun findNewestAvdDirectory(avdHome: File): File? { + val candidates = avdHome.listFiles()?.filter { + it.isDirectory && it.name.endsWith(".avd") + } ?: return null + return candidates.maxByOrNull { it.lastModified() } +} + + +fun findAvdDirectory(avdHome: File, deviceName: String): File? { + // Prefer the AVD pointer file when it exists: .ini contains the path + // to the corresponding .avd directory. + resolveAvdDirFromIni(avdHome, "$deviceName.ini")?.let { return it } + + val exact = File(avdHome, "$deviceName.avd") + if (exact.exists()) return exact + + // Gradle may add suffixes; try ini files first (more deterministic than + // scanning directories), then fall back to directory names. + val iniCandidates = avdHome.listFiles()?.filter { + it.isFile && it.name.endsWith(".ini") && ( + it.name == "$deviceName.ini" || + it.name.startsWith("${deviceName}_") || + it.name.startsWith(deviceName) + ) + }?.sortedBy { it.name } ?: emptyList() + + for (ini in iniCandidates) { + resolveAvdDirFromIni(avdHome, ini.name)?.let { return it } + } + + val dirCandidates = avdHome.listFiles()?.filter { + it.isDirectory && it.name.endsWith(".avd") && ( + it.name == "$deviceName.avd" || + it.name.startsWith("${deviceName}_") || + it.name.startsWith(deviceName) + ) + }?.sortedBy { it.name } ?: emptyList() + + return dirCandidates.firstOrNull() +} + + +fun resolveAvdDirFromIni(avdHome: File, iniFilename: String): File? { + val iniFile = File(avdHome, iniFilename) + if (!iniFile.exists()) return null + + for (line in iniFile.readLines()) { + if (line.startsWith("path=")) { + val pathValue = line.removePrefix("path=").trim() + if (pathValue.isEmpty()) return null + val path = File(pathValue) + return if (path.isAbsolute) path else File(avdHome, pathValue) + } + } + return null +} + + +fun setIniProperty(file: File, key: String, value: String): Boolean { + val newLine = "$key=$value" + val newline = System.lineSeparator() + + if (!file.exists()) { + file.parentFile?.mkdirs() + file.writeText(newLine + newline) + return true + } + + val lines = file.readLines().toMutableList() + var found = false + var changed = false + + for (i in lines.indices) { + if (lines[i].startsWith("$key=")) { + found = true + if (lines[i] != newLine) { + lines[i] = newLine + changed = true + } + } + } + + if (!found) { + lines.add(newLine) + changed = true + } + + if (changed) { + file.writeText(lines.joinToString(newline) + newline) + } + + return changed +} + dependencies { implementation("androidx.appcompat:appcompat:1.6.1") implementation("com.google.android.material:material:1.11.0") @@ -191,68 +399,70 @@ dependencies { // Create some custom tasks to copy Python and its standard library from // elsewhere in the repository. -androidComponents.onVariants { variant -> - val pyPlusVer = "python$pythonVersion" - generateTask(variant, variant.sources.assets!!) { - into("python") { - // Include files such as pyconfig.h are used by some of the tests. - into("include/$pyPlusVer") { - for (prefix in prefixes) { - from("$prefix/include/$pyPlusVer") +if (!isEmulatorSetupOnly) { + androidComponents.onVariants { variant -> + val pyPlusVer = "python$pythonVersion" + generateTask(variant, variant.sources.assets!!) { + into("python") { + // Include files such as pyconfig.h are used by some of the tests. + into("include/$pyPlusVer") { + for (prefix in prefixes) { + from("$prefix/include/$pyPlusVer") + } + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - } - into("lib/$pyPlusVer") { - // To aid debugging, the source directory takes priority when - // running inside a CPython source tree. - if (inSourceTree) { - from("$PYTHON_DIR/Lib") - } - for (prefix in prefixes) { - from("$prefix/lib/$pyPlusVer") - } + into("lib/$pyPlusVer") { + // To aid debugging, the source directory takes priority when + // running inside a CPython source tree. + if (inSourceTree) { + from("$PYTHON_DIR/Lib") + } + for (prefix in prefixes) { + from("$prefix/lib/$pyPlusVer") + } - into("site-packages") { - from("$projectDir/src/main/python") + into("site-packages") { + from("$projectDir/src/main/python") - val sitePackages = findProperty("python.sitePackages") as String? - if (!sitePackages.isNullOrEmpty()) { - if (!file(sitePackages).exists()) { - throw GradleException("$sitePackages does not exist") + val sitePackages = findProperty("python.sitePackages") as String? + if (!sitePackages.isNullOrEmpty()) { + if (!file(sitePackages).exists()) { + throw GradleException("$sitePackages does not exist") + } + from(sitePackages) } - from(sitePackages) } - } - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - exclude("**/__pycache__") - } + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + exclude("**/__pycache__") + } - into("cwd") { - val cwd = findProperty("python.cwd") as String? - if (!cwd.isNullOrEmpty()) { - if (!file(cwd).exists()) { - throw GradleException("$cwd does not exist") + into("cwd") { + val cwd = findProperty("python.cwd") as String? + if (!cwd.isNullOrEmpty()) { + if (!file(cwd).exists()) { + throw GradleException("$cwd does not exist") + } + from(cwd) } - from(cwd) } - } - // A filename ending with .gz will be automatically decompressed - // while building the APK. Avoid this by adding a dash to the end, - // and add an extra dash to any filenames that already end with one. - // This will be undone in MainActivity.kt. - rename(""".*(\.gz|-)""", "$0-") + // A filename ending with .gz will be automatically decompressed + // while building the APK. Avoid this by adding a dash to the end, + // and add an extra dash to any filenames that already end with one. + // This will be undone in MainActivity.kt. + rename(""".*(\.gz|-)""", "$0-") + } } - } - generateTask(variant, variant.sources.jniLibs!!) { - for ((prefix, abi) in abis.entries) { - into(abi) { - from("$prefix/lib") - include("libpython*.*.so") - include("lib*_python.so") + generateTask(variant, variant.sources.jniLibs!!) { + for ((prefix, abi) in abis.entries) { + into(abi) { + from("$prefix/lib") + include("libpython*.*.so") + include("lib*_python.so") + } } } } From 7f41bcfd7a13d37554f8e996936b3f47dd5353e5 Mon Sep 17 00:00:00 2001 From: Kaushalt2004 Date: Tue, 10 Feb 2026 10:43:02 +0530 Subject: [PATCH 2/2] gh-144418: add NEWS entry --- .../next/Build/2026-02-10-00-00-00.gh-issue-144418.Kaushal.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Build/2026-02-10-00-00-00.gh-issue-144418.Kaushal.rst diff --git a/Misc/NEWS.d/next/Build/2026-02-10-00-00-00.gh-issue-144418.Kaushal.rst b/Misc/NEWS.d/next/Build/2026-02-10-00-00-00.gh-issue-144418.Kaushal.rst new file mode 100644 index 00000000000000..e17c0b99d1b7bd --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-02-10-00-00-00.gh-issue-144418.Kaushal.rst @@ -0,0 +1 @@ +Ensure the Android testbed's Gradle-managed emulator has enough RAM to run the CPython test suite reliably.