diff --git a/builders/win-python-builder.psm1 b/builders/win-python-builder.psm1 index 33148d0b..0044cf4d 100644 --- a/builders/win-python-builder.psm1 +++ b/builders/win-python-builder.psm1 @@ -25,6 +25,7 @@ class WinPythonBuilder : PythonBuilder { [string] $InstallationTemplateName [string] $InstallationScriptName [string] $OutputArtifactName + [string] $PackageLocation WinPythonBuilder( [semver] $version, @@ -36,6 +37,80 @@ class WinPythonBuilder : PythonBuilder { $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.zip" } + [bool] UsesPackageManifest() { + <# + .SYNOPSIS + Python 3.16+ no longer ships legacy Windows installers (PEP 773), only packages listed in a release manifest. + #> + + return $this.Version.Major -eq 3 -and $this.Version.Minor -ge 16 + } + + [string] GetManifestPackageId([string] $package, [bool] $freethreaded) { + <# + .SYNOPSIS + Return the id of a package ("pythoncore" or "pythontest") in the Windows release manifest for this architecture. + #> + + $hardware = switch ($this.GetHardwareArchitecture()) { + "x86" { "32" } + "x64" { "64" } + "arm64" { "arm64" } + default { throw "Unsupported architecture: $($this.Architecture)" } + } + $suffix = if ($freethreaded) { "t" } else { "" } + + return "${package}-$($this.Version.Major).$($this.Version.Minor)${suffix}-${hardware}" + } + + [uri] GetManifestUri() { + <# + .SYNOPSIS + Return URI of the Windows release manifest, e.g. windows-3.16.0a1.json + #> + + $base = $this.GetBaseUri() + $versionName = $this.GetBaseVersion() + $nativeVersion = Convert-Version -version $this.Version + + return "${base}/${versionName}/windows-${nativeVersion}.json" + } + + [object] GetManifest() { + <# + .SYNOPSIS + Download and parse the Windows release manifest. + #> + + $manifestUri = $this.GetManifestUri() + Write-Host "Manifest URI: $manifestUri" + + return Invoke-RestMethod -Uri $manifestUri + } + + [string] DownloadPackage([object] $manifest, [string] $packageId) { + <# + .SYNOPSIS + Download a package listed in the Windows release manifest and validate its SHA-256 hash. + #> + + $package = $manifest.versions | Where-Object { $_.id -eq $packageId } + if ($null -eq $package) { + throw "Package '$packageId' not found in the Windows release manifest" + } + + Write-Host "Sources URI: $($package.url)" + $packagePath = Download-File -Uri $package.url -OutputFolder $this.WorkFolderLocation + Write-Debug "Done; Sources location: $packagePath" + + $actualHash = (Get-FileHash -Path $packagePath -Algorithm SHA256).Hash + if ($actualHash -ne $package.hash.sha256) { + throw "SHA-256 mismatch for $($package.url): expected $($package.hash.sha256), got $actualHash" + } + + return $packagePath + } + [string] GetPythonExtension() { <# .SYNOPSIS @@ -84,6 +159,21 @@ class WinPythonBuilder : PythonBuilder { Download Python installation executable into artifact location. #> + if ($this.UsesPackageManifest()) { + $manifest = $this.GetManifest() + + # The "pythontest" package is "pythoncore" plus Lib/test, for consistency with the legacy installers. + $packageId = $this.GetManifestPackageId("pythontest", $this.IsFreeThreaded()) + + # 3.16.0a1 has no free-threaded "pythontest" package, so fall back to "pythoncore" (no Lib/test) for those. + if (-not ($manifest.versions | Where-Object { $_.id -eq $packageId })) { + Write-Host "Package '$packageId' not found, using 'pythoncore' instead" + $packageId = $this.GetManifestPackageId("pythoncore", $this.IsFreeThreaded()) + } + + return $this.DownloadPackage($manifest, $packageId) + } + $sourceUri = $this.GetSourceUri() Write-Host "Sources URI: $sourceUri" @@ -99,8 +189,12 @@ class WinPythonBuilder : PythonBuilder { Create Python artifact installation script based on specified template. #> - $sourceUri = $this.GetSourceUri() - $pythonExecName = [IO.path]::GetFileName($sourceUri.AbsoluteUri) + if ($this.UsesPackageManifest()) { + $pythonExecName = [IO.path]::GetFileName($this.PackageLocation) + } else { + $sourceUri = $this.GetSourceUri() + $pythonExecName = [IO.path]::GetFileName($sourceUri.AbsoluteUri) + } $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File @@ -129,7 +223,7 @@ class WinPythonBuilder : PythonBuilder { #> Write-Host "Download Python $($this.Version) [$($this.Architecture)] executable..." - $this.Download() + $this.PackageLocation = $this.Download() Write-Host "Create installation script..." $this.CreateInstallationScript() diff --git a/installers/win-setup-template.ps1 b/installers/win-setup-template.ps1 index ad93a2dd..5ad78089 100644 --- a/installers/win-setup-template.ps1 +++ b/installers/win-setup-template.ps1 @@ -83,6 +83,7 @@ $PythonToolcachePath = Join-Path -Path $ToolcacheRoot -ChildPath "Python" $PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version $PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture +$IsZip = $PythonExecName -match "\.zip$" $IsMSI = $PythonExecName -match "msi" $IsFreeThreaded = $Architecture -match "-freethreaded" @@ -114,27 +115,38 @@ if ($null -ne $InstalledVersions) { Write-Host "No Python$MajorVersion.$MinorVersion.* found" } -Write-Host "Remove registry entries for Python ${MajorVersion}.${MinorVersion}(${Architecture})..." -Remove-RegistryEntries -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion +if (-not $IsZip) { + Write-Host "Remove registry entries for Python ${MajorVersion}.${MinorVersion}(${Architecture})..." + Remove-RegistryEntries -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion +} Write-Host "Create Python $Version folder in $PythonToolcachePath" New-Item -ItemType Directory -Path $PythonArchPath -Force | Out-Null -Write-Host "Copy Python binaries to $PythonArchPath" -Copy-Item -Path ./$PythonExecName -Destination $PythonArchPath | Out-Null +if ($IsZip) { + # Python 3.16+ is distributed as a package (PEP 773) that can be extracted and run as-is + Write-Host "Extract Python $Version in $PythonToolcachePath..." + Expand-Archive -Path ./$PythonExecName -DestinationPath $PythonArchPath -Force -Write-Host "Install Python $Version in $PythonToolcachePath..." -$ExecParams = Get-ExecParams -IsMSI $IsMSI -IsFreeThreaded $IsFreeThreaded -PythonArchPath $PythonArchPath + # Debug symbols are only in the "pythontest" package and were not installed by the legacy installers + Get-ChildItem -Path $PythonArchPath -Recurse -Filter "*.pdb" | Remove-Item -Force +} else { + Write-Host "Copy Python binaries to $PythonArchPath" + Copy-Item -Path ./$PythonExecName -Destination $PythonArchPath | Out-Null -cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet" -if ($LASTEXITCODE -ne 0) { - Throw "Error happened during Python installation" -} + Write-Host "Install Python $Version in $PythonToolcachePath..." + $ExecParams = Get-ExecParams -IsMSI $IsMSI -IsFreeThreaded $IsFreeThreaded -PythonArchPath $PythonArchPath -if ($IsFreeThreaded) { - # Delete python.exe and create a symlink to free-threaded exe - Remove-Item -Path "$PythonArchPath\python.exe" -Force - New-Item -Path "$PythonArchPath\python.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python${MajorVersion}.${MinorVersion}t.exe" + cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet" + if ($LASTEXITCODE -ne 0) { + Throw "Error happened during Python installation" + } + + if ($IsFreeThreaded) { + # Delete python.exe and create a symlink to free-threaded exe + Remove-Item -Path "$PythonArchPath\python.exe" -Force + New-Item -Path "$PythonArchPath\python.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python${MajorVersion}.${MinorVersion}t.exe" + } } Write-Host "Create `python3` symlink" diff --git a/tests/python-tests.ps1 b/tests/python-tests.ps1 index e1dc7a74..addd3b6e 100644 --- a/tests/python-tests.ps1 +++ b/tests/python-tests.ps1 @@ -66,6 +66,32 @@ Describe "Tests" { } } + # Windows free-threaded Python 3.16+ is installed from a "PythonCore" package, as there is no free-threaded "PythonTest" package with Lib/test + $SkipTestSuite = ($Platform -eq "win32") -and ($Architecture -match "-freethreaded") -and ([semver]$Version -ge [semver]"3.16.0-alpha.1") + It "Check if the standard library test suite (Lib/test) is installed" -Skip:$SkipTestSuite { + "python ./sources/stdlib-test-suite.py" | Should -ReturnZeroExitCode + } + + if ($Platform -eq "win32") { + It "pip.exe is from the toolcache" { + $pipLocation = (Get-Command "pip").Path + $pipLocation | Should -Not -BeNullOrEmpty + $expectedPath = Join-Path -Path $env:RUNNER_TOOL_CACHE -ChildPath "Python" + $pipLocation.startsWith($expectedPath) | Should -BeTrue + } + + # TODO: Python 3.16+ is extracted from a package, so it is not registered for the legacy py.exe launcher + $SkipPyLauncher = [semver]$Version -ge [semver]"3.16.0-alpha.1" + It "py.exe finds Python" -Skip:$SkipPyLauncher { + $suffix = if ($Architecture -match "-freethreaded") { "t" } else { "" } + $major, $minor = "$Version".Split(".")[0..1] + $pyLocation = py "-$major.$minor$suffix" -c "import sys; print(sys.executable)" + $LASTEXITCODE | Should -Be 0 + $expectedPath = Join-Path -Path $env:RUNNER_TOOL_CACHE -ChildPath "Python" + $pyLocation.startsWith($expectedPath) | Should -BeTrue + } + } + It "Run pip" { "pip install requests" | Should -ReturnZeroExitCode "pip uninstall requests -y" | Should -ReturnZeroExitCode diff --git a/tests/sources/stdlib-test-suite.py b/tests/sources/stdlib-test-suite.py new file mode 100644 index 00000000..0b1b9a13 --- /dev/null +++ b/tests/sources/stdlib-test-suite.py @@ -0,0 +1,19 @@ +import importlib.util +import os.path +import sys + + +def check_test_suite(): + spec = importlib.util.find_spec("test") + if spec is None or not spec.submodule_search_locations: + print("The 'test' package (Lib/test) is not installed") + sys.exit(1) + + test_dir = list(spec.submodule_search_locations)[0] + for name in ("support", "libregrtest"): + if not os.path.isdir(os.path.join(test_dir, name)): + print("Missing test.{} in {}".format(name, test_dir)) + sys.exit(1) + + +check_test_suite()