From 413f398cff6d1211650bd793caf5e51140e5e61e Mon Sep 17 00:00:00 2001 From: Jesse Natalie Date: Thu, 20 Jan 2022 05:50:02 -0800 Subject: [PATCH] ci/windows: Use 2 container stages The first container stage ("build") is for dependencies of the build. These are infrequently-changing things like Visual Studio, LLVM, git, and also meson. The second container stage ("test") currently depends on the first, and adds test dependencies like piglit. This lets us rev piglit without having to rebuild LLVM. Acked-by: Daniel Stone Part-of: --- .gitlab-ci.yml | 54 +++++++++++--- .gitlab-ci/image-tags.yml | 5 +- .../windows/{Dockerfile => Dockerfile_build} | 4 +- .gitlab-ci/windows/Dockerfile_test | 7 ++ .gitlab-ci/windows/mesa_container.ps1 | 4 +- .../{mesa_deps.ps1 => mesa_deps_build.ps1} | 70 +------------------ .gitlab-ci/windows/mesa_deps_test.ps1 | 70 +++++++++++++++++++ src/gallium/drivers/d3d12/ci/gitlab-ci.yml | 3 +- src/microsoft/ci/gitlab-ci.yml | 3 +- 9 files changed, 138 insertions(+), 82 deletions(-) rename .gitlab-ci/windows/{Dockerfile => Dockerfile_build} (84%) create mode 100644 .gitlab-ci/windows/Dockerfile_test rename .gitlab-ci/windows/{mesa_deps.ps1 => mesa_deps_build.ps1} (72%) create mode 100644 .gitlab-ci/windows/mesa_deps_test.ps1 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 73c5159962a..52fa75b9458 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -568,40 +568,78 @@ debian/arm_test: # then in the build stage by executing inside Docker. .windows-docker-vs2019: - extends: - - .set-image variables: MESA_IMAGE: "$CI_REGISTRY_IMAGE/${MESA_IMAGE_PATH}:${MESA_IMAGE_TAG}" - MESA_IMAGE_PATH: ${WINDOWS_X64_BUILD_PATH} - MESA_IMAGE_TAG: ${WINDOWS_X64_BUILD_TAG} MESA_UPSTREAM_IMAGE: "$CI_REGISTRY/$FDO_UPSTREAM_REPO/$MESA_IMAGE_PATH:${MESA_IMAGE_TAG}" -windows_build_vs2019: +.windows_container_build: inherit: default: false extends: - .container - .windows-docker-vs2019 - stage: container variables: GIT_STRATEGY: fetch # we do actually need the full repository though - timeout: 2h 30m # LLVM + piglit takes ages + MESA_BASE_IMAGE: None tags: - windows - shell - "1809" - mesa script: - - .\.gitlab-ci\windows\mesa_container.ps1 $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $MESA_IMAGE $MESA_UPSTREAM_IMAGE + - .\.gitlab-ci\windows\mesa_container.ps1 $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $MESA_IMAGE $MESA_UPSTREAM_IMAGE ${DOCKERFILE} ${MESA_BASE_IMAGE} + +windows_build_vs2019: + inherit: + default: false + extends: + - .windows_container_build + variables: + MESA_IMAGE_PATH: &windows_build_image_path ${WINDOWS_X64_BUILD_PATH} + MESA_IMAGE_TAG: &windows_build_image_tag ${WINDOWS_X64_BUILD_TAG} + DOCKERFILE: Dockerfile_build + timeout: 2h 30m # LLVM takes ages + +windows_test_vs2019: + inherit: + default: false + extends: + - .windows_container_build + # Don't want the .container rules + - .ci-run-policy + variables: + MESA_IMAGE_PATH: &windows_test_image_path ${WINDOWS_X64_TEST_PATH} + MESA_IMAGE_TAG: &windows_test_image_tag ${WINDOWS_X64_BUILD_TAG}--${WINDOWS_X64_TEST_TAG} + DOCKERFILE: Dockerfile_test + # Right now this only needs the VS install to get DXIL.dll. Maybe see about decoupling this at some point + MESA_BASE_IMAGE_PATH: *windows_build_image_path + MESA_BASE_IMAGE_TAG: *windows_build_image_tag + MESA_BASE_IMAGE: "$CI_REGISTRY_IMAGE/${MESA_BASE_IMAGE_PATH}:${MESA_BASE_IMAGE_TAG}" + script: + - .\.gitlab-ci\windows\mesa_container.ps1 $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $MESA_IMAGE $MESA_UPSTREAM_IMAGE Dockerfile_test ${MESA_BASE_IMAGE} + needs: + - windows_build_vs2019 .use-windows_build_vs2019: inherit: default: false extends: .windows-docker-vs2019 image: "$MESA_IMAGE" + variables: + MESA_IMAGE_PATH: *windows_build_image_path + MESA_IMAGE_TAG: *windows_build_image_tag needs: - windows_build_vs2019 +.use-windows_test_vs2019: + inherit: + default: false + extends: .windows-docker-vs2019 + image: "$MESA_IMAGE" + variables: + MESA_IMAGE_PATH: *windows_test_image_path + MESA_IMAGE_TAG: *windows_test_image_tag + # Git archive diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index 319661863cf..911c08b43cf 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -15,4 +15,7 @@ variables: KERNEL_ROOTFS_TAG: "2022-01-18-kernel" WINDOWS_X64_BUILD_PATH: "windows/x64_build" - WINDOWS_X64_BUILD_TAG: "2021-12-31-refactor" + WINDOWS_X64_BUILD_TAG: "2022-20-02-base_split" + + WINDOWS_X64_TEST_PATH: "windows/x64_test" + WINDOWS_X64_TEST_TAG: "2022-20-02-base_split" diff --git a/.gitlab-ci/windows/Dockerfile b/.gitlab-ci/windows/Dockerfile_build similarity index 84% rename from .gitlab-ci/windows/Dockerfile rename to .gitlab-ci/windows/Dockerfile_build index 430112ff1a4..f940504243d 100644 --- a/.gitlab-ci/windows/Dockerfile +++ b/.gitlab-ci/windows/Dockerfile_build @@ -9,5 +9,5 @@ ENV ErrorActionPreference='Stop' COPY mesa_deps_vs2019.ps1 C:\ RUN C:\mesa_deps_vs2019.ps1 -COPY mesa_deps.ps1 C:\ -RUN C:\mesa_deps.ps1 +COPY mesa_deps_build.ps1 C:\ +RUN C:\mesa_deps_build.ps1 diff --git a/.gitlab-ci/windows/Dockerfile_test b/.gitlab-ci/windows/Dockerfile_test new file mode 100644 index 00000000000..106e493358f --- /dev/null +++ b/.gitlab-ci/windows/Dockerfile_test @@ -0,0 +1,7 @@ +# escape=` + +ARG base_image +FROM ${base_image} + +COPY mesa_deps_test.ps1 C:\ +RUN C:\mesa_deps_test.ps1 diff --git a/.gitlab-ci/windows/mesa_container.ps1 b/.gitlab-ci/windows/mesa_container.ps1 index 7a2f6c2378b..cbb9e223508 100644 --- a/.gitlab-ci/windows/mesa_container.ps1 +++ b/.gitlab-ci/windows/mesa_container.ps1 @@ -6,6 +6,8 @@ $registry_username = $args[1] $registry_password = $args[2] $registry_user_image = $args[3] $registry_central_image = $args[4] +$build_dockerfile = $args[5] +$registry_base_image = $args[6] Set-Location -Path ".\.gitlab-ci\windows" @@ -39,7 +41,7 @@ if ($?) { } Write-Host "No image found at $registry_user_image or $registry_central_image; rebuilding" -docker --config "windows-docker.conf" build --no-cache -t "$registry_user_image" . +docker --config "windows-docker.conf" build --no-cache -t "$registry_user_image" -f "$build_dockerfile" --build-arg base_image="$registry_base_image" . if (!$?) { Write-Host "Container build failed" docker --config "windows-docker.conf" logout "$registry_uri" diff --git a/.gitlab-ci/windows/mesa_deps.ps1 b/.gitlab-ci/windows/mesa_deps_build.ps1 similarity index 72% rename from .gitlab-ci/windows/mesa_deps.ps1 rename to .gitlab-ci/windows/mesa_deps_build.ps1 index 597aed15b72..a771ca254bc 100644 --- a/.gitlab-ci/windows/mesa_deps.ps1 +++ b/.gitlab-ci/windows/mesa_deps_build.ps1 @@ -129,6 +129,8 @@ if (!$buildstatus) { Exit 1 } +# See https://gitlab.freedesktop.org/mesa/mesa/-/issues/3855 +# Until that's resolved, we need the vulkan-runtime as a build dependency to be able to run any unit tests on GL Get-Date Write-Host "Downloading Vulkan-Runtime" Invoke-WebRequest -Uri 'https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime.exe' -OutFile 'C:\vulkan-runtime.exe' | Out-Null @@ -140,73 +142,5 @@ if (!$?) { } Remove-Item C:\vulkan-runtime.exe -Force -Get-Date -Write-Host "Downloading Freeglut" - -$freeglut_zip = 'freeglut-MSVC.zip' -$freeglut_url = "https://www.transmissionzero.co.uk/files/software/development/GLUT/$freeglut_zip" - -For ($i = 0; $i -lt 5; $i++) { - Invoke-WebRequest -Uri $freeglut_url -OutFile $freeglut_zip - $freeglut_downloaded = $? - if ($freeglut_downloaded) { - Break - } -} - -if (!$freeglut_downloaded) { - Write-Host "Failed to download Freeglut" - Exit 1 -} - -Get-Date -Write-Host "Installing Freeglut" -Expand-Archive $freeglut_zip -DestinationPath C:\ -if (!$?) { - Write-Host "Failed to install Freeglut" - Exit 1 -} - -Get-Date -Write-Host "Downloading glext.h" -New-Item -ItemType Directory -Path ".\glext" -Name "GL" -$ProgressPreference = "SilentlyContinue" -Invoke-WebRequest -Uri 'https://www.khronos.org/registry/OpenGL/api/GL/glext.h' -OutFile '.\glext\GL\glext.h' | Out-Null - -Get-Date -Write-Host "Cloning Piglit" -git clone --no-progress --single-branch --no-checkout https://gitlab.freedesktop.org/mesa/piglit.git 'C:\src\piglit' -if (!$?) { - Write-Host "Failed to clone Piglit repository" - Exit 1 -} -Push-Location -Path C:\src\piglit -git checkout f7f2a6c2275cae023a27b6cc81be3dda8c99492d -Pop-Location - -Get-Date -$piglit_build = New-Item -ItemType Directory -Path "C:\src\piglit" -Name "build" -Push-Location -Path $piglit_build.FullName -Write-Host "Compiling Piglit" -cmd.exe /C 'C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="C:\Piglit" -DGLUT_INCLUDE_DIR=C:\freeglut\include -DGLUT_glut_LIBRARY_RELEASE=C:\freeglut\lib\x64\freeglut.lib -DGLEXT_INCLUDE_DIR=.\glext && ninja -j32' -$buildstatus = $? -ninja -j32 install | Out-Null -$installstatus = $? -Pop-Location -Remove-Item -Recurse -Path $piglit_build -if (!$buildstatus -Or !$installstatus) { - Write-Host "Failed to compile or install Piglit" - Exit 1 -} - -Copy-Item -Path C:\freeglut\bin\x64\freeglut.dll -Destination C:\Piglit\lib\piglit\bin\freeglut.dll - -Get-Date -Write-Host "Cloning spirv-samples" -git clone --no-progress --single-branch --no-checkout https://github.com/dneto0/spirv-samples.git C:\spirv-samples\ -Push-Location -Path C:\spirv-samples\ -git checkout 7ac0ad5a7fe0ec884faba1dc2916028d0268eeef -Pop-Location - Get-Date Write-Host "Complete" diff --git a/.gitlab-ci/windows/mesa_deps_test.ps1 b/.gitlab-ci/windows/mesa_deps_test.ps1 new file mode 100644 index 00000000000..441025db3e8 --- /dev/null +++ b/.gitlab-ci/windows/mesa_deps_test.ps1 @@ -0,0 +1,70 @@ +Get-Date +Write-Host "Downloading Freeglut" + +$freeglut_zip = 'freeglut-MSVC.zip' +$freeglut_url = "https://www.transmissionzero.co.uk/files/software/development/GLUT/$freeglut_zip" + +For ($i = 0; $i -lt 5; $i++) { + Invoke-WebRequest -Uri $freeglut_url -OutFile $freeglut_zip + $freeglut_downloaded = $? + if ($freeglut_downloaded) { + Break + } +} + +if (!$freeglut_downloaded) { + Write-Host "Failed to download Freeglut" + Exit 1 +} + +Get-Date +Write-Host "Installing Freeglut" +Expand-Archive $freeglut_zip -DestinationPath C:\ +if (!$?) { + Write-Host "Failed to install Freeglut" + Exit 1 +} + +Get-Date +Write-Host "Downloading glext.h" +New-Item -ItemType Directory -Path ".\glext" -Name "GL" +$ProgressPreference = "SilentlyContinue" +Invoke-WebRequest -Uri 'https://www.khronos.org/registry/OpenGL/api/GL/glext.h' -OutFile '.\glext\GL\glext.h' | Out-Null + +Get-Date +Write-Host "Cloning Piglit" +git clone --no-progress --single-branch --no-checkout https://gitlab.freedesktop.org/mesa/piglit.git 'C:\src\piglit' +if (!$?) { + Write-Host "Failed to clone Piglit repository" + Exit 1 +} +Push-Location -Path C:\src\piglit +git checkout f7f2a6c2275cae023a27b6cc81be3dda8c99492d +Pop-Location + +Get-Date +$piglit_build = New-Item -ItemType Directory -Path "C:\src\piglit" -Name "build" +Push-Location -Path $piglit_build.FullName +Write-Host "Compiling Piglit" +cmd.exe /C 'C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="C:\Piglit" -DGLUT_INCLUDE_DIR=C:\freeglut\include -DGLUT_glut_LIBRARY_RELEASE=C:\freeglut\lib\x64\freeglut.lib -DGLEXT_INCLUDE_DIR=.\glext && ninja -j32' +$buildstatus = $? +ninja -j32 install | Out-Null +$installstatus = $? +Pop-Location +Remove-Item -Recurse -Path $piglit_build +if (!$buildstatus -Or !$installstatus) { + Write-Host "Failed to compile or install Piglit" + Exit 1 +} + +Copy-Item -Path C:\freeglut\bin\x64\freeglut.dll -Destination C:\Piglit\lib\piglit\bin\freeglut.dll + +Get-Date +Write-Host "Cloning spirv-samples" +git clone --no-progress --single-branch --no-checkout https://github.com/dneto0/spirv-samples.git C:\spirv-samples\ +Push-Location -Path C:\spirv-samples\ +git checkout 7ac0ad5a7fe0ec884faba1dc2916028d0268eeef +Pop-Location + +Get-Date +Write-Host "Complete" diff --git a/src/gallium/drivers/d3d12/ci/gitlab-ci.yml b/src/gallium/drivers/d3d12/ci/gitlab-ci.yml index 43743bc68c3..8dd726a49da 100644 --- a/src/gallium/drivers/d3d12/ci/gitlab-ci.yml +++ b/src/gallium/drivers/d3d12/ci/gitlab-ci.yml @@ -1,13 +1,14 @@ .d3d12-test: extends: - .build-windows - - .use-windows_build_vs2019 + - .use-windows_test_vs2019 - .windows-test-rules stage: layered-backends dependencies: - windows-vs2019 needs: - windows-vs2019 + - windows_test_vs2019 variables: GIT_STRATEGY: none # testing doesn't build anything from source GALLIUM_DRIVER: d3d12 diff --git a/src/microsoft/ci/gitlab-ci.yml b/src/microsoft/ci/gitlab-ci.yml index c67037ca44d..7557fc219e3 100644 --- a/src/microsoft/ci/gitlab-ci.yml +++ b/src/microsoft/ci/gitlab-ci.yml @@ -1,13 +1,14 @@ test-spirv2dxil-windows: extends: - .build-windows - - .use-windows_build_vs2019 + - .use-windows_test_vs2019 - .windows-test-rules stage: layered-backends dependencies: - windows-vs2019 needs: - windows-vs2019 + - windows_test_vs2019 variables: GIT_STRATEGY: none # testing doesn't build anything from source script: