diff --git a/lib/LuxLib/test/runtests.jl b/lib/LuxLib/test/runtests.jl
index cc65b85f4..3e58328c0 100644
--- a/lib/LuxLib/test/runtests.jl
+++ b/lib/LuxLib/test/runtests.jl
@@ -20,7 +20,7 @@ if (BACKEND_GROUP == "all" || BACKEND_GROUP == "cuda")
     end
 end
 (BACKEND_GROUP == "all" || BACKEND_GROUP == "amdgpu") &&
-    push!(EXTRA_PKGS, PackageSpec(; name="AMDGPU"))
+    push!(EXTRA_PKGS, PackageSpec(; name="AMDGPU", version=v"1.0.4"))
 (BACKEND_GROUP == "all" || BACKEND_GROUP == "oneapi") &&
     push!(EXTRA_PKGS, PackageSpec(; name="oneAPI"))
 (BACKEND_GROUP == "all" || BACKEND_GROUP == "metal") &&
@@ -30,7 +30,6 @@ if !isempty(EXTRA_PKGS) || !isempty(EXTRA_DEV_PKGS)
     @info "Installing Extra Packages for testing" EXTRA_PKGS EXTRA_DEV_PKGS
     isempty(EXTRA_PKGS) || Pkg.add(EXTRA_PKGS)
     isempty(EXTRA_DEV_PKGS) || Pkg.develop(EXTRA_DEV_PKGS)
-    Pkg.update()
     Base.retry_load_extensions()
     Pkg.instantiate()
 end
diff --git a/lib/MLDataDevices/test/runtests.jl b/lib/MLDataDevices/test/runtests.jl
index 4b02862e3..4617a3a5d 100644
--- a/lib/MLDataDevices/test/runtests.jl
+++ b/lib/MLDataDevices/test/runtests.jl
@@ -15,7 +15,7 @@ if (BACKEND_GROUP == "all" || BACKEND_GROUP == "cuda")
     end
 end
 (BACKEND_GROUP == "all" || BACKEND_GROUP == "amdgpu") &&
-    push!(EXTRA_PKGS, PackageSpec(; name="AMDGPU"))
+    push!(EXTRA_PKGS, PackageSpec(; name="AMDGPU", version=v"1.0.4"))
 (BACKEND_GROUP == "all" || BACKEND_GROUP == "oneapi") &&
     push!(EXTRA_PKGS, PackageSpec(; name="oneAPI"))
 (BACKEND_GROUP == "all" || BACKEND_GROUP == "metal") &&
@@ -27,7 +27,6 @@ if !isempty(EXTRA_PKGS) || !isempty(EXTRA_DEV_PKGS)
     @info "Installing Extra Packages for testing" EXTRA_PKGS EXTRA_DEV_PKGS
     isempty(EXTRA_PKGS) || Pkg.add(EXTRA_PKGS)
     isempty(EXTRA_DEV_PKGS) || Pkg.develop(EXTRA_DEV_PKGS)
-    Pkg.update()
     Base.retry_load_extensions()
     Pkg.instantiate()
 end
diff --git a/lib/WeightInitializers/test/runtests.jl b/lib/WeightInitializers/test/runtests.jl
index 9de7d16bf..ff223653a 100644
--- a/lib/WeightInitializers/test/runtests.jl
+++ b/lib/WeightInitializers/test/runtests.jl
@@ -5,17 +5,20 @@ using InteractiveUtils, Hwloc
 
 const BACKEND_GROUP = lowercase(get(ENV, "BACKEND_GROUP", "All"))
 
-const EXTRA_PKGS = String[]
+const EXTRA_PKGS = PackageSpec[]
 
-(BACKEND_GROUP == "all" || BACKEND_GROUP == "cuda") && push!(EXTRA_PKGS, "CUDA")
-(BACKEND_GROUP == "all" || BACKEND_GROUP == "amdgpu") && push!(EXTRA_PKGS, "AMDGPU")
-(BACKEND_GROUP == "all" || BACKEND_GROUP == "metal") && push!(EXTRA_PKGS, "Metal")
-(BACKEND_GROUP == "all" || BACKEND_GROUP == "oneapi") && push!(EXTRA_PKGS, "oneAPI")
+(BACKEND_GROUP == "all" || BACKEND_GROUP == "cuda") &&
+    push!(EXTRA_PKGS, PackageSpec("CUDA"))
+(BACKEND_GROUP == "all" || BACKEND_GROUP == "amdgpu") &&
+    push!(EXTRA_PKGS, PackageSpec(; name="AMDGPU", version=v"1.0.4"))
+(BACKEND_GROUP == "all" || BACKEND_GROUP == "metal") &&
+    push!(EXTRA_PKGS, PackageSpec("Metal"))
+(BACKEND_GROUP == "all" || BACKEND_GROUP == "oneapi") &&
+    push!(EXTRA_PKGS, PackageSpec("oneAPI"))
 
 if !isempty(EXTRA_PKGS)
     @info "Installing Extra Packages for testing" EXTRA_PKGS=EXTRA_PKGS
     Pkg.add(EXTRA_PKGS)
-    Pkg.update()
     Base.retry_load_extensions()
     Pkg.instantiate()
 end
diff --git a/test/runtests.jl b/test/runtests.jl
index 4ae064c70..0e1935d46 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -44,13 +44,12 @@ if (BACKEND_GROUP == "all" || BACKEND_GROUP == "cuda")
     end
 end
 (BACKEND_GROUP == "all" || BACKEND_GROUP == "amdgpu") &&
-    push!(EXTRA_PKGS, Pkg.PackageSpec("AMDGPU"))
+    push!(EXTRA_PKGS, Pkg.PackageSpec(; name="AMDGPU", version=v"1.0.4"))
 
 if !isempty(EXTRA_PKGS) || !isempty(EXTRA_DEV_PKGS)
     @info "Installing Extra Packages for testing" EXTRA_PKGS EXTRA_DEV_PKGS
     isempty(EXTRA_PKGS) || Pkg.add(EXTRA_PKGS)
     isempty(EXTRA_DEV_PKGS) || Pkg.develop(EXTRA_DEV_PKGS)
-    Pkg.update()
     Base.retry_load_extensions()
     Pkg.instantiate()
     Pkg.precompile()