diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 617e97225f..c2fb86e794 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,7 +1,7 @@
 steps:
   - group: ":julia: CUDA GPU"
     steps:
-      - label: ":julia: Julia {{matrix.julia}} + CUDA GPU"
+      - label: ":julia: Julia {{matrix.julia}} + {{matrix.test_group}} + CUDA GPU"
         plugins:
           - JuliaCI/julia#v1:
               version: "{{matrix.julia}}"
@@ -16,13 +16,22 @@ steps:
           queue: "juliagpu"
           cuda: "*"
         env:
-          GROUP: "CUDA"
+          BACKEND_GROUP: "CUDA"
+          LUX_TEST_GROUP: "{{matrix.test_group}}"
         if: build.message !~ /\[skip tests\]/
         timeout_in_minutes: 240
         matrix:
           setup:
             julia:
               - "1"
+            test_group:
+              - "core_layers"
+              - "contrib"
+              - "helpers"
+              - "distributed"
+              - "normalize_layers"
+              - "others"
+              - "recurrent_layers"
 
   - group: ":telescope: Downstream CUDA"
     steps:
@@ -80,7 +89,7 @@ steps:
 
   - group: ":julia: AMD GPU"
     steps:
-      - label: ":julia: Julia: {{matrix.julia}} + AMD GPU"
+      - label: ":julia: Julia: {{matrix.julia}} + {{matrix.test_group}} + AMD GPU"
         plugins:
           - JuliaCI/julia#v1:
               version: "{{matrix.julia}}"
@@ -96,7 +105,8 @@ steps:
           JULIA_AMDGPU_CORE_MUST_LOAD: "1"
           JULIA_AMDGPU_HIP_MUST_LOAD: "1"
           JULIA_AMDGPU_DISABLE_ARTIFACTS: "1"
-          GROUP: "AMDGPU"
+          BACKEND_GROUP: "AMDGPU"
+          LUX_TEST_GROUP: "{{matrix.test_group}}"
         agents:
           queue: "juliagpu"
           rocm: "*"
@@ -107,6 +117,14 @@ steps:
           setup:
             julia:
               - "1"
+            test_group:
+              - "core_layers"
+              - "contrib"
+              - "helpers"
+              - "distributed"
+              - "normalize_layers"
+              - "others"
+              - "recurrent_layers"
 
   - group: ":telescope: Downstream AMD GPU"
     steps:
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index 2ad20dea15..3ffbb999b3 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -19,6 +19,14 @@ jobs:
       matrix:
         version:
           - "1"
+        test_group:
+          - "core_layers"
+          - "contrib"
+          - "helpers"
+          - "distributed"
+          - "normalize_layers"
+          - "others"
+          - "recurrent_layers"
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v2
@@ -37,7 +45,8 @@ jobs:
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
         env:
-          GROUP: "CPU"
+          BACKEND_GROUP: "CPU"
+          LUX_TEST_GROUP: ${{ matrix.test_group }}
           RETESTITEMS_NWORKERS: 4
           RETESTITEMS_NWORKER_THREADS: 2
       - uses: julia-actions/julia-processcoverage@v1
diff --git a/.github/workflows/Downgrade.yml b/.github/workflows/Downgrade.yml
index 269275ed5f..6f59392bcd 100644
--- a/.github/workflows/Downgrade.yml
+++ b/.github/workflows/Downgrade.yml
@@ -16,6 +16,14 @@ jobs:
     strategy:
       matrix:
         version: ['1']
+        test_group:
+          - "core_layers"
+          - "contrib"
+          - "helpers"
+          - "distributed"
+          - "normalize_layers"
+          - "others"
+          - "recurrent_layers"
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v2
@@ -27,7 +35,8 @@ jobs:
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
         env:
-          GROUP: "CPU"
+          BACKEND_GROUP: "CPU"
+          LUX_TEST_GROUP: ${{ matrix.test_group }}
           RETESTITEMS_NWORKERS: 4
           RETESTITEMS_NWORKER_THREADS: 2
       - uses: julia-actions/julia-processcoverage@v1
diff --git a/Manifest.toml b/Manifest.toml
index d404456db3..16cbc11851 100644
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -328,8 +328,8 @@ version = "0.1.20"
     Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [[deps.LuxLib]]
-deps = ["ArrayInterface", "ChainRulesCore", "FastBroadcast", "FastClosures", "KernelAbstractions", "LinearAlgebra", "LuxCore", "Markdown", "NNlib", "PrecompileTools", "Random", "Reexport", "Statistics"]
-git-tree-sha1 = "de9f3a2155d5fbf86bdf3391435876e1c203fc2b"
+deps = ["ArrayInterface", "ChainRulesCore", "FastBroadcast", "FastClosures", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "LuxCore", "Markdown", "NNlib", "PrecompileTools", "Random", "Reexport", "Statistics", "Strided"]
+git-tree-sha1 = "67c205a9b325f10e5751fc8f0c3ec2df042c6b3f"
 repo-rev = "ap/fused_dense"
 repo-url = "https://github.com/LuxDL/LuxLib.jl.git"
 uuid = "82251201-b29d-42c6-8e01-566dec8acb11"
@@ -413,6 +413,12 @@ git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
 uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
 version = "0.5.5+0"
 
+[[deps.PackageExtensionCompat]]
+git-tree-sha1 = "fb28e33b8a95c4cee25ce296c817d89cc2e53518"
+uuid = "65ce6f38-6b18-4e1d-a461-8949797d7930"
+version = "1.0.2"
+weakdeps = ["Requires", "TOML"]
+
 [[deps.PartialFunctions]]
 deps = ["MacroTools"]
 git-tree-sha1 = "47b49a4dbc23b76682205c646252c0f9e1eb75af"
@@ -554,6 +560,24 @@ git-tree-sha1 = "b518da45c50dfab8384125ba829f1739bda41034"
 uuid = "7792a7ef-975c-4747-a70f-980b88e8d1da"
 version = "0.5.5"
 
+[[deps.Strided]]
+deps = ["LinearAlgebra", "StridedViews", "TupleTools"]
+git-tree-sha1 = "40c69be0e1b72ee2f42923b7d1ff13e0b04e675c"
+uuid = "5e0ebb24-38b0-5f93-81fe-25c709ecae67"
+version = "2.0.4"
+
+[[deps.StridedViews]]
+deps = ["LinearAlgebra", "PackageExtensionCompat"]
+git-tree-sha1 = "5b765c4e401693ab08981989f74a36a010aa1d8e"
+uuid = "4db3bf67-4bd7-4b4e-b153-31dc3fb37143"
+version = "0.2.2"
+
+    [deps.StridedViews.extensions]
+    StridedViewsCUDAExt = "CUDA"
+
+    [deps.StridedViews.weakdeps]
+    CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+
 [[deps.SuiteSparse]]
 deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
 uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"
@@ -579,6 +603,11 @@ git-tree-sha1 = "eda08f7e9818eb53661b3deb74e3159460dfbc27"
 uuid = "8290d209-cae3-49c0-8002-c8c24d57dab5"
 version = "0.5.2"
 
+[[deps.TupleTools]]
+git-tree-sha1 = "41d61b1c545b06279871ef1a4b5fcb2cac2191cd"
+uuid = "9d95972d-f1c8-5527-a6e0-b4b365fa01f6"
+version = "1.5.0"
+
 [[deps.UUIDs]]
 deps = ["Random", "SHA"]
 uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
diff --git a/test/contrib/debug_tests.jl b/test/contrib/debug_tests.jl
index 803a09a3cb..82bb215e07 100644
--- a/test/contrib/debug_tests.jl
+++ b/test/contrib/debug_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Debugging Tools: DimensionMismatch" setup=[SharedTestSetup] begin
+@testitem "Debugging Tools: DimensionMismatch" setup=[SharedTestSetup] tags=[:contrib] begin
     using Logging
 
     rng = get_stable_rng(12345)
@@ -43,7 +43,7 @@
     end
 end
 
-@testitem "Debugging Tools: NaN" setup=[SharedTestSetup] begin
+@testitem "Debugging Tools: NaN" setup=[SharedTestSetup] tags=[:contrib] begin
     using Logging, ChainRulesCore
     import ChainRulesCore as CRC
 
diff --git a/test/contrib/freeze_tests.jl b/test/contrib/freeze_tests.jl
index 828331d614..d5f3cc30d4 100644
--- a/test/contrib/freeze_tests.jl
+++ b/test/contrib/freeze_tests.jl
@@ -1,4 +1,4 @@
-@testitem "All Parameter Freezing" setup=[SharedTestSetup] begin
+@testitem "All Parameter Freezing" setup=[SharedTestSetup] tags=[:contrib] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -61,7 +61,7 @@
     end
 end
 
-@testitem "Partial Freezing" setup=[SharedTestSetup] begin
+@testitem "Partial Freezing" setup=[SharedTestSetup] tags=[:contrib] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
diff --git a/test/contrib/map_tests.jl b/test/contrib/map_tests.jl
index c152746bb3..9264a00b6c 100644
--- a/test/contrib/map_tests.jl
+++ b/test/contrib/map_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Layer Map" setup=[SharedTestSetup] begin
+@testitem "Layer Map" setup=[SharedTestSetup] tags=[:contrib] begin
     using Setfield
 
     function zero_dense_params_1(l, ps, st, name)
diff --git a/test/contrib/share_parameters_tests.jl b/test/contrib/share_parameters_tests.jl
index b8ab24dee1..707489f798 100644
--- a/test/contrib/share_parameters_tests.jl
+++ b/test/contrib/share_parameters_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Parameter Sharing" setup=[SharedTestSetup] begin
+@testitem "Parameter Sharing" setup=[SharedTestSetup] tags=[:contrib] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
diff --git a/test/contrib/training_tests.jl b/test/contrib/training_tests.jl
index ded09da6c8..bdb9028527 100644
--- a/test/contrib/training_tests.jl
+++ b/test/contrib/training_tests.jl
@@ -1,4 +1,4 @@
-@testitem "TrainState" setup=[SharedTestSetup] begin
+@testitem "TrainState" setup=[SharedTestSetup] tags=[:contrib] begin
     using Optimisers
 
     rng = get_stable_rng(12345)
@@ -22,7 +22,7 @@
     end
 end
 
-@testitem "AbstractADTypes" setup=[SharedTestSetup] begin
+@testitem "AbstractADTypes" setup=[SharedTestSetup] tags=[:contrib] begin
     using ADTypes, Optimisers
 
     function _loss_function(model, ps, st, data)
diff --git a/test/core_tests.jl b/test/core_tests.jl
index d6c4c42f8f..8276aae8d4 100644
--- a/test/core_tests.jl
+++ b/test/core_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Functors Compatibility" setup=[SharedTestSetup] begin
+@testitem "Functors Compatibility" setup=[SharedTestSetup] tags=[:others] begin
     using Functors
 
     rng = get_stable_rng(12345)
diff --git a/test/helpers/compact_tests.jl b/test/helpers/compact_tests.jl
index caa37fe3d0..e057c77251 100644
--- a/test/helpers/compact_tests.jl
+++ b/test/helpers/compact_tests.jl
@@ -1,4 +1,4 @@
-@testitem "@compact" setup=[SharedTestSetup] begin
+@testitem "@compact" setup=[SharedTestSetup] tags=[:helpers] begin
     using ComponentArrays
 
     rng = get_stable_rng(12345)
diff --git a/test/layers/basic_tests.jl b/test/layers/basic_tests.jl
index 3c5c126fc5..d361a311cc 100644
--- a/test/layers/basic_tests.jl
+++ b/test/layers/basic_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Miscellaneous Layers" setup=[SharedTestSetup] begin
+@testitem "Miscellaneous Layers" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -72,7 +72,7 @@
     end
 end
 
-@testitem "Dense" setup=[SharedTestSetup] begin
+@testitem "Dense" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -148,7 +148,7 @@ end
     end
 end
 
-@testitem "Scale" setup=[SharedTestSetup] begin
+@testitem "Scale" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -218,7 +218,7 @@ end
     end
 end
 
-@testitem "Bilinear" setup=[SharedTestSetup] begin
+@testitem "Bilinear" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -312,7 +312,7 @@ end
     end
 end
 
-@testitem "Embedding" setup=[SharedTestSetup] begin
+@testitem "Embedding" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
diff --git a/test/layers/containers_tests.jl b/test/layers/containers_tests.jl
index 46177272d2..e35ba05944 100644
--- a/test/layers/containers_tests.jl
+++ b/test/layers/containers_tests.jl
@@ -1,5 +1,5 @@
 
-@testitem "SkipConnection" setup=[SharedTestSetup] begin
+@testitem "SkipConnection" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -31,7 +31,7 @@
     end
 end
 
-@testitem "Parallel" setup=[SharedTestSetup] begin
+@testitem "Parallel" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -149,7 +149,7 @@ end
     end
 end
 
-@testitem "PairwiseFusion" setup=[SharedTestSetup] begin
+@testitem "PairwiseFusion" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -198,7 +198,7 @@ end
     end
 end
 
-@testitem "BranchLayer" setup=[SharedTestSetup] begin
+@testitem "BranchLayer" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -232,7 +232,7 @@ end
     end
 end
 
-@testitem "Chain" setup=[SharedTestSetup] begin
+@testitem "Chain" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -308,7 +308,7 @@ end
     end
 end
 
-@testitem "Maxout" setup=[SharedTestSetup] begin
+@testitem "Maxout" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -370,7 +370,7 @@ end
     end
 end
 
-@testitem "Repeated" setup=[SharedTestSetup] begin
+@testitem "Repeated" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
diff --git a/test/layers/conv_tests.jl b/test/layers/conv_tests.jl
index 5ee530a230..bf92427693 100644
--- a/test/layers/conv_tests.jl
+++ b/test/layers/conv_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Pooling" setup=[SharedTestSetup] begin
+@testitem "Pooling" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, dev, ongpu) in MODES
@@ -94,7 +94,7 @@
     end
 end
 
-@testitem "CNN" setup=[SharedTestSetup] begin
+@testitem "CNN" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, dev, ongpu) in MODES
@@ -369,7 +369,7 @@ end
     end
 end
 
-@testitem "Upsample" setup=[SharedTestSetup] begin
+@testitem "Upsample" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, dev, ongpu) in MODES
@@ -443,7 +443,7 @@ end
     end
 end
 
-@testitem "PixelShuffle" setup=[SharedTestSetup] begin
+@testitem "PixelShuffle" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, dev, ongpu) in MODES
@@ -475,7 +475,7 @@ end
     end
 end
 
-@testitem "CrossCor" setup=[SharedTestSetup] begin
+@testitem "CrossCor" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, dev, ongpu) in MODES
@@ -555,7 +555,7 @@ end
     end
 end
 
-@testitem "ConvTranspose" setup=[SharedTestSetup] begin
+@testitem "ConvTranspose" setup=[SharedTestSetup] tags=[:core_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, dev, ongpu) in MODES
diff --git a/test/layers/dropout_tests.jl b/test/layers/dropout_tests.jl
index b45cb1ad5c..7c8abb618c 100644
--- a/test/layers/dropout_tests.jl
+++ b/test/layers/dropout_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Dropout" setup=[SharedTestSetup] begin
+@testitem "Dropout" setup=[SharedTestSetup] tags=[:normalize_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -28,7 +28,7 @@
     end
 end
 
-@testitem "AlphaDropout" setup=[SharedTestSetup] begin
+@testitem "AlphaDropout" setup=[SharedTestSetup] tags=[:normalize_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -59,7 +59,7 @@ end
     end
 end
 
-@testitem "VariationalHiddenDropout" setup=[SharedTestSetup] begin
+@testitem "VariationalHiddenDropout" setup=[SharedTestSetup] tags=[:normalize_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
diff --git a/test/layers/dynamic_expressions_tests.jl b/test/layers/dynamic_expressions_tests.jl
index 0a56ae7b5c..4d4d3bf5cd 100644
--- a/test/layers/dynamic_expressions_tests.jl
+++ b/test/layers/dynamic_expressions_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Dynamic Expressions" setup=[SharedTestSetup] begin
+@testitem "Dynamic Expressions" setup=[SharedTestSetup] tags=[:others] begin
     using DynamicExpressions
 
     operators = OperatorEnum(; binary_operators=[+, -, *], unary_operators=[cos])
diff --git a/test/layers/normalize_tests.jl b/test/layers/normalize_tests.jl
index c5b6bc89cb..1f6a926c0f 100644
--- a/test/layers/normalize_tests.jl
+++ b/test/layers/normalize_tests.jl
@@ -1,4 +1,4 @@
-@testitem "BatchNorm" setup=[SharedTestSetup] begin
+@testitem "BatchNorm" setup=[SharedTestSetup] tags=[:normalize_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -104,7 +104,7 @@
     end
 end
 
-@testitem "GroupNorm" setup=[SharedTestSetup] begin
+@testitem "GroupNorm" setup=[SharedTestSetup] tags=[:normalize_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -189,7 +189,7 @@ end
     end
 end
 
-@testitem "WeightNorm" setup=[SharedTestSetup] begin
+@testitem "WeightNorm" setup=[SharedTestSetup] tags=[:normalize_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -308,7 +308,7 @@ end
     end
 end
 
-@testitem "LayerNorm" setup=[SharedTestSetup] begin
+@testitem "LayerNorm" setup=[SharedTestSetup] tags=[:normalize_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -364,7 +364,7 @@ end
     end
 end
 
-@testitem "InstanceNorm" setup=[SharedTestSetup] begin
+@testitem "InstanceNorm" setup=[SharedTestSetup] tags=[:normalize_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
diff --git a/test/layers/recurrent_tests.jl b/test/layers/recurrent_tests.jl
index 7134540631..59ad5cfa8d 100644
--- a/test/layers/recurrent_tests.jl
+++ b/test/layers/recurrent_tests.jl
@@ -1,5 +1,4 @@
-
-@testitem "RNNCell" setup=[SharedTestSetup] begin
+@testitem "RNNCell" setup=[SharedTestSetup] tags=[:recurrent_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -58,7 +57,7 @@
     end
 end
 
-@testitem "LSTMCell" setup=[SharedTestSetup] begin
+@testitem "LSTMCell" setup=[SharedTestSetup] tags=[:recurrent_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -164,7 +163,7 @@ end
     end
 end
 
-@testitem "GRUCell" setup=[SharedTestSetup] begin
+@testitem "GRUCell" setup=[SharedTestSetup] tags=[:recurrent_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -234,7 +233,7 @@ end
     end
 end
 
-@testitem "StatefulRecurrentCell" setup=[SharedTestSetup] begin
+@testitem "StatefulRecurrentCell" setup=[SharedTestSetup] tags=[:recurrent_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -280,7 +279,7 @@ end
     end
 end
 
-@testitem "Recurrence" timeout=3000 setup=[SharedTestSetup] begin
+@testitem "Recurrence" timeout=3000 setup=[SharedTestSetup] tags=[:recurrent_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
diff --git a/test/misc_tests.jl b/test/misc_tests.jl
index d685161796..8eac52717b 100644
--- a/test/misc_tests.jl
+++ b/test/misc_tests.jl
@@ -1,54 +1,7 @@
-@testitem "Simple Zygote Second Order Derivative" setup=[SharedTestSetup] begin
-    # Add tests for BatchedRoutines here
-    rng = get_stable_rng()
-
-    @testset "$mode" for (mode, aType, dev, ongpu) in MODES
-        x = rand(rng, Float32, 1, 128) |> aType
-        nn = Dense(1 => 1)
-        ps, st = Lux.setup(rng, nn) |> dev
-        ps_ca = ComponentArray(ps)
-
-        function test_f(x, ps)
-            mz, back = Zygote.pullback(x -> first(nn(x, ps, st)), x)
-            ep = only(back(one.(mz)))
-            return sum(mz) + sum(ep)
-        end
-
-        @testset "Named Tuple Parameters" begin
-            @test_nowarn test_f(x, ps)
-
-            @test begin
-                y, back = Zygote.pullback(test_f, x, ps)
-                ∂x, ∂ps = back(one(y))
-                ∂x !== nothing && ∂ps !== nothing
-            end
-
-            # Weird Zygote Quirks
-            @test_broken begin
-                ∂x, ∂ps = Zygote.jacobian(test_f, x, ps)
-                ∂x !== nothing && ∂ps !== nothing
-            end
-        end
-
-        @testset "Component Array Parameters" begin
-            @test_nowarn test_f(x, ps_ca)
-
-            @test begin
-                y, back = Zygote.pullback(test_f, x, ps_ca)
-                ∂x, ∂ps = back(one(y))
-                ∂x !== nothing && ∂ps !== nothing
-            end
-
-            # Weird Zygote Quirks
-            @test begin
-                ∂x, ∂ps = Zygote.jacobian(test_f, x, ps_ca)
-                ∂x !== nothing && ∂ps !== nothing
-            end
-        end
-    end
-end
+# Previously we had a few Zygote.gradient over Zygote.gradient tests but those are now 
+# removed in favor of BatchedRoutines.jl
 
-@testitem "Tracing AD: AoS to SoA" setup=[SharedTestSetup] begin
+@testitem "Tracing AD: AoS to SoA" setup=[SharedTestSetup] tags=[:others] begin
     using ReverseDiff, Tracker
 
     rng = get_stable_rng()
diff --git a/test/qa_tests.jl b/test/qa_tests.jl
index 763f160396..9e19bde1b2 100644
--- a/test/qa_tests.jl
+++ b/test/qa_tests.jl
@@ -1,4 +1,4 @@
-@testitem "Aqua: Quality Assurance" begin
+@testitem "Aqua: Quality Assurance" tags=[:others] begin
     using Aqua, ChainRulesCore
 
     Aqua.test_all(Lux; piracies=false)
@@ -6,7 +6,7 @@
         Lux; treat_as_own=[ChainRulesCore.frule, ChainRulesCore.rrule, Core.kwcall])
 end
 
-@testitem "Explicit Imports: Quality Assurance" begin
+@testitem "Explicit Imports: Quality Assurance" tags=[:others] begin
     # Load all trigger packages
     import Lux, ComponentArrays, ReverseDiff, Flux, LuxAMDGPU, SimpleChains, Tracker, Zygote
 
diff --git a/test/runtests.jl b/test/runtests.jl
index 68d2fa348e..88f9abd411 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,45 +1,55 @@
 using ReTestItems
 
-ReTestItems.runtests(@__DIR__)
+const LUX_TEST_GROUP = lowercase(get(ENV, "LUX_TEST_GROUP", "all"))
+@info "Running tests for group: $LUX_TEST_GROUP"
+
+if LUX_TEST_GROUP == "all"
+    ReTestItems.runtests(@__DIR__)
+else
+    tag = Symbol(LUX_TEST_GROUP)
+    ReTestItems.runtests(@__DIR__; tags=[tag])
+end
 
 # Distributed Tests
 using MPI, Pkg, Test
 
-nprocs_str = get(ENV, "JULIA_MPI_TEST_NPROCS", "")
-nprocs = nprocs_str == "" ? clamp(Sys.CPU_THREADS, 2, 4) : parse(Int, nprocs_str)
-testdir = @__DIR__
-isdistributedtest(f) = endswith(f, "_distributedtest.jl")
-distributedtestfiles = String[]
-for (root, dirs, files) in walkdir(testdir)
-    for file in files
-        if isdistributedtest(file)
-            push!(distributedtestfiles, joinpath(root, file))
+if LUX_TEST_GROUP == "all" || LUX_TEST_GROUP == "distributed"
+    nprocs_str = get(ENV, "JULIA_MPI_TEST_NPROCS", "")
+    nprocs = nprocs_str == "" ? clamp(Sys.CPU_THREADS, 2, 4) : parse(Int, nprocs_str)
+    testdir = @__DIR__
+    isdistributedtest(f) = endswith(f, "_distributedtest.jl")
+    distributedtestfiles = String[]
+    for (root, dirs, files) in walkdir(testdir)
+        for file in files
+            if isdistributedtest(file)
+                push!(distributedtestfiles, joinpath(root, file))
+            end
         end
     end
-end
 
-@info "Running Distributed Tests with $nprocs processes"
+    @info "Running Distributed Tests with $nprocs processes"
 
-cur_proj = dirname(Pkg.project().path)
+    cur_proj = dirname(Pkg.project().path)
 
-include("setup_modes.jl")
+    include("setup_modes.jl")
 
-@testset "MODE: $(mode)" for (mode, aType, dev, ongpu) in MODES
-    if mode == "AMDGPU"
-        # AMDGPU needs to cause a deadlock, needs to be investigated
-        @test_broken 1 == 2
-        continue
-    end
-    backends = mode == "CUDA" ? ("mpi", "nccl") : ("mpi",)
-    for backend_type in backends
-        np = backend_type == "nccl" ? min(nprocs, length(CUDA.devices())) : nprocs
-        @testset "Backend: $(backend_type)" begin
-            @testset "$(basename(file))" for file in distributedtestfiles
-                @info "Running $file with $backend_type backend on $mode device"
-                run(`$(MPI.mpiexec()) -n $(np) $(Base.julia_cmd()) --color=yes \
-                     --code-coverage=user --project=$(cur_proj) --startup-file=no $(file) \
-                     $(mode) $(backend_type)`)
-                Test.@test true
+    @testset "MODE: $(mode)" for (mode, aType, dev, ongpu) in MODES
+        if mode == "AMDGPU"
+            # AMDGPU needs to cause a deadlock, needs to be investigated
+            @test_broken 1 == 2
+            continue
+        end
+        backends = mode == "CUDA" ? ("mpi", "nccl") : ("mpi",)
+        for backend_type in backends
+            np = backend_type == "nccl" ? min(nprocs, length(CUDA.devices())) : nprocs
+            @testset "Backend: $(backend_type)" begin
+                @testset "$(basename(file))" for file in distributedtestfiles
+                    @info "Running $file with $backend_type backend on $mode device"
+                    run(`$(MPI.mpiexec()) -n $(np) $(Base.julia_cmd()) --color=yes \
+                        --code-coverage=user --project=$(cur_proj) --startup-file=no $(file) \
+                        $(mode) $(backend_type)`)
+                    Test.@test true
+                end
             end
         end
     end
diff --git a/test/setup_modes.jl b/test/setup_modes.jl
index 7a55620286..ee40ea1e8e 100644
--- a/test/setup_modes.jl
+++ b/test/setup_modes.jl
@@ -2,11 +2,13 @@ using Lux, LuxCUDA, LuxAMDGPU
 
 CUDA.allowscalar(false)
 
-const GROUP = get(ENV, "GROUP", "All")
+const BACKEND_GROUP = get(ENV, "BACKEND_GROUP", "All")
 
-cpu_testing() = GROUP == "All" || GROUP == "CPU"
-cuda_testing() = (GROUP == "All" || GROUP == "CUDA") && LuxCUDA.functional()
-amdgpu_testing() = (GROUP == "All" || GROUP == "AMDGPU") && LuxAMDGPU.functional()
+cpu_testing() = BACKEND_GROUP == "All" || BACKEND_GROUP == "CPU"
+cuda_testing() = (BACKEND_GROUP == "All" || BACKEND_GROUP == "CUDA") && LuxCUDA.functional()
+function amdgpu_testing()
+    return (BACKEND_GROUP == "All" || BACKEND_GROUP == "AMDGPU") && LuxAMDGPU.functional()
+end
 
 const MODES = begin
     # Mode, Array Type, Device Function, GPU?
diff --git a/test/shared_testsetup.jl b/test/shared_testsetup.jl
index f8cd80b1c4..ce83387630 100644
--- a/test/shared_testsetup.jl
+++ b/test/shared_testsetup.jl
@@ -31,7 +31,7 @@ function _rocRAND_functional()
 end
 
 export @jet, @test_gradients, check_approx
-export GROUP, MODES, cpu_testing, cuda_testing, amdgpu_testing, get_default_rng,
+export BACKEND_GROUP, MODES, cpu_testing, cuda_testing, amdgpu_testing, get_default_rng,
        get_stable_rng, __display, _rocRAND_functional
 
 end
diff --git a/test/transform/flux_tests.jl b/test/transform/flux_tests.jl
index 3fa89f8137..584c05c441 100644
--- a/test/transform/flux_tests.jl
+++ b/test/transform/flux_tests.jl
@@ -1,4 +1,4 @@
-@testitem "FromFluxAdaptor" setup=[SharedTestSetup] begin
+@testitem "FromFluxAdaptor" setup=[SharedTestSetup] tags=[:others] begin
     import Flux
 
     from_flux = fdevice(::Lux.LuxCPUDevice) = Flux.cpu
diff --git a/test/transform/simple_chains_tests.jl b/test/transform/simple_chains_tests.jl
index 7f0f7366d0..007275a8ba 100644
--- a/test/transform/simple_chains_tests.jl
+++ b/test/transform/simple_chains_tests.jl
@@ -1,4 +1,4 @@
-@testitem "ToSimpleChainsAdaptor" setup=[SharedTestSetup] begin
+@testitem "ToSimpleChainsAdaptor" setup=[SharedTestSetup] tags=[:others] begin
     import SimpleChains: static
 
     lux_model = Chain(Conv((5, 5), 1 => 6, relu), MaxPool((2, 2)),
diff --git a/test/utils_tests.jl b/test/utils_tests.jl
index 51569edc40..4796a7bacd 100644
--- a/test/utils_tests.jl
+++ b/test/utils_tests.jl
@@ -1,4 +1,4 @@
-@testitem "replicate" setup=[SharedTestSetup] begin
+@testitem "replicate" setup=[SharedTestSetup] tags=[:others] begin
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
         _rng = get_default_rng(mode)
 
@@ -13,7 +13,7 @@
     end
 end
 
-@testitem "istraining" begin
+@testitem "istraining" tags=[:others] begin
     @test Lux.istraining(Val(true))
     @test !Lux.istraining(Val(false))
     @test !Lux.istraining((training=Val(false),))
@@ -22,7 +22,7 @@ end
     @test_throws MethodError Lux.istraining((training=true,))
 end
 
-@testitem "multigate" setup=[SharedTestSetup] begin
+@testitem "multigate" setup=[SharedTestSetup] tags=[:others] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -58,7 +58,7 @@ end
     end
 end
 
-@testitem "ComponentArrays" setup=[SharedTestSetup] begin
+@testitem "ComponentArrays" setup=[SharedTestSetup] tags=[:others] begin
     using Optimisers, Functors
 
     rng = get_stable_rng(12345)
@@ -103,7 +103,7 @@ end
     @test_nowarn Zygote.gradient(l2reg, ps)
 end
 
-@testitem "_init_hidden_state" setup=[SharedTestSetup] begin
+@testitem "_init_hidden_state" setup=[SharedTestSetup] tags=[:recurrent_layers] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES
@@ -114,7 +114,7 @@ end
     end
 end
 
-@testitem "FP Conversions" setup=[SharedTestSetup] begin
+@testitem "FP Conversions" setup=[SharedTestSetup] tags=[:others] begin
     rng = get_stable_rng(12345)
 
     @testset "$mode" for (mode, aType, device, ongpu) in MODES