diff --git a/fbgemm_gpu/CMakeLists.txt b/fbgemm_gpu/CMakeLists.txt
index 2016e36f7b..5f37ce9157 100644
--- a/fbgemm_gpu/CMakeLists.txt
+++ b/fbgemm_gpu/CMakeLists.txt
@@ -549,7 +549,7 @@ set(fbgemm_gpu_sources_static_cpu
     codegen/embedding_forward_quantized_host_cpu.cpp
     codegen/embedding_backward_dense_host_cpu.cpp
     codegen/embedding_bounds_check_host_cpu.cpp
-    src/permute_pooled_embedding_ops_split_cpu.cpp
+    src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_split_cpu.cpp
     src/jagged_tensor_ops/jagged_tensor_ops_autograd.cpp
     src/jagged_tensor_ops/jagged_tensor_ops_meta.cpp
     src/jagged_tensor_ops/jagged_tensor_ops_cpu.cpp
@@ -567,8 +567,8 @@ if(NOT FBGEMM_CPU_ONLY)
     codegen/embedding_bounds_check_host.cpp
     src/cumem_utils_host.cpp
     src/layout_transform_ops_gpu.cpp
-    src/permute_pooled_embedding_ops_gpu.cpp
-    src/permute_pooled_embedding_ops_split_gpu.cpp
+    src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_gpu.cpp
+    src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_split_gpu.cpp
     src/quantize_ops/quantize_ops_gpu.cpp
     src/sparse_ops/sparse_ops_gpu.cpp
     src/split_embeddings_utils.cpp
@@ -629,8 +629,8 @@ if(NOT FBGEMM_CPU_ONLY)
       src/jagged_tensor_ops/keyed_jagged_index_select_dim1.cu
       src/layout_transform_ops.cu
       src/metric_ops.cu
-      src/permute_pooled_embedding_ops_split.cu
-      src/permute_pooled_embedding_ops.cu
+      src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_split.cu
+      src/permute_pooled_embedding_ops/permute_pooled_embedding_ops.cu
       src/quantize_ops/quantize_bfloat16.cu
       src/quantize_ops/quantize_fp8_rowwise.cu
       src/quantize_ops/quantize_fused_8bit_rowwise.cu
diff --git a/fbgemm_gpu/src/permute_pooled_embedding_ops.cu b/fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops.cu
similarity index 100%
rename from fbgemm_gpu/src/permute_pooled_embedding_ops.cu
rename to fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops.cu
diff --git a/fbgemm_gpu/src/permute_pooled_embedding_ops_cpu.cpp b/fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_cpu.cpp
similarity index 100%
rename from fbgemm_gpu/src/permute_pooled_embedding_ops_cpu.cpp
rename to fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_cpu.cpp
diff --git a/fbgemm_gpu/src/permute_pooled_embedding_ops_gpu.cpp b/fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_gpu.cpp
similarity index 100%
rename from fbgemm_gpu/src/permute_pooled_embedding_ops_gpu.cpp
rename to fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_gpu.cpp
diff --git a/fbgemm_gpu/src/permute_pooled_embedding_ops_split.cu b/fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_split.cu
similarity index 100%
rename from fbgemm_gpu/src/permute_pooled_embedding_ops_split.cu
rename to fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_split.cu
diff --git a/fbgemm_gpu/src/permute_pooled_embedding_ops_split_cpu.cpp b/fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_split_cpu.cpp
similarity index 100%
rename from fbgemm_gpu/src/permute_pooled_embedding_ops_split_cpu.cpp
rename to fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_split_cpu.cpp
diff --git a/fbgemm_gpu/src/permute_pooled_embedding_ops_split_gpu.cpp b/fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_split_gpu.cpp
similarity index 100%
rename from fbgemm_gpu/src/permute_pooled_embedding_ops_split_gpu.cpp
rename to fbgemm_gpu/src/permute_pooled_embedding_ops/permute_pooled_embedding_ops_split_gpu.cpp