diff --git a/CMakeLists.txt b/CMakeLists.txt
index e6b97786888..6487ae7deb7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -28,19 +28,6 @@ function(CUDA_CONVERT_FLAGS EXISTING_TARGET)
     endif()
 endfunction()
 
-file(GLOB HEADERS torchvision/csrc/*.h)
-# Image extension
-file(GLOB IMAGE_HEADERS torchvision/csrc/cpu/image/*.h)
-file(GLOB IMAGE_SOURCES torchvision/csrc/cpu/image/*.cpp)
-file(GLOB OPERATOR_HEADERS torchvision/csrc/cpu/*.h)
-file(GLOB OPERATOR_SOURCES ${OPERATOR_HEADERS} torchvision/csrc/cpu/*.cpp ${IMAGE_HEADERS} ${IMAGE_SOURCES} ${HEADERS} torchvision/csrc/*.cpp)
-if(WITH_CUDA)
-  file(GLOB OPERATOR_HEADERS ${OPERATOR_HEADERS} torchvision/csrc/cuda/*.h)
-  file(GLOB OPERATOR_SOURCES ${OPERATOR_SOURCES} ${OPERATOR_HEADERS} torchvision/csrc/cuda/*.cu)
-endif()
-file(GLOB MODELS_HEADERS torchvision/csrc/models/*.h)
-file(GLOB MODELS_SOURCES torchvision/csrc/models/*.h torchvision/csrc/models/*.cpp)
-
 if(MSVC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4819")
   if(WITH_CUDA)
@@ -64,7 +51,17 @@ endif()
 include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
 
-add_library(${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${OPERATOR_SOURCES} ${IMAGE_SOURCES})
+set(TVCPP torchvision/csrc)
+list(APPEND ALLOW_LISTED ${TVCPP} ${TVCPP}/io/image ${TVCPP}/io/image/cpu ${TVCPP}/models ${TVCPP}/ops ${TVCPP}/ops/cpu)
+if(WITH_CUDA)
+    list(APPEND ALLOW_LISTED ${TVCPP}/ops/cuda)
+endif()
+
+FOREACH(DIR ${ALLOW_LISTED})
+    file(GLOB ALL_SOURCES ${ALL_SOURCES} ${DIR}/*.*)
+ENDFOREACH()
+
+add_library(${PROJECT_NAME} SHARED ${ALL_SOURCES})
 target_link_libraries(${PROJECT_NAME} PRIVATE ${TORCH_LIBRARIES} ${PNG_LIBRARY} ${JPEG_LIBRARIES} Python3::Python)
 set_target_properties(${PROJECT_NAME} PROPERTIES
   EXPORT_NAME TorchVision
@@ -95,13 +92,8 @@ install(EXPORT TorchVisionTargets
   NAMESPACE TorchVision::
   DESTINATION ${TORCHVISION_CMAKECONFIG_INSTALL_DIR})
 
-install(FILES ${HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME})
-install(FILES
-  ${OPERATOR_HEADERS}
-  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/cpu)
-if(WITH_CUDA)
-  install(FILES
-    ${OPERATOR_HEADERS}
-    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/cuda)
-endif()
-install(FILES ${MODELS_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/models)
+FOREACH(INPUT_DIR ${ALLOW_LISTED})
+    string(REPLACE "${TVCPP}" "${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}" OUTPUT_DIR ${INPUT_DIR})
+    file(GLOB INPUT_FILES ${INPUT_DIR}/*.*)
+    install(FILES ${INPUT_FILES} DESTINATION ${OUTPUT_DIR})
+ENDFOREACH()
diff --git a/setup.py b/setup.py
index 0d94e3ffb35..e3b1052ed2d 100644
--- a/setup.py
+++ b/setup.py
@@ -134,8 +134,9 @@ def get_extensions():
     this_dir = os.path.dirname(os.path.abspath(__file__))
     extensions_dir = os.path.join(this_dir, 'torchvision', 'csrc')
 
-    main_file = glob.glob(os.path.join(extensions_dir, '*.cpp'))
-    source_cpu = glob.glob(os.path.join(extensions_dir, 'cpu', '*.cpp'))
+    main_file = glob.glob(os.path.join(extensions_dir, '*.cpp')) + glob.glob(os.path.join(extensions_dir, 'ops',
+                                                                                          '*.cpp'))
+    source_cpu = glob.glob(os.path.join(extensions_dir, 'ops', 'cpu', '*.cpp'))
 
     is_rocm_pytorch = False
     if torch.__version__ >= '1.5':
@@ -146,17 +147,17 @@ def get_extensions():
         hipify_python.hipify(
             project_directory=this_dir,
             output_directory=this_dir,
-            includes="torchvision/csrc/cuda/*",
+            includes="torchvision/csrc/ops/cuda/*",
             show_detailed=True,
             is_pytorch_extension=True,
         )
-        source_cuda = glob.glob(os.path.join(extensions_dir, 'hip', '*.hip'))
+        source_cuda = glob.glob(os.path.join(extensions_dir, 'ops', 'hip', '*.hip'))
         # Copy over additional files
-        for file in glob.glob(r"torchvision/csrc/cuda/*.h"):
-            shutil.copy(file, "torchvision/csrc/hip")
+        for file in glob.glob(r"torchvision/csrc/ops/cuda/*.h"):
+            shutil.copy(file, "torchvision/csrc/ops/hip")
 
     else:
-        source_cuda = glob.glob(os.path.join(extensions_dir, 'cuda', '*.cu'))
+        source_cuda = glob.glob(os.path.join(extensions_dir, 'ops', 'cuda', '*.cu'))
 
     sources = main_file + source_cpu
     extension = CppExtension
@@ -309,8 +310,8 @@ def get_extensions():
             image_library += [jpeg_lib]
             image_include += [jpeg_include]
 
-    image_path = os.path.join(extensions_dir, 'cpu', 'image')
-    image_src = glob.glob(os.path.join(image_path, '*.cpp'))
+    image_path = os.path.join(extensions_dir, 'io', 'image')
+    image_src = glob.glob(os.path.join(image_path, '*.cpp')) + glob.glob(os.path.join(image_path, 'cpu', '*.cpp'))
 
     if png_found or jpeg_found:
         ext_modules.append(extension(
@@ -377,13 +378,13 @@ def get_extensions():
         print("ffmpeg library_dir: {}".format(ffmpeg_library_dir))
 
         # TorchVision base decoder + video reader
-        video_reader_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'cpu', 'video_reader')
+        video_reader_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'io', 'video_reader')
         video_reader_src = glob.glob(os.path.join(video_reader_src_dir, "*.cpp"))
-        base_decoder_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'cpu', 'decoder')
+        base_decoder_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'io', 'decoder')
         base_decoder_src = glob.glob(
             os.path.join(base_decoder_src_dir, "*.cpp"))
         # Torchvision video API
-        videoapi_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'cpu', 'video')
+        videoapi_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'io', 'video')
         videoapi_src = glob.glob(os.path.join(videoapi_src_dir, "*.cpp"))
         # exclude tests
         base_decoder_src = [x for x in base_decoder_src if '_test.cpp' not in x]
diff --git a/test/tracing/frcnn/test_frcnn_tracing.cpp b/test/tracing/frcnn/test_frcnn_tracing.cpp
index bd98c7866d8..d4746729d4b 100644
--- a/test/tracing/frcnn/test_frcnn_tracing.cpp
+++ b/test/tracing/frcnn/test_frcnn_tracing.cpp
@@ -1,8 +1,7 @@
 #include <ATen/ATen.h>
 #include <torch/script.h>
 #include <torch/torch.h>
-#include <torchvision/nms.h>
-#include <torchvision/roi_align.h>
+#include <torchvision/ops/nms.h>
 
 #ifdef _WIN32
 // Windows only
diff --git a/torchvision/csrc/cpu/image/image.h b/torchvision/csrc/cpu/image/image.h
deleted file mode 100644
index 3a652bef244..00000000000
--- a/torchvision/csrc/cpu/image/image.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#pragma once
-
-// Comment
-#include <torch/script.h>
-#include <torch/torch.h>
-#include "read_image_cpu.h"
-#include "read_write_file_cpu.h"
-#include "readjpeg_cpu.h"
-#include "readpng_cpu.h"
-#include "writejpeg_cpu.h"
-#include "writepng_cpu.h"
diff --git a/torchvision/csrc/cpu/decoder/audio_sampler.cpp b/torchvision/csrc/io/decoder/audio_sampler.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/audio_sampler.cpp
rename to torchvision/csrc/io/decoder/audio_sampler.cpp
diff --git a/torchvision/csrc/cpu/decoder/audio_sampler.h b/torchvision/csrc/io/decoder/audio_sampler.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/audio_sampler.h
rename to torchvision/csrc/io/decoder/audio_sampler.h
diff --git a/torchvision/csrc/cpu/decoder/audio_stream.cpp b/torchvision/csrc/io/decoder/audio_stream.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/audio_stream.cpp
rename to torchvision/csrc/io/decoder/audio_stream.cpp
diff --git a/torchvision/csrc/cpu/decoder/audio_stream.h b/torchvision/csrc/io/decoder/audio_stream.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/audio_stream.h
rename to torchvision/csrc/io/decoder/audio_stream.h
diff --git a/torchvision/csrc/cpu/decoder/cc_stream.cpp b/torchvision/csrc/io/decoder/cc_stream.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/cc_stream.cpp
rename to torchvision/csrc/io/decoder/cc_stream.cpp
diff --git a/torchvision/csrc/cpu/decoder/cc_stream.h b/torchvision/csrc/io/decoder/cc_stream.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/cc_stream.h
rename to torchvision/csrc/io/decoder/cc_stream.h
diff --git a/torchvision/csrc/cpu/decoder/decoder.cpp b/torchvision/csrc/io/decoder/decoder.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/decoder.cpp
rename to torchvision/csrc/io/decoder/decoder.cpp
diff --git a/torchvision/csrc/cpu/decoder/decoder.h b/torchvision/csrc/io/decoder/decoder.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/decoder.h
rename to torchvision/csrc/io/decoder/decoder.h
diff --git a/torchvision/csrc/cpu/decoder/defs.h b/torchvision/csrc/io/decoder/defs.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/defs.h
rename to torchvision/csrc/io/decoder/defs.h
diff --git a/torchvision/csrc/cpu/decoder/memory_buffer.cpp b/torchvision/csrc/io/decoder/memory_buffer.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/memory_buffer.cpp
rename to torchvision/csrc/io/decoder/memory_buffer.cpp
diff --git a/torchvision/csrc/cpu/decoder/memory_buffer.h b/torchvision/csrc/io/decoder/memory_buffer.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/memory_buffer.h
rename to torchvision/csrc/io/decoder/memory_buffer.h
diff --git a/torchvision/csrc/cpu/decoder/seekable_buffer.cpp b/torchvision/csrc/io/decoder/seekable_buffer.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/seekable_buffer.cpp
rename to torchvision/csrc/io/decoder/seekable_buffer.cpp
diff --git a/torchvision/csrc/cpu/decoder/seekable_buffer.h b/torchvision/csrc/io/decoder/seekable_buffer.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/seekable_buffer.h
rename to torchvision/csrc/io/decoder/seekable_buffer.h
diff --git a/torchvision/csrc/cpu/decoder/stream.cpp b/torchvision/csrc/io/decoder/stream.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/stream.cpp
rename to torchvision/csrc/io/decoder/stream.cpp
diff --git a/torchvision/csrc/cpu/decoder/stream.h b/torchvision/csrc/io/decoder/stream.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/stream.h
rename to torchvision/csrc/io/decoder/stream.h
diff --git a/torchvision/csrc/cpu/decoder/subtitle_sampler.cpp b/torchvision/csrc/io/decoder/subtitle_sampler.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/subtitle_sampler.cpp
rename to torchvision/csrc/io/decoder/subtitle_sampler.cpp
diff --git a/torchvision/csrc/cpu/decoder/subtitle_sampler.h b/torchvision/csrc/io/decoder/subtitle_sampler.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/subtitle_sampler.h
rename to torchvision/csrc/io/decoder/subtitle_sampler.h
diff --git a/torchvision/csrc/cpu/decoder/subtitle_stream.cpp b/torchvision/csrc/io/decoder/subtitle_stream.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/subtitle_stream.cpp
rename to torchvision/csrc/io/decoder/subtitle_stream.cpp
diff --git a/torchvision/csrc/cpu/decoder/subtitle_stream.h b/torchvision/csrc/io/decoder/subtitle_stream.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/subtitle_stream.h
rename to torchvision/csrc/io/decoder/subtitle_stream.h
diff --git a/torchvision/csrc/cpu/decoder/sync_decoder.cpp b/torchvision/csrc/io/decoder/sync_decoder.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/sync_decoder.cpp
rename to torchvision/csrc/io/decoder/sync_decoder.cpp
diff --git a/torchvision/csrc/cpu/decoder/sync_decoder.h b/torchvision/csrc/io/decoder/sync_decoder.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/sync_decoder.h
rename to torchvision/csrc/io/decoder/sync_decoder.h
diff --git a/torchvision/csrc/cpu/decoder/sync_decoder_test.cpp b/torchvision/csrc/io/decoder/sync_decoder_test.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/sync_decoder_test.cpp
rename to torchvision/csrc/io/decoder/sync_decoder_test.cpp
diff --git a/torchvision/csrc/cpu/decoder/time_keeper.cpp b/torchvision/csrc/io/decoder/time_keeper.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/time_keeper.cpp
rename to torchvision/csrc/io/decoder/time_keeper.cpp
diff --git a/torchvision/csrc/cpu/decoder/time_keeper.h b/torchvision/csrc/io/decoder/time_keeper.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/time_keeper.h
rename to torchvision/csrc/io/decoder/time_keeper.h
diff --git a/torchvision/csrc/cpu/decoder/util.cpp b/torchvision/csrc/io/decoder/util.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/util.cpp
rename to torchvision/csrc/io/decoder/util.cpp
diff --git a/torchvision/csrc/cpu/decoder/util.h b/torchvision/csrc/io/decoder/util.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/util.h
rename to torchvision/csrc/io/decoder/util.h
diff --git a/torchvision/csrc/cpu/decoder/util_test.cpp b/torchvision/csrc/io/decoder/util_test.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/util_test.cpp
rename to torchvision/csrc/io/decoder/util_test.cpp
diff --git a/torchvision/csrc/cpu/decoder/video_sampler.cpp b/torchvision/csrc/io/decoder/video_sampler.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/video_sampler.cpp
rename to torchvision/csrc/io/decoder/video_sampler.cpp
diff --git a/torchvision/csrc/cpu/decoder/video_sampler.h b/torchvision/csrc/io/decoder/video_sampler.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/video_sampler.h
rename to torchvision/csrc/io/decoder/video_sampler.h
diff --git a/torchvision/csrc/cpu/decoder/video_stream.cpp b/torchvision/csrc/io/decoder/video_stream.cpp
similarity index 100%
rename from torchvision/csrc/cpu/decoder/video_stream.cpp
rename to torchvision/csrc/io/decoder/video_stream.cpp
diff --git a/torchvision/csrc/cpu/decoder/video_stream.h b/torchvision/csrc/io/decoder/video_stream.h
similarity index 100%
rename from torchvision/csrc/cpu/decoder/video_stream.h
rename to torchvision/csrc/io/decoder/video_stream.h
diff --git a/torchvision/csrc/cpu/image/read_image_cpu.cpp b/torchvision/csrc/io/image/cpu/read_image_cpu.cpp
similarity index 100%
rename from torchvision/csrc/cpu/image/read_image_cpu.cpp
rename to torchvision/csrc/io/image/cpu/read_image_cpu.cpp
diff --git a/torchvision/csrc/cpu/image/read_image_cpu.h b/torchvision/csrc/io/image/cpu/read_image_cpu.h
similarity index 83%
rename from torchvision/csrc/cpu/image/read_image_cpu.h
rename to torchvision/csrc/io/image/cpu/read_image_cpu.h
index 6186d0d0d98..0f9b8d3be79 100644
--- a/torchvision/csrc/cpu/image/read_image_cpu.h
+++ b/torchvision/csrc/io/image/cpu/read_image_cpu.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <torch/torch.h>
-#include "image_read_mode.h"
+#include "../image_read_mode.h"
 
 C10_EXPORT torch::Tensor decode_image(
     const torch::Tensor& data,
diff --git a/torchvision/csrc/cpu/image/read_write_file_cpu.cpp b/torchvision/csrc/io/image/cpu/read_write_file_cpu.cpp
similarity index 100%
rename from torchvision/csrc/cpu/image/read_write_file_cpu.cpp
rename to torchvision/csrc/io/image/cpu/read_write_file_cpu.cpp
diff --git a/torchvision/csrc/cpu/image/read_write_file_cpu.h b/torchvision/csrc/io/image/cpu/read_write_file_cpu.h
similarity index 100%
rename from torchvision/csrc/cpu/image/read_write_file_cpu.h
rename to torchvision/csrc/io/image/cpu/read_write_file_cpu.h
diff --git a/torchvision/csrc/cpu/image/readjpeg_cpu.cpp b/torchvision/csrc/io/image/cpu/readjpeg_cpu.cpp
similarity index 100%
rename from torchvision/csrc/cpu/image/readjpeg_cpu.cpp
rename to torchvision/csrc/io/image/cpu/readjpeg_cpu.cpp
diff --git a/torchvision/csrc/cpu/image/readjpeg_cpu.h b/torchvision/csrc/io/image/cpu/readjpeg_cpu.h
similarity index 83%
rename from torchvision/csrc/cpu/image/readjpeg_cpu.h
rename to torchvision/csrc/io/image/cpu/readjpeg_cpu.h
index f05d05a9064..366eb11398b 100644
--- a/torchvision/csrc/cpu/image/readjpeg_cpu.h
+++ b/torchvision/csrc/io/image/cpu/readjpeg_cpu.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <torch/torch.h>
-#include "image_read_mode.h"
+#include "../image_read_mode.h"
 
 C10_EXPORT torch::Tensor decodeJPEG(
     const torch::Tensor& data,
diff --git a/torchvision/csrc/cpu/image/readpng_cpu.cpp b/torchvision/csrc/io/image/cpu/readpng_cpu.cpp
similarity index 100%
rename from torchvision/csrc/cpu/image/readpng_cpu.cpp
rename to torchvision/csrc/io/image/cpu/readpng_cpu.cpp
diff --git a/torchvision/csrc/cpu/image/readpng_cpu.h b/torchvision/csrc/io/image/cpu/readpng_cpu.h
similarity index 83%
rename from torchvision/csrc/cpu/image/readpng_cpu.h
rename to torchvision/csrc/io/image/cpu/readpng_cpu.h
index 9c74cb2c678..9798a615355 100644
--- a/torchvision/csrc/cpu/image/readpng_cpu.h
+++ b/torchvision/csrc/io/image/cpu/readpng_cpu.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <torch/torch.h>
-#include "image_read_mode.h"
+#include "../image_read_mode.h"
 
 C10_EXPORT torch::Tensor decodePNG(
     const torch::Tensor& data,
diff --git a/torchvision/csrc/cpu/image/writejpeg_cpu.cpp b/torchvision/csrc/io/image/cpu/writejpeg_cpu.cpp
similarity index 100%
rename from torchvision/csrc/cpu/image/writejpeg_cpu.cpp
rename to torchvision/csrc/io/image/cpu/writejpeg_cpu.cpp
diff --git a/torchvision/csrc/cpu/image/writejpeg_cpu.h b/torchvision/csrc/io/image/cpu/writejpeg_cpu.h
similarity index 100%
rename from torchvision/csrc/cpu/image/writejpeg_cpu.h
rename to torchvision/csrc/io/image/cpu/writejpeg_cpu.h
diff --git a/torchvision/csrc/cpu/image/writepng_cpu.cpp b/torchvision/csrc/io/image/cpu/writepng_cpu.cpp
similarity index 100%
rename from torchvision/csrc/cpu/image/writepng_cpu.cpp
rename to torchvision/csrc/io/image/cpu/writepng_cpu.cpp
diff --git a/torchvision/csrc/cpu/image/writepng_cpu.h b/torchvision/csrc/io/image/cpu/writepng_cpu.h
similarity index 100%
rename from torchvision/csrc/cpu/image/writepng_cpu.h
rename to torchvision/csrc/io/image/cpu/writepng_cpu.h
diff --git a/torchvision/csrc/cpu/image/image.cpp b/torchvision/csrc/io/image/image.cpp
similarity index 100%
rename from torchvision/csrc/cpu/image/image.cpp
rename to torchvision/csrc/io/image/image.cpp
diff --git a/torchvision/csrc/io/image/image.h b/torchvision/csrc/io/image/image.h
new file mode 100644
index 00000000000..94a5a7c42d6
--- /dev/null
+++ b/torchvision/csrc/io/image/image.h
@@ -0,0 +1,11 @@
+#pragma once
+
+// Comment
+#include <torch/script.h>
+#include <torch/torch.h>
+#include "cpu/read_image_cpu.h"
+#include "cpu/read_write_file_cpu.h"
+#include "cpu/readjpeg_cpu.h"
+#include "cpu/readpng_cpu.h"
+#include "cpu/writejpeg_cpu.h"
+#include "cpu/writepng_cpu.h"
diff --git a/torchvision/csrc/cpu/image/image_read_mode.h b/torchvision/csrc/io/image/image_read_mode.h
similarity index 100%
rename from torchvision/csrc/cpu/image/image_read_mode.h
rename to torchvision/csrc/io/image/image_read_mode.h
diff --git a/torchvision/csrc/cpu/image/jpegcommon.cpp b/torchvision/csrc/io/image/jpegcommon.cpp
similarity index 100%
rename from torchvision/csrc/cpu/image/jpegcommon.cpp
rename to torchvision/csrc/io/image/jpegcommon.cpp
diff --git a/torchvision/csrc/cpu/image/jpegcommon.h b/torchvision/csrc/io/image/jpegcommon.h
similarity index 100%
rename from torchvision/csrc/cpu/image/jpegcommon.h
rename to torchvision/csrc/io/image/jpegcommon.h
diff --git a/torchvision/csrc/cpu/video/Video.cpp b/torchvision/csrc/io/video/Video.cpp
similarity index 100%
rename from torchvision/csrc/cpu/video/Video.cpp
rename to torchvision/csrc/io/video/Video.cpp
diff --git a/torchvision/csrc/cpu/video/Video.h b/torchvision/csrc/io/video/Video.h
similarity index 100%
rename from torchvision/csrc/cpu/video/Video.h
rename to torchvision/csrc/io/video/Video.h
diff --git a/torchvision/csrc/cpu/video/register.cpp b/torchvision/csrc/io/video/register.cpp
similarity index 100%
rename from torchvision/csrc/cpu/video/register.cpp
rename to torchvision/csrc/io/video/register.cpp
diff --git a/torchvision/csrc/cpu/video_reader/VideoReader.cpp b/torchvision/csrc/io/video_reader/VideoReader.cpp
similarity index 100%
rename from torchvision/csrc/cpu/video_reader/VideoReader.cpp
rename to torchvision/csrc/io/video_reader/VideoReader.cpp
diff --git a/torchvision/csrc/cpu/video_reader/VideoReader.h b/torchvision/csrc/io/video_reader/VideoReader.h
similarity index 100%
rename from torchvision/csrc/cpu/video_reader/VideoReader.h
rename to torchvision/csrc/io/video_reader/VideoReader.h
diff --git a/torchvision/csrc/cpu/deform_conv2d_kernel.cpp b/torchvision/csrc/ops/cpu/deform_conv2d_kernel.cpp
similarity index 100%
rename from torchvision/csrc/cpu/deform_conv2d_kernel.cpp
rename to torchvision/csrc/ops/cpu/deform_conv2d_kernel.cpp
diff --git a/torchvision/csrc/cpu/nms_kernel.cpp b/torchvision/csrc/ops/cpu/nms_kernel.cpp
similarity index 100%
rename from torchvision/csrc/cpu/nms_kernel.cpp
rename to torchvision/csrc/ops/cpu/nms_kernel.cpp
diff --git a/torchvision/csrc/cpu/ps_roi_align_kernel.cpp b/torchvision/csrc/ops/cpu/ps_roi_align_kernel.cpp
similarity index 100%
rename from torchvision/csrc/cpu/ps_roi_align_kernel.cpp
rename to torchvision/csrc/ops/cpu/ps_roi_align_kernel.cpp
diff --git a/torchvision/csrc/cpu/ps_roi_pool_kernel.cpp b/torchvision/csrc/ops/cpu/ps_roi_pool_kernel.cpp
similarity index 100%
rename from torchvision/csrc/cpu/ps_roi_pool_kernel.cpp
rename to torchvision/csrc/ops/cpu/ps_roi_pool_kernel.cpp
diff --git a/torchvision/csrc/cpu/roi_align_kernel.cpp b/torchvision/csrc/ops/cpu/roi_align_kernel.cpp
similarity index 100%
rename from torchvision/csrc/cpu/roi_align_kernel.cpp
rename to torchvision/csrc/ops/cpu/roi_align_kernel.cpp
diff --git a/torchvision/csrc/cpu/roi_pool_kernel.cpp b/torchvision/csrc/ops/cpu/roi_pool_kernel.cpp
similarity index 100%
rename from torchvision/csrc/cpu/roi_pool_kernel.cpp
rename to torchvision/csrc/ops/cpu/roi_pool_kernel.cpp
diff --git a/torchvision/csrc/cuda/cuda_helpers.h b/torchvision/csrc/ops/cuda/cuda_helpers.h
similarity index 100%
rename from torchvision/csrc/cuda/cuda_helpers.h
rename to torchvision/csrc/ops/cuda/cuda_helpers.h
diff --git a/torchvision/csrc/cuda/deform_conv2d_kernel.cu b/torchvision/csrc/ops/cuda/deform_conv2d_kernel.cu
similarity index 100%
rename from torchvision/csrc/cuda/deform_conv2d_kernel.cu
rename to torchvision/csrc/ops/cuda/deform_conv2d_kernel.cu
diff --git a/torchvision/csrc/cuda/nms_kernel.cu b/torchvision/csrc/ops/cuda/nms_kernel.cu
similarity index 100%
rename from torchvision/csrc/cuda/nms_kernel.cu
rename to torchvision/csrc/ops/cuda/nms_kernel.cu
diff --git a/torchvision/csrc/cuda/ps_roi_align_kernel.cu b/torchvision/csrc/ops/cuda/ps_roi_align_kernel.cu
similarity index 100%
rename from torchvision/csrc/cuda/ps_roi_align_kernel.cu
rename to torchvision/csrc/ops/cuda/ps_roi_align_kernel.cu
diff --git a/torchvision/csrc/cuda/ps_roi_pool_kernel.cu b/torchvision/csrc/ops/cuda/ps_roi_pool_kernel.cu
similarity index 100%
rename from torchvision/csrc/cuda/ps_roi_pool_kernel.cu
rename to torchvision/csrc/ops/cuda/ps_roi_pool_kernel.cu
diff --git a/torchvision/csrc/cuda/roi_align_kernel.cu b/torchvision/csrc/ops/cuda/roi_align_kernel.cu
similarity index 100%
rename from torchvision/csrc/cuda/roi_align_kernel.cu
rename to torchvision/csrc/ops/cuda/roi_align_kernel.cu
diff --git a/torchvision/csrc/cuda/roi_pool_kernel.cu b/torchvision/csrc/ops/cuda/roi_pool_kernel.cu
similarity index 100%
rename from torchvision/csrc/cuda/roi_pool_kernel.cu
rename to torchvision/csrc/ops/cuda/roi_pool_kernel.cu
diff --git a/torchvision/csrc/deform_conv2d.cpp b/torchvision/csrc/ops/deform_conv2d.cpp
similarity index 100%
rename from torchvision/csrc/deform_conv2d.cpp
rename to torchvision/csrc/ops/deform_conv2d.cpp
diff --git a/torchvision/csrc/deform_conv2d.h b/torchvision/csrc/ops/deform_conv2d.h
similarity index 95%
rename from torchvision/csrc/deform_conv2d.h
rename to torchvision/csrc/ops/deform_conv2d.h
index 3f65036a9b1..e94636fb280 100644
--- a/torchvision/csrc/deform_conv2d.h
+++ b/torchvision/csrc/ops/deform_conv2d.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <ATen/ATen.h>
-#include "macros.h"
+#include "../macros.h"
 
 namespace vision {
 namespace ops {
diff --git a/torchvision/csrc/new_empty_tensor_op.cpp b/torchvision/csrc/ops/new_empty_tensor_op.cpp
similarity index 100%
rename from torchvision/csrc/new_empty_tensor_op.cpp
rename to torchvision/csrc/ops/new_empty_tensor_op.cpp
diff --git a/torchvision/csrc/new_empty_tensor_op.h b/torchvision/csrc/ops/new_empty_tensor_op.h
similarity index 90%
rename from torchvision/csrc/new_empty_tensor_op.h
rename to torchvision/csrc/ops/new_empty_tensor_op.h
index f1435517cba..6f22cfecda9 100644
--- a/torchvision/csrc/new_empty_tensor_op.h
+++ b/torchvision/csrc/ops/new_empty_tensor_op.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <ATen/ATen.h>
-#include "macros.h"
+#include "../macros.h"
 
 namespace vision {
 namespace ops {
diff --git a/torchvision/csrc/nms.cpp b/torchvision/csrc/ops/nms.cpp
similarity index 100%
rename from torchvision/csrc/nms.cpp
rename to torchvision/csrc/ops/nms.cpp
diff --git a/torchvision/csrc/nms.h b/torchvision/csrc/ops/nms.h
similarity index 90%
rename from torchvision/csrc/nms.h
rename to torchvision/csrc/ops/nms.h
index efe50acc136..8c75a242bff 100644
--- a/torchvision/csrc/nms.h
+++ b/torchvision/csrc/ops/nms.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <ATen/ATen.h>
-#include "macros.h"
+#include "../macros.h"
 
 namespace vision {
 namespace ops {
diff --git a/torchvision/csrc/ps_roi_align.cpp b/torchvision/csrc/ops/ps_roi_align.cpp
similarity index 100%
rename from torchvision/csrc/ps_roi_align.cpp
rename to torchvision/csrc/ops/ps_roi_align.cpp
diff --git a/torchvision/csrc/ps_roi_align.h b/torchvision/csrc/ops/ps_roi_align.h
similarity index 93%
rename from torchvision/csrc/ps_roi_align.h
rename to torchvision/csrc/ops/ps_roi_align.h
index b04359b70ed..cee53455f0c 100644
--- a/torchvision/csrc/ps_roi_align.h
+++ b/torchvision/csrc/ops/ps_roi_align.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <ATen/ATen.h>
-#include "macros.h"
+#include "../macros.h"
 
 namespace vision {
 namespace ops {
diff --git a/torchvision/csrc/ps_roi_pool.cpp b/torchvision/csrc/ops/ps_roi_pool.cpp
similarity index 100%
rename from torchvision/csrc/ps_roi_pool.cpp
rename to torchvision/csrc/ops/ps_roi_pool.cpp
diff --git a/torchvision/csrc/ps_roi_pool.h b/torchvision/csrc/ops/ps_roi_pool.h
similarity index 93%
rename from torchvision/csrc/ps_roi_pool.h
rename to torchvision/csrc/ops/ps_roi_pool.h
index c9ae946a4e2..2c2bab314d0 100644
--- a/torchvision/csrc/ps_roi_pool.h
+++ b/torchvision/csrc/ops/ps_roi_pool.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <ATen/ATen.h>
-#include "macros.h"
+#include "../macros.h"
 
 namespace vision {
 namespace ops {
diff --git a/torchvision/csrc/roi_align.cpp b/torchvision/csrc/ops/roi_align.cpp
similarity index 100%
rename from torchvision/csrc/roi_align.cpp
rename to torchvision/csrc/ops/roi_align.cpp
diff --git a/torchvision/csrc/roi_align.h b/torchvision/csrc/ops/roi_align.h
similarity index 93%
rename from torchvision/csrc/roi_align.h
rename to torchvision/csrc/ops/roi_align.h
index 84c357d9c5f..628c0fd33dd 100644
--- a/torchvision/csrc/roi_align.h
+++ b/torchvision/csrc/ops/roi_align.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <ATen/ATen.h>
-#include "macros.h"
+#include "../macros.h"
 
 namespace vision {
 namespace ops {
diff --git a/torchvision/csrc/roi_pool.cpp b/torchvision/csrc/ops/roi_pool.cpp
similarity index 100%
rename from torchvision/csrc/roi_pool.cpp
rename to torchvision/csrc/ops/roi_pool.cpp
diff --git a/torchvision/csrc/roi_pool.h b/torchvision/csrc/ops/roi_pool.h
similarity index 93%
rename from torchvision/csrc/roi_pool.h
rename to torchvision/csrc/ops/roi_pool.h
index ceebf201769..5ecdb711e1b 100644
--- a/torchvision/csrc/roi_pool.h
+++ b/torchvision/csrc/ops/roi_pool.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include <ATen/ATen.h>
-#include "macros.h"
+#include "../macros.h"
 
 namespace vision {
 namespace ops {