Skip to content

Commit

Permalink
Restructuring C++ project:
Browse files Browse the repository at this point in the history
Summary:
* Reduce unnecessary header inclusions in models and io.

* Move autocast to separate folder and hide autograd implementation in an anonymous namespace.

* Moving files in subfolders.

Reviewed By: fmassa

Differential Revision: D25461523

fbshipit-source-id: 756eeb6848aacaa474de4825ed4c1045d17e2cea
  • Loading branch information
datumbox authored and facebook-github-bot committed Dec 10, 2020
1 parent a0b13ca commit eed0807
Show file tree
Hide file tree
Showing 104 changed files with 318 additions and 302 deletions.
40 changes: 16 additions & 24 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,6 @@ function(CUDA_CONVERT_FLAGS EXISTING_TARGET)
endif()
endfunction()

file(GLOB HEADERS torchvision/csrc/*.h)
# Image extension
file(GLOB IMAGE_HEADERS torchvision/csrc/cpu/image/*.h)
file(GLOB IMAGE_SOURCES torchvision/csrc/cpu/image/*.cpp)
file(GLOB OPERATOR_HEADERS torchvision/csrc/cpu/*.h)
file(GLOB OPERATOR_SOURCES ${OPERATOR_HEADERS} torchvision/csrc/cpu/*.cpp ${IMAGE_HEADERS} ${IMAGE_SOURCES} ${HEADERS} torchvision/csrc/*.cpp)
if(WITH_CUDA)
file(GLOB OPERATOR_HEADERS ${OPERATOR_HEADERS} torchvision/csrc/cuda/*.h)
file(GLOB OPERATOR_SOURCES ${OPERATOR_SOURCES} ${OPERATOR_HEADERS} torchvision/csrc/cuda/*.cu)
endif()
file(GLOB MODELS_HEADERS torchvision/csrc/models/*.h)
file(GLOB MODELS_SOURCES torchvision/csrc/models/*.h torchvision/csrc/models/*.cpp)

if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4819")
if(WITH_CUDA)
Expand All @@ -64,7 +51,17 @@ endif()
include(GNUInstallDirs)
include(CMakePackageConfigHelpers)

add_library(${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${OPERATOR_SOURCES} ${IMAGE_SOURCES})
set(TVCPP torchvision/csrc)
list(APPEND ALLOW_LISTED ${TVCPP} ${TVCPP}/io/image ${TVCPP}/io/image/cpu ${TVCPP}/models ${TVCPP}/ops ${TVCPP}/ops/cpu)
if(WITH_CUDA)
list(APPEND ALLOW_LISTED ${TVCPP}/ops/cuda ${TVCPP}/ops/autocast)
endif()

FOREACH(DIR ${ALLOW_LISTED})
file(GLOB ALL_SOURCES ${ALL_SOURCES} ${DIR}/*.*)
ENDFOREACH()

add_library(${PROJECT_NAME} SHARED ${ALL_SOURCES})
target_link_libraries(${PROJECT_NAME} PRIVATE ${TORCH_LIBRARIES} ${PNG_LIBRARY} ${JPEG_LIBRARIES} Python3::Python)
set_target_properties(${PROJECT_NAME} PROPERTIES
EXPORT_NAME TorchVision
Expand Down Expand Up @@ -95,13 +92,8 @@ install(EXPORT TorchVisionTargets
NAMESPACE TorchVision::
DESTINATION ${TORCHVISION_CMAKECONFIG_INSTALL_DIR})

install(FILES ${HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME})
install(FILES
${OPERATOR_HEADERS}
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/cpu)
if(WITH_CUDA)
install(FILES
${OPERATOR_HEADERS}
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/cuda)
endif()
install(FILES ${MODELS_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/models)
FOREACH(INPUT_DIR ${ALLOW_LISTED})
string(REPLACE "${TVCPP}" "${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}" OUTPUT_DIR ${INPUT_DIR})
file(GLOB INPUT_FILES ${INPUT_DIR}/*.*)
install(FILES ${INPUT_FILES} DESTINATION ${OUTPUT_DIR})
ENDFOREACH()
2 changes: 2 additions & 0 deletions examples/cpp/hello_world/main.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
#include <iostream>
#include <torch/torch.h>
#include <torchvision/vision.h>
#include <torchvision/models/resnet.h>

int main()
Expand Down
27 changes: 15 additions & 12 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,9 @@ def get_extensions():
this_dir = os.path.dirname(os.path.abspath(__file__))
extensions_dir = os.path.join(this_dir, 'torchvision', 'csrc')

main_file = glob.glob(os.path.join(extensions_dir, '*.cpp'))
source_cpu = glob.glob(os.path.join(extensions_dir, 'cpu', '*.cpp'))
main_file = glob.glob(os.path.join(extensions_dir, '*.cpp')) + glob.glob(os.path.join(extensions_dir, 'ops',
'*.cpp'))
source_cpu = glob.glob(os.path.join(extensions_dir, 'ops', 'cpu', '*.cpp'))

is_rocm_pytorch = False
if torch.__version__ >= '1.5':
Expand All @@ -146,17 +147,19 @@ def get_extensions():
hipify_python.hipify(
project_directory=this_dir,
output_directory=this_dir,
includes="torchvision/csrc/cuda/*",
includes="torchvision/csrc/ops/cuda/*",
show_detailed=True,
is_pytorch_extension=True,
)
source_cuda = glob.glob(os.path.join(extensions_dir, 'hip', '*.hip'))
source_cuda = glob.glob(os.path.join(extensions_dir, 'ops', 'hip', '*.hip'))
# Copy over additional files
for file in glob.glob(r"torchvision/csrc/cuda/*.h"):
shutil.copy(file, "torchvision/csrc/hip")
for file in glob.glob(r"torchvision/csrc/ops/cuda/*.h"):
shutil.copy(file, "torchvision/csrc/ops/hip")

else:
source_cuda = glob.glob(os.path.join(extensions_dir, 'cuda', '*.cu'))
source_cuda = glob.glob(os.path.join(extensions_dir, 'ops', 'cuda', '*.cu'))

source_cuda += glob.glob(os.path.join(extensions_dir, 'ops', 'autocast', '*.cpp'))

sources = main_file + source_cpu
extension = CppExtension
Expand Down Expand Up @@ -309,8 +312,8 @@ def get_extensions():
image_library += [jpeg_lib]
image_include += [jpeg_include]

image_path = os.path.join(extensions_dir, 'cpu', 'image')
image_src = glob.glob(os.path.join(image_path, '*.cpp'))
image_path = os.path.join(extensions_dir, 'io', 'image')
image_src = glob.glob(os.path.join(image_path, '*.cpp')) + glob.glob(os.path.join(image_path, 'cpu', '*.cpp'))

if png_found or jpeg_found:
ext_modules.append(extension(
Expand Down Expand Up @@ -377,13 +380,13 @@ def get_extensions():
print("ffmpeg library_dir: {}".format(ffmpeg_library_dir))

# TorchVision base decoder + video reader
video_reader_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'cpu', 'video_reader')
video_reader_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'io', 'video_reader')
video_reader_src = glob.glob(os.path.join(video_reader_src_dir, "*.cpp"))
base_decoder_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'cpu', 'decoder')
base_decoder_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'io', 'decoder')
base_decoder_src = glob.glob(
os.path.join(base_decoder_src_dir, "*.cpp"))
# Torchvision video API
videoapi_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'cpu', 'video')
videoapi_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'io', 'video')
videoapi_src = glob.glob(os.path.join(videoapi_src_dir, "*.cpp"))
# exclude tests
base_decoder_src = [x for x in base_decoder_src if '_test.cpp' not in x]
Expand Down
5 changes: 2 additions & 3 deletions test/tracing/frcnn/test_frcnn_tracing.cpp
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
#include <ATen/ATen.h>
#include <torch/script.h>
#include <torch/torch.h>
#include <torchvision/nms.h>
#include <torchvision/roi_align.h>
#include <torchvision/vision.h>
#include <torchvision/ops/nms.h>

#ifdef _WIN32
// Windows only
Expand Down
11 changes: 0 additions & 11 deletions torchvision/csrc/cpu/image/image.h

This file was deleted.

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include "read_image_cpu.h"

#include "readjpeg_cpu.h"
#include "readpng_cpu.h"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#pragma once

#include <torch/torch.h>
#include "image_read_mode.h"
#include <torch/types.h>
#include "../image_read_mode.h"

C10_EXPORT torch::Tensor decode_image(
const torch::Tensor& data,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
#pragma once

#include <errno.h>
#include <sys/stat.h>
#include <torch/torch.h>
#include <torch/types.h>

C10_EXPORT torch::Tensor read_file(const std::string& filename);

Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
#include "readjpeg_cpu.h"

#include <ATen/ATen.h>

#if !JPEG_FOUND
torch::Tensor decodeJPEG(const torch::Tensor& data, ImageReadMode mode) {
TORCH_CHECK(
false, "decodeJPEG: torchvision not compiled with libjpeg support");
}
#else
#include <jpeglib.h>
#include <setjmp.h>
#include "jpegcommon.h"
#include "../jpegcommon.h"

struct torch_jpeg_mgr {
struct jpeg_source_mgr pub;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#pragma once

#include <torch/torch.h>
#include "image_read_mode.h"
#include <torch/types.h>
#include "../image_read_mode.h"

C10_EXPORT torch::Tensor decodeJPEG(
const torch::Tensor& data,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
#include "readpng_cpu.h"

#include <ATen/ATen.h>

#if !PNG_FOUND
torch::Tensor decodePNG(const torch::Tensor& data, ImageReadMode mode) {
TORCH_CHECK(false, "decodePNG: torchvision not compiled with libPNG support");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#pragma once

#include <torch/torch.h>
#include "image_read_mode.h"
#include <torch/types.h>
#include "../image_read_mode.h"

C10_EXPORT torch::Tensor decodePNG(
const torch::Tensor& data,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
#include "writejpeg_cpu.h"

#include <setjmp.h>
#include <string>

#if !JPEG_FOUND

torch::Tensor encodeJPEG(const torch::Tensor& data, int64_t quality) {
Expand All @@ -11,9 +8,7 @@ torch::Tensor encodeJPEG(const torch::Tensor& data, int64_t quality) {
}

#else

#include <jpeglib.h>
#include "jpegcommon.h"
#include "../jpegcommon.h"

torch::Tensor encodeJPEG(const torch::Tensor& data, int64_t quality) {
// Define compression structures and error handling
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#pragma once

#include <torch/torch.h>
#include <torch/types.h>

C10_EXPORT torch::Tensor encodeJPEG(const torch::Tensor& data, int64_t quality);
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
#include "writejpeg_cpu.h"

#include <setjmp.h>
#include <string>

#if !PNG_FOUND

torch::Tensor encodePNG(const torch::Tensor& data, int64_t compression_level) {
TORCH_CHECK(false, "encodePNG: torchvision not compiled with libpng support");
}

#else

#include <png.h>
#include <setjmp.h>

struct torch_mem_encode {
char* buffer;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/types.h>

C10_EXPORT torch::Tensor encodePNG(
const torch::Tensor& data,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@

#include "image.h"
#include <ATen/ATen.h>

#include <Python.h>

// If we are in a Windows environment, we need to define
Expand Down
8 changes: 8 additions & 0 deletions torchvision/csrc/io/image/image.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#pragma once

#include "cpu/read_image_cpu.h"
#include "cpu/read_write_file_cpu.h"
#include "cpu/readjpeg_cpu.h"
#include "cpu/readpng_cpu.h"
#include "cpu/writejpeg_cpu.h"
#include "cpu/writepng_cpu.h"
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#if JPEG_FOUND
#include "jpegcommon.h"
#include <string>

#if JPEG_FOUND
void torch_jpeg_error_exit(j_common_ptr cinfo) {
/* cinfo->err really points to a torch_jpeg_error_mgr struct, so coerce
* pointer */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
#if JPEG_FOUND
#include <jpeglib.h>
#include <setjmp.h>
#include <string>

static const JOCTET EOI_BUFFER[1] = {JPEG_EOI};
struct torch_jpeg_error_mgr {
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion torchvision/csrc/models/alexnet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/densenet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
2 changes: 0 additions & 2 deletions torchvision/csrc/models/googlenet.cpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
#include "googlenet.h"

#include "modelsimpl.h"

namespace vision {
namespace models {

Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/googlenet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/inception.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/mnasnet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/mobilenet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
6 changes: 1 addition & 5 deletions torchvision/csrc/models/modelsimpl.h
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
#pragma once

#include <torch/torch.h>

#ifndef TORCH_CHECK
#define TORCH_CHECK AT_CHECK
#endif
#include <torch/nn.h>

namespace vision {
namespace models {
Expand Down
2 changes: 0 additions & 2 deletions torchvision/csrc/models/resnet.cpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
#include "resnet.h"

#include "modelsimpl.h"

namespace vision {
namespace models {
namespace _resnetimpl {
Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/resnet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
Loading

0 comments on commit eed0807

Please sign in to comment.