Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Restructuring C++ project #3146

Merged
merged 1 commit into from
Dec 10, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 16 additions & 24 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,6 @@ function(CUDA_CONVERT_FLAGS EXISTING_TARGET)
endif()
endfunction()

file(GLOB HEADERS torchvision/csrc/*.h)
# Image extension
file(GLOB IMAGE_HEADERS torchvision/csrc/cpu/image/*.h)
file(GLOB IMAGE_SOURCES torchvision/csrc/cpu/image/*.cpp)
file(GLOB OPERATOR_HEADERS torchvision/csrc/cpu/*.h)
file(GLOB OPERATOR_SOURCES ${OPERATOR_HEADERS} torchvision/csrc/cpu/*.cpp ${IMAGE_HEADERS} ${IMAGE_SOURCES} ${HEADERS} torchvision/csrc/*.cpp)
if(WITH_CUDA)
file(GLOB OPERATOR_HEADERS ${OPERATOR_HEADERS} torchvision/csrc/cuda/*.h)
file(GLOB OPERATOR_SOURCES ${OPERATOR_SOURCES} ${OPERATOR_HEADERS} torchvision/csrc/cuda/*.cu)
endif()
file(GLOB MODELS_HEADERS torchvision/csrc/models/*.h)
file(GLOB MODELS_SOURCES torchvision/csrc/models/*.h torchvision/csrc/models/*.cpp)

if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4819")
if(WITH_CUDA)
Expand All @@ -64,7 +51,17 @@ endif()
include(GNUInstallDirs)
include(CMakePackageConfigHelpers)

add_library(${PROJECT_NAME} SHARED ${MODELS_SOURCES} ${OPERATOR_SOURCES} ${IMAGE_SOURCES})
set(TVCPP torchvision/csrc)
list(APPEND ALLOW_LISTED ${TVCPP} ${TVCPP}/io/image ${TVCPP}/io/image/cpu ${TVCPP}/models ${TVCPP}/ops ${TVCPP}/ops/cpu)
if(WITH_CUDA)
list(APPEND ALLOW_LISTED ${TVCPP}/ops/cuda ${TVCPP}/ops/autocast)
endif()

FOREACH(DIR ${ALLOW_LISTED})
file(GLOB ALL_SOURCES ${ALL_SOURCES} ${DIR}/*.*)
ENDFOREACH()

add_library(${PROJECT_NAME} SHARED ${ALL_SOURCES})
target_link_libraries(${PROJECT_NAME} PRIVATE ${TORCH_LIBRARIES} ${PNG_LIBRARY} ${JPEG_LIBRARIES} Python3::Python)
set_target_properties(${PROJECT_NAME} PROPERTIES
EXPORT_NAME TorchVision
Expand Down Expand Up @@ -95,13 +92,8 @@ install(EXPORT TorchVisionTargets
NAMESPACE TorchVision::
DESTINATION ${TORCHVISION_CMAKECONFIG_INSTALL_DIR})

install(FILES ${HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME})
install(FILES
${OPERATOR_HEADERS}
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/cpu)
if(WITH_CUDA)
install(FILES
${OPERATOR_HEADERS}
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/cuda)
endif()
install(FILES ${MODELS_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/models)
FOREACH(INPUT_DIR ${ALLOW_LISTED})
string(REPLACE "${TVCPP}" "${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}" OUTPUT_DIR ${INPUT_DIR})
file(GLOB INPUT_FILES ${INPUT_DIR}/*.*)
install(FILES ${INPUT_FILES} DESTINATION ${OUTPUT_DIR})
ENDFOREACH()
2 changes: 2 additions & 0 deletions examples/cpp/hello_world/main.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
#include <iostream>
#include <torch/torch.h>
#include <torchvision/vision.h>
#include <torchvision/models/resnet.h>

int main()
Expand Down
27 changes: 15 additions & 12 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,9 @@ def get_extensions():
this_dir = os.path.dirname(os.path.abspath(__file__))
extensions_dir = os.path.join(this_dir, 'torchvision', 'csrc')

main_file = glob.glob(os.path.join(extensions_dir, '*.cpp'))
source_cpu = glob.glob(os.path.join(extensions_dir, 'cpu', '*.cpp'))
main_file = glob.glob(os.path.join(extensions_dir, '*.cpp')) + glob.glob(os.path.join(extensions_dir, 'ops',
'*.cpp'))
source_cpu = glob.glob(os.path.join(extensions_dir, 'ops', 'cpu', '*.cpp'))

is_rocm_pytorch = False
if torch.__version__ >= '1.5':
Expand All @@ -146,17 +147,19 @@ def get_extensions():
hipify_python.hipify(
project_directory=this_dir,
output_directory=this_dir,
includes="torchvision/csrc/cuda/*",
includes="torchvision/csrc/ops/cuda/*",
show_detailed=True,
is_pytorch_extension=True,
)
source_cuda = glob.glob(os.path.join(extensions_dir, 'hip', '*.hip'))
source_cuda = glob.glob(os.path.join(extensions_dir, 'ops', 'hip', '*.hip'))
# Copy over additional files
for file in glob.glob(r"torchvision/csrc/cuda/*.h"):
shutil.copy(file, "torchvision/csrc/hip")
for file in glob.glob(r"torchvision/csrc/ops/cuda/*.h"):
shutil.copy(file, "torchvision/csrc/ops/hip")

else:
source_cuda = glob.glob(os.path.join(extensions_dir, 'cuda', '*.cu'))
source_cuda = glob.glob(os.path.join(extensions_dir, 'ops', 'cuda', '*.cu'))

source_cuda += glob.glob(os.path.join(extensions_dir, 'ops', 'autocast', '*.cpp'))

sources = main_file + source_cpu
extension = CppExtension
Expand Down Expand Up @@ -309,8 +312,8 @@ def get_extensions():
image_library += [jpeg_lib]
image_include += [jpeg_include]

image_path = os.path.join(extensions_dir, 'cpu', 'image')
image_src = glob.glob(os.path.join(image_path, '*.cpp'))
image_path = os.path.join(extensions_dir, 'io', 'image')
image_src = glob.glob(os.path.join(image_path, '*.cpp')) + glob.glob(os.path.join(image_path, 'cpu', '*.cpp'))

if png_found or jpeg_found:
ext_modules.append(extension(
Expand Down Expand Up @@ -377,13 +380,13 @@ def get_extensions():
print("ffmpeg library_dir: {}".format(ffmpeg_library_dir))

# TorchVision base decoder + video reader
video_reader_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'cpu', 'video_reader')
video_reader_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'io', 'video_reader')
video_reader_src = glob.glob(os.path.join(video_reader_src_dir, "*.cpp"))
base_decoder_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'cpu', 'decoder')
base_decoder_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'io', 'decoder')
base_decoder_src = glob.glob(
os.path.join(base_decoder_src_dir, "*.cpp"))
# Torchvision video API
videoapi_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'cpu', 'video')
videoapi_src_dir = os.path.join(this_dir, 'torchvision', 'csrc', 'io', 'video')
videoapi_src = glob.glob(os.path.join(videoapi_src_dir, "*.cpp"))
# exclude tests
base_decoder_src = [x for x in base_decoder_src if '_test.cpp' not in x]
Expand Down
5 changes: 2 additions & 3 deletions test/tracing/frcnn/test_frcnn_tracing.cpp
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
#include <ATen/ATen.h>
#include <torch/script.h>
#include <torch/torch.h>
#include <torchvision/nms.h>
#include <torchvision/roi_align.h>
#include <torchvision/vision.h>
#include <torchvision/ops/nms.h>

#ifdef _WIN32
// Windows only
Expand Down
11 changes: 0 additions & 11 deletions torchvision/csrc/cpu/image/image.h

This file was deleted.

File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include "read_image_cpu.h"

#include "readjpeg_cpu.h"
#include "readpng_cpu.h"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#pragma once

#include <torch/torch.h>
#include "image_read_mode.h"
#include <torch/types.h>
#include "../image_read_mode.h"

C10_EXPORT torch::Tensor decode_image(
const torch::Tensor& data,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
#pragma once

#include <errno.h>
#include <sys/stat.h>
#include <torch/torch.h>
#include <torch/types.h>

C10_EXPORT torch::Tensor read_file(const std::string& filename);

Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
#include "readjpeg_cpu.h"

#include <ATen/ATen.h>

#if !JPEG_FOUND
torch::Tensor decodeJPEG(const torch::Tensor& data, ImageReadMode mode) {
TORCH_CHECK(
false, "decodeJPEG: torchvision not compiled with libjpeg support");
}
#else
#include <jpeglib.h>
#include <setjmp.h>
#include "jpegcommon.h"
#include "../jpegcommon.h"

struct torch_jpeg_mgr {
struct jpeg_source_mgr pub;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#pragma once

#include <torch/torch.h>
#include "image_read_mode.h"
#include <torch/types.h>
#include "../image_read_mode.h"

C10_EXPORT torch::Tensor decodeJPEG(
const torch::Tensor& data,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
#include "readpng_cpu.h"

#include <ATen/ATen.h>

#if !PNG_FOUND
torch::Tensor decodePNG(const torch::Tensor& data, ImageReadMode mode) {
TORCH_CHECK(false, "decodePNG: torchvision not compiled with libPNG support");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#pragma once

#include <torch/torch.h>
#include "image_read_mode.h"
#include <torch/types.h>
#include "../image_read_mode.h"

C10_EXPORT torch::Tensor decodePNG(
const torch::Tensor& data,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
#include "writejpeg_cpu.h"

#include <setjmp.h>
#include <string>

#if !JPEG_FOUND

torch::Tensor encodeJPEG(const torch::Tensor& data, int64_t quality) {
Expand All @@ -11,9 +8,7 @@ torch::Tensor encodeJPEG(const torch::Tensor& data, int64_t quality) {
}

#else

#include <jpeglib.h>
#include "jpegcommon.h"
#include "../jpegcommon.h"

torch::Tensor encodeJPEG(const torch::Tensor& data, int64_t quality) {
// Define compression structures and error handling
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#pragma once

#include <torch/torch.h>
#include <torch/types.h>

C10_EXPORT torch::Tensor encodeJPEG(const torch::Tensor& data, int64_t quality);
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
#include "writejpeg_cpu.h"

#include <setjmp.h>
#include <string>

#if !PNG_FOUND

torch::Tensor encodePNG(const torch::Tensor& data, int64_t compression_level) {
TORCH_CHECK(false, "encodePNG: torchvision not compiled with libpng support");
}

#else

#include <png.h>
#include <setjmp.h>

struct torch_mem_encode {
char* buffer;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/types.h>

C10_EXPORT torch::Tensor encodePNG(
const torch::Tensor& data,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@

#include "image.h"
#include <ATen/ATen.h>

#include <Python.h>

// If we are in a Windows environment, we need to define
Expand Down
8 changes: 8 additions & 0 deletions torchvision/csrc/io/image/image.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#pragma once

#include "cpu/read_image_cpu.h"
#include "cpu/read_write_file_cpu.h"
#include "cpu/readjpeg_cpu.h"
#include "cpu/readpng_cpu.h"
#include "cpu/writejpeg_cpu.h"
#include "cpu/writepng_cpu.h"
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#if JPEG_FOUND
#include "jpegcommon.h"
#include <string>

#if JPEG_FOUND
void torch_jpeg_error_exit(j_common_ptr cinfo) {
/* cinfo->err really points to a torch_jpeg_error_mgr struct, so coerce
* pointer */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
#if JPEG_FOUND
#include <jpeglib.h>
#include <setjmp.h>
#include <string>

static const JOCTET EOI_BUFFER[1] = {JPEG_EOI};
struct torch_jpeg_error_mgr {
Expand Down
File renamed without changes.
2 changes: 1 addition & 1 deletion torchvision/csrc/models/alexnet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/densenet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
2 changes: 0 additions & 2 deletions torchvision/csrc/models/googlenet.cpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
#include "googlenet.h"

#include "modelsimpl.h"

namespace vision {
namespace models {

Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/googlenet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/inception.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/mnasnet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/mobilenet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
6 changes: 1 addition & 5 deletions torchvision/csrc/models/modelsimpl.h
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
#pragma once

#include <torch/torch.h>

#ifndef TORCH_CHECK
#define TORCH_CHECK AT_CHECK
#endif
#include <torch/nn.h>

namespace vision {
namespace models {
Expand Down
2 changes: 0 additions & 2 deletions torchvision/csrc/models/resnet.cpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
#include "resnet.h"

#include "modelsimpl.h"

namespace vision {
namespace models {
namespace _resnetimpl {
Expand Down
2 changes: 1 addition & 1 deletion torchvision/csrc/models/resnet.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#pragma once

#include <torch/torch.h>
#include <torch/nn.h>
#include "../macros.h"

namespace vision {
Expand Down
Loading