Add astc support (#433)

This is the start of astc support using `lib/astc-encoder`, a `git subrepo` clone of ARM's ASTC encoder. 

Since ktx-software now supports multiple encoders this patch extracts
out the common options into '--encode <encoding>' and removes redundant options.
Also adds enums for astc compressor into ktx.h.

Similarly the `--threads` option has been moved from the BasisU options to the
common options so it can be used for any encoder.

The `--bcmp` option has been renamed to `--etc1s` to reflect its actual use.

A common '--normal_mode' option has been added replacing the ETC1S encoder's `--normal_map`.
The new option is recognized by the ASTC and ETC1S encoders.

A very useful feature of uploading the output files of broken toktx-tests to a cloud storage service has been added to `.travis.yml`.

.github/workflows/main.yml

@@ -39,12 +39,14 @@ jobs:
       env:
         ANDROID_ABI: arm64-v8a
         ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }}
+        ASTC_ISA: "ISA_NEON=ON"
 
     - name: android_arm64-v8a
       run: ./ci_scripts/build_android.sh
       env:
         ANDROID_ABI: arm64-v8a
         ANDROID_NDK: ${{ steps.setup-ndk.outputs.ndk-path }}
+        ASTC_ISA: "ISA_NEON=ON"
 
     - name: android_armeabi-v7a
       run: ./ci_scripts/build_android.sh

.travis.yml

@@ -51,6 +51,10 @@ jobs:
 # Phase 3: Install cache components - Currently not using a cache
 # then the following phases:
 before_install:
+- echo "Running on the following travis CI runner"
+- uname -a
+- echo "CMake version on the runner is"
+- cmake --version
 - eval "${MATRIX_EVAL}"
 - echo -e "machine github.com\n  login $GITHUB_TOKEN" >> ~/.netrc # Prevent rate limiting on Git LFS.
 - cat ~/.netrc
@@ -103,6 +107,12 @@ script:
 after_script:
  - sleep 10
 
+# after_failure:
+#   - echo "Now uploading the failed tests"
+#   - ls -alsh ./tests/testimages/toktx*
+#   - tar -cvf failed-tests.tar ./tests/testimages/toktx*
+#   - curl --upload-file failed-tests.tar https://transfer.sh/toktx-failed-tests.tar
+
 # Errors in something run in after_success: don't cause the build to fail so don't use.
 
 before_deploy:

CMakeLists.txt

@@ -74,7 +74,7 @@ endif()
 if(APPLE)
     set(CMAKE_OSX_DEPLOYMENT_TARGET "10.11" CACHE STRING "macOS Deployment Target")
     if(IOS)
-        set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "10.0" CACHE STRING "iOS Deployment Target")
+        set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "11.0" CACHE STRING "iOS Deployment Target")
         set(CMAKE_XCODE_ATTRIBUTE_ONLY_ACTIVE_ARCH NO)
     endif()
 endif()
@@ -456,6 +456,7 @@ target_sources(
     ktx
 PRIVATE
     lib/basis_encode.cpp
+    lib/astc_encode.cpp
     ${BASISU_ENCODER_C_SRC}
     ${BASISU_ENCODER_CXX_SRC}
     lib/writer1.c
@@ -649,6 +650,49 @@ PUBLIC
 
 add_subdirectory(interface/basisu_c_binding)
 
+# Only one architecture is supported at once, if neither of
+# ISA_SSE41 and ISA_SSE2  are defined ISA_AVX2 is chosen.
+# If ISA_AVX2 fails to compile user must chose other x86 options.
+# On arm based systems ISA_NEON is default
+
+list(FIND CMAKE_OSX_ARCHITECTURES "$(ARCHS_STANDARD)" ASTC_BUILD_UNIVERSAL)
+
+if(${ASTC_BUILD_UNIVERSAL} EQUAL -1)
+    if (${ISA_NONE})
+        set(ASTC_LIB_NAME astcenc-none-static)
+    else()
+        if(CPU_ARCHITECTURE STREQUAL x86_64 OR CPU_ARCHITECTURE STREQUAL x86)
+            if (${ISA_SSE41})
+                set(ASTC_LIB_NAME astcenc-sse4.1-static)
+            elseif (${ISA_SSE2})
+                set(ASTC_LIB_NAME astcenc-sse2-static)
+            else()
+                set(ISA_AVX2 ON)
+                set(ASTC_LIB_NAME astcenc-avx2-static)
+            endif()
+            if(CPU_ARCHITECTURE STREQUAL x86)
+                set(ISA_NONE ON)
+                set(ISA_AVX2 OFF)
+                set(ASTCENC_POPCNT 0)
+                set(ASTC_LIB_NAME astcenc-none-static)
+            endif()
+        elseif(CPU_ARCHITECTURE STREQUAL armv8 OR CPU_ARCHITECTURE STREQUAL arm64)
+            set(ASTC_LIB_NAME astcenc-neon-static)
+        else()
+            message(STATUS "Unsupported ISA for ${CPU_ARCHITECTURE} arch, using ISA_NONE.")
+            set(ISA_NONE ON)
+        endif()
+    endif()
+else()
+    set(ASTC_LIB_NAME astcenc-static)
+endif()
+
+# astcenc
+set(CLI OFF) # Only build as library not the CLI astcencoder
+add_subdirectory(lib/astc-encoder)
+set_property(TARGET ${ASTC_LIB_NAME} PROPERTY POSITION_INDEPENDENT_CODE ON)
+target_link_libraries(ktx PRIVATE ${ASTC_LIB_NAME})
+
 # Tools
 if(KTX_FEATURE_TOOLS)
     add_subdirectory(tools)
@@ -662,13 +706,18 @@ if(KTX_FEATURE_DOC)
     include(cmake/docs.cmake)
 endif()
 
+set(KTX_INSTALL_TARGETS ktx)
+
+if(KTX_FEATURE_STATIC_LIBRARY)
+    list(APPEND KTX_INSTALL_TARGETS ${ASTC_LIB_NAME})
+endif()
 
 # Install
 
 if(APPLE OR LINUX)
     # Have library's name links as separate component
     set(KTX_NAMELINKS ON)
-    install(TARGETS ktx
+    install(TARGETS ${KTX_INSTALL_TARGETS}
         EXPORT KTXTargets
         LIBRARY
             DESTINATION ${CMAKE_INSTALL_LIBDIR}
@@ -678,7 +727,7 @@ if(APPLE OR LINUX)
             DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
             COMPONENT dev
     )
-    install(TARGETS ktx
+    install(TARGETS ${KTX_INSTALL_TARGETS}
         LIBRARY
             DESTINATION ${CMAKE_INSTALL_LIBDIR}
             COMPONENT library
@@ -687,7 +736,7 @@ if(APPLE OR LINUX)
 else()
     # No name links on Windows
     set(KTX_NAMELINKS OFF)
-    install(TARGETS ktx
+    install(TARGETS ${KTX_INSTALL_TARGETS}
         EXPORT KTXTargets
         ARCHIVE
             DESTINATION ${CMAKE_INSTALL_LIBDIR}

ci_scripts/build_android.sh

@@ -6,9 +6,10 @@ set -e
 
 # Fallback to arm64-v8a
 ANDROID_ABI=${ANDROID_ABI:-'arm64-v8a'}
+ASTC_ISA=${ASTC_ISA:-'ISA_NONE=ON'}
 
 # You need to set the following environment variables first
-# ANDROID_NDK= <Path to Android NDK> 
+# ANDROID_NDK= <Path to Android NDK>
 
 echo "Configure KTX-Software (Android $ANDROID_ABI Release)"
 cmake . -G Ninja -B "build-android-$ANDROID_ABI" \
@@ -17,7 +18,8 @@ cmake . -G Ninja -B "build-android-$ANDROID_ABI" \
 -DANDROID_NDK="$ANDROID_NDK" \
 -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \
 -DCMAKE_BUILD_TYPE=Release \
--DBASISU_SUPPORT_SSE=OFF
+-DBASISU_SUPPORT_SSE=OFF \
+-D${ASTC_ISA}
 
 pushd "build-android-$ANDROID_ABI"
 

ci_scripts/build_android_debug.sh

@@ -6,9 +6,10 @@ set -e
 
 # Fallback to arm64-v8a
 ANDROID_ABI=${ANDROID_ABI:-'arm64-v8a'}
+ASTC_ISA=${ASTC_ISA:-'ISA_NONE=ON'}
 
 # You need to set the following environment variables first
-# ANDROID_NDK= <Path to Android NDK> 
+# ANDROID_NDK= <Path to Android NDK>
 
 echo "Configure KTX-Software (Android $ANDROID_ABI Debug)"
 cmake . -G Ninja -B "build-android-$ANDROID_ABI-debug" \
@@ -17,7 +18,8 @@ cmake . -G Ninja -B "build-android-$ANDROID_ABI-debug" \
 -DANDROID_NDK="$ANDROID_NDK" \
 -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \
 -DCMAKE_BUILD_TYPE=Debug \
--DBASISU_SUPPORT_SSE=OFF
+-DBASISU_SUPPORT_SSE=OFF \
+-D${ASTC_ISA}
 
 pushd "build-android-$ANDROID_ABI-debug"
 

ci_scripts/build_macos.sh

@@ -63,8 +63,8 @@ echo "Configure KTX-Software (macOS x86_64) with SSE support"
 cmake -GXcode -Bbuild-macos-sse \
   -DCMAKE_OSX_ARCHITECTURES="x86_64" \
   -DKTX_FEATURE_LOADTEST_APPS=ON \
-  -DBASISU_SUPPORT_SSE=ON
-
+  -DBASISU_SUPPORT_SSE=ON \
+  -DISA_SSE41=ON
 
 # Cause the build pipes below to set the exit to the exit code of the
 # last program to exit non-zero.
@@ -117,7 +117,7 @@ popd
 #
 
 echo "Configure KTX-Software (iOS)"
-cmake -GXcode -Bbuild-ios -DCMAKE_SYSTEM_NAME=iOS -DKTX_FEATURE_LOADTEST_APPS=ON -DKTX_FEATURE_DOC=OFF
+cmake -GXcode -Bbuild-ios -DISA_NEON=ON -DCMAKE_SYSTEM_NAME=iOS -DKTX_FEATURE_LOADTEST_APPS=ON -DKTX_FEATURE_DOC=OFF
 pushd build-ios
 echo "Build KTX-Software (iOS Debug)"
 cmake --build . --config Debug  -- -sdk iphoneos CODE_SIGN_IDENTITY="" CODE_SIGNING_ALLOWED=NO CODE_SIGNING_REQUIRED=NO | handle_compiler_output

include/ktx.h

@@ -1092,8 +1092,137 @@ typedef enum ktx_pack_uastc_flag_bits_e {
 } ktx_pack_uastc_flag_bits_e;
 typedef ktx_uint32_t ktx_pack_uastc_flags;
 
+/**
+ * @~English
+ * @brief Options specifiying ASTC encoding quality levels.
+ */
+typedef enum ktx_pack_astc_quality_levels_e {
+    KTX_PACK_ASTC_QUALITY_LEVEL_FASTEST  = 0,
+        /*!< Fastest compression. */
+    KTX_PACK_ASTC_QUALITY_LEVEL_FAST   = 10,
+        /*!< Fast compression. */
+    KTX_PACK_ASTC_QUALITY_LEVEL_MEDIUM   = 60,
+        /*!< Medium compression. */
+    KTX_PACK_ASTC_QUALITY_LEVEL_THOROUGH   = 98,
+        /*!< Slower compression. */
+    KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE = 100,
+        /*!< Very slow compression. */
+    KTX_PACK_ASTC_QUALITY_LEVEL_MAX = KTX_PACK_ASTC_QUALITY_LEVEL_EXHAUSTIVE,
+        /*!< Maximum supported quality level. */
+} ktx_pack_astc_quality_levels_e;
+
+/**
+ * @~English
+ * @brief Options specifiying ASTC encoding block dimensions
+ */
+typedef enum ktx_pack_astc_block_dimension_e {
+    // 2D formats
+    KTX_PACK_ASTC_BLOCK_DIMENSION_4x4,                    //: 8.00 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_5x4,                    //: 6.40 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_5x5,                    //: 5.12 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_6x5,                    //: 4.27 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_6x6,                    //: 3.56 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_8x5,                    //: 3.20 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_8x6,                    //: 2.67 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_10x5,                   //: 2.56 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_10x6,                   //: 2.13 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_8x8,                    //: 2.00 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_10x8,                   //: 1.60 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_10x10,                  //: 1.28 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_12x10,                  //: 1.07 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_12x12,                  //: 0.89 bpp
+    // 3D formats
+    KTX_PACK_ASTC_BLOCK_DIMENSION_3x3x3,                  //: 4.74 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_4x3x3,                  //: 3.56 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x3,                  //: 2.67 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_4x4x4,                  //: 2.00 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_5x4x4,                  //: 1.60 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x4,                  //: 1.28 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_5x5x5,                  //: 1.02 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_6x5x5,                  //: 0.85 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x5,                  //: 0.71 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x6,                  //: 0.59 bpp
+    KTX_PACK_ASTC_BLOCK_DIMENSION_MAX = KTX_PACK_ASTC_BLOCK_DIMENSION_6x6x6
+        /*!< Maximum supported blocks. */
+} ktx_pack_astc_block_dimension_e;
+
+/**
+ * @~English
+ * @brief Options specifiying ASTC encoder profile function
+ */
+typedef enum ktx_pack_astc_encoder_function_e {
+    KTX_PACK_ASTC_ENCODER_FUNCTION_UNKNOWN,
+    KTX_PACK_ASTC_ENCODER_FUNCTION_SRGB,
+    KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR,
+    KTX_PACK_ASTC_ENCODER_FUNCTION_MAX = KTX_PACK_ASTC_ENCODER_FUNCTION_LINEAR
+} ktx_pack_astc_encoder_function_e;
+
+/**
+ * @~English
+ * @brief Options specifying ASTC encoder profile mode
+ *        This and function is used later to derive the profile.
+ */
+typedef enum ktx_pack_astc_encoder_mode_e {
+    KTX_PACK_ASTC_ENCODER_MODE_DEFAULT,
+    KTX_PACK_ASTC_ENCODER_MODE_LDR,
+    KTX_PACK_ASTC_ENCODER_MODE_HDR,
+    KTX_PACK_ASTC_ENCODER_MODE_MAX = KTX_PACK_ASTC_ENCODER_MODE_HDR
+} ktx_pack_astc_encoder_mode_e;
+
 extern KTX_API const ktx_uint32_t KTX_ETC1S_DEFAULT_COMPRESSION_LEVEL;
 
+/**
+ * @memberof ktxTexture
+ * @~English
+ * @brief Structure for passing extended parameters to
+ *        ktxTexture_CompressAstc.
+ *
+ * Passing a struct initialized to 0 (e.g. " = {};") will use the default
+ * values. Only those settings to be modified need be non-zero.
+ */
+typedef struct ktxAstcParams {
+    ktx_uint32_t structSize;
+        /*!< Size of this struct. Used so library can tell which version
+             of struct is being passed.
+         */
+    ktx_bool_t verbose;
+        /*!< If true, prints Astc encoder operation details to
+             @c stdout. Not recommended for GUI apps.
+         */
+    ktx_uint32_t threadCount;
+        /*!< Number of threads used for compression. Default is 1. */
+
+    /* astcenc params */
+    ktx_uint32_t blockDimension;
+        /*!< Combinations of block dimensions that astcenc supports
+          i.e. 6x6, 8x8, 6x5 etc*/
+
+    ktx_uint32_t function;
+        /*!< Can be {linear/srgb} from astcenc*/
+
+    ktx_uint32_t mode;
+        /*!< Can be {ldr/hdr} from astcenc*/
+
+    ktx_uint32_t qualityLevel;
+        /*!< astcenc supports -fastest, -fast, -medium, -thorough, -exhaustive*/
+
+    ktx_bool_t normalMap;
+        /*!< Tunes codec parameters for better quality on normal maps
+          In this mode normals are compressed to X,Y components
+          Discarding Z component, reader will need to generate Z
+          component in shaders.
+        */
+    char inputSwizzle[4];
+         /*!< A swizzle to provide as input to astcenc. It must match the regular
+             expression /^[rgba01]{4}$/.*/
+} ktxAstcParams;
+
+KTX_API KTX_error_code KTX_APIENTRY
+ktxTexture_CompressAstcEx(ktxTexture* This, ktxAstcParams* params);
+
+KTX_API KTX_error_code KTX_APIENTRY
+ktxTexture_CompressAstc(ktxTexture* This, ktx_uint32_t quality);
+
 /**
  * @memberof ktxTexture2
  * @~English
@@ -1648,4 +1777,3 @@ Initial release.
 */
 
 #endif /* KTX_H_A55A6F00956F42F3A137C11929827FE1 */
-