diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index b1db208a9..65682b173 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -42,7 +42,7 @@ Please: - Describe the focus area: transmission controls, offline storage, backend routing controls, reliability, performance, etc. - Add any other additional context. -We will review your question during our weekly community meeting or sooner. +We will review your question as soon as possible. --- diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml index 06bce4267..2d50517d8 100644 --- a/.github/workflows/build-android.yml +++ b/.github/workflows/build-android.yml @@ -17,13 +17,18 @@ on: - main - dev + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: runs-on: windows-latest name: Build for Android steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: false - name: Update submodules @@ -32,7 +37,7 @@ jobs: git config --global submodule.lib/modules.update none git -c protocol.version=2 submodule update --init --force --depth=1 - name: Setup Java - uses: actions/setup-java@v3 + uses: actions/setup-java@v5 with: distribution: 'adopt' java-version: '17' @@ -40,14 +45,13 @@ jobs: # Workaround for: 'Unable to decrypt local Maven settings credentials' run: rm $Env:USERPROFILE\.m2\settings.xml - name: Setup Android SDK - uses: android-actions/setup-android@v2 + uses: android-actions/setup-android@v3 - name: Install NDK run: | java -version gci env:* | sort-object name - new-item "C:\Users\runneradmin\.android\repositories.cfg" -ItemType "file" - echo yes | .\sdkmanager.bat "ndk-bundle" "cmake;3.10.2.4988404" "ndk;27.0.12077973" --sdk_root=$Env:ANDROID_SDK_ROOT - working-directory: ${{ env.ANDROID_SDK_ROOT }}\cmdline-tools\7.0\bin + new-item "$Env:USERPROFILE\.android\repositories.cfg" -ItemType "file" + echo yes | sdkmanager "ndk-bundle" "cmake;3.10.2.4988404" "ndk;27.0.12077973" --sdk_root=$Env:ANDROID_SDK_ROOT - name: Chocolatey run: | choco install --no-progress -y ninja diff --git a/.github/workflows/build-ios-mac.yml b/.github/workflows/build-ios-mac.yml index 80c5f981d..be9ec53f9 100644 --- a/.github/workflows/build-ios-mac.yml +++ b/.github/workflows/build-ios-mac.yml @@ -19,36 +19,42 @@ on: schedule: - cron: 0 2 * * 1-5 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: strategy: matrix: - os: [macos-13, macos-15] + os: [macos-14, macos-15] config: [release, debug] simulator: ["'iPhone 15'", "'iPad Pro (11-inch) (4th generation)'", "'iPhone 16'", "'iPad Air 11-inch (M2)'"] exclude: - - os: macos-13 + - os: macos-14 simulator: "'iPhone 16'" - - os: macos-13 + - os: macos-14 simulator: "'iPad Air 11-inch (M2)'" - os: macos-15 simulator: "'iPhone 15'" - os: macos-15 simulator: "'iPad Pro (11-inch) (4th generation)'" runs-on: ${{ matrix.os }} + timeout-minutes: 30 env: CMAKE_POLICY_VERSION_MINIMUM: "3.5" steps: - name: Grant write permissions to /usr/local run: | sudo chown -R $USER:staff /usr/local - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: 'true' continue-on-error: true - name: build run: | - if [[ "${{ matrix.os }}" == "macos-13" ]]; then + if [[ "${{ matrix.os }}" == "macos-14" ]]; then export IOS_DEPLOYMENT_TARGET=13.0; elif [[ "${{ matrix.os }}" == "macos-15" ]]; then export IOS_DEPLOYMENT_TARGET=15.0; diff --git a/.github/workflows/build-posix-latest.yml b/.github/workflows/build-posix-latest.yml index 2271a459b..657caa6e0 100644 --- a/.github/workflows/build-posix-latest.yml +++ b/.github/workflows/build-posix-latest.yml @@ -19,6 +19,11 @@ on: schedule: - cron: 0 2 * * 1-5 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: @@ -32,7 +37,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 continue-on-error: true - name: Test ${{ matrix.os }} ${{ matrix.config }} run: ./build-tests.sh ${{ matrix.config }} diff --git a/.github/workflows/build-ubuntu-2204.yml b/.github/workflows/build-ubuntu-2204.yml index e8f456bbc..e0ea03ac5 100644 --- a/.github/workflows/build-ubuntu-2204.yml +++ b/.github/workflows/build-ubuntu-2204.yml @@ -19,6 +19,11 @@ on: schedule: - cron: 0 2 * * 1-5 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: @@ -32,7 +37,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 continue-on-error: true - name: Test ${{ matrix.os }} ${{ matrix.config }} run: ./build-tests.sh ${{ matrix.config }} \ No newline at end of file diff --git a/.github/workflows/build-windows-clang.yaml.off b/.github/workflows/build-windows-clang.yaml.off index 9f2c790ea..2621613c1 100644 --- a/.github/workflows/build-windows-clang.yaml.off +++ b/.github/workflows/build-windows-clang.yaml.off @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 continue-on-error: true - name: Setup Tools diff --git a/.github/workflows/build-windows-vs2017.yaml.off b/.github/workflows/build-windows-vs2017.yaml.off index 1db4607db..a2d22827b 100644 --- a/.github/workflows/build-windows-vs2017.yaml.off +++ b/.github/workflows/build-windows-vs2017.yaml.off @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 continue-on-error: true - name: Setup Tools diff --git a/.github/workflows/build-windows-vs2022.yaml b/.github/workflows/build-windows-vs2022.yaml index a8a18394e..20605b442 100644 --- a/.github/workflows/build-windows-vs2022.yaml +++ b/.github/workflows/build-windows-vs2022.yaml @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 continue-on-error: true - name: Build diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 503cae673..3a55f2bc5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -14,6 +14,11 @@ on: schedule: - cron: '0 8 * * 1' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: analyze: name: Analyze @@ -34,7 +39,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 continue-on-error: true # Initializes the CodeQL tools for scanning. @@ -87,7 +92,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 continue-on-error: true - name: Update submodules @@ -102,21 +107,20 @@ jobs: languages: java - name: Setup Java - uses: actions/setup-java@v3 + uses: actions/setup-java@v5 with: distribution: 'adopt' java-version: '17' - name: Remove default github maven configuration run: rm $Env:USERPROFILE\.m2\settings.xml - name: Setup Android SDK - uses: android-actions/setup-android@v2 + uses: android-actions/setup-android@v3 - name: Install NDK run: | java -version gci env:* | sort-object name - new-item "C:\Users\runneradmin\.android\repositories.cfg" -ItemType "file" - echo yes | .\sdkmanager.bat "ndk-bundle" "cmake;3.10.2.4988404" "ndk;27.0.12077973" --sdk_root=$Env:ANDROID_SDK_ROOT - working-directory: ${{ env.ANDROID_SDK_ROOT }}\cmdline-tools\7.0\bin + new-item "$Env:USERPROFILE\.android\repositories.cfg" -ItemType "file" + echo yes | sdkmanager "ndk-bundle" "cmake;3.10.2.4988404" "ndk;27.0.12077973" --sdk_root=$Env:ANDROID_SDK_ROOT - name: Chocolatey run: | choco install --no-progress -y ninja diff --git a/.github/workflows/deploy-docs-pages.yml b/.github/workflows/deploy-docs-pages.yml new file mode 100644 index 000000000..09ecd2d35 --- /dev/null +++ b/.github/workflows/deploy-docs-pages.yml @@ -0,0 +1,78 @@ +name: Deploy documentation to GitHub Pages + +on: + push: + branches: + - main + paths: + - .github/workflows/deploy-docs-pages.yml + - .readthedocs.yaml + - docs/** + - lib/include/** + pull_request: + paths: + - .github/workflows/deploy-docs-pages.yml + - .readthedocs.yaml + - docs/** + - lib/include/** + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: github-pages + cancel-in-progress: false + +jobs: + build: + name: Build documentation + runs-on: ubuntu-24.04 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install --yes doxygen graphviz + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r docs/public/requirements.txt + + - name: Build Sphinx documentation + run: python -m sphinx -q -b html docs/public docs/public/_build/html + + - name: Upload Pages artifact + if: github.event_name != 'pull_request' + uses: actions/upload-pages-artifact@v3 + with: + path: docs/public/_build/html + + deploy: + name: Deploy documentation + if: github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-24.04 + + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Configure GitHub Pages + uses: actions/configure-pages@v5 + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 69b136fa1..912594c61 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -6,6 +6,11 @@ on: pull_request: branches: [ master, main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build: runs-on: ubuntu-latest @@ -13,7 +18,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 continue-on-error: true - name: install misspell diff --git a/.github/workflows/test-android-mac.yml.off b/.github/workflows/test-android-mac.yml.off index 56b87579a..96e17c4a5 100644 --- a/.github/workflows/test-android-mac.yml.off +++ b/.github/workflows/test-android-mac.yml.off @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true depth: 1 @@ -42,7 +42,7 @@ jobs: script: ./testandlog - name: Upload if: ${{ always() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: logcat path: ./lib/android_build/logcat.txt \ No newline at end of file diff --git a/.github/workflows/test-win-latest.yml b/.github/workflows/test-win-latest.yml index 760c88fb0..19b20ef53 100644 --- a/.github/workflows/test-win-latest.yml +++ b/.github/workflows/test-win-latest.yml @@ -19,6 +19,11 @@ on: schedule: - cron: 0 2 * * 1-5 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: test: name: Test on Windows ${{ matrix.arch }}-${{ matrix.build }} @@ -32,13 +37,13 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 continue-on-error: true - name: setup-msbuild - uses: microsoft/setup-msbuild@v1.1 + uses: microsoft/setup-msbuild@v2 with: - vs-version: '[16,)' + vs-version: '[17,)' - name: Test ${{ matrix.arch }} ${{ matrix.build }} shell: cmd diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 281bf4b1f..dec4558cd 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,11 +1,19 @@ version: 2 +build: + os: ubuntu-24.04 + tools: + python: "3.12" + apt_packages: + - doxygen + - graphviz + submodules: exclude: all python: - install: - - requirements: docs/public/requirements.txt + install: + - requirements: docs/public/requirements.txt sphinx: - configuration: docs/public/conf.py + configuration: docs/public/conf.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 959134a61..51d09802a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.1.0) -project(MSTelemetry) +project(MSTelemetry LANGUAGES C CXX) # Set installation prefix for macOS and Linux if(UNIX AND NOT DEFINED CMAKE_INSTALL_PREFIX) @@ -25,7 +25,7 @@ endif() # Enable ARC for obj-c on Apple if(APPLE) - message("-- BUILD_IOS: ${BUILD_IOS}") + message(STATUS "BUILD_IOS: ${BUILD_IOS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fobjc-arc") # iOS build options @@ -77,9 +77,9 @@ if(APPLE) OUTPUT_VARIABLE CMAKE_OSX_SYSROOT ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - message("-- CMAKE_OSX_SYSROOT ${CMAKE_OSX_SYSROOT}") - message("-- ARCHITECTURE: ${CMAKE_SYSTEM_PROCESSOR}") - message("-- PLATFORM: ${IOS_PLATFORM}") + message(STATUS "CMAKE_OSX_SYSROOT ${CMAKE_OSX_SYSROOT}") + message(STATUS "ARCHITECTURE: ${CMAKE_SYSTEM_PROCESSOR}") + message(STATUS "PLATFORM: ${IOS_PLATFORM}") else() if(${MAC_ARCH} STREQUAL "x86_64") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64") @@ -99,39 +99,42 @@ if(APPLE) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64 -arch arm64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64 -arch arm64") endif() - message("-- MAC_ARCH: ${MAC_ARCH}") + message(STATUS "MAC_ARCH: ${MAC_ARCH}") endif() endif() -message("-- CMAKE_SYSTEM_INFO_FILE: ${CMAKE_SYSTEM_INFO_FILE}") -message("-- CMAKE_SYSTEM_NAME: ${CMAKE_SYSTEM_NAME}") -message("-- CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}") -message("-- CMAKE_SYSTEM: ${CMAKE_SYSTEM}") -message("-- CMAKE_SYSTEM_VERSION: ${CMAKE_SYSTEM_VERSION}") -message("-- CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") -message("-- TARGET_ARCH: ${TARGET_ARCH}") -message("-- CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") +message(STATUS "CMAKE_SYSTEM_INFO_FILE: ${CMAKE_SYSTEM_INFO_FILE}") +message(STATUS "CMAKE_SYSTEM_NAME: ${CMAKE_SYSTEM_NAME}") +message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}") +message(STATUS "CMAKE_SYSTEM: ${CMAKE_SYSTEM}") +message(STATUS "CMAKE_SYSTEM_VERSION: ${CMAKE_SYSTEM_VERSION}") +message(STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") +message(STATUS "TARGET_ARCH: ${TARGET_ARCH}") +message(STATUS "CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") include(tools/ParseOsRelease.cmake) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") -set(WARN_FLAGS "/W4 /WX") + set(WARN_FLAGS "/W4 /WX") +elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + # -Wno-unknown-warning-option is Clang-only, omitted here + set(WARN_FLAGS "-Wall -Werror -Wextra -Wno-unused-parameter -Wno-unused-but-set-variable") else() -# No -pedantic -Wno-extra-semi -Wno-gnu-zero-variadic-macro-arguments -set(WARN_FLAGS "-Wall -Werror -Wextra -Wno-unused-parameter -Wno-unknown-warning-option -Wno-unused-but-set-variable") + # No -pedantic -Wno-extra-semi -Wno-gnu-zero-variadic-macro-arguments + set(WARN_FLAGS "-Wall -Werror -Wextra -Wno-unused-parameter -Wno-unknown-warning-option -Wno-unused-but-set-variable") endif() if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") # Using GCC with -s and -Wl linker flags - set(REL_FLAGS "-s -Wl,--gc-sections -Os ${WARN_FLAGS} -ffunction-sections -fdata-sections -fmerge-all-constants -ffast-math") + set(REL_FLAGS "-s -Wl,--gc-sections -Os ${WARN_FLAGS} -ffunction-sections -fdata-sections -fmerge-all-constants") elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") set(REL_FLAGS "${WARN_FLAGS}") elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") # AppleClang does not support -ffunction-sections and -fdata-sections with the -fembed-bitcode and -fembed-bitcode-marker - set(REL_FLAGS "-Os ${WARN_FLAGS} -fmerge-all-constants -ffast-math") + set(REL_FLAGS "-Os ${WARN_FLAGS} -fmerge-all-constants") else() # Using clang - strip unsupported GCC options - set(REL_FLAGS "-Os ${WARN_FLAGS} -ffunction-sections -fmerge-all-constants -ffast-math") + set(REL_FLAGS "-Os ${WARN_FLAGS} -ffunction-sections -fmerge-all-constants") endif() ## Uncomment this to reduce the volume of note warnings on RPi4 w/gcc-8 Ref. https://gcc.gnu.org/ml/gcc/2017-05/msg00073.html @@ -144,12 +147,12 @@ set(DBG_FLAGS "-ggdb -gdwarf-2 -O0 ${WARN_FLAGS} -fno-builtin-malloc -fno-built if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") #TODO: -fno-rtti - message("Building Release ...") + message(STATUS "Building Release ...") set(CMAKE_C_FLAGS "$ENV{CFLAGS} ${CMAKE_C_FLAGS} -std=c11 ${REL_FLAGS}") set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} ${CMAKE_CXX_FLAGS} -std=c++11 ${REL_FLAGS}") else() set(USE_TCMALLOC 1) - message("Building Debug ...") + message(STATUS "Building Debug ...") include(tools/FindTcmalloc.cmake) set(CMAKE_C_FLAGS "$ENV{CFLAGS} ${CMAKE_C_FLAGS} -std=c11 ${DBG_FLAGS}") set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} ${CMAKE_CXX_FLAGS} -std=c++11 ${DBG_FLAGS}") @@ -190,7 +193,7 @@ endif() set(PAL_IMPLEMENTATION ${DEFAULT_PAL_IMPLEMENTATION}) -message(STATUS "-- PAL implementation: ${PAL_IMPLEMENTATION}") +message(STATUS "PAL implementation: ${PAL_IMPLEMENTATION}") string(TOUPPER ${PAL_IMPLEMENTATION} PAL_IMPLEMENTATION_UPPER) add_definitions(-DMATSDK_PAL_${PAL_IMPLEMENTATION_UPPER}=1) @@ -222,7 +225,7 @@ add_definitions(-DNOMINMAX) set(SDK_VERSION_PREFIX "EVT") add_definitions("-DMATSDK_VERSION_PREFIX=\"${SDK_VERSION_PREFIX}\"") -set(MATSDK_API_VERSION "3.4") +set(MATSDK_API_VERSION "3.10") string(TIMESTAMP DAYNUMBER "%j") string(REGEX REPLACE "^00" "" DAYNUMBER ${DAYNUMBER}) string(REGEX REPLACE "^0" "" DAYNUMBER ${DAYNUMBER}) @@ -238,7 +241,7 @@ else() set(MATSDK_BUILD_VERSION ${MATSDK_API_VERSION}.${DAYNUMBER}.0) endif() -message(STATUS "-- SDK version: ${SDK_VERSION_PREFIX}-${MATSDK_BUILD_VERSION}") +message(STATUS "SDK version: ${SDK_VERSION_PREFIX}-${MATSDK_BUILD_VERSION}") ################################################################################################ # HTTP stack section @@ -286,9 +289,9 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") endif() if(BUILD_UNIT_TESTS OR BUILD_FUNC_TESTS) - message("Adding gtest") + message(STATUS "Adding gtest") add_library(gtest STATIC IMPORTED GLOBAL) - message("Adding gmock") + message(STATUS "Adding gmock") add_library(gmock STATIC IMPORTED GLOBAL) endif() @@ -320,7 +323,7 @@ if(BUILD_LIBRARY) endif() if(BUILD_UNIT_TESTS OR BUILD_FUNC_TESTS) - message("Building tests") + message(STATUS "Building tests") enable_testing() add_subdirectory(tests) endif() diff --git a/README.md b/README.md index c4da1e1c6..3ddcbb580 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,9 @@ Other resources to learn how to setup the build system: * **Supported** - these platforms are known to work well with the SDK in production. * **Covered by CI** - these platforms are tested as part of CI. +* For iOS simulator, CI covers representative supported simulator + configurations on the current macOS runner images rather than every + supported iOS 12+ runtime. ## Test diff --git a/Solutions/before.targets b/Solutions/before.targets index 63a526c90..052e5d77f 100644 --- a/Solutions/before.targets +++ b/Solutions/before.targets @@ -3,12 +3,12 @@ $(SolutionDir)\..\third_party\krabsetw\krabs;$(CustomIncludePath) - v141 - v142 - v143 - + + v145 + v143 + v142 + v141 v141 - $(PlatformToolset) $([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0')) $(LatestTargetPlatformVersion) diff --git a/Solutions/build.MIP.props b/Solutions/build.MIP.props index 9bd6e6d91..2419c4c36 100644 --- a/Solutions/build.MIP.props +++ b/Solutions/build.MIP.props @@ -1,20 +1,24 @@ - - - - - %(PreprocessorDefinitions);CONFIG_CUSTOM_H="config-MIP.h";MATSDK_SHARED_LIB=1; - - - ucrtbased.dll; %(DelayLoadDLLs) - - - - true - v141 - 14.1 - - - mip_ClientTelemetry - - - + + + + + %(PreprocessorDefinitions);CONFIG_CUSTOM_H="config-MIP.h";MATSDK_SHARED_LIB=1; + + + ucrtbased.dll; %(DelayLoadDLLs) + + + + true + $(DefaultPlatformToolset) + v145 + v143 + v142 + v141 + v141 + + + mip_ClientTelemetry + + + diff --git a/build-all.bat b/build-all.bat index 6ab0d4ab0..5b535bc08 100644 --- a/build-all.bat +++ b/build-all.bat @@ -1,9 +1,38 @@ @echo off cd %~dp0 -call tools\gen-version.cmd @setlocal ENABLEEXTENSIONS +set CUSTOM_PROPS= +if not "%~1"=="" ( + if not exist "%~f1" ( + goto custom_props_missing + ) + if /I not "%~x1"==".props" ( + if /I not "%~x1"==".targets" ( + goto custom_props_invalid_type + ) + ) + set CUSTOM_PROPS="/p:ForceImportBeforeCppTargets=%~f1" + echo Using custom properties file for the build: + echo %CUSTOM_PROPS% +) + +goto after_custom_props_validation + +:custom_props_missing +echo ERROR: Custom build input not found: %~1 +echo Pass an existing MSBuild .props or .targets file to ForceImportBeforeCppTargets. +exit /b 1 + +:custom_props_invalid_type +echo ERROR: Custom build input must be an MSBuild .props or .targets file: %~1 +echo Pass the MSBuild import file, not the CONFIG_CUSTOM_H header. +exit /b 1 + +:after_custom_props_validation +call tools\gen-version.cmd + echo Update all public submodules... git -c submodule."lib/modules".update=none submodule update --init --recursive @@ -17,13 +46,6 @@ if NOT EXIST %GTEST_PATH%\CMakeLists.txt ( git clone --depth 1 --branch release-1.12.1 https://github.com/google/googletest %GTEST_PATH% ) -set CUSTOM_PROPS= -if ("%~1"=="") goto skip -set CUSTOM_PROPS="/p:ForceImportBeforeCppTargets=%1" -echo Using custom properties file for the build: -echo %CUSTOM_PROPS% -:skip - if NOT DEFINED SKIP_MD_BUILD ( REM DLL and static /MD build REM Release diff --git a/build-tests-ios.sh b/build-tests-ios.sh index bf29a50b3..3e4a40f46 100755 --- a/build-tests-ios.sh +++ b/build-tests-ios.sh @@ -7,14 +7,67 @@ set -e ./build-ios.sh ${SKU} -# dyld_info /Users/runner/work/cpp_client_telemetry/cpp_client_telemetry/out/lib/libmat.a - cd tests/unittests xcrun simctl list devices available echo 'End of xcrun simctl list devices available' -xcodebuild test -scheme iOSUnitTests -destination "platform=iOS Simulator,name=$SIMULATOR" +# Resolve simulator UUID from simctl JSON using an exact name match. +# If the same device name exists across multiple iOS runtimes, pick the +# newest runtime and fail if that runtime still contains duplicate matches. +SIM_MATCH=$( + xcrun simctl list devices available --json | python3 -c ' +import json +import re +import sys + +simulator_name = sys.argv[1] +devices_by_runtime = json.load(sys.stdin).get("devices", {}) +matches = [] + +for runtime, devices in devices_by_runtime.items(): + if not runtime.startswith("com.apple.CoreSimulator.SimRuntime.iOS-"): + continue + + match = re.search(r"iOS-(\d+)(?:-(\d+))?$", runtime) + if match is None: + continue + + version = tuple(int(part) for part in match.groups(default="0")) + runtime_label = "iOS " + ".".join(str(part) for part in version) + + for device in devices: + if device.get("name") == simulator_name and device.get("isAvailable", True): + matches.append((version, runtime_label, device["udid"])) + +if not matches: + print(f"ERROR: No available simulator found for {simulator_name!r}", file=sys.stderr) + raise SystemExit(1) + +newest_version = max(version for version, _, _ in matches) +newest_matches = [(runtime_label, udid) for version, runtime_label, udid in matches if version == newest_version] + +if len(newest_matches) != 1: + print(f"ERROR: Multiple available simulators found for {simulator_name!r} in newest runtime:", file=sys.stderr) + for runtime_label, udid in newest_matches: + print(f" - {runtime_label}: {udid}", file=sys.stderr) + raise SystemExit(1) + +runtime_label, udid = newest_matches[0] +print(f"{udid}|{runtime_label}") +' "$SIMULATOR" +) +SIM_ID=${SIM_MATCH%%|*} +SIM_RUNTIME=${SIM_MATCH#*|} + +if [ -z "$SIM_ID" ]; then + echo "ERROR: No available simulator found for '$SIMULATOR'" + exit 1 +fi + +echo "Using simulator: $SIMULATOR ($SIM_RUNTIME, id=$SIM_ID)" + +xcodebuild test -scheme iOSUnitTests -destination "id=$SIM_ID" cd ../functests -xcodebuild test -scheme iOSFuncTests -destination "platform=iOS Simulator,name=$SIMULATOR" +xcodebuild test -scheme iOSFuncTests -destination "id=$SIM_ID" diff --git a/build-tests.cmd b/build-tests.cmd index b73f18689..e6dc4bf6a 100644 --- a/build-tests.cmd +++ b/build-tests.cmd @@ -1,8 +1,37 @@ @echo off cd %~dp0 -call tools\gen-version.cmd @setlocal ENABLEEXTENSIONS +set CUSTOM_PROPS= +if not "%~3"=="" ( + if not exist "%~f3" ( + goto custom_props_missing + ) + if /I not "%~x3"==".props" ( + if /I not "%~x3"==".targets" ( + goto custom_props_invalid_type + ) + ) + set CUSTOM_PROPS="/p:ForceImportBeforeCppTargets=%~f3" + echo Using custom properties file for the build: + echo %CUSTOM_PROPS% +) + +goto after_custom_props_validation + +:custom_props_missing +echo ERROR: Custom build input not found: %~3 +echo Pass an existing MSBuild .props or .targets file to ForceImportBeforeCppTargets. +exit /b 1 + +:custom_props_invalid_type +echo ERROR: Custom build input must be an MSBuild .props or .targets file: %~3 +echo Pass the MSBuild import file, not the CONFIG_CUSTOM_H header. +exit /b 1 + +:after_custom_props_validation +call tools\gen-version.cmd + if DEFINED GIT_PULL_TOKEN ( rd /s /q lib\modules git clone https://%GIT_PULL_TOKEN%:x-oauth-basic@github.com/microsoft/cpp_client_telemetry_modules.git lib\modules @@ -20,13 +49,6 @@ set PLAT=%1 REM Possible configurations: Release|Debug set CONFIGURATION=%2 -set CUSTOM_PROPS= -if ("%3"=="") goto skip -set CUSTOM_PROPS="/p:ForceImportBeforeCppTargets=%3" -echo Using custom properties file for the build: -echo %CUSTOM_PROPS% -:skip - set MAXCPUCOUNT=%NUMBER_OF_PROCESSORS% set SOLUTION=Solutions\MSTelemetrySDK.sln diff --git a/build-win.ps1 b/build-win.ps1 index 8f4ef80da..af4004aba 100644 --- a/build-win.ps1 +++ b/build-win.ps1 @@ -15,7 +15,17 @@ $cpuCount = $env:NUMBER_OF_PROCESSORS $actualCustomProps = "" if ($customProps -ne "") { - $actualCustomProps = "/p:ForceImportBeforeCppTargets=$customProps" + if (-not (Test-Path -LiteralPath $customProps -PathType Leaf)) { + throw "Custom build input not found: $customProps`nPass an existing MSBuild .props or .targets file to ForceImportBeforeCppTargets." + } + + $customPropsExtension = [System.IO.Path]::GetExtension($customProps) + if ($customPropsExtension -ine ".props" -and $customPropsExtension -ine ".targets") { + throw "Custom build input must be an MSBuild .props or .targets file: $customProps`nPass the MSBuild import file, not the CONFIG_CUSTOM_H header." + } + + $resolvedCustomProps = (Resolve-Path -LiteralPath $customProps).Path + $actualCustomProps = "/p:ForceImportBeforeCppTargets=$resolvedCustomProps" } $coreTargets = @("zlib") @@ -133,7 +143,18 @@ foreach ($arch in $archs) { } # Build! - & cmd /c "msbuild $solution /target:$targetStr /p:BuildProjectReferences=true /maxcpucount:$cpuCount /p:Configuration=$actualConfig /p:Platform=$actualArch $actualCustomProps" + $msbuildArgs = @( + $solution + "/target:$targetStr" + "/p:BuildProjectReferences=true" + "/maxcpucount:$cpuCount" + "/p:Configuration=$actualConfig" + "/p:Platform=$actualArch" + ) + if ($actualCustomProps -ne "") { + $msbuildArgs += $actualCustomProps + } + & msbuild @msbuildArgs echo "...Done!" echo "" diff --git a/cgmanifest.json b/cgmanifest.json new file mode 100644 index 000000000..b24f9900d --- /dev/null +++ b/cgmanifest.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/nlohmann/json", + "CommitHash": "9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/sqlite/sqlite", + "CommitHash": "60405cd15cdd085745101a29112043299d439cfa" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/madler/zlib", + "CommitHash": "cacf7f1d4e3d44d871b605da3b647f07d718623f" + } + }, + "DevelopmentDependency": false + } + ] +} \ No newline at end of file diff --git a/docs/List-of-OSS-Components.md b/docs/List-of-OSS-Components.md index 43d8480a3..094ea1472 100644 --- a/docs/List-of-OSS-Components.md +++ b/docs/List-of-OSS-Components.md @@ -6,7 +6,7 @@ These are the Open Source components used by Microsoft 1DS / C++ Client Telemetr ZLIB DATA COMPRESSION LIBRARY. -SDK maintains its own snapshot of the mainline ZLib with some Intel architecture performance optimizations [here](../zlib). +SDK maintains its own snapshot of the mainline ZLib [here](../zlib). It is the responsibility of product teams to ensure that a snapshot of zlib they use meets their product security and licensing requirements. ## [SQLite](https://www.sqlite.org/index.html) diff --git a/docs/Offline-storage-settings.md b/docs/Offline-storage-settings.md index 78f5b2aff..137cb5c34 100644 --- a/docs/Offline-storage-settings.md +++ b/docs/Offline-storage-settings.md @@ -11,6 +11,15 @@ There are several configurations that can alter the offline storage handler beha | CFG_INT_STORAGE_FULL_CHECK_TIME | int | 5000 | Sets the minimum time (ms) between storage full notifications. | CFG_BOOL_ENABLE_DB_DROP_IF_FULL | bool | false | When set to true, trim events if cache size reaches CFG_INT_CACHE_FILE_SIZE | CFG_STR_CACHE_FILE_PATH | string | %TEMP% | Sets the path for the cache file +| skipSqliteInitAndShutdown | string | unset | When set to `"true"`, the SDK skips its process-wide `sqlite3_initialize()` and `sqlite3_shutdown()` calls for offline storage. + +## Shared SQLite lifecycle ownership + +By default, the SDK initializes SQLite when the first offline-storage-backed instance starts and shuts SQLite down after the last such instance is released. + +Set `skipSqliteInitAndShutdown` to `"true"` only when your application already owns the global SQLite lifecycle, for example when another subsystem initializes SQLite before the SDK starts and shuts it down after every SDK instance has been destroyed. + +When this option is enabled, the application is responsible for calling `sqlite3_initialize()` before creating a `LogManager` that uses offline storage and for delaying `sqlite3_shutdown()` until all SDK offline storage instances have been released. ## Deprecated configurations diff --git a/docs/building-custom-SKU.md b/docs/building-custom-SKU.md index 5252e5e14..0668fbe3f 100644 --- a/docs/building-custom-SKU.md +++ b/docs/building-custom-SKU.md @@ -44,6 +44,8 @@ build-all.bat %CD%\Solutions\build.compact.props produces a custom compact SDK build. +The argument passed to `build-all.bat` must be an MSBuild `.props` or `.targets` file that sets the required preprocessor definitions. Do not pass the `config-*.h` header directly to `ForceImportBeforeCppTargets`. + How it works: **build.compact.props** - contains the preprocessor definition that is functionally equivalent to diff --git a/docs/cpp-start-ios.md b/docs/cpp-start-ios.md index 35313e7fa..ad80866e0 100644 --- a/docs/cpp-start-ios.md +++ b/docs/cpp-start-ios.md @@ -16,12 +16,38 @@ Run `build-ios.sh [clean] [release|debug]` script in the root folder of the sour Run `build-ios.sh [clean] [release|debug] [arm64|arm64e]`. +### Run iOS tests + +Run `./build-tests-ios.sh [release|debug] ""`. + +The script requires `python3` on `PATH` to parse `simctl --json` when resolving the simulator name. + +Example: + +```sh +./build-tests-ios.sh release "iPhone 17" +``` + +Use a simulator name returned by `xcrun simctl list devices available`. +The script resolves an exact device name from `simctl --json`, prefers the +newest installed iOS runtime for that device name, and fails if that newest +runtime still contains multiple matches. + +If Xcode reports that the requested simulator runtime is missing, install it +from Xcode > Settings > Components or run +`xcodebuild -downloadPlatform iOS -architectureVariant arm64`. + ## 3. Integrate the SDK into your C++ project -SDK package contains headers and library installed at the following locations: +SDK package contains headers and library installed at the following locations +by default: * Headers: /usr/local/include/mat -* Library: /usr/local/lib/${arch}/libmat.a +* Library: /usr/local/lib/libmat.a + +If you set a custom install prefix via +`CMAKE_OPTS="-DCMAKE_INSTALL_PREFIX=/path/to/install"`, the SDK is installed +under `/include/mat` and `/lib/libmat.a`. 1DS SDK is built using cmake, but you can explore building it with any other build system of your choice. diff --git a/docs/public/conf.py b/docs/public/conf.py index d38fa3f7b..1d29ebe99 100644 --- a/docs/public/conf.py +++ b/docs/public/conf.py @@ -16,7 +16,7 @@ # -- Project information ----------------------------------------------------- -project = 'Microsoft C++ Client Telemetry SDK"' +project = 'Microsoft C++ Client Telemetry SDK' copyright = 'Microsoft Corporation' author = 'Microsoft Corporation' @@ -28,8 +28,6 @@ # This is necessary so the readthedocs build works. It doesn't invoke the # Makefile, but just runs sphinx on this conf.py. import os -import shutil -import subprocess if not os.path.exists('doxyoutput'): os.makedirs('doxyoutput') @@ -61,7 +59,7 @@ primary_domain = "cpp" -higlight_language = "cpp" +highlight_language = "cpp" # Add any paths that contain templates here, relative to this directory. @@ -78,10 +76,9 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -#html_theme = "furo" -html_theme = "sphinx_rtd_theme" +html_theme = "furo" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file +html_static_path = [] diff --git a/docs/public/requirements.txt b/docs/public/requirements.txt index 7419647ac..92d9b3915 100644 --- a/docs/public/requirements.txt +++ b/docs/public/requirements.txt @@ -1,3 +1,4 @@ +sphinx<10 breathe exhale -furo \ No newline at end of file +furo diff --git a/examples/cpp/SampleCpp/SampleCpp.vcxproj b/examples/cpp/SampleCpp/SampleCpp.vcxproj index 500d966f6..ffa49633a 100644 --- a/examples/cpp/SampleCpp/SampleCpp.vcxproj +++ b/examples/cpp/SampleCpp/SampleCpp.vcxproj @@ -1080,7 +1080,6 @@ - diff --git a/examples/cpp/SampleCpp/SampleCpp.vcxproj.filters b/examples/cpp/SampleCpp/SampleCpp.vcxproj.filters index 9a605a027..ae87cc1f8 100644 --- a/examples/cpp/SampleCpp/SampleCpp.vcxproj.filters +++ b/examples/cpp/SampleCpp/SampleCpp.vcxproj.filters @@ -15,9 +15,6 @@ - - Header Files - Source Files diff --git a/examples/cpp/SampleCppMini/SampleCppMini.vcxproj b/examples/cpp/SampleCppMini/SampleCppMini.vcxproj index 88bca8d6d..424394f7e 100644 --- a/examples/cpp/SampleCppMini/SampleCppMini.vcxproj +++ b/examples/cpp/SampleCppMini/SampleCppMini.vcxproj @@ -1542,7 +1542,6 @@ - diff --git a/examples/cpp/SampleCppMini/SampleCppMini.vcxproj.filters b/examples/cpp/SampleCppMini/SampleCppMini.vcxproj.filters index ad3de7523..2df19ab39 100644 --- a/examples/cpp/SampleCppMini/SampleCppMini.vcxproj.filters +++ b/examples/cpp/SampleCppMini/SampleCppMini.vcxproj.filters @@ -14,11 +14,6 @@ rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - Header Files - - Source Files diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index a39e65b98..8824427b4 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,7 +1,7 @@ # Honor visibility properties for all target types cmake_policy(SET CMP0063 NEW) -include_directories( . ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include/public ${CMAKE_CURRENT_SOURCE_DIR}/include/public ${CMAKE_CURRENT_SOURCE_DIR}/include/mat ${CMAKE_CURRENT_SOURCE_DIR}/pal ${CMAKE_CURRENT_SOURCE_DIR}/utils ${CMAKE_CURRENT_SOURCE_DIR}/modules/exp ${CMAKE_CURRENT_SOURCE_DIR}/modules/dataviewer ${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard ${CMAKE_CURRENT_SOURCE_DIR}/modules/liveeventinspector ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/Reachability ${CMAKE_CURRENT_SOURCE_DIR}/modules/cds ${CMAKE_CURRENT_SOURCE_DIR}/modules/signals ${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer /usr/local/include ) +include_directories( . ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include/public ${CMAKE_CURRENT_SOURCE_DIR}/include/mat ${CMAKE_CURRENT_SOURCE_DIR}/pal ${CMAKE_CURRENT_SOURCE_DIR}/utils ${CMAKE_CURRENT_SOURCE_DIR}/modules/exp ${CMAKE_CURRENT_SOURCE_DIR}/modules/dataviewer ${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard ${CMAKE_CURRENT_SOURCE_DIR}/modules/liveeventinspector ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/Reachability ${CMAKE_CURRENT_SOURCE_DIR}/modules/cds ${CMAKE_CURRENT_SOURCE_DIR}/modules/signals ${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer /usr/local/include ) set(SRCS decorators/BaseDecorator.cpp packager/BondSplicer.cpp @@ -60,7 +60,7 @@ if(BUILD_AZMON) include(modules/azmon/CMakeLists.txt OPTIONAL) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/exp/) +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/exp/") list(APPEND SRCS modules/exp/afd/afdclient/AFDClientUtils.cpp modules/exp/afd/afdclient/AFDClient.cpp @@ -74,14 +74,14 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/exp/) ) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/dataviewer/) +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/dataviewer/") list(APPEND SRCS modules/dataviewer/DefaultDataViewer.cpp modules/dataviewer/OnDisableNotificationCollection.cpp ) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard/ AND BUILD_PRIVACYGUARD) +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard/" AND BUILD_PRIVACYGUARD) list(APPEND SRCS modules/privacyguard/PrivacyGuard.cpp modules/privacyguard/RegisteredFileTypes.cpp @@ -89,14 +89,14 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard/ AND BUILD_PRIVACYGUA ) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/liveeventinspector/ AND BUILD_LIVEEVENTINSPECTOR) +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/liveeventinspector/" AND BUILD_LIVEEVENTINSPECTOR) list(APPEND SRCS modules/liveeventinspector/LiveEventInspector.cpp modules/liveeventinspector/LiveEventInspector.hpp ) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/cds/ AND BUILD_CDS) +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/cds/" AND BUILD_CDS) add_definitions(-DHAVE_MAT_CDS) list(APPEND SRCS modules/cds/CdsFactory.hpp @@ -104,14 +104,14 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/cds/ AND BUILD_CDS) ) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/signals/ AND BUILD_SIGNALS) +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/signals/" AND BUILD_SIGNALS) list(APPEND SRCS modules/signals/Signals.cpp modules/signals/SignalsEncoder.cpp ) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer/ AND BUILD_SANITIZER) +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer/" AND BUILD_SANITIZER) list(APPEND SRCS modules/sanitizer/detectors/EmailAddressDetector.cpp modules/sanitizer/detectors/JwtDetector.cpp @@ -124,6 +124,15 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer/ AND BUILD_SANITIZER) modules/sanitizer/SanitizerTrie.cpp modules/sanitizer/SanitizerTrieNode.cpp ) + # Suppress -Wreorder for Sanitizer.cpp only: the submodule declares + # m_sanitizerprovider before m_notificationEventName in Sanitizer.hpp + # but the constructor init list reverses that order. The proper fix + # belongs in the lib/modules submodule; this scopes the suppression + # until that is done. + if(NOT MSVC) + set_source_files_properties(modules/sanitizer/Sanitizer.cpp + PROPERTIES COMPILE_FLAGS "-Wno-reorder") + endif() endif() if(PAL_IMPLEMENTATION STREQUAL "CPP11") @@ -172,7 +181,7 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11") ) endif() if(APPLE AND BUILD_OBJC_WRAPPER) - message("Include ObjC Wrappers") + message(STATUS "Include ObjC Wrappers") list(APPEND SRCS ../wrappers/obj-c/ODWLogger.mm ../wrappers/obj-c/ODWLogManager.mm @@ -180,19 +189,19 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11") ../wrappers/obj-c/ODWLogConfiguration.mm ../wrappers/obj-c/ODWSemanticContext.mm ) - if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/dataviewer/) + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/dataviewer/") list(APPEND SRCS ../wrappers/obj-c/ODWDiagnosticDataViewer.mm ) endif() - if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard/ AND BUILD_PRIVACYGUARD) + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard/" AND BUILD_PRIVACYGUARD) list(APPEND SRCS ../wrappers/obj-c/ODWCommonDataContext.mm ../wrappers/obj-c/ODWPrivacyGuard.mm ../wrappers/obj-c/ODWPrivacyGuardInitConfig.mm ) endif() - if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer/ AND BUILD_SANITIZER) + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer/" AND BUILD_SANITIZER) list(APPEND SRCS ../wrappers/obj-c/ODWSanitizerInitConfig.mm ../wrappers/obj-c/ODWSanitizer.mm @@ -201,7 +210,7 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11") endif() if(APPLE AND BUILD_SWIFT_WRAPPER) - message("Building Swift Wrappers") + message(STATUS "Building Swift Wrappers") # Run swift build for the Swift Wrappers Package string(TOLOWER ${CMAKE_BUILD_TYPE} LOWER_BUILD_TYPE) execute_process( @@ -213,9 +222,9 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11") ) if(SWIFT_BUILD_RESULT EQUAL 0) - message("Swift Wrappers build succeeded!") + message(STATUS "Swift Wrappers build succeeded!") else() - message(FATAL_ERROR, "Swift build failed with error code: ${SWIFT_BUILD_RESULT}") + message(FATAL_ERROR "Swift build failed with error code: ${SWIFT_BUILD_RESULT}") endif() endif() @@ -238,7 +247,7 @@ remove_definitions(-D_MBCS) ) # UTC module - if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/utc) + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/utc") list(APPEND SRCS modules/utc/desktop/UtcHelpers.cpp modules/utc/UtcTelemetrySystem.cpp @@ -250,7 +259,7 @@ else() endif() # Filtering module -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/filter) +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/filter") list(APPEND SRCS modules/filter/CompliantByDefaultEventFilterModule.cpp modules/filter/CompliantByDefaultFilterApi.cpp @@ -269,7 +278,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") endif() if(BUILD_SHARED_LIBS STREQUAL "ON") - message("-- Building shared SDK library") + message(STATUS "Building shared SDK library") # include(FindCURL) # find_package(CURL REQUIRED) @@ -304,13 +313,19 @@ if(BUILD_SHARED_LIBS STREQUAL "ON") # target_link_libraries(mat PUBLIC libsqlite3 libcurl.a libz.a libssl.a libcrypto.a "${SQLITE_LIBRARY}" "${CMAKE_THREAD_LIBS_INIT}" "${CMAKE_DL_LIBS}" ) install(TARGETS mat EXPORT mat LIBRARY DESTINATION ${INSTALL_LIB_DIR}) else() - message("-- Building static SDK library") + message(STATUS "Building static SDK library") add_library(mat STATIC ${SRCS}) if(LINK_STATIC_DEPENDS) if(PAL_IMPLEMENTATION STREQUAL "WIN32") target_link_libraries(mat ${LIBS} "${CMAKE_THREAD_LIBS_INIT}" "${CMAKE_DL_LIBS}" ) else() add_library(sqlite3 STATIC IMPORTED GLOBAL) + find_library(SQLITE3_STATIC_LIB NAMES libsqlite3.a + PATHS /usr/local/lib /usr/local/opt/sqlite/lib /opt/homebrew/opt/sqlite/lib + NO_DEFAULT_PATH) + if(SQLITE3_STATIC_LIB) + set_target_properties(sqlite3 PROPERTIES IMPORTED_LOCATION ${SQLITE3_STATIC_LIB}) + endif() add_library(z STATIC IMPORTED GLOBAL) # # TODO: allow adding "${Tcmalloc_LIBRARIES}" to target_link_libraries for memory leak debugging @@ -321,10 +336,6 @@ else() install(TARGETS mat EXPORT mat ARCHIVE DESTINATION ${INSTALL_LIB_DIR}) endif() -message("-- Library will be installed to ${INSTALL_LIB_DIR}") +message(STATUS "Library will be installed to ${INSTALL_LIB_DIR}") -#if(PAL_IMPLEMENTATION STREQUAL "CPP11") -# #target_link_libraries(mat PUBLIC libcurl.a libz.a libssl.a libcrypto.a "${SQLITE_LIBRARY}" "${CMAKE_THREAD_LIBS_INIT}" "${CMAKE_DL_LIBS}" ) -# #target_link_libraries(mat PUBLIC libsqlite3.a libz.a ${LIBS} "${CMAKE_THREAD_LIBS_INIT}" "${CMAKE_DL_LIBS}" ) -#endif() diff --git a/lib/android_build/maesdk/src/main/cpp/CMakeLists.txt b/lib/android_build/maesdk/src/main/cpp/CMakeLists.txt index dbea0dc98..f53d5403a 100644 --- a/lib/android_build/maesdk/src/main/cpp/CMakeLists.txt +++ b/lib/android_build/maesdk/src/main/cpp/CMakeLists.txt @@ -45,6 +45,7 @@ set(SRCS ${SDK_ROOT}/lib/bond/BondSerializer.cpp ${SDK_ROOT}/lib/callbacks/DebugSource.cpp ${SDK_ROOT}/lib/compression/HttpDeflateCompression.cpp + ${SDK_ROOT}/lib/decoder/PayloadDecoder.cpp ${SDK_ROOT}/lib/decorators/BaseDecorator.cpp ${SDK_ROOT}/lib/filter/EventFilterCollection.cpp ${SDK_ROOT}/lib/http/HttpClientFactory.cpp diff --git a/lib/api/LogManagerImpl.cpp b/lib/api/LogManagerImpl.cpp index fac8fdbd5..2f0e8933d 100644 --- a/lib/api/LogManagerImpl.cpp +++ b/lib/api/LogManagerImpl.cpp @@ -289,13 +289,7 @@ namespace MAT_NS_BEGIN if (m_httpClient == nullptr) { m_httpClient = HttpClientFactory::Create(); -#ifdef HAVE_MAT_WININET_HTTP_CLIENT - HttpClient_WinInet* client = static_cast(m_httpClient.get()); - if (client != nullptr) - { - client->SetMsRootCheck(m_logConfiguration[CFG_MAP_HTTP][CFG_BOOL_HTTP_MS_ROOT_CHECK]); - } -#endif + m_httpClient->ApplySettings(m_logConfiguration); } else { @@ -366,14 +360,10 @@ namespace MAT_NS_BEGIN /// void LogManagerImpl::Configure() { - // TODO: [maxgolov] - add other config params. -#ifdef HAVE_MAT_WININET_HTTP_CLIENT - HttpClient_WinInet* client = static_cast(m_httpClient.get()); - if (client != nullptr) + if (m_httpClient != nullptr) { - client->SetMsRootCheck(m_logConfiguration[CFG_MAP_HTTP][CFG_BOOL_HTTP_MS_ROOT_CHECK]); + m_httpClient->ApplySettings(m_logConfiguration); } -#endif } LogManagerImpl::~LogManagerImpl() noexcept diff --git a/lib/config/RuntimeConfig_Default.hpp b/lib/config/RuntimeConfig_Default.hpp index 27a8f6758..504aeefe3 100644 --- a/lib/config/RuntimeConfig_Default.hpp +++ b/lib/config/RuntimeConfig_Default.hpp @@ -33,6 +33,9 @@ namespace MAT_NS_BEGIN {/* Parameter that allows to split stats events by tenant */ {"split", false}, {"interval", 1800}, + /* Stats are disabled by default for the built-in shared token + to reduce OneCollector load (see #1420). Set to true to opt in. */ + {"enabled", false}, {"tokenProd", STATS_TOKEN_PROD}, {"tokenInt", STATS_TOKEN_INT}}}, {"utc", @@ -57,7 +60,11 @@ namespace MAT_NS_BEGIN , {"contentEncoding", "deflate"}, /* Optional parameter to require Microsoft Root CA */ - {CFG_BOOL_HTTP_MS_ROOT_CHECK, false}}}, + {CFG_BOOL_HTTP_MS_ROOT_CHECK, false}, + /* Optional parameter for SSL certificate verification (curl) */ + {CFG_BOOL_HTTP_SSL_VERIFY, true}, + /* Optional CA bundle path for OpenSSL-backed curl */ + {CFG_STR_HTTP_SSL_CAINFO, ""}}}, {CFG_MAP_TPM, { {CFG_INT_TPM_MAX_BLOB_BYTES, 2097152}, diff --git a/lib/http/HttpClient_Curl.cpp b/lib/http/HttpClient_Curl.cpp index 18ddabce9..b910cdf28 100644 --- a/lib/http/HttpClient_Curl.cpp +++ b/lib/http/HttpClient_Curl.cpp @@ -14,6 +14,7 @@ #include "utils/Utils.hpp" #include "HttpClient_Curl.hpp" +#include "ILogConfiguration.hpp" namespace MAT_NS_BEGIN { @@ -74,7 +75,13 @@ namespace MAT_NS_BEGIN { requestHeaders[header.first] = header.second; } - auto curlOperation = std::make_shared(curlRequest->m_method, curlRequest->m_url, callback, requestHeaders, curlRequest->m_body); + std::string sslCaInfo; + { + std::lock_guard lock(m_requestsMtx); + sslCaInfo = m_sslCaInfo; + } + + auto curlOperation = std::make_shared(curlRequest->m_method, curlRequest->m_url, callback, requestHeaders, curlRequest->m_body, false, HTTP_CONN_TIMEOUT, m_sslVerify, sslCaInfo); curlRequest->SetOperation(curlOperation); // The lifetime of curlOperation is guarnteed by the call to result.wait() in the d'tor. @@ -125,6 +132,20 @@ namespace MAT_NS_BEGIN { } } + void HttpClient_Curl::ApplySettings(ILogConfiguration& config) + { + SetSslVerification( + config[CFG_MAP_HTTP][CFG_BOOL_HTTP_SSL_VERIFY], + (const char *)config[CFG_MAP_HTTP][CFG_STR_HTTP_SSL_CAINFO]); + } + + void HttpClient_Curl::SetSslVerification(bool sslVerify, const std::string& caInfo) + { + m_sslVerify = sslVerify; + std::lock_guard lock(m_requestsMtx); + m_sslCaInfo = caInfo; + } + void HttpClient_Curl::EraseRequest(std::string const& id) { std::lock_guard lock(m_requestsMtx); diff --git a/lib/http/HttpClient_Curl.hpp b/lib/http/HttpClient_Curl.hpp index cb89ec7e6..c7a5bdecb 100644 --- a/lib/http/HttpClient_Curl.hpp +++ b/lib/http/HttpClient_Curl.hpp @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -55,12 +56,17 @@ class HttpClient_Curl : public IHttpClient { virtual void SendRequestAsync(IHttpRequest* request, IHttpResponseCallback* callback) override; virtual void CancelRequestAsync(std::string const& id) override; + virtual void ApplySettings(ILogConfiguration& config) override; + void SetSslVerification(bool sslVerify, const std::string& caInfo = ""); + private: void EraseRequest(std::string const& id); void AddRequest(IHttpRequest* request); std::mutex m_requestsMtx; std::map m_requests; + std::atomic m_sslVerify { true }; + std::string m_sslCaInfo; }; class CurlHttpOperation { @@ -91,7 +97,10 @@ class CurlHttpOperation { const std::vector& requestBody = std::vector(), // Default connectivity and response size options bool rawResponse = false, - size_t httpConnTimeout = HTTP_CONN_TIMEOUT) : + size_t httpConnTimeout = HTTP_CONN_TIMEOUT, + // SSL certificate verification options + bool sslVerify = true, + const std::string& sslCaInfo = "") : // Optional connection params rawResponse(rawResponse), @@ -100,6 +109,7 @@ class CurlHttpOperation { m_callback(callback), m_method(method), m_url(url), + m_sslCaInfo(sslCaInfo), // Local vars requestHeaders(requestHeaders), @@ -129,9 +139,11 @@ class CurlHttpOperation { // Specify target URL curl_easy_setopt(curl, CURLOPT_URL, m_url.c_str()); - // TODO: expose SSL cert verification opts via ILogConfiguration - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); // 1L - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); // 2L + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, sslVerify ? 1L : 0L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, sslVerify ? 2L : 0L); + if (!m_sslCaInfo.empty()) { + curl_easy_setopt(curl, CURLOPT_CAINFO, m_sslCaInfo.c_str()); + } // HTTP/2 please, fallback to HTTP/1.1 if not supported curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); @@ -423,6 +435,7 @@ class CurlHttpOperation { // Request values std::string m_method; std::string m_url; + std::string m_sslCaInfo; const std::map& requestHeaders; const std::vector& requestBody; struct curl_slist *m_headersChunk = nullptr; @@ -452,28 +465,12 @@ class CurlHttpOperation { */ static int WaitOnSocket(curl_socket_t sockfd, int for_recv, long timeout_ms) { - struct timeval tv; - fd_set infd, outfd, errfd; - int res; - - tv.tv_sec = timeout_ms / 1000; - tv.tv_usec = (timeout_ms % 1000) * 1000; - - FD_ZERO(&infd); - FD_ZERO(&outfd); - FD_ZERO(&errfd); - - FD_SET(sockfd, &errfd); /* always check for error */ - - if(for_recv) { - FD_SET(sockfd, &infd); - } else { - FD_SET(sockfd, &outfd); - } - - /* select() returns the number of signalled sockets or -1 */ - res = select((int)sockfd + 1, &infd, &outfd, &errfd, &tv); - return res; + struct pollfd pfd; + pfd.fd = sockfd; + pfd.events = for_recv ? POLLIN : POLLOUT; + // Cap timeout to max int value to avoid overflow in poll() + auto timeout = std::min(timeout_ms, static_cast(std::numeric_limits::max())); + return poll(&pfd, 1, static_cast(timeout)); } // Raw response buffer @@ -539,4 +536,3 @@ class CurlHttpOperation { #endif // HAVE_MAT_DEFAULT_HTTP_CLIENT #endif // HTTPCLIENTCURL_HPP - diff --git a/lib/http/HttpClient_WinInet.cpp b/lib/http/HttpClient_WinInet.cpp index 637b10778..eaefb2318 100644 --- a/lib/http/HttpClient_WinInet.cpp +++ b/lib/http/HttpClient_WinInet.cpp @@ -546,6 +546,11 @@ void HttpClient_WinInet::CancelAllRequests() /// Enforces MS-root server certificate check. /// /// if set to true [enforce verification that server cert is MS-Rooted]. +void HttpClient_WinInet::ApplySettings(ILogConfiguration& config) +{ + SetMsRootCheck(config[CFG_MAP_HTTP][CFG_BOOL_HTTP_MS_ROOT_CHECK]); +} + void HttpClient_WinInet::SetMsRootCheck(bool enforceMsRoot) { m_msRootCheck = enforceMsRoot; diff --git a/lib/http/HttpClient_WinInet.hpp b/lib/http/HttpClient_WinInet.hpp index e5936fcc3..7e9379ded 100644 --- a/lib/http/HttpClient_WinInet.hpp +++ b/lib/http/HttpClient_WinInet.hpp @@ -30,6 +30,8 @@ class HttpClient_WinInet : public IHttpClient { virtual void CancelRequestAsync(std::string const& id) final; virtual void CancelAllRequests() final; + virtual void ApplySettings(ILogConfiguration& config) override; + // Methods unique to WinInet implementation. void SetMsRootCheck(bool enforceMsRoot); bool IsMsRootCheckRequired(); diff --git a/lib/include/public/IHttpClient.hpp b/lib/include/public/IHttpClient.hpp index e9a71a210..89e5e6cf0 100644 --- a/lib/include/public/IHttpClient.hpp +++ b/lib/include/public/IHttpClient.hpp @@ -18,6 +18,7 @@ ///@cond INTERNAL_DOCS namespace MAT_NS_BEGIN { + class ILogConfiguration; /// /// The HttpHeaders class contains a set of HTTP headers. /// @@ -543,6 +544,14 @@ namespace MAT_NS_BEGIN virtual void CancelRequestAsync(std::string const& id) = 0; virtual void CancelAllRequests() {} + + /// + /// Apply HTTP settings from the log configuration. + /// Subclasses override to handle platform-specific options. + /// Default implementation is a no-op. + /// + /// The log configuration to read settings from. + virtual void ApplySettings(ILogConfiguration& /*config*/) {} }; /// @endcond diff --git a/lib/include/public/ILogConfiguration.hpp b/lib/include/public/ILogConfiguration.hpp index 952bc2651..1cb8103b8 100644 --- a/lib/include/public/ILogConfiguration.hpp +++ b/lib/include/public/ILogConfiguration.hpp @@ -361,6 +361,16 @@ namespace MAT_NS_BEGIN /// static constexpr const char* const CFG_BOOL_HTTP_COMPRESSION = "compress"; + /// + /// HTTP configuration: SSL certificate verification (peer + host) + /// + static constexpr const char* const CFG_BOOL_HTTP_SSL_VERIFY = "sslVerify"; + + /// + /// HTTP configuration: SSL CA bundle file path (for libcurl/OpenSSL) + /// + static constexpr const char* const CFG_STR_HTTP_SSL_CAINFO = "sslCaInfo"; + /// /// TPM configuration map /// diff --git a/lib/include/public/Version.hpp b/lib/include/public/Version.hpp index ce9e8fd79..850519486 100644 --- a/lib/include/public/Version.hpp +++ b/lib/include/public/Version.hpp @@ -6,8 +6,8 @@ #define MAT_VERSION_HPP // WARNING: DO NOT MODIFY THIS FILE! // This file has been automatically generated, manual changes will be lost. -#define BUILD_VERSION_STR "3.10.40.1" -#define BUILD_VERSION 3,10,40,1 +#define BUILD_VERSION_STR "3.10.100.1" +#define BUILD_VERSION 3,10,100,1 #ifndef RESOURCE_COMPILER_INVOKED #include "ctmacros.hpp" @@ -18,7 +18,7 @@ namespace MAT_NS_BEGIN { uint64_t const Version = ((uint64_t)3 << 48) | ((uint64_t)10 << 32) | - ((uint64_t)40 << 16) | + ((uint64_t)100 << 16) | ((uint64_t)1); } MAT_NS_END diff --git a/lib/offline/OfflineStorage_Room.cpp b/lib/offline/OfflineStorage_Room.cpp index ab7d43264..423aecde3 100644 --- a/lib/offline/OfflineStorage_Room.cpp +++ b/lib/offline/OfflineStorage_Room.cpp @@ -11,6 +11,19 @@ namespace { static constexpr bool s_throwExceptions = true; + + // RAII guard that deletes a JNI global class reference on all exit paths, + // including std::logic_error (ThrowLogic) and std::runtime_error (ThrowRuntime). + struct GlobalRefGuard { + JNIEnv* jni; + jclass* ref_ptr; + ~GlobalRefGuard() noexcept { + if (ref_ptr && *ref_ptr) { + jni->DeleteGlobalRef(*ref_ptr); + *ref_ptr = nullptr; + } + } + }; } namespace MAT_NS_BEGIN @@ -387,17 +400,23 @@ namespace MAT_NS_BEGIN { break; // out of r > c loop; no more records } - // we don't collect these here because GetObjectClass is - // less fragile than FindClass - jclass record_class = nullptr; - jfieldID id_id; - jfieldID tenantToken_id; - jfieldID latency_id; - jfieldID persistence_id; - jfieldID timestamp_id; - jfieldID retryCount_id; - jfieldID reservedUntil_id; - jfieldID blob_id; + // Field IDs are looked up once from the first record's class and reused. + // record_class is stored as a global reference so it remains valid across + // pushLocalFrame/popLocalFrame boundaries (local refs are freed on popLocalFrame, + // causing a JNI abort on ART if reused in subsequent iterations). + jclass record_class = nullptr; + jfieldID id_id = nullptr; + jfieldID tenantToken_id = nullptr; + jfieldID latency_id = nullptr; + jfieldID persistence_id = nullptr; + jfieldID timestamp_id = nullptr; + jfieldID retryCount_id = nullptr; + jfieldID reservedUntil_id = nullptr; + jfieldID blob_id = nullptr; + // RAII guard: deletes record_class global ref on all exit paths, + // including std::logic_error (ThrowLogic) and std::runtime_error + // (ThrowRuntime) which the catch block below would not otherwise clean up. + GlobalRefGuard record_class_guard{env.getInner(), &record_class}; // Set limits for conversion from int to enum int latency_lb = static_cast(EventLatency_Off); @@ -412,7 +431,14 @@ namespace MAT_NS_BEGIN ThrowLogic(env, "getAndReserve element"); if (!record_class) { - record_class = env->GetObjectClass(record); + // Promote to a global ref so it survives popLocalFrame on + // subsequent iterations. Freed by record_class_guard on exit. + jclass local_class = env->GetObjectClass(record); + record_class = static_cast(env->NewGlobalRef(local_class)); + if (!record_class) + { + MATSDK_THROW(std::runtime_error("NewGlobalRef failed")); + } id_id = env->GetFieldID(record_class, "id", "J"); ThrowLogic(env, "gar id"); tenantToken_id = env->GetFieldID(record_class, "tenantToken", @@ -663,9 +689,12 @@ namespace MAT_NS_BEGIN if (tokens > 0) { DroppedMap dropped; - jclass bt_class = nullptr; - jfieldID token_id; - jfieldID count_id; + // bt_class stored as a global ref to survive popLocalFrame across iterations. + jclass bt_class = nullptr; + jfieldID token_id = nullptr; + jfieldID count_id = nullptr; + // RAII guard: frees bt_class on all exit paths including exceptions. + GlobalRefGuard bt_class_guard{env.getInner(), &bt_class}; for (size_t index = 0; index < tokens; ++index) { env.pushLocalFrame(8); @@ -673,7 +702,14 @@ namespace MAT_NS_BEGIN ThrowRuntime(env, "Exception fetching element from results"); if (!bt_class) { - bt_class = env->GetObjectClass(byTenant); + // Promote to a global ref so it survives popLocalFrame. + // Freed by bt_class_guard on exit. + jclass local_class = env->GetObjectClass(byTenant); + bt_class = static_cast(env->NewGlobalRef(local_class)); + if (!bt_class) + { + MATSDK_THROW(std::runtime_error("NewGlobalRef failed")); + } token_id = env->GetFieldID(bt_class, "tenantToken", "Ljava/lang/String;"); ThrowLogic(env, "Error fetching tenantToken field id"); @@ -1160,15 +1196,18 @@ namespace MAT_NS_BEGIN "(ZIJ)[Lcom/microsoft/applications/events/StorageRecord;"); ThrowLogic(env, "getRecords method"); - jclass record_class = nullptr; - jfieldID id_id = nullptr; - jfieldID tenantToken_id; - jfieldID latency_id; - jfieldID persistence_id; - jfieldID timestamp_id; - jfieldID retryCount_id; - jfieldID reservedUntil_id; - jfieldID blob_id; + // record_class stored as a global ref to survive popLocalFrame across iterations. + jclass record_class = nullptr; + jfieldID id_id = nullptr; + jfieldID tenantToken_id = nullptr; + jfieldID latency_id = nullptr; + jfieldID persistence_id = nullptr; + jfieldID timestamp_id = nullptr; + jfieldID retryCount_id = nullptr; + jfieldID reservedUntil_id = nullptr; + jfieldID blob_id = nullptr; + // RAII guard: frees record_class on all exit paths including exceptions. + GlobalRefGuard record_class_guard{env.getInner(), &record_class}; auto java_records = static_cast(env->CallObjectMethod(m_room, method, @@ -1185,7 +1224,14 @@ namespace MAT_NS_BEGIN ThrowLogic(env, "access result element"); if (!record_class) { - record_class = env->GetObjectClass(record); + // Promote to a global ref so it survives popLocalFrame. + // Freed by record_class_guard on exit. + jclass local_class = env->GetObjectClass(record); + record_class = static_cast(env->NewGlobalRef(local_class)); + if (!record_class) + { + MATSDK_THROW(std::runtime_error("NewGlobalRef failed")); + } id_id = env->GetFieldID(record_class, "id", "J"); ThrowLogic(env, "id field"); tenantToken_id = env->GetFieldID(record_class, "tenantToken", diff --git a/lib/shared/Shared.vcxitems b/lib/shared/Shared.vcxitems index bf3d5df64..4b7c095ff 100644 --- a/lib/shared/Shared.vcxitems +++ b/lib/shared/Shared.vcxitems @@ -24,7 +24,6 @@ - diff --git a/lib/shared/Shared.vcxitems.filters b/lib/shared/Shared.vcxitems.filters index c488b869b..1868316bf 100644 --- a/lib/shared/Shared.vcxitems.filters +++ b/lib/shared/Shared.vcxitems.filters @@ -1,7 +1,6 @@  - diff --git a/lib/stats/Statistics.cpp b/lib/stats/Statistics.cpp index 2f421714c..4abf09563 100644 --- a/lib/stats/Statistics.cpp +++ b/lib/stats/Statistics.cpp @@ -7,6 +7,7 @@ #include "Statistics.hpp" #include "ILogManager.hpp" +#include "mat/config.h" #include "utils/Utils.hpp" #include @@ -60,12 +61,21 @@ namespace MAT_NS_BEGIN { return; } + std::string tenantToken = m_config.GetMetaStatsTenantToken(); + // Stats are disabled by default for the built-in shared token to + // reduce OneCollector load (see #1420). Custom tokens always send. + // Set config["metaStats"]["enabled"] = true to opt in. + bool isDefaultToken = (tenantToken == STATS_TOKEN_PROD || tenantToken == STATS_TOKEN_INT); + if (isDefaultToken && !static_cast(m_config[CFG_MAP_METASTATS_CONFIG]["enabled"])) + { + return; + } + std::vector< ::CsProtocol::Record> records; { LOCKGUARD(m_metaStats_mtx); records = m_metaStats.generateStatsEvent(rollupKind); } - std::string tenantToken = m_config.GetMetaStatsTenantToken(); for (auto& record : records) { diff --git a/tests/functests/AISendTests.cpp b/tests/functests/AISendTests.cpp index 180e9b074..dfd0bf185 100644 --- a/tests/functests/AISendTests.cpp +++ b/tests/functests/AISendTests.cpp @@ -118,7 +118,7 @@ class AISendTests : public ::testing::Test, } int port = server.addListeningPort(HTTP_PORT); std::ostringstream os; - os << "localhost:" << port; + os << "127.0.0.1:" << port; serverAddress = "http://" + os.str() + "/v2/track"; server.setServerName(os.str()); server.addHandler("/v2/track", *this); @@ -142,6 +142,9 @@ class AISendTests : public ::testing::Test, fileName += PATH_SEPARATOR_CHAR; fileName += TEST_STORAGE_FILENAME; std::remove(fileName.c_str()); + std::remove((fileName + "-wal").c_str()); + std::remove((fileName + "-shm").c_str()); + std::remove((fileName + "-journal").c_str()); } virtual void Initialize(DebugEventListener& debugListener, std::string const& path, bool compression) diff --git a/tests/functests/APITest.cpp b/tests/functests/APITest.cpp index 11524cd5a..0347807f6 100644 --- a/tests/functests/APITest.cpp +++ b/tests/functests/APITest.cpp @@ -302,7 +302,11 @@ static std::string GetStoragePath() static void CleanStorage() { - std::remove(GetStoragePath().c_str()); + std::string path = GetStoragePath(); + std::remove(path.c_str()); + std::remove((path + "-wal").c_str()); + std::remove((path + "-shm").c_str()); + std::remove((path + "-journal").c_str()); } #if 0 @@ -391,6 +395,10 @@ TEST(APITest, LogManager_Initialize_DebugEventListener) LogManager::GetLogger()->LogEvent(eventToLog); } LogManager::Flush(); + // Storage-full callback fires asynchronously; give it time to arrive + for (int i = 0; i < 50 && debugListener.storageFullPct.load() < 100; i++) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } EXPECT_GE(debugListener.storageFullPct.load(), (unsigned)100); LogManager::FlushAndTeardown(); @@ -404,8 +412,20 @@ TEST(APITest, LogManager_Initialize_DebugEventListener) debugListener.numSent = 0; debugListener.numLogged = 0; - CleanStorage(); + // Use a unique DB path for Phase 2/3 on every invocation. The SDK + // closes SQLite with sqlite3_close_v2(), which defers file-descriptor + // cleanup when prepared statements linger. If we reuse a fixed path + // and std::remove() the old files while deferred fds are still open, + // iOS emits "vnode unlinked while in use" and may invalidate the new + // DB's descriptors. A fresh, never-before-seen path sidesteps the + // problem entirely — no stale files, no collisions, no sleep needed. + static std::atomic s_phase2Counter{0}; + std::string phase2Path = GetStoragePath() + ".phase2." + + std::to_string(s_phase2Counter.fetch_add(1)); + configuration[CFG_STR_CACHE_FILE_PATH] = phase2Path; + configuration[CFG_INT_CACHE_FILE_SIZE] = 0; // No size limit for phase 2 ILogger *result = LogManager::Initialize(TEST_TOKEN, configuration); + LogManager::PauseTransmission(); // Pause before logging to avoid production uploads // Log some foo size_t numIterations = MAX_ITERATIONS; @@ -416,10 +436,6 @@ TEST(APITest, LogManager_Initialize_DebugEventListener) EXPECT_EQ(0u, debugListener.numDropped); EXPECT_EQ(0u, debugListener.numReject); - LogManager::UploadNow(); // Try to upload whatever we got - PAL::sleep(10000); // Give enough time to upload at least one event - EXPECT_NE(0u, debugListener.numSent); // Some posts must succeed within 500ms - LogManager::PauseTransmission(); // There could still be some pending at this point LogManager::Flush(); // Save all pending to disk numIterations = MAX_ITERATIONS; @@ -434,15 +450,27 @@ TEST(APITest, LogManager_Initialize_DebugEventListener) LogManager::Flush(); EXPECT_EQ(MAX_ITERATIONS, debugListener.numCached); + // Phase 3: resume transmission and upload the cached events LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::ResumeTransmission(); + LogManager::UploadNow(); + PAL::sleep(10000); // Give enough time to upload LogManager::FlushAndTeardown(); // Check that we sent all of logged + whatever left overs // prior to PauseTransmission EXPECT_GE(debugListener.numSent, debugListener.numLogged); + debugListener.printStats(); removeAllListeners(debugListener); + + // Best-effort cleanup. Assertions have already passed, so if + // sqlite3_close_v2 deferred cleanup triggers a vnode warning here + // it is harmless. + std::remove(phase2Path.c_str()); + std::remove((phase2Path + "-wal").c_str()); + std::remove((phase2Path + "-shm").c_str()); + std::remove((phase2Path + "-journal").c_str()); } #ifdef _WIN32 @@ -1180,10 +1208,12 @@ TEST(APITest, LogManager_BadNetwork_Test) // Clean temp file first const char *cacheFilePath = "bad-network.db"; std::string fileName = MAT::GetTempDirectory(); - fileName += "\\"; fileName += cacheFilePath; printf("remove %s\n", fileName.c_str()); std::remove(fileName.c_str()); + std::remove((fileName + "-wal").c_str()); + std::remove((fileName + "-shm").c_str()); + std::remove((fileName + "-journal").c_str()); for (auto url : { #if 0 /* [MG}: Temporary change to avoid GitHub Actions crash #92 */ diff --git a/tests/functests/BasicFuncTests.cpp b/tests/functests/BasicFuncTests.cpp index 51848d054..438411425 100644 --- a/tests/functests/BasicFuncTests.cpp +++ b/tests/functests/BasicFuncTests.cpp @@ -154,7 +154,7 @@ class BasicFuncTests : public ::testing::Test, } int port = server.addListeningPort(HTTP_PORT); std::ostringstream os; - os << "localhost:" << port; + os << "127.0.0.1:" << port; serverAddress = "http://" + os.str() + "/simple/"; server.setServerName(os.str()); server.addHandler("/simple/", *this); @@ -179,6 +179,11 @@ class BasicFuncTests : public ::testing::Test, fileName += PATH_SEPARATOR_CHAR; fileName += TEST_STORAGE_FILENAME; std::remove(fileName.c_str()); + // SQLite WAL mode creates companion journal files that must also + // be removed to avoid "vnode unlinked while in use" on iOS. + std::remove((fileName + "-wal").c_str()); + std::remove((fileName + "-shm").c_str()); + std::remove((fileName + "-journal").c_str()); } virtual void Initialize() @@ -196,16 +201,22 @@ class BasicFuncTests : public ::testing::Test, configuration[CFG_INT_RAM_QUEUE_SIZE] = 4096 * 20; configuration[CFG_STR_CACHE_FILE_PATH] = TEST_STORAGE_FILENAME; + configuration[CFG_INT_CACHE_FILE_SIZE] = 4096 * 1024; // 4MB default configuration[CFG_INT_MAX_TEARDOWN_TIME] = 2; // 2 seconds wait on shutdown + configuration[CFG_INT_STORAGE_FULL_PCT] = 75; // default + configuration[CFG_INT_STORAGE_FULL_CHECK_TIME] = 5000; // default 5s configuration[CFG_STR_COLLECTOR_URL] = serverAddress.c_str(); configuration[CFG_MAP_HTTP][CFG_BOOL_HTTP_COMPRESSION] = false; // disable compression for now + configuration[CFG_MAP_TPM][CFG_STR_TPM_BACKOFF] = "E,500,5000,2,1"; // faster retry for localhost tests configuration[CFG_MAP_METASTATS_CONFIG][CFG_INT_METASTATS_INTERVAL] = 30 * 60; // 30 mins + configuration[CFG_MAP_METASTATS_CONFIG]["enabled"] = true; // opt in to stats (disabled by default since #1420) configuration["name"] = __FILE__; configuration["version"] = "1.0.0"; configuration["config"] = { { "host", __FILE__ } }; // Host instance LogManager::Initialize(TEST_TOKEN, configuration); + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::SetLevelFilter(DIAG_LEVEL_DEFAULT, { DIAG_LEVEL_DEFAULT_MIN, DIAG_LEVEL_DEFAULT_MAX }); LogManager::ResumeTransmission(); @@ -257,15 +268,16 @@ class BasicFuncTests : public ::testing::Test, size_t lastIdx = 0; while ( ((PAL::getUtcSystemTimeMs()-start)<(1000* timeOutSec)) && (receivedEvents!=expected_count) ) { - /* Give time for our friendly HTTP server thread to process incoming request */ - std::this_thread::yield(); + /* Give time for HTTP server thread to process incoming request. + * sleep(10) instead of yield() reduces CPU contention on single-core + * iOS simulator runners and gives the network stack time to deliver. */ + PAL::sleep(10); { LOCKGUARD(mtx_requests); if (receivedRequests.size()) { size_t size = receivedRequests.size(); - //requests can come within 100 milisec sleep for (size_t index = lastIdx; index < size; index++) { auto request = receivedRequests.at(index); @@ -573,6 +585,7 @@ TEST_F(BasicFuncTests, sendNoPriorityEvents) event2.SetProperty("property2", "another value"); logger->LogEvent(event2); + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::UploadNow(); waitForEvents(1, 3); EXPECT_GE(receivedRequests.size(), (size_t)1); @@ -671,6 +684,7 @@ TEST_F(BasicFuncTests, sendDifferentPriorityEvents) logger->LogEvent(event2); + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::UploadNow(); // 2 x customer events + 1 x evt_stats on start waitForEvents(1, 3); @@ -718,6 +732,7 @@ TEST_F(BasicFuncTests, sendMultipleTenantsTogether) logger2->LogEvent(event2); + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::UploadNow(); // 2 x customer events + 1 x evt_stats on start @@ -748,6 +763,7 @@ TEST_F(BasicFuncTests, configDecorations) EventProperties event4("4th_event"); logger->LogEvent(event4); + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::UploadNow(); waitForEvents(2, 5); @@ -785,10 +801,11 @@ TEST_F(BasicFuncTests, restartRecoversEventsFromStorage) fooEvent.SetLatency(EventLatency_RealTime); fooEvent.SetPersistence(EventPersistence_Critical); LogManager::GetLogger()->LogEvent(fooEvent); + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::UploadNow(); // 1st request for realtime event - waitForEvents(3, 5); // start, first_event, second_event, ongoing, stop, start, fooEvent + waitForEvents(10, 5); // start, first_event, second_event, ongoing, stop, start, fooEvent // we drop two of the events during pause, though. EXPECT_GE(receivedRequests.size(), (size_t)1); if (receivedRequests.size() != 0) @@ -852,7 +869,7 @@ TEST_F(BasicFuncTests, storageFileSizeDoesntExceedConfiguredSize) { Initialize(); - waitForEvents(2, 8); + waitForEvents(5, 8); if (receivedRequests.size()) { auto payload = decodeRequest(receivedRequests[0], false); @@ -897,8 +914,9 @@ TEST_F(BasicFuncTests, sendMetaStatsOnStart) // Check Initialize(); LogManager::ResumeTransmission(); // ? + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::UploadNow(); - PAL::sleep(2000); + waitForEvents(5, 4); // (start + stop) + (2 events + start) auto r2 = records(); ASSERT_GE(r2.size(), (size_t)4); // (start + stop) + (2 events + start) @@ -927,8 +945,9 @@ TEST_F(BasicFuncTests, DiagLevelRequiredOnly_OneEventWithoutLevelOneWithButNotAl eventWithAllowedLevel.SetLevel(DIAG_LEVEL_REQUIRED); logger->LogEvent(eventWithAllowedLevel); + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::UploadNow(); - waitForEvents(1 /*timeout*/, 2 /*expected count*/); // Start and EventWithAllowedLevel + waitForEvents(5 /*timeout*/, 2 /*expected count*/); // Start and EventWithAllowedLevel ASSERT_EQ(records().size(), static_cast(2)); // Start and EventWithAllowedLevel @@ -970,8 +989,9 @@ TEST_F(BasicFuncTests, DiagLevelRequiredOnly_SendTwoEventsUpdateAllowedLevelsSen LogManager::SetLevelFilter(DIAG_LEVEL_OPTIONAL, { DIAG_LEVEL_OPTIONAL, DIAG_LEVEL_REQUIRED }); SendEventWithOptionalThenRequired(logger); + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::UploadNow(); - waitForEvents(2 /*timeout*/, 4 /*expected count*/); // Start and EventWithAllowedLevel + waitForEvents(5 /*timeout*/, 4 /*expected count*/); // Start and EventWithAllowedLevel auto sentRecords = records(); ASSERT_EQ(sentRecords.size(), static_cast(4)); // Start and EventWithAllowedLevel @@ -1140,6 +1160,7 @@ TEST_F(BasicFuncTests, killSwitchWorks) configuration[CFG_STR_COLLECTOR_URL] = serverAddress.c_str(); configuration[CFG_MAP_HTTP][CFG_BOOL_HTTP_COMPRESSION] = false; // disable compression for now configuration[CFG_MAP_METASTATS_CONFIG]["interval"] = 30 * 60; // 30 mins + configuration[CFG_MAP_METASTATS_CONFIG]["enabled"] = true; // opt in to stats (disabled by default since #1420) configuration["name"] = __FILE__; configuration["version"] = "1.0.0"; @@ -1173,7 +1194,8 @@ TEST_F(BasicFuncTests, killSwitchWorks) myLogger->LogEvent(event2); } } - // Try to upload and wait for 2 seconds to complete + // Try to upload and wait for completion + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::UploadNow(); PAL::sleep(2000); @@ -1222,6 +1244,7 @@ TEST_F(BasicFuncTests, killIsTemporary) configuration[CFG_STR_COLLECTOR_URL] = serverAddress.c_str(); configuration[CFG_MAP_HTTP][CFG_BOOL_HTTP_COMPRESSION] = false; // disable compression for now configuration[CFG_MAP_METASTATS_CONFIG]["interval"] = 30 * 60; // 30 mins + configuration[CFG_MAP_METASTATS_CONFIG]["enabled"] = true; // opt in to stats (disabled by default since #1420) configuration["name"] = __FILE__; configuration["version"] = "1.0.0"; diff --git a/tests/functests/CMakeLists.txt b/tests/functests/CMakeLists.txt index 656f8f866..0c2074e99 100644 --- a/tests/functests/CMakeLists.txt +++ b/tests/functests/CMakeLists.txt @@ -1,4 +1,4 @@ -message("--- functests") +message(STATUS "Building functests") set(SRCS APITest.cpp @@ -8,47 +8,40 @@ set(SRCS MultipleLogManagersTests.cpp ) -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard/ AND BUILD_PRIVACYGUARD) +if(EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/privacyguard/" AND BUILD_PRIVACYGUARD) add_definitions(-DHAVE_MAT_PRIVACYGUARD) list(APPEND SRCS - PrivacyGuardFuncTests.cpp + ${CMAKE_SOURCE_DIR}/lib/modules/privacyguard/tests/functests/PrivacyGuardFuncTests.cpp ) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer/ AND BUILD_SANITIZER) +if(EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/sanitizer/" AND BUILD_SANITIZER) list(APPEND SRCS - SanitizerFuncTests.cpp + ${CMAKE_SOURCE_DIR}/lib/modules/sanitizer/tests/functests/SanitizerFuncTests.cpp ) endif() -if(EXISTS ${CMAKE_SOURCE_DIR}/lib/modules/dataviewer/) +if(EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/dataviewer/") list(APPEND SRCS ${CMAKE_SOURCE_DIR}/lib/modules/dataviewer/tests/functests/DefaultDataViewerFuncTests.cpp ) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/liveeventinspector/ AND BUILD_LIVEEVENTINSPECTOR) +if(EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/liveeventinspector/" AND BUILD_LIVEEVENTINSPECTOR) add_definitions(-DHAVE_MAT_LIVEEVENTINSPECTOR) list(APPEND SRCS - LiveEventInspectorFuncTests.cpp + ${CMAKE_SOURCE_DIR}/lib/modules/liveeventinspector/tests/functests/LiveEventInspectorFuncTests.cpp ) endif() -if (EXISTS ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests) +if (EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/exp/tests") list(APPEND SRCS ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/ECSClientFuncTests.cpp ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/ECSClientRealworldFuncTests.cpp ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/ECSConfigCacheFuncTests.cpp ) - if (EXISTS ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/test.json) - if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.21") - # Use file(COPY_FILE ...) for CMake 3.21 and later - file(COPY_FILE ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/test.json ${CMAKE_BINARY_DIR}/test.json) - else() - # Use file(COPY ...) as an alternative for older versions - file(COPY ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/test.json - DESTINATION ${CMAKE_BINARY_DIR}) - endif() + if (EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/test.json") + configure_file("${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/functests/test.json" "${CMAKE_BINARY_DIR}/test.json" COPYONLY) endif() endif() @@ -63,11 +56,11 @@ endif() if(PAL_IMPLEMENTATION STREQUAL "WIN32") # Link against prebuilt libraries on Windows - message("--- WIN32: Linking against prebuilt libraries") - message("--- WIN32: ... ${CMAKE_BINARY_DIR}/gtest") - message("--- WIN32: ... ${CMAKE_BINARY_DIR}/gmock") - message("--- WIN32: ... ${CMAKE_BINARY_DIR}/zlib") - message("--- WIN32: ... ${CMAKE_BINARY_DIR}/sqlite") + message(STATUS "WIN32: Linking against prebuilt libraries") + message(STATUS "WIN32: ... ${CMAKE_BINARY_DIR}/gtest") + message(STATUS "WIN32: ... ${CMAKE_BINARY_DIR}/gmock") + message(STATUS "WIN32: ... ${CMAKE_BINARY_DIR}/zlib") + message(STATUS "WIN32: ... ${CMAKE_BINARY_DIR}/sqlite") # link_directories(${CMAKE_BINARY_DIR}/gtest/ ${CMAKE_BINARY_DIR}/gmock/ ${CMAKE_BINARY_DIR}/zlib/ ${CMAKE_BINARY_DIR}/sqlite/) include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/../../zlib ) target_link_libraries(FuncTests @@ -109,10 +102,9 @@ else() set (PLATFORM_LIBS "atomic") endif() - # Find libraries - message("--- Linking libraries! ") - message("Current Dir: ${CMAKE_CURRENT_SOURCE_DIR}") - message("Binary Dir: ${CMAKE_BINARY_DIR}") + message(STATUS "Linking libraries") + message(STATUS "Current Dir: ${CMAKE_CURRENT_SOURCE_DIR}") + message(STATUS "Binary Dir: ${CMAKE_BINARY_DIR}") set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE NEVER) diff --git a/tests/functests/FuncTests.vcxproj b/tests/functests/FuncTests.vcxproj index 50f4e2427..87b0b5b53 100644 --- a/tests/functests/FuncTests.vcxproj +++ b/tests/functests/FuncTests.vcxproj @@ -436,7 +436,6 @@ - diff --git a/tests/functests/FuncTests.vcxproj.filters b/tests/functests/FuncTests.vcxproj.filters index 22f2ac5d9..e19381a72 100644 --- a/tests/functests/FuncTests.vcxproj.filters +++ b/tests/functests/FuncTests.vcxproj.filters @@ -53,9 +53,6 @@ mocks - - mocks - mocks diff --git a/tests/functests/MultipleLogManagersTests.cpp b/tests/functests/MultipleLogManagersTests.cpp index 25f99f175..eac2bfd00 100644 --- a/tests/functests/MultipleLogManagersTests.cpp +++ b/tests/functests/MultipleLogManagersTests.cpp @@ -54,7 +54,7 @@ class RequestHandler : public HttpServer::Callback } private: - size_t m_count {}; + std::atomic m_count {}; int m_id ; }; @@ -63,9 +63,9 @@ class MultipleLogManagersTests : public ::testing::Test protected: std::string serverAddress; ILogConfiguration config1, config2, config3; - RequestHandler callback1 = RequestHandler(1); - RequestHandler callback2 = RequestHandler(2); - RequestHandler callback3 = RequestHandler(3); + RequestHandler callback1{1}; + RequestHandler callback2{2}; + RequestHandler callback3{3}; HttpServer server; @@ -74,7 +74,7 @@ class MultipleLogManagersTests : public ::testing::Test { int port = server.addListeningPort(0); std::ostringstream os; - os << "localhost:" << port; + os << "127.0.0.1:" << port; server.setServerName(os.str()); serverAddress = "http://" + os.str(); @@ -196,7 +196,7 @@ TEST_F(MultipleLogManagersTests, ThreeInstancesCoexist) lm2->GetLogController()->UploadNow(); lm3->GetLogController()->UploadNow(); - waitForRequestsMultipleLogManager(10000, 1, 1, 1); + waitForRequestsMultipleLogManager(20000, 1, 1, 1); lm1.reset(); lm2.reset(); @@ -224,7 +224,7 @@ TEST_F(MultipleLogManagersTests, MultiProcessesLogManager) CAPTURE_PERF_STATS("Events Sent"); lm->GetLogController()->UploadNow(); CAPTURE_PERF_STATS("Events Uploaded"); - waitForRequestsSingleLogManager(10000, 2); + waitForRequestsSingleLogManager(20000, 2); lm.reset(); CAPTURE_PERF_STATS("Log Manager deleted"); } diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index 891150d34..1b29db3bf 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -1,4 +1,4 @@ -message("--- unittests") +message(STATUS "Building unittests") set(SRCS AIJsonSerializerTests.cpp @@ -19,6 +19,7 @@ set(SRCS EventPropertiesTests.cpp GuidTests.cpp HttpClientCAPITests.cpp + HttpClientCurlTests.cpp HttpClientManagerTests.cpp HttpClientTests.cpp HttpDeflateCompressionTests.cpp @@ -53,7 +54,7 @@ set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS -Wno-deprecated-dec # Enable Azure Monitor unit tests when the module is present. # The AIJsonSerializer test sources are guarded by HAVE_MAT_AI. -if (EXISTS ${CMAKE_SOURCE_DIR}/lib/modules/azmon/AIJsonSerializer.hpp) +if (EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/azmon/AIJsonSerializer.hpp") add_definitions(-DHAVE_MAT_AI) endif() @@ -65,7 +66,7 @@ if (APPLE) endif() endif() -if (EXISTS ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests) +if (EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/exp/tests") list(APPEND SRCS ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/unittests/ECSConfigCacheTests.cpp ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests/unittests/ECSClientUtilsTests.cpp @@ -73,7 +74,7 @@ if (EXISTS ${CMAKE_SOURCE_DIR}/lib/modules/exp/tests) ) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard/ AND BUILD_PRIVACYGUARD) +if(EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/privacyguard/" AND BUILD_PRIVACYGUARD) add_definitions(-DHAVE_MAT_PRIVACYGUARD) list(APPEND SRCS ${CMAKE_SOURCE_DIR}/lib/modules/privacyguard/tests/unittests/InitializationConfigurationTests.cpp @@ -84,7 +85,7 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard/ AND BUILD_PRIVACYGUA ) endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer/ AND BUILD_SANITIZER) +if(EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/sanitizer/" AND BUILD_SANITIZER) list(APPEND SRCS ${CMAKE_SOURCE_DIR}/lib/modules/sanitizer/tests/unittests/SanitizerJwtTests.cpp ${CMAKE_SOURCE_DIR}/lib/modules/sanitizer/tests/unittests/SanitizerProviderTests.cpp @@ -96,7 +97,7 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer/ AND BUILD_SANITIZER) ) endif() -if(EXISTS ${CMAKE_SOURCE_DIR}/lib/modules/dataviewer/) +if(EXISTS "${CMAKE_SOURCE_DIR}/lib/modules/dataviewer/") list(APPEND SRCS ${CMAKE_SOURCE_DIR}/lib/modules/dataviewer/tests/unittests/DefaultDataViewerTests.cpp DataViewerCollectionTests.cpp @@ -114,11 +115,11 @@ endif() if(PAL_IMPLEMENTATION STREQUAL "WIN32") # Link against prebuilt libraries on Windows - message("--- WIN32: Linking against prebuilt libraries") - message("--- WIN32: ... ${CMAKE_BINARY_DIR}/gtest") - message("--- WIN32: ... ${CMAKE_BINARY_DIR}/gmock") - message("--- WIN32: ... ${CMAKE_BINARY_DIR}/zlib") - message("--- WIN32: ... ${CMAKE_BINARY_DIR}/sqlite") + message(STATUS "WIN32: Linking against prebuilt libraries") + message(STATUS "WIN32: ... ${CMAKE_BINARY_DIR}/gtest") + message(STATUS "WIN32: ... ${CMAKE_BINARY_DIR}/gmock") + message(STATUS "WIN32: ... ${CMAKE_BINARY_DIR}/zlib") + message(STATUS "WIN32: ... ${CMAKE_BINARY_DIR}/sqlite") # link_directories(${CMAKE_BINARY_DIR}/gtest/ ${CMAKE_BINARY_DIR}/gmock/ ${CMAKE_BINARY_DIR}/zlib/ ${CMAKE_BINARY_DIR}/sqlite/) include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/../../zlib ) target_link_libraries(UnitTests @@ -136,6 +137,9 @@ else() set (SQLITE3_LIB "/usr/local/lib/libsqlite3.a") elseif(EXISTS "/usr/local/opt/sqlite/lib/libsqlite3.a") set (SQLITE3_LIB "/usr/local/opt/sqlite/lib/libsqlite3.a") + elseif(EXISTS "/opt/homebrew/opt/sqlite/lib/libsqlite3.a") + # Apple Silicon homebrew installs to /opt/homebrew instead of /usr/local + set (SQLITE3_LIB "/opt/homebrew/opt/sqlite/lib/libsqlite3.a") else() set (SQLITE3_LIB "sqlite3") endif() @@ -158,10 +162,9 @@ else() set (PLATFORM_LIBS "atomic") endif() - # Find libraries - message("--- Linking libraries! ") - message("Current Dir: ${CMAKE_CURRENT_SOURCE_DIR}") - message("Binary Dir: ${CMAKE_BINARY_DIR}") + message(STATUS "Linking libraries") + message(STATUS "Current Dir: ${CMAKE_CURRENT_SOURCE_DIR}") + message(STATUS "Binary Dir: ${CMAKE_BINARY_DIR}") include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/../../lib/ ) @@ -179,8 +182,8 @@ else() ${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/googletest/build/lib/ ) - message("GTEST: ${LIBGTEST}") - message("GMOCK: ${LIBGMOCK}") + message(STATUS "GTEST: ${LIBGTEST}") + message(STATUS "GMOCK: ${LIBGMOCK}") target_link_libraries(UnitTests ${LIBGTEST} diff --git a/tests/unittests/HttpClientCurlTests.cpp b/tests/unittests/HttpClientCurlTests.cpp new file mode 100644 index 000000000..889d2ffe7 --- /dev/null +++ b/tests/unittests/HttpClientCurlTests.cpp @@ -0,0 +1,126 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +#include "mat/config.h" + +// These tests only apply to the curl HTTP client path (Linux, non-Apple, non-Android) +#if defined(MATSDK_PAL_CPP11) && !defined(_MSC_VER) && defined(HAVE_MAT_DEFAULT_HTTP_CLIENT) \ + && !defined(__APPLE__) && !defined(ANDROID) + +#include "common/Common.hpp" +#include "http/HttpClient_Curl.hpp" +#include "config/RuntimeConfig_Default.hpp" + +using namespace testing; +using namespace MAT; + +class HttpClientCurlTests : public ::testing::Test +{ +protected: + HttpClient_Curl m_client; +}; + +// --- SetSslVerification wiring --- + +TEST_F(HttpClientCurlTests, SslVerification_DefaultsToTrue) +{ + CurlHttpOperation op("GET", "https://example.com", nullptr); + ASSERT_NE(op.GetHandle(), nullptr); +} + +TEST_F(HttpClientCurlTests, CurlHttpOperation_ConstructsWithVerifyTrue) +{ + CurlHttpOperation op("GET", "https://example.com", nullptr, + std::map(), std::vector(), + false, 5, true, ""); + ASSERT_NE(op.GetHandle(), nullptr); +} + +TEST_F(HttpClientCurlTests, CurlHttpOperation_ConstructsWithVerifyFalse) +{ + CurlHttpOperation op("GET", "https://example.com", nullptr, + std::map(), std::vector(), + false, 5, false, ""); + ASSERT_NE(op.GetHandle(), nullptr); +} + +TEST_F(HttpClientCurlTests, CurlHttpOperation_ConstructsWithCaInfo) +{ + CurlHttpOperation op("GET", "https://example.com", nullptr, + std::map(), std::vector(), + false, 5, true, "/etc/ssl/certs/ca-certificates.crt"); + ASSERT_NE(op.GetHandle(), nullptr); +} + +// --- ILogConfiguration integration --- + +TEST(HttpClientCurlConfigTests, LogConfiguration_SslVerify_DefaultIsTrue) +{ + // defaultRuntimeConfig from RuntimeConfig_Default.hpp has the defaults + bool sslVerify = defaultRuntimeConfig[CFG_MAP_HTTP][CFG_BOOL_HTTP_SSL_VERIFY]; + EXPECT_TRUE(sslVerify); +} + +TEST(HttpClientCurlConfigTests, LogConfiguration_SslCaInfo_DefaultIsEmpty) +{ + const char* caInfo = defaultRuntimeConfig[CFG_MAP_HTTP][CFG_STR_HTTP_SSL_CAINFO]; + EXPECT_STREQ(caInfo, ""); +} + +TEST(HttpClientCurlConfigTests, LogConfiguration_SslVerify_CanBeDisabled) +{ + ILogConfiguration config; + config[CFG_MAP_HTTP][CFG_BOOL_HTTP_SSL_VERIFY] = false; + bool sslVerify = config[CFG_MAP_HTTP][CFG_BOOL_HTTP_SSL_VERIFY]; + EXPECT_FALSE(sslVerify); +} + +TEST(HttpClientCurlConfigTests, LogConfiguration_SslCaInfo_CanBeSet) +{ + ILogConfiguration config; + config[CFG_MAP_HTTP][CFG_STR_HTTP_SSL_CAINFO] = "/custom/ca-bundle.crt"; + const char* caInfo = config[CFG_MAP_HTTP][CFG_STR_HTTP_SSL_CAINFO]; + EXPECT_STREQ(caInfo, "/custom/ca-bundle.crt"); +} + +// --- ApplySettings integration --- + +TEST_F(HttpClientCurlTests, ApplySettings_ReadsSslConfigFromLogConfiguration) +{ + ILogConfiguration config; + config[CFG_MAP_HTTP][CFG_BOOL_HTTP_SSL_VERIFY] = false; + config[CFG_MAP_HTTP][CFG_STR_HTTP_SSL_CAINFO] = "/custom/ca.pem"; + m_client.ApplySettings(config); + // Verify indirectly -- constructing an operation should not fail + SUCCEED(); +} + +TEST_F(HttpClientCurlTests, ApplySettings_DefaultConfigEnablesVerification) +{ + ILogConfiguration config; + m_client.ApplySettings(config); + SUCCEED(); +} + +// --- Thread safety: SetSslVerification concurrent with reads --- + +TEST_F(HttpClientCurlTests, SetSslVerification_ConcurrentCallsNoRace) +{ + // Exercise the atomic + mutex path under contention. + // No assertions on output -- this is a sanitizer/TSAN target. + std::vector> futures; + for (int i = 0; i < 10; ++i) + { + futures.push_back(std::async(std::launch::async, [this, i]() { + m_client.SetSslVerification(i % 2 == 0, (i % 2 == 0) ? "/some/path" : ""); + })); + } + for (auto& f : futures) + { + f.get(); + } + SUCCEED(); +} + +#endif // MATSDK_PAL_CPP11 && !_MSC_VER && HAVE_MAT_DEFAULT_HTTP_CLIENT diff --git a/tests/unittests/HttpClientTests.cpp b/tests/unittests/HttpClientTests.cpp index 99d73248b..4b17bcce5 100644 --- a/tests/unittests/HttpClientTests.cpp +++ b/tests/unittests/HttpClientTests.cpp @@ -53,7 +53,7 @@ class HttpClientTests : public ::testing::Test, { _port = _server.addListeningPort(0); std::ostringstream os; - os << "localhost:" << _port; + os << "127.0.0.1:" << _port; _hostname = os.str(); _server.setServerName(_hostname); _server.addHandler("/simple/", *this); diff --git a/tests/unittests/OfflineStorageTests_Room.cpp b/tests/unittests/OfflineStorageTests_Room.cpp index 5e26154fe..5f5b56af6 100644 --- a/tests/unittests/OfflineStorageTests_Room.cpp +++ b/tests/unittests/OfflineStorageTests_Room.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #ifdef ANDROID #include #endif @@ -527,6 +528,107 @@ TEST_P(OfflineStorageTestsRoom, ReleaseActuallyReleases) { ); } +// Regression test for JNI stale local reference bug in GetAndReserveRecords, +// GetRecords, and ReleaseRecords. Each per-record iteration uses +// pushLocalFrame/popLocalFrame; the jclass obtained on iteration 0 must be a +// global reference to remain valid on iteration 1+. Without the fix, ART's JNI +// checker fires JniAbort (SIGABRT) on the second call to GetObjectClass / +// GetFieldID with the stale local ref. +TEST_P(OfflineStorageTestsRoom, MultiRecordIterationFieldIdValidity) +{ + auto now = PAL::getUtcSystemTimeMs(); + StorageRecordVector input; + // Store 3 records: enough to exercise iterations 0, 1, and 2 of the per-record + // loop, covering both the "first time" (class lookup) and "subsequent" paths. + for (size_t i = 0; i < 3; ++i) { + auto id = "reg-" + std::to_string(i); + input.emplace_back( + id, + id, + EventLatency_Normal, + EventPersistence_Normal, + now, + StorageBlob {static_cast(i + 1), 2, 3}); + } + offlineStorage->StoreRecords(input); + ASSERT_EQ(size_t { 3 }, offlineStorage->GetRecordCount(EventLatency_Normal)); + + // GetAndReserveRecords: all 3 records must be returned with correct field values. + StorageRecordVector found; + EXPECT_TRUE(offlineStorage->GetAndReserveRecords( + [&found](StorageRecord && record) -> bool { + found.push_back(std::move(record)); + return true; + }, 5000)); + ASSERT_EQ(size_t { 3 }, found.size()); + { + // Set-based check: return order is implementation-defined + // (SQLite/Room: insertion order; Memory: LIFO). + std::set blob0_values; + for (auto const& r : found) { + EXPECT_EQ(EventLatency_Normal, r.latency); + EXPECT_EQ(EventPersistence_Normal, r.persistence); + ASSERT_EQ(size_t { 3 }, r.blob.size()); + blob0_values.insert(r.blob[0]); + } + EXPECT_EQ((std::set{1, 2, 3}), blob0_values); + } + + // GetRecords: same 3 records readable via GetRecords (shutdown path). + // Memory's GetRecords delegates to GetAndReserveRecords, so it returns + // nothing when records are already reserved — skip that check for Memory. + if (implementation != StorageImplementation::Memory) { + auto shutdown_found = offlineStorage->GetRecords(true, EventLatency_Unspecified, 0); + ASSERT_EQ(size_t { 3 }, shutdown_found.size()); + std::set blob0_values; + for (auto const& r : shutdown_found) { + EXPECT_EQ(EventLatency_Normal, r.latency); + ASSERT_EQ(size_t { 3 }, r.blob.size()); + blob0_values.insert(r.blob[0]); + } + EXPECT_EQ((std::set{1, 2, 3}), blob0_values); + } + + // Un-reserve without using a retry slot so the retry loop below can start fresh. + { + std::vector initial_ids; + initial_ids.reserve(found.size()); + for (auto const& r : found) { + initial_ids.push_back(r.id); + } + bool fromMemory = false; + offlineStorage->ReleaseRecords(initial_ids, false, HttpHeaders(), fromMemory); + } + + // ReleaseRecords bt_class path: cycle GetAndReserveRecords + ReleaseRecords(true) + // GetMaximumRetryCount()+1 times. On the final cycle the Room impl drops the 3 + // records and returns a non-empty byTenant array, exercising the bt_class loop. + // Without the global-ref fix, iteration 1+ of that loop produces a JNI abort. + auto retries = configMock.GetMaximumRetryCount() + 1; + if (implementation != StorageImplementation::Memory) { + EXPECT_CALL(observerMock, OnStorageRecordsDropped(SizeIs(3))).WillOnce(Return()); + } + for (size_t retry = 0; retry < retries; ++retry) { + found.clear(); + offlineStorage->GetAndReserveRecords( + [&found](StorageRecord && record) -> bool { + found.push_back(std::move(record)); + return true; + }, 5000); + EXPECT_EQ(size_t { 3 }, found.size()) << "retry=" << retry; + std::vector ids; + ids.reserve(found.size()); + for (auto const& r : found) { + ids.push_back(r.id); + } + bool fromMemory = false; + offlineStorage->ReleaseRecords(ids, true, HttpHeaders(), fromMemory); + } + if (implementation != StorageImplementation::Memory) { + EXPECT_EQ(size_t { 0 }, offlineStorage->GetRecordCount(EventLatency_Normal)); + } +} + TEST_P(OfflineStorageTestsRoom, DeleteByToken) { StorageRecordVector records; diff --git a/tests/unittests/PalTests.cpp b/tests/unittests/PalTests.cpp index 1ec078916..15bb98e5e 100644 --- a/tests/unittests/PalTests.cpp +++ b/tests/unittests/PalTests.cpp @@ -85,7 +85,7 @@ TEST_F(PalTests, SystemTime) int64_t t1 = PAL::getUtcSystemTimeMs(); EXPECT_THAT(t1, Gt(t0 + 360)); - EXPECT_THAT(t1, Lt(t0 + 550)); + EXPECT_THAT(t1, Lt(t0 + 1000)); } TEST_F(PalTests, FormatUtcTimestampMsAsISO8601) @@ -103,7 +103,7 @@ TEST_F(PalTests, MonotonicTime) int64_t t1 = PAL::getMonotonicTimeMs(); EXPECT_THAT(t1 - t0, Gt(780)); - EXPECT_THAT(t1 - t0, Lt(950)); + EXPECT_THAT(t1 - t0, Lt(1500)); } TEST_F(PalTests, SemanticContextPopulation) diff --git a/tests/unittests/UnitTests.vcxproj b/tests/unittests/UnitTests.vcxproj index 579c52a83..12f44eae7 100644 --- a/tests/unittests/UnitTests.vcxproj +++ b/tests/unittests/UnitTests.vcxproj @@ -495,7 +495,6 @@ - diff --git a/tests/unittests/obj-c/ODWReachabilityTests.mm b/tests/unittests/obj-c/ODWReachabilityTests.mm index 501a3be7a..cc8a7c24e 100644 --- a/tests/unittests/obj-c/ODWReachabilityTests.mm +++ b/tests/unittests/obj-c/ODWReachabilityTests.mm @@ -13,11 +13,7 @@ #import #import -#import #import -#import -#import -#import @interface ODWReachabilityTests : XCTestCase @end @@ -110,4 +106,3 @@ - (void)testReachabilityForLocalWiFi } @end - diff --git a/third_party/Reachability/ODWReachability.m b/third_party/Reachability/ODWReachability.m index 0e13591d5..7947f7df0 100644 --- a/third_party/Reachability/ODWReachability.m +++ b/third_party/Reachability/ODWReachability.m @@ -29,11 +29,7 @@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF #import #import -#import #import -#import -#import -#import NSString *const kNetworkReachabilityChangedNotification = @"NetworkReachabilityChangedNotification"; diff --git a/third_party/cgmanifest.json b/third_party/cgmanifest.json new file mode 100644 index 000000000..dbda55d75 --- /dev/null +++ b/third_party/cgmanifest.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/tonymillion/Reachability", + "CommitHash": "788111ebf98e86368cd03da17f884015d2fc1543" + } + }, + "DevelopmentDependency": false + }, + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest.git", + "CommitHash": "58d77fa8070e8cec2dc1ed015d66b454c8d78850" + } + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/clibs/sha1/", + "CommitHash": "d5b29e11ec4871b78b370e8e00bd0cd56d798dda" + } + }, + "DevelopmentDependency": true + } + ] +} \ No newline at end of file diff --git a/tools/MakeDeb.cmake b/tools/MakeDeb.cmake index efec5ad32..4f839ab5b 100644 --- a/tools/MakeDeb.cmake +++ b/tools/MakeDeb.cmake @@ -24,7 +24,7 @@ set(CPACK_PACKAGE_VERSION_PATCH "${PATCH_VERSION}") # FIXME: add architecture name in file name set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}-${CPACK_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}") -message("-- Package name: ${CPACK_PACKAGE_FILE_NAME}.deb") +message(STATUS "Package name: ${CPACK_PACKAGE_FILE_NAME}.deb") #install(TARGETS ${MAT_SDK_LIB_DIR}/libMAT.a ARCHIVE DESTINATION lib/MAT COMPONENT headers) #install(FILES ${MAT_SDK_INC_DIR}/*.* DESTINATION include/MAT COMPONENT libraries) diff --git a/tools/MakeRpm.cmake b/tools/MakeRpm.cmake index c4db45de0..8d9cf0b67 100644 --- a/tools/MakeRpm.cmake +++ b/tools/MakeRpm.cmake @@ -18,7 +18,7 @@ set(CPACK_PACKAGE_NAME "mat_sdk") set(CPACK_PACKAGE_RELEASE "0") set(CPACK_PACKAGE_VENDOR "Microsoft") set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE}.${CMAKE_SYSTEM_PROCESSOR}") -message("-- Package name: ${CPACK_RPM_PACKAGE_FILE_NAME}.rpm") +message(STATUS "Package name: ${CPACK_PACKAGE_FILE_NAME}.rpm") #configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mat-sdk.spec.in" "${CMAKE_CURRENT_BINARY_DIR}/arka-sdk.spec" @ONLY IMMEDIATE) #set(CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_BINARY_DIR}/mat-sdk.spec") diff --git a/tools/MakeTgz.cmake b/tools/MakeTgz.cmake index bf159ab8d..44c308e32 100644 --- a/tools/MakeTgz.cmake +++ b/tools/MakeTgz.cmake @@ -24,8 +24,7 @@ file(GLOB ALL_TARGET_LIBS "${CMAKE_CURRENT_BINARY_DIR}/lib/libmat.*") install(FILES ${ALL_TARGET_LIBS} DESTINATION lib COMPONENT libraries) #install(FILES ${MAT_SDK_INC_DIR}/*.* DESTINATION include/mat COMPONENT libraries) -# FIXME: add architecture name in file name set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}-${CPACK_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}") -message("-- Package name: ${CPACK_PACKAGE_FILE_NAME}.tgz") +message(STATUS "Package name: ${CPACK_PACKAGE_FILE_NAME}.tgz") include(CPack) diff --git a/tools/ParseOsRelease.cmake b/tools/ParseOsRelease.cmake index 48bd0398a..61952f57f 100644 --- a/tools/ParseOsRelease.cmake +++ b/tools/ParseOsRelease.cmake @@ -1,6 +1,6 @@ # Parse /etc/os-release to determine Linux distro -if(EXISTS /etc/os-release) +if(EXISTS "/etc/os-release") file(STRINGS /etc/os-release OS_RELEASE) foreach(NameAndValue ${OS_RELEASE}) @@ -13,7 +13,7 @@ foreach(NameAndValue ${OS_RELEASE}) # Strip quotes from value string(REPLACE "\"" "" Value ${Value}) # Set the variable - message("-- /etc/os_release : ${Name}=${Value}") + message(STATUS "/etc/os-release : ${Name}=${Value}") set("OS_RELEASE_${Name}" "${Value}") endforeach() diff --git a/tools/ports/mstelemetry/v142-build.patch b/tools/ports/mstelemetry/v142-build.patch index 3913521ef..35cd0ec8e 100644 --- a/tools/ports/mstelemetry/v142-build.patch +++ b/tools/ports/mstelemetry/v142-build.patch @@ -3276,7 +3276,7 @@ diff --git a/zlib/contrib/vstudio/vc14/zlibvc.vcxproj b/zlib/contrib/vstudio/vc1 index 5ff55579..77518e48 100644 --- a/zlib/contrib/vstudio/vc14/zlibvc.vcxproj +++ b/zlib/contrib/vstudio/vc14/zlibvc.vcxproj -@@ -1,1218 +1,1218 @@ +@@ -1,1218 +1,1215 @@ - - - @@ -5669,15 +5669,12 @@ index 5ff55579..77518e48 100644 + + + -+ + -+ + + + + + -+ + + + diff --git a/tools/setup-buildtools.cmd b/tools/setup-buildtools.cmd index 5c4705a15..890467256 100644 --- a/tools/setup-buildtools.cmd +++ b/tools/setup-buildtools.cmd @@ -16,8 +16,9 @@ if ERRORLEVEL 0 ( vswhere -property installationPath ) -REM Install tools needed to build SDK with either Visual Studio or CMake -choco install -y cmake svn git llvm zip +REM Install baseline tools needed to build SDK with either Visual Studio or CMake +REM LLVM is optional and handled below when INSTALL_LLVM is defined. +choco install -y cmake svn git zip REM Try to autodetect Visual Studio call "%~dp0\vcvars.cmd" diff --git a/zlib/CMakeLists.txt b/zlib/CMakeLists.txt index 1a3c3ff7c..f23e529f7 100644 --- a/zlib/CMakeLists.txt +++ b/zlib/CMakeLists.txt @@ -3,7 +3,7 @@ set(CMAKE_ALLOW_LOOSE_LOOP_CONSTRUCTS ON) project(zlib C) -set(VERSION "1.2.8") +set(VERSION "1.2.11") option(ASM686 "Enable building i686 assembly implementation") option(AMD64 "Enable building amd64 assembly implementation") diff --git a/zlib/OWNERS b/zlib/OWNERS deleted file mode 100644 index 90ca6c07b..000000000 --- a/zlib/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -agl@chromium.org -cblume@chromium.org -gavinp@chromium.org -msarett@chromium.org - -# COMPONENT: Internals diff --git a/zlib/contrib/vstudio/vc14/zlibvc.vcxproj b/zlib/contrib/vstudio/vc14/zlibvc.vcxproj index ef193406d..0966b6c0f 100644 --- a/zlib/contrib/vstudio/vc14/zlibvc.vcxproj +++ b/zlib/contrib/vstudio/vc14/zlibvc.vcxproj @@ -1181,15 +1181,12 @@ bld_ml64.bat - - - diff --git a/zlib/crc32.c b/zlib/crc32.c index 9162429cc..9580440c0 100644 --- a/zlib/crc32.c +++ b/zlib/crc32.c @@ -28,8 +28,6 @@ # endif /* !DYNAMIC_CRC_TABLE */ #endif /* MAKECRCH */ -#include "deflate.h" -#include "x86.h" #include "zutil.h" /* for STDC and FAR definitions */ /* Definitions for doing the crc four data bytes at a time. */ @@ -442,28 +440,3 @@ uLong ZEXPORT crc32_combine64(crc1, crc2, len2) { return crc32_combine_(crc1, crc2, len2); } - -ZLIB_INTERNAL void crc_reset(deflate_state *const s) -{ - if (x86_cpu_enable_simd) { - crc_fold_init(s); - return; - } - s->strm->adler = crc32(0L, Z_NULL, 0); -} - -ZLIB_INTERNAL void crc_finalize(deflate_state *const s) -{ - if (x86_cpu_enable_simd) - s->strm->adler = crc_fold_512to32(s); -} - -ZLIB_INTERNAL void copy_with_crc(z_streamp strm, Bytef *dst, long size) -{ - if (x86_cpu_enable_simd) { - crc_fold_copy(strm->state, dst, strm->next_in, size); - return; - } - zmemcpy(dst, strm->next_in, size); - strm->adler = crc32(strm->adler, dst, size); -} diff --git a/zlib/crc_folding.c b/zlib/crc_folding.c deleted file mode 100644 index c4f0808a9..000000000 --- a/zlib/crc_folding.c +++ /dev/null @@ -1,495 +0,0 @@ -#ifndef ARCH_ARM -/* - * Compute the CRC32 using a parallelized folding approach with the PCLMULQDQ - * instruction. - * - * A white paper describing this algorithm can be found at: - * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf - * - * Copyright (C) 2013 Intel Corporation. All rights reserved. - * Authors: - * Wajdi Feghali - * Jim Guilford - * Vinodh Gopal - * Erdinc Ozturk - * Jim Kukunas - * - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#include "deflate.h" - -#include -#include -#include -#include - -#define CRC_LOAD(s) \ - do { \ - __m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0);\ - __m128i xmm_crc1 = _mm_loadu_si128((__m128i *)s->crc0 + 1);\ - __m128i xmm_crc2 = _mm_loadu_si128((__m128i *)s->crc0 + 2);\ - __m128i xmm_crc3 = _mm_loadu_si128((__m128i *)s->crc0 + 3);\ - __m128i xmm_crc_part = _mm_loadu_si128((__m128i *)s->crc0 + 4); - -#define CRC_SAVE(s) \ - _mm_storeu_si128((__m128i *)s->crc0 + 0, xmm_crc0);\ - _mm_storeu_si128((__m128i *)s->crc0 + 1, xmm_crc1);\ - _mm_storeu_si128((__m128i *)s->crc0 + 2, xmm_crc2);\ - _mm_storeu_si128((__m128i *)s->crc0 + 3, xmm_crc3);\ - _mm_storeu_si128((__m128i *)s->crc0 + 4, xmm_crc_part);\ - } while (0); - -ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) -{ - CRC_LOAD(s) - - xmm_crc0 = _mm_cvtsi32_si128(0x9db42487); - xmm_crc1 = _mm_setzero_si128(); - xmm_crc2 = _mm_setzero_si128(); - xmm_crc3 = _mm_setzero_si128(); - - CRC_SAVE(s) - - s->strm->adler = 0; -} - -local void fold_1(deflate_state *const s, - __m128i *xmm_crc0, __m128i *xmm_crc1, - __m128i *xmm_crc2, __m128i *xmm_crc3) -{ - const __m128i xmm_fold4 = _mm_set_epi32( - 0x00000001, 0x54442bd4, - 0x00000001, 0xc6e41596); - - __m128i x_tmp3; - __m128 ps_crc0, ps_crc3, ps_res; - - x_tmp3 = *xmm_crc3; - - *xmm_crc3 = *xmm_crc0; - *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); - *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); - ps_crc0 = _mm_castsi128_ps(*xmm_crc0); - ps_crc3 = _mm_castsi128_ps(*xmm_crc3); - ps_res = _mm_xor_ps(ps_crc0, ps_crc3); - - *xmm_crc0 = *xmm_crc1; - *xmm_crc1 = *xmm_crc2; - *xmm_crc2 = x_tmp3; - *xmm_crc3 = _mm_castps_si128(ps_res); -} - -local void fold_2(deflate_state *const s, - __m128i *xmm_crc0, __m128i *xmm_crc1, - __m128i *xmm_crc2, __m128i *xmm_crc3) -{ - const __m128i xmm_fold4 = _mm_set_epi32( - 0x00000001, 0x54442bd4, - 0x00000001, 0xc6e41596); - - __m128i x_tmp3, x_tmp2; - __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res31, ps_res20; - - x_tmp3 = *xmm_crc3; - x_tmp2 = *xmm_crc2; - - *xmm_crc3 = *xmm_crc1; - *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); - *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); - ps_crc3 = _mm_castsi128_ps(*xmm_crc3); - ps_crc1 = _mm_castsi128_ps(*xmm_crc1); - ps_res31= _mm_xor_ps(ps_crc3, ps_crc1); - - *xmm_crc2 = *xmm_crc0; - *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); - *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); - ps_crc0 = _mm_castsi128_ps(*xmm_crc0); - ps_crc2 = _mm_castsi128_ps(*xmm_crc2); - ps_res20= _mm_xor_ps(ps_crc0, ps_crc2); - - *xmm_crc0 = x_tmp2; - *xmm_crc1 = x_tmp3; - *xmm_crc2 = _mm_castps_si128(ps_res20); - *xmm_crc3 = _mm_castps_si128(ps_res31); -} - -local void fold_3(deflate_state *const s, - __m128i *xmm_crc0, __m128i *xmm_crc1, - __m128i *xmm_crc2, __m128i *xmm_crc3) -{ - const __m128i xmm_fold4 = _mm_set_epi32( - 0x00000001, 0x54442bd4, - 0x00000001, 0xc6e41596); - - __m128i x_tmp3; - __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res32, ps_res21, ps_res10; - - x_tmp3 = *xmm_crc3; - - *xmm_crc3 = *xmm_crc2; - *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); - *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); - ps_crc2 = _mm_castsi128_ps(*xmm_crc2); - ps_crc3 = _mm_castsi128_ps(*xmm_crc3); - ps_res32 = _mm_xor_ps(ps_crc2, ps_crc3); - - *xmm_crc2 = *xmm_crc1; - *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); - *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); - ps_crc1 = _mm_castsi128_ps(*xmm_crc1); - ps_crc2 = _mm_castsi128_ps(*xmm_crc2); - ps_res21= _mm_xor_ps(ps_crc1, ps_crc2); - - *xmm_crc1 = *xmm_crc0; - *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); - *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); - ps_crc0 = _mm_castsi128_ps(*xmm_crc0); - ps_crc1 = _mm_castsi128_ps(*xmm_crc1); - ps_res10= _mm_xor_ps(ps_crc0, ps_crc1); - - *xmm_crc0 = x_tmp3; - *xmm_crc1 = _mm_castps_si128(ps_res10); - *xmm_crc2 = _mm_castps_si128(ps_res21); - *xmm_crc3 = _mm_castps_si128(ps_res32); -} - -local void fold_4(deflate_state *const s, - __m128i *xmm_crc0, __m128i *xmm_crc1, - __m128i *xmm_crc2, __m128i *xmm_crc3) -{ - const __m128i xmm_fold4 = _mm_set_epi32( - 0x00000001, 0x54442bd4, - 0x00000001, 0xc6e41596); - - __m128i x_tmp0, x_tmp1, x_tmp2, x_tmp3; - __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3; - __m128 ps_t0, ps_t1, ps_t2, ps_t3; - __m128 ps_res0, ps_res1, ps_res2, ps_res3; - - x_tmp0 = *xmm_crc0; - x_tmp1 = *xmm_crc1; - x_tmp2 = *xmm_crc2; - x_tmp3 = *xmm_crc3; - - *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); - x_tmp0 = _mm_clmulepi64_si128(x_tmp0, xmm_fold4, 0x10); - ps_crc0 = _mm_castsi128_ps(*xmm_crc0); - ps_t0 = _mm_castsi128_ps(x_tmp0); - ps_res0 = _mm_xor_ps(ps_crc0, ps_t0); - - *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); - x_tmp1 = _mm_clmulepi64_si128(x_tmp1, xmm_fold4, 0x10); - ps_crc1 = _mm_castsi128_ps(*xmm_crc1); - ps_t1 = _mm_castsi128_ps(x_tmp1); - ps_res1 = _mm_xor_ps(ps_crc1, ps_t1); - - *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); - x_tmp2 = _mm_clmulepi64_si128(x_tmp2, xmm_fold4, 0x10); - ps_crc2 = _mm_castsi128_ps(*xmm_crc2); - ps_t2 = _mm_castsi128_ps(x_tmp2); - ps_res2 = _mm_xor_ps(ps_crc2, ps_t2); - - *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01); - x_tmp3 = _mm_clmulepi64_si128(x_tmp3, xmm_fold4, 0x10); - ps_crc3 = _mm_castsi128_ps(*xmm_crc3); - ps_t3 = _mm_castsi128_ps(x_tmp3); - ps_res3 = _mm_xor_ps(ps_crc3, ps_t3); - - *xmm_crc0 = _mm_castps_si128(ps_res0); - *xmm_crc1 = _mm_castps_si128(ps_res1); - *xmm_crc2 = _mm_castps_si128(ps_res2); - *xmm_crc3 = _mm_castps_si128(ps_res3); -} - -local const unsigned zalign(32) pshufb_shf_table[60] = { - 0x84838281,0x88878685,0x8c8b8a89,0x008f8e8d, /* shl 15 (16 - 1)/shr1 */ - 0x85848382,0x89888786,0x8d8c8b8a,0x01008f8e, /* shl 14 (16 - 3)/shr2 */ - 0x86858483,0x8a898887,0x8e8d8c8b,0x0201008f, /* shl 13 (16 - 4)/shr3 */ - 0x87868584,0x8b8a8988,0x8f8e8d8c,0x03020100, /* shl 12 (16 - 4)/shr4 */ - 0x88878685,0x8c8b8a89,0x008f8e8d,0x04030201, /* shl 11 (16 - 5)/shr5 */ - 0x89888786,0x8d8c8b8a,0x01008f8e,0x05040302, /* shl 10 (16 - 6)/shr6 */ - 0x8a898887,0x8e8d8c8b,0x0201008f,0x06050403, /* shl 9 (16 - 7)/shr7 */ - 0x8b8a8988,0x8f8e8d8c,0x03020100,0x07060504, /* shl 8 (16 - 8)/shr8 */ - 0x8c8b8a89,0x008f8e8d,0x04030201,0x08070605, /* shl 7 (16 - 9)/shr9 */ - 0x8d8c8b8a,0x01008f8e,0x05040302,0x09080706, /* shl 6 (16 -10)/shr10*/ - 0x8e8d8c8b,0x0201008f,0x06050403,0x0a090807, /* shl 5 (16 -11)/shr11*/ - 0x8f8e8d8c,0x03020100,0x07060504,0x0b0a0908, /* shl 4 (16 -12)/shr12*/ - 0x008f8e8d,0x04030201,0x08070605,0x0c0b0a09, /* shl 3 (16 -13)/shr13*/ - 0x01008f8e,0x05040302,0x09080706,0x0d0c0b0a, /* shl 2 (16 -14)/shr14*/ - 0x0201008f,0x06050403,0x0a090807,0x0e0d0c0b /* shl 1 (16 -15)/shr15*/ -}; - -local void partial_fold(deflate_state *const s, const size_t len, - __m128i *xmm_crc0, __m128i *xmm_crc1, - __m128i *xmm_crc2, __m128i *xmm_crc3, - __m128i *xmm_crc_part) -{ - - const __m128i xmm_fold4 = _mm_set_epi32( - 0x00000001, 0x54442bd4, - 0x00000001, 0xc6e41596); - const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080); - - __m128i xmm_shl, xmm_shr, xmm_tmp1, xmm_tmp2, xmm_tmp3; - __m128i xmm_a0_0, xmm_a0_1; - __m128 ps_crc3, psa0_0, psa0_1, ps_res; - - xmm_shl = _mm_load_si128((__m128i *)pshufb_shf_table + (len - 1)); - xmm_shr = xmm_shl; - xmm_shr = _mm_xor_si128(xmm_shr, xmm_mask3); - - xmm_a0_0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shl); - - *xmm_crc0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shr); - xmm_tmp1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shl); - *xmm_crc0 = _mm_or_si128(*xmm_crc0, xmm_tmp1); - - *xmm_crc1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shr); - xmm_tmp2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shl); - *xmm_crc1 = _mm_or_si128(*xmm_crc1, xmm_tmp2); - - *xmm_crc2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shr); - xmm_tmp3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shl); - *xmm_crc2 = _mm_or_si128(*xmm_crc2, xmm_tmp3); - - *xmm_crc3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shr); - *xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl); - *xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part); - - xmm_a0_1 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x10); - xmm_a0_0 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x01); - - ps_crc3 = _mm_castsi128_ps(*xmm_crc3); - psa0_0 = _mm_castsi128_ps(xmm_a0_0); - psa0_1 = _mm_castsi128_ps(xmm_a0_1); - - ps_res = _mm_xor_ps(ps_crc3, psa0_0); - ps_res = _mm_xor_ps(ps_res, psa0_1); - - *xmm_crc3 = _mm_castps_si128(ps_res); -} - -ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, - unsigned char *dst, const unsigned char *src, long len) -{ - unsigned long algn_diff; - __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; - - CRC_LOAD(s) - - if (len < 16) { - if (len == 0) - return; - goto partial; - } - - algn_diff = 0 - (uintptr_t)src & 0xF; - if (algn_diff) { - xmm_crc_part = _mm_loadu_si128((__m128i *)src); - _mm_storeu_si128((__m128i *)dst, xmm_crc_part); - - dst += algn_diff; - src += algn_diff; - len -= algn_diff; - - partial_fold(s, algn_diff, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, - &xmm_crc_part); - } - - while ((len -= 64) >= 0) { - xmm_t0 = _mm_load_si128((__m128i *)src); - xmm_t1 = _mm_load_si128((__m128i *)src + 1); - xmm_t2 = _mm_load_si128((__m128i *)src + 2); - xmm_t3 = _mm_load_si128((__m128i *)src + 3); - - fold_4(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); - - _mm_storeu_si128((__m128i *)dst, xmm_t0); - _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); - _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); - _mm_storeu_si128((__m128i *)dst + 3, xmm_t3); - - xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0); - xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1); - xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3); - - src += 64; - dst += 64; - } - - /* - * len = num bytes left - 64 - */ - if (len + 16 >= 0) { - len += 16; - - xmm_t0 = _mm_load_si128((__m128i *)src); - xmm_t1 = _mm_load_si128((__m128i *)src + 1); - xmm_t2 = _mm_load_si128((__m128i *)src + 2); - - fold_3(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); - - _mm_storeu_si128((__m128i *)dst, xmm_t0); - _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); - _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); - - xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0); - xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2); - - if (len == 0) - goto done; - - dst += 48; - src += 48; - } else if (len + 32 >= 0) { - len += 32; - - xmm_t0 = _mm_load_si128((__m128i *)src); - xmm_t1 = _mm_load_si128((__m128i *)src + 1); - - fold_2(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); - - _mm_storeu_si128((__m128i *)dst, xmm_t0); - _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); - - xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1); - - if (len == 0) - goto done; - - dst += 32; - src += 32; - } else if (len + 48 >= 0) { - len += 48; - - xmm_t0 = _mm_load_si128((__m128i *)src); - - fold_1(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); - - _mm_storeu_si128((__m128i *)dst, xmm_t0); - - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0); - - if (len == 0) - goto done; - - dst += 16; - src += 16; - } else { - len += 64; - if (len == 0) - goto done; - } - -partial: - -#if defined(_MSC_VER) - /* VS does not permit the use of _mm_set_epi64x in 32-bit builds */ - { - int32_t parts[4] = {0, 0, 0, 0}; - memcpy(&parts, src, len); - xmm_crc_part = _mm_set_epi32(parts[3], parts[2], parts[1], parts[0]); - } -#else - { - int64_t parts[2] = {0, 0}; - memcpy(&parts, src, len); - xmm_crc_part = _mm_set_epi64x(parts[1], parts[0]); - } -#endif - - _mm_storeu_si128((__m128i *)dst, xmm_crc_part); - partial_fold(s, len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, - &xmm_crc_part); -done: - CRC_SAVE(s) -} - -local const unsigned zalign(16) crc_k[] = { - 0xccaa009e, 0x00000000, /* rk1 */ - 0x751997d0, 0x00000001, /* rk2 */ - 0xccaa009e, 0x00000000, /* rk5 */ - 0x63cd6124, 0x00000001, /* rk6 */ - 0xf7011640, 0x00000001, /* rk7 */ - 0xdb710640, 0x00000001 /* rk8 */ -}; - -local const unsigned zalign(16) crc_mask[4] = { - 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000 -}; - -local const unsigned zalign(16) crc_mask2[4] = { - 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF -}; - -unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) -{ - const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask); - const __m128i xmm_mask2 = _mm_load_si128((__m128i *)crc_mask2); - - unsigned crc; - __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold; - - CRC_LOAD(s) - - /* - * k1 - */ - crc_fold = _mm_load_si128((__m128i *)crc_k); - - x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10); - xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01); - xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0); - xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_crc0); - - x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10); - xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01); - xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1); - xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_crc1); - - x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10); - xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01); - xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); - - /* - * k5 - */ - crc_fold = _mm_load_si128((__m128i *)crc_k + 1); - - xmm_crc0 = xmm_crc3; - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); - xmm_crc0 = _mm_srli_si128(xmm_crc0, 8); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); - - xmm_crc0 = xmm_crc3; - xmm_crc3 = _mm_slli_si128(xmm_crc3, 4); - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); - xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask2); - - /* - * k7 - */ - xmm_crc1 = xmm_crc3; - xmm_crc2 = xmm_crc3; - crc_fold = _mm_load_si128((__m128i *)crc_k + 2); - - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); - xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask); - - xmm_crc2 = xmm_crc3; - xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); - xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1); - - crc = _mm_extract_epi32(xmm_crc3, 2); - return ~crc; - CRC_SAVE(s) -} -#endif \ No newline at end of file diff --git a/zlib/deflate.c b/zlib/deflate.c index 3e76e185c..1ec761448 100644 --- a/zlib/deflate.c +++ b/zlib/deflate.c @@ -48,15 +48,8 @@ */ /* @(#) $Id$ */ -#include -#include "deflate.h" -#ifndef ARCH_ARM -#include "x86.h" -#else -int x86_cpu_enable_simd = 0; -int x86_check_features() { return 0; } -#endif +#include "deflate.h" const char deflate_copyright[] = " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; @@ -93,7 +86,7 @@ local block_state deflate_huff OF((deflate_state *s, int flush)); local void lm_init OF((deflate_state *s)); local void putShortMSB OF((deflate_state *s, uInt b)); local void flush_pending OF((z_streamp strm)); -unsigned ZLIB_INTERNAL read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); +local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); #ifdef ASMV # pragma message("Assembler code may have bugs -- use at your own risk") void match_init OF((void)); /* asm code initialization */ @@ -107,20 +100,6 @@ local void check_match OF((deflate_state *s, IPos start, IPos match, int length)); #endif -/* From crc32.c */ -extern void ZLIB_INTERNAL crc_reset(deflate_state *const s); -extern void ZLIB_INTERNAL crc_finalize(deflate_state *const s); -extern void ZLIB_INTERNAL copy_with_crc(z_streamp strm, Bytef *dst, long size); - -#ifdef _MSC_VER -#define INLINE __inline -#else -#define INLINE inline -#endif - -/* Inline optimisation */ -local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str); - /* =========================================================================== * Local data */ @@ -183,6 +162,7 @@ local const config configuration_table[10] = { */ #define UPDATE_HASH(s,h,c) (h = (((h)<hash_shift) ^ (c)) & s->hash_mask) + /* =========================================================================== * Insert string str in the dictionary and set match_head to the previous head * of the hash chain (the most recent string with same hash key). Return @@ -193,28 +173,17 @@ local const config configuration_table[10] = { * characters and the first MIN_MATCH bytes of str are valid (except for * the last MIN_MATCH-1 bytes of the input file). */ -local INLINE Pos insert_string_c(deflate_state *const s, const Pos str) -{ - Pos ret; - - UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]); #ifdef FASTEST - ret = s->head[s->ins_h]; +#define INSERT_STRING(s, str, match_head) \ + (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ + match_head = s->head[s->ins_h], \ + s->head[s->ins_h] = (Pos)(str)) #else - ret = s->prev[str & s->w_mask] = s->head[s->ins_h]; +#define INSERT_STRING(s, str, match_head) \ + (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ + match_head = s->prev[(str) & s->w_mask] = s->head[s->ins_h], \ + s->head[s->ins_h] = (Pos)(str)) #endif - s->head[s->ins_h] = str; - - return ret; -} - -local INLINE Pos insert_string(deflate_state *const s, const Pos str) -{ - if (x86_cpu_enable_simd) - return insert_string_sse(s, str); - return insert_string_c(s, str); -} - /* =========================================================================== * Initialize the hash table (avoiding 64K overflow for 16 bit systems). @@ -279,7 +248,6 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, const char *version; int stream_size; { - unsigned window_padding = 8; deflate_state *s; int wrap = 1; static const char my_version[] = ZLIB_VERSION; @@ -289,8 +257,6 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, * output size for (length,distance) codes is <= 24 bits. */ - x86_check_features(); - if (version == Z_NULL || version[0] != my_version[0] || stream_size != sizeof(z_stream)) { return Z_VERSION_ERROR; @@ -347,19 +313,12 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, s->w_size = 1 << s->w_bits; s->w_mask = s->w_size - 1; - if (x86_cpu_enable_simd) { - s->hash_bits = 15; - } else { - s->hash_bits = memLevel + 7; - } - + s->hash_bits = (uInt)memLevel + 7; s->hash_size = 1 << s->hash_bits; s->hash_mask = s->hash_size - 1; s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); - s->window = (Bytef *) ZALLOC(strm, - s->w_size + window_padding, - 2*sizeof(Byte)); + s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte)); s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); @@ -459,7 +418,11 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) str = s->strstart; n = s->lookahead - (MIN_MATCH-1); do { - insert_string(s, str); + UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); +#ifndef FASTEST + s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = (Pos)str; str++; } while (--n); s->strstart = str; @@ -885,7 +848,7 @@ int ZEXPORT deflate (strm, flush) #ifdef GZIP if (s->status == GZIP_STATE) { /* gzip header */ - crc_reset(s); + strm->adler = crc32(0L, Z_NULL, 0); put_byte(s, 31); put_byte(s, 139); put_byte(s, 8); @@ -1086,7 +1049,6 @@ int ZEXPORT deflate (strm, flush) /* Write the trailer */ #ifdef GZIP if (s->wrap == 2) { - crc_finalize(s); put_byte(s, (Byte)(strm->adler & 0xff)); put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); put_byte(s, (Byte)((strm->adler >> 16) & 0xff)); @@ -1199,7 +1161,7 @@ int ZEXPORT deflateCopy (dest, source) * allocating a large strm->next_in buffer and copying from it. * (See also flush_pending()). */ -ZLIB_INTERNAL unsigned read_buf(strm, buf, size) +local unsigned read_buf(strm, buf, size) z_streamp strm; Bytef *buf; unsigned size; @@ -1211,16 +1173,15 @@ ZLIB_INTERNAL unsigned read_buf(strm, buf, size) strm->avail_in -= len; + zmemcpy(buf, strm->next_in, len); + if (strm->state->wrap == 1) { + strm->adler = adler32(strm->adler, buf, len); + } #ifdef GZIP - if (strm->state->wrap == 2) - copy_with_crc(strm, buf, len); - else -#endif - { - zmemcpy(buf, strm->next_in, len); - if (strm->state->wrap == 1) - strm->adler = adler32(strm->adler, buf, len); + else if (strm->state->wrap == 2) { + strm->adler = crc32(strm->adler, buf, len); } +#endif strm->next_in += len; strm->total_in += len; @@ -1518,19 +1479,7 @@ local void check_match(s, start, match, length) * performed for at least two bytes (required for the zip translate_eol * option -- not supported here). */ -local void fill_window_c(deflate_state *s); - -local void fill_window(deflate_state *s) -{ - if (x86_cpu_enable_simd) { - fill_window_sse(s); - return; - } - - fill_window_c(s); -} - -local void fill_window_c(s) +local void fill_window(s) deflate_state *s; { unsigned n; @@ -1898,7 +1847,7 @@ local block_state deflate_fast(s, flush) */ hash_head = NIL; if (s->lookahead >= MIN_MATCH) { - hash_head = insert_string(s, s->strstart); + INSERT_STRING(s, s->strstart, hash_head); } /* Find the longest match, discarding those <= prev_length. @@ -1929,7 +1878,7 @@ local block_state deflate_fast(s, flush) s->match_length--; /* string at strstart already in table */ do { s->strstart++; - hash_head = insert_string(s, s->strstart); + INSERT_STRING(s, s->strstart, hash_head); /* strstart never exceeds WSIZE-MAX_MATCH, so there are * always MIN_MATCH bytes ahead. */ @@ -2001,7 +1950,7 @@ local block_state deflate_slow(s, flush) */ hash_head = NIL; if (s->lookahead >= MIN_MATCH) { - hash_head = insert_string(s, s->strstart); + INSERT_STRING(s, s->strstart, hash_head); } /* Find the longest match, discarding those <= prev_length. @@ -2052,7 +2001,7 @@ local block_state deflate_slow(s, flush) s->prev_length -= 2; do { if (++s->strstart <= max_insert) { - hash_head = insert_string(s, s->strstart); + INSERT_STRING(s, s->strstart, hash_head); } } while (--s->prev_length != 0); s->match_available = 0; @@ -2212,41 +2161,3 @@ local block_state deflate_huff(s, flush) FLUSH_BLOCK(s, 0); return block_done; } - -/* Safe to inline this as GCC/clang will use inline asm and Visual Studio will - * use intrinsic without extra params - */ -local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str) -{ -#ifndef ARCH_ARM - Pos ret; - unsigned *ip, val, h = 0; - - ip = (unsigned *)&s->window[str]; - val = *ip; - - if (s->level >= 6) - val &= 0xFFFFFF; - -/* Windows clang should use inline asm */ -#if defined(_MSC_VER) && !defined(__clang__) - h = _mm_crc32_u32(h, val); -#elif defined(__i386__) || defined(__amd64__) - __asm__ __volatile__ ( - "crc32 %1,%0\n\t" - : "+r" (h) - : "r" (val) - ); -#else - /* This should never happen */ - assert(0); -#endif - - ret = s->head[h & s->hash_mask]; - s->head[h & s->hash_mask] = str; - s->prev[str & s->w_mask] = ret; - return ret; -#else - return 0; -#endif -} diff --git a/zlib/deflate.h b/zlib/deflate.h index 5e969fe10..23ecdd312 100644 --- a/zlib/deflate.h +++ b/zlib/deflate.h @@ -109,7 +109,7 @@ typedef struct internal_state { ulg gzindex; /* where in extra, name, or comment */ Byte method; /* can only be DEFLATED */ int last_flush; /* value of flush param for previous deflate call */ - unsigned zalign(16) crc0[4 * 5]; + /* used by deflate.c: */ uInt w_size; /* LZ77 window size (32K by default) */ @@ -346,22 +346,4 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, flush = _tr_tally(s, distance, length) #endif -#ifndef ARCH_ARM -/* Functions that are SIMD optimised on x86 */ -void ZLIB_INTERNAL crc_fold_init(deflate_state* const s); -void ZLIB_INTERNAL crc_fold_copy(deflate_state* const s, - unsigned char* dst, - const unsigned char* src, - long len); -unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state* const s); - -void ZLIB_INTERNAL fill_window_sse(deflate_state* s); -#else -/* These functions are not available on ARM architectures */ -__inline void ZLIB_INTERNAL crc_fold_init(deflate_state* const s) { }; -__inline void ZLIB_INTERNAL crc_fold_copy(deflate_state* const s, unsigned char* dst, const unsigned char* src, long len) { }; -__inline unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state* const s) { return 0; }; -__inline void ZLIB_INTERNAL fill_window_sse(deflate_state* s) { }; -#endif - #endif /* DEFLATE_H */ diff --git a/zlib/fill_window_sse.c b/zlib/fill_window_sse.c deleted file mode 100644 index 97957b2f3..000000000 --- a/zlib/fill_window_sse.c +++ /dev/null @@ -1,177 +0,0 @@ -#ifndef ARCH_ARM -/* - * Fill Window with SSE2-optimized hash shifting - * - * Copyright (C) 2013 Intel Corporation - * Authors: - * Arjan van de Ven - * Jim Kukunas - * - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#include -#include "deflate.h" - -#define UPDATE_HASH(s,h,i) \ - {\ - if (s->level < 6) { \ - h = (3483 * (s->window[i]) +\ - 23081* (s->window[i+1]) +\ - 6954 * (s->window[i+2]) +\ - 20947* (s->window[i+3])) & s->hash_mask;\ - } else {\ - h = (25881* (s->window[i]) +\ - 24674* (s->window[i+1]) +\ - 25811* (s->window[i+2])) & s->hash_mask;\ - }\ - }\ - -extern int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); - -void fill_window_sse(deflate_state *s) -{ - const __m128i xmm_wsize = _mm_set1_epi16(s->w_size); - - register unsigned n; - register Posf *p; - unsigned more; /* Amount of free space at the end of the window. */ - uInt wsize = s->w_size; - - Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); - - do { - more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); - - /* Deal with !@#$% 64K limit: */ - if (sizeof(int) <= 2) { - if (more == 0 && s->strstart == 0 && s->lookahead == 0) { - more = wsize; - - } else if (more == (unsigned)(-1)) { - /* Very unlikely, but possible on 16 bit machine if - * strstart == 0 && lookahead == 1 (input done a byte at time) - */ - more--; - } - } - - /* If the window is almost full and there is insufficient lookahead, - * move the upper half to the lower one to make room in the upper half. - */ - if (s->strstart >= wsize+MAX_DIST(s)) { - - zmemcpy(s->window, s->window+wsize, (unsigned)wsize); - s->match_start -= wsize; - s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ - s->block_start -= (long) wsize; - - /* Slide the hash table (could be avoided with 32 bit values - at the expense of memory usage). We slide even when level == 0 - to keep the hash table consistent if we switch back to level > 0 - later. (Using level 0 permanently is not an optimal usage of - zlib, so we don't care about this pathological case.) - */ - n = s->hash_size; - p = &s->head[n]; - p -= 8; - do { - __m128i value, result; - - value = _mm_loadu_si128((__m128i *)p); - result = _mm_subs_epu16(value, xmm_wsize); - _mm_storeu_si128((__m128i *)p, result); - - p -= 8; - n -= 8; - } while (n > 0); - - n = wsize; -#ifndef FASTEST - p = &s->prev[n]; - p -= 8; - do { - __m128i value, result; - - value = _mm_loadu_si128((__m128i *)p); - result = _mm_subs_epu16(value, xmm_wsize); - _mm_storeu_si128((__m128i *)p, result); - - p -= 8; - n -= 8; - } while (n > 0); -#endif - more += wsize; - } - if (s->strm->avail_in == 0) break; - - /* If there was no sliding: - * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && - * more == window_size - lookahead - strstart - * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) - * => more >= window_size - 2*WSIZE + 2 - * In the BIG_MEM or MMAP case (not yet supported), - * window_size == input_size + MIN_LOOKAHEAD && - * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. - * Otherwise, window_size == 2*WSIZE so more >= 2. - * If there was sliding, more >= WSIZE. So in all cases, more >= 2. - */ - Assert(more >= 2, "more < 2"); - - n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); - s->lookahead += n; - - /* Initialize the hash value now that we have some input: */ - if (s->lookahead >= MIN_MATCH) { - uInt str = s->strstart; - s->ins_h = s->window[str]; - if (str >= 1) - UPDATE_HASH(s, s->ins_h, str + 1 - (MIN_MATCH-1)); -#if MIN_MATCH != 3 - Call UPDATE_HASH() MIN_MATCH-3 more times -#endif - } - /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, - * but this is not important since only literal bytes will be emitted. - */ - - } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); - - /* If the WIN_INIT bytes after the end of the current data have never been - * written, then zero those bytes in order to avoid memory check reports of - * the use of uninitialized (or uninitialised as Julian writes) bytes by - * the longest match routines. Update the high water mark for the next - * time through here. WIN_INIT is set to MAX_MATCH since the longest match - * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. - */ - if (s->high_water < s->window_size) { - ulg curr = s->strstart + (ulg)(s->lookahead); - ulg init; - - if (s->high_water < curr) { - /* Previous high water mark below current data -- zero WIN_INIT - * bytes or up to end of window, whichever is less. - */ - init = s->window_size - curr; - if (init > WIN_INIT) - init = WIN_INIT; - zmemzero(s->window + curr, (unsigned)init); - s->high_water = curr + init; - } - else if (s->high_water < (ulg)curr + WIN_INIT) { - /* High water mark at or above current data, but below current data - * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up - * to end of window, whichever is less. - */ - init = (ulg)curr + WIN_INIT - s->high_water; - if (init > s->window_size - s->high_water) - init = s->window_size - s->high_water; - zmemzero(s->window + s->high_water, (unsigned)init); - s->high_water += init; - } - } - - Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, - "not enough room for search"); -} -#endif diff --git a/zlib/names.h b/zlib/names.h index 4113d4e8d..a3ae764d1 100644 --- a/zlib/names.h +++ b/zlib/names.h @@ -136,16 +136,4 @@ /* An exported symbol that isn't handled by Z_PREFIX in zconf.h */ #define z_errmsg act_z_z_errmsg -/* Symbols added in simd.patch */ -#define copy_with_crc act_z_copy_with_crc -#define crc_finalize act_z_crc_finalize -#define crc_fold_512to32 act_z_crc_fold_512to32 -#define crc_fold_copy act_z_crc_fold_copy -#define crc_fold_init act_z_crc_fold_init -#define crc_reset act_z_crc_reset -#define fill_window_sse act_z_fill_window_sse -#define read_buf act_z_read_buf -#define x86_check_features act_z_x86_check_features -#define x86_cpu_enable_simd act_z_x86_cpu_enable_simd - #endif /* ZLIB_NAMES_H_ */ diff --git a/zlib/simd.patch b/zlib/simd.patch deleted file mode 100644 index 75828d26e..000000000 --- a/zlib/simd.patch +++ /dev/null @@ -1,1233 +0,0 @@ -diff --git a/crc32.c b/crc32.c -index 9580440c0e6b..9162429cc7b4 100644 ---- a/crc32.c -+++ b/crc32.c -@@ -28,6 +28,8 @@ - # endif /* !DYNAMIC_CRC_TABLE */ - #endif /* MAKECRCH */ - -+#include "deflate.h" -+#include "x86.h" - #include "zutil.h" /* for STDC and FAR definitions */ - - /* Definitions for doing the crc four data bytes at a time. */ -@@ -440,3 +442,28 @@ uLong ZEXPORT crc32_combine64(crc1, crc2, len2) - { - return crc32_combine_(crc1, crc2, len2); - } -+ -+ZLIB_INTERNAL void crc_reset(deflate_state *const s) -+{ -+ if (x86_cpu_enable_simd) { -+ crc_fold_init(s); -+ return; -+ } -+ s->strm->adler = crc32(0L, Z_NULL, 0); -+} -+ -+ZLIB_INTERNAL void crc_finalize(deflate_state *const s) -+{ -+ if (x86_cpu_enable_simd) -+ s->strm->adler = crc_fold_512to32(s); -+} -+ -+ZLIB_INTERNAL void copy_with_crc(z_streamp strm, Bytef *dst, long size) -+{ -+ if (x86_cpu_enable_simd) { -+ crc_fold_copy(strm->state, dst, strm->next_in, size); -+ return; -+ } -+ zmemcpy(dst, strm->next_in, size); -+ strm->adler = crc32(strm->adler, dst, size); -+} -diff --git a/crc_folding.c b/crc_folding.c -new file mode 100644 -index 000000000000..48d77744aaf4 ---- /dev/null -+++ b/crc_folding.c -@@ -0,0 +1,493 @@ -+/* -+ * Compute the CRC32 using a parallelized folding approach with the PCLMULQDQ -+ * instruction. -+ * -+ * A white paper describing this algorithm can be found at: -+ * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf -+ * -+ * Copyright (C) 2013 Intel Corporation. All rights reserved. -+ * Authors: -+ * Wajdi Feghali -+ * Jim Guilford -+ * Vinodh Gopal -+ * Erdinc Ozturk -+ * Jim Kukunas -+ * -+ * For conditions of distribution and use, see copyright notice in zlib.h -+ */ -+ -+#include "deflate.h" -+ -+#include -+#include -+#include -+#include -+ -+#define CRC_LOAD(s) \ -+ do { \ -+ __m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0);\ -+ __m128i xmm_crc1 = _mm_loadu_si128((__m128i *)s->crc0 + 1);\ -+ __m128i xmm_crc2 = _mm_loadu_si128((__m128i *)s->crc0 + 2);\ -+ __m128i xmm_crc3 = _mm_loadu_si128((__m128i *)s->crc0 + 3);\ -+ __m128i xmm_crc_part = _mm_loadu_si128((__m128i *)s->crc0 + 4); -+ -+#define CRC_SAVE(s) \ -+ _mm_storeu_si128((__m128i *)s->crc0 + 0, xmm_crc0);\ -+ _mm_storeu_si128((__m128i *)s->crc0 + 1, xmm_crc1);\ -+ _mm_storeu_si128((__m128i *)s->crc0 + 2, xmm_crc2);\ -+ _mm_storeu_si128((__m128i *)s->crc0 + 3, xmm_crc3);\ -+ _mm_storeu_si128((__m128i *)s->crc0 + 4, xmm_crc_part);\ -+ } while (0); -+ -+ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) -+{ -+ CRC_LOAD(s) -+ -+ xmm_crc0 = _mm_cvtsi32_si128(0x9db42487); -+ xmm_crc1 = _mm_setzero_si128(); -+ xmm_crc2 = _mm_setzero_si128(); -+ xmm_crc3 = _mm_setzero_si128(); -+ -+ CRC_SAVE(s) -+ -+ s->strm->adler = 0; -+} -+ -+local void fold_1(deflate_state *const s, -+ __m128i *xmm_crc0, __m128i *xmm_crc1, -+ __m128i *xmm_crc2, __m128i *xmm_crc3) -+{ -+ const __m128i xmm_fold4 = _mm_set_epi32( -+ 0x00000001, 0x54442bd4, -+ 0x00000001, 0xc6e41596); -+ -+ __m128i x_tmp3; -+ __m128 ps_crc0, ps_crc3, ps_res; -+ -+ x_tmp3 = *xmm_crc3; -+ -+ *xmm_crc3 = *xmm_crc0; -+ *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); -+ *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); -+ ps_crc0 = _mm_castsi128_ps(*xmm_crc0); -+ ps_crc3 = _mm_castsi128_ps(*xmm_crc3); -+ ps_res = _mm_xor_ps(ps_crc0, ps_crc3); -+ -+ *xmm_crc0 = *xmm_crc1; -+ *xmm_crc1 = *xmm_crc2; -+ *xmm_crc2 = x_tmp3; -+ *xmm_crc3 = _mm_castps_si128(ps_res); -+} -+ -+local void fold_2(deflate_state *const s, -+ __m128i *xmm_crc0, __m128i *xmm_crc1, -+ __m128i *xmm_crc2, __m128i *xmm_crc3) -+{ -+ const __m128i xmm_fold4 = _mm_set_epi32( -+ 0x00000001, 0x54442bd4, -+ 0x00000001, 0xc6e41596); -+ -+ __m128i x_tmp3, x_tmp2; -+ __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res31, ps_res20; -+ -+ x_tmp3 = *xmm_crc3; -+ x_tmp2 = *xmm_crc2; -+ -+ *xmm_crc3 = *xmm_crc1; -+ *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); -+ *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); -+ ps_crc3 = _mm_castsi128_ps(*xmm_crc3); -+ ps_crc1 = _mm_castsi128_ps(*xmm_crc1); -+ ps_res31= _mm_xor_ps(ps_crc3, ps_crc1); -+ -+ *xmm_crc2 = *xmm_crc0; -+ *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); -+ *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); -+ ps_crc0 = _mm_castsi128_ps(*xmm_crc0); -+ ps_crc2 = _mm_castsi128_ps(*xmm_crc2); -+ ps_res20= _mm_xor_ps(ps_crc0, ps_crc2); -+ -+ *xmm_crc0 = x_tmp2; -+ *xmm_crc1 = x_tmp3; -+ *xmm_crc2 = _mm_castps_si128(ps_res20); -+ *xmm_crc3 = _mm_castps_si128(ps_res31); -+} -+ -+local void fold_3(deflate_state *const s, -+ __m128i *xmm_crc0, __m128i *xmm_crc1, -+ __m128i *xmm_crc2, __m128i *xmm_crc3) -+{ -+ const __m128i xmm_fold4 = _mm_set_epi32( -+ 0x00000001, 0x54442bd4, -+ 0x00000001, 0xc6e41596); -+ -+ __m128i x_tmp3; -+ __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res32, ps_res21, ps_res10; -+ -+ x_tmp3 = *xmm_crc3; -+ -+ *xmm_crc3 = *xmm_crc2; -+ *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); -+ *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); -+ ps_crc2 = _mm_castsi128_ps(*xmm_crc2); -+ ps_crc3 = _mm_castsi128_ps(*xmm_crc3); -+ ps_res32 = _mm_xor_ps(ps_crc2, ps_crc3); -+ -+ *xmm_crc2 = *xmm_crc1; -+ *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); -+ *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); -+ ps_crc1 = _mm_castsi128_ps(*xmm_crc1); -+ ps_crc2 = _mm_castsi128_ps(*xmm_crc2); -+ ps_res21= _mm_xor_ps(ps_crc1, ps_crc2); -+ -+ *xmm_crc1 = *xmm_crc0; -+ *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); -+ *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); -+ ps_crc0 = _mm_castsi128_ps(*xmm_crc0); -+ ps_crc1 = _mm_castsi128_ps(*xmm_crc1); -+ ps_res10= _mm_xor_ps(ps_crc0, ps_crc1); -+ -+ *xmm_crc0 = x_tmp3; -+ *xmm_crc1 = _mm_castps_si128(ps_res10); -+ *xmm_crc2 = _mm_castps_si128(ps_res21); -+ *xmm_crc3 = _mm_castps_si128(ps_res32); -+} -+ -+local void fold_4(deflate_state *const s, -+ __m128i *xmm_crc0, __m128i *xmm_crc1, -+ __m128i *xmm_crc2, __m128i *xmm_crc3) -+{ -+ const __m128i xmm_fold4 = _mm_set_epi32( -+ 0x00000001, 0x54442bd4, -+ 0x00000001, 0xc6e41596); -+ -+ __m128i x_tmp0, x_tmp1, x_tmp2, x_tmp3; -+ __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3; -+ __m128 ps_t0, ps_t1, ps_t2, ps_t3; -+ __m128 ps_res0, ps_res1, ps_res2, ps_res3; -+ -+ x_tmp0 = *xmm_crc0; -+ x_tmp1 = *xmm_crc1; -+ x_tmp2 = *xmm_crc2; -+ x_tmp3 = *xmm_crc3; -+ -+ *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); -+ x_tmp0 = _mm_clmulepi64_si128(x_tmp0, xmm_fold4, 0x10); -+ ps_crc0 = _mm_castsi128_ps(*xmm_crc0); -+ ps_t0 = _mm_castsi128_ps(x_tmp0); -+ ps_res0 = _mm_xor_ps(ps_crc0, ps_t0); -+ -+ *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); -+ x_tmp1 = _mm_clmulepi64_si128(x_tmp1, xmm_fold4, 0x10); -+ ps_crc1 = _mm_castsi128_ps(*xmm_crc1); -+ ps_t1 = _mm_castsi128_ps(x_tmp1); -+ ps_res1 = _mm_xor_ps(ps_crc1, ps_t1); -+ -+ *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); -+ x_tmp2 = _mm_clmulepi64_si128(x_tmp2, xmm_fold4, 0x10); -+ ps_crc2 = _mm_castsi128_ps(*xmm_crc2); -+ ps_t2 = _mm_castsi128_ps(x_tmp2); -+ ps_res2 = _mm_xor_ps(ps_crc2, ps_t2); -+ -+ *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01); -+ x_tmp3 = _mm_clmulepi64_si128(x_tmp3, xmm_fold4, 0x10); -+ ps_crc3 = _mm_castsi128_ps(*xmm_crc3); -+ ps_t3 = _mm_castsi128_ps(x_tmp3); -+ ps_res3 = _mm_xor_ps(ps_crc3, ps_t3); -+ -+ *xmm_crc0 = _mm_castps_si128(ps_res0); -+ *xmm_crc1 = _mm_castps_si128(ps_res1); -+ *xmm_crc2 = _mm_castps_si128(ps_res2); -+ *xmm_crc3 = _mm_castps_si128(ps_res3); -+} -+ -+local const unsigned zalign(32) pshufb_shf_table[60] = { -+ 0x84838281,0x88878685,0x8c8b8a89,0x008f8e8d, /* shl 15 (16 - 1)/shr1 */ -+ 0x85848382,0x89888786,0x8d8c8b8a,0x01008f8e, /* shl 14 (16 - 3)/shr2 */ -+ 0x86858483,0x8a898887,0x8e8d8c8b,0x0201008f, /* shl 13 (16 - 4)/shr3 */ -+ 0x87868584,0x8b8a8988,0x8f8e8d8c,0x03020100, /* shl 12 (16 - 4)/shr4 */ -+ 0x88878685,0x8c8b8a89,0x008f8e8d,0x04030201, /* shl 11 (16 - 5)/shr5 */ -+ 0x89888786,0x8d8c8b8a,0x01008f8e,0x05040302, /* shl 10 (16 - 6)/shr6 */ -+ 0x8a898887,0x8e8d8c8b,0x0201008f,0x06050403, /* shl 9 (16 - 7)/shr7 */ -+ 0x8b8a8988,0x8f8e8d8c,0x03020100,0x07060504, /* shl 8 (16 - 8)/shr8 */ -+ 0x8c8b8a89,0x008f8e8d,0x04030201,0x08070605, /* shl 7 (16 - 9)/shr9 */ -+ 0x8d8c8b8a,0x01008f8e,0x05040302,0x09080706, /* shl 6 (16 -10)/shr10*/ -+ 0x8e8d8c8b,0x0201008f,0x06050403,0x0a090807, /* shl 5 (16 -11)/shr11*/ -+ 0x8f8e8d8c,0x03020100,0x07060504,0x0b0a0908, /* shl 4 (16 -12)/shr12*/ -+ 0x008f8e8d,0x04030201,0x08070605,0x0c0b0a09, /* shl 3 (16 -13)/shr13*/ -+ 0x01008f8e,0x05040302,0x09080706,0x0d0c0b0a, /* shl 2 (16 -14)/shr14*/ -+ 0x0201008f,0x06050403,0x0a090807,0x0e0d0c0b /* shl 1 (16 -15)/shr15*/ -+}; -+ -+local void partial_fold(deflate_state *const s, const size_t len, -+ __m128i *xmm_crc0, __m128i *xmm_crc1, -+ __m128i *xmm_crc2, __m128i *xmm_crc3, -+ __m128i *xmm_crc_part) -+{ -+ -+ const __m128i xmm_fold4 = _mm_set_epi32( -+ 0x00000001, 0x54442bd4, -+ 0x00000001, 0xc6e41596); -+ const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080); -+ -+ __m128i xmm_shl, xmm_shr, xmm_tmp1, xmm_tmp2, xmm_tmp3; -+ __m128i xmm_a0_0, xmm_a0_1; -+ __m128 ps_crc3, psa0_0, psa0_1, ps_res; -+ -+ xmm_shl = _mm_load_si128((__m128i *)pshufb_shf_table + (len - 1)); -+ xmm_shr = xmm_shl; -+ xmm_shr = _mm_xor_si128(xmm_shr, xmm_mask3); -+ -+ xmm_a0_0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shl); -+ -+ *xmm_crc0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shr); -+ xmm_tmp1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shl); -+ *xmm_crc0 = _mm_or_si128(*xmm_crc0, xmm_tmp1); -+ -+ *xmm_crc1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shr); -+ xmm_tmp2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shl); -+ *xmm_crc1 = _mm_or_si128(*xmm_crc1, xmm_tmp2); -+ -+ *xmm_crc2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shr); -+ xmm_tmp3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shl); -+ *xmm_crc2 = _mm_or_si128(*xmm_crc2, xmm_tmp3); -+ -+ *xmm_crc3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shr); -+ *xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl); -+ *xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part); -+ -+ xmm_a0_1 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x10); -+ xmm_a0_0 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x01); -+ -+ ps_crc3 = _mm_castsi128_ps(*xmm_crc3); -+ psa0_0 = _mm_castsi128_ps(xmm_a0_0); -+ psa0_1 = _mm_castsi128_ps(xmm_a0_1); -+ -+ ps_res = _mm_xor_ps(ps_crc3, psa0_0); -+ ps_res = _mm_xor_ps(ps_res, psa0_1); -+ -+ *xmm_crc3 = _mm_castps_si128(ps_res); -+} -+ -+ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, -+ unsigned char *dst, const unsigned char *src, long len) -+{ -+ unsigned long algn_diff; -+ __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; -+ -+ CRC_LOAD(s) -+ -+ if (len < 16) { -+ if (len == 0) -+ return; -+ goto partial; -+ } -+ -+ algn_diff = 0 - (uintptr_t)src & 0xF; -+ if (algn_diff) { -+ xmm_crc_part = _mm_loadu_si128((__m128i *)src); -+ _mm_storeu_si128((__m128i *)dst, xmm_crc_part); -+ -+ dst += algn_diff; -+ src += algn_diff; -+ len -= algn_diff; -+ -+ partial_fold(s, algn_diff, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, -+ &xmm_crc_part); -+ } -+ -+ while ((len -= 64) >= 0) { -+ xmm_t0 = _mm_load_si128((__m128i *)src); -+ xmm_t1 = _mm_load_si128((__m128i *)src + 1); -+ xmm_t2 = _mm_load_si128((__m128i *)src + 2); -+ xmm_t3 = _mm_load_si128((__m128i *)src + 3); -+ -+ fold_4(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); -+ -+ _mm_storeu_si128((__m128i *)dst, xmm_t0); -+ _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); -+ _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); -+ _mm_storeu_si128((__m128i *)dst + 3, xmm_t3); -+ -+ xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0); -+ xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1); -+ xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2); -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3); -+ -+ src += 64; -+ dst += 64; -+ } -+ -+ /* -+ * len = num bytes left - 64 -+ */ -+ if (len + 16 >= 0) { -+ len += 16; -+ -+ xmm_t0 = _mm_load_si128((__m128i *)src); -+ xmm_t1 = _mm_load_si128((__m128i *)src + 1); -+ xmm_t2 = _mm_load_si128((__m128i *)src + 2); -+ -+ fold_3(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); -+ -+ _mm_storeu_si128((__m128i *)dst, xmm_t0); -+ _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); -+ _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); -+ -+ xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0); -+ xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1); -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2); -+ -+ if (len == 0) -+ goto done; -+ -+ dst += 48; -+ src += 48; -+ } else if (len + 32 >= 0) { -+ len += 32; -+ -+ xmm_t0 = _mm_load_si128((__m128i *)src); -+ xmm_t1 = _mm_load_si128((__m128i *)src + 1); -+ -+ fold_2(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); -+ -+ _mm_storeu_si128((__m128i *)dst, xmm_t0); -+ _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); -+ -+ xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0); -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1); -+ -+ if (len == 0) -+ goto done; -+ -+ dst += 32; -+ src += 32; -+ } else if (len + 48 >= 0) { -+ len += 48; -+ -+ xmm_t0 = _mm_load_si128((__m128i *)src); -+ -+ fold_1(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); -+ -+ _mm_storeu_si128((__m128i *)dst, xmm_t0); -+ -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0); -+ -+ if (len == 0) -+ goto done; -+ -+ dst += 16; -+ src += 16; -+ } else { -+ len += 64; -+ if (len == 0) -+ goto done; -+ } -+ -+partial: -+ -+#if defined(_MSC_VER) -+ /* VS does not permit the use of _mm_set_epi64x in 32-bit builds */ -+ { -+ int32_t parts[4] = {0, 0, 0, 0}; -+ memcpy(&parts, src, len); -+ xmm_crc_part = _mm_set_epi32(parts[3], parts[2], parts[1], parts[0]); -+ } -+#else -+ { -+ int64_t parts[2] = {0, 0}; -+ memcpy(&parts, src, len); -+ xmm_crc_part = _mm_set_epi64x(parts[1], parts[0]); -+ } -+#endif -+ -+ _mm_storeu_si128((__m128i *)dst, xmm_crc_part); -+ partial_fold(s, len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, -+ &xmm_crc_part); -+done: -+ CRC_SAVE(s) -+} -+ -+local const unsigned zalign(16) crc_k[] = { -+ 0xccaa009e, 0x00000000, /* rk1 */ -+ 0x751997d0, 0x00000001, /* rk2 */ -+ 0xccaa009e, 0x00000000, /* rk5 */ -+ 0x63cd6124, 0x00000001, /* rk6 */ -+ 0xf7011640, 0x00000001, /* rk7 */ -+ 0xdb710640, 0x00000001 /* rk8 */ -+}; -+ -+local const unsigned zalign(16) crc_mask[4] = { -+ 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000 -+}; -+ -+local const unsigned zalign(16) crc_mask2[4] = { -+ 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF -+}; -+ -+unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) -+{ -+ const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask); -+ const __m128i xmm_mask2 = _mm_load_si128((__m128i *)crc_mask2); -+ -+ unsigned crc; -+ __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold; -+ -+ CRC_LOAD(s) -+ -+ /* -+ * k1 -+ */ -+ crc_fold = _mm_load_si128((__m128i *)crc_k); -+ -+ x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10); -+ xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01); -+ xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0); -+ xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_crc0); -+ -+ x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10); -+ xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01); -+ xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1); -+ xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_crc1); -+ -+ x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10); -+ xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01); -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2); -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); -+ -+ /* -+ * k5 -+ */ -+ crc_fold = _mm_load_si128((__m128i *)crc_k + 1); -+ -+ xmm_crc0 = xmm_crc3; -+ xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); -+ xmm_crc0 = _mm_srli_si128(xmm_crc0, 8); -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); -+ -+ xmm_crc0 = xmm_crc3; -+ xmm_crc3 = _mm_slli_si128(xmm_crc3, 4); -+ xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); -+ xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask2); -+ -+ /* -+ * k7 -+ */ -+ xmm_crc1 = xmm_crc3; -+ xmm_crc2 = xmm_crc3; -+ crc_fold = _mm_load_si128((__m128i *)crc_k + 2); -+ -+ xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); -+ xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask); -+ -+ xmm_crc2 = xmm_crc3; -+ xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); -+ xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1); -+ -+ crc = _mm_extract_epi32(xmm_crc3, 2); -+ return ~crc; -+ CRC_SAVE(s) -+} -diff --git a/deflate.c b/deflate.c -index 1ec761448de9..aa0c9c67a6dc 100644 ---- a/deflate.c -+++ b/deflate.c -@@ -48,8 +48,9 @@ - */ - - /* @(#) $Id$ */ -- -+#include - #include "deflate.h" -+#include "x86.h" - - const char deflate_copyright[] = - " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; -@@ -86,7 +87,7 @@ local block_state deflate_huff OF((deflate_state *s, int flush)); - local void lm_init OF((deflate_state *s)); - local void putShortMSB OF((deflate_state *s, uInt b)); - local void flush_pending OF((z_streamp strm)); --local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); -+unsigned ZLIB_INTERNAL read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); - #ifdef ASMV - # pragma message("Assembler code may have bugs -- use at your own risk") - void match_init OF((void)); /* asm code initialization */ -@@ -100,6 +101,20 @@ local void check_match OF((deflate_state *s, IPos start, IPos match, - int length)); - #endif - -+/* From crc32.c */ -+extern void ZLIB_INTERNAL crc_reset(deflate_state *const s); -+extern void ZLIB_INTERNAL crc_finalize(deflate_state *const s); -+extern void ZLIB_INTERNAL copy_with_crc(z_streamp strm, Bytef *dst, long size); -+ -+#ifdef _MSC_VER -+#define INLINE __inline -+#else -+#define INLINE inline -+#endif -+ -+/* Inline optimisation */ -+local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str); -+ - /* =========================================================================== - * Local data - */ -@@ -162,7 +177,6 @@ local const config configuration_table[10] = { - */ - #define UPDATE_HASH(s,h,c) (h = (((h)<hash_shift) ^ (c)) & s->hash_mask) - -- - /* =========================================================================== - * Insert string str in the dictionary and set match_head to the previous head - * of the hash chain (the most recent string with same hash key). Return -@@ -173,17 +187,28 @@ local const config configuration_table[10] = { - * characters and the first MIN_MATCH bytes of str are valid (except for - * the last MIN_MATCH-1 bytes of the input file). - */ -+local INLINE Pos insert_string_c(deflate_state *const s, const Pos str) -+{ -+ Pos ret; -+ -+ UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]); - #ifdef FASTEST --#define INSERT_STRING(s, str, match_head) \ -- (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ -- match_head = s->head[s->ins_h], \ -- s->head[s->ins_h] = (Pos)(str)) -+ ret = s->head[s->ins_h]; - #else --#define INSERT_STRING(s, str, match_head) \ -- (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ -- match_head = s->prev[(str) & s->w_mask] = s->head[s->ins_h], \ -- s->head[s->ins_h] = (Pos)(str)) -+ ret = s->prev[str & s->w_mask] = s->head[s->ins_h]; - #endif -+ s->head[s->ins_h] = str; -+ -+ return ret; -+} -+ -+local INLINE Pos insert_string(deflate_state *const s, const Pos str) -+{ -+ if (x86_cpu_enable_simd) -+ return insert_string_sse(s, str); -+ return insert_string_c(s, str); -+} -+ - - /* =========================================================================== - * Initialize the hash table (avoiding 64K overflow for 16 bit systems). -@@ -248,6 +273,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, - const char *version; - int stream_size; - { -+ unsigned window_padding = 8; - deflate_state *s; - int wrap = 1; - static const char my_version[] = ZLIB_VERSION; -@@ -257,6 +283,8 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, - * output size for (length,distance) codes is <= 24 bits. - */ - -+ x86_check_features(); -+ - if (version == Z_NULL || version[0] != my_version[0] || - stream_size != sizeof(z_stream)) { - return Z_VERSION_ERROR; -@@ -313,12 +341,19 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, - s->w_size = 1 << s->w_bits; - s->w_mask = s->w_size - 1; - -- s->hash_bits = (uInt)memLevel + 7; -+ if (x86_cpu_enable_simd) { -+ s->hash_bits = 15; -+ } else { -+ s->hash_bits = memLevel + 7; -+ } -+ - s->hash_size = 1 << s->hash_bits; - s->hash_mask = s->hash_size - 1; - s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); - -- s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte)); -+ s->window = (Bytef *) ZALLOC(strm, -+ s->w_size + window_padding, -+ 2*sizeof(Byte)); - s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); - s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); - -@@ -418,11 +453,7 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) - str = s->strstart; - n = s->lookahead - (MIN_MATCH-1); - do { -- UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); --#ifndef FASTEST -- s->prev[str & s->w_mask] = s->head[s->ins_h]; --#endif -- s->head[s->ins_h] = (Pos)str; -+ insert_string(s, str); - str++; - } while (--n); - s->strstart = str; -@@ -848,7 +879,7 @@ int ZEXPORT deflate (strm, flush) - #ifdef GZIP - if (s->status == GZIP_STATE) { - /* gzip header */ -- strm->adler = crc32(0L, Z_NULL, 0); -+ crc_reset(s); - put_byte(s, 31); - put_byte(s, 139); - put_byte(s, 8); -@@ -1049,6 +1080,7 @@ int ZEXPORT deflate (strm, flush) - /* Write the trailer */ - #ifdef GZIP - if (s->wrap == 2) { -+ crc_finalize(s); - put_byte(s, (Byte)(strm->adler & 0xff)); - put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); - put_byte(s, (Byte)((strm->adler >> 16) & 0xff)); -@@ -1161,7 +1193,7 @@ int ZEXPORT deflateCopy (dest, source) - * allocating a large strm->next_in buffer and copying from it. - * (See also flush_pending()). - */ --local unsigned read_buf(strm, buf, size) -+ZLIB_INTERNAL unsigned read_buf(strm, buf, size) - z_streamp strm; - Bytef *buf; - unsigned size; -@@ -1173,15 +1205,16 @@ local unsigned read_buf(strm, buf, size) - - strm->avail_in -= len; - -- zmemcpy(buf, strm->next_in, len); -- if (strm->state->wrap == 1) { -- strm->adler = adler32(strm->adler, buf, len); -- } - #ifdef GZIP -- else if (strm->state->wrap == 2) { -- strm->adler = crc32(strm->adler, buf, len); -- } -+ if (strm->state->wrap == 2) -+ copy_with_crc(strm, buf, len); -+ else - #endif -+ { -+ zmemcpy(buf, strm->next_in, len); -+ if (strm->state->wrap == 1) -+ strm->adler = adler32(strm->adler, buf, len); -+ } - strm->next_in += len; - strm->total_in += len; - -@@ -1479,7 +1512,19 @@ local void check_match(s, start, match, length) - * performed for at least two bytes (required for the zip translate_eol - * option -- not supported here). - */ --local void fill_window(s) -+local void fill_window_c(deflate_state *s); -+ -+local void fill_window(deflate_state *s) -+{ -+ if (x86_cpu_enable_simd) { -+ fill_window_sse(s); -+ return; -+ } -+ -+ fill_window_c(s); -+} -+ -+local void fill_window_c(s) - deflate_state *s; - { - unsigned n; -@@ -1847,7 +1892,7 @@ local block_state deflate_fast(s, flush) - */ - hash_head = NIL; - if (s->lookahead >= MIN_MATCH) { -- INSERT_STRING(s, s->strstart, hash_head); -+ hash_head = insert_string(s, s->strstart); - } - - /* Find the longest match, discarding those <= prev_length. -@@ -1878,7 +1923,7 @@ local block_state deflate_fast(s, flush) - s->match_length--; /* string at strstart already in table */ - do { - s->strstart++; -- INSERT_STRING(s, s->strstart, hash_head); -+ hash_head = insert_string(s, s->strstart); - /* strstart never exceeds WSIZE-MAX_MATCH, so there are - * always MIN_MATCH bytes ahead. - */ -@@ -1950,7 +1995,7 @@ local block_state deflate_slow(s, flush) - */ - hash_head = NIL; - if (s->lookahead >= MIN_MATCH) { -- INSERT_STRING(s, s->strstart, hash_head); -+ hash_head = insert_string(s, s->strstart); - } - - /* Find the longest match, discarding those <= prev_length. -@@ -2001,7 +2046,7 @@ local block_state deflate_slow(s, flush) - s->prev_length -= 2; - do { - if (++s->strstart <= max_insert) { -- INSERT_STRING(s, s->strstart, hash_head); -+ hash_head = insert_string(s, s->strstart); - } - } while (--s->prev_length != 0); - s->match_available = 0; -@@ -2161,3 +2206,37 @@ local block_state deflate_huff(s, flush) - FLUSH_BLOCK(s, 0); - return block_done; - } -+ -+/* Safe to inline this as GCC/clang will use inline asm and Visual Studio will -+ * use intrinsic without extra params -+ */ -+local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str) -+{ -+ Pos ret; -+ unsigned *ip, val, h = 0; -+ -+ ip = (unsigned *)&s->window[str]; -+ val = *ip; -+ -+ if (s->level >= 6) -+ val &= 0xFFFFFF; -+ -+/* Windows clang should use inline asm */ -+#if defined(_MSC_VER) && !defined(__clang__) -+ h = _mm_crc32_u32(h, val); -+#elif defined(__i386__) || defined(__amd64__) -+ __asm__ __volatile__ ( -+ "crc32 %1,%0\n\t" -+ : "+r" (h) -+ : "r" (val) -+ ); -+#else -+ /* This should never happen */ -+ assert(0); -+#endif -+ -+ ret = s->head[h & s->hash_mask]; -+ s->head[h & s->hash_mask] = str; -+ s->prev[str & s->w_mask] = ret; -+ return ret; -+} -diff --git a/deflate.h b/deflate.h -index 23ecdd312bc0..ab56df7663b6 100644 ---- a/deflate.h -+++ b/deflate.h -@@ -109,7 +109,7 @@ typedef struct internal_state { - ulg gzindex; /* where in extra, name, or comment */ - Byte method; /* can only be DEFLATED */ - int last_flush; /* value of flush param for previous deflate call */ -- -+ unsigned zalign(16) crc0[4 * 5]; - /* used by deflate.c: */ - - uInt w_size; /* LZ77 window size (32K by default) */ -@@ -346,4 +346,14 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, - flush = _tr_tally(s, distance, length) - #endif - -+/* Functions that are SIMD optimised on x86 */ -+void ZLIB_INTERNAL crc_fold_init(deflate_state* const s); -+void ZLIB_INTERNAL crc_fold_copy(deflate_state* const s, -+ unsigned char* dst, -+ const unsigned char* src, -+ long len); -+unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state* const s); -+ -+void ZLIB_INTERNAL fill_window_sse(deflate_state* s); -+ - #endif /* DEFLATE_H */ -diff --git a/fill_window_sse.c b/fill_window_sse.c -new file mode 100644 -index 000000000000..949ccce1ba9c ---- /dev/null -+++ b/fill_window_sse.c -@@ -0,0 +1,175 @@ -+/* -+ * Fill Window with SSE2-optimized hash shifting -+ * -+ * Copyright (C) 2013 Intel Corporation -+ * Authors: -+ * Arjan van de Ven -+ * Jim Kukunas -+ * -+ * For conditions of distribution and use, see copyright notice in zlib.h -+ */ -+ -+#include -+#include "deflate.h" -+ -+#define UPDATE_HASH(s,h,i) \ -+ {\ -+ if (s->level < 6) { \ -+ h = (3483 * (s->window[i]) +\ -+ 23081* (s->window[i+1]) +\ -+ 6954 * (s->window[i+2]) +\ -+ 20947* (s->window[i+3])) & s->hash_mask;\ -+ } else {\ -+ h = (25881* (s->window[i]) +\ -+ 24674* (s->window[i+1]) +\ -+ 25811* (s->window[i+2])) & s->hash_mask;\ -+ }\ -+ }\ -+ -+extern int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); -+ -+void fill_window_sse(deflate_state *s) -+{ -+ const __m128i xmm_wsize = _mm_set1_epi16(s->w_size); -+ -+ register unsigned n; -+ register Posf *p; -+ unsigned more; /* Amount of free space at the end of the window. */ -+ uInt wsize = s->w_size; -+ -+ Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); -+ -+ do { -+ more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); -+ -+ /* Deal with !@#$% 64K limit: */ -+ if (sizeof(int) <= 2) { -+ if (more == 0 && s->strstart == 0 && s->lookahead == 0) { -+ more = wsize; -+ -+ } else if (more == (unsigned)(-1)) { -+ /* Very unlikely, but possible on 16 bit machine if -+ * strstart == 0 && lookahead == 1 (input done a byte at time) -+ */ -+ more--; -+ } -+ } -+ -+ /* If the window is almost full and there is insufficient lookahead, -+ * move the upper half to the lower one to make room in the upper half. -+ */ -+ if (s->strstart >= wsize+MAX_DIST(s)) { -+ -+ zmemcpy(s->window, s->window+wsize, (unsigned)wsize); -+ s->match_start -= wsize; -+ s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ -+ s->block_start -= (long) wsize; -+ -+ /* Slide the hash table (could be avoided with 32 bit values -+ at the expense of memory usage). We slide even when level == 0 -+ to keep the hash table consistent if we switch back to level > 0 -+ later. (Using level 0 permanently is not an optimal usage of -+ zlib, so we don't care about this pathological case.) -+ */ -+ n = s->hash_size; -+ p = &s->head[n]; -+ p -= 8; -+ do { -+ __m128i value, result; -+ -+ value = _mm_loadu_si128((__m128i *)p); -+ result = _mm_subs_epu16(value, xmm_wsize); -+ _mm_storeu_si128((__m128i *)p, result); -+ -+ p -= 8; -+ n -= 8; -+ } while (n > 0); -+ -+ n = wsize; -+#ifndef FASTEST -+ p = &s->prev[n]; -+ p -= 8; -+ do { -+ __m128i value, result; -+ -+ value = _mm_loadu_si128((__m128i *)p); -+ result = _mm_subs_epu16(value, xmm_wsize); -+ _mm_storeu_si128((__m128i *)p, result); -+ -+ p -= 8; -+ n -= 8; -+ } while (n > 0); -+#endif -+ more += wsize; -+ } -+ if (s->strm->avail_in == 0) break; -+ -+ /* If there was no sliding: -+ * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && -+ * more == window_size - lookahead - strstart -+ * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) -+ * => more >= window_size - 2*WSIZE + 2 -+ * In the BIG_MEM or MMAP case (not yet supported), -+ * window_size == input_size + MIN_LOOKAHEAD && -+ * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. -+ * Otherwise, window_size == 2*WSIZE so more >= 2. -+ * If there was sliding, more >= WSIZE. So in all cases, more >= 2. -+ */ -+ Assert(more >= 2, "more < 2"); -+ -+ n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); -+ s->lookahead += n; -+ -+ /* Initialize the hash value now that we have some input: */ -+ if (s->lookahead >= MIN_MATCH) { -+ uInt str = s->strstart; -+ s->ins_h = s->window[str]; -+ if (str >= 1) -+ UPDATE_HASH(s, s->ins_h, str + 1 - (MIN_MATCH-1)); -+#if MIN_MATCH != 3 -+ Call UPDATE_HASH() MIN_MATCH-3 more times -+#endif -+ } -+ /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, -+ * but this is not important since only literal bytes will be emitted. -+ */ -+ -+ } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); -+ -+ /* If the WIN_INIT bytes after the end of the current data have never been -+ * written, then zero those bytes in order to avoid memory check reports of -+ * the use of uninitialized (or uninitialised as Julian writes) bytes by -+ * the longest match routines. Update the high water mark for the next -+ * time through here. WIN_INIT is set to MAX_MATCH since the longest match -+ * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. -+ */ -+ if (s->high_water < s->window_size) { -+ ulg curr = s->strstart + (ulg)(s->lookahead); -+ ulg init; -+ -+ if (s->high_water < curr) { -+ /* Previous high water mark below current data -- zero WIN_INIT -+ * bytes or up to end of window, whichever is less. -+ */ -+ init = s->window_size - curr; -+ if (init > WIN_INIT) -+ init = WIN_INIT; -+ zmemzero(s->window + curr, (unsigned)init); -+ s->high_water = curr + init; -+ } -+ else if (s->high_water < (ulg)curr + WIN_INIT) { -+ /* High water mark at or above current data, but below current data -+ * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up -+ * to end of window, whichever is less. -+ */ -+ init = (ulg)curr + WIN_INIT - s->high_water; -+ if (init > s->window_size - s->high_water) -+ init = s->window_size - s->high_water; -+ zmemzero(s->window + s->high_water, (unsigned)init); -+ s->high_water += init; -+ } -+ } -+ -+ Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, -+ "not enough room for search"); -+} -diff --git a/names.h b/names.h -index f18df5684dc5..3436baa4eb57 100644 ---- a/names.h -+++ b/names.h -@@ -152,4 +152,16 @@ - /* An exported symbol that isn't handled by Z_PREFIX in zconf.h */ - #define z_errmsg Cr_z_z_errmsg - -+/* Symbols added in simd.patch */ -+#define copy_with_crc Cr_z_copy_with_crc -+#define crc_finalize Cr_z_crc_finalize -+#define crc_fold_512to32 Cr_z_crc_fold_512to32 -+#define crc_fold_copy Cr_z_crc_fold_copy -+#define crc_fold_init Cr_z_crc_fold_init -+#define crc_reset Cr_z_crc_reset -+#define fill_window_sse Cr_z_fill_window_sse -+#define read_buf Cr_z_read_buf -+#define x86_check_features Cr_z_x86_check_features -+#define x86_cpu_enable_simd Cr_z_x86_cpu_enable_simd -+ - #endif /* THIRD_PARTY_ZLIB_NAMES_H_ */ -diff --git a/simd_stub.c b/simd_stub.c -new file mode 100644 -index 000000000000..c6d46051498f ---- /dev/null -+++ b/simd_stub.c -@@ -0,0 +1,35 @@ -+/* simd_stub.c -- stub implementations -+* Copyright (C) 2014 Intel Corporation -+* For conditions of distribution and use, see copyright notice in zlib.h -+*/ -+#include -+ -+#include "deflate.h" -+#include "x86.h" -+ -+int ZLIB_INTERNAL x86_cpu_enable_simd = 0; -+ -+void ZLIB_INTERNAL crc_fold_init(deflate_state *const s) { -+ assert(0); -+} -+ -+void ZLIB_INTERNAL crc_fold_copy(deflate_state *const s, -+ unsigned char *dst, -+ const unsigned char *src, -+ long len) { -+ assert(0); -+} -+ -+unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) { -+ assert(0); -+ return 0; -+} -+ -+void ZLIB_INTERNAL fill_window_sse(deflate_state *s) -+{ -+ assert(0); -+} -+ -+void x86_check_features(void) -+{ -+} -diff --git a/x86.c b/x86.c -new file mode 100644 -index 000000000000..e56fe8b85a39 ---- /dev/null -+++ b/x86.c -@@ -0,0 +1,92 @@ -+/* -+ * x86 feature check -+ * -+ * Copyright (C) 2013 Intel Corporation. All rights reserved. -+ * Author: -+ * Jim Kukunas -+ * -+ * For conditions of distribution and use, see copyright notice in zlib.h -+ */ -+ -+#include "x86.h" -+#include "zutil.h" -+ -+int ZLIB_INTERNAL x86_cpu_enable_simd = 0; -+ -+#ifndef _MSC_VER -+#include -+ -+pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; -+static void _x86_check_features(void); -+ -+void x86_check_features(void) -+{ -+ pthread_once(&cpu_check_inited_once, _x86_check_features); -+} -+ -+static void _x86_check_features(void) -+{ -+ int x86_cpu_has_sse2; -+ int x86_cpu_has_sse42; -+ int x86_cpu_has_pclmulqdq; -+ unsigned eax, ebx, ecx, edx; -+ -+ eax = 1; -+#ifdef __i386__ -+ __asm__ __volatile__ ( -+ "xchg %%ebx, %1\n\t" -+ "cpuid\n\t" -+ "xchg %1, %%ebx\n\t" -+ : "+a" (eax), "=S" (ebx), "=c" (ecx), "=d" (edx) -+ ); -+#else -+ __asm__ __volatile__ ( -+ "cpuid\n\t" -+ : "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) -+ ); -+#endif /* (__i386__) */ -+ -+ x86_cpu_has_sse2 = edx & 0x4000000; -+ x86_cpu_has_sse42 = ecx & 0x100000; -+ x86_cpu_has_pclmulqdq = ecx & 0x2; -+ -+ x86_cpu_enable_simd = x86_cpu_has_sse2 && -+ x86_cpu_has_sse42 && -+ x86_cpu_has_pclmulqdq; -+} -+#else -+#include -+#include -+ -+static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, -+ PVOID param, -+ PVOID *context); -+static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; -+ -+void x86_check_features(void) -+{ -+ InitOnceExecuteOnce(&cpu_check_inited_once, _x86_check_features, -+ NULL, NULL); -+} -+ -+static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, -+ PVOID param, -+ PVOID *context) -+{ -+ int x86_cpu_has_sse2; -+ int x86_cpu_has_sse42; -+ int x86_cpu_has_pclmulqdq; -+ int regs[4]; -+ -+ __cpuid(regs, 1); -+ -+ x86_cpu_has_sse2 = regs[3] & 0x4000000; -+ x86_cpu_has_sse42= regs[2] & 0x100000; -+ x86_cpu_has_pclmulqdq = regs[2] & 0x2; -+ -+ x86_cpu_enable_simd = x86_cpu_has_sse2 && -+ x86_cpu_has_sse42 && -+ x86_cpu_has_pclmulqdq; -+ return TRUE; -+} -+#endif /* _MSC_VER */ -diff --git a/x86.h b/x86.h -new file mode 100644 -index 000000000000..ebcf10ab09d2 ---- /dev/null -+++ b/x86.h -@@ -0,0 +1,15 @@ -+/* x86.h -- check for x86 CPU features -+* Copyright (C) 2013 Intel Corporation Jim Kukunas -+* For conditions of distribution and use, see copyright notice in zlib.h -+*/ -+ -+#ifndef X86_H -+#define X86_H -+ -+#include "zlib.h" -+ -+extern int x86_cpu_enable_simd; -+ -+void x86_check_features(void); -+ -+#endif /* X86_H */ -diff --git a/zutil.h b/zutil.h -index 80375b8b6109..4425bcf75eb3 100644 ---- a/zutil.h -+++ b/zutil.h -@@ -283,4 +283,10 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ - #define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ - (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) - -+#ifdef _MSC_VER -+#define zalign(x) __declspec(align(x)) -+#else -+#define zalign(x) __attribute__((aligned((x)))) -+#endif -+ - #endif /* ZUTIL_H */ diff --git a/zlib/simd_stub.c b/zlib/simd_stub.c deleted file mode 100644 index c6d460514..000000000 --- a/zlib/simd_stub.c +++ /dev/null @@ -1,35 +0,0 @@ -/* simd_stub.c -- stub implementations -* Copyright (C) 2014 Intel Corporation -* For conditions of distribution and use, see copyright notice in zlib.h -*/ -#include - -#include "deflate.h" -#include "x86.h" - -int ZLIB_INTERNAL x86_cpu_enable_simd = 0; - -void ZLIB_INTERNAL crc_fold_init(deflate_state *const s) { - assert(0); -} - -void ZLIB_INTERNAL crc_fold_copy(deflate_state *const s, - unsigned char *dst, - const unsigned char *src, - long len) { - assert(0); -} - -unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) { - assert(0); - return 0; -} - -void ZLIB_INTERNAL fill_window_sse(deflate_state *s) -{ - assert(0); -} - -void x86_check_features(void) -{ -} diff --git a/zlib/x86.c b/zlib/x86.c deleted file mode 100644 index 3a018d776..000000000 --- a/zlib/x86.c +++ /dev/null @@ -1,94 +0,0 @@ -#ifndef ARCH_ARM -/* - * x86 feature check - * - * Copyright (C) 2013 Intel Corporation. All rights reserved. - * Author: - * Jim Kukunas - * - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#include "x86.h" -#include "zutil.h" - -int ZLIB_INTERNAL x86_cpu_enable_simd = 0; - -#ifndef _MSC_VER -#include - -pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; -static void _x86_check_features(void); - -void x86_check_features(void) -{ - pthread_once(&cpu_check_inited_once, _x86_check_features); -} - -static void _x86_check_features(void) -{ - int x86_cpu_has_sse2; - int x86_cpu_has_sse42; - int x86_cpu_has_pclmulqdq; - unsigned eax, ebx, ecx, edx; - - eax = 1; -#ifdef __i386__ - __asm__ __volatile__ ( - "xchg %%ebx, %1\n\t" - "cpuid\n\t" - "xchg %1, %%ebx\n\t" - : "+a" (eax), "=S" (ebx), "=c" (ecx), "=d" (edx) - ); -#else - __asm__ __volatile__ ( - "cpuid\n\t" - : "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) - ); -#endif /* (__i386__) */ - - x86_cpu_has_sse2 = edx & 0x4000000; - x86_cpu_has_sse42 = ecx & 0x100000; - x86_cpu_has_pclmulqdq = ecx & 0x2; - - x86_cpu_enable_simd = x86_cpu_has_sse2 && - x86_cpu_has_sse42 && - x86_cpu_has_pclmulqdq; -} -#else -#include -#include - -static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, - PVOID param, - PVOID *context); -static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; - -void x86_check_features(void) -{ - InitOnceExecuteOnce(&cpu_check_inited_once, _x86_check_features, - NULL, NULL); -} - -static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, - PVOID param, - PVOID *context) -{ - int x86_cpu_has_sse2; - int x86_cpu_has_sse42; - int x86_cpu_has_pclmulqdq; - int regs[4]; - - __cpuid(regs, 1); - - x86_cpu_has_sse2 = regs[3] & 0x4000000; - x86_cpu_has_sse42= regs[2] & 0x100000; - x86_cpu_has_pclmulqdq = regs[2] & 0x2; - - x86_cpu_enable_simd = x86_cpu_has_sse2 && - x86_cpu_has_sse42 && - x86_cpu_has_pclmulqdq; - return TRUE; -} -#endif /* _MSC_VER */ -#endif diff --git a/zlib/x86.h b/zlib/x86.h deleted file mode 100644 index ebcf10ab0..000000000 --- a/zlib/x86.h +++ /dev/null @@ -1,15 +0,0 @@ -/* x86.h -- check for x86 CPU features -* Copyright (C) 2013 Intel Corporation Jim Kukunas -* For conditions of distribution and use, see copyright notice in zlib.h -*/ - -#ifndef X86_H -#define X86_H - -#include "zlib.h" - -extern int x86_cpu_enable_simd; - -void x86_check_features(void); - -#endif /* X86_H */ diff --git a/zlib/zutil.h b/zlib/zutil.h index 4425bcf75..80375b8b6 100644 --- a/zlib/zutil.h +++ b/zlib/zutil.h @@ -283,10 +283,4 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) -#ifdef _MSC_VER -#define zalign(x) __declspec(align(x)) -#else -#define zalign(x) __attribute__((aligned((x)))) -#endif - #endif /* ZUTIL_H */