First iteration of vcpkg build process

.gitignore

@@ -400,5 +400,5 @@ zlib/contrib/vstudio/vc14/x64/
 *.xcworkspacedata
 *.xcuserstate
 
-# Visual Code
+# Visual Studio Code-generated files
 build/.cmake/api/v1/query/client-vscode/query.json

.gitmodules

@@ -11,3 +11,7 @@
 	path = third_party/krabsetw
 	url = https://github.com/Microsoft/krabsetw
 	branch = master
+
+[submodule "third_party/googletest"]
+	path = third_party/googletest
+	url = https://github.com/google/googletest

build-gtest.sh

@@ -10,19 +10,24 @@ fi
 
 cd `dirname $0`
 
-# Use latest Google Test
+GTEST_PATH=googletest
+
+# Use latest Google Test for Ubuntu 20.04
+# TODO: switch all OS builds to Google Test located in third_party/googletest submodule
 if [ -f /etc/os-release ]; then
   source /etc/os-release
   # Use new Google Test on latest Ubuntu 20.04 : old one no longer compiles on 20
   if [ "$VERSION_ID" == "20.04" ]; then
     echo Running on Ubuntu 20.04
-    echo Clone googletest from google/googletest:master ...
-    rm -rf googletest
-    git clone https://github.com/google/googletest
+    GTEST_PATH=third_party/googletest
+    if [ ! "$(ls -A $GTEST_PATH)" ]; then      
+      echo Clone googletest from google/googletest:master ...
+      git clone https://github.com/google/googletest $GTEST_PATH
+    fi
   fi
 fi
 
-cd googletest
+pushd $GTEST_PATH
 set -evx
 env | sort
 rm -rf build
@@ -36,5 +41,6 @@ cmake -Dgtest_build_samples=OFF \
       -DBUILD_IOS=$IOS_BUILD \
       ..
 make
+popd
 # CTEST_OUTPUT_ON_FAILURE=1 make test
 # make install

build.sh

@@ -30,9 +30,54 @@ export NOROOT=true
 fi
 
 if [ "$1" == "release" ] || [ "$2" == "release" ] || [ "$3" == "release" ]; then
-BUILD_TYPE="Release"
+  BUILD_TYPE="Release"
 else
-BUILD_TYPE="Debug"
+  BUILD_TYPE="Debug"
+fi
+
+LINK_TYPE=
+CMAKE_OPTS="${CMAKE_OPTS:-DBUILD_SHARED_LIBS=OFF}"
+while getopts "h?vl:D:" opt; do
+    case "$opt" in
+    h|\?)
+        echo "Usage: build.sh [clean] [arm64|universal] [CUSTOM_CMAKE_CXX_FLAGS=x] [noroot] [release] [-h|-?] [-l (static|shared)] [-D CMAKE_OPTION] [-v]"
+        echo "                                                                                         "
+        echo "options:                                                                                 "
+        echo "                                                                                         "
+        echo "Positional options (1st three arguments):                                                "
+        echo "[clean]                  - perform clean build                                           "
+        echo "[arm64|universal]        - Apple platform build type. Not applicable to other OS.        "
+        echo "[CUSTOM_CMAKE_CXX_FLAGS] - custom CXX compiler flags                                     "
+        echo "[noroot]                 - custom CXX compiler flags                                     "
+        echo "[release]                - build for Release                                             "
+        echo "                                                                                         "
+        echo "Additional parameters:                                                                   "
+        echo " -h | -?                 - this help.                                                    "
+        echo " -l [static|shared]      - build static (default) or shared library.                     "
+        echo " -D [CMAKE_OPTION]       - additional options to pass to cmake. Could be multiple.       "
+        echo " -v                      - increase build verbosity (reserved for future use)            "
+        echo "                                                                                         "
+        echo "Environment variables:                                                                   "
+        echo "CMAKE_OPTS               - any additional cmake options.                                 "
+        echo "GIT_PULL_TOKEN           - authorization token for Microsoft-proprietary modules.        "
+        echo "Plus any other environment variables respected by CMake build system.                    "
+        exit 0
+        ;;
+    :)  echo "Invalid option: $OPTARG requires an argument" 1>&2
+        exit 0
+        ;;
+    v)  verbose=1
+        ;;
+    D)  CMAKE_OPTS="$CMAKE_OPTS -D$OPTARG"
+        ;;
+    l)  LINK_TYPE=$OPTARG
+        ;;
+    esac
+done
+shift $((OPTIND -1))
+
+if [ "$LINK_TYPE" == "shared" ]; then
+  CMAKE_OPTS="$CMAKE_OPTS -DBUILD_SHARED_LIBS=ON"
 fi
 
 if [ "$1" == "arm64" ] || [ "$2" == "arm64" ] || [ "$3" == "arm64" ]; then
@@ -62,21 +107,21 @@ export MACOSX_DEPLOYMENT_TARGET=10.10
 FILE=.buildtools
 OS_NAME=`uname -a`
 if [ ! -f $FILE ]; then
-case "$OS_NAME" in
- *Darwin*) tools/setup-buildtools-apple.sh ;;
- *Linux*)  [[ -z "$NOROOT" ]] && sudo tools/setup-buildtools.sh || echo "No root: skipping build tools installation." ;;
- *)        echo "WARNING: unsupported OS $OS_NAME , skipping build tools installation.."
-esac
-# Assume that the build tools have been successfully installed
-echo > $FILE
+  case "$OS_NAME" in
+    *Darwin*) tools/setup-buildtools-apple.sh ;;
+    *Linux*)  [[ -z "$NOROOT" ]] && sudo tools/setup-buildtools.sh || echo "No root: skipping build tools installation." ;;
+    *)        echo "WARNING: unsupported OS $OS_NAME , skipping build tools installation.."
+  esac
+  # Assume that the build tools have been successfully installed
+  echo > $FILE
 fi
 
 if [ -f /usr/bin/gcc ]; then
-echo "gcc   version: `gcc --version`"
+  echo "gcc   version: `gcc --version`"
 fi
 
 if [ -f /usr/bin/clang ]; then
-echo "clang version: `clang --version`"
+  echo "clang version: `clang --version`"
 fi
 
 # Skip Version.hpp changes
@@ -91,20 +136,19 @@ CMAKE_PACKAGE_TYPE=tgz
 
 # .deb package
 if [ -f /usr/bin/dpkg ]; then
-export CMAKE_PACKAGE_TYPE=deb
+  export CMAKE_PACKAGE_TYPE=deb
 fi
 
 # .rpm package
 if [ -f /usr/bin/rpmbuild ]; then
-export CMAKE_PACKAGE_TYPE=rpm
+  export CMAKE_PACKAGE_TYPE=rpm
 fi
 
-
 # Fail on error
 set -e
 
-
-cmake_cmd="cmake -DMAC_ARCH=$MAC_ARCH -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_PACKAGE_TYPE=$CMAKE_PACKAGE_TYPE -DCMAKE_CXX_FLAGS="${CUSTOM_CMAKE_CXX_FLAG}" .."
+# TODO: should this be improved to verify if the platform is Apple? Right now we unconditionally pass -DMAC_ARCH even if building for Windows or Linux.
+cmake_cmd="cmake -DMAC_ARCH=$MAC_ARCH -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_PACKAGE_TYPE=$CMAKE_PACKAGE_TYPE -DCMAKE_CXX_FLAGS="${CUSTOM_CMAKE_CXX_FLAG}" $CMAKE_OPTS .."
 echo $cmake_cmd
 eval $cmake_cmd
 # TODO: strip symbols to minimize (release-only)
@@ -125,13 +169,13 @@ make package
 
 # Debian / Ubuntu / Raspbian
 if [ -f /usr/bin/dpkg ]; then
-# Install new package
-[[ -z "$NOROOT" ]] && sudo dpkg -i *.deb || echo "No root: skipping package deployment."
+  # Install new package
+  [[ -z "$NOROOT" ]] && sudo dpkg -i *.deb || echo "No root: skipping package deployment."
 fi
 
 # RedHat / CentOS
 if [ -f /usr/bin/rpmbuild ]; then
-[[ -z "$NOROOT" ]] && sudo rpm -i --force -v *.rpm || echo "No root: skipping package deployment."
+  [[ -z "$NOROOT" ]] && sudo rpm -i --force -v *.rpm || echo "No root: skipping package deployment."
 fi
 
 # Install SDK headers and lib to /usr/local
@@ -141,10 +185,10 @@ fi
 ## strip -S --strip-unneeded --remove-section=.note.gnu.gold-version --remove-section=.comment --remove-section=.note --remove-section=.note.gnu.build-id --remove-section=.note.ABI-tag out/lib/libmat.so
 
 if [ "$CMAKE_PACKAGE_TYPE" == "tgz" ]; then
-cd ..
-MATSDK_INSTALL_DIR="${MATSDK_INSTALL_DIR:-/usr/local}"
-echo "+-----------------------------------------------------------------------------------+"
-echo " This step may prompt for your sudo password to deploy SDK to $MATSDK_INSTALL_DIR  "
-echo "+-----------------------------------------------------------------------------------+"
-[[ -z "$NOROOT" ]] && sudo ./install.sh $MATSDK_INSTALL_DIR || echo "No root: skipping package deployment."
+  cd ..
+  MATSDK_INSTALL_DIR="${MATSDK_INSTALL_DIR:-/usr/local}"
+  echo "+-----------------------------------------------------------------------------------+"
+  echo " This step may prompt for your sudo password to deploy SDK to $MATSDK_INSTALL_DIR  "
+  echo "+-----------------------------------------------------------------------------------+"
+  [[ -z "$NOROOT" ]] && sudo ./install.sh $MATSDK_INSTALL_DIR || echo "No root: skipping package deployment."
 fi

docs/building-with-vcpkg.md

@@ -0,0 +1,88 @@
+# Building 1DS C++ SDK with vcpkg
+
+vcpkg is a Microsoft cross-platform open source C++ package manager. Onboarding instructions for Windows, Linux and Mac OS X [available here](https://docs.microsoft.com/en-us/cpp/build/vcpkg). This document assumes that the customer build system is already configured to use vcpkg. 1DS C++ SDK maintainers provide a build recipe, `mstelemetry` port or CONTROL file for vcpkg. Mainline vcpkg repo is refreshed to point to latest stable open source release of 1DS C++ SDK. Public build of SDK does not include private submodules. However, local port / CONTROL file included in 1DS C++ SDK git repo allows to build SDK with additional Microsoft-proprietary private submodules.
+
+## Installing open source mstelemetry package
+
+The following command can be used to install the public open source release:
+
+```
+vcpkg install mstelemetry
+```
+
+That's it! The package should be compiled for the current OS.
+
+See instructions below to build the SDK with additional Microsoft-proprietary modules.
+
+## Windows build with submodules
+
+`cmd.exe` command line prompt commands:
+
+```
+git clone --recurse-submodules https://github.com/microsoft/cpp_client_telemetry
+cd cpp_client_telemetry
+vcpkg install --head --overlay-ports=%CD%\tools\ports mstelemetry
+```
+
+## POSIX (Linux and Mac) build with submodules
+
+Shell commands:
+
+```
+git clone --recurse-submodules https://github.com/microsoft/cpp_client_telemetry
+cd cpp_client_telemetry
+vcpkg install --head --overlay-ports=`pwd`/tools/ports mstelemetry
+```
+
+## Using response files to specify dependencies
+
+vcpkg allows to consolidate parameters passed to vcpkg in a response file. All 3rd party dependencies needed for 1DS SDK can be described and installed via response file.
+
+Example for Mac:
+```
+vcpkg install @tools/ports/mstelemetry/response_file_mac.txt
+```
+
+Example for Linux:
+```
+vcpkg install @tools/ports/mstelemetry/response_file_linux.txt
+```
+
+vcpkg build log files are created in `${VCPKG_INSTALL_DIR}/buildtrees/mstelemetry/build-[err|out].log` . Review the logs in case if you encounter package installation failures.
+
+## Using triplets
+
+In order to enable custom build flags - vcpkg triplets and custom environment variables may be used. Please see [triplets instruction here](https://vcpkg.readthedocs.io/en/latest/users/triplets/). Response file for a custom build, e.g. `response_file_linux_PRODUCTNAME.txt` may specify a custom triplet. For example, custom triplet controls if the library is built as static or dynamic. Default triplets may also be overridden with [custom triplets](https://vcpkg.readthedocs.io/en/latest/examples/overlay-triplets-linux-dynamic/#overlay-triplets-example). Custom triplets specific to various products must be maintained by product teams. Product teams may optionally decide to integrate their triplets in the mainline 1DS C++ SDK repo as-needed.
+
+## Build with vcpkg dependencies
+
+This section needs to be updated with more detailed info. Default `CMakeLists.txt` in top-level directory utilize the following dependencies:
+- OS-provided `sqlite3` library.
+- OS-provided `zlib` library.
+- SDK-provided snapshot of `nlohmann-json` header-only library.
+
+It is possible to adjust the build system to use vcpkg-installed dependencies instead.
+
+### nlohmann-json
+
+The package `nlohmann-json` provides CMake targets:
+```
+    find_package(nlohmann_json CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE nlohmann_json nlohmann_json::nlohmann_json)
+```
+
+### sqlite3
+
+The package `sqlite3` provides CMake targets:
+```
+    find_package(unofficial-sqlite3 CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE unofficial::sqlite3::sqlite3)
+```
+
+### zlib
+
+The package zlib is compatible with built-in CMake targets:
+```
+    find_package(ZLIB REQUIRED)
+    target_link_libraries(main PRIVATE ZLIB::ZLIB)
+```

examples/cpp/SampleCpp/CMakeLists.txt

@@ -4,17 +4,20 @@ project(SampleCpp)
 # Uncomment for building i386 binary on x86_64 system
 #set(CMAKE_SYSTEM_PROCESSOR i386)
 
+if (NOT TARGET_ARCH)
+  set(TARGET_ARCH ${CMAKE_SYSTEM_PROCESSOR})
+endif()
+
 # For ARM / Raspberry Pi 3 cross-compile
 # set(MAT_SDK_LIB       /usr/local/lib/armv7l-linux-gnu)
 
-# Point example to SDK dirs for x86_64 Desktop
-if(EXISTS "/usr/local/lib/libmat.a")
-# Use local libmat.a
-set(MAT_SDK_LIB	/usr/local/lib/)
-else()
-# Use architecture-specific libmat.a
-set(MAT_SDK_LIB	/usr/local/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu)
+# Search for static lib first
+find_path(MAT_SDK_LIB_PATH NAMES libmat.a PATHS /usr/lib /usr/local/lib /usr/local/lib/${TARGET_ARCH}-linux-gnu)
+if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+  # Search for .dylib on Mac OS X
+  find_path(MAT_SDK_DYLIB_PATH NAMES libmat.dylib PATHS /usr/lib /usr/local/lib)
 endif()
+find_library(LIBMAT NAMES libmat.a libmat.dylib HINTS ${MAT_SDK_LIB_PATH} ${MAT_SDK_DYLIB_PATH})
 
 set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -O0 -ggdb -gdwarf-2 -std=c11")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -ggdb -gdwarf-2 -std=c++11")
@@ -30,6 +33,7 @@ include_directories( . ${MAT_SDK_INCLUDE} )
 add_executable(SampleCpp main.cpp DebugCallback.cpp demo.c)
 source_group(" " REGULAR_EXPRESSION "")
 
+# NOTE: modify this sample if necessary to use vcpkg-provided sqlite library
 # Prefer linking to more recent local sqlite3
 if(EXISTS "/usr/local/lib/libsqlite3.a")
 set (SQLITE3_LIB "/usr/local/lib/libsqlite3.a")
@@ -51,4 +55,5 @@ endif()
 #tcmalloc turned off by default
 #target_link_libraries(SampleCpp ${MAT_SDK_LIB}/libmat.a curl z ${CMAKE_THREAD_LIBS_INIT} ${SQLITE3_LIB} dl tcmalloc)
 
-target_link_libraries(SampleCpp ${MAT_SDK_LIB}/libmat.a curl z ${CMAKE_THREAD_LIBS_INIT} ${SQLITE3_LIB} ${PLATFORM_LIBS} dl)
+# TODO: use add_library to allow linking against ${LIBMAT_STATIC}
+target_link_libraries(SampleCpp ${LIBMAT} curl z ${CMAKE_THREAD_LIBS_INIT} ${SQLITE3_LIB} ${PLATFORM_LIBS} dl)

googletest/README.MSFT

@@ -1,3 +1,5 @@
 This code was obtained from https://github.com/google/googletest/pull/1029/commits/ba638689e40fc60c2e5194fc777bf30541deeff8
 
-It has been approved by CELA for Skype internal use. These bits are not used in production, for internal testing only.
+It has been approved by CELA for internal use. These bits are not used in production, for internal testing only.
+
+This code must be deprecated and removed, permanently replaced by git submodule located at `third_party/googletest`.

install.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 MATSDK_INSTALL_DIR=$1
 echo "Install SDK to $MATSDK_INSTALL_DIR"
-mkdir -p $MATSDK_INSTALL_DIR/lib/mat
-cp out/lib/libmat.a $MATSDK_INSTALL_DIR/lib
+mkdir -p $MATSDK_INSTALL_DIR/lib
+cp out/lib/libmat.* $MATSDK_INSTALL_DIR/lib
 mkdir -p $MATSDK_INSTALL_DIR/include/mat
 cp lib/include/public/* $MATSDK_INSTALL_DIR/include/mat