diff --git a/CMakeLists.txt b/CMakeLists.txt
index 53a08a0759b..428f6d63979 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -10,31 +10,32 @@ if(NOT CHAKRACORE_BUILD_SH)
     option(INTL_ICU "Enable Intl" ON)
     option(EMBED_ICU "Build ICU within ChakraCore build" OFF)
     set(ICU_INCLUDE_PATH "" CACHE STRING "libicu iclude path")
-endif(NOT CHAKRACORE_BUILD_SH)
+else(NOT CHAKRACORE_BUILD_SH)
 
-# Keep CMake from caching static/shared library
-# option. Otherwise, CMake fails to update cached
-# references
+    # Keep CMake from caching static/shared library
+    # option. Otherwise, CMake fails to update cached
+    # references
 
-# todo: create a sub cmake file to take care of _SH uncaching...
-if(SHARED_LIBRARY_SH)
+    # todo: create a sub cmake file to take care of _SH uncaching...
+    if(SHARED_LIBRARY_SH)
     unset(SHARED_LIBRARY_SH CACHE)
     unset(STATIC_LIBRARY_SH CACHE)
-    unset(STATIC_LIBRARY CACHE)
-    set(SHARED_LIBRARY 1)
-endif()
+        unset(STATIC_LIBRARY CACHE)
+        set(SHARED_LIBRARY 1)
+    endif()
 
-if(STATIC_LIBRARY_SH)
-    unset(SHARED_LIBRARY_SH CACHE)
-    unset(STATIC_LIBRARY_SH CACHE)
-    unset(SHARED_LIBRARY CACHE)
-    set(STATIC_LIBRARY 1)
-endif()
+    if(STATIC_LIBRARY_SH)
+        unset(SHARED_LIBRARY_SH CACHE)
+        unset(STATIC_LIBRARY_SH CACHE)
+        unset(SHARED_LIBRARY CACHE)
+        set(STATIC_LIBRARY 1)
+    endif()
 
-if(LIBS_ONLY_BUILD_SH)
-    unset(LIBS_ONLY_BUILD_SH CACHE)
-    set(CC_LIBS_ONLY_BUILD 1)
-endif()
+    if(LIBS_ONLY_BUILD_SH)
+        unset(LIBS_ONLY_BUILD_SH CACHE)
+        set(CC_LIBS_ONLY_BUILD 1)
+    endif()
+endif(NOT CHAKRACORE_BUILD_SH)
 
 if(CC_USES_SYSTEM_ARCH_SH OR NOT CHAKRACORE_BUILD_SH)
     if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 26925290ddc..7c7695502aa 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -3,19 +3,52 @@ trigger:
 - release/*
 
 jobs:
-  - job: Linux
-    timeoutInMinutes: 0
-    pool:
-      vmImage: 'ubuntu-latest'
+  - job: CMake
+    timeoutInMinutes: 120
     strategy:
       matrix:
-        debug:
+        Linux.Debug:
+          image_name: 'ubuntu-latest'
+          deps: 'sudo apt-get install -y ninja-build clang libicu-dev'
+          build_type: 'Debug'
+          test_target: 'check'
+          libtype_flag: ''
+        Linux.ReleaseWithDebug:
+          image_name: 'ubuntu-latest'
+          deps: 'sudo apt-get install -y ninja-build clang libicu-dev'
+          build_type: 'RelWithDebInfo'
+          test_target: 'check'
+          libtype_flag: ''
+        Linux.Release:
+          image_name: 'ubuntu-latest'
+          deps: 'sudo apt-get install -y ninja-build clang libicu-dev'
+          build_type: 'Release'
+          test_target: 'all'
+          libtype_flag: ''
+        OSX.Debug:
+          image_name: 'macOS-latest'
+          deps: 'brew install ninja icu4c'
           build_type: 'Debug'
-        release_with_debug:
+          test_target: 'check'
+          libtype_flag: '-DSTATIC_LIBRARY=ON'
+        OSX.ReleaseWithDebug:
+          image_name: 'macOS-latest'
+          deps: 'brew install ninja icu4c'
           build_type: 'RelWithDebInfo'
+          test_target: 'check'
+          libtype_flag: ''
+        OSX.Release:
+          image_name: 'macOS-latest'
+          deps: 'brew install ninja icu4c'
+          build_type: 'Release'
+          test_target: 'all'
+          libtype_flag: ''
 
+    pool:
+      vmImage: $(image_name)
+      
     steps:
-    - script: sudo apt-get install -y ninja-build clang libicu-dev
+    - script: $(deps)
       displayName: 'Install dependencies'
   
     - script: |
@@ -24,13 +57,15 @@ jobs:
 
     - script: |
         cd build
-        cmake -GNinja -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang ..
+        cmake -GNinja -DCMAKE_BUILD_TYPE=$BUILD_TYPE $LIBTYPE -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang ..
       displayName: CMake
       env:
         BUILD_TYPE: $(build_type)
+        LIBTYPE: $(libtype_flag)
 
     - script: |
         cd build
-        ninja
-        ninja check
+        ninja $TARGET
       displayName: 'Build and test'
+      env:
+        TARGET: $(test_target)
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 932e8fc5bef..05aba09b1eb 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -15,3 +15,7 @@ add_custom_target(check
     DEPENDS ch
     )
 
+if (NOT STATIC_LIBRARY)
+    add_dependencies(check ChakraCore)
+endif()
+