Host with coreclr linked in (#36847)

* make all-inclusive coreclr_static.lib and link into singlefilehost

* Disable superhost linking on windows for now (issues with mixed Debug-on-Release builds)

* shim coreclr policy between singlefile and standalone hosts

* enable exports on singlefilehost executable

* export __progname, perhaps it will make FreeBSD happy.

* add environ too

* mergeable wks and jit

* ignore empty clr_path in single_file_bundle

* hide R2R specific assert behind R2R ifdef

* produce separate obj files from platform specific asm files for mergeable WKS - to make CMake happy

* remove libcoreclr and libclrjit from   netcoreapp/pkg

* temporary disable setting up GS cookie on OSX superhost

* renamed clrjit_static folder   dll->static

* extract JIT dllmain.cpp and eliminate clrjit_mergeable

* make coreclr_static dependent on def/exports files instead of coreclr.

* reducing double-build in cee_wks to just few files. Most of cee_wks is built once as an obj library and reused.

* bring back cee_wks_mergeable and push shared object lib stuff lower to cee_wks_core

* A comment about FreeBSD-specific workaround.

* move finding unwind libs to a CMake function

* since we have files compiled differently between embedded and standalone coreclr, we do not need EMBEDDED_RUNTIME marker.

* renamed host_resolver_t.h and host_resolver_t.cpp to not have "_t"

* path separating slash when forming CORECLR_STATIC_LIB_LOCATION

Author: VSadov

eng/native/functions.cmake

@@ -68,21 +68,46 @@ endfunction(get_include_directories)
 function(get_include_directories_asm IncludeDirectories)
     get_directory_property(dirs INCLUDE_DIRECTORIES)
 
-    if (CLR_CMAKE_HOST_ARCH_ARM AND WIN32)
-        list(APPEND INC_DIRECTORIES "-I ")
-    endif()
-
     foreach(dir IN LISTS dirs)
-      if (CLR_CMAKE_HOST_ARCH_ARM AND WIN32)
-        list(APPEND INC_DIRECTORIES ${dir};)
-      else()
-        list(APPEND INC_DIRECTORIES -I${dir})
-      endif()
+      list(APPEND INC_DIRECTORIES -I${dir};)
     endforeach()
 
     set(${IncludeDirectories} ${INC_DIRECTORIES} PARENT_SCOPE)
 endfunction(get_include_directories_asm)
 
+# Finds and returns unwind libs
+function(find_unwind_libs UnwindLibs)
+    if(CLR_CMAKE_HOST_ARCH_ARM)
+      find_library(UNWIND_ARCH NAMES unwind-arm)
+    endif()
+
+    if(CLR_CMAKE_HOST_ARCH_ARM64)
+      find_library(UNWIND_ARCH NAMES unwind-aarch64)
+    endif()
+
+    if(CLR_CMAKE_HOST_ARCH_AMD64)
+      find_library(UNWIND_ARCH NAMES unwind-x86_64)
+    endif()
+
+    if(NOT UNWIND_ARCH STREQUAL UNWIND_ARCH-NOTFOUND)
+       set(UNWIND_LIBS ${UNWIND_ARCH})
+    endif()
+
+    find_library(UNWIND_GENERIC NAMES unwind-generic)
+
+    if(NOT UNWIND_GENERIC STREQUAL UNWIND_GENERIC-NOTFOUND)
+      set(UNWIND_LIBS ${UNWIND_LIBS} ${UNWIND_GENERIC})
+    endif()
+
+    find_library(UNWIND NAMES unwind)
+
+    if(UNWIND STREQUAL UNWIND-NOTFOUND)
+      message(FATAL_ERROR "Cannot find libunwind. Try installing libunwind8-dev or libunwind-devel.")
+    endif()
+
+    set(${UnwindLibs} ${UNWIND_LIBS} ${UNWIND} PARENT_SCOPE)
+endfunction(find_unwind_libs)
+
 # Set the passed in RetSources variable to the list of sources with added current source directory
 # to form absolute paths.
 # The parameters after the RetSources are the input files.
@@ -118,10 +143,10 @@ function(preprocess_file inputFilename outputFilename)
                               PROPERTIES GENERATED TRUE)
 endfunction()
 
-# preprocess_compile_asm(ASM_FILES file1 [file2 ...] OUTPUT_OBJECTS [variableName])
+# preprocess_compile_asm(TARGET target ASM_FILES file1 [file2 ...] OUTPUT_OBJECTS [variableName])
 function(preprocess_compile_asm)
   set(options "")
-  set(oneValueArgs OUTPUT_OBJECTS)
+  set(oneValueArgs TARGET OUTPUT_OBJECTS)
   set(multiValueArgs ASM_FILES)
   cmake_parse_arguments(PARSE_ARGV 0 COMPILE_ASM "${options}" "${oneValueArgs}" "${multiValueArgs}")
 
@@ -135,28 +160,21 @@ function(preprocess_compile_asm)
     file(TO_CMAKE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${name}.asm" ASM_PREPROCESSED_FILE)
     preprocess_file(${ASM_FILE} ${ASM_PREPROCESSED_FILE})
 
-    # We do not pass any defines since we have already done pre-processing above
-    set (ASM_CMDLINE "-o ${CMAKE_CURRENT_BINARY_DIR}/${name}.obj ${ASM_PREPROCESSED_FILE}")
-
-    # Generate the batch file that will invoke the assembler
-    file(TO_CMAKE_PATH "${CMAKE_CURRENT_BINARY_DIR}/runasm_${name}.cmd" ASM_SCRIPT_FILE)
-
-    file(GENERATE OUTPUT "${ASM_SCRIPT_FILE}"
-        CONTENT "\"${CMAKE_ASM_MASM_COMPILER}\" -g ${ASM_INCLUDE_DIRECTORIES} ${ASM_CMDLINE}")
-
-    message("Generated  - ${ASM_SCRIPT_FILE}")
+    # Produce object file where CMake would store .obj files for an OBJECT library.
+    # ex: artifacts\obj\coreclr\Windows_NT.arm64.Debug\src\vm\wks\cee_wks.dir\Debug\AsmHelpers.obj
+    set (OBJ_FILE "${CMAKE_CURRENT_BINARY_DIR}/${COMPILE_ASM_TARGET}.dir/${CMAKE_CFG_INTDIR}/${name}.obj")
 
     # Need to compile asm file using custom command as include directories are not provided to asm compiler
-    add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${name}.obj
-                        COMMAND ${ASM_SCRIPT_FILE}
+    add_custom_command(OUTPUT ${OBJ_FILE}
+                        COMMAND "${CMAKE_ASM_MASM_COMPILER}" -g ${ASM_INCLUDE_DIRECTORIES} -o ${OBJ_FILE} ${ASM_PREPROCESSED_FILE}
                         DEPENDS ${ASM_PREPROCESSED_FILE}
-                        COMMENT "Assembling ${ASM_PREPROCESSED_FILE} - ${ASM_SCRIPT_FILE}")
+                        COMMENT "Assembling ${ASM_PREPROCESSED_FILE} ---> \"${CMAKE_ASM_MASM_COMPILER}\" -g ${ASM_INCLUDE_DIRECTORIES} -o ${OBJ_FILE} ${ASM_PREPROCESSED_FILE}")
 
     # mark obj as source that does not require compile
-    set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/${name}.obj PROPERTIES EXTERNAL_OBJECT TRUE)
+    set_source_files_properties(${OBJ_FILE} PROPERTIES EXTERNAL_OBJECT TRUE)
 
     # Add the generated OBJ in the dependency list so that it gets consumed during linkage
-    list(APPEND ASSEMBLED_OBJECTS ${CMAKE_CURRENT_BINARY_DIR}/${name}.obj)
+    list(APPEND ASSEMBLED_OBJECTS ${OBJ_FILE})
   endforeach()
 
   set(${COMPILE_ASM_OUTPUT_OBJECTS} ${ASSEMBLED_OBJECTS} PARENT_SCOPE)

src/coreclr/src/binder/CMakeLists.txt

@@ -83,7 +83,7 @@ convert_to_absolute_path(BINDER_SOURCES ${BINDER_SOURCES})
 convert_to_absolute_path(BINDER_CROSSGEN_SOURCES ${BINDER_CROSSGEN_SOURCES})
 
 add_library_clr(v3binder
-    STATIC
+    OBJECT
     ${BINDER_SOURCES}
 )
 add_dependencies(v3binder eventing_headers)

src/coreclr/src/classlibnative/bcltype/CMakeLists.txt

@@ -11,7 +11,7 @@ set(BCLTYPE_SOURCES
 )
 
 add_library_clr(bcltype
-    STATIC
+    OBJECT
     ${BCLTYPE_SOURCES}
 )
 

src/coreclr/src/classlibnative/float/CMakeLists.txt

@@ -8,7 +8,7 @@ set(FLOAT_SOURCES
 )
 
 add_library_clr(comfloat_wks
-    STATIC
+    OBJECT
     ${FLOAT_SOURCES}
 )
 

src/coreclr/src/debug/debug-pal/CMakeLists.txt

@@ -34,4 +34,4 @@ if(CLR_CMAKE_HOST_UNIX)
 
 endif(CLR_CMAKE_HOST_UNIX)
 
-_add_library(debug-pal STATIC ${TWO_WAY_PIPE_SOURCES})
+_add_library(debug-pal OBJECT ${TWO_WAY_PIPE_SOURCES})

src/coreclr/src/debug/di/CMakeLists.txt

@@ -59,7 +59,7 @@ if(CLR_CMAKE_HOST_WIN32)
 
     if ((CLR_CMAKE_TARGET_ARCH_ARM OR CLR_CMAKE_TARGET_ARCH_ARM64) AND NOT DEFINED CLR_CROSS_COMPONENTS_BUILD)
         convert_to_absolute_path(CORDBDI_SOURCES_ASM_FILE ${CORDBDI_SOURCES_ASM_FILE})
-        preprocess_compile_asm(ASM_FILES ${CORDBDI_SOURCES_ASM_FILE} OUTPUT_OBJECTS CORDBDI_SOURCES_ASM_FILE)
+        preprocess_compile_asm(TARGET cordbdi ASM_FILES ${CORDBDI_SOURCES_ASM_FILE} OUTPUT_OBJECTS CORDBDI_SOURCES_ASM_FILE)
     endif()
 elseif(CLR_CMAKE_HOST_UNIX)
 

src/coreclr/src/debug/ee/wks/CMakeLists.txt

@@ -9,9 +9,9 @@ if (CLR_CMAKE_TARGET_WIN32)
 
   if(CLR_CMAKE_HOST_ARCH_ARM OR CLR_CMAKE_HOST_ARCH_ARM64)
 
-    preprocess_compile_asm(ASM_FILES ${ASM_FILE} OUTPUT_OBJECTS ASM_OBJECTS)
+    preprocess_compile_asm(TARGET cordbee_wks ASM_FILES ${ASM_FILE} OUTPUT_OBJECTS ASM_OBJECTS)
 
-    add_library_clr(cordbee_wks ${CORDBEE_SOURCES_WKS} ${ASM_OBJECTS})
+    add_library_clr(cordbee_wks OBJECT ${CORDBEE_SOURCES_WKS} ${ASM_FILE} ${ASM_OBJECTS})
 
   else ()
 
@@ -23,14 +23,14 @@ if (CLR_CMAKE_TARGET_WIN32)
 
     set_source_files_properties(${ASM_FILE} PROPERTIES COMPILE_OPTIONS "${ASM_OPTIONS}")
 
-    add_library_clr(cordbee_wks ${CORDBEE_SOURCES_WKS} ${ASM_FILE})
+    add_library_clr(cordbee_wks OBJECT ${CORDBEE_SOURCES_WKS} ${ASM_FILE})
 
   endif()
 
 else ()
 
   if(CLR_CMAKE_HOST_ARCH_AMD64 OR CLR_CMAKE_HOST_ARCH_ARM OR CLR_CMAKE_HOST_ARCH_ARM64 OR CLR_CMAKE_HOST_ARCH_I386)
-    add_library_clr(cordbee_wks ${CORDBEE_SOURCES_WKS} ../${ARCH_SOURCES_DIR}/dbghelpers.S)
+    add_library_clr(cordbee_wks OBJECT ${CORDBEE_SOURCES_WKS} ../${ARCH_SOURCES_DIR}/dbghelpers.S)
   else()
     message(FATAL_ERROR "Unknown platform")
   endif()

src/coreclr/src/debug/ildbsymlib/CMakeLists.txt

@@ -10,5 +10,5 @@ set( ILDBSYMLIB_SOURCES
   symwrite.cpp
 )
 
-add_library_clr(ildbsymlib ${ILDBSYMLIB_SOURCES})
+add_library_clr(ildbsymlib OBJECT ${ILDBSYMLIB_SOURCES})
 

src/coreclr/src/dlls/mscordac/mscordac_unixexports.src

@@ -34,6 +34,8 @@ nativeStringResourceTable_mscorrc
 #PAL_GetPALDirectoryW
 #PAL_get_stdout
 #PAL_get_stderr
+#PAL_GetApplicationGroupId
+#PAL_GetTransportName
 #PAL_GetCurrentThread
 #PAL_GetCpuLimit
 #PAL_GetNativeExceptionHolderHead

src/coreclr/src/dlls/mscoree/coreclr/CMakeLists.txt

@@ -5,7 +5,8 @@ if (CLR_CMAKE_TARGET_WIN32)
 endif (CLR_CMAKE_TARGET_WIN32)
 
 if (CLR_CMAKE_HOST_WIN32)
-    preprocess_file(${DEF_SOURCES} ${CMAKE_CURRENT_BINARY_DIR}/coreclr.def)
+    set (DEF_FILE  ${CMAKE_CURRENT_BINARY_DIR}/coreclr.def)
+    preprocess_file(${DEF_SOURCES} ${DEF_FILE})
 
     list(APPEND CLR_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/coreclr.def)
 
@@ -65,8 +66,18 @@ add_library_clr(coreclr
     ${CLR_SOURCES}
 )
 
+add_library_clr(coreclr_static
+    STATIC
+    ${CLR_SOURCES}
+)
+
 add_custom_target(coreclr_exports DEPENDS ${EXPORTS_FILE})
+add_custom_target(coreclr_def DEPENDS ${DEF_FILE})
+
+add_dependencies(coreclr coreclr_def)
 add_dependencies(coreclr coreclr_exports)
+add_dependencies(coreclr_static coreclr_def)
+add_dependencies(coreclr_static coreclr_exports)
 
 set_property(TARGET coreclr APPEND_STRING PROPERTY LINK_FLAGS ${EXPORTS_LINKER_OPTION})
 set_property(TARGET coreclr APPEND_STRING PROPERTY LINK_DEPENDS ${EXPORTS_FILE})
@@ -75,10 +86,6 @@ if (CLR_CMAKE_HOST_UNIX)
     set(LIB_UNWINDER unwinder_wks)
 endif (CLR_CMAKE_HOST_UNIX)
 
-if(FEATURE_MERGE_JIT_AND_ENGINE)
-    set(CLRJIT_STATIC clrjit_static)
-endif(FEATURE_MERGE_JIT_AND_ENGINE)
-
 # IMPORTANT! Please do not rearrange the order of the libraries. The linker on Linux is
 # order dependent and changing the order can result in undefined symbols in the shared
 # library.
@@ -88,7 +95,6 @@ set(CORECLR_LIBRARIES
     cordbee_wks
     debug-pal
     ${LIB_UNWINDER}
-    cee_wks
     v3binder
     ${END_LIBRARY_GROUP} # End group of libraries that have circular references
     mdcompiler_wks
@@ -97,7 +103,6 @@ set(CORECLR_LIBRARIES
     mdhotdata_full
     bcltype
     ceefgen
-    ${CLRJIT_STATIC}
     comfloat_wks
     corguids
     gcinfo
@@ -159,7 +164,12 @@ if(FEATURE_EVENT_TRACE)
     endif(CLR_CMAKE_HOST_UNIX)
 endif(FEATURE_EVENT_TRACE)
 
-target_link_libraries(coreclr ${CORECLR_LIBRARIES})
+if(FEATURE_MERGE_JIT_AND_ENGINE)
+    set(CLRJIT_STATIC clrjit_static)
+endif(FEATURE_MERGE_JIT_AND_ENGINE)
+
+target_link_libraries(coreclr PUBLIC ${CORECLR_LIBRARIES} ${CLRJIT_STATIC} cee_wks cee_wks_core)
+target_link_libraries(coreclr_static PUBLIC ${CORECLR_LIBRARIES} clrjit_static cee_wks_mergeable cee_wks_core)
 
 # Create the runtime module index header file containing the coreclr build id
 # for xplat and the timestamp/size on Windows.
@@ -218,5 +228,8 @@ endif(CLR_CMAKE_TARGET_WIN32)
 # add the install targets
 install_clr(TARGETS coreclr ADDITIONAL_DESTINATION sharedFramework)
 
+# publish coreclr_static lib
+_install(TARGETS coreclr_static DESTINATION lib)
+
 # Enable profile guided optimization
 add_pgo(coreclr)

src/coreclr/src/dlls/mscoree/unixinterface.cpp

@@ -25,6 +25,9 @@
 // Holder for const wide strings
 typedef NewArrayHolder<const WCHAR> ConstWStringHolder;
 
+// Specifies whether coreclr is embedded or standalone
+extern bool g_coreclr_embedded;
+
 // Holder for array of wide strings
 class ConstWStringArrayHolder : public NewArrayHolder<LPCWSTR>
 {
@@ -171,8 +174,21 @@ int coreclr_initialize(
             unsigned int* domainId)
 {
     HRESULT hr;
+
+    LPCWSTR* propertyKeysW;
+    LPCWSTR* propertyValuesW;
+    BundleProbe* bundleProbe = nullptr;
+
+    ConvertConfigPropertiesToUnicode(
+        propertyKeys,
+        propertyValues,
+        propertyCount,
+        &propertyKeysW,
+        &propertyValuesW,
+        &bundleProbe);
+
 #ifdef TARGET_UNIX
-    DWORD error = PAL_InitializeCoreCLR(exePath);
+    DWORD error = PAL_InitializeCoreCLR(exePath, g_coreclr_embedded);
     hr = HRESULT_FROM_WIN32(error);
 
     // If PAL initialization failed, then we should return right away and avoid
@@ -190,18 +206,6 @@ int coreclr_initialize(
 
     ConstWStringHolder appDomainFriendlyNameW = StringToUnicode(appDomainFriendlyName);
 
-    LPCWSTR* propertyKeysW;
-    LPCWSTR* propertyValuesW;
-    BundleProbe* bundleProbe = nullptr;
-
-    ConvertConfigPropertiesToUnicode(
-        propertyKeys,
-        propertyValues,
-        propertyCount,
-        &propertyKeysW,
-        &propertyValuesW,
-        &bundleProbe);
-
     if (bundleProbe != nullptr)
     {
         static Bundle bundle(StringToUnicode(exePath), bundleProbe);

src/coreclr/src/dlls/mscorrc/CMakeLists.txt

@@ -19,7 +19,7 @@ if(CLR_CMAKE_HOST_WIN32)
 else()
     build_resources(${CMAKE_CURRENT_SOURCE_DIR}/include.rc mscorrc TARGET_CPP_FILE)
 
-    add_library_clr(mscorrc STATIC
+    add_library_clr(mscorrc OBJECT
         ${TARGET_CPP_FILE}
     )
 endif(CLR_CMAKE_HOST_WIN32)

src/coreclr/src/gcinfo/CMakeLists.txt

@@ -17,7 +17,7 @@ endif(CLR_CMAKE_TARGET_ARCH_I386)
 convert_to_absolute_path(GCINFO_SOURCES ${GCINFO_SOURCES})
 
 add_library_clr(gcinfo
-    STATIC
+    OBJECT
     ${GCINFO_SOURCES}
 )
 

src/coreclr/src/inc/CMakeLists.txt

@@ -58,7 +58,7 @@ if(FEATURE_JIT_PITCHING)
 endif(FEATURE_JIT_PITCHING)
 
 # Compile *_i.cpp to lib
-_add_library(corguids ${CORGUIDS_SOURCES})
+_add_library(corguids OBJECT ${CORGUIDS_SOURCES})
 
 # Binplace the inc files for packaging later.
 

src/coreclr/src/interop/CMakeLists.txt

@@ -31,6 +31,6 @@ endif(WIN32)
 convert_to_absolute_path(INTEROP_SOURCES ${INTEROP_SOURCES})
 
 add_library_clr(interop
-    STATIC
+    OBJECT
     ${INTEROP_SOURCES}
 )