Full kbuild support (atop of https://github.com/minipli-oss/open-gpu-kernel-modules/tree/fixes)

Makefile

@@ -6,6 +6,19 @@
 # To install the build kernel modules: run (as root) `make modules_install`
 ###########################################################################
 
+# Experimental use of the Linux kernel's kbuild system
+USE_KBUILD ?= 0
+export USE_KBUILD
+
+###########################################################################
+# rules
+###########################################################################
+
+.PHONY: all
+all: modules
+
+ifneq ($(USE_KBUILD),1)
+
 ###########################################################################
 # variables
 ###########################################################################
@@ -16,15 +29,8 @@ nv_kernel_o_binary         = kernel-open/nvidia/nv-kernel.o_binary
 nv_modeset_kernel_o        = src/nvidia-modeset/$(OUTPUTDIR)/nv-modeset-kernel.o
 nv_modeset_kernel_o_binary = kernel-open/nvidia-modeset/nv-modeset-kernel.o_binary
 
-###########################################################################
-# rules
-###########################################################################
-
 include utils.mk
 
-.PHONY: all
-all: modules
-
 ###########################################################################
 # nv-kernel.o is the OS agnostic portion of nvidia.ko
 ###########################################################################
@@ -48,22 +54,31 @@ $(nv_modeset_kernel_o):
 $(nv_modeset_kernel_o_binary): $(nv_modeset_kernel_o)
 	cd $(dir $@) && ln -sf ../../$^ $(notdir $@)
 
+# Make the OS agnostic binaries a prerequisite of the modules target
+modules: $(nv_kernel_o_binary) $(nv_modeset_kernel_o_binary)
+endif
 
 ###########################################################################
 # After the OS agnostic portions are built, descend into kernel-open/ and build
 # the kernel modules with kbuild.
 ###########################################################################
 
 .PHONY: modules
-modules: $(nv_kernel_o_binary) $(nv_modeset_kernel_o_binary)
+modules:
+ifeq ($(USE_KBUILD),1)
+# create nv-kernel.o and nv-modeset-kernel.o by building stub modules -- a
+# required hack to overcome kbuild limitations regarding the maximum number
+# of object files to link into a module.
+	$(MAKE) -C kernel-open modules NV_PREPARE_ONLY=1
+endif
 	$(MAKE) -C kernel-open modules
 
 ###########################################################################
 # Install the built kernel modules using kbuild.
 ###########################################################################
 
 .PHONY: modules_install
-modules_install:
+modules_install: modules
 	$(MAKE) -C kernel-open modules_install
 
 ###########################################################################
@@ -75,11 +90,15 @@ clean: nvidia.clean nvidia-modeset.clean kernel-open.clean
 
 .PHONY: nvidia.clean
 nvidia.clean:
+ifneq ($(USE_KBUILD),1)
 	$(MAKE) -C src/nvidia clean
+endif
 
 .PHONY: nvidia-modeset.clean
 nvidia-modeset.clean:
+ifneq ($(USE_KBUILD),1)
 	$(MAKE) -C src/nvidia-modeset clean
+endif
 
 .PHONY: kernel-open.clean
 kernel-open.clean:

README.md

@@ -57,6 +57,10 @@ DEBUG - Set this to "1" to build the kernel modules as debug.  By default, the
     build compiles without debugging information.  This also enables
     various debug log messages in the kernel modules.
 
+USE_KBUILD - Set this to "1" to build all sources using the linux kernel build
+    system.  This is needed to support features like RANDSTRUCT, kCFI or
+    grsecurity kernels.
+
 These variables can be set on the make command line.  E.g.,
 
     make modules -j$(nproc) NV_VERBOSE=1

kernel-open/Kbuild

@@ -39,9 +39,95 @@ NV_BUILD_TYPE ?= release
 #
 ASSIGN_PER_OBJ_CFLAGS = \
  $(foreach _cflags_variable, \
- $(notdir $(1)) $(1), \
+ $(sort $(notdir $(1)) $(1)), \
  $(eval $(addprefix CFLAGS_,$(_cflags_variable)) += $(2)))
 
+#
+# As ASSIGN_PER_OBJ_CFLAGS, just for per-object CFLAGS_REMOVE.
+#
+ASSIGN_PER_OBJ_CFLAGS_REMOVE = \
+ $(foreach _cflags_variable, \
+ $(sort $(notdir $(1)) $(1)), \
+ $(eval $(addprefix CFLAGS_REMOVE_,$(_cflags_variable)) += $(2)))
+
+#
+# As ASSIGN_PER_OBJ_CFLAGS, just for per-object LDFLAGS.
+#
+ASSIGN_PER_OBJ_LDFLAGS = \
+ $(foreach _ldflags_variable, \
+ $(sort $(notdir $(1)) $(1)), \
+ $(eval $(addprefix LDFLAGS_,$(_ldflags_variable)) += $(2)))
+
+#
+# As ASSIGN_PER_OBJ_CFLAGS, just for per-object OBJCOPYFLAGS.
+#
+ASSIGN_PER_OBJ_OBJCOPYFLAGS = \
+ $(foreach _objcopyflags_variable, \
+ $(sort $(notdir $(1)) $(1)i), \
+ $(eval $(addprefix OBJCOPYFLAGS_,$(_objcopyflags_variable)) += $(2)))
+
+
+#
+# Define rule for generating a source file containing identification information
+# for the build.
+#
+# $(1) string name
+# $(2) module name
+# $(3) prerequisite object files
+##############################################################################
+
+NV_BUILD_USER         ?= $(shell whoami)
+NV_BUILD_HOST         ?= $(shell hostname)
+
+ifndef TARGET_OS
+  TARGET_OS           := $(shell uname)
+endif
+
+ifndef TARGET_ARCH
+  ifneq ($(TARGET_OS),SunOS)
+    TARGET_ARCH       := $(shell uname -m)
+  else
+    TARGET_ARCH       := $(shell isainfo -n)
+  endif
+  TARGET_ARCH         := $(subst i386,x86,$(TARGET_ARCH))
+  TARGET_ARCH         := $(subst i486,x86,$(TARGET_ARCH))
+  TARGET_ARCH         := $(subst i586,x86,$(TARGET_ARCH))
+  TARGET_ARCH         := $(subst i686,x86,$(TARGET_ARCH))
+  TARGET_ARCH         := $(subst amd64,x86_64,$(TARGET_ARCH))
+endif
+
+DATE                  ?= date
+
+NVIDSTRING ?= g_nvid_string.c
+
+ifeq ($(NV_BUILD_TYPE),debug)
+  NVIDSTRING_BUILD_TYPE_STRING = Debug Build
+else ifeq ($(NV_BUILD_TYPE),develop)
+  NVIDSTRING_BUILD_TYPE_STRING = Develop Build
+else ifeq ($(NV_BUILD_TYPE),release)
+  NVIDSTRING_BUILD_TYPE_STRING = Release Build
+else
+  NVIDSTRING_BUILD_TYPE_STRING = Custom '$(NV_BUILD_TYPE)' Build
+endif
+
+# NV_VERSION_STRING is set as preprocessor define
+NVIDIA_NVID_VERSION := \"NV_VERSION_STRING\"
+
+define GENERATE_NVIDSTRING
+  $(1)_BUILD_NVID := NVIDIA $$(strip $(2)) for $$(TARGET_ARCH)  $$(NVIDIA_NVID_VERSION)
+  $(1)_BUILD_NVID := $$($$(strip $(1))_BUILD_NVID)  $$(NVIDSTRING_BUILD_TYPE_STRING)
+  ifneq ($$(NVIDIA_NVID_EXTRA),)
+    $(1)_BUILD_NVID := $$($$(strip $(1))_BUILD_NVID)  $$(NVIDIA_NVID_EXTRA)
+  endif
+  $(1)_BUILD_NVID := $$($$(strip $(1))_BUILD_NVID)  ($$(NV_BUILD_USER)@$$(NV_BUILD_HOST))
+  # g_nvid_string.c depends on all objects except g_nvid_string.o, and version.mk
+  # $(NVIDSTRING) evaluated early, to allow reusing it
+  $$(obj)/$(NVIDSTRING): $$(adprefix $$(obj)/,$$(filter-out $(NVIDSTRING:.c=.o), $(3))) $$(VERSION_MK)
+	@mkdir -p $$(dir $$@)
+	@echo "const char $(1)[] = \"nvidia id: $$($$(strip $(1))_BUILD_NVID)  `$$(DATE)`\";" > $$@
+	@echo "const char *const p$$(strip $(1)) = $(1) + 11;" >> $$@;
+endef
+
 
 #
 # Include the specifics of the individual NVIDIA kernel modules.
@@ -70,13 +156,39 @@ endif
 quiet_cmd_symlink = SYMLINK $@
  cmd_symlink = ln -sf $(abspath $<) $@
 
+#
+# Command to neuter .modinfo section for stub modules which are our hacky
+# attempt to make kbuild support nested compound objects.
+#
+# Intended usage:
+#
+#   # stub module
+#   obj-m += nv-kernel.stub.o
+#   nv-kernel.stub-y := ...
+#
+#   # real module wanting to make use of nv-kernel.o
+#   obj-m += ndidia.o
+#   nvidia-y += nv-kernel.clean.o ...
+#
 
-$(foreach _module, $(NV_KERNEL_MODULES), \
- $(eval include $(src)/$(_module)/$(_module).Kbuild))
+MOD_STUB := common/mod_stub.o
 
+# not really needed, but to have conftest be the very first
+NV_OBJECTS_DEPEND_ON_CONFTEST += $(MOD_STUB)
 
+$(obj)/%.clean.o: OBJCOPYFLAGS += --rename-section .modinfo=.discard.modinfo
+$(obj)/%.clean.o: $(obj)/%.stub.o FORCE
+	$(call if_changed,objcopy)
+
+# early flags, which may be filtered for USE_KBUILD=1 builds
 ccflags-y += -I$(src)/common/inc
 ccflags-y += -I$(src)
+
+
+$(foreach _module, $(NV_KERNEL_MODULES), \
+ $(eval include $(src)/$(_module)/$(_module).Kbuild))
+
+
 ccflags-y += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
 ccflags-y += -D__KERNEL__ -DMODULE -DNVRM
 ccflags-y += -DNV_VERSION_STRING=\"590.48.01\"

kernel-open/Makefile

@@ -64,12 +64,25 @@ else
             from CONFIG_CC_VERSION_TEXT in the kernel configuration.)
       else
           CC=$(cc_version_text)
+          CXX=$(subst cc,++,$(cc_version_text))
       endif
     endif
   endif
 
+  ifneq ($(USE_KBUILD),1)
+    # RANDSTRUCT is incompatible with how we build the OS-agnostic part, leading
+    # to calling the wrong callback functions from pure *ops structures at
+    # runtime.
+    ranstruct_enabled=$(firstword $(shell . $(KERNEL_OUTPUT)/.config; \
+                        echo "$$CONFIG_RANDSTRUCT$$CONFIG_GCC_PLUGIN_RANDSTRUCT"))
+    ifneq ($(ranstruct_enabled),)
+      $(error RANDSTRUCT enabled kernel is incompatible with binary objects!))
+    endif
+  endif
+
   CC ?= cc
   LD ?= ld
+  CXX ?= c++
   OBJDUMP ?= objdump
   AWK ?= awk
   # Bake the following awk program in a string. The program is needed to add C++
@@ -132,10 +145,18 @@ else
   KBUILD_PARAMS += NV_KERNEL_MODULES="$(NV_KERNEL_MODULES)"
   KBUILD_PARAMS += INSTALL_MOD_DIR="$(INSTALL_MOD_DIR)"
   KBUILD_PARAMS += NV_SPECTRE_V2=$(SPECTRE_V2_RETPOLINE)
+  ifeq ($(USE_KBUILD),1)
+    KBUILD_PARAMS += NV_PREPARE_ONLY=$(NV_PREPARE_ONLY)
+    ifeq ($(NV_PREPARE_ONLY),1)
+    KBUILD_PARAMS += KBUILD_MODPOST_NOFINAL=1
+    KBUILD_PARAMS += MODPOST=/bin/true
+    KBUILD_PARAMS += objtool-enabled=
+    endif
+  endif
 
   .PHONY: modules module clean clean_conftest modules_install
   modules clean modules_install:
-	@$(MAKE) "LD=$(LD)" "CC=$(CC)" "OBJDUMP=$(OBJDUMP)" \
+	@$(MAKE) "LD=$(LD)" "CC=$(CC)" "CXX=$(CXX)" "OBJDUMP=$(OBJDUMP)" \
 	  $(PAHOLE_VARIABLES) $(KBUILD_PARAMS) $@
 	@if [ "$@" = "modules" ]; then \
 	  for module in $(NV_KERNEL_MODULES); do \

kernel-open/common/mod_stub.c

@@ -0,0 +1,4 @@
+/* Stub module fragment for emulated nested compound object kbuild support */
+#include <linux/module.h>
+
+MODULE_LICENSE("Dual MIT/GPL");

kernel-open/conftest.sh

@@ -4919,6 +4919,38 @@ compile_test() {
             compile_check_conftest "$CODE" "NV_DRM_DRIVER_HAS_DATE" "" "types"
         ;;
 
+        drm_connector_helper_funcs_mode_valid_has_int_ret_type)
+            #
+            # Determine if the return type is 'int' for
+            # drm_connector_helper_funcs::mode_valid.
+            #
+            # It was changed to 'enum drm_mode_status' by commit 0993f1d0d8a1
+            # ("drm: Make the connector mode_valid() vfunc return a
+            # drm_mode_status enum") in v3.14.
+            #
+            CODE="
+            #include <drm/drm_atomic_helper.h>
+
+            #ifndef __same_type
+            #define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b))
+            #endif
+
+            /* BUILD_BUG_ON() from <linux/kernel.h> isn't working */
+            #define CONF_BUILD_BUG_ON(cond) \
+                char conf_bug_on_trigger[0 - !!(cond)]
+
+            /* We exploit the fact, that 'int' and 'enum' are compatible but
+             * 'enum e1' and 'enum e2' are not to cause a build error if the
+             * return type of drm_connector_helper_funcs::mode_valid is an enum.
+             */
+            enum conftest_enum { CONFTEST = -1 } conftest_enum;
+            const struct drm_connector_helper_funcs conftest_func;
+            CONF_BUILD_BUG_ON(!__same_type(conftest_func.mode_valid(NULL, NULL), conftest_enum));
+            "
+
+            compile_check_conftest "$CODE" "NV_DRM_CONNECTOR_HELPER_FUNCS_MODE_VALID_HAS_INT_RET_TYPE" "" "types"
+        ;;
+
         drm_connector_helper_funcs_mode_valid_has_const_mode_arg)
             #
             # Determine if the 'mode' pointer argument is const in
@@ -4932,8 +4964,9 @@ compile_test() {
             CODE="
             #include <drm/drm_atomic_helper.h>
 
-            static int conftest_drm_connector_mode_valid(struct drm_connector *connector,
-                                                         const struct drm_display_mode *mode) {
+            static enum drm_mode_status
+            conftest_drm_connector_mode_valid(struct drm_connector *connector,
+                                              const struct drm_display_mode *mode) {
                 return 0;
             }
 

kernel-open/nvidia-drm/nvidia-drm-connector.c

@@ -310,11 +310,16 @@ static int nv_drm_connector_get_modes(struct drm_connector *connector)
     return count;
 }
 
-static int nv_drm_connector_mode_valid(struct drm_connector    *connector,
+#ifdef NV_DRM_CONNECTOR_HELPER_FUNCS_MODE_VALID_HAS_INT_RET_TYPE
+static int
+#else
+static enum drm_mode_status
+#endif
+nv_drm_connector_mode_valid(struct drm_connector    *connector,
 #if defined(NV_DRM_CONNECTOR_HELPER_FUNCS_MODE_VALID_HAS_CONST_MODE_ARG)
-                                       const struct drm_display_mode *mode)
+                            const struct drm_display_mode *mode)
 #else
-                                       struct drm_display_mode *mode)
+                            struct drm_display_mode *mode)
 #endif
 {
     struct drm_device *dev = connector->dev;

kernel-open/nvidia-drm/nvidia-drm-sources.mk

@@ -107,5 +107,6 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += drm_format_info_has_is_yuv
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_gem_prime_mmap
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_output_poll_changed
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_date
+NV_CONFTEST_TYPE_COMPILE_TESTS += drm_connector_helper_funcs_mode_valid_has_int_ret_type
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_connector_helper_funcs_mode_valid_has_const_mode_arg
 NV_CONFTEST_TYPE_COMPILE_TESTS += drm_fb_create_takes_format_info

kernel-open/nvidia-drm/nvidia-drm.Kbuild

@@ -14,8 +14,10 @@ NVIDIA_DRM_SOURCES += nvidia-drm/nvidia-drm-linux.c
 
 NVIDIA_DRM_OBJECTS = $(patsubst %.c,%.o,$(NVIDIA_DRM_SOURCES))
 
+ifneq ($(NV_PREPARE_ONLY),1)
 obj-m += nvidia-drm.o
 nvidia-drm-y := $(NVIDIA_DRM_OBJECTS)
+endif
 
 NVIDIA_DRM_KO = nvidia-drm/nvidia-drm.ko