问题描述：
module在4.18的kernel上面无法在rmmod的时候会如下报错
[root@node-1 zjp]# insmod hello7.ko
[root@node-1 zjp]# rmmod hello7.ko
rmmod: ERROR: could not remove 'hello7': Device or resource busy
rmmod: ERROR: could not remove module hello7.ko: Device or resource busy
在rmmod卸载module的时候会调用delete_module，在return
EBUSY
The module is not "live" (i.e., it is still being initialized or is already marked for removal); or, the module has an init function but has no exit function, and O_TRUNC was not specified in flags.调查过程
查看已经安装的module信息
[root@node-1 zjp]# cat /proc/modules | grep hello7
hello7 262144 0 [permanent], Live 0xffff4f26408b0000 (OE)这个打印的代码
1005 if (mod->init != NULL && mod->exit == NULL) {
1006 printed_something = 1;
1007 seq_printf(m, "[permanent],");
1008 }
1009 卸载的时候
948 /* If it has an init func, it must have an exit func to unload */
949 if (mod->init && !mod->exit) {
950 forced = try_force_unload(flags);
951 if (!forced) {
952 /* This module can't be removed */
953 ret = -EBUSY;
954 goto out;
955 }
956 }
代码回去强制卸载如果调用 try_force_unload函数
811 #ifdef CONFIG_MODULE_FORCE_UNLOAD
812 static inline int try_force_unload(unsigned int flags)
813 {
814 int ret = (flags & O_TRUNC);
815 if (ret)
816 add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
817 return ret;
818 }
819 #else
820 static inline int try_force_unload(unsigned int flags)
821 {
822 return 0;
823 }
824 #endif /* CONFIG_MODULE_FORCE_UNLOAD */
4.18的系统没有配置这个
[root@node-1 zjp]# grep -r "CONFIG_MODULE_FORCE_UNLOAD" /boot/config-4.18.0-80.11.2.el8.es.2.aarch64
# CONFIG_MODULE_FORCE_UNLOAD is not set根本原因是 mod->exit为NULL LKM cleanup_module() not called
在测试的module的Makefile上添加 KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO
LINUX_KERNEL_PATH := /lib/modules/$(shell uname -r)/build #Linux内核源代码的绝对路径
KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO
all:
make -C $(LINUX_KERNEL_PATH) M=$(CURRENT_PATH) modules #编译模块了
clean:
make -C $(LINUX_KERNEL_PATH) M=$(CURRENT_PATH) clean #清理
⋊> ~/work-dir vim kernel-4.18.0/linux-4.18.0-80.1.2.el8_0/Makefile
500 # check for 'asm goto'
501 ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC) $(KBUILD_CFLAGS)), y)
502 CC_HAVE_ASM_GOTO := 1
503 KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO
504 KBUILD_AFLAGS += -DCC_HAVE_ASM_GOTO
505 endif
4.14的Makefile文件
# check for 'asm goto'
ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC) $(KBUILD_CFLAGS)), y)
KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO
KBUILD_AFLAGS += -DCC_HAVE_ASM_GOTO
endif/scripts/gcc-goto.sh内容
1 #!/bin/sh
2 # SPDX-License-Identifier: GPL-2.0
3 # Test for gcc 'asm goto' support
4 # Copyright (C) 2010, Jason Baron <jbaron@redhat.com>
5
6 cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y"
7 int main(void)
8 {
9 #if defined(__arm__) || defined(__aarch64__)
10 /*
11 * Not related to asm goto, but used by jump label
12 * and broken on some ARM GCC versions (see GCC Bug 48637).
13 */
14 static struct { int dummy; int state; } tp;
15 asm (".long %c0" :: "i" (&tp.state));
16 #endif
17
18 entry:
19 asm goto ("" :::: entry);
20 return 0;
21 }
22 END
抓取Makefile的执行过程
make V=1 SHELL='sh -x' > node-1-build.log 2>&1
+ /usr/bin/sh /usr/src/kernels/4.18.0-80.11.2.es.2.aarch64/scripts/gcc-goto.sh gcc -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \
-fno-strict-aliasing -fno-common -fshort-wchar \
-Werror-implicit-function-declaration \
-Wno-format-security \
-std=gnu89输出内容报错
<stdin>: In function ‘main’:
<stdin>:9:2: error: invalid 'asm': invalid operand prefix '%c'结果
得知4.18的编译环境是centos8的mock环境,4,18版本kernel两者的gcc版本差异很大,测试在centos8的mock环境编译的module是带有DCC_HAVE_ASM_GOTO,而4.14在build module的时候是没有带有 DCC_HAVE_ASM_GOTO;
参考链接：
[v1,5/7] arm64, jump label: detect %c support for ARM64
LKM cleanup_module() not called
编译参数打印
+ gcc -Wp,-MD,/root/zjp/zjp/.hello9.mod.o.d -nostdinc -isystem /usr/lib/gcc/aarch64-redhat-linux/4.8.5/include -I./arch/arm64/include -I./arch/arm64/include/generated -I./include -I./arch/arm64/include/uapi -I./arch/arm64/include/generated/uapi -I./include/uapi -I./include/generated/uapi -include ./include/linux/kconfig.h -include ./include/linux/compiler_types.h -D__KERNEL__ -mlittle-endian -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -fshort-wchar -Werror-implicit-function-declaration -Wno-format-security -std=gnu89 -fno-PIE -fno-PIE -mgeneral-regs-only -DCONFIG_AS_LSE=1 -fno-asynchronous-unwind-tables -fno-delete-null-pointer-checks -O2 -Wno-maybe-uninitialized --param=allow-store-data-races=0 -Wframe-larger-than=2048 -fstack-protector-strong -Wno-unused-but-set-variable -fno-omit-frame-pointer -fno-optimize-sibling-calls -fno-var-tracking-assignments -g -gdwarf-4 -pg -fno-inline-functions-called-once -Wdeclaration-after-statement -Wno-pointer-sign -fno-strict-overflow -fno-merge-all-constants -fmerge-constants -fno-stack-check -fconserve-stack -Werror=implicit-int -Werror=strict-prototypes '-DKBUILD_BASENAME="hello9.mod"' '-DKBUILD_MODNAME="hello9"' -DMODULE -c -o /root/zjp/zjp/hello9.mod.o /root/zjp/zjp/hello9.mod.c
+ scripts/basic/fixdep /root/zjp/zjp/.hello9.mod.o.d /root/zjp/zjp/hello9.mod.o 'gcc -Wp,-MD,/root/zjp/zjp/.hello9.mod.o.d -nostdinc -isystem /usr/lib/gcc/aarch64-redhat-linux/4.8.5/include -I./arch/arm64/include -I./arch/arm64/include/generated -I./include -I./arch/arm64/include/uapi -I./arch/arm64/include/generated/uapi -I./include/uapi -I./include/generated/uapi -include ./include/linux/kconfig.h -include ./include/linux/compiler_types.h -D__KERNEL__ -mlittle-endian -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs -fno-strict-aliasing -fno-common -fshort-wchar -Werror-implicit-function-declaration -Wno-format-security -std=gnu89 -fno-PIE -fno-PIE -mgeneral-regs-only -DCONFIG_AS_LSE=1 -fno-asynchronous-unwind-tables -fno-delete-null-pointer-checks -O2 -Wno-maybe-uninitialized --param=allow-store-data-races=0 -Wframe-larger-than=2048 -fstack-protector-strong -Wno-unused-but-set-variable -fno-omit-frame-pointer -fno-optimize-sibling-calls -fno-var-tracking-assignments -g -gdwarf-4 -pg -fno-inline-functions-called-once -Wdeclaration-after-statement -Wno-pointer-sign -fno-strict-overflow -fno-merge-all-constants -fmerge-constants -fno-stack-check -fconserve-stack -Werror=implicit-int -Werror=strict-prototypes -DKBUILD_BASENAME='\''"hello9.mod"'\'' -DKBUILD_MODNAME='\''"hello9"'\'' -DMODULE -c -o /root/zjp/zjp/hello9.mod.o /root/zjp/zjp/hello9.mod.c'