1 // SPDX-License-Identifier: GPL-2.0
6  *	- Rafael R. Reilova (moved everything from head.S),
8  *	- Channing Corn (tests & fixes),
9  *	- Andrew D. Balsa (code cleanup).
20 #include <asm/spec-ctrl.h>
24 #include <asm/processor-flags.h>
53 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
57 /* The current value of the SPEC_CTRL MSR with task-specific bits set */
89 	 * When KERNEL_IBRS this MSR is written on return-to-user, unless  in update_spec_ctrl_cond()
206 		hostval = ssbd_tif_to_spec_ctrl(ti->flags);  in x86_virt_spec_ctrl()
235 /* Default mitigation for MDS-affected CPUs */
271 		return -EINVAL;  in mds_cmdline()
296 /* Default mitigation for TAA-affected CPUs */
371 		return -EINVAL;  in tsx_async_abort_parse_cmdline()
428 	 * mitigations, disable KVM-only mitigation in that case.  in mmio_select_mitigation()
436 	 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can  in mmio_select_mitigation()
468 		return -EINVAL;  in mmio_stale_data_parse_cmdline()
520 		return -EINVAL;  in rfds_parse_cmdline()
593 	 * As these mitigations are inter-related and rely on VERW instruction  in md_clear_select_mitigation()
688 		return -EINVAL;  in srbds_parse_cmdline()
852 		return -EINVAL;  in gds_parse_cmdline()
895 	 * Consider SMAP to be non-functional as a mitigation on these  in smap_works_speculatively()
914 		 * path of a conditional swapgs with a user-controlled GS  in spectre_v1_select_mitigation()
939 			 * Enable lfences in the kernel entry (non-swapgs)  in spectre_v1_select_mitigation()
998 		return -EINVAL;  in retbleed_parse_cmdline()
1009 		} else if (!strcmp(str, "auto")) {  in retbleed_parse_cmdline()
1123 		 * software-based untraining so clear those in case some  in retbleed_select_mitigation()
1199 	return spectre_v2_bad_module ? " - vulnerable module loaded" : "";  in spectre_v2_module_string()
1266 	[SPECTRE_V2_USER_STRICT_PREFERRED]	= "User space: Mitigation: STIBP always-on protection",
1276 	{ "auto",		SPECTRE_V2_USER_CMD_AUTO,		false },
1321 	pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);  in spectre_v2_parse_user_cmdline()
1388 			"always-on" : "conditional");  in spectre_v2_user_select_mitigation()
1395 	 * Intel's Enhanced IBRS also protects against cross-thread branch target  in spectre_v2_user_select_mitigation()
1396 	 * injection in user-mode as the IBRS bit remains always set which  in spectre_v2_user_select_mitigation()
1397 	 * implicitly enables cross-thread protections.  However, in legacy IBRS  in spectre_v2_user_select_mitigation()
1400 	 * These modes therefore disable the implicit cross-thread protection,  in spectre_v2_user_select_mitigation()
1411 	 * If STIBP support is not being forced, check if STIBP always-on  in spectre_v2_user_select_mitigation()
1422 			pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n");  in spectre_v2_user_select_mitigation()
1456 	{ "auto",		SPECTRE_V2_CMD_AUTO,		  false },
1489 		pr_err("unknown option (%s). Switching to default mode\n", arg);  in spectre_v2_parse_cmdline()
1499 		pr_err("%s selected but not compiled in. Switching to AUTO select\n",  in spectre_v2_parse_cmdline()
1508 		pr_err("%s selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n",  in spectre_v2_parse_cmdline()
1516 		pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",  in spectre_v2_parse_cmdline()
1522 		pr_err("%s selected but not compiled in. Switching to AUTO select\n",  in spectre_v2_parse_cmdline()
1528 		pr_err("%s selected but not Intel CPU. Switching to AUTO select\n",  in spectre_v2_parse_cmdline()
1534 		pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n",  in spectre_v2_parse_cmdline()
1540 		pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n",  in spectre_v2_parse_cmdline()
1562 /* Disable in-kernel use of non-RSB RET predictors */
1597 	 * user-space-poisoned RSB entries.  in spectre_v2_determine_rsb_fill_type_at_vmexit()
1611 			pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n");  in spectre_v2_determine_rsb_fill_type_at_vmexit()
1656 		return -EINVAL;  in spectre_bhi_parse_cmdline()
1708 	 * If the CPU is not affected and the command line mode is NONE or AUTO  in spectre_v2_select_mitigation()
1805 	 * JMPs gets protection against BHI and Intramode-BTI, but RET  in spectre_v2_select_mitigation()
1806 	 * prediction from a non-RSB predictor is still a risk.  in spectre_v2_select_mitigation()
1828 	 *    which could have a user-poisoned BTB or BHB entry.  in spectre_v2_select_mitigation()
1833 	 *    When IBRS or eIBRS is enabled, the "user -> kernel" attack  in spectre_v2_select_mitigation()
1838 	 *    The "user -> user" attack scenario is mitigated by RSB filling.  in spectre_v2_select_mitigation()
1842 	 *    If the 'next' in-kernel return stack is shorter than 'prev',  in spectre_v2_select_mitigation()
1843 	 *    'next' could be tricked into speculating with a user-poisoned RSB  in spectre_v2_select_mitigation()
1846 	 *    The "user -> kernel" attack scenario is mitigated by SMEP and  in spectre_v2_select_mitigation()
1849 	 *    The "user -> user" scenario, also known as SpectreBHB, requires  in spectre_v2_select_mitigation()
1855 	 * FIXME: Is this pointless for retbleed-affected AMD?  in spectre_v2_select_mitigation()
1870 	 * the CPU supports Enhanced IBRS, kernel might un-intentionally not  in spectre_v2_select_mitigation()
1910 		mask & SPEC_CTRL_STIBP ? "always-on" : "off");  in update_stibp_strict()
1949 …n, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for…
1950 …n, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_ab…
1951 …n, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mm…
2034 	{ "auto",	SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
2067 			pr_err("unknown option (%s). Switching to default mode\n", arg);  in ssb_parse_cmdline()
2113 	 *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.  in __ssb_select_mitigation()
2114 	 *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass  in __ssb_select_mitigation()
2115 	 *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation  in __ssb_select_mitigation()
2153 	 * task, but for a non-current task delay setting the CPU  in task_update_spec_tif()
2167 		return -EPERM;  in l1d_flush_prctl_set()
2171 		set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);  in l1d_flush_prctl_set()
2174 		clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);  in l1d_flush_prctl_set()
2177 		return -ERANGE;  in l1d_flush_prctl_set()
2185 		return -ENXIO;  in ssb_prctl_set()
2191 			return -EPERM;  in ssb_prctl_set()
2209 			return -EPERM;  in ssb_prctl_set()
2215 		return -ERANGE;  in ssb_prctl_set()
2244 		 * updated, unless it was force-disabled by a previous prctl  in ib_prctl_set()
2253 			return -EPERM;  in ib_prctl_set()
2266 			return -EPERM;  in ib_prctl_set()
2279 		return -ERANGE;  in ib_prctl_set()
2295 		return -ENODEV;  in arch_prctl_spec_ctrl_set()
2315 	if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH))  in l1d_flush_prctl_get()
2375 		return -ENODEV;  in arch_prctl_spec_ctrl_get()
2394 /* Default mitigation for L1TF-affected CPUs */
2419 	if (c->x86 != 6)  in override_cache_bits()
2422 	switch (c->x86_vfm) {  in override_cache_bits()
2436 		if (c->x86_cache_bits < 44)  in override_cache_bits()
2437 			c->x86_cache_bits = 44;  in override_cache_bits()
2477 			e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {  in l1tf_select_mitigation()
2482 …pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help y…  in l1tf_select_mitigation()
2495 		return -EINVAL;  in l1tf_cmdline()
2551 		return -EINVAL;  in srso_parse_cmdline()
2557 	else if (!strcmp(str, "safe-ret"))  in srso_parse_cmdline()
2561 	else if (!strcmp(str, "ibpb-vmexit"))  in srso_parse_cmdline()
2570 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html …
2601 		pr_warn("IBPB-extending microcode not applied!\n");  in srso_select_mitigation()
2653 				 * software-based untraining so clear those in case some  in srso_select_mitigation()
2706 	[VMENTER_L1D_FLUSH_AUTO]		= "auto",
2821 		return "; STIBP: always-on";  in stibp_state()
2834 			return "; IBPB: always-on";  in ibpb_state()
2847 			return "; PBRSB-eIBRS: SW sequence";  in pbrsb_eibrs_state()
2849 			return "; PBRSB-eIBRS: Vulnerable";  in pbrsb_eibrs_state()
2851 		return "; PBRSB-eIBRS: Not affected";  in pbrsb_eibrs_state()
2908 			return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n");  in retbleed_show_state()