From 52cf2c6a80323655b786e7d54cc92ec3b2ed1a38 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:04 +0800 Subject: [PATCH 001/524] sw64: add build infrastructure Add Kbuild, Makefile, Kconfig and link script for SW64 build infrastructure. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kbuild | 7 + arch/sw_64/Kconfig | 645 +++++++++++++++++++++++++++++ arch/sw_64/Kconfig.debug | 53 +++ arch/sw_64/Makefile | 69 +++ arch/sw_64/Makefile.postlink | 36 ++ arch/sw_64/boot/.gitignore | 2 + arch/sw_64/boot/Makefile | 29 ++ arch/sw_64/boot/dts/Makefile | 27 ++ arch/sw_64/include/asm/Kbuild | 16 + arch/sw_64/include/uapi/asm/Kbuild | 5 + arch/sw_64/kernel/.gitignore | 2 + arch/sw_64/kernel/Makefile | 51 +++ arch/sw_64/kernel/vmlinux.lds.S | 113 +++++ 13 files changed, 1055 insertions(+) create mode 100644 arch/sw_64/Kbuild create mode 100644 arch/sw_64/Kconfig create mode 100644 arch/sw_64/Kconfig.debug create mode 100644 arch/sw_64/Makefile create mode 100644 arch/sw_64/Makefile.postlink create mode 100644 arch/sw_64/boot/.gitignore create mode 100644 arch/sw_64/boot/Makefile create mode 100644 arch/sw_64/boot/dts/Makefile create mode 100644 arch/sw_64/include/asm/Kbuild create mode 100644 arch/sw_64/include/uapi/asm/Kbuild create mode 100644 arch/sw_64/kernel/.gitignore create mode 100644 arch/sw_64/kernel/Makefile create mode 100644 arch/sw_64/kernel/vmlinux.lds.S diff --git a/arch/sw_64/Kbuild b/arch/sw_64/Kbuild new file mode 100644 index 000000000000..aa0bf0507406 --- /dev/null +++ b/arch/sw_64/Kbuild @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-y += kernel/ mm/ platform/ +obj-$(CONFIG_NET) += net/ +obj-$(CONFIG_KVM) += kvm/ +obj-$(CONFIG_MATHEMU) += math-emu/ + +obj-$(CONFIG_BUILTIN_DTB) += boot/dts/ diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig new file mode 100644 index 000000000000..0fd1be7195cc --- /dev/null +++ b/arch/sw_64/Kconfig @@ -0,0 +1,645 @@ +# SPDX-License-Identifier: GPL-2.0 +config SW64 + bool + default y + select ACPI + select ACPI_MCFG if (ACPI && PCI) + select ACPI_REDUCED_HARDWARE_ONLY + select ARCH_ATOMIC + select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI + select ARCH_HAS_ELF_RANDOMIZE + select ARCH_HAS_PHYS_TO_DMA + select ARCH_HAS_PMEM_API + select ARCH_HAS_PTE_DEVMAP + select ARCH_HAS_PTE_SPECIAL + select ARCH_HAS_SG_CHAIN + select ARCH_HAS_UACCESS_FLUSHCACHE + select ARCH_HAS_VM_GET_PAGE_PROT + select ARCH_HAS_ZONE_DEVICE + select ARCH_HAVE_NMI_SAFE_CMPXCHG + select ARCH_INLINE_READ_LOCK + select ARCH_INLINE_READ_LOCK_BH + select ARCH_INLINE_READ_LOCK_IRQ + select ARCH_INLINE_READ_LOCK_IRQSAVE + select ARCH_INLINE_READ_UNLOCK + select ARCH_INLINE_READ_UNLOCK_BH + select ARCH_INLINE_READ_UNLOCK_IRQ + select ARCH_INLINE_READ_UNLOCK_IRQRESTORE + select ARCH_INLINE_SPIN_LOCK + select ARCH_INLINE_SPIN_LOCK_BH + select ARCH_INLINE_SPIN_LOCK_IRQ + select ARCH_INLINE_SPIN_LOCK_IRQSAVE + select ARCH_INLINE_SPIN_TRYLOCK + select ARCH_INLINE_SPIN_TRYLOCK_BH + select ARCH_INLINE_SPIN_UNLOCK + select ARCH_INLINE_SPIN_UNLOCK_BH + select ARCH_INLINE_SPIN_UNLOCK_IRQ + select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE + select ARCH_INLINE_WRITE_LOCK + select ARCH_INLINE_WRITE_LOCK_BH + select ARCH_INLINE_WRITE_LOCK_IRQ + select ARCH_INLINE_WRITE_LOCK_IRQSAVE + select ARCH_INLINE_WRITE_UNLOCK + select ARCH_INLINE_WRITE_UNLOCK_BH + select ARCH_INLINE_WRITE_UNLOCK_IRQ + select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE + select ARCH_NO_PREEMPT + select ARCH_SUPPORTS_ACPI + select ARCH_SUPPORTS_ATOMIC_RMW + select ARCH_SUPPORTS_NUMA_BALANCING + select ARCH_SUPPORTS_UPROBES + select ARCH_USE_CMPXCHG_LOCKREF + select ARCH_USE_QUEUED_RWLOCKS + select ARCH_USE_QUEUED_SPINLOCKS + select ARCH_WANT_DEFAULT_BPF_JIT + select ARCH_WANT_FRAME_POINTERS + select ARCH_WANT_IPC_PARSE_VERSION + select AUDIT_ARCH + select COMMON_CLK + select DMA_OPS if PCI + select GENERIC_CLOCKEVENTS + select GENERIC_IRQ_LEGACY + select GENERIC_IRQ_MIGRATION if SMP + select GENERIC_IRQ_PROBE + select GENERIC_IRQ_SHOW + select GENERIC_PCI_IOMAP if PCI + select GENERIC_SMP_IDLE_THREAD + select GENERIC_STRNCPY_FROM_USER + select GENERIC_STRNLEN_USER + select GENERIC_TIME_VSYSCALL + select HANDLE_DOMAIN_IRQ + select HARDIRQS_SW_RESEND + select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_JUMP_LABEL + select HAVE_ARCH_KGDB + select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_TRACEHOOK + select HAVE_ARCH_TRANSPARENT_HUGEPAGE + select HAVE_ASM_MODVERSIONS + select HAVE_C_RECORDMCOUNT + select HAVE_DEBUG_BUGVERBOSE + select HAVE_DYNAMIC_FTRACE + select HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_EBPF_JIT + select HAVE_FAST_GUP + select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_TRACER + select HAVE_IDE + select HAVE_KPROBES + select HAVE_KPROBES_ON_FTRACE + select HAVE_KRETPROBES + select HAVE_LIVEPATCH if HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_MEMBLOCK + select HAVE_MEMBLOCK_NODE_MAP + select HAVE_MOD_ARCH_SPECIFIC + select HAVE_PCI + select HAVE_PCSPKR_PLATFORM + select HAVE_PERF_EVENTS + select HAVE_PERF_REGS + select HAVE_PERF_USER_STACK_DUMP + select HAVE_REGS_AND_STACK_ACCESS_API + select HAVE_RELIABLE_STACKTRACE if STACKTRACE + select HAVE_RSEQ + select HAVE_SYSCALL_TRACEPOINTS + select IRQ_FORCED_THREADING + select LOCK_MM_AND_FIND_VMA + select MEMORY_HOTPLUG_SPARSE if MEMORY_HOTPLUG + select MODULES_USE_ELF_RELA + select NO_BOOTMEM + select OF_EARLY_FLATTREE if OF + select OLD_SIGSUSPEND + select PCI_DOMAINS_GENERIC if PCI + select PCI_ECAM if (ACPI && PCI) + select PCI_MSI_ARCH_FALLBACKS if PCI_MSI + select PCI_SW64 if PCI + select SET_FS + select SPARSEMEM_EXTREME if SPARSEMEM + select SW64_IRQ_CPU + select SW64_IRQ_MSI if PCI_MSI + select SW64_IRQ_MSI_VT if PCI_MSI + select SW64_TIMER + select SWIOTLB + select THREAD_INFO_IN_TASK + +config LOCKDEP_SUPPORT + def_bool y + +config 64BIT + def_bool y + +config MMU + bool + default y + +config PGTABLE_LEVELS + int + default 4 + +config ARCH_SUPPORTS_HUGETLBFS + def_bool y + +config ARCH_ENABLE_MEMORY_HOTPLUG + bool + default y + +config ARCH_ENABLE_MEMORY_HOTREMOVE + bool + default y + +config ARCH_HAS_ILOG2_U32 + bool + default n + +config ARCH_HAS_ILOG2_U64 + bool + default n + +config GENERIC_GPIO + bool + +config GENERIC_CALIBRATE_DELAY + bool + default y + +config ZONE_DMA32 + bool + default y + +config NEED_DMA_MAP_STATE + def_bool y + +config NEED_SG_DMA_LENGTH + def_bool y + +config ARCH_WANT_HUGE_PMD_SHARE + def_bool y + +config GENERIC_ISA_DMA + bool + default y + +config NONCACHE_PAGE + bool + depends on SW64 + default y + +config AUDIT_ARCH + bool + +config SYS_HAS_EARLY_PRINTK + bool + +config HAVE_CSRRW + bool + +menu "System setup" + +menu "Machine Configuration" + +choice + prompt "Subarchitecture Configuration" + +config SUBARCH_C3B + bool "C3B" + +config SUBARCH_C4 + bool "C4" + select HAVE_CSRRW + select GENERIC_SCHED_CLOCK +endchoice + +choice + prompt "Uncore Configuration" + +config UNCORE_XUELANG + bool "Uncore for C3B" + depends on SUBARCH_C3B + help + Sunway cpu uncore for C3B + +config UNCORE_JUNZHANG + bool "Uncore for C4" + depends on SUBARCH_C4 + help + Sunway cpu uncore for C4 +endchoice + +choice + prompt "Platform Type" + +config PLATFORM_XUELANG + bool "Xuelang" + depends on UNCORE_XUELANG + select SPARSE_IRQ + select SYS_HAS_EARLY_PRINTK + select SW64_INTC_V2 + select I2C_SUNWAY if I2C + help + Sunway board chipset for C3B + +config PLATFORM_JUNZHANG + bool "JunZhang" + depends on UNCORE_JUNZHANG + select SPARSE_IRQ + select SYS_HAS_EARLY_PRINTK + help + Sunway board chipset for C4 + +endchoice + +config MIGHT_HAVE_PC_SERIO + bool "Use PC serio device i8042" + select ARCH_MIGHT_HAVE_PC_SERIO + default n + +endmenu + +menu "CPU Power Management" +source "drivers/cpufreq/Kconfig" + +config SW64_CPUAUTOPLUG + bool "sw64 CPU Autoplug interface" + depends on SW64_CPUFREQ + default y + help + Turns on the interface for SW64_CPU CPUAUTOPLUG. + +endmenu +# clear all implied options (don't want default values for those): +# Most of these machines have ISA slots; not exactly sure which don't, +# and this doesn't activate hordes of code, so do it always. +config ISA + bool + default y + help + Find out whether you have ISA slots on your motherboard. ISA is the + name of a bus system, i.e. the way the CPU talks to the other stuff + inside your box. Other bus systems are PCI, EISA, MicroChannel + (MCA) or VESA. ISA is an older system, now being displaced by PCI; + newer boards don't support it. If you have ISA, say Y, otherwise N. + +config ISA_DMA_API + bool + default y + +config PCI_DOMAINS + def_bool PCI + +config PCI_DOMAINS_GENERIC + def_bool PCI + +config PCI_SYSCALL + def_bool PCI + +config IOMMU_HELPER + def_bool PCI + +config PHYSICAL_START + hex "Physical address where the kernel starts" + default "0x900000" + help + This gives the physical address where the kernel starts, and it + is 0x10000 before _text. If you plan to use kernel for capturing + the crash dump change this value to start of the reserved region + (the "X" value as specified in the "crashkernel=YM@XM" command + line boot parameter passed to the panic-ed kernel). + +config KEXEC + bool "Kexec system call (EXPERIMENTAL)" + select KEXEC_CORE + help + kexec is a system call that implements the ability to shutdown your + current kernel, and to start another kernel. It is like a reboot + but it is independent of the system firmware. And like a reboot + you can start any kernel with it, not just Linux. + + The name comes from the similarity to the exec system call. + + It is an ongoing process to be certain the hardware in a machine + is properly shutdown, so do not be surprised if this code does not + initially work for you. As of this writing the exact hardware + interface is strongly in flux, so no good recommendation can be + made. + +config CRASH_DUMP + bool "Kernel crash dumps (EXPERIMENTAL)" + help + Generate crash dump after being started by kexec. + This should be normally only set in special crash dump kernels + which are loaded in the main kernel with kexec-tools into + a specially reserved region and then later executed after + a crash by kdump/kexec. The crash dump kernel must be compiled + to a memory address not used by the main kernel or firmware using + PHYSICAL_START. + +config SECCOMP + def_bool y + prompt "Enable seccomp to safely compute untrusted bytecode" + help + This kernel feature is useful for number crunching applications + that may need to compute untrusted bytecode during their + execution. By using pipes or other transports made available to + the process as file descriptors supporting the read/write + syscalls, it's possible to isolate those applications in + their own address space using seccomp. Once seccomp is + enabled via prctl(PR_SET_SECCOMP), it cannot be disabled + and the task is only allowed to execute a few safe syscalls + defined by each seccomp mode. + + If unsure, say Y. Only embedded should say N here. + +config GENERIC_HWEIGHT + bool + default y + +config SMP + bool "Symmetric multi-processing support" + depends on SW64 + select USE_GENERIC_SMP_HELPERS + help + This enables support for systems with more than one CPU. If you have + a system with only one CPU, like most personal computers, say N. If + you have a system with more than one CPU, say Y. + + If you say N here, the kernel will run on single and multiprocessor + machines, but will use only one CPU of a multiprocessor machine. If + you say Y here, the kernel will run on many, but not all, + singleprocessor machines. On a singleprocessor machine, the kernel + will run faster if you say N here. + + See also the SMP-HOWTO available at + . + + If you don't know what to do here, say N. + +config ARCH_PROC_KCORE_TEXT + def_bool y + +config HAVE_DEC_LOCK + bool "Use arch-specified dec_and_lock" + depends on SMP && !NUMA + default y + +config TRACE_IRQFLAGS_SUPPORT + def_bool y + +config ARCH_SUPPORTS_UPROBES + def_bool y + +config SCHED_SMT + bool "SMT scheduler support" + depends on SMP && SUBARCH_C4 + help + Improves the CPU scheduler's decision making when dealing with + MultiThreading at a cost of slightly increased overhead in some + places. If unsure say N here. + +config NR_CPUS + int "Maximum number of CPUs (2-256)" + range 2 256 + depends on SMP + default "64" if UNCORE_XUELANG + help + SW6 support can handle a maximum of 256 CPUs. + +config HOTPLUG_CPU + bool "Support for hot-pluggable CPUs" + depends on SMP + help + Say Y here to allow turning CPUs off and on. CPUs can be + controlled through /sys/devices/system/cpu. + ( Note: power management support will enable this option + automatically on SMP systems. ) + Say N if you want to disable CPU hotplug. + +config ARCH_SPARSEMEM_ENABLE + bool "Sparse Memory Support" + depends on SMP + select SPARSEMEM_VMEMMAP_ENABLE + +source "kernel/livepatch/Kconfig" + +config NUMA + bool "NUMA Support" + depends on SMP && !FLATMEM + select ACPI_NUMA if ACPI + help + Say Y to compile the kernel to support NUMA (Non-Uniform Memory + Access). This option is for configuring high-end multiprocessor + server machines. If in doubt, say N. + +config USE_PERCPU_NUMA_NODE_ID + def_bool y + depends on NUMA + +config NODES_SHIFT + int + default "7" + depends on NUMA + +config RELOCATABLE + bool "Relocatable kernel" + help + This builds a kernel image that retains relocation information + so it can be loaded someplace besides the default 1MB. + The relocations make the kernel binary about 15% larger, + but are discarded at runtime + +config RELOCATION_TABLE_SIZE + hex "Relocation table size" + depends on RELOCATABLE + range 0x0 0x01000000 + default "0x80000" + help + A table of relocation data will be appended to the kernel binary + and parsed at boot to fix up the relocated kernel. + + This option allows the amount of space reserved for the table to be + adjusted, although the default of 1Mb should be ok in most cases. + + The build will fail and a valid size suggested if this is too small. + + If unsure, leave at the default value. + +config RANDOMIZE_BASE + bool "Randomize the address of the kernel image" + depends on RELOCATABLE + help + Randomizes the physical and virtual address at which the + kernel image is loaded, as a security feature that + deters exploit attempts relying on knowledge of the location + of kernel internals. + + Entropy is generated using any coprocessor 0 registers available. + + The kernel will be offset by up to RANDOMIZE_BASE_MAX_OFFSET. + + If unsure, say N. + +config RANDOMIZE_BASE_MAX_OFFSET + hex "Maximum kASLR offset" if EXPERT + depends on RANDOMIZE_BASE + range 0x0 0x20000000 + default "0x10000000" + help + When kASLR is active, this provides the maximum offset that will + be applied to the kernel image. It should be set according to the + amount of physical RAM available in the target system minus + PHYSICAL_START and must be a power of 2. + + This is limited by the size of KTEXT space, 512Mb. The default is 256MB. + +config HZ + int "HZ of the short timer" + default 500 + +source "drivers/eisa/Kconfig" + +source "drivers/pcmcia/Kconfig" + +source "fs/Kconfig.binfmt" + +source "arch/sw_64/lib/Kconfig" + +endmenu + +menu "Boot options" + +config USE_OF + bool "Flattened Device Tree support" + select OF + select IRQ_DOMAIN + help + Include support for flattened device tree machine descriptions. + +config BUILTIN_DTB + bool "Embed DTB in kernel image" + depends on OF + default n + help + Embeds a device tree binary in the kernel image. + +config BUILTIN_DTB_NAME + string "Built in DTB" + depends on BUILTIN_DTB + help + Set the name of the DTB to embed, leave blank to pick one + automatically based on kernel configuration. + +config EFI + bool "UEFI runtime support" + select UCS2_STRING + select EFI_RUNTIME_WRAPPERS + default y + help + This option provides support for runtime services provided + by UEFI firmware (such as non-volatile variables, realtime + clock, and platform reset). A UEFI stub is also provided to + allow the kernel to be booted as an EFI application. This + is only useful on systems that have UEFI firmware. + +config DMI + bool "Enable support for SMBIOS (DMI) tables" + depends on EFI + default y + help + This enables SMBIOS/DMI feature for systems. + + This option is only useful on systems that have UEFI firmware. + However, even with this option, the resultant kernel should + continue to boot on existing non-UEFI platforms. + + NOTE: This does *NOT* enable or encourage the use of DMI quirks, + i.e., the practice of identifying the platform via DMI to + decide whether certain workarounds for buggy hardware and/or + firmware need to be enabled. This would require the DMI subsystem + to be enabled much earlier than we do on ARM, which is non-trivial. + +config CMDLINE_BOOL + bool "Built-in kernel command line" + help + Allow for specifying boot arguments to the kernel at + build time. On some systems (e.g. embedded ones), it is + necessary or convenient to provide some or all of the + kernel boot arguments with the kernel itself (that is, + to not rely on the boot loader to provide them.) + + To compile command line arguments into the kernel, + set this option to 'Y', then fill in the + boot arguments in CONFIG_CMDLINE. + + Systems with fully functional boot loaders (i.e. non-embedded) + should leave this option set to 'N'. + +config CMDLINE + string "Built-in kernel command string" + depends on CMDLINE_BOOL + default "" + help + Enter arguments here that should be compiled into the kernel + image and used at boot time. If the boot loader provides a + command line at boot time, it is appended to this string to + form the full kernel command line, when the system boots. + + However, you can use the CONFIG_CMDLINE_OVERRIDE option to + change this behavior. + + In most cases, the command line (whether built-in or provided + by the boot loader) should specify the device for the root + file system. + +config CMDLINE_OVERRIDE + bool "Built-in command line overrides boot loader arguments" + depends on CMDLINE_BOOL + help + Set this option to 'Y' to have the kernel ignore the boot loader + command line, and use ONLY the built-in command line. + + This is used to work around broken boot loaders. This should + be set to 'N' under normal conditions. + +config FORCE_MAX_ZONEORDER + int + default "16" if (HUGETLB_PAGE) + default "11" + help + The kernel memory allocator divides physically contiguous memory + blocks into "zones", where each zone is a power of two number of + pages. This option selects the largest power of two that the kernel + keeps in the memory allocator. If you need to allocate very large + blocks of physically contiguous memory, then you may need to + increase this value. + + This config option is actually maximum order plus one. For example, + a value of 11 means that the largest free memory block is 2^10 pages. + + We make sure that we can allocate up to a HugePage size for each configuration. + Hence we have : + MAX_ORDER = (PMD_SHIFT - PAGE_SHIFT) + 1 => PAGE_SHIFT - 2 + +endmenu + +source "drivers/firmware/Kconfig" + +menu "Power management options" + +source "kernel/power/Kconfig" + +source "drivers/acpi/Kconfig" + +config ARCH_SUSPEND_POSSIBLE + depends on SW64 + def_bool y + +config ARCH_HIBERNATION_POSSIBLE + depends on SW64 + def_bool y + +source "drivers/cpuidle/Kconfig" + +source "drivers/idle/Kconfig" + +endmenu + +source "arch/sw_64/kvm/Kconfig" diff --git a/arch/sw_64/Kconfig.debug b/arch/sw_64/Kconfig.debug new file mode 100644 index 000000000000..6cb3c2488b36 --- /dev/null +++ b/arch/sw_64/Kconfig.debug @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: GPL-2.0 +config EARLY_PRINTK + bool "Early printk" if EXPERT + depends on SYS_HAS_EARLY_PRINTK + default y + help + This option enables special console drivers which allow the kernel + to print messages very early in the bootup process. + + This is useful for kernel debugging when your machine crashes very + early before the console code is initialized. For normal operation, + it is not recommended because it looks ugly on some machines and + doesn't cooperate with an X server. You should normally say N here, + unless you want to debug such a crash. + +config UNA_PRINT + bool "Show debug info about user unalign memory access" + default n + +config MATHEMU + tristate "Kernel FP software completion" if DEBUG_KERNEL && !SMP + default y if !DEBUG_KERNEL || SMP + help + This option is required for IEEE compliant floating point arithmetic + on the SW. The only time you would ever not say Y is to say M in + order to debug the code. Say Y unless you know what you are doing. + +config STACKTRACE_SUPPORT + bool + default y + +config SW64_RRU + bool "Enable RRU(Remote Read User)" + depends on SW64 + default n + help + Duplicate user stdout and stderr to specific space. + Do not enable it in a production kernel. + +config SW64_RRK + bool "Enable RRK(Remote Read Kernel)" + depends on SW64 + default y + help + Duplicate kernel log to specific space. + Do not enable it in a production kernel. + +config DEBUG_MATCH + bool "instruction-flow and data-flow match debugfs interface" + depends on DEBUG_FS + default n + help + Turns on the DebugFS interface for instruction-flow and data-flow match. diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile new file mode 100644 index 000000000000..84f0dca5e9f7 --- /dev/null +++ b/arch/sw_64/Makefile @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# sw/Makefile +# +# This file is subject to the terms and conditions of the GNU General Public +# License. See the file "COPYING" in the main directory of this archive +# for more details. +# +# Copyright (C) 1994 by Linus Torvalds +# + + +archscripts: scripts_basic + $(Q)$(MAKE) $(build)=arch/sw_64/tools relocs + +archheaders: + $(Q)$(MAKE) $(build)=arch/sw_64/kernel/syscalls all + +NM := $(NM) -B +CCVERSION := $(shell $(CC) -dumpversion) +LDFLAGS_vmlinux := -static -N #-relax +CHECKFLAGS += -D__sw__ + +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux += --emit-relocs +endif + +CHECKFLAGS += -D__sw__ +cflags-y := -pipe -ffixed-8 -mno-fp-regs #-msmall-data +ifeq ($(CONFIG_SUBARCH_C4),y) + cflags-y += -fsw-rev +endif +cflags-y += $(call cc-option, -fno-jump-tables) + +cflags-y += $(cpuflags-y) + +KBUILD_CFLAGS += $(cflags-y) +KBUILD_DEFCONFIG = xuelang_defconfig + +head-y := arch/sw_64/kernel/head.o + +core-y += arch/sw_64/ +drivers-$(CONFIG_PCI) += arch/sw_64/pci/ +libs-y += arch/sw_64/lib/ + +# export what is needed by arch/sw_64/boot/Makefile +LIBS_Y := $(patsubst %/, %/lib.a, $(libs-y)) +export LIBS_Y + +boot := arch/sw_64/boot + +#Default target when executing make with no arguments +all: $(boot)/vmlinux.bin.gz + +$(boot)/vmlinux.bin.gz: vmlinux + $(Q)$(MAKE) $(build)=$(boot) $@ + +bootimage bootpfile bootpzfile: vmlinux + $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ + +archclean: + $(Q)$(MAKE) $(clean)=$(boot) + $(Q)$(MAKE) $(clean)=arch/sw_64/tools + +KBUILD_IMAGE := $(boot)/vmlinux.bin + +define archhelp + echo '* boot - Compressed kernel image (arch/sw_64/boot/vmlinux.bin.gz)' +endef diff --git a/arch/sw_64/Makefile.postlink b/arch/sw_64/Makefile.postlink new file mode 100644 index 000000000000..248844d141dd --- /dev/null +++ b/arch/sw_64/Makefile.postlink @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: GPL-2.0 +# =========================================================================== +# Post-link SW64 pass +# =========================================================================== +# +# 1. Insert relocations into vmlinux + +PHONY := __archpost +__archpost: + +-include include/config/auto.conf +include scripts/Kbuild.include + +CMD_RELOCS = arch/sw_64/tools/relocs +quiet_cmd_relocs = RELOCS $@ + cmd_relocs = $(CMD_RELOCS) $@ + +# `@true` prevents complaint when there is nothing to be done + +vmlinux: FORCE + @true +ifeq ($(CONFIG_RELOCATABLE),y) + $(call if_changed,relocs) +endif + +%.ko: FORCE + @true + +clean: + @true + +PHONY += FORCE clean + +FORCE: + +.PHONY: $(PHONY) diff --git a/arch/sw_64/boot/.gitignore b/arch/sw_64/boot/.gitignore new file mode 100644 index 000000000000..8a90e24c76ab --- /dev/null +++ b/arch/sw_64/boot/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +vmlinux diff --git a/arch/sw_64/boot/Makefile b/arch/sw_64/boot/Makefile new file mode 100644 index 000000000000..dd0976484649 --- /dev/null +++ b/arch/sw_64/boot/Makefile @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# arch/sw_64/boot/Makefile +# +# +# This file is subject to the terms and conditions of the GNU General Public +# License. See the file "COPYING" in the main directory of this archive +# for more details. +# +# Based on arch/arm64/boot/Makefile. +# + +OBJCOPYFLAGS_vmlinux.bin := -O binary + +targets := vmlinux vmlinux.bin vmlinux.bin.gz + +quiet_cmd_strip = STRIP $@ + cmd_strip = $(STRIP) -o $@ $< + +# Compressed kernel image +$(obj)/vmlinux.bin.gz: $(obj)/vmlinux.bin FORCE + $(call if_changed,gzip) + @echo ' Kernel $@ is ready' + +$(obj)/vmlinux: vmlinux FORCE + $(call if_changed,strip) + +$(obj)/vmlinux.bin: $(obj)/vmlinux FORCE + $(call if_changed,objcopy) diff --git a/arch/sw_64/boot/dts/Makefile b/arch/sw_64/boot/dts/Makefile new file mode 100644 index 000000000000..e32c159cab64 --- /dev/null +++ b/arch/sw_64/boot/dts/Makefile @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0 +# Built-in dtb + +ifeq ($(CONFIG_PLATFORM_XUELANG),y) +builtindtb-y := chip3 +endif + +ifeq ($(CONFIG_PLATFORM_JUNZHANG),y) +builtindtb-y := empty +endif + +ifeq ($(CONFIG_BUILTIN_DTB), y) +ifneq ($(CONFIG_BUILTIN_DTB_NAME),"") + builtindtb-y := $(patsubst "%",%,$(CONFIG_BUILTIN_DTB_NAME)) +endif + +obj-y += $(builtindtb-y).dtb.o +dtb-y := $(builtindtb-y).dtb + +# for CONFIG_OF_ALL_DTBS test +dtstree := $(srctree)/$(src) +dtb- := $(patsubst $(dtstree)/%.dts,%.dtb, $(wildcard $(dtstree)/*.dts)) +else +dtb-y := $(builtindtb-y).dtb +endif + +clean-files := *.dtb *.dtb.S diff --git a/arch/sw_64/include/asm/Kbuild b/arch/sw_64/include/asm/Kbuild new file mode 100644 index 000000000000..0dd0a704d8f1 --- /dev/null +++ b/arch/sw_64/include/asm/Kbuild @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 + +generic-y += clkdev.h +generic-y += export.h +generic-y += kvm_types.h +generic-y += mcs_spinlock.h +generic-y += param.h +generic-y += qrwlock.h +generic-y += qspinlock.h +generic-y += rwsem.h +generic-y += seccomp.h +generic-y += segment.h +generic-y += types.h +generic-y += user.h + +generated-y += syscall_table.h diff --git a/arch/sw_64/include/uapi/asm/Kbuild b/arch/sw_64/include/uapi/asm/Kbuild new file mode 100644 index 000000000000..15700040f138 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/Kbuild @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +# UAPI Header export list + +generic-y += kvm_para.h +generated-y += unistd_64.h diff --git a/arch/sw_64/kernel/.gitignore b/arch/sw_64/kernel/.gitignore new file mode 100644 index 000000000000..46c9537c5551 --- /dev/null +++ b/arch/sw_64/kernel/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +vmlinux.lds diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile new file mode 100644 index 000000000000..abf27ad19a94 --- /dev/null +++ b/arch/sw_64/kernel/Makefile @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux kernel. +# + +extra-y := vmlinux.lds +asflags-y := $(KBUILD_CFLAGS) +ccflags-y := -Wno-sign-compare + +ifdef CONFIG_FTRACE +CFLAGS_REMOVE_ftrace.o = -pg +CFLAGS_REMOVE_insn.o = -pg +CFLAGS_REMOVE_printk.o = -pg +endif + +obj-y := entry.o fpu.o traps.o process.o sys_sw64.o irq.o \ + irq_sw64.o signal.o setup.o ptrace.o time.o \ + systbls.o dup_print.o chip_setup.o \ + insn.o early_init.o topology.o cacheinfo.o \ + vdso.o vdso/ hmcall.o stacktrace.o idle.o reset.o \ + head.o termios.o + +obj-$(CONFIG_SUBARCH_C3B) += tc.o +obj-$(CONFIG_ACPI) += acpi.o +obj-$(CONFIG_SMP) += smp.o +obj-$(CONFIG_MODULES) += module.o +obj-$(CONFIG_PM) += pm.o +obj-$(CONFIG_SUSPEND) += suspend_asm.o suspend.o +obj-$(CONFIG_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_HIBERNATION) += hibernate_asm.o hibernate.o +obj-$(CONFIG_AUDIT) += audit.o +obj-$(CONFIG_RELOCATABLE) += relocate.o +obj-$(CONFIG_DEBUG_FS) += segvdbg.o unaligned.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o +obj-$(CONFIG_DEBUG_MATCH) += match.o + +ifndef CONFIG_PCI +obj-y += pci-noop.o +endif + +# Core logic support +obj-$(CONFIG_SW64_CPUAUTOPLUG) += cpuautoplug.o + +obj-$(CONFIG_CRASH_DUMP) += crash_dump.o +obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o +obj-$(CONFIG_KPROBES) += kprobes/ +obj-$(CONFIG_UPROBES) += uprobes.o +obj-$(CONFIG_EARLY_PRINTK) += early_printk.o +obj-$(CONFIG_KGDB) += kgdb.o +obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o diff --git a/arch/sw_64/kernel/vmlinux.lds.S b/arch/sw_64/kernel/vmlinux.lds.S new file mode 100644 index 000000000000..9b81b2c7afb8 --- /dev/null +++ b/arch/sw_64/kernel/vmlinux.lds.S @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#define RUNTIME_DISCARD_EXIT +#define EMITS_PT_NOTE +#define RO_EXCEPTION_TABLE_ALIGN 16 + +#include +#include +#include +#include +#include + +OUTPUT_FORMAT("elf64-sw_64") +OUTPUT_ARCH(sw_64) +ENTRY(__start) +PHDRS { text PT_LOAD; note PT_NOTE; } +jiffies = jiffies_64; +SECTIONS +{ + . = _TEXT_START; + + __start = .; + _text = .; /* Text and read-only data */ + _stext = .; + .text : { + HEAD_TEXT + TEXT_TEXT + SCHED_TEXT + LOCK_TEXT + IRQENTRY_TEXT + SOFTIRQENTRY_TEXT + KPROBES_TEXT + *(.fixup) + *(.gnu.warning) + } :text + _etext = .; /* End of text section */ + + RO_DATA(PAGE_SIZE) + + /* Will be freed after init */ + __init_begin = ALIGN(PAGE_SIZE); + INIT_TEXT_SECTION(PAGE_SIZE) + INIT_DATA_SECTION(16) + /* we have to discard exit text and such at runtime, not link time */ + .exit.text : + { + EXIT_TEXT + } + .exit.data : + { + EXIT_DATA + } + PERCPU_SECTION(L1_CACHE_BYTES) + + /* + * Align to THREAD_SIZE rather than PAGE_SIZE here so any padding page + * needed for the THREAD_SIZE aligned init_task gets freed after init + */ + . = ALIGN(THREAD_SIZE); + __init_end = .; + /* Freed after init ends here */ + + _sdata = .; /* Start of rw data section */ + _data = .; + RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) + .got : { +#ifdef CONFIG_RELOCATABLE + _got_start = .; +#endif + *(.got) +#ifdef CONFIG_RELOCATABLE + _got_end = .; +#endif + } + .sdata : { + *(.sdata) + } + _edata = .; /* End of data section */ + +#ifdef CONFIG_RELOCATABLE + _. = ALIGN(4); + .data.reloc : { + _relocation_start = .; + /* + * Space for relocation table + * This needs to be filled so that the + * relocs tool can overwrite the content. + * An invalid value is left at the start of the + * section to abort relocation if the table + * has not been filled in. + */ + LONG(0xFFFFFFFF); + FILL(0); + . += CONFIG_RELOCATION_TABLE_SIZE - 4; + _relocation_end = .; + } +#endif + BSS_SECTION(0, 0, 0) + _end = .; + + .mdebug 0 : { + *(.mdebug) + } + .note 0 : { + *(.note) + } + + STABS_DEBUG + DWARF_DEBUG + ELF_DETAILS + + DISCARDS +} -- Gitee From 03d5fac615979a72c39a2fea2466d4dff1c81f7c Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:05 +0800 Subject: [PATCH 002/524] sw64: add CPU definition headers Add common headers (CPU definition) for basic SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/core.h | 86 ++++++++++++++++++++++++ arch/sw_64/include/asm/cpu.h | 5 ++ arch/sw_64/include/asm/csr.h | 97 ++++++++++++++++++++++++++++ arch/sw_64/include/uapi/asm/regdef.h | 45 +++++++++++++ 4 files changed, 233 insertions(+) create mode 100644 arch/sw_64/include/asm/core.h create mode 100644 arch/sw_64/include/asm/cpu.h create mode 100644 arch/sw_64/include/asm/csr.h create mode 100644 arch/sw_64/include/uapi/asm/regdef.h diff --git a/arch/sw_64/include/asm/core.h b/arch/sw_64/include/asm/core.h new file mode 100644 index 000000000000..2b6748cec93d --- /dev/null +++ b/arch/sw_64/include/asm/core.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CORE_H +#define _ASM_SW64_CORE_H + +#include + +#define II_II0 0 +#define II_II1 1 +#define II_SLEEP 2 +#define II_WAKE 3 +#define II_NMII 6 + +#define II_RESET II_NMII + +#if defined(CONFIG_SUBARCH_C3B) + +#define DOMAIN_ID_BITS 2 +#define DOMAIN_ID_SHIFT 5 + +#define THREAD_ID_BITS 1 +#define THREAD_ID_SHIFT 31 + +#define CORE_ID_BITS 5 +#define CORE_ID_SHIFT 0 + +static inline bool core_is_ht(void) +{ + return 0; +} + +#elif defined(CONFIG_SUBARCH_C4) + +#define DOMAIN_ID_BITS 2 +#define DOMAIN_ID_SHIFT 12 + +#define THREAD_ID_BITS 1 +#define THREAD_ID_SHIFT 8 + +#define CORE_ID_BITS 6 +#define CORE_ID_SHIFT 0 + +static inline bool core_is_ht(void) +{ + return rdhtctl() == 0x3; +} + +#endif + +#define DOMAIN_ID_MASK (GENMASK(DOMAIN_ID_BITS - 1, 0) << DOMAIN_ID_SHIFT) +#define THREAD_ID_MASK (GENMASK(THREAD_ID_BITS - 1, 0) << THREAD_ID_SHIFT) +#define CORE_ID_MASK (GENMASK(CORE_ID_BITS - 1, 0) << CORE_ID_SHIFT) +#define MAX_CORES_PER_CPU (1 << CORE_ID_BITS) + +/* + * 0x00 ~ 0xff for hardware mm fault + */ + +#define MMCSR__TNV 0x0 +#define MMCSR__IACV 0x1 +#define MMCSR__FOR 0x2 +#define MMCSR__FOE 0x3 +#define MMCSR__FOW 0x4 + +#define MMCSR__BAD_DVA 0x6 +#define MMCSR__ACV1 0x7 +#define MMCSR__ACV0 0xc +#define MMCSR__BAD_IVA 0xf + +/* 0x100 ~ 0x1ff for match debug */ +#define MMCSR__DA_MATCH 0x100 +#define MMCSR__DV_MATCH 0x101 +#define MMCSR__DAV_MATCH 0x102 +#define MMCSR__IA_MATCH 0x103 +#define MMCSR__IDA_MATCH 0x104 +#define MMCSR__IV_MATCH 0x105 + + /* entry.S */ +extern void entArith(void); +extern void entIF(void); +extern void entInt(void); +extern void entMM(void); +extern void entSys(void); +extern void entUna(void); +/* head.S */ +extern void __smp_callin(unsigned long args); +#endif /* _ASM_SW64_CORE_H */ diff --git a/arch/sw_64/include/asm/cpu.h b/arch/sw_64/include/asm/cpu.h new file mode 100644 index 000000000000..4da30bb91d89 --- /dev/null +++ b/arch/sw_64/include/asm/cpu.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CPU_H +#define _ASM_SW64_CPU_H + +#endif /* _ASM_SW64_CPU_H */ diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h new file mode 100644 index 000000000000..0610384208a4 --- /dev/null +++ b/arch/sw_64/include/asm/csr.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CSR_H +#define _ASM_SW64_CSR_H + +#include + +#define CSR_EXC_SUM 0xd +#define CSR_INT_EN 0x1a +#define CSR_INT_STAT 0x1b +#define CSR_PCIE_MSI0_INT 0x1d +#define CSR_PCIE_MSI1_INT 0x1e +#define CSR_PCIE_MSI2_INT 0x1f +#define CSR_PCIE_MSI3_INT 0x20 +#define CSR_INT_VEC 0x2d +#define CSR_PCIE_MSI0_INTEN 0x35 +#define CSR_PCIE_MSI1_INTEN 0x36 +#define CSR_PCIE_MSI2_INTEN 0x37 +#define CSR_PCIE_MSI3_INTEN 0x38 +#define CSR_EXC_GPA 0x3b +#define CSR_EXC_PC 0xe +#define CSR_AS_INFO 0x3c +#define CSR_DS_STAT 0x48 +#define CSR_SOFTCID 0xc9 +#define CSR_DVA 0x54 +#define CSR_PTBR_SYS 0x68 +#define CSR_PTBR_USR 0x69 +#define CSR_APTP 0x6a +#define CSR_CID 0xc4 +#define CSR_WR_FREGS 0xc8 +#define CSR_SHTCLOCK 0xca +#define CSR_SHTCLOCK_OFFSET 0xcb + +#ifdef CONFIG_SUBARCH_C4 +#define CSR_IA_VPNMATCH 0xa +#define CSR_UPCR 0x15 +#define CSR_VPCR 0x16 +#define CSR_IA_MATCH 0x17 +#define CSR_IA_MASK 0x18 +#define CSR_IV_MATCH 0x19 +#define CSR_IA_UPNMATCH 0x3a +#define CSR_DC_CTLP 0x4e +#define CSR_DA_MATCH 0x51 +#define CSR_DA_MASK 0x52 +#define CSR_DA_MATCH_MODE 0x53 +#define CSR_DV_MATCH 0x56 +#define CSR_DV_MASK 0x57 +#define CSR_IDA_MATCH 0xc5 +#define CSR_IDA_MASK 0xc6 + +#define DA_MATCH_EN_S 4 +#define DV_MATCH_EN_S 6 +#define DAV_MATCH_EN_S 7 +#define DPM_MATCH 8 +#define DPM_MATCH_EN_S 10 +#define IDA_MATCH_EN_S 53 +#define IV_PM_EN_S 61 +#define IV_MATCH_EN_S 62 +#define IA_MATCH_EN_S 63 + +#endif + + +#ifdef CONFIG_HAVE_CSRRW +#define read_csr(x) \ + ({ unsigned long __val; \ + __asm__ __volatile__("csrr %0,%1" : "=r"(__val) : "i"(x)); \ + __val; }) + +#define write_csr(x, y) \ + ({ __asm__ __volatile__("csrw %0,%1" ::"r"(x), "i"(y)); }) + +#define write_csr_imb(x, y) \ + ({ __asm__ __volatile__("csrw %0,%1; imemb" ::"r"(x), "i"(y)); }) + + +#ifndef __ASSEMBLY__ +#include +static inline void update_ptbr_sys(unsigned long ptbr) +{ + imemb(); + write_csr_imb(ptbr, CSR_PTBR_SYS); +} +#endif +#else +#define read_csr(x) (0) +#define write_csr(x, y) do { } while (0) +#define write_csr_imb(x, y) do { } while (0) + +#ifndef __ASSEMBLY__ +static inline void update_ptbr_sys(unsigned long ptbr) +{ + wrptbr(ptbr); +} +#endif + +#endif +#endif /* _ASM_SW64_CSR_H */ diff --git a/arch/sw_64/include/uapi/asm/regdef.h b/arch/sw_64/include/uapi/asm/regdef.h new file mode 100644 index 000000000000..7460a987c726 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/regdef.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_REGDEF_H +#define _UAPI_ASM_SW64_REGDEF_H + +#define v0 $0 /* function return value */ + +#define t0 $1 /* temporary registers (caller-saved) */ +#define t1 $2 +#define t2 $3 +#define t3 $4 +#define t4 $5 +#define t5 $6 +#define t6 $7 +#define t7 $8 + +#define s0 $9 /* saved-registers (callee-saved registers) */ +#define s1 $10 +#define s2 $11 +#define s3 $12 +#define s4 $13 +#define s5 $14 +#define s6 $15 +#define fp s6 /* frame-pointer (s6 in frame-less procedures) */ + +#define a0 $16 /* argument registers (caller-saved) */ +#define a1 $17 +#define a2 $18 +#define a3 $19 +#define a4 $20 +#define a5 $21 + +#define t8 $22 /* more temps (caller-saved) */ +#define t9 $23 +#define t10 $24 +#define t11 $25 +#define ra $26 /* return address register */ +#define t12 $27 + +#define pv t12 /* procedure-variable register */ +#define AT $at /* assembler temporary */ +#define gp $29 /* global pointer */ +#define sp $30 /* stack pointer */ +#define zero $31 /* reads as zero, writes are noops */ + +#endif /* _UAPI_ASM_SW64_REGDEF_H */ -- Gitee From 3c3e8267b2a280958dd996f6abcefd8f1d2d4566 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:01 +0800 Subject: [PATCH 003/524] sw64: add atomic/locking headers Add common headers (atomic, bitops, barrier and locking) for basic SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/atomic.h | 547 +++++++++++++++++++++++++++++ arch/sw_64/include/asm/barrier.h | 30 ++ arch/sw_64/include/asm/bitops.h | 566 +++++++++++++++++++++++++++++++ arch/sw_64/include/asm/cmpxchg.h | 73 ++++ arch/sw_64/include/asm/percpu.h | 19 ++ arch/sw_64/include/asm/xchg.h | 485 ++++++++++++++++++++++++++ 6 files changed, 1720 insertions(+) create mode 100644 arch/sw_64/include/asm/atomic.h create mode 100644 arch/sw_64/include/asm/barrier.h create mode 100644 arch/sw_64/include/asm/bitops.h create mode 100644 arch/sw_64/include/asm/cmpxchg.h create mode 100644 arch/sw_64/include/asm/percpu.h create mode 100644 arch/sw_64/include/asm/xchg.h diff --git a/arch/sw_64/include/asm/atomic.h b/arch/sw_64/include/asm/atomic.h new file mode 100644 index 000000000000..4a68da09722c --- /dev/null +++ b/arch/sw_64/include/asm/atomic.h @@ -0,0 +1,547 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_ATOMIC_H +#define _ASM_SW64_ATOMIC_H + +#include +#include +#include + +/* + * Atomic operations that C can't guarantee us. Useful for + * resource counting etc... + * + * But use these as seldom as possible since they are much slower + * than regular operations. + */ + +#define ATOMIC_INIT(i) { (i) } +#define ATOMIC64_INIT(i) { (i) } + +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic64_read(v) READ_ONCE((v)->counter) + +#define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i)) +#define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i)) + +/* + * To get proper branch prediction for the main line, we must branch + * forward to code at the end of this object's .text section, then + * branch back to restart the operation. + */ +#define arch_atomic64_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), old, new)) +#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new)) + +#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), old, new)) +#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) + + +#ifdef CONFIG_SUBARCH_C3B +/** + * arch_atomic_fetch_add_unless - add unless the number is a given value + * @v: pointer of type atomic_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns the old value of @v. + */ +static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + int old, new, c; + unsigned long addr; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldw %0, 0(%3)\n" + " cmpeq %0, %5, %4\n" + " seleq %4, 1, $31, %4\n" + " wr_f %4\n" + " addw %0, %6, %1\n" + " lstw %1, 0(%3)\n" + " rd_f %1\n" + " beq %4, 2f\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (old), "=&r" (new), "=m" (v->counter), "=&r" (addr), "=&r" (c) + : "Ir" (u), "Ir" (a), "m" (v->counter)); + return old; +} +/** + * arch_atomic64_fetch_add_unless - add unless the number is a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns the old value of @v. + */ +static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u) +{ + long old, new, c; + unsigned long addr; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldl %0, 0(%3)\n" + " cmpeq %0, %5, %4\n" + " seleq %4, 1, $31, %4\n" + " wr_f %4\n" + " addl %0, %6, %1\n" + " lstl %1, 0(%3)\n" + " rd_f %1\n" + " beq %4, 2f\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (old), "=&r" (new), "=m" (v->counter), "=&r" (addr), "=&r" (c) + : "Ir" (u), "Ir" (a), "m" (v->counter)); + return old; +} +/* + * arch_atomic64_dec_if_positive - decrement by 1 if old value positive + * @v: pointer of type atomic_t + * + * The function returns the old value of *v minus 1, even if + * the atomic variable, v, was not decremented. + */ +static inline long arch_atomic64_dec_if_positive(atomic64_t *v) +{ + unsigned long old, temp1, addr, temp2; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldl %4, 0(%3)\n" + " cmple %4, 0, %0\n" + " seleq %0, 1, $31, %0\n" + " wr_f %0\n" + " subl %4, 1, %1\n" + " lstl %1, 0(%3)\n" + " rd_f %1\n" + " beq %0, 2f\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr), "=&r" (old) + : "m" (v->counter)); + return old - 1; +} + + + +#define ATOMIC_OP(op, asm_op) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ +{ \ + unsigned long temp1, temp2, addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldw %0, 0(%3)\n" \ + " ldi %1, 1\n" \ + " wr_f %1\n" \ + " " #asm_op " %0, %4, %0\n" \ + " lstw %0, 0(%3)\n" \ + " rd_f %0\n" \ + " beq %0, 2f\n" \ + ".subsection 2\n" \ + "2: br 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ +} \ + + +#define ATOMIC_OP_RETURN(op, asm_op) \ +static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ +{ \ + int temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldw %0, 0(%3)\n" \ + " ldi %1, 1\n" \ + " wr_f %1\n" \ + " " #asm_op " %0, %4, %1\n" \ + " " #asm_op " %0, %4, %0\n" \ + " lstw %1, 0(%3)\n" \ + " rd_f %1\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: br 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + + + +#define ATOMIC_FETCH_OP(op, asm_op) \ +static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ +{ \ + int temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldw %0, 0(%3)\n" \ + " ldi %1, 1\n" \ + " wr_f %1\n" \ + " " #asm_op " %0, %4, %1\n" \ + " lstw %1, 0(%3)\n" \ + " rd_f %1\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: br 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + + +#define ATOMIC64_OP(op, asm_op) \ +static inline void arch_atomic64_##op(long i, atomic64_t *v) \ +{ \ + unsigned long temp1, temp2, addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldl %0, 0(%3)\n" \ + " ldi %1, 1\n" \ + " wr_f %1\n" \ + " " #asm_op " %0, %4, %0\n" \ + " lstl %0, 0(%3)\n" \ + " rd_f %0\n" \ + " beq %0, 2f\n" \ + ".subsection 2\n" \ + "2: br 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ +} \ + + +#define ATOMIC64_OP_RETURN(op, asm_op) \ +static inline long arch_atomic64_##op##_return_relaxed(long i, atomic64_t *v)\ +{ \ + long temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldl %0, 0(%3)\n" \ + " ldi %1, 1\n" \ + " wr_f %1\n" \ + " " #asm_op " %0, %4, %1\n" \ + " " #asm_op " %0, %4, %0\n" \ + " lstl %1, 0(%3)\n" \ + " rd_f %1\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: br 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} + +#define ATOMIC64_FETCH_OP(op, asm_op) \ +static inline long arch_atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ +{ \ + long temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldl %0, 0(%3)\n" \ + " ldi %1, 1\n" \ + " wr_f %1\n" \ + " " #asm_op " %0, %4, %1\n" \ + " lstl %1, 0(%3)\n" \ + " rd_f %1\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: br 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + +#else /* !CONFIG_SUBARCH_C3B */ + +/** + * arch_atomic_fetch_add_unless - add unless the number is a given value + * @v: pointer of type atomic_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns the old value of @v. + */ +static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + int old, new, c; + unsigned long addr; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldw %0, 0(%3)\n" + " cmpeq %0, %5, %4\n" + " bne %4, 2f\n" + " addw %0, %6, %1\n" + " lstw %1, 0(%3)\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (old), "=&r" (new), "=m" (v->counter), "=&r" (addr), "=&r" (c) + : "Ir" (u), "Ir" (a), "m" (v->counter)); + return old; +} + +/** + * arch_atomic64_fetch_add_unless - add unless the number is a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns the old value of @v. + */ +static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u) +{ + long old, new, c; + unsigned long addr; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldl %0, 0(%3)\n" + " cmpeq %0, %5, %4\n" + " bne %4, 2f\n" + " addl %0, %6, %1\n" + " lstl %1, 0(%3)\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (old), "=&r" (new), "=m" (v->counter), "=&r" (addr), "=&r" (c) + : "Ir" (u), "Ir" (a), "m" (v->counter)); + return old; +} + +/* + * arch_atomic64_dec_if_positive - decrement by 1 if old value positive + * @v: pointer of type atomic_t + * + * The function returns the old value of *v minus 1, even if + * the atomic variable, v, was not decremented. + */ +static inline long arch_atomic64_dec_if_positive(atomic64_t *v) +{ + unsigned long old, temp1, addr, temp2; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldl %4, 0(%3)\n" + " cmple %4, 0, %0\n" + " bne %0, 2f\n" + " subl %4, 1, %1\n" + " lstl %1, 0(%3)\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr), "=&r" (old) + : "m" (v->counter)); + return old - 1; +} + +#define ATOMIC_OP(op, asm_op) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ +{ \ + unsigned long temp1, addr; \ + __asm__ __volatile__( \ + " ldi %2, %1\n" \ + "1: lldw %0, 0(%2)\n" \ + " " #asm_op " %0, %3, %0\n" \ + " lstw %0, 0(%2)\n" \ + " beq %0, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ +} \ + + +#define ATOMIC_OP_RETURN(op, asm_op) \ +static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ +{ \ + int temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldw %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " " #asm_op " %0, %4, %0\n" \ + " lstw %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + +#define ATOMIC_FETCH_OP(op, asm_op) \ +static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ +{ \ + int temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldw %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " lstw %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + + +#define ATOMIC64_OP(op, asm_op) \ +static inline void arch_atomic64_##op(long i, atomic64_t *v) \ +{ \ + unsigned long temp1, addr; \ + __asm__ __volatile__( \ + " ldi %2, %1\n" \ + "1: lldl %0, 0(%2)\n" \ + " " #asm_op " %0, %3, %0\n" \ + " lstl %0, 0(%2)\n" \ + " beq %0, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ +} \ + + +#define ATOMIC64_OP_RETURN(op, asm_op) \ +static inline long arch_atomic64_##op##_return_relaxed(long i, atomic64_t *v)\ +{ \ + long temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldl %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " " #asm_op " %0, %4, %0\n" \ + " lstl %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} + +#define ATOMIC64_FETCH_OP(op, asm_op) \ +static inline long arch_atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ +{ \ + long temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldl %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " lstl %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + +#endif /* CONFIG_SUBARCH_C3B */ + +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive + +#define ATOMIC_OPS(op) \ + ATOMIC_OP(op, op##w) \ + ATOMIC_OP_RETURN(op, op##w) \ + ATOMIC_FETCH_OP(op, op##w) \ + ATOMIC64_OP(op, op##l) \ + ATOMIC64_OP_RETURN(op, op##l) \ + ATOMIC64_FETCH_OP(op, op##l) \ + +ATOMIC_OPS(add) +ATOMIC_OPS(sub) + +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed + +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed + + + + +#undef ATOMIC_OPS + +#define ATOMIC_OPS(op, asm) \ + ATOMIC_OP(op, asm) \ + ATOMIC_FETCH_OP(op, asm) \ + ATOMIC64_OP(op, asm) \ + ATOMIC64_FETCH_OP(op, asm) \ + + +ATOMIC_OPS(and, and) +ATOMIC_OPS(andnot, bic) +ATOMIC_OPS(or, bis) +ATOMIC_OPS(xor, xor) + + +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed + +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed + + +#undef ATOMIC_OPS +#undef ATOMIC64_FETCH_OP +#undef ATOMIC64_OP_RETURN +#undef ATOMIC64_OP +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP + +#define arch_atomic_andnot arch_atomic_andnot +#define arch_atomic64_andnot arch_atomic64_andnot + +#endif /* _ASM_SW64_ATOMIC_H */ diff --git a/arch/sw_64/include/asm/barrier.h b/arch/sw_64/include/asm/barrier.h new file mode 100644 index 000000000000..bff199126c9f --- /dev/null +++ b/arch/sw_64/include/asm/barrier.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_BARRIER_H +#define _ASM_SW64_BARRIER_H + +#include + +#define mb() __asm__ __volatile__("memb" : : : "memory") + +#define rmb() __asm__ __volatile__("memb" : : : "memory") + +#if defined(CONFIG_SUBARCH_C3B) +#define wmb() __asm__ __volatile__("memb" : : : "memory") +#elif defined(CONFIG_SUBARCH_C4) +#define wmb() __asm__ __volatile__("wmemb" : : : "memory") +#endif + +#define imemb() __asm__ __volatile__("imemb" : : : "memory") + +#ifdef CONFIG_SMP +#define __ASM_SMP_MB "\tmemb\n" +#else +#define __ASM_SMP_MB +#endif + +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() + +#include + +#endif /* _ASM_SW64_BARRIER_H */ diff --git a/arch/sw_64/include/asm/bitops.h b/arch/sw_64/include/asm/bitops.h new file mode 100644 index 000000000000..b3cdabd95abf --- /dev/null +++ b/arch/sw_64/include/asm/bitops.h @@ -0,0 +1,566 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_BITOPS_H +#define _ASM_SW64_BITOPS_H + +#ifndef _LINUX_BITOPS_H +#error only can be included directly +#endif + +#include +#include + +#ifdef CONFIG_SUBARCH_C3B +/* + * These have to be done with inline assembly: that way the bit-setting + * is guaranteed to be atomic. All bit operations return 0 if the bit + * was cleared before the operation and != 0 if it was not. + * + * To get proper branch prediction for the main line, we must branch + * forward to code at the end of this object's .text section, then + * branch back to restart the operation. + * + * bit 0 is the LSB of addr; bit 64 is the LSB of (addr+1). + */ + +static inline void +set_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, temp2, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " ldi %1, 1\n" + " wr_f %1\n" + " bis %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " rd_f %0\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: br 1b\n" + ".previous" + : "=&r" (temp1), "=&r" (temp2), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + + +static inline void +clear_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, temp2, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " ldi %1, 1\n" + " wr_f %1\n" + " bic %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " rd_f %0\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: br 1b\n" + ".previous" + : "=&r" (temp1), "=&r" (temp2), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline void +change_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, temp2, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " ldi %1, 1\n" + " wr_f %1\n" + " xor %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " rd_f %0\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: br 1b\n" + ".previous" + : "=&r" (temp1), "=&r" (temp2), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline int +test_and_set_bit(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, temp2, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %4, %6\n" + "1: lldw %0, 0(%4)\n" + " and %0, %5, %3\n" + " seleq %3, 1, $31, %1\n" + " wr_f %1\n" + " bis %0, %5, %0\n" + " lstw %0, 0(%4)\n" + " rd_f %0\n" + " bne %3, 2f\n" // %3 is not zero, no need to set, return + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (temp1), "=&r" (temp2), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +static inline int +test_and_set_bit_lock(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, temp2, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %4, %6\n" + "1: lldw %0, 0(%4)\n" + " and %0, %5, %3\n" + " seleq %3, 1, $31, %1\n" + " wr_f %1\n" + " bis %0, %5, %0\n" + " lstw %0, 0(%4)\n" + " rd_f %0\n" + " bne %3, 2f\n" // %3 is not zero, no need to set, return + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (temp1), "=&r" (temp2), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +static inline int +test_and_clear_bit(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, temp2, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %4, %6\n" + "1: lldw %0, 0(%4)\n" + " and %0, %5, %3\n" + " selne %3, 1, $31, %1\n" //Note: here is SELNE!!! + " wr_f %1\n" + " bic %0, %5, %0\n" + " lstw %0, 0(%4)\n" + " rd_f %0\n" + " beq %3, 2f\n" // %3 is zero, no need to set, return + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (temp1), "=&r" (temp2), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +static inline int +test_and_change_bit(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " ldi %2, 1\n" + " wr_f %2\n" + " and %0, %4, %2\n" + " xor %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " rd_f %0\n" + " beq %0, 3f\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (temp), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +#else /* !CONFIG_SUBARCH_C3B */ +static inline void +set_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %2, %4\n" + "1: lldw %0, 0(%2)\n" + " bis %0, %3, %0\n" + " lstw %0, 0(%2)\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline void +clear_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %2, %4\n" + "1: lldw %0, 0(%2)\n" + " bic %0, %3, %0\n" + " lstw %0, 0(%2)\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline void +change_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %2, %4\n" + "1: lldw %0, 0(%2)\n" + " xor %0, %3, %0\n" + " lstw %0, 0(%2)\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline int +test_and_set_bit(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " and %0, %4, %2\n" + " bne %2, 2f\n" // %2 is not zero, no need to set, return + " bis %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +static inline int +test_and_set_bit_lock(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " and %0, %4, %2\n" + " bne %2, 2f\n" // %2 is not zero, no need to set, return + " bis %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +static inline int +test_and_clear_bit(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " and %0, %4, %2\n" + " beq %2, 2f\n" // %2 is zero, no need to set, return + " bic %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +static inline int +test_and_change_bit(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " and %0, %4, %2\n" + " xor %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " beq %0, 3f\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + + +#endif /* CONFIG_SUBARCH_C3B */ + +/* + * WARNING: non atomic version. + */ +static __always_inline void +arch___set_bit(unsigned long nr, volatile unsigned long *addr) +{ + int *m = ((int *) addr) + (nr >> 5); + + *m |= 1 << (nr & 31); +} + +#define smp_mb__before_clear_bit() smp_mb() +#define smp_mb__after_clear_bit() smp_mb() + +static inline void +clear_bit_unlock(unsigned long nr, volatile void *addr) +{ + smp_mb(); + clear_bit(nr, addr); +} + +static __always_inline void +arch___clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + int *m = ((int *) addr) + (nr >> 5); + + *m &= ~(1 << (nr & 31)); +} + +static inline void +__clear_bit_unlock(unsigned long nr, volatile void *addr) +{ + smp_mb(); + arch___clear_bit(nr, addr); +} + +static __always_inline void +arch___change_bit(unsigned long nr, volatile unsigned long *addr) +{ + int *m = ((int *) addr) + (nr >> 5); + + *m ^= 1 << (nr & 31); +} + +static __always_inline bool +arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = 1 << (nr & 0x1f); + int *m = ((int *) addr) + (nr >> 5); + int old = *m; + + *m = old | mask; + return (old & mask) != 0; +} + +static __always_inline bool +arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = 1 << (nr & 0x1f); + int *m = ((int *) addr) + (nr >> 5); + int old = *m; + + *m = old & ~mask; + return (old & mask) != 0; +} + +static __always_inline bool +arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long mask = 1 << (nr & 0x1f); + int *m = ((int *) addr) + (nr >> 5); + int old = *m; + + *m = old ^ mask; + return (old & mask) != 0; +} + +#define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire + +/* + * ffz = Find First Zero in word. Undefined if no zero exists, + * so code should check against ~0UL first.. + * + * Do a binary search on the bits. Due to the nature of large + * constants on the sw64, it is worthwhile to split the search. + */ +static inline unsigned long ffz_b(unsigned long x) +{ + unsigned long sum, x1, x2, x4; + + x = ~x & -~x; /* set first 0 bit, clear others */ + x1 = x & 0xAA; + x2 = x & 0xCC; + x4 = x & 0xF0; + sum = x2 ? 2 : 0; + sum += (x4 != 0) * 4; + sum += (x1 != 0); + + return sum; +} + +static inline unsigned long ffz(unsigned long word) +{ + return __kernel_cttz(~word); +} + +/* + * __ffs = Find First set bit in word. Undefined if no set bit exists. + */ +static inline unsigned long __ffs(unsigned long word) +{ + return __kernel_cttz(word); +} + +#ifdef __KERNEL__ + +/* + * ffs: find first bit set. This is defined the same way as + * the libc and compiler builtin ffs routines, therefore + * differs in spirit from the above __ffs. + */ + +static inline int ffs(int word) +{ + int result = __ffs(word) + 1; + + return word ? result : 0; +} + +/* + * fls: find last bit set. + */ +static inline int fls64(unsigned long word) +{ + return 64 - __kernel_ctlz(word); +} + +static inline unsigned long __fls(unsigned long x) +{ + return fls64(x) - 1; +} + +static inline int fls(int x) +{ + return fls64((unsigned int) x); +} + +/* + * hweightN: returns the hamming weight (i.e. the number + * of bits set) of a N-bit word + */ + +static inline unsigned long __arch_hweight64(unsigned long w) +{ + return __kernel_ctpop(w); +} + +static inline unsigned int __arch_hweight32(unsigned int w) +{ + return __arch_hweight64(w); +} + +static inline unsigned int __arch_hweight16(unsigned int w) +{ + return __arch_hweight64(w & 0xffff); +} + +static inline unsigned int __arch_hweight8(unsigned int w) +{ + return __arch_hweight64(w & 0xff); +} + +#include + +#endif /* __KERNEL__ */ + +#ifdef __KERNEL__ + +/* + * Every architecture must define this function. It's the fastest + * way of searching a 100-bit bitmap. It's guaranteed that at least + * one of the 100 bits is cleared. + */ +static inline unsigned long +sched_find_first_bit(const unsigned long b[2]) +{ + unsigned long b0, b1, ofs, tmp; + + b0 = b[0]; + b1 = b[1]; + ofs = (b0 ? 0 : 64); + tmp = (b0 ? b0 : b1); + + return __ffs(tmp) + ofs; +} + +#include + +#include + +#include + +#endif /* __KERNEL__ */ + +#endif /* _ASM_SW64_BITOPS_H */ diff --git a/arch/sw_64/include/asm/cmpxchg.h b/arch/sw_64/include/asm/cmpxchg.h new file mode 100644 index 000000000000..9f51d035313d --- /dev/null +++ b/arch/sw_64/include/asm/cmpxchg.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CMPXCHG_H +#define _ASM_SW64_CMPXCHG_H + +/* + * Atomic exchange routines. + */ + +#define __ASM__MB +#define ____xchg(type, args...) __arch_xchg ## type ## _local(args) +#define ____cmpxchg(type, args...) __cmpxchg ## type ## _local(args) +#include + +#define arch_xchg_local(ptr, x) \ +({ \ + __typeof__(*(ptr)) _x_ = (x); \ + (__typeof__(*(ptr))) __arch_xchg_local((ptr), (unsigned long)_x_, \ + sizeof(*(ptr))); \ +}) + +#define arch_cmpxchg_local(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \ + (unsigned long)_n_, \ + sizeof(*(ptr))); \ +}) + +#define arch_cmpxchg64_local(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + arch_cmpxchg_local((ptr), (o), (n)); \ +}) + +#ifdef CONFIG_SMP +#undef __ASM__MB +#define __ASM__MB "\tmemb\n" +#endif +#undef ____xchg +#undef ____cmpxchg +#undef _ASM_SW64_XCHG_H +#define ____xchg(type, args...) __arch_xchg ##type(args) +#define ____cmpxchg(type, args...) __cmpxchg ##type(args) +#include + +#define arch_xchg(ptr, x) \ +({ \ + __typeof__(*(ptr)) _x_ = (x); \ + (__typeof__(*(ptr))) __arch_xchg((ptr), (unsigned long)_x_, \ + sizeof(*(ptr))); \ +}) + +#define arch_cmpxchg(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ + (unsigned long)_n_, sizeof(*(ptr)));\ +}) + +#define arch_cmpxchg64(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + arch_cmpxchg((ptr), (o), (n)); \ +}) + +#undef __ASM__MB +#undef ____cmpxchg + +#define __HAVE_ARCH_CMPXCHG 1 + +#endif /* _ASM_SW64_CMPXCHG_H */ diff --git a/arch/sw_64/include/asm/percpu.h b/arch/sw_64/include/asm/percpu.h new file mode 100644 index 000000000000..3acdf36bcf55 --- /dev/null +++ b/arch/sw_64/include/asm/percpu.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_PERCPU_H +#define _ASM_SW64_PERCPU_H + +/* + * To calculate addresses of locally defined variables, GCC uses + * 32-bit displacement from the GP. Which doesn't work for per cpu + * variables in modules, as an offset to the kernel per cpu area is + * way above 4G. + * + * Always use weak definitions for percpu variables in modules. + */ +#if defined(MODULE) && defined(CONFIG_SMP) +#define ARCH_NEEDS_WEAK_PER_CPU +#endif + +#include + +#endif /* _ASM_SW64_PERCPU_H */ diff --git a/arch/sw_64/include/asm/xchg.h b/arch/sw_64/include/asm/xchg.h new file mode 100644 index 000000000000..38f067d5ed04 --- /dev/null +++ b/arch/sw_64/include/asm/xchg.h @@ -0,0 +1,485 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_XCHG_H +#define _ASM_SW64_XCHG_H + +#ifndef _ASM_SW64_CMPXCHG_H +#error Do not include xchg.h directly. Use cmpxchg.h +#endif +/* + * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code + * except that local version do not have the expensive memory barrier. + * So this file is included twice from asm/cmpxchg.h. + */ + +#if defined(CONFIG_SUBARCH_C3B) +/* + * Atomic exchange. + * Since it can be used to implement critical sections + * it must clobber "memory" (also for interrupts in UP). + */ + +static inline unsigned long +____xchg(_u8, volatile char *m, unsigned long val) +{ + unsigned long ret, tmp, addr64; + + __asm__ __volatile__( + + " andnot %4, 7, %3\n" + " inslb %1, %4, %1\n" + "1: lldl %2, 0(%3)\n" + " ldi %0, 1\n" + " wr_f %0\n" + " extlb %2, %4, %0\n" + " masklb %2, %4, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%3)\n" + " rd_f %2\n" + " beq %2, 2f\n" + ".subsection 2\n" + "2: br 1b\n" + ".previous" + : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) + : "r" ((long)m), "1" (val) : "memory"); + + return ret; +} + +static inline unsigned long +____xchg(_u16, volatile short *m, unsigned long val) +{ + unsigned long ret, tmp, addr64; + + __asm__ __volatile__( + " andnot %4, 7, %3\n" + " inslh %1, %4, %1\n" + "1: lldl %2, 0(%3)\n" + " ldi %0, 1\n" + " wr_f %0\n" + " extlh %2, %4, %0\n" + " masklh %2, %4, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%3)\n" + " rd_f %2\n" + " beq %2, 2f\n" + ".subsection 2\n" + "2: br 1b\n" + ".previous" + : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) + : "r" ((long)m), "1" (val) : "memory"); + + return ret; +} + +static inline unsigned long +____xchg(_u32, volatile int *m, unsigned long val) +{ + unsigned long dummy, addr; + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " ldi %1, 1\n" + " wr_f %1\n" + " bis $31, %4, %1\n" + " lstw %1, 0(%3)\n" + " rd_f %1\n" + " beq %1, 2f\n" + ".subsection 2\n" + "2: br 1b\n" + ".previous" + : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) + : "rI" (val), "m" (*m) : "memory"); + + return val; +} + +static inline unsigned long +____xchg(_u64, volatile long *m, unsigned long val) +{ + unsigned long dummy, addr; + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldl %0, 0(%3)\n" + " ldi %1, 1\n" + " wr_f %1\n" + " bis $31, %4, %1\n" + " lstl %1, 0(%3)\n" + " rd_f %1\n" + " beq %1, 2f\n" + ".subsection 2\n" + "2: br 1b\n" + ".previous" + : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) + : "rI" (val), "m" (*m) : "memory"); + + return val; +} + +/* + * Atomic compare and exchange. Compare OLD with MEM, if identical, + * store NEW in MEM. Return the initial value in MEM. Success is + * indicated by comparing RETURN with OLD. + * + * The memory barrier should be placed in SMP only when we actually + * make the change. If we don't change anything (so if the returned + * prev is equal to old) then we aren't acquiring anything new and + * we don't need any memory barrier as far I can tell. + */ + +static inline unsigned long +____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new) +{ + unsigned long prev, tmp, cmp, addr64; + + __asm__ __volatile__( + " andnot %5, 7, %4\n" + " inslb %1, %5, %1\n" + "1: lldl %2, 0(%4)\n" + " extlb %2, %5, %0\n" + " cmpeq %0, %6, %3\n" + " wr_f %3\n" + " masklb %2, %5, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%4)\n" + " rd_f %2\n" + " beq %3, 2f\n" + " beq %2, 3f\n" + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) + : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new) +{ + unsigned long prev, tmp, cmp, addr64; + + __asm__ __volatile__( + " andnot %5, 7, %4\n" + " inslh %1, %5, %1\n" + "1: lldl %2, 0(%4)\n" + " extlh %2, %5, %0\n" + " cmpeq %0, %6, %3\n" + " wr_f %3\n" + " masklh %2, %5, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%4)\n" + " rd_f %2\n" + " beq %3, 2f\n" + " beq %2, 3f\n" + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) + : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u32, volatile int *m, int old, int new) +{ + unsigned long prev, cmp, addr, tmp; + + __asm__ __volatile__( + " ldi %3, %7\n" + "1: lldw %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " wr_f %1\n" + " bis $31, %6, %4\n" + " lstw %4, 0(%3)\n" + " rd_f %4\n" + " beq %1, 2f\n" + " beq %4, 3f\n" + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) + : "r"((long) old), "r"(new), "m"(*m) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) +{ + unsigned long prev, cmp, addr, tmp; + + __asm__ __volatile__( + " ldi %3, %7\n" + "1: lldl %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " wr_f %1\n" + " bis $31, %6, %4\n" + " lstl %4, 0(%3)\n" + " rd_f %4\n" + " beq %1, 2f\n" + " beq %4, 3f\n" + "2:\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) + : "r"((long) old), "r"(new), "m"(*m) : "memory"); + + return prev; +} + +#elif defined(CONFIG_SUBARCH_C4) +/* + * Atomic exchange. + * Since it can be used to implement critical sections + * it must clobber "memory" (also for interrupts in UP). + */ + +static inline unsigned long +____xchg(_u8, volatile char *m, unsigned long val) +{ + unsigned long ret, tmp, addr64; + + __asm__ __volatile__( + " andnot %4, 7, %3\n" + " inslb %1, %4, %1\n" + "1: lldl %2, 0(%3)\n" + " extlb %2, %4, %0\n" + " masklb %2, %4, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%3)\n" + " beq %2, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) + : "r" ((long)m), "1" (val) : "memory"); + + return ret; +} + +static inline unsigned long +____xchg(_u16, volatile short *m, unsigned long val) +{ + unsigned long ret, tmp, addr64; + + __asm__ __volatile__( + " andnot %4, 7, %3\n" + " inslh %1, %4, %1\n" + "1: lldl %2, 0(%3)\n" + " extlh %2, %4, %0\n" + " masklh %2, %4, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%3)\n" + " beq %2, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) + : "r" ((long)m), "1" (val) : "memory"); + + return ret; +} + +static inline unsigned long +____xchg(_u32, volatile int *m, unsigned long val) +{ + unsigned long dummy, addr; + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " bis $31, %4, %1\n" + " lstw %1, 0(%3)\n" + " beq %1, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) + : "rI" (val), "m" (*m) : "memory"); + + return val; +} + +static inline unsigned long +____xchg(_u64, volatile long *m, unsigned long val) +{ + unsigned long dummy, addr; + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldl %0, 0(%3)\n" + " bis $31, %4, %1\n" + " lstl %1, 0(%3)\n" + " beq %1, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) + : "rI" (val), "m" (*m) : "memory"); + + return val; +} + +/* + * Atomic compare and exchange. Compare OLD with MEM, if identical, + * store NEW in MEM. Return the initial value in MEM. Success is + * indicated by comparing RETURN with OLD. + * + * The memory barrier should be placed in SMP only when we actually + * make the change. If we don't change anything (so if the returned + * prev is equal to old) then we aren't acquiring anything new and + * we don't need any memory barrier as far I can tell. + */ +static inline unsigned long +____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new) +{ + unsigned long prev, tmp, cmp, addr64; + + __asm__ __volatile__( + " andnot %5, 7, %4\n" + " inslb %1, %5, %1\n" + "1: lldl %2, 0(%4)\n" + " extlb %2, %5, %0\n" + " cmpeq %0, %6, %3\n" + " beq %3, 2f\n" + " masklb %2, %5, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%4)\n" + " beq %2, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) + : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new) +{ + unsigned long prev, tmp, cmp, addr64; + + __asm__ __volatile__( + " andnot %5, 7, %4\n" + " inslh %1, %5, %1\n" + "1: lldl %2, 0(%4)\n" + " extlh %2, %5, %0\n" + " cmpeq %0, %6, %3\n" + " beq %3, 2f\n" + " masklh %2, %5, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%4)\n" + " beq %2, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) + : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u32, volatile int *m, int old, int new) +{ + unsigned long prev, cmp, addr, tmp; + + __asm__ __volatile__( + " ldi %3, %7\n" + "1: lldw %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " beq %1, 2f\n" + " bis $31, %6, %4\n" + " lstw %4, 0(%3)\n" + " beq %4, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) + : "r"((long) old), "r"(new), "m"(*m) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) +{ + unsigned long prev, cmp, addr, tmp; + + __asm__ __volatile__( + " ldi %3, %7\n" + "1: lldl %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " beq %1, 2f\n" + " bis $31, %6, %4\n" + " lstl %4, 0(%3)\n" + " beq %4, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) + : "r"((long) old), "r"(new), "m"(*m) : "memory"); + + return prev; +} + +#endif + +/* This function doesn't exist, so you'll get a linker error + * if something tries to do an invalid xchg(). + */ +extern void __xchg_called_with_bad_pointer(void); + +static __always_inline unsigned long +____xchg(, volatile void *ptr, unsigned long x, int size) +{ + switch (size) { + case 1: + return ____xchg(_u8, ptr, x); + case 2: + return ____xchg(_u16, ptr, x); + case 4: + return ____xchg(_u32, ptr, x); + case 8: + return ____xchg(_u64, ptr, x); + } + __xchg_called_with_bad_pointer(); + return x; +} + +/* This function doesn't exist, so you'll get a linker error + * if something tries to do an invalid cmpxchg(). + */ +extern void __cmpxchg_called_with_bad_pointer(void); + +static __always_inline unsigned long ____cmpxchg(, volatile void *ptr, + unsigned long old, + unsigned long new, int size) +{ + switch (size) { + case 1: + return ____cmpxchg(_u8, ptr, old, new); + case 2: + return ____cmpxchg(_u16, ptr, old, new); + case 4: + return ____cmpxchg(_u32, ptr, old, new); + case 8: + return ____cmpxchg(_u64, ptr, old, new); + } + __cmpxchg_called_with_bad_pointer(); + return old; +} + +#endif /* _ASM_SW64_XCHG_H */ -- Gitee From d28251af0fe3cbaa7957b89ec618e1c048660a16 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:11 +0800 Subject: [PATCH 004/524] sw64: add common headers Add some other common headers for basic SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/asm-offsets.h | 1 + arch/sw_64/include/asm/asm-prototypes.h | 22 ++ arch/sw_64/include/asm/bug.h | 8 + arch/sw_64/include/asm/device.h | 13 ++ arch/sw_64/include/asm/hmcall.h | 236 ++++++++++++++++++++++ arch/sw_64/include/asm/hw_init.h | 167 +++++++++++++++ arch/sw_64/include/asm/idle.h | 7 + arch/sw_64/include/asm/insn.h | 97 +++++++++ arch/sw_64/include/asm/linkage.h | 9 + arch/sw_64/include/asm/word-at-a-time.h | 43 ++++ arch/sw_64/include/uapi/asm/bitsperlong.h | 9 + arch/sw_64/include/uapi/asm/byteorder.h | 7 + arch/sw_64/include/uapi/asm/compiler.h | 83 ++++++++ arch/sw_64/include/uapi/asm/errno.h | 128 ++++++++++++ arch/sw_64/include/uapi/asm/hmcall.h | 17 ++ arch/sw_64/include/uapi/asm/mman.h | 88 ++++++++ arch/sw_64/include/uapi/asm/param.h | 9 + arch/sw_64/include/uapi/asm/setup.h | 7 + 18 files changed, 951 insertions(+) create mode 100644 arch/sw_64/include/asm/asm-offsets.h create mode 100644 arch/sw_64/include/asm/asm-prototypes.h create mode 100644 arch/sw_64/include/asm/bug.h create mode 100644 arch/sw_64/include/asm/device.h create mode 100644 arch/sw_64/include/asm/hmcall.h create mode 100644 arch/sw_64/include/asm/hw_init.h create mode 100644 arch/sw_64/include/asm/idle.h create mode 100644 arch/sw_64/include/asm/insn.h create mode 100644 arch/sw_64/include/asm/linkage.h create mode 100644 arch/sw_64/include/asm/word-at-a-time.h create mode 100644 arch/sw_64/include/uapi/asm/bitsperlong.h create mode 100644 arch/sw_64/include/uapi/asm/byteorder.h create mode 100644 arch/sw_64/include/uapi/asm/compiler.h create mode 100644 arch/sw_64/include/uapi/asm/errno.h create mode 100644 arch/sw_64/include/uapi/asm/hmcall.h create mode 100644 arch/sw_64/include/uapi/asm/mman.h create mode 100644 arch/sw_64/include/uapi/asm/param.h create mode 100644 arch/sw_64/include/uapi/asm/setup.h diff --git a/arch/sw_64/include/asm/asm-offsets.h b/arch/sw_64/include/asm/asm-offsets.h new file mode 100644 index 000000000000..d370ee36a182 --- /dev/null +++ b/arch/sw_64/include/asm/asm-offsets.h @@ -0,0 +1 @@ +#include diff --git a/arch/sw_64/include/asm/asm-prototypes.h b/arch/sw_64/include/asm/asm-prototypes.h new file mode 100644 index 000000000000..67746d6bffb7 --- /dev/null +++ b/arch/sw_64/include/asm/asm-prototypes.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_ASM_PROTOTYPES_H +#define _ASM_SW64_ASM_PROTOTYPES_H + +#include +#include +#include +#include +#include + +#include + +extern void __divl(void); +extern void __reml(void); +extern void __divw(void); +extern void __remw(void); +extern void __divlu(void); +extern void __remlu(void); +extern void __divwu(void); +extern void __remwu(void); + +#endif /* _ASM_SW64_ASM_PROTOTYPES_H */ diff --git a/arch/sw_64/include/asm/bug.h b/arch/sw_64/include/asm/bug.h new file mode 100644 index 000000000000..4a179f236ccf --- /dev/null +++ b/arch/sw_64/include/asm/bug.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_BUG_H +#define _ASM_SW64_BUG_H + +#include +#include + +#endif /* _ASM_SW64_BUG_H */ diff --git a/arch/sw_64/include/asm/device.h b/arch/sw_64/include/asm/device.h new file mode 100644 index 000000000000..d999207e07d1 --- /dev/null +++ b/arch/sw_64/include/asm/device.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_DEVICE_H +#define _ASM_SW64_DEVICE_H + +struct dev_archdata { +#if defined(CONFIG_SUNWAY_IOMMU) || defined(CONFIG_SUNWAY_IOMMU_V2) + void *iommu; +#endif +}; + +struct pdev_archdata { +}; +#endif /* _ASM_SW64_DEVICE_H */ diff --git a/arch/sw_64/include/asm/hmcall.h b/arch/sw_64/include/asm/hmcall.h new file mode 100644 index 000000000000..e3bac3016740 --- /dev/null +++ b/arch/sw_64/include/asm/hmcall.h @@ -0,0 +1,236 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_HMCALL_H +#define _ASM_SW64_HMCALL_H + +/* + * Common HMC-code + */ +/* 0x0 - 0x3F : Kernel Level HMC routine */ +#define HMC_halt 0x00 +#define HMC_rdio64 0x01 +#define HMC_rdio32 0x02 +#define HMC_cpuid 0x03 +#define HMC_sleepen 0x05 +#define HMC_rdksp 0x06 +#define HMC_wrasid 0x08 +#define HMC_rdktp 0x09 +#define HMC_wrktp 0x0A +#define HMC_rdptbr 0x0B +#define HMC_wrptbr 0x0C +#define HMC_rdhtctl 0x0D +#define HMC_wrksp 0x0E +#define HMC_mtinten 0x0F +#define HMC_load_mm 0x11 +#define HMC_tbisasid 0x14 +#define HMC_tbivpn 0x19 +#define HMC_ret 0x1A +#define HMC_wrvpcr 0x29 +#define HMC_wrfen 0x2B +#define HMC_sflush 0x2F +#define HMC_entervm 0x31 +#define HMC_hcall 0x32 +#define HMC_tbi 0x33 +#define HMC_wrent 0x34 +#define HMC_swpipl 0x35 +#define HMC_rdps 0x36 +#define HMC_wrkgp 0x37 +#define HMC_wrusp 0x38 +#define HMC_rvpcr 0x39 +#define HMC_rdusp 0x3A +#define HMC_wrtimer 0x3B +#define HMC_whami 0x3C +#define HMC_retsys 0x3D +#define HMC_sendii 0x3E +#define HMC_rti 0x3F + + +/* 0x80 - 0xBF : User Level HMC routine */ +#include + +/* Following will be deprecated from user level invocation */ +#define HMC_rwreg 0x87 +#define HMC_sz_uflush 0xA8 +#define HMC_longtime 0xB1 + +#ifdef __KERNEL__ +#ifndef __ASSEMBLY__ + +#include +extern void __init fixup_hmcall(void); + +extern void halt(void) __noreturn; + +#define __CALL_HMC_VOID(NAME) \ +static inline void NAME(void) \ +{ \ + __asm__ __volatile__( \ + "sys_call %0 " : : "i" (HMC_ ## NAME)); \ +} + +#define __CALL_HMC_R0(NAME, TYPE) \ +static inline TYPE NAME(void) \ +{ \ + register TYPE __r0 __asm__("$0"); \ + __asm__ __volatile__( \ + "sys_call %1 # " #NAME \ + : "=r" (__r0) \ + : "i" (HMC_ ## NAME) \ + : "$1", "$16", "$22", "$23", "$24", "$25"); \ + return __r0; \ +} + +#define __CALL_HMC_W1(NAME, TYPE0) \ +static inline void NAME(TYPE0 arg0) \ +{ \ + register TYPE0 __r16 __asm__("$16") = arg0; \ + __asm__ __volatile__( \ + "sys_call %1 # "#NAME \ + : "=r"(__r16) \ + : "i"(HMC_ ## NAME), "0"(__r16) \ + : "$1", "$22", "$23", "$24", "$25"); \ +} + +#define __CALL_HMC_W2(NAME, TYPE0, TYPE1) \ +static inline void NAME(TYPE0 arg0, TYPE1 arg1) \ +{ \ + register TYPE0 __r16 __asm__("$16") = arg0; \ + register TYPE1 __r17 __asm__("$17") = arg1; \ + __asm__ __volatile__( \ + "sys_call %2 # "#NAME \ + : "=r"(__r16), "=r"(__r17) \ + : "i"(HMC_ ## NAME), "0"(__r16), "1"(__r17) \ + : "$1", "$22", "$23", "$24", "$25"); \ +} + +#define __CALL_HMC_RW1(NAME, RTYPE, TYPE0) \ +static inline RTYPE NAME(TYPE0 arg0) \ +{ \ + register RTYPE __r0 __asm__("$0"); \ + register TYPE0 __r16 __asm__("$16") = arg0; \ + __asm__ __volatile__( \ + "sys_call %2 # "#NAME \ + : "=r"(__r16), "=r"(__r0) \ + : "i"(HMC_ ## NAME), "0"(__r16) \ + : "$1", "$22", "$23", "$24", "$25"); \ + return __r0; \ +} + +#define __CALL_HMC_RW2(NAME, RTYPE, TYPE0, TYPE1) \ +static inline RTYPE NAME(TYPE0 arg0, TYPE1 arg1) \ +{ \ + register RTYPE __r0 __asm__("$0"); \ + register TYPE0 __r16 __asm__("$16") = arg0; \ + register TYPE1 __r17 __asm__("$17") = arg1; \ + __asm__ __volatile__( \ + "sys_call %3 # "#NAME \ + : "=r"(__r16), "=r"(__r17), "=r"(__r0) \ + : "i"(HMC_ ## NAME), "0"(__r16), "1"(__r17) \ + : "$1", "$22", "$23", "$24", "$25"); \ + return __r0; \ +} + +#define __CALL_HMC_RW3(NAME, RTYPE, TYPE0, TYPE1, TYPE2) \ +static inline RTYPE NAME(TYPE0 arg0, TYPE1 arg1, TYPE2 arg2) \ +{ \ + register RTYPE __r0 __asm__("$0"); \ + register TYPE0 __r16 __asm__("$16") = arg0; \ + register TYPE1 __r17 __asm__("$17") = arg1; \ + register TYPE2 __r18 __asm__("$18") = arg2; \ + __asm__ __volatile__( \ + "sys_call %4 # "#NAME \ + : "=r"(__r16), "=r"(__r17), "=r"(__r18), "=r"(__r0) \ + : "i"(HMC_ ## NAME), "0"(__r16), "1"(__r17), "2"(__r18) \ + : "$1", "$22", "$23", "$24", "$25"); \ + return __r0; \ +} + + +__CALL_HMC_VOID(imb); +__CALL_HMC_VOID(sflush); +__CALL_HMC_VOID(wrfen); +#define fpu_enable() wrfen() + +__CALL_HMC_VOID(sleepen); +__CALL_HMC_VOID(mtinten); + +__CALL_HMC_VOID(rdktp); +#define restore_ktp() rdktp() +__CALL_HMC_VOID(wrktp); +#define save_ktp() wrktp() + +__CALL_HMC_R0(rdps, unsigned long); + +__CALL_HMC_R0(rdusp, unsigned long); +__CALL_HMC_W1(wrusp, unsigned long); + +__CALL_HMC_R0(rdksp, unsigned long); +__CALL_HMC_W1(wrksp, unsigned long); +__CALL_HMC_R0(rdhtctl, unsigned long); + +/* + * Load a mm context. This is needed when we change the page + * table pointer(CSR:PTBR) or when we update the ASID. + * load_mm(asid, ptbr) + * + */ +__CALL_HMC_W2(load_mm, unsigned long, unsigned long); + +__CALL_HMC_W1(wrasid, unsigned long); +__CALL_HMC_R0(rdptbr, unsigned long); +__CALL_HMC_W1(wrptbr, unsigned long); + +__CALL_HMC_RW1(swpipl, unsigned long, unsigned long); +__CALL_HMC_R0(whami, unsigned long); +__CALL_HMC_RW1(rdio64, unsigned long, unsigned long); +__CALL_HMC_RW1(rdio32, unsigned int, unsigned long); +__CALL_HMC_W2(wrent, void*, unsigned long); +__CALL_HMC_W2(tbisasid, unsigned long, unsigned long); +__CALL_HMC_W1(wrkgp, unsigned long); +__CALL_HMC_RW2(wrperfmon, unsigned long, unsigned long, unsigned long); +__CALL_HMC_RW3(sendii, unsigned long, unsigned long, unsigned long, unsigned long); +__CALL_HMC_W1(wrtimer, unsigned long); +__CALL_HMC_RW3(tbivpn, unsigned long, unsigned long, unsigned long, unsigned long); +__CALL_HMC_RW2(cpuid, unsigned long, unsigned long, unsigned long); + +__CALL_HMC_W1(wrtp, unsigned long); +/* + * TB routines.. + */ +#define __tbi(nr, arg, arg1...) \ +({ \ + register unsigned long __r16 __asm__("$16") = (nr); \ + register unsigned long __r17 __asm__("$17"); arg; \ + __asm__ __volatile__( \ + "sys_call %3 #__tbi" \ + : "=r" (__r16), "=r" (__r17) \ + : "0" (__r16), "i" (HMC_tbi), ##arg1 \ + : "$0", "$1", "$22", "$23", "$24", "$25"); \ +}) + +#define tbi(x, y) __tbi(x, __r17 = (y), "1" (__r17)) + +/* Invalidate all TLB, only used by hypervisor */ +#define tbia() __tbi(-2, /* no second argument */) + +/* Invalidate TLB for all processes with currnet VPN */ +#define tbivp() __tbi(-1, /* no second argument */) + +/* Invalidate all TLB with current VPN */ +#define tbiv() __tbi(0, /* no second argument */) + +/* Invalidate ITLB of addr with current UPN and VPN */ +#define tbisi(addr) __tbi(1, __r17 = (addr), "1" (__r17)) + +/* Invalidate DTLB of addr with current UPN and VPN */ +#define tbisd(addr) __tbi(2, __r17 = (addr), "1" (__r17)) + +/* Invalidate TLB of addr with current UPN and VPN */ +#define tbis(addr) __tbi(3, __r17 = (addr), "1" (__r17)) + +/* Invalidate all user TLB with current UPN and VPN */ +#define tbiu() __tbi(4, /* no second argument */) + +#endif /* !__ASSEMBLY__ */ +#endif /* __KERNEL__ */ + +#endif /* _ASM_SW64_HMCALL_H */ diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h new file mode 100644 index 000000000000..2078c66d1c4f --- /dev/null +++ b/arch/sw_64/include/asm/hw_init.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_HW_INIT_H +#define _ASM_SW64_HW_INIT_H +#include +#include + +#include + +#define MMSIZE __va(0x2040) + +/* + * Descriptor for a cache + */ +struct cache_desc { + unsigned int size; /* Bytes per way */ + unsigned int sets; /* Number of lines per set */ + unsigned char ways; /* Number of ways */ + unsigned char linesz; /* Size of line in bytes */ + unsigned char flags; /* Flags describing cache properties */ +}; + +struct cpuinfo_sw64 { + unsigned long last_asid; + unsigned long last_vpn; + unsigned long ipi_count; + struct cache_desc icache; /* Primary I-cache */ + struct cache_desc dcache; /* Primary D or combined I/D cache */ + struct cache_desc scache; /* Secondary cache */ + struct cache_desc tcache; /* Tertiary/split secondary cache */ +} __aligned(SMP_CACHE_BYTES); + +struct cpu_desc_t { + __u8 model; + __u8 family; + __u8 chip_var; + __u8 arch_var; + __u8 arch_rev; + __u8 pa_bits; + __u8 va_bits; + char vendor_id[16]; + char model_id[64]; + unsigned long frequency; +} __randomize_layout; + +#define MAX_NUMSOCKETS 8 +struct socket_desc_t { + bool is_online; /* 1 for online, 0 for offline */ + int numcores; + unsigned long socket_mem; +}; + +enum memmap_types { + memmap_reserved, + memmap_pci, + memmap_initrd, + memmap_kvm, + memmap_crashkernel, + memmap_acpi, + memmap_use, + memmap_protected, +}; + +#define MAX_NUMMEMMAPS 64 +struct memmap_entry { + u64 addr; /* start of memory segment */ + u64 size; /* size of memory segment */ + enum memmap_types type; +}; + +extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; +extern void store_cpu_data(int cpu); + +extern struct cpu_desc_t cpu_desc; +extern struct socket_desc_t socket_desc[MAX_NUMSOCKETS]; +extern int memmap_nr; +extern struct memmap_entry memmap_map[MAX_NUMMEMMAPS]; +extern cpumask_t cpu_offline; +extern bool memblock_initialized; + +int __init add_memmap_region(u64 addr, u64 size, enum memmap_types type); +void __init process_memmap(void); + +static inline unsigned long get_cpu_freq(void) +{ + return cpu_desc.frequency; +} + +static inline void update_cpu_freq(unsigned long khz) +{ + cpu_desc.frequency = khz * 1000; +} + +#define EMUL_FLAG (0x1UL << 63) +#define MMSIZE_MASK (EMUL_FLAG - 1) + +DECLARE_STATIC_KEY_TRUE(run_mode_host_key); +DECLARE_STATIC_KEY_FALSE(run_mode_guest_key); +DECLARE_STATIC_KEY_FALSE(run_mode_emul_key); + +#define is_in_host() static_branch_likely(&run_mode_host_key) +#define is_in_guest() static_branch_unlikely(&run_mode_guest_key) +#define is_in_emul() static_branch_unlikely(&run_mode_emul_key) +#define is_guest_or_emul() !static_branch_likely(&run_mode_host_key) + +#define CPU_SW3231 0x31 +#define CPU_SW831 0x32 +#define CPU_SW8A 0x41 + +#define GET_TABLE_ENTRY 1 +#define GET_VENDOR_ID 2 +#define GET_MODEL 3 +#define GET_CPU_FREQ 4 +#define GET_CACHE_INFO 5 + +#define TABLE_ENTRY_MAX 32 +#define VENDOR_ID_MAX 2 +#define MODEL_MAX 8 +#define CACHE_INFO_MAX 4 + +#define L1_ICACHE 0 +#define L1_DCACHE 1 +#define L2_CACHE 2 +#define L3_CACHE 3 + +#define CPUID_ARCH_REV_MASK 0xf +#define CPUID_ARCH_REV(val) ((val) & CPUID_ARCH_REV_MASK) +#define CPUID_ARCH_VAR_SHIFT 4 +#define CPUID_ARCH_VAR_MASK (0xf << CPUID_ARCH_VAR_SHIFT) +#define CPUID_ARCH_VAR(val) \ + (((val) & CPUID_ARCH_VAR_MASK) >> CPUID_ARCH_VAR_SHIFT) +#define CPUID_CHIP_VAR_SHIFT 8 +#define CPUID_CHIP_VAR_MASK (0xf << CPUID_CHIP_VAR_SHIFT) +#define CPUID_CHIP_VAR(val) \ + (((val) & CPUID_CHIP_VAR_MASK) >> CPUID_CHIP_VAR_SHIFT) +#define CPUID_FAMILY_SHIFT 12 +#define CPUID_FAMILY_MASK (0xf << CPUID_FAMILY_SHIFT) +#define CPUID_FAMILY(val) \ + (((val) & CPUID_FAMILY_MASK) >> CPUID_FAMILY_SHIFT) +#define CPUID_MODEL_SHIFT 24 +#define CPUID_MODEL_MASK (0xff << CPUID_MODEL_SHIFT) +#define CPUID_MODEL(val) \ + (((val) & CPUID_MODEL_MASK) >> CPUID_MODEL_SHIFT) +#define CPUID_PA_BITS_SHIFT 32 +#define CPUID_PA_BITS_MASK (0x7fUL << CPUID_PA_BITS_SHIFT) +#define CPUID_PA_BITS(val) \ + (((val) & CPUID_PA_BITS_MASK) >> CPUID_PA_BITS_SHIFT) +#define CPUID_VA_BITS_SHIFT 39 +#define CPUID_VA_BITS_MASK (0x7fUL << CPUID_VA_BITS_SHIFT) +#define CPUID_VA_BITS(val) \ + (((val) & CPUID_VA_BITS_MASK) >> CPUID_VA_BITS_SHIFT) + + +#define CACHE_SIZE_SHIFT 0 +#define CACHE_SIZE_MASK (0xffffffffUL << CACHE_SIZE_SHIFT) +#define CACHE_SIZE(val) \ + (((val) & CACHE_SIZE_MASK) >> CACHE_SIZE_SHIFT) +#define CACHE_LINE_BITS_SHIFT 32 +#define CACHE_LINE_BITS_MASK (0xfUL << CACHE_LINE_BITS_SHIFT) +#define CACHE_LINE_BITS(val) \ + (((val) & CACHE_LINE_BITS_MASK) >> CACHE_LINE_BITS_SHIFT) +#define CACHE_INDEX_BITS_SHIFT 36 +#define CACHE_INDEX_BITS_MASK (0x3fUL << CACHE_INDEX_BITS_SHIFT) +#define CACHE_INDEX_BITS(val) \ + (((val) & CACHE_INDEX_BITS_MASK) >> CACHE_INDEX_BITS_SHIFT) +#define current_cpu_data cpu_data[smp_processor_id()] + +#endif /* _ASM_SW64_HW_INIT_H */ diff --git a/arch/sw_64/include/asm/idle.h b/arch/sw_64/include/asm/idle.h new file mode 100644 index 000000000000..95e145f25306 --- /dev/null +++ b/arch/sw_64/include/asm/idle.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_IDLE_H +#define _ASM_SW64_IDLE_H + +extern void arch_cpu_idle(void); + +#endif /* _ASM_SW64_IDLE_H */ diff --git a/arch/sw_64/include/asm/insn.h b/arch/sw_64/include/asm/insn.h new file mode 100644 index 000000000000..437cb48d1e93 --- /dev/null +++ b/arch/sw_64/include/asm/insn.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019, serveros, linyue + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef _ASM_SW64_INSN_H +#define _ASM_SW64_INSN_H +#include + +/* Register numbers */ +enum { + R26 = 26, + R27, + R28, + R31 = 31, +}; + +#define BR_MAX_DISP 0xfffff +/* SW64 instructions are always 32 bits. */ +#define SW64_INSN_SIZE 4 + +#define ___SW64_RA(a) (((a) & 0x1f) << 21) +#define ___SW64_RB(b) (((b) & 0x1f) << 16) +#define ___SW64_SIMP_RC(c) (((c) & 0x1f)) +#define ___SW64_ST_DISP(disp) (((disp) & 0xffff)) +#define ___SW64_SYSCALL_FUNC(func) ((func) & 0xff) +#define ___SW64_BR_DISP(disp) (((disp) & 0x1fffff)) + + +#define SW64_INSN_BIS 0x40000740 +#define SW64_INSN_CALL 0x04000000 +#define SW64_INSN_SYS_CALL 0x02000000 +#define SW64_INSN_BR 0x10000000 + +#define SW64_NOP (0x43ff075f) +#define SW64_BIS(a, b, c) (SW64_INSN_BIS | ___SW64_RA(a) | ___SW64_RB(b) | ___SW64_SIMP_RC(c)) +#define SW64_CALL(a, b, disp) (SW64_INSN_CALL | ___SW64_RA(a) | ___SW64_RB(b) | ___SW64_ST_DISP(disp)) +#define SW64_SYS_CALL(func) (SW64_INSN_SYS_CALL | ___SW64_SYSCALL_FUNC(func)) +#define SW64_BR(a, disp) (SW64_INSN_BR | ___SW64_RA(a) | ___SW64_BR_DISP(disp)) + +extern int sw64_insn_read(void *addr, u32 *insnp); +extern int sw64_insn_write(void *addr, u32 insn); +extern int sw64_insn_double_write(void *addr, u64 insn); +extern unsigned int sw64_insn_nop(void); +extern unsigned int sw64_insn_call(unsigned int ra, unsigned int rb); +extern unsigned int sw64_insn_sys_call(unsigned int num); +extern unsigned int sw64_insn_br(unsigned int ra, unsigned long pc, unsigned long new_pc); + +#define SW64_OPCODE_RA(opcode) ((opcode >> 21) & 0x1f) + +#define SW64_INSN(name, opcode, mask) \ +static inline bool sw64_insn_is_##name(u32 insn) \ +{ \ + return (insn & mask) == opcode; \ +} + +SW64_INSN(sys_call_b, 0x00000000, 0xfc000000); +SW64_INSN(sys_call, 0x00000001, 0xfc000000); +SW64_INSN(call, 0x04000000, 0xfc000000); +SW64_INSN(ret, 0x08000000, 0xfc000000); +SW64_INSN(jmp, 0x0c000000, 0xfc000000); +SW64_INSN(br, 0x10000000, 0xfc000000); +SW64_INSN(bsr, 0x14000000, 0xfc000000); +SW64_INSN(memb, 0x18000000, 0xfc00ffff); +SW64_INSN(imemb, 0x18000001, 0xfc00ffff); +SW64_INSN(rtc, 0x18000020, 0xfc00ffff); +SW64_INSN(halt, 0x18000080, 0xfc00ffff); +SW64_INSN(rd_f, 0x18001000, 0xfc00ffff); +SW64_INSN(beq, 0xc0000000, 0xfc000000); +SW64_INSN(bne, 0xc4000000, 0xfc000000); +SW64_INSN(blt, 0xc8000000, 0xfc000000); +SW64_INSN(ble, 0xcc000000, 0xfc000000); +SW64_INSN(bgt, 0xd0000000, 0xfc000000); +SW64_INSN(bge, 0xd4000000, 0xfc000000); +SW64_INSN(blbc, 0xd8000000, 0xfc000000); +SW64_INSN(blbs, 0xdc000000, 0xfc000000); +SW64_INSN(fbeq, 0xe0000000, 0xfc000000); +SW64_INSN(fbne, 0xe4000000, 0xfc000000); +SW64_INSN(fblt, 0xe8000000, 0xfc000000); +SW64_INSN(fble, 0xec000000, 0xfc000000); +SW64_INSN(fbgt, 0xf0000000, 0xfc000000); +SW64_INSN(fbge, 0xf4000000, 0xfc000000); +SW64_INSN(lldw, 0x20000000, 0xfc00f000); +SW64_INSN(lldl, 0x20001000, 0xfc00f000); + +#endif /* _ASM_SW64_INSN_H */ diff --git a/arch/sw_64/include/asm/linkage.h b/arch/sw_64/include/asm/linkage.h new file mode 100644 index 000000000000..85b279f6211e --- /dev/null +++ b/arch/sw_64/include/asm/linkage.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_LINKAGE_H +#define _ASM_SW64_LINKAGE_H + +#define cond_syscall(x) asm(".weak\t" #x "\n" #x " = sys_ni_syscall") +#define SYSCALL_ALIAS(alias, name) \ + asm (#alias " = " #name "\n\t.globl " #alias) + +#endif /* _ASM_SW64_LINKAGE_H */ diff --git a/arch/sw_64/include/asm/word-at-a-time.h b/arch/sw_64/include/asm/word-at-a-time.h new file mode 100644 index 000000000000..623efbec4429 --- /dev/null +++ b/arch/sw_64/include/asm/word-at-a-time.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_WORD_AT_A_TIME_H +#define _ASM_SW64_WORD_AT_A_TIME_H + +#include + +/* + * word-at-a-time interface for SW64. + */ + +/* + * We do not use the word_at_a_time struct on SW64, but it needs to be + * implemented to humour the generic code. + */ +struct word_at_a_time { + const unsigned long unused; +}; + +#define WORD_AT_A_TIME_CONSTANTS { 0 } + +/* Return nonzero if val has a zero */ +static inline unsigned long has_zero(unsigned long val, unsigned long *bits, const struct word_at_a_time *c) +{ + unsigned long zero_locations = __kernel_cmpgeb(0, val); + *bits = zero_locations; + return zero_locations; +} + +static inline unsigned long prep_zero_mask(unsigned long val, unsigned long bits, const struct word_at_a_time *c) +{ + return bits; +} + +#define create_zero_mask(bits) (bits) + +static inline unsigned long find_zero(unsigned long bits) +{ + return __kernel_cttz(bits); +} + +#define zero_bytemask(mask) ((2ul << (find_zero(mask) * 8)) - 1) + +#endif /* _ASM_SW64_WORD_AT_A_TIME_H */ diff --git a/arch/sw_64/include/uapi/asm/bitsperlong.h b/arch/sw_64/include/uapi/asm/bitsperlong.h new file mode 100644 index 000000000000..712c823e23d8 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/bitsperlong.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_BITSPERLONG_H +#define _UAPI_ASM_SW64_BITSPERLONG_H + +#define __BITS_PER_LONG 64 + +#include + +#endif /* _UAPI_ASM_SW64_BITSPERLONG_H */ diff --git a/arch/sw_64/include/uapi/asm/byteorder.h b/arch/sw_64/include/uapi/asm/byteorder.h new file mode 100644 index 000000000000..ededdd045e96 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/byteorder.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_BYTEORDER_H +#define _UAPI_ASM_SW64_BYTEORDER_H + +#include + +#endif /* _UAPI_ASM_SW64_BYTEORDER_H */ diff --git a/arch/sw_64/include/uapi/asm/compiler.h b/arch/sw_64/include/uapi/asm/compiler.h new file mode 100644 index 000000000000..64786df0f266 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/compiler.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_COMPILER_H +#define _UAPI_ASM_SW64_COMPILER_H + +/* + * Herein are macros we use when describing various patterns we want to GCC. + * In all cases we can get better schedules out of the compiler if we hide + * as little as possible inside inline assembly. However, we want to be + * able to know what we'll get out before giving up inline assembly. Thus + * these tests and macros. + */ + +#define __kernel_inslb(val, shift) \ +({ \ + unsigned long __kir; \ + __asm__("inslb %2, %1, %0" : "=r"(__kir) : "rI"(shift), "r"(val));\ + __kir; \ +}) + +#define __kernel_inslh(val, shift) \ +({ \ + unsigned long __kir; \ + __asm__("inslh %2, %1, %0" : "=r"(__kir) : "rI"(shift), "r"(val));\ + __kir; \ +}) + +#define __kernel_insll(val, shift) \ +({ \ + unsigned long __kir; \ + __asm__("insll %2, %1, %0" : "=r"(__kir) : "rI"(shift), "r"(val));\ + __kir; \ +}) + +#define __kernel_inshw(val, shift) \ +({ \ + unsigned long __kir; \ + __asm__("inshw %2, %1, %0" : "=r"(__kir) : "rI"(shift), "r"(val));\ + __kir; \ +}) + +#define __kernel_extlb(val, shift) \ +({ \ + unsigned long __kir; \ + __asm__("extlb %2, %1, %0" : "=r"(__kir) : "rI"(shift), "r"(val));\ + __kir; \ +}) + +#define __kernel_extlh(val, shift) \ +({ \ + unsigned long __kir; \ + __asm__("extlh %2, %1, %0" : "=r"(__kir) : "rI"(shift), "r"(val));\ + __kir; \ +}) + +#define __kernel_cmpgeb(a, b) \ +({ \ + unsigned long __kir; \ + __asm__("cmpgeb %r2, %1, %0" : "=r"(__kir) : "rI"(b), "rJ"(a)); \ + __kir; \ +}) + +#define __kernel_cttz(x) \ +({ \ + unsigned long __kir; \ + __asm__("cttz %1, %0" : "=r"(__kir) : "r"(x)); \ + __kir; \ +}) + +#define __kernel_ctlz(x) \ +({ \ + unsigned long __kir; \ + __asm__("ctlz %1, %0" : "=r"(__kir) : "r"(x)); \ + __kir; \ +}) + +#define __kernel_ctpop(x) \ +({ \ + unsigned long __kir; \ + __asm__("ctpop %1, %0" : "=r"(__kir) : "r"(x)); \ + __kir; \ +}) + +#endif /* _UAPI_ASM_SW64_COMPILER_H */ diff --git a/arch/sw_64/include/uapi/asm/errno.h b/arch/sw_64/include/uapi/asm/errno.h new file mode 100644 index 000000000000..969ee99ee86c --- /dev/null +++ b/arch/sw_64/include/uapi/asm/errno.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_ERRNO_H +#define _UAPI_ASM_SW64_ERRNO_H + +#include + +#undef EAGAIN /* 11 in errno-base.h */ + +#define EDEADLK 11 /* Resource deadlock would occur */ + +#define EAGAIN 35 /* Try again */ +#define EWOULDBLOCK EAGAIN /* Operation would block */ +#define EINPROGRESS 36 /* Operation now in progress */ +#define EALREADY 37 /* Operation already in progress */ +#define ENOTSOCK 38 /* Socket operation on non-socket */ +#define EDESTADDRREQ 39 /* Destination address required */ +#define EMSGSIZE 40 /* Message too long */ +#define EPROTOTYPE 41 /* Protocol wrong type for socket */ +#define ENOPROTOOPT 42 /* Protocol not available */ +#define EPROTONOSUPPORT 43 /* Protocol not supported */ +#define ESOCKTNOSUPPORT 44 /* Socket type not supported */ +#define EOPNOTSUPP 45 /* Operation not supported on transport endpoint */ +#define EPFNOSUPPORT 46 /* Protocol family not supported */ +#define EAFNOSUPPORT 47 /* Address family not supported by protocol */ +#define EADDRINUSE 48 /* Address already in use */ +#define EADDRNOTAVAIL 49 /* Cannot assign requested address */ +#define ENETDOWN 50 /* Network is down */ +#define ENETUNREACH 51 /* Network is unreachable */ +#define ENETRESET 52 /* Network dropped connection because of reset */ +#define ECONNABORTED 53 /* Software caused connection abort */ +#define ECONNRESET 54 /* Connection reset by peer */ +#define ENOBUFS 55 /* No buffer space available */ +#define EISCONN 56 /* Transport endpoint is already connected */ +#define ENOTCONN 57 /* Transport endpoint is not connected */ +#define ESHUTDOWN 58 /* Cannot send after transport endpoint shutdown */ +#define ETOOMANYREFS 59 /* Too many references: cannot splice */ +#define ETIMEDOUT 60 /* Connection timed out */ +#define ECONNREFUSED 61 /* Connection refused */ +#define ELOOP 62 /* Too many symbolic links encountered */ +#define ENAMETOOLONG 63 /* File name too long */ +#define EHOSTDOWN 64 /* Host is down */ +#define EHOSTUNREACH 65 /* No route to host */ +#define ENOTEMPTY 66 /* Directory not empty */ + +#define EUSERS 68 /* Too many users */ +#define EDQUOT 69 /* Quota exceeded */ +#define ESTALE 70 /* Stale NFS file handle */ +#define EREMOTE 71 /* Object is remote */ + +#define ENOLCK 77 /* No record locks available */ +#define ENOSYS 78 /* Function not implemented */ + +#define ENOMSG 80 /* No message of desired type */ +#define EIDRM 81 /* Identifier removed */ +#define ENOSR 82 /* Out of streams resources */ +#define ETIME 83 /* Timer expired */ +#define EBADMSG 84 /* Not a data message */ +#define EPROTO 85 /* Protocol error */ +#define ENODATA 86 /* No data available */ +#define ENOSTR 87 /* Device not a stream */ + +#define ENOPKG 92 /* Package not installed */ + +#define EILSEQ 116 /* Illegal byte sequence */ + +/* The following are just random noise.. */ +#define ECHRNG 88 /* Channel number out of range */ +#define EL2NSYNC 89 /* Level 2 not synchronized */ +#define EL3HLT 90 /* Level 3 halted */ +#define EL3RST 91 /* Level 3 reset */ + +#define ELNRNG 93 /* Link number out of range */ +#define EUNATCH 94 /* Protocol driver not attached */ +#define ENOCSI 95 /* No CSI structure available */ +#define EL2HLT 96 /* Level 2 halted */ +#define EBADE 97 /* Invalid exchange */ +#define EBADR 98 /* Invalid request descriptor */ +#define EXFULL 99 /* Exchange full */ +#define ENOANO 100 /* No anode */ +#define EBADRQC 101 /* Invalid request code */ +#define EBADSLT 102 /* Invalid slot */ + +#define EDEADLOCK EDEADLK + +#define EBFONT 104 /* Bad font file format */ +#define ENONET 105 /* Machine is not on the network */ +#define ENOLINK 106 /* Link has been severed */ +#define EADV 107 /* Advertise error */ +#define ESRMNT 108 /* Srmount error */ +#define ECOMM 109 /* Communication error on send */ +#define EMULTIHOP 110 /* Multihop attempted */ +#define EDOTDOT 111 /* RFS specific error */ +#define EOVERFLOW 112 /* Value too large for defined data type */ +#define ENOTUNIQ 113 /* Name not unique on network */ +#define EBADFD 114 /* File descriptor in bad state */ +#define EREMCHG 115 /* Remote address changed */ + +#define EUCLEAN 117 /* Structure needs cleaning */ +#define ENOTNAM 118 /* Not a XENIX named type file */ +#define ENAVAIL 119 /* No XENIX semaphores available */ +#define EISNAM 120 /* Is a named type file */ +#define EREMOTEIO 121 /* Remote I/O error */ + +#define ELIBACC 122 /* Can not access a needed shared library */ +#define ELIBBAD 123 /* Accessing a corrupted shared library */ +#define ELIBSCN 124 /* .lib section in a.out corrupted */ +#define ELIBMAX 125 /* Attempting to link in too many shared libraries */ +#define ELIBEXEC 126 /* Cannot exec a shared library directly */ +#define ERESTART 127 /* Interrupted system call should be restarted */ +#define ESTRPIPE 128 /* Streams pipe error */ + +#define ENOMEDIUM 129 /* No medium found */ +#define EMEDIUMTYPE 130 /* Wrong medium type */ +#define ECANCELED 131 /* Operation Cancelled */ +#define ENOKEY 132 /* Required key not available */ +#define EKEYEXPIRED 133 /* Key has expired */ +#define EKEYREVOKED 134 /* Key has been revoked */ +#define EKEYREJECTED 135 /* Key was rejected by service */ + +/* for robust mutexes */ +#define EOWNERDEAD 136 /* Owner died */ +#define ENOTRECOVERABLE 137 /* State not recoverable */ + +#define ERFKILL 138 /* Operation not possible due to RF-kill */ + +#define EHWPOISON 139 /* Memory page has hardware error */ + +#endif /* _UAPI_ASM_SW64_ERRNO_H */ diff --git a/arch/sw_64/include/uapi/asm/hmcall.h b/arch/sw_64/include/uapi/asm/hmcall.h new file mode 100644 index 000000000000..6867fb7b4d24 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/hmcall.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_HMCALL_H +#define _UAPI_ASM_SW64_HMCALL_H + +/* hmcall may be used in user mode */ + +#define HMC_bpt 0x80 +#define HMC_callsys 0x83 +#define HMC_imb 0x86 +#define HMC_rdtp 0x9E +#define HMC_wrtp 0x9F +#define HMC_rdunique HMC_rdtp +#define HMC_wrunique HMC_wrtp +#define HMC_gentrap 0xAA +#define HMC_wrperfmon 0xB0 + +#endif /* _UAPI_ASM_SW64_HMCALL_H */ diff --git a/arch/sw_64/include/uapi/asm/mman.h b/arch/sw_64/include/uapi/asm/mman.h new file mode 100644 index 000000000000..15cb7bfee3b1 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/mman.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_MMAN_H +#define _UAPI_ASM_SW64_MMAN_H + +#define PROT_READ 0x1 /* page can be read */ +#define PROT_WRITE 0x2 /* page can be written */ +#define PROT_EXEC 0x4 /* page can be executed */ +#define PROT_SEM 0x8 /* page may be used for atomic ops */ +#define PROT_NONE 0x0 /* page can not be accessed */ +#define PROT_GROWSDOWN 0x01000000 /* mprotect flag: extend change to start of growsdown vma */ +#define PROT_GROWSUP 0x02000000 /* mprotect flag: extend change to end of growsup vma */ + +#define MAP_TYPE 0x0f /* Mask for type of mapping */ +#define MAP_FIXED 0x100 /* Interpret addr exactly */ +#define MAP_ANONYMOUS 0x10 /* don't use a file */ + +/* not used by linux, may be deprecated */ +#define _MAP_HASSEMAPHORE 0x0200 +#define _MAP_INHERIT 0x0400 +#define _MAP_UNALIGNED 0x0800 + +/* These are linux-specific */ +#define MAP_GROWSDOWN 0x01000 /* stack-like segment */ +#define MAP_DENYWRITE 0x02000 /* ETXTBSY */ +#define MAP_EXECUTABLE 0x04000 /* mark it as an executable */ +#define MAP_LOCKED 0x08000 /* lock the mapping */ +#define MAP_NORESERVE 0x10000 /* don't check for reservations */ +#define MAP_POPULATE 0x20000 /* populate (prefault) pagetables */ +#define MAP_NONBLOCK 0x40000 /* do not block on IO */ +#define MAP_STACK 0x80000 /* give out an address that is best suited for process/thread stacks */ +#define MAP_HUGETLB 0x100000 /* create a huge page mapping */ +#define MAP_FIXED_NOREPLACE 0x200000 /* MAP_FIXED which doesn't unmap underlying mapping */ + +#define MS_ASYNC 1 /* sync memory asynchronously */ +#define MS_SYNC 2 /* synchronous memory sync */ +#define MS_INVALIDATE 4 /* invalidate the caches */ + +#define MCL_CURRENT 8192 /* lock all currently mapped pages */ +#define MCL_FUTURE 16384 /* lock all additions to address space */ +#define MCL_ONFAULT 32768 /* lock all pages that are faulted in */ + +#define MLOCK_ONFAULT 0x01 /* Lock pages in range after they are faulted in, do not prefault */ + +#define MADV_NORMAL 0 /* no further special treatment */ +#define MADV_RANDOM 1 /* expect random page references */ +#define MADV_SEQUENTIAL 2 /* expect sequential page references */ +#define MADV_WILLNEED 3 /* will need these pages */ +#define MADV_SPACEAVAIL 5 /* ensure resources are available */ +#define MADV_DONTNEED 6 /* don't need these pages */ + +/* common/generic parameters */ +#define MADV_FREE 8 /* free pages only if memory pressure */ +#define MADV_REMOVE 9 /* remove these pages & resources */ +#define MADV_DONTFORK 10 /* don't inherit across fork */ +#define MADV_DOFORK 11 /* do inherit across fork */ + +#define MADV_MERGEABLE 12 /* KSM may merge identical pages */ +#define MADV_UNMERGEABLE 13 /* KSM may not merge identical pages */ + +#define MADV_HUGEPAGE 14 /* Worth backing with hugepages */ +#define MADV_NOHUGEPAGE 15 /* Not worth backing with hugepages */ + +#define MADV_DONTDUMP 16 /* Explicity exclude from the core dump, + overrides the coredump filter bits */ +#define MADV_DODUMP 17 /* Clear the MADV_NODUMP flag */ + +#define MADV_WIPEONFORK 18 /* Zero memory on fork, child only */ +#define MADV_KEEPONFORK 19 /* Undo MADV_WIPEONFORK */ + +#define MADV_COLD 20 /* deactivate these pages */ +#define MADV_PAGEOUT 21 /* reclaim these pages */ + +#define MADV_POPULATE_READ 22 /* populate (prefault) page tables readable */ +#define MADV_POPULATE_WRITE 23 /* populate (prefault) page tables writable */ + +#define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ + +#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ + +/* compatibility flags */ +#define MAP_FILE 0 + + +#define PKEY_DISABLE_ACCESS 0x1 +#define PKEY_DISABLE_WRITE 0x2 +#define PKEY_ACCESS_MASK (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE) + +#endif /* _UAPI_ASM_SW64_MMAN_H */ diff --git a/arch/sw_64/include/uapi/asm/param.h b/arch/sw_64/include/uapi/asm/param.h new file mode 100644 index 000000000000..d38e8202dd97 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/param.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_PARAM_H +#define _UAPI_ASM_SW64_PARAM_H + +#define EXEC_PAGESIZE 8192 + +#include + +#endif /* _UAPI_ASM_SW64_PARAM_H */ diff --git a/arch/sw_64/include/uapi/asm/setup.h b/arch/sw_64/include/uapi/asm/setup.h new file mode 100644 index 000000000000..e6cca4525049 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/setup.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_SETUP_H +#define _UAPI_ASM_SW64_SETUP_H + +#define COMMAND_LINE_SIZE 2048 + +#endif /* _UAPI_ASM_SW64_SETUP_H */ -- Gitee From a06897e39893cc0c2a3fbd0f6c5dd195d562c78c Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 17:14:53 +0800 Subject: [PATCH 005/524] sw64: add ELF support Add ELF-related definition for basic SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/elf.h | 152 +++++++++++++++++++++++++++ arch/sw_64/include/uapi/asm/auxvec.h | 11 ++ 2 files changed, 163 insertions(+) create mode 100644 arch/sw_64/include/asm/elf.h create mode 100644 arch/sw_64/include/uapi/asm/auxvec.h diff --git a/arch/sw_64/include/asm/elf.h b/arch/sw_64/include/asm/elf.h new file mode 100644 index 000000000000..95ba89a1aa9d --- /dev/null +++ b/arch/sw_64/include/asm/elf.h @@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_ELF_H +#define _ASM_SW64_ELF_H +#ifdef __KERNEL__ +#include +#endif +/* Special values for the st_other field in the symbol table. */ + + +#define STO_SW64_NOPV 0x80 +#define STO_SW64_STD_GPLOAD 0x88 + +/* + * SW-64 ELF relocation types + */ +#define R_SW64_NONE 0 /* No reloc */ +#define R_SW64_REFLONG 1 /* Direct 32 bit */ +#define R_SW64_REFQUAD 2 /* Direct 64 bit */ +#define R_SW64_GPREL32 3 /* GP relative 32 bit */ +#define R_SW64_LITERAL 4 /* GP relative 16 bit w/optimization */ +#define R_SW64_LITUSE 5 /* Optimization hint for LITERAL */ +#define R_SW64_GPDISP 6 /* Add displacement to GP */ +#define R_SW64_BRADDR 7 /* PC+4 relative 23 bit shifted */ +#define R_SW64_HINT 8 /* PC+4 relative 16 bit shifted */ +#define R_SW64_SREL16 9 /* PC relative 16 bit */ +#define R_SW64_SREL32 10 /* PC relative 32 bit */ +#define R_SW64_SREL64 11 /* PC relative 64 bit */ +#define R_SW64_GPRELHIGH 17 /* GP relative 32 bit, high 16 bits */ +#define R_SW64_GPRELLOW 18 /* GP relative 32 bit, low 16 bits */ +#define R_SW64_GPREL16 19 /* GP relative 16 bit */ +#define R_SW64_COPY 24 /* Copy symbol at runtime */ +#define R_SW64_GLOB_DAT 25 /* Create GOT entry */ +#define R_SW64_JMP_SLOT 26 /* Create PLT entry */ +#define R_SW64_RELATIVE 27 /* Adjust by program base */ +#define R_SW64_BRSGP 28 +#define R_SW64_TLSGD 29 +#define R_SW64_TLS_LDM 30 +#define R_SW64_DTPMOD64 31 +#define R_SW64_GOTDTPREL 32 +#define R_SW64_DTPREL64 33 +#define R_SW64_DTPRELHI 34 +#define R_SW64_DTPRELLO 35 +#define R_SW64_DTPREL16 36 +#define R_SW64_GOTTPREL 37 +#define R_SW64_TPREL64 38 +#define R_SW64_TPRELHI 39 +#define R_SW64_TPRELLO 40 +#define R_SW64_TPREL16 41 +#define R_SW64_LITERAL_GOT 43 /* GP relative */ + +#define SHF_SW64_GPREL 0x10000000 + +/* Legal values for e_flags field of Elf64_Ehdr. */ + +#define EF_SW64_32BIT 1 /* All addresses are below 2GB */ + +/* + * ELF register definitions. + * + * For now, we just leave it at 33 (32 general regs + processor status word). + */ +#define ELF_NGREG 33 + +typedef unsigned long elf_greg_t; +typedef elf_greg_t elf_gregset_t[ELF_NGREG]; + +/* Same with user_fpsimd_state */ +#include +typedef struct user_fpsimd_state elf_fpregset_t; + +/* + * This is used to ensure we don't load something for the wrong architecture. + */ +#define elf_check_arch(x) ((x)->e_machine == EM_SW64) + +/* + * These are used to set parameters in the core dumps. + */ +#define ELF_CLASS ELFCLASS64 +#define ELF_DATA ELFDATA2LSB +#define ELF_ARCH EM_SW64 + +#define CORE_DUMP_USE_REGSET +#define ELF_EXEC_PAGESIZE PAGE_SIZE + +/* + * This is the location that an ET_DYN program is loaded if exec'ed. Typical + * use of this is to invoke "./ld.so someprog" to test out a new version of + * the loader. We need to make sure that it is out of the way of the program + * that it will "exec", and that there is sufficient room for the brk. + */ + +#define ELF_ET_DYN_BASE (TASK_UNMAPPED_BASE + 0x1000000) + +/* + * $0 is set by ld.so to a pointer to a function which might be + * registered using atexit. This provides a mean for the dynamic + * linker to call DT_FINI functions for shared libraries that have + * been loaded before the code runs. + + * So that we can use the same startup file with static executables, + * we start programs with a value of 0 to indicate that there is no + * such function. + */ + +#define ELF_PLAT_INIT(_r, load_addr) (_r->regs[0] = 0) + +/* + * The registers are laid out in pt_regs for HMCODE and syscall + * convenience. Re-order them for the linear elf_gregset_t. + */ + +#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 +struct linux_binprm; +extern int arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp); + +#ifdef __KERNEL__ +struct pt_regs; +struct task_struct; +extern void sw64_elf_core_copy_regs(elf_greg_t *dest, struct pt_regs *pt); +#define ELF_CORE_COPY_REGS(DEST, REGS) sw64_elf_core_copy_regs(DEST, REGS); + +/* + * This yields a mask that user programs can use to figure out what + * instruction set this CPU supports. + */ + +#define ELF_HWCAP 0 + +/* + * This yields a string that ld.so will use to load implementation + * specific libraries for optimization. This is more specific in + * intent than poking at uname or /proc/cpuinfo. + */ + +#define ELF_PLATFORM ("sw_64") + + +/* update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes */ +#define ARCH_DLINFO \ +do { \ + NEW_AUX_ENT(AT_SYSINFO_EHDR, \ + (elf_addr_t)current->mm->context.vdso); \ +} while (0) + +struct mm_struct; +extern unsigned long arch_randomize_brk(struct mm_struct *mm); +#define arch_randomize_brk arch_randomize_brk +#endif + +#endif /* _ASM_SW64_ELF_H */ diff --git a/arch/sw_64/include/uapi/asm/auxvec.h b/arch/sw_64/include/uapi/asm/auxvec.h new file mode 100644 index 000000000000..309a8294be7a --- /dev/null +++ b/arch/sw_64/include/uapi/asm/auxvec.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_AUXVEC_H +#define _UAPI_ASM_SW64_AUXVEC_H + +/* VDSO location. */ +#define AT_SYSINFO_EHDR 33 + +/* entries in ARCH_DLINFO */ +#define AT_VECTOR_SIZE_ARCH 1 + +#endif /* _UAPI_ASM_SW64_AUXVEC_H */ -- Gitee From 5acd2a315005350e07123a4ecefc7afb735ce5a8 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:13 +0800 Subject: [PATCH 006/524] sw64: add some other headers Add some other uncommon headers for basic SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/ast2400.h | 168 +++++++++++++++++++++++++ arch/sw_64/include/asm/socket.h | 11 ++ arch/sw_64/include/uapi/asm/fcntl.h | 58 +++++++++ arch/sw_64/include/uapi/asm/ioctl.h | 19 +++ arch/sw_64/include/uapi/asm/ioctls.h | 128 +++++++++++++++++++ arch/sw_64/include/uapi/asm/resource.h | 16 +++ arch/sw_64/include/uapi/asm/socket.h | 161 ++++++++++++++++++++++++ arch/sw_64/include/uapi/asm/sockios.h | 17 +++ arch/sw_64/include/uapi/asm/stat.h | 50 ++++++++ arch/sw_64/include/uapi/asm/termbits.h | 167 ++++++++++++++++++++++++ arch/sw_64/include/uapi/asm/termios.h | 70 +++++++++++ 11 files changed, 865 insertions(+) create mode 100644 arch/sw_64/include/asm/ast2400.h create mode 100644 arch/sw_64/include/asm/socket.h create mode 100644 arch/sw_64/include/uapi/asm/fcntl.h create mode 100644 arch/sw_64/include/uapi/asm/ioctl.h create mode 100644 arch/sw_64/include/uapi/asm/ioctls.h create mode 100644 arch/sw_64/include/uapi/asm/resource.h create mode 100644 arch/sw_64/include/uapi/asm/socket.h create mode 100644 arch/sw_64/include/uapi/asm/sockios.h create mode 100644 arch/sw_64/include/uapi/asm/stat.h create mode 100644 arch/sw_64/include/uapi/asm/termbits.h create mode 100644 arch/sw_64/include/uapi/asm/termios.h diff --git a/arch/sw_64/include/asm/ast2400.h b/arch/sw_64/include/asm/ast2400.h new file mode 100644 index 000000000000..5f4cc84ff3a8 --- /dev/null +++ b/arch/sw_64/include/asm/ast2400.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015 Weiqiang Su + * + * Both AST2400D and AST2400F package variants are supported. + */ + +#ifndef _ASM_SW64_AST2400_H +#define _ASM_SW64_AST2400_H + +#include + +/* Logical Device Numbers (LDN). */ +#define AST2400_FDC 0x00 /* Floppy */ +#define AST2400_PP 0x01 /* Parallel port */ +#define AST2400_SP1 0x02 /* Com1 */ +#define AST2400_SP2 0x03 /* Com2 & IR */ +#define AST2400_KBC 0x05 /* PS/2 keyboard and mouse */ +#define AST2400_CIR 0x06 +#define AST2400_GPIO6789_V 0x07 +#define AST2400_WDT1_GPIO01A_V 0x08 +#define AST2400_GPIO1234567_V 0x09 +#define AST2400_ACPI 0x0A +#define AST2400_HWM_FPLED 0x0B /* Hardware monitor & front LED */ +#define AST2400_VID 0x0D +#define AST2400_CIRWKUP 0x0E /* CIR wakeup */ +#define AST2400_GPIO_PP_OD 0x0F /* GPIO Push-Pull/Open drain select */ +#define AST2400_SVID 0x14 +#define AST2400_DSLP 0x16 /* Deep sleep */ +#define AST2400_GPIOA_LDN 0x17 + +/* virtual LDN for GPIO and WDT */ +#define AST2400_WDT1 ((0 << 8) | AST2400_WDT1_GPIO01A_V) + +#define AST2400_GPIOBASE ((0 << 8) | AST2400_WDT1_GPIO01A_V) //? + +#define AST2400_GPIO0 ((1 << 8) | AST2400_WDT1_GPIO01A_V) +#define AST2400_GPIO1 ((1 << 8) | AST2400_GPIO1234567_V) +#define AST2400_GPIO2 ((2 << 8) | AST2400_GPIO1234567_V) +#define AST2400_GPIO3 ((3 << 8) | AST2400_GPIO1234567_V) +#define AST2400_GPIO4 ((4 << 8) | AST2400_GPIO1234567_V) +#define AST2400_GPIO5 ((5 << 8) | AST2400_GPIO1234567_V) +#define AST2400_GPIO6 ((6 << 8) | AST2400_GPIO1234567_V) +#define AST2400_GPIO7 ((7 << 8) | AST2400_GPIO1234567_V) +#define AST2400_GPIO8 ((0 << 8) | AST2400_GPIO6789_V) +#define AST2400_GPIO9 ((1 << 8) | AST2400_GPIO6789_V) +#define AST2400_GPIOA ((2 << 8) | AST2400_WDT1_GPIO01A_V) + +#define SUPERIO_PNP_PORT 0x2E +#define SUPERIO_CHIPID 0xC333 + +struct device_operations; +typedef struct pnp_device { + unsigned int port; + unsigned int device; + + struct device_operations *ops; +} *device_t; + +struct pnp_mode_ops { + void (*enter_conf_mode)(device_t dev); + void (*exit_conf_mode)(device_t dev); +}; + + +struct device_operations { + void (*read_resources)(device_t dev); + void (*set_resources)(device_t dev); + void (*enable_resources)(device_t dev); + void (*init)(device_t dev); + void (*final)(device_t dev); + void (*enable)(device_t dev); + void (*disable)(device_t dev); + + const struct pnp_mode_ops *ops_pnp_mode; +}; + +/* PNP helper operations */ +struct io_info { + unsigned int mask, set; +}; + +struct pnp_info { + bool enabled; /* set if we should enable the device */ + struct pnp_device pnp_device; + unsigned int function; /* Must be at least 16 bits (virtual LDNs)! */ +}; + +/* Chip operations */ +struct chip_operations { + void (*enable_dev)(struct device *dev); + void (*init)(void *chip_info); + void (*final)(void *chip_info); + unsigned int initialized : 1; + unsigned int finalized : 1; + const char *name; +}; + +typedef struct superio_ast2400_device { + struct device *dev; + const char *name; + unsigned int enabled : 1; /* set if we should enable the device */ + unsigned int superio_ast2400_efir; /* extended function index register */ + unsigned int superio_ast2400_efdr; /* extended function data register */ + struct chip_operations *chip_ops; + const void *chip_info; +} *superio_device_t; + + +static inline void pnp_enter_conf_mode_a5a5(device_t dev) +{ + outb(0xa5, dev->port); + outb(0xa5, dev->port); +} + +static inline void pnp_exit_conf_mode_aa(device_t dev) +{ + outb(0xaa, dev->port); +} + +/* PNP config mode wrappers */ + +static inline void pnp_enter_conf_mode(device_t dev) +{ + if (dev->ops->ops_pnp_mode) + dev->ops->ops_pnp_mode->enter_conf_mode(dev); +} + +static inline void pnp_exit_conf_mode(device_t dev) +{ + if (dev->ops->ops_pnp_mode) + dev->ops->ops_pnp_mode->exit_conf_mode(dev); +} + +/* PNP device operations */ +static inline u8 pnp_read_config(device_t dev, u8 reg) +{ + outb(reg, dev->port); + return inb(dev->port + 1); +} + +static inline void pnp_write_config(device_t dev, u8 reg, u8 value) +{ + outb(reg, dev->port); + outb(value, dev->port + 1); +} + +static inline void pnp_set_logical_device(device_t dev) +{ + pnp_write_config(dev, 0x07, dev->device & 0xff); +// pnp_write_config(dev, 0x07, 0x3); +} + +static inline void pnp_set_enable(device_t dev, int enable) +{ + u8 tmp; + + tmp = pnp_read_config(dev, 0x30); + + if (enable) + tmp |= 1; + else + tmp &= ~1; + + pnp_write_config(dev, 0x30, tmp); +} + +#endif /* _ASM_SW64_AST2400_H */ diff --git a/arch/sw_64/include/asm/socket.h b/arch/sw_64/include/asm/socket.h new file mode 100644 index 000000000000..e87043467775 --- /dev/null +++ b/arch/sw_64/include/asm/socket.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SOCKET_H +#define _ASM_SW64_SOCKET_H + +#include + +/* O_NONBLOCK clashes with the bits used for socket types. Therefore we + * have to define SOCK_NONBLOCK to a different value here. + */ +#define SOCK_NONBLOCK 0x40000000 +#endif /* _ASM_SW64_SOCKET_H */ diff --git a/arch/sw_64/include/uapi/asm/fcntl.h b/arch/sw_64/include/uapi/asm/fcntl.h new file mode 100644 index 000000000000..be2daae2cc4d --- /dev/null +++ b/arch/sw_64/include/uapi/asm/fcntl.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_FCNTL_H +#define _UAPI_ASM_SW64_FCNTL_H + +#define O_CREAT 01000 /* not fcntl */ +#define O_TRUNC 02000 /* not fcntl */ +#define O_EXCL 04000 /* not fcntl */ +#define O_NOCTTY 010000 /* not fcntl */ + +#define O_NONBLOCK 00004 +#define O_APPEND 00010 +#define O_DSYNC 040000 /* used to be O_SYNC, see below */ +#define O_DIRECTORY 0100000 /* must be a directory */ +#define O_NOFOLLOW 0200000 /* don't follow links */ +#define O_LARGEFILE 0400000 /* will be set by the kernel on every open */ +#define O_DIRECT 02000000 /* direct disk access */ +#define O_NOATIME 04000000 +#define O_CLOEXEC 010000000 /* set close_on_exec */ +/* + * Before Linux 2.6.33 only O_DSYNC semantics were implemented, but using + * the O_SYNC flag. We continue to use the existing numerical value + * for O_DSYNC semantics now, but using the correct symbolic name for it. + * This new value is used to request true Posix O_SYNC semantics. It is + * defined in this strange way to make sure applications compiled against + * new headers get at least O_DSYNC semantics on older kernels. + * + * This has the nice side-effect that we can simply test for O_DSYNC + * wherever we do not care if O_DSYNC or O_SYNC is used. + * + * Note: __O_SYNC must never be used directly. + */ +#define __O_SYNC 020000000 +#define O_SYNC (__O_SYNC|O_DSYNC) + +#define O_PATH 040000000 +#define __O_TMPFILE 0100000000 + +#define F_GETLK 7 +#define F_SETLK 8 +#define F_SETLKW 9 + +#define F_SETOWN 5 /* for sockets. */ +#define F_GETOWN 6 /* for sockets. */ +#define F_SETSIG 10 /* for sockets. */ +#define F_GETSIG 11 /* for sockets. */ + +/* for posix fcntl() and lockf() */ +#define F_RDLCK 1 +#define F_WRLCK 2 +#define F_UNLCK 8 + +/* for old implementation of bsd flock () */ +#define F_EXLCK 16 /* or 3 */ +#define F_SHLCK 32 /* or 4 */ + +#include + +#endif /* _UAPI_ASM_SW64_FCNTL_H */ diff --git a/arch/sw_64/include/uapi/asm/ioctl.h b/arch/sw_64/include/uapi/asm/ioctl.h new file mode 100644 index 000000000000..fb5267b034fc --- /dev/null +++ b/arch/sw_64/include/uapi/asm/ioctl.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_IOCTL_H +#define _UAPI_ASM_SW64_IOCTL_H + +#define _IOC_SIZEBITS 13 +#define _IOC_DIRBITS 3 + +/* + * Direction bits _IOC_NONE could be 0, but legacy version gives it a bit. + * And this turns out useful to catch old ioctl numbers in header files for + * us. + */ +#define _IOC_NONE 1U +#define _IOC_READ 2U +#define _IOC_WRITE 4U + +#include + +#endif /* _UAPI_ASM_SW64_IOCTL_H */ diff --git a/arch/sw_64/include/uapi/asm/ioctls.h b/arch/sw_64/include/uapi/asm/ioctls.h new file mode 100644 index 000000000000..36a7fc205aa7 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/ioctls.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_IOCTLS_H +#define _UAPI_ASM_SW64_IOCTLS_H + +#include + +#define FIOCLEX _IO('f', 1) +#define FIONCLEX _IO('f', 2) +#define FIOASYNC _IOW('f', 125, int) +#define FIONBIO _IOW('f', 126, int) +#define FIONREAD _IOR('f', 127, int) +#define TIOCINQ FIONREAD +#define FIOQSIZE _IOR('f', 128, loff_t) + +#define TIOCGETP _IOR('t', 8, struct sgttyb) +#define TIOCSETP _IOW('t', 9, struct sgttyb) +#define TIOCSETN _IOW('t', 10, struct sgttyb) /* TIOCSETP wo flush */ + +#define TIOCSETC _IOW('t', 17, struct tchars) +#define TIOCGETC _IOR('t', 18, struct tchars) +#define TCGETS _IOR('t', 19, struct termios) +#define TCSETS _IOW('t', 20, struct termios) +#define TCSETSW _IOW('t', 21, struct termios) +#define TCSETSF _IOW('t', 22, struct termios) + +#define TCGETA _IOR('t', 23, struct termio) +#define TCSETA _IOW('t', 24, struct termio) +#define TCSETAW _IOW('t', 25, struct termio) +#define TCSETAF _IOW('t', 28, struct termio) + +#define TCSBRK _IO('t', 29) +#define TCXONC _IO('t', 30) +#define TCFLSH _IO('t', 31) + +#define TCGETS2 _IOR('T', 42, struct termios2) +#define TCSETS2 _IOW('T', 43, struct termios2) +#define TCSETSW2 _IOW('T', 44, struct termios2) +#define TCSETSF2 _IOW('T', 45, struct termios2) + +#define TIOCSWINSZ _IOW('t', 103, struct winsize) +#define TIOCGWINSZ _IOR('t', 104, struct winsize) +#define TIOCSTART _IO('t', 110) /* start output, like ^Q */ +#define TIOCSTOP _IO('t', 111) /* stop output, like ^S */ +#define TIOCOUTQ _IOR('t', 115, int) /* output queue size */ + +#define TIOCGLTC _IOR('t', 116, struct ltchars) +#define TIOCSLTC _IOW('t', 117, struct ltchars) +#define TIOCSPGRP _IOW('t', 118, int) +#define TIOCGPGRP _IOR('t', 119, int) + +#define TIOCEXCL 0x540C +#define TIOCNXCL 0x540D +#define TIOCSCTTY 0x540E + +#define TIOCSTI 0x5412 +#define TIOCMGET 0x5415 +#define TIOCMBIS 0x5416 +#define TIOCMBIC 0x5417 +#define TIOCMSET 0x5418 +#define TIOCM_LE 0x001 +#define TIOCM_DTR 0x002 +#define TIOCM_RTS 0x004 +#define TIOCM_ST 0x008 +#define TIOCM_SR 0x010 +#define TIOCM_CTS 0x020 +#define TIOCM_CAR 0x040 +#define TIOCM_RNG 0x080 +#define TIOCM_DSR 0x100 +#define TIOCM_CD TIOCM_CAR +#define TIOCM_RI TIOCM_RNG +#define TIOCM_OUT1 0x2000 +#define TIOCM_OUT2 0x4000 +#define TIOCM_LOOP 0x8000 + +#define TIOCGSOFTCAR 0x5419 +#define TIOCSSOFTCAR 0x541A +#define TIOCLINUX 0x541C +#define TIOCCONS 0x541D +#define TIOCGSERIAL 0x541E +#define TIOCSSERIAL 0x541F +#define TIOCPKT 0x5420 +#define TIOCPKT_DATA 0 +#define TIOCPKT_FLUSHREAD 1 +#define TIOCPKT_FLUSHWRITE 2 +#define TIOCPKT_STOP 4 +#define TIOCPKT_START 8 +#define TIOCPKT_NOSTOP 16 +#define TIOCPKT_DOSTOP 32 +#define TIOCPKT_IOCTL 64 + + +#define TIOCNOTTY 0x5422 +#define TIOCSETD 0x5423 +#define TIOCGETD 0x5424 +#define TCSBRKP 0x5425 /* Needed for POSIX tcsendbreak() */ +#define TIOCSBRK 0x5427 /* BSD compatibility */ +#define TIOCCBRK 0x5428 /* BSD compatibility */ +#define TIOCGSID 0x5429 /* Return the session ID of FD */ +#define TIOCGRS485 _IOR('T', 0x2E, struct serial_rs485) +#define TIOCSRS485 _IOWR('T', 0x2F, struct serial_rs485) +#define TIOCGPTN _IOR('T', 0x30, unsigned int) /* Get Pty Number (of pty-mux device) */ +#define TIOCSPTLCK _IOW('T', 0x31, int) /* Lock/unlock Pty */ +#define TIOCGDEV _IOR('T', 0x32, unsigned int) /* Get primary device node of /dev/console */ +#define TIOCSIG _IOW('T', 0x36, int) /* Generate signal on Pty slave */ +#define TIOCVHANGUP 0x5437 +#define TIOCGPKT _IOR('T', 0x38, int) /* Get packet mode state */ +#define TIOCGPTLCK _IOR('T', 0x39, int) /* Get Pty lock state */ +#define TIOCGEXCL _IOR('T', 0x40, int) /* Get exclusive mode state */ +#define TIOCGPTPEER _IO('T', 0x41) /* Safely open the slave */ +#define TIOCGISO7816 _IOR('T', 0x42, struct serial_iso7816) +#define TIOCSISO7816 _IOWR('T', 0x43, struct serial_iso7816) + +#define TIOCSERCONFIG 0x5453 +#define TIOCSERGWILD 0x5454 +#define TIOCSERSWILD 0x5455 +#define TIOCGLCKTRMIOS 0x5456 +#define TIOCSLCKTRMIOS 0x5457 +#define TIOCSERGSTRUCT 0x5458 /* For debugging only */ +#define TIOCSERGETLSR 0x5459 /* Get line status register */ +/* ioctl (fd, TIOCSERGETLSR, &result) where result may be as below */ +#define TIOCSER_TEMT 0x01 /* Transmitter physically empty */ +#define TIOCSERGETMULTI 0x545A /* Get multiport config */ +#define TIOCSERSETMULTI 0x545B /* Set multiport config */ + +#define TIOCMIWAIT 0x545C /* wait for a change on serial input line(s) */ +#define TIOCGICOUNT 0x545D /* read serial port inline interrupt counts */ + +#endif /* _UAPI_ASM_SW64_IOCTLS_H */ diff --git a/arch/sw_64/include/uapi/asm/resource.h b/arch/sw_64/include/uapi/asm/resource.h new file mode 100644 index 000000000000..2e1ce8f6ee64 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/resource.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_RESOURCE_H +#define _UAPI_ASM_SW64_RESOURCE_H + +/* + * SW-64/Linux-specific ordering of these four resource limit IDs, + * the rest comes from the generic header: + */ +#define RLIMIT_NOFILE 6 /* max number of open files */ +#define RLIMIT_AS 7 /* address space limit */ +#define RLIMIT_NPROC 8 /* max number of processes */ +#define RLIMIT_MEMLOCK 9 /* max locked-in-memory address space */ + +#include + +#endif /* _UAPI_ASM_SW64_RESOURCE_H */ diff --git a/arch/sw_64/include/uapi/asm/socket.h b/arch/sw_64/include/uapi/asm/socket.h new file mode 100644 index 000000000000..1094d11fff5b --- /dev/null +++ b/arch/sw_64/include/uapi/asm/socket.h @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_SOCKET_H +#define _UAPI_ASM_SW64_SOCKET_H + +#include +#include + +/* For setsockopt(2) */ +/* + * Note: we only bother about making the SOL_SOCKET options + * same as legacy, as that's all that "normal" programs are + * likely to set. We don't necessarily want to be binary + * compatible with _everything_. + */ +#define SOL_SOCKET 0xffff + +#define SO_DEBUG 0x0001 +#define SO_REUSEADDR 0x0004 +#define SO_KEEPALIVE 0x0008 +#define SO_DONTROUTE 0x0010 +#define SO_BROADCAST 0x0020 +#define SO_LINGER 0x0080 +#define SO_OOBINLINE 0x0100 +#define SO_REUSEPORT 0x0200 + +#define SO_TYPE 0x1008 +#define SO_ERROR 0x1007 +#define SO_SNDBUF 0x1001 +#define SO_RCVBUF 0x1002 +#define SO_SNDBUFFORCE 0x100a +#define SO_RCVBUFFORCE 0x100b +#define SO_RCVLOWAT 0x1010 +#define SO_SNDLOWAT 0x1011 +#define SO_RCVTIMEO_OLD 0x1012 +#define SO_SNDTIMEO_OLD 0x1013 +#define SO_ACCEPTCONN 0x1014 +#define SO_PROTOCOL 0x1028 +#define SO_DOMAIN 0x1029 + +/* linux-specific, might as well be the same as on i386 */ +#define SO_NO_CHECK 11 +#define SO_PRIORITY 12 +#define SO_BSDCOMPAT 14 + +#define SO_PASSCRED 17 +#define SO_PEERCRED 18 +#define SO_BINDTODEVICE 25 + +/* Socket filtering */ +#define SO_ATTACH_FILTER 26 +#define SO_DETACH_FILTER 27 +#define SO_GET_FILTER SO_ATTACH_FILTER + +#define SO_PEERNAME 28 + +#define SO_PEERSEC 30 +#define SO_PASSSEC 34 + +/* Security levels - as per NRL IPv6 - don't actually do anything */ +#define SO_SECURITY_AUTHENTICATION 19 +#define SO_SECURITY_ENCRYPTION_TRANSPORT 20 +#define SO_SECURITY_ENCRYPTION_NETWORK 21 + +#define SO_MARK 36 + +#define SO_RXQ_OVFL 40 + +#define SO_WIFI_STATUS 41 +#define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 42 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 43 + +#define SO_LOCK_FILTER 44 +#define SO_SELECT_ERR_QUEUE 45 +#define SO_BUSY_POLL 46 +#define SO_MAX_PACING_RATE 47 +#define SO_BPF_EXTENSIONS 48 +#define SO_INCOMING_CPU 49 +#define SO_ATTACH_BPF 50 +#define SO_DETACH_BPF SO_DETACH_FILTER + +#define SO_ATTACH_REUSEPORT_CBPF 51 +#define SO_ATTACH_REUSEPORT_EBPF 52 + +#define SO_CNX_ADVICE 53 + +#define SCM_TIMESTAMPING_OPT_STATS 54 + +#define SO_MEMINFO 55 + +#define SO_INCOMING_NAPI_ID 56 + +#define SO_COOKIE 57 + +#define SCM_TIMESTAMPING_PKTINFO 58 + +#define SO_PEERGROUPS 59 + +#define SO_ZEROCOPY 60 + +#define SO_TXTIME 61 +#define SCM_TXTIME SO_TXTIME + +#define SO_BINDTOIFINDEX 62 + +#define SO_TIMESTAMP_OLD 29 +#define SO_TIMESTAMPNS_OLD 35 +#define SO_TIMESTAMPING_OLD 37 + +#define SO_TIMESTAMP_NEW 63 +#define SO_TIMESTAMPNS_NEW 64 +#define SO_TIMESTAMPING_NEW 65 + +#define SO_RCVTIMEO_NEW 66 +#define SO_SNDTIMEO_NEW 67 + +#define SO_DETACH_REUSEPORT_BPF 68 + +#define SO_PREFER_BUSY_POLL 69 +#define SO_BUSY_POLL_BUDGET 70 + +#define SO_NETNS_COOKIE 71 + +#define SO_BUF_LOCK 72 + +#define SO_RESERVE_MEM 73 + +#define SO_TXREHASH 74 + +#define SO_RCVMARK 75 + +#define SO_PASSPIDFD 76 +#define SO_PEERPIDFD 77 + +#if !defined(__KERNEL__) + +#if __BITS_PER_LONG == 64 +#define SO_TIMESTAMP SO_TIMESTAMP_OLD +#define SO_TIMESTAMPNS SO_TIMESTAMPNS_OLD +#define SO_TIMESTAMPING SO_TIMESTAMPING_OLD + +#define SO_RCVTIMEO SO_RCVTIMEO_OLD +#define SO_SNDTIMEO SO_SNDTIMEO_OLD +#else +#define SO_TIMESTAMP (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMP_OLD : SO_TIMESTAMP_NEW) +#define SO_TIMESTAMPNS (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMPNS_OLD : SO_TIMESTAMPNS_NEW) +#define SO_TIMESTAMPING (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMPING_OLD : SO_TIMESTAMPING_NEW) + +#define SO_RCVTIMEO (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_RCVTIMEO_OLD : SO_RCVTIMEO_NEW) +#define SO_SNDTIMEO (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_SNDTIMEO_OLD : SO_SNDTIMEO_NEW) +#endif + +#define SCM_TIMESTAMP SO_TIMESTAMP +#define SCM_TIMESTAMPNS SO_TIMESTAMPNS +#define SCM_TIMESTAMPING SO_TIMESTAMPING + +#endif + +#endif /* _UAPI_ASM_SW64_SOCKET_H */ diff --git a/arch/sw_64/include/uapi/asm/sockios.h b/arch/sw_64/include/uapi/asm/sockios.h new file mode 100644 index 000000000000..88e89dcf8300 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/sockios.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_SOCKIOS_H +#define _UAPI_ASM_SW64_SOCKIOS_H + +/* Socket-level I/O control calls. */ + +#define FIOGETOWN _IOR('f', 123, int) +#define FIOSETOWN _IOW('f', 124, int) + +#define SIOCATMARK _IOR('s', 7, int) +#define SIOCSPGRP _IOW('s', 8, pid_t) +#define SIOCGPGRP _IOR('s', 9, pid_t) + +#define SIOCGSTAMP_OLD 0x8906 /* Get stamp (timeval) */ +#define SIOCGSTAMPNS_OLD 0x8907 /* Get stamp (timespec) */ + +#endif /* _UAPI_ASM_SW64_SOCKIOS_H */ diff --git a/arch/sw_64/include/uapi/asm/stat.h b/arch/sw_64/include/uapi/asm/stat.h new file mode 100644 index 000000000000..677a75f1cf5b --- /dev/null +++ b/arch/sw_64/include/uapi/asm/stat.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_STAT_H +#define _UAPI_ASM_SW64_STAT_H + +struct stat { + unsigned int st_dev; + unsigned int st_ino; + unsigned int st_mode; + unsigned int st_nlink; + unsigned int st_uid; + unsigned int st_gid; + unsigned int st_rdev; + long st_size; + unsigned long st_atime; + unsigned long st_mtime; + unsigned long st_ctime; + unsigned int st_blksize; + unsigned int st_blocks; + unsigned int st_flags; + unsigned int st_gen; +}; + +/* The stat64 structure increases the size of dev_t, blkcnt_t, adds + * nanosecond resolution times, and padding for expansion. + */ + +struct stat64 { + unsigned long st_dev; + unsigned long st_ino; + unsigned long st_rdev; + long st_size; + unsigned long st_blocks; + + unsigned int st_mode; + unsigned int st_uid; + unsigned int st_gid; + unsigned int st_blksize; + unsigned int st_nlink; + unsigned int __pad0; + + unsigned long st_atime; + unsigned long st_atime_nsec; + unsigned long st_mtime; + unsigned long st_mtime_nsec; + unsigned long st_ctime; + unsigned long st_ctime_nsec; + long __unused[3]; +}; + +#endif /* _UAPI_ASM_SW64_STAT_H */ diff --git a/arch/sw_64/include/uapi/asm/termbits.h b/arch/sw_64/include/uapi/asm/termbits.h new file mode 100644 index 000000000000..a71aaf33c26c --- /dev/null +++ b/arch/sw_64/include/uapi/asm/termbits.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_TERMBITS_H +#define _UAPI_ASM_SW64_TERMBITS_H + +#include + +typedef unsigned int tcflag_t; + +/* + * termios type and macro definitions. Be careful about adding stuff + * to this file since it's used in GNU libc and there are strict rules + * concerning namespace pollution. + */ + +#define NCCS 19 +struct termios { + tcflag_t c_iflag; /* input mode flags */ + tcflag_t c_oflag; /* output mode flags */ + tcflag_t c_cflag; /* control mode flags */ + tcflag_t c_lflag; /* local mode flags */ + cc_t c_cc[NCCS]; /* control characters */ + cc_t c_line; /* line discipline (== c_cc[19]) */ + speed_t c_ispeed; /* input speed */ + speed_t c_ospeed; /* output speed */ +}; + +/* SW64 has identical termios and termios2 */ + +struct termios2 { + tcflag_t c_iflag; /* input mode flags */ + tcflag_t c_oflag; /* output mode flags */ + tcflag_t c_cflag; /* control mode flags */ + tcflag_t c_lflag; /* local mode flags */ + cc_t c_cc[NCCS]; /* control characters */ + cc_t c_line; /* line discipline (== c_cc[19]) */ + speed_t c_ispeed; /* input speed */ + speed_t c_ospeed; /* output speed */ +}; + +/* SW64 has matching termios and ktermios */ + +struct ktermios { + tcflag_t c_iflag; /* input mode flags */ + tcflag_t c_oflag; /* output mode flags */ + tcflag_t c_cflag; /* control mode flags */ + tcflag_t c_lflag; /* local mode flags */ + cc_t c_cc[NCCS]; /* control characters */ + cc_t c_line; /* line discipline (== c_cc[19]) */ + speed_t c_ispeed; /* input speed */ + speed_t c_ospeed; /* output speed */ +}; + +/* c_cc characters */ +#define VEOF 0 +#define VEOL 1 +#define VEOL2 2 +#define VERASE 3 +#define VWERASE 4 +#define VKILL 5 +#define VREPRINT 6 +#define VSWTC 7 +#define VINTR 8 +#define VQUIT 9 +#define VSUSP 10 +#define VSTART 12 +#define VSTOP 13 +#define VLNEXT 14 +#define VDISCARD 15 +#define VMIN 16 +#define VTIME 17 + +/* c_iflag bits */ +#define IXON 0x0200 +#define IXOFF 0x0400 +#define IUCLC 0x1000 +#define IMAXBEL 0x2000 +#define IUTF8 0x4000 + +/* c_oflag bits */ +#define ONLCR 0x00002 +#define OLCUC 0x00004 +#define NLDLY 0x00300 +#define NL0 0x00000 +#define NL1 0x00100 +#define NL2 0x00200 +#define NL3 0x00300 +#define TABDLY 0x00c00 +#define TAB0 0x00000 +#define TAB1 0x00400 +#define TAB2 0x00800 +#define TAB3 0x00c00 +#define CRDLY 0x03000 +#define CR0 0x00000 +#define CR1 0x01000 +#define CR2 0x02000 +#define CR3 0x03000 +#define FFDLY 0x04000 +#define FF0 0x00000 +#define FF1 0x04000 +#define BSDLY 0x08000 +#define BS0 0x00000 +#define BS1 0x08000 +#define VTDLY 0x10000 +#define VT0 0x00000 +#define VT1 0x10000 +/* + * Should be equivalent to TAB3, see description of TAB3 in + * POSIX.1-2008, Ch. 11.2.3 "Output Modes" + */ +#define XTABS TAB3 + +/* c_cflag bit meaning */ +#define CBAUD 0x0000001f +#define CBAUDEX 0x00000000 +#define BOTHER 0x0000001f +#define B57600 0x00000010 +#define B115200 0x00000011 +#define B230400 0x00000012 +#define B460800 0x00000013 +#define B500000 0x00000014 +#define B576000 0x00000015 +#define B921600 0x00000016 +#define B1000000 0x00000017 +#define B1152000 0x00000018 +#define B1500000 0x00000019 +#define B2000000 0x0000001a +#define B2500000 0x0000001b +#define B3000000 0x0000001c +#define B3500000 0x0000001d +#define B4000000 0x0000001e +#define CSIZE 0x00000300 +#define CS5 0x00000000 +#define CS6 0x00000100 +#define CS7 0x00000200 +#define CS8 0x00000300 +#define CSTOPB 0x00000400 +#define CREAD 0x00000800 +#define PARENB 0x00001000 +#define PARODD 0x00002000 +#define HUPCL 0x00004000 +#define CLOCAL 0x00008000 +#define CIBAUD 0x001f0000 + +/* c_lflag bits */ +#define ISIG 0x00000080 +#define ICANON 0x00000100 +#define XCASE 0x00004000 +#define ECHO 0x00000008 +#define ECHOE 0x00000002 +#define ECHOK 0x00000004 +#define ECHONL 0x00000010 +#define NOFLSH 0x80000000 +#define TOSTOP 0x00400000 +#define ECHOCTL 0x00000040 +#define ECHOPRT 0x00000020 +#define ECHOKE 0x00000001 +#define FLUSHO 0x00800000 +#define PENDIN 0x20000000 +#define IEXTEN 0x00000400 +#define EXTPROC 0x10000000 + +/* Values for the OPTIONAL_ACTIONS argument to `tcsetattr'. */ +#define TCSANOW 0 +#define TCSADRAIN 1 +#define TCSAFLUSH 2 + +#endif /* _UAPI_ASM_SW64_TERMBITS_H */ diff --git a/arch/sw_64/include/uapi/asm/termios.h b/arch/sw_64/include/uapi/asm/termios.h new file mode 100644 index 000000000000..62f4b40551b2 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/termios.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_TERMIOS_H +#define _UAPI_ASM_SW64_TERMIOS_H + +#include +#include + +struct sgttyb { + char sg_ispeed; + char sg_ospeed; + char sg_erase; + char sg_kill; + short sg_flags; +}; + +struct tchars { + char t_intrc; + char t_quitc; + char t_startc; + char t_stopc; + char t_eofc; + char t_brkc; +}; + +struct ltchars { + char t_suspc; + char t_dsuspc; + char t_rprntc; + char t_flushc; + char t_werasc; + char t_lnextc; +}; + +struct winsize { + unsigned short ws_row; + unsigned short ws_col; + unsigned short ws_xpixel; + unsigned short ws_ypixel; +}; + +#define NCC 8 +struct termio { + unsigned short c_iflag; /* input mode flags */ + unsigned short c_oflag; /* output mode flags */ + unsigned short c_cflag; /* control mode flags */ + unsigned short c_lflag; /* local mode flags */ + unsigned char c_line; /* line discipline */ + unsigned char c_cc[NCC]; /* control characters */ +}; + +/* + * c_cc characters in the termio structure. Oh, how I love being + * backwardly compatible. Notice that character 4 and 5 are + * interpreted differently depending on whether ICANON is set in + * c_lflag. If it's set, they are used as _VEOF and _VEOL, otherwise + * as _VMIN and V_TIME. This is for compatibility with sysV)... + */ +#define _VINTR 0 +#define _VQUIT 1 +#define _VERASE 2 +#define _VKILL 3 +#define _VEOF 4 +#define _VMIN 4 +#define _VEOL 5 +#define _VTIME 5 +#define _VEOL2 6 +#define _VSWTC 7 + + +#endif /* _UAPI_ASM_SW64_TERMIOS_H */ -- Gitee From 88e578a872625e0e14044765185e58ffd83826c3 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:02 +0800 Subject: [PATCH 007/524] sw64: add boot and setup routines Add basic boot, setup and reset routines for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/platform.h | 32 + arch/sw_64/include/asm/setup.h | 51 ++ arch/sw_64/include/asm/sw64_init.h | 50 ++ arch/sw_64/include/uapi/asm/bootparam.h | 22 + arch/sw_64/kernel/cacheinfo.c | 99 +++ arch/sw_64/kernel/chip_setup.c | 245 ++++++ arch/sw_64/kernel/early_init.c | 11 + arch/sw_64/kernel/head.S | 112 +++ arch/sw_64/kernel/hmcall.c | 131 +++ arch/sw_64/kernel/reset.c | 120 +++ arch/sw_64/kernel/setup.c | 1061 +++++++++++++++++++++++ 11 files changed, 1934 insertions(+) create mode 100644 arch/sw_64/include/asm/platform.h create mode 100644 arch/sw_64/include/asm/setup.h create mode 100644 arch/sw_64/include/asm/sw64_init.h create mode 100644 arch/sw_64/include/uapi/asm/bootparam.h create mode 100644 arch/sw_64/kernel/cacheinfo.c create mode 100644 arch/sw_64/kernel/chip_setup.c create mode 100644 arch/sw_64/kernel/early_init.c create mode 100644 arch/sw_64/kernel/head.S create mode 100644 arch/sw_64/kernel/hmcall.c create mode 100644 arch/sw_64/kernel/reset.c create mode 100644 arch/sw_64/kernel/setup.c diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h new file mode 100644 index 000000000000..ad54cdc772e1 --- /dev/null +++ b/arch/sw_64/include/asm/platform.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_PLATFORM_H +#define _ASM_SW64_PLATFORM_H + +#include +#if defined(CONFIG_UNCORE_XUELANG) +#include +#elif defined(CONFIG_UNCORE_JUNZHANG) +#include +#endif + +#ifdef CONFIG_EFI +#define BIOS_VERSION_GUID EFI_GUID(0xc47a23c3, 0xcebb, 0x4cc9, 0xa5, 0xe2, 0xde, 0xd0, 0x8f, 0xe4, 0x20, 0xb5) + +#define BIOS_SUPPORT_RESET_CLALLBACK(bios_version) ((bios_version) != NULL) + +extern unsigned long bios_version; + +#endif + +extern struct boot_params *sunway_boot_params; + +extern void sw64_halt(void); +extern void sw64_poweroff(void); +extern void sw64_restart(void); +extern void (*pm_restart)(void); +extern void (*pm_halt)(void); +extern int i2c_set_adapter(void); +extern void cpld_write(uint8_t slave_addr, uint8_t reg, uint8_t data); +extern void fix_jm585_reset(void); + +#endif /* _ASM_SW64_PLATFORM_H */ diff --git a/arch/sw_64/include/asm/setup.h b/arch/sw_64/include/asm/setup.h new file mode 100644 index 000000000000..2d557b349555 --- /dev/null +++ b/arch/sw_64/include/asm/setup.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SETUP_H +#define _ASM_SW64_SETUP_H + +#include + +/* + * We leave one page for the initial stack page, and one page for + * the initial process structure. Also, the console eats 3 MB for + * the initial bootloader (one of which we can reclaim later). + */ +#define BOOT_PCB 0x20000000 +#define BOOT_ADDR 0x20000000 +/* Remove when official MILO sources have ELF support: */ +#define BOOT_SIZE (16 * 1024) + +#define KERNEL_START_PHYS CONFIG_PHYSICAL_START +#define KERNEL_START (__START_KERNEL_map + CONFIG_PHYSICAL_START) + +/* INIT_STACK may be used for merging lwk to kernel*/ +#define INIT_STACK (KERNEL_START + 0x02000) + +/* + * This is setup by the secondary bootstrap loader. Because + * the zero page is zeroed out as soon as the vm system is + * initialized, we need to copy things out into a more permanent + * place. + */ +#define PARAM (KERNEL_START + 0x0A000) +#define COMMAND_LINE ((char *)(KERNEL_START + 0x0B000)) +#define INITRD_START (*(unsigned long *)(PARAM + 0x100)) +#define INITRD_SIZE (*(unsigned long *)(PARAM + 0x108)) +#define DTB_START (*(unsigned long *)(PARAM + 0x118)) + +#define _TEXT_START (KERNEL_START + 0x10000) + +#define COMMAND_LINE_OFF (0x10000UL - 0xB000UL) +#define INITRD_START_OFF (0x10000UL - 0xA100UL) +#define INITRD_SIZE_OFF (0x10000UL - 0xA108UL) + +/* Motherboard Configuration Tables */ +#define MB_CONFIG_START 0x908000 +#define MB_MCLK (MB_CONFIG_START + 0x1) +#define MB_EXTCLK (MB_CONFIG_START + 0x11) + +#ifndef __ASSEMBLY__ +#include +extern struct boot_params *sunway_boot_params; +#endif + +#endif /* _ASM_SW64_SETUP_H */ diff --git a/arch/sw_64/include/asm/sw64_init.h b/arch/sw_64/include/asm/sw64_init.h new file mode 100644 index 000000000000..86ddd2cb65f8 --- /dev/null +++ b/arch/sw_64/include/asm/sw64_init.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SW64_INIT_H +#define _ASM_SW64_SW64_INIT_H + +#include +#include + +#include + +struct sw64_early_init_ops { + void (*setup_core_map)(struct cpumask *cpumask); + unsigned long (*get_node_mem)(int nodeid); + void (*get_smp_info)(void); +}; + +struct sw64_pci_init_ops { + int (*map_irq)(const struct pci_dev *dev, u8 slot, u8 pin); + unsigned long (*get_rc_enable)(unsigned long node); + void (*hose_init)(struct pci_controller *hose); + void (*set_rc_piu)(unsigned long node, unsigned long index); + int (*check_pci_linkup)(unsigned long node, unsigned long index); + void (*set_intx)(unsigned long node, unsigned long index, + unsigned long int_conf); +}; + + +struct sw64_chip_init_ops { + struct sw64_early_init_ops early_init; + struct sw64_pci_init_ops pci_init; + void (*fixup)(void); +}; + +struct sw64_chip_ops { + int (*get_cpu_num)(void); + void (*device_interrupt)(unsigned long irq_info); + void (*suspend)(bool wake); + void (*fixup)(void); +}; + +extern void sw64_init_noop(void); +extern void setup_chip_ops(void); +extern struct sw64_chip_ops *sw64_chip; +extern struct sw64_chip_init_ops *sw64_chip_init; +#ifdef CONFIG_PM +extern struct syscore_ops io_syscore_ops; +#endif + +DECLARE_PER_CPU(unsigned long, hard_node_id); + +#endif /* _ASM_SW64_SW64_INIT_H */ diff --git a/arch/sw_64/include/uapi/asm/bootparam.h b/arch/sw_64/include/uapi/asm/bootparam.h new file mode 100644 index 000000000000..6ce75d65e86e --- /dev/null +++ b/arch/sw_64/include/uapi/asm/bootparam.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_BOOTPARAM_H +#define _UAPI_ASM_SW64_BOOTPARAM_H + +#ifndef __ASSEMBLY__ + +#include + +struct boot_params { + __u64 initrd_start; /* logical address of initrd */ + __u64 initrd_size; /* size of initrd */ + __u64 dtb_start; /* logical address of dtb */ + __u64 efi_systab; /* logical address of EFI system table */ + __u64 efi_memmap; /* logical address of EFI memory map */ + __u64 efi_memmap_size; /* size of EFI memory map */ + __u64 efi_memdesc_size; /* size of an EFI memory map descriptor */ + __u64 efi_memdesc_version; /* memory descriptor version */ + __u64 cmdline; /* logical address of cmdline */ +}; +#endif + +#endif /* _UAPI_ASM_SW64_BOOTPARAM_H */ diff --git a/arch/sw_64/kernel/cacheinfo.c b/arch/sw_64/kernel/cacheinfo.c new file mode 100644 index 000000000000..e340c53690a9 --- /dev/null +++ b/arch/sw_64/kernel/cacheinfo.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SW64 cacheinfo support + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +#include + +/* Populates leaf and increments to next leaf */ +#define populate_cache(cache, leaf, c_level, c_type, c_id) \ +do { \ + leaf->id = c_id; \ + leaf->attributes = CACHE_ID; \ + leaf->type = c_type; \ + leaf->level = c_level; \ + leaf->coherency_line_size = c->cache.linesz; \ + leaf->number_of_sets = c->cache.sets; \ + leaf->ways_of_associativity = c->cache.ways; \ + leaf->size = c->cache.size; \ + leaf++; \ +} while (0) + +int init_cache_level(unsigned int cpu) +{ + struct cpuinfo_sw64 *c = &cpu_data[cpu]; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + int levels = 0, leaves = 0; + + /* + * If Dcache is not set, we assume the cache structures + * are not properly initialized. + */ + if (c->dcache.size) + levels += 1; + else + return -ENOENT; + + + leaves += (c->icache.size) ? 2 : 1; + + if (c->scache.size) { + levels++; + leaves++; + } + + if (c->tcache.size) { + levels++; + leaves++; + } + + this_cpu_ci->num_levels = levels; + this_cpu_ci->num_leaves = leaves; + return 0; +} + +int populate_cache_leaves(unsigned int cpu) +{ + struct cpuinfo_sw64 *c = &cpu_data[cpu]; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *this_leaf = this_cpu_ci->info_list; + struct cpu_topology *topo = &cpu_topology[cpu]; + + if (c->icache.size) { + cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); + populate_cache(dcache, this_leaf, 1, CACHE_TYPE_DATA, cpu); + cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); + populate_cache(icache, this_leaf, 1, CACHE_TYPE_INST, cpu); + + } else { + cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); + populate_cache(dcache, this_leaf, 1, CACHE_TYPE_UNIFIED, cpu); + } + + if (c->scache.size) { + cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); + populate_cache(scache, this_leaf, 2, CACHE_TYPE_UNIFIED, cpu); + } + + if (c->tcache.size) { + cpumask_copy(&this_leaf->shared_cpu_map, topology_llc_cpumask(cpu)); + populate_cache(tcache, this_leaf, 3, CACHE_TYPE_UNIFIED, topo->package_id); + } + + this_cpu_ci->cpu_map_populated = true; + + return 0; +} diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c new file mode 100644 index 000000000000..b8c359db2ef6 --- /dev/null +++ b/arch/sw_64/kernel/chip_setup.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include + +struct sw64_chip_ops *sw64_chip; +struct sw64_chip_init_ops *sw64_chip_init; + +static int get_cpu_nums(void) +{ + if (is_guest_or_emul()) + return 1; + + return __get_cpu_nums(); +} + +static unsigned long __init get_node_mem(int nodeid) +{ + + if (is_guest_or_emul()) + return *(unsigned long *)MMSIZE & MMSIZE_MASK; + + return __get_node_mem(nodeid); +} + +static void __init setup_core_map(struct cpumask *cpumask) +{ + int i, j, cpu_num, cpuid, max_cores_per_cpu; + unsigned long coreonline; + + cpu_num = get_cpu_nums(); + cpuid = 0; + for (i = 0; i < cpu_num; i++) { + coreonline = sw64_io_read(i, CORE_ONLINE); + max_cores_per_cpu = MAX_CORES_PER_CPU; + + if (is_guest_or_emul()) + max_cores_per_cpu = 64; + + for (j = 0; j < max_cores_per_cpu; j++) { + if (coreonline & (1UL << j)) { + __cpu_to_rcid[cpuid] = (i << DOMAIN_ID_SHIFT) | (j << CORE_ID_SHIFT); + cpuid++; + } + } + } + + if (is_in_host() && core_is_ht()) { + for (i = 0; i < cpuid; i++) + __cpu_to_rcid[cpuid + i] = __cpu_to_rcid[i] | (1 << THREAD_ID_SHIFT); + + cpuid = cpuid + i; + } + + while (cpuid < NR_CPUS) { + __cpu_to_rcid[cpuid] = -1; + cpuid++; + } +} + +#ifdef CONFIG_PM +static void i2c_srst(void) +{ + sw64_io_write(0, I2C0_SRST_L, 0x0); + sw64_io_write(0, I2C0_SRST_L, 0x1); + + sw64_io_write(0, I2C1_SRST_L, 0x0); + sw64_io_write(0, I2C1_SRST_L, 0x1); + + sw64_io_write(0, I2C2_SRST_L, 0x0); + sw64_io_write(0, I2C2_SRST_L, 0x1); +} + +static void pcie_save(void) +{ + struct pci_controller *hose; + struct piu_saved *piu_save; + unsigned long node, index; + unsigned long i; + + for (hose = hose_head; hose; hose = hose->next) { + piu_save = kzalloc(sizeof(*piu_save), GFP_KERNEL); + + node = hose->node; + index = hose->index; + hose->sysdata = piu_save; + + piu_save->piuconfig0 = read_piu_ior0(node, index, PIUCONFIG0); + piu_save->piuconfig1 = read_piu_ior1(node, index, PIUCONFIG1); + piu_save->epdmabar = read_piu_ior0(node, index, EPDMABAR); + piu_save->msiaddr = read_piu_ior0(node, index, MSIADDR); + + if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { + for (i = 0; i < 256; i++) { + piu_save->msiconfig[i] = read_piu_ior0(node, index, + MSICONFIG0 + (i << 7)); + } + } + + piu_save->iommuexcpt_ctrl = read_piu_ior0(node, index, IOMMUEXCPT_CTRL); + piu_save->dtbaseaddr = read_piu_ior0(node, index, DTBASEADDR); + + piu_save->intaconfig = read_piu_ior0(node, index, INTACONFIG); + piu_save->intbconfig = read_piu_ior0(node, index, INTBCONFIG); + piu_save->intcconfig = read_piu_ior0(node, index, INTCCONFIG); + piu_save->intdconfig = read_piu_ior0(node, index, INTDCONFIG); + piu_save->pmeintconfig = read_piu_ior0(node, index, PMEINTCONFIG); + piu_save->aererrintconfig = read_piu_ior0(node, index, AERERRINTCONFIG); + piu_save->hpintconfig = read_piu_ior0(node, index, HPINTCONFIG); + + } +} + +static void pcie_restore(void) +{ + struct pci_controller *hose; + struct piu_saved *piu_save; + unsigned long node, index; + u32 rc_misc_ctrl; + unsigned int value; + unsigned long i; + + for (hose = hose_head; hose; hose = hose->next) { + node = hose->node; + index = hose->index; + piu_save = hose->sysdata; + + write_piu_ior0(node, index, PIUCONFIG0, piu_save->piuconfig0); + write_piu_ior1(node, index, PIUCONFIG1, piu_save->piuconfig1); + write_piu_ior0(node, index, EPDMABAR, piu_save->epdmabar); + write_piu_ior0(node, index, MSIADDR, piu_save->msiaddr); + + if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { + for (i = 0; i < 256; i++) { + write_piu_ior0(node, index, MSICONFIG0 + (i << 7), + piu_save->msiconfig[i]); + } + } + + write_piu_ior0(node, index, IOMMUEXCPT_CTRL, piu_save->iommuexcpt_ctrl); + write_piu_ior0(node, index, DTBASEADDR, piu_save->dtbaseaddr); + + write_piu_ior0(node, index, INTACONFIG, piu_save->intaconfig); + write_piu_ior0(node, index, INTBCONFIG, piu_save->intbconfig); + write_piu_ior0(node, index, INTCCONFIG, piu_save->intcconfig); + write_piu_ior0(node, index, INTDCONFIG, piu_save->intdconfig); + write_piu_ior0(node, index, PMEINTCONFIG, piu_save->pmeintconfig); + write_piu_ior0(node, index, AERERRINTCONFIG, piu_save->aererrintconfig); + write_piu_ior0(node, index, HPINTCONFIG, piu_save->hpintconfig); + + /* Enable DBI_RO_WR_EN */ + rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); + write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); + + /* Fix up DEVICE_ID_VENDOR_ID register */ + value = (PCI_DEVICE_ID_SW64_ROOT_BRIDGE << 16) | PCI_VENDOR_ID_JN; + write_rc_conf(node, index, RC_VENDOR_ID, value); + + /* Set PCI-E root class code */ + value = read_rc_conf(node, index, RC_REVISION_ID); + write_rc_conf(node, index, RC_REVISION_ID, (PCI_CLASS_BRIDGE_HOST << 16) | value); + + /* Disable DBI_RO_WR_EN */ + write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); + } + +} + +static unsigned long saved_dvc_int, saved_long_time; + +static inline void intpu_save(void) +{ + switch (cpu_desc.model) { + case CPU_SW831: + saved_long_time = __io_read_longtime(0); + default: + break; + } +} + +static inline void intpu_restore(void) +{ + switch (cpu_desc.model) { + case CPU_SW831: + __io_write_longtime(0, saved_long_time); + __io_write_longtime_start_en(0, 0x1); + break; + default: + pr_info("long time start is disable!"); + break; + } +} + +static inline void spbu_save(void) +{ + saved_dvc_int = sw64_io_read(0, MCU_DVC_INT_EN); +} + +static inline void spbu_restore(void) +{ + i2c_srst(); + sw64_io_write(0, MCU_DVC_INT_EN, saved_dvc_int); +} + +static int io_suspend(void) +{ + spbu_save(); + intpu_save(); + pcie_save(); + + return 0; +} + +static void io_resume(void) +{ + pcie_restore(); + intpu_restore(); + spbu_restore(); +} +#endif /* CONFIG_PM */ + +static struct sw64_chip_init_ops chip_init_ops = { + .early_init = { + .setup_core_map = setup_core_map, + .get_node_mem = get_node_mem, + }, +}; + +static struct sw64_chip_ops chip_ops = { + .get_cpu_num = get_cpu_nums, +}; + +void __init setup_chip_ops(void) +{ + sw64_chip_init = &chip_init_ops; + sw64_chip = &chip_ops; + setup_chip_pci_ops(); +#ifdef CONFIG_PM + io_syscore_ops.suspend = io_suspend; + io_syscore_ops.resume = io_resume; +#endif +} diff --git a/arch/sw_64/kernel/early_init.c b/arch/sw_64/kernel/early_init.c new file mode 100644 index 000000000000..2ec7a3e99443 --- /dev/null +++ b/arch/sw_64/kernel/early_init.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include + +asmlinkage __visible void __init sw64_start_kernel(void) +{ + fixup_hmcall(); + save_ktp(); + start_kernel(); +} diff --git a/arch/sw_64/kernel/head.S b/arch/sw_64/kernel/head.S new file mode 100644 index 000000000000..fd0fbfbcf5b6 --- /dev/null +++ b/arch/sw_64/kernel/head.S @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * initial boot stuff.. At this point, the bootloader has already + * switched into HMcode, and loaded us at the correct address + * (START_ADDR). So there isn't much left for us to do: just set up + * the kernel global pointer and jump to the kernel entry-point. + */ + +#include +#include +#include +#include + +__HEAD + .globl _stext + .set noreorder + .globl __start + .ent __start +_stext: +__start: + .prologue 0 + br $27, 1f +1: ldgp $29, 0($27) + /* We need to get current_task_info loaded up... */ + ldi $8, init_task + ldl $30, TASK_STACK($8) + /* ... and find our stack ... */ + ldi $30, ASM_THREAD_SIZE($30) + + /* ... and then we can clear bss data. */ + ldi $16, __bss_start + ldi $18, __bss_stop + subl $18, $16, $18 + mov $31, $17 + call $26, __constant_c_memset +#ifdef CONFIG_RELOCATABLE + ldi $30, -8($30) + stl $29, 0($30) + /* Copy kernel and apply the relocations */ + call $26, relocate_kernel + ldl $29, 0($30) + addl $29, $0, $29 + addl $8, $0, $8 + ldi $30, 8($30) + /* Repoint the sp into the new kernel image */ + addl $30, $0, $30 +#endif + /* ... and then we can start the kernel. */ + call $26, sw64_start_kernel + sys_call HMC_halt + .end __start + +#ifdef CONFIG_SMP + .align 3 + .globl __smp_callin + .ent __smp_callin + /* On entry here the PCB of the idle task for this processor + * has been loaded. We've arranged for the tilde_pcb[x] for + * this process to contain the PCBB of the target idle task. + */ +__smp_callin: + .prologue 1 + br $27, 2f # we copy this from above "br $27 1f" +2: ldgp $29, 0($27) # First order of business, load the GP. + + bis $31, $31, $16 # invalidate all TLB with current VPN + sys_call HMC_tbi + +#if defined(CONFIG_SUBARCH_C3B) + sys_call HMC_whami # Get hard cid + ldi $1, __cpu_to_rcid + ldi $2, 0($31) + ldi $4, CONFIG_NR_CPUS +3: ldw $3, 0($1) + cmpeq $3, $0, $3 + bne $3, 4f + addl $1, 4, $1 + addl $2, 1, $2 + cmpeq $2, $4, $5 + bne $5, 5f + br $31, 3b +4: ldi $0, 0($2) +#else + rcid $0 +#endif + + ldi $2, idle_task_pointer + s8addl $0, $2, $2 + ldl $8, 0($2) # Get ksp of idle thread + sys_call HMC_wrktp + + ldl $30, TASK_STACK($8) + ldi $30, ASM_THREAD_SIZE($30) + + call $26, smp_callin +5: + sys_call HMC_halt + .end __smp_callin +#endif /* CONFIG_SMP */ + # + # It is handy, on occasion, to make halt actually just loop. + # Putting it here means we dont have to recompile the whole + # kernel. + # + + .align 3 + .globl halt + .ent halt +halt: + .prologue 0 + sys_call HMC_halt + .end halt diff --git a/arch/sw_64/kernel/hmcall.c b/arch/sw_64/kernel/hmcall.c new file mode 100644 index 000000000000..d2054a930bd7 --- /dev/null +++ b/arch/sw_64/kernel/hmcall.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arch/sw_64/kernel/hmcall.c + * + * Copyright (C) 2022 WXIAT + * Author: He Sheng + */ + +#include +#include + +#define A0(func) (((HMC_##func & 0xFF) >> 6) & 0x1) +#define A1(func) ((((HMC_##func & 0xFF)>>6) & 0x2) >> 1) +#define A2(func) ((HMC_##func & 0x3F) << 7) + +#define T(func) ((A0(func) ^ A1(func)) & 0x1) +#define B0(func) ((T(func) | A0(func)) << 13) +#define B1(func) (((~T(func) & 1) | A1(func)) << 14) + +#define PRI_BASE 0x10000UL + +#define HMCALL_ENTRY(func) (PRI_BASE | B1(func) | B0(func) | A2(func)) + + +static inline void fixup_rdtp(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(rdtp)); + + entry[0] = 0x181ffec7; /* pri_rcsr $0, CSR__TID */ + entry[1] = 0x1ee00000; /* pri_ret $23 */ +} + +static inline void fixup_wrtp(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(wrtp)); + + entry[0] = 0x1a1fffc7; /* pri_wcsr $16, CSR__TID */ + entry[1] = 0x1ee00000; /* pri_ret $23 */ +} + +static inline void fixup_tbiasid(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(tbisasid)); + + entry[0] = 0x18fffe47; /* pri_rcsr p7, CSR__DTB_PCR*/ + entry[1] = 0x4a05c905; /* sll r16, CSR__DTB_PCR__UPN__S, p5 */ + entry[2] = 0xf89f03ff; /* ldi p4, CSR__DTB_PCR__UPN__M */ + entry[3] = 0x4885c904; /* sll p4, CSR__DTB_PCR__UPN__S, p4 */ + entry[4] = 0x40e40724; /* bic p7, p4, p4 */ + entry[5] = 0x40850745; /* bis p4, p5, p5 */ + entry[6] = 0x18bfff47; /* pri_wcsr p5, CSR__DTB_PCR */ + entry[7] = 0x1a3fff46; /* pri_wcsr r17, CSR__DTB_IS */ + entry[8] = 0x18ffff47; /* pri_wcsr p7, CSR__DTB_PCR */ + entry[9] = 0x4a04e906; /* sll r16, CSR__UPCR_UPN__UPN__S, p6 */ + entry[10] = 0x189ffe22; /* pri_rcsr p4, CSR__UPCR_UPN */ + entry[11] = 0x18dfff22; /* pri_wcsr p6, CSR__UPCR_UPN */ + entry[12] = 0x1a3fff06; /* pri_wcsr r17, CSR__ITB_IS */ + entry[13] = 0x1bffff15; /* pri_wcsr r31, CSR__IC_FLUSH */ + entry[14] = 0x189fff22; /* pri_wcsr p4, CSR__UPCR_UPN */ + entry[15] = 0x1ef00000; /* pri_ret/b p23 */ +} + +static inline void fixup_wrasid(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(wrasid)); + + entry[0] = 0x18fffe47; /* pri_rcsr p7, CSR__DTB_PCR*/ + entry[1] = 0x4a05c905; /* sll r16, CSR__DTB_PCR__UPN__S, p5 */ + entry[2] = 0xf89f03ff; /* ldi p4, CSR__DTB_PCR__UPN__M */ + entry[3] = 0x4885c904; /* sll p4, CSR__DTB_PCR__UPN__S, p4 */ + entry[4] = 0x40e40724; /* bic p7, p4, p4 */ + entry[5] = 0x40850745; /* bis p4, p5, p5 */ + entry[6] = 0x18bfff47; /* pri_wcsr p5, CSR__DTB_PCR */ + entry[7] = 0x4a04e906; /* sll r16, CSR__UPCR_UPN__UPN__S, p6 */ + entry[8] = 0x18dfff22; /* pri_wcsr p4, CSR__UPCR_UPN */ + entry[9] = 0x1ef00000; /* pri_ret/b p23 */ +} + +static inline void fixup_rdktp(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(rdktp)); + + entry[0] = 0x95161000; /* pri_ldl/p $8, VC__KTP(vcpucb) */ + entry[1] = 0x1ee00000; /* pri_ret $23 */ +} + +static inline void fixup_wrktp(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(wrktp)); + + entry[0] = 0xb5161000; /* pri_stl/p $8, VC__KTP(vcpucb) */ + entry[1] = 0x1ee00000; /* pri_ret $23 */ +} + +static inline void fixup_rdusp(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(rdusp)); + + entry[0] = 0x94161018; /* pri_ldl/p $0, VC__USP(vcpucb) */ + entry[1] = 0x1ee00000; /* pri_ret $23 */ +} + +static inline void fixup_wrusp(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(wrusp)); + + entry[0] = 0xb6161018; /* pri_stl/p $16, VC__USP(vcpucb) */ + entry[1] = 0x1ee00000; /* pri_ret $23 */ +} + +void __init fixup_hmcall(void) +{ +#if defined(CONFIG_SUBARCH_C3B) + fixup_rdtp(); + fixup_wrtp(); + fixup_tbiasid(); + fixup_wrasid(); + fixup_rdktp(); + fixup_wrktp(); + fixup_rdusp(); + fixup_wrusp(); + imemb(); +#endif +} + +#undef A0 +#undef A1 +#undef A2 +#undef T +#undef B0 +#undef B1 diff --git a/arch/sw_64/kernel/reset.c b/arch/sw_64/kernel/reset.c new file mode 100644 index 000000000000..955339557a7a --- /dev/null +++ b/arch/sw_64/kernel/reset.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2022 Sunway Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +void fix_jm585_reset(void) +{ + struct pci_dev *pdev; + struct pci_controller *hose; + int val; + + pdev = pci_get_device(PCI_VENDOR_ID_JMICRON, + 0x0585, NULL); + if (pdev) { + hose = pci_bus_to_pci_controller(pdev->bus); + val = read_rc_conf(hose->node, hose->index, + RC_PORT_LINK_CTL); + write_rc_conf(hose->node, hose->index, + RC_PORT_LINK_CTL, val | 0x8); + write_rc_conf(hose->node, hose->index, + RC_PORT_LINK_CTL, val); + } + +} +static void default_halt(void) +{ + local_irq_disable(); + + pr_notice("\n\n** You can safely turn off the power now **\n\n"); + + while (true) + arch_cpu_idle(); +} + +static void default_poweroff(void) +{ + /* No point in taking interrupts anymore. */ + local_irq_disable(); +#ifdef CONFIG_EFI + efi.reset_system(EFI_RESET_SHUTDOWN, EFI_SUCCESS, 0, NULL); +#endif + while (true) + arch_cpu_idle(); +} + +static void default_restart(void) +{ + /* No point in taking interrupts anymore. */ + local_irq_disable(); + + fix_jm585_reset(); +#ifdef CONFIG_EFI + if (efi_capsule_pending(NULL)) + efi_reboot(REBOOT_WARM, NULL); + else + efi_reboot(REBOOT_COLD, NULL); +#endif + + while (true) + arch_cpu_idle(); +} + +void (*pm_restart)(void); + +void (*pm_power_off)(void); +EXPORT_SYMBOL(pm_power_off); + +void (*pm_halt)(void); + +void machine_halt(void) +{ +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif + pm_halt(); +} + +void machine_power_off(void) +{ +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif + pm_power_off(); +} + +void machine_restart(char *command) +{ +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif + do_kernel_restart(command); + pm_restart(); +} + +static int __init sw64_reboot_setup(void) +{ + pm_restart = default_restart; + pm_power_off = default_poweroff; + pm_halt = default_halt; + + return 0; +} +arch_initcall(sw64_reboot_setup); diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c new file mode 100644 index 000000000000..0c1ddb9b46d7 --- /dev/null +++ b/arch/sw_64/kernel/setup.c @@ -0,0 +1,1061 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Bootup setup stuff. + */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_MAGIC_SYSRQ +#include +#include +#endif +#ifdef CONFIG_DEBUG_FS +#include +#endif +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "proto.h" + +#undef DEBUG_DISCONTIG +#ifdef DEBUG_DISCONTIG +#define DBGDCONT(args...) pr_debug(args) +#else +#define DBGDCONT(args...) +#endif + +int __cpu_to_rcid[NR_CPUS]; /* Map logical to physical */ +EXPORT_SYMBOL(__cpu_to_rcid); + +DEFINE_PER_CPU(unsigned long, hard_node_id) = { 0 }; +static DEFINE_PER_CPU(struct cpu, cpu_devices); + +#ifdef CONFIG_SUBARCH_C3B +#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) +struct cma *sw64_kvm_cma; +EXPORT_SYMBOL(sw64_kvm_cma); + +static phys_addr_t kvm_mem_size; +static phys_addr_t kvm_mem_base; + +struct gen_pool *sw64_kvm_pool; +EXPORT_SYMBOL(sw64_kvm_pool); +#endif +#endif + +static inline int phys_addr_valid(unsigned long addr) +{ + /* + * At this point memory probe has not been done such that max_pfn + * and other physical address variables cannot be used, so let's + * roughly judge physical address based on arch specific bit. + */ + return !(addr >> (cpu_desc.pa_bits - 1)); +} + +extern struct atomic_notifier_head panic_notifier_list; +static int sw64_panic_event(struct notifier_block *, unsigned long, void *); +static struct notifier_block sw64_panic_block = { + sw64_panic_event, + NULL, + INT_MAX /* try to do it first */ +}; + +/* the value is IOR: CORE_ONLIE*/ +cpumask_t core_start = CPU_MASK_NONE; + +static struct resource data_resource = { + .name = "Kernel data", + .start = 0, + .end = 0, + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM +}; + +static struct resource code_resource = { + .name = "Kernel code", + .start = 0, + .end = 0, + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM +}; + +static struct resource bss_resource = { + .name = "Kernel bss", + .start = 0, + .end = 0, + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM +}; + +/* A collection of per-processor data. */ +struct cpuinfo_sw64 cpu_data[NR_CPUS]; +EXPORT_SYMBOL(cpu_data); + +DEFINE_STATIC_KEY_TRUE(run_mode_host_key); +DEFINE_STATIC_KEY_FALSE(run_mode_guest_key); +DEFINE_STATIC_KEY_FALSE(run_mode_emul_key); +struct cpu_desc_t cpu_desc; +struct socket_desc_t socket_desc[MAX_NUMSOCKETS]; +int memmap_nr; +struct memmap_entry memmap_map[MAX_NUMMEMMAPS]; +bool memblock_initialized; + +cpumask_t cpu_offline = CPU_MASK_NONE; + +static char command_line[COMMAND_LINE_SIZE] __initdata; +#ifdef CONFIG_CMDLINE_BOOL +static char builtin_cmdline[COMMAND_LINE_SIZE] __initdata = CONFIG_CMDLINE; +#endif + +/* boot_params */ +struct boot_params *sunway_boot_params = (struct boot_params *) (PARAM + 0x100); + +/* + * The format of "screen_info" is strange, and due to early + * i386-setup code. This is just enough to make the console + * code think we're on a VGA color display. + */ + +struct screen_info screen_info = { + .orig_x = 0, + .orig_y = 25, + .orig_video_cols = 80, + .orig_video_lines = 25, + .orig_video_isVGA = 1, + .orig_video_points = 16 +}; +EXPORT_SYMBOL(screen_info); + +/* + * Move global data into per-processor storage. + */ +void store_cpu_data(int cpu) +{ + cpu_data[cpu].last_asid = ASID_FIRST_VERSION; +} + +#ifdef CONFIG_KEXEC + +void *kexec_control_page; + +#define KTEXT_MAX KERNEL_IMAGE_SIZE + +static void __init kexec_control_page_init(void) +{ + phys_addr_t addr; + + addr = memblock_phys_alloc_range(KEXEC_CONTROL_PAGE_SIZE, PAGE_SIZE, + 0, KTEXT_MAX); + kexec_control_page = (void *)(__START_KERNEL_map + addr); +} + +/* + * reserve_crashkernel() - reserves memory are for crash kernel + * + * This function reserves memory area given in "crashkernel=" kernel command + * line parameter. The memory reserved is used by a dump capture kernel when + * primary kernel is crashing. + */ +static void __init reserve_crashkernel(void) +{ + unsigned long long crash_size, crash_base; + int ret; + + ret = parse_crashkernel(boot_command_line, mem_desc.size, + &crash_size, &crash_base); + if (ret || !crash_size) + return; + + if (!crash_size) { + pr_warn("size of crash kernel memory unspecified, no memory reserved for crash kernel\n"); + return; + } + if (!crash_base) { + pr_warn("base of crash kernel memory unspecified, no memory reserved for crash kernel\n"); + return; + } + + if (!memblock_is_region_memory(crash_base, crash_size)) + memblock_add(crash_base, crash_size); + + ret = memblock_reserve(crash_base, crash_size); + if (ret < 0) { + pr_warn("crashkernel reservation failed - memory is in use [mem %#018llx-%#018llx]\n", + crash_base, crash_base + crash_size - 1); + return; + } + + pr_info("Reserving %ldMB of memory at %ldMB for crashkernel (System RAM: %ldMB)\n", + (unsigned long)(crash_size >> 20), + (unsigned long)(crash_base >> 20), + (unsigned long)(mem_desc.size >> 20)); + + ret = add_memmap_region(crash_base, crash_size, memmap_crashkernel); + if (ret) + pr_warn("Add crash kernel area [mem %#018llx-%#018llx] to memmap region failed.\n", + crash_base, crash_base + crash_size - 1); + + if (crash_base >= KERNEL_IMAGE_SIZE) + pr_warn("Crash base should be less than %#x\n", KERNEL_IMAGE_SIZE); + + crashk_res.start = crash_base; + crashk_res.end = crash_base + crash_size - 1; + insert_resource(&iomem_resource, &crashk_res); +} +#else /* !defined(CONFIG_KEXEC) */ +static void __init reserve_crashkernel(void) {} +static void __init kexec_control_page_init(void) {} +#endif /* !defined(CONFIG_KEXEC) */ + +/* + * I/O resources inherited from PeeCees. Except for perhaps the + * turbochannel SWs, everyone has these on some sort of SuperIO chip. + * + * ??? If this becomes less standard, move the struct out into the + * machine vector. + */ + +static void __init +reserve_std_resources(void) +{ + static struct resource standard_io_resources[] = { + { .name = "rtc", .start = -1, .end = -1 }, + { .name = "dma1", .start = 0x00, .end = 0x1f }, + { .name = "pic1", .start = 0x20, .end = 0x3f }, + { .name = "timer", .start = 0x40, .end = 0x5f }, + { .name = "keyboard", .start = 0x60, .end = 0x6f }, + { .name = "dma page reg", .start = 0x80, .end = 0x8f }, + { .name = "pic2", .start = 0xa0, .end = 0xbf }, + { .name = "dma2", .start = 0xc0, .end = 0xdf }, + }; + + struct resource *io = &ioport_resource; + size_t i; + + if (hose_head) { + struct pci_controller *hose; + + for (hose = hose_head; hose; hose = hose->next) + if (hose->index == 0) { + io = hose->io_space; + break; + } + } + + /* Fix up for the Jensen's queer RTC placement. */ + standard_io_resources[0].start = RTC_PORT(0); + standard_io_resources[0].end = RTC_PORT(0) + 0x10; + + for (i = 0; i < ARRAY_SIZE(standard_io_resources); ++i) + request_resource(io, standard_io_resources+i); +} + +static int __init parse_memmap_one(char *p) +{ + char *oldp; + u64 start_at, mem_size; + int ret; + + if (!p) + return -EINVAL; + + if (!strncmp(p, "exactmap", 8)) { + pr_err("\"memmap=exactmap\" not valid on sw64\n"); + return 0; + } + + oldp = p; + mem_size = memparse(p, &p); + if (p == oldp) + return -EINVAL; + + if (*p == '@') { + pr_err("\"memmap=nn@ss\" invalid on sw64\n"); + } else if (*p == '#') { + pr_err("\"memmap=nn#ss\" (force ACPI data) invalid on sw64\n"); + } else if (*p == '$') { + start_at = memparse(p + 1, &p); + ret = add_memmap_region(start_at, mem_size, memmap_reserved); + if (ret) + return ret; + } else { + return -EINVAL; + } + return *p == '\0' ? 0 : -EINVAL; +} + +static int __init setup_memmap(char *str) +{ + while (str) { + char *k = strchr(str, ','); + + if (k) + *k++ = 0; + + parse_memmap_one(str); + str = k; + } + + return 0; +} +early_param("memmap", setup_memmap); + +static int __init setup_cpuoffline(char *p) +{ + cpulist_parse(p, &cpu_offline); + cpumask_clear_cpu(0, &cpu_offline); + return 0; +} +early_param("cpuoffline", setup_cpuoffline); + +#ifdef CONFIG_BLK_DEV_INITRD +static void * __init move_initrd(unsigned long mem_limit) +{ + void *start; + unsigned long size; + + size = initrd_end - initrd_start; + start = memblock_alloc_from(PAGE_ALIGN(size), PAGE_SIZE, 0); + if (!start || __pa(start) + size > mem_limit) { + initrd_start = initrd_end = 0; + return NULL; + } + memmove(start, (void *)initrd_start, size); + initrd_start = (unsigned long)start; + initrd_end = initrd_start + size; + pr_info("initrd moved to 0x%px\n", start); + return start; +} +#else +static void * __init move_initrd(unsigned long mem_limit) +{ + return NULL; +} +#endif + +static bool __init memmap_range_valid(phys_addr_t base, phys_addr_t *size) +{ + if (base > memblock_end_of_DRAM()) + return false; + + if ((base + *size) > memblock_end_of_DRAM()) + *size = memblock_end_of_DRAM() - base; + + return true; +} + +void __init process_memmap(void) +{ + static int i; // Make it static so we won't start over again every time. + int ret; + phys_addr_t base, size; + unsigned long dma_end __maybe_unused = (MAX_DMA32_PFN << PAGE_SHIFT); + + if (!memblock_initialized) + return; + + for (; i < memmap_nr; i++) { + base = memmap_map[i].addr; + size = memmap_map[i].size; + switch (memmap_map[i].type) { + case memmap_reserved: + if (!memmap_range_valid(base, &size)) { + pr_err("reserved memmap region [mem %#018llx-%#018llx] beyond end of memory (%#018llx)\n", + base, base + size - 1, memblock_end_of_DRAM()); + } else { + pr_info("reserved memmap region [mem %#018llx-%#018llx]\n", + base, base + size - 1); + ret = memblock_mark_nomap(base, size); + if (ret) + pr_err("reserve memmap region [mem %#018llx-%#018llx] failed\n", + base, base + size - 1); + else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (base < dma_end)) + pr_warn("memmap region [mem %#018llx-%#018llx] overlapped with DMA32 region\n", + base, base + size - 1); + } + break; + case memmap_pci: + if (!memmap_range_valid(base, &size)) { + pr_err("pci memmap region [mem %#018llx-%#018llx] beyond end of memory (%#018llx)\n", + base, base + size - 1, memblock_end_of_DRAM()); + } else { + pr_info("pci memmap region [mem %#018llx-%#018llx]\n", + base, base + size - 1); + ret = memblock_mark_nomap(base, size); + if (ret) + pr_err("reserve memmap region [mem %#018llx-%#018llx] failed\n", + base, base + size - 1); + } + break; + case memmap_initrd: + if ((base + size) > memblock_end_of_DRAM()) { + phys_addr_t old_base = base; + + base = (unsigned long) move_initrd(memblock_end_of_DRAM()); + if (!base) { + pr_err("initrd memmap region [mem %#018llx-%#018llx] extends beyond end of memory (%#018llx)\n", + old_base, old_base + size - 1, memblock_end_of_DRAM()); + break; + } + memmap_map[i].addr = base; + } + pr_info("initrd memmap region [mem %#018llx-%#018llx]\n", base, base + size - 1); + ret = memblock_reserve(base, size); + if (ret) + pr_err("reserve memmap region [mem %#018llx-%#018llx] failed\n", + base, base + size - 1); + break; + case memmap_kvm: + case memmap_crashkernel: + /* kvm and crashkernel are handled elsewhere, skip */ + break; + case memmap_acpi: + pr_err("ACPI memmap region is not supported.\n"); + break; + case memmap_use: + pr_err("Force usage memmap region is not supported.\n"); + break; + case memmap_protected: + pr_err("Protected memmap region is not supported.\n"); + break; + default: + pr_err("Unknown type of memmap region.\n"); + } + } +} + +int __init add_memmap_region(u64 addr, u64 size, enum memmap_types type) +{ + if (memmap_nr >= ARRAY_SIZE(memmap_map)) { + pr_err("Ooops! Too many entries in the memory map!\n"); + return -EPERM; + } + + if (addr + size <= addr) { + pr_warn("Trying to add an invalid memory region, skipped\n"); + return -EINVAL; + } + + memmap_map[memmap_nr].addr = addr; + memmap_map[memmap_nr].size = size; + memmap_map[memmap_nr].type = type; + memmap_nr++; + + process_memmap(); + + return 0; +} + +static struct resource* __init +insert_ram_resource(u64 start, u64 end, bool reserved) +{ + struct resource *res = + kzalloc(sizeof(struct resource), GFP_ATOMIC); + if (!res) + return NULL; + if (reserved) { + res->name = "reserved"; + res->flags = IORESOURCE_MEM; + } else { + res->name = "System RAM"; + res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + } + res->start = start; + res->end = end; + if (insert_resource(&iomem_resource, res)) { + kfree(res); + return NULL; + } + return res; +} + +static int __init request_standard_resources(void) +{ + struct memblock_region *mblk; + + extern char _text[], _etext[]; + extern char _sdata[], _edata[]; + extern char __bss_start[], __bss_stop[]; + + for_each_mem_region(mblk) { + if (!memblock_is_nomap(mblk)) + insert_ram_resource(mblk->base, + mblk->base + mblk->size - 1, 0); + else + insert_ram_resource(mblk->base, + mblk->base + mblk->size - 1, 1); + } + + code_resource.start = __pa_symbol(_text); + code_resource.end = __pa_symbol(_etext)-1; + data_resource.start = __pa_symbol(_sdata); + data_resource.end = __pa_symbol(_edata)-1; + bss_resource.start = __pa_symbol(__bss_start); + bss_resource.end = __pa_symbol(__bss_stop)-1; + + insert_resource(&iomem_resource, &code_resource); + insert_resource(&iomem_resource, &data_resource); + insert_resource(&iomem_resource, &bss_resource); + + return 0; +} +subsys_initcall(request_standard_resources); + +#ifdef CONFIG_NUMA +extern void cpu_set_node(void); +#endif + +static void __init show_socket_mem_layout(void) +{ + int i; + phys_addr_t base, size, end; + + base = 0; + + pr_info("Socket memory layout:\n"); + for (i = 0; i < MAX_NUMSOCKETS; i++) { + if (socket_desc[i].is_online) { + size = socket_desc[i].socket_mem; + end = base + size - 1; + pr_info("Socket %d: [mem %#018llx-%#018llx], size %llu\n", + i, base, end, size); + base = end + 1; + } + } + pr_info("Reserved memory size for Socket 0: %#lx\n", NODE0_START); +} + +int page_is_ram(unsigned long pfn) +{ + pfn <<= PAGE_SHIFT; + + return pfn >= mem_desc.base && pfn < (mem_desc.base + mem_desc.size); +} + +static int __init topology_init(void) +{ + int i, ret; + + for_each_possible_cpu(i) { + struct cpu *cpu = &per_cpu(cpu_devices, i); + +#ifdef CONFIG_HOTPLUG_CPU + if (i != 0) + cpu->hotpluggable = 1; +#endif + ret = register_cpu(cpu, i); + if (unlikely(ret)) + pr_warn("Warning: %s: register_cpu %d failed (%d)\n", + __func__, i, ret); + } + + return 0; +} +subsys_initcall(topology_init); + +static void __init setup_machine_fdt(void) +{ +#ifdef CONFIG_USE_OF + void *dt_virt; + const char *name; + + /* Give a chance to select kernel builtin DTB firstly */ + if (IS_ENABLED(CONFIG_BUILTIN_DTB)) + dt_virt = (void *)__dtb_start; + else { + dt_virt = (void *)sunway_boot_params->dtb_start; + if (virt_to_phys(dt_virt) < virt_to_phys(__bss_stop)) { + pr_emerg("BUG: DTB has been corrupted by kernel image!\n"); + while (true) + cpu_relax(); + } + } + + if (!phys_addr_valid(__boot_pa(dt_virt)) || + !early_init_dt_scan(dt_virt)) { + pr_crit("\n" + "Error: invalid device tree blob at virtual address %px\n" + "The dtb must be 8-byte aligned and must not exceed 2 MB in size\n" + "\nPlease check your bootloader.", + dt_virt); + + while (true) + cpu_relax(); + } + + name = of_flat_dt_get_machine_name(); + if (!name) + return; + + pr_info("Machine model: %s\n", name); +#else + pr_info("Kernel disable device tree support.\n"); + return; +#endif +} + +void __init device_tree_init(void) +{ + unflatten_and_copy_device_tree(); + sunway_boot_params->dtb_start = (__u64)initial_boot_params; +} + +static void __init setup_cpu_info(void) +{ + int i; + struct cache_desc *c; + unsigned long val; + + val = cpuid(GET_TABLE_ENTRY, 0); + cpu_desc.model = CPUID_MODEL(val); + cpu_desc.family = CPUID_FAMILY(val); + cpu_desc.chip_var = CPUID_CHIP_VAR(val); + cpu_desc.arch_var = CPUID_ARCH_VAR(val); + cpu_desc.arch_rev = CPUID_ARCH_REV(val); + cpu_desc.pa_bits = CPUID_PA_BITS(val); + cpu_desc.va_bits = CPUID_VA_BITS(val); + + for (i = 0; i < VENDOR_ID_MAX; i++) { + val = cpuid(GET_VENDOR_ID, i); + memcpy(cpu_desc.vendor_id + (i * 8), &val, 8); + } + + for (i = 0; i < MODEL_MAX; i++) { + val = cpuid(GET_MODEL, i); + memcpy(cpu_desc.model_id + (i * 8), &val, 8); + } + + cpu_desc.frequency = cpuid(GET_CPU_FREQ, 0) * 1000UL * 1000UL; + + for (i = 0; i < NR_CPUS; i++) { + c = &(cpu_data[i].icache); + val = cpuid(GET_CACHE_INFO, L1_ICACHE); + c->size = CACHE_SIZE(val); + c->linesz = 1 << (CACHE_LINE_BITS(val)); + c->sets = 1 << (CACHE_INDEX_BITS(val)); + c->ways = c->size / c->sets / c->linesz; + + c = &(cpu_data[i].dcache); + val = cpuid(GET_CACHE_INFO, L1_DCACHE); + c->size = CACHE_SIZE(val); + c->linesz = 1 << (CACHE_LINE_BITS(val)); + c->sets = 1 << (CACHE_INDEX_BITS(val)); + c->ways = c->size / c->sets / c->linesz; + + c = &(cpu_data[i].scache); + val = cpuid(GET_CACHE_INFO, L2_CACHE); + c->size = CACHE_SIZE(val); + c->linesz = 1 << (CACHE_LINE_BITS(val)); + c->sets = 1 << (CACHE_INDEX_BITS(val)); + c->ways = c->size / c->sets / c->linesz; + + c = &(cpu_data[i].tcache); + val = cpuid(GET_CACHE_INFO, L3_CACHE); + c->size = CACHE_SIZE(val); + c->linesz = 1 << (CACHE_LINE_BITS(val)); + c->sets = 1 << (CACHE_INDEX_BITS(val)); + c->ways = c->size / c->sets / c->linesz; + } +} + +static void __init setup_run_mode(void) +{ + if (*(unsigned long *)MMSIZE) { + static_branch_disable(&run_mode_host_key); + if (*(unsigned long *)MMSIZE & EMUL_FLAG) { + pr_info("run mode: emul\n"); + static_branch_disable(&run_mode_guest_key); + static_branch_enable(&run_mode_emul_key); + + } else { + pr_info("run mode: guest\n"); + static_branch_enable(&run_mode_guest_key); + static_branch_disable(&run_mode_emul_key); + } + } else { + pr_info("run mode: host\n"); + static_branch_enable(&run_mode_host_key); + static_branch_disable(&run_mode_guest_key); + static_branch_disable(&run_mode_emul_key); + } +} + +static void __init setup_socket_info(void) +{ + int i; + int numsockets = sw64_chip->get_cpu_num(); + + memset(socket_desc, 0, MAX_NUMSOCKETS * sizeof(struct socket_desc_t)); + + for (i = 0; i < numsockets; i++) { + socket_desc[i].is_online = 1; + if (sw64_chip_init->early_init.get_node_mem) + socket_desc[i].socket_mem = sw64_chip_init->early_init.get_node_mem(i); + } +} + +#ifdef CONFIG_BLK_DEV_INITRD +static void __init reserve_mem_for_initrd(void) +{ + int ret; + + initrd_start = sunway_boot_params->initrd_start; + if (initrd_start) { + initrd_start = __pa(initrd_start) + PAGE_OFFSET; + initrd_end = initrd_start + sunway_boot_params->initrd_size; + pr_info("Initial ramdisk at: 0x%px (%llu bytes)\n", + (void *)initrd_start, sunway_boot_params->initrd_size); + + ret = add_memmap_region(__pa(initrd_start), initrd_end - initrd_start, memmap_initrd); + if (ret) + pr_err("Add initrd area [mem %#018lx-%#018lx] to memmap region failed.\n", + __pa(initrd_start), __pa(initrd_end - 1)); + } +} +#endif /* CONFIG_BLK_DEV_INITRD */ + +#ifdef CONFIG_SUBARCH_C3B +#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) +static int __init early_kvm_reserved_mem(char *p) +{ + if (!p) { + pr_err("Config string not provided\n"); + return -EINVAL; + } + + kvm_mem_size = memparse(p, &p); + if (*p != '@') + return -EINVAL; + kvm_mem_base = memparse(p + 1, &p); + return 0; +} +early_param("kvm_mem", early_kvm_reserved_mem); + +void __init sw64_kvm_reserve(void) +{ + kvm_cma_declare_contiguous(kvm_mem_base, kvm_mem_size, 0, + PAGE_SIZE, 0, "sw64_kvm_cma", &sw64_kvm_cma); +} +#endif +#endif + +void __init +setup_arch(char **cmdline_p) +{ + /** + * Work around the unaligned access exception to parse ACPI + * tables in the following function acpi_boot_table_init(). + */ + trap_init(); + + jump_label_init(); + setup_cpu_info(); + setup_run_mode(); + setup_chip_ops(); + setup_socket_info(); + show_socket_mem_layout(); + sw64_chip_init->early_init.setup_core_map(&core_start); + if (is_guest_or_emul()) + get_vt_smp_info(); + + setup_sched_clock(); + + setup_machine_fdt(); + + /* Register a call for panic conditions. */ + atomic_notifier_chain_register(&panic_notifier_list, + &sw64_panic_block); + + callback_init(); + + /* command line */ + if (!sunway_boot_params->cmdline) + sunway_boot_params->cmdline = (unsigned long)COMMAND_LINE; + + strscpy(boot_command_line, (char *)sunway_boot_params->cmdline, COMMAND_LINE_SIZE); + +#if IS_ENABLED(CONFIG_CMDLINE_BOOL) +#if IS_ENABLED(CONFIG_CMDLINE_OVERRIDE) + strscpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE); + strscpy((char *)sunway_boot_params->cmdline, boot_command_line, COMMAND_LINE_SIZE); +#else + if (builtin_cmdline[0]) { + /* append builtin to boot loader cmdline */ + strlcat(boot_command_line, " ", COMMAND_LINE_SIZE); + strlcat(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE); + } +#endif /* CMDLINE_EXTEND */ +#endif + + strscpy(command_line, boot_command_line, COMMAND_LINE_SIZE); + *cmdline_p = command_line; + + /* + * Process command-line arguments. + */ + parse_early_param(); + + /* Find our memory. */ + mem_detect(); + +#ifdef CONFIG_PCI + reserve_mem_for_pci(); +#endif + +#ifdef CONFIG_BLK_DEV_INITRD + reserve_mem_for_initrd(); +#endif + + sw64_memblock_init(); + + reserve_crashkernel(); + + /* Reserve large chunks of memory for use by CMA for KVM. */ +#ifdef CONFIG_SUBARCH_C3B +#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) + sw64_kvm_reserve(); +#endif +#endif + + efi_init(); + + /* Try to upgrade ACPI tables via initrd */ + acpi_table_upgrade(); + + /* Parse the ACPI tables for possible boot-time configuration */ + acpi_boot_table_init(); + +#ifdef CONFIG_SMP + setup_smp(); +#else + store_cpu_data(0); +#endif + + sw64_numa_init(); + + memblock_dump_all(); + + sparse_init(); + + zone_sizes_init(); + + paging_init(); + + kexec_control_page_init(); + + /* + * Initialize the machine. Usually has to do with setting up + * DMA windows and the like. + */ + sw64_init_arch(); + + /* Reserve standard resources. */ + reserve_std_resources(); + + /* + * Give us a default console. TGA users will see nothing until + * chr_dev_init is called, rather late in the boot sequence. + */ + +#ifdef CONFIG_VT +#if defined(CONFIG_VGA_CONSOLE) + conswitchp = &vga_con; +#elif defined(CONFIG_DUMMY_CONSOLE) + conswitchp = &dummy_con; +#endif +#endif + + /* Default root filesystem to sda2. */ + ROOT_DEV = MKDEV(SCSI_DISK0_MAJOR, 2); + + if (acpi_disabled) { +#ifdef CONFIG_NUMA + cpu_set_node(); +#endif + device_tree_init(); + } +} + + +static int +show_cpuinfo(struct seq_file *f, void *slot) +{ + int i; + unsigned long cpu_freq; + + cpu_freq = cpuid(GET_CPU_FREQ, 0); + + for_each_online_cpu(i) { + /* + * glibc reads /proc/cpuinfo to determine the number of + * online processors, looking for lines beginning with + * "processor". Give glibc what it expects. + */ + seq_printf(f, "processor\t: %u\n" + "vendor_id\t: %s\n" + "cpu family\t: %d\n" + "model\t\t: %u\n" + "model name\t: %s CPU @ %lu.%lu%luGHz\n" + "cpu variation\t: %u\n" + "cpu revision\t: %u\n", + i, cpu_desc.vendor_id, cpu_desc.family, + cpu_desc.model, cpu_desc.model_id, + cpu_freq / 1000, (cpu_freq % 1000) / 100, + (cpu_freq % 100) / 10, + cpu_desc.arch_var, cpu_desc.arch_rev); + seq_printf(f, "cpu MHz\t\t: %lu.00\n" + "cache size\t: %u KB\n" + "physical id\t: %d\n" + "bogomips\t: %lu.%02lu\n", + get_cpu_freq() / 1000 / 1000, cpu_data[i].tcache.size >> 10, + cpu_topology[i].package_id, + loops_per_jiffy / (500000/HZ), + (loops_per_jiffy / (5000/HZ)) % 100); + + seq_printf(f, "flags\t\t: fpu simd vpn upn cpuid\n"); + seq_printf(f, "page size\t: %d\n", 8192); + seq_printf(f, "cache_alignment\t: %d\n", cpu_data[i].tcache.linesz); + seq_printf(f, "address sizes\t: %u bits physical, %u bits virtual\n\n", + cpu_desc.pa_bits, cpu_desc.va_bits); + } + return 0; +} + +/* + * We show only CPU #0 info. + */ +static void * +c_start(struct seq_file *f, loff_t *pos) +{ + return *pos < 1 ? (void *)1 : NULL; +} + +static void * +c_next(struct seq_file *f, void *v, loff_t *pos) +{ + (*pos)++; + return NULL; +} + +static void +c_stop(struct seq_file *f, void *v) +{ +} + +const struct seq_operations cpuinfo_op = { + .start = c_start, + .next = c_next, + .stop = c_stop, + .show = show_cpuinfo, +}; + + +static int +sw64_panic_event(struct notifier_block *this, unsigned long event, void *ptr) +{ + return NOTIFY_DONE; +} + +static __init int add_pcspkr(void) +{ + struct platform_device *pd; + int ret; + + pd = platform_device_alloc("pcspkr", -1); + if (!pd) + return -ENOMEM; + + ret = platform_device_add(pd); + if (ret) + platform_device_put(pd); + + return ret; +} +device_initcall(add_pcspkr); + +#ifdef CONFIG_DEBUG_FS +struct dentry *sw64_debugfs_dir; +EXPORT_SYMBOL(sw64_debugfs_dir); + +static int __init debugfs_sw64(void) +{ + struct dentry *d; + + d = debugfs_create_dir("sw64", NULL); + if (!d) + return -ENOMEM; + sw64_debugfs_dir = d; + return 0; +} +arch_initcall(debugfs_sw64); +#endif + +#ifdef CONFIG_OF +static int __init sw64_of_init(void) +{ + of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); + return 0; +} +core_initcall(sw64_of_init); +#endif + +#ifdef CONFIG_SUBARCH_C3B +#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) +static int __init sw64_kvm_pool_init(void) +{ + int status = 0; + unsigned long kvm_pool_virt; + struct page *base_page, *end_page, *p; + + if (!sw64_kvm_cma) + goto out; + + kvm_pool_virt = (unsigned long)kvm_mem_base; + + sw64_kvm_pool = gen_pool_create(PAGE_SHIFT, -1); + if (!sw64_kvm_pool) + goto out; + + status = gen_pool_add_virt(sw64_kvm_pool, kvm_pool_virt, kvm_mem_base, + kvm_mem_size, -1); + if (status < 0) { + pr_err("failed to add memory chunks to sw64 kvm pool\n"); + gen_pool_destroy(sw64_kvm_pool); + sw64_kvm_pool = NULL; + goto out; + } + gen_pool_set_algo(sw64_kvm_pool, gen_pool_best_fit, NULL); + + base_page = pfn_to_page(kvm_mem_base >> PAGE_SHIFT); + end_page = pfn_to_page((kvm_mem_base + kvm_mem_size - 1) >> PAGE_SHIFT); + + p = base_page; + while (p <= end_page && page_ref_count(p) == 0) { + set_page_count(p, 1); + page_mapcount_reset(p); + SetPageReserved(p); + p++; + } + + return status; + +out: + return -ENOMEM; +} +core_initcall_sync(sw64_kvm_pool_init); +#endif +#endif -- Gitee From 612888d046fba450ed6ebb50998163c10a69ffd8 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:38 +0800 Subject: [PATCH 008/524] sw64: add topology setup routine Add topology setup for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/topology.h | 71 ++++++++++ arch/sw_64/kernel/topology.c | 212 ++++++++++++++++++++++++++++++ 2 files changed, 283 insertions(+) create mode 100644 arch/sw_64/include/asm/topology.h create mode 100644 arch/sw_64/kernel/topology.c diff --git a/arch/sw_64/include/asm/topology.h b/arch/sw_64/include/asm/topology.h new file mode 100644 index 000000000000..25ec7b9e9431 --- /dev/null +++ b/arch/sw_64/include/asm/topology.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_TOPOLOGY_H +#define _ASM_SW64_TOPOLOGY_H + +#include +#include +#include +#include +#include +#include + +extern struct cpu_topology cpu_topology[NR_CPUS]; + +#define topology_physical_package_id(cpu) (cpu_topology[cpu].package_id) +#define topology_core_id(cpu) (cpu_topology[cpu].core_id) +#define topology_core_cpumask(cpu) (&cpu_topology[cpu].core_sibling) +#define topology_sibling_cpumask(cpu) (&cpu_topology[cpu].thread_sibling) +#define topology_llc_cpumask(cpu) (&cpu_topology[cpu].llc_sibling) + +void init_cpu_topology(void); +void store_cpu_topology(int cpuid); +void remove_cpu_topology(int cpuid); +const struct cpumask *cpu_coregroup_mask(int cpu); + +static inline int rcid_to_thread_id(int rcid) +{ + return (rcid & THREAD_ID_MASK) >> THREAD_ID_SHIFT; +} + +static inline int rcid_to_core_id(int rcid) +{ + return (rcid & CORE_ID_MASK) >> CORE_ID_SHIFT; +} + +static inline int rcid_to_domain_id(int rcid) +{ + return (rcid & DOMAIN_ID_MASK) >> DOMAIN_ID_SHIFT; +} + +#ifdef CONFIG_NUMA + +#ifndef CONFIG_DEBUG_PER_CPU_MAPS +extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; +/* Returns a pointer to the cpumask of CPUs on Node 'node'. */ +#define cpumask_of_node(node) ((node) == NUMA_NO_NODE ? \ + cpu_all_mask : \ + node_to_cpumask_map[node]) +#else +extern const struct cpumask *cpumask_of_node(int node); +#endif /* CONFIG_DEBUG_PER_CPU_MAPS */ + +extern void numa_add_cpu(unsigned int cpu); +extern void numa_remove_cpu(unsigned int cpu); +extern void numa_store_cpu_info(unsigned int cpu); +extern int __node_distance(int from, int to); +#define node_distance(a, b) __node_distance(a, b) +#define parent_node(node) (node) +#define cpumask_of_pcibus(bus) (cpu_online_mask) +#else /* !CONFIG_NUMA */ +static inline void numa_add_cpu(unsigned int cpu) { } +static inline void numa_remove_cpu(unsigned int cpu) { } +static inline void numa_store_cpu_info(unsigned int cpu) { } +#endif /* CONFIG_NUMA */ + +extern void get_vt_smp_info(void); + +#include + +static inline void arch_fix_phys_package_id(int num, u32 slot) { } + +#endif /* _ASM_SW64_TOPOLOGY_H */ diff --git a/arch/sw_64/kernel/topology.c b/arch/sw_64/kernel/topology.c new file mode 100644 index 000000000000..8371c013446f --- /dev/null +++ b/arch/sw_64/kernel/topology.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include + +static int __init parse_dt_topology(void) +{ + return 0; +} + +/* + * cpu topology table + */ +struct cpu_topology cpu_topology[NR_CPUS]; +EXPORT_SYMBOL_GPL(cpu_topology); + +int topo_nr_threads, topo_nr_cores, topo_nr_maxcpus; + +static int topo_nr_cpus; +static int topo_threads[NR_CPUS]; +static int topo_cores[NR_CPUS]; +static int topo_packages[NR_CPUS]; + +void __init get_vt_smp_info(void) +{ + unsigned long smp_info; + + smp_info = sw64_io_read(0, SMP_INFO); + if (smp_info == -1UL) + smp_info = 0; + topo_nr_threads = (smp_info >> VT_THREADS_SHIFT) & VT_THREADS_MASK; + topo_nr_cores = (smp_info >> VT_CORES_SHIFT) & VT_CORES_MASK; + topo_nr_maxcpus = (smp_info >> VT_MAX_CPUS_SHIFT) & VT_MAX_CPUS_MASK; +} + +static void __init init_topo_threads(void) +{ + int i, j; + + if (topo_nr_threads == 0) + topo_nr_threads = 1; + + for (i = 0; i < topo_nr_cpus; i += topo_nr_threads) { + for (j = 0; j < topo_nr_threads; j++) + topo_threads[i+j] = j; + } +} + +static void __init init_topo_cores(void) +{ + int i, j; + + if (topo_nr_cores == 0) + topo_nr_cores = topo_nr_cpus; + + for (i = 0; i < topo_nr_cpus; i += topo_nr_cores) { + for (j = 0; j < topo_nr_cores; j++) + topo_cores[i+j] = j; + } +} + +static void __init init_topo_packages(void) +{ + int i, j, packet_index = 0; + int topo_nr_packages = topo_nr_cpus / (topo_nr_cores * topo_nr_threads); + int div_package = topo_nr_cpus / topo_nr_packages; + + for (i = 0; i < topo_nr_cpus; i += div_package) { + for (j = 0 ; j < div_package; j++) + topo_packages[i+j] = packet_index; + packet_index++; + } + if (packet_index > topo_nr_packages) + pr_err("topo_cores init failed.\n"); +} + +static void __init init_topology_array(void) +{ + topo_nr_cpus = num_present_cpus(); + if (topo_nr_maxcpus > topo_nr_cpus) + topo_nr_cpus = topo_nr_maxcpus; + init_topo_threads(); + init_topo_cores(); + init_topo_packages(); +} + +const struct cpumask *cpu_coregroup_mask(int cpu) +{ + return topology_llc_cpumask(cpu); +} + +static void update_siblings_masks(int cpu) +{ + struct cpu_topology *cpu_topo = &cpu_topology[cpu]; + int sib; + + /* update core and thread sibling masks */ + for_each_online_cpu(sib) { + struct cpu_topology *sib_topo = &cpu_topology[sib]; + + if (cpu_topo->package_id == sib_topo->package_id) { + cpumask_set_cpu(cpu, &sib_topo->core_sibling); + cpumask_set_cpu(sib, &cpu_topo->core_sibling); + cpumask_set_cpu(cpu, &sib_topo->llc_sibling); + cpumask_set_cpu(sib, &cpu_topo->llc_sibling); + + if (cpu_topo->core_id == sib_topo->core_id) { + cpumask_set_cpu(cpu, &sib_topo->thread_sibling); + cpumask_set_cpu(sib, &cpu_topo->thread_sibling); + } + } + } +} + +void store_cpu_topology(int cpu) +{ + struct cpu_topology *cpu_topo = &cpu_topology[cpu]; + + if (cpu_topo->package_id != -1) + goto topology_populated; + + if (is_guest_or_emul()) { + cpu_topo->package_id = topo_packages[cpu]; + cpu_topo->core_id = topo_cores[cpu]; + cpu_topo->thread_id = topo_threads[cpu]; + goto topology_populated; + } + + cpu_topo->package_id = rcid_to_domain_id(cpu_to_rcid(cpu)); + cpu_topo->core_id = rcid_to_core_id(cpu_to_rcid(cpu)); + cpu_topo->thread_id = rcid_to_thread_id(cpu_to_rcid(cpu)); + + pr_debug("CPU%u: socket %d core %d thread %d\n", + cpu, cpu_topo->package_id, cpu_topo->core_id, + cpu_topo->thread_id); + +topology_populated: + update_siblings_masks(cpu); +} + +static void clear_cpu_topology(int cpu) +{ + struct cpu_topology *cpu_topo = &cpu_topology[cpu]; + + cpumask_clear(&cpu_topo->llc_sibling); + cpumask_set_cpu(cpu, &cpu_topo->llc_sibling); + + cpumask_clear(&cpu_topo->core_sibling); + cpumask_set_cpu(cpu, &cpu_topo->core_sibling); + cpumask_clear(&cpu_topo->thread_sibling); + cpumask_set_cpu(cpu, &cpu_topo->thread_sibling); +} + +static void __init reset_cpu_topology(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct cpu_topology *cpu_topo = &cpu_topology[cpu]; + + cpu_topo->thread_id = -1; + cpu_topo->core_id = 0; + cpu_topo->package_id = -1; + + clear_cpu_topology(cpu); + } +} + +void remove_cpu_topology(int cpu) +{ + int sibling; + + for_each_cpu(sibling, topology_core_cpumask(cpu)) + cpumask_clear_cpu(cpu, topology_core_cpumask(sibling)); + for_each_cpu(sibling, topology_sibling_cpumask(cpu)) + cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling)); + for_each_cpu(sibling, topology_llc_cpumask(cpu)) + cpumask_clear_cpu(cpu, topology_llc_cpumask(sibling)); + + clear_cpu_topology(cpu); +} + +#ifdef CONFIG_ACPI +static int __init parse_acpi_topology(void) +{ + return 0; +} +#else +static inline int __init parse_acpi_topology(void) +{ + return -EINVAL; +} +#endif + +void __init init_cpu_topology(void) +{ + reset_cpu_topology(); + + if (is_guest_or_emul()) + init_topology_array(); + /* + * Discard anything that was parsed if we hit an error so we + * don't use partial information. + */ + if (!acpi_disabled && parse_acpi_topology()) + reset_cpu_topology(); + else if (of_have_populated_dt() && parse_dt_topology()) + reset_cpu_topology(); +} -- Gitee From 6175b272d8c186c3f8472ff208439f7a415a3a27 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:38 +0800 Subject: [PATCH 009/524] sw64: add timer support Add timer for basic SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/tc.h | 16 ++++++++++++++++ arch/sw_64/include/asm/timer.h | 11 +++++++++++ arch/sw_64/include/asm/timex.h | 23 +++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 arch/sw_64/include/asm/tc.h create mode 100644 arch/sw_64/include/asm/timer.h create mode 100644 arch/sw_64/include/asm/timex.h diff --git a/arch/sw_64/include/asm/tc.h b/arch/sw_64/include/asm/tc.h new file mode 100644 index 000000000000..aa39c3528e3f --- /dev/null +++ b/arch/sw_64/include/asm/tc.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_TC_H +#define _ASM_SW64_TC_H + +static inline unsigned long rdtc(void) +{ + unsigned long ret; + + __asm__ __volatile__ ("rtc %0" : "=r"(ret)); + return ret; +} + +extern void tc_sync_clear(void); +extern void tc_sync_ready(void *ignored); +extern void tc_sync_set(void); +#endif /* _ASM_SW64_TC_H */ diff --git a/arch/sw_64/include/asm/timer.h b/arch/sw_64/include/asm/timer.h new file mode 100644 index 000000000000..9ea9e0a538d0 --- /dev/null +++ b/arch/sw_64/include/asm/timer.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_TIMER_H +#define _ASM_SW64_TIMER_H + +extern void sw64_setup_clocksource(void); + +extern void sw64_setup_timer(void); + +extern void __init setup_sched_clock(void); + +#endif /* _ASM_SW64_TIMER_H */ diff --git a/arch/sw_64/include/asm/timex.h b/arch/sw_64/include/asm/timex.h new file mode 100644 index 000000000000..a5760bf8abd4 --- /dev/null +++ b/arch/sw_64/include/asm/timex.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_TIMEX_H +#define _ASM_SW64_TIMEX_H + +#include + +/* With only one or two oddballs, we use the RTC as the ticker, selecting + * the 32.768kHz reference clock, which nicely divides down to our HZ. + */ +#define CLOCK_TICK_RATE 32768 + +/* + * Standard way to access the cycle counter. + */ + +typedef unsigned long cycles_t; + +static inline cycles_t get_cycles(void) +{ + return rdtc(); +} + +#endif /* _ASM_SW64_TIMEX_H */ -- Gitee From c98a70f14a188552b484f7c1d50809419a5182e8 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:16 +0800 Subject: [PATCH 010/524] sw64: add irq handling support Add interrupt handling mechanism for basic SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hardirq.h | 24 +++++++ arch/sw_64/include/asm/hw_irq.h | 16 +++++ arch/sw_64/include/asm/irq.h | 31 +++++++++ arch/sw_64/include/asm/irq_impl.h | 48 +++++++++++++ arch/sw_64/include/asm/irqflags.h | 55 +++++++++++++++ arch/sw_64/kernel/irq.c | 108 ++++++++++++++++++++++++++++++ arch/sw_64/kernel/irq_sw64.c | 84 +++++++++++++++++++++++ arch/sw_64/kernel/time.c | 63 +++++++++++++++++ 8 files changed, 429 insertions(+) create mode 100644 arch/sw_64/include/asm/hardirq.h create mode 100644 arch/sw_64/include/asm/hw_irq.h create mode 100644 arch/sw_64/include/asm/irq.h create mode 100644 arch/sw_64/include/asm/irq_impl.h create mode 100644 arch/sw_64/include/asm/irqflags.h create mode 100644 arch/sw_64/kernel/irq.c create mode 100644 arch/sw_64/kernel/irq_sw64.c create mode 100644 arch/sw_64/kernel/time.c diff --git a/arch/sw_64/include/asm/hardirq.h b/arch/sw_64/include/asm/hardirq.h new file mode 100644 index 000000000000..03368c3659dd --- /dev/null +++ b/arch/sw_64/include/asm/hardirq.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_HARDIRQ_H +#define _ASM_SW64_HARDIRQ_H + +void ack_bad_irq(unsigned int irq); +#define ack_bad_irq ack_bad_irq + +#include + +#define __ARCH_IRQ_STAT +typedef struct { + u16 __softirq_pending; + unsigned int timer_irqs_event; +} ____cacheline_aligned irq_cpustat_t; + +DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); + +#define inc_irq_stat(member) this_cpu_inc(irq_stat.member) +#define arch_irq_stat_cpu arch_irq_stat_cpu +#define arch_irq_stat arch_irq_stat +extern u64 arch_irq_stat_cpu(unsigned int cpu); +extern u64 arch_irq_stat(void); + +#endif /* _ASM_SW64_HARDIRQ_H */ diff --git a/arch/sw_64/include/asm/hw_irq.h b/arch/sw_64/include/asm/hw_irq.h new file mode 100644 index 000000000000..3cfc725f7517 --- /dev/null +++ b/arch/sw_64/include/asm/hw_irq.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_HW_IRQ_H +#define _ASM_SW64_HW_IRQ_H + +#include + +extern volatile unsigned long irq_err_count; +DECLARE_PER_CPU(unsigned long, irq_pmi_count); + +#define ACTUAL_NR_IRQS NR_IRQS + +#ifdef CONFIG_PCI_MSI +typedef unsigned int vector_irq_t[PERCPU_MSI_IRQS]; +DECLARE_PER_CPU(vector_irq_t, vector_irq); +#endif +#endif /* _ASM_SW64_HW_IRQ_H */ diff --git a/arch/sw_64/include/asm/irq.h b/arch/sw_64/include/asm/irq.h new file mode 100644 index 000000000000..b3ac4105c29e --- /dev/null +++ b/arch/sw_64/include/asm/irq.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_IRQ_H +#define _ASM_SW64_IRQ_H + +/* + * arch/sw/include/asm/irq.h + * + * (C) 2012 OSKernel JN + */ + +#include + +#define NR_VECTORS_PERCPU 256 +#define NR_IRQS_LEGACY 16 +#define NR_IRQS ((NR_VECTORS_PERCPU + NR_IRQS_LEGACY) * NR_CPUS) + +static inline int irq_canonicalize(int irq) +{ + /* + * XXX is this true for all Sw? The old serial driver + * did it this way for years without any complaints, so.... + */ + return ((irq == 2) ? 9 : irq); +} + +struct pt_regs; +extern void (*perf_irq)(unsigned long vector, struct pt_regs *regs); +extern void fixup_irqs(void); +extern void sw64_timer_interrupt(void); + +#endif /* _ASM_SW64_IRQ_H */ diff --git a/arch/sw_64/include/asm/irq_impl.h b/arch/sw_64/include/asm/irq_impl.h new file mode 100644 index 000000000000..797af433a126 --- /dev/null +++ b/arch/sw_64/include/asm/irq_impl.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file contains declarations and inline functions for interfacing + * with the IRQ handling routines in irq.c. + */ + +#ifndef _ASM_SW64_IRQ_IMPL_H +#define _ASM_SW64_IRQ_IMPL_H + +#include +#include +#include + +#include + +#define SW64_PCIE0_INT_BASE 17 +#define SW64_PCIE0_MSI_BASE 21 + +#define SW64_PCIE1_INT_BASE 277 +#define SW64_PCIE1_MSI_BASE 281 + +#define RTC_IRQ 8 +#define SWI2C_IRQ 14 + +enum sw64_irq_type { + INT_IPI = 1, + INT_PC0 = 2, + INT_PC1 = 3, + INT_INTx = 5, + INT_MSI = 6, + INT_MT = 7, + INT_RTC = 9, + INT_FAULT = 10, + INT_VT_SERIAL = 12, + INT_VT_HOTPLUG = 13, + INT_DEV = 17, + INT_NMI = 18, + INT_LEGACY = 31, +}; + +extern struct irqaction timer_irqaction; +extern void init_rtc_irq(irq_handler_t handler); +extern void handle_irq(int irq); +extern void handle_ipi(struct pt_regs *regs); +extern void __init sw64_init_irq(void); +extern irqreturn_t timer_interrupt(int irq, void *dev); + +#endif /* _ASM_SW64_IRQ_IMPL_H */ diff --git a/arch/sw_64/include/asm/irqflags.h b/arch/sw_64/include/asm/irqflags.h new file mode 100644 index 000000000000..b4440f25a51d --- /dev/null +++ b/arch/sw_64/include/asm/irqflags.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_IRQFLAGS_H +#define _ASM_SW64_IRQFLAGS_H + +#include + +#define IPL_MIN 0 +#define IPL_MAX 7 + +#define getipl() (rdps() & 7) +#define setipl(ipl) ((void) swpipl(ipl)) + +static inline unsigned long arch_local_save_flags(void) +{ + return rdps(); +} + +static inline void arch_local_irq_disable(void) +{ + setipl(IPL_MAX); + barrier(); +} + +static inline unsigned long arch_local_irq_save(void) +{ + unsigned long flags = swpipl(IPL_MAX); + + barrier(); + return flags; +} + +static inline void arch_local_irq_enable(void) +{ + barrier(); + setipl(IPL_MIN); +} + +static inline void arch_local_irq_restore(unsigned long flags) +{ + barrier(); + setipl(flags); + barrier(); +} + +static inline bool arch_irqs_disabled_flags(unsigned long flags) +{ + return flags > IPL_MIN; +} + +static inline bool arch_irqs_disabled(void) +{ + return arch_irqs_disabled_flags(getipl()); +} + +#endif /* _ASM_SW64_IRQFLAGS_H */ diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c new file mode 100644 index 000000000000..126fe2f70495 --- /dev/null +++ b/arch/sw_64/kernel/irq.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/arch/sw_64/kernel/irq.c + * + * Copyright (C) 1995 Linus Torvalds + * + * This file contains the code used by various IRQ handling routines: + * asking for different IRQ's should be done through these routines + * instead of just grabbing them. Thus setups with different IRQ numbers + * shouldn't result in any weird surprises, and installing new handlers + * should be easier. + */ + +#include +#include +#include +#include + +volatile unsigned long irq_err_count; +DEFINE_PER_CPU(unsigned long, irq_pmi_count); +DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); +EXPORT_PER_CPU_SYMBOL(irq_stat); + +void ack_bad_irq(unsigned int irq) +{ + irq_err_count++; + pr_crit("Unexpected IRQ trap at vector %u\n", irq); +} + +u64 arch_irq_stat_cpu(unsigned int cpu) +{ + u64 sum = per_cpu(irq_stat, cpu).timer_irqs_event; + + return sum; +} + +u64 arch_irq_stat(void) +{ + return 0; +} + +int arch_show_interrupts(struct seq_file *p, int prec) +{ + int j; + + seq_printf(p, "%*s: ", prec, "TIMER"); + for_each_online_cpu(j) + seq_printf(p, "%10u", per_cpu(irq_stat, j).timer_irqs_event); + seq_puts(p, "\n"); + +#ifdef CONFIG_SMP + seq_printf(p, "%*s: ", prec, "IPI"); + for_each_online_cpu(j) + seq_printf(p, "%10lu ", cpu_data[j].ipi_count); + seq_puts(p, "\n"); +#endif + seq_printf(p, "%*s: ", prec, "PMI"); + for_each_online_cpu(j) + seq_printf(p, "%10lu ", per_cpu(irq_pmi_count, j)); + seq_puts(p, "\n"); + + seq_printf(p, "ERR: %10lu\n", irq_err_count); + return 0; +} + +/* + * handle_irq handles all normal device IRQ's (the special + * SMP cross-CPU interrupts have their own specific + * handlers). + */ + +#define MAX_ILLEGAL_IRQS 16 + +void +handle_irq(int irq) +{ + /* + * We ack quickly, we don't want the irq controller + * thinking we're snobs just because some other CPU has + * disabled global interrupts (we have already done the + * INT_ACK cycles, it's too late to try to pretend to the + * controller that we aren't taking the interrupt). + * + * 0 return value means that this irq is already being + * handled by some other CPU. (or is disabled) + */ + static unsigned int illegal_count; + struct irq_desc *desc = irq_to_desc(irq); + + if (!desc || ((unsigned int) irq > ACTUAL_NR_IRQS && + illegal_count < MAX_ILLEGAL_IRQS)) { + irq_err_count++; + illegal_count++; + pr_crit("device_interrupt: invalid interrupt %d\n", irq); + return; + } + + irq_enter(); + generic_handle_irq_desc(desc); + irq_exit(); +} + +#ifdef CONFIG_HOTPLUG_CPU +void fixup_irqs(void) +{ + irq_migrate_all_off_this_cpu(); +} +#endif diff --git a/arch/sw_64/kernel/irq_sw64.c b/arch/sw_64/kernel/irq_sw64.c new file mode 100644 index 000000000000..989d55ee1b1b --- /dev/null +++ b/arch/sw_64/kernel/irq_sw64.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SW64 specific irq code. + */ + +#include +#include + +#include +#include + +void __init +init_IRQ(void) +{ + /* + * Just in case the platform init_irq() causes interrupts/mchecks + * (as is the case with RAWHIDE, at least). + */ + if (is_in_host()) { + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); + } + + wrent(entInt, 0); + + sw64_init_irq(); + irqchip_init(); +} + +DEFINE_SPINLOCK(irq_lock); + +static void +__enable_irq(struct irq_data *d) +{ +} + +static void +__disable_irq(struct irq_data *d) +{ +} + +static unsigned int +__startup_irq(struct irq_data *d) +{ + __enable_irq(d); + return 0; +} + +static void +__mask_and_ack_irq(struct irq_data *d) +{ + spin_lock(&irq_lock); + __disable_irq(d); + spin_unlock(&irq_lock); +} + +struct irq_chip sw64_irq_chip = { + .name = "SW64_NODE", + .irq_startup = __startup_irq, + .irq_unmask = __enable_irq, + .irq_mask = __disable_irq, + .irq_mask_ack = __mask_and_ack_irq, +}; + +void __weak arch_init_msi_domain(struct irq_domain *parent) {} + +int __init arch_early_irq_init(void) +{ + int i; + + for (i = 0; i < NR_IRQS; ++i) { + irq_set_chip_and_handler(i, &sw64_irq_chip, handle_level_irq); + irq_set_status_flags(i, IRQ_LEVEL); + } + arch_init_msi_domain(NULL); + return 0; +} + +int __init arch_probe_nr_irqs(void) +{ + return NR_IRQS_LEGACY; +} diff --git a/arch/sw_64/kernel/time.c b/arch/sw_64/kernel/time.c new file mode 100644 index 000000000000..533a6a14c200 --- /dev/null +++ b/arch/sw_64/kernel/time.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include +#include + +#include "proto.h" + +DEFINE_SPINLOCK(rtc_lock); +EXPORT_SYMBOL(rtc_lock); + +#define TICK_SIZE (tick_nsec / 1000) + +/* + * Shift amount by which scaled_ticks_per_cycle is scaled. Shifting + * by 48 gives us 16 bits for HZ while keeping the accuracy good even + * for large CPU clock rates. + */ +#define FIX_SHIFT 48 + +unsigned long est_cycle_freq; + +#ifdef CONFIG_IRQ_WORK + +DEFINE_PER_CPU(u8, irq_work_pending); + +#define set_irq_work_pending_flag() __this_cpu_write(irq_work_pending, 1) +#define test_irq_work_pending() __this_cpu_read(irq_work_pending) +#define clear_irq_work_pending() __this_cpu_write(irq_work_pending, 0) + +void arch_irq_work_raise(void) +{ + set_irq_work_pending_flag(); +} + +#else /* CONFIG_IRQ_WORK */ + +#define test_irq_work_pending() 0 +#define clear_irq_work_pending() + +#endif /* CONFIG_IRQ_WORK */ + +void __init +time_init(void) +{ + unsigned long cycle_freq; + + cycle_freq = get_cpu_freq(); + + pr_info("CPU Cycle frequency = %ld Hz\n", cycle_freq); + + /* Register clocksource */ + sw64_setup_clocksource(); + of_clk_init(NULL); + /* Startup the timer source. */ + sw64_setup_timer(); + /* Calibrate the delay loop directly */ + lpj_fine = cycle_freq / HZ; +} -- Gitee From 1d153f954decb3fbf703a821a7325b2bc2c4abdb Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:07 +0800 Subject: [PATCH 011/524] sw64: add exception handling support Add exception handling mechanism for basic SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kdebug.h | 15 + arch/sw_64/include/uapi/asm/gentrap.h | 38 + arch/sw_64/include/uapi/asm/sysinfo.h | 20 + arch/sw_64/kernel/traps.c | 1542 +++++++++++++++++++++++++ arch/sw_64/kernel/unaligned.c | 80 ++ 5 files changed, 1695 insertions(+) create mode 100644 arch/sw_64/include/asm/kdebug.h create mode 100644 arch/sw_64/include/uapi/asm/gentrap.h create mode 100644 arch/sw_64/include/uapi/asm/sysinfo.h create mode 100644 arch/sw_64/kernel/traps.c create mode 100644 arch/sw_64/kernel/unaligned.c diff --git a/arch/sw_64/include/asm/kdebug.h b/arch/sw_64/include/asm/kdebug.h new file mode 100644 index 000000000000..73793057c3e8 --- /dev/null +++ b/arch/sw_64/include/asm/kdebug.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KDEBUG_H +#define _ASM_SW64_KDEBUG_H + +#include + +enum die_val { + DIE_OOPS = 1, + DIE_BREAK, + DIE_SSTEPBP, + DIE_UPROBE, + DIE_UPROBE_XOL, +}; + +#endif /* _ASM_SW64_KDEBUG_H */ diff --git a/arch/sw_64/include/uapi/asm/gentrap.h b/arch/sw_64/include/uapi/asm/gentrap.h new file mode 100644 index 000000000000..3786b8b52add --- /dev/null +++ b/arch/sw_64/include/uapi/asm/gentrap.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_GENTRAP_H +#define _UAPI_ASM_SW64_GENTRAP_H + +/* + * Definitions for gentrap causes. They are generated by user-level + * programs and therefore should be compatible with the corresponding + * legacy definitions. + */ +#define GEN_INTOVF -1 /* integer overflow */ +#define GEN_INTDIV -2 /* integer division by zero */ +#define GEN_FLTOVF -3 /* fp overflow */ +#define GEN_FLTDIV -4 /* fp division by zero */ +#define GEN_FLTUND -5 /* fp underflow */ +#define GEN_FLTINV -6 /* invalid fp operand */ +#define GEN_FLTINE -7 /* inexact fp operand */ +#define GEN_DECOVF -8 /* decimal overflow (for COBOL??) */ +#define GEN_DECDIV -9 /* decimal division by zero */ +#define GEN_DECINV -10 /* invalid decimal operand */ +#define GEN_ROPRAND -11 /* reserved operand */ +#define GEN_ASSERTERR -12 /* assertion error */ +#define GEN_NULPTRERR -13 /* null pointer error */ +#define GEN_STKOVF -14 /* stack overflow */ +#define GEN_STRLENERR -15 /* string length error */ +#define GEN_SUBSTRERR -16 /* substring error */ +#define GEN_RANGERR -17 /* range error */ +#define GEN_SUBRNG -18 +#define GEN_SUBRNG1 -19 +#define GEN_SUBRNG2 -20 +#define GEN_SUBRNG3 -21 /* these report range errors for */ +#define GEN_SUBRNG4 -22 /* subscripting (indexing) at levels 0..7 */ +#define GEN_SUBRNG5 -23 +#define GEN_SUBRNG6 -24 +#define GEN_SUBRNG7 -25 + +/* the remaining codes (-26..-1023) are reserved. */ + +#endif /* _UAPI_ASM_SW64_GENTRAP_H */ diff --git a/arch/sw_64/include/uapi/asm/sysinfo.h b/arch/sw_64/include/uapi/asm/sysinfo.h new file mode 100644 index 000000000000..667405c3447c --- /dev/null +++ b/arch/sw_64/include/uapi/asm/sysinfo.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * include/asm/sysinfo.h + */ + +#ifndef _UAPI_ASM_SW64_SYSINFO_H +#define _UAPI_ASM_SW64_SYSINFO_H + +#define GSI_IEEE_FP_CONTROL 45 + +#define SSI_IEEE_FP_CONTROL 14 +#define SSI_IEEE_RAISE_EXCEPTION 1001 /* linux specific */ + +#define UAC_BITMASK 7 +#define UAC_NOPRINT 1 +#define UAC_NOFIX 2 +#define UAC_SIGBUS 4 +#define PR_NOFIX 4 /* do not fix up unaligned accesses */ + +#endif /* _UAPI_ASM_SW64_SYSINFO_H */ diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c new file mode 100644 index 000000000000..a30e18ad1f00 --- /dev/null +++ b/arch/sw_64/kernel/traps.c @@ -0,0 +1,1542 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arch/sw_64/kernel/traps.c + * + * (C) Copyright 1994 Linus Torvalds + */ + +/* + * This file initializes the trap entry points + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "proto.h" + +enum SW64_IF_TYPES { + IF_BREAKPOINT = 0, + IF_RESERVED, + IF_GENTRAP, + IF_FEN, + IF_OPDEC, + IF_SIMDEMU, +}; + +void show_regs(struct pt_regs *regs) +{ + show_regs_print_info(KERN_DEFAULT); + + printk(KERN_DEFAULT "pc = [<%016lx>] ra = [<%016lx>] ps = %04lx %s\n", + regs->pc, regs->regs[26], regs->ps, print_tainted()); + printk(KERN_DEFAULT "pc is at %pSR\n", (void *)regs->pc); + printk(KERN_DEFAULT "ra is at %pSR\n", (void *)regs->regs[26]); + printk(KERN_DEFAULT "v0 = %016lx t0 = %016lx t1 = %016lx\n", + regs->regs[0], regs->regs[1], regs->regs[2]); + printk(KERN_DEFAULT "t2 = %016lx t3 = %016lx t4 = %016lx\n", + regs->regs[3], regs->regs[4], regs->regs[5]); + printk(KERN_DEFAULT "t5 = %016lx t6 = %016lx t7 = %016lx\n", + regs->regs[6], regs->regs[7], regs->regs[8]); + + printk(KERN_DEFAULT "s0 = %016lx s1 = %016lx s2 = %016lx\n", + regs->regs[9], regs->regs[10], regs->regs[11]); + printk(KERN_DEFAULT "s3 = %016lx s4 = %016lx s5 = %016lx\n", + regs->regs[12], regs->regs[13], regs->regs[14]); + printk(KERN_DEFAULT "s6 = %016lx\n", + regs->regs[15]); + + printk(KERN_DEFAULT "a0 = %016lx a1 = %016lx a2 = %016lx\n", + regs->regs[16], regs->regs[17], regs->regs[18]); + printk(KERN_DEFAULT "a3 = %016lx a4 = %016lx a5 = %016lx\n", + regs->regs[19], regs->regs[20], regs->regs[21]); + printk(KERN_DEFAULT "t8 = %016lx t9 = %016lx t10 = %016lx\n", + regs->regs[22], regs->regs[23], regs->regs[24]); + printk(KERN_DEFAULT "t11= %016lx pv = %016lx at = %016lx\n", + regs->regs[25], regs->regs[27], regs->regs[28]); + printk(KERN_DEFAULT "gp = %016lx sp = %016lx\n", regs->regs[29], regs->regs[30]); +} + +static void show_code(unsigned int *pc) +{ + long i; + unsigned int insn; + + printk(KERN_DEFAULT "Code:"); + for (i = -6; i < 2; i++) { + if (__get_user(insn, (unsigned int __user *)pc + i)) + break; + printk(KERN_DEFAULT "%c%08x%c", i ? ' ' : '<', insn, i ? ' ' : '>'); + } + printk(KERN_DEFAULT "\n"); +} + +static DEFINE_SPINLOCK(die_lock); + +void die(char *str, struct pt_regs *regs, long err) +{ + static int die_counter; + unsigned long flags; + int ret; + + oops_enter(); + + spin_lock_irqsave(&die_lock, flags); + console_verbose(); + bust_spinlocks(1); + + pr_emerg("%s [#%d]\n", str, ++die_counter); + + ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV); + + print_modules(); + show_regs(regs); + show_code((unsigned int *)regs->pc); + show_stack(current, NULL, KERN_EMERG); + + bust_spinlocks(0); + add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); + spin_unlock_irqrestore(&die_lock, flags); + oops_exit(); + + if (kexec_should_crash(current)) + crash_kexec(regs); + if (in_interrupt()) + panic("Fatal exception in interrupt"); + if (panic_on_oops) + panic("Fatal exception"); + + if (ret != NOTIFY_STOP) + make_task_dead(SIGSEGV); +} + +#ifndef CONFIG_MATHEMU +static long dummy_emul(void) +{ + return 0; +} + +long (*sw64_fp_emul_imprecise)(struct pt_regs *regs, unsigned long writemask) = (void *)dummy_emul; +EXPORT_SYMBOL_GPL(sw64_fp_emul_imprecise); + +long (*sw64_fp_emul)(unsigned long pc) = (void *)dummy_emul; +EXPORT_SYMBOL_GPL(sw64_fp_emul); +#else +long sw64_fp_emul_imprecise(struct pt_regs *regs, unsigned long writemask); +long sw64_fp_emul(unsigned long pc); +#endif + +asmlinkage void +do_entArith(unsigned long summary, unsigned long write_mask, + struct pt_regs *regs) +{ + long si_code = FPE_FLTINV; + + if (summary & 1) { + /* Software-completion summary bit is set, so try to + * emulate the instruction. If the processor supports + * precise exceptions, we don't have to search. + */ + si_code = sw64_fp_emul(regs->pc - 4); + if (si_code == 0) + return; + } + + if (!user_mode(regs)) + die("Arithmetic fault", regs, 0); + + /*summary<39> means integer divide by zero in C4.*/ + if ((summary >> 39) & 1) + si_code = FPE_INTDIV; + + force_sig_fault(SIGFPE, si_code, (void __user *)regs->pc); +} + +void simd_emulate(unsigned int inst, unsigned long va) +{ + unsigned long *fp; + int instr_opc, reg; + + instr_opc = (inst >> 26) & 0x3f; + reg = (inst >> 21) & 0x1f; + fp = (unsigned long *) va; + + switch (instr_opc) { + case 0x0d: /* vldd */ + sw64_write_simd_fp_reg_d(reg, fp[0], fp[1], fp[2], fp[3]); + return; + + case 0x0f: /* vstd */ + sw64_read_simd_fp_m_d(reg, fp); + return; + } +} + +/* + * BPT/GENTRAP/OPDEC make regs->pc = exc_pc + 4. debugger should + * do something necessary to handle it correctly. + */ +asmlinkage void +do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) +{ + int signo, code; + unsigned int inst, type; + + type = inst_type & 0xffffffff; + inst = inst_type >> 32; + + if (type == IF_SIMDEMU) { + simd_emulate(inst, va); + return; + } + + if (!user_mode(regs) && type != IF_OPDEC) { + if (type == IF_BREAKPOINT) { + /* support kgdb */ + notify_die(0, "kgdb trap", regs, 0, 0, SIGTRAP); + return; + } + die((type == IF_RESERVED ? "Kernel Bug" : "Instruction fault"), + regs, type); + } + + switch (type) { + case IF_BREAKPOINT: /* gdb do pc-4 for sigtrap */ + force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc); + return; + + case IF_GENTRAP: + regs->pc -= 4; + switch ((long)regs->regs[16]) { + case GEN_INTOVF: + signo = SIGFPE; + code = FPE_INTOVF; + break; + case GEN_INTDIV: + signo = SIGFPE; + code = FPE_INTDIV; + break; + case GEN_FLTOVF: + signo = SIGFPE; + code = FPE_FLTOVF; + break; + case GEN_FLTDIV: + signo = SIGFPE; + code = FPE_FLTDIV; + break; + case GEN_FLTUND: + signo = SIGFPE; + code = FPE_FLTUND; + break; + case GEN_FLTINV: + signo = SIGFPE; + code = FPE_FLTINV; + break; + case GEN_FLTINE: + signo = SIGFPE; + code = FPE_FLTRES; + break; + case GEN_ROPRAND: + signo = SIGFPE; + code = FPE_FLTUNK; + break; + + case GEN_DECOVF: + case GEN_DECDIV: + case GEN_DECINV: + case GEN_ASSERTERR: + case GEN_NULPTRERR: + case GEN_STKOVF: + case GEN_STRLENERR: + case GEN_SUBSTRERR: + case GEN_RANGERR: + case GEN_SUBRNG: + case GEN_SUBRNG1: + case GEN_SUBRNG2: + case GEN_SUBRNG3: + case GEN_SUBRNG4: + case GEN_SUBRNG5: + case GEN_SUBRNG6: + case GEN_SUBRNG7: + default: + regs->pc += 4; + signo = SIGTRAP; + code = TRAP_UNK; + break; + } + + force_sig_fault(signo, code, (void __user *)regs->pc); + return; + + case IF_FEN: + fpu_enable(); + return; + + case IF_OPDEC: + switch (inst) { +#ifdef CONFIG_KPROBES + case BREAK_KPROBE: + if (notify_die(DIE_BREAK, "kprobe", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) + return; + break; + case BREAK_KPROBE_SS: + if (notify_die(DIE_SSTEPBP, "single_step", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) + return; + break; +#endif +#ifdef CONFIG_UPROBES + case UPROBE_BRK_UPROBE: + if (notify_die(DIE_UPROBE, "uprobe", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) + return; + break; + case UPROBE_BRK_UPROBE_XOL: + if (notify_die(DIE_UPROBE_XOL, "uprobe_xol", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) + return; +#endif + } + + if (user_mode(regs)) + regs->pc -= 4; + else + die("Instruction fault", regs, type); + break; + + default: /* unexpected instruction-fault type */ + regs->pc -= 4; + break; + } + + force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->pc); +} + +asmlinkage void +do_entUna(void *va, unsigned long opcode, unsigned long reg, + struct pt_regs *regs) +{ + long error; + unsigned long tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + unsigned long pc = regs->pc - 4; + + /* + * We don't want to use the generic get/put unaligned macros as + * we want to trap exceptions. Only if we actually get an + * exception will we decide whether we should have caught it. + */ + + switch (opcode) { + case 0x21: + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 1(%3)\n" + " extlh %1, %3, %1\n" + " exthh %2, %3, %2\n" + "3:\n" + ".section __ex_table,\"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + regs->regs[reg] = tmp1 | tmp2; + return; + + case 0x22: + __asm__ __volatile__( + "1: ldl_u %1,0(%3)\n" + "2: ldl_u %2,3(%3)\n" + " extlw %1,%3,%1\n" + " exthw %2,%3,%2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + regs->regs[reg] = (int)(tmp1 | tmp2); + return; + + case 0x23: /* ldl */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + regs->regs[reg] = tmp1 | tmp2; + return; + + case 0x29: /* sth */ + __asm__ __volatile__( + " zap %6, 2, %1\n" + " srl %6, 8, %2\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %2, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %1, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(regs->regs[reg]), "0"(0)); + + if (error) + goto got_exception; + return; + + case 0x2a: /* stw */ + __asm__ __volatile__( + " zapnot %6, 0x1, %1\n" + " srl %6, 8, %2\n" + " zapnot %2, 0x1,%2\n" + " srl %6, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %6, 24, %4\n" + " zapnot %4, 0x1, %4\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3: stb %3, 0x2(%5)\n" + "4: stb %4, 0x3(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi $31, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(regs->regs[reg]), "0"(0)); + + if (error) + goto got_exception; + return; + + case 0x2b: /* stl */ + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(regs->regs[reg]), "0"(0)); + + if (error) + goto got_exception; + return; + } + + pr_warn("Bad unaligned kernel access at %016lx: %p %lx %lu\n", + pc, va, opcode, reg); + make_task_dead(SIGSEGV); + +got_exception: + /* Ok, we caught the exception, but we don't want it. Is there + * someone to pass it along to? + */ + if (fixup_exception(regs, pc)) { + pr_info("Forwarding unaligned exception at %lx (%lx)\n", + pc, regs->pc); + return; + } + + /* + * Yikes! No one to forward the exception to. + * Since the registers are in a weird format, dump them ourselves. + */ + + die("Unhandled unaligned exception", regs, error); +} + +/* + * Handle user-level unaligned fault. Handling user-level unaligned + * faults is *extremely* slow and produces nasty messages. A user + * program *should* fix unaligned faults ASAP. + * + * Notice that we have (almost) the regular kernel stack layout here, + * so finding the appropriate registers is a little more difficult + * than in the kernel case. + * + * Finally, we handle regular integer load/stores only. In + * particular, load-linked/store-conditionally and floating point + * load/stores are not supported. The former make no sense with + * unaligned faults (they are guaranteed to fail) and I don't think + * the latter will occur in any decent program. + * + * Sigh. We *do* have to handle some FP operations, because GCC will + * uses them as temporary storage for integer memory to memory copies. + * However, we need to deal with stt/ldt and sts/lds only. + */ +#define OP_INT_MASK (1L << 0x22 | 1L << 0x2a | /* ldw stw */ \ + 1L << 0x23 | 1L << 0x2b | /* ldl stl */ \ + 1L << 0x21 | 1L << 0x29 | /* ldhu sth */ \ + 1L << 0x20 | 1L << 0x28) /* ldbu stb */ + +asmlinkage void +do_entUnaUser(void __user *va, unsigned long opcode, + unsigned long reg, struct pt_regs *regs) +{ +#ifdef CONFIG_UNA_PRINT + static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5); +#endif + + unsigned long tmp1, tmp2, tmp3, tmp4; + unsigned long fake_reg, *reg_addr = &fake_reg; + int si_code; + long error; + unsigned long tmp, tmp5, tmp6, tmp7, tmp8, vb; + unsigned long fp[4]; + unsigned long instr, instr_op, value; + +#ifdef CONFIG_DEBUG_FS + /* + * If command name is specified, record some information + * to debugfs. + */ + if (unaligned_task[0] && !strcmp(unaligned_task, current->comm)) { + int idx; + + idx = unaligned_count % UNA_MAX_ENTRIES; + unaligned[idx].va = (unsigned long)va; + unaligned[idx].pc = regs->pc; + unaligned_count++; + } +#endif + + /* Check the UAC bits to decide what the user wants us to do + * with the unaliged access. + */ + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, + 1, regs, regs->pc - 4); + +#ifdef CONFIG_UNA_PRINT + if (!(current_thread_info()->status & TS_UAC_NOPRINT)) { + if (__ratelimit(&ratelimit)) { + pr_info("%s(%d): unaligned trap at %016lx: %p %lx %ld\n", + current->comm, task_pid_nr(current), + regs->pc - 4, va, opcode, reg); + } + } +#endif + if ((current_thread_info()->status & TS_UAC_SIGBUS)) + goto give_sigbus; + /* Not sure why you'd want to use this, but... */ + if ((current_thread_info()->status & TS_UAC_NOFIX)) + return; + + /* Don't bother reading ds in the access check since we already + * know that this came from the user. Also rely on the fact that + * the page at TASK_SIZE is unmapped and so can't be touched anyway. + */ + if ((unsigned long)va >= TASK_SIZE) + goto give_sigsegv; + + if ((1L << opcode) & OP_INT_MASK) { + /* it's an integer load/store */ + if (reg < 31) { + reg_addr = ®s->regs[reg]; + } else { + /* zero "register" */ + fake_reg = 0; + } + } + + get_user(instr, (__u32 *)(regs->pc - 4)); + instr_op = (instr >> 26) & 0x3f; + + get_user(value, (__u64 *)va); + + switch (instr_op) { + + case 0x0c: /* vlds */ + if ((unsigned long)va << 61 == 0) { + __asm__ __volatile__( + "1: ldl %1, 0(%5)\n" + "2: ldl %2, 8(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); + + return; + } else { + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + tmp1 = tmp1 | tmp4; + tmp2 = tmp5 | tmp3; + + sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); + + return; + } + case 0x0a: /* ldse */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + tmp = tmp1 | tmp2; + tmp = tmp | (tmp << 32); + + sw64_write_simd_fp_reg_s(reg, tmp, tmp); + + return; + + case 0x0d: /* vldd */ + if ((unsigned long)va << 61 == 0) { + __asm__ __volatile__( + "1: ldl %1, 0(%5)\n" + "2: ldl %2, 8(%5)\n" + "3: ldl %3, 16(%5)\n" + "4: ldl %4, 24(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi %4, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + sw64_write_simd_fp_reg_d(reg, tmp1, tmp2, tmp3, tmp4); + + return; + } else { + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + tmp7 = tmp1 | tmp4; //f0 + tmp8 = tmp5 | tmp3; //f1 + + vb = ((unsigned long)(va))+16; + + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(vb), "0"(0)); + + if (error) + goto give_sigsegv; + + tmp = tmp1 | tmp4; // f2 + tmp2 = tmp5 | tmp3; // f3 + + sw64_write_simd_fp_reg_d(reg, tmp7, tmp8, tmp, tmp2); + return; + } + + case 0x0b: /* ldde */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + tmp = tmp1 | tmp2; + + sw64_write_simd_fp_reg_d(reg, tmp, tmp, tmp, tmp); + return; + + case 0x09: /* ldwe */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + sw64_write_simd_fp_reg_ldwe(reg, (int)(tmp1 | tmp2)); + + return; + + case 0x0e: /* vsts */ + sw64_read_simd_fp_m_s(reg, fp); + if ((unsigned long)va << 61 == 0) { + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); + + if (error) + goto give_sigsegv; + + return; + } else { + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(fp[0]), "0"(0)); + + if (error) + goto give_sigsegv; + + + vb = ((unsigned long)va) + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[1]), "0"(0)); + + if (error) + goto give_sigsegv; + + return; + } + + case 0x0f: /* vstd */ + sw64_read_simd_fp_m_d(reg, fp); + if ((unsigned long)va << 61 == 0) { + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); + + if (error) + goto give_sigsegv; + + vb = ((unsigned long)va)+16; + + + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(vb), "r"(fp[2]), "r"(fp[3]), "0"(0)); + + if (error) + goto give_sigsegv; + + return; + } else { + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(fp[0]), "0"(0)); + + if (error) + goto give_sigsegv; + + vb = ((unsigned long)va) + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[1]), "0"(0)); + + if (error) + goto give_sigsegv; + + vb = vb + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[2]), "0"(0)); + + if (error) + goto give_sigsegv; + + vb = vb + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[3]), "0"(0)); + + if (error) + goto give_sigsegv; + + return; + } + } + switch (opcode) { + case 0x21: /* ldhu */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 1(%3)\n" + " extlh %1, %3, %1\n" + " exthh %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = tmp1 | tmp2; + break; + + case 0x26: /* flds */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + sw64_write_fp_reg_s(reg, tmp1 | tmp2); + return; + + case 0x27: /* fldd */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + sw64_write_fp_reg(reg, tmp1 | tmp2); + return; + + case 0x22: /* ldw */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = (int)(tmp1 | tmp2); + break; + + case 0x23: /* ldl */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = tmp1 | tmp2; + break; + + /* Note that the store sequences do not indicate that they change + * memory because it _should_ be affecting nothing in this context. + * (Otherwise we have other, much larger, problems.) + */ + case 0x29: /* sth with stb */ + __asm__ __volatile__( + " zap %6, 2, %1\n" + " srl %6, 8, %2\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %2, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %1, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + return; + + case 0x2e: /* fsts*/ + fake_reg = sw64_read_fp_reg_s(reg); + fallthrough; + + case 0x2a: /* stw with stb*/ + __asm__ __volatile__( + " zapnot %6, 0x1, %1\n" + " srl %6, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %6, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %6, 24, %4\n" + " zapnot %4, 0x1, %4\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3: stb %3, 0x2(%5)\n" + "4: stb %4, 0x3(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi $31, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + return; + + case 0x2f: /* fstd */ + fake_reg = sw64_read_fp_reg(reg); + fallthrough; + + case 0x2b: /* stl */ + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + return; + + default: + /* What instruction were you trying to use, exactly? */ + goto give_sigbus; + } + + return; + +give_sigsegv: + regs->pc -= 4; /* make pc point to faulting insn */ + + /* We need to replicate some of the logic in mm/fault.c, + * since we don't have access to the fault code in the + * exception handling return path. + */ + if ((unsigned long)va >= TASK_SIZE) + si_code = SEGV_ACCERR; + else { + struct mm_struct *mm = current->mm; + + down_read(&mm->mmap_lock); + if (find_vma(mm, (unsigned long)va)) + si_code = SEGV_ACCERR; + else + si_code = SEGV_MAPERR; + up_read(&mm->mmap_lock); + } + force_sig_fault(SIGSEGV, si_code, va); + return; + +give_sigbus: + regs->pc -= 4; + force_sig_fault(SIGBUS, BUS_ADRALN, va); +} + +asmlinkage void do_entSys(struct pt_regs *regs) +{ + long ret = -ENOSYS; + unsigned long nr; + unsigned long ti_flags = current_thread_info()->flags; + + regs->orig_r0 = regs->regs[0]; + regs->orig_r19 = regs->regs[19]; + nr = regs->regs[0]; + + if (ti_flags & _TIF_SYSCALL_WORK) { + nr = syscall_trace_enter(); + if (nr == NO_SYSCALL) + goto syscall_out; + regs->orig_r0 = regs->regs[0]; + regs->orig_r19 = regs->regs[19]; + } + + if (nr < __NR_syscalls) { + syscall_fn_t syscall_fn = sys_call_table[nr]; + + ret = syscall_fn(regs->regs[16], regs->regs[17], regs->regs[18], + regs->regs[19], regs->regs[20], regs->regs[21]); + } + + if ((nr != __NR_sigreturn) && (nr != __NR_rt_sigreturn)) { + if (likely((ret >= 0) || regs->orig_r0 == NO_SYSCALL)) + syscall_set_return_value(current, regs, 0, ret); + else + syscall_set_return_value(current, regs, ret, 0); + } + +syscall_out: + rseq_syscall(regs); + + if (ti_flags & _TIF_SYSCALL_WORK) + syscall_trace_leave(); +} + +void +trap_init(void) +{ + /* Tell HMcode what global pointer we want in the kernel. */ + register unsigned long gptr __asm__("$29"); + wrkgp(gptr); + + wrent(entArith, 1); + wrent(entMM, 2); + wrent(entIF, 3); + wrent(entUna, 4); + wrent(entSys, 5); +#ifdef CONFIG_EFI + if (smp_processor_id() == 0) + wrent((void *)entSuspend, 6); +#endif +} diff --git a/arch/sw_64/kernel/unaligned.c b/arch/sw_64/kernel/unaligned.c new file mode 100644 index 000000000000..40a17fb9cbd2 --- /dev/null +++ b/arch/sw_64/kernel/unaligned.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include + +unsigned long unaligned_count; +char unaligned_task[TASK_COMM_LEN]; +struct unaligned_stat unaligned[UNA_MAX_ENTRIES]; + +static ssize_t unaligned_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + + unaligned_count = 0; + size = min(sizeof(unaligned_task), len); + if (copy_from_user(unaligned_task, user_buf, size)) + return -EFAULT; + unaligned_task[size - 1] = '\0'; + + return len; +} + +static int unaligned_show(struct seq_file *m, void *v) +{ + int i, idx, nr; + + if (!unaligned_task[0]) { + seq_puts(m, "No task traced\n"); + return 0; + } + seq_printf(m, "Task command:\t\t%s\n", unaligned_task); + seq_printf(m, "Unaligned count:\t%ld\n", unaligned_count); + if (!unaligned_count) + return 0; + nr = 0; + idx = unaligned_count % UNA_MAX_ENTRIES; + seq_printf(m, "Latest %d unaligned stat:\nNo.\tVA\t\tPC\n", UNA_MAX_ENTRIES); + if (unaligned_count >= UNA_MAX_ENTRIES) { + for (i = idx; i < UNA_MAX_ENTRIES; i++) + seq_printf(m, "%d\t%#lx\t%#lx\n", + nr++, unaligned[i].va, unaligned[i].pc); + } + for (i = 0; i < idx; i++) + seq_printf(m, "%d\t%#lx\t%#lx\n", + nr++, unaligned[i].va, unaligned[i].pc); + return 0; +} + +static int unaligned_open(struct inode *inode, struct file *file) +{ + return single_open(file, unaligned_show, NULL); +} + +static const struct file_operations unaligned_fops = { + .read = seq_read, + .write = unaligned_set, + .open = unaligned_open, + .llseek = default_llseek, +}; + +static int __init unaligned_init(void) +{ + struct dentry *unaligned; + + if (!sw64_debugfs_dir) + return -ENODEV; + + unaligned = debugfs_create_file("unaligned", 0644, + sw64_debugfs_dir, NULL, + &unaligned_fops); + if (!unaligned) + return -ENOMEM; + + return 0; +} + +late_initcall(unaligned_init); -- Gitee From a200349132d0b6aa865d34c88c1fce88dc1b8fdf Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:31 +0800 Subject: [PATCH 012/524] sw64: add process management Add process management support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/current.h | 19 + arch/sw_64/include/asm/processor.h | 100 ++++ arch/sw_64/include/asm/ptrace.h | 92 +++ arch/sw_64/include/asm/switch_to.h | 60 ++ arch/sw_64/include/asm/thread_info.h | 148 +++++ arch/sw_64/include/uapi/asm/ptrace.h | 56 ++ arch/sw_64/kernel/idle.c | 35 ++ arch/sw_64/kernel/process.c | 109 ++++ arch/sw_64/kernel/ptrace.c | 858 +++++++++++++++++++++++++++ 9 files changed, 1477 insertions(+) create mode 100644 arch/sw_64/include/asm/current.h create mode 100644 arch/sw_64/include/asm/processor.h create mode 100644 arch/sw_64/include/asm/ptrace.h create mode 100644 arch/sw_64/include/asm/switch_to.h create mode 100644 arch/sw_64/include/asm/thread_info.h create mode 100644 arch/sw_64/include/uapi/asm/ptrace.h create mode 100644 arch/sw_64/kernel/idle.c create mode 100644 arch/sw_64/kernel/process.c create mode 100644 arch/sw_64/kernel/ptrace.c diff --git a/arch/sw_64/include/asm/current.h b/arch/sw_64/include/asm/current.h new file mode 100644 index 000000000000..862caabb9c70 --- /dev/null +++ b/arch/sw_64/include/asm/current.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CURRENT_H +#define _ASM_SW64_CURRENT_H + +#ifndef __ASSEMBLY__ + +struct task_struct; +static __always_inline struct task_struct *get_current(void) +{ + register struct task_struct *tp __asm__("$8"); + + return tp; +} + +#define current get_current() + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_SW64_CURRENT_H */ diff --git a/arch/sw_64/include/asm/processor.h b/arch/sw_64/include/asm/processor.h new file mode 100644 index 000000000000..ec68fe6cc6f2 --- /dev/null +++ b/arch/sw_64/include/asm/processor.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * include/asm-sw64/processor.h + * + * Copyright (C) 1994 Linus Torvalds + */ + +#ifndef _ASM_SW64_PROCESSOR_H +#define _ASM_SW64_PROCESSOR_H + +#include /* for ADDR_LIMIT_32BIT */ +#include + +#define task_pt_regs(task) \ + ((struct pt_regs *) (task->stack + THREAD_SIZE) - 1) + +/* + * Returns current instruction pointer ("program counter"). + */ +#define current_text_addr() \ + ({ void *__pc; __asm__ ("br %0, .+4" : "=r"(__pc)); __pc; }) + +/* + * SW64 does have an arch_pick_mmap_layout() + */ +#define HAVE_ARCH_PICK_MMAP_LAYOUT 1 + +/* + * We have a 52-bit user address space: 4PB user VM... + */ +#define TASK_SIZE (0x10000000000000UL) +#define UNMAPPED_BASE (TASK_SIZE >> 6) +#define STACK_TOP \ + (current->personality & ADDR_LIMIT_32BIT ? 0x80000000 : 0x00120000000UL) + +#define STACK_TOP_MAX 0x00120000000UL + +/* This decides where the kernel will search for a free chunk of vm + * space during mmap's. + */ +#define TASK_UNMAPPED_BASE \ + ((current->personality & ADDR_LIMIT_32BIT) ? 0x40000000 : UNMAPPED_BASE) + +struct thread_struct { + struct user_fpsimd_state fpstate; + /* Callee-saved registers */ + unsigned long ra; + unsigned long sp; + unsigned long s[7]; /* s0 ~ s6 */ +}; +#define INIT_THREAD { } + +struct task_struct; +struct pt_regs; + +/* Do necessary setup to start up a newly executed thread. */ +extern void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp); + +/* Free all resources held by a thread. */ +extern void release_thread(struct task_struct *dead_task); + +unsigned long __get_wchan(struct task_struct *p); + +#define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc) + +#define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[30]) + +#define cpu_relax() barrier() + +#define ARCH_HAS_PREFETCH +#define ARCH_HAS_PREFETCHW +#define ARCH_HAS_SPINLOCK_PREFETCH + +#ifndef CONFIG_SMP +/* Nothing to prefetch. */ +#define spin_lock_prefetch(lock) do { } while (0) +#endif + +static inline void prefetch(const void *ptr) +{ + __builtin_prefetch(ptr, 0, 3); +} + +static inline void prefetchw(const void *ptr) +{ + __builtin_prefetch(ptr, 1, 3); +} + +#ifdef CONFIG_SMP +static inline void spin_lock_prefetch(const void *ptr) +{ + __builtin_prefetch(ptr, 1, 3); +} +#endif + +static inline void wait_for_interrupt(void) +{ + __asm__ __volatile__ ("halt"); +} +#endif /* _ASM_SW64_PROCESSOR_H */ diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h new file mode 100644 index 000000000000..964f4fc730f2 --- /dev/null +++ b/arch/sw_64/include/asm/ptrace.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_PTRACE_H +#define _ASM_SW64_PTRACE_H + +#include +#include +#include + +#define NO_SYSCALL _AC(-1, UL) + +#ifdef __KERNEL__ +#ifndef __ASSEMBLY__ + +/* + * This struct defines the way the registers are stored on the + * kernel stack during a system call or other kernel entry + */ + +struct pt_regs { + union { + struct user_pt_regs user_regs; + struct { + unsigned long regs[31]; + unsigned long pc; + unsigned long ps; + }; + }; + unsigned long orig_r0; + unsigned long orig_r19; + /* These are saved by HMcode: */ + unsigned long hm_ps; + unsigned long hm_pc; + unsigned long hm_gp; + unsigned long hm_r16; + unsigned long hm_r17; + unsigned long hm_r18; +}; + +#define arch_has_single_step() (1) +#define user_mode(regs) (((regs)->ps & 8) != 0) +#define instruction_pointer(regs) ((regs)->pc) +#define profile_pc(regs) instruction_pointer(regs) +#define user_stack_pointer(pt_regs) ((pt_regs)->regs[30]) +#define kernel_stack_pointer(regs) ((unsigned long)((regs) + 1)) +#define instruction_pointer_set(regs, val) ((regs)->pc = val) + +#define force_successful_syscall_return() (current_pt_regs()->orig_r0 = NO_SYSCALL) + +#define MAX_REG_OFFSET (offsetof(struct pt_regs, orig_r0)) + +extern short regoffsets[]; + +extern unsigned long syscall_trace_enter(void); +extern void syscall_trace_leave(void); + +/** + * regs_get_register() - get register value from its offset + * @regs: pt_regs from which register value is gotten + * @offset: offset of the register. + * + * regs_get_register returns the value of a register whose offset from @regs. + * The @offset is the offset of the register in struct pt_regs. + * If @offset is bigger than MAX_REG_OFFSET, this returns 0. + */ +static inline u64 regs_get_register(struct pt_regs *regs, unsigned int offset) +{ + if (unlikely(offset > MAX_REG_OFFSET)) + return 0; + + return *(unsigned long *)((unsigned long)regs + offset); +} +extern int regs_query_register_offset(const char *name); +extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, + unsigned int n); + +static inline int is_syscall_success(struct pt_regs *regs) +{ + return !regs->regs[19]; +} + +static inline long regs_return_value(struct pt_regs *regs) +{ + if ((regs->orig_r0 == NO_SYSCALL) || is_syscall_success(regs)) + return regs->regs[0]; + else + return -regs->regs[0]; +} + +#endif /* !__ASSEMBLY__ */ +#endif /* __KERNEL__ */ + +#endif /* _ASM_SW64_PTRACE_H */ diff --git a/arch/sw_64/include/asm/switch_to.h b/arch/sw_64/include/asm/switch_to.h new file mode 100644 index 000000000000..5e2db4b9e266 --- /dev/null +++ b/arch/sw_64/include/asm/switch_to.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SWITCH_TO_H +#define _ASM_SW64_SWITCH_TO_H + +#include + +extern void __fpstate_save(struct task_struct *save_to); +extern void __fpstate_restore(struct task_struct *restore_from); +extern struct task_struct *__switch_to(struct task_struct *prev, + struct task_struct *next); +extern void restore_da_match_after_sched(void); + +static inline void aux_save(struct task_struct *task) +{ + struct pcb_struct *pcb; + + if (likely(!(task->flags & PF_KTHREAD))) { + pcb = &task_thread_info(task)->pcb; + pcb->tp = rtid(); + __fpstate_save(task); + } +} + +static inline void aux_restore(struct task_struct *task) +{ + struct pcb_struct *pcb; + + if (likely(!(task->flags & PF_KTHREAD))) { + pcb = &task_thread_info(task)->pcb; + wrtp(pcb->tp); + __fpstate_restore(task); + } +} + +static inline void __switch_to_aux(struct task_struct *prev, + struct task_struct *next) +{ + aux_save(prev); + aux_restore(next); +} + + +#define switch_to(prev, next, last) \ +do { \ + struct task_struct *__prev = (prev); \ + struct task_struct *__next = (next); \ + __switch_to_aux(__prev, __next); \ + (last) = __switch_to(__prev, __next); \ +} while (0) + + +/* TODO: finish_arch_switch has been removed from arch-independent code. */ + +/* + * finish_arch_switch will be called after switch_to + */ +#define finish_arch_post_lock_switch restore_da_match_after_sched + + +#endif /* _ASM_SW64_SWITCH_TO_H */ diff --git a/arch/sw_64/include/asm/thread_info.h b/arch/sw_64/include/asm/thread_info.h new file mode 100644 index 000000000000..4f3b837e2e90 --- /dev/null +++ b/arch/sw_64/include/asm/thread_info.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_THREAD_INFO_H +#define _ASM_SW64_THREAD_INFO_H + +#ifdef __KERNEL__ + +#ifndef __ASSEMBLY__ +#include +#include + +typedef struct { + unsigned long seg; +} mm_segment_t; + + +struct pcb_struct { + unsigned long tp; + unsigned long da_match, da_mask; + unsigned long dv_match, dv_mask; + union { + unsigned long dc_ctl; + unsigned long match_ctl; + }; + unsigned long ia_match, ia_mask; + unsigned long iv_match; + unsigned long ida_match, ida_mask; +}; + +struct thread_info { + struct pcb_struct pcb; /* hmcode state */ + + unsigned int flags; /* low level flags */ + unsigned int ieee_state; /* see fpu.h */ + + mm_segment_t addr_limit; /* thread address space */ + unsigned int cpu; /* current CPU */ + int preempt_count; /* 0 => preemptible, <0 => BUG */ + unsigned int status; /* thread-synchronous flags */ + + int bpt_nsaved; + unsigned long bpt_addr[2]; /* breakpoint handling */ + unsigned int bpt_insn[2]; +#ifdef CONFIG_DYNAMIC_FTRACE + unsigned long dyn_ftrace_addr; +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + unsigned long dyn_ftrace_regs_addr; +#endif +#endif +}; + +static __always_inline u64 rtid(void) +{ + u64 val; + + asm volatile("rtid %0" : "=r" (val) : :); + return val; +} + +/* + * Macros/functions for gaining access to the thread information structure. + */ +#define INIT_THREAD_INFO(tsk) \ +{ \ + .addr_limit = KERNEL_DS, \ + .preempt_count = INIT_PREEMPT_COUNT, \ +} + + +#endif /* __ASSEMBLY__ */ + +/* Thread information allocation. */ +#define THREAD_SIZE_ORDER 1 +#define THREAD_SIZE (2 * PAGE_SIZE) + +/* + * Thread information flags: + * - these are process state flags and used from assembly + * - pending work-to-be-done flags come first and must be assigned to be + * within bits 0 to 7 to fit in and immediate operand. + * + * TIF_SYSCALL_TRACE is known to be 0 via blbs. + */ +#define TIF_SYSCALL_TRACE 0 /* syscall trace active */ +#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */ +#define TIF_SIGPENDING 2 /* signal pending */ +#define TIF_NEED_RESCHED 3 /* rescheduling necessary */ +#define TIF_SYSCALL_AUDIT 4 /* syscall audit active */ +#define TIF_UPROBE 5 /* uprobe breakpoint or singlestep */ +#define TIF_PATCH_PENDING 6 /* pending live patching update */ +#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ +#define TIF_DIE_IF_KERNEL 9 /* dik recursion lock */ +#define TIF_SYSCALL_TRACEPOINT 10 +#define TIF_SECCOMP 11 /* secure computing */ +#define TIF_MEMDIE 13 /* is terminating due to OOM killer */ +#define TIF_POLLING_NRFLAG 14 /* idle is polling for TIF_NEED_RESCHED */ + +#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) +#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) +#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) +#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_PATCH_PENDING (1 << TIF_PATCH_PENDING) +#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) +#define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) +#define _TIF_SECCOMP (1 << TIF_SECCOMP) +#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) +#define _TIF_UPROBE (1 << TIF_UPROBE) +#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) + +/* Work to do on interrupt/exception return. */ +#define _TIF_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ + _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ + _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL) + +#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ + _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP) + +/* Work to do on any return to userspace. */ +#define _TIF_ALLWORK_MASK (_TIF_WORK_MASK | _TIF_SYSCALL_TRACE) + +#define TS_UAC_NOPRINT 0x0001 /* ! Preserve the following three */ +#define TS_UAC_NOFIX 0x0002 /* ! flags as they match */ +#define TS_UAC_SIGBUS 0x0004 /* ! userspace part of 'prctl' */ + +#define SET_UNALIGN_CTL(task, value) ({ \ + __u32 status = task_thread_info(task)->status & ~UAC_BITMASK; \ + if (value & PR_UNALIGN_NOPRINT) \ + status |= TS_UAC_NOPRINT; \ + if (value & PR_UNALIGN_SIGBUS) \ + status |= TS_UAC_SIGBUS; \ + if (value & PR_NOFIX) /* sw-specific */ \ + status |= TS_UAC_NOFIX; \ + task_thread_info(task)->status = status; \ + 0; }) + +#define GET_UNALIGN_CTL(task, value) ({ \ + __u32 status = task_thread_info(task)->status & ~UAC_BITMASK; \ + __u32 res = 0; \ + if (status & TS_UAC_NOPRINT) \ + res |= PR_UNALIGN_NOPRINT; \ + if (status & TS_UAC_SIGBUS) \ + res |= PR_UNALIGN_SIGBUS; \ + if (status & TS_UAC_NOFIX) \ + res |= PR_NOFIX; \ + put_user(res, (int __user *)(value)); \ + }) + +#endif /* __KERNEL__ */ +#endif /* _ASM_SW64_THREAD_INFO_H */ diff --git a/arch/sw_64/include/uapi/asm/ptrace.h b/arch/sw_64/include/uapi/asm/ptrace.h new file mode 100644 index 000000000000..3fd53450e418 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/ptrace.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_PTRACE_H +#define _UAPI_ASM_SW64_PTRACE_H + +#include + +#ifndef __ASSEMBLY__ +/* + * User structures for general purpose, floating point and debug registers. + */ +struct user_pt_regs { + __u64 regs[31]; + __u64 pc; + __u64 pstate; +}; + +/* 256 bits aligned for simd */ +struct fpreg { + __u64 v[4] __attribute__((aligned(32))); +}; + +struct user_fpsimd_state { + struct fpreg fp[31]; + __u64 fpcr; + __u64 __reserved[3]; +}; +#endif + +/* PTRACE_ATTACH is 16 */ +/* PTRACE_DETACH is 17 */ + +#define PT_REG_BASE 0 +#define PT_REG_END 30 +#define PT_FPREG_BASE 32 +#define PT_FPREG_END 62 +#define PT_FPCR 63 +#define PT_PC 64 +#define PT_TP 65 +#define PT_UNIQUE PT_TP +#define PT_VECREG_BASE 67 +#define PT_VECREG_END 161 +#define PT_F31_V1 98 +#define PT_F31_V2 130 +#define PT_DA_MATCH 163 +#define PT_DA_MASK 164 +#define PT_DV_MATCH 165 +#define PT_DV_MASK 166 +#define PT_DC_CTL 167 +#define PT_MATCH_CTL 167 +#define PT_IA_MATCH 168 +#define PT_IA_MASK 169 +#define PT_IV_MATCH 170 +#define PT_IDA_MATCH 171 +#define PT_IDA_MASK 172 + +#endif /* _UAPI_ASM_SW64_PTRACE_H */ diff --git a/arch/sw_64/kernel/idle.c b/arch/sw_64/kernel/idle.c new file mode 100644 index 000000000000..d26bdc405b53 --- /dev/null +++ b/arch/sw_64/kernel/idle.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sw64 idle loop support. + * + */ +#include +#include +#include +#include +#include + +void arch_cpu_idle(void) +{ + local_irq_enable(); + cpu_relax(); + + if (is_in_guest()) { + if (!need_resched()) + hcall(HCALL_HALT, 0, 0, 0); + } else { + asm( + ".globl __idle_start\n" + "__idle_start = .\n" + "ldw $1, %0($8)\n" + "srl $1, %1, $1\n" + "blbs $1, $need_resched\n" + "halt\n" + ".globl __idle_end\n" + "__idle_end = .\n" + "$need_resched:" + :: "i"(TI_FLAGS), "i"(TIF_NEED_RESCHED) + : "$1"); + } + local_irq_disable(); +} diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c new file mode 100644 index 000000000000..fa58a0de4368 --- /dev/null +++ b/arch/sw_64/kernel/process.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file handles the architecture-dependent parts of process handling. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "proto.h" + +/* + * Re-start a thread when doing execve() + */ +void +start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) +{ + regs->pc = pc; + regs->ps = 8; + regs->regs[30] = sp; +} +EXPORT_SYMBOL(start_thread); + + +void +flush_thread(void) +{ + /* Arrange for each exec'ed process to start off with a clean slate + * with respect to the FPU. This is all exceptions disabled. + */ + current_thread_info()->ieee_state = 0; + wrfpcr(FPCR_INIT | ieee_swcr_to_fpcr(0)); + + /* Clean slate for TLS. */ + current_thread_info()->pcb.tp = 0; +} + +void +release_thread(struct task_struct *dead_task) +{ +} + +int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) +{ + /* + * aux_save() has to read the current TLS pointer from CSR:TID as it + * may be out-of-sync with the saved value. + */ + aux_save(src); + *dst = *src; + return 0; +} + +/* + * Copy architecture-specific thread state + */ + +int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) +{ + unsigned long clone_flags = args->flags; + unsigned long usp = args->stack; + unsigned long tls = args->tls; + struct thread_info *childti = task_thread_info(p); + struct pt_regs *childregs = task_pt_regs(p); + struct pt_regs *regs = current_pt_regs(); + + extern void ret_from_fork(void); + extern void ret_from_kernel_thread(void); + + p->thread.sp = (unsigned long) childregs; + + if (unlikely(args->fn)) { + /* kernel thread */ + memset(childregs, 0, sizeof(struct pt_regs)); + p->thread.ra = (unsigned long) ret_from_kernel_thread; + p->thread.s[0] = (unsigned long) args->fn; /* function */ + p->thread.s[1] = (unsigned long) args->fn_arg; + return 0; + } + + /* + * Note: if CLONE_SETTLS is not set, then we must inherit the + * value from the parent, which will have been set by the block + * copy in dup_task_struct. This is non-intuitive, but is + * required for proper operation in the case of a threaded + * application calling fork. + */ + if (clone_flags & CLONE_SETTLS) + childti->pcb.tp = tls; + else + regs->regs[20] = 0; + *childregs = *regs; + if (usp) + childregs->regs[30] = usp; + syscall_set_return_value(NULL, childregs, 0, 0); + p->thread.ra = (unsigned long) ret_from_fork; + return 0; +} + +unsigned long arch_randomize_brk(struct mm_struct *mm) +{ + return randomize_page(mm->brk, 0x02000000); +} diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c new file mode 100644 index 000000000000..070e27ee2567 --- /dev/null +++ b/arch/sw_64/kernel/ptrace.c @@ -0,0 +1,858 @@ +// SPDX-License-Identifier: GPL-2.0 +/* ptrace.c */ +/* By Ross Biro 1/23/92 */ +/* edited by Linus Torvalds */ +/* mangled further by Bob Manson (manson@santafe.edu) */ +/* more mutilation by David Mosberger (davidm@azstarnet.com) */ + +#include +#include +#include +#include + +#include + +#include "proto.h" +#include + +#define CREATE_TRACE_POINTS +#include + +#define BREAKINST 0x00000080 /* sys_call bpt */ + +/* + * does not yet catch signals sent when the child dies. + * in exit.c or in signal.c. + */ + +/* + * Processes always block with the following stack-layout: + * + * +================================+ <---- task + 2*PAGE_SIZE + * | HMcode saved frame (ps, pc, | ^ + * | gp, a0, a1, a2) | | + * +================================+ | struct pt_regs + * | | | + * | frame generated by SAVE_ALL | | + * | | v + * +================================+ + */ + +/* + * The following table maps a register index into the stack offset at + * which the register is saved. Register indices are 0-31 for integer + * regs, 32-63 for fp regs, and 64 for the pc. Notice that sp and + * zero have no stack-slot and need to be treated specially (see + * get_reg/put_reg below). + */ +#define PCB_OFF(var) offsetof(struct pcb_struct, var) + +static int pcboff[] = { + [PT_TP] = PCB_OFF(tp), + [PT_DA_MATCH] = PCB_OFF(da_match), + [PT_DA_MASK] = PCB_OFF(da_mask), + [PT_DV_MATCH] = PCB_OFF(dv_match), + [PT_DV_MASK] = PCB_OFF(dv_mask), + [PT_DC_CTL] = PCB_OFF(dc_ctl), + [PT_MATCH_CTL] = PCB_OFF(match_ctl), + [PT_IA_MATCH] = PCB_OFF(ia_match), + [PT_IA_MASK] = PCB_OFF(ia_mask), + [PT_IV_MATCH] = PCB_OFF(iv_match), + [PT_IDA_MATCH] = PCB_OFF(ida_match), + [PT_IDA_MASK] = PCB_OFF(ida_mask) +}; + +static unsigned long zero; + +/* + * Get address of register REGNO in task TASK. + */ + +static unsigned long * +get_reg_addr(struct task_struct *task, unsigned long regno) +{ + void *addr; + int fno, vno; + + switch (regno) { + case PT_UNIQUE: + case PT_DA_MATCH: + case PT_DA_MASK: + case PT_DV_MATCH: + case PT_DV_MASK: + case PT_MATCH_CTL: + case PT_IA_MATCH: + case PT_IA_MASK: + case PT_IV_MATCH: + case PT_IDA_MATCH: + case PT_IDA_MASK: + addr = (void *)task_thread_info(task) + pcboff[regno]; + break; + case PT_REG_BASE ... PT_REG_END: + addr = &task_pt_regs(task)->regs[regno]; + break; + case PT_FPREG_BASE ... PT_FPREG_END: + fno = regno - PT_FPREG_BASE; + addr = &task->thread.fpstate.fp[fno].v[0]; + break; + case PT_VECREG_BASE ... PT_VECREG_END: + /* + * return addr for zero value if we catch vectors of f31 + * v0 and v3 of f31 are not in this range so ignore them + */ + if (regno == PT_F31_V1 || regno == PT_F31_V2) { + addr = &zero; + break; + } + fno = (regno - PT_VECREG_BASE) & 0x1f; + vno = 1 + ((regno - PT_VECREG_BASE) >> 5); + addr = &task->thread.fpstate.fp[fno].v[vno]; + break; + case PT_FPCR: + addr = &task->thread.fpstate.fpcr; + break; + case PT_PC: + addr = (void *)task_pt_regs(task) + PT_REGS_PC; + break; + default: + addr = &zero; + } + + return addr; +} + +/* + * Get contents of register REGNO in task TASK. + */ +unsigned long +get_reg(struct task_struct *task, unsigned long regno) +{ + return *get_reg_addr(task, regno); +} + +/* + * Write contents of register REGNO in task TASK. + */ +static int +put_reg(struct task_struct *task, unsigned long regno, unsigned long data) +{ + *get_reg_addr(task, regno) = data; + return 0; +} + +static inline int +read_int(struct task_struct *task, unsigned long addr, int *data) +{ + int copied = access_process_vm(task, addr, data, sizeof(int), FOLL_FORCE); + + return (copied == sizeof(int)) ? 0 : -EIO; +} + +static inline int +write_int(struct task_struct *task, unsigned long addr, int data) +{ + int copied = access_process_vm(task, addr, &data, sizeof(int), + FOLL_FORCE | FOLL_WRITE); + return (copied == sizeof(int)) ? 0 : -EIO; +} + +/* + * Set breakpoint. + */ +int +ptrace_set_bpt(struct task_struct *child) +{ + int displ, i, res, reg_b, nsaved = 0; + unsigned int insn, op_code; + unsigned long pc; + + pc = get_reg(child, PT_PC); + res = read_int(child, pc, (int *)&insn); + if (res < 0) + return res; + + op_code = insn >> 26; + /* br bsr beq bne blt ble bgt bge blbc blbs fbeq fbne fblt fble fbgt fbge */ + if ((1UL << op_code) & 0x3fff000000000030UL) { + /* + * It's a branch: instead of trying to figure out + * whether the branch will be taken or not, we'll put + * a breakpoint at either location. This is simpler, + * more reliable, and probably not a whole lot slower + * than the alternative approach of emulating the + * branch (emulation can be tricky for fp branches). + */ + displ = ((s32)(insn << 11)) >> 9; + task_thread_info(child)->bpt_addr[nsaved++] = pc + 4; + if (displ) /* guard against unoptimized code */ + task_thread_info(child)->bpt_addr[nsaved++] + = pc + 4 + displ; + /*call ret jmp*/ + } else if (op_code >= 0x1 && op_code <= 0x3) { + reg_b = (insn >> 16) & 0x1f; + task_thread_info(child)->bpt_addr[nsaved++] = get_reg(child, reg_b); + } else { + task_thread_info(child)->bpt_addr[nsaved++] = pc + 4; + } + + /* install breakpoints: */ + for (i = 0; i < nsaved; ++i) { + res = read_int(child, task_thread_info(child)->bpt_addr[i], + (int *)&insn); + if (res < 0) + return res; + task_thread_info(child)->bpt_insn[i] = insn; + res = write_int(child, task_thread_info(child)->bpt_addr[i], + BREAKINST); + if (res < 0) + return res; + } + task_thread_info(child)->bpt_nsaved = nsaved; + return 0; +} + +/* + * Ensure no single-step breakpoint is pending. Returns non-zero + * value if child was being single-stepped. + */ +int +ptrace_cancel_bpt(struct task_struct *child) +{ + int i, nsaved = task_thread_info(child)->bpt_nsaved; + + task_thread_info(child)->bpt_nsaved = 0; + + if (nsaved > 2) { + pr_info("%s: bogus nsaved: %d!\n", __func__, nsaved); + nsaved = 2; + } + + for (i = 0; i < nsaved; ++i) { + write_int(child, task_thread_info(child)->bpt_addr[i], + task_thread_info(child)->bpt_insn[i]); + } + return (nsaved != 0); +} + +void user_enable_single_step(struct task_struct *child) +{ + /* Mark single stepping. */ + task_thread_info(child)->bpt_nsaved = -1; +} + +void user_disable_single_step(struct task_struct *child) +{ + ptrace_cancel_bpt(child); +} + +/* + * Called by kernel/ptrace.c when detaching.. + * + * Make sure the single step bit is not set. + */ +void ptrace_disable(struct task_struct *child) +{ + user_disable_single_step(child); +} + +static int gpr_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + return membuf_write(&to, task_pt_regs(target), sizeof(struct user_pt_regs)); +} + +static int gpr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, + task_pt_regs(target), 0, sizeof(struct user_pt_regs)); +} + +static int fpr_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + + return membuf_write(&to, &target->thread.fpstate, + sizeof(struct user_fpsimd_state)); +} + +static int fpr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpstate, 0, + sizeof(struct user_fpsimd_state)); +} + +enum sw64_regset { + REGSET_GPR, + REGSET_FPR, +}; + +static const struct user_regset sw64_regsets[] = { + [REGSET_GPR] = { + .core_note_type = NT_PRSTATUS, + .n = ELF_NGREG, + .size = sizeof(elf_greg_t), + .align = sizeof(elf_greg_t), + .regset_get = gpr_get, + .set = gpr_set + }, + [REGSET_FPR] = { + .core_note_type = NT_PRFPREG, + .n = sizeof(struct user_fpsimd_state) / sizeof(u64), + .size = sizeof(u64), + .align = sizeof(u64), + .regset_get = fpr_get, + .set = fpr_set + }, +}; + +static const struct user_regset_view user_sw64_view = { + .name = "sw64", .e_machine = EM_SW64, + .regsets = sw64_regsets, .n = ARRAY_SIZE(sw64_regsets) +}; + +const struct user_regset_view *task_user_regset_view(struct task_struct *task) +{ + return &user_sw64_view; +} + +long arch_ptrace(struct task_struct *child, long request, + unsigned long addr, unsigned long data) +{ + unsigned long tmp; + size_t copied; + long ret; + + switch (request) { + /* When I and D space are separate, these will need to be fixed. */ + case PTRACE_PEEKTEXT: /* read word at location addr. */ + case PTRACE_PEEKDATA: + copied = access_process_vm(child, addr, &tmp, sizeof(tmp), FOLL_FORCE); + ret = -EIO; + if (copied != sizeof(tmp)) + break; + + force_successful_syscall_return(); + ret = tmp; + break; + + /* Read register number ADDR. */ + case PTRACE_PEEKUSR: + force_successful_syscall_return(); + ret = get_reg(child, addr); + break; + + /* When I and D space are separate, this will have to be fixed. */ + case PTRACE_POKETEXT: /* write the word at location addr. */ + case PTRACE_POKEDATA: + ret = generic_ptrace_pokedata(child, addr, data); + break; + + case PTRACE_POKEUSR: /* write the specified register */ + ret = put_reg(child, addr, data); + break; + default: + ret = ptrace_request(child, request, addr, data); + break; + } + return ret; +} + +asmlinkage unsigned long syscall_trace_enter(void) +{ + unsigned long ret = 0; + struct pt_regs *regs = current_pt_regs(); + + if (test_thread_flag(TIF_SYSCALL_TRACE) && + ptrace_report_syscall_entry(regs)) + return NO_SYSCALL; + +#ifdef CONFIG_SECCOMP + /* Do seccomp after ptrace, to catch any tracer changes. */ + if (secure_computing() == -1) + return NO_SYSCALL; +#endif + + if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_enter(regs, regs->regs[0]); + audit_syscall_entry(regs->regs[0], regs->regs[16], regs->regs[17], regs->regs[18], regs->regs[19]); + return ret ?: regs->regs[0]; +} + +asmlinkage void +syscall_trace_leave(void) +{ + struct pt_regs *regs = current_pt_regs(); + + audit_syscall_exit(regs); + if (test_thread_flag(TIF_SYSCALL_TRACE)) + ptrace_report_syscall_exit(regs, 0); + if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_exit(regs, regs_return_value(regs)); +} + +#ifdef CONFIG_SUBARCH_C3B +static long rwcsr(int rw, unsigned long csr, unsigned long value) +{ + register unsigned long __r0 __asm__("$0"); + register unsigned long __r16 __asm__("$16") = rw; + register unsigned long __r17 __asm__("$17") = csr; + register unsigned long __r18 __asm__("$18") = value; + + __asm__ __volatile__( + "sys_call %4" + : "=r"(__r0), "=r"(__r16), "=r"(__r17), "=r"(__r18) + : "i"(HMC_rwreg), "1"(__r16), "2"(__r17), "3"(__r18) + : "$1", "$22", "$23", "$24", "$25"); + + return __r0; +} + +#define RCSR 0 +#define WCSR 1 + +#define CSR_DA_MATCH 0 +#define CSR_DA_MASK 1 +#define CSR_IA_MATCH 2 +#define CSR_IA_MASK 3 +#define CSR_IDA_MATCH 6 +#define CSR_IDA_MASK 7 +#define CSR_DC_CTL 11 +#define CSR_DV_MATCH 15 +#define CSR_DV_MASK 16 + +#define DV_MATCH_EN_S 19 +#define DAV_MATCH_EN_S 20 + +int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_regs *regs) +{ + unsigned long dc_ctl; + unsigned long value; + + pr_info("%s: pid %d, name = %s,cause = %#lx, mmcsr = %#lx, address = %#lx, pc %#lx\n", + __func__, current->pid, current->comm, cause, mmcsr, address, regs->pc); + + switch (mmcsr) { + case MMCSR__DA_MATCH: + case MMCSR__DV_MATCH: + case MMCSR__DAV_MATCH: + show_regs(regs); + + if (!(current->ptrace & PT_PTRACED)) { + pr_notice(" pid %d %s not be ptraced, return\n", current->pid, current->comm); + if (mmcsr == MMCSR__DA_MATCH) + rwcsr(WCSR, CSR_DA_MATCH, 0); //clear da_match + if (mmcsr == MMCSR__DV_MATCH) { + value = rwcsr(RCSR, CSR_DV_MATCH, 0); + pr_notice("value is %#lx\n", value); + value = rwcsr(RCSR, CSR_DV_MASK, 0); + pr_notice("value is %#lx\n", value); + dc_ctl = rwcsr(RCSR, CSR_DC_CTL, 0); + dc_ctl &= ~(0x1UL << DV_MATCH_EN_S); + rwcsr(WCSR, CSR_DC_CTL, dc_ctl); + } + if (mmcsr == MMCSR__DAV_MATCH) { + dc_ctl = rwcsr(RCSR, CSR_DC_CTL, 0); + dc_ctl &= ~((0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + rwcsr(WCSR, CSR_DC_CTL, dc_ctl); + rwcsr(WCSR, CSR_DA_MATCH, 0); //clear da_match + } + task_thread_info(current)->pcb.da_match = 0; + task_thread_info(current)->pcb.dv_match = 0; + task_thread_info(current)->pcb.dc_ctl = 0; + return 1; + } + + if (mmcsr == MMCSR__DA_MATCH) { + rwcsr(WCSR, CSR_DA_MATCH, 0); //clear da_match + task_thread_info(current)->pcb.da_match = 0; + } + if (mmcsr == MMCSR__DV_MATCH) { + dc_ctl = rwcsr(RCSR, CSR_DC_CTL, 0); + dc_ctl &= ~(0x1UL << DV_MATCH_EN_S); + rwcsr(WCSR, CSR_DC_CTL, dc_ctl); + } + if (mmcsr == MMCSR__DAV_MATCH) { + dc_ctl = rwcsr(RCSR, CSR_DC_CTL, 0); + dc_ctl &= ~((0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + rwcsr(WCSR, CSR_DC_CTL, dc_ctl); + rwcsr(WCSR, CSR_DA_MATCH, 0); //clear da_match + } + task_thread_info(current)->pcb.dv_match = 0; + task_thread_info(current)->pcb.dc_ctl = 0; + pr_notice("do_page_fault: want to send SIGTRAP, pid = %d\n", current->pid); + force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void *) address); + return 1; + + case MMCSR__IA_MATCH: + rwcsr(WCSR, CSR_IA_MATCH, 0); //clear ia_match + return 1; + case MMCSR__IDA_MATCH: + rwcsr(WCSR, CSR_IDA_MATCH, 0); //clear ida_match + return 1; + } + + return 0; +} + +void restore_da_match_after_sched(void) +{ + unsigned long dc_ctl_mode; + unsigned long dc_ctl; + struct pcb_struct *pcb = &task_thread_info(current)->pcb; + + rwcsr(WCSR, CSR_DA_MATCH, 0); + rwcsr(WCSR, CSR_DA_MASK, pcb->da_mask); + rwcsr(WCSR, CSR_DA_MATCH, pcb->da_match); + dc_ctl_mode = pcb->dc_ctl; + dc_ctl = rwcsr(RCSR, CSR_DC_CTL, 0); + dc_ctl &= ~((0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + dc_ctl |= ((dc_ctl_mode << DV_MATCH_EN_S) & ((0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S))); + if (dc_ctl_mode & 0x1) { + rwcsr(WCSR, CSR_DV_MATCH, pcb->dv_match); + rwcsr(WCSR, CSR_DV_MASK, pcb->dv_mask); + rwcsr(WCSR, CSR_DC_CTL, dc_ctl); + } +} + +#elif defined(CONFIG_SUBARCH_C4) +int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_regs *regs) +{ + kernel_siginfo_t info; + unsigned long match_ctl, ia_match; + sigval_t sw64_value; + + pr_info("%s: pid %d, name = %s, cause = %#lx, mmcsr = %#lx, address = %#lx, pc %#lx\n", + __func__, current->pid, current->comm, cause, mmcsr, address, regs->pc); + + switch (mmcsr) { + case MMCSR__DA_MATCH: + case MMCSR__DV_MATCH: + case MMCSR__DAV_MATCH: + case MMCSR__IA_MATCH: + case MMCSR__IDA_MATCH: + case MMCSR__IV_MATCH: + show_regs(regs); + + if (!(current->ptrace & PT_PTRACED)) { + pr_notice(" pid %d %s not be ptraced, return\n", current->pid, current->comm); + if (mmcsr == MMCSR__DA_MATCH) { + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x3UL << DA_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + task_thread_info(current)->pcb.match_ctl &= ~0x1; + task_thread_info(current)->pcb.da_match = 0; + } + if (mmcsr == MMCSR__DV_MATCH) { + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x1UL << DV_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 1); + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__DAV_MATCH) { + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 | (0x1 << 1) | (0x1 << 2)); + task_thread_info(current)->pcb.da_match = 0; + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__IA_MATCH) { + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IA_MATCH_EN_S) | (0x7ffffffffffffUL << 2)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 3); + task_thread_info(current)->pcb.ia_match = 0; + } + if (mmcsr == MMCSR__IV_MATCH) { + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + write_csr(0, CSR_IV_MATCH); // clear iv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 4); + task_thread_info(current)->pcb.ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + task_thread_info(current)->pcb.iv_match = 0; + } + if (mmcsr == MMCSR__IDA_MATCH) { + write_csr(0, CSR_IDA_MATCH); // clear ida_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 5); + task_thread_info(current)->pcb.ida_match = 0; + } + return 1; + } + + info.si_signo = SIGTRAP; + info.si_addr = (void *) address; + sw64_value.sival_ptr = (void *)(regs->pc); + info.si_value = sw64_value; + info.si_code = TRAP_HWBKPT; + + if (mmcsr == MMCSR__DA_MATCH) { + info.si_errno = 1; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x3UL << DA_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + task_thread_info(current)->pcb.match_ctl &= ~0x1; + task_thread_info(current)->pcb.da_match = 0; + } + if (mmcsr == MMCSR__DV_MATCH) { + info.si_errno = 2; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x1UL << DV_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 1); + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__DAV_MATCH) { + info.si_errno = 3; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 | (0x1 << 1) | (0x1 << 2)); + task_thread_info(current)->pcb.da_match = 0; + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__IA_MATCH) { + info.si_errno = 4; + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IA_MATCH_EN_S) | (0x7ffffffffffffUL << 2)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 3); + task_thread_info(current)->pcb.ia_match = 0; + } + if (mmcsr == MMCSR__IV_MATCH) { + info.si_errno = 5; + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + write_csr(0, CSR_IV_MATCH); // clear iv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 4); + task_thread_info(current)->pcb.ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + task_thread_info(current)->pcb.iv_match = 0; + } + if (mmcsr == MMCSR__IDA_MATCH) { + info.si_errno = 6; + write_csr(0, CSR_IDA_MATCH); // clear ida_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 5); + task_thread_info(current)->pcb.ida_match = 0; + } + pr_notice("do_page_fault: want to send SIGTRAP, pid = %d\n", current->pid); + force_sig_info(&info); + return 1; + } + + return 0; +} + +/* + *pcb->match_ctl: + * [0] DA_MATCH + * [1] DV_MATCH + * [2] DAV_MATCH + * [3] IA_MATCH + * [4] IV_MATCH + * [5] IDA_MATCH + * [8:9] match_ctl_mode + * + */ +#define DA_MATCH 0x1 +#define DV_MATCH 0x2 +#define DAV_MATCH 0x4 +#define IA_MATCH 0x8 +#define IV_MATCH 0x10 +#define IDA_MATCH 0x20 + +void restore_da_match_after_sched(void) +{ + unsigned long match_ctl_mode; + unsigned long match_ctl; + struct pcb_struct *pcb = &task_thread_info(current)->pcb; + unsigned long vpn, upn; + + if (!pcb->match_ctl) + return; + pr_info("Restroe MATCH status, pid: %d\n", current->pid); + + if (pcb->match_ctl & DA_MATCH) { + write_csr(pcb->da_match, CSR_DA_MATCH); + write_csr(pcb->da_mask, CSR_DA_MASK); + match_ctl_mode = (pcb->match_ctl >> 8) & 0x3; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + match_ctl |= (match_ctl_mode << DA_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH); + write_csr(match_ctl, CSR_DC_CTLP); + pr_info("da_match:%#lx da_mask:%#lx match_ctl:%#lx\n", pcb->da_match, pcb->da_mask, match_ctl); + } + + if (pcb->match_ctl & DV_MATCH) { + write_csr(pcb->dv_match, CSR_DV_MATCH); + write_csr(pcb->dv_mask, CSR_DV_MASK); + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + match_ctl |= (0x1UL << DV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH); + write_csr(match_ctl, CSR_DC_CTLP); + pr_info("dv_match:%#lx dv_mask:%#lx match_ctl:%#lx\n", pcb->dv_match, pcb->dv_mask, match_ctl); + } + + if (pcb->match_ctl & DAV_MATCH) { + write_csr(pcb->da_match, CSR_DA_MATCH); + write_csr(pcb->da_mask, CSR_DA_MASK); + write_csr(pcb->dv_match, CSR_DV_MATCH); + write_csr(pcb->dv_mask, CSR_DV_MASK); + write_csr(0xfffffffff, CSR_DA_MATCH_MODE); + match_ctl_mode = (pcb->match_ctl >> 8) & 0x3; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + match_ctl |= (match_ctl_mode << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) + | (0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) + | (0x3UL << DPM_MATCH); + write_csr(match_ctl, CSR_DC_CTLP); + pr_info("da_match:%#lx da_mask:%#lx dv_match:%#lx dv_mask:%#lx match_ctl:%#lx\n", + pcb->da_match, pcb->da_mask, pcb->dv_match, pcb->dv_mask, match_ctl); + } + + if (pcb->match_ctl & IA_MATCH) { + pcb->ia_match |= (0x1UL << IA_MATCH_EN_S) | 0x3; + pcb->ia_mask |= 0x3; + write_csr(pcb->ia_match, CSR_IA_MATCH); + write_csr(pcb->ia_mask, CSR_IA_MASK); + vpn = read_csr(CSR_VPCR) >> 44; + vpn &= 0x3ff; + upn = read_csr(CSR_UPCR); + upn &= 0x3ff; + write_csr(((0x3ff << 18) | vpn), CSR_IA_VPNMATCH); + write_csr(((0x3ff << 18) | upn), CSR_IA_UPNMATCH); + pr_info("ia_match:%#lx ia_mask:%#lx\n", pcb->ia_match, pcb->ia_mask); + } + if (pcb->match_ctl & IV_MATCH) { + pcb->ia_match |= (0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S) | 0x3; + write_csr(pcb->ia_match, CSR_IA_MATCH); + write_csr(pcb->iv_match, CSR_IV_MATCH); + pr_info("ia_match:%#lx iv_match:%#lx\n", pcb->ia_match, pcb->iv_match); + } + if (pcb->match_ctl & IDA_MATCH) { + pcb->ida_match |= (0x1UL << IDA_MATCH_EN_S) | 0x3; + pcb->ida_mask |= 0x3; + write_csr(pcb->ida_match, CSR_IDA_MATCH); + write_csr(pcb->ida_mask, CSR_IDA_MASK); + pr_info("ida_match:%#lx ida_mask:%#lx\n", pcb->ida_match, pcb->ida_mask); + } +} +#endif + +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define GPR_OFFSET_NAME(r) { \ + .name = "r" #r, \ + .offset = offsetof(struct pt_regs, regs[r]) \ +} + +#define REG_OFFSET_NAME(r) { \ + .name = #r, \ + .offset = offsetof(struct pt_regs, r) \ +} + +#define REG_OFFSET_END { \ + .name = NULL, \ + .offset = 0 \ +} + +static const struct pt_regs_offset regoffset_table[] = { + GPR_OFFSET_NAME(0), + GPR_OFFSET_NAME(1), + GPR_OFFSET_NAME(2), + GPR_OFFSET_NAME(3), + GPR_OFFSET_NAME(4), + GPR_OFFSET_NAME(5), + GPR_OFFSET_NAME(6), + GPR_OFFSET_NAME(7), + GPR_OFFSET_NAME(8), + GPR_OFFSET_NAME(9), + GPR_OFFSET_NAME(10), + GPR_OFFSET_NAME(11), + GPR_OFFSET_NAME(12), + GPR_OFFSET_NAME(13), + GPR_OFFSET_NAME(14), + GPR_OFFSET_NAME(15), + GPR_OFFSET_NAME(16), + GPR_OFFSET_NAME(17), + GPR_OFFSET_NAME(18), + GPR_OFFSET_NAME(19), + GPR_OFFSET_NAME(20), + GPR_OFFSET_NAME(21), + GPR_OFFSET_NAME(22), + GPR_OFFSET_NAME(23), + GPR_OFFSET_NAME(24), + GPR_OFFSET_NAME(25), + GPR_OFFSET_NAME(26), + GPR_OFFSET_NAME(27), + GPR_OFFSET_NAME(28), + GPR_OFFSET_NAME(29), + GPR_OFFSET_NAME(30), + REG_OFFSET_NAME(pc), + REG_OFFSET_NAME(ps), + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +static int regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) +{ + unsigned long ksp = kernel_stack_pointer(regs); + + return (addr & ~(THREAD_SIZE - 1)) == (ksp & ~(THREAD_SIZE - 1)); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs:pt_regs which contains kernel stack pointer. + * @n:stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specifined by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long addr; + + addr = kernel_stack_pointer(regs) + n * sizeof(long); + if (!regs_within_kernel_stack(regs, addr)) + return 0; + return *(unsigned long *)addr; +} -- Gitee From f953f913d3ff39252a176e1e9ac3f8d0a034cd58 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 9 Jan 2024 12:27:52 +0800 Subject: [PATCH 013/524] sw64: add hardware match support Add hardware match mechanism for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/match.c | 551 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 551 insertions(+) create mode 100644 arch/sw_64/kernel/match.c diff --git a/arch/sw_64/kernel/match.c b/arch/sw_64/kernel/match.c new file mode 100644 index 000000000000..3926391270da --- /dev/null +++ b/arch/sw_64/kernel/match.c @@ -0,0 +1,551 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#include +#include +#include + + +char da_match_buf[1024], dv_match_buf[1024], dav_match_buf[1024]; +char ia_match_buf[1024], iv_match_buf[1024], ida_match_buf[1024]; + +unsigned long da_match_cf1, da_match_cf2, da_match_cf3; +unsigned long dv_match_cf1, dv_match_cf2, dv_match_cf3; +unsigned long dav_match_cf1, dav_match_cf2, dav_match_cf3, + dav_match_cf4, dav_match_cf5; +unsigned long ia_match_cf1, ia_match_cf2, ia_match_cf3, ia_match_cf4; +unsigned long iv_match_cf1, iv_match_cf2; +unsigned long ida_match_cf1, ida_match_cf2; + +static int da_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", da_match_buf); + return 0; +} + +static int dv_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", dv_match_buf); + return 0; +} + +static int dav_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", dav_match_buf); + return 0; +} + +static int ia_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", ia_match_buf); + return 0; +} + +static int iv_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", iv_match_buf); + return 0; +} + +static int ida_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", ida_match_buf); + return 0; +} + +static int da_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, da_match_show, NULL); +} + +static int dv_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, dv_match_show, NULL); +} + +static int dav_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, dav_match_show, NULL); +} + +static int ia_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, ia_match_show, NULL); +} + +static int iv_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, iv_match_show, NULL); +} + +static int ida_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, ida_match_show, NULL); +} + +static void +write_da_match(void *i) +{ + unsigned long dc_ctl; + + write_csr(da_match_cf1, CSR_DA_MATCH); + write_csr(da_match_cf2, CSR_DA_MASK); + dc_ctl = read_csr(CSR_DC_CTLP); + dc_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) + | (0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) + | (0x3UL << DPM_MATCH)); + dc_ctl |= da_match_cf3; + write_csr(dc_ctl, CSR_DC_CTLP); +} + +static void +write_dv_match(void *i) +{ + unsigned long dc_ctl; + + write_csr(dv_match_cf1, CSR_DV_MATCH); + write_csr(dv_match_cf2, CSR_DV_MASK); + dc_ctl = read_csr(CSR_DC_CTLP); + dc_ctl &= ~((0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) + | (0x3UL << DPM_MATCH)); + dc_ctl |= ((0x1UL << DV_MATCH_EN_S) | dv_match_cf3); + write_csr(dc_ctl, CSR_DC_CTLP); +} + +static void +write_dav_match(void *i) +{ + unsigned long dc_ctl; + + write_csr(dav_match_cf1, CSR_DA_MATCH); + write_csr(dav_match_cf2, CSR_DA_MASK); + write_csr(dav_match_cf3, CSR_DV_MATCH); + write_csr(dav_match_cf4, CSR_DV_MASK); + dc_ctl = read_csr(CSR_DC_CTLP); + dc_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) + | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH)); + dc_ctl |= ((0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S) + | dav_match_cf5); + write_csr(dc_ctl, CSR_DC_CTLP); +} + +static void +write_ia_match(void *i) +{ + ia_match_cf1 |= (0x1UL << IA_MATCH_EN_S); + write_csr_imb(ia_match_cf1, CSR_IA_MATCH); + write_csr_imb(ia_match_cf2, CSR_IA_MASK); + write_csr(((0x3ffUL << 18) | ia_match_cf3), CSR_IA_VPNMATCH); + write_csr(((0x3ffUL << 18) | ia_match_cf4), CSR_IA_UPNMATCH); +} + +static void +write_iv_match(void *i) +{ + unsigned long ia_match_tmp; + + ia_match_tmp = read_csr(CSR_IA_MATCH); + ia_match_tmp &= ~(0x1UL << IV_PM_EN_S); + ia_match_tmp |= ((((iv_match_cf2 >> IV_PM_EN_S) & 0x1) << IV_PM_EN_S) + | (iv_match_cf2 & 0x3) | (0x1UL << IV_MATCH_EN_S)); + write_csr_imb(iv_match_cf1, CSR_IV_MATCH); + write_csr_imb(ia_match_tmp, CSR_IA_MATCH); +} + +static void +write_ida_match(void *i) +{ + + ida_match_cf1 |= (0x1UL << IDA_MATCH_EN_S); + write_csr(ida_match_cf1, CSR_IDA_MATCH); + write_csr(ida_match_cf2, CSR_IDA_MASK); +} + +static ssize_t da_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(da_match_buf) - 1, len); + if (copy_from_user(da_match_buf, user_buf, size)) + return -EFAULT; + + da_match_buf[size] = '\0'; + strcpy(tmp, da_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &da_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &da_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &da_match_cf3); + if (err) + return err; + + if (on_each_cpu(write_da_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + + return len; +} + +static ssize_t dv_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(dv_match_buf) - 1, len); + if (copy_from_user(dv_match_buf, user_buf, size)) + return -EFAULT; + + dv_match_buf[size] = '\0'; + strcpy(tmp, dv_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &dv_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &dv_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &dv_match_cf3); + if (err) + return err; + + if (on_each_cpu(write_dv_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + + return len; +} + +static ssize_t dav_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[500]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[500]; + int err; + char *ret = NULL; + + size = min(sizeof(dav_match_buf) - 1, len); + if (copy_from_user(dav_match_buf, user_buf, size)) + return -EFAULT; + + dav_match_buf[size] = '\0'; + strcpy(tmp, dav_match_buf); + p = tmp; + + for (i = 0 ; i < 5; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[500] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &dav_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &dav_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &dav_match_cf3); + if (err) + return err; + + err = kstrtoul(&tmp1[300], 0, &dav_match_cf4); + if (err) + return err; + + err = kstrtoul(&tmp1[400], 0, &dav_match_cf5); + if (err) + return err; + + + if (on_each_cpu(write_dav_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + return len; +} + +static ssize_t ia_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(ia_match_buf) - 1, len); + if (copy_from_user(ia_match_buf, user_buf, size)) + return -EFAULT; + + ia_match_buf[size] = '\0'; + strcpy(tmp, ia_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &ia_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &ia_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &ia_match_cf3); + if (err) + return err; + + err = kstrtoul(&tmp1[300], 0, &ia_match_cf4); + if (err) + return err; + + if (on_each_cpu(write_ia_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + return len; +} + +static ssize_t iv_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(ia_match_buf) - 1, len); + if (copy_from_user(ia_match_buf, user_buf, size)) + return -EFAULT; + + ia_match_buf[size] = '\0'; + strcpy(tmp, ia_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &iv_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &iv_match_cf2); + if (err) + return err; + + if (on_each_cpu(write_iv_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + return len; +} + + +static ssize_t ida_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(ida_match_buf) - 1, len); + if (copy_from_user(ida_match_buf, user_buf, size)) + return -EFAULT; + + ida_match_buf[size] = '\0'; + strcpy(tmp, ida_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &ida_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &ida_match_cf2); + if (err) + return err; + + if (on_each_cpu(write_ida_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + + return len; +} + +static const struct file_operations set_da_match_fops = { + .open = da_match_open, + .read = seq_read, + .write = da_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_dv_match_fops = { + .open = dv_match_open, + .read = seq_read, + .write = dv_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_dav_match_fops = { + .open = dav_match_open, + .read = seq_read, + .write = dav_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_ia_match_fops = { + .open = ia_match_open, + .read = seq_read, + .write = ia_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_iv_match_fops = { + .open = iv_match_open, + .read = seq_read, + .write = iv_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + + +static const struct file_operations set_ida_match_fops = { + .open = ida_match_open, + .read = seq_read, + .write = ida_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __init match_debugfs_init(void) +{ + struct dentry *match_entry; + + if (!sw64_debugfs_dir) + return -ENODEV; + + match_entry = debugfs_create_file("da_match", 0600, + sw64_debugfs_dir, NULL, + &set_da_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("dv_match", 0600, + sw64_debugfs_dir, NULL, + &set_dv_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("dav_match", 0600, + sw64_debugfs_dir, NULL, + &set_dav_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("ia_match", 0600, + sw64_debugfs_dir, NULL, + &set_ia_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("iv_match", 0600, + sw64_debugfs_dir, NULL, + &set_iv_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("ida_match", 0600, + sw64_debugfs_dir, NULL, + &set_ida_match_fops); + if (!match_entry) + return -ENOMEM; + + return 0; +} +late_initcall(match_debugfs_init); -- Gitee From ffe17e12803c77e0434e8564419e467326c79204 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:27 +0800 Subject: [PATCH 014/524] sw64: add memory management Add memory management support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cache.h | 13 + arch/sw_64/include/asm/cacheflush.h | 13 + arch/sw_64/include/asm/memory.h | 35 ++ arch/sw_64/include/asm/mmu.h | 10 + arch/sw_64/include/asm/mmu_context.h | 136 ++++ arch/sw_64/include/asm/mmzone.h | 17 + arch/sw_64/include/asm/page.h | 71 +++ arch/sw_64/include/asm/pgalloc.h | 51 ++ arch/sw_64/include/asm/pgtable-4level.h | 32 + arch/sw_64/include/asm/pgtable.h | 789 ++++++++++++++++++++++++ arch/sw_64/include/asm/sparsemem.h | 9 + arch/sw_64/include/asm/tlb.h | 13 + arch/sw_64/include/asm/tlbflush.h | 94 +++ arch/sw_64/include/asm/vmalloc.h | 5 + arch/sw_64/mm/Makefile | 16 + arch/sw_64/mm/extable.c | 25 + arch/sw_64/mm/fault.c | 305 +++++++++ arch/sw_64/mm/init.c | 339 ++++++++++ arch/sw_64/mm/mmap.c | 102 +++ arch/sw_64/mm/physaddr.c | 39 ++ 20 files changed, 2114 insertions(+) create mode 100644 arch/sw_64/include/asm/cache.h create mode 100644 arch/sw_64/include/asm/cacheflush.h create mode 100644 arch/sw_64/include/asm/memory.h create mode 100644 arch/sw_64/include/asm/mmu.h create mode 100644 arch/sw_64/include/asm/mmu_context.h create mode 100644 arch/sw_64/include/asm/mmzone.h create mode 100644 arch/sw_64/include/asm/page.h create mode 100644 arch/sw_64/include/asm/pgalloc.h create mode 100644 arch/sw_64/include/asm/pgtable-4level.h create mode 100644 arch/sw_64/include/asm/pgtable.h create mode 100644 arch/sw_64/include/asm/sparsemem.h create mode 100644 arch/sw_64/include/asm/tlb.h create mode 100644 arch/sw_64/include/asm/tlbflush.h create mode 100644 arch/sw_64/include/asm/vmalloc.h create mode 100644 arch/sw_64/mm/Makefile create mode 100644 arch/sw_64/mm/extable.c create mode 100644 arch/sw_64/mm/fault.c create mode 100644 arch/sw_64/mm/init.c create mode 100644 arch/sw_64/mm/mmap.c create mode 100644 arch/sw_64/mm/physaddr.c diff --git a/arch/sw_64/include/asm/cache.h b/arch/sw_64/include/asm/cache.h new file mode 100644 index 000000000000..6a6ce4e99265 --- /dev/null +++ b/arch/sw_64/include/asm/cache.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * include/asm/cache.h + */ +#ifndef _ASM_SW64_CACHE_H +#define _ASM_SW64_CACHE_H + +#define L1_CACHE_SHIFT 7 +#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) + +#define SMP_CACHE_BYTES L1_CACHE_BYTES + +#endif /* _ASM_SW64_CACHE_H */ diff --git a/arch/sw_64/include/asm/cacheflush.h b/arch/sw_64/include/asm/cacheflush.h new file mode 100644 index 000000000000..0d49830b8493 --- /dev/null +++ b/arch/sw_64/include/asm/cacheflush.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CACHEFLUSH_H +#define _ASM_SW64_CACHEFLUSH_H + +/* + * DCache: PIPT + * ICache: + * - C3B is VIVT with ICTAG, support coherence. + * - C4 is VIPT + */ +#include + +#endif /* _ASM_SW64_CACHEFLUSH_H */ diff --git a/arch/sw_64/include/asm/memory.h b/arch/sw_64/include/asm/memory.h new file mode 100644 index 000000000000..b2b7492ae477 --- /dev/null +++ b/arch/sw_64/include/asm/memory.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_MEMORY_H +#define _ASM_SW64_MEMORY_H + +#ifdef CONFIG_NUMA +#include +#endif + +#define MIN_MEMORY_BLOCK_SIZE_VM_MEMHP (1UL << 30) +#define NODE0_START (_TEXT_START - __START_KERNEL_map) + +#define MAX_PHYSMEM_BITS 48 + +struct mem_desc_t { + unsigned long phys_base; /* start address of physical memory */ + unsigned long phys_size; /* size of physical memory */ + phys_addr_t base; /* start address of memory managed by kernel */ + phys_addr_t size; /* size of memory managed by kernel */ +}; +extern struct mem_desc_t mem_desc; + +struct numa_node_desc_t { + phys_addr_t base; + phys_addr_t size; +}; +extern struct numa_node_desc_t numa_nodes_desc[]; + +void __init callback_init(void); +void __init mem_detect(void); +void __init sw64_memblock_init(void); +void __init zone_sizes_init(void); +void __init sw64_numa_init(void); +void __init sw64_memory_present(void); + +#endif /* _ASM_SW64_MEMORY_H */ diff --git a/arch/sw_64/include/asm/mmu.h b/arch/sw_64/include/asm/mmu.h new file mode 100644 index 000000000000..f24219fac654 --- /dev/null +++ b/arch/sw_64/include/asm/mmu.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_MMU_H +#define _ASM_SW64_MMU_H + +/* The sw64 MMU context is one "unsigned long" bitmap per CPU*/ +typedef struct { + unsigned long asid[NR_CPUS]; + void *vdso; +} mm_context_t; +#endif /* _ASM_SW64_MMU_H */ diff --git a/arch/sw_64/include/asm/mmu_context.h b/arch/sw_64/include/asm/mmu_context.h new file mode 100644 index 000000000000..420ad5f745be --- /dev/null +++ b/arch/sw_64/include/asm/mmu_context.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_MMU_CONTEXT_H +#define _ASM_SW64_MMU_CONTEXT_H + +#include + +#include +#include + +/* + * The maximum ASID's the processor supports. + */ + +#if defined(CONFIG_SUBARCH_C3B) || defined(CONFIG_SUBARCH_C4) +#define ASID_BITS 10 +#endif + +#include +#define last_asid(cpu) (cpu_data[cpu].last_asid) + +#define ASID_FIRST_VERSION (1UL << ASID_BITS) +#define ASID_MASK ((1UL << ASID_BITS) - 1) + +#define cpu_asid(cpu, mm) ((mm)->context.asid[cpu] & ASID_MASK) + +static inline bool asid_valid(struct mm_struct *mm, unsigned int cpu) +{ + return !((mm->context.asid[cpu] ^ last_asid(cpu)) & ~ASID_MASK); +} + +/* + * NOTE! The way this is set up, the high bits of the "last_asid" (and + * the "mm->context.asid[cpu]") are the ASID _version_ code. A version + * of 0 is always considered invalid, so to invalidate another process + * you only need to do "p->mm->context.asid[cpu] = 0". + * + * If we need more ASID's than the processor has, we invalidate the old + * user TLB's (tbivp()) and start a new ASID version. That will force a + * new asid for any other processes the next time they want to run. + */ + +static inline void __get_new_mm_context(struct mm_struct *mm, long cpu) +{ + unsigned long asid = last_asid(cpu); + + if (!(++asid & ASID_MASK)) + tbivp(); + mm->context.asid[cpu] = last_asid(cpu) = asid; + +} + +static inline void +switch_mm_irqs_off(struct mm_struct *prev_mm, struct mm_struct *next_mm, + struct task_struct *next) +{ + /* Check if our ASID is of an older version, and thus invalid. */ + unsigned long asid, ptbr; + long cpu = smp_processor_id(); + + if (!asid_valid(next_mm, cpu)) + __get_new_mm_context(next_mm, cpu); + + /* Update CSR:UPN and CSR:PTBR. Another thread may have allocated + * a new mm->context[asid] (via flush_tlb_mm) without the ASID serial + * number wrapping. We have no way to detect when this is needed. + */ + asid = cpu_asid(cpu, next_mm); + ptbr = virt_to_pfn(next_mm->pgd); + load_mm(asid, ptbr); + cpumask_set_cpu(cpu, mm_cpumask(next_mm)); +} + +#define switch_mm_irqs_off switch_mm_irqs_off + +static inline void +switch_mm(struct mm_struct *prev_mm, struct mm_struct *next_mm, + struct task_struct *tsk) +{ + unsigned long flags; + + local_irq_save(flags); + switch_mm_irqs_off(prev_mm, next_mm, tsk); + local_irq_restore(flags); +} + +#define activate_mm(prev, next) switch_mm(prev, next, current) +#define deactivate_mm(tsk, mm) do { } while (0) + +static inline int init_new_context(struct task_struct *tsk, + struct mm_struct *mm) +{ + int i; + + for_each_possible_cpu(i) + mm->context.asid[i] = 0; + return 0; +} + +static inline void destroy_context(struct mm_struct *mm) +{ + /* Nothing to do. */ +} + +static inline void enter_lazy_tlb(struct mm_struct *mm, + struct task_struct *tsk) +{ +} + +static inline int arch_dup_mmap(struct mm_struct *oldmm, + struct mm_struct *mm) +{ + return 0; +} + +static inline void arch_exit_mmap(struct mm_struct *mm) +{ +} + +static inline void arch_unmap(struct mm_struct *mm, unsigned long start, + unsigned long end) +{ +} + +static inline void arch_bprm_mm_init(struct mm_struct *mm, + struct vm_area_struct *vma) +{ +} + +static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, + bool write, bool execute, + bool foreign) +{ + /* by default, allow everything */ + return true; +} +#endif /* _ASM_SW64_MMU_CONTEXT_H */ diff --git a/arch/sw_64/include/asm/mmzone.h b/arch/sw_64/include/asm/mmzone.h new file mode 100644 index 000000000000..363e2bc98a95 --- /dev/null +++ b/arch/sw_64/include/asm/mmzone.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_MMZONE_H +#define _ASM_SW64_MMZONE_H + +#include + +/* + * Following are macros that are specific to this numa platform. + */ + +extern pg_data_t *node_data[]; + +#ifdef CONFIG_NUMA +#define NODE_DATA(nid) (node_data[(nid)]) +#endif + +#endif /* _ASM_SW64_MMZONE_H */ diff --git a/arch/sw_64/include/asm/page.h b/arch/sw_64/include/asm/page.h new file mode 100644 index 000000000000..68b4f2fc1b48 --- /dev/null +++ b/arch/sw_64/include/asm/page.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_PAGE_H +#define _ASM_SW64_PAGE_H + +#include +#include + +/* PAGE_SHIFT determines the page size */ +#define PAGE_SHIFT 13 +#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) +#define PAGE_MASK (~(PAGE_SIZE - 1)) + +#define HPAGE_SHIFT PMD_SHIFT +#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) +#define HPAGE_MASK (~(HPAGE_SIZE - 1)) +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) + +#define HUGE_MAX_HSTATE 2 + +#ifdef __KERNEL__ +#ifndef __ASSEMBLY__ + +extern void clear_page(void *page); +#define clear_user_page(page, vaddr, pg) clear_page(page) + +#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ + alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE + +extern void copy_page(void *_to, void *_from); +#define copy_user_page(to, from, vaddr, pg) copy_page(to, from) + +typedef struct page *pgtable_t; + +extern unsigned long __phys_addr(unsigned long addr); +#ifdef CONFIG_SUBARCH_C3B +extern unsigned long __boot_phys_addr(unsigned long addr); +#else +#define __boot_phys_addr(x) __phys_addr(x) +#endif + +#endif /* !__ASSEMBLY__ */ + +#define KERNEL_IMAGE_SIZE (512 * 1024 * 1024) + +#include + +#define __START_KERNEL_map PAGE_OFFSET + +#define __pa(x) __phys_addr((unsigned long)(x)) +#define __va(x) ((void *)((unsigned long) (x) | PAGE_OFFSET)) + +#define __boot_pa(x) __boot_phys_addr((unsigned long)(x)) +#define __boot_va(x) __va(x) + +#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) +#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) + +#define virt_to_pfn(vaddr) (PHYS_PFN(__pa(vaddr))) +#define pfn_to_virt(pfn) (__va(PFN_PHYS(pfn))) + +#ifdef CONFIG_FLATMEM +#define pfn_valid(pfn) ((pfn) < max_mapnr) +#endif /* CONFIG_FLATMEM */ + +#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_NON_EXEC +#include +#include +#endif + +#endif /* _ASM_SW64_PAGE_H */ diff --git a/arch/sw_64/include/asm/pgalloc.h b/arch/sw_64/include/asm/pgalloc.h new file mode 100644 index 000000000000..1cc03e3be5b6 --- /dev/null +++ b/arch/sw_64/include/asm/pgalloc.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_PGALLOC_H +#define _ASM_SW64_PGALLOC_H + +#include +#include +#include /* for pte_{alloc,free}_one */ + +/* + * Allocate and free page tables. The xxx_kernel() versions are + * used to allocate a kernel page table - this turns on ASN bits + * if any. + */ + +static inline void +pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte) +{ + unsigned long pfn = page_to_pfn(pte); + + set_pmd(pmd, __pmd((pfn << _PFN_SHIFT) | _PAGE_TABLE)); +} + +static inline void +pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) +{ + unsigned long pfn = virt_to_pfn(pte); + + set_pmd(pmd, __pmd((pfn << _PFN_SHIFT) | _PAGE_TABLE)); +} + +static inline void +pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) +{ + unsigned long pfn = virt_to_pfn(pmd); + + set_pud(pud, __pud((pfn << _PFN_SHIFT) | _PAGE_TABLE)); +} + +static inline void +p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) +{ + unsigned long pfn = virt_to_pfn(pud); + + set_p4d(p4d, __p4d((pfn << _PFN_SHIFT) | _PAGE_TABLE)); +} + +extern pgd_t *pgd_alloc(struct mm_struct *mm); + +#define check_pgt_cache() do { } while (0) + +#endif /* _ASM_SW64_PGALLOC_H */ diff --git a/arch/sw_64/include/asm/pgtable-4level.h b/arch/sw_64/include/asm/pgtable-4level.h new file mode 100644 index 000000000000..719e2c5377e3 --- /dev/null +++ b/arch/sw_64/include/asm/pgtable-4level.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_PGTABLE_4LEVEL_H +#define _ASM_SW64_PGTABLE_4LEVEL_H + +#ifdef __KERNEL__ +#ifndef __ASSEMBLY__ +/* + * These are used to make use of C type-checking.. + */ +typedef struct { unsigned long pte; } pte_t; +typedef struct { unsigned long pmd; } pmd_t; +typedef struct { unsigned long pgd; } pgd_t; +typedef struct { unsigned long pud; } pud_t; +typedef struct { unsigned long pgprot; } pgprot_t; + +#define pte_val(x) ((x).pte) +#define pmd_val(x) ((x).pmd) +#define pgd_val(x) ((x).pgd) +#define pud_val(x) ((x).pud) +#define pgprot_val(x) ((x).pgprot) + +#define __pte(x) ((pte_t) { (x) }) +#define __pmd(x) ((pmd_t) { (x) }) +#define __pud(x) ((pud_t) { (x) }) +#define __pgd(x) ((pgd_t) { (x) }) +#define __pgprot(x) ((pgprot_t) { (x) }) +#endif /* !__ASSEMBLY__ */ + +#define PAGE_OFFSET 0xfff0000000000000 + +#endif +#endif /* _ASM_SW64_PGTABLE_4LEVEL_H */ diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h new file mode 100644 index 000000000000..0b1f825eb74c --- /dev/null +++ b/arch/sw_64/include/asm/pgtable.h @@ -0,0 +1,789 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_PGTABLE_H +#define _ASM_SW64_PGTABLE_H + + +#include + +/* + * This file contains the functions and defines necessary to modify and use + * the sw64 page table tree. + * + * This hopefully works with any standard sw64 page-size, as defined + * in (currently 8192). + */ +#include +#include + +#include +#include +#include /* For TASK_SIZE */ +#include + +struct mm_struct; +struct vm_area_struct; + +static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) +{ + *pmdp = pmd; +} + +static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmdval) +{ + set_pmd(pmdp, pmdval); +} + +static inline void set_pud(pud_t *pudp, pud_t pud) +{ + *pudp = pud; +} + +static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) +{ + *p4dp = p4d; +} +/* PGDIR_SHIFT determines what a forth-level page table entry can map */ +#define PGDIR_SHIFT (PAGE_SHIFT + 3 * (PAGE_SHIFT - 3)) +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE - 1)) + +/* PUD_SHIFT determines the size of the area a third-level page table can map */ +#define PUD_SHIFT (PAGE_SHIFT + 2 * (PAGE_SHIFT - 3)) +#define PUD_SIZE (1UL << PUD_SHIFT) +#define PUD_MASK (~(PUD_SIZE-1)) + +/* PMD_SHIFT determines the size of the area a second-level page table can map */ +#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - 3)) +#define PMD_SIZE (1UL << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE - 1)) + +#define CONT_PMD_SHIFT 6 +#define CONT_PMDS (1 << CONT_PMD_SHIFT) +#define CONT_PMD_SIZE (CONT_PMDS * PMD_SIZE) +#define CONT_PMD_MASK (~(CONT_PMD_SIZE - 1)) + +/* + * Entries per page directory level: the sw64 is three-level, with + * all levels having a one-page page table. + */ +#define PTRS_PER_PTE (1UL << (PAGE_SHIFT - 3)) +#define PTRS_PER_PMD (1UL << (PAGE_SHIFT - 3)) +#define PTRS_PER_PGD (1UL << (PAGE_SHIFT - 3)) +#define PTRS_PER_PUD (1UL << (PAGE_SHIFT - 3)) + +#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) +#define FIRST_USER_ADDRESS 0UL + +/* Number of pointers that fit on a page: this will go away. */ +#define PTRS_PER_PAGE (1UL << (PAGE_SHIFT - 3)) + +#define VMALLOC_START (-2 * PGDIR_SIZE) +#ifndef CONFIG_SPARSEMEM_VMEMMAP +#define VMALLOC_END (-PGDIR_SIZE) +#else +#define VMEMMAP_END (-PGDIR_SIZE) +#define vmemmap ((struct page *)VMEMMAP_END - (1UL << (MAX_PHYSMEM_BITS - PAGE_SHIFT))) +#define VMALLOC_END ((unsigned long)vmemmap) +#endif + +/* + * HMcode-imposed page table bits + */ +#if defined(CONFIG_SUBARCH_C3B) + +#define _PAGE_VALID 0x0001 +#define _PAGE_PRESENT _PAGE_VALID +#define _PAGE_FOR 0x0002 /* used for page protection (fault on read) */ +#define _PAGE_FOW 0x0004 /* used for page protection (fault on write) */ +#define _PAGE_FOE 0x0008 /* used for page protection (fault on exec) */ +#define _PAGE_ASM 0x0010 +#define _PAGE_CONT 0x0020 /* used for 256M page size bit */ +#define _PAGE_LEAF 0x0040 /* used for 8M page size bit */ +#define _PAGE_PROTNONE 0x0080 /* used for numa page balancing */ +#define _PAGE_SPECIAL 0x0100 +#define _PAGE_KRE 0x0400 /* xxx - see below on the "accessed" bit */ +#define _PAGE_URE 0x0800 /* xxx */ +#define _PAGE_KWE 0x4000 /* used to do the dirty bit in software */ +#define _PAGE_UWE 0x8000 /* used to do the dirty bit in software */ + +/* .. and these are ours ... */ +#define _PAGE_DIRTY 0x20000 +#define _PAGE_ACCESSED 0x40000 + +#define _PAGE_SPLITTING 0x200000 /* For Transparent Huge Page */ +#define _PAGE_DEVMAP 0x400000 /* For ZONE DEVICE page */ + +#define _PAGE_BIT_FOW 2 /* bit of _PAGE_FOW */ +#define _PAGE_BIT_ACCESSED 18 /* bit of _PAGE_ACCESSED */ +#define _PAGE_BIT_SPLITTING 21 /* bit of _PAGE_SPLITTING */ +#define _PAGE_BIT_DEVMAP 22 /* bit of _PAGE_DEVMAP */ +/* + * NOTE! The "accessed" bit isn't necessarily exact: it can be kept exactly + * by software (use the KRE/URE/KWE/UWE bits appropriately), but I'll fake it. + * Under Linux/sw64, the "accessed" bit just means "read", and I'll just use + * the KRE/URE bits to watch for it. That way we don't need to overload the + * KWE/UWE bits with both handling dirty and accessed. + * + * Note that the kernel uses the accessed bit just to check whether to page + * out a page or not, so it doesn't have to be exact anyway. + */ + +/* Used for swap PTEs only. */ +#define _PAGE_SWP_EXCLUSIVE _BITUL(5) + +#define __DIRTY_BITS (_PAGE_DIRTY | _PAGE_KWE | _PAGE_UWE) +#define __ACCESS_BITS (_PAGE_ACCESSED | _PAGE_KRE | _PAGE_URE) + +#define _PFN_SHIFT 28 + +/* + * All the normal masks have the "page accessed" bits on, as any time they are used, + * the page is accessed. They are cleared only by the page-out routines + */ +#define PAGE_NONE __pgprot(__ACCESS_BITS | _PAGE_FOR | _PAGE_FOW | _PAGE_FOE | _PAGE_PROTNONE) +#define PAGE_SHARED __pgprot(_PAGE_VALID | __ACCESS_BITS) +#define PAGE_COPY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW) +#define PAGE_READONLY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW) +#define PAGE_KERNEL __pgprot(_PAGE_VALID | _PAGE_ASM | _PAGE_KRE | _PAGE_KWE) +#define _PAGE_NORMAL(x) __pgprot(_PAGE_VALID | __ACCESS_BITS | (x)) + +#define page_valid_kern(x) (0) + +#elif defined(CONFIG_SUBARCH_C4) + +#define _PAGE_VALID 0x0001 +#define _PAGE_PRESENT _PAGE_VALID +#define _PAGE_FOR 0x0002 /* used for page protection (fault on read) */ +#define _PAGE_FOW 0x0004 /* used for page protection (fault on write) */ +#define _PAGE_FOE 0x0008 /* used for page protection (fault on exec) */ +#define _PAGE_FIXED 0x0010 +#define _PAGE_CONT 0x0020 /* used for 512M page size bit*/ +#define _PAGE_LEAF 0x0040 /* used for huge page bit */ +#define _PAGE_PCD 0x0080 /* used for page cache disabled */ + +/* and these are sw definition */ +#define _PAGE_WCD 0x0100 +#define _PAGE_ACCESSED 0x0200 +#define _PAGE_SPLITTING 0x0400 /* For Transparent Huge Page */ +#define _PAGE_SPECIAL 0x0800 +#define _PAGE_DEVMAP 0x1000 /* For ZONE DEVICE page */ +#define _PAGE_KERN 0x2000 +#define _PAGE_DIRTY _BITUL(62) +#define _PAGE_PROTNONE _BITUL(63) +#define _PAGE_BIT_FOW 2 /* bit of _PAGE_FOW */ +#define _PAGE_BIT_ACCESSED 9 /* bit of _PAGE_ACCESSED */ +#define _PAGE_BIT_SPLITTING 10 /* bit of _PAGE_SPLITTING */ +#define _PAGE_BIT_DEVMAP 12 /* bit of _PAGE_DEVMAP */ + +/* Used for swap PTEs only. */ +#define _PAGE_SWP_EXCLUSIVE _BITUL(5) + +#define __DIRTY_BITS _PAGE_DIRTY +#define __ACCESS_BITS _PAGE_ACCESSED + +#define _PFN_SHIFT 24 + +/* + * All the normal masks have the "page accessed" bits on, as any time they are used, + * the page is accessed. They are cleared only by the page-out routines + */ +#define PAGE_NONE __pgprot(__ACCESS_BITS | _PAGE_FOR | _PAGE_FOW | _PAGE_FOE | _PAGE_LEAF | _PAGE_PROTNONE) +#define PAGE_SHARED __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_LEAF) +#define PAGE_COPY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW | _PAGE_LEAF) +#define PAGE_READONLY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW | _PAGE_LEAF) +#define PAGE_KERNEL __pgprot(_PAGE_VALID | _PAGE_KERN | _PAGE_LEAF) +#define _PAGE_NORMAL(x) __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_LEAF | (x)) + +#define page_valid_kern(x) ((x & (_PAGE_VALID | _PAGE_KERN)) == (_PAGE_VALID | _PAGE_KERN)) +#endif + +#define PFN_PTE_SHIFT _PFN_SHIFT + +#define _PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) +#define _PFN_MASK (GENMASK(_PFN_BITS - 1, 0) << _PFN_SHIFT) + +#define _PAGE_TABLE (_PAGE_VALID | __DIRTY_BITS | __ACCESS_BITS) +#define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL | _PAGE_LEAF | _PAGE_CONT) + +#define _PAGE_P(x) _PAGE_NORMAL((x) | _PAGE_FOW) +#define _PAGE_S(x) _PAGE_NORMAL(x) + +/* + * pgprot_noncached() is only for infiniband pci support, and a real + * implementation for RAM would be more complicated. + */ +#define pgprot_noncached(prot) (prot) + +/* + * ZERO_PAGE is a global shared page that is always zero: used + * for zero-mapped memory areas etc.. + */ +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; +#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) + +static inline void set_pte(pte_t *ptep, pte_t pteval) +{ + *ptep = pteval; + + if (page_valid_kern(pte_val(pteval))) { + mb(); + if ((pte_val(pteval) & _PAGE_FOE) == 0) + imemb(); + } +} + +static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) +{ + pte_t pte; + + pte_val(pte) = (pfn << _PFN_SHIFT) | pgprot_val(prot); + return pte; +} + +static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) +{ + pmd_t pmd; + + pmd_val(pmd) = (pfn << _PFN_SHIFT) | pgprot_val(prot); + return pmd; +} +static inline pud_t pfn_pud(unsigned long pfn, pgprot_t pgprot) +{ + pud_t pud; + + pud_val(pud) = (pfn << _PFN_SHIFT) | pgprot_val(pgprot); + return pud; +} + +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) +{ + pte_val(pte) = (pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot); + return pte; +} + +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + pmd_val(pmd) = (pmd_val(pmd) & _PAGE_CHG_MASK) | pgprot_val(newprot); + return pmd; +} + +/* + * Conversion functions: convert a page and protection to a page entry, + * and a page entry and page directory to the page they refer to. + */ +#define page_to_pa(page) (page_to_pfn(page) << PAGE_SHIFT) + +#define p4d_pfn(p4d) ((p4d_val(p4d) & _PFN_MASK) >> _PFN_SHIFT) +#define pud_pfn(pud) ((pud_val(pud) & _PFN_MASK) >> _PFN_SHIFT) +#define pmd_pfn(pmd) ((pmd_val(pmd) & _PFN_MASK) >> _PFN_SHIFT) +#define pte_pfn(pte) ((pte_val(pte) & _PFN_MASK) >> _PFN_SHIFT) + +#define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d)) +#define pud_page(pud) pfn_to_page(pud_pfn(pud)) +#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd)) +#define pte_page(pte) pfn_to_page(pte_pfn(pte)) + +#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + return (pmd_t *)pfn_to_virt(pud_pfn(pud)); +} + +static inline pud_t *p4d_pgtable(p4d_t p4d) +{ + return (pud_t *)pfn_to_virt(p4d_pfn(p4d)); +} + +static inline unsigned long p4d_page_vaddr(p4d_t p4d) +{ + return (unsigned long)pfn_to_virt(p4d_pfn(p4d)); +} + +static inline unsigned long pud_page_vaddr(pud_t pud) +{ + return (unsigned long)pfn_to_virt(pud_pfn(pud)); +} + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)pfn_to_virt(pmd_pfn(pmd)); +} + +static inline int pte_none(pte_t pte) +{ + return !pte_val(pte); +} + +static inline int pte_valid(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_VALID); +} + +static inline int pte_present(pte_t pte) +{ + return !!(pte_val(pte) & (_PAGE_VALID | _PAGE_PROTNONE)); +} + +static inline int pte_huge(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_LEAF); +} + +static inline void pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pte_val(*ptep) = 0; +} + +#define pte_accessible(mm, pte) \ + (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte)) + +static inline int pmd_none(pmd_t pmd) +{ + return !pmd_val(pmd); +} + +static inline int pmd_bad(pmd_t pmd) +{ + return (pmd_val(pmd) & ~_PFN_MASK) != _PAGE_TABLE; +} + +static inline int pmd_present(pmd_t pmd) +{ + /* + * Checking for _PAGE_LEAF is needed too because + * split_huge_page will temporarily clear the valid bit (but + * the _PAGE_LEAF flag will remain set at all times while the + * _PAGE_VALID bit is clear). + */ + return !!(pmd_val(pmd) & (_PAGE_VALID | _PAGE_PROTNONE | _PAGE_LEAF)); +} + +static inline void pmd_clear(pmd_t *pmdp) +{ + pmd_val(*pmdp) = 0; +} + +static inline int pmd_dirty(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_DIRTY); +} + +#define pmd_young pmd_young +static inline int pmd_young(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_ACCESSED); +} + +#define __HAVE_ARCH_PMD_WRITE +#define pmd_write pmd_write +static inline int pmd_write(pmd_t pmd) +{ + return !(pmd_val(pmd) & _PAGE_FOW); +} + +static inline pmd_t pmd_wrprotect(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_FOW; + return pmd; +} + +static inline pmd_t pmd_mkinvalid(pmd_t pmd) +{ + pmd_val(pmd) &= ~(_PAGE_VALID | _PAGE_PROTNONE); + return pmd; +} + +static inline pmd_t pmd_mkclean(pmd_t pmd) +{ + pmd_val(pmd) &= ~(__DIRTY_BITS); + pmd_val(pmd) |= _PAGE_FOW; + return pmd; +} + +static inline pmd_t pmd_mkold(pmd_t pmd) +{ + pmd_val(pmd) &= ~(__ACCESS_BITS); + return pmd; +} + +static inline pmd_t pmd_mkwrite_novma(pmd_t pmd) +{ + pmd_val(pmd) &= ~_PAGE_FOW; + return pmd; +} + +static inline pmd_t pmd_mkdirty(pmd_t pmd) +{ + pmd_val(pmd) |= __DIRTY_BITS; + return pmd; +} + +static inline pmd_t pmd_mkdevmap(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_DEVMAP; + return pmd; +} + +static inline pmd_t pmd_mkyoung(pmd_t pmd) +{ + pmd_val(pmd) |= __ACCESS_BITS; + return pmd; +} + +static inline pmd_t pmd_mkhuge(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_LEAF; + return pmd; +} + +static inline pmd_t pmd_mkcont(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_CONT; + return pmd; +} + +static inline int pud_none(pud_t pud) +{ + return !pud_val(pud); +} + +static inline int pud_bad(pud_t pud) +{ + return (pud_val(pud) & ~_PFN_MASK) != _PAGE_TABLE; +} + +static inline int pud_present(pud_t pud) +{ + return !!(pud_val(pud) & _PAGE_VALID); +} + +static inline void pud_clear(pud_t *pudp) +{ + pud_val(*pudp) = 0; +} + +static inline pud_t pud_mkhuge(pud_t pud) +{ + pud_val(pud) |= _PAGE_LEAF; + return pud; +} + +static inline int p4d_none(p4d_t p4d) +{ + return !p4d_val(p4d); +} + +static inline int p4d_bad(p4d_t p4d) +{ + return (p4d_val(p4d) & ~_PFN_MASK) != _PAGE_TABLE; +} + +static inline int p4d_present(p4d_t p4d) +{ + return !!(p4d_val(p4d) & _PAGE_VALID); +} + +static inline void p4d_clear(p4d_t *p4dp) +{ + p4d_val(*p4dp) = 0; +} + +static inline pte_t pmd_pte(pmd_t pmd) +{ + return __pte(pmd_val(pmd)); +} + +static inline pmd_t pte_pmd(pte_t pte) +{ + return __pmd(pte_val(pte)); +} + +/* + * The following only work if pte_present() is true. + * Undefined behaviour if not.. + */ +static inline int pte_write(pte_t pte) +{ + return !(pte_val(pte) & _PAGE_FOW); +} + +static inline int pte_dirty(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_DIRTY); +} + +static inline int pte_young(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_ACCESSED); +} + +static inline int pte_special(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_SPECIAL); +} + +static inline int pte_cont(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_CONT); +} + +static inline pte_t pte_wrprotect(pte_t pte) +{ + pte_val(pte) |= _PAGE_FOW; + return pte; +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + pte_val(pte) &= ~(__DIRTY_BITS); + pte_val(pte) |= _PAGE_FOW; + return pte; +} + +static inline pte_t pte_mkold(pte_t pte) +{ + pte_val(pte) &= ~(__ACCESS_BITS); + return pte; +} + +static inline pte_t pte_mkwrite_novma(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_FOW; + return pte; +} + +static inline pte_t pte_mkdirty(pte_t pte) +{ + pte_val(pte) |= __DIRTY_BITS; + return pte; +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + pte_val(pte) |= __ACCESS_BITS; + return pte; +} + +static inline pte_t pte_mkhuge(pte_t pte) +{ + pte_val(pte) |= _PAGE_LEAF; + return pte; +} + +static inline pte_t pte_mkspecial(pte_t pte) +{ + pte_val(pte) |= _PAGE_SPECIAL; + return pte; +} + +static inline pte_t pte_mkdevmap(pte_t pte) +{ + pte_val(pte) |= _PAGE_SPECIAL; + return pte; +} + +#ifdef CONFIG_NUMA_BALANCING +/* + * See the comment in include/asm-generic/pgtable.h + */ +static inline int pte_protnone(pte_t pte) +{ + return (pte_val(pte) & (_PAGE_PROTNONE | _PAGE_VALID)) + == _PAGE_PROTNONE; +} + +static inline int pmd_protnone(pmd_t pmd) +{ + return (pmd_val(pmd) & (_PAGE_PROTNONE | _PAGE_VALID)) + == _PAGE_PROTNONE; +} +#endif + +#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP +static inline int pte_devmap(pte_t a) +{ + return (pte_val(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP; +} +#endif + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + +/* We don't have hardware dirty/accessed bits, generic_pmdp_establish is fine.*/ +#define pmdp_establish generic_pmdp_establish + +static inline int pmd_trans_splitting(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_SPLITTING); +} + +static inline int pmd_trans_cont(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_CONT); +} + +static inline int pmd_trans_huge(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_LEAF); +} + +static inline int has_transparent_hugepage(void) +{ + return 1; +} + +#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP +static inline int pmd_devmap(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_DEVMAP); +} + +#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD +static inline int pud_devmap(pud_t pud) +{ + return !!(pud_val(pud) & _PAGE_DEVMAP); +} +#else +static inline int pud_devmap(pud_t pud) +{ + return 0; +} +#endif + +static inline int pgd_devmap(pgd_t pgd) +{ + return 0; +} +#endif +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#define __HAVE_ARCH_PMDP_GET_AND_CLEAR +static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp) +{ + unsigned long pmd_val = xchg(&pmdp->pmd, 0); + pmd_t pmd = (pmd_t){pmd_val}; + return pmd; +} + +#define __HAVE_ARCH_PMDP_SET_WRPROTECT +static inline void pmdp_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pmd_t *pmdp) +{ + set_bit(_PAGE_BIT_FOW, (unsigned long *)pmdp); +} + +#define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), (prot)) + +#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS +extern int pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty); + +#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG +extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp); + +#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH +extern int pmdp_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp); + +#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH +extern void pmdp_splitting_flush(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp); + +extern pgd_t swapper_pg_dir[1024]; + +/* + * The sw64 doesn't have any external MMU info: the kernel page + * tables contain all the necessary information. + */ +#define update_mmu_cache(vma, address, ptep) do { } while (0) +#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) + +static inline void update_mmu_cache_range(struct vm_fault *vmf, + struct vm_area_struct *vma, unsigned long address, + pte_t *ptep, unsigned int nr) +{ +} + +#if defined(CONFIG_SUBARCH_C3B) + +/* + * Encode and decode a swap entry: + * + * Format of swap PTE: + * bit 0: _PAGE_VALID (must be zero) + * bit 6: _PAGE_LEAF (must be zero) + * bit 7: _PAGE_PROTNONE (must be zero) + * bits 8-15: swap type + * bits 16-63: swap offset + */ +#define __SWP_TYPE_SHIFT 8 +#define __SWP_TYPE_BITS 8 + +#elif defined(CONFIG_SUBARCH_C4) + +/* + * Encode and decode a swap entry: + * + * Format of swap PTE: + * bit 0: _PAGE_VALID (must be zero) + * bit 6: _PAGE_LEAF (must be zero) + * bits 7-11: swap type + * bits 12-58: swap offset + * bit 63: _PAGE_PROTNONE (must be zero) + */ +#define __SWP_TYPE_SHIFT 7 +#define __SWP_TYPE_BITS 5 + +#endif + +#define __SWP_OFFSET_BITS 47 +#define __SWP_TYPE_MASK ((1UL << __SWP_TYPE_BITS) - 1) +#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) +#define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1) + +#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK) +#define __swp_offset(x) (((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK) +#define __swp_entry(type, offset) \ + ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) }) + +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) +#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) + +static inline int pte_swp_exclusive(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_SWP_EXCLUSIVE); +} + +static inline pte_t pte_swp_mkexclusive(pte_t pte) +{ + pte_val(pte) |= _PAGE_SWP_EXCLUSIVE; + return pte; +} + +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_SWP_EXCLUSIVE; + return pte; +} + +#define kern_addr_valid(addr) (1) + +#define pte_ERROR(e) \ + pr_err("%s: %d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) +#define pmd_ERROR(e) \ + pr_err("%s: %d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) +#define pud_ERROR(e) \ + pr_err("%s: %d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) +#define pgd_ERROR(e) \ + pr_err("%s: %d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) +extern void paging_init(void); + +/* We have our own get_unmapped_area to cope with ADDR_LIMIT_32BIT. */ +#define HAVE_ARCH_UNMAPPED_AREA + +#endif /* _ASM_SW64_PGTABLE_H */ diff --git a/arch/sw_64/include/asm/sparsemem.h b/arch/sw_64/include/asm/sparsemem.h new file mode 100644 index 000000000000..a60e757f3838 --- /dev/null +++ b/arch/sw_64/include/asm/sparsemem.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SPARSEMEM_H +#define _ASM_SW64_SPARSEMEM_H + +#include + +#define SECTION_SIZE_BITS 28 + +#endif /* _ASM_SW64_SPARSEMEM_H */ diff --git a/arch/sw_64/include/asm/tlb.h b/arch/sw_64/include/asm/tlb.h new file mode 100644 index 000000000000..08c8f4f97de1 --- /dev/null +++ b/arch/sw_64/include/asm/tlb.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_TLB_H +#define _ASM_SW64_TLB_H + +#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm) + +#include + +#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) +#define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd) +#define __pud_free_tlb(tlb, pud, address) pud_free((tlb)->mm, pud) + +#endif /* _ASM_SW64_TLB_H */ diff --git a/arch/sw_64/include/asm/tlbflush.h b/arch/sw_64/include/asm/tlbflush.h new file mode 100644 index 000000000000..73995d9663a6 --- /dev/null +++ b/arch/sw_64/include/asm/tlbflush.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_TLBFLUSH_H +#define _ASM_SW64_TLBFLUSH_H + +#include +#include +#include +#include +#include +#include +#include + +static inline void local_flush_tlb_all(void) +{ + tbiv(); +} + +static inline void local_flush_tlb_mm(struct mm_struct *mm) +{ + int cpu; + unsigned long flags; + + local_irq_save(flags); + + cpu = smp_processor_id(); + if (!asid_valid(mm, cpu)) { + cpumask_clear_cpu(cpu, mm_cpumask(mm)); + goto out; + } + + if (current->mm == mm) { + __get_new_mm_context(mm, cpu); + wrasid(cpu_asid(cpu, mm)); + } else { + mm->context.asid[cpu] = 0; + cpumask_clear_cpu(cpu, mm_cpumask(mm)); + } +out: + local_irq_restore(flags); +} + +static inline void +local_flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) +{ + int cpu; + struct mm_struct *mm; + + cpu = smp_processor_id(); + mm = vma->vm_mm; + + if (asid_valid(mm, cpu)) + tbisasid(cpu_asid(cpu, mm), addr); + else + cpumask_clear_cpu(cpu, mm_cpumask(mm)); +} + +/* + * It flushes the whole user tlb now. + */ +static inline void +local_flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + local_flush_tlb_mm(vma->vm_mm); +} + +/* + * There is no way to invalidate kernel pages only, so it has to + * inlvalidate all mapping. + */ +static inline void +local_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + local_flush_tlb_all(); +} + + +#ifdef CONFIG_SMP +extern void flush_tlb_all(void); +extern void flush_tlb_mm(struct mm_struct *mm); +extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); +#else +#define flush_tlb_all() local_flush_tlb_all() +#define flush_tlb_mm(mm) local_flush_tlb_mm(mm) +#define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr) +#define flush_tlb_range(vma, start, end) local_flush_tlb_range(vma, start, end) +#define flush_tlb_kernel_range(start, end) local_flush_tlb_kernel_range(start, end) + +#endif /* CONFIG_SMP */ + +#endif /* _ASM_SW64_TLBFLUSH_H */ diff --git a/arch/sw_64/include/asm/vmalloc.h b/arch/sw_64/include/asm/vmalloc.h new file mode 100644 index 000000000000..a76d1133d6c6 --- /dev/null +++ b/arch/sw_64/include/asm/vmalloc.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_VMALLOC_H +#define _ASM_SW64_VMALLOC_H + +#endif /* _ASM_SW64_VMALLOC_H */ diff --git a/arch/sw_64/mm/Makefile b/arch/sw_64/mm/Makefile new file mode 100644 index 000000000000..8b9d6e4d2ebf --- /dev/null +++ b/arch/sw_64/mm/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux sw_64-specific parts of the memory manager. +# + +#ccflags-y := -Werror + +obj-y := init.o fault.o physaddr.o mmap.o extable.o + +obj-$(CONFIG_NUMA) += numa.o +ifeq ($(CONFIG_SUBARCH_C4),y) +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage_c4.o +else +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +endif +obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += thp.o diff --git a/arch/sw_64/mm/extable.c b/arch/sw_64/mm/extable.c new file mode 100644 index 000000000000..d2678e12a1b1 --- /dev/null +++ b/arch/sw_64/mm/extable.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +int fixup_exception(struct pt_regs *regs, unsigned long pc) +{ + const struct exception_table_entry *fixup; + + fixup = search_exception_tables(pc); + if (fixup) { + unsigned int valreg = fixup->fixup.bits.valreg; + unsigned int errreg = fixup->fixup.bits.errreg; + + if (valreg != 31) + regs->regs[valreg] = 0; + if (errreg != 31) + regs->regs[errreg] = -EFAULT; + pc += fixup->fixup.bits.nextinsn; + regs->pc = pc; + + return 1; + } + return 0; +} diff --git a/arch/sw_64/mm/fault.c b/arch/sw_64/mm/fault.c new file mode 100644 index 000000000000..e76560a7edca --- /dev/null +++ b/arch/sw_64/mm/fault.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 1995 Linus Torvalds + */ + +#include +#include +#include + +#include + +__read_mostly bool segv_debug_enabled; + +#ifdef CONFIG_KPROBES +static inline int notify_page_fault(struct pt_regs *regs, unsigned long mmcsr) +{ + int ret = 0; + /* kprobe_running() needs smp_processor_id() */ + if (!user_mode(regs)) { + preempt_disable(); + if (kprobe_running() && kprobe_fault_handler(regs, mmcsr)) + ret = 1; + preempt_enable(); + } + return ret; +} +#else +static inline int notify_page_fault(struct pt_regs *regs, unsigned long mmcsr) +{ + return 0; +} +#endif + +extern void die(char *, struct pt_regs *, long); +extern void show_regs(struct pt_regs *regs); + +void show_all_vma(void) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + + MA_STATE(mas, 0, 0, 0); + + if (!mm) + return; + + mas.tree = &mm->mm_mt; + + for (int i = 0; (vma = mas_find(&mas, ULONG_MAX)) != NULL; i++) { + unsigned long start = vma->vm_start; + unsigned long end = vma->vm_end; + struct file *file = vma->vm_file; + + if (file) + pr_info("vma[%d]: [%#lx, %#lx], len = %#lx, flags = %#lx, file = %s, name = %s\n", + i, start, end, (end - start), vma->vm_flags, + file->f_path.dentry->d_name.name, current->comm); + else + pr_info("vma[%d]: [%#lx, %#lx], len = %#lx, flags = %#lx, name = %s\n", + i, start, end, (end - start), vma->vm_flags, current->comm); + } +} + +/* + * This routine handles page faults. It determines the address, + * and the problem, and then passes it off to handle_mm_fault(). + * + * mmcsr: + * 0 = translation not valid + * 1 = access violation + * 2 = fault-on-read + * 3 = fault-on-execute + * 4 = fault-on-write + * + * cause: + * -1 = instruction fetch + * 0 = load + * 1 = store + * + * Registers $9 through $15 are saved in a block just prior to `regs' and + * are saved and restored around the call to allow exception code to + * modify them. + */ + +unsigned long show_va_to_pa(struct mm_struct *mm, unsigned long addr) +{ + pgd_t *pgd = NULL; + p4d_t *p4d = NULL; + pud_t *pud = NULL; + pmd_t *pmd = NULL; + pte_t *pte = NULL; + unsigned long ret = 0UL; + + pgd = pgd_offset(mm, addr); + if (pgd_none(*pgd)) { + ret = 0; + pr_debug("addr = %#lx, pgd = %#lx\n", addr, pgd_val(*pgd)); + goto out; + } + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) { + ret = 0; + pr_debug("addr = %#lx, pgd = %#lx, p4d = %#lx\n", + addr, pgd_val(*pgd), p4d_val(*p4d)); + goto out; + } + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) { + ret = 0; + pr_debug("addr = %#lx, pgd = %#lx, pud = %#lx\n", + addr, pgd_val(*pgd), pud_val(*pud)); + goto out; + } + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) { + ret = 0; + pr_debug("addr = %#lx, pgd = %#lx, pud = %#lx, pmd = %#lx\n", + addr, pgd_val(*pgd), pud_val(*pud), pmd_val(*pmd)); + goto out; + + } + pte = pte_offset_map(pmd, addr); + if (pte_present(*pte)) { + ret = (unsigned long)pfn_to_virt(pte_pfn(*pte)); + pr_debug("addr = %#lx, pgd = %#lx, pud = %#lx, pmd = %#lx, pte = %#lx, ret = %#lx\n", + addr, *(unsigned long *)pgd, *(unsigned long *)pud, + *(unsigned long *)pmd, *(unsigned long *)pte, ret); + } +out: + return ret; +} + +extern int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_regs *regs); + +asmlinkage void notrace +do_page_fault(unsigned long address, unsigned long mmcsr, + long cause, struct pt_regs *regs) +{ + struct vm_area_struct *vma; + struct mm_struct *mm = current->mm; + int si_code = SEGV_MAPERR; + vm_fault_t fault; + unsigned int flags = FAULT_FLAG_DEFAULT; + + if (notify_page_fault(regs, mmcsr)) + return; + + if (unlikely(mmcsr >= MMCSR__DA_MATCH)) { + if (do_match(address, mmcsr, cause, regs) == 1) + return; + } + + if (unlikely(mmcsr == MMCSR__ACV1)) { + if (!user_mode(regs)) + goto no_context; + else { + mmap_read_unlock(mm); + goto bad_area; + } + } + + /* + * If we're in an interrupt context, or have no user context, + * we must not take the fault. + */ + if (!mm || faulthandler_disabled()) + goto no_context; + + if (user_mode(regs)) + flags |= FAULT_FLAG_USER; + + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); + +retry: + vma = lock_mm_and_find_vma(mm, address, regs); + if (!vma) + goto bad_area_nosemaphore; + + /* + * Ok, we have a good vm_area for this memory access, so + * we can handle it. + */ + si_code = SEGV_ACCERR; + if (cause < 0) { + if (!(vma->vm_flags & VM_EXEC)) + goto bad_area; + } else if (!cause) { + /* Allow reads even for write-only mappings */ + if (!(vma->vm_flags & (VM_READ | VM_WRITE))) + goto bad_area; + } else { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + flags |= FAULT_FLAG_WRITE; + } + + /* + * If for any reason at all we couldn't handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ + fault = handle_mm_fault(vma, address, flags, regs); + + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + goto no_context; + return; + } + + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + + if (unlikely(fault & VM_FAULT_ERROR)) { + if (fault & VM_FAULT_OOM) + goto out_of_memory; + else if (fault & VM_FAULT_SIGSEGV) + goto bad_area; + else if (fault & VM_FAULT_SIGBUS) + goto do_sigbus; + BUG(); + } + + if (fault & VM_FAULT_MAJOR) { + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, + regs, address); + current->maj_flt++; + } else { + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, + regs, address); + current->min_flt++; + } + + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; + + /* No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + + goto retry; + } + + mmap_read_unlock(mm); + + return; + + /* + * Something tried to access memory that isn't in our memory map. + * Fix it, but check if it's kernel or user first. + */ + bad_area: + mmap_read_unlock(mm); + + bad_area_nosemaphore: + if (user_mode(regs)) + goto do_sigsegv; + + no_context: + /* Are we prepared to handle this fault as an exception? */ + if (fixup_exception(regs, regs->pc)) + return; + + /* + * Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. + */ + pr_alert("Unable to handle kernel paging request at virtual address %016lx\n", + address); + die("Oops", regs, cause); + make_task_dead(SIGKILL); + + /* + * We ran out of memory, or some other thing happened to us that + * made us unable to handle the page fault gracefully. + */ + out_of_memory: + mmap_read_unlock(mm); + if (!user_mode(regs)) + goto no_context; + pagefault_out_of_memory(); + return; + + do_sigbus: + mmap_read_unlock(mm); + /* + * Send a sigbus, regardless of whether we were in kernel + * or user mode. + */ + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *) address); + if (!user_mode(regs)) + goto no_context; + return; + + do_sigsegv: + force_sig_fault(SIGSEGV, si_code, (void __user *) address); + + if (unlikely(segv_debug_enabled)) { + pr_info("fault: want to send_segv: pid %d, cause = %#lx, mmcsr = %#lx, address = %#lx, pc %#lx\n", + current->pid, cause, mmcsr, address, regs->pc); + show_regs(regs); + show_all_vma(); + } +} diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c new file mode 100644 index 000000000000..ca761b602ab6 --- /dev/null +++ b/arch/sw_64/mm/init.c @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 1995 Linus Torvalds + */ + +/* 2.3.x zone allocator, 1999 Andrea Arcangeli */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +struct mem_desc_t mem_desc; +#ifndef CONFIG_NUMA +struct numa_node_desc_t numa_nodes_desc[1]; +#endif /* CONFIG_NUMA */ + +/* + * empty_zero_page is a special page that is used for + * zero-initialized data and COW. + */ +unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; +EXPORT_SYMBOL(empty_zero_page); +pg_data_t *node_data[MAX_NUMNODES] __read_mostly; +EXPORT_SYMBOL(node_data); + +pgd_t swapper_pg_dir[1024] __aligned(PAGE_SIZE); +static pud_t vmalloc_pud[1024] __aligned(PAGE_SIZE); + +static phys_addr_t mem_start; +static phys_addr_t mem_size_limit; + +#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE +unsigned long memory_block_size_bytes(void) +{ + if (is_in_guest()) + return MIN_MEMORY_BLOCK_SIZE_VM_MEMHP; + else + return MIN_MEMORY_BLOCK_SIZE; +} +#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ + +static int __init setup_mem_size(char *p) +{ + char *oldp; + unsigned long start, size; + + start = 0; + oldp = p; + size = memparse(p, &p); + if (p == oldp) + return -EINVAL; + + if (*p == '@') + start = memparse(p + 1, &p); + + mem_start = start; + mem_size_limit = size; + return 0; +} +early_param("mem", setup_mem_size); + +#if defined(CONFIG_SUBARCH_C3B) +pgd_t * +pgd_alloc(struct mm_struct *mm) +{ + pgd_t *ret, *init; + + ret = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + init = pgd_offset(&init_mm, 0UL); + if (ret) + pgd_val(ret[PTRS_PER_PGD-2]) = pgd_val(init[PTRS_PER_PGD-2]); + + return ret; +} +#elif defined(CONFIG_SUBARCH_C4) +pgd_t * +pgd_alloc(struct mm_struct *mm) +{ + pgd_t *ret; + + ret = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + + return ret; +} +#endif + +/* Set up initial PCB, VPTB, and other such nicities. */ + +static inline void +switch_to_system_map(void) +{ + memset(swapper_pg_dir, 0, PAGE_SIZE); + update_ptbr_sys(virt_to_phys(swapper_pg_dir)); + tbiv(); +} + +void __init callback_init(void) +{ + pgd_t *pgd; + p4d_t *p4d; + + switch_to_system_map(); + + /* Allocate one PGD and one PUD. */ + pgd = pgd_offset_k(VMALLOC_START); + p4d = p4d_offset(pgd, VMALLOC_START); + p4d_populate(&init_mm, p4d, (pud_t *)vmalloc_pud); +} + +void __init zone_sizes_init(void) +{ + unsigned long max_zone_pfns[MAX_NR_ZONES]; + + memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); + +#ifdef CONFIG_ZONE_DMA32 + max_zone_pfns[ZONE_DMA32] = min(MAX_DMA32_PFN, max_low_pfn); +#endif + max_zone_pfns[ZONE_NORMAL] = max_low_pfn; + + free_area_init(max_zone_pfns); +} + +/* + * paging_init() sets up the memory map. + */ +void __init paging_init(void) +{ +} + +void __init mem_detect(void) +{ + int i; + + mem_desc.phys_base = 0; + for (i = 0; i < MAX_NUMSOCKETS; i++) { + if (socket_desc[i].is_online) + mem_desc.phys_size += socket_desc[i].socket_mem; + } + + if (mem_start >= NODE0_START) { + mem_desc.base = mem_start; + } else { + mem_desc.base = NODE0_START; + mem_size_limit -= NODE0_START - mem_start; + } + + if (mem_size_limit && mem_size_limit < mem_desc.phys_size - NODE0_START) + mem_desc.size = mem_size_limit; + else + mem_desc.size = mem_desc.phys_size - NODE0_START; +} + +void __init sw64_memblock_init(void) +{ + memblock_add(mem_desc.base, mem_desc.size); + + memblock_remove(1ULL << MAX_PHYSMEM_BITS, PHYS_ADDR_MAX); + + max_pfn = max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); + + memblock_allow_resize(); + memblock_initialized = true; + process_memmap(); + + /* Make sure kernel text is in memory range. */ + memblock_add(__pa_symbol(_text), _end - _text); + memblock_reserve(__pa_symbol(_text), _end - _text); + + /* Make sure initrd is in memory range. */ + if (sunway_boot_params->initrd_start) { + phys_addr_t base = __boot_pa(sunway_boot_params->initrd_start); + phys_addr_t size = sunway_boot_params->initrd_size; + + memblock_add(base, size); + memblock_reserve(base, size); + } + + /* end of DRAM range may have been changed */ + max_pfn = max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); +} + +#ifndef CONFIG_NUMA +void __init sw64_numa_init(void) +{ + const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES); + u64 nd_pa; + void *nd; + int tnid; + + memblock_set_node(mem_desc.base, mem_desc.size, &memblock.memory, 0); + nd_pa = memblock_phys_alloc(nd_size, SMP_CACHE_BYTES); + nd = __va(nd_pa); + + /* report and initialize */ + pr_info("NODE_DATA [mem %#018llx-%#018llx]\n", + nd_pa, nd_pa + nd_size - 1); + tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT); + if (tnid != 0) + pr_info("NODE_DATA(%d) on node %d\n", 0, tnid); + + node_data[0] = nd; + memset(NODE_DATA(0), 0, sizeof(pg_data_t)); + NODE_DATA(0)->node_id = 0; + NODE_DATA(0)->node_start_pfn = mem_desc.base >> PAGE_SHIFT; + NODE_DATA(0)->node_spanned_pages = mem_desc.size >> PAGE_SHIFT; + node_set_online(0); +} +#endif /* CONFIG_NUMA */ + +void __init +mem_init(void) +{ + set_max_mapnr(max_low_pfn); + high_memory = (void *) __va(max_low_pfn * PAGE_SIZE); +#ifdef CONFIG_SWIOTLB + swiotlb_init(true, SWIOTLB_VERBOSE); +#endif + memblock_free_all(); +} + +#ifdef CONFIG_SPARSEMEM_VMEMMAP +int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, + struct vmem_altmap *altmap) +{ + return vmemmap_populate_basepages(start, end, node, altmap); +} + +void vmemmap_free(unsigned long start, unsigned long end, + struct vmem_altmap *altmap) +{ +} +#endif + +#ifdef CONFIG_HAVE_MEMBLOCK +#ifndef MIN_MEMBLOCK_ADDR +#define MIN_MEMBLOCK_ADDR __pa(PAGE_OFFSET) +#endif +#ifndef MAX_MEMBLOCK_ADDR +#define MAX_MEMBLOCK_ADDR ((phys_addr_t)~0) +#endif +void __init early_init_dt_add_memory_arch(u64 base, u64 size) +{ + const u64 phys_offset = MIN_MEMBLOCK_ADDR; + + if (acpi_disabled) { + if (!PAGE_ALIGNED(base)) { + if (size < PAGE_SIZE - (base & ~PAGE_MASK)) { + pr_warn("Ignoring memory block 0x%llx - 0x%llx\n", + base, base + size); + return; + } + size -= PAGE_SIZE - (base & ~PAGE_MASK); + base = PAGE_ALIGN(base); + } + size &= PAGE_MASK; + + if (base > MAX_MEMBLOCK_ADDR) { + pr_warn("Ignoring memory block 0x%llx - 0x%llx\n", + base, base + size); + return; + } + + if (base + size - 1 > MAX_MEMBLOCK_ADDR) { + pr_warn("Ignoring memory range 0x%llx - 0x%llx\n", + ((u64)MAX_MEMBLOCK_ADDR) + 1, base + size); + size = MAX_MEMBLOCK_ADDR - base + 1; + } + + if (base + size < phys_offset) { + pr_warn("Ignoring memory block 0x%llx - 0x%llx\n", + base, base + size); + return; + } + + if (base < phys_offset) { + pr_warn("Ignoring memory range 0x%llx - 0x%llx\n", + base, phys_offset); + size -= phys_offset - base; + base = phys_offset; + } + memblock_add(base, size); + } else + return; +} +#endif + +#ifdef CONFIG_MEMORY_HOTPLUG +int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params) +{ + unsigned long start_pfn = start >> PAGE_SHIFT; + unsigned long nr_pages = size >> PAGE_SHIFT; + int ret; + + ret = __add_pages(nid, start_pfn, nr_pages, params); + if (ret) + pr_warn("%s: Problem encountered in __add_pages() as ret=%d\n", + __func__, ret); + + return ret; +} + +void arch_remove_memory(int nid, u64 start, u64 size, + struct vmem_altmap *altmap) +{ + unsigned long start_pfn = start >> PAGE_SHIFT; + unsigned long nr_pages = size >> PAGE_SHIFT; + + __remove_pages(start_pfn, nr_pages, altmap); +} +#endif + +static const pgprot_t protection_map[16] = { + [VM_NONE] = _PAGE_P(_PAGE_FOE | _PAGE_FOW | + _PAGE_FOR), + [VM_READ] = _PAGE_P(_PAGE_FOE | _PAGE_FOW), + [VM_WRITE] = _PAGE_P(_PAGE_FOE), + [VM_WRITE | VM_READ] = _PAGE_P(_PAGE_FOE), + [VM_EXEC] = _PAGE_P(_PAGE_FOW | _PAGE_FOR), + [VM_EXEC | VM_READ] = _PAGE_P(_PAGE_FOW), + [VM_EXEC | VM_WRITE] = _PAGE_P(0), + [VM_EXEC | VM_WRITE | VM_READ] = _PAGE_P(0), + [VM_SHARED] = _PAGE_S(_PAGE_FOE | _PAGE_FOW | + _PAGE_FOR), + [VM_SHARED | VM_READ] = _PAGE_S(_PAGE_FOE | _PAGE_FOW), + [VM_SHARED | VM_WRITE] = _PAGE_S(_PAGE_FOE), + [VM_SHARED | VM_WRITE | VM_READ] = _PAGE_S(_PAGE_FOE), + [VM_SHARED | VM_EXEC] = _PAGE_S(_PAGE_FOW | _PAGE_FOR), + [VM_SHARED | VM_EXEC | VM_READ] = _PAGE_S(_PAGE_FOW), + [VM_SHARED | VM_EXEC | VM_WRITE] = _PAGE_S(0), + [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = _PAGE_S(0) +}; +DECLARE_VM_GET_PAGE_PROT diff --git a/arch/sw_64/mm/mmap.c b/arch/sw_64/mm/mmap.c new file mode 100644 index 000000000000..a7a189fc36d6 --- /dev/null +++ b/arch/sw_64/mm/mmap.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include + +#include + +unsigned long +arch_get_unmapped_area(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + struct vm_unmapped_area_info info; + unsigned long limit; + + /* Support 32 bit heap. */ + if (current->personality & ADDR_LIMIT_32BIT) + limit = 0x80000000; + else + limit = TASK_SIZE; + + if (len > limit) + return -ENOMEM; + + if (flags & MAP_FIXED) { + if (addr + len > TASK_SIZE) + return -EINVAL; + + return addr; + } + + if (addr) { + addr = PAGE_ALIGN(addr); + + vma = find_vma(mm, addr); + if (TASK_SIZE - len >= addr && + (!vma || addr + len <= vm_start_gap(vma))) + return addr; + } + + info.flags = 0; + info.length = len; + info.low_limit = mm->mmap_base; + info.high_limit = limit; + info.align_mask = 0; + info.align_offset = pgoff << PAGE_SHIFT; + + return vm_unmapped_area(&info); +} + +unsigned long arch_mmap_rnd(void) +{ + unsigned long rnd; + + /* 8MB for 32bit, 256MB for 64bit */ + if (current->personality & ADDR_LIMIT_32BIT) + rnd = get_random_long() & 0x7ffffful; + else + rnd = get_random_long() & 0xffffffful; + + return rnd << PAGE_SHIFT; +} + +/* + * This function, called very early during the creation of a new process VM + * image, sets up which VM layout function to use: + */ +void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) +{ + unsigned long random_factor = 0UL; + + if (current->flags & PF_RANDOMIZE) + random_factor = arch_mmap_rnd(); + + /* + * Fall back to the standard layout if the personality bit is set, or + * if the expected stack growth is unlimited: + */ + mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; + mm->get_unmapped_area = arch_get_unmapped_area; +} + +SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, unsigned long, fd, + unsigned long, off) +{ + unsigned long ret = -EINVAL; + + if ((off + PAGE_ALIGN(len)) < off) + goto out; + if (off & ~PAGE_MASK) + goto out; + ret = ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); + out: + return ret; +} diff --git a/arch/sw_64/mm/physaddr.c b/arch/sw_64/mm/physaddr.c new file mode 100644 index 000000000000..3c6ecb8ee86a --- /dev/null +++ b/arch/sw_64/mm/physaddr.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +unsigned long __phys_addr(unsigned long addr) +{ + VIRTUAL_BUG_ON(addr < PAGE_OFFSET); + addr &= ~PAGE_OFFSET; + VIRTUAL_BUG_ON(!phys_addr_valid(addr)); + return addr; +} +EXPORT_SYMBOL(__phys_addr); + +bool __virt_addr_valid(unsigned long addr) +{ + if (addr < PAGE_OFFSET) + return false; + addr &= ~PAGE_OFFSET; + return pfn_valid(addr >> PAGE_SHIFT); +} +EXPORT_SYMBOL(__virt_addr_valid); + +#ifdef CONFIG_SUBARCH_C3B +#define LEGACY_BOOT_VA 0xffffffff80000000 +unsigned long __boot_phys_addr(unsigned long addr) +{ + if (addr >= LEGACY_BOOT_VA) { + addr &= ~LEGACY_BOOT_VA; + VIRTUAL_BUG_ON(addr >= KERNEL_IMAGE_SIZE); + } else { + VIRTUAL_BUG_ON(addr < PAGE_OFFSET); + addr &= ~PAGE_OFFSET; + VIRTUAL_BUG_ON(!phys_addr_valid(addr)); + } + return addr; +} +#endif -- Gitee From a82d7e560d05c26bd9cb368307d08585f13cbc1f Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:14 +0800 Subject: [PATCH 015/524] sw64: add hugetlb support Add hugetlb support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hugetlb.h | 43 +++ arch/sw_64/mm/hugetlbpage.c | 313 +++++++++++++++++++++ arch/sw_64/mm/hugetlbpage_c4.c | 452 +++++++++++++++++++++++++++++++ arch/sw_64/mm/thp.c | 55 ++++ 4 files changed, 863 insertions(+) create mode 100644 arch/sw_64/include/asm/hugetlb.h create mode 100644 arch/sw_64/mm/hugetlbpage.c create mode 100644 arch/sw_64/mm/hugetlbpage_c4.c create mode 100644 arch/sw_64/mm/thp.c diff --git a/arch/sw_64/include/asm/hugetlb.h b/arch/sw_64/include/asm/hugetlb.h new file mode 100644 index 000000000000..f4c8cbe0891a --- /dev/null +++ b/arch/sw_64/include/asm/hugetlb.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_HUGETLB_H +#define _ASM_SW64_HUGETLB_H + +#include + +#ifdef CONFIG_SUBARCH_C4 +#define __HAVE_ARCH_HUGE_PTE_CLEAR +extern void huge_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long sz); + +#define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT +extern void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned long sz); + +#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR +extern pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep); + +#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH +extern pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, + pte_t *ptep); + +#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT +extern void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep); + +#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS +extern int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, pte_t pte, int dirty); + +#define arch_make_huge_pte arch_make_huge_pte +extern pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, + vm_flags_t flags); + +#define set_huge_swap_pte_at set_huge_swap_pte_at +extern void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned long sz); +#endif + +#include + +#endif /* _ASM_SW64_HUGETLB_H */ diff --git a/arch/sw_64/mm/hugetlbpage.c b/arch/sw_64/mm/hugetlbpage.c new file mode 100644 index 000000000000..fae1fa8bf7df --- /dev/null +++ b/arch/sw_64/mm/hugetlbpage.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SW64 Huge TLB Page Support for Kernel. + */ + +#include +#include +#include +#include + +#include +#include + +/* + * pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal + * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry. + * Otherwise, returns 0. + */ +int pmd_huge(pmd_t pmd) +{ + return !pmd_none(pmd) && + (pmd_val(pmd) & (_PAGE_VALID | _PAGE_LEAF)) != _PAGE_VALID; +} + +int pud_huge(pud_t pud) +{ + return 0; +} + +pte_t *sw64_256m_hugepte_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr) +{ + int i; + struct page *page; + pmd_t *pmd; + pte_t *pte = NULL; + + pmd = pmd_alloc(mm, pud, addr); + if (pmd == NULL) + return NULL; + + pte = pte_alloc_map(mm, pmd, addr); + if (pte == NULL) + return NULL; + + page = virt_to_page(pte); + pmd_val(*pmd) = pmd_val(*pmd) | _PAGE_LEAF | _PAGE_CONT; + for (i = 1; i < 32; i++) + pmd_val(*(pmd+i)) = pmd_val(*pmd); + return pte; +} + +pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pte_t *pte = NULL; + + pgd = pgd_offset(mm, addr); + p4d = p4d_alloc(mm, pgd, addr); + pud = pud_alloc(mm, p4d, addr); + if (pud) { + if (sz == PMD_SIZE) { + if (want_pmd_share(vma, addr) && pud_none(*pud)) + pte = huge_pmd_share(mm, vma, addr, pud); + else + pte = (pte_t *)pmd_alloc(mm, pud, addr); + } else if (sz == (PMD_SIZE << 5)) { + pte = sw64_256m_hugepte_alloc(mm, pud, addr); + } else { + pr_warn("Unsupported page size %lx\n", sz); + return NULL; + } + } + BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte)); + + return pte; +} + +pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, + unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd = NULL; + pte_t *pte = NULL; + + pgd = pgd_offset(mm, addr); + if (pgd_present(*pgd)) { + p4d = p4d_offset(pgd, addr); + if (p4d_present(*p4d)) { + pud = pud_offset(p4d, addr); + if (pud_present(*pud)) { + pmd = pmd_offset(pud, addr); + if (!pmd_present(*pmd)) + return NULL; + if (pmd_val(*pmd) & _PAGE_CONT) + pte = pte_offset_map(pmd, addr); + else + pte = (pte_t *) pmd; + } + } + } + return pte; +} + +static inline int sw64_huge_pmd_bad(pmd_t pmd) +{ + return !(((pmd_val(pmd) & ~_PFN_MASK) == _PAGE_TABLE) || + ((pmd_val(pmd) & _PAGE_CONT) == _PAGE_CONT)); +} + +static inline int sw64_huge_pmd_none_or_clear_bad(pmd_t *pmd) +{ + if (pmd_none(*pmd)) + return 1; + if (unlikely(sw64_huge_pmd_bad(*pmd))) { + pmd_clear_bad(pmd); + return 1; + } + return 0; +} + +static void sw64_huge_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long addr) +{ + if ((((unsigned long)pmd & 0xffUL) == 0) && + ((pmd_val(*pmd) & _PAGE_CONT) == _PAGE_CONT)) { + pgtable_t token = pmd_pgtable(*pmd); + + pmd_clear(pmd); + pte_free_tlb(tlb, token, addr); + mm_dec_nr_ptes(tlb->mm); + } else { + pmd_clear(pmd); + } +} + +static inline void sw64_huge_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, + unsigned long addr, unsigned long end, + unsigned long floor, unsigned long ceiling) +{ + pmd_t *pmd; + unsigned long next; + unsigned long start; + + start = addr; + pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (sw64_huge_pmd_none_or_clear_bad(pmd)) + continue; + sw64_huge_free_pte_range(tlb, pmd, addr); + } while (pmd++, addr = next, addr != end); + + start &= PUD_MASK; + if (start < floor) + return; + if (ceiling) { + ceiling &= PUD_MASK; + if (!ceiling) + return; + } + if (end - 1 > ceiling - 1) + return; + + pmd = pmd_offset(pud, start); + pud_clear(pud); + pmd_free_tlb(tlb, pmd, start); + mm_dec_nr_pmds(tlb->mm); +} + +static inline void sw64_huge_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, + unsigned long addr, unsigned long end, + unsigned long floor, unsigned long ceiling) +{ + pud_t *pud; + unsigned long next; + unsigned long start; + + start = addr; + pud = pud_offset(p4d, addr); + do { + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(pud)) + continue; + sw64_huge_free_pmd_range(tlb, pud, addr, next, floor, ceiling); + } while (pud++, addr = next, addr != end); + + start &= PGDIR_MASK; + if (start < floor) + return; + if (ceiling) { + ceiling &= PGDIR_MASK; + if (!ceiling) + return; + } + if (end - 1 > ceiling - 1) + return; + + pud = pud_offset(p4d, start); + p4d_clear(p4d); + pud_free_tlb(tlb, pud, start); + mm_dec_nr_puds(tlb->mm); +} + +#ifdef CONFIG_HUGETLB_PAGE +static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct vm_unmapped_area_info info; + + info.flags = 0; + info.length = len; + info.low_limit = current->mm->mmap_legacy_base; + info.high_limit = TASK_SIZE; + info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.align_offset = 0; + return vm_unmapped_area(&info); +} + +static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, + unsigned long addr0, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct vm_unmapped_area_info info; + unsigned long addr; + + info.flags = VM_UNMAPPED_AREA_TOPDOWN; + info.length = len; + info.low_limit = PAGE_SIZE; + info.high_limit = current->mm->mmap_base; + info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.align_offset = 0; + addr = vm_unmapped_area(&info); + + /* + * A failed mmap() very likely causes application failure, + * so fall back to the bottom-up function here. This scenario + * can happen with large stack limits and large mmap() + * allocations. + */ + if (addr & ~PAGE_MASK) { + VM_BUG_ON(addr != -ENOMEM); + info.flags = 0; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = TASK_SIZE; + addr = vm_unmapped_area(&info); + } + + return addr; +} + +unsigned long +hugetlb_get_unmapped_area(struct file *file, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + + if (len & ~huge_page_mask(h)) + return -EINVAL; + if (len > TASK_SIZE) + return -ENOMEM; + + if (flags & MAP_FIXED) { + if (prepare_hugepage_range(file, addr, len)) + return -EINVAL; + return addr; + } + + if (addr) { + addr = ALIGN(addr, huge_page_size(h)); + vma = find_vma(mm, addr); + if (TASK_SIZE - len >= addr && + (!vma || addr + len <= vma->vm_start)) + return addr; + } + if (mm->get_unmapped_area == arch_get_unmapped_area) + return hugetlb_get_unmapped_area_bottomup(file, addr, len, + pgoff, flags); + else + return hugetlb_get_unmapped_area_topdown(file, addr, len, + pgoff, flags); +} + +#if (defined(CONFIG_FORCE_MAX_ZONEORDER) && (CONFIG_FORCE_MAX_ZONEORDER >= 16)) +static __init int sw64_256m_hugetlb_init(void) +{ + if (!size_to_hstate(1UL << (PMD_SHIFT + 5))) + hugetlb_add_hstate(PMD_SHIFT + 5 - PAGE_SHIFT); + return 0; +} +arch_initcall(sw64_256m_hugetlb_init); +#endif +#endif /* CONFIG_HUGETLB_PAGE */ + +bool __init arch_hugetlb_valid_size(unsigned long size) +{ + switch (size) { + case PMD_SIZE: + case (PMD_SIZE<<5): + return true; + } + + return false; +} diff --git a/arch/sw_64/mm/hugetlbpage_c4.c b/arch/sw_64/mm/hugetlbpage_c4.c new file mode 100644 index 000000000000..913389cd2577 --- /dev/null +++ b/arch/sw_64/mm/hugetlbpage_c4.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SW_64 Huge TLB Page Support for Kernel. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/* + * pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal + * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry. + * Otherwise, returns 0. + */ +int pmd_huge(pmd_t pmd) +{ + return !pmd_none(pmd) && + (pmd_val(pmd) & (_PAGE_PRESENT|_PAGE_LEAF)) != _PAGE_PRESENT; +} + +int pud_huge(pud_t pud) +{ + return !pud_none(pud) && + (pud_val(pud) & (_PAGE_PRESENT|_PAGE_LEAF)) != _PAGE_PRESENT; +} +EXPORT_SYMBOL(pud_huge); + +/* + * Select all bits except the pfn + */ +static inline pgprot_t pte_pgprot(pte_t pte) +{ + unsigned long pfn = pte_pfn(pte); + + return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); +} + +static inline int num_contig_ptes(unsigned long size, size_t *pgsize) +{ + int contig_ptes = 0; + + *pgsize = size; + + switch (size) { + case PUD_SIZE: + case PMD_SIZE: + contig_ptes = 1; + break; + case CONT_PMD_SIZE: + *pgsize = PMD_SIZE; + contig_ptes = CONT_PMDS; + break; + default: + break; + } + + return contig_ptes; +} + +static pte_t get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + unsigned long pgsize, unsigned long ncontig) +{ + pte_t orig_pte = huge_ptep_get(ptep); + unsigned long i; + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { + pte_t pte = ptep_get_and_clear(mm, addr, ptep); + + if (pte_dirty(pte)) + orig_pte = pte_mkdirty(orig_pte); + + if (pte_young(pte)) + orig_pte = pte_mkyoung(orig_pte); + } + + return orig_pte; +} + +static pte_t get_clear_contig_flush(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long pgsize, + unsigned long ncontig) +{ + pte_t orig_pte = get_and_clear(mm, addr, ptep, pgsize, ncontig); + struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0); + unsigned long i, saddr = addr; + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) + pte_clear(mm, addr, ptep); + + flush_tlb_range(&vma, saddr, addr); + return orig_pte; +} + +pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pte_t *pte = NULL; + + pgd = pgd_offset(mm, addr); + p4d = p4d_alloc(mm, pgd, addr); + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return NULL; + + if (sz == PUD_SIZE) { + pte = (pte_t *)pud; + } else if (sz == PMD_SIZE) { + if (want_pmd_share(vma, addr) && pud_none(*pud)) + pte = huge_pmd_share(mm, vma, addr, pud); + else + pte = (pte_t *)pmd_alloc(mm, pud, addr); + } else if (sz == (PMD_SIZE * CONT_PMDS)) { + pte = (pte_t *)pmd_alloc(mm, pud, addr); + WARN_ON(addr & (sz - 1)); + } + + WARN_ON(pte && !pte_none(*pte) && !pte_huge(*pte)); + return pte; +} + +pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, + unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd = NULL; + + pgd = pgd_offset(mm, addr); + if (!pgd_present(*pgd)) + return NULL; + + p4d = p4d_offset(pgd, addr); + if (!p4d_present(*p4d)) + return NULL; + + pud = pud_offset(p4d, addr); + + if (sz != PUD_SIZE && pud_none(*pud)) + return NULL; + /* hugepage or swap? */ + if (pud_huge(*pud) || !pud_present(*pud)) + return (pte_t *)pud; + /* table; check the next level */ + + if (sz == CONT_PMD_SIZE) + addr &= CONT_PMD_MASK; + + pmd = pmd_offset(pud, addr); + if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) && + pmd_none(*pmd)) + return NULL; + if (pmd_huge(*pmd) || !pmd_present(*pmd)) + return (pte_t *)pmd; + + return NULL; +} + +pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) +{ + size_t pagesize = 1UL << shift; + + if (pagesize == CONT_PMD_SIZE) { + entry = pmd_pte(pmd_mkcont(pte_pmd(entry))); + } else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) { + pr_warn("%s: unrecognized huge page size 0x%lx\n", + __func__, pagesize); + } + return entry; +} + +void huge_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long sz) +{ + int i, ncontig; + size_t pgsize; + + ncontig = num_contig_ptes(sz, &pgsize); + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) + pte_clear(mm, addr, ptep); +} + +void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned long sz) +{ + size_t pgsize; + int i; + int ncontig; + unsigned long pfn; + pgprot_t hugeprot; + + /* + * Code needs to be expanded to handle huge swap and migration + * entries. Needed for HUGETLB and MEMORY_FAILURE. + */ + WARN_ON(!pte_present(pte)); + + if (!pte_cont(pte)) { + set_pte_at(mm, addr, ptep, pte); + return; + } + + ncontig = num_contig_ptes(sz, &pgsize); + pfn = pte_pfn(pte); + hugeprot = pte_pgprot(pte); + + get_and_clear(mm, addr, ptep, pgsize, ncontig); + + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); +} + +void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned long sz) +{ + int i, ncontig; + size_t pgsize; + + ncontig = num_contig_ptes(sz, &pgsize); + + for (i = 0; i < ncontig; i++, ptep++) + set_pte(ptep, pte); +} + +void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + unsigned long pfn; + pgprot_t hugeprot; + int ncontig, i; + size_t pgsize; + pte_t pte; + + if (!pte_cont(READ_ONCE(*ptep))) { + ptep_set_wrprotect(mm, addr, ptep); + return; + } + + ncontig = CONT_PMDS; + + pte = get_and_clear(mm, addr, ptep, pgsize, ncontig); + pte = pte_wrprotect(pte); + + hugeprot = pte_pgprot(pte); + pfn = pte_pfn(pte); + + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); +} + +pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + int ncontig; + size_t pgsize; + pte_t orig_pte = huge_ptep_get(ptep); + + if (!pte_cont(orig_pte)) + return ptep_get_and_clear(mm, addr, ptep); + + ncontig = CONT_PMDS; + + return get_and_clear(mm, addr, ptep, pgsize, ncontig); +} + +pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + struct mm_struct *mm = vma->vm_mm; + size_t pgsize; + int ncontig; + + if (!pte_cont(READ_ONCE(*ptep))) + return ptep_clear_flush(vma, addr, ptep); + + ncontig = CONT_PMDS; + return get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig); +} + +static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig) +{ + int i; + + if (pte_write(pte) != pte_write(huge_ptep_get(ptep))) + return 1; + + for (i = 0; i < ncontig; i++) { + pte_t orig_pte = huge_ptep_get(ptep + i); + + if (pte_dirty(pte) != pte_dirty(orig_pte)) + return 1; + + if (pte_young(pte) != pte_young(orig_pte)) + return 1; + } + + return 0; +} + +int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, int dirty) +{ + int ncontig, i; + size_t pgsize = 0; + unsigned long pfn = pte_pfn(pte); + pgprot_t hugeprot; + pte_t orig_pte; + + if (!pte_cont(pte)) + return ptep_set_access_flags(vma, addr, ptep, pte, dirty); + + ncontig = CONT_PMDS; + + if (!__cont_access_flags_changed(ptep, pte, ncontig)) + return 0; + + orig_pte = get_and_clear(vma->vm_mm, addr, ptep, pgsize, ncontig); + flush_tlb_fix_spurious_fault(vma, addr, ptep); + + /* Make sure we don't lose the dirty or young state */ + if (pte_dirty(orig_pte)) + pte = pte_mkdirty(pte); + + if (pte_young(orig_pte)) + pte = pte_mkyoung(pte); + + hugeprot = pte_pgprot(pte); + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot)); + + return 1; +} + +#ifdef CONFIG_HUGETLB_PAGE +static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct vm_unmapped_area_info info; + + info.flags = 0; + info.length = len; + info.low_limit = current->mm->mmap_legacy_base; + info.high_limit = TASK_SIZE; + info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.align_offset = 0; + return vm_unmapped_area(&info); +} + +static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, + unsigned long addr0, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct vm_unmapped_area_info info; + unsigned long addr; + + info.flags = VM_UNMAPPED_AREA_TOPDOWN; + info.length = len; + info.low_limit = PAGE_SIZE; + info.high_limit = current->mm->mmap_base; + info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.align_offset = 0; + addr = vm_unmapped_area(&info); + + /* + * A failed mmap() very likely causes application failure, + * so fall back to the bottom-up function here. This scenario + * can happen with large stack limits and large mmap() + * allocations. + */ + if (addr & ~PAGE_MASK) { + VM_BUG_ON(addr != -ENOMEM); + info.flags = 0; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = TASK_SIZE; + addr = vm_unmapped_area(&info); + } + + return addr; +} + + unsigned long +hugetlb_get_unmapped_area(struct file *file, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + + if (len & ~huge_page_mask(h)) + return -EINVAL; + if (len > TASK_SIZE) + return -ENOMEM; + + if (flags & MAP_FIXED) { + if (prepare_hugepage_range(file, addr, len)) + return -EINVAL; + return addr; + } + + if (addr) { + addr = ALIGN(addr, huge_page_size(h)); + vma = find_vma(mm, addr); + if (TASK_SIZE - len >= addr && + (!vma || addr + len <= vma->vm_start)) + return addr; + } + if (mm->get_unmapped_area == arch_get_unmapped_area) + return hugetlb_get_unmapped_area_bottomup(file, addr, len, + pgoff, flags); + else + return hugetlb_get_unmapped_area_topdown(file, addr, len, + pgoff, flags); +} +#endif /* CONFIG_HUGETLB_PAGE */ + +static __init int setup_hugepagesz(char *opt) +{ + unsigned long ps = memparse(opt, &opt); + + switch (ps) { + case PUD_SIZE: + case PMD_SIZE * CONT_PMDS: + case PMD_SIZE: + hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT); + return 1; + } + + pr_err("hugepagesz: Unsupported page size %lu M\n", + ps >> 20); + return 0; +} +__setup("hugepagesz=", setup_hugepagesz); diff --git a/arch/sw_64/mm/thp.c b/arch/sw_64/mm/thp.c new file mode 100644 index 000000000000..833bb59f79d0 --- /dev/null +++ b/arch/sw_64/mm/thp.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +int pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty) +{ + int changed = !pmd_same(*pmdp, entry); + + VM_BUG_ON(address & ~HPAGE_PMD_MASK); + + if (changed && dirty) { + *pmdp = entry; + flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); + } + + return changed; +} +int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp) +{ + int ret = 0; + + if (pmd_young(*pmdp)) + ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, + (unsigned long *)pmdp); + return ret; +} + +int pmdp_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) +{ + int young; + + VM_BUG_ON(address & ~HPAGE_PMD_MASK); + + young = pmdp_test_and_clear_young(vma, address, pmdp); + if (young) + flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); + + return young; +} +void pmdp_splitting_flush(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) +{ + int set; + + VM_BUG_ON(address & ~HPAGE_PMD_MASK); + set = !test_and_set_bit(_PAGE_BIT_SPLITTING, (unsigned long *)pmdp); + if (set) { + /* need tlb flush only to serialize against gup-fast */ + flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); + } +} -- Gitee From d274fb65a3b6eeea7dadac576470a220e8761f14 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:37 +0800 Subject: [PATCH 016/524] sw64: add system call support Add system call support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/syscall.h | 82 ++++ arch/sw_64/include/asm/uaccess.h | 311 +++++++++++++++ arch/sw_64/include/asm/unistd.h | 27 ++ arch/sw_64/include/uapi/asm/unistd.h | 12 + arch/sw_64/kernel/sys_sw64.c | 151 +++++++ arch/sw_64/kernel/syscalls/Makefile | 32 ++ arch/sw_64/kernel/syscalls/syscall.tbl | 528 +++++++++++++++++++++++++ arch/sw_64/kernel/systbls.S | 15 + 8 files changed, 1158 insertions(+) create mode 100644 arch/sw_64/include/asm/syscall.h create mode 100644 arch/sw_64/include/asm/uaccess.h create mode 100644 arch/sw_64/include/asm/unistd.h create mode 100644 arch/sw_64/include/uapi/asm/unistd.h create mode 100644 arch/sw_64/kernel/sys_sw64.c create mode 100644 arch/sw_64/kernel/syscalls/Makefile create mode 100644 arch/sw_64/kernel/syscalls/syscall.tbl create mode 100644 arch/sw_64/kernel/systbls.S diff --git a/arch/sw_64/include/asm/syscall.h b/arch/sw_64/include/asm/syscall.h new file mode 100644 index 000000000000..a821bf68be16 --- /dev/null +++ b/arch/sw_64/include/asm/syscall.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SYSCALL_H +#define _ASM_SW64_SYSCALL_H + +#include + +#ifndef __ASSEMBLY__ + +typedef long (*syscall_fn_t)(ulong, ulong, ulong, ulong, ulong, ulong); + +extern syscall_fn_t sys_call_table[]; + +static inline int syscall_get_nr(struct task_struct *task, + struct pt_regs *regs) +{ + return regs->regs[0]; +} + +static inline long +syscall_get_error(struct task_struct *task, struct pt_regs *regs) +{ + return regs->regs[19] ? -regs->regs[0] : 0; +} + +static inline long syscall_get_return_value(struct task_struct *task, + struct pt_regs *regs) +{ + return regs->regs[0]; +} + +static inline void syscall_set_return_value(struct task_struct *task, + struct pt_regs *regs, + int error, long val) +{ + if (error) { + regs->regs[0] = -error; + regs->regs[19] = 1; + } else { + regs->regs[0] = val; + regs->regs[19] = 0; + } +} + +static inline void syscall_rollback(struct task_struct *task, + struct pt_regs *regs) +{ + regs->regs[0] = regs->orig_r0; + regs->regs[19] = regs->orig_r19; +} + +static inline void syscall_get_arguments(struct task_struct *task, + struct pt_regs *regs, + unsigned long *args) +{ + *args++ = regs->regs[16]; + *args++ = regs->regs[17]; + *args++ = regs->regs[18]; + *args++ = regs->regs[19]; + *args++ = regs->regs[20]; + *args = regs->regs[21]; +} + +static inline void syscall_set_arguments(struct task_struct *task, + struct pt_regs *regs, + const unsigned long *args) +{ + regs->regs[16] = *args++; + regs->regs[17] = *args++; + regs->regs[18] = *args++; + regs->regs[19] = *args++; + regs->regs[20] = *args++; + regs->regs[21] = *args; +} + +static inline int syscall_get_arch(struct task_struct *task) +{ + return AUDIT_ARCH_SW64; +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* _ASM_SW64_SYSCALL_H */ diff --git a/arch/sw_64/include/asm/uaccess.h b/arch/sw_64/include/asm/uaccess.h new file mode 100644 index 000000000000..f6b119f7fa78 --- /dev/null +++ b/arch/sw_64/include/asm/uaccess.h @@ -0,0 +1,311 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UACCESS_H +#define _ASM_SW64_UACCESS_H + +#include + +/* + * The fs value determines whether argument validity checking should be + * performed or not. If get_fs() == USER_DS, checking is performed, with + * get_fs() == KERNEL_DS, checking is bypassed. + * + * Or at least it did once upon a time. Nowadays it is a mask that + * defines which bits of the address space are off limits. This is a + * wee bit faster than the above. + * + * For historical reasons, these macros are grossly misnamed. + */ + +#define KERNEL_DS ((mm_segment_t) { 0UL }) +#define USER_DS ((mm_segment_t) { -0x10000000000000UL }) + +#define get_fs() (current_thread_info()->addr_limit) +#define get_ds() (KERNEL_DS) +#define set_fs(x) (current_thread_info()->addr_limit = (x)) + +#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) + +/* + * These are the main single-value transfer routines. They automatically + * use the right size if we just have the right pointer type. + * + * As the sw64 uses the same address space for kernel and user + * data, we can just do these as direct assignments. (Of course, the + * exception handling means that it's no longer "just"...) + * + * Careful to not + * (a) re-use the arguments for side effects (sizeof/typeof is ok) + * (b) require any knowledge of processes at this stage + */ +#define put_user(x, ptr) \ + __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) +#define get_user(x, ptr) \ + __get_user_check((x), (ptr), sizeof(*(ptr))) + +/* + * The "__xxx" versions do not do address space checking, useful when + * doing multiple accesses to the same area (the programmer has to do the + * checks by hand with "access_ok()") + */ +#define __put_user(x, ptr) \ + __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) +#define __get_user(x, ptr) \ + __get_user_nocheck((x), (ptr), sizeof(*(ptr))) +/* + * The "ldi %1, 2b-1b(%0)" bits are magic to get the assembler to + * encode the bits we need for resolving the exception. See the + * more extensive comments with fixup_inline_exception below for + * more information. + */ + +extern void __get_user_unknown(void); + +#define __get_user_nocheck(x, ptr, size) \ +({ \ + long __gu_err = 0; \ + unsigned long __gu_val; \ + __chk_user_ptr(ptr); \ + switch (size) { \ + case 1: \ + __get_user_8(ptr); \ + break; \ + case 2: \ + __get_user_16(ptr); \ + break; \ + case 4: \ + __get_user_32(ptr); \ + break; \ + case 8: \ + __get_user_64(ptr); \ + break; \ + default: \ + __get_user_unknown(); \ + break; \ + } \ + (x) = (__force __typeof__(*(ptr))) __gu_val; \ + __gu_err; \ +}) + +#define __get_user_check(x, ptr, size) \ +({ \ + long __gu_err = -EFAULT; \ + unsigned long __gu_val = 0; \ + const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ + if (__access_ok(__gu_addr, size)) { \ + __gu_err = 0; \ + switch (size) { \ + case 1: \ + __get_user_8(__gu_addr); \ + break; \ + case 2: \ + __get_user_16(__gu_addr); \ + break; \ + case 4: \ + __get_user_32(__gu_addr); \ + break; \ + case 8: \ + __get_user_64(__gu_addr); \ + break; \ + default: \ + __get_user_unknown(); \ + break; \ + } \ + } \ + (x) = (__force __typeof__(*(ptr))) __gu_val; \ + __gu_err; \ +}) + +struct __large_struct { unsigned long buf[100]; }; +#define __m(x) (*(struct __large_struct __user *)(x)) + +#define __get_user_64(addr) \ + __asm__("1: ldl %0,%2\n" \ + "2:\n" \ + ".section __ex_table,\"a\"\n" \ + " .long 1b - .\n" \ + " ldi %0, 2b-1b(%1)\n" \ + ".previous" \ + : "=r"(__gu_val), "=r"(__gu_err) \ + : "m"(__m(addr)), "1"(__gu_err)) + +#define __get_user_32(addr) \ + __asm__("1: ldw %0,%2\n" \ + "2:\n" \ + ".section __ex_table,\"a\"\n" \ + " .long 1b - .\n" \ + " ldi %0, 2b-1b(%1)\n" \ + ".previous" \ + : "=r"(__gu_val), "=r"(__gu_err) \ + : "m"(__m(addr)), "1"(__gu_err)) + +#define __get_user_16(addr) \ + __asm__("1: ldhu %0,%2\n" \ + "2:\n" \ + ".section __ex_table,\"a\"\n" \ + " .long 1b - .\n" \ + " ldi %0, 2b-1b(%1)\n" \ + ".previous" \ + : "=r"(__gu_val), "=r"(__gu_err) \ + : "m"(__m(addr)), "1"(__gu_err)) + +#define __get_user_8(addr) \ + __asm__("1: ldbu %0,%2\n" \ + "2:\n" \ + ".section __ex_table,\"a\"\n" \ + " .long 1b - .\n" \ + " ldi %0, 2b-1b(%1)\n" \ + ".previous" \ + : "=r"(__gu_val), "=r"(__gu_err) \ + : "m"(__m(addr)), "1"(__gu_err)) + +extern void __put_user_unknown(void); + +#define __put_user_nocheck(x, ptr, size) \ +({ \ + long __pu_err = 0; \ + __chk_user_ptr(ptr); \ + switch (size) { \ + case 1: \ + __put_user_8(x, ptr); \ + break; \ + case 2: \ + __put_user_16(x, ptr); \ + break; \ + case 4: \ + __put_user_32(x, ptr); \ + break; \ + case 8: \ + __put_user_64(x, ptr); \ + break; \ + default: \ + __put_user_unknown(); \ + break; \ + } \ + __pu_err; \ +}) + +#define __put_user_check(x, ptr, size) \ +({ \ + long __pu_err = -EFAULT; \ + __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ + if (__access_ok(__pu_addr, size)) { \ + __pu_err = 0; \ + switch (size) { \ + case 1: \ + __put_user_8(x, __pu_addr); \ + break; \ + case 2: \ + __put_user_16(x, __pu_addr); \ + break; \ + case 4: \ + __put_user_32(x, __pu_addr); \ + break; \ + case 8: \ + __put_user_64(x, __pu_addr); \ + break; \ + default: \ + __put_user_unknown(); \ + break; \ + } \ + } \ + __pu_err; \ +}) + +/* + * The "__put_user_xx()" macros tell gcc they read from memory + * instead of writing: this is because they do not write to + * any memory gcc knows about, so there are no aliasing issues + */ +#define __put_user_64(x, addr) \ +__asm__ __volatile__("1: stl %r2, %1\n" \ + "2:\n" \ + ".section __ex_table, \"a\"\n" \ + " .long 1b - .\n" \ + " ldi $31, 2b-1b(%0)\n" \ + ".previous" \ + : "=r"(__pu_err) \ + : "m" (__m(addr)), "rJ" (x), "0"(__pu_err)) + +#define __put_user_32(x, addr) \ +__asm__ __volatile__("1: stw %r2, %1\n" \ + "2:\n" \ + ".section __ex_table, \"a\"\n" \ + " .long 1b - .\n" \ + " ldi $31, 2b-1b(%0)\n" \ + ".previous" \ + : "=r"(__pu_err) \ + : "m"(__m(addr)), "rJ"(x), "0"(__pu_err)) + +#define __put_user_16(x, addr) \ +__asm__ __volatile__("1: sth %r2, %1\n" \ + "2:\n" \ + ".section __ex_table, \"a\"\n" \ + " .long 1b - .\n" \ + " ldi $31, 2b-1b(%0)\n" \ + ".previous" \ + : "=r"(__pu_err) \ + : "m"(__m(addr)), "rJ"(x), "0"(__pu_err)) + +#define __put_user_8(x, addr) \ +__asm__ __volatile__("1: stb %r2, %1\n" \ + "2:\n" \ + ".section __ex_table, \"a\"\n" \ + " .long 1b - .\n" \ + " ldi $31, 2b-1b(%0)\n" \ + ".previous" \ + : "=r"(__pu_err) \ + : "m"(__m(addr)), "rJ"(x), "0"(__pu_err)) + +/* + * Complex access routines + */ + +extern long __copy_user(void *to, const void *from, long len); + +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long len) +{ + return __copy_user(to, (__force const void *)from, len); +} + +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long len) +{ + return __copy_user((__force void *)to, from, len); +} +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER + +extern long __clear_user(void __user *to, long len); + +static inline long +clear_user(void __user *to, long len) +{ + if (__access_ok(to, len)) + len = __clear_user(to, len); + return len; +} + +#define user_addr_max() (uaccess_kernel() ? ~0UL : TASK_SIZE) + +extern long strncpy_from_user(char *dest, const char __user *src, long count); +extern __must_check long strlen_user(const char __user *str); +extern __must_check long strnlen_user(const char __user *str, long n); + +#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE +struct page; +void memcpy_page_flushcache(char *to, struct page *page, size_t offset, + size_t len); +extern unsigned long __must_check __copy_user_flushcache(void *to, + const void __user *from, unsigned long n); + +static inline int +__copy_from_user_flushcache(void *dst, const void __user *src, unsigned long size) +{ + kasan_check_write(dst, size); + return __copy_user_flushcache(dst, src, size); +} +#endif + +#include +#endif /* _ASM_SW64_UACCESS_H */ diff --git a/arch/sw_64/include/asm/unistd.h b/arch/sw_64/include/asm/unistd.h new file mode 100644 index 000000000000..6d1b8d1e2011 --- /dev/null +++ b/arch/sw_64/include/asm/unistd.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNISTD_H +#define _ASM_SW64_UNISTD_H + +#include + +#define NR_SYSCALLS __NR_syscalls +#define NR_syscalls NR_SYSCALLS + +#define __ARCH_WANT_NEW_STAT +#define __ARCH_WANT_OLD_READDIR +#define __ARCH_WANT_STAT64 +#define __ARCH_WANT_SYS_GETHOSTNAME +#define __ARCH_WANT_SYS_FADVISE64 +#define __ARCH_WANT_SYS_GETPGRP +#define __ARCH_WANT_SYS_OLD_GETRLIMIT +#define __ARCH_WANT_SYS_OLDUMOUNT +#define __ARCH_WANT_SYS_SIGPENDING +#define __ARCH_WANT_SYS_UTIME +#define __ARCH_WANT_SYS_FORK +#define __ARCH_WANT_SYS_VFORK +#define __ARCH_WANT_SYS_CLONE +#define __ARCH_WANT_SYS_SOCKETCALL +#define __ARCH_WANT_SYS_SIGPROCMASK +#define __ARCH_WANT_SYS_CLONE3 + +#endif /* _ASM_SW64_UNISTD_H */ diff --git a/arch/sw_64/include/uapi/asm/unistd.h b/arch/sw_64/include/uapi/asm/unistd.h new file mode 100644 index 000000000000..be844b2be9d5 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/unistd.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_UNISTD_H +#define _UAPI_ASM_SW64_UNISTD_H + +/* + * These are traditionally the names uses for generic system calls + */ +#define __NR_umount __NR_umount2 + +#include + +#endif /* _UAPI_ASM_SW64_UNISTD_H */ diff --git a/arch/sw_64/kernel/sys_sw64.c b/arch/sw_64/kernel/sys_sw64.c new file mode 100644 index 000000000000..d0198aef554d --- /dev/null +++ b/arch/sw_64/kernel/sys_sw64.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +SYSCALL_DEFINE5(getsysinfo, unsigned long, op, void __user *, buffer, + unsigned long, nbytes, int __user *, start, void __user *, arg) +{ + unsigned long w; + + switch (op) { + case GSI_IEEE_FP_CONTROL: + /* Return current software fp control & status bits. */ + /* Note that DU doesn't verify available space here. */ + + w = current_thread_info()->ieee_state & IEEE_SW_MASK; + w = swcr_update_status(w, rdfpcr()); + if (put_user(w, (unsigned long __user *) buffer)) + return -EFAULT; + return 0; + default: + break; + } + + return -EOPNOTSUPP; +} + +SYSCALL_DEFINE5(setsysinfo, unsigned long, op, void __user *, buffer, + unsigned long, nbytes, int __user *, start, void __user *, arg) +{ + switch (op) { + case SSI_IEEE_FP_CONTROL: { + unsigned long swcr, fpcr; + unsigned int *state; + + /* + * Sw_64 Architecture Handbook 4.7.7.3: + * To be fully IEEE compiant, we must track the current IEEE + * exception state in software, because spurious bits can be + * set in the trap shadow of a software-complete insn. + */ + + if (get_user(swcr, (unsigned long __user *)buffer)) + return -EFAULT; + state = ¤t_thread_info()->ieee_state; + + /* Update softare trap enable bits. */ + *state = (*state & ~IEEE_SW_MASK) | (swcr & IEEE_SW_MASK); + + /* Update the real fpcr. */ + fpcr = rdfpcr() & FPCR_DYN_MASK; + fpcr |= ieee_swcr_to_fpcr(swcr); + wrfpcr(fpcr); + + return 0; + } + + case SSI_IEEE_RAISE_EXCEPTION: { + unsigned long exc, swcr, fpcr, fex; + unsigned int *state; + + if (get_user(exc, (unsigned long __user *)buffer)) + return -EFAULT; + state = ¤t_thread_info()->ieee_state; + exc &= IEEE_STATUS_MASK; + + /* Update softare trap enable bits. */ + swcr = (*state & IEEE_SW_MASK) | exc; + *state |= exc; + + /* Update the real fpcr. */ + fpcr = rdfpcr(); + fpcr |= ieee_swcr_to_fpcr(swcr); + wrfpcr(fpcr); + + /* If any exceptions set by this call, and are unmasked, + * send a signal. Old exceptions are not signaled. + */ + fex = (exc >> IEEE_STATUS_TO_EXCSUM_SHIFT) & swcr; + if (fex) { + int si_code = FPE_FLTUNK; + + if (fex & IEEE_TRAP_ENABLE_DNO) + si_code = FPE_FLTUND; + if (fex & IEEE_TRAP_ENABLE_INE) + si_code = FPE_FLTRES; + if (fex & IEEE_TRAP_ENABLE_UNF) + si_code = FPE_FLTUND; + if (fex & IEEE_TRAP_ENABLE_OVF) + si_code = FPE_FLTOVF; + if (fex & IEEE_TRAP_ENABLE_DZE) + si_code = FPE_FLTDIV; + if (fex & IEEE_TRAP_ENABLE_INV) + si_code = FPE_FLTINV; + + send_sig_fault(SIGFPE, si_code, (void __user *)NULL, current); + } + return 0; + } + default: + break; + } + + return -EOPNOTSUPP; +} + +SYSCALL_DEFINE2(odd_getpriority, int, which, int, who) +{ + int prio = sys_getpriority(which, who); + + if (prio >= 0) { + /* Return value is the unbiased priority, i.e. 20 - prio. + * This does result in negative return values, so signal + * no error. + */ + force_successful_syscall_return(); + prio = 20 - prio; + } + return prio; +} + +SYSCALL_DEFINE0(getxuid) +{ + current_pt_regs()->regs[20] = sys_geteuid(); + return sys_getuid(); +} + +SYSCALL_DEFINE0(getxgid) +{ + current_pt_regs()->regs[20] = sys_getegid(); + return sys_getgid(); +} + +SYSCALL_DEFINE0(getxpid) +{ + current_pt_regs()->regs[20] = sys_getppid(); + return sys_getpid(); +} + +SYSCALL_DEFINE0(sw64_pipe) +{ + int fd[2]; + int res = do_pipe_flags(fd, 0); + + if (!res) { + /* The return values are in $0 and $20. */ + current_pt_regs()->regs[20] = fd[1]; + res = fd[0]; + } + return res; +} diff --git a/arch/sw_64/kernel/syscalls/Makefile b/arch/sw_64/kernel/syscalls/Makefile new file mode 100644 index 000000000000..cdfe761d7282 --- /dev/null +++ b/arch/sw_64/kernel/syscalls/Makefile @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0 +kapi := arch/$(SRCARCH)/include/generated/asm +uapi := arch/$(SRCARCH)/include/generated/uapi/asm + +$(shell mkdir -p $(uapi) $(kapi)) + +syscall := $(src)/syscall.tbl +syshdr := $(srctree)/scripts/syscallhdr.sh +systbl := $(srctree)/scripts/syscalltbl.sh + +quiet_cmd_syshdr = SYSHDR $@ + cmd_syshdr = $(CONFIG_SHELL) $(syshdr) --emit-nr $< $@ + +quiet_cmd_systbl = SYSTBL $@ + cmd_systbl = $(CONFIG_SHELL) $(systbl) $< $@ + +$(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE + $(call if_changed,syshdr) + +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE + $(call if_changed,systbl) + +uapisyshdr-y += unistd_64.h +kapisyshdr-y += syscall_table.h + +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) + +PHONY += all +all: $(uapisyshdr-y) $(kapisyshdr-y) + @: diff --git a/arch/sw_64/kernel/syscalls/syscall.tbl b/arch/sw_64/kernel/syscalls/syscall.tbl new file mode 100644 index 000000000000..fdf9e4cb03eb --- /dev/null +++ b/arch/sw_64/kernel/syscalls/syscall.tbl @@ -0,0 +1,528 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# system call numbers and entry vectors for sw64 +# +# The format is: +# +# +# The is always "common" for this file +# +#0 is unused +1 common exit sys_exit +2 common fork sys_fork +3 common read sys_read +4 common write sys_write +#5 is unused +6 common close sys_close +#7 is unused +#8 is unused +9 common link sys_link +10 common unlink sys_unlink +#11 is unused +12 common chdir sys_chdir +13 common fchdir sys_fchdir +14 common mknod sys_mknod +15 common chmod sys_chmod +16 common chown sys_chown +17 common brk sys_brk +#18 is unused +19 common lseek sys_lseek +20 common getxpid sys_getxpid +#21 is unused +22 common umount2 sys_umount +23 common setuid sys_setuid +24 common getxuid sys_getxuid +#25 is unused +26 common ptrace sys_ptrace +#27 is unused +#28 is unused +#29 is unused +#30 is unused +#31 is unused +#32 is unused +33 common access sys_access +#34 is unused +#35 is unused +36 common sync sys_sync +37 common kill sys_kill +#38 is unused +39 common setpgid sys_setpgid +#40 is unused +41 common dup sys_dup +42 common pipe sys_sw64_pipe +#43 is unused +#44 is unused +45 common open sys_open +#46 is unused +47 common getxgid sys_getxgid +48 common odd_sigprocmask sys_odd_sigprocmask +#49 is unused +#50 is unused +51 common acct sys_acct +52 common sigpending sys_sigpending +#53 is unused +54 common ioctl sys_ioctl +#55 is unused +#56 is unused +57 common symlink sys_symlink +58 common readlink sys_readlink +59 common execve sys_execve +60 common umask sys_umask +61 common chroot sys_chroot +#62 is unused +63 common getpgrp sys_getpgrp +#64 is unused +#65 is unused +66 common vfork sys_vfork +67 common stat sys_newstat +68 common lstat sys_newlstat +#69 is unused +#70 is unused +71 common mmap sys_mmap +#72 is unused +73 common munmap sys_munmap +74 common mprotect sys_mprotect +75 common madvise sys_madvise +76 common vhangup sys_vhangup +#77 is unused +#78 is unused +79 common getgroups sys_getgroups +80 common setgroups sys_setgroups +#81 is unused +82 common setpgrp sys_setpgid +#83 is unused +#84 is unused +#85 is unused +#86 is unused +87 common gethostname sys_gethostname +88 common sethostname sys_sethostname +#89 is unused +90 common dup2 sys_dup2 +91 common fstat sys_newfstat +92 common fcntl sys_fcntl +#93 is unused +94 common poll sys_poll +95 common fsync sys_fsync +96 common setpriority sys_setpriority +97 common socket sys_socket +98 common connect sys_connect +99 common accept sys_accept +100 common odd_getpriority sys_odd_getpriority +101 common send sys_send +102 common recv sys_recv +103 common sigreturn sys_sigreturn +104 common bind sys_bind +105 common setsockopt sys_setsockopt +106 common listen sys_listen +#107 is unused +#108 is unused +#109 is unused +#110 is unused +111 common sigsuspend sys_sigsuspend +#112 is unused +113 common recvmsg sys_recvmsg +114 common sendmsg sys_sendmsg +#115 is unused +#116 is unused +#117 is unused +118 common getsockopt sys_getsockopt +119 common socketcall sys_socketcall +120 common readv sys_readv +121 common writev sys_writev +#122 is unused +123 common fchown sys_fchown +124 common fchmod sys_fchmod +125 common recvfrom sys_recvfrom +126 common setreuid sys_setreuid +127 common setregid sys_setregid +128 common rename sys_rename +129 common truncate sys_truncate +130 common ftruncate sys_ftruncate +131 common flock sys_flock +132 common setgid sys_setgid +133 common sendto sys_sendto +134 common shutdown sys_shutdown +135 common socketpair sys_socketpair +136 common mkdir sys_mkdir +137 common rmdir sys_rmdir +#138 is unused +#139 is unused +#140 is unused +141 common getpeername sys_getpeername +#142 is unused +#143 is unused +144 common getrlimit sys_getrlimit +145 common setrlimit sys_setrlimit +#146 is unused +147 common setsid sys_setsid +148 common quotactl sys_quotactl +#149 is unused +150 common getsockname sys_getsockname +#151 is unused +#152 is unused +#153 is unused +#154 is unused +#155 is unused +156 common sigaction sys_odd_sigaction +#157 is unused +#158 is unused +#159 is unused +#160 is unused +#161 is unused +#162 is unused +#163 is unused +#164 is unused +#165 is unused +166 common setdomainname sys_setdomainname +#167 is unused +#168 is unused +#169 is unused +170 common bpf sys_bpf +171 common userfaultfd sys_userfaultfd +172 common membarrier sys_membarrier +173 common mlock2 sys_mlock2 +174 common getpid sys_getpid +175 common getppid sys_getppid +176 common getuid sys_getuid +177 common geteuid sys_geteuid +178 common getgid sys_getgid +179 common getegid sys_getegid +180 common epoll_pwait2 sys_epoll_pwait2 +181 common mount_setattr sys_mount_setattr +182 common quotactl_fd sys_quotactl_fd +183 common landlock_create_ruleset sys_landlock_create_ruleset +184 common landlock_add_rule sys_landlock_add_rule +185 common landlock_restrict_self sys_landlock_restrict_self +# 186 reserved for memfd_secret +187 common process_mrelease sys_process_mrelease +188 common futex_waitv sys_futex_waitv +189 common set_mempolicy_home_node sys_ni_syscall +190 common cachestat sys_cachestat +191 common fchmodat2 sys_fchmodat2 +#192 is unused +#193 is unused +#194 is unused +#195 is unused +#196 is unused +#197 is unused +#198 is unused +#199 is unused +200 common msgctl sys_old_msgctl +201 common msgget sys_msgget +202 common msgrcv sys_msgrcv +203 common msgsnd sys_msgsnd +204 common semctl sys_old_semctl +205 common semget sys_semget +206 common semop sys_semop +#207 is unused +208 common lchown sys_lchown +209 common shmat sys_shmat +210 common shmctl sys_old_shmctl +211 common shmdt sys_shmdt +212 common shmget sys_shmget +#213 is unused +#214 is unused +#215 is unused +#216 is unused +217 common msync sys_msync +#218 is unused +#219 is unused +#220 is unused +#221 is unused +#222 is unused +#223 is unused +#224 is unused +#225 is unused +#226 is unused +#227 is unused +#228 is unused +229 common statfs64 sys_statfs64 +230 common fstatfs64 sys_fstatfs64 +#231 is unused +#232 is unused +233 common getpgid sys_getpgid +234 common getsid sys_getsid +235 common sigaltstack sys_sigaltstack +#236 is unused +#237 is unused +#238 is unused +#239 is unused +#240 is unused +#241 is unused +#242 is unused +#243 is unused +#244 is unused +#245 is unused +#246 is unused +#247 is unused +#248 is unused +#249 is unused +#250 is unused +#251 is unused +#252 is unused +#253 is unused +254 common sysfs sys_sysfs +#255 is unused +256 common getsysinfo sys_getsysinfo +257 common setsysinfo sys_setsysinfo +#258 is unused +#259 is unused +#260 is unused +#261 is unused +#262 is unused +#263 is unused +#264 is unused +#265 is unused +#266 is unused +#267 is unused +#268 is unused +#269 is unused +#270 is unused +271 common pidfd_send_signal sys_pidfd_send_signal +272 common io_uring_setup sys_io_uring_setup +273 common io_uring_enter sys_io_uring_enter +274 common io_uring_register sys_io_uring_register +275 common open_tree sys_open_tree +276 common move_mount sys_move_mount +277 common fsopen sys_fsopen +278 common fsconfig sys_fsconfig +279 common fsmount sys_fsmount +280 common fspick sys_fspick +281 common pidfd_open sys_pidfd_open +282 common clone3 sys_clone3 +283 common close_range sys_close_range +284 common openat2 sys_openat2 +285 common pidfd_getfd sys_pidfd_getfd +286 common faccessat2 sys_faccessat2 +287 common process_madvise sys_process_madvise +288 common pkey_mprotect sys_pkey_mprotect +289 common pkey_alloc sys_pkey_alloc +290 common pkey_free sys_pkey_free +#291 is unused +#292 is unused +#293 is unused +#294 is unused +#295 is unused +#296 is unused +#297 is unused +298 common getpriority sys_getpriority +299 common sigprocmask sys_sigprocmask +300 common bdflush sys_ni_syscall +#301 is unused +302 common mount sys_mount +#303 is unused +304 common swapoff sys_swapoff +305 common getdents sys_getdents +306 common create_module sys_ni_syscall +307 common init_module sys_init_module +308 common delete_module sys_delete_module +309 common get_kernel_syms sys_ni_syscall +310 common syslog sys_syslog +311 common reboot sys_reboot +312 common clone sys_clone +313 common uselib sys_uselib +314 common mlock sys_mlock +315 common munlock sys_munlock +316 common mlockall sys_mlockall +317 common munlockall sys_munlockall +318 common sysinfo sys_sysinfo +#319 is unused +#320 is unused +321 common oldumount sys_oldumount +322 common swapon sys_swapon +323 common times sys_times +324 common personality sys_personality +325 common setfsuid sys_setfsuid +326 common setfsgid sys_setfsgid +327 common ustat sys_ustat +328 common statfs sys_statfs +329 common fstatfs sys_fstatfs +330 common sched_setparam sys_sched_setparam +331 common sched_getparam sys_sched_getparam +332 common sched_setscheduler sys_sched_setscheduler +333 common sched_getscheduler sys_sched_getscheduler +334 common sched_yield sys_sched_yield +335 common sched_get_priority_max sys_sched_get_priority_max +336 common sched_get_priority_min sys_sched_get_priority_min +337 common sched_rr_get_interval sys_sched_rr_get_interval +338 common afs_syscall sys_ni_syscall +339 common uname sys_newuname +340 common nanosleep sys_nanosleep +341 common mremap sys_mremap +342 common nfsservctl sys_ni_syscall +343 common setresuid sys_setresuid +344 common getresuid sys_getresuid +345 common pciconfig_read sys_pciconfig_read +346 common pciconfig_write sys_pciconfig_write +347 common query_module sys_ni_syscall +348 common prctl sys_prctl +349 common pread64 sys_pread64 +350 common pwrite64 sys_pwrite64 +351 common rt_sigreturn sys_rt_sigreturn +352 common rt_sigaction sys_rt_sigaction +353 common rt_sigprocmask sys_rt_sigprocmask +354 common rt_sigpending sys_rt_sigpending +355 common rt_sigtimedwait sys_rt_sigtimedwait +356 common rt_sigqueueinfo sys_rt_sigqueueinfo +357 common rt_sigsuspend sys_rt_sigsuspend +358 common select sys_select +359 common gettimeofday sys_gettimeofday +360 common settimeofday sys_settimeofday +361 common getitimer sys_getitimer +362 common setitimer sys_setitimer +363 common utimes sys_utimes +364 common getrusage sys_getrusage +365 common wait4 sys_wait4 +366 common adjtimex sys_adjtimex +367 common getcwd sys_getcwd +368 common capget sys_capget +369 common capset sys_capset +370 common sendfile sys_sendfile64 +371 common setresgid sys_setresgid +372 common getresgid sys_getresgid +373 common dipc sys_ni_syscall +374 common pivot_root sys_pivot_root +375 common mincore sys_mincore +376 common pciconfig_iobase sys_pciconfig_iobase +377 common getdents64 sys_getdents64 +378 common gettid sys_gettid +379 common readahead sys_readahead +#380 is unused +381 common tkill sys_tkill +382 common setxattr sys_setxattr +383 common lsetxattr sys_lsetxattr +384 common fsetxattr sys_fsetxattr +385 common getxattr sys_getxattr +386 common lgetxattr sys_lgetxattr +387 common fgetxattr sys_fgetxattr +388 common listxattr sys_listxattr +389 common llistxattr sys_llistxattr +390 common flistxattr sys_flistxattr +391 common removexattr sys_removexattr +392 common lremovexattr sys_lremovexattr +393 common fremovexattr sys_fremovexattr +394 common futex sys_futex +395 common sched_setaffinity sys_sched_setaffinity +396 common sched_getaffinity sys_sched_getaffinity +397 common tuxcall sys_ni_syscall +398 common io_setup sys_io_setup +399 common io_destroy sys_io_destroy +400 common io_getevents sys_io_getevents +401 common io_submit sys_io_submit +402 common io_cancel sys_io_cancel +403 common io_pgetevents sys_io_pgetevents +404 common rseq sys_rseq +405 common exit_group sys_exit_group +406 common lookup_dcookie sys_lookup_dcookie +407 common epoll_create sys_epoll_create +408 common epoll_ctl sys_epoll_ctl +409 common epoll_wait sys_epoll_wait +410 common remap_file_pages sys_remap_file_pages +411 common set_tid_address sys_set_tid_address +412 common restart_syscall sys_restart_syscall +413 common fadvise64 sys_fadvise64 +414 common timer_create sys_timer_create +415 common timer_settime sys_timer_settime +416 common timer_gettime sys_timer_gettime +417 common timer_getoverrun sys_timer_getoverrun +418 common timer_delete sys_timer_delete +419 common clock_settime sys_clock_settime +420 common clock_gettime sys_clock_gettime +421 common clock_getres sys_clock_getres +422 common clock_nanosleep sys_clock_nanosleep +423 common semtimedop sys_semtimedop +424 common tgkill sys_tgkill +425 common stat64 sys_stat64 +426 common lstat64 sys_lstat64 +427 common fstat64 sys_fstat64 +428 common vserver sys_ni_syscall +429 common mbind sys_mbind +430 common get_mempolicy sys_get_mempolicy +431 common set_mempolicy sys_set_mempolicy +432 common mq_open sys_mq_open +433 common mq_unlink sys_mq_unlink +434 common mq_timedsend sys_mq_timedsend +435 common mq_timedreceive sys_mq_timedreceive +436 common mq_notify sys_mq_notify +437 common mq_getsetattr sys_mq_getsetattr +438 common waitid sys_waitid +439 common add_key sys_add_key +440 common request_key sys_request_key +441 common keyctl sys_keyctl +442 common ioprio_set sys_ioprio_set +443 common ioprio_get sys_ioprio_get +444 common inotify_init sys_inotify_init +445 common inotify_add_watch sys_inotify_add_watch +446 common inotify_rm_watch sys_inotify_rm_watch +447 common fdatasync sys_fdatasync +448 common kexec_load sys_kexec_load +449 common migrate_pages sys_migrate_pages +450 common openat sys_openat +451 common mkdirat sys_mkdirat +452 common mknodat sys_mknodat +453 common fchownat sys_fchownat +454 common futimesat sys_futimesat +455 common fstatat64 sys_fstatat64 +456 common unlinkat sys_unlinkat +457 common renameat sys_renameat +458 common linkat sys_linkat +459 common symlinkat sys_symlinkat +460 common readlinkat sys_readlinkat +461 common fchmodat sys_fchmodat +462 common faccessat sys_faccessat +463 common pselect6 sys_pselect6 +464 common ppoll sys_ppoll +465 common unshare sys_unshare +466 common set_robust_list sys_set_robust_list +467 common get_robust_list sys_get_robust_list +468 common splice sys_splice +469 common sync_file_range sys_sync_file_range +470 common tee sys_tee +471 common vmsplice sys_vmsplice +472 common move_pages sys_move_pages +473 common getcpu sys_getcpu +474 common epoll_pwait sys_epoll_pwait +475 common utimensat sys_utimensat +476 common signalfd sys_signalfd +477 common timerfd sys_ni_syscall +478 common eventfd sys_eventfd +479 common recvmmsg sys_recvmmsg +480 common fallocate sys_fallocate +481 common timerfd_create sys_timerfd_create +482 common timerfd_settime sys_timerfd_settime +483 common timerfd_gettime sys_timerfd_gettime +484 common signalfd4 sys_signalfd4 +485 common eventfd2 sys_eventfd2 +486 common epoll_create1 sys_epoll_create1 +487 common dup3 sys_dup3 +488 common pipe2 sys_pipe2 +489 common inotify_init1 sys_inotify_init1 +490 common preadv sys_preadv +491 common pwritev sys_pwritev +492 common rt_tgsigqueueinfo sys_rt_tgsigqueueinfo +493 common perf_event_open sys_perf_event_open +494 common fanotify_init sys_fanotify_init +495 common fanotify_mark sys_fanotify_mark +496 common prlimit64 sys_prlimit64 +497 common name_to_handle_at sys_name_to_handle_at +498 common open_by_handle_at sys_open_by_handle_at +499 common clock_adjtime sys_clock_adjtime +500 common syncfs sys_syncfs +501 common setns sys_setns +502 common accept4 sys_accept4 +503 common sendmmsg sys_sendmmsg +504 common process_vm_readv sys_process_vm_readv +505 common process_vm_writev sys_process_vm_writev +506 common kcmp sys_kcmp +507 common finit_module sys_finit_module +508 common sched_setattr sys_sched_setattr +509 common sched_getattr sys_sched_getattr +510 common renameat2 sys_renameat2 +511 common getrandom sys_getrandom +512 common memfd_create sys_memfd_create +513 common execveat sys_execveat +514 common seccomp sys_seccomp +515 common copy_file_range sys_copy_file_range +516 common preadv2 sys_preadv2 +517 common pwritev2 sys_pwritev2 +518 common statx sys_statx diff --git a/arch/sw_64/kernel/systbls.S b/arch/sw_64/kernel/systbls.S new file mode 100644 index 000000000000..010ca3f8e016 --- /dev/null +++ b/arch/sw_64/kernel/systbls.S @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arch/sw_64/kernel/systbls.S + * + * The system call table. + */ + +#include + +#define __SYSCALL(nr, entry) .quad entry + .data + .align 3 + .globl sys_call_table +sys_call_table: +#include -- Gitee From 84cf6884467f20796f39301f7b613e0f2b00e5e5 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:34 +0800 Subject: [PATCH 017/524] sw64: add signal handling support Add ucontext/sigcontext definition and signal handling support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/signal.h | 28 ++ arch/sw_64/include/uapi/asm/sigcontext.h | 34 ++ arch/sw_64/include/uapi/asm/siginfo.h | 10 + arch/sw_64/include/uapi/asm/signal.h | 119 +++++++ arch/sw_64/include/uapi/asm/ucontext.h | 14 + arch/sw_64/kernel/signal.c | 378 +++++++++++++++++++++++ 6 files changed, 583 insertions(+) create mode 100644 arch/sw_64/include/asm/signal.h create mode 100644 arch/sw_64/include/uapi/asm/sigcontext.h create mode 100644 arch/sw_64/include/uapi/asm/siginfo.h create mode 100644 arch/sw_64/include/uapi/asm/signal.h create mode 100644 arch/sw_64/include/uapi/asm/ucontext.h create mode 100644 arch/sw_64/kernel/signal.c diff --git a/arch/sw_64/include/asm/signal.h b/arch/sw_64/include/asm/signal.h new file mode 100644 index 000000000000..4dc3b6510b86 --- /dev/null +++ b/arch/sw_64/include/asm/signal.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SIGNAL_H +#define _ASM_SW64_SIGNAL_H + +#include + +/* Digital Unix defines 64 signals. Most things should be clean enough + * to redefine this at will, if care is taken to make libc match. + */ + +#define _NSIG 64 +#define _NSIG_BPW 64 +#define _NSIG_WORDS (_NSIG / _NSIG_BPW) + +typedef unsigned long old_sigset_t; /* at least 32 bits */ + +typedef struct { + unsigned long sig[_NSIG_WORDS]; +} sigset_t; + +struct odd_sigaction { + __sighandler_t sa_handler; + old_sigset_t sa_mask; + int sa_flags; +}; + +#include +#endif /* _ASM_SW64_SIGNAL_H */ diff --git a/arch/sw_64/include/uapi/asm/sigcontext.h b/arch/sw_64/include/uapi/asm/sigcontext.h new file mode 100644 index 000000000000..08a081470383 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/sigcontext.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_SIGCONTEXT_H +#define _UAPI_ASM_SW64_SIGCONTEXT_H + +/* + * Signal context structure + * + * The context is saved before a signal handler is invoked, and it is + * restored by sys_sigreturn / sys_rt_sigreturn. + */ +struct sigcontext { + long sc_onstack; + long sc_mask; + long sc_pc; + long sc_ps; + long sc_regs[32]; + long sc_ownedfp; + long sc_fpregs[128]; /* SIMD-FP */ + unsigned long sc_fpcr; + /* TODO: Following are unused, to be removed and synced with libc */ + unsigned long sc_fp_control; + unsigned long sc_reserved1, sc_reserved2; + unsigned long sc_ssize; + char *sc_sbase; + unsigned long sc_traparg_a0; + unsigned long sc_traparg_a1; + unsigned long sc_traparg_a2; + unsigned long sc_fp_trap_pc; + unsigned long sc_fp_trigger_sum; + unsigned long sc_fp_trigger_inst; +}; + + +#endif /* _UAPI_ASM_SW64_SIGCONTEXT_H */ diff --git a/arch/sw_64/include/uapi/asm/siginfo.h b/arch/sw_64/include/uapi/asm/siginfo.h new file mode 100644 index 000000000000..f47fb917c9b2 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/siginfo.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_SIGINFO_H +#define _UAPI_ASM_SW64_SIGINFO_H + +#define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) + +#include + + +#endif /* _UAPI_ASM_SW64_SIGINFO_H */ diff --git a/arch/sw_64/include/uapi/asm/signal.h b/arch/sw_64/include/uapi/asm/signal.h new file mode 100644 index 000000000000..0d7a935fe37c --- /dev/null +++ b/arch/sw_64/include/uapi/asm/signal.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_SIGNAL_H +#define _UAPI_ASM_SW64_SIGNAL_H + +#include + +/* Avoid too many header ordering problems. */ +struct siginfo; + +#ifndef __KERNEL__ +/* Here we must cater to libcs that poke about in kernel headers. */ + +#define NSIG 32 +typedef unsigned long sigset_t; + +#endif /* __KERNEL__ */ + + +/* + * Linux/sw64 different signal numbers that Linux/i386. + */ +#define SIGHUP 1 +#define SIGINT 2 +#define SIGQUIT 3 +#define SIGILL 4 +#define SIGTRAP 5 +#define SIGABRT 6 +#define SIGEMT 7 +#define SIGFPE 8 +#define SIGKILL 9 +#define SIGBUS 10 +#define SIGSEGV 11 +#define SIGSYS 12 +#define SIGPIPE 13 +#define SIGALRM 14 +#define SIGTERM 15 +#define SIGURG 16 +#define SIGSTOP 17 +#define SIGTSTP 18 +#define SIGCONT 19 +#define SIGCHLD 20 +#define SIGTTIN 21 +#define SIGTTOU 22 +#define SIGIO 23 +#define SIGXCPU 24 +#define SIGXFSZ 25 +#define SIGVTALRM 26 +#define SIGPROF 27 +#define SIGWINCH 28 +#define SIGINFO 29 +#define SIGUSR1 30 +#define SIGUSR2 31 + +#define SIGPOLL SIGIO +#define SIGPWR SIGINFO +#define SIGIOT SIGABRT + +/* These should not be considered constants from userland. */ +#define SIGRTMIN 32 +#define SIGRTMAX _NSIG + +/* + * SA_FLAGS values: + * + * SA_ONSTACK indicates that a registered stack_t will be used. + * SA_RESTART flag to get restarting signals (which were the default long ago) + * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop. + * SA_RESETHAND clears the handler when the signal is delivered. + * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies. + * SA_NODEFER prevents the current signal from being masked in the handler. + * + * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single + * Unix names RESETHAND and NODEFER respectively. + */ + +#define SA_ONSTACK 0x00000001 +#define SA_RESTART 0x00000002 +#define SA_NOCLDSTOP 0x00000004 +#define SA_NODEFER 0x00000008 +#define SA_RESETHAND 0x00000010 +#define SA_NOCLDWAIT 0x00000020 +#define SA_SIGINFO 0x00000040 + +#define SA_ONESHOT SA_RESETHAND +#define SA_NOMASK SA_NODEFER + +#define MINSIGSTKSZ 4096 +#define SIGSTKSZ 16384 + +#define SIG_BLOCK 1 /* for blocking signals */ +#define SIG_UNBLOCK 2 /* for unblocking signals */ +#define SIG_SETMASK 3 /* for setting the signal mask */ + +#include + +#ifndef __KERNEL__ +/* Here we must cater to libcs that poke about in kernel headers. */ + +struct sigaction { + union { + __sighandler_t _sa_handler; + void (*_sa_sigaction)(int sig, struct siginfo *info, void *ucontext); + } _u; + sigset_t sa_mask; + int sa_flags; +}; + +#define sa_handler _u._sa_handler +#define sa_sigaction _u._sa_sigaction + +#endif /* __KERNEL__ */ + +typedef struct sigaltstack { + void __user *ss_sp; + int ss_flags; + size_t ss_size; +} stack_t; + +#endif /* _UAPI_ASM_SW64_SIGNAL_H */ diff --git a/arch/sw_64/include/uapi/asm/ucontext.h b/arch/sw_64/include/uapi/asm/ucontext.h new file mode 100644 index 000000000000..c5d6e24e3e5f --- /dev/null +++ b/arch/sw_64/include/uapi/asm/ucontext.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_UCONTEXT_H +#define _UAPI_ASM_SW64_UCONTEXT_H + +struct ucontext { + unsigned long uc_flags; + struct ucontext *uc_link; + old_sigset_t uc_old_sigmask; + stack_t uc_stack; + struct sigcontext uc_mcontext; + sigset_t uc_sigmask; /* mask last for extensibility */ +}; + +#endif /* _UAPI_ASM_SW64_UCONTEXT_H */ diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c new file mode 100644 index 000000000000..496f33bb1c89 --- /dev/null +++ b/arch/sw_64/kernel/signal.c @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/arch/sw_64/kernel/signal.c + * + * Copyright (C) 1995 Linus Torvalds + * + * 1997-11-02 Modified for POSIX.1b signals by Richard Henderson + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "proto.h" + + +#define DEBUG_SIG 0 + +#define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP))) + +SYSCALL_DEFINE2(odd_sigprocmask, int, how, unsigned long, newmask) +{ + sigset_t oldmask; + sigset_t mask; + unsigned long res; + + siginitset(&mask, newmask & _BLOCKABLE); + res = sigprocmask(how, &mask, &oldmask); + if (!res) { + force_successful_syscall_return(); + res = oldmask.sig[0]; + } + return res; +} + +SYSCALL_DEFINE3(odd_sigaction, int, sig, + const struct odd_sigaction __user *, act, + struct odd_sigaction __user *, oact) +{ + struct k_sigaction new_ka, old_ka; + old_sigset_t mask; + int ret; + + if (act) { + if (!access_ok(act, sizeof(*act)) || + __get_user(new_ka.sa.sa_handler, &act->sa_handler) || + __get_user(new_ka.sa.sa_flags, &act->sa_flags) || + __get_user(mask, &act->sa_mask)) + return -EFAULT; + siginitset(&new_ka.sa.sa_mask, mask); + } + + ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); + + if (!ret && oact) { + if (!access_ok(oact, sizeof(*oact)) || + __put_user(old_ka.sa.sa_handler, &oact->sa_handler) || + __put_user(old_ka.sa.sa_flags, &oact->sa_flags) || + __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask)) + return -EFAULT; + } + + return ret; +} + +/* + * Do a signal return; undo the signal stack. + */ + +#if _NSIG_WORDS > 1 +# error "Non SA_SIGINFO frame needs rearranging" +#endif + +struct rt_sigframe { + struct siginfo info; + struct ucontext uc; +}; + +/* + * If this changes, userland unwinders that Know Things about our signal + * frame will break. Do not undertake lightly. It also implies an ABI + * change wrt the size of siginfo_t, which may cause some pain. + */ +extern char compile_time_assert + [offsetof(struct rt_sigframe, uc.uc_mcontext) == 176 ? 1 : -1]; + +static long +restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) +{ + long err = __get_user(regs->pc, &sc->sc_pc); + + err |= __copy_from_user(regs, sc->sc_regs, sizeof_field(struct pt_regs, regs)); + /* simd-fp */ + err |= __copy_from_user(¤t->thread.fpstate, &sc->sc_fpregs, + offsetof(struct user_fpsimd_state, fpcr)); + err |= __get_user(current->thread.fpstate.fpcr, &sc->sc_fpcr); + + if (likely(!err)) + __fpstate_restore(current); + + return err; +} + +/* + * Note that this syscall is also used by setcontext(3) to install + * a given sigcontext. This because it's impossible to set *all* + * registers and transfer control from userland. + */ + +SYSCALL_DEFINE1(sigreturn, struct sigcontext __user *, sc) +{ + struct pt_regs *regs = current_pt_regs(); + sigset_t set; + + force_successful_syscall_return(); + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + /* Verify that it's a good sigcontext before using it */ + if (!access_ok(sc, sizeof(*sc))) + goto give_sigsegv; + if (__get_user(set.sig[0], &sc->sc_mask)) + goto give_sigsegv; + + set_current_blocked(&set); + + if (restore_sigcontext(sc, regs)) + goto give_sigsegv; + + /* Send SIGTRAP if we're single-stepping: */ + if (ptrace_cancel_bpt(current)) { + force_sig_fault(SIGTRAP, TRAP_BRKPT, + (void __user *)regs->pc); + } + return regs->regs[0]; + +give_sigsegv: + force_sig(SIGSEGV); + return 0; +} + +SYSCALL_DEFINE1(rt_sigreturn, struct rt_sigframe __user *, frame) +{ + struct pt_regs *regs = current_pt_regs(); + sigset_t set; + + force_successful_syscall_return(); + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + /* Verify that it's a good ucontext_t before using it */ + if (!access_ok(&frame->uc, sizeof(frame->uc))) + goto give_sigsegv; + if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) + goto give_sigsegv; + + set_current_blocked(&set); + + if (restore_sigcontext(&frame->uc.uc_mcontext, regs)) + goto give_sigsegv; + + if (restore_altstack(&frame->uc.uc_stack)) + goto give_sigsegv; + + /* Send SIGTRAP if we're single-stepping: */ + if (ptrace_cancel_bpt(current)) { + force_sig_fault(SIGTRAP, TRAP_BRKPT, + (void __user *)regs->pc); + } + return regs->regs[0]; + +give_sigsegv: + force_sig(SIGSEGV); + return 0; +} + + +/* + * Set up a signal frame. + */ + +static inline void __user * +get_sigframe(struct ksignal *ksig, unsigned long sp, size_t frame_size) +{ + return (void __user *)((sigsp(sp, ksig) - frame_size) & -32ul); +} + +static long +setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, + unsigned long mask) +{ + long err = 0; + + err |= __put_user(on_sig_stack((unsigned long)sc), &sc->sc_onstack); + err |= __put_user(mask, &sc->sc_mask); + err |= __put_user(regs->pc, &sc->sc_pc); + err |= __put_user(8, &sc->sc_ps); + + err |= __copy_to_user(sc->sc_regs, regs, sizeof_field(struct pt_regs, regs)); + err |= __put_user(0, sc->sc_regs+31); + /* simd-fp */ + __fpstate_save(current); + err |= __copy_to_user(&sc->sc_fpregs, ¤t->thread.fpstate, + offsetof(struct user_fpsimd_state, fpcr)); + err |= __put_user(current->thread.fpstate.fpcr, &sc->sc_fpcr); + + return err; +} + +static int +setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) +{ + unsigned long err = 0; + struct rt_sigframe __user *frame; + + frame = get_sigframe(ksig, regs->regs[30], sizeof(*frame)); + if (!access_ok(frame, sizeof(*frame))) + return -EFAULT; + + if (ksig->ka.sa.sa_flags & SA_SIGINFO) + err |= copy_siginfo_to_user(&frame->info, &ksig->info); + + /* Create the ucontext. */ + err |= __put_user(0, &frame->uc.uc_flags); + err |= __put_user(0, &frame->uc.uc_link); + err |= __put_user(set->sig[0], &frame->uc.uc_old_sigmask); + err |= __save_altstack(&frame->uc.uc_stack, regs->regs[30]); + err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, set->sig[0]); + err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); + if (err) + return -EFAULT; + + /* "Return" to the handler */ + regs->regs[26] = VDSO_SYMBOL(current->mm->context.vdso, rt_sigreturn); + regs->regs[27] = regs->pc = (unsigned long) ksig->ka.sa.sa_handler; + regs->regs[16] = ksig->sig; /* a0: signal number */ + if (ksig->ka.sa.sa_flags & SA_SIGINFO) { + /* a1: siginfo pointer, a2: ucontext pointer */ + regs->regs[17] = (unsigned long) &frame->info; + regs->regs[18] = (unsigned long) &frame->uc; + } else { + /* a1: exception code, a2: sigcontext pointer */ + regs->regs[17] = 0; + regs->regs[18] = (unsigned long) &frame->uc.uc_mcontext; + } + regs->regs[30] = (unsigned long) frame; + +#if DEBUG_SIG + pr_info("SIG deliver (%s:%d): sp=%p pc=%p ra=%p\n", + current->comm, current->pid, frame, regs->pc, regs->regs[26]); +#endif + + return 0; +} + +/* + * OK, we're invoking a handler. + */ +static inline void +handle_signal(struct ksignal *ksig, struct pt_regs *regs) +{ + sigset_t *oldset = sigmask_to_save(); + int ret; + + rseq_signal_deliver(ksig, regs); + + ret = setup_rt_frame(ksig, oldset, regs); + + signal_setup_done(ret, ksig, 0); +} + +/* + * Note that 'init' is a special process: it doesn't get signals it doesn't + * want to handle. Thus you cannot kill init even with a SIGKILL even by + * mistake. + * + * Note that we go through the signals twice: once to check the signals that + * the kernel can handle, and then we build all the user-level signal handling + * stack-frames in one go after that. + */ +static void +do_signal(struct pt_regs *regs) +{ + unsigned long single_stepping = ptrace_cancel_bpt(current); + struct ksignal ksig; + + /* This lets the debugger run, ... */ + if (get_signal(&ksig)) { + /* ... so re-check the single stepping. */ + single_stepping |= ptrace_cancel_bpt(current); + /* Whee! Actually deliver the signal. */ + if (regs->orig_r0 != NO_SYSCALL) { + switch (syscall_get_error(current, regs)) { + case -ERESTARTSYS: + if (!(ksig.ka.sa.sa_flags & SA_RESTART)) { + regs->regs[0] = EINTR; + break; + } + fallthrough; + case -ERESTARTNOINTR: + /* reset v0 and a3 and replay syscall */ + regs->regs[0] = regs->orig_r0; + regs->regs[19] = regs->orig_r19; + regs->pc -= 4; + break; + case -ERESTARTNOHAND: + case -ERESTART_RESTARTBLOCK: + regs->regs[0] = EINTR; + break; + } + regs->orig_r0 = NO_SYSCALL; + } + handle_signal(&ksig, regs); + } else { + single_stepping |= ptrace_cancel_bpt(current); + if (regs->orig_r0 != NO_SYSCALL) { + switch (syscall_get_error(current, regs)) { + case -ERESTARTSYS: + case -ERESTARTNOINTR: + case -ERESTARTNOHAND: + /* Reset v0 and a3 and replay syscall. */ + regs->regs[0] = regs->orig_r0; + regs->regs[19] = regs->orig_r19; + regs->pc -= 4; + break; + case -ERESTART_RESTARTBLOCK: + /* Set v0 to the restart_syscall and replay */ + regs->regs[0] = __NR_restart_syscall; + regs->pc -= 4; + break; + } + regs->orig_r0 = NO_SYSCALL; + } + restore_saved_sigmask(); + } + if (single_stepping) + ptrace_set_bpt(current); /* re-set breakpoint */ +} + +asmlinkage void +do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) +{ + do { + local_irq_enable(); + + if (thread_flags & _TIF_NEED_RESCHED) + schedule(); + + if (thread_flags & _TIF_UPROBE) { + unsigned long pc = regs->pc; + + uprobe_notify_resume(regs); + sw64_fix_uretprobe(regs, pc - 4); + } + + if (thread_flags & _TIF_PATCH_PENDING) + klp_update_patch_state(current); + + if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs); + + if (thread_flags & _TIF_NOTIFY_RESUME) + resume_user_mode_work(regs); + + local_irq_disable(); + thread_flags = READ_ONCE(current_thread_info()->flags); + } while (thread_flags & _TIF_WORK_MASK); +} -- Gitee From 4feba22198f0ccc72ad4e16e8864c4b3adb6adaa Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:08 +0800 Subject: [PATCH 018/524] sw64: add FPU support Add FPU and floating-point emulation support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/fpu.h | 91 ++ arch/sw_64/include/asm/sfp-machine.h | 69 + arch/sw_64/include/uapi/asm/fpu.h | 233 +++ arch/sw_64/kernel/fpu.S | 111 ++ arch/sw_64/math-emu/Makefile | 10 + arch/sw_64/math-emu/math.c | 2255 ++++++++++++++++++++++++++ arch/sw_64/math-emu/qrnnd.S | 133 ++ arch/sw_64/math-emu/sfp-util.h | 41 + 8 files changed, 2943 insertions(+) create mode 100644 arch/sw_64/include/asm/fpu.h create mode 100644 arch/sw_64/include/asm/sfp-machine.h create mode 100644 arch/sw_64/include/uapi/asm/fpu.h create mode 100644 arch/sw_64/kernel/fpu.S create mode 100644 arch/sw_64/math-emu/Makefile create mode 100644 arch/sw_64/math-emu/math.c create mode 100644 arch/sw_64/math-emu/qrnnd.S create mode 100644 arch/sw_64/math-emu/sfp-util.h diff --git a/arch/sw_64/include/asm/fpu.h b/arch/sw_64/include/asm/fpu.h new file mode 100644 index 000000000000..a0b0ff5af168 --- /dev/null +++ b/arch/sw_64/include/asm/fpu.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_FPU_H +#define _ASM_SW64_FPU_H + +#include +#ifdef __KERNEL__ + +/* + * The following two functions don't need trapb/excb instructions + * around the mf_fpcr/mt_fpcr instructions because (a) the kernel + * never generates arithmetic faults and (b) sys_call instructions + * are implied trap barriers. + */ + +static inline unsigned long +rdfpcr(void) +{ + unsigned long ret; + unsigned long fp[4] __aligned(32); + + __asm__ __volatile__ ( + " vstd $f0, %0\n\t" + " rfpcr $f0\n\t" + " fimovd $f0, %1\n\t" + " vldd $f0, %0\n\t" + : "=m"(*fp), "=r"(ret)); + + return ret; +} + +static inline void +wrfpcr(unsigned long val) +{ + unsigned long tmp; + unsigned long fp[4] __aligned(32); + + __asm__ __volatile__ ( + " vstd $f0, %0\n\t" + " ifmovd %2, $f0\n\t" + " wfpcr $f0\n\t" + " and %2, 0x3, %1\n\t" + " beq %1, 1f\n\t" + " subl %1, 1, %1\n\t" + " beq %1, 2f\n\t" + " subl %1, 1, %1\n\t" + " beq %1, 3f\n\t" + " setfpec3\n\t" + " br 6f\n\t" + "1: setfpec0\n\t" + " br 6f\n\t" + "2: setfpec1\n\t" + " br 6f\n\t" + "3: setfpec2\n\t" + "6: vldd $f0, %0\n\t" + : "=m"(*fp), "=&r"(tmp) : "r"(val)); +} + +static inline unsigned long +swcr_update_status(unsigned long swcr, unsigned long fpcr) +{ + /* + * SW64 implements most of the bits in hardware. Collect + * the acrued exception bits from the real fpcr. + */ + swcr &= ~(IEEE_STATUS_MASK0 | IEEE_STATUS_MASK1 + | IEEE_STATUS_MASK2 | IEEE_STATUS_MASK3); + swcr |= (fpcr >> 35) & IEEE_STATUS_MASK0; + swcr |= (fpcr >> 13) & IEEE_STATUS_MASK1; + swcr |= (fpcr << 14) & IEEE_STATUS_MASK2; + swcr |= (fpcr << 36) & IEEE_STATUS_MASK3; + return swcr; +} + +extern unsigned long sw64_read_fp_reg(unsigned long reg); +extern void sw64_write_fp_reg(unsigned long reg, unsigned long val); +extern unsigned long sw64_read_fp_reg_s(unsigned long reg); +extern void sw64_write_fp_reg_s(unsigned long reg, unsigned long val); + + +extern void sw64_write_simd_fp_reg_s(unsigned long reg, + unsigned long f0, unsigned long f1); +extern void sw64_write_simd_fp_reg_d(unsigned long reg, + unsigned long f0, unsigned long f1, + unsigned long f2, unsigned long f3); +extern void sw64_write_simd_fp_reg_ldwe(unsigned long reg, int a); +extern void sw64_read_simd_fp_m_s(unsigned long reg, unsigned long *fp_value); +extern void sw64_read_simd_fp_m_d(unsigned long reg, unsigned long *fp_value); + +#endif /* __KERNEL__ */ + +#endif /* _ASM_SW64_FPU_H */ diff --git a/arch/sw_64/include/asm/sfp-machine.h b/arch/sw_64/include/asm/sfp-machine.h new file mode 100644 index 000000000000..156bebc9c515 --- /dev/null +++ b/arch/sw_64/include/asm/sfp-machine.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Machine-dependent software floating-point definitions. + * sw64 kernel version. + * Copyright (C) 1997,1998,1999 Free Software Foundation, Inc. + * This file is part of the GNU C Library. + * Contributed by Richard Henderson (rth@cygnus.com), + * Jakub Jelinek (jakub@redhat.com) and + * David S. Miller (davem@redhat.com). + */ + +#ifndef _ASM_SW64_SFP_MACHINE_H +#define _ASM_SW64_SFP_MACHINE_H + +#define _FP_W_TYPE_SIZE 64 +#define _FP_W_TYPE unsigned long +#define _FP_WS_TYPE signed long +#define _FP_I_TYPE long + +#define _FP_MUL_MEAT_S(R, X, Y) \ + _FP_MUL_MEAT_1_imm(_FP_WFRACBITS_S, R, X, Y) +#define _FP_MUL_MEAT_D(R, X, Y) \ + _FP_MUL_MEAT_1_wide(_FP_WFRACBITS_D, R, X, Y, umul_ppmm) +#define _FP_MUL_MEAT_Q(R, X, Y) \ + _FP_MUL_MEAT_2_wide(_FP_WFRACBITS_Q, R, X, Y, umul_ppmm) + +#define _FP_DIV_MEAT_S(R, X, Y) _FP_DIV_MEAT_1_imm(S, R, X, Y, _FP_DIV_HELP_imm) +#define _FP_DIV_MEAT_D(R, X, Y) _FP_DIV_MEAT_1_udiv(D, R, X, Y) +#define _FP_DIV_MEAT_Q(R, X, Y) _FP_DIV_MEAT_2_udiv(Q, R, X, Y) + +#define _FP_NANFRAC_S _FP_QNANBIT_S +#define _FP_NANFRAC_D _FP_QNANBIT_D +#define _FP_NANFRAC_Q _FP_QNANBIT_Q +#define _FP_NANSIGN_S 1 +#define _FP_NANSIGN_D 1 +#define _FP_NANSIGN_Q 1 + +#define _FP_KEEPNANFRACP 1 + +/* Sw_64 Architecture Handbook, 4.7.10.4 sais that + * we should prefer any type of NaN in Fb, then Fa. + */ +#define _FP_CHOOSENAN(fs, wc, R, X, Y, OP) \ +do { \ + R##_s = Y##_s; \ + _FP_FRAC_COPY_##wc(R, X); \ + R##_c = FP_CLS_NAN; \ +} while (0) + +/* Obtain the current rounding mode. */ +#define FP_ROUNDMODE mode +#define FP_RND_NEAREST (FPCR_DYN_NORMAL >> FPCR_DYN_SHIFT) +#define FP_RND_ZERO (FPCR_DYN_CHOPPED >> FPCR_DYN_SHIFT) +#define FP_RND_PINF (FPCR_DYN_PLUS >> FPCR_DYN_SHIFT) +#define FP_RND_MINF (FPCR_DYN_MINUS >> FPCR_DYN_SHIFT) + +/* Exception flags. */ +#define FP_EX_INVALID IEEE_TRAP_ENABLE_INV +#define FP_EX_OVERFLOW IEEE_TRAP_ENABLE_OVF +#define FP_EX_UNDERFLOW IEEE_TRAP_ENABLE_UNF +#define FP_EX_DIVZERO IEEE_TRAP_ENABLE_DZE +#define FP_EX_INEXACT IEEE_TRAP_ENABLE_INE +#define FP_EX_DENORM IEEE_TRAP_ENABLE_DNO + +#define FP_DENORM_ZERO (swcr & IEEE_MAP_DMZ) + +/* We write the results always */ +#define FP_INHIBIT_RESULTS 0 + +#endif /* _ASM_SW64_SFP_MACHINE_H */ diff --git a/arch/sw_64/include/uapi/asm/fpu.h b/arch/sw_64/include/uapi/asm/fpu.h new file mode 100644 index 000000000000..8945816c542b --- /dev/null +++ b/arch/sw_64/include/uapi/asm/fpu.h @@ -0,0 +1,233 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_FPU_H +#define _UAPI_ASM_SW64_FPU_H + +/* + * SW-64 floating-point control register defines: + */ +#define FPCR_DNOD (1UL << 47) /* denorm INV trap disable */ +#ifdef CONFIG_SUBARCH_C3B +#define FPCR_DNZ (1UL << 48) /* denorms to zero */ +#else +#define FPCR_DNOE (1UL << 48) /* hardware denormal support */ +#endif +#define FPCR_INVD (1UL << 49) /* invalid op disable (opt.) */ +#define FPCR_DZED (1UL << 50) /* division by zero disable (opt.) */ +#define FPCR_OVFD (1UL << 51) /* overflow disable (optional) */ +#define FPCR_INV (1UL << 52) /* invalid operation */ +#define FPCR_DZE (1UL << 53) /* division by zero */ +#define FPCR_OVF (1UL << 54) /* overflow */ +#define FPCR_UNF (1UL << 55) /* underflow */ +#define FPCR_INE (1UL << 56) /* inexact */ +#define FPCR_IOV (1UL << 57) /* integer overflow */ +#define FPCR_UNDZ (1UL << 60) /* underflow to zero (opt.) */ +#define FPCR_UNFD (1UL << 61) /* underflow disable (opt.) */ +#define FPCR_INED (1UL << 62) /* inexact disable (opt.) */ +#define FPCR_SUM (1UL << 63) /* summary bit */ + +#define FPCR_DYN_SHIFT 58 /* first dynamic rounding mode bit */ +#define FPCR_DYN_CHOPPED (0x0UL << FPCR_DYN_SHIFT) /* towards 0 */ +#define FPCR_DYN_MINUS (0x1UL << FPCR_DYN_SHIFT) /* towards -INF */ +#define FPCR_DYN_NORMAL (0x2UL << FPCR_DYN_SHIFT) /* towards nearest */ +#define FPCR_DYN_PLUS (0x3UL << FPCR_DYN_SHIFT) /* towards +INF */ +#define FPCR_DYN_MASK (0x3UL << FPCR_DYN_SHIFT) + +#define FPCR_MASK 0xffff800000000000L + +#ifdef CONFIG_SUBARCH_C3B +#define FPCR_INIT FPCR_DYN_NORMAL +#else +#define FPCR_INIT (FPCR_DYN_NORMAL | FPCR_DNOE) +#endif + +/* status bit coming from hardware fpcr . definde by fire3 */ +#define FPCR_STATUS_INV0 (1UL << 52) +#define FPCR_STATUS_DZE0 (1UL << 53) +#define FPCR_STATUS_OVF0 (1UL << 54) +#define FPCR_STATUS_UNF0 (1UL << 55) +#define FPCR_STATUS_INE0 (1UL << 56) +#define FPCR_STATUS_OVI0 (1UL << 57) + +#define FPCR_STATUS_INV1 (1UL << 36) +#define FPCR_STATUS_DZE1 (1UL << 37) +#define FPCR_STATUS_OVF1 (1UL << 38) +#define FPCR_STATUS_UNF1 (1UL << 39) +#define FPCR_STATUS_INE1 (1UL << 40) +#define FPCR_STATUS_OVI1 (1UL << 41) + +#define FPCR_STATUS_INV2 (1UL << 20) +#define FPCR_STATUS_DZE2 (1UL << 21) +#define FPCR_STATUS_OVF2 (1UL << 22) +#define FPCR_STATUS_UNF2 (1UL << 23) +#define FPCR_STATUS_INE2 (1UL << 24) +#define FPCR_STATUS_OVI2 (1UL << 25) + +#define FPCR_STATUS_INV3 (1UL << 4) +#define FPCR_STATUS_DZE3 (1UL << 5) +#define FPCR_STATUS_OVF3 (1UL << 6) +#define FPCR_STATUS_UNF3 (1UL << 7) +#define FPCR_STATUS_INE3 (1UL << 8) +#define FPCR_STATUS_OVI3 (1UL << 9) + +#define FPCR_STATUS_MASK0 (FPCR_STATUS_INV0 | FPCR_STATUS_DZE0 | \ + FPCR_STATUS_OVF0 | FPCR_STATUS_UNF0 | \ + FPCR_STATUS_INE0 | FPCR_STATUS_OVI0) + +#define FPCR_STATUS_MASK1 (FPCR_STATUS_INV1 | FPCR_STATUS_DZE1 | \ + FPCR_STATUS_OVF1 | FPCR_STATUS_UNF1 | \ + FPCR_STATUS_INE1 | FPCR_STATUS_OVI1) + +#define FPCR_STATUS_MASK2 (FPCR_STATUS_INV2 | FPCR_STATUS_DZE2 | \ + FPCR_STATUS_OVF2 | FPCR_STATUS_UNF2 | \ + FPCR_STATUS_INE2 | FPCR_STATUS_OVI2) + +#define FPCR_STATUS_MASK3 (FPCR_STATUS_INV3 | FPCR_STATUS_DZE3 | \ + FPCR_STATUS_OVF3 | FPCR_STATUS_UNF3 | \ + FPCR_STATUS_INE3 | FPCR_STATUS_OVI3) + + +/* + * IEEE trap enables are implemented in software. These per-thread + * bits are stored in the "ieee_state" field of "struct thread_info". + * Thus, the bits are defined so as not to conflict with the + * floating-point enable bit (which is architected). + */ +#define IEEE_TRAP_ENABLE_INV (1UL << 1) /* invalid op */ +#define IEEE_TRAP_ENABLE_DZE (1UL << 2) /* division by zero */ +#define IEEE_TRAP_ENABLE_OVF (1UL << 3) /* overflow */ +#define IEEE_TRAP_ENABLE_UNF (1UL << 4) /* underflow */ +#define IEEE_TRAP_ENABLE_INE (1UL << 5) /* inexact */ +#define IEEE_TRAP_ENABLE_DNO (1UL << 6) /* denorm */ +#define IEEE_TRAP_ENABLE_MASK (IEEE_TRAP_ENABLE_INV | IEEE_TRAP_ENABLE_DZE |\ + IEEE_TRAP_ENABLE_OVF | IEEE_TRAP_ENABLE_UNF |\ + IEEE_TRAP_ENABLE_INE | IEEE_TRAP_ENABLE_DNO) + +/* Denorm and Underflow flushing */ +#define IEEE_MAP_DMZ (1UL << 12) /* Map denorm inputs to zero */ +#define IEEE_MAP_UMZ (1UL << 13) /* Map underflowed outputs to zero */ + +#define IEEE_MAP_MASK (IEEE_MAP_DMZ | IEEE_MAP_UMZ) + +/* status bits coming from fpcr: */ +#define IEEE_STATUS_INV (1UL << 17) +#define IEEE_STATUS_DZE (1UL << 18) +#define IEEE_STATUS_OVF (1UL << 19) +#define IEEE_STATUS_UNF (1UL << 20) +#define IEEE_STATUS_INE (1UL << 21) +#define IEEE_STATUS_DNO (1UL << 22) + + +#define IEEE_STATUS_MASK (IEEE_STATUS_INV | IEEE_STATUS_DZE | \ + IEEE_STATUS_OVF | IEEE_STATUS_UNF | \ + IEEE_STATUS_INE | IEEE_STATUS_DNO) + +#define IEEE_SW_MASK (IEEE_TRAP_ENABLE_MASK | \ + IEEE_STATUS_MASK | IEEE_MAP_MASK) + +#define IEEE_CURRENT_RM_SHIFT 32 +#define IEEE_CURRENT_RM_MASK (3UL << IEEE_CURRENT_RM_SHIFT) + +#define IEEE_STATUS_TO_EXCSUM_SHIFT 16 + +#define IEEE_INHERIT (1UL << 63) /* inherit on thread create? */ + +/* ieee_state expand to surport simd added by fire3 */ + +#define IEEE_STATUS_INV0 (1UL << 17) +#define IEEE_STATUS_DZE0 (1UL << 18) +#define IEEE_STATUS_OVF0 (1UL << 19) +#define IEEE_STATUS_UNF0 (1UL << 20) +#define IEEE_STATUS_INE0 (1UL << 21) +#define IEEE_STATUS_DNO0 (1UL << 22) +#define IEEE_STATUS_MASK0 (IEEE_STATUS_INV0 | IEEE_STATUS_DZE0 | \ + IEEE_STATUS_OVF0 | IEEE_STATUS_UNF0 | \ + IEEE_STATUS_INE0 | IEEE_STATUS_DNO0) + +#define IEEE_STATUS0_TO_EXCSUM_SHIFT 16 + +#define IEEE_STATUS_INV1 (1UL << 23) +#define IEEE_STATUS_DZE1 (1UL << 24) +#define IEEE_STATUS_OVF1 (1UL << 25) +#define IEEE_STATUS_UNF1 (1UL << 26) +#define IEEE_STATUS_INE1 (1UL << 27) +#define IEEE_STATUS_DNO1 (1UL << 28) +#define IEEE_STATUS_MASK1 (IEEE_STATUS_INV1 | IEEE_STATUS_DZE1 | \ + IEEE_STATUS_OVF1 | IEEE_STATUS_UNF1 | \ + IEEE_STATUS_INE1 | IEEE_STATUS_DNO1) + +#define IEEE_STATUS1_TO_EXCSUM_SHIFT 22 + +#define IEEE_STATUS_INV2 (1UL << 34) +#define IEEE_STATUS_DZE2 (1UL << 35) +#define IEEE_STATUS_OVF2 (1UL << 36) +#define IEEE_STATUS_UNF2 (1UL << 37) +#define IEEE_STATUS_INE2 (1UL << 38) +#define IEEE_STATUS_DNO2 (1UL << 39) +#define IEEE_STATUS_MASK2 (IEEE_STATUS_INV2 | IEEE_STATUS_DZE2 | \ + IEEE_STATUS_OVF2 | IEEE_STATUS_UNF2 | \ + IEEE_STATUS_INE2 | IEEE_STATUS_DNO2) + +#define IEEE_STATUS2_TO_EXCSUM_SHIFT 33 + +#define IEEE_STATUS_INV3 (1UL << 40) +#define IEEE_STATUS_DZE3 (1UL << 41) +#define IEEE_STATUS_OVF3 (1UL << 42) +#define IEEE_STATUS_UNF3 (1UL << 43) +#define IEEE_STATUS_INE3 (1UL << 44) +#define IEEE_STATUS_DNO3 (1UL << 45) +#define IEEE_STATUS_MASK3 (IEEE_STATUS_INV3 | IEEE_STATUS_DZE3 | \ + IEEE_STATUS_OVF3 | IEEE_STATUS_UNF3 | \ + IEEE_STATUS_INE3 | IEEE_STATUS_DNO3) + +#define IEEE_STATUS3_TO_EXCSUM_SHIFT 39 + + +/* + * Convert the software IEEE trap enable and status bits into the + * hardware fpcr format. + * + * Digital Unix engineers receive my thanks for not defining the + * software bits identical to the hardware bits. The chip designers + * receive my thanks for making all the not-implemented fpcr bits + * RAZ forcing us to use system calls to read/write this value. + */ +static inline unsigned long +ieee_swcr_to_fpcr(unsigned long sw) +{ + unsigned long fp; + + fp = (sw & IEEE_STATUS_MASK0) << 35; + fp |= (sw & IEEE_STATUS_MASK1) << 13; + fp |= (sw & IEEE_STATUS_MASK2) >> 14; + fp |= (sw & IEEE_STATUS_MASK3) >> 36; + + fp |= (sw & IEEE_MAP_DMZ) << 36; + fp |= (sw & IEEE_STATUS_MASK0 ? FPCR_SUM : 0); + fp |= (sw & IEEE_STATUS_MASK1 ? FPCR_SUM : 0); + fp |= (sw & IEEE_STATUS_MASK2 ? FPCR_SUM : 0); + fp |= (sw & IEEE_STATUS_MASK3 ? FPCR_SUM : 0); + fp |= (~sw & (IEEE_TRAP_ENABLE_INV + | IEEE_TRAP_ENABLE_DZE + | IEEE_TRAP_ENABLE_OVF)) << 48; + fp |= (~sw & (IEEE_TRAP_ENABLE_UNF | IEEE_TRAP_ENABLE_INE)) << 57; + fp |= (sw & IEEE_MAP_UMZ ? FPCR_UNDZ | FPCR_UNFD : 0); + fp |= (~sw & IEEE_TRAP_ENABLE_DNO) << 41; + return fp; +} + +static inline unsigned long +ieee_fpcr_to_swcr(unsigned long fp) +{ + unsigned long sw; + + sw = (fp >> 35) & IEEE_STATUS_MASK; + sw |= (fp >> 36) & IEEE_MAP_DMZ; + sw |= (~fp >> 48) & (IEEE_TRAP_ENABLE_INV + | IEEE_TRAP_ENABLE_DZE + | IEEE_TRAP_ENABLE_OVF); + sw |= (~fp >> 57) & (IEEE_TRAP_ENABLE_UNF | IEEE_TRAP_ENABLE_INE); + sw |= (fp >> 47) & IEEE_MAP_UMZ; + sw |= (~fp >> 41) & IEEE_TRAP_ENABLE_DNO; + return sw; +} +#endif /* _UAPI_ASM_SW64_FPU_H */ diff --git a/arch/sw_64/kernel/fpu.S b/arch/sw_64/kernel/fpu.S new file mode 100644 index 000000000000..ddc988681fdd --- /dev/null +++ b/arch/sw_64/kernel/fpu.S @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include + + .text + .set noat +ENTRY(__fpstate_save) + /* a0: prev task */ +#ifdef CONFIG_SUBARCH_C4 + csrr $1, CSR_WR_FREGS + beq $1, out +#endif + vstd $f0, TASK_THREAD_F0(a0) + vstd $f1, TASK_THREAD_F1(a0) + vstd $f2, TASK_THREAD_F2(a0) + vstd $f3, TASK_THREAD_F3(a0) + vstd $f4, TASK_THREAD_F4(a0) + vstd $f5, TASK_THREAD_F5(a0) + vstd $f6, TASK_THREAD_F6(a0) + vstd $f7, TASK_THREAD_F7(a0) + vstd $f8, TASK_THREAD_F8(a0) + vstd $f9, TASK_THREAD_F9(a0) + vstd $f10, TASK_THREAD_F10(a0) + vstd $f11, TASK_THREAD_F11(a0) + vstd $f12, TASK_THREAD_F12(a0) + vstd $f13, TASK_THREAD_F13(a0) + vstd $f14, TASK_THREAD_F14(a0) + vstd $f15, TASK_THREAD_F15(a0) + vstd $f16, TASK_THREAD_F16(a0) + vstd $f17, TASK_THREAD_F17(a0) + vstd $f18, TASK_THREAD_F18(a0) + vstd $f19, TASK_THREAD_F19(a0) + vstd $f20, TASK_THREAD_F20(a0) + vstd $f21, TASK_THREAD_F21(a0) + vstd $f22, TASK_THREAD_F22(a0) + vstd $f23, TASK_THREAD_F23(a0) + vstd $f24, TASK_THREAD_F24(a0) + vstd $f25, TASK_THREAD_F25(a0) + vstd $f26, TASK_THREAD_F26(a0) + vstd $f27, TASK_THREAD_F27(a0) + rfpcr $f0 + vstd $f28, TASK_THREAD_F28(a0) + vstd $f29, TASK_THREAD_F29(a0) + vstd $f30, TASK_THREAD_F30(a0) + fstd $f0, TASK_THREAD_FPCR(a0) + vldd $f0, TASK_THREAD_F0(a0) +out: + ret +END(__fpstate_save) + +ENTRY(__fpstate_restore) + /* a0: next task */ + fldd $f0, TASK_THREAD_FPCR(a0) + wfpcr $f0 + fimovd $f0, t1 + and t1, 0x3, t1 + beq t1, $setfpec_0 + subl t1, 0x1, t1 + beq t1, $setfpec_1 + subl t1, 0x1, t1 + beq t1, $setfpec_2 + setfpec3 + br $setfpec_over +$setfpec_0: + setfpec0 + br $setfpec_over +$setfpec_1: + setfpec1 + br $setfpec_over +$setfpec_2: + setfpec2 +$setfpec_over: + vldd $f0, TASK_THREAD_F0(a0) + vldd $f1, TASK_THREAD_F1(a0) + vldd $f2, TASK_THREAD_F2(a0) + vldd $f3, TASK_THREAD_F3(a0) + vldd $f4, TASK_THREAD_F4(a0) + vldd $f5, TASK_THREAD_F5(a0) + vldd $f6, TASK_THREAD_F6(a0) + vldd $f7, TASK_THREAD_F7(a0) + vldd $f8, TASK_THREAD_F8(a0) + vldd $f9, TASK_THREAD_F9(a0) + vldd $f10, TASK_THREAD_F10(a0) + vldd $f11, TASK_THREAD_F11(a0) + vldd $f12, TASK_THREAD_F12(a0) + vldd $f13, TASK_THREAD_F13(a0) + vldd $f14, TASK_THREAD_F14(a0) + vldd $f15, TASK_THREAD_F15(a0) + vldd $f16, TASK_THREAD_F16(a0) + vldd $f17, TASK_THREAD_F17(a0) + vldd $f18, TASK_THREAD_F18(a0) + vldd $f19, TASK_THREAD_F19(a0) + vldd $f20, TASK_THREAD_F20(a0) + vldd $f21, TASK_THREAD_F21(a0) + vldd $f22, TASK_THREAD_F22(a0) + vldd $f23, TASK_THREAD_F23(a0) + vldd $f24, TASK_THREAD_F24(a0) + vldd $f25, TASK_THREAD_F25(a0) + vldd $f26, TASK_THREAD_F26(a0) + vldd $f27, TASK_THREAD_F27(a0) + vldd $f28, TASK_THREAD_F28(a0) + vldd $f29, TASK_THREAD_F29(a0) + vldd $f30, TASK_THREAD_F30(a0) +#ifdef CONFIG_SUBARCH_C4 + csrw $31, CSR_WR_FREGS +#endif + ret +END(__fpstate_restore) diff --git a/arch/sw_64/math-emu/Makefile b/arch/sw_64/math-emu/Makefile new file mode 100644 index 000000000000..72e750d138e6 --- /dev/null +++ b/arch/sw_64/math-emu/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the FPU instruction emulation. +# + +ccflags-y := -w + +obj-$(CONFIG_MATHEMU) += math-emu.o + +math-emu-objs := math.o qrnnd.o diff --git a/arch/sw_64/math-emu/math.c b/arch/sw_64/math-emu/math.c new file mode 100644 index 000000000000..b578752f0730 --- /dev/null +++ b/arch/sw_64/math-emu/math.c @@ -0,0 +1,2255 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Modify History + * + * who when what + * --- ---- ---- + * stone 2004-09-02 Add SIMD floating emulation code + * fire3 2008-12-27 Add SIMD floating emulation code for SW64 + */ + +#include + +#include + +#include "sfp-util.h" + +#include +#include +#include + +/* + * This is for sw64 + */ + +#define IEEE_E_STATUS_MASK IEEE_STATUS_MASK +#define IEEE_E_STATUS_TO_EXCSUM_SHIFT 0 +#define SW64_FP_DENOMAL 1 /* A denormal data */ +#define SW64_FP_NORMAL 0 /* A denormal data */ +#define SW64_FP_NAN 2 + +#define SW64_FP_NAN_S(X, val) \ +do { \ + union _FP_UNION_S *_flo = \ + (union _FP_UNION_S *)(val); \ + \ + X##_f = _flo->bits.frac; \ + X##_e = _flo->bits.exp; \ + X##_s = _flo->bits.sign; \ + \ + switch (X##_e) { \ + case 255: \ + if (_FP_FRAC_ZEROP_1(X)) \ + X##_c = SW64_FP_NORMAL; \ + else \ + X##_c = SW64_FP_NAN; \ + break; \ + default: \ + X##_c = SW64_FP_NORMAL; \ + break; \ + } \ +} while (0) + + +#define SW64_FP_NAN_D(X, val) \ +do { \ + union _FP_UNION_D *_flo = \ + (union _FP_UNION_D *)(val); \ + \ + X##_f = _flo->bits.frac; \ + X##_e = _flo->bits.exp; \ + X##_s = _flo->bits.sign; \ + \ + switch (X##_e) { \ + case 2047: \ + if (_FP_FRAC_ZEROP_1(X)) \ + X##_c = SW64_FP_NORMAL; \ + else \ + X##_c = SW64_FP_NAN; \ + break; \ + default: \ + X##_c = SW64_FP_NORMAL; \ + break; \ + } \ +} while (0) + + + +#define SW64_FP_NORMAL_S(X, val) \ +do { \ + union _FP_UNION_S *_flo = \ + (union _FP_UNION_S *)(val); \ + \ + X##_f = _flo->bits.frac; \ + X##_e = _flo->bits.exp; \ + X##_s = _flo->bits.sign; \ + \ + switch (X##_e) { \ + case 0: \ + if (_FP_FRAC_ZEROP_1(X)) \ + X##_c = SW64_FP_NORMAL; \ + else \ + X##_c = SW64_FP_DENOMAL; \ + break; \ + default: \ + X##_c = SW64_FP_NORMAL; \ + break; \ + } \ +} while (0) + +#define SW64_FP_NORMAL_D(X, val) \ +do { \ + union _FP_UNION_D *_flo = \ + (union _FP_UNION_D *)(val); \ + \ + X##_f = _flo->bits.frac; \ + X##_e = _flo->bits.exp; \ + X##_s = _flo->bits.sign; \ + \ + switch (X##_e) { \ + case 0: \ + if (_FP_FRAC_ZEROP_1(X)) \ + X##_c = SW64_FP_NORMAL; \ + else \ + X##_c = SW64_FP_DENOMAL; \ + break; \ + default: \ + X##_c = SW64_FP_NORMAL; \ + break; \ + } \ +} while (0) + +/* Operation Code for SW64 */ +#define OP_SIMD_1 0x1A +#define OP_SIMD_2 0x1B +#define OP_SIMD_MUL_ADD 0x1B +#define OP_SIMD_NORMAL 0x1A +#define OP_MUL_ADD 0x19 + +#define FNC_FMAS 0x0 +#define FNC_FMAD 0x1 +#define FNC_FMSS 0x2 +#define FNC_FMSD 0x3 +#define FNC_FNMAS 0x4 +#define FNC_FNMAD 0x5 +#define FNC_FNMSS 0x6 +#define FNC_FNMSD 0x7 + +#define FNC_VADDS 0x80 +#define FNC_VADDD 0x81 +#define FNC_VSUBS 0x82 +#define FNC_VSUBD 0x83 +#define FNC_VMULS 0x84 +#define FNC_VMULD 0x85 +#define FNC_VDIVS 0x86 +#define FNC_VDIVD 0x87 +#define FNC_VSQRTS 0x88 +#define FNC_VSQRTD 0x89 + +#define FNC_VFCMPEQ 0x8c +#define FNC_VFCMPLE 0x8d +#define FNC_VFCMPLT 0x8e +#define FNC_VFCMPUN 0x8f + +#define FNC_VCPYS 0x90 +#define FNC_VCPYSE 0x91 +#define FNC_VCPYSN 0x92 + +#define FNC_VMAS 0x0 +#define FNC_VMAD 0x1 +#define FNC_VMSS 0x2 +#define FNC_VMSD 0x3 +#define FNC_VNMAS 0x4 +#define FNC_VNMAD 0x5 +#define FNC_VNMSS 0x6 +#define FNC_VNMSD 0x7 + +long simd_fp_emul_s(unsigned long pc); +long simd_fp_emul_d(unsigned long pc); +long mul_add_fp_emul(unsigned long pc); +long simd_cmp_emul_d(unsigned long pc); + +long simd_mul_add_fp_emul_d(unsigned long pc); +long simd_mul_add_fp_emul_s(unsigned long pc); + +void read_fp_reg_s(unsigned long reg, unsigned long *p0, + unsigned long *p1, unsigned long *p2, unsigned long *p3); +void read_fp_reg_d(unsigned long reg, unsigned long *val_p0, + unsigned long *p1, unsigned long *p2, unsigned long *p3); +void write_fp_reg_s(unsigned long reg, unsigned long val_p0, + unsigned long p1, unsigned long p2, unsigned long p3); +void write_fp_reg_d(unsigned long reg, unsigned long val_p0, + unsigned long p1, unsigned long p2, unsigned long p3); +#define LOW_64_WORKING 1 +#define HIGH_64_WORKING 2 + +/* + * End for sw64 + */ + +#define OPC_HMC 0x00 +#define OPC_INTA 0x10 +#define OPC_INTL 0x11 +#define OPC_INTS 0x12 +#define OPC_INTM 0x13 +#define OPC_FLTC 0x14 +#define OPC_FLTV 0x15 +#define OPC_FLTI 0x16 +#define OPC_FLTL 0x17 +#define OPC_MISC 0x18 +#define OPC_JSR 0x1a + +#define FOP_SRC_S 0 +#define FOP_SRC_T 2 +#define FOP_SRC_Q 3 + +#define FOP_FNC_ADDx 0 +#define FOP_FNC_CVTQL 0 +#define FOP_FNC_SUBx 1 +#define FOP_FNC_MULx 2 +#define FOP_FNC_DIVx 3 +#define FOP_FNC_CMPxUN 4 +#define FOP_FNC_CMPxEQ 5 +#define FOP_FNC_CMPxLT 6 +#define FOP_FNC_CMPxLE 7 +#define FOP_FNC_SQRTx 11 +#define FOP_FNC_CVTxS 12 +#define FOP_FNC_CVTxT 14 +#define FOP_FNC_CVTxQ 15 + +/* this is for sw64 added by fire3*/ +#define FOP_FNC_ADDS 0 +#define FOP_FNC_ADDD 1 +#define FOP_FNC_SUBS 2 +#define FOP_FNC_SUBD 3 +#define FOP_FNC_MULS 4 +#define FOP_FNC_MULD 5 +#define FOP_FNC_DIVS 6 +#define FOP_FNC_DIVD 7 +#define FOP_FNC_SQRTS 8 +#define FOP_FNC_SQRTD 9 + +#define FOP_FNC_CMPEQ 0x10 +#define FOP_FNC_CMPLE 0x11 +#define FOP_FNC_CMPLT 0x12 +#define FOP_FNC_CMPUN 0x13 + +#define FOP_FNC_CVTSD 0x20 +#define FOP_FNC_CVTDS 0x21 +#define FOP_FNC_CVTLS 0x2D +#define FOP_FNC_CVTLD 0x2F +#define FOP_FNC_CVTDL 0x27 +#define FOP_FNC_CVTDL_G 0x22 +#define FOP_FNC_CVTDL_P 0x23 +#define FOP_FNC_CVTDL_Z 0x24 +#define FOP_FNC_CVTDL_N 0x25 + +#define FOP_FNC_CVTWL 0x28 +#define FOP_FNC_CVTLW 0x29 + +/* fire3 added end */ + + +#define MISC_TRAPB 0x0000 +#define MISC_EXCB 0x0400 + +extern unsigned long sw64_read_fp_reg(unsigned long reg); +extern void sw64_write_fp_reg(unsigned long reg, unsigned long val); +extern unsigned long sw64_read_fp_reg_s(unsigned long reg); +extern void sw64_write_fp_reg_s(unsigned long reg, unsigned long val); + + +#ifdef MODULE + +MODULE_DESCRIPTION("FP Software completion module"); + +extern long (*sw64_fp_emul_imprecise)(struct pt_regs *regs, unsigned long write_mask); +extern long (*sw64_fp_emul)(unsigned long pc); + +static long (*save_emul_imprecise)(struct pt_regs *regs, unsigned long write_mask); +static long (*save_emul)(unsigned long pc); + +long do_sw_fp_emul_imprecise(struct pt_regs *regs, unsigned long write_mask); +long do_sw_fp_emul(unsigned long pc); + +int init_module(void) +{ + save_emul_imprecise = sw64_fp_emul_imprecise; + save_emul = sw64_fp_emul; + sw64_fp_emul_imprecise = do_sw_fp_emul_imprecise; + sw64_fp_emul = do_sw_fp_emul; + return 0; +} + +void cleanup_module(void) +{ + sw64_fp_emul_imprecise = save_emul_imprecise; + sw64_fp_emul = save_emul; +} + +#undef sw64_fp_emul_imprecise +#define sw64_fp_emul_imprecise do_sw_fp_emul_imprecise +#undef sw64_fp_emul +#define sw64_fp_emul do_sw_fp_emul + +#endif /* MODULE */ + + +/* + * Emulate the floating point instruction at address PC. Returns -1 if the + * instruction to be emulated is illegal (such as with the opDEC trap), else + * the SI_CODE for a SIGFPE signal, else 0 if everything's ok. + * + * Notice that the kernel does not and cannot use FP regs. This is good + * because it means that instead of saving/restoring all fp regs, we simply + * stick the result of the operation into the appropriate register. + */ +long sw64_fp_emul(unsigned long pc) +{ + FP_DECL_EX; + FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); + FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); + + unsigned long fa, fb, fc, func, mode, mode_bk, src; + unsigned long res, va, vb, vc, swcr, fpcr; + __u32 insn; + long si_code; + unsigned long opcode; + + get_user(insn, (__u32 *)pc); + opcode = (insn >> 26) & 0x3f; + fc = (insn >> 0) & 0x1f; /* destination register */ + fb = (insn >> 16) & 0x1f; + fa = (insn >> 21) & 0x1f; + func = (insn >> 5) & 0xff; + fpcr = rdfpcr(); + mode = (fpcr >> FPCR_DYN_SHIFT) & 0x3; + pr_debug("======= Entering Floating mathe emulation =====\n"); + pr_debug("Floating math emulation insn = %#lx, opcode=%d, func=%d\n", insn, opcode, func); + pr_debug("SW64 hardware fpcr = %#lx\n", fpcr); + swcr = swcr_update_status(current_thread_info()->ieee_state, fpcr); + pr_debug("SW64 software swcr = %#lx\n", swcr); + pr_debug("fa:%#lx,fb:%#lx,fc:%#lx,func:%#lx,mode:%#lx\n", fa, fb, fc, func, mode); + + if (opcode == OP_SIMD_NORMAL) { /* float simd math */ + if (func == FNC_VADDS || func == FNC_VSUBS || func == FNC_VSQRTS + || func == FNC_VMULS || func == FNC_VDIVS) + si_code = simd_fp_emul_s(pc); + if (func == FNC_VADDD || func == FNC_VSUBD || func == FNC_VSQRTD + || func == FNC_VMULD || func == FNC_VDIVD) + si_code = simd_fp_emul_d(pc); + if (func == FNC_VFCMPUN || func == FNC_VFCMPLT || func == FNC_VFCMPLE + || func == FNC_VFCMPEQ) + si_code = simd_cmp_emul_d(pc); + return si_code; + } + if (opcode == OP_SIMD_MUL_ADD) {/* simd mul and add */ + func = (insn >> 10) & 0x3f; + if (func == FNC_VMAS || func == FNC_VMSS || func == FNC_VNMAS + || func == FNC_VNMSS) { + si_code = simd_mul_add_fp_emul_s(pc); + return si_code; + } + + if (func == FNC_VMAD || func == FNC_VMSD || func == FNC_VNMAD + || func == FNC_VNMSD) { + si_code = simd_mul_add_fp_emul_d(pc); + return si_code; + } + func = (insn >> 5) & 0xff; + } + + if (opcode == OP_MUL_ADD) { + si_code = mul_add_fp_emul(pc); + return si_code; + } + switch (func) { + case FOP_FNC_SUBS: + va = sw64_read_fp_reg_s(fa); + vb = sw64_read_fp_reg_s(fb); + FP_UNPACK_SP(SA, &va); + FP_UNPACK_SP(SB, &vb); + FP_SUB_S(SR, SA, SB); + goto pack_s; + + case FOP_FNC_SUBD: + va = sw64_read_fp_reg(fa); + vb = sw64_read_fp_reg(fb); + FP_UNPACK_DP(DA, &va); + FP_UNPACK_DP(DB, &vb); + FP_SUB_D(DR, DA, DB); + goto pack_d; + + case FOP_FNC_ADDS: + va = sw64_read_fp_reg_s(fa); + vb = sw64_read_fp_reg_s(fb); + FP_UNPACK_SP(SA, &va); + FP_UNPACK_SP(SB, &vb); + FP_ADD_S(SR, SA, SB); + goto pack_s; + + case FOP_FNC_ADDD: + va = sw64_read_fp_reg(fa); + vb = sw64_read_fp_reg(fb); + FP_UNPACK_DP(DA, &va); + FP_UNPACK_DP(DB, &vb); + FP_ADD_D(DR, DA, DB); + goto pack_d; + + case FOP_FNC_MULS: + va = sw64_read_fp_reg_s(fa); + vb = sw64_read_fp_reg_s(fb); + FP_UNPACK_SP(SA, &va); + FP_UNPACK_SP(SB, &vb); + FP_MUL_S(SR, SA, SB); + goto pack_s; + + case FOP_FNC_MULD: + va = sw64_read_fp_reg(fa); + vb = sw64_read_fp_reg(fb); + FP_UNPACK_DP(DA, &va); + FP_UNPACK_DP(DB, &vb); + FP_MUL_D(DR, DA, DB); + goto pack_d; + + case FOP_FNC_DIVS: + pr_debug("FOP_FNC_DIVS\n"); + va = sw64_read_fp_reg_s(fa); + vb = sw64_read_fp_reg_s(fb); + FP_UNPACK_SP(SA, &va); + FP_UNPACK_SP(SB, &vb); + FP_DIV_S(SR, SA, SB); + goto pack_s; + + case FOP_FNC_DIVD: + pr_debug("FOP_FNC_DIVD\n"); + va = sw64_read_fp_reg(fa); + vb = sw64_read_fp_reg(fb); + FP_UNPACK_DP(DA, &va); + FP_UNPACK_DP(DB, &vb); + FP_DIV_D(DR, DA, DB); + goto pack_d; + + case FOP_FNC_SQRTS: + va = sw64_read_fp_reg_s(fa); + vb = sw64_read_fp_reg_s(fb); + FP_UNPACK_SP(SA, &va); + FP_UNPACK_SP(SB, &vb); + FP_SQRT_S(SR, SB); + goto pack_s; + case FOP_FNC_SQRTD: + va = sw64_read_fp_reg(fa); + vb = sw64_read_fp_reg(fb); + FP_UNPACK_DP(DA, &va); + FP_UNPACK_DP(DB, &vb); + FP_SQRT_D(DR, DB); + goto pack_d; + } + + + va = sw64_read_fp_reg(fa); + vb = sw64_read_fp_reg(fb); + if ((func & ~0xf) == FOP_FNC_CMPEQ) { + va = sw64_read_fp_reg(fa); + vb = sw64_read_fp_reg(fb); + + FP_UNPACK_RAW_DP(DA, &va); + FP_UNPACK_RAW_DP(DB, &vb); + if (!DA_e && !_FP_FRAC_ZEROP_1(DA)) { + FP_SET_EXCEPTION(FP_EX_DENORM); + if (FP_DENORM_ZERO) + _FP_FRAC_SET_1(DA, _FP_ZEROFRAC_1); + } + if (!DB_e && !_FP_FRAC_ZEROP_1(DB)) { + FP_SET_EXCEPTION(FP_EX_DENORM); + if (FP_DENORM_ZERO) + _FP_FRAC_SET_1(DB, _FP_ZEROFRAC_1); + } + FP_CMP_D(res, DA, DB, 3); + vc = 0x4000000000000000; + /* CMPTEQ, CMPTUN don't trap on QNaN, while CMPTLT and CMPTLE do */ + if (res == 3 && (((func == FOP_FNC_CMPLT) || (func == FOP_FNC_CMPLE)) + || FP_ISSIGNAN_D(DA) || FP_ISSIGNAN_D(DB))) { + pr_debug("CMPLT CMPLE:func:%d, trap on QNaN.", func); + FP_SET_EXCEPTION(FP_EX_INVALID); + } + switch (func) { + case FOP_FNC_CMPUN: + if (res != 3) + vc = 0; + break; + case FOP_FNC_CMPEQ: + if (res) + vc = 0; + break; + case FOP_FNC_CMPLT: + if (res != -1) + vc = 0; + break; + case FOP_FNC_CMPLE: + if ((long)res > 0) + vc = 0; + break; + } + goto done_d; + } + FP_UNPACK_DP(DA, &va); + FP_UNPACK_DP(DB, &vb); + + if (func == FOP_FNC_CVTSD) { + vb = sw64_read_fp_reg_s(fb); + FP_UNPACK_SP(SB, &vb); + DR_c = DB_c; + DR_s = DB_s; + DR_e = DB_e + (1024 - 128); + DR_f = SB_f << (52 - 23); + goto pack_d; + } + + if (func == FOP_FNC_CVTDS) { + FP_CONV(S, D, 1, 1, SR, DB); + goto pack_s; + } + + if (func == FOP_FNC_CVTDL || func == FOP_FNC_CVTDL_G || func == FOP_FNC_CVTDL_P + || func == FOP_FNC_CVTDL_Z || func == FOP_FNC_CVTDL_N) { + mode_bk = mode; + if (func == FOP_FNC_CVTDL_Z) + mode = 0x0UL; + else if (func == FOP_FNC_CVTDL_N) + mode = 0x1UL; + else if (func == FOP_FNC_CVTDL_G) + mode = 0x2UL; + else if (func == FOP_FNC_CVTDL_P) + mode = 0x3UL; + + if (DB_c == FP_CLS_NAN && (_FP_FRAC_HIGH_RAW_D(DB) & _FP_QNANBIT_D)) { + /* AAHB Table B-2 says QNaN should not trigger INV */ + vc = 0; + } else + FP_TO_INT_ROUND_D(vc, DB, 64, 2); + mode = mode_bk; + goto done_d; + } + + vb = sw64_read_fp_reg(fb); + + switch (func) { + case FOP_FNC_CVTLW: + /* + * Notice: We can get here only due to an integer + * overflow. Such overflows are reported as invalid + * ops. We return the result the hw would have + * computed. + */ + vc = ((vb & 0xc0000000) << 32 | /* sign and msb */ + (vb & 0x3fffffff) << 29); /* rest of the int */ + FP_SET_EXCEPTION(FP_EX_INVALID); + goto done_d; + + case FOP_FNC_CVTLS: + FP_FROM_INT_S(SR, ((long)vb), 64, long); + goto pack_s; + + case FOP_FNC_CVTLD: + FP_FROM_INT_D(DR, ((long)vb), 64, long); + goto pack_d; + } + goto bad_insn; + + +pack_s: + FP_PACK_SP(&vc, SR); + + if ((_fex & FP_EX_UNDERFLOW) && (swcr & IEEE_MAP_UMZ)) + vc = 0; + pr_debug("SW64 Emulation S-floating _fex=%#lx, va=%#lx, vb=%#lx, vc=%#lx\n", _fex, va, vb, vc); + pr_debug("SW64 Emulation S-floating mode=%#lx,func=%#lx, swcr=%#lx\n", mode, func, swcr); + sw64_write_fp_reg_s(fc, vc); + goto done; + +pack_d: + FP_PACK_DP(&vc, DR); + if ((_fex & FP_EX_UNDERFLOW) && (swcr & IEEE_MAP_UMZ)) + vc = 0; + pr_debug("SW64 Emulation D-floating _fex=%#lx, va=%#lx, vb=%#lx, vc=%#lx\n", _fex, va, vb, vc); + pr_debug("SW64 Emulation D-floating mode=%#lx,func=%#lx, swcr=%#lx\n", mode, func, swcr); +done_d: + sw64_write_fp_reg(fc, vc); + goto done; + + /* + * Take the appropriate action for each possible + * floating-point result: + * + * - Set the appropriate bits in the FPCR + * - If the specified exception is enabled in the FPCR, + * return. The caller (entArith) will dispatch + * the appropriate signal to the translated program. + * + * In addition, properly track the exception state in software + * as described in the SW64 Architecture Handbook section 4.7.7.3. + */ +done: + if (_fex) { + /* Record exceptions in software control word. */ + swcr |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr |= ieee_swcr_to_fpcr(swcr); + pr_debug("SW64 before write fpcr = %#lx\n", fpcr); + wrfpcr(fpcr); + + /* Do we generate a signal? */ + _fex = _fex & swcr & IEEE_TRAP_ENABLE_MASK; + si_code = 0; + if (_fex) { + if (_fex & IEEE_TRAP_ENABLE_DNO) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_INE) + si_code = FPE_FLTRES; + if (_fex & IEEE_TRAP_ENABLE_UNF) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_OVF) + si_code = FPE_FLTOVF; + if (_fex & IEEE_TRAP_ENABLE_DZE) + si_code = FPE_FLTDIV; + if (_fex & IEEE_TRAP_ENABLE_INV) + si_code = FPE_FLTINV; + } + + return si_code; + } + + /* + * We used to write the destination register here, but DEC FORTRAN + * requires that the result *always* be written... so we do the write + * immediately after the operations above. + */ + + return 0; + +bad_insn: + pr_err("%s: Invalid FP insn %#x at %#lx\n", __func__, insn, pc); + return -1; +} + +long sw64_fp_emul_imprecise(struct pt_regs *regs, unsigned long write_mask) +{ + unsigned long trigger_pc = regs->pc - 4; + unsigned long insn, opcode, rc, si_code = 0; + + + /* + * Turn off the bits corresponding to registers that are the + * target of instructions that set bits in the exception + * summary register. We have some slack doing this because a + * register that is the target of a trapping instruction can + * be written at most once in the trap shadow. + * + * Branches, jumps, TRAPBs, EXCBs and calls to HMcode all + * bound the trap shadow, so we need not look any further than + * up to the first occurrence of such an instruction. + */ + while (write_mask) { + get_user(insn, (__u32 *)(trigger_pc)); + opcode = insn >> 26; + rc = insn & 0x1f; + + switch (opcode) { + case OPC_HMC: + case OPC_JSR: + case 0x30 ... 0x3f: /* branches */ + goto egress; + + case OPC_MISC: + switch (insn & 0xffff) { + case MISC_TRAPB: + case MISC_EXCB: + goto egress; + + default: + break; + } + break; + + case OPC_INTA: + case OPC_INTL: + case OPC_INTS: + case OPC_INTM: + write_mask &= ~(1UL << rc); + break; + + case OPC_FLTC: + case OPC_FLTV: + case OPC_FLTI: + case OPC_FLTL: + write_mask &= ~(1UL << (rc + 32)); + break; + } + if (!write_mask) { + /* Re-execute insns in the trap-shadow. */ + regs->pc = trigger_pc + 4; + si_code = sw64_fp_emul(trigger_pc); + goto egress; + } + trigger_pc -= 4; + } + +egress: + return si_code; +} + +#define WORKING_PART_0 0 +#define WORKING_PART_1 1 +#define WORKING_PART_2 2 +#define WORKING_PART_3 3 + + +/* + * This is for sw64 + */ + +long simd_cmp_emul_d(unsigned long pc) +{ + FP_DECL_EX; + FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); FP_DECL_D(DC); + unsigned long fa, fb, fc, func, mode, src; + unsigned long res, va, vb, vc, swcr, fpcr; + __u32 insn; + long si_code; + + unsigned long va_p0, va_p1, va_p2, va_p3; + unsigned long vb_p0, vb_p1, vb_p2, vb_p3; + unsigned long vc_p0, vc_p1, vc_p2, vc_p3; + unsigned long fex_p0, fex_p1, fex_p2, fex_p3; + + int working_part; + + get_user(insn, (__u32 *)pc); + fc = (insn >> 0) & 0x1f; /* destination register */ + fb = (insn >> 16) & 0x1f; + fa = (insn >> 21) & 0x1f; + func = (insn >> 5) & 0xff; + fpcr = rdfpcr(); + mode = (fpcr >> FPCR_DYN_SHIFT) & 0x3; + + pr_debug("======== Entering SIMD floating-CMP math emulation =======\n"); + pr_debug("hardware fpcr = %#lx\n", fpcr); + swcr = swcr_update_status(current_thread_info()->ieee_state, fpcr); + pr_debug("software swcr = %#lx\n", swcr); + pr_debug("fa:%#lx,fb:%#lx,fc:%#lx,func:%#lx,mode:%#lx\n", fa, fb, fc, func, mode); + read_fp_reg_d(fa, &va_p0, &va_p1, &va_p2, &va_p3); + read_fp_reg_d(fb, &vb_p0, &vb_p1, &vb_p2, &vb_p3); + read_fp_reg_d(fc, &vc_p0, &vc_p1, &vc_p2, &vc_p3); + pr_debug("va_p0:%#lx, va_p1:%#lx, va_p2:%#lx, va_p3:%#lx\n", va_p0, va_p1, va_p2, va_p3); + pr_debug("vb_p0:%#lx, vb_p1:%#lx, vb_p2:%#lx, vb_p3:%#lx\n", vb_p0, vb_p1, vb_p2, vb_p3); + pr_debug("vc_p0:%#lx, vc_p1:%#lx, vc_p2:%#lx, vc_p3:%#lx\n", vc_p0, vc_p1, vc_p2, vc_p3); + working_part = WORKING_PART_0; +simd_working: + _fex = 0; + switch (working_part) { + case WORKING_PART_0: + pr_debug("WORKING_PART_0\n"); + va = va_p0; + vb = vb_p0; + vc = vc_p0; + break; + case WORKING_PART_1: + pr_debug("WORKING_PART_1\n"); + va = va_p1; + vb = vb_p1; + vc = vc_p1; + break; + case WORKING_PART_2: + pr_debug("WORKING_PART_2\n"); + va = va_p2; + vb = vb_p2; + vc = vc_p2; + break; + case WORKING_PART_3: + pr_debug("WORKING_PART_3\n"); + va = va_p3; + vb = vb_p3; + vc = vc_p3; + break; + } + pr_debug("Before unpack va:%#lx, vb:%#lx\n", va, vb); + FP_UNPACK_RAW_DP(DA, &va); + FP_UNPACK_RAW_DP(DB, &vb); + pr_debug("DA_e:%d, _FP_FRAC_ZEROP_1(DA):%d\n", DA_e, _FP_FRAC_ZEROP_1(DA)); + pr_debug("DB_e:%d, _FP_FRAC_ZEROP_1(DB):%d\n", DA_e, _FP_FRAC_ZEROP_1(DA)); + pr_debug("DA iszero:%d, DB iszero:%d\n", ((!DA_e && _FP_FRAC_ZEROP_1(DA)) ? 1 : 0), + ((!DB_e && _FP_FRAC_ZEROP_1(DB)))); + if (!DA_e && !_FP_FRAC_ZEROP_1(DA)) { + FP_SET_EXCEPTION(FP_EX_DENORM); + if (FP_DENORM_ZERO) + _FP_FRAC_SET_1(DA, _FP_ZEROFRAC_1); + } + if (!DB_e && !_FP_FRAC_ZEROP_1(DB)) { + FP_SET_EXCEPTION(FP_EX_DENORM); + if (FP_DENORM_ZERO) + _FP_FRAC_SET_1(DB, _FP_ZEROFRAC_1); + } + FP_CMP_D(res, DA, DB, 3); + vc = 0x4000000000000000; + /* CMPTEQ, CMPTUN don't trap on QNaN, while CMPTLT and CMPTLE do */ + if (res == 3 && (((func == FOP_FNC_CMPLT) || (func == FOP_FNC_CMPLE)) + || FP_ISSIGNAN_D(DA) || FP_ISSIGNAN_D(DB))) { + pr_debug("CMPLT CMPLE:func:%d, trap on QNaN.", func); + FP_SET_EXCEPTION(FP_EX_INVALID); + } + pr_debug("res:%d\n", res); + switch (func) { + case FNC_VFCMPUN: + if (res != 3) + vc = 0; + break; + case FNC_VFCMPEQ: + if (res) + vc = 0; + break; + case FNC_VFCMPLT: + if (res != -1) + vc = 0; + break; + case FNC_VFCMPLE: + if ((long)res > 0) + vc = 0; + break; + } +next_working_s: + switch (working_part) { + case WORKING_PART_0: + working_part = WORKING_PART_1; + vc_p0 = vc; + fex_p0 = _fex; + goto simd_working; + case WORKING_PART_1: + working_part = WORKING_PART_2; + vc_p1 = vc; + fex_p1 = _fex; + goto simd_working; + case WORKING_PART_2: + working_part = WORKING_PART_3; + vc_p2 = vc; + fex_p2 = _fex; + goto simd_working; + case WORKING_PART_3: + vc_p3 = vc; + fex_p3 = _fex; + goto done; + } +done: + if (fex_p0 || fex_p1 || fex_p2 || fex_p3) { + unsigned long fpcr_p0, fpcr_p1, fpcr_p2, fpcr_p3; + unsigned long swcr_p0, swcr_p1, swcr_p2, swcr_p3; + + fpcr_p0 = fpcr_p1 = fpcr_p2 = fpcr_p3 = 0; + swcr_p0 = swcr_p1 = swcr_p2 = swcr_p3 = swcr; + /* manage fpcr_p0 */ + if (fex_p0) { + swcr_p0 |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p0 = fpcr; + fpcr_p0 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p0 |= ieee_swcr_to_fpcr(swcr_p0); + } + + if (fex_p1) { + swcr_p1 |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p1 = fpcr; + fpcr_p1 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p1 |= ieee_swcr_to_fpcr(swcr_p1); + } + + if (fex_p2) { + swcr_p2 |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p2 = fpcr; + fpcr_p2 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p2 |= ieee_swcr_to_fpcr(swcr_p2); + } + + if (fex_p3) { + swcr_p3 |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p3 = fpcr; + fpcr_p3 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p3 |= ieee_swcr_to_fpcr(swcr_p3); + } + + fpcr = fpcr_p0 | fpcr_p1 | fpcr_p2 | fpcr_p3; + pr_debug("fex_p0 = %#lx\n", fex_p0); + pr_debug("fex_p1 = %#lx\n", fex_p1); + pr_debug("fex_p2 = %#lx\n", fex_p2); + pr_debug("fex_p3 = %#lx\n", fex_p3); + pr_debug("SIMD emulation almost finished.before write fpcr = %#lx\n", fpcr); + wrfpcr(fpcr); + pr_debug("Before write fp: vc_p0=%#lx, vc_p1=%#lx, vc_p2=%#lx, vc_p3=%#lx\n", vc_p0, vc_p1, vc_p2, vc_p3); + write_fp_reg_d(fc, vc_p0, vc_p1, vc_p2, vc_p3); + + /* Do we generate a signal? */ + _fex = (fex_p0 & swcr & IEEE_TRAP_ENABLE_MASK) | (fex_p1 & swcr & IEEE_TRAP_ENABLE_MASK) + | (fex_p2 & swcr & IEEE_TRAP_ENABLE_MASK) | (fex_p3 & swcr & IEEE_TRAP_ENABLE_MASK); + si_code = 0; + if (_fex) { + if (_fex & IEEE_TRAP_ENABLE_DNO) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_INE) + si_code = FPE_FLTRES; + if (_fex & IEEE_TRAP_ENABLE_UNF) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_OVF) + si_code = FPE_FLTOVF; + if (_fex & IEEE_TRAP_ENABLE_DZE) + si_code = FPE_FLTDIV; + if (_fex & IEEE_TRAP_ENABLE_INV) + si_code = FPE_FLTINV; + } + pr_debug("SIMD finished.. si_code:%#lx\n", si_code); + return si_code; + + } + pr_debug("SIMD finished.. si_code:%#lx\n", si_code); + return 0; + +bad_insn: + pr_err("%s: Invalid FP insn %#x at %#lx\n", __func__, insn, pc); + return -1; +} + + +long simd_fp_emul_d(unsigned long pc) +{ + FP_DECL_EX; + FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); FP_DECL_D(DC); + unsigned long fa, fb, fc, func, mode, src; + unsigned long res, va, vb, vc, swcr, fpcr; + __u32 insn; + long si_code; + + unsigned long va_p0, va_p1, va_p2, va_p3; + unsigned long vb_p0, vb_p1, vb_p2, vb_p3; + unsigned long vc_p0, vc_p1, vc_p2, vc_p3; + unsigned long fex_p0, fex_p1, fex_p2, fex_p3; + + int working_part; + + get_user(insn, (__u32 *)pc); + fc = (insn >> 0) & 0x1f; /* destination register */ + fb = (insn >> 16) & 0x1f; + fa = (insn >> 21) & 0x1f; + func = (insn >> 5) & 0xff; + fpcr = rdfpcr(); + mode = (fpcr >> FPCR_DYN_SHIFT) & 0x3; + + pr_debug("======== Entering SIMD D-floating math emulation =======\n"); + pr_debug("hardware fpcr = %#lx\n", fpcr); + swcr = swcr_update_status(current_thread_info()->ieee_state, fpcr); + pr_debug("software swcr = %#lx\n", swcr); + pr_debug("fa:%#lx,fb:%#lx,fc:%#lx,func:%#lx,mode:%#lx\n", fa, fb, fc, func, mode); + read_fp_reg_d(fa, &va_p0, &va_p1, &va_p2, &va_p3); + read_fp_reg_d(fb, &vb_p0, &vb_p1, &vb_p2, &vb_p3); + read_fp_reg_d(fc, &vc_p0, &vc_p1, &vc_p2, &vc_p3); + pr_debug("va_p0:%#lx, va_p1:%#lx, va_p2:%#lx, va_p3:%#lx\n", va_p0, va_p1, va_p2, va_p3); + pr_debug("vb_p0:%#lx, vb_p1:%#lx, vb_p2:%#lx, vb_p3:%#lx\n", vb_p0, vb_p1, vb_p2, vb_p3); + pr_debug("vc_p0:%#lx, vc_p1:%#lx, vc_p2:%#lx, vc_p3:%#lx\n", vc_p0, vc_p1, vc_p2, vc_p3); + working_part = WORKING_PART_0; +simd_working: + _fex = 0; + switch (working_part) { + case WORKING_PART_0: + pr_debug("WORKING_PART_0\n"); + va = va_p0; + vb = vb_p0; + vc = vc_p0; + if ((fpcr & FPCR_STATUS_MASK0) == 0) { + SW64_FP_NORMAL_D(DA, &va); + SW64_FP_NORMAL_D(DB, &vb); + if ((DA_c == SW64_FP_NORMAL) && (DB_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("LOW: DA_c = %#lx, DB_c = %#lx\n", DA_c, DB_c); + } else { + SW64_FP_NAN_D(DA, &va); + SW64_FP_NAN_D(DB, &vb); + if (((DA_c == SW64_FP_NAN) || (DB_c == SW64_FP_NAN))) + goto next_working_s; + } + break; + case WORKING_PART_1: + pr_debug("WORKING_PART_1\n"); + va = va_p1; + vb = vb_p1; + vc = vc_p1; + if ((fpcr & FPCR_STATUS_MASK1) == 0) { + SW64_FP_NORMAL_D(DA, &va); + SW64_FP_NORMAL_D(DB, &vb); + if ((DA_c == SW64_FP_NORMAL) && (DB_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("HIGH: DA_c = %#lx, DB_c = %#lx\n", DA_c, DB_c); + } else { + SW64_FP_NAN_D(DA, &va); + SW64_FP_NAN_D(DB, &vb); + if (((DA_c == SW64_FP_NAN) || (DB_c == SW64_FP_NAN))) + goto next_working_s; + } + + break; + case WORKING_PART_2: + pr_debug("WORKING_PART_2\n"); + va = va_p2; + vb = vb_p2; + vc = vc_p2; + if ((fpcr & FPCR_STATUS_MASK2) == 0) { + SW64_FP_NORMAL_D(DA, &va); + SW64_FP_NORMAL_D(DB, &vb); + if ((DA_c == SW64_FP_NORMAL) && (DB_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("HIGH: DA_c = %#lx, DB_c = %#lx\n", DA_c, DB_c); + } else { + SW64_FP_NAN_D(DA, &va); + SW64_FP_NAN_D(DB, &vb); + if (((DA_c == SW64_FP_NAN) || (DB_c == SW64_FP_NAN))) + goto next_working_s; + } + break; + case WORKING_PART_3: + pr_debug("WORKING_PART_3\n"); + va = va_p3; + vb = vb_p3; + vc = vc_p3; + if ((fpcr & FPCR_STATUS_MASK3) == 0) { + SW64_FP_NORMAL_D(DA, &va); + SW64_FP_NORMAL_D(DB, &vb); + if ((DA_c == SW64_FP_NORMAL) && (DB_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("HIGH: DA_c = %#lx, DB_c = %#lx\n", DA_c, DB_c); + } else { + SW64_FP_NAN_D(DA, &va); + SW64_FP_NAN_D(DB, &vb); + if (((DA_c == SW64_FP_NAN) || (DB_c == SW64_FP_NAN))) + goto next_working_s; + } + break; + } + + FP_UNPACK_DP(DA, &va); + FP_UNPACK_DP(DB, &vb); + + switch (func) { + case FNC_VSUBD: + pr_debug("FNC_VSUBD\n"); + FP_SUB_D(DR, DA, DB); + goto pack_d; + case FNC_VMULD: + pr_debug("FNC_VMULD\n"); + FP_MUL_D(DR, DA, DB); + goto pack_d; + case FNC_VADDD: + pr_debug("FNC_VADDD\n"); + FP_ADD_D(DR, DA, DB); + goto pack_d; + case FNC_VDIVD: + pr_debug("FNC_VDIVD\n"); + FP_DIV_D(DR, DA, DB); + goto pack_d; + case FNC_VSQRTD: + pr_debug("FNC_VSQRTD\n"); + FP_SQRT_D(DR, DB); + goto pack_d; + } +pack_d: + FP_PACK_DP(&vc, DR); + if ((_fex & FP_EX_UNDERFLOW) && (swcr & IEEE_MAP_UMZ)) { + pr_debug("pack_d, vc=0 !!!!\n"); + vc = 0; + } + + pr_debug("SW64 SIMD Emulation D-floating _fex=%#lx, va=%#lx, vb=%#lx, vc=%#lx\n", _fex, va, vb, vc); + pr_debug("SW64 SIMD Emulation D-floating mode=%#lx,func=%#lx, swcr=%#lx\n", mode, func, swcr); +next_working_s: + switch (working_part) { + case WORKING_PART_0: + working_part = WORKING_PART_1; + vc_p0 = vc; + fex_p0 = _fex; + goto simd_working; + case WORKING_PART_1: + working_part = WORKING_PART_2; + vc_p1 = vc; + fex_p1 = _fex; + goto simd_working; + case WORKING_PART_2: + working_part = WORKING_PART_3; + vc_p2 = vc; + fex_p2 = _fex; + goto simd_working; + case WORKING_PART_3: + vc_p3 = vc; + fex_p3 = _fex; + goto done; + } +done: + if (fex_p0 || fex_p1 || fex_p2 || fex_p3) { + unsigned long fpcr_p0, fpcr_p1, fpcr_p2, fpcr_p3; + unsigned long swcr_p0, swcr_p1, swcr_p2, swcr_p3; + + fpcr_p0 = fpcr_p1 = fpcr_p2 = fpcr_p3 = 0; + swcr_p0 = swcr_p1 = swcr_p2 = swcr_p3 = swcr; + /* manage fpcr_p0 */ + if (fex_p0) { + swcr_p0 |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p0 = fpcr; + fpcr_p0 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p0 |= ieee_swcr_to_fpcr(swcr_p0); + } + + if (fex_p1) { + swcr_p1 |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p1 = fpcr; + fpcr_p1 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p1 |= ieee_swcr_to_fpcr(swcr_p1); + } + + if (fex_p2) { + swcr_p2 |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p2 = fpcr; + fpcr_p2 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p2 |= ieee_swcr_to_fpcr(swcr_p2); + } + + if (fex_p3) { + swcr_p3 |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p3 = fpcr; + fpcr_p3 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p3 |= ieee_swcr_to_fpcr(swcr_p3); + } + + fpcr = fpcr_p0 | fpcr_p1 | fpcr_p2 | fpcr_p3; + pr_debug("fex_p0 = %#lx\n", fex_p0); + pr_debug("fex_p1 = %#lx\n", fex_p1); + pr_debug("fex_p2 = %#lx\n", fex_p2); + pr_debug("fex_p3 = %#lx\n", fex_p3); + pr_debug("SIMD emulation almost finished.before write fpcr = %#lx\n", fpcr); + wrfpcr(fpcr); + pr_debug("Before write fp: vp_p0=%#lx, vc_p1=%#lx, vc_p2=%#lx, vc_p3=%#lx\n", vc_p0, vc_p1, vc_p2, vc_p3); + write_fp_reg_d(fc, vc_p0, vc_p1, vc_p2, vc_p3); + + /* Do we generate a signal? */ + _fex = (fex_p0 & swcr & IEEE_TRAP_ENABLE_MASK) | (fex_p1 & swcr & IEEE_TRAP_ENABLE_MASK) + | (fex_p2 & swcr & IEEE_TRAP_ENABLE_MASK) | (fex_p3 & swcr & IEEE_TRAP_ENABLE_MASK); + si_code = 0; + if (_fex) { + if (_fex & IEEE_TRAP_ENABLE_DNO) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_INE) + si_code = FPE_FLTRES; + if (_fex & IEEE_TRAP_ENABLE_UNF) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_OVF) + si_code = FPE_FLTOVF; + if (_fex & IEEE_TRAP_ENABLE_DZE) + si_code = FPE_FLTDIV; + if (_fex & IEEE_TRAP_ENABLE_INV) + si_code = FPE_FLTINV; + } + pr_debug("SIMD finished.. si_code:%#lx\n", si_code); + return si_code; + } + pr_debug("SIMD finished.. si_code:%#lx\n", si_code); + return 0; + +bad_insn: + pr_err("%s: Invalid FP insn %#x at %#lx\n", __func__, insn, pc); + return -1; +} + +long simd_fp_emul_s(unsigned long pc) +{ + FP_DECL_EX; + FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); + + unsigned long fa, fb, fc, func, mode, src; + unsigned long res, va, vb, vc, swcr, fpcr; + __u32 insn; + long si_code; + + unsigned long va_p0, va_p1, va_p2, va_p3; + unsigned long vb_p0, vb_p1, vb_p2, vb_p3; + unsigned long vc_p0, vc_p1, vc_p2, vc_p3; + unsigned long fex_p0, fex_p1, fex_p2, fex_p3; + + int working_part; + + get_user(insn, (__u32 *)pc); + fc = (insn >> 0) & 0x1f; /* destination register */ + fb = (insn >> 16) & 0x1f; + fa = (insn >> 21) & 0x1f; + func = (insn >> 5) & 0xff; + fpcr = rdfpcr(); + mode = (fpcr >> FPCR_DYN_SHIFT) & 0x3; + + pr_debug("======== Entering SIMD S-floating math emulation =======\n"); + pr_debug("hardware fpcr = %#lx\n", fpcr); + swcr = swcr_update_status(current_thread_info()->ieee_state, fpcr); + pr_debug("software swcr = %#lx\n", swcr); + pr_debug("fa:%#lx,fb:%#lx,fc:%#lx,func:%#lx,mode:%#lx\n", fa, fb, fc, func, mode); + read_fp_reg_s(fa, &va_p0, &va_p1, &va_p2, &va_p3); + read_fp_reg_s(fb, &vb_p0, &vb_p1, &vb_p2, &vb_p3); + read_fp_reg_s(fc, &vc_p0, &vc_p1, &vc_p2, &vc_p3); + pr_debug("va_p0:%#lx, va_p1:%#lx, va_p2:%#lx, va_p3:%#lx\n", va_p0, va_p1, va_p2, va_p3); + pr_debug("vb_p0:%#lx, vb_p1:%#lx, vb_p2:%#lx, vb_p3:%#lx\n", vb_p0, vb_p1, vb_p2, vb_p3); + pr_debug("vc_p0:%#lx, vc_p1:%#lx, vc_p2:%#lx, vc_p3:%#lx\n", vc_p0, vc_p1, vc_p2, vc_p3); + working_part = WORKING_PART_0; +simd_working: + _fex = 0; + switch (working_part) { + case WORKING_PART_0: + pr_debug("WORKING_PART_0\n"); + va = va_p0; + vb = vb_p0; + vc = vc_p0; + if ((fpcr & FPCR_STATUS_MASK0) == 0) { + SW64_FP_NORMAL_S(SA, &va); + SW64_FP_NORMAL_S(SB, &vb); + if ((SA_c == SW64_FP_NORMAL) && (SB_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("PART0: SA_c = %#lx, SB_c = %#lx\n", SA_c, SB_c); + } else { + SW64_FP_NAN_S(SA, &va); + SW64_FP_NAN_S(SB, &vb); + if ((SA_c == SW64_FP_NAN) && (SB_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + case WORKING_PART_1: + pr_debug("WORKING_PART_1\n"); + va = va_p1; + vb = vb_p1; + vc = vc_p1; + if ((fpcr & FPCR_STATUS_MASK1) == 0) { + SW64_FP_NORMAL_S(SA, &va); + SW64_FP_NORMAL_S(SB, &vb); + if ((SA_c == SW64_FP_NORMAL) && (SB_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("PART1: SA_c = %#lx, SB_c = %#lx\n", SA_c, SB_c); + } else { + SW64_FP_NAN_S(SA, &va); + SW64_FP_NAN_S(SB, &vb); + if ((SA_c == SW64_FP_NAN) && (SB_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + case WORKING_PART_2: + pr_debug("WORKING_PART_2\n"); + va = va_p2; + vb = vb_p2; + vc = vc_p2; + if ((fpcr & FPCR_STATUS_MASK2) == 0) { + SW64_FP_NORMAL_S(SA, &va); + SW64_FP_NORMAL_S(SB, &vb); + if ((SA_c == SW64_FP_NORMAL) && (SB_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("PART2: SA_c = %#lx, SB_c = %#lx\n", SA_c, SB_c); + } else { + SW64_FP_NAN_S(SA, &va); + SW64_FP_NAN_S(SB, &vb); + if ((SA_c == SW64_FP_NAN) && (SB_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + case WORKING_PART_3: + pr_debug("WORKING_PART_3\n"); + va = va_p3; + vb = vb_p3; + vc = vc_p3; + if ((fpcr & FPCR_STATUS_MASK3) == 0) { + SW64_FP_NORMAL_S(SA, &va); + SW64_FP_NORMAL_S(SB, &vb); + if ((SA_c == SW64_FP_NORMAL) && (SB_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("PART3: SA_c = %#lx, SB_c = %#lx\n", SA_c, SB_c); + } else { + SW64_FP_NAN_S(SA, &va); + SW64_FP_NAN_S(SB, &vb); + if ((SA_c == SW64_FP_NAN) && (SB_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + + } + + FP_UNPACK_SP(SA, &va); + FP_UNPACK_SP(SB, &vb); + + switch (func) { + case FNC_VSUBS: + pr_debug("FNC_VSUBS\n"); + FP_SUB_S(SR, SA, SB); + goto pack_s; + case FNC_VMULS: + pr_debug("FNC_VMULS\n"); + FP_MUL_S(SR, SA, SB); + goto pack_s; + case FNC_VADDS: + pr_debug("FNC_VADDS\n"); + FP_ADD_S(SR, SA, SB); + goto pack_s; + case FNC_VDIVS: + pr_debug("FNC_VDIVS\n"); + FP_DIV_S(SR, SA, SB); + goto pack_s; + case FNC_VSQRTS: + pr_debug("FNC_VSQRTS\n"); + FP_SQRT_S(SR, SB); + goto pack_s; + } +pack_s: + FP_PACK_SP(&vc, SR); + if ((_fex & FP_EX_UNDERFLOW) && (swcr & IEEE_MAP_UMZ)) { + pr_debug("pack_s, vc=0 !!!!\n"); + vc = 0; + } + + pr_debug("SW64 SIMD Emulation S-floating _fex=%#lx, va=%#lx, vb=%#lx, vc=%#lx\n", _fex, va, vb, vc); + pr_debug("SW64 SIMD Emulation S-floating mode=%#lx,func=%#lx, swcr=%#lx\n", mode, func, swcr); +next_working_s: + switch (working_part) { + case WORKING_PART_0: + working_part = WORKING_PART_1; + vc_p0 = vc; + fex_p0 = _fex; + goto simd_working; + case WORKING_PART_1: + working_part = WORKING_PART_2; + vc_p1 = vc; + fex_p1 = _fex; + goto simd_working; + case WORKING_PART_2: + working_part = WORKING_PART_3; + vc_p2 = vc; + fex_p2 = _fex; + goto simd_working; + case WORKING_PART_3: + vc_p3 = vc; + fex_p3 = _fex; + goto done; + } +done: + if (fex_p0 || fex_p1 || fex_p2 || fex_p3) { + unsigned long fpcr_p0, fpcr_p1, fpcr_p2, fpcr_p3; + unsigned long swcr_p0, swcr_p1, swcr_p2, swcr_p3; + + fpcr_p0 = fpcr_p1 = fpcr_p2 = fpcr_p3 = 0; + swcr_p0 = swcr_p1 = swcr_p2 = swcr_p3 = swcr; + /* manage fpcr_p0 */ + if (fex_p0) { + swcr_p0 |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p0 = fpcr; + fpcr_p0 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p0 |= ieee_swcr_to_fpcr(swcr_p0); + pr_debug("fex_p0: fpcr_p0:%#lx\n", fpcr_p0); + } + + if (fex_p1) { + swcr_p1 |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p1 = fpcr; + fpcr_p1 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p1 |= ieee_swcr_to_fpcr(swcr_p1); + pr_debug("fex_p1: fpcr_p1:%#lx\n", fpcr_p1); + } + + if (fex_p2) { + swcr_p2 |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p2 = fpcr; + fpcr_p2 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p2 |= ieee_swcr_to_fpcr(swcr_p2); + pr_debug("fex_p2: fpcr_p2:%#lx\n", fpcr_p2); + } + + if (fex_p3) { + swcr_p3 |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p3 = fpcr; + fpcr_p3 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p3 |= ieee_swcr_to_fpcr(swcr_p3); + pr_debug("fex_p3: fpcr_p3:%#lx\n", fpcr_p3); + } + + fpcr = fpcr_p0 | fpcr_p1 | fpcr_p2 | fpcr_p3; + pr_debug("fex_p0 = %#lx\n", fex_p0); + pr_debug("fex_p1 = %#lx\n", fex_p1); + pr_debug("fex_p2 = %#lx\n", fex_p2); + pr_debug("fex_p3 = %#lx\n", fex_p3); + pr_debug("SIMD emulation almost finished.before write fpcr = %#lx\n", fpcr); + wrfpcr(fpcr); + + pr_debug("Before write fp: vc_p0=%#lx, vc_p1=%#lx, vc_p2=%#lx, vc_p3=%#lx\n", vc_p0, vc_p1, vc_p2, vc_p3); + write_fp_reg_s(fc, vc_p0, vc_p1, vc_p2, vc_p3); + + /* Do we generate a signal? */ + _fex = (fex_p0 & swcr & IEEE_TRAP_ENABLE_MASK) | (fex_p1 & swcr & IEEE_TRAP_ENABLE_MASK) + | (fex_p2 & swcr & IEEE_TRAP_ENABLE_MASK) | (fex_p3 & swcr & IEEE_TRAP_ENABLE_MASK); + si_code = 0; + if (_fex) { + if (_fex & IEEE_TRAP_ENABLE_DNO) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_INE) + si_code = FPE_FLTRES; + if (_fex & IEEE_TRAP_ENABLE_UNF) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_OVF) + si_code = FPE_FLTOVF; + if (_fex & IEEE_TRAP_ENABLE_DZE) + si_code = FPE_FLTDIV; + if (_fex & IEEE_TRAP_ENABLE_INV) + si_code = FPE_FLTINV; + } + pr_debug("SIMD finished.. si_code:%#lx\n", si_code); + return si_code; + } + pr_debug("SIMD finished.. si_code:%#lx\n", si_code); + return 0; + +bad_insn: + pr_err("%s: Invalid FP insn %#x at %#lx\n", __func__, insn, pc); + return -1; + +} + +static inline unsigned long negative_value(unsigned long va) +{ + return (va ^ 0x8000000000000000UL); +} + +static inline unsigned long s_negative_value(unsigned long va) +{ + return (va ^ 0x80000000UL); +} + +/* + * sw64 mul-add floating emulation + */ +long mul_add_fp_emul(unsigned long pc) +{ + FP_DECL_EX; + FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SC); FP_DECL_S(S_TMP); FP_DECL_S(SR); + FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DC); FP_DECL_D(D_TMP); FP_DECL_D(DR); + FP_DECL_S(S_ZERO); + FP_DECL_D(D_ZERO); + FP_DECL_S(S_TMP2); + FP_DECL_D(D_TMP2); + + unsigned long fa, fb, fc, fd, func, mode, src; + unsigned long res, va, vb, vc, vd, vtmp, vtmp2, swcr, fpcr; + __u32 insn; + long si_code; + unsigned long vzero = 0; + + get_user(insn, (__u32 *)pc); + fd = (insn >> 0) & 0x1f; /* destination register */ + fc = (insn >> 5) & 0x1f; + fb = (insn >> 16) & 0x1f; + fa = (insn >> 21) & 0x1f; + func = (insn >> 10) & 0x3f; + + fpcr = rdfpcr(); + mode = (fpcr >> FPCR_DYN_SHIFT) & 0x3; + + pr_debug("===== Entering SW64 MUL-ADD Emulation =====\n"); + pr_debug("hardware fpcr = %#lx\n", fpcr); + swcr = swcr_update_status(current_thread_info()->ieee_state, fpcr); + pr_debug("software swcr = %#lx\n", swcr); + + if (func == FNC_FMAS || func == FNC_FMSS || func == FNC_FNMAS || func == FNC_FNMSS) { + va = sw64_read_fp_reg_s(fa); + vb = sw64_read_fp_reg_s(fb); + vc = sw64_read_fp_reg_s(fc); + FP_UNPACK_SP(SA, &va); + FP_UNPACK_SP(SB, &vb); + FP_UNPACK_SP(SC, &vc); + FP_UNPACK_SP(S_ZERO, &vzero); + } + if (func == FNC_FMAD || func == FNC_FMSD || func == FNC_FNMAD || func == FNC_FNMSD) { + va = sw64_read_fp_reg(fa); + vb = sw64_read_fp_reg(fb); + vc = sw64_read_fp_reg(fc); + FP_UNPACK_DP(DA, &va); + FP_UNPACK_DP(DB, &vb); + FP_UNPACK_DP(DC, &vc); + FP_UNPACK_DP(D_ZERO, &vzero); + } + pr_debug("va = %#lx, vb = %#lx, vc = %#lx\n", va, vb, vc); + switch (func) { + case FNC_FMAS: + FP_MUL_S(S_TMP, SA, SB); + FP_ADD_S(SR, S_TMP, SC); + goto pack_s; + case FNC_FMSS: + FP_MUL_S(S_TMP, SA, SB); + FP_SUB_S(SR, S_TMP, SC); + goto pack_s; + case FNC_FNMAS: /* (-va*vb) + vc */ + va = s_negative_value(va); + FP_UNPACK_SP(SA, &va); + FP_MUL_S(S_TMP, SA, SB); + FP_ADD_S(SR, S_TMP, SC); + goto pack_s; + case FNC_FNMSS: /* (-va*vb) - vc */ + va = s_negative_value(va); + FP_UNPACK_SP(SA, &va); + FP_MUL_S(S_TMP, SA, SB); + FP_SUB_S(SR, S_TMP, SC); + goto pack_s; + case FNC_FMAD: + FP_MUL_D(D_TMP, DA, DB); + FP_ADD_D(DR, D_TMP, DC); + goto pack_d; + case FNC_FMSD: + FP_MUL_D(D_TMP, DA, DB); + FP_SUB_D(DR, D_TMP, DC); + goto pack_d; + case FNC_FNMAD: + va = negative_value(va); + FP_UNPACK_DP(DA, &va); + FP_MUL_D(D_TMP, DA, DB); + FP_ADD_D(DR, D_TMP, DC); + goto pack_d; + case FNC_FNMSD: + va = negative_value(va); + FP_UNPACK_DP(DA, &va); + FP_MUL_D(D_TMP, DA, DB); + FP_SUB_D(DR, D_TMP, DC); + goto pack_d; + default: + goto bad_insn; + + } +pack_s: + FP_PACK_SP(&vd, SR); + if ((_fex & FP_EX_UNDERFLOW) && (swcr & IEEE_MAP_UMZ)) + vd = 0; + sw64_write_fp_reg_s(fd, vd); + goto done; + +pack_d: + FP_PACK_DP(&vd, DR); + if ((_fex & FP_EX_UNDERFLOW) && (swcr & IEEE_MAP_UMZ)) + vd = 0; + sw64_write_fp_reg(fd, vd); + +done: + pr_debug("vd = %#lx\n", vd); + if (_fex) { + /* Record exceptions in software control word. */ + swcr |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr |= ieee_swcr_to_fpcr(swcr); + wrfpcr(fpcr); /** wrfpcr will destroy vector register! */ + if (func == FNC_FMAS || func == FNC_FMSS || func == FNC_FNMAS || func == FNC_FNMSS) + sw64_write_fp_reg_s(fd, vd); + if (func == FNC_FMAD || func == FNC_FMSD || func == FNC_FNMAD || func == FNC_FNMSD) + sw64_write_fp_reg(fd, vd); + + /* Do we generate a signal? */ + _fex = _fex & swcr & IEEE_TRAP_ENABLE_MASK; + si_code = 0; + if (_fex) { + if (_fex & IEEE_TRAP_ENABLE_DNO) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_INE) + si_code = FPE_FLTRES; + if (_fex & IEEE_TRAP_ENABLE_UNF) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_OVF) + si_code = FPE_FLTOVF; + if (_fex & IEEE_TRAP_ENABLE_DZE) + si_code = FPE_FLTDIV; + if (_fex & IEEE_TRAP_ENABLE_INV) + si_code = FPE_FLTINV; + } + + return si_code; + } + + /* + * We used to write the destination register here, but DEC FORTRAN + * requires that the result *always* be written... so we do the write + * immediately after the operations above. + */ + + return 0; + +bad_insn: + pr_err("%s: Invalid FP insn %#x at %#lx\n", __func__, insn, pc); + return -1; +} + + +long simd_mul_add_fp_emul_s(unsigned long pc) +{ + FP_DECL_EX; + FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SC); FP_DECL_S(S_TMP); FP_DECL_S(SR); + FP_DECL_S(S_ZERO); + FP_DECL_S(S_TMP2); + + unsigned long fa, fb, fc, fd, func, mode, src; + unsigned long res, va, vb, vc, vd, vtmp, vtmp2, swcr, fpcr; + __u32 insn; + long si_code; + unsigned long vzero = 0; + + get_user(insn, (__u32 *)pc); + fd = (insn >> 0) & 0x1f; /* destination register */ + fc = (insn >> 5) & 0x1f; + fb = (insn >> 16) & 0x1f; + fa = (insn >> 21) & 0x1f; + func = (insn >> 10) & 0x3f; + + fpcr = rdfpcr(); + mode = (fpcr >> FPCR_DYN_SHIFT) & 0x3; + + unsigned long va_p0, va_p1, va_p2, va_p3; + unsigned long vb_p0, vb_p1, vb_p2, vb_p3; + unsigned long vc_p0, vc_p1, vc_p2, vc_p3; + unsigned long vd_p0, vd_p1, vd_p2, vd_p3; + unsigned long fex_p0, fex_p1, fex_p2, fex_p3; + + int working_part; + + pr_debug("======== Entering SIMD S-floating mul-add emulation =======\n"); + swcr = swcr_update_status(current_thread_info()->ieee_state, fpcr); + pr_debug("software swcr = %#lx\n", swcr); + pr_debug("hardware fpcr = %#lx\n", fpcr); + read_fp_reg_s(fa, &va_p0, &va_p1, &va_p2, &va_p3); + read_fp_reg_s(fb, &vb_p0, &vb_p1, &vb_p2, &vb_p3); + read_fp_reg_s(fc, &vc_p0, &vc_p1, &vc_p2, &vc_p3); + read_fp_reg_s(fd, &vd_p0, &vd_p1, &vd_p2, &vd_p3); + pr_debug("va_p0:%#lx, va_p1:%#lx, va_p2:%#lx, va_p3:%#lx\n", va_p0, va_p1, va_p2, va_p3); + pr_debug("vb_p0:%#lx, vb_p1:%#lx, vb_p2:%#lx, vb_p3:%#lx\n", vb_p0, vb_p1, vb_p2, vb_p3); + pr_debug("vc_p0:%#lx, vc_p1:%#lx, vc_p2:%#lx, vc_p3:%#lx\n", vc_p0, vc_p1, vc_p2, vc_p3); + pr_debug("vd_p0:%#lx, vd_p1:%#lx, vd_p2:%#lx, vd_p3:%#lx\n", vd_p0, vd_p1, vd_p2, vd_p3); + working_part = WORKING_PART_0; +simd_working: + _fex = 0; + switch (working_part) { + case WORKING_PART_0: + pr_debug("WORKING_PART_0\n"); + va = va_p0; + vb = vb_p0; + vc = vc_p0; + pr_debug("FPCR_STATUS_MASK0 : %#lx, fpcr :%#lx\n", FPCR_STATUS_MASK0, fpcr); + if ((fpcr & FPCR_STATUS_MASK0) == 0) { + SW64_FP_NORMAL_S(SA, &va); + SW64_FP_NORMAL_S(SB, &vb); + SW64_FP_NORMAL_S(SC, &vc); + if ((SA_c == SW64_FP_NORMAL) && (SB_c == SW64_FP_NORMAL) && (SC_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("LOW: SA_c = %#lx, SB_c = %#lx\n", SA_c, SB_c); + } else { + SW64_FP_NAN_S(SA, &va); + SW64_FP_NAN_S(SB, &vb); + if ((SA_c == SW64_FP_NAN) && (SB_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + case WORKING_PART_1: + pr_debug("WORKING_PART_1\n"); + va = va_p1; + vb = vb_p1; + vc = vc_p1; + pr_debug("FPCR_STATUS_MASK1 : %#lx, fpcr :%#lx\n", FPCR_STATUS_MASK0, fpcr); + if ((fpcr & FPCR_STATUS_MASK1) == 0) { + SW64_FP_NORMAL_S(SA, &va); + SW64_FP_NORMAL_S(SB, &vb); + SW64_FP_NORMAL_S(SC, &vc); + if ((SA_c == SW64_FP_NORMAL) && (SB_c == SW64_FP_NORMAL) && (SC_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("HIGH: SA_c = %#lx, SB_c = %#lx\n", SA_c, SB_c); + } else { + SW64_FP_NAN_S(SA, &va); + SW64_FP_NAN_S(SB, &vb); + if ((SA_c == SW64_FP_NAN) && (SB_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + case WORKING_PART_2: + pr_debug("WORKING_PART_2\n"); + va = va_p2; + vb = vb_p2; + vc = vc_p2; + if ((fpcr & FPCR_STATUS_MASK2) == 0) { + SW64_FP_NORMAL_S(SA, &va); + SW64_FP_NORMAL_S(SB, &vb); + SW64_FP_NORMAL_S(SC, &vc); + if ((SA_c == SW64_FP_NORMAL) && (SB_c == SW64_FP_NORMAL) && (SC_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("HIGH: SA_c = %#lx, SB_c = %#lx\n", SA_c, SB_c); + } else { + SW64_FP_NAN_S(SA, &va); + SW64_FP_NAN_S(SB, &vb); + if ((SA_c == SW64_FP_NAN) && (SB_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + case WORKING_PART_3: + pr_debug("WORKING_PART_3\n"); + va = va_p3; + vb = vb_p3; + vc = vc_p3; + if ((fpcr & FPCR_STATUS_MASK3) == 0) { + SW64_FP_NORMAL_S(SA, &va); + SW64_FP_NORMAL_S(SB, &vb); + SW64_FP_NORMAL_S(SC, &vc); + if ((SA_c == SW64_FP_NORMAL) && (SB_c == SW64_FP_NORMAL) && (SC_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("HIGH: SA_c = %#lx, SB_c = %#lx\n", SA_c, SB_c); + } else { + SW64_FP_NAN_S(SA, &va); + SW64_FP_NAN_S(SB, &vb); + if ((SA_c == SW64_FP_NAN) && (SB_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + } + + FP_UNPACK_SP(SA, &va); + FP_UNPACK_SP(SB, &vb); + FP_UNPACK_SP(SC, &vc); + FP_UNPACK_SP(S_ZERO, &vzero); + switch (func) { + case FNC_FMAS: + FP_MUL_S(S_TMP, SA, SB); + FP_ADD_S(SR, S_TMP, SC); + goto pack_s; + case FNC_FMSS: + FP_MUL_S(S_TMP, SA, SB); + FP_SUB_S(SR, S_TMP, SC); + goto pack_s; + case FNC_FNMAS: /* (-va*vb) + vc */ + va = s_negative_value(va); + FP_UNPACK_SP(SA, &va); + FP_MUL_S(S_TMP, SA, SB); + FP_ADD_S(SR, S_TMP, SC); + goto pack_s; + case FNC_FNMSS: /* (-va*vb) - vc */ + va = s_negative_value(va); + FP_UNPACK_SP(SA, &va); + FP_MUL_S(S_TMP, SA, SB); + FP_SUB_S(SR, S_TMP, SC); + goto pack_s; + default: + goto bad_insn; + } + +pack_s: + FP_PACK_SP(&vd, SR); + if ((_fex & FP_EX_UNDERFLOW) && (swcr & IEEE_MAP_UMZ)) + vd = 0; + pr_debug("SW64 SIMD Emulation S-floating _fex=%#lx, va=%#lx, vb=%#lx, vc=%#lx\n", _fex, va, vb, vc); + pr_debug("SW64 SIMD Emulation S-floating mode=%#lx,func=%#lx, swcr=%#lx\n", mode, func, swcr); +next_working_s: + switch (working_part) { + case WORKING_PART_0: + working_part = WORKING_PART_1; + vd_p0 = vd; + fex_p0 = _fex; + goto simd_working; + case WORKING_PART_1: + working_part = WORKING_PART_2; + vd_p1 = vd; + fex_p1 = _fex; + goto simd_working; + case WORKING_PART_2: + working_part = WORKING_PART_3; + vd_p2 = vd; + fex_p2 = _fex; + goto simd_working; + case WORKING_PART_3: + vd_p3 = vd; + fex_p3 = _fex; + goto done; + } +done: + if (fex_p0 || fex_p1 || fex_p2 || fex_p3) { + unsigned long fpcr_p0, fpcr_p1, fpcr_p2, fpcr_p3; + unsigned long swcr_p0, swcr_p1, swcr_p2, swcr_p3; + + fpcr_p0 = fpcr_p1 = fpcr_p2 = fpcr_p3 = 0; + swcr_p0 = swcr_p1 = swcr_p2 = swcr_p3 = swcr; + /* manage fpcr_p0 */ + if (fex_p0) { + swcr_p0 |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p0 = fpcr; + fpcr_p0 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p0 |= ieee_swcr_to_fpcr(swcr_p0); + } + + if (fex_p1) { + swcr_p1 |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p1 = fpcr; + fpcr_p1 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p1 |= ieee_swcr_to_fpcr(swcr_p1); + } + + if (fex_p2) { + swcr_p2 |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p2 = fpcr; + fpcr_p2 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p2 |= ieee_swcr_to_fpcr(swcr_p2); + } + + if (fex_p3) { + swcr_p3 |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p3 = fpcr; + fpcr_p3 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p3 |= ieee_swcr_to_fpcr(swcr_p3); + } + + fpcr = fpcr_p0 | fpcr_p1 | fpcr_p2 | fpcr_p3; + pr_debug("fex_p0 = %#lx\n", fex_p0); + pr_debug("fex_p1 = %#lx\n", fex_p1); + pr_debug("fex_p2 = %#lx\n", fex_p2); + pr_debug("fex_p3 = %#lx\n", fex_p3); + pr_debug("SIMD emulation almost finished.before write fpcr = %#lx\n", fpcr); + wrfpcr(fpcr); + pr_debug("Before write fp: vp_p0=%#lx, vc_p1=%#lx, vc_p2=%#lx, vc_p3=%#lx\n", vc_p0, vc_p1, vc_p2, vc_p3); + write_fp_reg_s(fd, vd_p0, vd_p1, vd_p2, vd_p3); /* write to fd */ + + /* Do we generate a signal? */ + _fex = (fex_p0 & swcr & IEEE_TRAP_ENABLE_MASK) | (fex_p1 & swcr & IEEE_TRAP_ENABLE_MASK) + | (fex_p2 & swcr & IEEE_TRAP_ENABLE_MASK) | (fex_p3 & swcr & IEEE_TRAP_ENABLE_MASK); + si_code = 0; + if (_fex) { + if (_fex & IEEE_TRAP_ENABLE_DNO) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_INE) + si_code = FPE_FLTRES; + if (_fex & IEEE_TRAP_ENABLE_UNF) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_OVF) + si_code = FPE_FLTOVF; + if (_fex & IEEE_TRAP_ENABLE_DZE) + si_code = FPE_FLTDIV; + if (_fex & IEEE_TRAP_ENABLE_INV) + si_code = FPE_FLTINV; + } + pr_debug("SIMD finished.. si_code:%#lx\n", si_code); + return si_code; + + } + pr_debug("SIMD finished.. si_code:%#lx\n", si_code); + return 0; + +bad_insn: + pr_err("%s: Invalid FP insn %#x at %#lx\n", __func__, insn, pc); + return -1; +} + +long simd_mul_add_fp_emul_d(unsigned long pc) +{ + FP_DECL_EX; + FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DC); FP_DECL_D(D_TMP); FP_DECL_D(DR); + FP_DECL_D(D_ZERO); + FP_DECL_D(D_TMP2); + + unsigned long fa, fb, fc, fd, func, mode, src; + unsigned long res, va, vb, vc, vd, vtmp, vtmp2, swcr, fpcr; + __u32 insn; + long si_code; + unsigned long vzero = 0; + + get_user(insn, (__u32 *)pc); + fd = (insn >> 0) & 0x1f; /* destination register */ + fc = (insn >> 5) & 0x1f; + fb = (insn >> 16) & 0x1f; + fa = (insn >> 21) & 0x1f; + func = (insn >> 10) & 0x3f; + + fpcr = rdfpcr(); + mode = (fpcr >> FPCR_DYN_SHIFT) & 0x3; + + unsigned long va_p0, va_p1, va_p2, va_p3; + unsigned long vb_p0, vb_p1, vb_p2, vb_p3; + unsigned long vc_p0, vc_p1, vc_p2, vc_p3; + unsigned long vd_p0, vd_p1, vd_p2, vd_p3; + unsigned long fex_p0, fex_p1, fex_p2, fex_p3; + + int working_part; + + pr_debug("======== Entering SIMD D-floating mul-add emulation =======\n"); + pr_debug("hardware fpcr = %#lx\n", fpcr); + swcr = swcr_update_status(current_thread_info()->ieee_state, fpcr); + pr_debug("software swcr = %#lx\n", swcr); + read_fp_reg_d(fa, &va_p0, &va_p1, &va_p2, &va_p3); + read_fp_reg_d(fb, &vb_p0, &vb_p1, &vb_p2, &vb_p3); + read_fp_reg_d(fc, &vc_p0, &vc_p1, &vc_p2, &vc_p3); + read_fp_reg_d(fd, &vd_p0, &vd_p1, &vd_p2, &vd_p3); + pr_debug("va_p0:%#lx, va_p1:%#lx, va_p2:%#lx, va_p3:%#lx\n", va_p0, va_p1, va_p2, va_p3); + pr_debug("vb_p0:%#lx, vb_p1:%#lx, vb_p2:%#lx, vb_p3:%#lx\n", vb_p0, vb_p1, vb_p2, vb_p3); + pr_debug("vc_p0:%#lx, vc_p1:%#lx, vc_p2:%#lx, vc_p3:%#lx\n", vc_p0, vc_p1, vc_p2, vc_p3); + pr_debug("vd_p0:%#lx, vd_p1:%#lx, vd_p2:%#lx, vd_p3:%#lx\n", vd_p0, vd_p1, vd_p2, vd_p3); + working_part = WORKING_PART_0; +simd_working: + _fex = 0; + switch (working_part) { + case WORKING_PART_0: + pr_debug("WORKING_PART_0\n"); + va = va_p0; + vb = vb_p0; + vc = vc_p0; + vd = vd_p0; + if ((fpcr & FPCR_STATUS_MASK0) == 0) { + SW64_FP_NORMAL_D(DA, &va); + SW64_FP_NORMAL_D(DB, &vb); + SW64_FP_NORMAL_D(DC, &vc); + if ((DA_c == SW64_FP_NORMAL) && (DB_c == SW64_FP_NORMAL) && (DC_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("LOW: DA_c = %#lx, DB_c = %#lx\n", DA_c, DB_c); + } else { + SW64_FP_NAN_D(DA, &va); + SW64_FP_NAN_D(DB, &vb); + SW64_FP_NAN_D(DC, &vc); + if ((DA_c == SW64_FP_NAN) && (DB_c == SW64_FP_NAN) && (DC_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + case WORKING_PART_1: + pr_debug("WORKING_PART_1\n"); + va = va_p1; + vb = vb_p1; + vc = vc_p1; + vd = vd_p1; + if ((fpcr & FPCR_STATUS_MASK1) == 0) { + SW64_FP_NORMAL_D(DA, &va); + SW64_FP_NORMAL_D(DB, &vb); + SW64_FP_NORMAL_D(DC, &vc); + if ((DA_c == SW64_FP_NORMAL) && (DB_c == SW64_FP_NORMAL) && (DC_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("HIGH: DA_c = %#lx, DB_c = %#lx\n", DA_c, DB_c); + } else { + SW64_FP_NAN_D(DA, &va); + SW64_FP_NAN_D(DB, &vb); + SW64_FP_NAN_D(DC, &vc); + if ((DA_c == SW64_FP_NAN) && (DB_c == SW64_FP_NAN) && (DC_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + case WORKING_PART_2: + pr_debug("WORKING_PART_2\n"); + va = va_p2; + vb = vb_p2; + vc = vc_p2; + vd = vd_p2; + if ((fpcr & FPCR_STATUS_MASK2) == 0) { + SW64_FP_NORMAL_D(DA, &va); + SW64_FP_NORMAL_D(DB, &vb); + SW64_FP_NORMAL_D(DC, &vc); + if ((DA_c == SW64_FP_NORMAL) && (DB_c == SW64_FP_NORMAL) && (DC_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("HIGH: DA_c = %#lx, DB_c = %#lx\n", DA_c, DB_c); + } else { + SW64_FP_NAN_D(DA, &va); + SW64_FP_NAN_D(DB, &vb); + SW64_FP_NAN_D(DC, &vc); + if ((DA_c == SW64_FP_NAN) && (DB_c == SW64_FP_NAN) && (DC_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + case WORKING_PART_3: + pr_debug("WORKING_PART_3\n"); + va = va_p3; + vb = vb_p3; + vc = vc_p3; + vd = vd_p3; + if ((fpcr & FPCR_STATUS_MASK3) == 0) { + SW64_FP_NORMAL_D(DA, &va); + SW64_FP_NORMAL_D(DB, &vb); + SW64_FP_NORMAL_D(DC, &vc); + if ((DA_c == SW64_FP_NORMAL) && (DB_c == SW64_FP_NORMAL) && (DC_c == SW64_FP_NORMAL)) + goto next_working_s; + else + pr_debug("HIGH: DA_c = %#lx, DB_c = %#lx\n", DA_c, DB_c); + } else { + SW64_FP_NAN_D(DA, &va); + SW64_FP_NAN_D(DB, &vb); + SW64_FP_NAN_D(DC, &vc); + if ((DA_c == SW64_FP_NAN) && (DB_c == SW64_FP_NAN) && (DC_c == SW64_FP_NAN)) + goto next_working_s; + } + break; + } + + FP_UNPACK_DP(DA, &va); + FP_UNPACK_DP(DB, &vb); + FP_UNPACK_DP(DC, &vc); + FP_UNPACK_DP(D_ZERO, &vzero); + + switch (func) { + case FNC_FMAD: + FP_MUL_D(D_TMP, DA, DB); + FP_ADD_D(DR, D_TMP, DC); + goto pack_d; + case FNC_FMSD: + FP_MUL_D(D_TMP, DA, DB); + FP_SUB_D(DR, D_TMP, DC); + goto pack_d; + case FNC_FNMAD: + va = negative_value(va); + FP_UNPACK_DP(DA, &va); + FP_MUL_D(D_TMP, DA, DB); + FP_ADD_D(DR, D_TMP, DC); + goto pack_d; + case FNC_FNMSD: + va = negative_value(va); + FP_UNPACK_DP(DA, &va); + FP_MUL_D(D_TMP, DA, DB); + FP_SUB_D(DR, D_TMP, DC); + + goto pack_d; + default: + goto bad_insn; + } + +pack_d: + FP_PACK_DP(&vd, DR); + if ((_fex & FP_EX_UNDERFLOW) && (swcr & IEEE_MAP_UMZ)) + vd = 0; + pr_debug("SW64 SIMD Emulation D-floating _fex=%#lx, va=%#lx, vb=%#lx, vc=%#lx\n", _fex, va, vb, vc); + pr_debug("SW64 SIMD Emulation D-floating mode=%#lx,func=%#lx, swcr=%#lx\n", mode, func, swcr); +next_working_s: + switch (working_part) { + case WORKING_PART_0: + working_part = WORKING_PART_1; + vd_p0 = vd; + fex_p0 = _fex; + goto simd_working; + case WORKING_PART_1: + working_part = WORKING_PART_2; + vd_p1 = vd; + fex_p1 = _fex; + goto simd_working; + case WORKING_PART_2: + working_part = WORKING_PART_3; + vd_p2 = vd; + fex_p2 = _fex; + goto simd_working; + case WORKING_PART_3: + vd_p3 = vd; + fex_p3 = _fex; + goto done; + } +done: + if (fex_p0 || fex_p1 || fex_p2 || fex_p3) { + unsigned long fpcr_p0, fpcr_p1, fpcr_p2, fpcr_p3; + unsigned long swcr_p0, swcr_p1, swcr_p2, swcr_p3; + + fpcr_p0 = fpcr_p1 = fpcr_p2 = fpcr_p3 = 0; + swcr_p0 = swcr_p1 = swcr_p2 = swcr_p3 = swcr; + /* manage fpcr_p0 */ + if (fex_p0) { + swcr_p0 |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p0 = fpcr; + fpcr_p0 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p0 |= ieee_swcr_to_fpcr(swcr_p0); + } + + if (fex_p1) { + swcr_p1 |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p1 = fpcr; + fpcr_p1 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p1 |= ieee_swcr_to_fpcr(swcr_p1); + } + + if (fex_p2) { + swcr_p2 |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p2 = fpcr; + fpcr_p2 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p2 |= ieee_swcr_to_fpcr(swcr_p2); + } + + if (fex_p3) { + swcr_p3 |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + current_thread_info()->ieee_state + |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + + /* Update hardware control register. */ + fpcr_p3 = fpcr; + fpcr_p3 &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr_p3 |= ieee_swcr_to_fpcr(swcr_p3); + } + + fpcr = fpcr_p0 | fpcr_p1 | fpcr_p2 | fpcr_p3; + pr_debug("fex_p0 = %#lx\n", fex_p0); + pr_debug("fex_p1 = %#lx\n", fex_p1); + pr_debug("fex_p2 = %#lx\n", fex_p2); + pr_debug("fex_p3 = %#lx\n", fex_p3); + pr_debug("SIMD emulation almost finished.before write fpcr = %#lx\n", fpcr); + wrfpcr(fpcr); + + pr_debug("Before write fp: vp_p0=%#lx, vc_p1=%#lx, vc_p2=%#lx, vc_p3=%#lx\n", vc_p0, vc_p1, vc_p2, vc_p3); + write_fp_reg_d(fd, vd_p0, vd_p1, vd_p2, vd_p3); /* write to fd */ + + /* Do we generate a signal? */ + _fex = (fex_p0 & swcr & IEEE_TRAP_ENABLE_MASK) | (fex_p1 & swcr & IEEE_TRAP_ENABLE_MASK) + | (fex_p2 & swcr & IEEE_TRAP_ENABLE_MASK) | (fex_p3 & swcr & IEEE_TRAP_ENABLE_MASK); + si_code = 0; + if (_fex) { + if (_fex & IEEE_TRAP_ENABLE_DNO) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_INE) + si_code = FPE_FLTRES; + if (_fex & IEEE_TRAP_ENABLE_UNF) + si_code = FPE_FLTUND; + if (_fex & IEEE_TRAP_ENABLE_OVF) + si_code = FPE_FLTOVF; + if (_fex & IEEE_TRAP_ENABLE_DZE) + si_code = FPE_FLTDIV; + if (_fex & IEEE_TRAP_ENABLE_INV) + si_code = FPE_FLTINV; + } + pr_debug("SIMD finished.. si_code:%#lx\n", si_code); + return si_code; + } + pr_debug("SIMD finished.. si_code:%#lx\n", si_code); + return 0; + +bad_insn: + pr_err("%s: Invalid FP insn %#x at %#lx\n", __func__, insn, pc); + return -1; +} + +void read_fp_reg_s(unsigned long reg, unsigned long *val_p0, + unsigned long *val_p1, unsigned long *val_p2, unsigned long *val_p3) +{ + unsigned long fp[2]; + + sw64_read_simd_fp_m_s(reg, fp); + *val_p0 = fp[0] & 0xffffffffUL; + *val_p1 = (fp[0] >> 32) & 0xffffffffUL; + *val_p2 = fp[1] & 0xffffffffUL; + *val_p3 = (fp[1] >> 32) & 0xffffffffUL; +} + +void read_fp_reg_d(unsigned long reg, unsigned long *val_p0, + unsigned long *val_p1, unsigned long *val_p2, unsigned long *val_p3) +{ + unsigned long fp[4]; + + sw64_read_simd_fp_m_d(reg, fp); + *val_p0 = fp[0]; + *val_p1 = fp[1]; + *val_p2 = fp[2]; + *val_p3 = fp[3]; +} + +void write_fp_reg_s(unsigned long reg, unsigned long val_p0, + unsigned long val_p1, unsigned long val_p2, unsigned long val_p3) +{ + unsigned long fp[2]; + + fp[0] = ((val_p1 & 0xffffffffUL) << 32) | (val_p0 & 0xffffffffUL); + fp[1] = ((val_p3 & 0xffffffffUL) << 32) | (val_p2 & 0xffffffffUL); + sw64_write_simd_fp_reg_s(reg, fp[0], fp[1]); +} + +void write_fp_reg_d(unsigned long reg, unsigned long val_p0, + unsigned long val_p1, unsigned long val_p2, unsigned long val_p3) +{ + sw64_write_simd_fp_reg_d(reg, val_p0, val_p1, val_p2, val_p3); +} diff --git a/arch/sw_64/math-emu/qrnnd.S b/arch/sw_64/math-emu/qrnnd.S new file mode 100644 index 000000000000..1e732f2e68c0 --- /dev/null +++ b/arch/sw_64/math-emu/qrnnd.S @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + # __udiv_qrnnd + # Copyright (C) 1992, 1994, 1995, 2000 Free Software Foundation, Inc. + + # This file is part of GCC. + + .set noreorder + .set noat + + .text + + .globl __udiv_qrnnd + .ent __udiv_qrnnd +__udiv_qrnnd: + .frame $30, 0, $26, 0 + .prologue 0 + + # ldiq $2,16 + ldi $2, 16($31) + blt $19, $largedivisor + +$loop1: cmplt $18, 0, $3 + addl $17, $17, $17 + bis $17, $3, $17 + addl $18, $18, $18 + cmpule $19, $17, $20 + subl $17, $19, $3 + selne $20, $3, $17, $17 + bis $18, $20, $18 + cmplt $18, 0, $3 + addl $17, $17, $17 + bis $17, $3, $17 + addl $18, $18, $18 + cmpule $19, $17, $20 + subl $17, $19, $3 + selne $20, $3, $17, $17 + bis $18, $20, $18 + cmplt $18, 0, $3 + addl $17, $17, $17 + bis $17, $3, $17 + addl $18, $18, $18 + cmpule $19, $17, $20 + subl $17, $19, $3 + selne $20, $3, $17, $17 + bis $18, $20, $18 + cmplt $18, 0, $3 + addl $17, $17, $17 + bis $17, $3, $17 + addl $18, $18, $18 + cmpule $19, $17, $20 + subl $17, $19, $3 + selne $20, $3, $17, $17 + bis $18, $20, $18 + subl $2, 1, $2 + bgt $2, $loop1 + stl $17, 0($16) + bis $31, $18, $0 + ret $31, ($26), 1 + +$largedivisor: + and $18, 1, $4 + + srl $18, 1, $18 + sll $17, 63, $3 + or $3, $18, $18 + srl $17, 1, $17 + + and $19, 1, $6 + srl $19, 1, $5 + addl $5, $6, $5 + +$loop2: cmplt $18, 0, $3 + addl $17, $17, $17 + bis $17, $3, $17 + addl $18, $18, $18 + cmpule $5, $17, $20 + subl $17, $5, $3 + selne $20, $3, $17, $17 + bis $18, $20, $18 + cmplt $18, 0, $3 + addl $17, $17, $17 + bis $17, $3, $17 + addl $18, $18, $18 + cmpule $5, $17, $20 + subl $17, $5, $3 + selne $20, $3, $17, $17 + bis $18, $20, $18 + cmplt $18, 0, $3 + addl $17, $17, $17 + bis $17, $3, $17 + addl $18, $18, $18 + cmpule $5, $17, $20 + subl $17, $5, $3 + selne $20, $3, $17, $17 + bis $18, $20, $18 + cmplt $18, 0, $3 + addl $17, $17, $17 + bis $17, $3, $17 + addl $18, $18, $18 + cmpule $5, $17, $20 + subl $17, $5, $3 + selne $20, $3, $17, $17 + bis $18, $20, $18 + subl $2, 1, $2 + bgt $2, $loop2 + + addl $17, $17, $17 + addl $4, $17, $17 + bne $6, $Odd + stl $17, 0($16) + bis $31, $18, $0 + ret $31, ($26), 1 + +$Odd: + # q' in $18. r' in $17 + addl $17, $18, $17 + + cmpult $17, $18, $3 # $3 := carry from addl + subl $17, $19, $at + addl $18, $3, $18 + selne $3, $at, $17, $17 + + cmpult $17, $19, $3 + addl $18, 1, $at + seleq $3, $at, $18, $18 + subl $17, $19, $at + seleq $3, $at, $17, $17 + + stl $17, 0($16) + bis $31, $18, $0 + ret $31, ($26), 1 + + .end __udiv_qrnnd diff --git a/arch/sw_64/math-emu/sfp-util.h b/arch/sw_64/math-emu/sfp-util.h new file mode 100644 index 000000000000..0769c0223e0d --- /dev/null +++ b/arch/sw_64/math-emu/sfp-util.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SW64_MATH_EMU_SFP_UTIL_H +#define _SW64_MATH_EMU_SFP_UTIL_H + +#include +#include +#include +#include +#include + +#define add_ssaaaa(sh, sl, ah, al, bh, bl) \ + ((sl) = (al) + (bl), (sh) = (ah) + (bh) + ((sl) < (al))) + +#define sub_ddmmss(sh, sl, ah, al, bh, bl) \ + ((sl) = (al) - (bl), (sh) = (ah) - (bh) - ((al) < (bl))) + +#define umul_ppmm(wh, wl, u, v) \ + __asm__ ("mull %2, %3, %1; umulh %2, %3, %0" \ + : "=r" ((UDItype)(wh)), \ + "=&r" ((UDItype)(wl)) \ + : "r" ((UDItype)(u)), \ + "r" ((UDItype)(v))) + +#define udiv_qrnnd(q, r, n1, n0, d) \ +do { unsigned long __r; \ + (q) = __udiv_qrnnd(&__r, (n1), (n0), (d)); \ + (r) = __r; \ +} while (0) +extern unsigned long __udiv_qrnnd(unsigned long *, unsigned long, + unsigned long, unsigned long); + +#define UDIV_NEEDS_NORMALIZATION 1 + +#define abort() goto bad_insn + +#ifndef __LITTLE_ENDIAN +#define __LITTLE_ENDIAN -1 +#endif +#define __BYTE_ORDER __LITTLE_ENDIAN + +#endif /* _SW64_MATH_EMU_SFP_UTIL_H */ -- Gitee From 2e3eacc535fa0c4d5a4668f901bfa8bcbc520e21 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:15 +0800 Subject: [PATCH 019/524] sw64: add basic IO support Add basic IO support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/early_ioremap.h | 29 ++ arch/sw_64/include/asm/io.h | 288 ++++++++++++++++ arch/sw_64/include/asm/sw64io.h | 109 ++++++ arch/sw_64/include/asm/uncore_io_junzhang.h | 201 +++++++++++ .../include/asm/uncore_io_ops_junzhang.h | 39 +++ .../sw_64/include/asm/uncore_io_ops_xuelang.h | 65 ++++ arch/sw_64/include/asm/uncore_io_xuelang.h | 323 ++++++++++++++++++ 7 files changed, 1054 insertions(+) create mode 100644 arch/sw_64/include/asm/early_ioremap.h create mode 100644 arch/sw_64/include/asm/io.h create mode 100644 arch/sw_64/include/asm/sw64io.h create mode 100644 arch/sw_64/include/asm/uncore_io_junzhang.h create mode 100644 arch/sw_64/include/asm/uncore_io_ops_junzhang.h create mode 100644 arch/sw_64/include/asm/uncore_io_ops_xuelang.h create mode 100644 arch/sw_64/include/asm/uncore_io_xuelang.h diff --git a/arch/sw_64/include/asm/early_ioremap.h b/arch/sw_64/include/asm/early_ioremap.h new file mode 100644 index 000000000000..172b96a401cb --- /dev/null +++ b/arch/sw_64/include/asm/early_ioremap.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_EARLY_IOREMAP_H +#define _ASM_SW64_EARLY_IOREMAP_H + +#include +#include + +static inline void __iomem * +early_ioremap(unsigned long phys_addr, unsigned long size) +{ + unsigned long y = 0; + + if (phys_addr >= __START_KERNEL_map) { + y = (unsigned long) phys_to_virt(__pa(phys_addr)); + } else { + y = phys_addr; + y |= PAGE_OFFSET; + } + + return (void __iomem *) y; +} +#define early_memremap(phys_addr, size) early_ioremap(phys_addr, size) + +static inline void early_iounmap(volatile void __iomem *addr, unsigned long size) +{ +} +#define early_memunmap(addr, size) early_iounmap(addr, size) + +#endif /* _ASM_SW64_EARLY_IOREMAP_H */ diff --git a/arch/sw_64/include/asm/io.h b/arch/sw_64/include/asm/io.h new file mode 100644 index 000000000000..2b045be5257e --- /dev/null +++ b/arch/sw_64/include/asm/io.h @@ -0,0 +1,288 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_IO_H +#define _ASM_SW64_IO_H + +#ifdef __KERNEL__ + +#include +#include +#include +#include + +/* The generic header contains only prototypes. Including it ensures that + * the implementation we have here matches that interface. + */ +#include + +/* We don't use IO slowdowns on the sw64, but.. */ +#define __SLOW_DOWN_IO do { } while (0) +#define SLOW_DOWN_IO do { } while (0) + +#define page_to_phys(page) page_to_pa(page) + +/* Maximum PIO space address supported? */ +#define IO_SPACE_LIMIT 0xffffffffffffffff + +/* + * Generic IO read/write. These perform native-endian accesses. + */ + +#define __raw_writeb __raw_writeb +static inline void __raw_writeb(u8 val, volatile void __iomem *addr) +{ + asm volatile("stb %0, 0(%1)" : : "r" (val), "r" (addr)); +} + +#define __raw_writew __raw_writew +static inline void __raw_writew(u16 val, volatile void __iomem *addr) +{ + asm volatile("sth %0, 0(%1)" : : "r" (val), "r" (addr)); +} + +#define __raw_writel __raw_writel +static inline void __raw_writel(u32 val, volatile void __iomem *addr) +{ + asm volatile("stw %0, 0(%1)" : : "r" (val), "r" (addr)); +} + +#define __raw_writeq __raw_writeq +static inline void __raw_writeq(u64 val, volatile void __iomem *addr) +{ + asm volatile("stl %0, 0(%1)" : : "r" (val), "r" (addr)); +} + +#define __raw_readb __raw_readb +static inline u8 __raw_readb(const volatile void __iomem *addr) +{ + u8 val; + + asm volatile("ldbu %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} + +#define __raw_readw __raw_readw +static inline u16 __raw_readw(const volatile void __iomem *addr) +{ + u16 val; + + asm volatile("ldhu %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} + +#define __raw_readl __raw_readl +static inline u32 __raw_readl(const volatile void __iomem *addr) +{ + u32 val; + + asm volatile("ldw %0, 0(%1)\n" + "zapnot %0, 0xf, %0\n" + : "=r" (val) : "r" (addr)); + return val; +} + +#define __raw_readq __raw_readq +static inline u64 __raw_readq(const volatile void __iomem *addr) +{ + u64 val; + + asm volatile("ldl %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} + +/* IO barriers */ + +#define __iormb() rmb() +#define __iowmb() wmb() +#define mmiowb() do { } while (0) + +/* + * Relaxed I/O memory access primitives. These follow the Device memory + * ordering rules but do not guarantee any ordering relative to Normal memory + * accesses. + */ +#define readb_relaxed(c) __raw_readb(c) +#define readw_relaxed(c) __raw_readw(c) +#define readl_relaxed(c) __raw_readl(c) +#define readq_relaxed(c) __raw_readq(c) + +#define writeb_relaxed(v, c) __raw_writeb((v), (c)) +#define writew_relaxed(v, c) __raw_writew((v), (c)) +#define writel_relaxed(v, c) __raw_writel((v), (c)) +#define writeq_relaxed(v, c) __raw_writeq((v), (c)) + +/* + * I/O memory access primitives. Reads are ordered relative to any + * following Normal memory access. Writes are ordered relative to any prior + * Normal memory access. + */ +#define readb(c) ({ u8 __v = readb_relaxed(c); __iormb(); __v; }) +#define readw(c) ({ u16 __v = readw_relaxed(c); __iormb(); __v; }) +#define readl(c) ({ u32 __v = readl_relaxed(c); __iormb(); __v; }) +#define readq(c) ({ u64 __v = readq_relaxed(c); __iormb(); __v; }) + +#define writeb(v, c) ({ __iowmb(); writeb_relaxed((v), (c)); }) +#define writew(v, c) ({ __iowmb(); writew_relaxed((v), (c)); }) +#define writel(v, c) ({ __iowmb(); writel_relaxed((v), (c)); }) +#define writeq(v, c) ({ __iowmb(); writeq_relaxed((v), (c)); }) +/* + * We always have external versions of these routines. + */ +extern u8 inb(unsigned long port); +extern u16 inw(unsigned long port); +extern u32 inl(unsigned long port); +extern void outb(u8 b, unsigned long port); +extern void outw(u16 b, unsigned long port); +extern void outl(u32 b, unsigned long port); +#define inb inb +#define inw inw +#define inl inl +#define outb outb +#define outw outw +#define outl outl + +static inline void __iomem *__ioremap(phys_addr_t addr, size_t size, + pgprot_t prot) +{ + unsigned long tmp = addr | PAGE_OFFSET; + + return (void __iomem *)(tmp); +} + +#define ioremap(addr, size) __ioremap((addr), (size), PAGE_KERNEL) +#define ioremap_nocache(addr, size) __ioremap((addr), (size), PAGE_KERNEL) +#define ioremap_cache(addr, size) __ioremap((addr), (size), PAGE_KERNEL) +#define ioremap_uc ioremap_nocache + +#define ioport_map ioport_map +#define ioport_unmap ioport_unmap + +static inline void __iounmap(volatile void __iomem *addr) +{ +} + +#define iounmap __iounmap + +#define ioread16be(p) be16_to_cpu(ioread16(p)) +#define ioread32be(p) be32_to_cpu(ioread32(p)) +#define iowrite16be(v, p) iowrite16(cpu_to_be16(v), (p)) +#define iowrite32be(v, p) iowrite32(cpu_to_be32(v), (p)) + +#define inb_p inb +#define inw_p inw +#define inl_p inl +#define outb_p outb +#define outw_p outw +#define outl_p outl + + +/* + * String version of IO memory access ops: + */ +#define memcpy_fromio memcpy_fromio +extern void memcpy_fromio(void *buffer, const volatile void __iomem *addr, long len); + +#define memcpy_toio memcpy_toio +extern void memcpy_toio(volatile void __iomem *addr, const void *buffer, long len); + +extern void _memset_c_io(volatile void __iomem *addr, unsigned long c, long len); + +#define memset_io memset_io +static inline void memset_io(volatile void __iomem *addr, u8 c, long len) +{ + _memset_c_io(addr, 0x0101010101010101UL * c, len); +} + +static inline void memsetw_io(volatile void __iomem *addr, u16 c, long len) +{ + _memset_c_io(addr, 0x0001000100010001UL * c, len); +} + +/* + * String versions of in/out ops: + */ +extern void insb(unsigned long port, void *dst, unsigned long count); +extern void insw(unsigned long port, void *dst, unsigned long count); +extern void insl(unsigned long port, void *dst, unsigned long count); +extern void outsb(unsigned long port, const void *src, unsigned long count); +extern void outsw(unsigned long port, const void *src, unsigned long count); +extern void outsl(unsigned long port, const void *src, unsigned long count); + +#define insb insb +#define insw insw +#define insl insl +#define outsb outsb +#define outsw outsw +#define outsl outsl + +/* + * These defines will override the defaults when doing RTC queries + */ + +#define RTC_PORT(x) (0x70 + (x)) +#define RTC_ALWAYS_BCD 0 + +/* + * Convert a physical pointer to a virtual kernel pointer for /dev/mem + * access + */ +#define xlate_dev_mem_ptr(p) __va(p) + +/* + * Convert a virtual cached pointer to an uncached pointer + */ +#define xlate_dev_kmem_ptr(p) p + +/* + * These get provided from since sw64 does not + * select GENERIC_IOMAP. + */ +#define ioread8 ioread8 +#define ioread16 ioread16 +#define ioread32 ioread32 +#define ioread64 ioread64 +#define iowrite8 iowrite8 +#define iowrite16 iowrite16 +#define iowrite32 iowrite32 +#define iowrite64 iowrite64 +#define ioread64be ioread64be +#define iowrite64be iowrite64be +#define ioread8_rep ioread8_rep +#define ioread16_rep ioread16_rep +#define ioread32_rep ioread32_rep +#define iowrite8_rep iowrite8_rep +#define iowrite16_rep iowrite16_rep +#define iowrite32_rep iowrite32_rep +#define pci_iounmap pci_iounmap + +#include + +/* + * Change addresses as seen by the kernel (virtual) to addresses as + * seen by a device (bus), and vice versa. + * + * Note that this only works for a limited range of kernel addresses, + * and very well may not span all memory. Consider this interface + * deprecated in favour of the DMA-mapping API. + */ +static inline unsigned long __deprecated virt_to_bus(void *address) +{ + return virt_to_phys(address); +} +#define isa_virt_to_bus virt_to_bus + +static inline void * __deprecated bus_to_virt(unsigned long address) +{ + void *virt; + + /* This check is a sanity check but also ensures that bus address 0 + * maps to virtual address 0 which is useful to detect null pointers + * (the NCR driver is much simpler if NULL pointers are preserved). + */ + virt = phys_to_virt(address); + return (long)address <= 0 ? NULL : virt; +} +#define isa_bus_to_virt bus_to_virt + +#endif /* __KERNEL__ */ + +#endif /* _ASM_SW64_IO_H */ diff --git a/arch/sw_64/include/asm/sw64io.h b/arch/sw_64/include/asm/sw64io.h new file mode 100644 index 000000000000..d52cd8cc86bf --- /dev/null +++ b/arch/sw_64/include/asm/sw64io.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SW64IO_H +#define _ASM_SW64_SW64IO_H + +#include +#include + +#if defined(CONFIG_UNCORE_XUELANG) +#include +#endif + +#if defined(CONFIG_UNCORE_JUNZHANG) +#include +#endif + +#define MK_RC_CFG(nid, idx) \ + (SW64_PCI_IO_BASE((nid), (idx)) | PCI_RC_CFG) +#define MK_PIU_IOR0(nid, idx) \ + (SW64_PCI_IO_BASE((nid), (idx)) | PCI_IOR0_BASE) +#define MK_PIU_IOR1(nid, idx) \ + (SW64_PCI_IO_BASE((nid), (idx)) | PCI_IOR1_BASE) + +static inline unsigned int +read_rc_conf(unsigned long node, unsigned long rc, + unsigned int offset) +{ + void __iomem *addr; + + addr = __va(MK_RC_CFG(node, rc) | offset); + return readl(addr); +} + +static inline void +write_rc_conf(unsigned long node, unsigned long rc, + unsigned int offset, unsigned int data) +{ + void __iomem *addr; + + addr = __va(MK_RC_CFG(node, rc) | offset); + writel(data, addr); +} + +static inline unsigned long +read_piu_ior0(unsigned long node, unsigned long rc, + unsigned int reg) +{ + void __iomem *addr; + + addr = __va(MK_PIU_IOR0(node, rc) + reg); + return readq(addr); +} + +static inline void +write_piu_ior0(unsigned long node, unsigned long rc, + unsigned int reg, unsigned long data) +{ + void __iomem *addr; + + addr = __va(MK_PIU_IOR0(node, rc) + reg); + writeq(data, addr); +} + +static inline unsigned long +read_piu_ior1(unsigned long node, unsigned long rc, + unsigned int reg) +{ + void __iomem *addr; + + addr = __va(MK_PIU_IOR1(node, rc) + reg); + return readq(addr); +} + +static inline void +write_piu_ior1(unsigned long node, unsigned long rc, + unsigned int reg, unsigned long data) +{ + void __iomem *addr; + + addr = __va(MK_PIU_IOR1(node, rc) + reg); + writeq(data, addr); +} + +static inline unsigned long +sw64_io_read(unsigned long node, unsigned long reg) +{ + void __iomem *addr; + + addr = __va(SW64_IO_BASE(node) | reg); + return readq(addr); +} + +static inline void +sw64_io_write(unsigned long node, unsigned long reg, unsigned long data) +{ + void __iomem *addr; + + addr = __va(SW64_IO_BASE(node) | reg); + writeq(data, addr); +} + +#if defined(CONFIG_UNCORE_XUELANG) +#include +#endif + +#if defined(CONFIG_UNCORE_JUNZHANG) +#include +#endif + +#endif /* _ASM_SW64_SW64IO_H */ diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h new file mode 100644 index 000000000000..37cfe1fd6807 --- /dev/null +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNCORE_IO_JUNZHANG_H +#define _ASM_SW64_UNCORE_IO_JUNZHANG_H + +#include + +#define IO_BASE (0x1UL << 47) +#define PCI_BASE (0x1UL << 43) +#define PCI_IOR0_BASE (0x2UL << 32) +#define PCI_IOR1_BASE (0x3UL << 32) + +#define PCI_RC_CFG (0x5UL << 32) + +#define PCI_EP_CFG (0x3UL << 33) +#define PCI_LEGACY_IO (0x1UL << 32) +#define PCI_LEGACY_IO_SIZE (0x100000000UL) +#define PCI_MEM_UNPRE 0x0UL +#define PCI_32BIT_VT_MEMIO (0xc0000000UL) +#define PCI_32BIT_MEMIO (0xe0000000UL) +#define PCI_32BIT_MEMIO_SIZE (0x20000000UL) +#define PCI_64BIT_MEMIO (0x1UL << 39) +#define PCI_64BIT_MEMIO_SIZE (0x8000000000UL) + +#define IO_RC_SHIFT 40 +#define IO_NODE_SHIFT 44 +#define IO_MARK_BIT 47 + +#define VT_MAX_CPUS_SHIFT 0 +#define VT_MAX_CPUS_MASK 0x3ff +#define VT_CORES_SHIFT 10 +#define VT_CORES_MASK 0x3ff +#define VT_THREADS_SHIFT 20 +#define VT_THREADS_MASK 0xfff + +#define QEMU_PRINTF_BUFF_BASE (IO_BASE | SPBU_BASE | 0x40000UL) + +/* MSIConfig */ +#define MSICONFIG_VALID (0x1UL << 63) +#define MSICONFIG_EN (0x1UL << 62) +#define MSICONFIG_VECTOR_SHIFT 10 + +#define MSIX_MSG_ADDR (0xfff00000UL) + +#define SW64_PCI_IO_BASE(m, n) \ + (IO_BASE | ((m) << IO_NODE_SHIFT) | PCI_BASE | ((n) << IO_RC_SHIFT)) +#define SW64_IO_BASE(x) (IO_BASE | ((x) << IO_NODE_SHIFT)) + +#define SW64_PCI0_BUS 0 +#define PCI0_BUS SW64_PCI0_BUS + +#define MAX_NR_NODES 0x2 +#define MAX_NR_RCS 0x6 + +#define SPBU_BASE (0x3UL << 36) +#define INTPU_BASE (0x3aUL << 32) +#define IIC0_BASE (0x31UL << 32) +#define SPI_BASE (0x32UL << 32) +#define UART_BASE (0x33UL << 32) +#define IIC1_BASE (0x34UL << 32) +#define IIC2_BASE (0x35UL << 32) +#define GPIO_BASE (0x36UL << 32) +#define LPC_BASE (0x37UL << 32) +#define LPC_LEGACY_IO (0x1UL << 28 | IO_BASE | LPC_BASE) +#define LPC_MEM_IO (0x2UL << 28 | IO_BASE | LPC_BASE) +#define LPC_FIRMWARE_IO (0x3UL << 28 | IO_BASE | LPC_BASE) +#define PCI_VT_LEGACY_IO (IO_BASE | PCI_BASE | PCI_LEGACY_IO) + +#define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) +#define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) + +#define PIUCONFIG0_INIT_VAL 0x38016 + +/*-----------------------addr-----------------------*/ +/* INTPU REG */ +enum { + DEVINT_MISS = INTPU_BASE | 0x100UL, + MT_INT_CONFIG = INTPU_BASE | 0x300UL, + DEV_INT_CONFIG = INTPU_BASE | 0x480UL, + FMT_ERR = INTPU_BASE | 0x700UL, + FAULT_INT_CONFIG = INTPU_BASE | 0x780UL, + SERR_CNTTH = INTPU_BASE | 0x880UL, + SPBUSERR_CNT = INTPU_BASE | 0x900UL, + IRUSERR_CNT = INTPU_BASE | 0xa80UL, + ERRRPT_EN = INTPU_BASE | 0xb00UL, + IINT_MISS_VECTOR0 = INTPU_BASE | 0x1080UL, + IINT_MISS_VECTOR1 = INTPU_BASE | 0x1100UL, + IINT_MISS = INTPU_BASE | 0x1180UL, + IINT_MISS_RPTEN = INTPU_BASE | 0x1200UL, + DEVINT_MISS_RPTEN = INTPU_BASE | 0x1280UL, + ECCSERR = INTPU_BASE | 0x1300UL, + ECCSERR_RPTEN = INTPU_BASE | 0x1380UL, + ECCMERR = INTPU_BASE | 0x1400UL, + ECCMERR_RPTEN = INTPU_BASE | 0x1480UL, + DEVINT_WKEN = INTPU_BASE | 0x1500UL, + ADR_INT_CONFIG = INTPU_BASE | 0x1580UL, + DEVINTWK_INTEN = INTPU_BASE | 0x1600UL, +}; + +/* SPBU CSR */ +enum { + SMP_INFO = SPBU_BASE | 0x80UL, + INIT_CTL = SPBU_BASE | 0x680UL, + CORE_ONLINE = SPBU_BASE | 0x780UL, + DLI_RLTD_FAULT = SPBU_BASE | 0x980UL, + DLI_RLTD_FAULT_EN = SPBU_BASE | 0xa00UL, + DLI_RLTD_FAULT_INTEN = SPBU_BASE | 0xa80UL, + CFG_INFO = SPBU_BASE | 0x1100UL, + IO_START = SPBU_BASE | 0x1300UL, + I2C0_SRST_L = SPBU_BASE | 0x1900UL, + I2C1_SRST_L = SPBU_BASE | 0x1980UL, + I2C2_SRST_L = SPBU_BASE | 0x1a00UL, + MCU_DVC_INT = SPBU_BASE | 0x3000UL, + MCU_DVC_INT_EN = SPBU_BASE | 0x3080UL, + SI_FAULT_STAT = SPBU_BASE | 0x3100UL, + SI_FAULT_STAT_EN = SPBU_BASE | 0x3180UL, + SI_FAULT_INT_EN = SPBU_BASE | 0x3200UL, + ADR_CTL = SPBU_BASE | 0x3600UL, + MC_ONLINE = SPBU_BASE | 0x3780UL, + PIU_TOP0_CONFIG = SPBU_BASE | 0x4c80UL, + PIU_TOP1_CONFIG = SPBU_BASE | 0x4d00UL, + SOFT_INFO0 = SPBU_BASE | 0xa000UL, +}; + +/*--------------------------offset-----------------------------------*/ +/* PIU IOR0 */ +enum { + PIUCONFIG0 = 0x0UL, + EPDMABAR = 0x80UL, + IOMMUSEGITEM0 = 0x100UL, + IOMMUEXCPT_CTRL = 0x2100UL, + MSIADDR = 0x2180UL, + MSICONFIG0 = 0x2200UL, + INTACONFIG = 0xa200UL, + INTBCONFIG = 0xa280UL, + INTCCONFIG = 0xa300UL, + INTDCONFIG = 0xa380UL, + AERERRINTCONFIG = 0xa400UL, + AERERRMSICONFIG = 0xa480UL, + PMEINTCONFIG = 0xa500UL, + PMEMSICONFIG = 0xa580UL, + HPINTCONFIG = 0xa600UL, + HPMSICONFIG = 0xa680UL, + DTBASEADDR = 0xb000UL, + DTLB_FLUSHALL = 0xb080UL, + DTLB_FLUSHDEV = 0xb100UL, + PTLB_FLUSHALL = 0xb180UL, + PTLB_FLUSHDEV = 0xb200UL, + PTLB_FLUSHVADDR = 0xb280UL, + PCACHE_FLUSHALL = 0xb300UL, + PCACHE_FLUSHDEV = 0xb380UL, + PCACHE_FLUSHPADDR = 0xb400UL, + TIMEOUT_CONFIG = 0xb480UL, + IOMMUEXCPT_STATUS = 0xb500UL, + IOMMUPAGE_PADDR1 = 0xb580UL, + IOMMUPAGE_PADDR2 = 0xb600UL, + IOMMUPAGE_PADDR3 = 0xb680UL, + PTLB_ACCESS = 0xb700UL, + PTLB_ITEM_TAG = 0xb780UL, + PTLB_ITEM_DATA = 0xb800UL, + PCACHE_ACCESS = 0xb880UL, + PCACHE_ITEM_TAG = 0xb900UL, + PCACHE_ITEM_DATA0 = 0xb980UL, +}; + +/* PIU IOR1 */ +enum { + PIUCONFIG1 = 0x0UL, + ERRENABLE = 0x880UL, + RCDEBUGINF1 = 0xc80UL, + DCACONTROL = 0x1a00UL, + DEVICEID0 = 0x1a80UL, +}; + +/* RC */ +enum { + RC_VENDOR_ID = 0x0UL, + RC_COMMAND = 0x80UL, + RC_REVISION_ID = 0x100UL, + RC_PRIMARY_BUS = 0x300UL, + RC_MSI_CONTROL = 0xa00UL, + RC_EXP_DEVCAP = 0xe80UL, + RC_EXP_DEVCTL = 0xf00UL, + RC_SLOT_CTRL = 0x1100UL, + RC_LINK_STAT = 0x1000UL, + RC_CONTROL = 0X1180UL, + RC_STATUS = 0X1200UL, + RC_EXP_DEVCTL2 = 0x1300UL, + RC_PORT_LINK_CTL = 0xe200UL, + RC_ORDER_RULE_CTL = 0x11680UL, + RC_MISC_CONTROL_1 = 0x11780UL, + RC_PHY_INT_REG = 0x80000UL, + RC_PHY_EXT_GEN1 = 0x82400UL, + RC_PHY_EXT_GEN2 = 0x82480UL, +}; +/* GPIO */ +enum { + GPIO_SWPORTA_DR = GPIO_BASE | 0x0UL, + GPIO_SWPORTA_DDR = GPIO_BASE | 0x200UL, +}; +/*--------------------------------------------------------------------------*/ +#endif /* _ASM_SW64_UNCORE_IO_JUNZHANG_H */ diff --git a/arch/sw_64/include/asm/uncore_io_ops_junzhang.h b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h new file mode 100644 index 000000000000..95a3b5c80531 --- /dev/null +++ b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H +#define _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H + +static inline int __get_cpu_nums(void) +{ + int cpus; + unsigned long cfg_info; + + cfg_info = sw64_io_read(0, CFG_INFO); + cfg_info = (cfg_info >> 33) & 0x3; + cpus = 1 << cfg_info; + + return cpus; +} + +static inline unsigned long __get_node_mem(int node) +{ + unsigned long node_mem; + unsigned long total_mem; + + total_mem = sw64_io_read(node, CFG_INFO) >> 3; + total_mem = (total_mem & 0xffff) << 28; + node_mem = total_mem / __get_cpu_nums(); + + return node_mem; +} + +#define __io_read_longtime(node) (0UL) +#define __io_write_longtime(node, data) do { } while (0) +#define __io_write_longtime_start_en(node, data) do { } while (0) + +static inline void +__io_write_fault_int_en(int node, unsigned long data) +{ + sw64_io_write(node, FAULT_INT_CONFIG, data); +} + +#endif /* _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H */ diff --git a/arch/sw_64/include/asm/uncore_io_ops_xuelang.h b/arch/sw_64/include/asm/uncore_io_ops_xuelang.h new file mode 100644 index 000000000000..9336e473211d --- /dev/null +++ b/arch/sw_64/include/asm/uncore_io_ops_xuelang.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNCORE_IO_OPS_XUELANG_H +#define _ASM_SW64_UNCORE_IO_OPS_XUELANG_H + +static inline int __get_cpu_nums(void) +{ + int cpus; + unsigned long trkmode; + + trkmode = sw64_io_read(0, TRKMODE); + trkmode = (trkmode >> 6) & 0x3; + cpus = 1 << trkmode; + + return cpus; +} + +static inline unsigned long __get_node_mem(int node) +{ + unsigned long node_mem; + unsigned long mc_config; + unsigned long mc_online; + unsigned long mc_cap; + unsigned long mc_num; + + mc_config = sw64_io_read(node, MC_CAP_CFG) & 0xf; + mc_cap = (1UL << mc_config) << 28; + mc_online = sw64_io_read(node, MC_ONLINE) & 0xff; + mc_num = __kernel_ctpop(mc_online); + node_mem = mc_cap * mc_num; + + return node_mem; +} + +static inline unsigned long +__io_read_longtime(int node) +{ + return sw64_io_read(node, LONG_TIME); +} + +static inline void +__io_write_longtime(int node, unsigned long data) +{ + sw64_io_write(node, LONG_TIME, data); +} + +static inline void +__io_write_longtime_start_en(int node, unsigned long data) +{ + sw64_io_write(node, LONG_TIME_START_EN, data); +} + +static inline void +__io_write_fault_int_en(int node, unsigned long data) +{ + sw64_io_write(node, DUAL_CG0_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG1_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG2_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG3_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG4_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG5_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG6_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG7_FAULT_INTEN, data); +} + +#endif /* _ASM_SW64_UNCORE_IO_OPS_XUELANG_H */ diff --git a/arch/sw_64/include/asm/uncore_io_xuelang.h b/arch/sw_64/include/asm/uncore_io_xuelang.h new file mode 100644 index 000000000000..aeaadec5be16 --- /dev/null +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -0,0 +1,323 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNCORE_IO_XUELANG_H +#define _ASM_SW64_UNCORE_IO_XUELANG_H + +#include + +#define IO_BASE (0x1UL << 47) +#define PCI_BASE (0x1UL << 43) +#define PCI_IOR0_BASE (0x2UL << 32) +#define PCI_IOR1_BASE (0x3UL << 32) + +#define PCI_RC_CFG (0x5UL << 32) + +#define PCI_EP_CFG (0x3UL << 33) +#define PCI_LEGACY_IO (0x1UL << 32) +#define PCI_LEGACY_IO_SIZE (0x100000000UL) +#define PCI_MEM_UNPRE 0x0UL +#define PCI_32BIT_MEMIO (0xe0000000UL) +#define PCI_32BIT_MEMIO_SIZE (0x20000000UL) +#define PCI_64BIT_MEMIO (0x1UL << 39) +#define PCI_64BIT_MEMIO_SIZE (0x8000000000UL) + +#define IO_RC_SHIFT 40 +#define IO_NODE_SHIFT 44 +#define IO_MARK_BIT 47 + +#define VT_MAX_CPUS_SHIFT 0 +#define VT_MAX_CPUS_MASK 0x3ff +#define VT_CORES_SHIFT 10 +#define VT_CORES_MASK 0x3ff +#define VT_THREADS_SHIFT 20 +#define VT_THREADS_MASK 0xfff + +#define QEMU_PRINTF_BUFF_BASE (IO_BASE | MCU_BASE | 0x40000UL) + +/* MSIConfig */ +#define MSICONFIG_VALID (0x1UL << 63) +#define MSICONFIG_EN (0x1UL << 62) +#define MSICONFIG_VECTOR_SHIFT 10 + +#define MSIX_MSG_ADDR (0x91abc0UL) + +#define SW64_PCI_IO_BASE(m, n) \ + (IO_BASE | ((m) << IO_NODE_SHIFT) | PCI_BASE | ((n) << IO_RC_SHIFT)) +#define SW64_IO_BASE(x) (IO_BASE | ((x) << IO_NODE_SHIFT)) + +#define SW64_PCI0_BUS 0 +#define PCI0_BUS SW64_PCI0_BUS + +#define MAX_NR_NODES 0x2 +#define MAX_NR_RCS 0x6 + +#define MCU_BASE (0x3UL << 36) +#define CAB0_BASE (0x10UL << 32) +#define INTPU_BASE (0x2aUL << 32) +#define IIC0_BASE (0x31UL << 32) +#define SPI_BASE (0x32UL << 32) +#define UART_BASE (0x33UL << 32) +#define IIC1_BASE (0x34UL << 32) +#define IIC2_BASE (0x35UL << 32) +#define GPIO_BASE (0x36UL << 32) +#define LPC_BASE (0x37UL << 32) +#define LPC_LEGACY_IO (0x1UL << 28 | IO_BASE | LPC_BASE) +#define LPC_MEM_IO (0x2UL << 28 | IO_BASE | LPC_BASE) +#define LPC_FIRMWARE_IO (0x3UL << 28 | IO_BASE | LPC_BASE) +#define DLIA_BASE (0x20UL << 32) +#define DLIB_BASE (0x21UL << 32) +#define DLIC_BASE (0x22UL << 32) +#define DLI_PHY_CTL (0x10UL << 24) +#define PCI_VT_LEGACY_IO (IO_BASE | PCI_BASE | PCI_LEGACY_IO) + +#define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x1UL << 10) +#define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x1UL << 10) + +#define PIUCONFIG0_INIT_VAL 0x38056 + +/*-----------------------addr-----------------------*/ +/* CAB0 REG */ +enum { + TRKMODE = CAB0_BASE | 0x80UL, +}; + +/* DLIA IO REG */ +enum { + DLIA_BWTEST_PAT = DLIA_BASE | 0x100980UL, + DLIA_PHY_VLDLANE = DLIA_BASE | DLI_PHY_CTL | 0x300UL, +}; + +/* DLIB IO REG */ +enum { + DLIB_BWTEST_PAT = DLIB_BASE | 0x100980UL, + DLIB_PHY_VLDLANE = DLIB_BASE | DLI_PHY_CTL | 0x300UL, +}; + +/* DLIC IO REG */ +enum { + DLIC_BWTEST_PAT = DLIC_BASE | 0x100980UL, + DLIC_PHY_VLDLANE = DLIC_BASE | DLI_PHY_CTL | 0x300UL, +}; +/* INTPU REG */ +enum { + LCORE_SLEEPY = INTPU_BASE | 0x0UL, + LCORE_SLEEP = INTPU_BASE | 0x80UL, + DEVICE_MISS = INTPU_BASE | 0x100UL, + LONG_TIME = INTPU_BASE | 0x180UL, + LCORE_IDLE = INTPU_BASE | 0x280UL, + MT_INT_CONFIG = INTPU_BASE | 0x300UL, + DEV_INT_CONFIG = INTPU_BASE | 0x480UL, + FMT_ERR = INTPU_BASE | 0x700UL, + FAULT_INT_CONFIG = INTPU_BASE | 0x780UL, + SERR_CNTTH = INTPU_BASE | 0x880UL, + MCUSERR_CNT = INTPU_BASE | 0x900UL, + IRUSERR_CNT = INTPU_BASE | 0xa80UL, + ERRRPT_EN = INTPU_BASE | 0xb00UL, + IINT_MISS_VECTOR = INTPU_BASE | 0x1100UL, + IINT_MIS = INTPU_BASE | 0x1180UL, + IINT_MISS_RPTEN = INTPU_BASE | 0x1200UL, + DEVINT_MISS_RPTEN = INTPU_BASE | 0x1280UL, + ECCSERR = INTPU_BASE | 0x1300UL, + ECCSERR_RPTEN = INTPU_BASE | 0x1380UL, + ECCMERR = INTPU_BASE | 0x1400UL, + ECCMERR_RPTEN = INTPU_BASE | 0x1480UL, + DEVINT_WKEN = INTPU_BASE | 0x1500UL, + NMI_INT_CONFIG = INTPU_BASE | 0x1580UL, + DEVINTWK_INTEN = INTPU_BASE | 0x1600UL, +}; + +/* MC IO REG */ +enum { + CFGDEC = 0x400UL, + CFGCR = 0x480UL, + INIT_CTRL = 0x580UL, + CFGERR = 0xd00UL, + FSMSTAT = 0xe00UL, + PUB_INTERFACE = 0x1000UL, + POWERCTRL = 0x1080UL, + CFGMR0 = 0x1280UL, + CFGMR1 = 0x1300UL, + CFGMR2 = 0x1380UL, + CFGMR3 = 0x1400UL, + PERF_CTRL = 0x1480UL, + MC_PERF0 = 0x1500UL, + CFGMR4 = 0x1800UL, + CFGMR5 = 0x1880UL, + CFGMR6 = 0x1900UL, + MC_CTRL = 0x1c00UL, + MEMSERR_P = 0x1c80UL, + MEMSERR = 0x1d00UL, +}; + +/* MCU CSR */ +enum { + SMP_INFO = MCU_BASE | 0x80UL, + INIT_CTL = MCU_BASE | 0x680UL, + MT_STATE = MCU_BASE | 0x700UL, + CORE_ONLINE = MCU_BASE | 0x780UL, + MT_INT = MCU_BASE | 0x800UL, + MT_INT_END = MCU_BASE | 0x880UL, + CPU_ID = MCU_BASE | 0x900UL, + DLI_RLTD_FAULT = MCU_BASE | 0x980UL, + DLI_RLTD_FAULT_EN = MCU_BASE | 0xa00UL, + DLI_RLTD_FAULT_INTEN = MCU_BASE | 0xa80UL, + FAULT_SOURCE = MCU_BASE | 0xb00UL, + INT_SOURCE = MCU_BASE | 0xb80UL, + CORE_STATE0 = MCU_BASE | 0xc00UL, + CORE_STATE1 = MCU_BASE | 0xc80UL, + CFG_INFO = MCU_BASE | 0x1100UL, + MC_CAP_CFG = MCU_BASE | 0x1180UL, + IO_START = MCU_BASE | 0x1300UL, + UART_ONLINE = MCU_BASE | 0x1780UL, + I2C0_SRST_L = MCU_BASE | 0x1900UL, + I2C1_SRST_L = MCU_BASE | 0x1980UL, + I2C2_SRST_L = MCU_BASE | 0x1a00UL, + MCU_DVC_INT = MCU_BASE | 0x3000UL, + MCU_DVC_INT_EN = MCU_BASE | 0x3080UL, + SI_FAULT_STAT = MCU_BASE | 0x3100UL, + SI_FAULT_EN = MCU_BASE | 0x3180UL, + SI_FAULT_INT_EN = MCU_BASE | 0x3200UL, + FIFO_SYNSEL = MCU_BASE | 0x3400UL, + CPU_INFO = MCU_BASE | 0x3480UL, + WAKEUP_CTL = MCU_BASE | 0x3500UL, + FLAGREG = MCU_BASE | 0x3580UL, + NMI_CTL = MCU_BASE | 0x3600UL, + PIUPLL_CNT = MCU_BASE | 0x3680UL, + MC_ONLINE = MCU_BASE | 0x3780UL, + FLASH_INFO = MCU_BASE | 0x3800UL, + RTPUSROMCNT = MCU_BASE | 0x3880UL, + CLU_LV1_SEL = MCU_BASE | 0x3a80UL, + CLU_LV2_SEL = MCU_BASE | 0x3b00UL, + CLK_CTL = MCU_BASE | 0x3b80UL, + SLEEP_WAIT_CNT = MCU_BASE | 0x4980UL, + CHIP_ID = MCU_BASE | 0x4b00UL, + PIU_TOP0_CONFIG = MCU_BASE | 0x4c80UL, + PIU_TOP1_CONFIG = MCU_BASE | 0x4d00UL, + LVDS_CTL = MCU_BASE | 0x4d80UL, + LPC_DMAREQ_TOTH = MCU_BASE | 0x5100UL, + DLI_ONLINE = MCU_BASE | 0x6180UL, + LPC_DMAREQ_HADR = MCU_BASE | 0x6200UL, + PIU_PHY_SRST_H = MCU_BASE | 0x6280UL, + CLK_SEL_PCIE0 = MCU_BASE | 0x6280UL, + CLK_SEL_PCIE1 = MCU_BASE | 0x6300UL, + CLK_SEL_PCIE2 = MCU_BASE | 0x6380UL, + CLK_SEL_PCIE3 = MCU_BASE | 0x6400UL, + CLK_SEL_PCIE4 = MCU_BASE | 0x6480UL, + CLK_SEL_PCIE5 = MCU_BASE | 0x6500UL, + PERST_N_PCIE0 = MCU_BASE | 0x6680UL, + PERST_N_PCIE1 = MCU_BASE | 0x6700UL, + PERST_N_PCIE2 = MCU_BASE | 0x6780UL, + PERST_N_PCIE3 = MCU_BASE | 0x6800UL, + PERST_N_PCIE4 = MCU_BASE | 0x6880UL, + PERST_N_PCIE5 = MCU_BASE | 0x6900UL, + BUTTON_RST_N_PCIE0 = MCU_BASE | 0x6a80UL, + BUTTON_RST_N_PCIE1 = MCU_BASE | 0x6b00UL, + BUTTON_RST_N_PCIE2 = MCU_BASE | 0x6b80UL, + BUTTON_RST_N_PCIE3 = MCU_BASE | 0x6c00UL, + BUTTON_RST_N_PCIE4 = MCU_BASE | 0x6c80UL, + BUTTON_RST_N_PCIE5 = MCU_BASE | 0x6d00UL, + DUAL_CG0_FAULT = MCU_BASE | 0x6d80UL, + DUAL_CG1_FAULT = MCU_BASE | 0x6e00UL, + DUAL_CG2_FAULT = MCU_BASE | 0x6e80UL, + DUAL_CG3_FAULT = MCU_BASE | 0x6f00UL, + DUAL_CG4_FAULT = MCU_BASE | 0x6f80UL, + DUAL_CG5_FAULT = MCU_BASE | 0x7000UL, + DUAL_CG6_FAULT = MCU_BASE | 0x7080UL, + DUAL_CG7_FAULT = MCU_BASE | 0x7100UL, + DUAL_CG0_FAULT_EN = MCU_BASE | 0x7180UL, + DUAL_CG1_FAULT_EN = MCU_BASE | 0x7200UL, + DUAL_CG2_FAULT_EN = MCU_BASE | 0x7280UL, + DUAL_CG3_FAULT_EN = MCU_BASE | 0x7300UL, + DUAL_CG4_FAULT_EN = MCU_BASE | 0x7380UL, + DUAL_CG5_FAULT_EN = MCU_BASE | 0x7400UL, + DUAL_CG6_FAULT_EN = MCU_BASE | 0x7480UL, + DUAL_CG7_FAULT_EN = MCU_BASE | 0x7500UL, + DUAL_CG0_FAULT_INTEN = MCU_BASE | 0x7580UL, + DUAL_CG1_FAULT_INTEN = MCU_BASE | 0x7600UL, + DUAL_CG2_FAULT_INTEN = MCU_BASE | 0x7680UL, + DUAL_CG3_FAULT_INTEN = MCU_BASE | 0x7700UL, + DUAL_CG4_FAULT_INTEN = MCU_BASE | 0x7780UL, + DUAL_CG5_FAULT_INTEN = MCU_BASE | 0x7800UL, + DUAL_CG6_FAULT_INTEN = MCU_BASE | 0x7880UL, + DUAL_CG7_FAULT_INTEN = MCU_BASE | 0x7900UL, + SOFT_INFO0 = MCU_BASE | 0x7f00UL, + LONG_TIME_START_EN = MCU_BASE | 0x9000UL, +}; + +/*--------------------------offset-----------------------------------*/ +/* PIU IOR0 */ +enum { + PIUCONFIG0 = 0x0UL, + EPDMABAR = 0x80UL, + IOMMUSEGITEM0 = 0x100UL, + IOMMUEXCPT_CTRL = 0x2100UL, + MSIADDR = 0x2180UL, + MSICONFIG0 = 0x2200UL, + INTACONFIG = 0xa200UL, + INTBCONFIG = 0xa280UL, + INTCCONFIG = 0xa300UL, + INTDCONFIG = 0xa380UL, + AERERRINTCONFIG = 0xa400UL, + AERERRMSICONFIG = 0xa480UL, + PMEINTCONFIG = 0xa500UL, + PMEMSICONFIG = 0xa580UL, + HPINTCONFIG = 0xa600UL, + HPMSICONFIG = 0xa680UL, + DTBASEADDR = 0xb000UL, + DTLB_FLUSHALL = 0xb080UL, + DTLB_FLUSHDEV = 0xb100UL, + PTLB_FLUSHALL = 0xb180UL, + PTLB_FLUSHDEV = 0xb200UL, + PTLB_FLUSHVADDR = 0xb280UL, + PCACHE_FLUSHALL = 0xb300UL, + PCACHE_FLUSHDEV = 0xb380UL, + PCACHE_FLUSHPADDR = 0xb400UL, + TIMEOUT_CONFIG = 0xb480UL, + IOMMUEXCPT_STATUS = 0xb500UL, + IOMMUPAGE_PADDR1 = 0xb580UL, + IOMMUPAGE_PADDR2 = 0xb600UL, + IOMMUPAGE_PADDR3 = 0xb680UL, + PTLB_ACCESS = 0xb700UL, + PTLB_ITEM_TAG = 0xb780UL, + PTLB_ITEM_DATA = 0xb800UL, + PCACHE_ACCESS = 0xb880UL, + PCACHE_ITEM_TAG = 0xb900UL, + PCACHE_ITEM_DATA0 = 0xb980UL, +}; + +/* PIU IOR1 */ +enum { + PIUCONFIG1 = 0x0UL, + ERRENABLE = 0x880UL, + RCDEBUGINF1 = 0xc80UL, + DCACONTROL = 0x1a00UL, + DEVICEID0 = 0x1a80UL, +}; + +/* RC */ +enum { + RC_VENDOR_ID = 0x0UL, + RC_COMMAND = 0x80UL, + RC_REVISION_ID = 0x100UL, + RC_PRIMARY_BUS = 0x300UL, + RC_MSI_CONTROL = 0xa00UL, + RC_EXP_DEVCAP = 0xe80UL, + RC_EXP_DEVCTL = 0xf00UL, + RC_SLOT_CTRL = 0x1100UL, + RC_LINK_STAT = 0x1000UL, + RC_CONTROL = 0X1180UL, + RC_STATUS = 0X1200UL, + RC_EXP_DEVCTL2 = 0x1300UL, + RC_PORT_LINK_CTL = 0xe200UL, + RC_ORDER_RULE_CTL = 0x11680UL, + RC_MISC_CONTROL_1 = 0x11780UL, + RC_PHY_INT_REG = 0x80000UL, + RC_PHY_EXT_GEN1 = 0x82400UL, + RC_PHY_EXT_GEN2 = 0x82480UL, +}; +/* GPIO */ +enum { + GPIO_SWPORTA_DR = GPIO_BASE | 0x0UL, + GPIO_SWPORTA_DDR = GPIO_BASE | 0x200UL, +}; +/*--------------------------------------------------------------------------*/ +#endif /* _ASM_SW64_UNCORE_IO_XUELANG_H */ -- Gitee From fbf73c1db29fc73833f1c4cde144bf3af6bd95fa Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:27 +0800 Subject: [PATCH 020/524] sw64: add module support Add kernel module support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/module.h | 17 ++ arch/sw_64/kernel/module.c | 279 ++++++++++++++++++++++++++++++++ 2 files changed, 296 insertions(+) create mode 100644 arch/sw_64/include/asm/module.h create mode 100644 arch/sw_64/kernel/module.c diff --git a/arch/sw_64/include/asm/module.h b/arch/sw_64/include/asm/module.h new file mode 100644 index 000000000000..d1663aab4097 --- /dev/null +++ b/arch/sw_64/include/asm/module.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_MODULE_H +#define _ASM_SW64_MODULE_H + +#include + +struct mod_arch_specific { + unsigned int gotsecindex; +}; + +#define ARCH_SHF_SMALL SHF_SW64_GPREL + +#ifdef MODULE +asm(".section .got, \"aw\", @progbits; .align 3; .previous"); +#endif + +#endif /* _ASM_SW64_MODULE_H */ diff --git a/arch/sw_64/kernel/module.c b/arch/sw_64/kernel/module.c new file mode 100644 index 000000000000..67264e3644a7 --- /dev/null +++ b/arch/sw_64/kernel/module.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#define DEBUGP(fmt...) + +/* Allocate the GOT at the end of the core sections. */ + +struct got_entry { + struct got_entry *next; + Elf64_Sxword r_addend; + int got_offset; +}; + +static inline void +process_reloc_for_got(Elf64_Rela *rela, + struct got_entry *chains, Elf64_Xword *poffset) +{ + unsigned long r_sym = ELF64_R_SYM(rela->r_info); + unsigned long r_type = ELF64_R_TYPE(rela->r_info); + Elf64_Sxword r_addend = rela->r_addend; + struct got_entry *g; + + if (r_type != R_SW64_LITERAL) + return; + + for (g = chains + r_sym; g ; g = g->next) + if (g->r_addend == r_addend) { + if (g->got_offset == 0) { + g->got_offset = *poffset; + *poffset += 8; + } + goto found_entry; + } + + g = kmalloc(sizeof(*g), GFP_KERNEL); + g->next = chains[r_sym].next; + g->r_addend = r_addend; + g->got_offset = *poffset; + *poffset += 8; + chains[r_sym].next = g; + + found_entry: + /* + * Trick: most of the ELF64_R_TYPE field is unused. There are + * 42 valid relocation types, and a 32-bit field. Co-opt the + * bits above 256 to store the got offset for this reloc. + */ + rela->r_info |= g->got_offset << 8; +} + +int +module_frob_arch_sections(Elf64_Ehdr *hdr, Elf64_Shdr *sechdrs, + char *secstrings, struct module *me) +{ + struct got_entry *chains; + Elf64_Rela *rela; + Elf64_Shdr *esechdrs, *symtab, *s, *got; + unsigned long nsyms, nrela, i; + + esechdrs = sechdrs + hdr->e_shnum; + symtab = got = NULL; + + /* Find out how large the symbol table is. Allocate one got_entry + * head per symbol. Normally this will be enough, but not always. + * We'll chain different offsets for the symbol down each head. + */ + for (s = sechdrs; s < esechdrs; ++s) + if (s->sh_type == SHT_SYMTAB) + symtab = s; + else if (!strcmp(".got", secstrings + s->sh_name)) { + got = s; + me->arch.gotsecindex = s - sechdrs; + } + + if (!symtab) { + pr_err("module %s: no symbol table\n", me->name); + return -ENOEXEC; + } + if (!got) { + pr_err("module %s: no got section\n", me->name); + return -ENOEXEC; + } + + nsyms = symtab->sh_size / sizeof(Elf64_Sym); + chains = kcalloc(nsyms, sizeof(struct got_entry), GFP_KERNEL); + if (!chains) { + pr_err("module %s: no memory for symbol chain buffer\n", + me->name); + return -ENOMEM; + } + + got->sh_size = 0; + got->sh_addralign = 8; + got->sh_type = SHT_NOBITS; + + /* Examine all LITERAL relocations to find out what GOT entries + * are required. This sizes the GOT section as well. + */ + for (s = sechdrs; s < esechdrs; ++s) + if (s->sh_type == SHT_RELA) { + nrela = s->sh_size / sizeof(Elf64_Rela); + rela = (void *)hdr + s->sh_offset; + for (i = 0; i < nrela; ++i) + process_reloc_for_got(rela+i, chains, + &got->sh_size); + } + + /* Free the memory we allocated. */ + for (i = 0; i < nsyms; ++i) { + struct got_entry *g, *n; + + for (g = chains[i].next; g ; g = n) { + n = g->next; + kfree(g); + } + } + kfree(chains); + + return 0; +} + +int +apply_relocate_add(Elf64_Shdr *sechdrs, const char *strtab, + unsigned int symindex, unsigned int relsec, + struct module *me) +{ + Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr; + unsigned long i, n = sechdrs[relsec].sh_size / sizeof(*rela); + Elf64_Sym *symtab, *sym; + void *base, *location; + unsigned long got, gp; + + DEBUGP("Applying relocate section %u to %u\n", relsec, + sechdrs[relsec].sh_info); + + base = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr; + symtab = (Elf64_Sym *)sechdrs[symindex].sh_addr; + + /* The small sections were sorted to the end of the segment. + * The following should definitely cover them. + */ + got = sechdrs[me->arch.gotsecindex].sh_addr; + gp = got + 0x8000; + + for (i = 0; i < n; i++) { + unsigned long r_sym = ELF64_R_SYM(rela[i].r_info); + unsigned long r_type = ELF64_R_TYPE(rela[i].r_info); + unsigned long r_got_offset = r_type >> 8; + unsigned long value, hi, lo; + + r_type &= 0xff; + + /* This is where to make the change. */ + location = base + rela[i].r_offset; + + /* This is the symbol it is referring to. Note that all + * unresolved symbols have been resolved. + */ + sym = symtab + r_sym; + value = sym->st_value + rela[i].r_addend; + + switch (r_type) { + case R_SW64_NONE: + break; + case R_SW64_REFLONG: + *(u32 *)location = value; + break; + case R_SW64_REFQUAD: + /* BUG() can produce misaligned relocations. */ + ((u32 *)location)[0] = value; + ((u32 *)location)[1] = value >> 32; + break; + case R_SW64_GPREL32: + value -= gp; + if ((int)value != value) + goto reloc_overflow; + *(u32 *)location = value; + break; + case R_SW64_LITERAL: + hi = got + r_got_offset; + lo = hi - gp; + if ((short)lo != lo) { + unsigned long over_offset = (lo + 0x8000) >> 16; + + if ((over_offset & 0x8000) == 0) { + *(u16 *)(location - 0x4) = over_offset; + *(u16 *)location = lo - ((over_offset << 16) + gp); + *(u64 *)hi = value; + } else { + goto reloc_overflow; + } + } else { + *(u16 *)location = lo; + *(u64 *)hi = value; + } + break; + case R_SW64_LITERAL_GOT: + /* empty for now need to fill */ + break; + case R_SW64_LITUSE: + break; + case R_SW64_GPDISP: + value = gp - (u64)location; + lo = (short)value; + hi = (int)(value - lo); + if (hi + lo != value) + goto reloc_overflow; + *(u16 *)location = hi >> 16; + *(u16 *)(location + rela[i].r_addend) = lo; + break; + case R_SW64_BRSGP: + /* + * BRSGP is only allowed to bind to local symbols. + * If the section is undef, this means that the + * value was resolved from somewhere else. + */ + if (sym->st_shndx == SHN_UNDEF) + goto reloc_overflow; + if ((sym->st_other & STO_SW64_STD_GPLOAD) == + STO_SW64_STD_GPLOAD) + /* Omit the prologue. */ + value += 8; + fallthrough; + case R_SW64_BRADDR: + value -= (u64)location + 4; + if (value & 3) + goto reloc_overflow; + value = (long)value >> 2; + if (value + (1<<21) >= 1<<22) + goto reloc_overflow; + value &= 0x1fffff; + value |= *(u32 *)location & ~0x1fffff; + *(u32 *)location = value; + break; + case R_SW64_HINT: + break; + case R_SW64_SREL32: + value -= (u64)location; + if ((int)value != value) + goto reloc_overflow; + *(u32 *)location = value; + break; + case R_SW64_SREL64: + value -= (u64)location; + *(u64 *)location = value; + break; + case R_SW64_GPRELHIGH: + value = (long)(value - gp + 0x8000) >> 16; + if ((short) value != value) + goto reloc_overflow; + *(u16 *)location = value; + break; + case R_SW64_GPRELLOW: + value -= gp; + *(u16 *)location = value; + break; + case R_SW64_GPREL16: + value -= gp; + if ((short) value != value) + goto reloc_overflow; + *(u16 *)location = value; + break; + default: + pr_err("module %s: Unknown relocation: %lu\n", me->name, r_type); + return -ENOEXEC; +reloc_overflow: + if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION) + pr_err("module %s: Relocation (type %lu) overflow vs section %d\n", + me->name, r_type, sym->st_shndx); + else + pr_err("module %s: Relocation (type %lu) overflow vs %s\n", + me->name, r_type, strtab + sym->st_name); + return -ENOEXEC; + } + } + + return 0; +} -- Gitee From 9c2d6d9772cdc2ec71009341f0046e30bc861fa9 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:33 +0800 Subject: [PATCH 021/524] sw64: add some common routines Add some other common routines for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/debug.h | 38 ++++ arch/sw_64/include/asm/extable.h | 47 +++++ arch/sw_64/include/asm/futex.h | 168 +++++++++++++++++ arch/sw_64/kernel/asm-offsets.c | 240 ++++++++++++++++++++++++ arch/sw_64/kernel/audit.c | 61 ++++++ arch/sw_64/kernel/early_printk.c | 183 ++++++++++++++++++ arch/sw_64/kernel/entry.S | 306 +++++++++++++++++++++++++++++++ 7 files changed, 1043 insertions(+) create mode 100644 arch/sw_64/include/asm/debug.h create mode 100644 arch/sw_64/include/asm/extable.h create mode 100644 arch/sw_64/include/asm/futex.h create mode 100644 arch/sw_64/kernel/asm-offsets.c create mode 100644 arch/sw_64/kernel/audit.c create mode 100644 arch/sw_64/kernel/early_printk.c create mode 100644 arch/sw_64/kernel/entry.S diff --git a/arch/sw_64/include/asm/debug.h b/arch/sw_64/include/asm/debug.h new file mode 100644 index 000000000000..8db5a8bb9ab7 --- /dev/null +++ b/arch/sw_64/include/asm/debug.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 Mao Minkai + * Author: Mao Minkai + * + * This code is taken from arch/mips/include/asm/debug.h + * Copyright (C) 2015 Imagination Technologies + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#ifndef _ASM_SW64_DEBUG_H +#define _ASM_SW64_DEBUG_H + +#include + +/* + * sw64_debugfs_dir corresponds to the "sw_64" directory at the top level + * of the DebugFS hierarchy. SW64-specific DebugFS entries should be + * placed beneath this directory. + */ +extern struct dentry *sw64_debugfs_dir; + +#define UNA_MAX_ENTRIES 64 + +struct unaligned_stat { + unsigned long pc; + unsigned long va; +}; + +extern char unaligned_task[]; +extern unsigned long unaligned_count; +extern struct unaligned_stat unaligned[]; + +#endif /* _ASM_SW64_DEBUG_H */ diff --git a/arch/sw_64/include/asm/extable.h b/arch/sw_64/include/asm/extable.h new file mode 100644 index 000000000000..42f872ce6c3b --- /dev/null +++ b/arch/sw_64/include/asm/extable.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_EXTABLE_H +#define _ASM_SW64_EXTABLE_H + +/* + * About the exception table: + * + * - insn is a 32-bit pc-relative offset from the faulting insn. + * - nextinsn is a 16-bit offset off of the faulting instruction + * (not off of the *next* instruction as branches are). + * - errreg is the register in which to place -EFAULT. + * - valreg is the final target register for the load sequence + * and will be zeroed. + * + * Either errreg or valreg may be $31, in which case nothing happens. + * + * The exception fixup information "just so happens" to be arranged + * as in a MEM format instruction. This lets us emit our three + * values like so: + * + * lda valreg, nextinsn(errreg) + * + */ + +struct exception_table_entry { + signed int insn; + union exception_fixup { + unsigned int unit; + struct { + signed int nextinsn : 16; + unsigned int errreg : 5; + unsigned int valreg : 5; + } bits; + } fixup; +}; + +#define ARCH_HAS_RELATIVE_EXTABLE + +extern int fixup_exception(struct pt_regs *regs, unsigned long pc); + +#define swap_ex_entry_fixup(a, b, tmp, delta) \ + do { \ + (a)->fixup.unit = (b)->fixup.unit; \ + (b)->fixup.unit = (tmp).fixup.unit; \ + } while (0) + +#endif /* _ASM_SW64_EXTABLE_H */ diff --git a/arch/sw_64/include/asm/futex.h b/arch/sw_64/include/asm/futex.h new file mode 100644 index 000000000000..783799813980 --- /dev/null +++ b/arch/sw_64/include/asm/futex.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_FUTEX_H +#define _ASM_SW64_FUTEX_H + +#ifdef __KERNEL__ + +#include +#include +#include +#include + +#ifdef CONFIG_SUBARCH_C3B + +#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg, tmp) \ + __asm__ __volatile__( \ + "1: lldw %0, 0(%3)\n" \ + " ldi %2, 1\n" \ + " wr_f %2\n" \ + insn \ + "2: lstw %1, 0(%3)\n" \ + " rd_f %1\n" \ + " beq %1, 4f\n" \ + " bis $31, $31, %1\n" \ + "3: .subsection 2\n" \ + "4: br 1b\n" \ + " .previous\n" \ + " .section __ex_table, \"a\"\n" \ + " .long 1b-.\n" \ + " ldi $31, 3b-1b(%1)\n" \ + " .long 2b-.\n" \ + " ldi $31, 3b-2b(%1)\n" \ + " .previous\n" \ + : "=&r" (oldval), "=&r"(ret), "=&r"(tmp) \ + : "r" (uaddr), "r"(oparg) \ + : "memory") + +static inline int +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) +{ + int ret = 0, cmp; + u32 prev, tmp; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + __asm__ __volatile__ ( + "1: lldw %1, 0(%4)\n" + " cmpeq %1, %5, %2\n" + " wr_f %2\n" + " bis $31, %6, %3\n" + "2: lstw %3, 0(%4)\n" + " rd_f %3\n" + " beq %2, 3f\n" + " beq %3, 4f\n" + "3: .subsection 2\n" + "4: br 1b\n" + " .previous\n" + " .section __ex_table, \"a\"\n" + " .long 1b-.\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b-.\n" + " ldi $31, 3b-2b(%0)\n" + " .previous\n" + : "+r"(ret), "=&r"(prev), "=&r"(cmp), "=&r"(tmp) + : "r"(uaddr), "r"((long)(int)oldval), "r"(newval) + : "memory"); + + *uval = prev; + return ret; +} +#else /* !CONFIG_SUBARCH_C3B */ + +#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg, tmp) \ + __asm__ __volatile__( \ + "1: lldw %0, 0(%3)\n" \ + insn \ + "2: lstw %1, 0(%3)\n" \ + " beq %1, 4f\n" \ + " bis $31, $31, %1\n" \ + "3: .subsection 2\n" \ + "4: lbr 1b\n" \ + " .previous\n" \ + " .section __ex_table, \"a\"\n" \ + " .long 1b-.\n" \ + " ldi $31, 3b-1b(%1)\n" \ + " .long 2b-.\n" \ + " ldi $31, 3b-2b(%1)\n" \ + " .previous\n" \ + : "=&r" (oldval), "=&r"(ret), "=&r"(tmp) \ + : "r" (uaddr), "r"(oparg) \ + : "memory") + + +static inline int +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) +{ + int ret = 0, cmp; + u32 prev, tmp; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + __asm__ __volatile__ ( + "1: lldw %1, 0(%4)\n" + " cmpeq %1, %5, %2\n" + " beq %2, 3f\n" + " bis $31, %6, %3\n" + "2: lstw %3, 0(%4)\n" + " beq %3, 4f\n" + "3: .subsection 2\n" + "4: lbr 1b\n" + " .previous\n" + " .section __ex_table, \"a\"\n" + " .long 1b-.\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b-.\n" + " ldi $31, 3b-2b(%0)\n" + " .previous\n" + : "+r"(ret), "=&r"(prev), "=&r"(cmp), "=&r"(tmp) + : "r"(uaddr), "r"((long)(int)oldval), "r"(newval) + : "memory"); + + *uval = prev; + return ret; +} +#endif /* CONFIG_SUBARCH_C3B */ + +static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, + u32 __user *uaddr) +{ + int oldval = 0, ret; + unsigned long tmp; + + pagefault_disable(); + + switch (op) { + case FUTEX_OP_SET: + __futex_atomic_op("mov %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_ADD: + __futex_atomic_op("addw %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_OR: + __futex_atomic_op("or %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_ANDN: + __futex_atomic_op("andnot %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_XOR: + __futex_atomic_op("xor %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + default: + ret = -ENOSYS; + } + + pagefault_enable(); + + if (!ret) + *oval = oldval; + + return ret; +} + +#endif /* __KERNEL__ */ + +#endif /* _ASM_SW64_FUTEX_H */ diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c new file mode 100644 index 000000000000..41310a8a7af1 --- /dev/null +++ b/arch/sw_64/kernel/asm-offsets.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Generate definitions needed by assembly language modules. + * This code generates raw asm output which is post-processed to extract + * and format the required data. + */ + +#define GENERATING_ASM_OFFSETS /* asm/smp.h */ +#include +#include +#include +#include + +#include +#include + +#include "traps.c" +#include "signal.c" + +void foo(void) +{ + DEFINE(ASM_THREAD_SIZE, THREAD_SIZE); + DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); + BLANK(); + + DEFINE(TASK_BLOCKED, offsetof(struct task_struct, blocked)); + DEFINE(TASK_CRED, offsetof(struct task_struct, cred)); + DEFINE(TASK_REAL_PARENT, offsetof(struct task_struct, real_parent)); + DEFINE(TASK_GROUP_LEADER, offsetof(struct task_struct, group_leader)); + DEFINE(TASK_TGID, offsetof(struct task_struct, tgid)); + DEFINE(TASK_STACK, offsetof(struct task_struct, stack)); +#ifdef CONFIG_SMP + DEFINE(TASK_CPU, offsetof(struct task_struct, thread_info.cpu)); +#endif + BLANK(); + + OFFSET(PSTATE_REGS, processor_state, regs); + OFFSET(PSTATE_FPREGS, processor_state, fpregs); + OFFSET(PSTATE_FPCR, processor_state, fpcr); + OFFSET(PSTATE_KTP, processor_state, ktp); +#ifdef CONFIG_HIBERNATION + OFFSET(PSTATE_SP, processor_state, sp); +#endif + OFFSET(PBE_ADDR, pbe, address); + OFFSET(PBE_ORIG_ADDR, pbe, orig_address); + OFFSET(PBE_NEXT, pbe, next); + OFFSET(CALLEE_R9, callee_saved_regs, r9); + OFFSET(CALLEE_R10, callee_saved_regs, r10); + OFFSET(CALLEE_R11, callee_saved_regs, r11); + OFFSET(CALLEE_R12, callee_saved_regs, r12); + OFFSET(CALLEE_R13, callee_saved_regs, r13); + OFFSET(CALLEE_R14, callee_saved_regs, r14); + OFFSET(CALLEE_R15, callee_saved_regs, r15); + OFFSET(CALLEE_RA, callee_saved_regs, ra); + OFFSET(CALLEE_F2, callee_saved_fpregs, f2); + OFFSET(CALLEE_F3, callee_saved_fpregs, f3); + OFFSET(CALLEE_F4, callee_saved_fpregs, f4); + OFFSET(CALLEE_F5, callee_saved_fpregs, f5); + OFFSET(CALLEE_F6, callee_saved_fpregs, f6); + OFFSET(CALLEE_F7, callee_saved_fpregs, f7); + OFFSET(CALLEE_F8, callee_saved_fpregs, f8); + OFFSET(CALLEE_F9, callee_saved_fpregs, f9); + BLANK(); + DEFINE(CRED_UID, offsetof(struct cred, uid)); + DEFINE(CRED_EUID, offsetof(struct cred, euid)); + DEFINE(CRED_GID, offsetof(struct cred, gid)); + DEFINE(CRED_EGID, offsetof(struct cred, egid)); + BLANK(); + + DEFINE(PT_REGS_SIZE, sizeof(struct pt_regs)); + DEFINE(PT_REGS_R0, offsetof(struct pt_regs, regs[0])); + DEFINE(PT_REGS_R1, offsetof(struct pt_regs, regs[1])); + DEFINE(PT_REGS_R2, offsetof(struct pt_regs, regs[2])); + DEFINE(PT_REGS_R3, offsetof(struct pt_regs, regs[3])); + DEFINE(PT_REGS_R4, offsetof(struct pt_regs, regs[4])); + DEFINE(PT_REGS_R5, offsetof(struct pt_regs, regs[5])); + DEFINE(PT_REGS_R6, offsetof(struct pt_regs, regs[6])); + DEFINE(PT_REGS_R7, offsetof(struct pt_regs, regs[7])); + DEFINE(PT_REGS_R8, offsetof(struct pt_regs, regs[8])); + DEFINE(PT_REGS_R9, offsetof(struct pt_regs, regs[9])); + DEFINE(PT_REGS_R10, offsetof(struct pt_regs, regs[10])); + DEFINE(PT_REGS_R11, offsetof(struct pt_regs, regs[11])); + DEFINE(PT_REGS_R12, offsetof(struct pt_regs, regs[12])); + DEFINE(PT_REGS_R13, offsetof(struct pt_regs, regs[13])); + DEFINE(PT_REGS_R14, offsetof(struct pt_regs, regs[14])); + DEFINE(PT_REGS_R15, offsetof(struct pt_regs, regs[15])); + DEFINE(PT_REGS_R16, offsetof(struct pt_regs, regs[16])); + DEFINE(PT_REGS_R17, offsetof(struct pt_regs, regs[17])); + DEFINE(PT_REGS_R18, offsetof(struct pt_regs, regs[18])); + DEFINE(PT_REGS_R19, offsetof(struct pt_regs, regs[19])); + DEFINE(PT_REGS_R20, offsetof(struct pt_regs, regs[20])); + DEFINE(PT_REGS_R21, offsetof(struct pt_regs, regs[21])); + DEFINE(PT_REGS_R22, offsetof(struct pt_regs, regs[22])); + DEFINE(PT_REGS_R23, offsetof(struct pt_regs, regs[23])); + DEFINE(PT_REGS_R24, offsetof(struct pt_regs, regs[24])); + DEFINE(PT_REGS_R25, offsetof(struct pt_regs, regs[25])); + DEFINE(PT_REGS_R26, offsetof(struct pt_regs, regs[26])); + DEFINE(PT_REGS_R27, offsetof(struct pt_regs, regs[27])); + DEFINE(PT_REGS_R28, offsetof(struct pt_regs, regs[28])); + DEFINE(PT_REGS_GP, offsetof(struct pt_regs, regs[29])); + DEFINE(PT_REGS_SP, offsetof(struct pt_regs, regs[30])); + DEFINE(PT_REGS_PC, offsetof(struct pt_regs, pc)); + DEFINE(PT_REGS_PS, offsetof(struct pt_regs, ps)); + DEFINE(PT_REGS_ORIG_R0, offsetof(struct pt_regs, orig_r0)); + DEFINE(PT_REGS_ORIG_R19, offsetof(struct pt_regs, orig_r19)); + DEFINE(PT_REGS_HM_PS, offsetof(struct pt_regs, hm_ps)); + DEFINE(PT_REGS_HM_PC, offsetof(struct pt_regs, hm_pc)); + DEFINE(PT_REGS_HM_GP, offsetof(struct pt_regs, hm_gp)); + DEFINE(PT_REGS_HM_R16, offsetof(struct pt_regs, hm_r16)); + DEFINE(PT_REGS_HM_R17, offsetof(struct pt_regs, hm_r17)); + DEFINE(PT_REGS_HM_R18, offsetof(struct pt_regs, hm_r18)); + BLANK(); + + DEFINE(KVM_REGS_SIZE, sizeof(struct kvm_regs)); + DEFINE(KVM_REGS_R0, offsetof(struct kvm_regs, r0)); + DEFINE(KVM_REGS_R1, offsetof(struct kvm_regs, r1)); + DEFINE(KVM_REGS_R2, offsetof(struct kvm_regs, r2)); + DEFINE(KVM_REGS_R3, offsetof(struct kvm_regs, r3)); + DEFINE(KVM_REGS_R4, offsetof(struct kvm_regs, r4)); + DEFINE(KVM_REGS_R5, offsetof(struct kvm_regs, r5)); + DEFINE(KVM_REGS_R6, offsetof(struct kvm_regs, r6)); + DEFINE(KVM_REGS_R7, offsetof(struct kvm_regs, r7)); + DEFINE(KVM_REGS_R8, offsetof(struct kvm_regs, r8)); + DEFINE(KVM_REGS_R9, offsetof(struct kvm_regs, r9)); + DEFINE(KVM_REGS_R10, offsetof(struct kvm_regs, r10)); + DEFINE(KVM_REGS_R11, offsetof(struct kvm_regs, r11)); + DEFINE(KVM_REGS_R12, offsetof(struct kvm_regs, r12)); + DEFINE(KVM_REGS_R13, offsetof(struct kvm_regs, r13)); + DEFINE(KVM_REGS_R14, offsetof(struct kvm_regs, r14)); + DEFINE(KVM_REGS_R15, offsetof(struct kvm_regs, r15)); + DEFINE(KVM_REGS_R19, offsetof(struct kvm_regs, r19)); + DEFINE(KVM_REGS_R20, offsetof(struct kvm_regs, r20)); + DEFINE(KVM_REGS_R21, offsetof(struct kvm_regs, r21)); + DEFINE(KVM_REGS_R22, offsetof(struct kvm_regs, r22)); + DEFINE(KVM_REGS_R23, offsetof(struct kvm_regs, r23)); + DEFINE(KVM_REGS_R24, offsetof(struct kvm_regs, r24)); + DEFINE(KVM_REGS_R25, offsetof(struct kvm_regs, r25)); + DEFINE(KVM_REGS_R26, offsetof(struct kvm_regs, r26)); + DEFINE(KVM_REGS_R27, offsetof(struct kvm_regs, r27)); + DEFINE(KVM_REGS_R28, offsetof(struct kvm_regs, r28)); + DEFINE(KVM_REGS_FPCR, offsetof(struct kvm_regs, fpcr)); + DEFINE(KVM_REGS_F0, offsetof(struct kvm_regs, fp[0 * 4])); + DEFINE(KVM_REGS_F1, offsetof(struct kvm_regs, fp[1 * 4])); + DEFINE(KVM_REGS_F2, offsetof(struct kvm_regs, fp[2 * 4])); + DEFINE(KVM_REGS_F3, offsetof(struct kvm_regs, fp[3 * 4])); + DEFINE(KVM_REGS_F4, offsetof(struct kvm_regs, fp[4 * 4])); + DEFINE(KVM_REGS_F5, offsetof(struct kvm_regs, fp[5 * 4])); + DEFINE(KVM_REGS_F6, offsetof(struct kvm_regs, fp[6 * 4])); + DEFINE(KVM_REGS_F7, offsetof(struct kvm_regs, fp[7 * 4])); + DEFINE(KVM_REGS_F8, offsetof(struct kvm_regs, fp[8 * 4])); + DEFINE(KVM_REGS_F9, offsetof(struct kvm_regs, fp[9 * 4])); + DEFINE(KVM_REGS_F10, offsetof(struct kvm_regs, fp[10 * 4])); + DEFINE(KVM_REGS_F11, offsetof(struct kvm_regs, fp[11 * 4])); + DEFINE(KVM_REGS_F12, offsetof(struct kvm_regs, fp[12 * 4])); + DEFINE(KVM_REGS_F13, offsetof(struct kvm_regs, fp[13 * 4])); + DEFINE(KVM_REGS_F14, offsetof(struct kvm_regs, fp[14 * 4])); + DEFINE(KVM_REGS_F15, offsetof(struct kvm_regs, fp[15 * 4])); + DEFINE(KVM_REGS_F16, offsetof(struct kvm_regs, fp[16 * 4])); + DEFINE(KVM_REGS_F17, offsetof(struct kvm_regs, fp[17 * 4])); + DEFINE(KVM_REGS_F18, offsetof(struct kvm_regs, fp[18 * 4])); + DEFINE(KVM_REGS_F19, offsetof(struct kvm_regs, fp[19 * 4])); + DEFINE(KVM_REGS_F20, offsetof(struct kvm_regs, fp[20 * 4])); + DEFINE(KVM_REGS_F21, offsetof(struct kvm_regs, fp[21 * 4])); + DEFINE(KVM_REGS_F22, offsetof(struct kvm_regs, fp[22 * 4])); + DEFINE(KVM_REGS_F23, offsetof(struct kvm_regs, fp[23 * 4])); + DEFINE(KVM_REGS_F24, offsetof(struct kvm_regs, fp[24 * 4])); + DEFINE(KVM_REGS_F25, offsetof(struct kvm_regs, fp[25 * 4])); + DEFINE(KVM_REGS_F26, offsetof(struct kvm_regs, fp[26 * 4])); + DEFINE(KVM_REGS_F27, offsetof(struct kvm_regs, fp[27 * 4])); + DEFINE(KVM_REGS_F28, offsetof(struct kvm_regs, fp[28 * 4])); + DEFINE(KVM_REGS_F29, offsetof(struct kvm_regs, fp[29 * 4])); + DEFINE(KVM_REGS_F30, offsetof(struct kvm_regs, fp[30 * 4])); + DEFINE(KVM_REGS_PS, offsetof(struct kvm_regs, ps)); + DEFINE(KVM_REGS_PC, offsetof(struct kvm_regs, pc)); + DEFINE(KVM_REGS_GP, offsetof(struct kvm_regs, gp)); + DEFINE(KVM_REGS_R16, offsetof(struct kvm_regs, r16)); + DEFINE(KVM_REGS_R17, offsetof(struct kvm_regs, r17)); + DEFINE(KVM_REGS_R18, offsetof(struct kvm_regs, r18)); + BLANK(); + + DEFINE(VCPU_RET_SIZE, sizeof(struct vcpu_run_ret_stack)); + DEFINE(VCPU_RET_RA, offsetof(struct vcpu_run_ret_stack, ra)); + DEFINE(VCPU_RET_R0, offsetof(struct vcpu_run_ret_stack, r0)); + BLANK(); + + DEFINE(HOST_INT_SIZE, sizeof(struct host_int_args)); + DEFINE(HOST_INT_R18, offsetof(struct host_int_args, r18)); + DEFINE(HOST_INT_R17, offsetof(struct host_int_args, r17)); + DEFINE(HOST_INT_R16, offsetof(struct host_int_args, r16)); + BLANK(); + + OFFSET(TASK_THREAD, task_struct, thread); + OFFSET(TASK_THREAD_F0, task_struct, thread.fpstate.fp[0]); + OFFSET(TASK_THREAD_F1, task_struct, thread.fpstate.fp[1]); + OFFSET(TASK_THREAD_F2, task_struct, thread.fpstate.fp[2]); + OFFSET(TASK_THREAD_F3, task_struct, thread.fpstate.fp[3]); + OFFSET(TASK_THREAD_F4, task_struct, thread.fpstate.fp[4]); + OFFSET(TASK_THREAD_F5, task_struct, thread.fpstate.fp[5]); + OFFSET(TASK_THREAD_F6, task_struct, thread.fpstate.fp[6]); + OFFSET(TASK_THREAD_F7, task_struct, thread.fpstate.fp[7]); + OFFSET(TASK_THREAD_F8, task_struct, thread.fpstate.fp[8]); + OFFSET(TASK_THREAD_F9, task_struct, thread.fpstate.fp[9]); + OFFSET(TASK_THREAD_F10, task_struct, thread.fpstate.fp[10]); + OFFSET(TASK_THREAD_F11, task_struct, thread.fpstate.fp[11]); + OFFSET(TASK_THREAD_F12, task_struct, thread.fpstate.fp[12]); + OFFSET(TASK_THREAD_F13, task_struct, thread.fpstate.fp[13]); + OFFSET(TASK_THREAD_F14, task_struct, thread.fpstate.fp[14]); + OFFSET(TASK_THREAD_F15, task_struct, thread.fpstate.fp[15]); + OFFSET(TASK_THREAD_F16, task_struct, thread.fpstate.fp[16]); + OFFSET(TASK_THREAD_F17, task_struct, thread.fpstate.fp[17]); + OFFSET(TASK_THREAD_F18, task_struct, thread.fpstate.fp[18]); + OFFSET(TASK_THREAD_F19, task_struct, thread.fpstate.fp[19]); + OFFSET(TASK_THREAD_F20, task_struct, thread.fpstate.fp[20]); + OFFSET(TASK_THREAD_F21, task_struct, thread.fpstate.fp[21]); + OFFSET(TASK_THREAD_F22, task_struct, thread.fpstate.fp[22]); + OFFSET(TASK_THREAD_F23, task_struct, thread.fpstate.fp[23]); + OFFSET(TASK_THREAD_F24, task_struct, thread.fpstate.fp[24]); + OFFSET(TASK_THREAD_F25, task_struct, thread.fpstate.fp[25]); + OFFSET(TASK_THREAD_F26, task_struct, thread.fpstate.fp[26]); + OFFSET(TASK_THREAD_F27, task_struct, thread.fpstate.fp[27]); + OFFSET(TASK_THREAD_F28, task_struct, thread.fpstate.fp[28]); + OFFSET(TASK_THREAD_F29, task_struct, thread.fpstate.fp[29]); + OFFSET(TASK_THREAD_F30, task_struct, thread.fpstate.fp[30]); + OFFSET(TASK_THREAD_FPCR, task_struct, thread.fpstate.fpcr); + BLANK(); + OFFSET(TASK_THREAD_RA, task_struct, thread.ra); + OFFSET(TASK_THREAD_SP, task_struct, thread.sp); + OFFSET(TASK_THREAD_S0, task_struct, thread.s[0]); + OFFSET(TASK_THREAD_S1, task_struct, thread.s[1]); + OFFSET(TASK_THREAD_S2, task_struct, thread.s[2]); + OFFSET(TASK_THREAD_S3, task_struct, thread.s[3]); + OFFSET(TASK_THREAD_S4, task_struct, thread.s[4]); + OFFSET(TASK_THREAD_S5, task_struct, thread.s[5]); + OFFSET(TASK_THREAD_S6, task_struct, thread.s[6]); + BLANK(); + DEFINE(ASM_THREAD_SIZE, THREAD_SIZE); + BLANK(); + DEFINE(RT_SIGFRAME_SIZE, sizeof(struct rt_sigframe)); + OFFSET(RT_SIGFRAME_MCTX, rt_sigframe, uc.uc_mcontext); +} diff --git a/arch/sw_64/kernel/audit.c b/arch/sw_64/kernel/audit.c new file mode 100644 index 000000000000..dcf58deee3e2 --- /dev/null +++ b/arch/sw_64/kernel/audit.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include + +static unsigned int dir_class[] = { +#include +~0U +}; + +static unsigned int read_class[] = { +#include +~0U +}; + +static unsigned int write_class[] = { +#include +~0U +}; + +static unsigned int chattr_class[] = { +#include +~0U +}; + +static unsigned int signal_class[] = { +#include +~0U +}; + +int audit_classify_arch(int arch) +{ + return 0; +} + +int audit_classify_syscall(int abi, unsigned int syscall) +{ + switch (syscall) { + case __NR_open: + return 2; + case __NR_openat: + return 3; + case __NR_execve: + return 5; + default: + return 0; + } +} + +static int __init audit_classes_init(void) +{ + audit_register_class(AUDIT_CLASS_WRITE, write_class); + audit_register_class(AUDIT_CLASS_READ, read_class); + audit_register_class(AUDIT_CLASS_DIR_WRITE, dir_class); + audit_register_class(AUDIT_CLASS_CHATTR, chattr_class); + audit_register_class(AUDIT_CLASS_SIGNAL, signal_class); + return 0; +} + +device_initcall(audit_classes_init); diff --git a/arch/sw_64/kernel/early_printk.c b/arch/sw_64/kernel/early_printk.c new file mode 100644 index 000000000000..66af1165e89b --- /dev/null +++ b/arch/sw_64/kernel/early_printk.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include + +static unsigned long early_serial_base; /* ttyS0 */ + +#define XMTRDY 0x20 + +#define DLAB 0x80 + +#define TXR 0 /* Transmit register (WRITE) */ +#define RXR 0 /* Receive register (READ) */ +#define IER 1 /* Interrupt Enable */ +#define IIR 2 /* Interrupt ID */ +#define FCR 2 /* FIFO control */ +#define LCR 3 /* Line control */ +#define MCR 4 /* Modem control */ +#define LSR 5 /* Line Status */ +#define MSR 6 /* Modem Status */ +#define DLL 0 /* Divisor Latch Low */ +#define DLH 1 /* Divisor latch High */ + +static void mem32_serial_out(unsigned long addr, int offset, int value) +{ + void __iomem *vaddr = (void __iomem *)addr; + + offset = offset << 9; + + writel(value, vaddr + offset); +} + +static unsigned int mem32_serial_in(unsigned long addr, int offset) +{ + void __iomem *vaddr = (void __iomem *)addr; + + offset = offset << 9; + + return readl(vaddr + offset); +} + +static unsigned int (*serial_in)(unsigned long addr, int offset) = mem32_serial_in; +static void (*serial_out)(unsigned long addr, int offset, int value) = mem32_serial_out; + +static int early_serial_putc(unsigned char ch) +{ + unsigned int timeout = 0xffff; + + while ((serial_in(early_serial_base, LSR) & XMTRDY) == 0 && --timeout) + cpu_relax(); + serial_out(early_serial_base, TXR, ch); + + return timeout ? 0 : -1; +} + +static void early_serial_write(struct console *con, const char *s, unsigned int n) +{ + while (*s && n-- > 0) { + if (*s == '\n') + early_serial_putc('\r'); + early_serial_putc(*s); + s++; + } +} + +static unsigned int uart_get_refclk(void) +{ + return 24000000UL; +} + +static unsigned int uart_calculate_baudrate_divisor(unsigned long baudrate) +{ + unsigned int refclk = uart_get_refclk(); + + return (1 + (2 * refclk) / (baudrate * 16)) / 2; +} + +static __init void early_serial_hw_init(unsigned long baud) +{ + unsigned char c; + unsigned long divisor = uart_calculate_baudrate_divisor(baud); + + serial_out(early_serial_base, LCR, 0x3); /* 8n1 */ + serial_out(early_serial_base, IER, 0); /* no interrupt */ + serial_out(early_serial_base, FCR, 0); /* no fifo */ + serial_out(early_serial_base, MCR, 0x3); /* DTR + RTS */ + + c = serial_in(early_serial_base, LCR); + serial_out(early_serial_base, LCR, c | DLAB); + serial_out(early_serial_base, DLL, divisor & 0xff); + serial_out(early_serial_base, DLH, (divisor >> 8) & 0xff); + serial_out(early_serial_base, LCR, c & ~DLAB); +} + +#define DEFAULT_BAUD 115200 + +static __init void early_serial_init(char *s) +{ + unsigned long baud = DEFAULT_BAUD; + int err; + + if (*s == ',') + ++s; + + if (*s) { + unsigned int port; + static const long bases[] __initconst = { 0xfff0803300000000ULL, + 0xfff0903300000000ULL }; + + if (!strncmp(s, "ttyS", 4)) + s += 4; + err = kstrtouint(s, 10, &port); + if (err || port > 1) + port = 0; + early_serial_base = bases[port]; + s += strcspn(s, ","); + if (*s == ',') + s++; + } + + if (*s) { + err = kstrtoul(s, 0, &baud); + if (err || baud == 0) + baud = DEFAULT_BAUD; + } + + /* These will always be IO based ports */ + serial_in = mem32_serial_in; + serial_out = mem32_serial_out; + + /* Set up the HW */ + early_serial_hw_init(baud); +} + +static struct console early_serial_console = { + .name = "early", + .write = early_serial_write, + .flags = CON_PRINTBUFFER, + .index = -1, +}; + +static void early_console_register(struct console *con, int keep_early) +{ + if (con->index != -1) { + pr_crit("ERROR: earlyprintk= %s already used\n", + con->name); + return; + } + early_console = con; + + if (keep_early) + early_console->flags &= ~CON_BOOT; + else + early_console->flags |= CON_BOOT; + + register_console(early_console); +} + +static int __init setup_early_printk(char *buf) +{ + int keep; + + if (!buf) + return 0; + + if (early_console) + return 0; + + keep = (strstr(buf, "keep") != NULL); + + if (!strncmp(buf, "serial", 6)) { + buf += 6; + early_serial_init(buf); + early_console_register(&early_serial_console, keep); + if (!strncmp(buf, ",ttyS", 5)) + buf += 5; + } + + return 0; +} + +early_param("earlyprintk", setup_early_printk); diff --git a/arch/sw_64/kernel/entry.S b/arch/sw_64/kernel/entry.S new file mode 100644 index 000000000000..59c2ff4eb915 --- /dev/null +++ b/arch/sw_64/kernel/entry.S @@ -0,0 +1,306 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Kernel entry-points. + */ + +#include +#include +#include +#include +#include +#include + + .text + .set noat +/* + * This defines the normal kernel pt-regs layout. + * + * regs 9-15 preserved by C code, saving to pt_regs will make + * them easier to be accessed in an unified way. + * regs 16-18 saved by HMcode + * regs 29-30 saved and set up by HMcode + */ + + .macro SAVE_ALL + ldi $sp, -PT_REGS_HM_PS($sp) + stl $0, PT_REGS_R0($sp) + stl $1, PT_REGS_R1($sp) + stl $2, PT_REGS_R2($sp) + stl $3, PT_REGS_R3($sp) + stl $4, PT_REGS_R4($sp) + stl $28, PT_REGS_R28($sp) + stl $5, PT_REGS_R5($sp) + stl $6, PT_REGS_R6($sp) + stl $7, PT_REGS_R7($sp) + stl $8, PT_REGS_R8($sp) + stl $9, PT_REGS_R9($sp) + stl $10, PT_REGS_R10($sp) + stl $11, PT_REGS_R11($sp) + stl $12, PT_REGS_R12($sp) + stl $13, PT_REGS_R13($sp) + stl $14, PT_REGS_R14($sp) + stl $15, PT_REGS_R15($sp) + stl $19, PT_REGS_R19($sp) + stl $20, PT_REGS_R20($sp) + stl $21, PT_REGS_R21($sp) + stl $22, PT_REGS_R22($sp) + stl $23, PT_REGS_R23($sp) + stl $24, PT_REGS_R24($sp) + stl $25, PT_REGS_R25($sp) + stl $26, PT_REGS_R26($sp) + stl $27, PT_REGS_R27($sp) + ldl $1, PT_REGS_HM_R16($sp) + ldl $2, PT_REGS_HM_R17($sp) + ldl $3, PT_REGS_HM_R18($sp) + ldl $4, PT_REGS_HM_GP($sp) + ldl $5, PT_REGS_HM_PC($sp) + ldl $6, PT_REGS_HM_PS($sp) + stl $1, PT_REGS_R16($sp) + stl $2, PT_REGS_R17($sp) + stl $3, PT_REGS_R18($sp) + stl $4, PT_REGS_GP($sp) + stl $5, PT_REGS_PC($sp) + stl $6, PT_REGS_PS($sp) + and $6, 0x8, $7 + beq $7, 1f + sys_call HMC_rdusp + br 2f +1: ldi $0, PT_REGS_SIZE($sp) +2: stl $0, PT_REGS_SP($sp) + ldi $1, NO_SYSCALL + stl $1, PT_REGS_ORIG_R0($sp) + sys_call HMC_rdktp + .endm + + .macro RESTORE_ALL + ldl $16, PT_REGS_SP($sp) + /* skip wrusp if returning to kernel */ + blt $16, 1f + sys_call HMC_wrusp +1: ldl $1, PT_REGS_R16($sp) + ldl $2, PT_REGS_R17($sp) + ldl $3, PT_REGS_R18($sp) + ldl $4, PT_REGS_GP($sp) + ldl $5, PT_REGS_PC($sp) + ldl $6, PT_REGS_PS($sp) + stl $1, PT_REGS_HM_R16($sp) + stl $2, PT_REGS_HM_R17($sp) + stl $3, PT_REGS_HM_R18($sp) + stl $4, PT_REGS_HM_GP($sp) + stl $5, PT_REGS_HM_PC($sp) + stl $6, PT_REGS_HM_PS($sp) + ldl $0, PT_REGS_R0($sp) + ldl $1, PT_REGS_R1($sp) + ldl $2, PT_REGS_R2($sp) + ldl $3, PT_REGS_R3($sp) + ldl $4, PT_REGS_R4($sp) + ldl $5, PT_REGS_R5($sp) + ldl $6, PT_REGS_R6($sp) + ldl $7, PT_REGS_R7($sp) + ldl $8, PT_REGS_R8($sp) + ldl $9, PT_REGS_R9($sp) + ldl $10, PT_REGS_R10($sp) + ldl $11, PT_REGS_R11($sp) + ldl $12, PT_REGS_R12($sp) + ldl $13, PT_REGS_R13($sp) + ldl $14, PT_REGS_R14($sp) + ldl $15, PT_REGS_R15($sp) + ldl $19, PT_REGS_R19($sp) + ldl $20, PT_REGS_R20($sp) + ldl $21, PT_REGS_R21($sp) + ldl $22, PT_REGS_R22($sp) + ldl $23, PT_REGS_R23($sp) + ldl $24, PT_REGS_R24($sp) + ldl $25, PT_REGS_R25($sp) + ldl $26, PT_REGS_R26($sp) + ldl $27, PT_REGS_R27($sp) + ldl $28, PT_REGS_R28($sp) + ldi $sp, PT_REGS_HM_PS($sp) + .endm + +/* + * Non-syscall kernel entry points. + */ + + .align 4 + .globl entInt + .ent entInt +entInt: + SAVE_ALL + mov $sp, $19 + call $26, do_entInt + br ret_from_sys_call + .end entInt + + .align 4 + .globl entArith + .ent entArith +entArith: + SAVE_ALL + mov $sp, $18 + call $26, do_entArith + br ret_from_sys_call + .end entArith + + .align 4 + .globl entMM + .ent entMM +entMM: + SAVE_ALL + mov $sp, $19 + call $26, do_page_fault + br ret_from_sys_call + .end entMM + + .align 4 + .globl entIF + .ent entIF +entIF: + SAVE_ALL + mov $sp, $18 + call $26, do_entIF + br ret_from_sys_call + .end entIF + +/* + * Handle unalignment exception. + * We don't handle the "gp" register correctly, but if we fault on a + * gp-register unaligned load/store, something is _very_ wrong in the + * kernel anyway. + */ + .align 4 + .globl entUna + .ent entUna +entUna: + SAVE_ALL + mov $sp, $19 + ldl $0, PT_REGS_PS($sp) + and $0, 8, $0 /* user mode ? */ + beq $0, 1f + call $26, do_entUnaUser /* return to ret_from_syscall */ + br ret_from_sys_call +1: ldl $9, PT_REGS_GP($sp) + call $26, do_entUna + stl $9, PT_REGS_GP($sp) + RESTORE_ALL + sys_call HMC_rti + .end entUna + +/* + * The system call entry point is special. Most importantly, it looks + * like a function call to userspace as far as clobbered registers. We + * do preserve the argument registers (for syscall restarts) and $26 + * (for leaf syscall functions). + * + * So much for theory. We don't take advantage of this yet. + * + * Note that a0-a2 are not saved by HMcode as with the other entry points. + */ + + .align 4 + .globl entSys + .ent entSys +entSys: + SAVE_ALL + stl $16, PT_REGS_R16($sp) + stl $17, PT_REGS_R17($sp) + stl $18, PT_REGS_R18($sp) + mov $sp, $16 + call $26, do_entSys + br ret_from_sys_call + .end entSys + + .align 4 + .globl ret_from_sys_call + .ent ret_from_sys_call +ret_from_sys_call: +#ifdef CONFIG_SUBARCH_C3B + fillcs 0($sp) /* prefetch */ + fillcs 128($sp) /* prefetch */ +#endif + br $27, 1f +1: ldgp $29, 0($27) + /* Make sure need_resched and sigpending don't change between + sampling and the rti. */ + ldi $16, 7 + sys_call HMC_swpipl + ldl $0, PT_REGS_PS($sp) + and $0, 8, $0 + beq $0, restore_all +ret_to_user: + ldw $17, TI_FLAGS($8) + and $17, _TIF_WORK_MASK, $2 + beq $2, restore_all + mov $sp, $16 + call $26, do_notify_resume +restore_all: + RESTORE_ALL + sys_call HMC_rti + .end ret_from_sys_call + +/* + * Integer register context switch + * The callee-saved registers must be saved and restored. + * + * a0: previous task_struct (must be preserved across the switch) + * a1: next task_struct + * + * The value of a0 must be preserved by this function, as that's how + * arguments are passed to schedule_tail. + */ + .align 4 + .globl __switch_to + .ent __switch_to +__switch_to: + .prologue 0 + /* Save context into prev->thread */ + stl $26, TASK_THREAD_RA($16) + stl $30, TASK_THREAD_SP($16) + stl $9, TASK_THREAD_S0($16) + stl $10, TASK_THREAD_S1($16) + stl $11, TASK_THREAD_S2($16) + stl $12, TASK_THREAD_S3($16) + stl $13, TASK_THREAD_S4($16) + stl $14, TASK_THREAD_S5($16) + stl $15, TASK_THREAD_S6($16) + /* Restore context from next->thread */ + ldl $26, TASK_THREAD_RA($17) + ldl $30, TASK_THREAD_SP($17) + ldl $9, TASK_THREAD_S0($17) + ldl $10, TASK_THREAD_S1($17) + ldl $11, TASK_THREAD_S2($17) + ldl $12, TASK_THREAD_S3($17) + ldl $13, TASK_THREAD_S4($17) + ldl $14, TASK_THREAD_S5($17) + ldl $15, TASK_THREAD_S6($17) + mov $17, $8 + sys_call HMC_wrktp + mov $16, $0 + ret + .end __switch_to + +/* + * New processes begin life here. + */ + + .globl ret_from_fork + .align 4 + .ent ret_from_fork +ret_from_fork: + call $26, schedule_tail + br ret_from_sys_call + .end ret_from_fork + +/* + * ... and new kernel threads - here + */ + .align 4 + .globl ret_from_kernel_thread + .ent ret_from_kernel_thread +ret_from_kernel_thread: + call $26, schedule_tail + mov $9, $27 + mov $10, $16 + call $26, ($9) + br ret_to_user + .end ret_from_kernel_thread -- Gitee From 7ad9397b8443b608d6e91fcea41aaece980fa389 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:34 +0800 Subject: [PATCH 022/524] sw64: add some other routines Add some uncommon routines for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/dup_print.c | 88 +++++++++++++++++++++++++++++++++++ arch/sw_64/kernel/proc_misc.c | 25 ++++++++++ arch/sw_64/kernel/proto.h | 18 +++++++ arch/sw_64/kernel/segvdbg.c | 26 +++++++++++ arch/sw_64/kernel/tc.c | 36 ++++++++++++++ arch/sw_64/kernel/termios.c | 62 ++++++++++++++++++++++++ 6 files changed, 255 insertions(+) create mode 100644 arch/sw_64/kernel/dup_print.c create mode 100644 arch/sw_64/kernel/proc_misc.c create mode 100644 arch/sw_64/kernel/proto.h create mode 100644 arch/sw_64/kernel/segvdbg.c create mode 100644 arch/sw_64/kernel/tc.c create mode 100644 arch/sw_64/kernel/termios.c diff --git a/arch/sw_64/kernel/dup_print.c b/arch/sw_64/kernel/dup_print.c new file mode 100644 index 000000000000..439ac75feb01 --- /dev/null +++ b/arch/sw_64/kernel/dup_print.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#include +#include + +#ifdef CONFIG_SW64_RRK + +#define KERNEL_PRINTK_BUFF_BASE (0x700000UL + __START_KERNEL_map) + +static DEFINE_SPINLOCK(printk_lock); + +unsigned long sw64_printk_offset; +#define PRINTK_SIZE 0x100000UL + +int sw64_printk(const char *fmt, va_list args) +{ + char *sw64_printk_buf; + int printed_len = 0; + unsigned long flags; + + spin_lock_irqsave(&printk_lock, flags); + + sw64_printk_buf = (char *)(KERNEL_PRINTK_BUFF_BASE + sw64_printk_offset); + + if (sw64_printk_offset >= (PRINTK_SIZE-1024)) { //printk wrapped + sw64_printk_offset = 0; + sw64_printk_buf = (char *)(KERNEL_PRINTK_BUFF_BASE + sw64_printk_offset); + memset(sw64_printk_buf, 0, PRINTK_SIZE); + printed_len += vscnprintf(sw64_printk_buf, 1024, fmt, args); + } else { + printed_len += vscnprintf(sw64_printk_buf, 1024, fmt, args); + if (is_in_emul()) { + void __iomem *addr = __va(QEMU_PRINTF_BUFF_BASE); + u64 data = ((u64)sw64_printk_buf & 0xffffffffUL) + | ((u64)printed_len << 32); + *(u64 *)addr = data; + } + } + sw64_printk_offset += printed_len; + spin_unlock_irqrestore(&printk_lock, flags); + return printed_len; +} +#endif + +#ifdef CONFIG_SW64_RRU +#include + +static DEFINE_SPINLOCK(printf_lock); +#define USER_PRINT_BUFF_BASE (0x600000UL + __START_KERNEL_map) +#define USER_PRINT_BUFF_LEN 0x100000UL +#define USER_MESSAGE_MAX_LEN 0x100000UL +unsigned long sw64_printf_offset; +int sw64_user_printf(const char __user *buf, int len) +{ + static char *user_printf_buf; + unsigned long flags; + + if (current->pid <= 0) + return 0; + + /* + * do not write large (fake) message which may not be from + * STDOUT/STDERR any more as file descriptor could be duplicated + * in a pipe. + */ + if (len > USER_MESSAGE_MAX_LEN) + return 0; + + spin_lock_irqsave(&printf_lock, flags); + user_printf_buf = (char *)(USER_PRINT_BUFF_BASE + sw64_printf_offset); + + if (sw64_printf_offset == 0) + memset(user_printf_buf, 0, USER_PRINT_BUFF_LEN); + + if ((sw64_printf_offset + len) > USER_PRINT_BUFF_LEN) { + sw64_printf_offset = 0; + user_printf_buf = (char *)(USER_PRINT_BUFF_BASE + sw64_printf_offset); + memset(user_printf_buf, 0, USER_PRINT_BUFF_LEN); + } + copy_from_user(user_printf_buf, buf, len); + sw64_printf_offset += len; + spin_unlock_irqrestore(&printf_lock, flags); + return 0; +} +#endif diff --git a/arch/sw_64/kernel/proc_misc.c b/arch/sw_64/kernel/proc_misc.c new file mode 100644 index 000000000000..ca107ec1e05e --- /dev/null +++ b/arch/sw_64/kernel/proc_misc.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +extern const struct seq_operations cpu_active_mask_op; +static int cpu_active_mask_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &cpu_active_mask_op); +} + +static const struct file_operations proc_cpu_active_mask_operations = { + .open = cpu_active_mask_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int __init proc_cpu_active_mask_init(void) +{ + proc_create("cpu_active_mask", 0, NULL, &proc_cpu_active_mask_operations); + return 0; +} +fs_initcall(proc_cpu_active_mask_init); diff --git a/arch/sw_64/kernel/proto.h b/arch/sw_64/kernel/proto.h new file mode 100644 index 000000000000..d7222334d1b9 --- /dev/null +++ b/arch/sw_64/kernel/proto.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SW64_KERNEL_PROTO_H +#define _SW64_KERNEL_PROTO_H + +#include +#include +#include +#include + +/* ptrace.c */ +extern int ptrace_set_bpt(struct task_struct *child); +extern int ptrace_cancel_bpt(struct task_struct *child); + +/* traps.c */ +extern void show_regs(struct pt_regs *regs); +extern void die(char *str, struct pt_regs *regs, long err); + +#endif /* _SW64_KERNEL_PROTO_H */ diff --git a/arch/sw_64/kernel/segvdbg.c b/arch/sw_64/kernel/segvdbg.c new file mode 100644 index 000000000000..148d639a08db --- /dev/null +++ b/arch/sw_64/kernel/segvdbg.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Zhi Tongze + * Author: Zhi Tongze + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#include + +#include + +extern bool segv_debug_enabled; + +static int __init segv_debug_init(void) +{ + if (!sw64_debugfs_dir) + return -ENODEV; + + debugfs_create_bool("segv_debug", 0644, + sw64_debugfs_dir, &segv_debug_enabled); + return 0; +} +late_initcall(segv_debug_init); diff --git a/arch/sw_64/kernel/tc.c b/arch/sw_64/kernel/tc.c new file mode 100644 index 000000000000..f2de5ac3d9dc --- /dev/null +++ b/arch/sw_64/kernel/tc.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019, serveros, linyue + */ + + +#include +#include + +/* + * Entry/exit counters that make sure that both CPUs + * run the measurement code at once: + */ +unsigned long time_sync; + +DEFINE_PER_CPU(u64, tc_offset); + +void tc_sync_clear(void) +{ + time_sync = 0; +} + +void tc_sync_ready(void *ignored) +{ + /* make sure we can see time_sync been set to 0 */ + smp_mb(); + while (!time_sync) + cpu_relax(); + + __this_cpu_write(tc_offset, time_sync - rdtc()); +} + +void tc_sync_set(void) +{ + time_sync = rdtc() + __this_cpu_read(tc_offset); +} diff --git a/arch/sw_64/kernel/termios.c b/arch/sw_64/kernel/termios.c new file mode 100644 index 000000000000..5c76a513c896 --- /dev/null +++ b/arch/sw_64/kernel/termios.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +/* + * Translate a "termio" structure into a "termios". Ugh. + */ + +int user_termio_to_kernel_termios(struct ktermios *a_termios, struct termio __user *u_termio) +{ + struct ktermios *k_termios = (a_termios); + struct termio k_termio; + int canon, ret; + + ret = copy_from_user(&k_termio, u_termio, sizeof(k_termio)); + if (!ret) { + /* Overwrite only the low bits. */ + *(unsigned short *)&k_termios->c_iflag = k_termio.c_iflag; + *(unsigned short *)&k_termios->c_oflag = k_termio.c_oflag; + *(unsigned short *)&k_termios->c_cflag = k_termio.c_cflag; + *(unsigned short *)&k_termios->c_lflag = k_termio.c_lflag; + canon = k_termio.c_lflag & ICANON; + + k_termios->c_cc[VINTR] = k_termio.c_cc[_VINTR]; + k_termios->c_cc[VQUIT] = k_termio.c_cc[_VQUIT]; + k_termios->c_cc[VERASE] = k_termio.c_cc[_VERASE]; + k_termios->c_cc[VKILL] = k_termio.c_cc[_VKILL]; + k_termios->c_cc[VEOL2] = k_termio.c_cc[_VEOL2]; + k_termios->c_cc[VSWTC] = k_termio.c_cc[_VSWTC]; + k_termios->c_cc[canon ? VEOF : VMIN] = k_termio.c_cc[_VEOF]; + k_termios->c_cc[canon ? VEOL : VTIME] = k_termio.c_cc[_VEOL]; + } + return ret; +} + +/* + * Translate a "termios" structure into a "termio". Ugh. + * + * Note the "fun" _VMIN overloading. + */ +int kernel_termios_to_user_termio(struct termio __user *u_termio, struct ktermios *a_termios) +{ + struct ktermios *k_termios = (a_termios); + struct termio k_termio; + int canon; + + k_termio.c_iflag = k_termios->c_iflag; + k_termio.c_oflag = k_termios->c_oflag; + k_termio.c_cflag = k_termios->c_cflag; + canon = (k_termio.c_lflag = k_termios->c_lflag) & ICANON; + + k_termio.c_line = k_termios->c_line; + k_termio.c_cc[_VINTR] = k_termios->c_cc[VINTR]; + k_termio.c_cc[_VQUIT] = k_termios->c_cc[VQUIT]; + k_termio.c_cc[_VERASE] = k_termios->c_cc[VERASE]; + k_termio.c_cc[_VKILL] = k_termios->c_cc[VKILL]; + k_termio.c_cc[_VEOF] = k_termios->c_cc[canon ? VEOF : VMIN]; + k_termio.c_cc[_VEOL] = k_termios->c_cc[canon ? VEOL : VTIME]; + k_termio.c_cc[_VEOL2] = k_termios->c_cc[VEOL2]; + k_termio.c_cc[_VSWTC] = k_termios->c_cc[VSWTC]; + + return copy_to_user(u_termio, &k_termio, sizeof(k_termio)); +} -- Gitee From a2ba5e28d79ca7dca8e52f026008d30841c6e072 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:25 +0800 Subject: [PATCH 023/524] sw64: add some library functions Add some library functions for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/checksum.h | 126 ++++ arch/sw_64/include/asm/delay.h | 11 + arch/sw_64/include/asm/string.h | 54 ++ arch/sw_64/include/asm/xor.h | 857 +++++++++++++++++++++ arch/sw_64/include/uapi/asm/swab.h | 43 ++ arch/sw_64/lib/Kconfig | 47 ++ arch/sw_64/lib/Makefile | 53 ++ arch/sw_64/lib/checksum.c | 147 ++++ arch/sw_64/lib/clear_page.S | 46 ++ arch/sw_64/lib/clear_user.S | 102 +++ arch/sw_64/lib/copy_page.S | 71 ++ arch/sw_64/lib/copy_user.S | 106 +++ arch/sw_64/lib/csum_ipv6_magic.S | 113 +++ arch/sw_64/lib/csum_partial_copy.c | 154 ++++ arch/sw_64/lib/deep-clear_page.S | 53 ++ arch/sw_64/lib/deep-clear_user.S | 52 ++ arch/sw_64/lib/deep-copy_page.S | 60 ++ arch/sw_64/lib/deep-copy_template.S | 301 ++++++++ arch/sw_64/lib/deep-copy_template_c4.S | 108 +++ arch/sw_64/lib/deep-copy_user.S | 53 ++ arch/sw_64/lib/deep-memcpy.S | 24 + arch/sw_64/lib/deep-memset.S | 97 +++ arch/sw_64/lib/deep-set_template.S | 133 ++++ arch/sw_64/lib/deep-set_template_c4.S | 93 +++ arch/sw_64/lib/divide.S | 190 +++++ arch/sw_64/lib/fls.c | 33 + arch/sw_64/lib/fpreg.c | 992 +++++++++++++++++++++++++ arch/sw_64/lib/iomap.c | 477 ++++++++++++ arch/sw_64/lib/iomap_copy.c | 52 ++ arch/sw_64/lib/memcpy.S | 201 +++++ arch/sw_64/lib/memmove.S | 148 ++++ arch/sw_64/lib/memset.S | 153 ++++ arch/sw_64/lib/strcpy.S | 131 ++++ arch/sw_64/lib/strncpy.S | 156 ++++ arch/sw_64/lib/uaccess_flushcache.c | 42 ++ arch/sw_64/lib/udelay.c | 59 ++ 36 files changed, 5538 insertions(+) create mode 100644 arch/sw_64/include/asm/checksum.h create mode 100644 arch/sw_64/include/asm/delay.h create mode 100644 arch/sw_64/include/asm/string.h create mode 100644 arch/sw_64/include/asm/xor.h create mode 100644 arch/sw_64/include/uapi/asm/swab.h create mode 100644 arch/sw_64/lib/Kconfig create mode 100644 arch/sw_64/lib/Makefile create mode 100644 arch/sw_64/lib/checksum.c create mode 100644 arch/sw_64/lib/clear_page.S create mode 100644 arch/sw_64/lib/clear_user.S create mode 100644 arch/sw_64/lib/copy_page.S create mode 100644 arch/sw_64/lib/copy_user.S create mode 100644 arch/sw_64/lib/csum_ipv6_magic.S create mode 100644 arch/sw_64/lib/csum_partial_copy.c create mode 100644 arch/sw_64/lib/deep-clear_page.S create mode 100644 arch/sw_64/lib/deep-clear_user.S create mode 100644 arch/sw_64/lib/deep-copy_page.S create mode 100644 arch/sw_64/lib/deep-copy_template.S create mode 100644 arch/sw_64/lib/deep-copy_template_c4.S create mode 100644 arch/sw_64/lib/deep-copy_user.S create mode 100644 arch/sw_64/lib/deep-memcpy.S create mode 100644 arch/sw_64/lib/deep-memset.S create mode 100644 arch/sw_64/lib/deep-set_template.S create mode 100644 arch/sw_64/lib/deep-set_template_c4.S create mode 100644 arch/sw_64/lib/divide.S create mode 100644 arch/sw_64/lib/fls.c create mode 100644 arch/sw_64/lib/fpreg.c create mode 100644 arch/sw_64/lib/iomap.c create mode 100644 arch/sw_64/lib/iomap_copy.c create mode 100644 arch/sw_64/lib/memcpy.S create mode 100644 arch/sw_64/lib/memmove.S create mode 100644 arch/sw_64/lib/memset.S create mode 100644 arch/sw_64/lib/strcpy.S create mode 100644 arch/sw_64/lib/strncpy.S create mode 100644 arch/sw_64/lib/uaccess_flushcache.c create mode 100644 arch/sw_64/lib/udelay.c diff --git a/arch/sw_64/include/asm/checksum.h b/arch/sw_64/include/asm/checksum.h new file mode 100644 index 000000000000..7f3768290402 --- /dev/null +++ b/arch/sw_64/include/asm/checksum.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CHECKSUM_H +#define _ASM_SW64_CHECKSUM_H + +#include + +#define extll(x, y, z) \ + ({__asm__ __volatile__("extll %1, %2, %0" : "=r" (z) \ + : "r" (x), "r" (y)); }) + +#define exthl(x, y, z) \ + ({__asm__ __volatile__("exthl %1, %2, %0" : "=r" (z) \ + : "r" (x), "r" (y)); }) + +#define maskll(x, y, z) \ + ({__asm__ __volatile__("maskll %1, %2, %0" : "=r" (z) \ + : "r" (x), "r" (y)); }) + +#define maskhl(x, y, z) \ + ({__asm__ __volatile__("maskhl %1, %2, %0" : "=r" (z) \ + : "r" (x), "r" (y)); }) + +#define insll(x, y, z) \ + ({__asm__ __volatile__("insll %1, %2, %0" : "=r" (z) \ + : "r" (x), "r" (y)); }) + +#define inshl(x, y, z) \ + ({__asm__ __volatile__("inshl %1, %2, %0" : "=r" (z) \ + : "r" (x), "r" (y)); }) + +/* + * This is a version of ip_compute_csum() optimized for IP headers, + * which always checksum on 4 octet boundaries. + */ +extern __sum16 ip_fast_csum(const void *iph, unsigned int ihl); + +/* + * computes the checksum of the TCP/UDP pseudo-header + * returns a 16-bit checksum, already complemented + */ +__sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, + __u32 len, __u8 proto, __wsum sum); + +__wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, + __u32 len, __u8 proto, __wsum sum); + +/* + * computes the checksum of a memory block at buff, length len, + * and adds in "sum" (32-bit) + * + * returns a 32-bit number suitable for feeding into itself + * or csum_tcpudp_magic + * + * this function must be called with even lengths, except + * for the last fragment, which may be odd + * + * it's best to have buff aligned on a 32-bit boundary + */ +extern __wsum csum_partial(const void *buff, int len, __wsum sum); + +/* + * the same as csum_partial, but copies from src while it + * checksums + * + * here even more important to align src and dst on a 32-bit (or even + * better 64-bit) boundary + */ +#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER +#define _HAVE_ARCH_CSUM_AND_COPY +__wsum csum_and_copy_from_user(const void __user *src, void *dst, int len); + +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len); + +/* + * this routine is used for miscellaneous IP-like checksums, mainly + * in icmp.c + */ + +extern __sum16 ip_compute_csum(const void *buff, int len); + +/* + * Fold a partial checksum without adding pseudo headers + */ + +static inline __sum16 csum_fold(__wsum csum) +{ + u32 sum = (__force u32)csum; + + sum = (sum & 0xffff) + (sum >> 16); + sum = (sum & 0xffff) + (sum >> 16); + return (__force __sum16)~sum; +} + +#define _HAVE_ARCH_IPV6_CSUM +extern __sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, __u32 len, + __u8 proto, __wsum sum); + +static inline unsigned short from64to16(unsigned long x) +{ + /* + * Using extract instructions is a bit more efficient + * than the original shift/bitmask version. + */ + + union { + unsigned long ul; + unsigned int ui[2]; + unsigned short us[4]; + } in_v, tmp_v, out_v; + + in_v.ul = x; + tmp_v.ul = (unsigned long)in_v.ui[0] + (unsigned long)in_v.ui[1]; + + /* + * Since the bits of tmp_v.sh[3] are going to always be zero, + * we don't have to bother to add that in. + */ + out_v.ul = (unsigned long)tmp_v.us[0] + (unsigned long)tmp_v.us[1] + + (unsigned long)tmp_v.us[2]; + + /* Similarly, out_v.us[2] is always zero for the final add. */ + return out_v.us[0] + out_v.us[1]; +} + +#endif /* _ASM_SW64_CHECKSUM_H */ diff --git a/arch/sw_64/include/asm/delay.h b/arch/sw_64/include/asm/delay.h new file mode 100644 index 000000000000..f4080753e954 --- /dev/null +++ b/arch/sw_64/include/asm/delay.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_DELAY_H +#define _ASM_SW64_DELAY_H + +extern void __delay(unsigned long loops); +extern void udelay(unsigned long usecs); + +extern void ndelay(unsigned long nsecs); +#define ndelay ndelay + +#endif /* _ASM_SW64_DELAY_H */ diff --git a/arch/sw_64/include/asm/string.h b/arch/sw_64/include/asm/string.h new file mode 100644 index 000000000000..87d93f4cd4d5 --- /dev/null +++ b/arch/sw_64/include/asm/string.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_STRING_H +#define _ASM_SW64_STRING_H + +#ifdef __KERNEL__ + +/* + * GCC of any recent vintage doesn't do stupid things with bcopy. + * EGCS 1.1 knows all about expanding memcpy inline, others don't. + * + * Similarly for a memset with data = 0. + */ + +#define __HAVE_ARCH_MEMCPY +extern void *memcpy(void *dest, const void *src, size_t n); +/* For backward compatibility with modules. Unused otherwise. */ +extern void *__memcpy(void *dest, const void *src, size_t n); + +#define __HAVE_ARCH_MEMMOVE +extern void *memmove(void *dest, const void *src, size_t n); + +#define __HAVE_ARCH_MEMSET +extern void *__constant_c_memset(void *s, unsigned long c, size_t n); +extern void *___memset(void *s, int c, size_t n); +extern void *__memset(void *s, int c, size_t n); +extern void *memset(void *s, int c, size_t n); + +#define __HAVE_ARCH_STRCPY +extern char *strcpy(char *dest, const char *src); + +#define __HAVE_ARCH_STRNCPY +extern char *strncpy(char *dest, const char *src, size_t n); + +/* The following routine is like memset except that it writes 16-bit + * aligned values. The DEST and COUNT parameters must be even for + * correct operation. + */ + +#define __HAVE_ARCH_MEMSETW +extern void *__memsetw(void *dest, unsigned short c, size_t count); + +#define memsetw(s, c, n) \ +(__builtin_constant_p(c) \ + ? __constant_c_memset((s), 0x0001000100010001UL * (unsigned short)(c), (n)) \ + : __memsetw((s), (c), (n))) + +#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE +#define __HAVE_ARCH_MEMCPY_FLUSHCACHE +void memcpy_flushcache(void *dst, const void *src, size_t cnt); +#endif + +#endif /* __KERNEL__ */ + +#endif /* _ASM_SW64_STRING_H */ diff --git a/arch/sw_64/include/asm/xor.h b/arch/sw_64/include/asm/xor.h new file mode 100644 index 000000000000..0aff8804f503 --- /dev/null +++ b/arch/sw_64/include/asm/xor.h @@ -0,0 +1,857 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Optimized RAID-5 checksumming functions. + */ + +#ifndef _ASM_SW64_XOR_H +#define _ASM_SW64_XOR_H + +extern void xor_sw64_2(unsigned long bytes, unsigned long *__restrict p1, + const unsigned long *__restrict p2); +extern void xor_sw64_3(unsigned long bytes, unsigned long *__restrict p1, + const unsigned long *__restrict p2, + const unsigned long *__restrict p3); +extern void xor_sw64_4(unsigned long bytes, unsigned long *__restrict p1, + const unsigned long *__restrict p2, + const unsigned long *__restrict p3, + const unsigned long *__restrict p4); +extern void xor_sw64_5(unsigned long bytes, unsigned long *__restrict p1, + const unsigned long *__restrict p2, + const unsigned long *__restrict p3, + const unsigned long *__restrict p4, + const unsigned long *__restrict p5); + +extern void xor_sw64_prefetch_2(unsigned long bytes, unsigned long *__restrict p1, + const unsigned long *__restrict p2); +extern void xor_sw64_prefetch_3(unsigned long bytes, unsigned long *__restrict p1, + const unsigned long *__restrict p2, + const unsigned long *__restrict p3); +extern void xor_sw64_prefetch_4(unsigned long bytes, unsigned long *__restrict p1, + const unsigned long *__restrict p2, + const unsigned long *__restrict p3, + const unsigned long *__restrict p4); +extern void xor_sw64_prefetch_5(unsigned long bytes, unsigned long *__restrict p1, + const unsigned long *__restrict p2, + const unsigned long *__restrict p3, + const unsigned long *__restrict p4, + const unsigned long *__restrict p5); + +asm(" \n\ + .text \n\ + .align 3 \n\ + .ent xor_sw64_2 \n\ +xor_sw64_2: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + .align 4 \n\ +2: \n\ + ldl $0, 0($17) \n\ + ldl $1, 0($18) \n\ + ldl $2, 8($17) \n\ + ldl $3, 8($18) \n\ + \n\ + ldl $4, 16($17) \n\ + ldl $5, 16($18) \n\ + ldl $6, 24($17) \n\ + ldl $7, 24($18) \n\ + \n\ + ldl $19, 32($17) \n\ + ldl $20, 32($18) \n\ + ldl $21, 40($17) \n\ + ldl $22, 40($18) \n\ + \n\ + ldl $23, 48($17) \n\ + ldl $24, 48($18) \n\ + ldl $25, 56($17) \n\ + xor $0, $1, $0 # 7 cycles from $1 load \n\ + \n\ + ldl $27, 56($18) \n\ + xor $2, $3, $2 \n\ + stl $0, 0($17) \n\ + xor $4, $5, $4 \n\ + \n\ + stl $2, 8($17) \n\ + xor $6, $7, $6 \n\ + stl $4, 16($17) \n\ + xor $19, $20, $19 \n\ + \n\ + stl $6, 24($17) \n\ + xor $21, $22, $21 \n\ + stl $19, 32($17) \n\ + xor $23, $24, $23 \n\ + \n\ + stl $21, 40($17) \n\ + xor $25, $27, $25 \n\ + stl $23, 48($17) \n\ + subl $16, 1, $16 \n\ + \n\ + stl $25, 56($17) \n\ + addl $17, 64, $17 \n\ + addl $18, 64, $18 \n\ + bgt $16, 2b \n\ + \n\ + ret \n\ + .end xor_sw64_2 \n\ + \n\ + .align 3 \n\ + .ent xor_sw64_3 \n\ +xor_sw64_3: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + .align 4 \n\ +3: \n\ + ldl $0, 0($17) \n\ + ldl $1, 0($18) \n\ + ldl $2, 0($19) \n\ + ldl $3, 8($17) \n\ + \n\ + ldl $4, 8($18) \n\ + ldl $6, 16($17) \n\ + ldl $7, 16($18) \n\ + ldl $21, 24($17) \n\ + \n\ + ldl $22, 24($18) \n\ + ldl $24, 32($17) \n\ + ldl $25, 32($18) \n\ + ldl $5, 8($19) \n\ + \n\ + ldl $20, 16($19) \n\ + ldl $23, 24($19) \n\ + ldl $27, 32($19) \n\ + \n\ + xor $0, $1, $1 # 8 cycles from $0 load \n\ + xor $3, $4, $4 # 6 cycles from $4 load \n\ + xor $6, $7, $7 # 6 cycles from $7 load \n\ + xor $21, $22, $22 # 5 cycles from $22 load \n\ + \n\ + xor $1, $2, $2 # 9 cycles from $2 load \n\ + xor $24, $25, $25 # 5 cycles from $25 load \n\ + stl $2, 0($17) \n\ + xor $4, $5, $5 # 6 cycles from $5 load \n\ + \n\ + stl $5, 8($17) \n\ + xor $7, $20, $20 # 7 cycles from $20 load \n\ + stl $20, 16($17) \n\ + xor $22, $23, $23 # 7 cycles from $23 load \n\ + \n\ + stl $23, 24($17) \n\ + xor $25, $27, $27 # 7 cycles from $27 load \n\ + stl $27, 32($17) \n\ + \n\ + ldl $0, 40($17) \n\ + ldl $1, 40($18) \n\ + ldl $3, 48($17) \n\ + ldl $4, 48($18) \n\ + \n\ + ldl $6, 56($17) \n\ + ldl $7, 56($18) \n\ + ldl $2, 40($19) \n\ + ldl $5, 48($19) \n\ + \n\ + ldl $20, 56($19) \n\ + xor $0, $1, $1 # 4 cycles from $1 load \n\ + xor $3, $4, $4 # 5 cycles from $4 load \n\ + xor $6, $7, $7 # 5 cycles from $7 load \n\ + \n\ + xor $1, $2, $2 # 4 cycles from $2 load \n\ + xor $4, $5, $5 # 5 cycles from $5 load \n\ + stl $2, 40($17) \n\ + xor $7, $20, $20 # 4 cycles from $20 load \n\ + \n\ + stl $5, 48($17) \n\ + subl $16, 1, $16 \n\ + stl $20, 56($17) \n\ + addl $19, 64, $19 \n\ + \n\ + addl $18, 64, $18 \n\ + addl $17, 64, $17 \n\ + bgt $16, 3b \n\ + ret \n\ + .end xor_sw64_3 \n\ + \n\ + .align 3 \n\ + .ent xor_sw64_4 \n\ +xor_sw64_4: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + .align 4 \n\ +4: \n\ + ldl $0, 0($17) \n\ + ldl $1, 0($18) \n\ + ldl $2, 0($19) \n\ + ldl $3, 0($20) \n\ + \n\ + ldl $4, 8($17) \n\ + ldl $5, 8($18) \n\ + ldl $6, 8($19) \n\ + ldl $7, 8($20) \n\ + \n\ + ldl $21, 16($17) \n\ + ldl $22, 16($18) \n\ + ldl $23, 16($19) \n\ + ldl $24, 16($20) \n\ + \n\ + ldl $25, 24($17) \n\ + xor $0, $1, $1 # 6 cycles from $1 load \n\ + ldl $27, 24($18) \n\ + xor $2, $3, $3 # 6 cycles from $3 load \n\ + \n\ + ldl $0, 24($19) \n\ + xor $1, $3, $3 \n\ + ldl $1, 24($20) \n\ + xor $4, $5, $5 # 7 cycles from $5 load \n\ + \n\ + stl $3, 0($17) \n\ + xor $6, $7, $7 \n\ + xor $21, $22, $22 # 7 cycles from $22 load \n\ + xor $5, $7, $7 \n\ + \n\ + stl $7, 8($17) \n\ + xor $23, $24, $24 # 7 cycles from $24 load \n\ + ldl $2, 32($17) \n\ + xor $22, $24, $24 \n\ + \n\ + ldl $3, 32($18) \n\ + ldl $4, 32($19) \n\ + ldl $5, 32($20) \n\ + xor $25, $27, $27 # 8 cycles from $27 load \n\ + \n\ + ldl $6, 40($17) \n\ + ldl $7, 40($18) \n\ + ldl $21, 40($19) \n\ + ldl $22, 40($20) \n\ + \n\ + stl $24, 16($17) \n\ + xor $0, $1, $1 # 9 cycles from $1 load \n\ + xor $2, $3, $3 # 5 cycles from $3 load \n\ + xor $27, $1, $1 \n\ + \n\ + stl $1, 24($17) \n\ + xor $4, $5, $5 # 5 cycles from $5 load \n\ + ldl $23, 48($17) \n\ + ldl $24, 48($18) \n\ + \n\ + ldl $25, 48($19) \n\ + xor $3, $5, $5 \n\ + ldl $27, 48($20) \n\ + ldl $0, 56($17) \n\ + \n\ + ldl $1, 56($18) \n\ + ldl $2, 56($19) \n\ + xor $6, $7, $7 # 8 cycles from $6 load \n\ + ldl $3, 56($20) \n\ + \n\ + stl $5, 32($17) \n\ + xor $21, $22, $22 # 8 cycles from $22 load \n\ + xor $7, $22, $22 \n\ + xor $23, $24, $24 # 5 cycles from $24 load \n\ + \n\ + stl $22, 40($17) \n\ + xor $25, $27, $27 # 5 cycles from $27 load \n\ + xor $24, $27, $27 \n\ + xor $0, $1, $1 # 5 cycles from $1 load \n\ + \n\ + stl $27, 48($17) \n\ + xor $2, $3, $3 # 4 cycles from $3 load \n\ + xor $1, $3, $3 \n\ + subl $16, 1, $16 \n\ + \n\ + stl $3, 56($17) \n\ + addl $20, 64, $20 \n\ + addl $19, 64, $19 \n\ + addl $18, 64, $18 \n\ + \n\ + addl $17, 64, $17 \n\ + bgt $16, 4b \n\ + ret \n\ + .end xor_sw64_4 \n\ + \n\ + .align 3 \n\ + .ent xor_sw64_5 \n\ +xor_sw64_5: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + .align 4 \n\ +5: \n\ + ldl $0, 0($17) \n\ + ldl $1, 0($18) \n\ + ldl $2, 0($19) \n\ + ldl $3, 0($20) \n\ + \n\ + ldl $4, 0($21) \n\ + ldl $5, 8($17) \n\ + ldl $6, 8($18) \n\ + ldl $7, 8($19) \n\ + \n\ + ldl $22, 8($20) \n\ + ldl $23, 8($21) \n\ + ldl $24, 16($17) \n\ + ldl $25, 16($18) \n\ + \n\ + ldl $27, 16($19) \n\ + xor $0, $1, $1 # 6 cycles from $1 load \n\ + ldl $28, 16($20) \n\ + xor $2, $3, $3 # 6 cycles from $3 load \n\ + \n\ + ldl $0, 16($21) \n\ + xor $1, $3, $3 \n\ + ldl $1, 24($17) \n\ + xor $3, $4, $4 # 7 cycles from $4 load \n\ + \n\ + stl $4, 0($17) \n\ + xor $5, $6, $6 # 7 cycles from $6 load \n\ + xor $7, $22, $22 # 7 cycles from $22 load \n\ + xor $6, $23, $23 # 7 cycles from $23 load \n\ + \n\ + ldl $2, 24($18) \n\ + xor $22, $23, $23 \n\ + ldl $3, 24($19) \n\ + xor $24, $25, $25 # 8 cycles from $25 load \n\ + \n\ + stl $23, 8($17) \n\ + xor $25, $27, $27 # 8 cycles from $27 load \n\ + ldl $4, 24($20) \n\ + xor $28, $0, $0 # 7 cycles from $0 load \n\ + \n\ + ldl $5, 24($21) \n\ + xor $27, $0, $0 \n\ + ldl $6, 32($17) \n\ + ldl $7, 32($18) \n\ + \n\ + stl $0, 16($17) \n\ + xor $1, $2, $2 # 6 cycles from $2 load \n\ + ldl $22, 32($19) \n\ + xor $3, $4, $4 # 4 cycles from $4 load \n\ + \n\ + ldl $23, 32($20) \n\ + xor $2, $4, $4 \n\ + ldl $24, 32($21) \n\ + ldl $25, 40($17) \n\ + \n\ + ldl $27, 40($18) \n\ + ldl $28, 40($19) \n\ + ldl $0, 40($20) \n\ + xor $4, $5, $5 # 7 cycles from $5 load \n\ + \n\ + stl $5, 24($17) \n\ + xor $6, $7, $7 # 7 cycles from $7 load \n\ + ldl $1, 40($21) \n\ + ldl $2, 48($17) \n\ + \n\ + ldl $3, 48($18) \n\ + xor $7, $22, $22 # 7 cycles from $22 load \n\ + ldl $4, 48($19) \n\ + xor $23, $24, $24 # 6 cycles from $24 load \n\ + \n\ + ldl $5, 48($20) \n\ + xor $22, $24, $24 \n\ + ldl $6, 48($21) \n\ + xor $25, $27, $27 # 7 cycles from $27 load \n\ + \n\ + stl $24, 32($17) \n\ + xor $27, $28, $28 # 8 cycles from $28 load \n\ + ldl $7, 56($17) \n\ + xor $0, $1, $1 # 6 cycles from $1 load \n\ + \n\ + ldl $22, 56($18) \n\ + ldl $23, 56($19) \n\ + ldl $24, 56($20) \n\ + ldl $25, 56($21) \n\ + \n\ + xor $28, $1, $1 \n\ + xor $2, $3, $3 # 9 cycles from $3 load \n\ + xor $3, $4, $4 # 9 cycles from $4 load \n\ + xor $5, $6, $6 # 8 cycles from $6 load \n\ + \n\ + stl $1, 40($17) \n\ + xor $4, $6, $6 \n\ + xor $7, $22, $22 # 7 cycles from $22 load \n\ + xor $23, $24, $24 # 6 cycles from $24 load \n\ + \n\ + stl $6, 48($17) \n\ + xor $22, $24, $24 \n\ + subl $16, 1, $16 \n\ + xor $24, $25, $25 # 8 cycles from $25 load \n\ + \n\ + stl $25, 56($17) \n\ + addl $21, 64, $21 \n\ + addl $20, 64, $20 \n\ + addl $19, 64, $19 \n\ + \n\ + addl $18, 64, $18 \n\ + addl $17, 64, $17 \n\ + bgt $16, 5b \n\ + ret \n\ + .end xor_sw64_5 \n\ + \n\ + .align 3 \n\ + .ent xor_sw64_prefetch_2 \n\ +xor_sw64_prefetch_2: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + \n\ + fillde 0($17) \n\ + fillde 0($18) \n\ + \n\ + fillde 64($17) \n\ + fillde 64($18) \n\ + \n\ + fillde 128($17) \n\ + fillde 128($18) \n\ + \n\ + fillde 192($17) \n\ + fillde 192($18) \n\ + .align 4 \n\ +2: \n\ + ldl $0, 0($17) \n\ + ldl $1, 0($18) \n\ + ldl $2, 8($17) \n\ + ldl $3, 8($18) \n\ + \n\ + ldl $4, 16($17) \n\ + ldl $5, 16($18) \n\ + ldl $6, 24($17) \n\ + ldl $7, 24($18) \n\ + \n\ + ldl $19, 32($17) \n\ + ldl $20, 32($18) \n\ + ldl $21, 40($17) \n\ + ldl $22, 40($18) \n\ + \n\ + ldl $23, 48($17) \n\ + ldl $24, 48($18) \n\ + ldl $25, 56($17) \n\ + ldl $27, 56($18) \n\ + \n\ + fillde 256($17) \n\ + xor $0, $1, $0 # 8 cycles from $1 load \n\ + fillde 256($18) \n\ + xor $2, $3, $2 \n\ + \n\ + stl $0, 0($17) \n\ + xor $4, $5, $4 \n\ + stl $2, 8($17) \n\ + xor $6, $7, $6 \n\ + \n\ + stl $4, 16($17) \n\ + xor $19, $20, $19 \n\ + stl $6, 24($17) \n\ + xor $21, $22, $21 \n\ + \n\ + stl $19, 32($17) \n\ + xor $23, $24, $23 \n\ + stl $21, 40($17) \n\ + xor $25, $27, $25 \n\ + \n\ + stl $23, 48($17) \n\ + subl $16, 1, $16 \n\ + stl $25, 56($17) \n\ + addl $17, 64, $17 \n\ + \n\ + addl $18, 64, $18 \n\ + bgt $16, 2b \n\ + ret \n\ + .end xor_sw64_prefetch_2 \n\ + \n\ + .align 3 \n\ + .ent xor_sw64_prefetch_3 \n\ +xor_sw64_prefetch_3: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + \n\ + fillde 0($17) \n\ + fillde 0($18) \n\ + fillde 0($19) \n\ + \n\ + fillde 64($17) \n\ + fillde 64($18) \n\ + fillde 64($19) \n\ + \n\ + fillde 128($17) \n\ + fillde 128($18) \n\ + fillde 128($19) \n\ + \n\ + fillde 192($17) \n\ + fillde 192($18) \n\ + fillde 192($19) \n\ + .align 4 \n\ +3: \n\ + ldl $0, 0($17) \n\ + ldl $1, 0($18) \n\ + ldl $2, 0($19) \n\ + ldl $3, 8($17) \n\ + \n\ + ldl $4, 8($18) \n\ + ldl $6, 16($17) \n\ + ldl $7, 16($18) \n\ + ldl $21, 24($17) \n\ + \n\ + ldl $22, 24($18) \n\ + ldl $24, 32($17) \n\ + ldl $25, 32($18) \n\ + ldl $5, 8($19) \n\ + \n\ + ldl $20, 16($19) \n\ + ldl $23, 24($19) \n\ + ldl $27, 32($19) \n\ + \n\ + xor $0, $1, $1 # 8 cycles from $0 load \n\ + xor $3, $4, $4 # 7 cycles from $4 load \n\ + xor $6, $7, $7 # 6 cycles from $7 load \n\ + xor $21, $22, $22 # 5 cycles from $22 load \n\ + \n\ + xor $1, $2, $2 # 9 cycles from $2 load \n\ + xor $24, $25, $25 # 5 cycles from $25 load \n\ + stl $2, 0($17) \n\ + xor $4, $5, $5 # 6 cycles from $5 load \n\ + \n\ + stl $5, 8($17) \n\ + xor $7, $20, $20 # 7 cycles from $20 load \n\ + stl $20, 16($17) \n\ + xor $22, $23, $23 # 7 cycles from $23 load \n\ + \n\ + stl $23, 24($17) \n\ + xor $25, $27, $27 # 7 cycles from $27 load \n\ + stl $27, 32($17) \n\ + \n\ + ldl $0, 40($17) \n\ + ldl $1, 40($18) \n\ + ldl $3, 48($17) \n\ + ldl $4, 48($18) \n\ + \n\ + ldl $6, 56($17) \n\ + ldl $7, 56($18) \n\ + ldl $2, 40($19) \n\ + ldl $5, 48($19) \n\ + \n\ + ldl $20, 56($19) \n\ + fillde 256($17) \n\ + fillde 256($18) \n\ + fillde 256($19) \n\ + \n\ + xor $0, $1, $1 # 6 cycles from $1 load \n\ + xor $3, $4, $4 # 5 cycles from $4 load \n\ + xor $6, $7, $7 # 5 cycles from $7 load \n\ + xor $1, $2, $2 # 4 cycles from $2 load \n\ + \n\ + xor $4, $5, $5 # 5 cycles from $5 load \n\ + xor $7, $20, $20 # 4 cycles from $20 load \n\ + stl $2, 40($17) \n\ + subl $16, 1, $16 \n\ + \n\ + stl $5, 48($17) \n\ + addl $19, 64, $19 \n\ + stl $20, 56($17) \n\ + addl $18, 64, $18 \n\ + \n\ + addl $17, 64, $17 \n\ + bgt $16, 3b \n\ + ret \n\ + .end xor_sw64_prefetch_3 \n\ + \n\ + .align 3 \n\ + .ent xor_sw64_prefetch_4 \n\ +xor_sw64_prefetch_4: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + \n\ + fillde 0($17) \n\ + fillde 0($18) \n\ + fillde 0($19) \n\ + fillde 0($20) \n\ + \n\ + fillde 64($17) \n\ + fillde 64($18) \n\ + fillde 64($19) \n\ + fillde 64($20) \n\ + \n\ + fillde 128($17) \n\ + fillde 128($18) \n\ + fillde 128($19) \n\ + fillde 128($20) \n\ + \n\ + fillde 192($17) \n\ + fillde 192($18) \n\ + fillde 192($19) \n\ + fillde 192($20) \n\ + .align 4 \n\ +4: \n\ + ldl $0, 0($17) \n\ + ldl $1, 0($18) \n\ + ldl $2, 0($19) \n\ + ldl $3, 0($20) \n\ + \n\ + ldl $4, 8($17) \n\ + ldl $5, 8($18) \n\ + ldl $6, 8($19) \n\ + ldl $7, 8($20) \n\ + \n\ + ldl $21, 16($17) \n\ + ldl $22, 16($18) \n\ + ldl $23, 16($19) \n\ + ldl $24, 16($20) \n\ + \n\ + ldl $25, 24($17) \n\ + xor $0, $1, $1 # 6 cycles from $1 load \n\ + ldl $27, 24($18) \n\ + xor $2, $3, $3 # 6 cycles from $3 load \n\ + \n\ + ldl $0, 24($19) \n\ + xor $1, $3, $3 \n\ + ldl $1, 24($20) \n\ + xor $4, $5, $5 # 7 cycles from $5 load \n\ + \n\ + stl $3, 0($17) \n\ + xor $6, $7, $7 \n\ + xor $21, $22, $22 # 7 cycles from $22 load \n\ + xor $5, $7, $7 \n\ + \n\ + stl $7, 8($17) \n\ + xor $23, $24, $24 # 7 cycles from $24 load \n\ + ldl $2, 32($17) \n\ + xor $22, $24, $24 \n\ + \n\ + ldl $3, 32($18) \n\ + ldl $4, 32($19) \n\ + ldl $5, 32($20) \n\ + xor $25, $27, $27 # 8 cycles from $27 load \n\ + \n\ + ldl $6, 40($17) \n\ + ldl $7, 40($18) \n\ + ldl $21, 40($19) \n\ + ldl $22, 40($20) \n\ + \n\ + stl $24, 16($17) \n\ + xor $0, $1, $1 # 9 cycles from $1 load \n\ + xor $2, $3, $3 # 5 cycles from $3 load \n\ + xor $27, $1, $1 \n\ + \n\ + stl $1, 24($17) \n\ + xor $4, $5, $5 # 5 cycles from $5 load \n\ + ldl $23, 48($17) \n\ + xor $3, $5, $5 \n\ + \n\ + ldl $24, 48($18) \n\ + ldl $25, 48($19) \n\ + ldl $27, 48($20) \n\ + ldl $0, 56($17) \n\ + \n\ + ldl $1, 56($18) \n\ + ldl $2, 56($19) \n\ + ldl $3, 56($20) \n\ + xor $6, $7, $7 # 8 cycles from $6 load \n\ + \n\ + fillde 256($17) \n\ + xor $21, $22, $22 # 8 cycles from $22 load \n\ + fillde 256($18) \n\ + xor $7, $22, $22 \n\ + \n\ + fillde 256($19) \n\ + xor $23, $24, $24 # 6 cycles from $24 load \n\ + fillde 256($20) \n\ + xor $25, $27, $27 # 6 cycles from $27 load \n\ + \n\ + stl $5, 32($17) \n\ + xor $24, $27, $27 \n\ + xor $0, $1, $1 # 7 cycles from $1 load \n\ + xor $2, $3, $3 # 6 cycles from $3 load \n\ + \n\ + stl $22, 40($17) \n\ + xor $1, $3, $3 \n\ + stl $27, 48($17) \n\ + subl $16, 1, $16 \n\ + \n\ + stl $3, 56($17) \n\ + addl $20, 64, $20 \n\ + addl $19, 64, $19 \n\ + addl $18, 64, $18 \n\ + \n\ + addl $17, 64, $17 \n\ + bgt $16, 4b \n\ + ret \n\ + .end xor_sw64_prefetch_4 \n\ + \n\ + .align 3 \n\ + .ent xor_sw64_prefetch_5 \n\ +xor_sw64_prefetch_5: \n\ + .prologue 0 \n\ + srl $16, 6, $16 \n\ + \n\ + fillde 0($17) \n\ + fillde 0($18) \n\ + fillde 0($19) \n\ + fillde 0($20) \n\ + fillde 0($21) \n\ + \n\ + fillde 64($17) \n\ + fillde 64($18) \n\ + fillde 64($19) \n\ + fillde 64($20) \n\ + fillde 64($21) \n\ + \n\ + fillde 128($17) \n\ + fillde 128($18) \n\ + fillde 128($19) \n\ + fillde 128($20) \n\ + fillde 128($21) \n\ + \n\ + fillde 192($17) \n\ + fillde 192($18) \n\ + fillde 192($19) \n\ + fillde 192($20) \n\ + fillde 192($21) \n\ + .align 4 \n\ +5: \n\ + ldl $0, 0($17) \n\ + ldl $1, 0($18) \n\ + ldl $2, 0($19) \n\ + ldl $3, 0($20) \n\ + \n\ + ldl $4, 0($21) \n\ + ldl $5, 8($17) \n\ + ldl $6, 8($18) \n\ + ldl $7, 8($19) \n\ + \n\ + ldl $22, 8($20) \n\ + ldl $23, 8($21) \n\ + ldl $24, 16($17) \n\ + ldl $25, 16($18) \n\ + \n\ + ldl $27, 16($19) \n\ + xor $0, $1, $1 # 6 cycles from $1 load \n\ + ldl $28, 16($20) \n\ + xor $2, $3, $3 # 6 cycles from $3 load \n\ + \n\ + ldl $0, 16($21) \n\ + xor $1, $3, $3 \n\ + ldl $1, 24($17) \n\ + xor $3, $4, $4 # 7 cycles from $4 load \n\ + \n\ + stl $4, 0($17) \n\ + xor $5, $6, $6 # 7 cycles from $6 load \n\ + xor $7, $22, $22 # 7 cycles from $22 load \n\ + xor $6, $23, $23 # 7 cycles from $23 load \n\ + \n\ + ldl $2, 24($18) \n\ + xor $22, $23, $23 \n\ + ldl $3, 24($19) \n\ + xor $24, $25, $25 # 8 cycles from $25 load \n\ + \n\ + stl $23, 8($17) \n\ + xor $25, $27, $27 # 8 cycles from $27 load \n\ + ldl $4, 24($20) \n\ + xor $28, $0, $0 # 7 cycles from $0 load \n\ + \n\ + ldl $5, 24($21) \n\ + xor $27, $0, $0 \n\ + ldl $6, 32($17) \n\ + ldl $7, 32($18) \n\ + \n\ + stl $0, 16($17) \n\ + xor $1, $2, $2 # 6 cycles from $2 load \n\ + ldl $22, 32($19) \n\ + xor $3, $4, $4 # 4 cycles from $4 load \n\ + \n\ + ldl $23, 32($20) \n\ + xor $2, $4, $4 \n\ + ldl $24, 32($21) \n\ + ldl $25, 40($17) \n\ + \n\ + ldl $27, 40($18) \n\ + ldl $28, 40($19) \n\ + ldl $0, 40($20) \n\ + xor $4, $5, $5 # 7 cycles from $5 load \n\ + \n\ + stl $5, 24($17) \n\ + xor $6, $7, $7 # 7 cycles from $7 load \n\ + ldl $1, 40($21) \n\ + ldl $2, 48($17) \n\ + \n\ + ldl $3, 48($18) \n\ + xor $7, $22, $22 # 7 cycles from $22 load \n\ + ldl $4, 48($19) \n\ + xor $23, $24, $24 # 6 cycles from $24 load \n\ + \n\ + ldl $5, 48($20) \n\ + xor $22, $24, $24 \n\ + ldl $6, 48($21) \n\ + xor $25, $27, $27 # 7 cycles from $27 load \n\ + \n\ + stl $24, 32($17) \n\ + xor $27, $28, $28 # 8 cycles from $28 load \n\ + ldl $7, 56($17) \n\ + xor $0, $1, $1 # 6 cycles from $1 load \n\ + \n\ + ldl $22, 56($18) \n\ + ldl $23, 56($19) \n\ + ldl $24, 56($20) \n\ + ldl $25, 56($21) \n\ + \n\ + fillde 256($17) \n\ + xor $28, $1, $1 \n\ + fillde 256($18) \n\ + xor $2, $3, $3 # 9 cycles from $3 load \n\ + \n\ + fillde 256($19) \n\ + xor $3, $4, $4 # 9 cycles from $4 load \n\ + fillde 256($20) \n\ + xor $5, $6, $6 # 8 cycles from $6 load \n\ + \n\ + stl $1, 40($17) \n\ + xor $4, $6, $6 \n\ + xor $7, $22, $22 # 7 cycles from $22 load \n\ + xor $23, $24, $24 # 6 cycles from $24 load \n\ + \n\ + stl $6, 48($17) \n\ + xor $22, $24, $24 \n\ + fillde 256($21) \n\ + xor $24, $25, $25 # 8 cycles from $25 load \n\ + \n\ + stl $25, 56($17) \n\ + subl $16, 1, $16 \n\ + addl $21, 64, $21 \n\ + addl $20, 64, $20 \n\ + \n\ + addl $19, 64, $19 \n\ + addl $18, 64, $18 \n\ + addl $17, 64, $17 \n\ + bgt $16, 5b \n\ + \n\ + ret \n\ + .end xor_sw64_prefetch_5 \n\ +"); + +static struct xor_block_template xor_block_sw64 = { + .name = "sw64", + .do_2 = xor_sw64_2, + .do_3 = xor_sw64_3, + .do_4 = xor_sw64_4, + .do_5 = xor_sw64_5, +}; + +static struct xor_block_template xor_block_sw64_prefetch = { + .name = "sw64 prefetch", + .do_2 = xor_sw64_prefetch_2, + .do_3 = xor_sw64_prefetch_3, + .do_4 = xor_sw64_prefetch_4, + .do_5 = xor_sw64_prefetch_5, +}; + +/* For grins, also test the generic routines. */ +#include + +#undef XOR_TRY_TEMPLATES +#define XOR_TRY_TEMPLATES \ + do { \ + xor_speed(&xor_block_8regs); \ + xor_speed(&xor_block_32regs); \ + xor_speed(&xor_block_sw64); \ + xor_speed(&xor_block_sw64_prefetch); \ + } while (0) + +/* Force the use of sw64_prefetch as it is significantly + * faster in the cold cache case. + */ +#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_sw64_prefetch) + +#endif /* _ASM_SW64_XOR_H */ diff --git a/arch/sw_64/include/uapi/asm/swab.h b/arch/sw_64/include/uapi/asm/swab.h new file mode 100644 index 000000000000..275661b346ac --- /dev/null +++ b/arch/sw_64/include/uapi/asm/swab.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_SWAB_H +#define _UAPI_ASM_SW64_SWAB_H + +#include +#include +#include + +#ifdef __GNUC__ + +static inline __attribute_const__ __u32 __arch_swab32(__u32 x) +{ + /* + * Unfortunately, we can't use the 6 instruction sequence + * on sw64 since the latency of the UNPKBW is 3, which is + * pretty hard to hide. Just in case a future implementation + * has a lower latency, here's the sequence (also by Mike Burrows) + * + * UNPKBW a0, v0 v0: 00AA00BB00CC00DD + * SLL v0, 24, a0 a0: BB00CC00DD000000 + * BIS v0, a0, a0 a0: BBAACCBBDDCC00DD + * EXTWL a0, 6, v0 v0: 000000000000BBAA + * ZAP a0, 0xf3, a0 a0: 00000000DDCC0000 + * ADDL a0, v0, v0 v0: ssssssssDDCCBBAA + */ + + __u64 t0, t1, t2, t3; + + t0 = __kernel_inshw(x, 7); /* t0 : 0000000000AABBCC */ + t1 = __kernel_inslh(x, 3); /* t1 : 000000CCDD000000 */ + t1 |= t0; /* t1 : 000000CCDDAABBCC */ + t2 = t1 >> 16; /* t2 : 0000000000CCDDAA */ + t0 = t1 & 0xFF00FF00; /* t0 : 00000000DD00BB00 */ + t3 = t2 & 0x00FF00FF; /* t3 : 0000000000CC00AA */ + t1 = t0 + t3; /* t1 : ssssssssDDCCBBAA */ + + return t1; +} +#define __arch_swab32 __arch_swab32 + +#endif /* __GNUC__ */ + +#endif /* _UAPI_ASM_SW64_SWAB_H */ diff --git a/arch/sw_64/lib/Kconfig b/arch/sw_64/lib/Kconfig new file mode 100644 index 000000000000..e22751a457ce --- /dev/null +++ b/arch/sw_64/lib/Kconfig @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: GPL-2.0 +menu "Library optimization options" + +config DEEP_CLEAR_PAGE + bool "Clear Page with SIMD optimization" + default y + help + This option enables the use of SIMD version of clear page routine. + Say N if you want to use the generic version. + +config DEEP_CLEAR_USER + bool "Clear User with SIMD optimization" + default y + help + This option enables the use of SIMD version of clear user routine. + Say N if you want to use the generic version. + +config DEEP_COPY_PAGE + bool "Copy Page with SIMD optimization" + default y + help + This option enables the use of SIMD version of copy page routine. + Say N if you want to use the generic version. + +config DEEP_COPY_USER + bool "Copy User with SIMD optimization" + default y + help + This option enables the use of SIMD version of copy user routine. + Say N if you want to use the generic version. + + +config DEEP_MEMCPY + bool "Memory Copy with SIMD optimization" + default y + help + This option enables the use of SIMD version of memory copy routine. + Say N if you want to use the generic version. + +config DEEP_MEMSET + bool "Memory Set with SIMD optimization" + default y + help + This option enables the use of SIMD version of memory set routine. + Say N if you want to use the generic version. + +endmenu diff --git a/arch/sw_64/lib/Makefile b/arch/sw_64/lib/Makefile new file mode 100644 index 000000000000..e6455bb51139 --- /dev/null +++ b/arch/sw_64/lib/Makefile @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for sw-specific library files.. +# + +asflags-y := $(KBUILD_CFLAGS) +ccflags-y := -Werror + +lib-y = __divlu.o __remlu.o __divwu.o __remwu.o \ + udelay.o \ + memmove.o \ + checksum.o \ + csum_partial_copy.o \ + fpreg.o \ + strcpy.o \ + strncpy.o \ + fls.o \ + csum_ipv6_magic.o + +lib-clear_page-y := clear_page.o +lib-clear_page-$(CONFIG_DEEP_CLEAR_PAGE) := deep-clear_page.o + +lib-clear_user-y := clear_user.o +lib-clear_user-$(CONFIG_DEEP_CLEAR_USER) := deep-clear_user.o + +lib-copy_page-y := copy_page.o +lib-copy_page-$(CONFIG_DEEP_COPY_PAGE) := deep-copy_page.o + +lib-copy_user-y := copy_user.o +lib-copy_user-$(CONFIG_DEEP_COPY_USER) := deep-copy_user.o + +lib-memcpy-y := memcpy.o +lib-memcpy-$(CONFIG_DEEP_MEMCPY) := deep-memcpy.o + +lib-memset-y := memset.o +lib-memset-$(CONFIG_DEEP_MEMSET) := deep-memset.o + +lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o + +lib-y += $(lib-clear_page-y) $(lib-clear_user-y) $(lib-copy_page-y) $(lib-copy_user-y) $(lib-memcpy-y) $(lib-memset-y) + +obj-y = iomap.o +obj-y += iomap_copy.o + +# The division routines are built from single source, with different defines. +AFLAGS___divlu.o = -DDIV +AFLAGS___remlu.o = -DREM +AFLAGS___divwu.o = -DDIV -DINTSIZE +AFLAGS___remwu.o = -DREM -DINTSIZE + +$(addprefix $(obj)/,__divlu.o __remlu.o __divwu.o __remwu.o): \ + $(src)/divide.S FORCE + $(call if_changed_rule,as_o_S) diff --git a/arch/sw_64/lib/checksum.c b/arch/sw_64/lib/checksum.c new file mode 100644 index 000000000000..d1314caa15bf --- /dev/null +++ b/arch/sw_64/lib/checksum.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file contains network checksum routines that are better done + * in an architecture-specific manner due to speed.. + * Comments in other versions indicate that the algorithms are from RFC1071 + */ +#include +#include +#include +#include + +/* + * computes the checksum of the TCP/UDP pseudo-header + * returns a 16-bit checksum, already complemented. + */ +__sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, + __u32 len, __u8 proto, __wsum sum) +{ + return (__force __sum16)~from64to16( + (__force u64)saddr + (__force u64)daddr + + (__force u64)sum + ((len + proto) << 8)); +} +EXPORT_SYMBOL(csum_tcpudp_magic); + +__wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, + __u32 len, __u8 proto, __wsum sum) +{ + unsigned long result; + + result = (__force u64)saddr + (__force u64)daddr + + (__force u64)sum + ((len + proto) << 8); + + /* + * Fold down to 32-bits so we don't lose in the typedef-less + * network stack. + * + * 64 to 33 + */ + result = (result & 0xffffffff) + (result >> 32); + /* 33 to 32 */ + result = (result & 0xffffffff) + (result >> 32); + return (__force __wsum)result; +} +EXPORT_SYMBOL(csum_tcpudp_nofold); + +/* + * Do a 64-bit checksum on an arbitrary memory area.. + */ +static inline unsigned long do_csum(const unsigned char *buff, int len) +{ + const unsigned long *dst = (unsigned long *)buff; + unsigned long doff = 7 & (unsigned long) dst; + unsigned long checksum = 0; + unsigned long word, patch; + unsigned long partial_dest, second_dest; + + len -= 8; + + if (!doff) { + while (len > 0) { + word = *dst; + checksum += word; + checksum += (checksum < word); + dst++; + len -= 8; + } + + len += 8; + word = *dst; + + if (len != 8) + maskll(word, len, word); + + checksum += word; + checksum += (checksum < word); + } else { + dst = (unsigned long *)((unsigned long)dst & (~7UL)); + word = *dst; + inshl(word, 8 - doff, partial_dest); + dst++; + + while (len >= 0) { + word = *dst; + insll(word, 8 - doff, second_dest); + patch = partial_dest | second_dest; + checksum += patch; + checksum += (checksum < patch); + inshl(word, 8 - doff, partial_dest); + dst++; + len -= 8; + } + + len += 8; + word = *dst; + insll(word, 8 - doff, second_dest); + patch = partial_dest | second_dest; + maskll(patch, len, patch); + checksum += patch; + checksum += (checksum < patch); + } + + return from64to16(checksum); +} + +/* + * This is a version of ip_compute_csum() optimized for IP headers, + * which always checksum on 4 octet boundaries. + */ +__sum16 ip_fast_csum(const void *iph, unsigned int ihl) +{ + return (__force __sum16)~do_csum(iph, ihl*4); +} +EXPORT_SYMBOL(ip_fast_csum); + +/* + * computes the checksum of a memory block at buff, length len, + * and adds in "sum" (32-bit) + * + * returns a 32-bit number suitable for feeding into itself + * or csum_tcpudp_magic + * + * this function must be called with even lengths, except + * for the last fragment, which may be odd + * + * it's best to have buff aligned on a 32-bit boundary + */ +__wsum csum_partial(const void *buff, int len, __wsum sum) +{ + unsigned long result = do_csum(buff, len); + + /* add in old sum, and carry.. */ + result += (__force u32)sum; + /* 32+c bits -> 32 bits */ + result = (result & 0xffffffff) + (result >> 32); + return (__force __wsum)result; +} +EXPORT_SYMBOL(csum_partial); + +/* + * this routine is used for miscellaneous IP-like checksums, mainly + * in icmp.c + */ +__sum16 ip_compute_csum(const void *buff, int len) +{ + return (__force __sum16)~from64to16(do_csum(buff, len)); +} +EXPORT_SYMBOL(ip_compute_csum); diff --git a/arch/sw_64/lib/clear_page.S b/arch/sw_64/lib/clear_page.S new file mode 100644 index 000000000000..e1cc7cddfd2f --- /dev/null +++ b/arch/sw_64/lib/clear_page.S @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Zero an entire page. + */ +#include + .text + .align 4 + .global clear_page + .ent clear_page +clear_page: + .prologue 0 + + ldi $0, 64 + +/* Optimize by GUOY from SOC 2013-06-04 */ +1: + + stl_nc $31, 0x0($16) + stl_nc $31, 0x8($16) + stl_nc $31, 0x10($16) + stl_nc $31, 0x18($16) + + stl_nc $31, 0x20($16) + stl_nc $31, 0x28($16) + stl_nc $31, 0x30($16) + stl_nc $31, 0x38($16) + + stl_nc $31, 0x40($16) + stl_nc $31, 0x48($16) + stl_nc $31, 0x50($16) + stl_nc $31, 0x58($16) + + stl_nc $31, 0x60($16) + stl_nc $31, 0x68($16) + subl $0, 1, $0 + + stl_nc $31, 0x70($16) + stl_nc $31, 0x78($16) + addl $16, 128, $16 + bne $0, 1b + + memb + ret + + .end clear_page + EXPORT_SYMBOL(clear_page) diff --git a/arch/sw_64/lib/clear_user.S b/arch/sw_64/lib/clear_user.S new file mode 100644 index 000000000000..5ac77fc8ca0d --- /dev/null +++ b/arch/sw_64/lib/clear_user.S @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Contributed by Richard Henderson + * + * Zero user space, handling exceptions as we go. + * + * We have to make sure that $0 is always up-to-date and contains the + * right "bytes left to zero" value (and that it is updated only _after_ + * a successful copy). There is also some rather minor exception setup + * stuff. + * + */ +#include +/* Allow an exception for an insn; exit if we get one. */ +#define EX(x,y...) \ + 99: x,##y; \ + .section __ex_table,"a"; \ + .long 99b - .; \ + ldi $31, $exception-99b($31); \ + .previous + + .set noat + .set noreorder + .align 4 + + .globl __clear_user + .ent __clear_user + .frame $30, 0, $26 + .prologue 0 +__clear_user: + and $17, $17, $0 + and $16, 7, $4 + beq $0, $zerolength + addl $0, $4, $1 + and $1, 7, $2 + srl $1, 3, $1 + beq $4, $loop + + subl $4, 8, $4 + addl $0, $4, $0 + beq $1, $oneword + +$head: + EX(stb $31, 0($16)) + addl $16, 1, $16 + addl $4, 1, $4 + bne $4, $head + subl $1, 1, $1 + br $loop + unop + +$oneword: + EX(stb $31, 0($16)) + addl $16, 1, $16 + addl $4, 1, $4 + bne $4, $oneword + clr $0 + +$zerolength: +$exception: + ret $31, ($26), 1 + +$loop: + and $1, 3, $4 + beq $4, 1f + +0: EX(stl $31, 0($16)) + subl $0, 8, $0 + subl $4, 1, $4 + addl $16, 8, $16 + bne $4, 0b + unop + +1: bic $1, 3, $1 + beq $1, $tail + +2: EX(stl $31, 0($16)) + subl $0, 8, $0 + EX(stl $31, 8($16)) + subl $0, 8, $0 + EX(stl $31, 16($16)) + subl $0, 8, $0 + EX(stl $31, 24($16)) + subl $0, 8, $0 + subl $1, 4, $1 + addl $16, 32, $16 + bne $1, 2b + +$tail: + bne $2, 1f + ret $31, ($26), 1 + +1: + EX(stb $31, 0($16)) + addl $16, 1, $16 + subl $2, 1, $2 + bne $2, 1b + clr $0 + ret $31, ($26), 1 + + .end __clear_user + EXPORT_SYMBOL(__clear_user) diff --git a/arch/sw_64/lib/copy_page.S b/arch/sw_64/lib/copy_page.S new file mode 100644 index 000000000000..898472c36c80 --- /dev/null +++ b/arch/sw_64/lib/copy_page.S @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arch/sw/lib/copy_page.S + * + * Copy an entire page. + */ +#include + + .text + .align 4 + .global copy_page + .ent copy_page +copy_page: + .prologue 0 + + ldi $18, 64 + +/* Optimize by GUOY from SOC 2013-06-04 */ +1: + ldl $0, 0($17) + ldl $1, 8($17) + ldl $2, 16($17) + ldl $3, 24($17) + + stl_nc $0, 0($16) + stl_nc $1, 8($16) + stl_nc $2, 16($16) + stl_nc $3, 24($16) + + ldl $4, 32($17) + ldl $5, 40($17) + ldl $6, 48($17) + ldl $7, 56($17) + + stl_nc $4, 32($16) + stl_nc $5, 40($16) + stl_nc $6, 48($16) + stl_nc $7, 56($16) + + ldl $0, 64($17) + ldl $1, 72($17) + ldl $2, 80($17) + ldl $3, 88($17) + + stl_nc $0, 64($16) + stl_nc $1, 72($16) + stl_nc $2, 80($16) + stl_nc $3, 88($16) + + ldl $4, 96($17) + ldl $5, 104($17) + ldl $6, 112($17) + ldl $7, 120($17) + + stl_nc $4, 96($16) + stl_nc $5, 104($16) + stl_nc $6, 112($16) + stl_nc $7, 120($16) + + ldwe $f31, 3 * 0x80($17) + subl $18, 1, $18 + addl $17, 128, $17 + + addl $16, 128, $16 + bne $18, 1b + + memb + ret + + .end copy_page + EXPORT_SYMBOL(copy_page) diff --git a/arch/sw_64/lib/copy_user.S b/arch/sw_64/lib/copy_user.S new file mode 100644 index 000000000000..2c3dd0b5656c --- /dev/null +++ b/arch/sw_64/lib/copy_user.S @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copy to/from user space, handling exceptions as we go.. This + * isn't exactly pretty. + * + * This is essentially the same as "memcpy()", but with a few twists. + * Notably, we have to make sure that $0 is always up-to-date and + * contains the right "bytes left to copy" value (and that it is updated + * only _after_ a successful copy). There is also some rather minor + * exception setup stuff.. + */ +#include +/* Allow an exception for an insn; exit if we get one. */ +#define EXI(x,y...) \ + 99: x,##y; \ + .section __ex_table, "a"; \ + .long 99b - .; \ + ldi $31, $exitin-99b($31); \ + .previous + +#define EXO(x,y...) \ + 99: x, ##y; \ + .section __ex_table, "a"; \ + .long 99b - .; \ + ldi $31, $exitout-99b($31); \ + .previous + + .set noat + .align 4 + .globl __copy_user + .ent __copy_user +__copy_user: + .prologue 0 + and $18, $18, $0 + and $16, 7, $3 + beq $0, $35 + beq $3, $36 + subl $3, 8, $3 + .align 4 +$37: + EXI(ldbu $1, 0($17)) + EXO(stb $1, 0($16)) + addl $3, 1, $3 + subl $0, 1, $0 + addl $16, 1, $16 + addl $17, 1, $17 + beq $0, $41 + bne $3, $37 +$36: + and $17, 7, $1 + bic $0, 7, $4 + beq $1, $43 + beq $4, $48 + EXI(ldl_u $3, 0($17)) + .align 4 +$50: + EXI(ldl_u $2, 8($17)) + subl $4, 8, $4 + extll $3, $17, $3 + exthl $2, $17, $1 + bis $3, $1, $1 + EXO(stl $1,0($16)) + addl $17, 8, $17 + subl $0, 8, $0 + addl $16, 8, $16 + bis $2, $2, $3 + bne $4, $50 +$48: + beq $0, $41 + .align 4 +$57: + EXI(ldbu $1, 0($17)) + EXO(stb $1, 0($16)) + subl $0, 1, $0 + addl $16, 1, $16 + addl $17, 1, $17 + bne $0, $57 + br $31, $41 + .align 4 +$43: + beq $4, $65 + .align 4 +$66: + EXI(ldl $1, 0($17)) + subl $4, 8, $4 + EXO(stl $1,0($16)) + addl $17, 8, $17 + subl $0, 8, $0 + addl $16, 8, $16 + bne $4, $66 +$65: + beq $0, $41 + EXI(ldbu $1, 0($17)) + EXO(stb $1, 0($16)) + addl $17, 1, $17 + addl $16, 1, $16 + subl $0, 1, $0 + br $31, $65 +$41: +$35: +$exitin: +$exitout: + ret $31, ($26), 1 + + .end __copy_user + EXPORT_SYMBOL(__copy_user) diff --git a/arch/sw_64/lib/csum_ipv6_magic.S b/arch/sw_64/lib/csum_ipv6_magic.S new file mode 100644 index 000000000000..755e1c13cb25 --- /dev/null +++ b/arch/sw_64/lib/csum_ipv6_magic.S @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Contributed by Richard Henderson + * + * unsigned short csum_ipv6_magic(struct in6_addr *saddr, + * struct in6_addr *daddr, __u32 len, + * unsigned short proto, unsigned int csum); + * + * Misalignment handling (which costs 16 instructions / 8 cycles) + * added by Ivan Kokshaysky + */ +#include + .globl csum_ipv6_magic + .align 4 + .ent csum_ipv6_magic + .frame $30, 0, $26, 0 +csum_ipv6_magic: + .prologue 0 + + ldl_u $0, 0($16) + zapnot $20, 15, $20 + exthl $18, 1, $4 + ldl_u $21, 7($16) + + extlb $18, 1, $5 + ldl_u $1, 8($16) + extlb $18, 2, $6 + ldl_u $22, 15($16) + + extlb $18, 3, $18 + ldl_u $2, 0($17) + sra $4, 32, $4 + ldl_u $23, 7($17) + + extll $0, $16, $0 + ldl_u $3, 8($17) + exthl $21, $16, $21 + ldl_u $24, 15($17) + + sll $5, 16, $5 + or $0, $21, $0 + extll $1, $16, $1 + addl $20, $0, $20 + + exthl $22, $16, $22 + cmpult $20, $0, $0 + sll $6, 8, $6 + or $1, $22, $1 + + extll $2, $17, $2 + or $4, $18, $18 + exthl $23, $17, $23 + or $5, $6, $5 + + extll $3, $17, $3 + or $2, $23, $2 + exthl $24, $17, $24 + or $18, $5, $18 + + exthh $19, 7, $7 + or $3, $24, $3 + extlb $19, 1, $19 + addl $20, $1, $20 + + or $19, $7, $19 + cmpult $20, $1, $1 + sll $19, 48, $19 + + sra $19, 32, $19 + addl $20, $2, $20 + cmpult $20, $2, $2 + addl $20, $3, $20 + + cmpult $20, $3, $3 + addl $20, $18, $20 + cmpult $20, $18, $18 + addl $20, $19, $20 + + cmpult $20, $19, $19 + addl $0, $1, $0 + addl $2, $3, $2 + addl $18, $19, $18 + + addl $0, $2, $0 + addl $20, $18, $20 + addl $0, $20, $0 + unop + + extlh $0, 2, $2 + zapnot $0, 3, $3 + extlh $0, 4, $1 + addl $2, $3, $3 + + extlh $0, 6, $0 + addl $3, $1, $3 + addl $0, $3, $0 + unop + + extlh $0, 2, $1 + zapnot $0, 3, $0 + addl $0, $1, $0 + unop + + extlh $0, 2, $1 + zapnot $0, 3, $0 + addl $0, $1, $0 + not $0, $0 + + zapnot $0, 3, $0 + ret + + .end csum_ipv6_magic + EXPORT_SYMBOL(csum_ipv6_magic) diff --git a/arch/sw_64/lib/csum_partial_copy.c b/arch/sw_64/lib/csum_partial_copy.c new file mode 100644 index 000000000000..1a8c18757e09 --- /dev/null +++ b/arch/sw_64/lib/csum_partial_copy.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * csum_partial_copy - do IP checksumming and copy + * + * (C) Copyright 1996 Linus Torvalds + * + * Don't look at this too closely - you'll go mad. The things + * we do for performance.. + */ + +#include +#include +#include +#include + + +#define ldl_u(x, y) \ + __asm__ __volatile__("ldl_u %0, %1":"=r" (x):"m" (*(const unsigned long *)(y))) + +#define stl_u(x, y) \ + __asm__ __volatile__("stl_u %1, %0":"=m" (*(unsigned long *)(y)):"r" (x)) + +static inline void stll_u(unsigned long data, unsigned long *dst) +{ + int i = 0; + unsigned long doff = (unsigned long)dst & 7; + + for (; doff < 8; i++, doff++) + *((char *)dst + i) = *((char *)&data + i); +} + +static inline void sthl_u(unsigned long data, unsigned long *dst) +{ + int i = 0; + unsigned long doff = (unsigned long)dst & 7; + + for (; i < doff; i++) + *((char *)dst + 8 - doff + i) = *((char *)&data + 8 - doff + i); +} + +#define __get_word(insn, x, ptr) \ +({ \ + long __guu_err; \ + __asm__ __volatile__( \ + "1: "#insn" %0,%2\n" \ + "2:\n" \ + ".section __ex_table,\"a\"\n" \ + " .long 1b - .\n" \ + " ldi %0,2b-1b(%1)\n" \ + ".previous" \ + : "=r"(x), "=r"(__guu_err) \ + : "m"(__m(ptr)), "1"(0)); \ + __guu_err; \ +}) + +static inline unsigned long +csum_partial_cfu_dest_aligned(const unsigned long __user *src, + unsigned long *dst, long len) +{ + unsigned long word; + unsigned long checksum = ~0U; + int err = 0; + + err = __copy_from_user(dst, src, len+8); + + while (len > 0) { + word = *dst; + checksum += word; + checksum += (checksum < word); + dst++; + len -= 8; + } + len += 8; + word = *dst; + + if (len != 8) + maskll(word, len, word); + checksum += word; + checksum += (checksum < word); + + return checksum; +} + +static inline unsigned long +csum_partial_cfu_dest_unaligned(const unsigned long __user *src, + unsigned long *dst, unsigned long doff, long len) +{ + unsigned long word, patch; + unsigned long partial_dest, second_dest; + unsigned long checksum = ~0U; + int err = 0; + + err = __copy_from_user(dst, src, len+8); + + dst = (unsigned long *)((unsigned long)dst & (~7UL)); + word = *dst; + inshl(word, 8 - doff, partial_dest); + dst++; + + while (len >= 0) { + word = *dst; + insll(word, 8 - doff, second_dest); + patch = partial_dest | second_dest; + checksum += patch; + checksum += (checksum < patch); + inshl(word, 8 - doff, partial_dest); + dst++; + len -= 8; + } + + len += 8; + word = *dst; + insll(word, 8 - doff, second_dest); + patch = partial_dest | second_dest; + maskll(patch, len, patch); + checksum += patch; + checksum += (checksum < patch); + + return checksum; +} + +static __wsum __csum_and_copy(const void __user *src, void *dst, int len) +{ + unsigned long checksum; + unsigned long doff = 7 & (unsigned long) dst; + + if (!doff) { + checksum = csum_partial_cfu_dest_aligned( + (const unsigned long __user *) src, + (unsigned long *) dst, len-8); + } else { + checksum = csum_partial_cfu_dest_unaligned( + (const unsigned long __user *) src, + (unsigned long *) dst, doff, len-8); + } + return (__force __wsum)from64to16(checksum); +} + +__wsum +csum_and_copy_from_user(const void __user *src, void *dst, int len) +{ + if (!access_ok(src, len)) + return 0; + return __csum_and_copy(src, dst, len); +} +EXPORT_SYMBOL(csum_and_copy_from_user); + +__wsum +csum_partial_copy_nocheck(const void *src, void *dst, int len) +{ + return __csum_and_copy((__force const void __user *)src, + dst, len); +} +EXPORT_SYMBOL(csum_partial_copy_nocheck); diff --git a/arch/sw_64/lib/deep-clear_page.S b/arch/sw_64/lib/deep-clear_page.S new file mode 100644 index 000000000000..52a3db33fc17 --- /dev/null +++ b/arch/sw_64/lib/deep-clear_page.S @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Zero an entire page. + */ +#include + .text + .align 4 + .global clear_page + .ent clear_page +clear_page: + .prologue 0 + + ldi $0,64 + +/* Optimize by GUOY from SOC 2013-06-04 */ +1: + +/* + stl_nc $31,0x0($16) + stl_nc $31,0x8($16) + stl_nc $31,0x10($16) + stl_nc $31,0x18($16) + + stl_nc $31,0x20($16) + stl_nc $31,0x28($16) + stl_nc $31,0x30($16) + stl_nc $31,0x38($16) + + stl_nc $31,0x40($16) + stl_nc $31,0x48($16) + stl_nc $31,0x50($16) + stl_nc $31,0x58($16) + + stl_nc $31,0x60($16) + stl_nc $31,0x68($16) + stl_nc $31,0x70($16) + stl_nc $31,0x78($16) +*/ + + vstd_nc $f31, 0x0($16) + vstd_nc $f31, 0x20($16) + subl $0, 1, $0 + vstd_nc $f31, 0x40($16) + + vstd_nc $f31, 0x60($16) + addl $16, 128, $16 + bne $0, 1b + + memb + ret + + .end clear_page + EXPORT_SYMBOL(clear_page) diff --git a/arch/sw_64/lib/deep-clear_user.S b/arch/sw_64/lib/deep-clear_user.S new file mode 100644 index 000000000000..c81418ed99a2 --- /dev/null +++ b/arch/sw_64/lib/deep-clear_user.S @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Contributed by Mao Minkai + * + * Zero user space, handling exceptions as we go. + * + * We have to make sure that $0 is always up-to-date and contains the + * right "bytes left to zero" value (and that it is updated only _after_ + * a successful copy). There is also some rather minor exception setup + * stuff. + * + */ +#include +/* Allow an exception for an insn; exit if we get one. */ +#define FIXUP_LDST(x,y...) \ + 99: x,##y; \ + .section __ex_table,"a"; \ + .long 99b - .; \ + ldi $31, $out-99b($31); \ + .previous + +/* + * $7: SIMD status + * 0: not in simd loop + * 1: in simd loop + * 2: in simd_u loop + * $18: bytes left to copy + * + */ + .globl __clear_user + .ent __clear_user +__clear_user: + .prologue 0 + bis $31, $31, $7 + mov $17, $18 + bis $31, $31, $17 +#if defined(CONFIG_SUBARCH_C3B) +#include "deep-set_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-set_template_c4.S" +#endif +$out: + bis $31, $18, $0 + beq $7, $return + +$restore_simd: + RESTORE_SIMD_REGS + +$return: + ret + .end __clear_user + EXPORT_SYMBOL(__clear_user) diff --git a/arch/sw_64/lib/deep-copy_page.S b/arch/sw_64/lib/deep-copy_page.S new file mode 100644 index 000000000000..a9b9d97f318a --- /dev/null +++ b/arch/sw_64/lib/deep-copy_page.S @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arch/sw/lib/copy_page.S + * + * Copy an entire page. + */ +#include +#include + + .text + .align 4 + .global copy_page + .ent copy_page +copy_page: + .prologue 0 + + ldi $18, 64 + subl $sp, 0x60, $sp + ldi $4, 0x40($sp) + stl $4, 0($sp) + bic $4, 0x1f, $4 + vstd $f16, 0($4) +#ifdef CONFIG_SUBARCH_C4 + csrr $5, CSR_WR_FREGS +#endif + +/* Optimize by GUOY from SOC 2013-06-04 */ +1: + vldd $f16, 0($17) + vstd_nc $f16, 0($16) + + vldd $f16, 32($17) + vstd_nc $f16, 32($16) + + vldd $f16, 64($17) + vstd_nc $f16, 64($16) + + vldd $f16, 96($17) + vstd_nc $f16, 96($16) + + ldwe $f31, 5*0x80($17) + subl $18, 1, $18 + addl $17, 128, $17 + + addl $16, 128, $16 + bne $18, 1b + + memb + ldl $4, 0($sp) + ldi $4, 0x40($sp) + bic $4, 0x1f, $4 + vldd $f16, 0($4) +#ifdef CONFIG_SUBARCH_C4 + csrw $5, CSR_WR_FREGS +#endif + addl $sp, 0x60, $sp + ret + + .end copy_page + EXPORT_SYMBOL(copy_page) diff --git a/arch/sw_64/lib/deep-copy_template.S b/arch/sw_64/lib/deep-copy_template.S new file mode 100644 index 000000000000..7705eb3f36d4 --- /dev/null +++ b/arch/sw_64/lib/deep-copy_template.S @@ -0,0 +1,301 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * template for memcpy and copy_user with SIMD + * + * $4: 8-byte misalignment of src when dest is 8-byte aligned + * $5: 32-byte misalignment of src when dest is 32-byte aligned + * $7: SIMD status + * 0: not in simd loop + * 1: in simd loop + * 2: in simd_u loop + * $16: latest dest, clobbered + * $17: latest src, clobbered + * $18: bytes left to copy + * + */ + +#define NC_STORE_THRESHOLD 2048 + +#define SAVE_SIMD_REGS \ + ldi $sp, -0x60($sp); \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vstd $f1, 0($23); \ + vstd $f2, 0x20($23); \ + ldi $7, 1 + +#define RESTORE_SIMD_REGS \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vldd $f1, 0($23); \ + vldd $f2, 0x20($23); \ + ldi $sp, 0x60($sp); \ + bis $31, $31, $7 + +#define SAVE_SIMD_U_REGS \ + ldi $sp, -0xc0($sp); \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vstd $f1, 0($23); \ + vstd $f2, 0x20($23); \ + vstd $f4, 0x40($23); \ + vstd $f5, 0x60($23); \ + vstd $f3, 0x80($23); \ + ldi $7, 2 + +#define RESTORE_SIMD_U_REGS \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vldd $f1, 0($23); \ + vldd $f2, 0x20($23); \ + vldd $f4, 0x40($23); \ + vldd $f5, 0x60($23); \ + vldd $f3, 0x80($23); \ + ldi $sp, 0xc0($sp); \ + bis $31, $31, $7 + + ble $18, $out + and $16, 7, $1 + beq $1, $dest_aligned_8 + +$byte_loop_head: + FIXUP_LDST( ldbu $2, 0($17) ) + FIXUP_LDST( stb $2, 0($16) ) + subl $18, 1, $18 + addl $17, 1, $17 + addl $16, 1, $16 + ble $18, $out + and $16, 7, $1 + bne $1, $byte_loop_head + +$dest_aligned_8: + and $17, 7, $4 + cmplt $18, 16, $1 + bne $1, $quad_loop_end + and $16, 31, $1 + beq $1, $dest_aligned_32 + cmplt $18, 64, $1 + bne $1, $simd_end + bne $4, $quad_u_loop_head + +$quad_loop_head: + FIXUP_LDST( ldl $2, 0($17) ) + FIXUP_LDST( stl $2, 0($16) ) + addl $16, 8, $16 + addl $17, 8, $17 + subl $18, 8, $18 + and $16, 31, $1 + beq $1, $dest_aligned_32 + br $31, $quad_loop_head + +$dest_aligned_32: + cmplt $18, 64, $1 + bne $1, $simd_end + and $17, 31, $5 + bne $5, $prep_simd_u_loop + +$prep_simd_loop: + SAVE_SIMD_REGS + ldi $1, NC_STORE_THRESHOLD($31) + cmple $18, $1, $1 + bne $1, $simd_loop + + .align 4 +$simd_loop_nc: + FIXUP_LDST( vldd $f1, 0($17) ) + FIXUP_LDST( vldd $f2, 32($17) ) + FIXUP_LDST( vstd_nc $f1, 0($16) ) + FIXUP_LDST( vstd_nc $f2, 32($16) ) + subl $18, 64, $18 + addl $17, 64, $17 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_loop_nc + memb # required for _nc store instructions + br $31, $simd_loop_end + + .align 4 +$simd_loop: + FIXUP_LDST( vldd $f1, 0($17) ) + FIXUP_LDST( vldd $f2, 32($17) ) + FIXUP_LDST( vstd $f1, 0($16) ) + FIXUP_LDST( vstd $f2, 32($16) ) + subl $18, 64, $18 + addl $17, 64, $17 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_loop + +$simd_loop_end: + cmplt $18, 32, $1 + bne $1, $no_more_simd + FIXUP_LDST( vldd $f1, 0($17) ) + FIXUP_LDST( vstd $f1, 0($16) ) + subl $18, 32, $18 + addl $17, 32, $17 + addl $16, 32, $16 + +$no_more_simd: + RESTORE_SIMD_REGS + +$simd_end: + ble $18, $out + cmplt $18, 16, $1 + bne $1, $quad_loop_end + bne $4, $prep_quad_u_loop_tail + + .align 4 +$quad_loop_tail: + FIXUP_LDST( ldl $2, 0($17) ) + FIXUP_LDST( ldl $3, 8($17) ) + FIXUP_LDST( stl $2, 0($16) ) + FIXUP_LDST( stl $3, 8($16) ) + subl $18, 16, $18 + addl $17, 16, $17 + addl $16, 16, $16 + cmplt $18, 16, $1 + beq $1, $quad_loop_tail + +$quad_loop_end: + ble $18, $out + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + bne $4, $move_one_quad_u + +$move_one_quad: + FIXUP_LDST( ldl $2, 0($17) ) + FIXUP_LDST( stl $2, 0($16) ) + subl $18, 8, $18 + addl $17, 8, $17 + addl $16, 8, $16 + ble $18, $out + + .align 3 +$byte_loop_tail: + FIXUP_LDST( ldbu $2, 0($17) ) + FIXUP_LDST( stb $2, 0($16) ) + subl $18, 1, $18 + addl $17, 1, $17 + addl $16, 1, $16 + bgt $18, $byte_loop_tail + br $31, $out + +/* misaligned src and dst */ +$quad_u_loop_head: + FIXUP_LDST( ldl_u $2, 0($17) ) + FIXUP_LDST( ldl_u $3, 7($17) ) + extll $2, $4, $2 + exthl $3, $4, $3 + bis $2, $3, $2 + FIXUP_LDST( stl $2, 0($16) ) + addl $16, 8, $16 + addl $17, 8, $17 + subl $18, 8, $18 + and $16, 31, $1 + beq $1, $dest_aligned_32 + br $31, $quad_u_loop_head + +$prep_simd_u_loop: + SAVE_SIMD_U_REGS + andnot $17, 31, $3 + ldi $2, 256($31) + sll $5, 3, $1 + subl $2, $1, $2 + sll $1, 29, $1 + sll $2, 29, $2 + ifmovd $1, $f1 + ifmovd $2, $f2 + FIXUP_LDST( vldd $f4, 0($3) ) + ldi $1, NC_STORE_THRESHOLD($31) + cmple $18, $1, $1 + bne $1, $simd_u_loop + + .align 4 +$simd_u_loop_nc: + FIXUP_LDST( vldd $f5, 32($3) ) + srlow $f4, $f1, $f4 + sllow $f5, $f2, $f3 + vlogfc $f3, $f4, $f31, $f3 + FIXUP_LDST( vstd_nc $f3, 0($16) ) + FIXUP_LDST( vldd $f4, 64($3) ) + srlow $f5, $f1, $f5 + sllow $f4, $f2, $f3 + vlogfc $f5, $f3, $f31, $f5 + FIXUP_LDST( vstd_nc $f5, 32($16) ) + subl $18, 64, $18 + addl $3, 64, $3 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_u_loop_nc + memb # required for _nc store instructions + br $31, $simd_u_loop_end + + .align 4 +$simd_u_loop: + FIXUP_LDST( vldd $f5, 32($3) ) + srlow $f4, $f1, $f4 + sllow $f5, $f2, $f3 + vlogfc $f4, $f3, $f31, $f3 + FIXUP_LDST( vstd $f3, 0($16) ) + FIXUP_LDST( vldd $f4, 64($3) ) + srlow $f5, $f1, $f5 + sllow $f4, $f2, $f3 + vlogfc $f5, $f3, $f31, $f3 + FIXUP_LDST( vstd $f3, 32($16) ) + subl $18, 64, $18 + addl $3, 64, $3 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_u_loop + +$simd_u_loop_end: + cmplt $18, 32, $1 + bne $1, $no_more_simd_u + FIXUP_LDST( vldd $f5, 32($3) ) + srlow $f4, $f1, $f4 + sllow $f5, $f2, $f3 + vlogfc $f4, $f3, $f31, $f3 + FIXUP_LDST( vstd $f3, 0($16) ) + subl $18, 32, $18 + addl $3, 32, $3 + addl $16, 32, $16 + +$no_more_simd_u: + RESTORE_SIMD_U_REGS + bis $3, $5, $17 + br $31, $simd_end + +$prep_quad_u_loop_tail: + FIXUP_LDST( ldl_u $2, 0($17) ) + .align 4 +$quad_u_loop_tail: + FIXUP_LDST( ldl_u $3, 8($17) ) + extll $2, $4, $22 + exthl $3, $4, $23 + bis $22, $23, $22 + FIXUP_LDST( stl $22, 0($16) ) + FIXUP_LDST( ldl_u $2, 16($17) ) + extll $3, $4, $24 + exthl $2, $4, $25 + bis $24, $25, $24 + FIXUP_LDST( stl $24, 8($16) ) + subl $18, 16, $18 + addl $17, 16, $17 + addl $16, 16, $16 + cmplt $18, 16, $1 + beq $1, $quad_u_loop_tail + br $31, $quad_loop_end + +$move_one_quad_u: + FIXUP_LDST( ldl_u $2, 0($17) ) + FIXUP_LDST( ldl_u $3, 8($17) ) + extll $2, $4, $22 + exthl $3, $4, $23 + bis $22, $23, $22 + FIXUP_LDST( stl $22, 0($16) ) + subl $18, 8, $18 + addl $17, 8, $17 + addl $16, 8, $16 + ble $18, $out + br $31, $byte_loop_tail diff --git a/arch/sw_64/lib/deep-copy_template_c4.S b/arch/sw_64/lib/deep-copy_template_c4.S new file mode 100644 index 000000000000..e0740874dfa3 --- /dev/null +++ b/arch/sw_64/lib/deep-copy_template_c4.S @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * template for memcpy and copy_user with SIMD + * + * $7: SIMD status + * 0: not in simd loop + * 1: in simd and simd_u loop + * $16: latest dest, clobbered + * $17: latest src, clobbered + * $18: bytes left to copy + * + */ + +#define SAVE_SIMD_REGS \ + ldi $sp, -0x60($sp); \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vstd $f1, 0($23); \ + vstd $f2, 0x20($23); \ + ldi $7, 1 + +#define RESTORE_SIMD_REGS \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vldd $f1, 0($23); \ + vldd $f2, 0x20($23); \ + ldi $sp, 0x60($sp); \ + bis $31, $31, $7 + + + ble $18, $out + + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + cmplt $18, 16, $1 + bne $1, $quad_loop_end + cmplt $18, 32, $1 + bne $1, $simd_end + +$prep_simd_loop: + SAVE_SIMD_REGS + cmplt $18, 64, $1 + bne $1, $simd_loop_end + + .align 4 +$simd_loop: + FIXUP_LDST( vldd $f1, 0($17) ) + FIXUP_LDST( vldd $f2, 32($17) ) + FIXUP_LDST( vstd $f1, 0($16) ) + FIXUP_LDST( vstd $f2, 32($16) ) + subl $18, 64, $18 + addl $17, 64, $17 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_loop + +$simd_loop_end: + cmplt $18, 32, $1 + bne $1, $no_more_simd + FIXUP_LDST( vldd $f1, 0($17) ) + FIXUP_LDST( vstd $f1, 0($16) ) + subl $18, 32, $18 + addl $17, 32, $17 + addl $16, 32, $16 + +$no_more_simd: + RESTORE_SIMD_REGS + +$simd_end: + ble $18, $out + cmplt $18, 16, $1 + bne $1, $quad_loop_end + + .align 4 +$quad_loop_tail: + FIXUP_LDST( ldl $2, 0($17) ) + FIXUP_LDST( ldl $3, 8($17) ) + FIXUP_LDST( stl $2, 0($16) ) + FIXUP_LDST( stl $3, 8($16) ) + subl $18, 16, $18 + addl $17, 16, $17 + addl $16, 16, $16 + cmplt $18, 16, $1 + beq $1, $quad_loop_tail + +$quad_loop_end: + ble $18, $out + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + +$move_one_quad: + FIXUP_LDST( ldl $2, 0($17) ) + FIXUP_LDST( stl $2, 0($16) ) + subl $18, 8, $18 + addl $17, 8, $17 + addl $16, 8, $16 + ble $18, $out + + .align 3 +$byte_loop_tail: + FIXUP_LDST( ldbu $2, 0($17) ) + FIXUP_LDST( stb $2, 0($16) ) + subl $18, 1, $18 + addl $17, 1, $17 + addl $16, 1, $16 + bgt $18, $byte_loop_tail + br $31, $out diff --git a/arch/sw_64/lib/deep-copy_user.S b/arch/sw_64/lib/deep-copy_user.S new file mode 100644 index 000000000000..b79f8f3f0f4a --- /dev/null +++ b/arch/sw_64/lib/deep-copy_user.S @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +/* Allow an exception for an insn; exit if we get one. */ +#define FIXUP_LDST(x, y) \ + 99: x, y; \ + .section __ex_table, "a"; \ + .long 99b - .; \ + ldi $31, $out-99b($31); \ + .previous + +/* + * $7: SIMD status for C3B + * 0: not in simd loop + * 1: in simd loop + * 2: in simd_u loop + * $7: SIMD status for C4 + * 0: not in simd loop + * 1: in simd and simd_u loop + * $18: bytes left to copy + * + */ + .globl __copy_user + .ent __copy_user +__copy_user: + .prologue 0 + .set noreorder + bis $31, $31, $7 +#if defined(CONFIG_SUBARCH_C3B) +#include "deep-copy_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-copy_template_c4.S" +#endif +$out: + bis $31, $18, $0 + beq $7, $return + subl $7, 1, $7 + beq $7, $restore_simd + +#if defined(CONFIG_SUBARCH_C3B) +$restore_simd_u: + RESTORE_SIMD_U_REGS + br $31, $return +#endif + +$restore_simd: + RESTORE_SIMD_REGS + +$return: + ret + .end __copy_user + EXPORT_SYMBOL(__copy_user) diff --git a/arch/sw_64/lib/deep-memcpy.S b/arch/sw_64/lib/deep-memcpy.S new file mode 100644 index 000000000000..78a6bd85cf01 --- /dev/null +++ b/arch/sw_64/lib/deep-memcpy.S @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +#define FIXUP_LDST(x, y) \ + x, y + + .globl memcpy + .ent memcpy +memcpy: + .frame $30, 0, $26, 0 + .prologue 0 + mov $16, $0 +#if defined(CONFIG_SUBARCH_C3B) +#include "deep-copy_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-copy_template_c4.S" +#endif +$out: + ret + .end memcpy + EXPORT_SYMBOL(memcpy) +__memcpy = memcpy +.globl __memcpy diff --git a/arch/sw_64/lib/deep-memset.S b/arch/sw_64/lib/deep-memset.S new file mode 100644 index 000000000000..c6b5355beec6 --- /dev/null +++ b/arch/sw_64/lib/deep-memset.S @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Optimized memset() for SW64 with SIMD instructions + * + * Copyright (C) Mao Minkai + * Author: Mao Minkai + * + * Fill SIZE bytes pointed to by SRC with CHAR. + * + * Input: + * $16: SRC, clobbered + * $17: CHAR, clobbered + * $18: SIZE, clobbered + * + * Output: + * $0: SRC + * + * Temporaries: + * $1: unaligned parts of addr (0 means aligned addr), tmp data + * $2: tmp data + * $3: tmp data + * $4: tmp data + * $5: compare result + * $f10: 32 bytes data (manually saved) + * + */ + +#include +#include + +#define FIXUP_LDST(x, y) \ + x, y + + .set noat + .set noreorder + .text + .align 4 + .globl memset + .globl __memset + .globl ___memset + .globl __memsetw + .globl __constant_c_memset + .ent ___memset +___memset: + .frame $30, 0, $26, 0 + .prologue 0 + +#ifdef CONFIG_SUBARCH_C4 + csrr $6, CSR_WR_FREGS +#endif +/* expand 1 byte data to 8 bytes */ + and $17, 0xff, $17 + sll $17, 8, $4 + bis $17, $4, $17 + sll $17, 16, $4 + bis $17, $4, $17 + sll $17, 32, $4 + bis $17, $4, $17 + +__constant_c_memset: + bis $31, $31, $7 + bis $31, $16, $0 +#if defined(CONFIG_SUBARCH_C3B) +#include "deep-set_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-set_template_c4.S" +#endif +$out: +#ifdef CONFIG_SUBARCH_C4 + csrw $6, CSR_WR_FREGS +#endif + ret + + .end ___memset + EXPORT_SYMBOL(___memset) + + .align 5 + .ent __memsetw +__memsetw: + .prologue 0 + + inslh $17, 0, $1 + inslh $17, 2, $2 + inslh $17, 4, $3 + bis $1, $2, $1 + inslh $17, 6, $4 + bis $1, $3, $1 + bis $1, $4, $17 + br $31, __constant_c_memset + + .end __memsetw + EXPORT_SYMBOL(__memsetw) + +memset = ___memset +EXPORT_SYMBOL(memset) +__memset = ___memset +EXPORT_SYMBOL(__memset) diff --git a/arch/sw_64/lib/deep-set_template.S b/arch/sw_64/lib/deep-set_template.S new file mode 100644 index 000000000000..f9073d638468 --- /dev/null +++ b/arch/sw_64/lib/deep-set_template.S @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * template for memcpy and copy_user with SIMD + * + * $7: SIMD status + * 0: not in simd loop + * 1: in simd loop + * 2: in simd_u loop + * $16: latest dest, clobbered + * $17: 8-byte data to set + * $18: bytes left to copy + * + */ + +#define NC_STORE_THRESHOLD 2048 + +#define SAVE_SIMD_REGS \ + ldi $sp, -0x40($sp); \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vstd $f1, 0($23); \ + ldi $7, 1 + +#define RESTORE_SIMD_REGS \ + vldd $f1, 0($23); \ + ldi $sp, 0x40($sp); \ + bis $31, $31, $7 + + ble $18, $out + and $16, 7, $1 + beq $1, $dest_aligned_8 + + .align 3 +$byte_loop_head: + FIXUP_LDST( stb $17, 0($16) ) + subl $18, 1, $18 + addl $16, 1, $16 + ble $18, $out + and $16, 7, $1 + bne $1, $byte_loop_head + +$dest_aligned_8: + cmplt $18, 16, $1 + bne $1, $quad_loop_end + and $16, 31, $1 + beq $1, $dest_aligned_32 + cmplt $18, 64, $1 + bne $1, $simd_end + + .align 3 +$quad_loop_head: + FIXUP_LDST( stl $17, 0($16) ) + addl $16, 8, $16 + subl $18, 8, $18 + and $16, 31, $1 + beq $1, $dest_aligned_32 + br $31, $quad_loop_head + +$dest_aligned_32: + cmplt $18, 64, $1 + bne $1, $simd_end + +$prep_simd_loop: + SAVE_SIMD_REGS + ifmovd $17, $f1 + vcpyf $f1, $f1 + ldi $1, NC_STORE_THRESHOLD($31) + cmple $18, $1, $1 + bne $1, $simd_loop + + .align 3 +$simd_loop_nc: + FIXUP_LDST( vstd_nc $f1, 0($16) ) + FIXUP_LDST( vstd_nc $f1, 32($16) ) + subl $18, 64, $18 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_loop_nc + memb # required for _nc store instructions + br $31, $simd_loop_end + + .align 3 +$simd_loop: + FIXUP_LDST( vstd $f1, 0($16) ) + FIXUP_LDST( vstd $f1, 32($16) ) + subl $18, 64, $18 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_loop + +$simd_loop_end: + cmplt $18, 32, $1 + bne $1, $no_more_simd + FIXUP_LDST( vstd $f1, 0($16) ) + subl $18, 32, $18 + addl $16, 32, $16 + +$no_more_simd: + RESTORE_SIMD_REGS + +$simd_end: + ble $18, $out + cmplt $18, 16, $1 + bne $1, $quad_loop_end + + .align 3 +$quad_loop_tail: + FIXUP_LDST( stl $17, 0($16) ) + FIXUP_LDST( stl $17, 8($16) ) + subl $18, 16, $18 + addl $16, 16, $16 + cmplt $18, 16, $1 + beq $1, $quad_loop_tail + +$quad_loop_end: + ble $18, $out + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + +$move_one_quad: + FIXUP_LDST( stl $17, 0($16) ) + subl $18, 8, $18 + addl $16, 8, $16 + ble $18, $out + + .align 3 +$byte_loop_tail: + FIXUP_LDST( stb $17, 0($16) ) + subl $18, 1, $18 + addl $16, 1, $16 + bgt $18, $byte_loop_tail + br $31, $out diff --git a/arch/sw_64/lib/deep-set_template_c4.S b/arch/sw_64/lib/deep-set_template_c4.S new file mode 100644 index 000000000000..2b1bcab8fec9 --- /dev/null +++ b/arch/sw_64/lib/deep-set_template_c4.S @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * template for memset and clear_user with SIMD + * + * $7: SIMD status + * 0: not in simd loop + * 1: in simd loop + * $16: latest dest, clobbered + * $17: 8-byte data to set + * $18: bytes left to copy + * + */ + +#define SAVE_SIMD_REGS \ + ldi $sp, -0x40($sp); \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vstd $f1, 0($23); \ + ldi $7, 1 + +#define RESTORE_SIMD_REGS \ + vldd $f1, 0($23); \ + ldi $sp, 0x40($sp); \ + bis $31, $31, $7 + + ble $18, $out + + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + cmplt $18, 16, $1 + bne $1, $quad_loop_end + cmplt $18, 32, $1 + bne $1, $simd_end + +$prep_simd_loop: + SAVE_SIMD_REGS + ifmovd $17, $f1 + vcpyf $f1, $f1 + cmplt $18, 64, $1 + bne $1, $simd_loop_end + + .align 3 +$simd_loop: + FIXUP_LDST( vstd $f1, 0($16) ) + FIXUP_LDST( vstd $f1, 32($16) ) + subl $18, 64, $18 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_loop + +$simd_loop_end: + cmplt $18, 32, $1 + bne $1, $no_more_simd + FIXUP_LDST( vstd $f1, 0($16) ) + subl $18, 32, $18 + addl $16, 32, $16 + +$no_more_simd: + RESTORE_SIMD_REGS + +$simd_end: + ble $18, $out + cmplt $18, 16, $1 + bne $1, $quad_loop_end + + .align 3 +$quad_loop_tail: + FIXUP_LDST( stl $17, 0($16) ) + FIXUP_LDST( stl $17, 8($16) ) + subl $18, 16, $18 + addl $16, 16, $16 + cmplt $18, 16, $1 + beq $1, $quad_loop_tail + +$quad_loop_end: + ble $18, $out + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + +$move_one_quad: + FIXUP_LDST( stl $17, 0($16) ) + subl $18, 8, $18 + addl $16, 8, $16 + ble $18, $out + + .align 3 +$byte_loop_tail: + FIXUP_LDST( stb $17, 0($16) ) + subl $18, 1, $18 + addl $16, 1, $16 + bgt $18, $byte_loop_tail + br $31, $out diff --git a/arch/sw_64/lib/divide.S b/arch/sw_64/lib/divide.S new file mode 100644 index 000000000000..ceef343a6084 --- /dev/null +++ b/arch/sw_64/lib/divide.S @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * (C) 1995 Linus Torvalds + * + * The sw64 chip doesn't provide hardware division, so we have to do it + * by hand. The compiler expects the functions + * + * __divlu: 64-bit unsigned long divide + * __remlu: 64-bit unsigned long remainder + * __divls/__remqs: signed 64-bit + * __divwu/__remlu: unsigned 32-bit + * __divws/__remls: signed 32-bit + * + * These are not normal C functions: instead of the normal + * calling sequence, these expect their arguments in registers + * $24 and $25, and return the result in $27. Register $28 may + * be clobbered (assembly temporary), anything else must be saved. + * + * In short: painful. + * + * This is a rather simple bit-at-a-time algorithm: it's very good + * at dividing random 64-bit numbers, but the more usual case where + * the divisor is small is handled better by the DEC algorithm + * using lookup tables. This uses much less memory, though, and is + * nicer on the cache.. Besides, I don't know the copyright status + * of the DEC code. + */ + +/* + * My temporaries: + * $0 - current bit + * $1 - shifted divisor + * $2 - modulus/quotient + * + * $23 - return address + * $24 - dividend + * $25 - divisor + * + * $27 - quotient/modulus + * $28 - compare status + */ +#include + +#define halt .long 0 + +/* + * Select function type and registers + */ +#define mask $0 +#define divisor $1 +#define compare $28 +#define tmp1 $3 +#define tmp2 $4 + +#ifdef DIV +#define DIV_ONLY(x,y...) x, ##y +#define MOD_ONLY(x,y...) +#define func(x) __div##x +#define modulus $2 +#define quotient $27 +#define GETSIGN(x) xor $24, $25, x +#define STACK 48 +#else +#define DIV_ONLY(x,y...) +#define MOD_ONLY(x,y...) x, ##y +#define func(x) __rem##x +#define modulus $27 +#define quotient $2 +#define GETSIGN(x) bis $24, $24, x +#define STACK 32 +#endif + +/* + * For 32-bit operations, we need to extend to 64-bit + */ +#ifdef INTSIZE +#define ufunction func(wu) +#define sfunction func(w) +#define LONGIFY(x) zapnot x, 15, x +#define SLONGIFY(x) addw x, 0, x +#else +#define ufunction func(lu) +#define sfunction func(l) +#define LONGIFY(x) +#define SLONGIFY(x) +#endif + +.set noat +.align 3 +.globl ufunction +.ent ufunction +ufunction: + subl $30, STACK, $30 + .frame $30, STACK, $23 + .prologue 0 + +7: stl $1, 0($30) + bis $25, $25, divisor + stl $2, 8($30) + bis $24, $24, modulus + stl $0, 16($30) + bis $31, $31, quotient + LONGIFY(divisor) + stl tmp1, 24($30) + LONGIFY(modulus) + bis $31, 1, mask + DIV_ONLY(stl tmp2, 32($30)) + beq divisor, 9f # div by zero + +#ifdef INTSIZE + /* + * shift divisor left, using 3-bit shifts for + * 32-bit divides as we can't overflow. Three-bit + * shifts will result in looping three times less + * here, but can result in two loops more later. + * Thus using a large shift isn't worth it (and + * s8add pairs better than a sll..) + */ +1: cmpult divisor, modulus, compare + s8addl divisor, $31, divisor + s8addl mask, $31, mask + bne compare, 1b +#else +1: cmpult divisor, modulus, compare + blt divisor, 2f + addl divisor, divisor, divisor + addl mask, mask, mask + bne compare, 1b +#endif + + /* ok, start to go right again.. */ +2: DIV_ONLY(addl quotient, mask, tmp2) + srl mask, 1, mask + cmpule divisor, modulus, compare + subl modulus, divisor, tmp1 + DIV_ONLY(selne compare, tmp2, quotient, quotient) + srl divisor, 1, divisor + selne compare, tmp1, modulus, modulus + bne mask, 2b + +9: ldl $1, 0($30) + ldl $2, 8($30) + ldl $0, 16($30) + ldl tmp1, 24($30) + DIV_ONLY(ldl tmp2, 32($30)) + addl $30, STACK, $30 + ret $31, ($23), 1 + .end ufunction + EXPORT_SYMBOL(ufunction) +/* + * Uhh.. Ugly signed division. I'd rather not have it at all, but + * it's needed in some circumstances. There are different ways to + * handle this, really. This does: + * -a / b = a / -b = -(a / b) + * -a % b = -(a % b) + * a % -b = a % b + * which is probably not the best solution, but at least should + * have the property that (x/y)*y + (x%y) = x. + */ +.align 3 +.globl sfunction +.ent sfunction +sfunction: + subl $30, STACK, $30 + .frame $30, STACK, $23 + .prologue 0 + bis $24, $25, $28 + SLONGIFY($28) + bge $28, 7b + stl $24, 0($30) + subl $31, $24, $28 + stl $25, 8($30) + sellt $24, $28, $24, $24 # abs($24) + stl $23, 16($30) + subl $31, $25, $28 + stl tmp1, 24($30) + sellt $25, $28, $25, $25 # abs($25) + bsr $23, ufunction + ldl $24, 0($30) + ldl $25, 8($30) + GETSIGN($28) + subl $31, $27, tmp1 + SLONGIFY($28) + ldl $23, 16($30) + sellt $28, tmp1, $27, $27 + ldl tmp1, 24($30) + addl $30, STACK, $30 + ret $31, ($23), 1 + .end sfunction + EXPORT_SYMBOL(sfunction) diff --git a/arch/sw_64/lib/fls.c b/arch/sw_64/lib/fls.c new file mode 100644 index 000000000000..aa4231f7e472 --- /dev/null +++ b/arch/sw_64/lib/fls.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +/* This is fls(x)-1, except zero is held to zero. This allows most + * efficient input into extbl, plus it allows easy handling of fls(0)=0. + */ + +const unsigned char __flsm1_tab[256] = { + 0, + 0, + 1, 1, + 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, +}; +EXPORT_SYMBOL(__flsm1_tab); diff --git a/arch/sw_64/lib/fpreg.c b/arch/sw_64/lib/fpreg.c new file mode 100644 index 000000000000..178870310908 --- /dev/null +++ b/arch/sw_64/lib/fpreg.c @@ -0,0 +1,992 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) Copyright 1998 Linus Torvalds + */ + +#include +#include + +#define STT(reg, val) \ + asm volatile("fimovd $f"#reg", %0" : "=r"(val)) +#define STS(reg, val) \ + asm volatile("fimovs $f"#reg", %0" : "=r"(val)) +#define LDT(reg, val) \ + asm volatile("ifmovd %0, $f"#reg : : "r"(val)) +#define LDS(reg, val) \ + asm volatile("ifmovs %0, $f"#reg : : "r"(val)) +#define VLDD(reg, val) \ + asm volatile("vldd $f"#reg", %0" : : "m"(val) : "memory") +#define VSTD(reg, val) \ + asm volatile("vstd $f"#reg", %0" : "=m"(val) : : "memory") +#define VLDS(reg, val) \ + asm volatile("vlds $f"#reg", %0" : : "m"(val) : "memory") +#define LDWE(reg, val) \ + asm volatile("ldwe $f"#reg", %0" : : "m"(val) : "memory") +#define VSTS(reg, val) \ + asm volatile("vsts $f"#reg", %0" : "=m"(val) : : "memory") +#define STDH(reg, val) \ + asm volatile("vstd $f"#reg", %0" : "=m"(val) : : "memory") + +void +sw64_write_simd_fp_reg_s(unsigned long reg, unsigned long f0, unsigned long f1) +{ + + unsigned long tmpa[4] __aligned(16); + + tmpa[0] = f0; + tmpa[1] = f1; + + switch (reg) { + case 0: + VLDS(0, *tmpa); + break; + case 1: + VLDS(1, *tmpa); + break; + case 2: + VLDS(2, *tmpa); + break; + case 3: + VLDS(3, *tmpa); + break; + case 4: + VLDS(4, *tmpa); + break; + case 5: + VLDS(5, *tmpa); + break; + case 6: + VLDS(6, *tmpa); + break; + case 7: + VLDS(7, *tmpa); + break; + case 8: + VLDS(8, *tmpa); + break; + case 9: + VLDS(9, *tmpa); + break; + case 10: + VLDS(10, *tmpa); + break; + case 11: + VLDS(11, *tmpa); + break; + case 12: + VLDS(12, *tmpa); + break; + case 13: + VLDS(13, *tmpa); + break; + case 14: + VLDS(14, *tmpa); + break; + case 15: + VLDS(15, *tmpa); + break; + case 16: + VLDS(16, *tmpa); + break; + case 17: + VLDS(17, *tmpa); + break; + case 18: + VLDS(18, *tmpa); + break; + case 19: + VLDS(19, *tmpa); + break; + case 20: + VLDS(20, *tmpa); + break; + case 21: + VLDS(21, *tmpa); + break; + case 22: + VLDS(22, *tmpa); + break; + case 23: + VLDS(23, *tmpa); + break; + case 24: + VLDS(24, *tmpa); + break; + case 25: + VLDS(25, *tmpa); + break; + case 26: + VLDS(26, *tmpa); + break; + case 27: + VLDS(27, *tmpa); + break; + case 28: + VLDS(28, *tmpa); + break; + case 29: + VLDS(29, *tmpa); + break; + case 30: + VLDS(30, *tmpa); + break; + case 31: + break; + } + +} + + +void sw64_write_simd_fp_reg_d(unsigned long reg, unsigned long f0, + unsigned long f1, unsigned long f2, unsigned long f3) +{ + unsigned long tmpa[4] __aligned(32); + + tmpa[0] = f0; + tmpa[1] = f1; + tmpa[2] = f2; + tmpa[3] = f3; + + switch (reg) { + case 0: + VLDD(0, *tmpa); + break; + case 1: + VLDD(1, *tmpa); + break; + case 2: + VLDD(2, *tmpa); + break; + case 3: + VLDD(3, *tmpa); + break; + case 4: + VLDD(4, *tmpa); + break; + case 5: + VLDD(5, *tmpa); + break; + case 6: + VLDD(6, *tmpa); + break; + case 7: + VLDD(7, *tmpa); + break; + case 8: + VLDD(8, *tmpa); + break; + case 9: + VLDD(9, *tmpa); + break; + case 10: + VLDD(10, *tmpa); + break; + case 11: + VLDD(11, *tmpa); + break; + case 12: + VLDD(12, *tmpa); + break; + case 13: + VLDD(13, *tmpa); + break; + case 14: + VLDD(14, *tmpa); + break; + case 15: + VLDD(15, *tmpa); + break; + case 16: + VLDD(16, *tmpa); + break; + case 17: + VLDD(17, *tmpa); + break; + case 18: + VLDD(18, *tmpa); + break; + case 19: + VLDD(19, *tmpa); + break; + case 20: + VLDD(20, *tmpa); + break; + case 21: + VLDD(21, *tmpa); + break; + case 22: + VLDD(22, *tmpa); + break; + case 23: + VLDD(23, *tmpa); + break; + case 24: + VLDD(24, *tmpa); + break; + case 25: + VLDD(25, *tmpa); + break; + case 26: + VLDD(26, *tmpa); + break; + case 27: + VLDD(27, *tmpa); + break; + case 28: + VLDD(28, *tmpa); + break; + case 29: + VLDD(29, *tmpa); + break; + case 30: + VLDD(30, *tmpa); + break; + case 31: + break; + } + + +} + + +void sw64_write_simd_fp_reg_ldwe(unsigned long reg, int a) +{ + switch (reg) { + case 0: + LDWE(0, a); + break; + case 1: + LDWE(1, a); + break; + case 2: + LDWE(2, a); + break; + case 3: + LDWE(3, a); + break; + case 4: + LDWE(4, a); + break; + case 5: + LDWE(5, a); + break; + case 6: + LDWE(6, a); + break; + case 7: + LDWE(7, a); + break; + case 8: + LDWE(8, a); + break; + case 9: + LDWE(9, a); + break; + case 10: + LDWE(10, a); + break; + case 11: + LDWE(11, a); + break; + case 12: + LDWE(12, a); + break; + case 13: + LDWE(13, a); + break; + case 14: + LDWE(14, a); + break; + case 15: + LDWE(15, a); + break; + case 16: + LDWE(16, a); + break; + case 17: + LDWE(17, a); + break; + case 18: + LDWE(18, a); + break; + case 19: + LDWE(19, a); + break; + case 20: + LDWE(20, a); + break; + case 21: + LDWE(21, a); + break; + case 22: + LDWE(22, a); + break; + case 23: + LDWE(23, a); + break; + case 24: + LDWE(24, a); + break; + case 25: + LDWE(25, a); + break; + case 26: + LDWE(26, a); + break; + case 27: + LDWE(27, a); + break; + case 28: + LDWE(28, a); + break; + case 29: + LDWE(29, a); + break; + case 30: + LDWE(30, a); + break; + case 31: + break; + } +} + + +void sw64_read_simd_fp_m_s(unsigned long reg, unsigned long *fp_value) +{ + volatile unsigned long tmpa[2] __aligned(16); + + switch (reg) { + case 0: + VSTS(0, *tmpa); + break; + case 1: + VSTS(1, *tmpa); + break; + case 2: + VSTS(2, *tmpa); + break; + case 3: + VSTS(3, *tmpa); + break; + case 4: + VSTS(4, *tmpa); + break; + case 5: + VSTS(5, *tmpa); + break; + case 6: + VSTS(6, *tmpa); + break; + case 7: + VSTS(7, *tmpa); + break; + case 8: + VSTS(8, *tmpa); + break; + case 9: + VSTS(9, *tmpa); + break; + case 10: + VSTS(10, *tmpa); + break; + case 11: + VSTS(11, *tmpa); + break; + case 12: + VSTS(12, *tmpa); + break; + case 13: + VSTS(13, *tmpa); + break; + case 14: + VSTS(14, *tmpa); + break; + case 15: + VSTS(15, *tmpa); + break; + case 16: + VSTS(16, *tmpa); + break; + case 17: + VSTS(17, *tmpa); + break; + case 18: + VSTS(18, *tmpa); + break; + case 19: + VSTS(19, *tmpa); + break; + case 20: + VSTS(20, *tmpa); + break; + case 21: + VSTS(21, *tmpa); + break; + case 22: + VSTS(22, *tmpa); + break; + case 23: + VSTS(23, *tmpa); + break; + case 24: + VSTS(24, *tmpa); + break; + case 25: + VSTS(25, *tmpa); + break; + case 26: + VSTS(26, *tmpa); + break; + case 27: + VSTS(27, *tmpa); + break; + case 28: + VSTS(28, *tmpa); + break; + case 29: + VSTS(29, *tmpa); + break; + case 30: + VSTS(30, *tmpa); + break; + case 31: + VSTS(31, *tmpa); + break; + } + + *fp_value = tmpa[0]; + *(fp_value+1) = tmpa[1]; +} + +void sw64_read_simd_fp_m_d(unsigned long reg, unsigned long *fp_value) +{ + volatile unsigned long tmpa[4] __aligned(32); + + switch (reg) { + case 0: + VSTD(0, *tmpa); + break; + case 1: + VSTD(1, *tmpa); + break; + case 2: + VSTD(2, *tmpa); + break; + case 3: + VSTD(3, *tmpa); + break; + case 4: + VSTD(4, *tmpa); + break; + case 5: + VSTD(5, *tmpa); + break; + case 6: + VSTD(6, *tmpa); + break; + case 7: + VSTD(7, *tmpa); + break; + case 8: + VSTD(8, *tmpa); + break; + case 9: + VSTD(9, *tmpa); + break; + case 10: + VSTD(10, *tmpa); + break; + case 11: + VSTD(11, *tmpa); + break; + case 12: + VSTD(12, *tmpa); + break; + case 13: + VSTD(13, *tmpa); + break; + case 14: + VSTD(14, *tmpa); + break; + case 15: + VSTD(15, *tmpa); + break; + case 16: + VSTD(16, *tmpa); + break; + case 17: + VSTD(17, *tmpa); + break; + case 18: + VSTD(18, *tmpa); + break; + case 19: + VSTD(19, *tmpa); + break; + case 20: + VSTD(20, *tmpa); + break; + case 21: + VSTD(21, *tmpa); + break; + case 22: + VSTD(22, *tmpa); + break; + case 23: + VSTD(23, *tmpa); + break; + case 24: + VSTD(24, *tmpa); + break; + case 25: + VSTD(25, *tmpa); + break; + case 26: + VSTD(26, *tmpa); + break; + case 27: + VSTD(27, *tmpa); + break; + case 28: + VSTD(28, *tmpa); + break; + case 29: + VSTD(29, *tmpa); + break; + case 30: + VSTD(30, *tmpa); + break; + case 31: + VSTD(31, *tmpa); + break; + } + + *fp_value = tmpa[0]; + *(fp_value+1) = tmpa[1]; + *(fp_value+2) = tmpa[2]; + *(fp_value+3) = tmpa[3]; +} + +unsigned long sw64_read_fp_reg(unsigned long reg) +{ + unsigned long val; + + switch (reg) { + case 0: + STT(0, val); + break; + case 1: + STT(1, val); + break; + case 2: + STT(2, val); + break; + case 3: + STT(3, val); + break; + case 4: + STT(4, val); + break; + case 5: + STT(5, val); + break; + case 6: + STT(6, val); + break; + case 7: + STT(7, val); + break; + case 8: + STT(8, val); + break; + case 9: + STT(9, val); + break; + case 10: + STT(10, val); + break; + case 11: + STT(11, val); + break; + case 12: + STT(12, val); + break; + case 13: + STT(13, val); + break; + case 14: + STT(14, val); + break; + case 15: + STT(15, val); + break; + case 16: + STT(16, val); + break; + case 17: + STT(17, val); + break; + case 18: + STT(18, val); + break; + case 19: + STT(19, val); + break; + case 20: + STT(20, val); + break; + case 21: + STT(21, val); + break; + case 22: + STT(22, val); + break; + case 23: + STT(23, val); + break; + case 24: + STT(24, val); + break; + case 25: + STT(25, val); + break; + case 26: + STT(26, val); + break; + case 27: + STT(27, val); + break; + case 28: + STT(28, val); + break; + case 29: + STT(29, val); + break; + case 30: + STT(30, val); + break; + case 31: + STT(31, val); + break; + default: + return 0; + } + + return val; +} +EXPORT_SYMBOL(sw64_read_fp_reg); + +void sw64_write_fp_reg(unsigned long reg, unsigned long val) +{ + switch (reg) { + case 0: + LDT(0, val); + break; + case 1: + LDT(1, val); + break; + case 2: + LDT(2, val); + break; + case 3: + LDT(3, val); + break; + case 4: + LDT(4, val); + break; + case 5: + LDT(5, val); + break; + case 6: + LDT(6, val); + break; + case 7: + LDT(7, val); + break; + case 8: + LDT(8, val); + break; + case 9: + LDT(9, val); + break; + case 10: + LDT(10, val); + break; + case 11: + LDT(11, val); + break; + case 12: + LDT(12, val); + break; + case 13: + LDT(13, val); + break; + case 14: + LDT(14, val); + break; + case 15: + LDT(15, val); + break; + case 16: + LDT(16, val); + break; + case 17: + LDT(17, val); + break; + case 18: + LDT(18, val); + break; + case 19: + LDT(19, val); + break; + case 20: + LDT(20, val); + break; + case 21: + LDT(21, val); + break; + case 22: + LDT(22, val); + break; + case 23: + LDT(23, val); + break; + case 24: + LDT(24, val); + break; + case 25: + LDT(25, val); + break; + case 26: + LDT(26, val); + break; + case 27: + LDT(27, val); + break; + case 28: + LDT(28, val); + break; + case 29: + LDT(29, val); + break; + case 30: + LDT(30, val); + break; + case 31: + LDT(31, val); + break; + } +} +EXPORT_SYMBOL(sw64_write_fp_reg); + +unsigned long sw64_read_fp_reg_s(unsigned long reg) +{ + unsigned long val; + + switch (reg) { + case 0: + STS(0, val); + break; + case 1: + STS(1, val); + break; + case 2: + STS(2, val); + break; + case 3: + STS(3, val); + break; + case 4: + STS(4, val); + break; + case 5: + STS(5, val); + break; + case 6: + STS(6, val); + break; + case 7: + STS(7, val); + break; + case 8: + STS(8, val); + break; + case 9: + STS(9, val); + break; + case 10: + STS(10, val); + break; + case 11: + STS(11, val); + break; + case 12: + STS(12, val); + break; + case 13: + STS(13, val); + break; + case 14: + STS(14, val); + break; + case 15: + STS(15, val); + break; + case 16: + STS(16, val); + break; + case 17: + STS(17, val); + break; + case 18: + STS(18, val); + break; + case 19: + STS(19, val); + break; + case 20: + STS(20, val); + break; + case 21: + STS(21, val); + break; + case 22: + STS(22, val); + break; + case 23: + STS(23, val); + break; + case 24: + STS(24, val); + break; + case 25: + STS(25, val); + break; + case 26: + STS(26, val); + break; + case 27: + STS(27, val); + break; + case 28: + STS(28, val); + break; + case 29: + STS(29, val); + break; + case 30: + STS(30, val); + break; + case 31: + STS(31, val); + break; + default: + return 0; + } + + return val; +} +EXPORT_SYMBOL(sw64_read_fp_reg_s); + +void sw64_write_fp_reg_s(unsigned long reg, unsigned long val) +{ + switch (reg) { + case 0: + LDS(0, val); + break; + case 1: + LDS(1, val); + break; + case 2: + LDS(2, val); + break; + case 3: + LDS(3, val); + break; + case 4: + LDS(4, val); + break; + case 5: + LDS(5, val); + break; + case 6: + LDS(6, val); + break; + case 7: + LDS(7, val); + break; + case 8: + LDS(8, val); + break; + case 9: + LDS(9, val); + break; + case 10: + LDS(10, val); + break; + case 11: + LDS(11, val); + break; + case 12: + LDS(12, val); + break; + case 13: + LDS(13, val); + break; + case 14: + LDS(14, val); + break; + case 15: + LDS(15, val); + break; + case 16: + LDS(16, val); + break; + case 17: + LDS(17, val); + break; + case 18: + LDS(18, val); + break; + case 19: + LDS(19, val); + break; + case 20: + LDS(20, val); + break; + case 21: + LDS(21, val); + break; + case 22: + LDS(22, val); + break; + case 23: + LDS(23, val); + break; + case 24: + LDS(24, val); + break; + case 25: + LDS(25, val); + break; + case 26: + LDS(26, val); + break; + case 27: + LDS(27, val); + break; + case 28: + LDS(28, val); + break; + case 29: + LDS(29, val); + break; + case 30: + LDS(30, val); + break; + case 31: + LDS(31, val); + break; + } +} +EXPORT_SYMBOL(sw64_write_fp_reg_s); diff --git a/arch/sw_64/lib/iomap.c b/arch/sw_64/lib/iomap.c new file mode 100644 index 000000000000..d9c66a89131e --- /dev/null +++ b/arch/sw_64/lib/iomap.c @@ -0,0 +1,477 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Sw_64 IO and memory functions. + */ + +#include + +#include +#include + +/* + * Here comes the sw64 implementation of the IOMAP interfaces. + */ +unsigned int ioread8(const void __iomem *addr) +{ + return readb(addr); +} +EXPORT_SYMBOL(ioread8); + +unsigned int ioread16(const void __iomem *addr) +{ + return readw(addr); +} +EXPORT_SYMBOL(ioread16); + +unsigned int ioread32(const void __iomem *addr) +{ + return readl(addr); +} +EXPORT_SYMBOL(ioread32); + +void iowrite8(u8 b, void __iomem *addr) +{ + writeb(b, addr); +} +EXPORT_SYMBOL(iowrite8); + +void iowrite16(u16 b, void __iomem *addr) +{ + writew(b, addr); +} +EXPORT_SYMBOL(iowrite16); + +void iowrite32(u32 b, void __iomem *addr) +{ + writel(b, addr); +} +EXPORT_SYMBOL(iowrite32); + +u8 inb(unsigned long port) +{ + return ioread8(ioport_map(port, 1)); +} +EXPORT_SYMBOL(inb); + +u16 inw(unsigned long port) +{ + return ioread16(ioport_map(port, 2)); +} +EXPORT_SYMBOL(inw); + +u32 inl(unsigned long port) +{ + return ioread32(ioport_map(port, 4)); +} +EXPORT_SYMBOL(inl); + +void outb(u8 b, unsigned long port) +{ + iowrite8(b, ioport_map(port, 1)); +} +EXPORT_SYMBOL(outb); + +void outw(u16 b, unsigned long port) +{ + iowrite16(b, ioport_map(port, 2)); +} +EXPORT_SYMBOL(outw); + +void outl(u32 b, unsigned long port) +{ + iowrite32(b, ioport_map(port, 4)); +} +EXPORT_SYMBOL(outl); + + +/* + * Read COUNT 8-bit bytes from port PORT into memory starting at SRC. + */ +void ioread8_rep(const void __iomem *port, void *dst, unsigned long count) +{ + while ((unsigned long)dst & 0x3) { + if (!count) + return; + count--; + *(unsigned char *)dst = ioread8(port); + dst += 1; + } + + while (count >= 4) { + unsigned int w; + + count -= 4; + w = ioread8(port); + w |= ioread8(port) << 8; + w |= ioread8(port) << 16; + w |= ioread8(port) << 24; + *(unsigned int *)dst = w; + dst += 4; + } + + while (count) { + --count; + *(unsigned char *)dst = ioread8(port); + dst += 1; + } +} +EXPORT_SYMBOL(ioread8_rep); + +void insb(unsigned long port, void *dst, unsigned long count) +{ + ioread8_rep(ioport_map(port, 1), dst, count); +} +EXPORT_SYMBOL(insb); + +/* + * Read COUNT 16-bit words from port PORT into memory starting at + * SRC. SRC must be at least short aligned. This is used by the + * IDE driver to read disk sectors. Performance is important, but + * the interfaces seems to be slow: just using the inlined version + * of the inw() breaks things. + */ +void ioread16_rep(const void __iomem *port, void *dst, unsigned long count) +{ + if (unlikely((unsigned long)dst & 0x3)) { + if (!count) + return; + BUG_ON((unsigned long)dst & 0x1); + count--; + *(unsigned short *)dst = ioread16(port); + dst += 2; + } + + while (count >= 2) { + unsigned int w; + + count -= 2; + w = ioread16(port); + w |= ioread16(port) << 16; + *(unsigned int *)dst = w; + dst += 4; + } + + if (count) + *(unsigned short *)dst = ioread16(port); +} +EXPORT_SYMBOL(ioread16_rep); + +void insw(unsigned long port, void *dst, unsigned long count) +{ + ioread16_rep(ioport_map(port, 2), dst, count); +} +EXPORT_SYMBOL(insw); + + +/* + * Read COUNT 32-bit words from port PORT into memory starting at + * SRC. Now works with any alignment in SRC. Performance is important, + * but the interfaces seems to be slow: just using the inlined version + * of the inl() breaks things. + */ +void ioread32_rep(const void __iomem *port, void *dst, unsigned long count) +{ + if (unlikely((unsigned long)dst & 0x3)) { + while (count--) { + struct S { int x __packed; }; + ((struct S *)dst)->x = ioread32(port); + dst += 4; + } + } else { + /* Buffer 32-bit aligned. */ + while (count--) { + *(unsigned int *)dst = ioread32(port); + dst += 4; + } + } +} +EXPORT_SYMBOL(ioread32_rep); + +void insl(unsigned long port, void *dst, unsigned long count) +{ + ioread32_rep(ioport_map(port, 4), dst, count); +} +EXPORT_SYMBOL(insl); + + +/* + * Like insb but in the opposite direction. + * Don't worry as much about doing aligned memory transfers: + * doing byte reads the "slow" way isn't nearly as slow as + * doing byte writes the slow way (no r-m-w cycle). + */ +void iowrite8_rep(void __iomem *port, const void *xsrc, unsigned long count) +{ + const unsigned char *src = xsrc; + + while (count--) + iowrite8(*src++, port); +} +EXPORT_SYMBOL(iowrite8_rep); + +void outsb(unsigned long port, const void *src, unsigned long count) +{ + iowrite8_rep(ioport_map(port, 1), src, count); +} +EXPORT_SYMBOL(outsb); + + +/* + * Like insw but in the opposite direction. This is used by the IDE + * driver to write disk sectors. Performance is important, but the + * interfaces seems to be slow: just using the inlined version of the + * outw() breaks things. + */ +void iowrite16_rep(void __iomem *port, const void *src, unsigned long count) +{ + if (unlikely((unsigned long)src & 0x3)) { + if (!count) + return; + BUG_ON((unsigned long)src & 0x1); + iowrite16(*(unsigned short *)src, port); + src += 2; + --count; + } + + while (count >= 2) { + unsigned int w; + + count -= 2; + w = *(unsigned int *)src; + src += 4; + iowrite16(w >> 0, port); + iowrite16(w >> 16, port); + } + + if (count) + iowrite16(*(unsigned short *)src, port); +} +EXPORT_SYMBOL(iowrite16_rep); + +void outsw(unsigned long port, const void *src, unsigned long count) +{ + iowrite16_rep(ioport_map(port, 2), src, count); +} +EXPORT_SYMBOL(outsw); + + +/* + * Like insl but in the opposite direction. This is used by the IDE + * driver to write disk sectors. Works with any alignment in SRC. + * Performance is important, but the interfaces seems to be slow: + * just using the inlined version of the outl() breaks things. + */ +void iowrite32_rep(void __iomem *port, const void *src, unsigned long count) +{ + if (unlikely((unsigned long)src & 0x3)) { + while (count--) { + struct S { int x __packed; }; + iowrite32(((struct S *)src)->x, port); + src += 4; + } + } else { + /* Buffer 32-bit aligned. */ + while (count--) { + iowrite32(*(unsigned int *)src, port); + src += 4; + } + } +} +EXPORT_SYMBOL(iowrite32_rep); + +void outsl(unsigned long port, const void *src, unsigned long count) +{ + iowrite32_rep(ioport_map(port, 4), src, count); +} +EXPORT_SYMBOL(outsl); + + +/* + * Copy data from IO memory space to "real" memory space. + * This needs to be optimized. + */ +void memcpy_fromio(void *to, const volatile void __iomem *from, long count) +{ + /* + * Optimize co-aligned transfers. Everything else gets handled + * a byte at a time. + */ + + if (count >= 8 && ((u64)to & 7) == ((u64)from & 7)) { + count -= 8; + do { + *(u64 *)to = __raw_readq(from); + count -= 8; + to += 8; + from += 8; + } while (count >= 0); + count += 8; + } + + if (count >= 4 && ((u64)to & 3) == ((u64)from & 3)) { + count -= 4; + do { + *(u32 *)to = __raw_readl(from); + count -= 4; + to += 4; + from += 4; + } while (count >= 0); + count += 4; + } + + if (count >= 2 && ((u64)to & 1) == ((u64)from & 1)) { + count -= 2; + do { + *(u16 *)to = __raw_readw(from); + count -= 2; + to += 2; + from += 2; + } while (count >= 0); + count += 2; + } + + while (count > 0) { + *(u8 *) to = __raw_readb(from); + count--; + to++; + from++; + } + mb(); +} +EXPORT_SYMBOL(memcpy_fromio); + + +/* + * Copy data from "real" memory space to IO memory space. + * This needs to be optimized. + */ +void memcpy_toio(volatile void __iomem *to, const void *from, long count) +{ + /* + * Optimize co-aligned transfers. Everything else gets handled + * a byte at a time. + * FIXME -- align FROM. + */ + + if (count >= 8 && ((u64)to & 7) == ((u64)from & 7)) { + count -= 8; + do { + __raw_writeq(*(const u64 *)from, to); + count -= 8; + to += 8; + from += 8; + } while (count >= 0); + count += 8; + } + + if (count >= 4 && ((u64)to & 3) == ((u64)from & 3)) { + count -= 4; + do { + __raw_writel(*(const u32 *)from, to); + count -= 4; + to += 4; + from += 4; + } while (count >= 0); + count += 4; + } + + if (count >= 2 && ((u64)to & 1) == ((u64)from & 1)) { + count -= 2; + do { + __raw_writew(*(const u16 *)from, to); + count -= 2; + to += 2; + from += 2; + } while (count >= 0); + count += 2; + } + + while (count > 0) { + __raw_writeb(*(const u8 *) from, to); + count--; + to++; + from++; + } + mb(); +} +EXPORT_SYMBOL(memcpy_toio); + + +/* + * "memset" on IO memory space. + */ +void _memset_c_io(volatile void __iomem *to, unsigned long c, long count) +{ + /* Handle any initial odd byte */ + if (count > 0 && ((u64)to & 1)) { + __raw_writeb(c, to); + to++; + count--; + } + + /* Handle any initial odd halfword */ + if (count >= 2 && ((u64)to & 2)) { + __raw_writew(c, to); + to += 2; + count -= 2; + } + + /* Handle any initial odd word */ + if (count >= 4 && ((u64)to & 4)) { + __raw_writel(c, to); + to += 4; + count -= 4; + } + + /* + * Handle all full-sized quadwords: we're aligned + * (or have a small count) + */ + count -= 8; + if (count >= 0) { + do { + __raw_writeq(c, to); + to += 8; + count -= 8; + } while (count >= 0); + } + count += 8; + + /* The tail is word-aligned if we still have count >= 4 */ + if (count >= 4) { + __raw_writel(c, to); + to += 4; + count -= 4; + } + + /* The tail is half-word aligned if we have count >= 2 */ + if (count >= 2) { + __raw_writew(c, to); + to += 2; + count -= 2; + } + + /* And finally, one last byte.. */ + if (count) + __raw_writeb(c, to); + mb(); +} +EXPORT_SYMBOL(_memset_c_io); + +void __iomem *ioport_map(unsigned long port, unsigned int size) +{ + unsigned long io_offset; + + if (port < 0x100000) { + io_offset = is_in_host() ? LPC_LEGACY_IO : PCI_VT_LEGACY_IO; + port = port | io_offset; + } + + return __va(port); +} +EXPORT_SYMBOL(ioport_map); + +void ioport_unmap(void __iomem *addr) +{ +} +EXPORT_SYMBOL(ioport_unmap); diff --git a/arch/sw_64/lib/iomap_copy.c b/arch/sw_64/lib/iomap_copy.c new file mode 100644 index 000000000000..1c75bd602d7e --- /dev/null +++ b/arch/sw_64/lib/iomap_copy.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +/** + * __iowrite32_copy - copy data to MMIO space, in 32-bit units + * @to: destination, in MMIO space (must be 32-bit aligned) + * @from: source (must be 32-bit aligned) + * @count: number of 32-bit quantities to copy + * + * Copy data from kernel space to MMIO space, in units of 32 bits at a + * time. Order of access is not guaranteed, nor is a memory barrier + * performed afterwards. + */ +void __iowrite32_copy(void __iomem *to, + const void *from, + size_t count) +{ + u32 __iomem *dst = to; + const u32 *src = from; + const u32 *end = src + count; + + while (src < end) { + __raw_writel(*src++, dst++); + mb(); + } + +} + +/** + * __iowrite64_copy - copy data to MMIO space, in 64-bit or 32-bit units + * @to: destination, in MMIO space (must be 64-bit aligned) + * @from: source (must be 64-bit aligned) + * @count: number of 64-bit quantities to copy + * + * Copy data from kernel space to MMIO space, in units of 32 or 64 bits at a + * time. Order of access is not guaranteed, nor is a memory barrier + * performed afterwards. + */ +void __iowrite64_copy(void __iomem *to, + const void *from, + size_t count) +{ + u64 __iomem *dst = to; + const u64 *src = from; + const u64 *end = src + count; + + while (src < end) { + __raw_writeq(*src++, dst++); + mb(); + } +} diff --git a/arch/sw_64/lib/memcpy.S b/arch/sw_64/lib/memcpy.S new file mode 100644 index 000000000000..31c422b393ee --- /dev/null +++ b/arch/sw_64/lib/memcpy.S @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Reasonably optimized memcpy() routine for the sw64 + * + * - memory accessed as aligned quadwords only + * - uses bcmpge to compare 8 bytes in parallel + * + * Temp usage notes: + * $1, $2, - scratch + */ +#include + .set noreorder + .set noat + + .align 4 + .globl memcpy + .ent memcpy +memcpy: + .frame $30, 0, $26, 0 + .prologue 0 + + mov $16, $0 + ble $18, $nomoredata + xor $16, $17, $1 + and $1, 7, $1 + + bne $1, $misaligned + /* source and dest are same mod 8 address */ + and $16, 7, $1 + beq $1, $both_0mod8 + + /* + * source and dest are same misalignment. move a byte at a time + * until a 0mod8 alignment for both is reached. + * At least one byte more to move + */ + +$head_align: + ldbu $1, 0($17) + subl $18, 1, $18 + addl $17, 1, $17 + stb $1, 0($16) + addl $16, 1, $16 + and $16, 7, $1 + ble $18, $nomoredata + bne $1, $head_align + +$both_0mod8: + cmple $18, 127, $1 + bne $1, $no_unroll + and $16, 63, $1 + beq $1, $do_unroll + +$single_head_quad: + ldl $1, 0($17) + subl $18, 8, $18 + addl $17, 8, $17 + + stl $1, 0($16) + addl $16, 8, $16 + and $16, 63, $1 + bne $1, $single_head_quad + +$do_unroll: + addl $16, 64, $7 + cmple $18, 127, $1 + bne $1, $tail_quads + +$unroll_body: + #wh64 ($7) + fillde 0($7) + + ldl $6, 0($17) + + ldl $4, 8($17) + ldl $5, 16($17) + addl $7, 64, $7 + + ldl $3, 24($17) + addl $16, 64, $1 + + addl $17, 32, $17 + stl $6, 0($16) + + stl $4, 8($16) + stl $5, 16($16) + subl $18, 192, $2 + + stl $3, 24($16) + addl $16, 32, $16 + + ldl $6, 0($17) + ldl $4, 8($17) + #cmovlt $2, $1, $7 + sellt $2, $1, $7, $7 + + ldl $5, 16($17) + ldl $3, 24($17) + addl $16, 32, $16 + subl $18, 64, $18 + + addl $17, 32, $17 + stl $6, -32($16) + stl $4, -24($16) + cmple $18, 63, $1 + + stl $5, -16($16) + stl $3, -8($16) + beq $1, $unroll_body + +$tail_quads: +$no_unroll: + .align 4 + subl $18, 8, $18 + blt $18, $less_than_8 + +$move_a_quad: + ldl $1, 0($17) + subl $18, 8, $18 + addl $17, 8, $17 + + stl $1, 0($16) + addl $16, 8, $16 + bge $18, $move_a_quad + +$less_than_8: + .align 4 + addl $18, 8, $18 + ble $18, $nomoredata + + /* Trailing bytes */ +$tail_bytes: + subl $18, 1, $18 + ldbu $1, 0($17) + addl $17, 1, $17 + + stb $1, 0($16) + addl $16, 1, $16 + bgt $18, $tail_bytes + + /* branching to exit takes 3 extra cycles, so replicate exit here */ + ret $31, ($26), 1 + +$misaligned: + mov $0, $4 + and $0, 7, $1 + beq $1, $dest_0mod8 + +$aligndest: + ble $18, $nomoredata + ldbu $1, 0($17) + subl $18, 1, $18 + addl $17, 1, $17 + + stb $1, 0($4) + addl $4, 1, $4 + and $4, 7, $1 + bne $1, $aligndest + + /* Source has unknown alignment, but dest is known to be 0mod8 */ +$dest_0mod8: + subl $18, 8, $18 + blt $18, $misalign_tail + ldl_u $3, 0($17) + +$mis_quad: + ldl_u $16, 8($17) + extll $3, $17, $3 + exthl $16, $17, $1 + bis $3, $1, $1 + + subl $18, 8, $18 + addl $17, 8, $17 + stl $1, 0($4) + mov $16, $3 + + addl $4, 8, $4 + bge $18, $mis_quad + +$misalign_tail: + addl $18, 8, $18 + ble $18, $nomoredata + +$misalign_byte: + ldbu $1, 0($17) + subl $18, 1, $18 + addl $17, 1, $17 + + stb $1, 0($4) + addl $4, 1, $4 + bgt $18, $misalign_byte + + +$nomoredata: + ret $31, ($26), 1 + + .end memcpy + EXPORT_SYMBOL(memcpy) +/* For backwards module compatibility. */ +__memcpy = memcpy +.globl __memcpy diff --git a/arch/sw_64/lib/memmove.S b/arch/sw_64/lib/memmove.S new file mode 100644 index 000000000000..3e34fcd5b217 --- /dev/null +++ b/arch/sw_64/lib/memmove.S @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Barely optimized memmove routine for sw64. + * This is hand-massaged output from the original memcpy.c. We defer to + * memcpy whenever possible; the backwards copy loops are not unrolled. + */ +#include + .set noat + .set noreorder + .text + + .align 4 + .globl memmove + .ent memmove +memmove: + ldgp $29, 0($27) + unop + .prologue 1 + + addl $16, $18, $4 + addl $17, $18, $5 + cmpule $4, $17, $1 # dest + n <= src + cmpule $5, $16, $2 # dest >= src + n + + bis $1, $2, $1 + mov $16, $0 + xor $16, $17, $2 + bne $1, memcpy # samegp + + and $2, 7, $2 # Test for src/dest co-alignment. + and $16, 7, $1 + cmpule $16, $17, $3 + bne $3, $memmove_up # dest < src + + and $4, 7, $1 + bne $2, $misaligned_dn + unop + beq $1, $skip_aligned_byte_loop_head_dn + +$aligned_byte_loop_head_dn: + ldi $4, -1($4) + ldi $5, -1($5) + unop + ble $18, $egress + + ldbu $1, 0($5) + ldi $18, -1($18) + stb $1, 0($4) + + and $4, 7, $6 + bne $6, $aligned_byte_loop_head_dn + +$skip_aligned_byte_loop_head_dn: + ldi $18, -8($18) + blt $18, $skip_aligned_word_loop_dn + +$aligned_word_loop_dn: + ldl $1, -8($5) + ldi $5, -8($5) + ldi $18, -8($18) + + stl $1, -8($4) + ldi $4, -8($4) + bge $18, $aligned_word_loop_dn + +$skip_aligned_word_loop_dn: + ldi $18, 8($18) + bgt $18, $byte_loop_tail_dn + unop + ret $31, ($26), 1 + + .align 4 +$misaligned_dn: + fnop + unop + beq $18, $egress + +$byte_loop_tail_dn: + ldbu $1, -1($5) + ldi $5, -1($5) + ldi $4, -1($4) + + ldi $18, -1($18) + stb $1, 0($4) + + bgt $18, $byte_loop_tail_dn + br $egress + +$memmove_up: + mov $16, $4 + mov $17, $5 + bne $2, $misaligned_up + beq $1, $skip_aligned_byte_loop_head_up + +$aligned_byte_loop_head_up: + unop + ble $18, $egress + ldbu $1, 0($5) + + ldi $18, -1($18) + + ldi $5, 1($5) + stb $1, 0($4) + ldi $4, 1($4) + + and $4, 7, $6 + bne $6, $aligned_byte_loop_head_up + +$skip_aligned_byte_loop_head_up: + ldi $18, -8($18) + blt $18, $skip_aligned_word_loop_up + +$aligned_word_loop_up: + ldl $1, 0($5) + ldi $5, 8($5) + ldi $18, -8($18) + + stl $1, 0($4) + ldi $4, 8($4) + bge $18, $aligned_word_loop_up + +$skip_aligned_word_loop_up: + ldi $18, 8($18) + bgt $18, $byte_loop_tail_up + unop + ret $31, ($26), 1 + + .align 4 +$misaligned_up: + fnop + unop + beq $18, $egress + +$byte_loop_tail_up: + ldbu $1, 0($5) + ldi $18, -1($18) + + stb $1, 0($4) + + ldi $5, 1($5) + ldi $4, 1($4) + bgt $18, $byte_loop_tail_up + +$egress: + ret $31, ($26), 1 + + .end memmove + EXPORT_SYMBOL(memmove) diff --git a/arch/sw_64/lib/memset.S b/arch/sw_64/lib/memset.S new file mode 100644 index 000000000000..dbc4d775c7ea --- /dev/null +++ b/arch/sw_64/lib/memset.S @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This is an efficient (and small) implementation of the C library "memset()" + * function for the sw. + * + * (C) Copyright 1996 Linus Torvalds + * + * This routine is "moral-ware": you are free to use it any way you wish, and + * the only obligation I put on you is a moral one: if you make any improvements + * to the routine, please send me your improvements for me to use similarly. + * + * The scheduling comments are according to the documentation (and done by + * hand, so they might well be incorrect, please do tell me about it..) + */ + +#include + + .set noat + .set noreorder +.text + .globl memset + .globl __memset + .globl ___memset + .globl __memsetw + .globl __constant_c_memset + + .ent ___memset +.align 5 +___memset: + .frame $30, 0, $26, 0 + .prologue 0 + + and $17, 255, $1 + inslb $17, 1, $17 + bis $17, $1, $17 + sll $17, 16, $1 + + bis $17, $1, $17 + sll $17, 32, $1 + bis $17, $1, $17 + ldl_u $31, 0($30) + +.align 5 +__constant_c_memset: + addl $18, $16, $6 + bis $16, $16, $0 + xor $16, $6, $1 + ble $18, end + + bic $1, 7, $1 + beq $1, within_one_quad + and $16, 7, $3 + beq $3, aligned + + bis $16, $16, $5 + subl $3, 8, $3 + addl $18, $3, $18 + subl $16, $3, $16 + + eqv $3, $31, $3 + addl $3, 1, $3 +unaligned_start_loop: + stb $17, 0($5) + subl $3, 1, $3 + addl $5, 1, $5 + bgt $3, unaligned_start_loop + + +.align 4 +aligned: + sra $18, 3, $3 + and $18, 7, $18 + bis $16, $16, $5 + beq $3, no_quad + +/*added by JJ*/ + ldi $3, -8($3) + blt $3, nounrol + +.align 3 +wloop: + fillde 256($5) + stl $17, 0($5) + stl $17, 8($5) + stl $17, 16($5) + stl $17, 24($5) + subl $3, 8, $3 + stl $17, 32($5) + stl $17, 40($5) + stl $17, 48($5) + stl $17, 56($5) + addl $5, 0x40, $5 + bge $3, wloop + +nounrol: + addl $3, 8, $3 + beq $3, no_quad +/*end JJ*/ + +.align 3 +loop: + stl $17, 0($5) + subl $3, 1, $3 + addl $5, 8, $5 + bne $3, loop + +no_quad: + bis $31, $31, $31 + beq $18, end + and $6, 7, $6 +no_quad_loop: + stb $17, 0($5) + subl $6, 1, $6 + addl $5, 1, $5 + bgt $6, no_quad_loop + ret $31, ($26), 1 + +.align 3 +within_one_quad: + bis $18, $18, $1 + bis $16, $16, $5 +within_one_quad_loop: + stb $17, 0($5) + subl $1, 1, $1 + addl $5, 1, $5 + bgt $1, within_one_quad_loop + +end: + ret $31, ($26), 1 + .end ___memset + EXPORT_SYMBOL(___memset) + + .align 5 + .ent __memsetw +__memsetw: + .prologue 0 + + inslh $17, 0, $1 + inslh $17, 2, $2 + inslh $17, 4, $3 + or $1, $2, $1 + inslh $17, 6, $4 + or $1, $3, $1 + or $1, $4, $17 + br __constant_c_memset + + .end __memsetw + EXPORT_SYMBOL(__memsetw) + +memset = ___memset +EXPORT_SYMBOL(memset) +__memset = ___memset +EXPORT_SYMBOL(__memset) diff --git a/arch/sw_64/lib/strcpy.S b/arch/sw_64/lib/strcpy.S new file mode 100644 index 000000000000..61b6141f88e2 --- /dev/null +++ b/arch/sw_64/lib/strcpy.S @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Optimized strcpy() for SW64 + + * Copyright (C) Mao Minkai + * Author: Mao Minkai + * + * Copy a null-terminated string from SRC to DST. + * + * Input: + * $16: DST, clobbered + * $17: SRC, clobbered + * + * Output: + * $0: DST + * + * Temporaries: + * $1: unaligned parts of addr (0 means aligned addr) + * $4: current data to copy (could have 1 byte or 8 bytes) + * $5: parts of current data, compare result + * $6: number of bytes left to copy + * + * Tag naming: + * co: SRC and DST are co-aligned + * mis: SRC and DST are not co-aligned + * a: SRC or DST has aligned address + * una: SRC or DST has unaligned address + * + */ + +#include + + .text + .align 4 + .globl strcpy + .ent strcpy +strcpy: + .frame $30, 0, $26 + .prologue 0 + + bis $31, $16, $0 # set return value + + xor $16, $17, $1 + and $1, 7, $1 + bne $1, $mis_aligned + +/* src and dst are co-aligned */ + and $16, 7, $1 + bne $1, $co_una_head + +/* do the copy in loop, for (co)-aligned src and dst with (a)ligned addr */ +$co_a_loop: + ldl $4, 0($17) + cmpgeb $31, $4, $5 + bne $5, $tail_loop # we find null + stl $4, 0($16) + addl $17, 8, $17 + addl $16, 8, $16 + br $31, $co_a_loop + +/* src and dst are co-aligned but have unaligned address */ +$co_una_head: + ldl_u $4, 0($17) + extll $4, $16, $4 + cmpgeb $31, $4, $5 + bne $5, $tail_loop # we find null + ldi $6, 8($31) + subl $6, $1, $6 + addl $17, $6, $17 # prepare addr of middle part + +/* copy the unaligned part in loop */ +$co_una_head_loop: + stb $4, 0($16) + addl $16, 1, $16 + subl $6, 1, $6 + beq $6, $co_a_loop + addl $4, 1, $4 + br $31, $co_una_head_loop + +/* src and dst are not co-aligned */ +$mis_aligned: + and $16, 7, $1 + beq $1, $mis_a_dst + ldi $6, 8($31) + subl $6, $1, $6 + +/* copy the first few bytes to make dst aligned */ +$mis_una_head_loop: + bis $31, $31, $6 + ldbu $4, 0($17) + stb $4, 0($16) + beq $4, $out # we have reached null, return + addl $17, 1, $17 + addl $16, 1, $16 + subl $6, 1, $6 + beq $6, $mis_a_dst + br $31, $mis_una_head_loop + +/* dst has aligned addr */ +$mis_a_dst: + and $17, 7, $1 + +$mis_a_dst_loop: + ldl_u $4, 0($17) + ldl_u $5, 7($17) + extll $4, $1, $4 + exthl $5, $1, $5 + bis $4, $5, $4 + cmpgeb $31, $4, $5 + bne $5, $tail_loop # we find null + stl $4, 0($16) + addl $17, 8, $17 + addl $16, 8, $16 + br $31, $mis_a_dst_loop + +/* we have find null in the last few bytes, copy one byte each time */ +$tail_loop: + ldbu $4, 0($17) + stb $4, 0($16) + beq $4, $out # we have reached null, return + addl $17, 1, $17 + addl $16, 1, $16 + br $31, $tail_loop + +/* copy is done, return */ +$out: + ret + + .end strcpy + EXPORT_SYMBOL(strcpy) diff --git a/arch/sw_64/lib/strncpy.S b/arch/sw_64/lib/strncpy.S new file mode 100644 index 000000000000..f50c70599bb4 --- /dev/null +++ b/arch/sw_64/lib/strncpy.S @@ -0,0 +1,156 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Optimized strncpy() for SW64 + + * Copyright (C) Mao Minkai + * Author: Mao Minkai + * + * Copy a string from SRC to DST. At most SIZE bytes are coppied. + * + * Input: + * $16: DST, clobbered + * $17: SRC, clobbered + * $18: SIZE, clobbered + * + * Output: + * $0: DST + * + * Temporaries: + * $1: unaligned parts of addr (0 means aligned addr) + * $4: current data to copy (could have 1 byte or 8 bytes) + * $5: parts of current data, compare result + * $6: number of bytes left to copy in head + * + * Tag naming: + * co: SRC and DST are co-aligned + * mis: SRC and DST are not co-aligned + * a: SRC or DST has aligned address + * una: SRC or DST has unaligned address + * + */ + +#include + + .text + .align 4 + .globl strncpy + .ent strncpy +strncpy: + .frame $30, 0, $26 + .prologue 0 + + bis $31, $16, $0 # set return value + beq $18, $out # return if size is 0 + cmplt $18, 8, $5 # size less than 8, do 1-byte copy + bne $5, $tail_loop + + xor $16, $17, $1 + and $1, 7, $1 + bne $1, $mis_aligned + +/* src and dst are co-aligned */ + and $16, 7, $1 + bne $1, $co_una_head + +/* do the copy in loop, for (co)-aligned src and dst with (a)ligned addr */ +$co_a_loop: + ldl $4, 0($17) + cmpgeb $31, $4, $5 + bne $5, $tail_loop # we find null + subl $18, 8, $5 + blt $5, $tail_loop # we have fewer than 8 bytes to copy + stl $4, 0($16) + subl $18, 8, $18 + beq $18, $out + addl $17, 8, $17 + addl $16, 8, $16 + br $31, $co_a_loop + +/* src and dst are co-aligned but have unaligned address */ +$co_una_head: + ldl_u $4, 0($17) + extll $4, $16, $4 + cmpgeb $31, $4, $5 + bne $5, $tail_loop # we find null + ldi $6, 8($31) + subl $6, $1, $6 + addl $17, $6, $17 # prepare addr of middle part + subl $18, $6, $18 # sub bytes going to be copy + +/* copy the unaligned part in loop */ +$co_una_head_loop: + stb $4, 0($16) + addl $16, 1, $16 + subl $6, 1, $6 + beq $6, $co_a_loop + addl $4, 1, $4 + br $31, $co_una_head_loop + +/* src and dst are not co-aligned */ +$mis_aligned: + and $16, 7, $1 + beq $1, $mis_a_dst + +$mis_una_head: + ldi $6, 8($31) + subl $6, $1, $6 + +/* copy the first few bytes to make dst aligned */ +$mis_una_head_loop: + ldbu $4, 0($17) + stb $4, 0($16) + subl $18, 1, $18 + beq $18, $out + beq $4, $null_padding # we have reached null + addl $17, 1, $17 + addl $16, 1, $16 + subl $6, 1, $6 + beq $6, $mis_a_dst + br $31, $mis_una_head_loop + +/* dst has aligned addr */ +$mis_a_dst: + and $17, 7, $1 + +$mis_a_dst_loop: + ldl_u $4, 0($17) + ldl_u $5, 7($17) + extll $4, $1, $4 + exthl $5, $1, $5 + bis $4, $5, $4 + cmpgeb $31, $4, $5 + bne $5, $tail_loop # we find null + subl $18, 8, $5 + blt $5, $tail_loop # we have fewer than 8 bytes to copy + stl $4, 0($16) + subl $18, 8, $18 + beq $5, $out + addl $17, 8, $17 + addl $16, 8, $16 + br $31, $mis_a_dst_loop + +/* we have find null in the last few bytes, copy one byte each time */ +$tail_loop: + ldbu $4, 0($17) + stb $4, 0($16) + subl $18, 1, $18 + beq $18, $out + beq $4, $null_padding # we have reached null + addl $17, 1, $17 + addl $16, 1, $16 + br $31, $tail_loop + +$null_padding: + addl $16, 1, $16 + subl $18, 1, $18 + stb $31, 0($16) + beq $18, $out + br $31, $null_padding + +/* copy is done, return */ +$out: + ret + + .end strncpy + EXPORT_SYMBOL(strncpy) diff --git a/arch/sw_64/lib/uaccess_flushcache.c b/arch/sw_64/lib/uaccess_flushcache.c new file mode 100644 index 000000000000..353d5ac15248 --- /dev/null +++ b/arch/sw_64/lib/uaccess_flushcache.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include + +void memcpy_flushcache(void *dst, const void *src, size_t cnt) +{ + memcpy(dst, src, cnt); + flush_cache_all(); +} +EXPORT_SYMBOL_GPL(memcpy_flushcache); + +void memcpy_page_flushcache(char *to, struct page *page, size_t offset, + size_t len) +{ + memcpy_flushcache(to, page_address(page) + offset, len); +} + +unsigned long __copy_user_flushcache(void *to, const void __user *from, + unsigned long n) +{ + unsigned long rc = __copy_from_user(to, from, n); + + flush_cache_all(); + return rc; +} + +#ifdef CONFIG_ARCH_HAS_PMEM_API +void arch_wb_cache_pmem(void *addr, size_t size) +{ + flush_cache_all(); +} +EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); + +void arch_invalidate_pmem(void *addr, size_t size) +{ + flush_cache_all(); +} +EXPORT_SYMBOL_GPL(arch_invalidate_pmem); +#endif diff --git a/arch/sw_64/lib/udelay.c b/arch/sw_64/lib/udelay.c new file mode 100644 index 000000000000..59ca8a97d748 --- /dev/null +++ b/arch/sw_64/lib/udelay.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 1993, 2000 Linus Torvalds + * + * Delay routines, using a pre-computed "loops_per_jiffy" value. + */ + +#include + +/* + * Use only for very small delays (< 1 msec). + * + * The active part of our cycle counter is only 32-bits wide, and + * we're treating the difference between two marks as signed. On + * a 1GHz box, that's about 2 seconds. + */ +void __delay(unsigned long loops) +{ + unsigned long tmp; + + __asm__ __volatile__( + " rtc %0\n" + " addl %1,%0,%1\n" + "1: rtc %0\n" + " subl %1,%0,%0\n" + " bgt %0,1b" + : "=&r" (tmp), "=r" (loops) : "1"(loops)); +} +EXPORT_SYMBOL(__delay); + +void udelay(unsigned long usecs) +{ + unsigned long loops = usecs * get_cpu_freq() / 1000000; + unsigned long tmp; + + __asm__ __volatile__( + " rtc %0\n" + " addl %1,%0,%1\n" + "1: rtc %0\n" + " subl %1,%0,%0\n" + " bgt %0,1b" + : "=&r" (tmp), "=r" (loops) : "1"(loops)); +} +EXPORT_SYMBOL(udelay); + +void ndelay(unsigned long nsecs) +{ + unsigned long loops = nsecs * get_cpu_freq() / 1000000000; + unsigned long tmp; + + __asm__ __volatile__( + " rtc %0\n" + " addl %1,%0,%1\n" + "1: rtc %0\n" + " subl %1,%0,%0\n" + " bgt %0,1b" + : "=&r" (tmp), "=r" (loops) : "1"(loops)); +} +EXPORT_SYMBOL(ndelay); -- Gitee From 2b45317b0715d26c1d78aa4fbb19032d8ca5ca0c Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:39 +0800 Subject: [PATCH 024/524] sw64: add VDSO support Add VDSO support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/vdso.h | 116 ++++++++++++++ arch/sw_64/kernel/vdso.c | 143 ++++++++++++++++++ arch/sw_64/kernel/vdso/.gitignore | 4 + arch/sw_64/kernel/vdso/Makefile | 74 +++++++++ arch/sw_64/kernel/vdso/so2s.sh | 4 + arch/sw_64/kernel/vdso/vdso.S | 30 ++++ arch/sw_64/kernel/vdso/vdso.lds.S | 89 +++++++++++ arch/sw_64/kernel/vdso/vgettimeofday.c | 201 +++++++++++++++++++++++++ arch/sw_64/kernel/vdso/vrt_sigreturn.S | 68 +++++++++ 9 files changed, 729 insertions(+) create mode 100644 arch/sw_64/include/asm/vdso.h create mode 100644 arch/sw_64/kernel/vdso.c create mode 100644 arch/sw_64/kernel/vdso/.gitignore create mode 100644 arch/sw_64/kernel/vdso/Makefile create mode 100755 arch/sw_64/kernel/vdso/so2s.sh create mode 100644 arch/sw_64/kernel/vdso/vdso.S create mode 100644 arch/sw_64/kernel/vdso/vdso.lds.S create mode 100644 arch/sw_64/kernel/vdso/vgettimeofday.c create mode 100644 arch/sw_64/kernel/vdso/vrt_sigreturn.S diff --git a/arch/sw_64/include/asm/vdso.h b/arch/sw_64/include/asm/vdso.h new file mode 100644 index 000000000000..7a2e23c648f3 --- /dev/null +++ b/arch/sw_64/include/asm/vdso.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 SW64 Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef _ASM_SW64_VDSO_H +#define _ASM_SW64_VDSO_H + +#ifdef __KERNEL__ + +/* + * Default link address for the vDSO. + * Since we randomise the VDSO mapping, there's little point in trying + * to prelink this. + */ +#define VDSO_LBASE 0x0 + +#ifndef __ASSEMBLY__ + +#include +#include +#include +#define VDSO_SYMBOL(base, name) \ +({ \ + extern const unsigned long __vdso_##name; \ + ((unsigned long)(base) + __vdso_##name); \ +}) + + +struct vdso_data { + u64 xtime_sec; + u64 xtime_nsec; + u64 wall_to_mono_sec; + u64 wall_to_mono_nsec; + u32 cs_shift; + u32 cs_mult; + u64 cs_cycle_last; + u64 cs_mask; + s32 tz_minuteswest; + s32 tz_dsttime; + u32 seq_count; +}; + +static inline unsigned long get_vdso_base(void) +{ + unsigned long addr, tmp; + __asm__ __volatile__( + " br %1, 1f\n" + "1: ldi %0, 0(%1)\n" + : "=r" (addr), "=&r" (tmp) + ::); + + addr &= ~(PAGE_SIZE - 1); + return addr; +} + +static inline const struct vdso_data *get_vdso_data(void) +{ + return (const struct vdso_data *)(get_vdso_base() - PAGE_SIZE); +} + +static inline u32 vdso_data_read_begin(const struct vdso_data *data) +{ + u32 seq; + + while (true) { + seq = READ_ONCE(data->seq_count); + if (likely(!(seq & 1))) { + /* Paired with smp_wmb() in vdso_data_write_*(). */ + smp_rmb(); + return seq; + } + + cpu_relax(); + } +} + +static inline bool vdso_data_read_retry(const struct vdso_data *data, + u32 start_seq) +{ + /* Paired with smp_wmb() in vdso_data_write_*(). */ + smp_rmb(); + return unlikely(data->seq_count != start_seq); +} + +static inline void vdso_data_write_begin(struct vdso_data *data) +{ + ++data->seq_count; + + /* Ensure sequence update is written before other data page values. */ + smp_wmb(); +} + +static inline void vdso_data_write_end(struct vdso_data *data) +{ + /* Ensure data values are written before updating sequence again. */ + smp_wmb(); + ++data->seq_count; +} + + +#endif /* !__ASSEMBLY__ */ + +#endif /* __KERNEL__ */ +#endif /* _ASM_SW64_VDSO_H */ diff --git a/arch/sw_64/kernel/vdso.c b/arch/sw_64/kernel/vdso.c new file mode 100644 index 000000000000..b4126cbaa4bd --- /dev/null +++ b/arch/sw_64/kernel/vdso.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +#include +#include +#include +#include + +#include + +extern char vdso_start, vdso_end; +static unsigned long vdso_pages; +static struct page **vdso_pagelist; + +/* + * The vDSO data page. + */ +static union { + struct vdso_data data; + u8 page[PAGE_SIZE]; +} vdso_data_store __page_aligned_data; +struct vdso_data *vdso_data = &vdso_data_store.data; + +static struct vm_special_mapping vdso_spec[2]; + +static int __init vdso_init(void) +{ + int i; + + if (memcmp(&vdso_start, "\177ELF", 4)) { + pr_err("vDSO is not a valid ELF object!\n"); + return -EINVAL; + } + + vdso_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT; + pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n", + vdso_pages + 1, vdso_pages, &vdso_start, 1L, vdso_data); + + /* Allocate the vDSO pagelist, plus a page for the data. */ + vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *), + GFP_KERNEL); + if (vdso_pagelist == NULL) + return -ENOMEM; + + /* Grab the vDSO data page. */ + vdso_pagelist[0] = virt_to_page(vdso_data); + + /* Grab the vDSO code pages. */ + for (i = 0; i < vdso_pages; i++) + vdso_pagelist[i + 1] = virt_to_page(&vdso_start + i * PAGE_SIZE); + + /* Populate the special mapping structures */ + vdso_spec[0] = (struct vm_special_mapping) { + .name = "[vvar]", + .pages = vdso_pagelist, + }; + + vdso_spec[1] = (struct vm_special_mapping) { + .name = "[vdso]", + .pages = &vdso_pagelist[1], + }; + + return 0; +} +arch_initcall(vdso_init); + +int arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp) +{ + struct mm_struct *mm = current->mm; + unsigned long vdso_base, vdso_text_len, vdso_mapping_len; + void *ret; + + vdso_text_len = vdso_pages << PAGE_SHIFT; + /* Be sure to map the data page */ + vdso_mapping_len = vdso_text_len + PAGE_SIZE; + + if (down_write_killable(&mm->mmap_lock)) + return -EINTR; + vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); + if (IS_ERR_VALUE(vdso_base)) { + ret = ERR_PTR(vdso_base); + goto up_fail; + } + ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE, + VM_READ|VM_MAYREAD, + &vdso_spec[0]); + if (IS_ERR(ret)) + goto up_fail; + + vdso_base += PAGE_SIZE; + mm->context.vdso = (void *)vdso_base; + ret = _install_special_mapping(mm, vdso_base, vdso_text_len, + VM_READ|VM_EXEC| + VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, + &vdso_spec[1]); + if (IS_ERR(ret)) + goto up_fail; + + up_write(&mm->mmap_lock); + return 0; + +up_fail: + mm->context.vdso = NULL; + up_write(&mm->mmap_lock); + return PTR_ERR(ret); +} + +void update_vsyscall(struct timekeeper *tk) +{ + vdso_data_write_begin(vdso_data); + + vdso_data->xtime_sec = tk->xtime_sec; + vdso_data->xtime_nsec = tk->tkr_mono.xtime_nsec; + vdso_data->wall_to_mono_sec = tk->wall_to_monotonic.tv_sec; + vdso_data->wall_to_mono_nsec = tk->wall_to_monotonic.tv_nsec; + vdso_data->cs_shift = tk->tkr_mono.shift; + + vdso_data->cs_mult = tk->tkr_mono.mult; + vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last; + vdso_data->cs_mask = tk->tkr_mono.mask; + + vdso_data_write_end(vdso_data); +} + +void update_vsyscall_tz(void) +{ + vdso_data->tz_minuteswest = sys_tz.tz_minuteswest; + vdso_data->tz_dsttime = sys_tz.tz_dsttime; +} diff --git a/arch/sw_64/kernel/vdso/.gitignore b/arch/sw_64/kernel/vdso/.gitignore new file mode 100644 index 000000000000..2b6a8b0ed7ca --- /dev/null +++ b/arch/sw_64/kernel/vdso/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +vdso.lds +vdso.so.dbg.tmp +vdso-syms.S diff --git a/arch/sw_64/kernel/vdso/Makefile b/arch/sw_64/kernel/vdso/Makefile new file mode 100644 index 000000000000..190cc345dbb9 --- /dev/null +++ b/arch/sw_64/kernel/vdso/Makefile @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: GPL-2.0 +# Symbols present in the vdso +vdso-syms = rt_sigreturn gettimeofday + +# Files to link into the vdso +obj-vdso = $(patsubst %, v%.o, $(vdso-syms)) + +# Build rules +targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds vdso-syms.S +obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) + +obj-y += vdso.o vdso-syms.o +extra-y += vdso.lds +CPPFLAGS_vdso.lds += -P -C -U$(ARCH) + +# vDSO code runs in userspace and -pg doesn't help with profiling anyway. +CFLAGS_REMOVE_vdso.o = -pg +CFLAGS_REMOVE_vrt_sigreturn.o = -pg +CFLAGS_REMOVE_vgettimeofday.o = -pg + +ifdef CONFIG_FEEDBACK_COLLECT +# vDSO code runs in userspace, not collecting feedback data. +CFLAGS_REMOVE_vdso.o = -ffeedback-generate +CFLAGS_REMOVE_vrt_sigreturn.o = -ffeedback-generate +CFLAGS_REMOVE_vgettimeofday.o = -ffeedback-generate +endif + +# Disable gcov profiling for VDSO code +GCOV_PROFILE := n + +# Force dependency +$(obj)/vdso.o: $(obj)/vdso.so + +# link rule for the .so file, .lds has to be first +SYSCFLAGS_vdso.so.dbg = $(c_flags) +$(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) FORCE + $(call if_changed,vdsold) +SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ + $(call cc-ldoption, -Wl$(comma)--hash-style=both) + +$(obj)/vdso-syms.S: $(obj)/vdso.so FORCE + $(call if_changed,so2s) + +# strip rule for the .so file +$(obj)/%.so: OBJCOPYFLAGS := -S +$(obj)/%.so: $(obj)/%.so.dbg FORCE + $(call if_changed,objcopy) + +# actual build commands +# The DSO images are built using a special linker script +# Add -lgcc so tilepro gets static muldi3 and lshrdi3 definitions. +# Make sure only to export the intended __vdso_xxx symbol offsets. +quiet_cmd_vdsold = VDSOLD $@ + cmd_vdsold = $(CC) $(KCFLAGS) -nostdlib $(SYSCFLAGS_$(@F)) \ + -Wl,-T,$(filter-out FORCE,$^) -o $@.tmp -lgcc && \ + $(CROSS_COMPILE)objcopy \ + $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ + rm $@.tmp + +# Extracts symbol offsets from the VDSO, converting them into an assembly file +# that contains the same symbols at the same offsets. +quiet_cmd_so2s = SO2S $@ + cmd_so2s = $(NM) -D $< | $(srctree)/$(src)/so2s.sh > $@ + +# install commands for the unstripped file +quiet_cmd_vdso_install = INSTALL $@ + cmd_vdso_install = cp $(obj)/$@.dbg $(MODLIB)/vdso/$@ + +vdso.so: $(obj)/vdso.so.dbg + @mkdir -p $(MODLIB)/vdso + $(call cmd,vdso_install) + + +vdso_install: vdso.so diff --git a/arch/sw_64/kernel/vdso/so2s.sh b/arch/sw_64/kernel/vdso/so2s.sh new file mode 100755 index 000000000000..e1763af8e730 --- /dev/null +++ b/arch/sw_64/kernel/vdso/so2s.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ + +grep "__vdso_" | sed 's/\([0-9a-f]*\) T \([a-z0-9_]*\)\(@@LINUX_.*\)*/.globl\t\2\n\2:\n.quad\t0x\1/' diff --git a/arch/sw_64/kernel/vdso/vdso.S b/arch/sw_64/kernel/vdso/vdso.S new file mode 100644 index 000000000000..edd9be27db9d --- /dev/null +++ b/arch/sw_64/kernel/vdso/vdso.S @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +#include +#include + + __PAGE_ALIGNED_DATA + + .globl vdso_start, vdso_end + .balign PAGE_SIZE +vdso_start: + .incbin "arch/sw_64/kernel/vdso/vdso.so" + .balign PAGE_SIZE +vdso_end: + + .previous diff --git a/arch/sw_64/kernel/vdso/vdso.lds.S b/arch/sw_64/kernel/vdso/vdso.lds.S new file mode 100644 index 000000000000..de1782ccb7b6 --- /dev/null +++ b/arch/sw_64/kernel/vdso/vdso.lds.S @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * GNU linker script for the VDSO library. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Heavily based on the vDSO linker scripts for other archs. + */ + +#include +#include +#include + +OUTPUT_FORMAT("elf64-sw_64") +OUTPUT_ARCH(sw_64) + +SECTIONS +{ + PROVIDE(_vdso_data = . - PAGE_SIZE); + . = VDSO_LBASE + SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + + . = ALIGN(16); + .text : { *(.text*) } + PROVIDE (__etext = .); + PROVIDE (_etext = .); + PROVIDE (etext = .); + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + + .dynamic : { *(.dynamic) } :text :dynamic + + .rodata : { *(.rodata*) } :text + + _end = .; + PROVIDE(end = .); + + /DISCARD/ : { + *(.note.GNU-stack) + *(.data .data.* .gnu.linkonce.d.* .sdata*) + *(.bss .sbss .dynbss .dynsbss) + } +} + +/* + * We must supply the ELF program headers explicitly to get just one + * PT_LOAD segment, and set the flags explicitly to make segments read-only. + */ +PHDRS +{ + text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + +/* + * This controls what symbols we export from the DSO. + */ +VERSION +{ + LINUX_2.6 { + global: + __vdso_rt_sigreturn; + __vdso_gettimeofday; + __vdso_clock_gettime; + local: *; + }; +} diff --git a/arch/sw_64/kernel/vdso/vgettimeofday.c b/arch/sw_64/kernel/vdso/vgettimeofday.c new file mode 100644 index 000000000000..0aa16e988e88 --- /dev/null +++ b/arch/sw_64/kernel/vdso/vgettimeofday.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or + * NON INFRINGEMENT. See the GNU General Public License for + * more details. + */ + +#include + +#include +#include +#include +#include + +static __always_inline int syscall_fallback(clockid_t clkid, struct timespec64 *ts) +{ + register int r0 asm("$0"); + register unsigned long r19 asm("$19"); + asm volatile( + " mov %0, $16\n" + " mov %1, $17\n" + " ldi $0, %2\n" + " sys_call %3\n" + :: "r"(clkid), "r"(ts), "i"(__NR_clock_gettime), "i"(HMC_callsys) + : "$0", "$16", "$17", "$19"); + if (unlikely(r19)) + return -r0; + else + return r0; +} + +static __always_inline int do_realtime_coarse(struct timespec64 *ts, + const struct vdso_data *data) +{ + u32 start_seq; + + do { + start_seq = vdso_data_read_begin(data); + + ts->tv_sec = data->xtime_sec; + ts->tv_nsec = data->xtime_nsec >> data->cs_shift; + } while (vdso_data_read_retry(data, start_seq)); + + return 0; +} + + +static __always_inline int do_monotonic_coarse(struct timespec64 *ts, + const struct vdso_data *data) +{ + u32 start_seq; + u64 to_mono_sec; + u64 to_mono_nsec; + + do { + start_seq = vdso_data_read_begin(data); + + ts->tv_sec = data->xtime_sec; + ts->tv_nsec = data->xtime_nsec >> data->cs_shift; + + to_mono_sec = data->wall_to_mono_sec; + to_mono_nsec = data->wall_to_mono_nsec; + } while (vdso_data_read_retry(data, start_seq)); + + ts->tv_sec += to_mono_sec; + timespec64_add_ns(ts, to_mono_nsec); + + return 0; +} + +#if defined(CONFIG_SUBARCH_C3B) +static __always_inline u64 read_longtime(void) +{ + register unsigned long __r0 __asm__("$0"); + + __asm__ __volatile__( + "sys_call %1" : "=r"(__r0) : "i" (HMC_longtime)); + + return __r0; +} +#elif defined(CONFIG_SUBARCH_C4) +static __always_inline u64 read_longtime(void) +{ + return read_csr(CSR_SHTCLOCK); +} +#endif + +static __always_inline u64 get_ns(const struct vdso_data *data) +{ + u64 cycle_now, delta, nsec; + + cycle_now = read_longtime(); + delta = (cycle_now - data->cs_cycle_last) & data->cs_mask; + + nsec = (delta * data->cs_mult) + data->xtime_nsec; + nsec >>= data->cs_shift; + + return nsec; +} + + +static __always_inline int do_realtime(struct timespec64 *ts, + const struct vdso_data *data) +{ + u32 start_seq; + u64 ns; + + do { + start_seq = vdso_data_read_begin(data); + + ts->tv_sec = data->xtime_sec; + ns = get_ns(data); + } while (vdso_data_read_retry(data, start_seq)); + + ts->tv_nsec = 0; + timespec64_add_ns(ts, ns); + + return 0; +} + +static __always_inline int do_monotonic(struct timespec64 *ts, + const struct vdso_data *data) +{ + u32 start_seq; + u64 ns; + u64 to_mono_sec; + u64 to_mono_nsec; + + do { + start_seq = vdso_data_read_begin(data); + + ts->tv_sec = data->xtime_sec; + ns = get_ns(data); + + to_mono_sec = data->wall_to_mono_sec; + to_mono_nsec = data->wall_to_mono_nsec; + } while (vdso_data_read_retry(data, start_seq)); + + ts->tv_sec += to_mono_sec; + ts->tv_nsec = 0; + timespec64_add_ns(ts, ns + to_mono_nsec); + + return 0; +} + + +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + const struct vdso_data *data = get_vdso_data(); + struct timespec64 ts; + int ret; + + ret = do_realtime(&ts, data); + if (ret) + return ret; + + if (tv) { + tv->tv_sec = ts.tv_sec; + tv->tv_usec = ts.tv_nsec / 1000; + } + + if (tz) { + tz->tz_minuteswest = data->tz_minuteswest; + tz->tz_dsttime = data->tz_dsttime; + } + + return 0; +} + +int __vdso_clock_gettime(clockid_t clkid, struct timespec64 *ts) +{ + const struct vdso_data *data = get_vdso_data(); + int ret; + + switch (clkid) { + case CLOCK_REALTIME_COARSE: + ret = do_realtime_coarse(ts, data); + break; + case CLOCK_MONOTONIC_COARSE: + ret = do_monotonic_coarse(ts, data); + break; + case CLOCK_REALTIME: + ret = do_realtime(ts, data); + break; + case CLOCK_MONOTONIC: + ret = do_monotonic(ts, data); + break; + default: + /* fall back to a syscall */ + ret = syscall_fallback(clkid, ts); + } + + return ret; +} diff --git a/arch/sw_64/kernel/vdso/vrt_sigreturn.S b/arch/sw_64/kernel/vdso/vrt_sigreturn.S new file mode 100644 index 000000000000..cdbf6501ad64 --- /dev/null +++ b/arch/sw_64/kernel/vdso/vrt_sigreturn.S @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Sigreturn trampoline for returning from a signal when the SA_RESTORER + * flag is not set. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +#include +#include +#include +#include + + .text + + .macro SIGCONTEXT_REGS_I base, from = 0 + .cfi_offset \from, \base + (4 + \from) * 8 + .if 30 - \from + SIGCONTEXT_REGS_I \base, "(\from + 1)" + .endif + .endm + + .macro SIGCONTEXT_REGS_F base, from = 32 + .cfi_offset \from, \base + (4 + 32 + 1) * 8 + (\from - 32) * 32 + .if 62 - \from + SIGCONTEXT_REGS_F \base, "(\from + 1)" + .endif + .endm + + .macro SIGCONTEXT_REGS_V base, from = 67 + .cfi_offset \from, \base + (4 + 32 + 1) * 8 + ((\from - 67) & 0x1f) * 32 + (((\from - 67) >> 5) + 1) * 8 + .if 161 - \from + SIGCONTEXT_REGS_V \base, "(\from + 1)" + .endif + .endm + + .macro SIGCONTEXT_REGS base + SIGCONTEXT_REGS_I \base + SIGCONTEXT_REGS_F \base + SIGCONTEXT_REGS_V \base + .cfi_offset 63, \base + (4 + 32 + 1) * 8 + 32 * 32 + .cfi_offset 64, \base + 2 * 8 + .endm + + .cfi_startproc + .cfi_return_column 64 + .cfi_signal_frame + SIGCONTEXT_REGS (-RT_SIGFRAME_SIZE + RT_SIGFRAME_MCTX) + .cfi_def_cfa_offset RT_SIGFRAME_SIZE + + nop +ENTRY(__vdso_rt_sigreturn) + mov $sp, $16 + ldi $0, __NR_rt_sigreturn + sys_call HMC_callsys +ENDPROC(__vdso_rt_sigreturn) + .cfi_endproc -- Gitee From 7c9a29e523bd1ce176a813451ffe2e1ba5773933 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:35 +0800 Subject: [PATCH 025/524] sw64: add SMP support Add Symmetric Multi-Processing (SMP) support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/smp.h | 95 ++++++ arch/sw_64/kernel/smp.c | 578 +++++++++++++++++++++++++++++++++++ 2 files changed, 673 insertions(+) create mode 100644 arch/sw_64/include/asm/smp.h create mode 100644 arch/sw_64/kernel/smp.c diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h new file mode 100644 index 000000000000..3a2fcf62b30c --- /dev/null +++ b/arch/sw_64/include/asm/smp.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SMP_H +#define _ASM_SW64_SMP_H + +#include +#include +#include +#include +#include + +#include +#include +#include + +/* HACK: Cabrio WHAMI return value is bogus if more than 8 bits used.. :-( */ + +extern cpumask_t core_start; + +static inline unsigned long +read_vpcr(void) +{ + register unsigned long __r0 __asm__("$0"); + __asm__ __volatile__( + "sys_call %1 #rvpcr" + : "=r"(__r0) + : "i" (0x39) + : "$1", "$22", "$23", "$24", "$25"); + return __r0; +} + +#ifdef CONFIG_SMP +/* SMP initialization hook for setup_arch */ +void __init setup_smp(void); + +#include + +/* smp reset control block */ +struct smp_rcb_struct { + void (*restart_entry)(unsigned long args); + unsigned long restart_args; + unsigned long ready; + unsigned long init_done; +}; + +#define INIT_SMP_RCB ((struct smp_rcb_struct *) __va(0x820000UL)) + + +#ifdef GENERATING_ASM_OFFSETS +#define raw_smp_processor_id() (0) +#else +#include +#define raw_smp_processor_id() (*((unsigned int *)((void *)current + TASK_CPU))) +#endif +#define hard_smp_processor_id() cpu_to_rcid(raw_smp_processor_id()) + +/* The map from sequential logical cpu number to hard cid. */ +extern int __cpu_to_rcid[NR_CPUS]; +#define cpu_to_rcid(cpu) __cpu_to_rcid[cpu] +#define cpu_physical_id(cpu) __cpu_to_rcid[cpu] + +extern unsigned long tidle_pcb[NR_CPUS]; +extern void arch_send_call_function_single_ipi(int cpu); +extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); + +#ifdef CONFIG_HOTPLUG_CPU +int __cpu_disable(void); +void __cpu_die(unsigned int cpu); +#endif /* CONFIG_HOTPLUG_CPU */ + +#else /* CONFIG_SMP */ +#define hard_smp_processor_id() 0 +#define smp_call_function_on_cpu(func, info, wait, cpu) ({ 0; }) +/* The map from sequential logical cpu number to hard cid. */ +extern int __cpu_to_rcid[NR_CPUS]; +#define cpu_to_rcid(cpu) __cpu_to_rcid[0] +#define cpu_physical_id(cpu) __cpu_to_rcid[0] +#endif /* CONFIG_SMP */ + +#define NO_PROC_ID (-1) + +static inline void send_ipi(int cpu, unsigned long type) +{ + int rcid; + + rcid = cpu_to_rcid(cpu); + + if (is_in_guest()) + hcall(HCALL_IVI, rcid, type, 0); + else + sendii(rcid, type, 0); +} + +#define reset_cpu(cpu) send_ipi((cpu), II_RESET) + +#endif /* _ASM_SW64_SMP_H */ diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c new file mode 100644 index 000000000000..6d1aab4be1c0 --- /dev/null +++ b/arch/sw_64/kernel/smp.c @@ -0,0 +1,578 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/arch/sw_64/kernel/smp.c + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "proto.h" + +struct smp_rcb_struct *smp_rcb; + +extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; + +int smp_booted; + +void *idle_task_pointer[NR_CPUS]; + +/* State of each CPU */ +DEFINE_PER_CPU(int, cpu_state) = { 0 }; + +/* A collection of single bit ipi messages. */ +static struct { + unsigned long bits ____cacheline_aligned; +} ipi_data[NR_CPUS] __cacheline_aligned; + +enum ipi_message_type { + IPI_RESCHEDULE, + IPI_CALL_FUNC, + IPI_CPU_STOP, +}; + +int smp_num_cpus = 1; /* Number that came online. */ +EXPORT_SYMBOL(smp_num_cpus); + +#define send_sleep_interrupt(cpu) send_ipi((cpu), II_SLEEP) +#define send_wakeup_interrupt(cpu) send_ipi((cpu), II_WAKE) + +/* + * Where secondaries begin a life of C. + */ +void smp_callin(void) +{ + int cpuid = smp_processor_id(); + + local_irq_disable(); + + if (cpu_online(cpuid)) { + pr_err("??, cpu 0x%x already present??\n", cpuid); + BUG(); + } + set_cpu_online(cpuid, true); + + /* clear ksp, usp */ + wrksp(0); + wrusp(0); + + /* Set trap vectors. */ + trap_init(); + + /* Set interrupt vector. */ + if (is_in_host()) { + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); + } + wrent(entInt, 0); + + /* Get our local ticker going. */ + sw64_setup_timer(); + + /* All kernel threads share the same mm context. */ + mmgrab(&init_mm); + current->active_mm = &init_mm; + /* update csr:ptbr */ + update_ptbr_sys(virt_to_phys(init_mm.pgd)); + + /* inform the notifiers about the new cpu */ + notify_cpu_starting(cpuid); + + per_cpu(cpu_state, cpuid) = CPU_ONLINE; + per_cpu(hard_node_id, cpuid) = rcid_to_domain_id(cpu_to_rcid(cpuid)); + + /* Must have completely accurate bogos. */ + local_irq_enable(); + + /* Cpu0 init preempt_count at start_kernel, other smp cpus do here. */ + preempt_disable(); + + cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); +} + + +/* + * Set ready for secondary cpu. + */ +static inline void set_secondary_ready(int cpuid) +{ + smp_rcb->ready = cpuid; +} + +/* + * Convince the hmcode to have a secondary cpu begin execution. + */ +static int secondary_cpu_start(int cpuid, struct task_struct *idle) +{ + unsigned long timeout; + /* + * Precalculate the target ksp. + */ + idle_task_pointer[cpuid] = idle; + + set_cpu_online(cpuid, false); + wmb(); + + set_secondary_ready(cpuid); + + /* Wait 10 seconds for secondary cpu. */ + timeout = jiffies + 10*HZ; + while (time_before(jiffies, timeout)) { + if (cpu_online(cpuid)) + goto started; + udelay(10); + barrier(); + } + pr_err("SMP: Processor %d failed to start.\n", cpuid); + return -1; + +started: + store_cpu_topology(cpuid); + numa_add_cpu(cpuid); + return 0; +} + +/* + * Bring one cpu online. + */ +static int smp_boot_one_cpu(int cpuid, struct task_struct *idle) +{ + per_cpu(cpu_state, cpuid) = CPU_UP_PREPARE; + + return secondary_cpu_start(cpuid, idle); +} + +static void __init process_nr_cpu_ids(void) +{ + int i; + + for (i = nr_cpu_ids; i < NR_CPUS; i++) { + set_cpu_possible(i, false); + set_cpu_present(i, false); + } + + nr_cpu_ids = num_possible_cpus(); +} + +void __init smp_rcb_init(void) +{ + smp_rcb = INIT_SMP_RCB; + memset(smp_rcb, 0, sizeof(struct smp_rcb_struct)); + /* Setup SMP_RCB fields that uses to activate secondary CPU */ + smp_rcb->restart_entry = __smp_callin; + smp_rcb->init_done = 0xDEADBEEFUL; + mb(); +} + +/* + * Called from setup_arch. Detect an SMP system and which processors + * are present. + */ +void __init setup_smp(void) +{ + int i = 0, num = 0; + + init_cpu_possible(cpu_none_mask); + + /* For unified kernel, NR_CPUS is the maximum possible value */ + for (; i < NR_CPUS; i++) { + if (cpu_to_rcid(i) != -1) { + set_cpu_possible(num, true); + store_cpu_data(num); + if (!cpumask_test_cpu(i, &cpu_offline)) + set_cpu_present(num, true); + num++; + } + } + + process_nr_cpu_ids(); + + pr_info("Detected %u possible CPU(s), %u CPU(s) are present\n", + nr_cpu_ids, num_present_cpus()); + + smp_rcb_init(); +} +/* + * Called by smp_init prepare the secondaries + */ +void __init smp_prepare_cpus(unsigned int max_cpus) +{ + unsigned int cpu; + /* Take care of some initial bookkeeping. */ + memset(ipi_data, 0, sizeof(ipi_data)); + + init_cpu_topology(); + store_cpu_topology(smp_processor_id()); + numa_add_cpu(smp_processor_id()); + + for_each_possible_cpu(cpu) { + numa_store_cpu_info(cpu); + } + + /* Nothing to do on a UP box, or when told not to. */ + if (nr_cpu_ids == 1 || max_cpus == 0) { + init_cpu_possible(cpumask_of(0)); + init_cpu_present(cpumask_of(0)); + pr_info("SMP mode deactivated.\n"); + return; + } + + pr_info("SMP starting up secondaries.\n"); +} + +void smp_prepare_boot_cpu(void) +{ + int me = smp_processor_id(); + + per_cpu(cpu_state, me) = CPU_ONLINE; +} + +int vt_cpu_up(unsigned int cpu, struct task_struct *tidle) +{ + pr_info("%s: cpu = %d\n", __func__, cpu); + + wmb(); + smp_rcb->ready = 0; + if (smp_booted) { + /* irq must be disabled before reset vCPU */ + reset_cpu(cpu); + } + smp_boot_one_cpu(cpu, tidle); + + return cpu_online(cpu) ? 0 : -EIO; +} + +#ifdef CONFIG_SUBARCH_C3B +DECLARE_STATIC_KEY_FALSE(use_tc_as_sched_clock); +#endif + +int __cpu_up(unsigned int cpu, struct task_struct *tidle) +{ + if (is_in_guest()) + return vt_cpu_up(cpu, tidle); + + wmb(); + smp_rcb->ready = 0; + + /* send wake up signal */ + send_wakeup_interrupt(cpu); + /* send reset signal */ + if (smp_booted) { + if (is_in_host()) { + reset_cpu(cpu); + } else { + while (1) + cpu_relax(); + } + } + smp_boot_one_cpu(cpu, tidle); + +#ifdef CONFIG_SUBARCH_C3B + if (static_branch_likely(&use_tc_as_sched_clock)) { + if (smp_booted) { + tc_sync_clear(); + smp_call_function_single(cpu, tc_sync_ready, NULL, 0); + tc_sync_set(); + } + } +#endif + + return cpu_online(cpu) ? 0 : -EIO; +} + +void __init smp_cpus_done(unsigned int max_cpus) +{ + smp_booted = 1; + pr_info("SMP: Total of %d processors activated.\n", num_online_cpus()); +} + +int setup_profiling_timer(unsigned int multiplier) +{ + return -EINVAL; +} + + +static void send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation) +{ + int i; + + mb(); + for_each_cpu(i, to_whom) + set_bit(operation, &ipi_data[i].bits); + + mb(); + for_each_cpu(i, to_whom) + send_ipi(i, II_II0); +} + +static void ipi_cpu_stop(int cpu) +{ + local_irq_disable(); + set_cpu_online(cpu, false); + while (1) + wait_for_interrupt(); +} + +void handle_ipi(struct pt_regs *regs) +{ + int cpu = smp_processor_id(); + unsigned long *pending_ipis = &ipi_data[cpu].bits; + unsigned long ops; + + mb(); /* Order interrupt and bit testing. */ + while ((ops = xchg(pending_ipis, 0)) != 0) { + mb(); /* Order bit clearing and data access. */ + do { + unsigned long which; + + which = ops & -ops; + ops &= ~which; + which = __ffs(which); + + switch (which) { + case IPI_RESCHEDULE: + scheduler_ipi(); + break; + + case IPI_CALL_FUNC: + irq_enter(); + generic_smp_call_function_interrupt(); + irq_exit(); + break; + + case IPI_CPU_STOP: + ipi_cpu_stop(cpu); + break; + + default: + pr_crit("Unknown IPI on CPU %d: %lu\n", cpu, which); + break; + } + } while (ops); + + mb(); /* Order data access and bit testing. */ + } + + cpu_data[cpu].ipi_count++; +} + +void arch_smp_send_reschedule(int cpu) +{ + send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); +} +EXPORT_SYMBOL(arch_smp_send_reschedule); + +void smp_send_stop(void) +{ + unsigned long timeout; + + if (num_online_cpus() > 1) { + cpumask_t mask; + + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(smp_processor_id(), &mask); + + if (system_state <= SYSTEM_RUNNING) + pr_crit("SMP: stopping secondary CPUs\n"); + send_ipi_message(&mask, IPI_CPU_STOP); + } + + /* Wait up to one second for other CPUs to stop */ + timeout = USEC_PER_SEC; + while (num_online_cpus() > 1 && timeout--) + udelay(1); + + if (num_online_cpus() > 1) + pr_warn("SMP: failed to stop secondary CPUs %*pbl\n", + cpumask_pr_args(cpu_online_mask)); +} + +void arch_send_call_function_ipi_mask(const struct cpumask *mask) +{ + send_ipi_message(mask, IPI_CALL_FUNC); +} + +void arch_send_call_function_single_ipi(int cpu) +{ + send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC); +} + +static void ipi_flush_tlb_all(void *ignored) +{ + local_flush_tlb_all(); +} + +void flush_tlb_all(void) +{ + /* Although we don't have any data to pass, we do want to + * synchronize with the other processors. + */ + on_each_cpu(ipi_flush_tlb_all, NULL, 1); +} + +static void ipi_flush_tlb_mm(void *x) +{ + local_flush_tlb_mm((struct mm_struct *)x); +} + +void flush_tlb_mm(struct mm_struct *mm) +{ + + /* happens as a result of exit_mmap() + * Shall we clear mm->context.asid[] here? + */ + if (atomic_read(&mm->mm_users) == 0) + return; + + preempt_disable(); + + if (atomic_read(&mm->mm_users) != 1 || mm != current->mm) { + on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1); + } else { + int cpu, this_cpu = smp_processor_id(); + + for_each_online_cpu(cpu) { + if (cpu != this_cpu && mm->context.asid[cpu]) + mm->context.asid[cpu] = 0; + } + local_flush_tlb_mm(mm); + } + + preempt_enable(); +} +EXPORT_SYMBOL(flush_tlb_mm); + +struct flush_tlb_info { + struct vm_area_struct *vma; + unsigned long addr; +#define start addr + unsigned long end; +}; + +static void ipi_flush_tlb_page(void *x) +{ + struct flush_tlb_info *info = x; + + local_flush_tlb_page(info->vma, info->addr); +} + +void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) +{ + struct mm_struct *mm = vma->vm_mm; + + preempt_disable(); + + if (atomic_read(&mm->mm_users) != 1 || mm != current->mm) { + struct flush_tlb_info info = { + .vma = vma, + .addr = addr, + }; + on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_page, &info, 1); + } else { + int cpu, this_cpu = smp_processor_id(); + + for_each_online_cpu(cpu) { + if (cpu != this_cpu && mm->context.asid[cpu]) + mm->context.asid[cpu] = 0; + } + local_flush_tlb_page(vma, addr); + } + + preempt_enable(); +} +EXPORT_SYMBOL(flush_tlb_page); + +/* It always flush the whole user tlb by now. To be optimized. */ +void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) +{ + flush_tlb_mm(vma->vm_mm); +} +EXPORT_SYMBOL(flush_tlb_range); + +static void ipi_flush_tlb_kernel_range(void *x) +{ + struct flush_tlb_info *info = x; + + local_flush_tlb_kernel_range(info->start, info->end); +} + +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + struct flush_tlb_info info = { + .start = start, + .end = end, + }; + + on_each_cpu(ipi_flush_tlb_kernel_range, &info, 1); +} +EXPORT_SYMBOL(flush_tlb_kernel_range); + +#ifdef CONFIG_HOTPLUG_CPU +int __cpu_disable(void) +{ + int cpu = smp_processor_id(); + + set_cpu_online(cpu, false); + remove_cpu_topology(cpu); + numa_remove_cpu(cpu); + clear_tasks_mm_cpumask(cpu); + return 0; +} + +void __cpu_die(unsigned int cpu) +{ + /* We don't do anything here: idle task is faking death itself. */ + unsigned int i; + + for (i = 0; i < 10; i++) { + /* They ack this in play_dead by setting CPU_DEAD */ + if (per_cpu(cpu_state, cpu) == CPU_DEAD) { + if (system_state == SYSTEM_RUNNING) + pr_info("CPU %u is now offline\n", cpu); + smp_rcb->ready = 0; + return; + } + msleep(100); + } + pr_err("CPU %u didn't die...\n", cpu); +} + +void arch_cpu_idle_dead(void) +{ + idle_task_exit(); + mb(); + __this_cpu_write(cpu_state, CPU_DEAD); + fixup_irqs(); + local_irq_disable(); + + if (is_in_guest()) { + hcall(HCALL_SET_CLOCKEVENT, 0, 0, 0); + hcall(HCALL_STOP, 0, 0, 0); + } else { + wrtimer(0); + } + +#ifdef CONFIG_SUSPEND + sleepen(); + send_sleep_interrupt(smp_processor_id()); + while (1) + asm("nop"); +#else + asm volatile("memb"); + asm volatile("halt"); +#endif +} +#endif -- Gitee From 2321cc2e188081cc46cd5ebe5b0f1e9af84f6332 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:28 +0800 Subject: [PATCH 026/524] sw64: add NUMA support Add Non Uniform Memory Access (NUMA) support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/numa.h | 36 +++ arch/sw_64/mm/numa.c | 466 ++++++++++++++++++++++++++++++++++ 2 files changed, 502 insertions(+) create mode 100644 arch/sw_64/include/asm/numa.h create mode 100644 arch/sw_64/mm/numa.c diff --git a/arch/sw_64/include/asm/numa.h b/arch/sw_64/include/asm/numa.h new file mode 100644 index 000000000000..a2e3171caff1 --- /dev/null +++ b/arch/sw_64/include/asm/numa.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SW64_NUMA_H +#define _ASM_SW64_NUMA_H + +#include +#include + +#ifdef CONFIG_NUMA +extern nodemask_t numa_nodes_parsed __initdata; +extern int numa_off; + +struct numa_memblk { + u64 start; + u64 end; + int nid; +}; + +#define NR_NODE_MEMBLKS (MAX_NUMNODES*2) +struct numa_meminfo { + int nr_blks; + struct numa_memblk blk[NR_NODE_MEMBLKS]; +}; +extern int __init numa_add_memblk(int nodeid, u64 start, u64 end); +extern void numa_clear_node(unsigned int cpu); +extern void __init numa_set_distance(int from, int to, int distance); +extern void __init early_map_cpu_to_node(unsigned int cpu, int nid); + +#else /* CONFIG_NUMA */ + +static inline void numa_clear_node(unsigned int cpu) { } +static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { } + +#endif /* CONFIG_NUMA */ + +#endif /* _ASM_SW64_NUMA_H */ diff --git a/arch/sw_64/mm/numa.c b/arch/sw_64/mm/numa.c new file mode 100644 index 000000000000..fcf1f97a7840 --- /dev/null +++ b/arch/sw_64/mm/numa.c @@ -0,0 +1,466 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DISCONTIGMEM NUMA sw64 support. + */ + +#include +#include +#include +#include + +#include + +int cpu_to_node_map[NR_CPUS]; +cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; +EXPORT_SYMBOL(node_to_cpumask_map); + +struct numa_node_desc_t numa_nodes_desc[MAX_NUMNODES]; +nodemask_t numa_nodes_parsed __initdata; + +static int numa_distance_cnt; +static u8 *numa_distance; +int numa_off; + +static __init int numa_setup(char *opt) +{ + if (!opt) + return -EINVAL; + if (!strncmp(opt, "off", 3)) + numa_off = 1; + return 0; +} +early_param("numa", numa_setup); + +/* + * Allocate node_to_cpumask_map based on number of available nodes + * Requires node_possible_map to be valid. + * + * Note: cpumask_of_node() is not valid until after this is done. + * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) + */ +static void __init setup_node_to_cpumask_map(void) +{ + int node; + + /* setup nr_node_ids if not done yet */ + if (nr_node_ids == MAX_NUMNODES) + setup_nr_node_ids(); + + /* allocate and clear the mapping */ + for (node = 0; node < nr_node_ids; node++) { + alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); + cpumask_clear(node_to_cpumask_map[node]); + } + + /* cpumask_of_node() will now work */ + pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids); +} + +/** + * numa_add_memblk - Set node id to memblk + * @nid: NUMA node ID of the new memblk + * @start: Start address of the new memblk + * @end: End address of the new memblk + * + * RETURNS: + * 0 on success, -errno on failure. + */ +int __init numa_add_memblk(int nid, u64 start, u64 end) +{ + int ret; + + ret = memblock_set_node(start, (end - start), &memblock.memory, nid); + if (ret < 0) { + pr_err("memblock [0x%llx - 0x%llx] failed to add on node %d\n", + start, (end - 1), nid); + return ret; + } + + node_set(nid, numa_nodes_parsed); + return ret; +} + +/** + * Initialize NODE_DATA for a node on the local memory + */ +static void __init setup_node_data(int nid, unsigned long start_pfn, unsigned long end_pfn) +{ + const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES); + u64 nd_pa; + void *nd; + int tnid; + + if (start_pfn >= end_pfn) + pr_info("Initmem setup node %d []\n", nid); + + nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid); + nd = __va(nd_pa); + + /* report and initialize */ + pr_info("NODE_DATA [mem %#018llx-%#018llx]\n", + nd_pa, nd_pa + nd_size - 1); + tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT); + if (tnid != nid) + pr_info("NODE_DATA(%d) on node %d\n", nid, tnid); + + node_data[nid] = nd; + memset(NODE_DATA(nid), 0, sizeof(pg_data_t)); + NODE_DATA(nid)->node_id = nid; + NODE_DATA(nid)->node_start_pfn = start_pfn; + NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn; +} + +/** + * numa_free_distance + * + * Free current distance table. + */ +void __init numa_free_distance(void) +{ + size_t size; + + if (!numa_distance) + return; + + size = numa_distance_cnt * numa_distance_cnt * + sizeof(numa_distance[0]); + + memblock_free(numa_distance, size); + numa_distance_cnt = 0; + numa_distance = NULL; +} + +/** + * + * Create a new NUMA distance table. + * + */ +static int __init numa_alloc_distance(void) +{ + size_t size; + phys_addr_t phys; + int i, j; + + size = nr_node_ids * nr_node_ids * sizeof(numa_distance[0]); + phys = memblock_phys_alloc(size, PAGE_SIZE); + if (WARN_ON(!phys)) + return -ENOMEM; + + numa_distance = __va(phys); + numa_distance_cnt = nr_node_ids; + + /* fill with the default distances */ + for (i = 0; i < numa_distance_cnt; i++) + for (j = 0; j < numa_distance_cnt; j++) { + numa_distance[i * numa_distance_cnt + j] = i == j ? + LOCAL_DISTANCE : REMOTE_DISTANCE; + } + + pr_info("Initialized distance table, cnt=%d\n", numa_distance_cnt); + + return 0; +} + +/** + * numa_set_distance - Set inter node NUMA distance from node to node. + * @from: the 'from' node to set distance + * @to: the 'to' node to set distance + * @distance: NUMA distance + * + * Set the distance from node @from to @to to @distance. + * If distance table doesn't exist, a warning is printed. + * + * If @from or @to is higher than the highest known node or lower than zero + * or @distance doesn't make sense, the call is ignored. + * + */ +void __init numa_set_distance(int from, int to, int distance) +{ + if (!numa_distance) { + pr_warn_once("Warning: distance table not allocated yet\n"); + return; + } + + if (from >= numa_distance_cnt || to >= numa_distance_cnt || + from < 0 || to < 0) { + pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n", + from, to, distance); + return; + } + + if ((u8)distance != distance || + (from == to && distance != LOCAL_DISTANCE)) { + pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n", + from, to, distance); + return; + } + + numa_distance[from * numa_distance_cnt + to] = distance; +} + +/** + * Return NUMA distance @from to @to + */ +int __node_distance(int from, int to) +{ + if (from >= numa_distance_cnt || to >= numa_distance_cnt) + return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE; + return numa_distance[from * numa_distance_cnt + to]; +} +EXPORT_SYMBOL(__node_distance); + +static int __init numa_register_nodes(void) +{ + int nid; + struct memblock_region *mblk; + + /* Check that valid nid is set to memblks */ + for_each_mem_region(mblk) { + pr_info("memblk node %d [mem %#018llx-%#018llx]\n", + mblk->nid, mblk->base, + mblk->base + mblk->size - 1); + if (mblk->nid == NUMA_NO_NODE || mblk->nid >= MAX_NUMNODES) { + pr_warn("Warning: invalid memblk node %d [mem %#018llx-%#018llx]\n", + mblk->nid, mblk->base, + mblk->base + mblk->size - 1); + return -EINVAL; + } + } + + /* Finally register nodes */ + for_each_node_mask(nid, numa_nodes_parsed) { + unsigned long start_pfn, end_pfn; + + get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); + setup_node_data(nid, start_pfn, end_pfn); + node_set_online(nid); + } + + /* Setup online nodes to actual nodes */ + node_possible_map = numa_nodes_parsed; + + return 0; +} + +static int __init numa_init(int (*init_func)(void)) +{ + int ret; + + nodes_clear(numa_nodes_parsed); + nodes_clear(node_possible_map); + nodes_clear(node_online_map); + numa_free_distance(); + + ret = numa_alloc_distance(); + if (ret < 0) + return ret; + + ret = init_func(); + if (ret < 0) + return ret; + + if (nodes_empty(numa_nodes_parsed)) { + pr_info("No NUMA configuration found\n"); + return -EINVAL; + } + + ret = numa_register_nodes(); + if (ret < 0) + return ret; + + setup_node_to_cpumask_map(); + + return 0; +} + +static void __init get_numa_info_socket(void) +{ + int i; + + phys_addr_t base = 0; + + for (i = 0; i < MAX_NUMSOCKETS; i++) { + if (socket_desc[i].is_online) { + numa_nodes_desc[i].base = base; + numa_nodes_desc[i].size = socket_desc[i].socket_mem; + base += numa_nodes_desc[i].size; + } + } +} + +static int __init manual_numa_init(void) +{ + int ret, nid; + struct memblock_region *mblk; + phys_addr_t node_base, node_size, node_end; + + if (numa_off) { + pr_info("NUMA disabled\n"); /* Forced off on command line. */ + pr_info("Faking one node at [mem %#018llx-%#018llx]\n", + memblock_start_of_DRAM(), memblock_end_of_DRAM() - 1); + for_each_mem_region(mblk) { + ret = numa_add_memblk(0, mblk->base, mblk->base + mblk->size); + if (!ret) + continue; + + pr_err("NUMA init failed\n"); + return ret; + } + } else { + get_numa_info_socket(); + + for (nid = 0; nid < MAX_NUMNODES; nid++) { + node_base = numa_nodes_desc[nid].base; + node_size = numa_nodes_desc[nid].size; + node_end = node_base + node_size; + ret = 0; + + if (!node_end) + continue; + + for_each_mem_region(mblk) { + if (mblk->base >= node_base && mblk->base < node_end) { + if (mblk->base + mblk->size < node_end) + ret = numa_add_memblk(nid, mblk->base, mblk->base + mblk->size); + else + ret = numa_add_memblk(nid, mblk->base, node_end); + } + } + + if (!node_size) { + memblock_add_node(node_base, node_size, nid, MEMBLOCK_NONE); + node_set(nid, numa_nodes_parsed); + pr_info("Setup empty node %d from %#llx\n", nid, node_base); + } + + if (!ret) + continue; + + pr_err("NUMA init failed for node %d, [mem %#018llx-%#018llx]", + nid, node_base, node_end - 1); + } + } + + return 0; +} + +void __init sw64_numa_init(void) +{ + if (!numa_off) { + if (!acpi_disabled && !numa_init(acpi_numa_init)) + return; + if (acpi_disabled && !numa_init(of_numa_init)) + return; + } + + numa_init(manual_numa_init); +} + +void cpu_set_node(void) +{ + int i; + + if (numa_off) { + for (i = 0; i < nr_cpu_ids; i++) + cpu_to_node_map[i] = 0; + } else { + int rr, default_node, cid; + + rr = first_node(node_online_map); + for (i = 0; i < nr_cpu_ids; i++) { + cid = cpu_to_rcid(i); + default_node = rcid_to_domain_id(cid); + if (node_online(default_node)) { + cpu_to_node_map[i] = default_node; + } else { + cpu_to_node_map[i] = rr; + rr = next_node(rr, node_online_map); + if (rr == MAX_NUMNODES) + rr = first_node(node_online_map); + } + } + } + /* + * Setup numa_node for cpu 0 before per_cpu area for booting. + * Actual setup of numa_node will be done in native_smp_prepare_cpus(). + */ + set_cpu_numa_node(0, cpu_to_node_map[0]); +} + +void numa_store_cpu_info(unsigned int cpu) +{ + set_cpu_numa_node(cpu, cpu_to_node_map[cpu]); +} + +void __init early_map_cpu_to_node(unsigned int cpu, int nid) +{ + /* fallback to node 0 */ + if (nid < 0 || nid >= MAX_NUMNODES || numa_off) + nid = 0; + + cpu_to_node_map[cpu] = nid; + + /* + * We should set the numa node of cpu0 as soon as possible, because it + * has already been set up online before. cpu_to_node(0) will soon be + * called. + */ + if (!cpu) + set_cpu_numa_node(cpu, nid); +} + +#ifdef CONFIG_DEBUG_PER_CPU_MAPS +/* + * Returns a pointer to the bitmask of CPUs on Node 'node'. + */ +const struct cpumask *cpumask_of_node(int node) +{ + + if (node == NUMA_NO_NODE) { + pr_warn("%s: NUMA_NO_NODE\n", __func__); + return cpu_all_mask; + } + + if (WARN_ON(node < 0 || node >= nr_node_ids)) { + pr_warn("%s: invalid node %d\n", __func__, node); + return cpu_none_mask; + } + + if (WARN_ON(node_to_cpumask_map[node] == NULL)) { + pr_warn("%s: uninitialized node %d\n", __func__, node); + return cpu_online_mask; + } + + return node_to_cpumask_map[node]; +} +EXPORT_SYMBOL(cpumask_of_node); +#endif + +static void numa_update_cpu(unsigned int cpu, bool remove) +{ + int nid = cpu_to_node(cpu); + + if (nid == NUMA_NO_NODE) + return; + + if (remove) + cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]); + else + cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); +} + +void numa_add_cpu(unsigned int cpu) +{ + numa_update_cpu(cpu, false); +} + +void numa_remove_cpu(unsigned int cpu) +{ + numa_update_cpu(cpu, true); +} + +void numa_clear_node(unsigned int cpu) +{ + numa_remove_cpu(cpu); + set_cpu_numa_node(cpu, NUMA_NO_NODE); +} -- Gitee From 8100955e1a377dc461c7b98cede3c4d59de6e51f Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:05 +0800 Subject: [PATCH 027/524] sw64: add default configs Add default config files for SW64 based xuelang and junzhang platforms. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/configs/junzhang_defconfig | 667 +++++++++++++++++++++ arch/sw_64/configs/kata_guest_defconfig | 633 ++++++++++++++++++++ arch/sw_64/configs/kata_xuelang_defconfig | 616 ++++++++++++++++++++ arch/sw_64/configs/xuelang_defconfig | 668 ++++++++++++++++++++++ 4 files changed, 2584 insertions(+) create mode 100644 arch/sw_64/configs/junzhang_defconfig create mode 100644 arch/sw_64/configs/kata_guest_defconfig create mode 100644 arch/sw_64/configs/kata_xuelang_defconfig create mode 100644 arch/sw_64/configs/xuelang_defconfig diff --git a/arch/sw_64/configs/junzhang_defconfig b/arch/sw_64/configs/junzhang_defconfig new file mode 100644 index 000000000000..4f25770ca193 --- /dev/null +++ b/arch/sw_64/configs/junzhang_defconfig @@ -0,0 +1,667 @@ +CONFIG_LOCALVERSION="-junzhang" +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +# CONFIG_CROSS_MEMORY_ATTACH is not set +CONFIG_USELIB=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_BPF_SYSCALL=y +CONFIG_BPF_JIT=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_IKHEADERS=y +CONFIG_LOG_BUF_SHIFT=18 +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_NAMESPACES=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_RELAY=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_EXPERT=y +CONFIG_KALLSYMS_ALL=y +CONFIG_PERF_EVENTS=y +CONFIG_DEBUG_PERF_USE_VMALLOC=y +CONFIG_SUBARCH_C4=y +CONFIG_SMP=y +CONFIG_SCHED_SMT=y +CONFIG_NR_CPUS=64 +CONFIG_ARCH_SPARSEMEM_ENABLE=y +CONFIG_NUMA=y +CONFIG_HZ=100 +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_USE_OF=y +CONFIG_FIRMWARE_MEMMAP=y +CONFIG_DMI_SYSFS=m +CONFIG_ACPI_TAD=y +# CONFIG_CPU_IDLE is not set +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_SCSI=m +CONFIG_VHOST_VSOCK=m +CONFIG_VHOST_CROSS_ENDIAN_LEGACY=y +CONFIG_KPROBES=y +CONFIG_JUMP_LABEL=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_OSF_PARTITION=y +CONFIG_BSD_DISKLABEL=y +CONFIG_MINIX_SUBPARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_LDM_PARTITION=y +CONFIG_SGI_PARTITION=y +CONFIG_ULTRIX_PARTITION=y +# CONFIG_COMPAT_BRK is not set +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_CMA_AREAS=7 +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=y +CONFIG_UNIX=y +CONFIG_UNIX_DIAG=y +CONFIG_TLS=m +CONFIG_TLS_DEVICE=y +CONFIG_XFRM_USER=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_SUB_POLICY=y +CONFIG_XFRM_STATISTICS=y +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_FIB_TRIE_STATS=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_IP_MROUTE=y +CONFIG_NET_IPVTI=m +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_UDP_DIAG=m +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_MD5SIG=y +CONFIG_IPV6=m +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_MIP6=m +CONFIG_IPV6_ILA=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_GRE=m +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_MROUTE=y +CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y +CONFIG_IPV6_PIMSM_V2=y +CONFIG_IPV6_SEG6_LWTUNNEL=y +CONFIG_IPV6_SEG6_HMAC=y +CONFIG_NETFILTER=y +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_OSF=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_L2TP=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_NF_TABLES_IPV4=y +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_LOG_ARP=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BRIDGE=m +CONFIG_VLAN_8021Q=m +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_SKBPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_DEFAULT=y +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=m +CONFIG_NETLINK_DIAG=m +CONFIG_CGROUP_NET_PRIO=y +# CONFIG_WIRELESS is not set +CONFIG_PCI=y +CONFIG_PCIEPORTBUS=y +CONFIG_PCIEAER=y +# CONFIG_PCIEASPM is not set +CONFIG_PCI_MSI=y +CONFIG_PCI_IOV=y +CONFIG_UEVENT_HELPER=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +# CONFIG_STANDALONE is not set +# CONFIG_PREVENT_FIRMWARE_BUILD is not set +CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_JEDECPROBE=y +CONFIG_MTD_CFI_ADV_OPTIONS=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_CFI_STAA=y +CONFIG_MTD_ROM=y +CONFIG_MTD_ABSENT=y +CONFIG_MTD_COMPLEX_MAPPINGS=y +CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_PHYSMAP_OF=y +CONFIG_MTD_PLATRAM=y +CONFIG_MTD_SPI_NOR=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=5000000 +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_FC=y +CONFIG_NVME_TARGET=y +CONFIG_NVME_TARGET_LOOP=y +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_FC=y +CONFIG_NVME_TARGET_FCLOOP=y +CONFIG_RAID_ATTRS=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_ST=y +CONFIG_BLK_DEV_SR=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_SCSI_SPI_ATTRS=y +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_LIBSAS=y +CONFIG_SCSI_SAS_ATA=y +CONFIG_SCSI_SRP_ATTRS=y +CONFIG_ISCSI_TCP=m +CONFIG_ISCSI_BOOT_SYSFS=y +CONFIG_SCSI_CXGB3_ISCSI=m +CONFIG_SCSI_CXGB4_ISCSI=m +CONFIG_SCSI_BNX2_ISCSI=m +CONFIG_MEGARAID_SAS=m +CONFIG_SCSI_MPT3SAS=m +CONFIG_SCSI_DH=y +CONFIG_SCSI_DH_RDAC=y +CONFIG_SCSI_DH_HP_SW=y +CONFIG_SCSI_DH_EMC=y +CONFIG_SCSI_DH_ALUA=y +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +# CONFIG_ATA_SFF is not set +CONFIG_MD=y +CONFIG_MD_LINEAR=m +CONFIG_MD_MULTIPATH=m +CONFIG_MD_FAULTY=m +CONFIG_BCACHE=m +CONFIG_BCACHE_DEBUG=y +CONFIG_BCACHE_CLOSURES_DEBUG=y +CONFIG_BLK_DEV_DM=m +CONFIG_DM_DEBUG=y +CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING=y +CONFIG_DM_DEBUG_BLOCK_STACK_TRACING=y +CONFIG_DM_UNSTRIPED=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_WRITECACHE=m +CONFIG_DM_ERA=m +CONFIG_DM_MIRROR=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_RAID=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_DELAY=m +CONFIG_DM_UEVENT=y +CONFIG_DM_FLAKEY=m +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_SWITCH=m +CONFIG_DM_LOG_WRITES=m +CONFIG_DM_INTEGRITY=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_ISCSI_TARGET=m +CONFIG_NET_FC=y +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_VIRTIO_NET=y +# CONFIG_NET_VENDOR_3COM is not set +# CONFIG_NET_VENDOR_AMD is not set +# CONFIG_NET_VENDOR_ARC is not set +CONFIG_CAVIUM_PTP=y +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_EZCHIP is not set +CONFIG_E100=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_IGB=y +CONFIG_IGBVF=m +CONFIG_IXGBE=m +CONFIG_IXGBEVF=m +CONFIG_I40E=y +CONFIG_I40EVF=y +# CONFIG_NET_VENDOR_MARVELL is not set +CONFIG_MLX4_EN=y +CONFIG_MLX5_CORE=m +CONFIG_MLX5_FPGA=y +CONFIG_MLX5_CORE_EN=y +CONFIG_MLXSW_CORE=y +CONFIG_MLXSW_PCI=y +CONFIG_MLXSW_I2C=y +CONFIG_MLXSW_MINIMAL=y +# CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_QUALCOMM is not set +# CONFIG_NET_VENDOR_RENESAS is not set +# CONFIG_NET_VENDOR_ROCKER is not set +# CONFIG_NET_VENDOR_SAMSUNG is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SMSC is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set +# CONFIG_NET_VENDOR_VIA is not set +# CONFIG_NET_VENDOR_WIZNET is not set +# CONFIG_WLAN is not set +CONFIG_INPUT_FF_MEMLESS=y +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_EVDEV=y +# CONFIG_KEYBOARD_ATKBD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO_SERPORT is not set +CONFIG_SERIO_LIBPS2=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +# CONFIG_SERIAL_8250_PCI is not set +CONFIG_SERIAL_8250_SUNWAY=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_VIRTIO_CONSOLE=y +# CONFIG_HW_RANDOM is not set +# CONFIG_I2C_COMPAT is not set +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_MUX=y +CONFIG_SPI=y +CONFIG_SPI_SPIDEV=y +CONFIG_SENSORS_PVT=y +CONFIG_SENSORS_LM75=y +CONFIG_SSB=y +CONFIG_DRM=y +CONFIG_DRM_RADEON=y +CONFIG_DRM_AST=y +CONFIG_DRM_VIRTIO_GPU=y +CONFIG_LCD_CLASS_DEVICE=y +# CONFIG_VGA_CONSOLE is not set +CONFIG_USB=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_STORAGE=y +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_MLX4_INFINIBAND=m +CONFIG_MLX5_INFINIBAND=m +CONFIG_INFINIBAND_MTHCA=m +# CONFIG_INFINIBAND_MTHCA_DEBUG is not set +CONFIG_INFINIBAND_IPOIB=m +CONFIG_INFINIBAND_IPOIB_CM=y +CONFIG_RTC_CLASS=y +# CONFIG_RTC_NVMEM is not set +# CONFIG_RTC_INTF_PROC is not set +CONFIG_RTC_DRV_PCF8523=y +CONFIG_UIO=y +CONFIG_UIO_PCI_GENERIC=m +CONFIG_VIRTIO_PCI=y +# CONFIG_VIRTIO_PCI_LEGACY is not set +CONFIG_VIRTIO_MMIO=y +CONFIG_STAGING=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_EXT4_DEBUG=y +CONFIG_XFS_FS=y +CONFIG_GFS2_FS=y +CONFIG_FANOTIFY=y +CONFIG_QUOTA=y +CONFIG_FUSE_FS=y +CONFIG_FSCACHE=y +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_FAT_DEFAULT_UTF8=y +CONFIG_NTFS_FS=y +CONFIG_NTFS_RW=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_CONFIGFS_FS=y +# CONFIG_MISC_FILESYSTEMS is not set +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_SWAP=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_NFS_V4_1_MIGRATION=y +CONFIG_ROOT_NFS=y +CONFIG_NFS_FSCACHE=y +CONFIG_NFS_USE_LEGACY_DNS=y +CONFIG_NFSD=m +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_SCSILAYOUT=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=y +CONFIG_NLS_CODEPAGE_950=y +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_INFINIBAND=y +CONFIG_SECURITY_PATH=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_CRYPTO_ECHAINIV=y +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_DEFLATE=y +CONFIG_CRYPTO_LZO=y +# CONFIG_CRYPTO_HW is not set +CONFIG_CONSOLE_LOGLEVEL_QUIET=7 +# CONFIG_FRAME_POINTER is not set +CONFIG_SCHEDSTATS=y +# CONFIG_RCU_TRACE is not set diff --git a/arch/sw_64/configs/kata_guest_defconfig b/arch/sw_64/configs/kata_guest_defconfig new file mode 100644 index 000000000000..8122155c1276 --- /dev/null +++ b/arch/sw_64/configs/kata_guest_defconfig @@ -0,0 +1,633 @@ +CONFIG_LOCALVERSION="-xuelang" +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +# CONFIG_CROSS_MEMORY_ATTACH is not set +CONFIG_USELIB=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_LOG_BUF_SHIFT=18 +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_NAMESPACES=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_RELAY=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_EXPERT=y +CONFIG_KALLSYMS_ALL=y +CONFIG_BPF_SYSCALL=y +CONFIG_PERF_EVENTS=y +CONFIG_DEBUG_PERF_USE_VMALLOC=y +# CONFIG_COMPAT_BRK is not set +CONFIG_CPUFREQ_DEBUGFS=y +# CONFIG_LOCK_MEMB is not set +CONFIG_SMP=y +CONFIG_ARCH_SPARSEMEM_ENABLE=y +CONFIG_NUMA=y +CONFIG_HZ=100 +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_BINFMT_MISC=y +CONFIG_USE_OF=y +CONFIG_SW64_BUILTIN_DTB=y +CONFIG_SW64_BUILTIN_DTB_NAME="chip_vt" +CONFIG_FIRMWARE_MEMMAP=y +CONFIG_DMI_SYSFS=m +CONFIG_GOOGLE_FIRMWARE=y +CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE=y +CONFIG_SW64_SUSPEND_DEEPSLEEP_BOOTCORE=y +# CONFIG_CPU_IDLE is not set +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_SCSI=m +CONFIG_VHOST_VSOCK=m +CONFIG_VHOST_CROSS_ENDIAN_LEGACY=y +CONFIG_KPROBES=y +CONFIG_JUMP_LABEL=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_OSF_PARTITION=y +CONFIG_BSD_DISKLABEL=y +CONFIG_MINIX_SUBPARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_LDM_PARTITION=y +CONFIG_SGI_PARTITION=y +CONFIG_ULTRIX_PARTITION=y +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=y +CONFIG_UNIX=y +CONFIG_UNIX_DIAG=y +CONFIG_TLS=m +CONFIG_TLS_DEVICE=y +CONFIG_XFRM_USER=y +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_SUB_POLICY=y +CONFIG_XFRM_STATISTICS=y +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_FIB_TRIE_STATS=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_IP_MROUTE=y +CONFIG_NET_IPVTI=m +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_UDP_DIAG=m +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_MD5SIG=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_MIP6=m +CONFIG_IPV6_ILA=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_GRE=m +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_MROUTE=y +CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y +CONFIG_IPV6_PIMSM_V2=y +CONFIG_IPV6_SEG6_LWTUNNEL=y +CONFIG_IPV6_SEG6_HMAC=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=m +CONFIG_NF_LOG_NETDEV=m +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=m +CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_OBJREF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_OSF=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_L2TP=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_NF_TABLES_IPV4=y +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_FLOW_TABLE_IPV4=m +CONFIG_NF_LOG_ARP=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_TARGET_CLUSTERIP=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NF_TABLES_BRIDGE=y +CONFIG_NF_LOG_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BRIDGE=y +CONFIG_VLAN_8021Q=m +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_CBQ=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_DSMARK=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_SKBPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_DEFAULT=y +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_TCINDEX=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_RSVP=m +CONFIG_NET_CLS_RSVP6=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=y +CONFIG_VSOCKETS_DIAG=m +CONFIG_VIRTIO_VSOCKETS=y +CONFIG_NETLINK_DIAG=m +CONFIG_CGROUP_NET_PRIO=y +CONFIG_BPF_JIT=y +# CONFIG_WIRELESS is not set +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +# CONFIG_STANDALONE is not set +# CONFIG_PREVENT_FIRMWARE_BUILD is not set +CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_JEDECPROBE=y +CONFIG_MTD_CFI_ADV_OPTIONS=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_CFI_STAA=y +CONFIG_MTD_ROM=y +CONFIG_MTD_ABSENT=y +CONFIG_MTD_COMPLEX_MAPPINGS=y +CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_PHYSMAP_OF=y +CONFIG_MTD_PLATRAM=y +CONFIG_MTD_SPI_NOR=y +CONFIG_OF_OVERLAY=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=5000000 +CONFIG_VIRTIO_BLK=y +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_FC=y +CONFIG_NVME_TARGET=y +CONFIG_NVME_TARGET_LOOP=y +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_FC=y +CONFIG_NVME_TARGET_FCLOOP=y +CONFIG_RAID_ATTRS=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_ST=y +CONFIG_BLK_DEV_SR=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_SCSI_SPI_ATTRS=y +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_LIBSAS=y +CONFIG_SCSI_SAS_ATA=y +CONFIG_SCSI_SRP_ATTRS=y +CONFIG_ISCSI_TCP=m +CONFIG_ISCSI_BOOT_SYSFS=y +CONFIG_SCSI_DH=y +CONFIG_SCSI_DH_RDAC=y +CONFIG_SCSI_DH_HP_SW=y +CONFIG_SCSI_DH_EMC=y +CONFIG_SCSI_DH_ALUA=y +CONFIG_ATA=y +# CONFIG_ATA_SFF is not set +CONFIG_MD=y +CONFIG_MD_LINEAR=m +CONFIG_MD_MULTIPATH=m +CONFIG_MD_FAULTY=m +CONFIG_BCACHE=m +CONFIG_BCACHE_DEBUG=y +CONFIG_BCACHE_CLOSURES_DEBUG=y +CONFIG_BLK_DEV_DM=m +CONFIG_DM_DEBUG=y +CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING=y +CONFIG_DM_DEBUG_BLOCK_STACK_TRACING=y +CONFIG_DM_UNSTRIPED=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_WRITECACHE=m +CONFIG_DM_ERA=m +CONFIG_DM_MIRROR=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_RAID=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_DELAY=m +CONFIG_DM_UEVENT=y +CONFIG_DM_FLAKEY=m +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_SWITCH=m +CONFIG_DM_LOG_WRITES=m +CONFIG_DM_INTEGRITY=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_ISCSI_TARGET=m +CONFIG_INPUT_FF_MEMLESS=y +CONFIG_INPUT_POLLDEV=y +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_EVDEV=y +# CONFIG_KEYBOARD_ATKBD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO_SERPORT is not set +CONFIG_SERIO_LIBPS2=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_SUNWAY=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_VIRTIO_CONSOLE=y +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=y +# CONFIG_DEVPORT is not set +# CONFIG_I2C_COMPAT is not set +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_MUX=y +CONFIG_SPI=y +CONFIG_SPI_SPIDEV=y +CONFIG_SENSORS_PVT=y +CONFIG_SENSORS_LM75=y +CONFIG_SSB=y +CONFIG_SUNWAY_SUPERIO_AST2400=y +CONFIG_DRM=y +CONFIG_DRM_VIRTIO_GPU=y +CONFIG_FIRMWARE_EDID=y +CONFIG_LCD_CLASS_DEVICE=y +# CONFIG_VGA_CONSOLE is not set +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_LOGO=y +CONFIG_USB=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_STORAGE=y +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_RTC_CLASS=y +# CONFIG_RTC_NVMEM is not set +# CONFIG_RTC_INTF_PROC is not set +CONFIG_RTC_DRV_PCF8523=y +CONFIG_UIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_STAGING=y +CONFIG_SW64_LPC_INTC=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_EXT4_DEBUG=y +CONFIG_XFS_FS=y +CONFIG_GFS2_FS=y +CONFIG_FANOTIFY=y +CONFIG_QUOTA=y +CONFIG_AUTOFS4_FS=y +CONFIG_FUSE_FS=y +CONFIG_OVERLAY_FS=y +CONFIG_OVERLAY_FS_INDEX=y +CONFIG_OVERLAY_FS_XINO_AUTO=y +CONFIG_OVERLAY_FS_METACOPY=y +CONFIG_FSCACHE=y +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_FAT_DEFAULT_UTF8=y +CONFIG_NTFS_FS=y +CONFIG_NTFS_RW=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_CONFIGFS_FS=y +CONFIG_SQUASHFS=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_SWAP=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_NFS_V4_1_MIGRATION=y +CONFIG_ROOT_NFS=y +CONFIG_NFS_FSCACHE=y +CONFIG_NFS_USE_LEGACY_DNS=y +CONFIG_NFSD=m +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_SCSILAYOUT=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_9P_FS=y +CONFIG_9P_FSCACHE=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_9P_FS_SECURITY=y +CONFIG_NLS_CODEPAGE_437=m +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=y +CONFIG_NLS_CODEPAGE_950=y +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_INFINIBAND=y +CONFIG_SECURITY_PATH=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_ECHAINIV=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_DEFLATE=y +CONFIG_CRYPTO_LZO=y +# CONFIG_CRYPTO_HW is not set +CONFIG_CONSOLE_LOGLEVEL_QUIET=7 +# CONFIG_ENABLE_MUST_CHECK is not set +# CONFIG_FRAME_POINTER is not set +CONFIG_SCHEDSTATS=y +# CONFIG_RCU_TRACE is not set diff --git a/arch/sw_64/configs/kata_xuelang_defconfig b/arch/sw_64/configs/kata_xuelang_defconfig new file mode 100644 index 000000000000..f553f0e71dbf --- /dev/null +++ b/arch/sw_64/configs/kata_xuelang_defconfig @@ -0,0 +1,616 @@ +CONFIG_LOCALVERSION="-xuelang" +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +# CONFIG_CROSS_MEMORY_ATTACH is not set +CONFIG_USELIB=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_LOG_BUF_SHIFT=18 +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_HUGETLB=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_NAMESPACES=y +CONFIG_USER_NS=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_RELAY=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_EXPERT=y +CONFIG_KALLSYMS_ALL=y +CONFIG_PERF_EVENTS=y +CONFIG_DEBUG_PERF_USE_VMALLOC=y +# CONFIG_COMPAT_BRK is not set +CONFIG_CPUFREQ_DEBUGFS=y +# CONFIG_LOCK_MEMB is not set +CONFIG_SMP=y +CONFIG_HOTPLUG_CPU=y +CONFIG_ARCH_SPARSEMEM_ENABLE=y +CONFIG_NUMA=y +CONFIG_HZ=100 +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_USE_OF=y +CONFIG_FIRMWARE_MEMMAP=y +CONFIG_DMI_SYSFS=m +# CONFIG_SUSPEND is not set +# CONFIG_CPU_IDLE is not set +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_VSOCK=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_OSF_PARTITION=y +CONFIG_BSD_DISKLABEL=y +CONFIG_MINIX_SUBPARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_LDM_PARTITION=y +CONFIG_SGI_PARTITION=y +CONFIG_ULTRIX_PARTITION=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=y +CONFIG_UNIX=y +CONFIG_UNIX_DIAG=y +CONFIG_TLS=m +CONFIG_TLS_DEVICE=y +CONFIG_XFRM_USER=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_SUB_POLICY=y +CONFIG_XFRM_STATISTICS=y +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_FIB_TRIE_STATS=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_IP_MROUTE=y +CONFIG_NET_IPVTI=m +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_UDP_DIAG=m +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_MD5SIG=y +CONFIG_IPV6=m +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_MIP6=m +CONFIG_IPV6_ILA=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_GRE=m +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_MROUTE=y +CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y +CONFIG_IPV6_PIMSM_V2=y +CONFIG_IPV6_SEG6_LWTUNNEL=y +CONFIG_IPV6_SEG6_HMAC=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=m +CONFIG_NF_LOG_NETDEV=m +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=m +CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_OBJREF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_OSF=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_L2TP=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_IP_VS_PROTO_TCP=y +CONFIG_IP_VS_PROTO_UDP=y +CONFIG_IP_VS_RR=m +CONFIG_IP_VS_NFCT=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_FLOW_TABLE_IPV4=m +CONFIG_NF_LOG_ARP=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_TARGET_CLUSTERIP=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NF_LOG_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BRIDGE=m +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_VLAN_8021Q=m +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_CBQ=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_DSMARK=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_SKBPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_DEFAULT=y +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_TCINDEX=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_RSVP=m +CONFIG_NET_CLS_RSVP6=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=y +CONFIG_VSOCKETS_DIAG=m +CONFIG_NETLINK_DIAG=m +CONFIG_CGROUP_NET_PRIO=y +CONFIG_BPF_JIT=y +CONFIG_NET_DROP_MONITOR=m +# CONFIG_WIRELESS is not set +CONFIG_CAIF=m +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +# CONFIG_STANDALONE is not set +# CONFIG_PREVENT_FIRMWARE_BUILD is not set +CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_JEDECPROBE=y +CONFIG_MTD_CFI_ADV_OPTIONS=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_CFI_STAA=y +CONFIG_MTD_ROM=y +CONFIG_MTD_ABSENT=y +CONFIG_MTD_COMPLEX_MAPPINGS=y +CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_PHYSMAP_OF=y +CONFIG_MTD_PLATRAM=y +CONFIG_MTD_SPI_NOR=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=5000000 +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_FC=y +CONFIG_NVME_TARGET=y +CONFIG_NVME_TARGET_LOOP=y +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_FC=y +CONFIG_NVME_TARGET_FCLOOP=y +CONFIG_RAID_ATTRS=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_ST=y +CONFIG_BLK_DEV_SR=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_SCSI_SPI_ATTRS=y +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_LIBSAS=y +CONFIG_SCSI_SAS_ATA=y +CONFIG_SCSI_SRP_ATTRS=y +CONFIG_ISCSI_TCP=m +CONFIG_ISCSI_BOOT_SYSFS=y +CONFIG_SCSI_DH=y +CONFIG_SCSI_DH_RDAC=y +CONFIG_SCSI_DH_HP_SW=y +CONFIG_SCSI_DH_EMC=y +CONFIG_SCSI_DH_ALUA=y +CONFIG_ATA=y +# CONFIG_ATA_SFF is not set +CONFIG_MD=y +CONFIG_MD_LINEAR=m +CONFIG_MD_MULTIPATH=m +CONFIG_MD_FAULTY=m +CONFIG_BCACHE=m +CONFIG_BCACHE_DEBUG=y +CONFIG_BCACHE_CLOSURES_DEBUG=y +CONFIG_BLK_DEV_DM=m +CONFIG_DM_DEBUG=y +CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING=y +CONFIG_DM_DEBUG_BLOCK_STACK_TRACING=y +CONFIG_DM_UNSTRIPED=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_WRITECACHE=m +CONFIG_DM_ERA=m +CONFIG_DM_MIRROR=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_RAID=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_DELAY=m +CONFIG_DM_UEVENT=y +CONFIG_DM_FLAKEY=m +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_SWITCH=m +CONFIG_DM_LOG_WRITES=m +CONFIG_DM_INTEGRITY=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_ISCSI_TARGET=m +CONFIG_INPUT_FF_MEMLESS=y +CONFIG_INPUT_POLLDEV=y +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_EVDEV=y +# CONFIG_KEYBOARD_ATKBD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO_SERPORT is not set +CONFIG_SERIO_LIBPS2=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_SUNWAY=y +# CONFIG_HW_RANDOM is not set +# CONFIG_I2C_COMPAT is not set +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_MUX=y +CONFIG_SPI=y +CONFIG_SPI_SPIDEV=y +CONFIG_SSB=y +CONFIG_DRM=y +CONFIG_FIRMWARE_EDID=y +CONFIG_LCD_CLASS_DEVICE=y +# CONFIG_VGA_CONSOLE is not set +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_LOGO=y +CONFIG_USB=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_STORAGE=y +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_RTC_CLASS=y +# CONFIG_RTC_NVMEM is not set +# CONFIG_RTC_INTF_PROC is not set +CONFIG_RTC_DRV_PCF8523=y +CONFIG_UIO=y +CONFIG_STAGING=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_EXT4_DEBUG=y +CONFIG_XFS_FS=y +CONFIG_GFS2_FS=y +CONFIG_BTRFS_FS=m +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_FANOTIFY=y +CONFIG_QUOTA=y +CONFIG_AUTOFS4_FS=y +CONFIG_FUSE_FS=y +CONFIG_OVERLAY_FS=m +CONFIG_FSCACHE=y +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_FAT_DEFAULT_UTF8=y +CONFIG_NTFS_FS=y +CONFIG_NTFS_RW=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_CONFIGFS_FS=y +# CONFIG_MISC_FILESYSTEMS is not set +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_SWAP=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_NFS_V4_1_MIGRATION=y +CONFIG_ROOT_NFS=y +CONFIG_NFS_FSCACHE=y +CONFIG_NFS_USE_LEGACY_DNS=y +CONFIG_NFSD=m +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_SCSILAYOUT=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_NLS_CODEPAGE_437=m +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=y +CONFIG_NLS_CODEPAGE_950=y +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_INFINIBAND=y +CONFIG_SECURITY_PATH=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_ECHAINIV=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_DEFLATE=y +CONFIG_CRYPTO_LZO=y +# CONFIG_CRYPTO_HW is not set +CONFIG_CONSOLE_LOGLEVEL_QUIET=7 +# CONFIG_ENABLE_MUST_CHECK is not set +CONFIG_SCHEDSTATS=y +# CONFIG_RCU_TRACE is not set diff --git a/arch/sw_64/configs/xuelang_defconfig b/arch/sw_64/configs/xuelang_defconfig new file mode 100644 index 000000000000..b1c0101d0089 --- /dev/null +++ b/arch/sw_64/configs/xuelang_defconfig @@ -0,0 +1,668 @@ +CONFIG_LOCALVERSION="-xuelang" +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +# CONFIG_CROSS_MEMORY_ATTACH is not set +CONFIG_USELIB=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_BPF_SYSCALL=y +CONFIG_BPF_JIT=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_IKHEADERS=y +CONFIG_LOG_BUF_SHIFT=18 +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_NAMESPACES=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_RELAY=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_EXPERT=y +CONFIG_KALLSYMS_ALL=y +CONFIG_PERF_EVENTS=y +CONFIG_DEBUG_PERF_USE_VMALLOC=y +CONFIG_SMP=y +CONFIG_ARCH_SPARSEMEM_ENABLE=y +CONFIG_NUMA=y +CONFIG_HZ=100 +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_USE_OF=y +CONFIG_FIRMWARE_MEMMAP=y +CONFIG_DMI_SYSFS=m +CONFIG_ACPI_TAD=y +# CONFIG_CPU_IDLE is not set +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_SCSI=m +CONFIG_VHOST_VSOCK=m +CONFIG_VHOST_CROSS_ENDIAN_LEGACY=y +CONFIG_KPROBES=y +CONFIG_JUMP_LABEL=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_OSF_PARTITION=y +CONFIG_BSD_DISKLABEL=y +CONFIG_MINIX_SUBPARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_LDM_PARTITION=y +CONFIG_SGI_PARTITION=y +CONFIG_ULTRIX_PARTITION=y +# CONFIG_COMPAT_BRK is not set +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_CMA_AREAS=7 +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=y +CONFIG_UNIX=y +CONFIG_UNIX_DIAG=y +CONFIG_TLS=m +CONFIG_TLS_DEVICE=y +CONFIG_XFRM_USER=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_SUB_POLICY=y +CONFIG_XFRM_STATISTICS=y +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_FIB_TRIE_STATS=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_IP_MROUTE=y +CONFIG_NET_IPVTI=m +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_UDP_DIAG=m +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_MD5SIG=y +CONFIG_IPV6=m +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_MIP6=m +CONFIG_IPV6_ILA=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_GRE=m +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_MROUTE=y +CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y +CONFIG_IPV6_PIMSM_V2=y +CONFIG_IPV6_SEG6_LWTUNNEL=y +CONFIG_IPV6_SEG6_HMAC=y +CONFIG_NETFILTER=y +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_OSF=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_L2TP=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_NF_TABLES_IPV4=y +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_LOG_ARP=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BRIDGE=m +CONFIG_VLAN_8021Q=m +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_SKBPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_DEFAULT=y +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=m +CONFIG_NETLINK_DIAG=m +CONFIG_CGROUP_NET_PRIO=y +# CONFIG_WIRELESS is not set +CONFIG_PCI=y +CONFIG_PCIEPORTBUS=y +CONFIG_PCIEAER=y +# CONFIG_PCIEASPM is not set +CONFIG_PCI_MSI=y +CONFIG_PCI_IOV=y +CONFIG_UEVENT_HELPER=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +# CONFIG_STANDALONE is not set +# CONFIG_PREVENT_FIRMWARE_BUILD is not set +CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_JEDECPROBE=y +CONFIG_MTD_CFI_ADV_OPTIONS=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_CFI_STAA=y +CONFIG_MTD_ROM=y +CONFIG_MTD_ABSENT=y +CONFIG_MTD_COMPLEX_MAPPINGS=y +CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_PHYSMAP_OF=y +CONFIG_MTD_PLATRAM=y +CONFIG_MTD_SPI_NOR=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=5000000 +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_FC=y +CONFIG_NVME_TARGET=y +CONFIG_NVME_TARGET_LOOP=y +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_FC=y +CONFIG_NVME_TARGET_FCLOOP=y +CONFIG_RAID_ATTRS=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_ST=y +CONFIG_BLK_DEV_SR=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_SCSI_SPI_ATTRS=y +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_LIBSAS=y +CONFIG_SCSI_SAS_ATA=y +CONFIG_SCSI_SRP_ATTRS=y +CONFIG_ISCSI_TCP=m +CONFIG_ISCSI_BOOT_SYSFS=y +CONFIG_SCSI_CXGB3_ISCSI=m +CONFIG_SCSI_CXGB4_ISCSI=m +CONFIG_SCSI_BNX2_ISCSI=m +CONFIG_MEGARAID_SAS=m +CONFIG_SCSI_MPT3SAS=m +CONFIG_SCSI_DH=y +CONFIG_SCSI_DH_RDAC=y +CONFIG_SCSI_DH_HP_SW=y +CONFIG_SCSI_DH_EMC=y +CONFIG_SCSI_DH_ALUA=y +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +# CONFIG_ATA_SFF is not set +CONFIG_MD=y +CONFIG_MD_LINEAR=m +CONFIG_MD_MULTIPATH=m +CONFIG_MD_FAULTY=m +CONFIG_BCACHE=m +CONFIG_BCACHE_DEBUG=y +CONFIG_BCACHE_CLOSURES_DEBUG=y +CONFIG_BLK_DEV_DM=m +CONFIG_DM_DEBUG=y +CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING=y +CONFIG_DM_DEBUG_BLOCK_STACK_TRACING=y +CONFIG_DM_UNSTRIPED=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_WRITECACHE=m +CONFIG_DM_ERA=m +CONFIG_DM_MIRROR=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_RAID=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_DELAY=m +CONFIG_DM_UEVENT=y +CONFIG_DM_FLAKEY=m +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_SWITCH=m +CONFIG_DM_LOG_WRITES=m +CONFIG_DM_INTEGRITY=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_ISCSI_TARGET=m +CONFIG_NET_FC=y +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_VIRTIO_NET=y +# CONFIG_NET_VENDOR_3COM is not set +# CONFIG_NET_VENDOR_AMD is not set +# CONFIG_NET_VENDOR_ARC is not set +CONFIG_CAVIUM_PTP=y +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_EZCHIP is not set +CONFIG_E100=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_IGB=y +CONFIG_IGBVF=m +CONFIG_IXGBE=m +CONFIG_IXGBEVF=m +CONFIG_I40E=y +CONFIG_I40EVF=y +# CONFIG_NET_VENDOR_MARVELL is not set +CONFIG_MLX4_EN=y +CONFIG_MLX5_CORE=m +CONFIG_MLX5_FPGA=y +CONFIG_MLX5_CORE_EN=y +CONFIG_MLXSW_CORE=y +CONFIG_MLXSW_PCI=y +CONFIG_MLXSW_I2C=y +CONFIG_MLXSW_MINIMAL=y +# CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_QUALCOMM is not set +# CONFIG_NET_VENDOR_RENESAS is not set +# CONFIG_NET_VENDOR_ROCKER is not set +# CONFIG_NET_VENDOR_SAMSUNG is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SMSC is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set +# CONFIG_NET_VENDOR_VIA is not set +# CONFIG_NET_VENDOR_WIZNET is not set +# CONFIG_WLAN is not set +CONFIG_INPUT_FF_MEMLESS=y +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_EVDEV=y +# CONFIG_KEYBOARD_ATKBD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO_SERPORT is not set +CONFIG_SERIO_LIBPS2=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +# CONFIG_SERIAL_8250_PCI is not set +CONFIG_SERIAL_8250_SUNWAY=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_VIRTIO_CONSOLE=y +# CONFIG_HW_RANDOM is not set +# CONFIG_I2C_COMPAT is not set +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_MUX=y +CONFIG_SPI=y +CONFIG_SPI_CHIP3=y +CONFIG_SPI_SPIDEV=y +CONFIG_SENSORS_PVT=y +CONFIG_SENSORS_LM75=y +CONFIG_SSB=y +CONFIG_DRM=y +CONFIG_DRM_RADEON=y +CONFIG_DRM_AST=y +CONFIG_DRM_VIRTIO_GPU=y +CONFIG_LCD_CLASS_DEVICE=y +# CONFIG_VGA_CONSOLE is not set +CONFIG_USB=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_STORAGE=y +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_MLX4_INFINIBAND=m +CONFIG_MLX5_INFINIBAND=m +CONFIG_INFINIBAND_MTHCA=m +# CONFIG_INFINIBAND_MTHCA_DEBUG is not set +CONFIG_INFINIBAND_IPOIB=m +CONFIG_INFINIBAND_IPOIB_CM=y +CONFIG_RTC_CLASS=y +# CONFIG_RTC_NVMEM is not set +# CONFIG_RTC_INTF_PROC is not set +CONFIG_RTC_DRV_PCF8523=y +CONFIG_UIO=y +CONFIG_UIO_PCI_GENERIC=m +CONFIG_VIRTIO_PCI=y +# CONFIG_VIRTIO_PCI_LEGACY is not set +CONFIG_VIRTIO_MMIO=y +CONFIG_STAGING=y +CONFIG_IOMMU_DEFAULT_PASSTHROUGH=y +CONFIG_SUNWAY_IOMMU=y +CONFIG_SW64_LPC_INTC=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_EXT4_DEBUG=y +CONFIG_XFS_FS=y +CONFIG_GFS2_FS=y +CONFIG_FANOTIFY=y +CONFIG_QUOTA=y +CONFIG_FUSE_FS=y +CONFIG_FSCACHE=y +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_FAT_DEFAULT_UTF8=y +CONFIG_NTFS_FS=y +CONFIG_NTFS_RW=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_CONFIGFS_FS=y +# CONFIG_MISC_FILESYSTEMS is not set +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_SWAP=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_NFS_V4_1_MIGRATION=y +CONFIG_ROOT_NFS=y +CONFIG_NFS_FSCACHE=y +CONFIG_NFS_USE_LEGACY_DNS=y +CONFIG_NFSD=m +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_SCSILAYOUT=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=y +CONFIG_NLS_CODEPAGE_950=y +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_INFINIBAND=y +CONFIG_SECURITY_PATH=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_CRYPTO_ECHAINIV=y +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_DEFLATE=y +CONFIG_CRYPTO_LZO=y +# CONFIG_CRYPTO_HW is not set +CONFIG_CONSOLE_LOGLEVEL_QUIET=7 +# CONFIG_FRAME_POINTER is not set +CONFIG_SCHEDSTATS=y +# CONFIG_RCU_TRACE is not set -- Gitee From e5d2ae63923c5a127971547cf87f4e23a4d2e0a2 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:29 +0800 Subject: [PATCH 028/524] sw64: add PCI support Add basic PCI support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 163 ++++++++++ arch/sw_64/include/asm/pci_impl.h | 27 ++ arch/sw_64/kernel/pci-noop.c | 138 ++++++++ arch/sw_64/pci/Makefile | 8 + arch/sw_64/pci/pci-legacy.c | 508 ++++++++++++++++++++++++++++++ arch/sw_64/pci/pci-sysfs.c | 359 +++++++++++++++++++++ arch/sw_64/pci/pci.c | 436 +++++++++++++++++++++++++ 7 files changed, 1639 insertions(+) create mode 100644 arch/sw_64/include/asm/pci.h create mode 100644 arch/sw_64/include/asm/pci_impl.h create mode 100644 arch/sw_64/kernel/pci-noop.c create mode 100644 arch/sw_64/pci/Makefile create mode 100644 arch/sw_64/pci/pci-legacy.c create mode 100644 arch/sw_64/pci/pci-sysfs.c create mode 100644 arch/sw_64/pci/pci.c diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h new file mode 100644 index 000000000000..21bfcef21c5f --- /dev/null +++ b/arch/sw_64/include/asm/pci.h @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_PCI_H +#define _ASM_SW64_PCI_H + +#ifdef __KERNEL__ + +#include +#include +#include + +/* + * The following structure is used to manage multiple PCI busses. + */ + +struct pci_dev; +struct pci_bus; +struct resource; +struct sunway_iommu; +struct page; + +struct piu_saved { + unsigned long piuconfig0; + unsigned long piuconfig1; + unsigned long epdmabar; + unsigned long msiaddr; + unsigned long msiconfig[256]; + unsigned long iommuexcpt_ctrl; + unsigned long dtbaseaddr; + unsigned long hpintconfig; + unsigned long pmeintconfig; + unsigned long aererrintconfig; + unsigned long intaconfig; + unsigned long intbconfig; + unsigned long intcconfig; + unsigned long intdconfig; +}; + +/* A controller. Used to manage multiple PCI busses. */ +struct pci_controller { + struct pci_controller *next; + struct pci_bus *bus; + struct resource *io_space; + struct resource *mem_space; + struct resource *pre_mem_space; + struct resource *busn_space; + unsigned long sparse_mem_base; + unsigned long dense_mem_base; + unsigned long sparse_io_base; + unsigned long dense_io_base; + + /* This one's for the kernel only. It's in KSEG somewhere. */ + void __iomem *ep_config_space_base; + void __iomem *rc_config_space_base; + + unsigned long index; + unsigned long node; + DECLARE_BITMAP(piu_msiconfig, 256); + int int_irq; + int service_irq; + /* For compatibility with current (as of July 2003) pciutils + * and XFree86. Eventually will be removed. + */ + unsigned int need_domain_info; + bool iommu_enable; + struct sunway_iommu *pci_iommu; + int first_busno; + int last_busno; + int self_busno; + void *sysdata; +}; + +/* Override the logic in pci_scan_bus for skipping already-configured + * bus numbers. + */ + +#define pcibios_assign_all_busses() (pci_has_flag(PCI_REASSIGN_ALL_BUS)) + +#define PCIBIOS_MIN_IO 0 +#define PCIBIOS_MIN_MEM 0 + +extern void __init sw64_init_pci(void); +extern void __init sw64_device_interrupt(unsigned long vector); +extern void __init sw64_init_irq(void); +extern void __init sw64_init_arch(void); +extern struct pci_ops sw64_pci_ops; +extern int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); +extern struct pci_controller *hose_head; +#ifdef CONFIG_PCI_SW64 +extern void __init setup_chip_pci_ops(void); +#else +#define setup_chip_pci_ops() do { } while (0) +#endif + +extern struct pci_controller *pci_bus_to_pci_controller(const struct pci_bus *bus); +extern struct pci_controller *bus_num_to_pci_controller(unsigned long bus_num); + +extern void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge); +extern void sw64_pci_root_bridge_scan_finish_up(struct pci_host_bridge *bridge); +extern int sw64_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); + +#ifdef CONFIG_PCI_DOMAINS +static inline int pci_proc_domain(struct pci_bus *bus) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + return hose->need_domain_info; +} +#endif + +#ifdef CONFIG_NUMA +static inline int __pcibus_to_node(const struct pci_bus *bus) +{ + struct pci_controller *hose; + + hose = pci_bus_to_pci_controller(bus); + if (!node_online(hose->node)) + return next_node_in(hose->node, node_online_map); + else + return hose->node; +} +#define pcibus_to_node(bus) __pcibus_to_node(bus) +#endif + +#endif /* __KERNEL__ */ + +/* Values for the `which' argument to sys_pciconfig_iobase. */ +#define IOBASE_HOSE 0 +#define IOBASE_SPARSE_MEM 1 +#define IOBASE_DENSE_MEM 2 +#define IOBASE_SPARSE_IO 3 +#define IOBASE_DENSE_IO 4 +#define IOBASE_ROOT_BUS 5 +#define IOBASE_FROM_HOSE 0x10000 + +extern int pci_legacy_read(struct pci_bus *bus, loff_t port, u32 *val, + size_t count); +extern int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, + size_t count); +extern int pci_mmap_legacy_page_range(struct pci_bus *bus, + struct vm_area_struct *vma, + enum pci_mmap_state mmap_state); +extern void pci_adjust_legacy_attr(struct pci_bus *bus, + enum pci_mmap_state mmap_type); +#define HAVE_PCI_LEGACY 1 + +extern int pci_create_resource_files(struct pci_dev *dev); +extern void pci_remove_resource_files(struct pci_dev *dev); +extern void __init reserve_mem_for_pci(void); +extern int chip_pcie_configure(struct pci_controller *hose); + +#define PCI_VENDOR_ID_JN 0x5656 +#define PCI_DEVICE_ID_SW64_ROOT_BRIDGE 0x3231 +#define PCI_DEVICE_ID_JN_PCIESW 0x1000 +#define PCI_DEVICE_ID_JN_PCIEUSIP 0x1200 +#define PCI_DEVICE_ID_JN_PCIE2PCI 0x1314 + +#define NR_IRQ_VECTORS NR_IRQS + +#define LAST_DEVICE_VECTOR 31 + +#define PCITODMA_OFFSET 0x0 /*0 offset*/ + +#endif /* _ASM_SW64_PCI_H */ diff --git a/arch/sw_64/include/asm/pci_impl.h b/arch/sw_64/include/asm/pci_impl.h new file mode 100644 index 000000000000..aa17a69b73f8 --- /dev/null +++ b/arch/sw_64/include/asm/pci_impl.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file contains declarations and inline functions for interfacing + * with the PCI initialization routines. + */ +#ifndef _SW64_KERNEL_PCI_IMPL_H +#define _SW64_KERNEL_PCI_IMPL_H + +#include + +struct pci_dev; +struct pci_controller; + +/* The hose list. */ +extern struct pci_controller *hose_head, **hose_tail; + +extern void common_init_pci(void); +extern struct pci_controller *alloc_pci_controller(void); +extern struct resource *alloc_resource(void); + +extern unsigned long size_for_memory(unsigned long max); + +extern const struct dma_map_ops sw64_dma_direct_ops; + +extern struct cma *sw64_kvm_cma; +extern struct gen_pool *sw64_kvm_pool; +#endif /* _SW64_KERNEL_PCI_IMPL_H */ diff --git a/arch/sw_64/kernel/pci-noop.c b/arch/sw_64/kernel/pci-noop.c new file mode 100644 index 000000000000..abfba92fa6a9 --- /dev/null +++ b/arch/sw_64/kernel/pci-noop.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/arch/sw/kernel/pci-noop.c + * + * Stub PCI interfaces for NO PCI kernels. + */ + +#include +#include +#include +#include + +/* + * The PCI controller list. + */ + +struct pci_controller *hose_head, **hose_tail = &hose_head; + +struct pci_controller * __init +alloc_pci_controller(void) +{ + struct pci_controller *hose; + + hose = memblock_alloc(sizeof(*hose), SMP_CACHE_BYTES); + + *hose_tail = hose; + hose_tail = &hose->next; + + return hose; +} + +struct resource * __init +alloc_resource(void) +{ + struct resource *res; + + res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES); + + return res; +} + +asmlinkage long +sys_pciconfig_iobase(long which, unsigned long bus, unsigned long dfn) +{ + return -ENODEV; +} + +asmlinkage long +sys_pciconfig_read(unsigned long bus, unsigned long dfn, + unsigned long off, unsigned long len, void *buf) +{ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + else + return -ENODEV; +} + +asmlinkage long +sys_pciconfig_write(unsigned long bus, unsigned long dfn, + unsigned long off, unsigned long len, void *buf) +{ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + else + return -ENODEV; +} + +static void *sw64_noop_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, + unsigned long attrs) +{ + void *ret; + + if (!dev || *dev->dma_mask >= 0xffffffffUL) + gfp &= ~GFP_DMA; + ret = (void *)__get_free_pages(gfp, get_order(size)); + if (ret) { + memset(ret, 0, size); + *dma_handle = virt_to_phys(ret); + } + return ret; +} + +static void sw64_noop_free_coherent(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t dma_addr, + unsigned long attrs) +{ + free_pages((unsigned long)cpu_addr, get_order(size)); +} + +static dma_addr_t sw64_noop_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + return page_to_pa(page) + offset; +} + +static int sw64_noop_map_sg(struct device *dev, struct scatterlist *sgl, int nents, + enum dma_data_direction dir, unsigned long attrs) +{ + int i; + struct scatterlist *sg; + + for_each_sg(sgl, sg, nents, i) { + void *va; + + BUG_ON(!sg_page(sg)); + va = sg_virt(sg); + sg_dma_address(sg) = (dma_addr_t)virt_to_phys(va); + sg_dma_len(sg) = sg->length; + } + + return nents; +} + +static int sw64_noop_supported(struct device *dev, u64 mask) +{ + return mask < 0x00ffffffUL ? 0 : 1; +} + +const struct dma_map_ops sw64_noop_ops = { + .alloc = sw64_noop_alloc_coherent, + .free = sw64_noop_free_coherent, + .map_page = sw64_noop_map_page, + .map_sg = sw64_noop_map_sg, + .dma_supported = sw64_noop_supported, +}; + +const struct dma_map_ops *dma_ops = &sw64_noop_ops; +EXPORT_SYMBOL(dma_ops); + +void __init common_init_pci(void) +{ +} + +void __init sw64_init_arch(void) { } +void __init sw64_init_irq(void) { } diff --git a/arch/sw_64/pci/Makefile b/arch/sw_64/pci/Makefile new file mode 100644 index 000000000000..327efb163b12 --- /dev/null +++ b/arch/sw_64/pci/Makefile @@ -0,0 +1,8 @@ +PDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux kernel. +# + +obj-y += pci.o pci-legacy.o pci-sysfs.o +obj-$(CONFIG_ACPI) += acpi.o +obj-$(CONFIG_PCI_MSI) += msi.o diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c new file mode 100644 index 000000000000..2a44463db0a4 --- /dev/null +++ b/arch/sw_64/pci/pci-legacy.c @@ -0,0 +1,508 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include +#include + +unsigned long rc_linkup; + +/* + * The PCI controller list. + */ + +struct pci_controller *hose_head, **hose_tail = &hose_head; +static void __init pcibios_reserve_legacy_regions(struct pci_bus *bus); + +static int __init +pcibios_init(void) +{ + if (acpi_disabled) + sw64_init_pci(); + return 0; +} +subsys_initcall(pcibios_init); + +void __init pcibios_claim_one_bus(struct pci_bus *b) +{ + struct pci_dev *dev; + struct pci_bus *child_bus; + + list_for_each_entry(dev, &b->devices, bus_list) { + int i; + + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + struct resource *r = &dev->resource[i]; + + if (r->parent || !r->start || !r->flags) + continue; + if (r->flags & IORESOURCE_PCI_FIXED) { + if (pci_claim_resource(dev, i) == 0) + continue; + + pci_claim_bridge_resource(dev, i); + } + } + } + + list_for_each_entry(child_bus, &b->children, node) + pcibios_claim_one_bus(child_bus); +} + +static void __init +pcibios_claim_console_setup(void) +{ + struct pci_bus *b; + + list_for_each_entry(b, &pci_root_buses, node) + pcibios_claim_one_bus(b); +} + +int __weak chip_pcie_configure(struct pci_controller *hose) +{ + return 0; +} + +unsigned char last_bus = PCI0_BUS; +void __init common_init_pci(void) +{ + struct pci_controller *hose; + struct pci_host_bridge *bridge; + struct pci_bus *bus; + unsigned int init_busnr; + int need_domain_info = 0; + int ret; + unsigned long offset; + + /* Scan all of the recorded PCI controllers. */ + hose = hose_head; + for (hose = hose_head; hose; hose = hose->next) { + bridge = pci_alloc_host_bridge(0); + if (!bridge) + continue; + hose->busn_space->start = last_bus; + init_busnr = (0xff << 16) + ((last_bus + 1) << 8) + (last_bus); + write_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS, init_busnr); + offset = hose->mem_space->start - PCI_32BIT_MEMIO; + if (is_in_host()) + hose->first_busno = last_bus + 1; + else + hose->first_busno = last_bus; + pci_add_resource_offset(&bridge->windows, hose->mem_space, offset); + pci_add_resource_offset(&bridge->windows, hose->io_space, hose->io_space->start); + pci_add_resource_offset(&bridge->windows, hose->pre_mem_space, 0); + pci_add_resource_offset(&bridge->windows, hose->busn_space, 0); + bridge->dev.parent = NULL; + bridge->sysdata = hose; + bridge->busnr = hose->busn_space->start; + bridge->ops = &sw64_pci_ops; + bridge->swizzle_irq = pci_common_swizzle; + bridge->map_irq = sw64_map_irq; + + ret = pci_scan_root_bus_bridge(bridge); + if (ret) { + pci_free_host_bridge(bridge); + continue; + } + + bus = hose->bus = bridge->bus; + hose->need_domain_info = need_domain_info; + + if (is_in_host()) + last_bus = chip_pcie_configure(hose); + else + while (pci_find_bus(pci_domain_nr(bus), last_bus)) + last_bus++; + + hose->last_busno = hose->busn_space->end = last_bus; + init_busnr = read_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS); + init_busnr &= ~(0xff << 16); + init_busnr |= last_bus << 16; + write_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS, init_busnr); + pci_bus_update_busn_res_end(bus, last_bus); + last_bus++; + } + + pcibios_claim_console_setup(); + + if (is_in_host()) { + list_for_each_entry(bus, &pci_root_buses, node) + pcibios_reserve_legacy_regions(bus); + } + + pr_info("SW arch assign unassigned resources.\n"); + + pci_assign_unassigned_resources(); + + for (hose = hose_head; hose; hose = hose->next) { + bus = hose->bus; + if (bus) + pci_bus_add_devices(bus); + } +} + +struct pci_controller * __init +alloc_pci_controller(void) +{ + struct pci_controller *hose; + + hose = memblock_alloc(sizeof(*hose), SMP_CACHE_BYTES); + + *hose_tail = hose; + hose_tail = &hose->next; + + return hose; +} + +struct resource * __init +alloc_resource(void) +{ + struct resource *res; + + res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES); + + return res; +} + +static void __init pcibios_reserve_legacy_regions(struct pci_bus *bus) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + resource_size_t offset; + struct resource *res; + + pr_debug("Reserving legacy ranges for domain %04x\n", pci_domain_nr(bus)); + + /* Check for IO */ + if (!(hose->io_space->flags & IORESOURCE_IO)) + goto no_io; + offset = (unsigned long)hose->io_space->start; + res = kzalloc(sizeof(struct resource), GFP_KERNEL); + BUG_ON(res == NULL); + res->name = "Legacy IO"; + res->flags = IORESOURCE_IO; + res->start = offset; + res->end = (offset + 0xfff) & 0xfffffffffffffffful; + pr_debug("Candidate legacy IO: %pR\n", res); + if (request_resource(hose->io_space, res)) { + pr_debug("PCI %04x:%02x Cannot reserve Legacy IO %pR\n", + pci_domain_nr(bus), bus->number, res); + kfree(res); + } + +no_io: + return; +} + +/* PCIe RC operations */ +int sw6_pcie_read_rc_cfg(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + u32 data; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + void __iomem *cfg_iobase = hose->rc_config_space_base; + + if (IS_ENABLED(CONFIG_PCI_DEBUG)) + pr_debug("rc read addr:%px bus %d, devfn %#x, where %#x size=%d\t", + cfg_iobase + ((where & ~3) << 5), bus->number, devfn, where, size); + + if ((uintptr_t)where & (size - 1)) { + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + if (unlikely(devfn > 0)) { + *val = ~0; + return PCIBIOS_DEVICE_NOT_FOUND; + } + + data = readl(cfg_iobase + ((where & ~3) << 5)); + + switch (size) { + case 1: + *val = (data >> (8 * (where & 0x3))) & 0xff; + break; + case 2: + *val = (data >> (8 * (where & 0x2))) & 0xffff; + break; + default: + *val = data; + break; + } + + if (IS_ENABLED(CONFIG_PCI_DEBUG)) + pr_debug("*val %#x\n ", *val); + + return PCIBIOS_SUCCESSFUL; +} + +int sw6_pcie_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + u32 data; + u32 shift = 8 * (where & 3); + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + void __iomem *cfg_iobase = (void *)hose->rc_config_space_base; + + if ((uintptr_t)where & (size - 1)) + return PCIBIOS_BAD_REGISTER_NUMBER; + + switch (size) { + case 1: + data = readl(cfg_iobase + ((where & ~3) << 5)); + data &= ~(0xff << shift); + data |= (val & 0xff) << shift; + break; + case 2: + data = readl(cfg_iobase + ((where & ~3) << 5)); + data &= ~(0xffff << shift); + data |= (val & 0xffff) << shift; + break; + default: + data = val; + break; + } + + if (IS_ENABLED(CONFIG_PCI_DEBUG)) + pr_debug("rc write addr:%px bus %d, devfn %#x, where %#x *val %#x size %d\n", + cfg_iobase + ((where & ~3) << 5), bus->number, devfn, where, val, size); + + writel(data, cfg_iobase + ((where & ~3) << 5)); + + return PCIBIOS_SUCCESSFUL; +} + +int sw6_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + int ret = PCIBIOS_DEVICE_NOT_FOUND; + + if (is_guest_or_emul()) + return pci_generic_config_read(bus, devfn, where, size, val); + + hose->self_busno = hose->busn_space->start; + + if (unlikely(bus->number == hose->self_busno)) { + ret = sw6_pcie_read_rc_cfg(bus, devfn, where, size, val); + } else { + if (test_bit(hose->node * 8 + hose->index, &rc_linkup)) + ret = pci_generic_config_read(bus, devfn, where, size, val); + else + return ret; + } + return ret; +} + +int sw6_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + if (is_guest_or_emul()) + return pci_generic_config_write(bus, devfn, where, size, val); + + hose->self_busno = hose->busn_space->start; + + if (unlikely(bus->number == hose->self_busno)) + return sw6_pcie_write_rc_cfg(bus, devfn, where, size, val); + else + return pci_generic_config_write(bus, devfn, where, size, val); +} + +/* + *sw6_pcie_valid_device - Check if a valid device is present on bus + *@bus: PCI Bus structure + *@devfn: device/function + * + *Return: 'true' on success and 'false' if invalid device is found + */ +static bool sw6_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + if (is_in_host()) { + /* Only one device down on each root complex */ + if (bus->number == hose->self_busno && devfn > 0) + return false; + } + + return true; +} + +/* + *sw6_pcie_map_bus - Get configuration base + *@bus: PCI Bus structure + *@devfn: Device/function + *@where: Offset from base + * + *Return: Base address of the configuration space needed to be + *accessed. + */ +static void __iomem *sw6_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + void __iomem *cfg_iobase; + unsigned long relbus; + + if (!sw6_pcie_valid_device(bus, devfn)) + return NULL; + + relbus = (bus->number << 24) | (devfn << 16) | where; + + cfg_iobase = hose->ep_config_space_base + relbus; + + if (IS_ENABLED(CONFIG_PCI_DEBUG)) + pr_debug("addr:%px bus %d, devfn %d, where %d\n", + cfg_iobase, bus->number, devfn, where); + return cfg_iobase; +} + +struct pci_ops sw64_pci_ops = { + .map_bus = sw6_pcie_map_bus, + .read = sw6_pcie_config_read, + .write = sw6_pcie_config_write, +}; + +int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + return sw64_chip_init->pci_init.map_irq(dev, slot, pin); +} + +static void __init +sw64_init_host(unsigned long node, unsigned long index) +{ + struct pci_controller *hose; + int ret = 0; + + hose = alloc_pci_controller(); + if (!hose) { + pr_warn("alloc NODE %ld RC %ld hose failed\n", node, index); + return; + } + hose->iommu_enable = false; + hose->io_space = alloc_resource(); + hose->mem_space = alloc_resource(); + hose->pre_mem_space = alloc_resource(); + hose->busn_space = alloc_resource(); + hose->index = index; + hose->node = node; + + sw64_chip_init->pci_init.hose_init(hose); + + if (sw64_chip_init->pci_init.set_rc_piu) + sw64_chip_init->pci_init.set_rc_piu(node, index); + + ret = sw64_chip_init->pci_init.check_pci_linkup(node, index); + if (ret == 0) { + /* Root Complex downstream port is link up */ + set_bit(node * 8 + index, &rc_linkup); //8-bit per node + } +} + +void __weak set_devint_wken(int node) {} +void __weak set_adr_int(int node) {} + +void __init sw64_init_arch(void) +{ + if (IS_ENABLED(CONFIG_PCI)) { + unsigned long node, cpu_num; + unsigned long rc_enable; + char id[8], msg[64]; + int i; + + cpu_num = sw64_chip->get_cpu_num(); + + for (node = 0; node < cpu_num; node++) { + if (is_in_host()) { + set_devint_wken(node); + set_adr_int(node); + } + } + + if (!acpi_disabled) + return; + + pr_info("SW arch PCI initialize!\n"); + for (node = 0; node < cpu_num; node++) { + rc_enable = sw64_chip_init->pci_init.get_rc_enable(node); + if (rc_enable == 0) { + pr_notice("PCIe is disabled on node %ld\n", node); + continue; + } + for (i = 0; i < MAX_NR_RCS; i++) { + if ((rc_enable >> i) & 0x1) + sw64_init_host(node, i); + } + if ((rc_linkup >> node * 8) & 0xff) { + memset(msg, 0, 64); + sprintf(msg, "Node %ld: RC [ ", node); + for (i = 0; i < MAX_NR_RCS; i++) { + if ((rc_linkup >> (i + node * 8)) & 1) { + memset(id, 0, 8); + sprintf(id, "%d ", i); + strcat(msg, id); + } + } + strcat(msg, "] link up"); + pr_info("%s\n", msg); + } else { + pr_info("Node %ld: no RC link up\n", node); + } + } + } +} + +void __weak set_pcieport_service_irq(int node, int index) {} + +static void __init sw64_init_intx(struct pci_controller *hose) +{ + unsigned long int_conf, node, val_node; + unsigned long index, irq; + int rcid; + + node = hose->node; + index = hose->index; + + if (!node_online(node)) + val_node = next_node_in(node, node_online_map); + else + val_node = node; + irq = irq_alloc_descs_from(NR_IRQS_LEGACY, 2, val_node); + WARN_ON(irq < 0); + irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_level_irq); + irq_set_status_flags(irq, IRQ_LEVEL); + hose->int_irq = irq; + irq_set_chip_and_handler(irq + 1, &dummy_irq_chip, handle_level_irq); + hose->service_irq = irq + 1; + rcid = cpu_to_rcid(0); + + pr_info_once("INTx are directed to node %d core %d.\n", + ((rcid >> 6) & 0x3), (rcid & 0x1f)); + int_conf = 1UL << 62 | rcid; /* rebase all intx on the first logical cpu */ + if (sw64_chip_init->pci_init.set_intx) + sw64_chip_init->pci_init.set_intx(node, index, int_conf); + + set_pcieport_service_irq(node, index); +} + +void __init sw64_init_irq(void) +{ + struct pci_controller *hose; + + /* Scan all of the recorded PCI controllers. */ + hose = hose_head; + for (hose = hose_head; hose; hose = hose->next) + sw64_init_intx(hose); +} + +void __init +sw64_init_pci(void) +{ + pci_add_flags(PCI_REASSIGN_ALL_BUS); + common_init_pci(); + pci_clear_flags(PCI_REASSIGN_ALL_BUS); +} diff --git a/arch/sw_64/pci/pci-sysfs.c b/arch/sw_64/pci/pci-sysfs.c new file mode 100644 index 000000000000..5b52a534fa80 --- /dev/null +++ b/arch/sw_64/pci/pci-sysfs.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: GPL-2.0 +/** + * Sw_64 PCI resource files. + * + * Loosely based on generic HAVE_PCI_MMAP implementation in + * drivers/pci/pci-sysfs.c + */ + +#include + +static int hose_mmap_page_range(struct pci_controller *hose, + struct vm_area_struct *vma, + enum pci_mmap_state mmap_type, int sparse) +{ + unsigned long base; + + if (mmap_type == pci_mmap_mem) + base = sparse ? hose->sparse_mem_base : hose->dense_mem_base; + else + base = sparse ? hose->sparse_io_base : hose->dense_io_base; + + vma->vm_pgoff |= base >> PAGE_SHIFT; + + return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + vma->vm_end - vma->vm_start, + vma->vm_page_prot); +} + +static int __pci_mmap_fits(struct pci_dev *pdev, int num, + struct vm_area_struct *vma, int sparse) +{ + unsigned long nr, start, size; + int shift = sparse ? 5 : 0; + + nr = vma_pages(vma); + start = vma->vm_pgoff; + size = ((pci_resource_len(pdev, num) - 1) >> (PAGE_SHIFT - shift)) + 1; + + if (start < size && size - start >= nr) + return 1; + WARN(1, "process \"%s\" tried to map%s 0x%08lx-0x%08lx on %s BAR %d (size 0x%08lx)\n", + current->comm, sparse ? " sparse" : "", start, start + nr, + pci_name(pdev), num, size); + return 0; +} + +/** + * pci_mmap_resource - map a PCI resource into user memory space + * @kobj: kobject for mapping + * @attr: struct bin_attribute for the file being mapped + * @vma: struct vm_area_struct passed into the mmap + * @sparse: address space type + * + * Use the bus mapping routines to map a PCI resource into userspace. + */ +static int pci_mmap_resource(struct kobject *kobj, + struct bin_attribute *attr, + struct vm_area_struct *vma, int sparse) +{ + struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + struct resource *res = attr->private; + enum pci_mmap_state mmap_type; + struct pci_bus_region bar; + int i; + + for (i = 0; i < PCI_ROM_RESOURCE; i++) + if (res == &pdev->resource[i]) + break; + if (i >= PCI_ROM_RESOURCE) + return -ENODEV; + + if (res->flags & IORESOURCE_MEM && iomem_is_exclusive(res->start)) + return -EINVAL; + + if (!__pci_mmap_fits(pdev, i, vma, sparse)) + return -EINVAL; + + pcibios_resource_to_bus(pdev->bus, &bar, res); + vma->vm_pgoff += bar.start >> (PAGE_SHIFT - (sparse ? 5 : 0)); + mmap_type = res->flags & IORESOURCE_MEM ? pci_mmap_mem : pci_mmap_io; + + return hose_mmap_page_range(pdev->sysdata, vma, mmap_type, sparse); +} + +static int pci_mmap_resource_sparse(struct file *filp, struct kobject *kobj, + struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + return pci_mmap_resource(kobj, attr, vma, 1); +} + +static int pci_mmap_resource_dense(struct file *filp, struct kobject *kobj, + struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + return pci_mmap_resource(kobj, attr, vma, 0); +} + +/** + * pci_remove_resource_files - cleanup resource files + * @dev: dev to cleanup + * + * If we created resource files for @dev, remove them from sysfs and + * free their resources. + */ +void pci_remove_resource_files(struct pci_dev *pdev) +{ + int i; + + for (i = 0; i < PCI_ROM_RESOURCE; i++) { + struct bin_attribute *res_attr; + + res_attr = pdev->res_attr[i]; + if (res_attr) { + sysfs_remove_bin_file(&pdev->dev.kobj, res_attr); + kfree(res_attr); + } + + res_attr = pdev->res_attr_wc[i]; + if (res_attr) { + sysfs_remove_bin_file(&pdev->dev.kobj, res_attr); + kfree(res_attr); + } + } +} + +static int sparse_mem_mmap_fits(struct pci_dev *pdev, int num) +{ + struct pci_bus_region bar; + struct pci_controller *hose = pci_bus_to_pci_controller(pdev->bus); + long dense_offset; + unsigned long sparse_size; + + pcibios_resource_to_bus(pdev->bus, &bar, &pdev->resource[num]); + + /* + * All core logic chips have 4G sparse address space, except + * CIA which has 16G (see xxx_SPARSE_MEM and xxx_DENSE_MEM + * definitions in asm/core_xxx.h files). This corresponds + * to 128M or 512M of the bus space. + */ + dense_offset = (long)(hose->dense_mem_base - hose->sparse_mem_base); + sparse_size = dense_offset >= 0x400000000UL ? 0x20000000 : 0x8000000; + + return bar.end < sparse_size; +} + +static int pci_create_one_attr(struct pci_dev *pdev, int num, char *name, + char *suffix, struct bin_attribute *res_attr, + unsigned long sparse) +{ + size_t size = pci_resource_len(pdev, num); + + sprintf(name, "resource%d%s", num, suffix); + res_attr->mmap = sparse ? pci_mmap_resource_sparse : + pci_mmap_resource_dense; + res_attr->attr.name = name; + res_attr->attr.mode = 0600; + res_attr->size = sparse ? size << 5 : size; + res_attr->private = &pdev->resource[num]; + return sysfs_create_bin_file(&pdev->dev.kobj, res_attr); +} + +static int pci_create_attr(struct pci_dev *pdev, int num) +{ + /* allocate attribute structure, piggyback attribute name */ + int retval, nlen1, nlen2 = 0, res_count = 1; + unsigned long sparse_base, dense_base; + struct bin_attribute *attr; + struct pci_controller *hose = pci_bus_to_pci_controller(pdev->bus); + char *suffix, *attr_name; + + suffix = ""; + nlen1 = 10; + + if (pdev->resource[num].flags & IORESOURCE_MEM) { + sparse_base = hose->sparse_mem_base; + dense_base = hose->dense_mem_base; + if (sparse_base && !sparse_mem_mmap_fits(pdev, num)) { + sparse_base = 0; + suffix = "_dense"; + nlen1 = 16; /* resourceN_dense */ + } + } else { + sparse_base = hose->sparse_io_base; + dense_base = hose->dense_io_base; + } + + if (sparse_base) { + suffix = "_sparse"; + nlen1 = 17; + if (dense_base) { + nlen2 = 16; /* resourceN_dense */ + res_count = 2; + } + } + + attr = kzalloc(sizeof(*attr) * res_count + nlen1 + nlen2, GFP_ATOMIC); + if (!attr) + return -ENOMEM; + + attr_name = (char *)(attr + res_count); + pdev->res_attr[num] = attr; + retval = pci_create_one_attr(pdev, num, attr_name, suffix, attr, + sparse_base); + if (retval || res_count == 1) + return retval; + + /* Create dense file */ + attr_name += nlen1; + attr++; + pdev->res_attr_wc[num] = attr; + return pci_create_one_attr(pdev, num, attr_name, "_dense", attr, 0); +} + +/** + * pci_create_resource_files - create resource files in sysfs for @dev + * @dev: dev in question + * + * Walk the resources in @dev creating files for each resource available. + */ +int pci_create_resource_files(struct pci_dev *pdev) +{ + int i; + int retval; + + /* Expose the PCI resources from this device as files */ + for (i = 0; i < PCI_ROM_RESOURCE; i++) { + + /* skip empty resources */ + if (!pci_resource_len(pdev, i)) + continue; + + retval = pci_create_attr(pdev, i); + if (retval) { + pci_remove_resource_files(pdev); + return retval; + } + } + return 0; +} + +/* Legacy I/O bus mapping stuff. */ + +static int __legacy_mmap_fits(struct pci_controller *hose, + struct vm_area_struct *vma, + unsigned long res_size, int sparse) +{ + unsigned long nr, start, size; + + nr = vma_pages(vma); + start = vma->vm_pgoff; + size = ((res_size - 1) >> PAGE_SHIFT) + 1; + + if (start < size && size - start >= nr) + return 1; + WARN(1, "process \"%s\" tried to map%s 0x%08lx-0x%08lx on hose %ld (size 0x%08lx)\n", + current->comm, sparse ? " sparse" : "", start, start + nr, + hose->index, size); + return 0; +} + +static inline int has_sparse(struct pci_controller *hose, + enum pci_mmap_state mmap_type) +{ + unsigned long base; + + base = (mmap_type == pci_mmap_mem) ? hose->sparse_mem_base : + hose->sparse_io_base; + + return base != 0; +} + +int pci_mmap_legacy_page_range(struct pci_bus *bus, struct vm_area_struct *vma, + enum pci_mmap_state mmap_type) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + int sparse = has_sparse(hose, mmap_type); + unsigned long res_size; + + res_size = (mmap_type == pci_mmap_mem) ? bus->legacy_mem->size : + bus->legacy_io->size; + if (!__legacy_mmap_fits(hose, vma, res_size, sparse)) + return -EINVAL; + + return hose_mmap_page_range(hose, vma, mmap_type, sparse); +} + +/** + * pci_adjust_legacy_attr - adjustment of legacy file attributes + * @b: bus to create files under + * @mmap_type: I/O port or memory + * + * Adjust file name and size for sparse mappings. + */ +void pci_adjust_legacy_attr(struct pci_bus *bus, enum pci_mmap_state mmap_type) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + if (!has_sparse(hose, mmap_type)) + return; + + if (mmap_type == pci_mmap_mem) { + bus->legacy_mem->attr.name = "legacy_mem_sparse"; + bus->legacy_mem->size <<= 5; + } else { + bus->legacy_io->attr.name = "legacy_io_sparse"; + bus->legacy_io->size <<= 5; + } +} + +/* Legacy I/O bus read/write functions */ +int pci_legacy_read(struct pci_bus *bus, loff_t port, u32 *val, size_t size) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + port += hose->io_space->start; + + switch (size) { + case 1: + *((u8 *)val) = inb(port); + return 1; + case 2: + if (port & 1) + return -EINVAL; + *((u16 *)val) = inw(port); + return 2; + case 4: + if (port & 3) + return -EINVAL; + *((u32 *)val) = inl(port); + return 4; + } + return -EINVAL; +} + +int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, size_t size) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + port += hose->io_space->start; + + switch (size) { + case 1: + outb(port, val); + return 1; + case 2: + if (port & 1) + return -EINVAL; + outw(port, val); + return 2; + case 4: + if (port & 3) + return -EINVAL; + outl(port, val); + return 4; + } + return -EINVAL; +} diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c new file mode 100644 index 000000000000..3db9816e19f1 --- /dev/null +++ b/arch/sw_64/pci/pci.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#include +#include + +/* + * raw_pci_read/write - Platform-specific PCI config space access. + */ +int raw_pci_read(unsigned int domain, unsigned int bus, unsigned int devfn, + int reg, int len, u32 *val) +{ + struct pci_bus *bus_tmp = pci_find_bus(domain, bus); + + if (bus_tmp) + return bus_tmp->ops->read(bus_tmp, devfn, reg, len, val); + + return -EINVAL; +} + +int raw_pci_write(unsigned int domain, unsigned int bus, unsigned int devfn, + int reg, int len, u32 val) +{ + struct pci_bus *bus_tmp = pci_find_bus(domain, bus); + + if (bus_tmp) + return bus_tmp->ops->write(bus_tmp, devfn, reg, len, val); + + return -EINVAL; +} + +resource_size_t pcibios_default_alignment(void) +{ + if (is_in_guest()) + return PAGE_SIZE; + else + return 0; +} + +/** + * Just declaring that the power-of-ten prefixes are actually the + * power-of-two ones doesn't make it true :) + */ +#define KB 1024 +#define MB (1024*KB) +#define GB (1024*MB) + +resource_size_t pcibios_align_resource(void *data, const struct resource *res, + resource_size_t size, resource_size_t align) +{ + struct pci_dev *dev = data; + struct pci_controller *hose = pci_bus_to_pci_controller(dev->bus); + unsigned long alignto; + resource_size_t start = res->start; + + if (res->flags & IORESOURCE_IO) { + /* Make sure we start at our min on all hoses */ + if (start - hose->io_space->start < PCIBIOS_MIN_IO) + start = PCIBIOS_MIN_IO + hose->io_space->start; + /* + * Put everything into 0x00-0xff region modulo 0x400 + */ + if (start & 0x300) + start = (start + 0x3ff) & ~0x3ff; + } else if (res->flags & IORESOURCE_MEM) { + /* Make sure we start at our min on all hoses */ + if (start - hose->mem_space->start < PCIBIOS_MIN_MEM) + start = PCIBIOS_MIN_MEM + hose->mem_space->start; + /* + * The following holds at least for the Low Cost + * SW64 implementation of the PCI interface: + * + * In sparse memory address space, the first + * octant (16MB) of every 128MB segment is + * aliased to the very first 16 MB of the + * address space (i.e., it aliases the ISA + * memory address space). Thus, we try to + * avoid allocating PCI devices in that range. + * Can be allocated in 2nd-7th octant only. + * Devices that need more than 112MB of + * address space must be accessed through + * dense memory space only! + */ + + /* Align to multiple of size of minimum base. */ + alignto = max_t(resource_size_t, 0x1000UL, align); + start = ALIGN(start, alignto); + if (hose->sparse_mem_base && size <= 7 * 16*MB) { + if (((start / (16*MB)) & 0x7) == 0) { + start &= ~(128*MB - 1); + start += 16*MB; + start = ALIGN(start, alignto); + } + if (start/(128*MB) != (start + size - 1)/(128*MB)) { + start &= ~(128*MB - 1); + start += (128 + 16)*MB; + start = ALIGN(start, alignto); + } + } + } + + return start; +} + +#undef KB +#undef MB +#undef GB + +char *pcibios_setup(char *str) +{ + return str; +} + +void pcibios_fixup_bus(struct pci_bus *bus) +{ + /* Propagate hose info into the subordinate devices. */ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + struct pci_dev *dev = bus->self; + + if (!dev || bus->number == hose->first_busno) { + bus->resource[0] = hose->io_space; + bus->resource[1] = hose->mem_space; + bus->resource[2] = hose->pre_mem_space; + } +} + +/** + * Provide information on locations of various I/O regions in physical + * memory. Do this on a per-card basis so that we choose the right hose. + */ +asmlinkage long sys_pciconfig_iobase(long which, unsigned long bus, unsigned long dfn) +{ + struct pci_controller *hose; + + hose = bus_num_to_pci_controller(bus); + if (hose == NULL) + return -ENODEV; + + switch (which & ~IOBASE_FROM_HOSE) { + case IOBASE_HOSE: + return hose->index; + case IOBASE_SPARSE_MEM: + return hose->sparse_mem_base; + case IOBASE_DENSE_MEM: + return hose->dense_mem_base; + case IOBASE_SPARSE_IO: + return hose->sparse_io_base; + case IOBASE_DENSE_IO: + return hose->dense_io_base; + case IOBASE_ROOT_BUS: + return hose->bus->number; + } + + return -EOPNOTSUPP; +} + +void pci_iounmap(struct pci_dev *dev, void __iomem *addr) +{ +} +EXPORT_SYMBOL(pci_iounmap); + +void __init reserve_mem_for_pci(void) +{ + int ret; + unsigned long base = PCI_32BIT_MEMIO; + + ret = add_memmap_region(base, PCI_32BIT_MEMIO_SIZE, memmap_pci); + if (ret) { + pr_err("reserved pages for pcie memory space failed\n"); + return; + } + + pr_info("reserved pages for pcie memory space %lx:%lx\n", base >> PAGE_SHIFT, + (base + PCI_32BIT_MEMIO_SIZE) >> PAGE_SHIFT); +} + +const struct dma_map_ops *dma_ops; +EXPORT_SYMBOL(dma_ops); + +/* Quirks */ +static void quirk_isa_bridge(struct pci_dev *dev) +{ + dev->class = PCI_CLASS_BRIDGE_ISA << 8; +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82378, quirk_isa_bridge); + +/* + * Early fix up the Root Complex settings + */ +static void fixup_root_complex(struct pci_dev *dev) +{ + int i; + struct pci_bus *bus = dev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + hose->self_busno = hose->busn_space->start; + + if (likely(bus->number == hose->self_busno)) { + if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) { + /* Check Root Complex port again */ + dev->is_hotplug_bridge = 0; + dev->current_state = PCI_D0; + } + + dev->class &= 0xff; + dev->class |= PCI_CLASS_BRIDGE_PCI << 8; + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + dev->resource[i].start = 0; + dev->resource[i].end = 0; + dev->resource[i].flags = IORESOURCE_PCI_FIXED; + } + } + atomic_inc(&dev->enable_cnt); + + dev->no_msi = 1; +} + +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, fixup_root_complex); + +static int setup_bus_dma_cb(struct pci_dev *pdev, void *data) +{ + pdev->dev.bus_dma_limit = DMA_BIT_MASK(32); + return 0; +} + +static void fix_bus_dma_limit(struct pci_dev *dev) +{ + pci_walk_bus(dev->subordinate, setup_bus_dma_cb, NULL); + pr_info("Set zx200 bus_dma_limit to 32-bit\n"); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ZHAOXIN, 0x071f, fix_bus_dma_limit); + +#ifdef CONFIG_DCA +static void enable_sw_dca(struct pci_dev *dev) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(dev->bus); + unsigned long node, rc_index, dca_ctl, dca_conf; + int i; + + if (dev->class >> 8 != PCI_CLASS_NETWORK_ETHERNET) + return; + + node = hose->node; + rc_index = hose->index; + + for (i = 0; i < 256; i++) { + dca_conf = read_piu_ior1(node, rc_index, DEVICEID0 + (i << 7)); + if (dca_conf >> 63) + continue; + else { + dca_conf = (1UL << 63) | (dev->bus->number << 8) | dev->devfn; + pr_info("dca device index %d, dca_conf = %#lx\n", i, dca_conf); + write_piu_ior1(node, rc_index, DEVICEID0 + (i << 7), dca_conf); + break; + } + } + + dca_ctl = read_piu_ior1(node, rc_index, DCACONTROL); + if (dca_ctl & 0x1) { + dca_ctl = 0x2; + write_piu_ior1(node, rc_index, DCACONTROL, dca_ctl); + pr_info("Node %ld RC %ld enable DCA 1.0\n", node, rc_index); + } +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, enable_sw_dca); +#endif + +/** + * There are some special aspects to the Root Complex of Sunway: + * 1. Root Complex config space base addr is different + * from EP config space base addr. + * 2. For the case of multiple Root Complex, different + * Root Complex have config space base addr. + * + * These means that even if multiple Root Complex share + * the same segment group number, their bus numbers can + * still overlap. + * + * But due to a Xorg related issue, we can not overlap + * the bus numbers of multiple Root Complex. So, after + * scanning the Root Complex, use "last_bus" to record + * the next bus number of the current maximum used bus + * number, and use it as the start bus number of the + * next Root Complex to be scanned. + * + * A question: when there is too much RCs, may 256 bus + * numbers be insufficient? + */ +static unsigned char last_bus; + +void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge) +{ + struct pci_controller *hose = NULL; + struct resource_entry *entry = NULL; + struct pci_bus *bus = bridge->bus; + unsigned long flags = 0; + unsigned int init_busnr = 0; + + hose = pci_bus_to_pci_controller(bus); + + resource_list_for_each_entry(entry, &bridge->windows) { + flags = entry->res->flags; + if (flags & IORESOURCE_IO) { + entry->offset = entry->res->start; + hose->io_space = entry->res; + } else if (flags & IORESOURCE_BUS) { + entry->res->start = last_bus; + hose->busn_space = entry->res; + } else if (flags & IORESOURCE_MEM) { + if (!(flags & IORESOURCE_PREFETCH)) { + entry->offset = entry->res->start - PCI_32BIT_MEMIO; + hose->mem_space = entry->res; + } else + hose->pre_mem_space = entry->res; + } + } + + /** + * We scan Root Complex and update bus num in kernel, + * not in firmware. Firmware just pass 0x0-0xff via _CRS. + * + * So, need to update bus num of pci host bridge here. + */ + bridge->busnr = last_bus; + dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus), last_bus); + + /** + * At this point, pci_bus has been created and use old + * bridge->busnr, so need to update bus->number here. + */ + bus->number = last_bus; + + bridge->swizzle_irq = pci_common_swizzle; + bridge->map_irq = sw64_pci_map_irq; + + init_busnr = (0xff << 16) + ((last_bus + 1) << 8) + (last_bus); + write_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS, init_busnr); + + hose->first_busno = last_bus + (is_in_host() ? 1 : 0); + + pci_add_flags(PCI_REASSIGN_ALL_BUS); +} + +static void sw64_pci_root_bridge_reserve_legacy_io(struct pci_host_bridge *bridge) +{ + struct pci_bus *bus = bridge->bus; + struct resource_entry *entry = NULL; + struct resource *res = NULL; + + resource_list_for_each_entry(entry, &bridge->windows) { + if (!(entry->res->flags & IORESOURCE_IO)) + continue; + + res = kzalloc(sizeof(struct resource), GFP_KERNEL); + if (res == NULL) { + pr_err("alloc resource for legacy io out of mem\n"); + return; + } + + res->name = "legacy io"; + res->flags = IORESOURCE_IO; + res->start = entry->res->start; + res->end = (res->start + 0xFFF) & 0xFFFFFFFFFFFFFFFFUL; + + pr_info("reserving legacy io %pR for domain %04x\n", + res, pci_domain_nr(bus)); + if (request_resource(entry->res, res)) { + pr_err("pci %04x:%02x reserve legacy io %pR failed\n", + pci_domain_nr(bus), bus->number, res); + kfree(res); + } + } +} + +void sw64_pci_root_bridge_scan_finish_up(struct pci_host_bridge *bridge) +{ + struct pci_controller *hose = NULL; + struct pci_bus *bus = NULL; + unsigned int init_busnr = 0; + + bus = bridge->bus; + + hose = pci_bus_to_pci_controller(bus); + hose->bus = bus; + + if (is_in_host()) + last_bus = chip_pcie_configure(hose); + else { + while (pci_find_bus(pci_domain_nr(bus), last_bus)) + last_bus++; + } + + hose->last_busno = last_bus; + hose->busn_space->end = last_bus; + + init_busnr = read_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS); + init_busnr &= ~(0xff << 16); + init_busnr |= last_bus << 16; + write_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS, init_busnr); + + pci_bus_update_busn_res_end(bus, last_bus); + last_bus++; + + pr_info("bus number update to %u\n", last_bus); + + if (is_in_host()) + sw64_pci_root_bridge_reserve_legacy_io(bridge); + + /** + * Root Complex of SW64 does not support ASPM, causing + * control field(_OSC) unable to be updated. + * + * Related logic can be found in "negotiate_os_control". + */ + bridge->native_aer = 1; + bridge->native_pme = 1; + + /** + * Since some buggy firmwares may configure invalid bridge bus numbers, + * the kernel re-assigns all PCI bus numbers when scan Root Complex. + * + * However, users may trigger a pci bus rescan in the userspace by the + * command below: + * + * > echo 1 > /sys/bus/pci/rescan + * + * Unexpected errors may occur on the endpoint devices due to the re-assign + * bus numbers of upstream bridges. + * + * To work around this problem, the flag PCI_REASSIGN_ALL_BUS is set before + * scanning Root Complex and cleared after scanning Root Complex. + */ + pci_clear_flags(PCI_REASSIGN_ALL_BUS); +} -- Gitee From adc5beeee53fca94287c96c85a120cc76f060ec8 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:28 +0800 Subject: [PATCH 029/524] sw64: add MSI support Add basic MSI support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/msi.h | 93 ++++++++++++++++++++++++++++++++++++ arch/sw_64/pci/msi.c | 21 ++++++++ 2 files changed, 114 insertions(+) create mode 100644 arch/sw_64/include/asm/msi.h create mode 100644 arch/sw_64/pci/msi.c diff --git a/arch/sw_64/include/asm/msi.h b/arch/sw_64/include/asm/msi.h new file mode 100644 index 000000000000..dbf6f81843be --- /dev/null +++ b/arch/sw_64/include/asm/msi.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_MSI_H +#define _ASM_SW64_MSI_H + +#include + +#define NR_VECTORS NR_IRQS +#define NR_IRQ_VECTORS NR_IRQS + +#define AUTO_ASSIGN 0 + +#define LAST_DEVICE_VECTOR 31 + +#define MSI_OFFSET 0x44 + +#define NUM_MSI_IRQS 256 + +#define PERCPU_MSI_IRQS 256 + +#define VT_MSIX_MSG_ADDR (0x8000fee00000UL) +#define VT_MSIX_ADDR_DEST_ID_SHIFT 12 +#define VT_MSIX_ADDR_DEST_ID_MASK (0xff << VT_MSIX_ADDR_DEST_ID_SHIFT) +#define VT_MSIX_ADDR_DEST_ID(dest) \ + (((dest) << VT_MSIX_ADDR_DEST_ID_SHIFT) & VT_MSIX_ADDR_DEST_ID_MASK) + + +#ifdef CONFIG_PCI_MSI +extern void vt_sw64_vector_free_irqs(unsigned int virq, unsigned int nr_irqs); +extern int sw64_setup_vt_msi_irqs(struct pci_dev *dev, int nvec, int type); +extern bool find_free_cpu_vector(const struct cpumask *search_mask, + int *found_cpu, int *found_vector); +extern int msi_compose_msg(unsigned int irq, struct msi_msg *msg); +extern void sw64_irq_noop(struct irq_data *d); +extern struct irq_chip sw64_irq_chip; +extern void handle_pci_msi_interrupt(unsigned long type, + unsigned long vector, + unsigned long pci_msi1_addr); + +#define MSI_ADDR_BASE_HI 0 +#define MSI_ADDR_BASE_LO 0x91abc0 + +#define MSI_ADDR_SHIFT 20 +#define MSI_ADDR_DEST_ID_SHIFT 10 + +struct sw64_msi_chip_data { + spinlock_t cdata_lock; + union { + unsigned long msi_config; + unsigned long msiaddr; + }; + unsigned long rc_node; + unsigned long rc_index; + unsigned int msi_config_index; + unsigned int dst_cpu; + unsigned int vector; + unsigned int prev_cpu; + unsigned int prev_vector; + unsigned int multi_msi; + bool move_in_progress; +}; + +static inline int rcid_to_msicid(int rcid) +{ + int msicid = 0; + + msicid |= (rcid_to_domain_id(rcid) << 7); + msicid |= (rcid_to_thread_id(rcid) << 6); + msicid |= (rcid_to_core_id(rcid) << 0); + + return msicid; +} + +extern void arch_init_msi_domain(struct irq_domain *domain); +enum irq_alloc_type { + IRQ_ALLOC_TYPE_MSI, + IRQ_ALLOC_TYPE_MSIX, + IRQ_ALLOC_TYPE_INTX, +}; +struct irq_alloc_info { + struct msi_desc *desc; + enum irq_alloc_type type; + struct pci_dev *msi_dev; + irq_hw_number_t hwirq; +}; +typedef struct irq_alloc_info msi_alloc_info_t; +#else /* !CONFIG_PCI_MSI */ +static inline void handle_pci_msi_interrupt(unsigned long type, + unsigned long vector, unsigned long pci_msi1_addr) +{ + pr_warn("SW arch disable CONFIG_PCI_MSI option.\n"); +} +#endif /* CONFIG_PCI_MSI */ +#endif /* _ASM_SW64_MSI_H */ diff --git a/arch/sw_64/pci/msi.c b/arch/sw_64/pci/msi.c new file mode 100644 index 000000000000..fc2c122c37ef --- /dev/null +++ b/arch/sw_64/pci/msi.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +int msi_compose_msg(unsigned int irq, struct msi_msg *msg) +{ + msg->address_hi = (unsigned int)(MSIX_MSG_ADDR >> 32); + msg->address_lo = (unsigned int)(MSIX_MSG_ADDR & 0xffffffff); + msg->data = irq; + return irq; +} + +void sw64_irq_noop(struct irq_data *d) +{ +} + +void arch_teardown_msi_irq(unsigned int irq) +{ +} -- Gitee From 22edce0e586a1a28f9de26f47b993792a963c943 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:06 +0800 Subject: [PATCH 030/524] sw64: add device trees Add device trees for SW64 based chip3 platform and virtual machines (including an empty device tree for platforms that are under development). Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/chip3.dts | 240 ++++++++++++++++++++++++++++++++ arch/sw_64/boot/dts/chip_vt.dts | 55 ++++++++ arch/sw_64/boot/dts/empty.dts | 15 ++ 3 files changed, 310 insertions(+) create mode 100644 arch/sw_64/boot/dts/chip3.dts create mode 100644 arch/sw_64/boot/dts/chip_vt.dts create mode 100644 arch/sw_64/boot/dts/empty.dts diff --git a/arch/sw_64/boot/dts/chip3.dts b/arch/sw_64/boot/dts/chip3.dts new file mode 100644 index 000000000000..082506393ac9 --- /dev/null +++ b/arch/sw_64/boot/dts/chip3.dts @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Default device tree; + */ + +/dts-v1/; +/ { + compatible = "sunway,chip3"; + model = "chip3"; + #address-cells = <2>; + #size-cells = <2>; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + clocks { + i2cclk: i2cclk { + compatible = "fixed-clock"; + clock-frequency = <25000000>; + #clock-cells = <0>; + clock-output-names = "i2cclk_25mhz"; + }; + spiclk: spiclk { + compatible = "fixed-clock"; + clock-frequency = <25000000>; + #clock-cells = <0>; + clock-output-names = "spiclk_25mhz"; + }; + + }; + + intc: interrupt-controller { + compatible = "sw64,sw6_irq_controller"; + interrupt-controller; + #interrupt-cells = <1>; + }; + + lpc_intc: interrupt-controller@0x8037 { + compatible = "sw64,lpc_intc"; + reg = <0x8037 0x40000000 0x0 0x8000>; + interrupt-controller; + #interrupt-cells = <1>; + interrupt-parent = <&intc>; + interrupts = <2>; + }; + + uart: serial0@8033 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sw6,sunway-apb-uart"; + reg = <0x8033 0x0 0x0 0x1000>; + interrupt-parent=<&intc>; + interrupts = <3>; + reg-shift = <9>; + reg-io-width = <4>; + clock-frequency = <24000000>; + status = "okay"; + }; + + serial1@9033 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sw6,sunway-apb-uart"; + reg = <0x9033 0x0 0x0 0x1000>; + reg-shift = <9>; + reg-io-width = <4>; + clock-frequency = <24000000>; + status = "okay"; + }; + + + i2c0@0x8031 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "snps,designware-i2c"; + reg = <0x8031 0x0 0x0 0x8000>; + clock-frequency = <100000>; + clocks = <&i2cclk>; + interrupt-parent=<&intc>; + interrupts = <5>; + status = "okay"; + }; + + i2c1@0x8034 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,designware-i2c"; + reg = <0x8034 0x0 0x0 0x8000>; + clock-frequency = <100000>; + clocks = <&i2cclk>; + interrupt-parent=<&intc>; + interrupts = <6>; + status = "okay"; + }; + + i2c2@0x8035 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,designware-i2c"; + reg = <0x8035 0x0 0x0 0x8000>; + clock-frequency = <100000>; + clocks = <&i2cclk>; + interrupt-parent=<&intc>; + interrupts = <7>; + status = "okay"; + + rtc: pcf8523@68 { + compatible = "nxp,pcf8523"; + reg = <0x68>; + }; + + lm75: at30tse752a@48 { + compatible = "microchip,tcn75"; + reg = <0x48>; + }; + }; + + pvt: pvt@0x8030 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sw64,pvt-vol"; + reg = <0x8030 0x0 0x0 0x7c00>; + status = "okay"; + }; + + spi: spi@0x8032 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sunway,chip3-spi"; + reg = <0x8032 0x0 0x0 0x8000>; + clocks = <&spiclk>; + interrupt-parent=<&intc>; + interrupts = <4>; + status = "okay"; + + flash@0 { + compatible = "winbond,w25q32dw", "jedec,spi-flash"; + spi-max-frequency = <25000000>; + m25p,fast-read; + spi-cpha; + spi-cpol; + poll_mode = <1>; /* poll_mode:1 interrupt mode: 0 */ + reg-io-width = <2>; + reg = <0 0 0 0 >; /* 0: flash chip selected bit */ + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "spares0"; + reg = <0 0x400000>; + }; + }; + }; + + flash@1 { + compatible = "winbond,w25q32dw", "jedec,spi-flash"; + spi-max-frequency = <25000000>; + m25p,fast-read; + spi-cpha; + spi-cpol; + poll_mode = <1>; /* poll_mode:1 interrupt mode: 0 */ + reg-io-width = <2>; + reg = <1 0 0 0 >; /* 1: flash chip selected bit */ + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "spares1"; + reg = <0 0x400000>; + }; + }; + }; + }; + + lpc: lpc@0x8037 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sunway,chip3_lpc"; + reg = <0x8037 0x40000000 0x0 0x8000>; + status = "okay"; + + }; + + ipmi-kcs@0x8037 { + #address-cells = <2>; + #size-cells = <2>; + device_type = "ipmi"; + compatible = "ipmi-kcs"; + reg = <0x8037 0x10000ca2 0x0 0x10>; + reg-size = <1>; + reg-spacing = <1>; + reg-shift = <0>; + status = "disabled"; + }; + + ipmi-bt@0x8037 { + #address-cells = <2>; + #size-cells = <2>; + device_type = "ipmi"; + compatible = "ipmi-bt"; + reg = <0x8037 0x100000e4 0x0 0x10>; + interrupt-parent=<&lpc_intc>; + interrupts = <10>; + reg-size = <1>; + reg-spacing = <1>; + reg-shift = <0>; + status = "disabled"; + }; + + gpio: gpio@8036 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "snps,sw-gpio"; + reg = <0x8036 0x0 0x0 0x8000>; + status = "okay"; + + porta: gpio-contraller@0 { + compatible = "snps,dw-apb-gpio-port"; + gpio-controller; + #gpio-cells = <2>; + snps,nr-gpios = <8>; + reg = <0 0 0 0>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent=<&intc>; + interrupts = <0>; + }; + }; + + }; +}; diff --git a/arch/sw_64/boot/dts/chip_vt.dts b/arch/sw_64/boot/dts/chip_vt.dts new file mode 100644 index 000000000000..f26285367f98 --- /dev/null +++ b/arch/sw_64/boot/dts/chip_vt.dts @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Default device tree; + */ + +/dts-v1/; +/ { + compatible = "sunway,chip3"; + model = "chip3"; + #address-cells = <2>; + #size-cells = <2>; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + intc: interrupt-controller{ + compatible = "sw64,sw6_irq_vt_controller"; + interrupt-controller; + #interrupt-cells = <1>; + }; + + uart: serial0@8801 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "ns16550a"; + reg = <0x8801 0x3f8 0x0 0x10>; + interrupt-parent=<&intc>; + interrupts = <12>; + reg-shift = <0>; + reg-io-width = <1>; + clock-frequency = <24000000>; + status = "okay"; + }; + misc: misc0@8036 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sw6,sunway-ged"; + reg = <0x8036 0x0 0x0 0x20>; + interrupt-parent=<&intc>; + interrupts = <13>; + reg-shift = <0>; + reg-io-width = <8>; + clock-frequency = <24000000>; + status = "okay"; + }; + fw_cfg: fw_cfg@8049 { + dma-coherent; + reg = <0x8049 0x20000000 0x0 0x18>; + compatible = "qemu,fw-cfg-mmio"; + }; + }; +}; diff --git a/arch/sw_64/boot/dts/empty.dts b/arch/sw_64/boot/dts/empty.dts new file mode 100644 index 000000000000..f8fe34e29641 --- /dev/null +++ b/arch/sw_64/boot/dts/empty.dts @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Default device tree; + */ + +/dts-v1/; +/ { + compatible = "sunway,chip3"; + model = "chip3"; + #address-cells = <2>; + #size-cells = <2>; + + soc { + }; +}; -- Gitee From 7606640a73836a20ab87f0fe5154f4a86db34f28 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:00 +0800 Subject: [PATCH 031/524] sw64: add ACPI support Add basic ACPI support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/acenv.h | 40 +++++ arch/sw_64/include/asm/acpi.h | 117 +++++++++++++ arch/sw_64/kernel/acpi.c | 304 +++++++++++++++++++++++++++++++++ arch/sw_64/pci/acpi.c | 245 ++++++++++++++++++++++++++ 4 files changed, 706 insertions(+) create mode 100644 arch/sw_64/include/asm/acenv.h create mode 100644 arch/sw_64/include/asm/acpi.h create mode 100644 arch/sw_64/kernel/acpi.c create mode 100644 arch/sw_64/pci/acpi.c diff --git a/arch/sw_64/include/asm/acenv.h b/arch/sw_64/include/asm/acenv.h new file mode 100644 index 000000000000..53b2898718fe --- /dev/null +++ b/arch/sw_64/include/asm/acenv.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SW64_ACENV_H +#define _ASM_SW64_ACENV_H + +#define COMPILER_DEPENDENT_INT64 long +#define COMPILER_DEPENDENT_UINT64 unsigned long + +/* + * Calling conventions: + * + * ACPI_SYSTEM_XFACE - Interfaces to host OS (handlers, threads) + * ACPI_EXTERNAL_XFACE - External ACPI interfaces + * ACPI_INTERNAL_XFACE - Internal ACPI interfaces + * ACPI_INTERNAL_VAR_XFACE - Internal variable-parameter list interfaces + */ +#define ACPI_SYSTEM_XFACE +#define ACPI_EXTERNAL_XFACE +#define ACPI_INTERNAL_XFACE +#define ACPI_INTERNAL_VAR_XFACE + +/* Asm macros */ +#define ACPI_FLUSH_CPU_CACHE() + +int __acpi_acquire_global_lock(unsigned int *lock); +int __acpi_release_global_lock(unsigned int *lock); + +#define ACPI_ACQUIRE_GLOBAL_LOCK(facs, Acq) \ + ((Acq) = __acpi_acquire_global_lock(&facs->global_lock)) + +#define ACPI_RELEASE_GLOBAL_LOCK(facs, Acq) \ + ((Acq) = __acpi_release_global_lock(&facs->global_lock)) + +/* + * Math helper asm macros + */ +#define ACPI_DIV_64_BY_32(n_hi, n_lo, d32, q32, r32) + +#define ACPI_SHIFT_RIGHT_64(n_hi, n_lo) +#endif /* _ASM_SW64_ACENV_H */ diff --git a/arch/sw_64/include/asm/acpi.h b/arch/sw_64/include/asm/acpi.h new file mode 100644 index 000000000000..ef46f481e1fd --- /dev/null +++ b/arch/sw_64/include/asm/acpi.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SW64_ACPI_H +#define _ASM_SW64_ACPI_H + +#include +#include +#include +#include + +#ifdef CONFIG_ACPI +extern int acpi_noirq; +extern int acpi_strict; +extern int acpi_disabled; +extern int acpi_pci_disabled; + +/* _ASM_SW64_PDC_H */ +#define ACPI_PDC_P_FFH (0x0001) +#define ACPI_PDC_C_C1_HALT (0x0002) +#define ACPI_PDC_T_FFH (0x0004) +#define ACPI_PDC_SMP_C1PT (0x0008) +#define ACPI_PDC_SMP_C2C3 (0x0010) +#define ACPI_PDC_SMP_P_SWCOORD (0x0020) +#define ACPI_PDC_SMP_C_SWCOORD (0x0040) +#define ACPI_PDC_SMP_T_SWCOORD (0x0080) +#define ACPI_PDC_C_C1_FFH (0x0100) +#define ACPI_PDC_C_C2C3_FFH (0x0200) +#define ACPI_PDC_SMP_P_HWCOORD (0x0800) + +#define ACPI_PDC_EST_CAPABILITY_SMP (ACPI_PDC_SMP_C1PT | \ + ACPI_PDC_C_C1_HALT | \ + ACPI_PDC_P_FFH) + +#define ACPI_PDC_EST_CAPABILITY_SWSMP (ACPI_PDC_SMP_C1PT | \ + ACPI_PDC_C_C1_HALT | \ + ACPI_PDC_SMP_P_SWCOORD | \ + ACPI_PDC_SMP_P_HWCOORD | \ + ACPI_PDC_P_FFH) + +#define ACPI_PDC_C_CAPABILITY_SMP (ACPI_PDC_SMP_C2C3 | \ + ACPI_PDC_SMP_C1PT | \ + ACPI_PDC_C_C1_HALT | \ + ACPI_PDC_C_C1_FFH | \ + ACPI_PDC_C_C2C3_FFH) + +#define ACPI_TABLE_UPGRADE_MAX_PHYS MEMBLOCK_ALLOC_ACCESSIBLE + +/** + * Use the number 64 is just because this number is the most + * frequently used number in other architectures. Actually, + * SW64 does not have fixmap area in memory layout. + */ +#define NR_FIX_BTMAPS 64 + +static inline void disable_acpi(void) +{ + acpi_disabled = 1; + acpi_pci_disabled = 1; + acpi_noirq = 1; +} + +static inline void enable_acpi(void) +{ + acpi_disabled = 0; + acpi_pci_disabled = 0; + acpi_noirq = 0; +} + +static inline void acpi_noirq_set(void) +{ + acpi_noirq = 1; +} + +static inline void acpi_disable_pci(void) +{ + acpi_pci_disabled = 1; + acpi_noirq_set(); +} + +static inline bool acpi_has_cpu_in_madt(void) +{ + return true; +} + +/* Low-level suspend routine. */ +extern int (*acpi_suspend_lowlevel)(void); +extern unsigned long long arch_acpi_wakeup_start; + +/* Physical address to resume after wakeup */ +#define acpi_wakeup_address arch_acpi_wakeup_start + +/* + * Check if the CPU can handle C2 and deeper + */ +static inline unsigned int acpi_processor_cstate_check(unsigned int max_cstate) +{ + return max_cstate; +} + +static inline bool arch_has_acpi_pdc(void) +{ + return false; +} + +static inline void arch_acpi_set_pdc_bits(u32 *buf) +{ +} +#else /* !CONFIG_ACPI */ + +static inline void acpi_noirq_set(void) { } +static inline void acpi_disable_pci(void) { } +static inline void disable_acpi(void) { } + +#endif /* !CONFIG_ACPI */ + +#define acpi_unlazy_tlb(x) +#endif /* _ASM_SW64_ACPI_H */ diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c new file mode 100644 index 000000000000..9779d4bdea0d --- /dev/null +++ b/arch/sw_64/kernel/acpi.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include + +#include + +#ifdef CONFIG_ACPI_HOTPLUG_CPU +#include +#endif + +int acpi_disabled = 1; +EXPORT_SYMBOL(acpi_disabled); + +int acpi_noirq = 1; /* skip ACPI IRQ initialization */ +int acpi_pci_disabled = 1; /* skip ACPI PCI scan and IRQ initialization */ +EXPORT_SYMBOL(acpi_pci_disabled); + +static bool param_acpi_on __initdata; +static bool param_acpi_off __initdata; + +int acpi_strict; +u64 arch_acpi_wakeup_start; +u64 acpi_saved_sp_s3; + +#define MAX_LOCAL_APIC 256 + +#define PREFIX "ACPI: " +/* + * The default interrupt routing model is PIC (8259). This gets + * overridden if IOAPICs are enumerated (below). + */ +enum acpi_irq_model_id acpi_irq_model = ACPI_IRQ_MODEL_IOSAPIC; +void __iomem *__init __acpi_map_table(unsigned long phys, unsigned long size) +{ + if (!phys || !size) + return NULL; + + return early_ioremap(phys, size); +} +void __init __acpi_unmap_table(void __iomem *map, unsigned long size) +{ + if (!map || !size) + return; + + early_iounmap(map, size); +} +/* + * Following __acpi_xx functions should be implemented for sepecific cpu. + */ +int acpi_gsi_to_irq(u32 gsi, unsigned int *irqp) +{ + if (irqp != NULL) + *irqp = acpi_register_gsi(NULL, gsi, -1, -1); + + return 0; +} +EXPORT_SYMBOL_GPL(acpi_gsi_to_irq); + +int acpi_isa_irq_to_gsi(unsigned int isa_irq, u32 *gsi) +{ + if (gsi) + *gsi = isa_irq; + + return 0; +} + +int (*acpi_suspend_lowlevel)(void); + +/* + * success: return IRQ number (>=0) + * failure: return < 0 + */ +static struct irq_domain *irq_default_domain; +int acpi_register_gsi(struct device *dev, u32 gsi, int trigger, int polarity) +{ + u32 irq; + + irq = irq_find_mapping(irq_default_domain, gsi); + + return irq; +} +EXPORT_SYMBOL_GPL(acpi_register_gsi); + +void acpi_unregister_gsi(u32 gsi) +{ + +} +EXPORT_SYMBOL_GPL(acpi_unregister_gsi); + +/* + * ACPI based hotplug support for CPU + */ +#ifdef CONFIG_ACPI_HOTPLUG_CPU +/* wrapper to silence section mismatch warning */ +int __ref acpi_map_lsapic(acpi_handle handle, int physid, int *pcpu) +{ + return 0; +} +EXPORT_SYMBOL(acpi_map_lsapic); + +int acpi_unmap_lsapic(int cpu) +{ + return 0; +} +EXPORT_SYMBOL(acpi_unmap_lsapic); +#endif /* CONFIG_ACPI_HOTPLUG_CPU */ + +u8 acpi_checksum(u8 *table, u32 length) +{ + u8 ret = 0; + + while (length--) { + ret += *table; + table++; + } + return -ret; +} + +static int __init parse_acpi(char *arg) +{ + if (!arg) + return -EINVAL; + + /* disable both ACPI table parsing and interpreter */ + if (strcmp(arg, "off") == 0) + param_acpi_off = true; + else if (strcmp(arg, "on") == 0) /* prefer ACPI over device tree */ + param_acpi_on = true; + else + return -EINVAL; /* Core will printk when we return error. */ + + return 0; +} +early_param("acpi", parse_acpi); + +/* + * __acpi_acquire_global_lock + * will always return -1 indicating owning the lock. + * + * __acpi_release_global_lock will always return 0 indicating + * no acquring request pending. + */ +int __acpi_acquire_global_lock(unsigned int *lock) +{ + return -1; +} + +int __acpi_release_global_lock(unsigned int *lock) +{ + return 0; +} + +#ifdef CONFIG_ACPI_NUMA +static int rcid_to_cpu(int physical_id) +{ + int i; + + for (i = 0; i < NR_CPUS; ++i) { + if (__cpu_to_rcid[i] == physical_id) + return i; + } + + /* physical id not found */ + return -1; +} + +/* Callback for Proximity Domain -> CPUID mapping */ +void __init +acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) +{ + int pxm, node; + int cpu; // logical core id + + if (srat_disabled()) + return; + if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) { + bad_srat(); + return; + } + if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0) + return; + pxm = pa->proximity_domain_lo; + if (acpi_srat_revision >= 2) { + pxm |= (pa->proximity_domain_hi[0] << 8); + pxm |= (pa->proximity_domain_hi[1] << 16); + pxm |= (pa->proximity_domain_hi[2] << 24); + } + + node = acpi_map_pxm_to_node(pxm); + if (node < 0) { + pr_err("SRAT: Too many proximity domains %x\n", pxm); + bad_srat(); + return; + } + + if (pa->apic_id >= CONFIG_NR_CPUS) { + pr_err("SRAT: PXM %u -> CPU 0x%02x -> Node %u skipped apicid that is too big\n", pxm, pa->apic_id, node); + return; + } + + /* Record the mapping from logical core id to node id */ + cpu = rcid_to_cpu(pa->apic_id); + if (cpu < 0) { + pr_err("SRAT: Can not find the logical id for physical Core 0x%02x\n", pa->apic_id); + return; + } + + early_map_cpu_to_node(cpu, node); + + node_set(node, numa_nodes_parsed); + pr_info("SRAT: PXM %u -> CPU 0x%02x -> Node %u\n", pxm, pa->apic_id, node); +} + +#ifdef CONFIG_MEMORY_HOTPLUG +static inline int save_add_info(void) { return 1; } +#else +static inline int save_add_info(void) { return 0; } +#endif + +#endif + +void __init arch_reserve_mem_area(acpi_physical_address addr, size_t size) +{ +} + +#ifdef CONFIG_ACPI_HOTPLUG_CPU +static int acpi_map_cpu2node(acpi_handle handle, int cpu, int physid) +{ +#ifdef CONFIG_ACPI_NUMA + int nid; + + nid = acpi_get_node(handle); + if (nid != NUMA_NO_NODE) { + set_cpuid_to_node(cpu, nid); + node_set(nid, numa_nodes_parsed); + } +#endif + return 0; +} + +int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id, + int *pcpu) +{ + int cpu; + struct acpi_madt_local_apic *processor; + + processor = kzalloc(sizeof(struct acpi_madt_local_apic), GFP_KERNEL); + processor->id = physid; + processor->processor_id = acpi_id; + processor->lapic_flags = ACPI_MADT_ENABLED; + + cpu = set_processor_mask(processor); + if (cpu < 0) { + pr_info(PREFIX "Unable to map lapic to logical cpu number\n"); + return cpu; + } + + acpi_map_cpu2node(handle, cpu, physid); + + *pcpu = cpu; + return 0; +} +EXPORT_SYMBOL(acpi_map_cpu); + +int acpi_unmap_cpu(int cpu) +{ +#ifdef CONFIG_ACPI_NUMA + set_cpuid_to_node(cpu, NUMA_NO_NODE); +#endif + set_cpu_present(cpu, false); + num_processors--; + + pr_info("cpu%d hot remove!\n", cpu); + + return 0; +} +EXPORT_SYMBOL(acpi_unmap_cpu); +#endif /* CONFIG_ACPI_HOTPLUG_CPU */ + +void __init acpi_boot_table_init(void) +{ + /** + * ACPI is disabled by default. + * ACPI is only enabled when firmware passes ACPI table + * and sets boot parameter "acpi=on". + */ + if (param_acpi_on) + enable_acpi(); + + /* + * If acpi_disabled, bail out + */ + if (!acpi_disabled) { + pr_warn("Currently, ACPI is an experimental feature!\n"); + if (acpi_table_init()) { + pr_err("Failed to init ACPI tables\n"); + disable_acpi(); + } else + pr_info("Successfully parsed ACPI table\n"); + } +} diff --git a/arch/sw_64/pci/acpi.c b/arch/sw_64/pci/acpi.c new file mode 100644 index 000000000000..1353994320b3 --- /dev/null +++ b/arch/sw_64/pci/acpi.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +struct pci_root_info { + struct acpi_pci_root_info info; + struct pci_config_window *cfg; +}; + +static void pci_acpi_release_root_info(struct acpi_pci_root_info *ci) +{ + struct pci_root_info *pci_ri; + + pci_ri = container_of(ci, struct pci_root_info, info); + pci_ecam_free(pci_ri->cfg); + kfree(ci->ops); + kfree(pci_ri); +} + +int acpi_pci_bus_find_domain_nr(struct pci_bus *bus) +{ + struct pci_config_window *cfg = bus->sysdata; + struct acpi_device *adev = to_acpi_device(cfg->parent); + struct acpi_pci_root *root = acpi_driver_data(adev); + + return root->segment; +} + +/** + * Lookup the MCFG table entry corresponding to the current + * PCI host controller, and set up config space mapping. + */ +static struct pci_config_window * +pci_acpi_setup_ecam_mapping(struct acpi_pci_root *root) +{ + struct device *dev = &root->device->dev; + struct pci_config_window *cfg = NULL; + const struct pci_ecam_ops *ecam_ops = NULL; + struct resource *bus_res = &root->secondary; + struct resource cfg_res; + struct acpi_device *adev = NULL; + int ret = 0, bus_shift = 0; + u16 seg = root->segment; + + ret = pci_mcfg_lookup(root, &cfg_res, &ecam_ops); + if (ret < 0) { + dev_err(dev, "%04x:%pR ECAM region not found\n", seg, bus_res); + return NULL; + } + + /** + * Do the quirk of bus shift here, since we can not + * know the ECAM addr in MCFG table when fill mcfg_quirks + */ + bus_shift = ecam_ops->bus_shift; + cfg_res.start = root->mcfg_addr + (bus_res->start << bus_shift); + cfg_res.end = cfg_res.start + ((resource_size(bus_res)) << bus_shift) - 1; + cfg_res.flags = IORESOURCE_MEM; + + /** + * ECAM area considered as the mem resource of the current + * PCI host controller, we'd better record this resource + * in ACPI namespace(_CRS). + */ + adev = acpi_resource_consumer(&cfg_res); + if (adev) + dev_info(dev, "ECAM area %pR reserved by %s\n", &cfg_res, + dev_name(&adev->dev)); + else + dev_info(dev, "Note: ECAM area %pR not reserved in ACPI namespace\n", + &cfg_res); + + cfg = pci_ecam_create(dev, &cfg_res, bus_res, ecam_ops); + if (IS_ERR(cfg)) { + dev_err(dev, "%04x:%pR error %ld mapping ECAM\n", seg, bus_res, + PTR_ERR(cfg)); + return NULL; + } + + return cfg; +} + +static int pci_acpi_prepare_root_resources(struct acpi_pci_root_info *ci) +{ + int status = 0; + acpi_status rc; + unsigned long long mem_space_base = 0; + struct resource_entry *entry = NULL, *tmp = NULL; + struct acpi_device *device = ci->bridge; + + /** + * Get host bridge resources via _CRS method, the return value + * is the num of resource parsed. + */ + status = acpi_pci_probe_root_resources(ci); + if (status > 0) { + /** + * To distinguish between mem and pre_mem, firmware only pass the + * lower 32bits of mem via acpi and use vendor specific "MEMH" to + * record the upper 32 bits of mem. + * + * Get the upper 32 bits here. + */ + rc = acpi_evaluate_integer(ci->bridge->handle, + "MEMH", NULL, &mem_space_base); + if (rc != AE_OK) { + dev_err(&device->dev, "unable to retrieve MEMH\n"); + return -EEXIST; + } + + resource_list_for_each_entry_safe(entry, tmp, &ci->resources) { + if (entry->res->flags & IORESOURCE_MEM) { + if (!(entry->res->end & 0xFFFFFFFF00000000ULL)) { + /* Patch the mem resource with upper 32 bits */ + entry->res->start |= (mem_space_base << 32); + entry->res->end |= (mem_space_base << 32); + } else { + /** + * Add PREFETCH and MEM_64 flags for pre_mem, + * so that we can distinguish between mem and + * pre_mem. + */ + entry->res->flags |= IORESOURCE_PREFETCH; + entry->res->flags |= IORESOURCE_MEM_64; + } + } + + dev_dbg(&device->dev, + "host bridge resource: 0x%llx-0x%llx flags [0x%lx]\n", + entry->res->start, entry->res->end, entry->res->flags); + } + return status; + } + + /** + * If not successfully parse resources, destroy + * resources which have been parsed. + */ + resource_list_for_each_entry_safe(entry, tmp, &ci->resources) { + dev_info(&device->dev, + "host bridge resource(ignored): 0x%llx-0x%llx flags [0x%lx]\n", + entry->res->start, entry->res->end, entry->res->flags); + resource_list_destroy_entry(entry); + } + + return 0; +} + +/** + * This function is called from ACPI code and used to + * setup PCI host controller. + */ +struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root) +{ + struct pci_bus *bus = NULL, *child = NULL; + struct pci_root_info *pci_ri = NULL; + struct acpi_pci_root_ops *root_ops = NULL; + int domain = root->segment; + int busnum = root->secondary.start; + + pci_ri = kzalloc(sizeof(*pci_ri), GFP_KERNEL); + if (!pci_ri) + goto out_of_mem_0; + + root_ops = kzalloc(sizeof(*root_ops), GFP_KERNEL); + if (!root_ops) + goto out_of_mem_1; + + pci_ri->cfg = pci_acpi_setup_ecam_mapping(root); + if (!pci_ri->cfg) + goto setup_ecam_err; + + root_ops->release_info = pci_acpi_release_root_info; + root_ops->prepare_resources = pci_acpi_prepare_root_resources; + root_ops->pci_ops = (struct pci_ops *)&pci_ri->cfg->ops->pci_ops; + + bus = pci_find_bus(domain, busnum); + if (bus) { + memcpy(bus->sysdata, pci_ri->cfg, sizeof(struct pci_config_window)); + kfree(pci_ri->cfg); + kfree(pci_ri); + kfree(root_ops); + } else { + bus = acpi_pci_root_create(root, root_ops, &pci_ri->info, pci_ri->cfg); + + /** + * No need to do kfree here, because acpi_pci_root_create will free + * mem alloced when it cannot create pci_bus. + */ + if (!bus) + return NULL; + + /* Some quirks for pci controller of Sunway after scanning Root Complex */ + sw64_pci_root_bridge_scan_finish_up(pci_find_host_bridge(bus)); + + pci_bus_size_bridges(bus); + pci_bus_assign_resources(bus); + + list_for_each_entry(child, &bus->children, node) + pcie_bus_configure_settings(child); + } + + return bus; + +setup_ecam_err: + kfree(root_ops); +out_of_mem_1: + kfree(pci_ri); +out_of_mem_0: + pr_warn("RC [%04x:%02x:] failed (out of memory or setup ecam error)!\n", + domain, busnum); + + return NULL; +} + +int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge) +{ + if (!acpi_disabled) { + struct pci_config_window *cfg = bridge->sysdata; + struct acpi_device *adev = to_acpi_device(cfg->parent); + struct pci_controller *hose = cfg->priv; + struct device *bus_dev = &bridge->bus->dev; + + ACPI_COMPANION_SET(&bridge->dev, adev); + set_dev_node(bus_dev, hose->node); + + /* Some quirks for pci controller of Sunway before scanning Root Complex */ + sw64_pci_root_bridge_prepare(bridge); + } + + return 0; +} + +void pcibios_add_bus(struct pci_bus *bus) +{ + acpi_pci_add_bus(bus); +} + +void pcibios_remove_bus(struct pci_bus *bus) +{ + acpi_pci_remove_bus(bus); +} -- Gitee From 8d872e3258a384d2035cd12ce2b00ea23be36f01 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:06 +0800 Subject: [PATCH 032/524] sw64: add DMA support Add DMA support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/dma-direct.h | 15 ++ arch/sw_64/include/asm/dma-mapping.h | 14 ++ arch/sw_64/include/asm/dma.h | 350 +++++++++++++++++++++++++++ 3 files changed, 379 insertions(+) create mode 100644 arch/sw_64/include/asm/dma-direct.h create mode 100644 arch/sw_64/include/asm/dma-mapping.h create mode 100644 arch/sw_64/include/asm/dma.h diff --git a/arch/sw_64/include/asm/dma-direct.h b/arch/sw_64/include/asm/dma-direct.h new file mode 100644 index 000000000000..dee1680b8f6d --- /dev/null +++ b/arch/sw_64/include/asm/dma-direct.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_DMA_DIRECT_H +#define _ASM_SW64_DMA_DIRECT_H + +static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) +{ + return paddr; +} + +static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) +{ + return daddr; +} + +#endif /* _ASM_SW64_DMA_DIRECT_H */ diff --git a/arch/sw_64/include/asm/dma-mapping.h b/arch/sw_64/include/asm/dma-mapping.h new file mode 100644 index 000000000000..65795f8e5792 --- /dev/null +++ b/arch/sw_64/include/asm/dma-mapping.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_DMA_MAPPING_H +#define _ASM_SW64_DMA_MAPPING_H + + +extern const struct dma_map_ops *dma_ops; + +static inline const struct dma_map_ops *get_arch_dma_ops(void) +{ + return dma_ops; +} + + +#endif /* _ASM_SW64_DMA_MAPPING_H */ diff --git a/arch/sw_64/include/asm/dma.h b/arch/sw_64/include/asm/dma.h new file mode 100644 index 000000000000..cf6a9cf75233 --- /dev/null +++ b/arch/sw_64/include/asm/dma.h @@ -0,0 +1,350 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * include/asm-sw_64/dma.h + * + * This is essentially the same as the i386 DMA stuff, as the SW64PCs + * use ISA-compatible dma. The only extension is support for high-page + * registers that allow to set the top 8 bits of a 32-bit DMA address. + * This register should be written last when setting up a DMA address + * as this will also enable DMA across 64 KB boundaries. + */ + +/* $Id: dma.h,v 1.7 1992/12/14 00:29:34 root Exp root $ + * linux/include/asm/dma.h: Defines for using and allocating dma channels. + * Written by Hennus Bergman, 1992. + * High DMA channel support & info by Hannu Savolainen + * and John Boyd, Nov. 1992. + */ + +#ifndef _ASM_SW64_DMA_H +#define _ASM_SW64_DMA_H + +#include +#include + +#define dma_outb outb +#define dma_inb inb + +/* + * NOTES about DMA transfers: + * + * controller 1: channels 0-3, byte operations, ports 00-1F + * controller 2: channels 4-7, word operations, ports C0-DF + * + * - ALL registers are 8 bits only, regardless of transfer size + * - channel 4 is not used - cascades 1 into 2. + * - channels 0-3 are byte - addresses/counts are for physical bytes + * - channels 5-7 are word - addresses/counts are for physical words + * - transfers must not cross physical 64K (0-3) or 128K (5-7) boundaries + * - transfer count loaded to registers is 1 less than actual count + * - controller 2 offsets are all even (2x offsets for controller 1) + * - page registers for 5-7 don't use data bit 0, represent 128K pages + * - page registers for 0-3 use bit 0, represent 64K pages + * + * DMA transfers are limited to the lower 16MB of _physical_ memory. + * Note that addresses loaded into registers must be _physical_ addresses, + * not logical addresses (which may differ if paging is active). + * + * Address mapping for channels 0-3: + * + * A23 ... A16 A15 ... A8 A7 ... A0 (Physical addresses) + * | ... | | ... | | ... | + * | ... | | ... | | ... | + * | ... | | ... | | ... | + * P7 ... P0 A7 ... A0 A7 ... A0 + * | Page | Addr MSB | Addr LSB | (DMA registers) + * + * Address mapping for channels 5-7: + * + * A23 ... A17 A16 A15 ... A9 A8 A7 ... A1 A0 (Physical addresses) + * | ... | \ \ ... \ \ \ ... \ \ + * | ... | \ \ ... \ \ \ ... \ (not used) + * | ... | \ \ ... \ \ \ ... \ + * P7 ... P1 (0) A7 A6 ... A0 A7 A6 ... A0 + * | Page | Addr MSB | Addr LSB | (DMA registers) + * + * Again, channels 5-7 transfer _physical_ words (16 bits), so addresses + * and counts _must_ be word-aligned (the lowest address bit is _ignored_ at + * the hardware level, so odd-byte transfers aren't possible). + * + * Transfer count (_not # bytes_) is limited to 64K, represented as actual + * count - 1 : 64K => 0xFFFF, 1 => 0x0000. Thus, count is always 1 or more, + * and up to 128K bytes may be transferred on channels 5-7 in one operation. + * + */ + +#define MAX_DMA_CHANNELS 8 + +/* + * ISA DMA limitations on sw64 platforms, + + * These may be due to SIO (PCI<->ISA bridge) chipset limitation, or + * just a wiring limit. + */ + +/* + * Maximum address for all the others is the complete 32-bit bus + * address space. + */ +#define MAX_ISA_DMA_ADDRESS 0x100000000UL + +#define MAX_DMA32_PFN (1UL << (32 - PAGE_SHIFT)) + +/* + * If we have the iommu, we don't have any address limitations on DMA. + * Otherwise (Nautilus, RX164), we have to have 0-16 Mb DMA zone + * like i386. + */ +#define MAX_DMA_ADDRESS ~0UL + +/* 8237 DMA controllers */ +#define IO_DMA1_BASE 0x00 /* 8 bit slave DMA, channels 0..3 */ +#define IO_DMA2_BASE 0xC0 /* 16 bit master DMA, ch 4(=slave input)..7 */ + +/* DMA controller registers */ +#define DMA1_CMD_REG 0x08 /* command register (w) */ +#define DMA1_STAT_REG 0x08 /* status register (r) */ +#define DMA1_REQ_REG 0x09 /* request register (w) */ +#define DMA1_MASK_REG 0x0A /* single-channel mask (w) */ +#define DMA1_MODE_REG 0x0B /* mode register (w) */ +#define DMA1_CLEAR_FF_REG 0x0C /* clear pointer flip-flop (w) */ +#define DMA1_TEMP_REG 0x0D /* Temporary Register (r) */ +#define DMA1_RESET_REG 0x0D /* Master Clear (w) */ +#define DMA1_CLR_MASK_REG 0x0E /* Clear Mask */ +#define DMA1_MASK_ALL_REG 0x0F /* all-channels mask (w) */ +#define DMA1_EXT_MODE_REG (0x400 | DMA1_MODE_REG) + +#define DMA2_CMD_REG 0xD0 /* command register (w) */ +#define DMA2_STAT_REG 0xD0 /* status register (r) */ +#define DMA2_REQ_REG 0xD2 /* request register (w) */ +#define DMA2_MASK_REG 0xD4 /* single-channel mask (w) */ +#define DMA2_MODE_REG 0xD6 /* mode register (w) */ +#define DMA2_CLEAR_FF_REG 0xD8 /* clear pointer flip-flop (w) */ +#define DMA2_TEMP_REG 0xDA /* Temporary Register (r) */ +#define DMA2_RESET_REG 0xDA /* Master Clear (w) */ +#define DMA2_CLR_MASK_REG 0xDC /* Clear Mask */ +#define DMA2_MASK_ALL_REG 0xDE /* all-channels mask (w) */ +#define DMA2_EXT_MODE_REG (0x400 | DMA2_MODE_REG) + +#define DMA_ADDR_0 0x00 /* DMA address registers */ +#define DMA_ADDR_1 0x02 +#define DMA_ADDR_2 0x04 +#define DMA_ADDR_3 0x06 +#define DMA_ADDR_4 0xC0 +#define DMA_ADDR_5 0xC4 +#define DMA_ADDR_6 0xC8 +#define DMA_ADDR_7 0xCC + +#define DMA_CNT_0 0x01 /* DMA count registers */ +#define DMA_CNT_1 0x03 +#define DMA_CNT_2 0x05 +#define DMA_CNT_3 0x07 +#define DMA_CNT_4 0xC2 +#define DMA_CNT_5 0xC6 +#define DMA_CNT_6 0xCA +#define DMA_CNT_7 0xCE + +#define DMA_PAGE_0 0x87 /* DMA page registers */ +#define DMA_PAGE_1 0x83 +#define DMA_PAGE_2 0x81 +#define DMA_PAGE_3 0x82 +#define DMA_PAGE_5 0x8B +#define DMA_PAGE_6 0x89 +#define DMA_PAGE_7 0x8A + +#define DMA_HIPAGE_0 (0x400 | DMA_PAGE_0) +#define DMA_HIPAGE_1 (0x400 | DMA_PAGE_1) +#define DMA_HIPAGE_2 (0x400 | DMA_PAGE_2) +#define DMA_HIPAGE_3 (0x400 | DMA_PAGE_3) +#define DMA_HIPAGE_4 (0x400 | DMA_PAGE_4) +#define DMA_HIPAGE_5 (0x400 | DMA_PAGE_5) +#define DMA_HIPAGE_6 (0x400 | DMA_PAGE_6) +#define DMA_HIPAGE_7 (0x400 | DMA_PAGE_7) + +#define DMA_MODE_READ 0x44 /* I/O to memory, no autoinit, increment, single mode */ +#define DMA_MODE_WRITE 0x48 /* memory to I/O, no autoinit, increment, single mode */ +#define DMA_MODE_CASCADE 0xC0 /* pass thru DREQ->HRQ, DACK<-HLDA only */ + +#define DMA_AUTOINIT 0x10 + +extern spinlock_t dma_spin_lock; + +static inline unsigned long claim_dma_lock(void) +{ + unsigned long flags; + + spin_lock_irqsave(&dma_spin_lock, flags); + return flags; +} + +static inline void release_dma_lock(unsigned long flags) +{ + spin_unlock_irqrestore(&dma_spin_lock, flags); +} + +/* enable/disable a specific DMA channel */ +static inline void enable_dma(unsigned int dmanr) +{ + if (dmanr <= 3) + dma_outb(dmanr, DMA1_MASK_REG); + else + dma_outb(dmanr & 3, DMA2_MASK_REG); +} + +static inline void disable_dma(unsigned int dmanr) +{ + if (dmanr <= 3) + dma_outb(dmanr | 4, DMA1_MASK_REG); + else + dma_outb((dmanr & 3) | 4, DMA2_MASK_REG); +} + +/* Clear the 'DMA Pointer Flip Flop'. + * Write 0 for LSB/MSB, 1 for MSB/LSB access. + * Use this once to initialize the FF to a known state. + * After that, keep track of it. :-) + * --- In order to do that, the DMA routines below should --- + * --- only be used while interrupts are disabled! --- + */ +static inline void clear_dma_ff(unsigned int dmanr) +{ + if (dmanr <= 3) + dma_outb(0, DMA1_CLEAR_FF_REG); + else + dma_outb(0, DMA2_CLEAR_FF_REG); +} + +/* set mode (above) for a specific DMA channel */ +static inline void set_dma_mode(unsigned int dmanr, char mode) +{ + if (dmanr <= 3) + dma_outb(mode | dmanr, DMA1_MODE_REG); + else + dma_outb(mode | (dmanr & 3), DMA2_MODE_REG); +} + +/* set extended mode for a specific DMA channel */ +static inline void set_dma_ext_mode(unsigned int dmanr, char ext_mode) +{ + if (dmanr <= 3) + dma_outb(ext_mode | dmanr, DMA1_EXT_MODE_REG); + else + dma_outb(ext_mode | (dmanr & 3), DMA2_EXT_MODE_REG); +} + +/* Set only the page register bits of the transfer address. + * This is used for successive transfers when we know the contents of + * the lower 16 bits of the DMA current address register. + */ +static inline void set_dma_page(unsigned int dmanr, unsigned int pagenr) +{ + switch (dmanr) { + case 0: + dma_outb(pagenr, DMA_PAGE_0); + dma_outb((pagenr >> 8), DMA_HIPAGE_0); + break; + case 1: + dma_outb(pagenr, DMA_PAGE_1); + dma_outb((pagenr >> 8), DMA_HIPAGE_1); + break; + case 2: + dma_outb(pagenr, DMA_PAGE_2); + dma_outb((pagenr >> 8), DMA_HIPAGE_2); + break; + case 3: + dma_outb(pagenr, DMA_PAGE_3); + dma_outb((pagenr >> 8), DMA_HIPAGE_3); + break; + case 5: + dma_outb(pagenr & 0xfe, DMA_PAGE_5); + dma_outb((pagenr >> 8), DMA_HIPAGE_5); + break; + case 6: + dma_outb(pagenr & 0xfe, DMA_PAGE_6); + dma_outb((pagenr >> 8), DMA_HIPAGE_6); + break; + case 7: + dma_outb(pagenr & 0xfe, DMA_PAGE_7); + dma_outb((pagenr >> 8), DMA_HIPAGE_7); + break; + } +} + + +/* Set transfer address & page bits for specific DMA channel. + * Assumes dma flipflop is clear. + */ +static inline void set_dma_addr(unsigned int dmanr, unsigned int a) +{ + if (dmanr <= 3) { + dma_outb(a & 0xff, ((dmanr & 3) << 1) + IO_DMA1_BASE); + dma_outb((a >> 8) & 0xff, ((dmanr & 3) << 1) + IO_DMA1_BASE); + } else { + dma_outb((a >> 1) & 0xff, ((dmanr & 3) << 2) + IO_DMA2_BASE); + dma_outb((a >> 9) & 0xff, ((dmanr & 3) << 2) + IO_DMA2_BASE); + } + set_dma_page(dmanr, a >> 16); /* set hipage last to enable 32-bit mode */ +} + + +/* Set transfer size (max 64k for DMA1..3, 128k for DMA5..7) for + * a specific DMA channel. + * You must ensure the parameters are valid. + * NOTE: from a manual: "the number of transfers is one more + * than the initial word count"! This is taken into account. + * Assumes dma flip-flop is clear. + * NOTE 2: "count" represents _bytes_ and must be even for channels 5-7. + */ +static inline void set_dma_count(unsigned int dmanr, unsigned int count) +{ + count--; + if (dmanr <= 3) { + dma_outb(count & 0xff, ((dmanr & 3) << 1) + 1 + IO_DMA1_BASE); + dma_outb((count >> 8) & 0xff, ((dmanr & 3) << 1) + 1 + IO_DMA1_BASE); + } else { + dma_outb((count >> 1) & 0xff, ((dmanr & 3) << 2) + 2 + IO_DMA2_BASE); + dma_outb((count >> 9) & 0xff, ((dmanr & 3) << 2) + 2 + IO_DMA2_BASE); + } +} + + +/* Get DMA residue count. After a DMA transfer, this + * should return zero. Reading this while a DMA transfer is + * still in progress will return unpredictable results. + * If called before the channel has been used, it may return 1. + * Otherwise, it returns the number of _bytes_ left to transfer. + * + * Assumes DMA flip-flop is clear. + */ +static inline int get_dma_residue(unsigned int dmanr) +{ + unsigned int io_port = (dmanr <= 3) ? + ((dmanr & 3) << 1) + 1 + IO_DMA1_BASE : + ((dmanr & 3) << 2) + 2 + IO_DMA2_BASE; + + /* using short to get 16-bit wrap around */ + unsigned short count; + + count = 1 + dma_inb(io_port); + count += dma_inb(io_port) << 8; + + return (dmanr <= 3) ? count : (count << 1); +} + + +/* These are in kernel/dma.c: */ +extern int request_dma(unsigned int dmanr, const char *device_id); /* reserve a DMA channel */ +extern void free_dma(unsigned int dmanr); /* release it again */ +#define KERNEL_HAVE_CHECK_DMA +extern int check_dma(unsigned int dmanr); + +/* From PCI */ + +#ifdef CONFIG_PCI +extern int isa_dma_bridge_buggy; +#else +#define isa_dma_bridge_buggy (0) +#endif + + +#endif /* _ASM_SW64_DMA_H */ -- Gitee From 266fee9e40e7754877f346feaff4176010666f98 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:07 +0800 Subject: [PATCH 033/524] sw64: add EFI support Add basic EFI support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/dmi.h | 30 ++++++++++++++++++++++++++++++ arch/sw_64/include/asm/efi.h | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 arch/sw_64/include/asm/dmi.h create mode 100644 arch/sw_64/include/asm/efi.h diff --git a/arch/sw_64/include/asm/dmi.h b/arch/sw_64/include/asm/dmi.h new file mode 100644 index 000000000000..05e80c9a3a76 --- /dev/null +++ b/arch/sw_64/include/asm/dmi.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arch/sw_64/include/asm/dmi.h + * + * Copyright (C) 2019 Deepin Limited. + * Porting by: Deepin Kernel Team (kernel@deepin.com) + * + * based on arch/x864/include/asm/dmi.h + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#ifndef _ASM_SW64_DMI_H +#define _ASM_SW64_DMI_H + +#include +#include +#include +#include + +/* Use early IO mappings for DMI because it's initialized early */ +#define dmi_early_remap(x, l) early_ioremap(x, l) +#define dmi_early_unmap(x, l) early_iounmap(x, l) +#define dmi_remap(x, l) early_ioremap(x, l) +#define dmi_unmap(x) early_iounmap(x, 0) +#define dmi_alloc(l) kzalloc(l, GFP_KERNEL) + +#endif /* _ASM_SW64_DMI_H */ diff --git a/arch/sw_64/include/asm/efi.h b/arch/sw_64/include/asm/efi.h new file mode 100644 index 000000000000..34d5637e23c2 --- /dev/null +++ b/arch/sw_64/include/asm/efi.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_EFI_H +#define _ASM_SW64_EFI_H + +#include +#include +#ifdef CONFIG_EFI +extern void efi_init(void); +extern unsigned long entSuspend; + +#define SLEEP_ENTRY_GUID EFI_GUID(0x59cb76bb, 0x9c3a, 0x4c8f, 0xbd, 0x5c, 0xc0, 0x0f, 0x20, 0x61, 0x18, 0x4b) + +#else +#define efi_init() +#define efi_idmap_init() +#endif + +#define arch_efi_call_virt_setup() +#define arch_efi_call_virt_teardown() + +#define ARCH_EFI_IRQ_FLAGS_MASK 0x00000001 + +/* arch specific definitions used by the stub code */ + +/* + * AArch64 requires the DTB to be 8-byte aligned in the first 512MiB from + * start of kernel and may not cross a 2MiB boundary. We set alignment to + * 2MiB so we know it won't cross a 2MiB boundary. + */ +#define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */ +#define MAX_FDT_OFFSET SZ_512M + +#define efi_call_early(f, ...) sys_table_arg->boottime->f(__VA_ARGS__) + +#endif /* _ASM_SW64_EFI_H */ -- Gitee From 1b1c260f2bb86cadfd9c0b7c4fe6a9ab5b7f1972 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:21 +0800 Subject: [PATCH 034/524] sw64: add KVM support Add KVM support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hcall.h | 41 + arch/sw_64/include/asm/kvm_asm.h | 38 + arch/sw_64/include/asm/kvm_cma.h | 11 + arch/sw_64/include/asm/kvm_emulate.h | 46 + arch/sw_64/include/asm/kvm_host.h | 225 ++++ arch/sw_64/include/asm/kvm_mmio.h | 17 + arch/sw_64/include/asm/kvm_mmu.h | 131 +++ arch/sw_64/include/asm/kvm_para.h | 26 + arch/sw_64/include/asm/kvm_timer.h | 9 + arch/sw_64/include/asm/vcpu.h | 106 ++ arch/sw_64/include/uapi/asm/kvm.h | 131 +++ arch/sw_64/kvm/Kconfig | 49 + arch/sw_64/kvm/Makefile | 20 + arch/sw_64/kvm/emulate.c | 128 +++ arch/sw_64/kvm/entry.S | 263 +++++ arch/sw_64/kvm/handle_exit.c | 85 ++ arch/sw_64/kvm/irq.h | 12 + arch/sw_64/kvm/kvm_cma.c | 269 +++++ arch/sw_64/kvm/kvm_core3.c | 419 +++++++ arch/sw_64/kvm/kvm_core4.c | 132 +++ arch/sw_64/kvm/kvm_timer.c | 83 ++ arch/sw_64/kvm/mmio.c | 89 ++ arch/sw_64/kvm/mmu.c | 1561 ++++++++++++++++++++++++++ arch/sw_64/kvm/perf.c | 27 + arch/sw_64/kvm/sw64.c | 592 ++++++++++ arch/sw_64/kvm/trace.h | 62 + arch/sw_64/kvm/vmem.c | 183 +++ 27 files changed, 4755 insertions(+) create mode 100644 arch/sw_64/include/asm/hcall.h create mode 100644 arch/sw_64/include/asm/kvm_asm.h create mode 100644 arch/sw_64/include/asm/kvm_cma.h create mode 100644 arch/sw_64/include/asm/kvm_emulate.h create mode 100644 arch/sw_64/include/asm/kvm_host.h create mode 100644 arch/sw_64/include/asm/kvm_mmio.h create mode 100644 arch/sw_64/include/asm/kvm_mmu.h create mode 100644 arch/sw_64/include/asm/kvm_para.h create mode 100644 arch/sw_64/include/asm/kvm_timer.h create mode 100644 arch/sw_64/include/asm/vcpu.h create mode 100644 arch/sw_64/include/uapi/asm/kvm.h create mode 100644 arch/sw_64/kvm/Kconfig create mode 100644 arch/sw_64/kvm/Makefile create mode 100644 arch/sw_64/kvm/emulate.c create mode 100644 arch/sw_64/kvm/entry.S create mode 100644 arch/sw_64/kvm/handle_exit.c create mode 100644 arch/sw_64/kvm/irq.h create mode 100644 arch/sw_64/kvm/kvm_cma.c create mode 100644 arch/sw_64/kvm/kvm_core3.c create mode 100644 arch/sw_64/kvm/kvm_core4.c create mode 100644 arch/sw_64/kvm/kvm_timer.c create mode 100644 arch/sw_64/kvm/mmio.c create mode 100644 arch/sw_64/kvm/mmu.c create mode 100644 arch/sw_64/kvm/perf.c create mode 100644 arch/sw_64/kvm/sw64.c create mode 100644 arch/sw_64/kvm/trace.h create mode 100644 arch/sw_64/kvm/vmem.c diff --git a/arch/sw_64/include/asm/hcall.h b/arch/sw_64/include/asm/hcall.h new file mode 100644 index 000000000000..bded05779db7 --- /dev/null +++ b/arch/sw_64/include/asm/hcall.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_HCALL_H +#define _ASM_SW64_HCALL_H + +#define HMC_hcall 0x32 +/* HCALL must > 0 */ +enum HCALL_TYPE { + HCALL_HALT = 10, + HCALL_NOTIFY = 11, + HCALL_SHUTDOWN = 12, + HCALL_SET_CLOCKEVENT = 13, + HCALL_IVI = 14, /* interrupt between virtual cpu */ + HCALL_TBI = 15, /* tlb flush for virtual cpu */ + HCALL_STOP = 16, /* indicate virtual cpu stopped */ + HCALL_RESTART = 17, /* indicate virtual cpu restarted */ + HCALL_MSI = 18, /* guest request msi intr */ + HCALL_MSIX = 19, /* guest request msix intr */ + HCALL_SWNET = 20, /* guest request swnet service */ + HCALL_SWNET_IRQ = 21, /* guest request swnet intr */ + HCALL_FATAL_ERROR = 22, /* guest fatal error, issued by hmcode */ + HCALL_MEMHOTPLUG = 23, /* guest memory hotplug event */ + NR_HCALL +}; + +static inline unsigned long hcall(unsigned long hcall, unsigned long arg0, + unsigned long arg1, unsigned long arg2) +{ + register unsigned long __r0 __asm__("$0"); + register unsigned long __r16 __asm__("$16") = hcall; + register unsigned long __r17 __asm__("$17") = arg0; + register unsigned long __r18 __asm__("$18") = arg1; + register unsigned long __r19 __asm__("$19") = arg2; + __asm__ __volatile__( + "sys_call %5 " + : "=r"(__r16), "=r"(__r17), "=r"(__r18), "=r"(__r19), "=r"(__r0) + : "i"(HMC_hcall), "0"(__r16), "1"(__r17), "2"(__r18), "3"(__r19) + : "$1", "$22", "$23", "$24", "$25"); + return __r0; +} + +#endif /* _ASM_SW64_HCALL_H */ diff --git a/arch/sw_64/include/asm/kvm_asm.h b/arch/sw_64/include/asm/kvm_asm.h new file mode 100644 index 000000000000..fd1b25018fc8 --- /dev/null +++ b/arch/sw_64/include/asm/kvm_asm.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_ASM_H +#define _ASM_SW64_KVM_ASM_H + +#define SW64_KVM_EXIT_HOST_INTR 0 +#define SW64_KVM_EXIT_IO 1 +#define SW64_KVM_MIGRATION_SET_DIRTY 2 +#define SW64_KVM_MIGRATION_SET_DIRTY_HM 3 +#define SW64_KVM_EXIT_HALT 10 +#define SW64_KVM_EXIT_SHUTDOWN 12 +#define SW64_KVM_EXIT_TIMER 13 +#define SW64_KVM_EXIT_IPI 14 +#define SW64_KVM_EXIT_STOP 16 +#define SW64_KVM_EXIT_RESTART 17 +#define SW64_KVM_EXIT_APT_FAULT 18 +#define SW64_KVM_EXIT_FATAL_ERROR 22 +#define SW64_KVM_EXIT_MEMHOTPLUG 23 +#define SW64_KVM_EXIT_DEBUG 24 + + +#define kvm_sw64_exception_type \ + {0, "HOST_INTR" }, \ + {1, "IO" }, \ + {10, "HALT" }, \ + {12, "SHUTDOWN" }, \ + {13, "TIMER" }, \ + {14, "IPI" }, \ + {16, "STOP" }, \ + {17, "RESTART" }, \ + {18, "APT_FAULT" }, \ + {22, "FATAL_ERROR" }, \ + {23, "MEMHOTPLUG" }, \ + {24, "DEBUG" } + + +#include + +#endif /* _ASM_SW64_KVM_ASM_H */ diff --git a/arch/sw_64/include/asm/kvm_cma.h b/arch/sw_64/include/asm/kvm_cma.h new file mode 100644 index 000000000000..d50ba599ceb7 --- /dev/null +++ b/arch/sw_64/include/asm/kvm_cma.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_CMA_H +#define _ASM_SW64_KVM_CMA_H + +#include + +extern int __init kvm_cma_declare_contiguous(phys_addr_t base, + phys_addr_t size, phys_addr_t limit, + phys_addr_t alignment, unsigned int order_per_bit, + const char *name, struct cma **res_cma); +#endif /* _ASM_SW64_KVM_CMA_H */ diff --git a/arch/sw_64/include/asm/kvm_emulate.h b/arch/sw_64/include/asm/kvm_emulate.h new file mode 100644 index 000000000000..915aa6c0bce2 --- /dev/null +++ b/arch/sw_64/include/asm/kvm_emulate.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_EMULATE_H +#define _ASM_SW64_KVM_EMULATE_H + +#include +#include + +#define R(x) ((size_t) &((struct kvm_regs *)0)->x) + +static int reg_offsets[32] = { + R(r0), R(r1), R(r2), R(r3), R(r4), R(r5), R(r6), R(r7), R(r8), + R(r9), R(r10), R(r11), R(r12), R(r13), R(r14), R(r15), + R(r16), R(r17), R(r18), + R(r19), R(r20), R(r21), R(r22), R(r23), R(r24), R(r25), R(r26), + R(r27), R(r28), R(gp), + 0, 0, +}; + + +static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num, + unsigned long val) +{ + void *regs_ptr = (void *)&vcpu->arch.regs; + + regs_ptr += reg_offsets[reg_num]; + *(unsigned long *)regs_ptr = val; +} + +static inline unsigned long vcpu_get_reg(struct kvm_vcpu *vcpu, u8 reg_num) +{ + void *regs_ptr = (void *)&vcpu->arch.regs; + + if (reg_num == 31) + return 0; + regs_ptr += reg_offsets[reg_num]; + return *(unsigned long *)regs_ptr; +} + +void sw64_decode(struct kvm_vcpu *vcpu, unsigned int insn, + struct kvm_run *run); + +unsigned int interrupt_pending(struct kvm_vcpu *vcpu, bool *more); +void clear_vcpu_irq(struct kvm_vcpu *vcpu); +void inject_vcpu_irq(struct kvm_vcpu *vcpu, unsigned int irq); +void try_deliver_interrupt(struct kvm_vcpu *vcpu, unsigned int irq, bool more); +#endif /* _ASM_SW64_KVM_EMULATE_H */ diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h new file mode 100644 index 000000000000..09a995218a2c --- /dev/null +++ b/arch/sw_64/include/asm/kvm_host.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_HOST_H +#define _ASM_SW64_KVM_HOST_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#define last_vpn(cpu) (cpu_data[cpu].last_vpn) + +#ifdef CONFIG_SUBARCH_C3B +#define VPN_BITS 8 +#define GUEST_RESET_PC 0xffffffff80011100 +#endif + +#ifdef CONFIG_SUBARCH_C4 +#define VPN_BITS 10 +#define GUEST_RESET_PC 0xfff0000000011002 +#endif + +#define VPN_FIRST_VERSION (1UL << VPN_BITS) +#define VPN_MASK ((1UL << VPN_BITS) - 1) +#define VPN_SHIFT (64 - VPN_BITS) + +#define KVM_MAX_VCPUS 64 +#define KVM_INTERNAL_MEM_SLOTS (KVM_MEM_SLOTS_NUM - 512) + +#define KVM_HALT_POLL_NS_DEFAULT 0 +#define KVM_IRQCHIP_NUM_PINS 256 +/* KVM Hugepage definitions for sw64 */ +#define KVM_NR_PAGE_SIZES 3 +#define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9) +#define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x)) +#define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x)) +#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) +#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) + +/* + * The architecture supports 48-bit GPA as input to the addtional stage translations. + */ +#define KVM_PHYS_SHIFT (48) +#define KVM_PHYS_SIZE (_AC(1, ULL) << KVM_PHYS_SHIFT) +#define KVM_PHYS_MASK (KVM_PHYS_SIZE - _AC(1, ULL)) + +struct kvm_arch_memory_slot { + unsigned long host_phys_addr; + bool valid; +}; + +struct kvm_arch { + unsigned long host_phys_addr; + unsigned long size; + + /* segment table */ + unsigned long *seg_pgd; + + struct swvm_mem mem; + /* Addtional stage page table*/ + pgd_t *pgd; +}; + +#define KVM_NR_MEM_OBJS 40 + +/* + * We don't want allocation failures within the mmu code, so we preallocate + * enough memory for a single page fault in a cache. + */ +struct kvm_mmu_memory_cache { + int nobjs; + void *objects[KVM_NR_MEM_OBJS]; +}; + +struct kvm_vcpu_arch { + struct kvm_regs regs __aligned(32); + struct vcpucb vcb; + struct task_struct *tsk; + unsigned int pcpu_id; /* current running pcpu id */ + + /* Virtual clock device */ + struct hrtimer hrt; + unsigned long timer_next_event; + unsigned long vtimer_freq; + + int first_run; + int halted; + int stopped; + int restart; + + /* Pending virtual interrupts */ + DECLARE_BITMAP(irqs_pending, SWVM_IRQS); + unsigned long vpnc[NR_CPUS]; + + /* Detect first run of a vcpu */ + bool has_run_once; + + /* WAIT executed */ + int wait; + + /* vcpu power-off state */ + bool power_off; + + /* Don't run the guest (internal implementation need) */ + bool pause; + + struct kvm_decode mmio_decode; + + /* Cache some mmu pages needed inside spinlock regions */ + struct kvm_mmu_memory_cache mmu_page_cache; + + /* guest live migration */ + unsigned long migration_mark; + unsigned long shtclock; +}; + +struct vmem_info { + unsigned long start; + size_t size; + atomic_t refcnt; +}; + +struct kvm_vm_stat { + struct kvm_vm_stat_generic generic; +}; + +struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; + u64 pid; + u64 exits; + u64 io_exits; + u64 mmio_exits; + u64 migration_set_dirty; + u64 shutdown_exits; + u64 restart_exits; + u64 stop_exits; + u64 ipi_exits; + u64 timer_exits; + u64 debug_exits; +#ifdef CONFIG_KVM_MEMHOTPLUG + u64 memhotplug_exits; +#endif + u64 fatal_error_exits; + u64 halt_exits; + u64 halt_successful_poll; + u64 halt_attempted_poll; + u64 halt_wakeup; + u64 halt_poll_success_ns; + u64 halt_poll_fail_ns; + u64 halt_poll_invalid; + u64 signal_exits; + u64 steal; + u64 st_max; + u64 utime; + u64 stime; + u64 gtime; +}; + +#ifdef CONFIG_KVM_MEMHOTPLUG +void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr); +#endif +#ifdef CONFIG_SUBARCH_C4 +#define KVM_ARCH_WANT_MMU_NOTIFIER +#endif +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); +int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable); +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); + +void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat); +void check_vcpu_requests(struct kvm_vcpu *vcpu); +void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu); +int vmem_init(void); +void vmem_exit(void); +int __sw64_vcpu_run(unsigned long vcb_pa, struct kvm_regs *regs, + struct hcall_args *args); +int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, + int exception_index, struct hcall_args *hargs); +void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type); +static inline void kvm_arch_hardware_disable(void) {} +static inline void kvm_arch_sync_events(struct kvm *kvm) {} +static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} +static inline void kvm_arch_free_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} + +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); + +int kvm_sw64_perf_init(void); +int kvm_sw64_perf_teardown(void); +void kvm_flush_tlb_all(void); +void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn); +int kvm_sw64_init_vm(struct kvm *kvm); +void kvm_sw64_destroy_vm(struct kvm *kvm); +int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu); +long kvm_sw64_set_vcb(struct file *filp, unsigned long arg); +long kvm_sw64_get_vcb(struct file *filp, unsigned long arg); + +void update_aptp(unsigned long pgd); +void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu); +#endif /* _ASM_SW64_KVM_HOST_H */ diff --git a/arch/sw_64/include/asm/kvm_mmio.h b/arch/sw_64/include/asm/kvm_mmio.h new file mode 100644 index 000000000000..c87b259e9395 --- /dev/null +++ b/arch/sw_64/include/asm/kvm_mmio.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_MMIO_H +#define _ASM_SW64_KVM_MMIO_H + +#include +#include + +struct kvm_decode { + unsigned long rt; + bool sign_extend; +}; + +int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); +int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, + struct hcall_args *hargs); + +#endif /* _ASM_SW64_KVM_MMIO_H */ diff --git a/arch/sw_64/include/asm/kvm_mmu.h b/arch/sw_64/include/asm/kvm_mmu.h new file mode 100644 index 000000000000..f4493de934ba --- /dev/null +++ b/arch/sw_64/include/asm/kvm_mmu.h @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_MMU_H +#define _ASM_SW64_KVM_MMU_H + +#define AF_ACCESS_TYPE_SHIFT 55 +#define AF_INV_LEVEL_SHIFT 53 +#define AF_FAULT_STATUS_SHIFT 48 + +#define AF_ACCESS_TYPE_MASK 0x3 +#define AF_INV_LEVEL_MASK 0x3 +#define AF_FAULT_STATUS_MASK 0x1f +#define AF_ENTRY_ADDR_MASK ((0x1UL << AF_FAULT_STATUS_SHIFT) - 1) + +/* access type defination */ +#define AF_READ_ACCESS_TYPE 0x1 +#define AF_WRITE_ACCESS_TYPE 0x2 +#define AF_EXEC_ACCESS_TYPE 0x3 + +/* invalid page level */ +#define AF_INV_LEVEL_1 0 +#define AF_INV_LEVEL_2 1 +#define AF_INV_LEVEL_3 2 +#define AF_INV_LEVEL_4 3 + +/* fault status */ +#define AF_STATUS_MISCONFIG 0x1 +#define AF_STATUS_FOR 0x2 +#define AF_STATUS_FOW 0x4 +#define AF_STATUS_FOE 0x8 +#define AF_STATUS_INV 0x10 + +#define KVM_MMU_CACHE_MIN_PAGES 2 + +static inline void kvm_set_aptpte_readonly(pte_t *pte) +{ + pte_val(*pte) |= _PAGE_FOW; +} + +static inline bool kvm_aptpte_readonly(pte_t *pte) +{ + return (pte_val(*pte) & _PAGE_FOW) == _PAGE_FOW; +} + +static inline void kvm_set_aptpmd_readonly(pmd_t *pmd) +{ + pmd_val(*pmd) |= _PAGE_FOW; +} + +static inline bool kvm_aptpmd_readonly(pmd_t *pmd) +{ + return (pmd_val(*pmd) & _PAGE_FOW) == _PAGE_FOW; +} + +static inline void kvm_set_aptpud_readonly(pud_t *pud) +{ + pud_val(*pud) |= _PAGE_FOW; +} + +static inline bool kvm_aptpud_readonly(pud_t *pud) +{ + return (pud_val(*pud) & _PAGE_FOW) == _PAGE_FOW; +} + +static inline pte_t kvm_pte_mkwrite(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_FOW; + return pte; +} + +static inline pte_t kvm_pte_mkexec(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_FOE; + return pte; +} + +static inline bool kvm_pte_exec(pte_t *pte) +{ + return !(pte_val(*pte) & _PAGE_FOE); +} + +static inline pmd_t kvm_pmd_mkwrite(pmd_t pmd) +{ + pmd_val(pmd) &= ~_PAGE_FOW; + return pmd; +} + +static inline pmd_t kvm_pmd_mkexec(pmd_t pmd) +{ + pmd_val(pmd) &= ~_PAGE_FOE; + return pmd; +} + +static inline bool kvm_pmd_exec(pmd_t *pmd) +{ + return !(pmd_val(*pmd) & _PAGE_FOE); +} + +static inline pud_t kvm_pud_mkwrite(pud_t pud) +{ + pud_val(pud) &= ~_PAGE_FOW; + return pud; +} + +static inline pud_t kvm_pud_mkexec(pud_t pud) +{ + pud_val(pud) &= ~_PAGE_FOE; + return pud; +} + +static inline bool kvm_pud_exec(pud_t *pud) +{ + return !(pud_val(*pud) & _PAGE_FOE); +} + +void kvm_core4_commit_memory_region(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + const struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change); +void kvm_core4_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot); +void kvm_core4_flush_shadow_all(struct kvm *kvm); +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); +void kvm_handle_apt_fault(struct kvm_vcpu *vcpu); +int kvm_alloc_addtional_stage_pgd(struct kvm *kvm); +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot); +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run); +void apt_unmap_vm(struct kvm *kvm); +#endif /* _ASM_SW64_KVM_MMU_H */ diff --git a/arch/sw_64/include/asm/kvm_para.h b/arch/sw_64/include/asm/kvm_para.h new file mode 100644 index 000000000000..442f1c7d9f83 --- /dev/null +++ b/arch/sw_64/include/asm/kvm_para.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_PARA_H +#define _ASM_SW64_KVM_PARA_H + +#include + +#define HMC_hcall 0x32 + +static inline unsigned long kvm_hypercall3(unsigned long num, + unsigned long arg0, + unsigned long arg1, + unsigned long arg2) +{ + register unsigned long __r0 __asm__("$0"); + register unsigned long __r16 __asm__("$16") = num; + register unsigned long __r17 __asm__("$17") = arg0; + register unsigned long __r18 __asm__("$18") = arg1; + register unsigned long __r19 __asm__("$19") = arg2; + __asm__ __volatile__( + "sys_call %5" + : "=r"(__r16), "=r"(__r17), "=r"(__r18), "=r"(__r19), "=r"(__r0) + : "i"(HMC_hcall), "0"(__r16), "1"(__r17), "2"(__r18), "3"(__r19) + : "$1", "$22", "$23", "$24", "$25"); + return __r0; +} +#endif /* _ASM_SW64_KVM_PARA_H */ diff --git a/arch/sw_64/include/asm/kvm_timer.h b/arch/sw_64/include/asm/kvm_timer.h new file mode 100644 index 000000000000..8080873c684f --- /dev/null +++ b/arch/sw_64/include/asm/kvm_timer.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_TIMER_H +#define _ASM_SW64_KVM_TIMER_H + +void set_timer(struct kvm_vcpu *vcpu, unsigned long delta); +void set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq); +enum hrtimer_restart clockdev_fn(struct hrtimer *timer); + +#endif /* _ASM_SW64_KVM_TIMER_H */ diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h new file mode 100644 index 000000000000..c4e3caacbc70 --- /dev/null +++ b/arch/sw_64/include/asm/vcpu.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_VCPU_H +#define _ASM_SW64_VCPU_H + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_SUBARCH_C3B + +struct vcpucb { + unsigned long go_flag; + unsigned long pcbb; + unsigned long ksp; + unsigned long usp; + unsigned long kgp; + unsigned long ent_arith; + unsigned long ent_if; + unsigned long ent_int; + unsigned long ent_mm; + unsigned long ent_sys; + unsigned long ent_una; + unsigned long stack_pc; + unsigned long new_a0; + unsigned long new_a1; + unsigned long new_a2; + unsigned long soft_cid; + unsigned long csr_save; + unsigned long wakeup_magic; + unsigned long host_vcpucb; + unsigned long upcr; + unsigned long vpcr; + unsigned long dtb_vpcr; + unsigned long guest_ksp; + unsigned long guest_usp; + unsigned long vcpu_irq_disabled; + unsigned long vcpu_irq; + unsigned long ptbr; + unsigned long soft_tid; + unsigned long int_stat1; + unsigned long int_stat2; + unsigned long int_stat3; + unsigned long reset_entry; + unsigned long pvcpu; + unsigned long exit_reason; + unsigned long ipaddr; + unsigned long vcpu_irq_vector; + unsigned long pri_base; + unsigned long stack_pc_dfault; + unsigned long guest_p20; + unsigned long guest_dfault_double; + unsigned long guest_irqs_pending; + unsigned long guest_hm_r30; + unsigned long migration_mark; + unsigned long guest_longtime; + unsigned long guest_longtime_offset; + unsigned long reserved[3]; +}; + +#else + +struct vcpucb { + unsigned long ktp; + unsigned long pcbb; + unsigned long ksp; + unsigned long usp; + unsigned long kgp; + unsigned long ent_arith; + unsigned long ent_if; + unsigned long ent_int; + unsigned long ent_mm; + unsigned long ent_sys; + unsigned long ent_una; + unsigned long stack_pc; + unsigned long new_a0; + unsigned long new_a1; + unsigned long new_a2; + unsigned long soft_cid; + unsigned long csr_save; + unsigned long wakeup_magic; + unsigned long host_vcpucb; + unsigned long upcr; + unsigned long vpcr; + unsigned long dtb_vpcr; + unsigned long dtb_upcr; + unsigned long guest_ksp; + unsigned long guest_usp; + unsigned long vcpu_irq_disabled; + unsigned long vcpu_irq; + unsigned long ptbr_usr; + unsigned long ptbr_sys; + unsigned long soft_tid; + unsigned long int_stat0; + unsigned long int_stat1; + unsigned long int_stat2; + unsigned long int_stat3; + unsigned long reset_entry; + unsigned long pvcpu; + unsigned long exit_reason; + unsigned long ipaddr; + unsigned long vcpu_pc_save; + unsigned long shtclock_offset; + unsigned long reserved[8]; +}; +#endif + +#endif /* __ASSEMBLY__ */ +#endif /* _ASM_SW64_VCPU_H */ diff --git a/arch/sw_64/include/uapi/asm/kvm.h b/arch/sw_64/include/uapi/asm/kvm.h new file mode 100644 index 000000000000..2253475deaa5 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/kvm.h @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_KVM_H +#define _UAPI_ASM_SW64_KVM_H + +/* + * KVM SW specific structures and definitions. + */ +#define SWVM_IRQS 256 +#define IRQ_PENDING_INTX_SHIFT 16 +#define IRQ_PENDING_MSI_VECTORS_SHIFT 17 + +enum SW64_KVM_IRQ { + SW64_KVM_IRQ_IPI = 27, + SW64_KVM_IRQ_TIMER = 9, + SW64_KVM_IRQ_KBD = 29, + SW64_KVM_IRQ_MOUSE = 30, +}; + +#define SWVM_VM_TYPE_DEFAULT 0 +#define SWVM_VM_TYPE_PHYVCPU 1 +#define __KVM_HAVE_IRQ_LINE + +#define SWVM_NUM_NUMA_MEMBANKS 1 +#define KVM_NR_IRQCHIPS 1 +/* + * for KVM_GET_REGS and KVM_SET_REGS + */ +struct kvm_regs { + unsigned long r0; + unsigned long r1; + unsigned long r2; + unsigned long r3; + + unsigned long r4; + unsigned long r5; + unsigned long r6; + unsigned long r7; + + unsigned long r8; + unsigned long r9; + unsigned long r10; + unsigned long r11; + + unsigned long r12; + unsigned long r13; + unsigned long r14; + unsigned long r15; + + unsigned long r19; + unsigned long r20; + unsigned long r21; + unsigned long r22; + + unsigned long r23; + unsigned long r24; + unsigned long r25; + unsigned long r26; + + unsigned long r27; + unsigned long r28; + unsigned long __padding0; + unsigned long fpcr; + + unsigned long fp[124]; + /* These are saved by HMcode: */ + unsigned long ps; + unsigned long pc; + unsigned long gp; + unsigned long r16; + unsigned long r17; + unsigned long r18; +}; + + +/* + * return stack for __sw64_vcpu_run + */ +struct vcpu_run_ret_stack { + unsigned long ra; + unsigned long r0; +}; + +struct host_int_args { + unsigned long r18; + unsigned long r17; + unsigned long r16; +}; + +/* + * for KVM_GET_FPU and KVM_SET_FPU + */ +struct kvm_fpu { +}; + +struct hcall_args { + unsigned long arg0, arg1, arg2; +}; + +struct phyvcpu_hcall_args { + unsigned long call; + struct hcall_args args; +}; + +struct kvm_debug_exit_arch { + unsigned long epc; +}; + +/* for KVM_SET_GUEST_DEBUG */ +struct kvm_guest_debug_arch { +}; + +/* definition of registers in kvm_run */ +struct kvm_sync_regs { +}; + +/* dummy definition */ +struct kvm_sregs { +}; + +struct swvm_mem_bank { + unsigned long guest_phys_addr; + unsigned long host_phys_addr; + unsigned long host_addr; + unsigned long size; +}; + +struct swvm_mem { + struct swvm_mem_bank membank[SWVM_NUM_NUMA_MEMBANKS]; +}; + +#endif /* _UAPI_ASM_SW64_KVM_H */ diff --git a/arch/sw_64/kvm/Kconfig b/arch/sw_64/kvm/Kconfig new file mode 100644 index 000000000000..b7e43d0bae51 --- /dev/null +++ b/arch/sw_64/kvm/Kconfig @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# KVM configuration +# +source "virt/kvm/Kconfig" + +menuconfig VIRTUALIZATION + bool "Virtualization" + help + Say Y here to get to see options for using your Linux host to run + other operating systems inside virtual machines (guests). + This option alone does not add any kernel code. + + If you say N, all options in this submenu will be skipped and disabled. + +if VIRTUALIZATION + +config KVM + tristate "Kernel-based Virtual Machine (KVM) support" + select PREEMPT_NOTIFIERS + select CMA + depends on NET + select HAVE_KVM_EVENTFD + select HAVE_KVM_IRQCHIP + select HAVE_KVM_IRQ_ROUTING + select HAVE_KVM_IRQFD + select HAVE_KVM_MSI + select KVM_VFIO + select MMU_NOTIFIER + select KVM_GENERIC_DIRTYLOG_READ_PROTECT + select TUN + select GENERIC_ALLOCATOR + select KVM_GENERIC_DIRTYLOG_READ_PROTECT + help + Support for hosting Guest kernels. + We don't support KVM with 3-level page tables yet. + + If unsure, say N. + +config KVM_MEMHOTPLUG + bool "Memory hotplug support for guest" + depends on KVM && MEMORY_HOTPLUG && SUBARCH_C3B + help + Provides memory hotplug support for SW64 guest. + + +source "drivers/vhost/Kconfig" + +endif # VIRTUALIZATION diff --git a/arch/sw_64/kvm/Makefile b/arch/sw_64/kvm/Makefile new file mode 100644 index 000000000000..8111014c5cca --- /dev/null +++ b/arch/sw_64/kvm/Makefile @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for Kernel-based Virtual Machine module +# + +ccflags-y += -I $(srctree)/$(src) + +include $(srctree)/virt/kvm/Makefile.kvm + +obj-$(CONFIG_KVM) += kvm.o + +kvm-y += sw64.o +kvm-y += entry.o +kvm-y += emulate.o +kvm-y += mmio.o +kvm-y += kvm_timer.o +kvm-y += handle_exit.o +kvm-y += perf.o +kvm-$(CONFIG_SUBARCH_C3B) += kvm_core3.o kvm_cma.o +kvm-$(CONFIG_SUBARCH_C4) += kvm_core4.o mmu.o diff --git a/arch/sw_64/kvm/emulate.c b/arch/sw_64/kvm/emulate.c new file mode 100644 index 000000000000..fc37461b97a0 --- /dev/null +++ b/arch/sw_64/kvm/emulate.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 yangzh + * linhn + */ +#include +#include +#include +#include + +void sw64_decode(struct kvm_vcpu *vcpu, unsigned int insn, struct kvm_run *run) +{ + int opc, ra; + +#ifdef CONFIG_SUBARCH_C3B + opc = (insn >> 26) & 0x3f; + ra = (insn >> 21) & 0x1f; +#elif defined(CONFIG_SUBARCH_C4) + unsigned long ds_stat, exc_sum; + + ds_stat = read_csr(CSR_DS_STAT); + exc_sum = read_csr(CSR_EXC_SUM); + + opc = (ds_stat >> 4) & 0x3f; + ra = (exc_sum >> 8) & 0x1f; +#endif + + switch (opc) { + case 0x20: /* LDBU */ + run->mmio.is_write = 0; + run->mmio.len = 1; + vcpu->arch.mmio_decode.rt = ra; + break; + case 0x21: /* LDHU */ + run->mmio.is_write = 0; + run->mmio.len = 2; + vcpu->arch.mmio_decode.rt = ra; + break; + case 0x22: /* LDW */ + run->mmio.is_write = 0; + run->mmio.len = 4; + vcpu->arch.mmio_decode.rt = ra; + break; + case 0x23: /* LDL */ + case 0x24: /* LDL_U */ + run->mmio.is_write = 0; + run->mmio.len = 8; + vcpu->arch.mmio_decode.rt = ra; + break; + case 0x28: /* STB */ + run->mmio.is_write = 1; + *(unsigned long *)run->mmio.data = vcpu_get_reg(vcpu, ra) & 0xffUL; + run->mmio.len = 1; + break; + case 0x29: /* STH */ + run->mmio.is_write = 1; + *(unsigned long *)run->mmio.data = vcpu_get_reg(vcpu, ra) & 0xffffUL; + run->mmio.len = 2; + break; + case 0x2a: /* STW */ + run->mmio.is_write = 1; + *(unsigned long *)run->mmio.data = vcpu_get_reg(vcpu, ra) & 0xffffffffUL; + run->mmio.len = 4; + break; + case 0x2b: /* STL */ + case 0x2c: /* STL_U */ + run->mmio.is_write = 1; + *(unsigned long *)run->mmio.data = vcpu_get_reg(vcpu, ra); + run->mmio.len = 8; + break; + default: + pr_info("Miss done opc %d\n", opc); + break; + } +} + +/* + * Virtual Interrupts. + */ +unsigned int interrupt_pending(struct kvm_vcpu *vcpu, bool *more) +{ + unsigned int irq; + DECLARE_BITMAP(blk, SWVM_IRQS); + + bitmap_copy(blk, vcpu->arch.irqs_pending, SWVM_IRQS); + + irq = find_last_bit(blk, SWVM_IRQS); + + return irq; +} + +void clear_vcpu_irq(struct kvm_vcpu *vcpu) +{ + vcpu->arch.vcb.vcpu_irq = 0xffffffffffffffffUL; +} + +void inject_vcpu_irq(struct kvm_vcpu *vcpu, unsigned int irq) +{ + vcpu->arch.vcb.vcpu_irq = irq; +} + +/* + * This actually diverts the Guest to running an interrupt handler, once an + * interrupt has been identified by interrupt_pending(). + */ +void try_deliver_interrupt(struct kvm_vcpu *vcpu, unsigned int irq, bool more) +{ + BUG_ON(irq >= SWVM_IRQS); + + /* Otherwise we check if they have interrupts disabled. */ + if (vcpu->arch.vcb.vcpu_irq_disabled) { + clear_vcpu_irq(vcpu); + return; + } + + /* If they don't have a handler (yet?), we just ignore it */ + if (vcpu->arch.vcb.ent_int != 0) { + /* OK, mark it no longer pending and deliver it. */ + clear_bit(irq, (vcpu->arch.irqs_pending)); + /* + * set_guest_interrupt() takes the interrupt descriptor and a + * flag to say whether this interrupt pushes an error code onto + * the stack as well: virtual interrupts never do. + */ + inject_vcpu_irq(vcpu, irq); + } +} diff --git a/arch/sw_64/kvm/entry.S b/arch/sw_64/kvm/entry.S new file mode 100644 index 000000000000..a61ecc387d26 --- /dev/null +++ b/arch/sw_64/kvm/entry.S @@ -0,0 +1,263 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 + */ + .text +#include +#include +#include +#include + + .set noat + +/* + * r16: physical address of guest kvm_vcpu.arch.vcb + * r17: pointer to guest kvm_vcpu.arch.kvm_regs + * r18: pointer to hcall args + */ +ENTRY(__sw64_vcpu_run) + /* save host fpregs */ + rfpcr $f0 + fstd $f0, TASK_THREAD_FPCR($8) + vstd $f2, TASK_THREAD_F2($8) + vstd $f3, TASK_THREAD_F3($8) + vstd $f4, TASK_THREAD_F4($8) + vstd $f5, TASK_THREAD_F5($8) + vstd $f6, TASK_THREAD_F6($8) + vstd $f7, TASK_THREAD_F7($8) + vstd $f8, TASK_THREAD_F8($8) + vstd $f9, TASK_THREAD_F9($8) + + ldi sp, -VCPU_RET_SIZE(sp) + /* save host pt_regs to current kernel stack */ + ldi sp, -PT_REGS_SIZE(sp) + stl $9, PT_REGS_R9(sp) + stl $8, PT_REGS_R8(sp) + stl $10, PT_REGS_R10(sp) + stl $11, PT_REGS_R11(sp) + stl $12, PT_REGS_R12(sp) + stl $13, PT_REGS_R13(sp) + stl $14, PT_REGS_R14(sp) + stl $15, PT_REGS_R15(sp) + stl $26, PT_REGS_R26(sp) + + /* restore guest switch stack from guest kvm_regs struct */ + ldl $0, KVM_REGS_R0($17) + ldl $1, KVM_REGS_R1($17) + /* restore $2 later */ + ldl $3, KVM_REGS_R3($17) + ldl $4, KVM_REGS_R4($17) + ldl $5, KVM_REGS_R5($17) + ldl $6, KVM_REGS_R6($17) + ldl $7, KVM_REGS_R7($17) + ldl $8, KVM_REGS_R8($17) + ldl $9, KVM_REGS_R9($17) + ldl $10, KVM_REGS_R10($17) + ldl $11, KVM_REGS_R11($17) + ldl $12, KVM_REGS_R12($17) + ldl $13, KVM_REGS_R13($17) + ldl $14, KVM_REGS_R14($17) + ldl $15, KVM_REGS_R15($17) + ldl $19, KVM_REGS_R19($17) + ldl $20, KVM_REGS_R20($17) + ldl $21, KVM_REGS_R21($17) + ldl $22, KVM_REGS_R22($17) + ldl $23, KVM_REGS_R23($17) + ldl $24, KVM_REGS_R24($17) + ldl $25, KVM_REGS_R25($17) + ldl $26, KVM_REGS_R26($17) + ldl $27, KVM_REGS_R27($17) + ldl $28, KVM_REGS_R28($17) + + fldd $f0, KVM_REGS_FPCR($17) + wfpcr $f0 + fimovd $f0, $2 + and $2, 0x3, $2 + beq $2, $g_setfpec_0 + subl $2, 0x1, $2 + beq $2, $g_setfpec_1 + subl $2, 0x1, $2 + beq $2, $g_setfpec_2 + setfpec3 + br $g_setfpec_over +$g_setfpec_0: + setfpec0 + br $g_setfpec_over +$g_setfpec_1: + setfpec1 + br $g_setfpec_over +$g_setfpec_2: + setfpec2 +$g_setfpec_over: + ldl $2, KVM_REGS_R2($17) + vldd $f0, KVM_REGS_F0($17) + vldd $f1, KVM_REGS_F1($17) + vldd $f2, KVM_REGS_F2($17) + vldd $f3, KVM_REGS_F3($17) + vldd $f4, KVM_REGS_F4($17) + vldd $f5, KVM_REGS_F5($17) + vldd $f6, KVM_REGS_F6($17) + vldd $f7, KVM_REGS_F7($17) + vldd $f8, KVM_REGS_F8($17) + vldd $f9, KVM_REGS_F9($17) + vldd $f10, KVM_REGS_F10($17) + vldd $f11, KVM_REGS_F11($17) + vldd $f12, KVM_REGS_F12($17) + vldd $f13, KVM_REGS_F13($17) + vldd $f14, KVM_REGS_F14($17) + vldd $f15, KVM_REGS_F15($17) + vldd $f16, KVM_REGS_F16($17) + vldd $f17, KVM_REGS_F17($17) + vldd $f18, KVM_REGS_F18($17) + vldd $f19, KVM_REGS_F19($17) + vldd $f20, KVM_REGS_F20($17) + vldd $f21, KVM_REGS_F21($17) + vldd $f22, KVM_REGS_F22($17) + vldd $f23, KVM_REGS_F23($17) + vldd $f24, KVM_REGS_F24($17) + vldd $f25, KVM_REGS_F25($17) + vldd $f26, KVM_REGS_F26($17) + vldd $f27, KVM_REGS_F27($17) + vldd $f28, KVM_REGS_F28($17) + vldd $f29, KVM_REGS_F29($17) + vldd $f30, KVM_REGS_F30($17) + + ldi $17, KVM_REGS_PS($17) + + /* enter guest */ + /* r16 = guest vcpucb pointer */ + /* r17 = base of guest kvm_regs.ps, saved/restored by hmcode */ + + /* enter guest now */ + sys_call 0x31 + /* exit guest now */ + + ldi $17, -KVM_REGS_PS($17) /* r17: base of kvm_regs */ + + vstd $f0, KVM_REGS_F0($17) + vstd $f1, KVM_REGS_F1($17) + vstd $f2, KVM_REGS_F2($17) + vstd $f3, KVM_REGS_F3($17) + vstd $f4, KVM_REGS_F4($17) + vstd $f5, KVM_REGS_F5($17) + vstd $f6, KVM_REGS_F6($17) + vstd $f7, KVM_REGS_F7($17) + vstd $f8, KVM_REGS_F8($17) + vstd $f9, KVM_REGS_F9($17) + vstd $f10, KVM_REGS_F10($17) + vstd $f11, KVM_REGS_F11($17) + vstd $f12, KVM_REGS_F12($17) + vstd $f13, KVM_REGS_F13($17) + vstd $f14, KVM_REGS_F14($17) + vstd $f15, KVM_REGS_F15($17) + vstd $f16, KVM_REGS_F16($17) + vstd $f17, KVM_REGS_F17($17) + vstd $f18, KVM_REGS_F18($17) + vstd $f19, KVM_REGS_F19($17) + vstd $f20, KVM_REGS_F20($17) + vstd $f21, KVM_REGS_F21($17) + vstd $f22, KVM_REGS_F22($17) + vstd $f23, KVM_REGS_F23($17) + vstd $f24, KVM_REGS_F24($17) + vstd $f25, KVM_REGS_F25($17) + vstd $f26, KVM_REGS_F26($17) + vstd $f27, KVM_REGS_F27($17) + vstd $f28, KVM_REGS_F28($17) + vstd $f29, KVM_REGS_F29($17) + vstd $f30, KVM_REGS_F30($17) + + rfpcr $f0 + fstd $f0, KVM_REGS_FPCR($17) + + /* don't save r0 Hmcode have saved r0 for us */ + stl $1, KVM_REGS_R1($17) + stl $2, KVM_REGS_R2($17) + stl $3, KVM_REGS_R3($17) + stl $4, KVM_REGS_R4($17) + stl $5, KVM_REGS_R5($17) + stl $6, KVM_REGS_R6($17) + stl $7, KVM_REGS_R7($17) + stl $8, KVM_REGS_R8($17) + stl $9, KVM_REGS_R9($17) + stl $10, KVM_REGS_R10($17) + stl $11, KVM_REGS_R11($17) + stl $12, KVM_REGS_R12($17) + stl $13, KVM_REGS_R13($17) + stl $14, KVM_REGS_R14($17) + stl $15, KVM_REGS_R15($17) + stl $19, KVM_REGS_R19($17) + stl $20, KVM_REGS_R20($17) + stl $21, KVM_REGS_R21($17) + stl $22, KVM_REGS_R22($17) + stl $23, KVM_REGS_R23($17) + stl $24, KVM_REGS_R24($17) + stl $25, KVM_REGS_R25($17) + stl $26, KVM_REGS_R26($17) + stl $27, KVM_REGS_R27($17) + stl $28, KVM_REGS_R28($17) + + /* restore host regs from host sp */ + ldl $8, PT_REGS_R8(sp) + ldl $9, PT_REGS_R9(sp) + ldl $10, PT_REGS_R10(sp) + ldl $11, PT_REGS_R11(sp) + ldl $12, PT_REGS_R12(sp) + ldl $13, PT_REGS_R13(sp) + ldl $14, PT_REGS_R14(sp) + ldl $15, PT_REGS_R15(sp) + ldl $26, PT_REGS_R26(sp) + ldi sp, PT_REGS_SIZE(sp) + + /* restore host fpregs */ + fldd $f0, TASK_THREAD_FPCR($8) + wfpcr $f0 + fimovd $f0, $2 + and $2, 0x3, $2 + beq $2, $setfpec_0 + subl $2, 0x1, $2 + beq $2, $setfpec_1 + subl $2, 0x1, $2 + beq $2, $setfpec_2 + setfpec3 + br $setfpec_over +$setfpec_0: + setfpec0 + br $setfpec_over +$setfpec_1: + setfpec1 + br $setfpec_over +$setfpec_2: + setfpec2 +$setfpec_over: + vldd $f2, TASK_THREAD_F2($8) + vldd $f3, TASK_THREAD_F3($8) + vldd $f4, TASK_THREAD_F4($8) + vldd $f5, TASK_THREAD_F5($8) + vldd $f6, TASK_THREAD_F6($8) + vldd $f7, TASK_THREAD_F7($8) + vldd $f8, TASK_THREAD_F8($8) + vldd $f9, TASK_THREAD_F9($8) + + /* if $0 > 0, handle hcall */ + bgt $0, $ret_to + + stl $26, VCPU_RET_RA(sp) + stl $0, VCPU_RET_R0(sp) + + /* Hmcode will setup in */ + /* restore $16 $17 $18, do interrupt trick */ + ldi sp, -(HOST_INT_SIZE + PT_REGS_SIZE)(sp) + ldl $16, HOST_INT_R16(sp) + ldl $17, HOST_INT_R17(sp) + ldl $18, HOST_INT_R18(sp) + ldi sp, (HOST_INT_SIZE + PT_REGS_SIZE)(sp) + + ldi $19, -PT_REGS_SIZE(sp) + call $26, do_entInt + ldl $26, VCPU_RET_RA(sp) + ldl $0, VCPU_RET_R0(sp) +$ret_to: + /* ret($0) indicate hcall number */ + ldi sp, VCPU_RET_SIZE(sp) /* pop stack */ + ret diff --git a/arch/sw_64/kvm/handle_exit.c b/arch/sw_64/kvm/handle_exit.c new file mode 100644 index 000000000000..69b97860db88 --- /dev/null +++ b/arch/sw_64/kvm/handle_exit.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 yangzh + * linhn + */ +#include +#include +#include +#include +#include +#include +#include + +int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, + int exception_index, struct hcall_args *hargs) +{ + gfn_t gfn __maybe_unused; + + switch (exception_index) { + case SW64_KVM_EXIT_IO: + vcpu->stat.io_exits++; + return io_mem_abort(vcpu, run, hargs); + case SW64_KVM_MIGRATION_SET_DIRTY_HM: + case SW64_KVM_MIGRATION_SET_DIRTY: + vcpu->stat.migration_set_dirty++; + gfn = hargs->arg2 >> 24; + mutex_lock(&vcpu->kvm->slots_lock); + kvm_vcpu_mark_page_dirty(vcpu, gfn); + mutex_unlock(&vcpu->kvm->slots_lock); + return 1; + case SW64_KVM_EXIT_HALT: + vcpu->stat.halt_exits++; + vcpu->arch.halted = 1; + kvm_vcpu_block(vcpu); + return 1; + case SW64_KVM_EXIT_SHUTDOWN: + vcpu->stat.shutdown_exits++; + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SHUTDOWN; + return 0; + case SW64_KVM_EXIT_RESTART: + vcpu->stat.restart_exits++; + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET; + return 0; + case SW64_KVM_EXIT_STOP: + vcpu->stat.stop_exits++; + vcpu->arch.halted = 1; + memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); + kvm_vcpu_block(vcpu); + return 1; + case SW64_KVM_EXIT_TIMER: + vcpu->stat.timer_exits++; + set_timer(vcpu, hargs->arg0); + return 1; + case SW64_KVM_EXIT_IPI: + vcpu->stat.ipi_exits++; + vcpu_send_ipi(vcpu, hargs->arg0, hargs->arg1); + return 1; + case SW64_KVM_EXIT_DEBUG: + vcpu->stat.debug_exits++; + vcpu->run->exit_reason = KVM_EXIT_DEBUG; + vcpu->run->debug.arch.epc = vcpu->arch.regs.pc; + return 0; +#ifdef CONFIG_KVM_MEMHOTPLUG + case SW64_KVM_EXIT_MEMHOTPLUG: + vcpu->stat.memhotplug_exits++; + vcpu_mem_hotplug(vcpu, hargs->arg0); + return 1; +#endif +#ifdef CONFIG_SUBARCH_C4 + case SW64_KVM_EXIT_APT_FAULT: + return kvm_handle_guest_abort(vcpu, run); +#endif + case SW64_KVM_EXIT_FATAL_ERROR: + vcpu->stat.fatal_error_exits++; + pr_err("Guest fatal error: Reason=[%lx], EXC_PC=[%lx], DVA=[%lx]", hargs->arg0, hargs->arg1, hargs->arg2); + vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; + vcpu->run->hw.hardware_exit_reason = hargs->arg0; + return 0; + } + + return 1; +} diff --git a/arch/sw_64/kvm/irq.h b/arch/sw_64/kvm/irq.h new file mode 100644 index 000000000000..9268ab6af492 --- /dev/null +++ b/arch/sw_64/kvm/irq.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * irq.h: in kernel interrupt controller related definitions + */ + +#ifndef _SW64_KVM_IRQ_H +#define _SW64_KVM_IRQ_H +static inline int irqchip_in_kernel(struct kvm *kvm) +{ + return 1; +} +#endif /* _SW64_KVM_IRQ_H */ diff --git a/arch/sw_64/kvm/kvm_cma.c b/arch/sw_64/kvm/kvm_cma.c new file mode 100644 index 000000000000..de04eb5d20d7 --- /dev/null +++ b/arch/sw_64/kvm/kvm_cma.c @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Contiguous Memory Allocator for KVM + * + * This program is modified on the basis of CMA, to achieve cross-node + * memory reservation, as well as reserved memory information statistics. + */ + +#define pr_fmt(fmt) "kvm_cma: " fmt + +#include +#include +#include +#include +#include +#include + +#include "../../../mm/cma.h" +#include "../../../mm/internal.h" + +struct cma kvm_cma_areas[MAX_CMA_AREAS]; +unsigned int kvm_cma_area_count; + +static void __init init_kvm_cma_reserved_pageblock(struct page *page) +{ + unsigned int i = pageblock_nr_pages; + struct page *p = page; + + do { + __ClearPageReserved(p); + set_page_count(p, 0); + } while (++p, --i); + + set_pageblock_migratetype(page, MIGRATE_ISOLATE); + + if (pageblock_order >= MAX_ORDER) { + i = pageblock_nr_pages; + p = page; + do { + set_page_refcounted(p); + __free_pages(p, MAX_ORDER - 1); + p += MAX_ORDER_NR_PAGES; + } while (i -= MAX_ORDER_NR_PAGES); + } else { + set_page_refcounted(page); + __free_pages(page, pageblock_order); + } + + adjust_managed_page_count(page, pageblock_nr_pages); +} + +static int __init kvm_cma_activate_area(struct cma *cma) +{ + int bitmap_size = BITS_TO_LONGS(cma_bitmap_maxno(cma)) * sizeof(long); + unsigned long base_pfn = cma->base_pfn, pfn = base_pfn; + unsigned int i = cma->count >> pageblock_order; + + cma->bitmap = kzalloc(bitmap_size, GFP_KERNEL); + + if (!cma->bitmap) { + cma->count = 0; + return -ENOMEM; + } + + WARN_ON_ONCE(!pfn_valid(pfn)); + + do { + unsigned int j; + + base_pfn = pfn; + + for (j = pageblock_nr_pages; j; --j, pfn++) + WARN_ON_ONCE(!pfn_valid(pfn)); + + init_kvm_cma_reserved_pageblock(pfn_to_page(base_pfn)); + } while (--i); + + spin_lock_init(&cma->lock); + + return 0; +} + +static int __init kvm_cma_init_reserved_areas(void) +{ + int i; + + for (i = 0; i < kvm_cma_area_count; i++) { + int ret = kvm_cma_activate_area(&kvm_cma_areas[i]); + + if (ret) + return ret; + } + + return 0; +} +core_initcall(kvm_cma_init_reserved_areas); + +/** + * kvm_cma_init_reserved_mem() - create custom contiguous area + * from reserved memory + * @base: Base address of the reserved area + * @size: Size of the reserved area (in bytes), + * @order_per_bit: Order of pages represented by one bit on bitmap. + * @name: The name of the area. If this parameter is NULL, the name of + * the area will be set to "cmaN", where N is a running counter of + * used areas. + * @res_cma: Pointer to store the created cma region. + * + * This function creates custom contiguous area from already reserved memory. + */ +int __init kvm_cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, + unsigned int order_per_bit, const char *name, + struct cma **res_cma) +{ + struct cma *cma; + phys_addr_t alignment; + + /* Sanity checks */ + if (kvm_cma_area_count == ARRAY_SIZE(kvm_cma_areas)) { + pr_err("Not enough slots for CMA reserved regions!\n"); + return -ENOSPC; + } + + if (!size || !memblock_is_region_reserved(base, size)) + return -EINVAL; + + /* ensure minimal alignment required by mm core */ + alignment = PAGE_SIZE << + max_t(unsigned long, MAX_ORDER - 1, pageblock_order); + + /* alignment should be aligned with order_per_bit */ + if (!IS_ALIGNED(alignment >> PAGE_SHIFT, 1 << order_per_bit)) + return -EINVAL; + + if (ALIGN(base, alignment) != base || ALIGN(size, alignment) != size) + return -EINVAL; + + /* + * Each reserved area must be initialised later, when more kernel + * subsystems (like slab allocator) are available. + */ + cma = &kvm_cma_areas[kvm_cma_area_count]; + + if (name) + snprintf(cma->name, CMA_MAX_NAME, name); + else + snprintf(cma->name, CMA_MAX_NAME, "cma%d\n", cma_area_count); + + cma->base_pfn = PFN_DOWN(base); + cma->count = size >> PAGE_SHIFT; + cma->order_per_bit = order_per_bit; + *res_cma = cma; + kvm_cma_area_count++; + totalcma_pages += (size / PAGE_SIZE); + + return 0; +} + +/** + * kvm_cma_declare_contiguous() - reserve contiguous area for VM + * @base: Base address of the reserved area optional, + * @size: Size of the reserved area (in bytes), + * @limit: End address of the reserved memory (optional, 0 for any). + * @alignment: Alignment for the CMA area, should be power of 2 or zero + * @order_per_bit: Order of pages represented by one bit on bitmap. + * @name: The name of the area. See function cma_init_reserved_mem() + * @res_cma: Pointer to store the created cma region. + * + * This function reserves memory from early allocator. It should be + * called by arch specific code once the early allocator (memblock or bootmem) + * has been activated and all other subsystems have already allocated/reserved + * memory. This function allows to create custom reserved areas. + */ +int __init kvm_cma_declare_contiguous(phys_addr_t base, + phys_addr_t size, phys_addr_t limit, + phys_addr_t alignment, unsigned int order_per_bit, + const char *name, struct cma **res_cma) +{ + phys_addr_t memblock_end = memblock_end_of_DRAM(); + phys_addr_t highmem_start; + int ret = 0; + + /* + * We can't use __pa(high_memory) directly, since high_memory + * isn't a valid direct map VA, and DEBUG_VIRTUAL will (validly) + * complain. Find the boundary by adding one to the last valid + * address. + */ + highmem_start = __pa(high_memory - 1) + 1; + + if (!size) + return -EINVAL; + + if (alignment && !is_power_of_2(alignment)) + return -EINVAL; + + /* + * Sanitise input arguments. + * Pages both ends in CMA area could be merged into adjacent unmovable + * migratetype page by page allocator's buddy algorithm. In the case, + * you couldn't get a contiguous memory, which is not what we want. + */ + alignment = max(alignment, (phys_addr_t)PAGE_SIZE << + max_t(unsigned long, MAX_ORDER - 1, pageblock_order)); + if (base & (alignment - 1)) { + ret = -EINVAL; + pr_err("Region at %pa must be aligned to %pa bytes\n", + &base, &alignment); + goto err; + } + base = ALIGN(base, alignment); + size = ALIGN(size, alignment); + limit &= ~(alignment - 1); + + if (!base) { + pr_err("Base address of region must be needed!\n"); + goto err; + } + + /* size should be aligned with order_per_bit */ + if (!IS_ALIGNED(size >> PAGE_SHIFT, 1 << order_per_bit)) + return -EINVAL; + + /* + * The request region must not cross the low/high memory boundary. + */ + if (base < highmem_start && base + size > highmem_start) { + ret = -EINVAL; + pr_err("Region at %pa defined on low/high memory boundary (%pa)\n", + &base, &highmem_start); + goto err; + } + + /* + * If the limit is unspecified or above the memblock end, its effective + * value will be the memblock end. Set it explicitly to simplify further + * checks. + */ + if (limit == 0 || limit > memblock_end) + limit = memblock_end; + + if (base + size > limit) { + ret = -EINVAL; + pr_err("Size (%pa) of region at %pa exceeds limit (%pa)\n", + &size, &base, &limit); + goto err; + } + + /* Reserve memory */ + if (memblock_is_region_reserved(base, size) || + memblock_reserve(base, size) < 0) { + ret = -EBUSY; + goto err; + } + ret = kvm_cma_init_reserved_mem(base, size, order_per_bit, + name, res_cma); + if (ret) + goto free_mem; + + pr_info("Reserved %ld MiB at %pa\n", (unsigned long)size / SZ_1M, + &base); + return 0; + +free_mem: + memblock_free((void *)base, size); +err: + pr_err("Failed to reserve %ld MiB\n", (unsigned long)size / SZ_1M); + return ret; +} diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c new file mode 100644 index 000000000000..f7e9150d40e0 --- /dev/null +++ b/arch/sw_64/kvm/kvm_core3.c @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 yangzh + * linhn + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "trace.h" +#include "vmem.c" + +__read_mostly bool bind_vcpu_enabled; + +#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) +static int __init bind_vcpu_init(void) +{ + if (!sw64_debugfs_dir) + return -ENODEV; + debugfs_create_bool("bind_vcpu", 0644, + sw64_debugfs_dir, &bind_vcpu_enabled); + return 0; +} + +static void bind_vcpu_exit(void) +{ + bind_vcpu_enabled = false; +} +#else +static int __init bind_vcpu_init(void) +{ + return 0; +} + +static void bind_vcpu_exit(void) { } + +#endif + +static unsigned long longtime_offset; + +#ifdef CONFIG_KVM_MEMHOTPLUG +static unsigned long get_vpcr(struct kvm_vcpu *vcpu, u64 vpn) +{ + unsigned long base; + + base = virt_to_phys(vcpu->kvm->arch.seg_pgd); + return base | ((vpn & VPN_MASK) << 44); +} +#else +static unsigned long get_vpcr(struct kvm_vcpu *vcpu, u64 vpn) +{ + unsigned long base, size; + + base = vcpu->kvm->arch.host_phys_addr; + size = vcpu->kvm->arch.size; + return (base >> 23) | ((size >> 23) << 16) | ((vpn & VPN_MASK) << 44); +} +#endif + +void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.vcb.vpcr == 0) { + vcpu->arch.vcb.vpcr = get_vpcr(vcpu, 0); +#ifndef CONFIG_KVM_MEMHOTPLUG + if (unlikely(bind_vcpu_enabled)) { + int nid; + unsigned long end; + + end = vcpu->kvm->arch.host_phys_addr + vcpu->kvm->arch.size; + nid = pfn_to_nid(PHYS_PFN(vcpu->kvm->arch.host_phys_addr)); + if (pfn_to_nid(PHYS_PFN(end)) == nid) + set_cpus_allowed_ptr(vcpu->arch.tsk, cpumask_of_node(nid)); + } +#endif + vcpu->arch.vcb.upcr = 0x7; + } +} + +void kvm_flush_tlb_all(void) +{ + tbia(); +} + +void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) +{ + vcpu->arch.vcb.vpcr = ((vcpu->arch.vcb.vpcr) & (~(VPN_MASK << 44))) | (vpn << 44); + vcpu->arch.vcb.dtb_vpcr = ((vcpu->arch.vcb.dtb_vpcr) & (~(VPN_MASK << VPN_SHIFT))) | (vpn << VPN_SHIFT); +} + +int kvm_sw64_init_vm(struct kvm *kvm) +{ +#ifdef CONFIG_KVM_MEMHOTPLUG + unsigned long *seg_pgd; + + if (kvm->arch.seg_pgd != NULL) { + kvm_err("kvm_arch already initialized?\n"); + return -EINVAL; + } + + seg_pgd = alloc_pages_exact(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO); + if (!seg_pgd) + return -ENOMEM; + + kvm->arch.seg_pgd = seg_pgd; + #endif + return 0; +} + +void kvm_sw64_destroy_vm(struct kvm *kvm) +{ + #ifdef CONFIG_KVM_MEMHOTPLUG + void *seg_pgd = NULL; + + if (kvm->arch.seg_pgd) { + seg_pgd = READ_ONCE(kvm->arch.seg_pgd); + kvm->arch.seg_pgd = NULL; + } + + if (seg_pgd) + free_pages_exact(seg_pgd, PAGE_SIZE); + #endif + kvm_destroy_vcpus(kvm); +} + +#ifdef CONFIG_KVM_MEMHOTPLUG +static void setup_segment_table(struct kvm *kvm, + struct kvm_memory_slot *memslot, unsigned long addr, size_t size) +{ + unsigned long *seg_pgd = kvm->arch.seg_pgd; + unsigned long num_of_entry; + unsigned long base_hpa = addr; + unsigned long i; + + num_of_entry = round_up(size, 1 << 30) >> 30; + + for (i = 0; i < num_of_entry; i++) { + *seg_pgd = base_hpa + (i << 30); + seg_pgd++; + } +} +#endif + +int kvm_arch_prepare_memory_region(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + unsigned long addr; + struct file *vm_file; + struct vm_area_struct *vma; + struct vmem_info *info; + struct kvm_userspace_memory_region new_mem; + struct kvm_userspace_memory_region *mem = &new_mem; + unsigned long ret; + size_t size; + + mem->flags = new->flags; + mem->guest_phys_addr = ((new->base_gfn) << PAGE_SHIFT); + mem->memory_size = ((new->npages) << PAGE_SHIFT); + mem->userspace_addr = new->userspace_addr; + + if (change == KVM_MR_FLAGS_ONLY || change == KVM_MR_DELETE) + return 0; + + if (test_bit(IO_MARK_BIT, (unsigned long *)(&(mem->guest_phys_addr)))) + return 0; + + if (test_bit(IO_MARK_BIT + 1, (unsigned long *)(&(mem->guest_phys_addr)))) + return 0; + +#ifndef CONFIG_KVM_MEMHOTPLUG + if (mem->guest_phys_addr) { + pr_info("%s, No KVM MEMHOTPLUG support!\n", __func__); + return 0; + } +#endif + if (!sw64_kvm_pool) + return -ENOMEM; + + pr_info("%s: %#llx %#llx, user addr: %#llx\n", __func__, + mem->guest_phys_addr, mem->memory_size, mem->userspace_addr); + + vma = find_vma(current->mm, mem->userspace_addr); + if (!vma) + return -ENOMEM; + vm_file = vma->vm_file; + + if (!vm_file) { + info = kzalloc(sizeof(struct vmem_info), GFP_KERNEL); + + size = round_up(mem->memory_size, 8<<20); + addr = gen_pool_alloc(sw64_kvm_pool, size); + if (!addr) + return -ENOMEM; + vm_munmap(mem->userspace_addr, mem->memory_size); + ret = vm_mmap(vm_file, mem->userspace_addr, mem->memory_size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, 0); + if ((long)ret < 0) + return ret; + + vma = find_vma(current->mm, mem->userspace_addr); + if (!vma) + return -ENOMEM; + + info->start = addr; + info->size = size; + vma->vm_private_data = (void *) info; + + vma->vm_ops = &vmem_vm_ops; + vma->vm_ops->open(vma); + + ret = vmem_vm_insert_page(vma); + if ((int)ret < 0) + return ret; + } else { + info = vm_file->private_data; + addr = info->start; + } + + pr_info("guest phys addr = %#lx, size = %#lx\n", + addr, vma->vm_end - vma->vm_start); + + kvm->arch.host_phys_addr = (u64)addr; + kvm->arch.size = round_up(mem->memory_size, 8<<20); + + memset(__va(addr), 0, 0x2000000); + + return 0; +} + +/* + * kvm_mark_migration write the mark on every vcpucbs of the kvm, which tells + * the system to do migration while the mark is on, and flush all vcpu's tlbs + * at the beginning of the migration. + */ +void kvm_mark_migration(struct kvm *kvm, int mark) +{ + struct kvm_vcpu *vcpu; + unsigned long cpu; + + kvm_for_each_vcpu(cpu, vcpu, kvm) + vcpu->arch.vcb.migration_mark = mark << 2; + + kvm_flush_remote_tlbs(kvm); +} + +void kvm_arch_commit_memory_region(struct kvm *kvm, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + /* + * At this point memslot has been committed and there is an + * allocated dirty_bitmap[], dirty pages will be tracked while the + * memory slot is write protected. + */ + + + /* If it's the first time dirty logging, flush all vcpu tlbs. */ + if ((change == KVM_MR_FLAGS_ONLY) && (new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + kvm_mark_migration(kvm, 1); +} + +int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) +{ + unsigned long addr = vcpu->kvm->arch.host_phys_addr; + + hrtimer_cancel(&vcpu->arch.hrt); + vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; + vcpu->arch.vcb.vcpu_irq_disabled = 1; + vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ + vcpu->arch.power_off = 0; + memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); + + if (vcpu->vcpu_id == 0) + memset(__va(addr), 0, 0x2000000); + + return 0; +} + +long kvm_sw64_get_vcb(struct file *filp, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + + if (vcpu->arch.vcb.migration_mark) { + unsigned long result = sw64_io_read(0, LONG_TIME) + + vcpu->arch.vcb.guest_longtime_offset; + vcpu->arch.vcb.guest_longtime = result; + vcpu->arch.vcb.guest_irqs_pending = vcpu->arch.irqs_pending[0]; + } + + if (copy_to_user((void __user *)arg, &(vcpu->arch.vcb), sizeof(struct vcpucb))) + return -EINVAL; + + return 0; +} + +long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) +{ + unsigned long result; + struct kvm_vcpu *vcpu = filp->private_data; + struct vcpucb *kvm_vcb; + + kvm_vcb = memdup_user((void __user *)arg, sizeof(*kvm_vcb)); + memcpy(&(vcpu->arch.vcb), kvm_vcb, sizeof(struct vcpucb)); + + if (vcpu->arch.vcb.migration_mark) { + /* updated vpcr needed by destination vm */ + vcpu->arch.vcb.vpcr = get_vpcr(vcpu, 0); + /* synchronize the longtime of source and destination */ + if (vcpu->arch.vcb.soft_cid == 0) { + result = sw64_io_read(0, LONG_TIME); + vcpu->arch.vcb.guest_longtime_offset = vcpu->arch.vcb.guest_longtime - result; + longtime_offset = vcpu->arch.vcb.guest_longtime_offset; + } else + vcpu->arch.vcb.guest_longtime_offset = longtime_offset; + + set_timer(vcpu, 200000000); + vcpu->arch.vcb.migration_mark = 0; + } + + return 0; +} + +#ifdef CONFIG_KVM_MEMHOTPLUG +void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr) +{ + struct kvm *kvm = vcpu->kvm; + struct kvm_memory_slot *slot; + unsigned long start_pfn = start_addr >> PAGE_SHIFT; + + kvm_for_each_memslot(slot, kvm_memslots(kvm)) { + if (start_pfn == slot->base_gfn) { + unsigned long *seg_pgd; + unsigned long num_of_entry = slot->npages >> 17; + unsigned long base_hpa = slot->arch.host_phys_addr; + unsigned long i; + + seg_pgd = kvm->arch.seg_pgd + (start_pfn >> 17); + for (i = 0; i < num_of_entry; i++) { + *seg_pgd = base_hpa + (i << 30); + seg_pgd++; + } + } + } +} +#endif + +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) +{ +} + +void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, gfn_t gfn_offset, + unsigned long mask) +{ +} + +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) +{ +} + +void kvm_arch_flush_shadow_all(struct kvm *kvm) +{ +} + +void update_aptp(unsigned long pgd) +{ +} + +static int __init kvm_core3_init(void) +{ + int i, ret; + + bind_vcpu_init(); + + ret = vmem_init(); + if (unlikely(ret)) + goto out; + + for (i = 0; i < NR_CPUS; i++) + last_vpn(i) = VPN_FIRST_VERSION; + + ret = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); + + if (likely(!ret)) + return 0; + + vmem_exit(); +out: + bind_vcpu_exit(); + return ret; +} + +static void __exit kvm_core3_exit(void) +{ + kvm_exit(); + vmem_exit(); + bind_vcpu_exit(); +} + +module_init(kvm_core3_init); +module_exit(kvm_core3_exit); diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c new file mode 100644 index 000000000000..08d28a365a3b --- /dev/null +++ b/arch/sw_64/kvm/kvm_core4.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 yangzh + * linhn + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include "trace.h" + +static unsigned long shtclock_offset; + +void update_aptp(unsigned long pgd) +{ + imemb(); + write_csr_imb(pgd, CSR_APTP); +} + +void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) +{ + vcpu->arch.vcb.vpcr = vpn << 44; + vcpu->arch.vcb.dtb_vpcr = vpn; +} + +void kvm_flush_tlb_all(void) +{ + tbivpn(-1, 0, 0); +} + +int kvm_sw64_init_vm(struct kvm *kvm) +{ + return kvm_alloc_addtional_stage_pgd(kvm); +} + +void kvm_sw64_destroy_vm(struct kvm *kvm) +{ + kvm_destroy_vcpus(kvm); +} + +int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.has_run_once) + apt_unmap_vm(vcpu->kvm); + + hrtimer_cancel(&vcpu->arch.hrt); + vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; + vcpu->arch.vcb.vcpu_irq_disabled = 1; + vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ + vcpu->arch.power_off = 0; + memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); + + return 0; +} + +long kvm_sw64_get_vcb(struct file *filp, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + + if (vcpu->arch.migration_mark) + vcpu->arch.shtclock = read_csr(CSR_SHTCLOCK) + + vcpu->arch.vcb.shtclock_offset; + if (copy_to_user((void __user *)arg, &(vcpu->arch.vcb), sizeof(struct vcpucb))) + return -EINVAL; + + return 0; +} + +long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + struct vcpucb *kvm_vcb; + + kvm_vcb = memdup_user((void __user *)arg, sizeof(*kvm_vcb)); + memcpy(&(vcpu->arch.vcb), kvm_vcb, sizeof(struct vcpucb)); + + if (vcpu->arch.migration_mark) { + /* synchronize the longtime of source and destination */ + if (vcpu->arch.vcb.soft_cid == 0) + shtclock_offset = vcpu->arch.shtclock - read_csr(CSR_SHTCLOCK); + vcpu->arch.vcb.shtclock_offset = shtclock_offset; + set_timer(vcpu, 200000000); + vcpu->arch.migration_mark = 0; + } + return 0; +} + +int kvm_arch_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *memslot, + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) +{ + return 0; +} + +void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu) +{ +} + +static int __init kvm_core4_init(void) +{ + int i, ret; + + for (i = 0; i < NR_CPUS; i++) + last_vpn(i) = VPN_FIRST_VERSION; + + ret = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); + + if (ret) + return ret; + + return 0; +} + +static void __exit kvm_core4_exit(void) +{ + kvm_exit(); +} + +module_init(kvm_core4_init); +module_exit(kvm_core4_exit); diff --git a/arch/sw_64/kvm/kvm_timer.c b/arch/sw_64/kvm/kvm_timer.c new file mode 100644 index 000000000000..895be63cd8d1 --- /dev/null +++ b/arch/sw_64/kvm/kvm_timer.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 yangzh + */ +#include +#include +#include +#include + +/* + * The Guest Clock. + * + * There are two sources of virtual interrupts. We saw one in lguest_user.c: + * the Launcher sending interrupts for virtual devices. The other is the Guest + * timer interrupt. + * + * The Guest uses the LHCALL_SET_CLOCKEVENT hypercall to tell us how long to + * the next timer interrupt (in ticks). We use the high-resolution timer + * infrastructure to set a callback at that time. + * + * 0 means "turn off the clock". + */ + +void set_timer(struct kvm_vcpu *vcpu, unsigned long delta) +{ + ktime_t expires; + + if (unlikely(delta == 0)) { + /* Clock event device is shutting down. */ + hrtimer_cancel(&vcpu->arch.hrt); + return; + } + + /* Convert clock event device ticks to nanoseconds */ + delta = delta * NSEC_PER_SEC; + do_div(delta, vcpu->arch.vtimer_freq); + + /* + * We use wallclock time here, so the Guest might not be running for + * all the time between now and the timer interrupt it asked for. This + * is almost always the right thing to do. + */ + + expires = ktime_add_ns(ktime_get_real(), delta); + vcpu->arch.timer_next_event = expires; + hrtimer_start(&vcpu->arch.hrt, expires, HRTIMER_MODE_ABS); +} + +/* And this is the routine when we want to set an interrupt for the Guest. */ +void set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) +{ + /* + * Next time the Guest runs, the core code will see if it can deliver + * this interrupt. + */ + set_bit(irq, (vcpu->arch.irqs_pending)); + + /* + * Make sure it sees it; it might be asleep (eg. halted), or running + * the Guest right now, in which case kick_process() will knock it out. + */ + kvm_vcpu_kick(vcpu); +} + +enum hrtimer_restart clockdev_fn(struct hrtimer *timer) +{ + struct kvm_vcpu *vcpu; + ktime_t now, delta; + + vcpu = container_of(timer, struct kvm_vcpu, arch.hrt); + + now = ktime_get_real(); + + if (now < vcpu->arch.timer_next_event) { + delta = vcpu->arch.timer_next_event - now; + hrtimer_forward_now(timer, delta); + return HRTIMER_RESTART; + } + + set_interrupt(vcpu, SW64_KVM_IRQ_TIMER); + return HRTIMER_NORESTART; +} diff --git a/arch/sw_64/kvm/mmio.c b/arch/sw_64/kvm/mmio.c new file mode 100644 index 000000000000..21ad89722f9a --- /dev/null +++ b/arch/sw_64/kvm/mmio.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 yangzh + * linhn + */ +#include +#include +#include +#include + +static unsigned long mmio_read_buf(char *buf, unsigned int len) +{ + unsigned long data = 0; + union { + u16 hword; + u32 word; + u64 dword; + } tmp; + + switch (len) { + case 1: + data = buf[0]; + break; + case 2: + memcpy(&tmp.hword, buf, len); + data = tmp.hword; + break; + case 4: + memcpy(&tmp.word, buf, len); + data = tmp.word; + break; + case 8: + memcpy(&tmp.dword, buf, len); + data = tmp.dword; + break; + } + + return data; +} + +int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + unsigned long data; + unsigned int len; + + if (!run->mmio.is_write) { + len = run->mmio.len; + if (len > sizeof(unsigned long)) + return -EINVAL; + + data = mmio_read_buf(run->mmio.data, len); + vcpu_set_reg(vcpu, vcpu->arch.mmio_decode.rt, data); + } + + vcpu->stat.mmio_exits++; + vcpu->arch.regs.pc += 4; + + return 0; +} + +int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, + struct hcall_args *hargs) +{ + int ret; + +#ifdef CONFIG_SUBARCH_C3B + run->mmio.phys_addr = hargs->arg1 & 0xfffffffffffffUL; + sw64_decode(vcpu, hargs->arg2, run); +#elif defined(CONFIG_SUBARCH_C4) + run->mmio.phys_addr = read_csr(CSR_DVA) & 0xfffffffffffffUL; + sw64_decode(vcpu, 0, run); +#endif + if (run->mmio.is_write) + ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, run->mmio.data); + else + ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, + run->mmio.len, run->mmio.data); + + if (!ret) { + /* We handled the access successfully in the kernel. */ + kvm_handle_mmio_return(vcpu, run); + return 1; + } + + run->exit_reason = KVM_EXIT_MMIO; + return 0; +} diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c new file mode 100644 index 000000000000..573d32b171d6 --- /dev/null +++ b/arch/sw_64/kvm/mmu.c @@ -0,0 +1,1561 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 - os kernal + * Author: lff + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#define KVM_APT_FLAG_LOGGING_ACTIVE (1UL << 1) + +static bool memslot_is_logging(struct kvm_memory_slot *memslot) +{ + return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); +} + +/* + * Return values of kvm_handle_mmio_page_fault and mmu.page_fault: + * RET_AF_RETRY: let CPU fault again on the address. + * RET_AF_EMULATE: mmio page fault, emulate the instruction directly. + * + * For kvm_handle_mmio_page_fault only: + * RET_AF_INVALID: the spte is invalid, let the real page fault path update it. + */ +enum { + RET_AF_RETRY = 0, + RET_AF_EMULATE = 1, + RET_AF_INVALID = 2, +}; + +/** + * apt_dissolve_pmd() - clear and flush huge PMD entry + * @kvm: pointer to kvm structure. + * @addr: IPA + * @pmd: pmd pointer for IPA + * + * Function clears a PMD entry, flushes TLBs. + */ +static void apt_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd) +{ + int i; + + if (!pmd_trans_huge(*pmd)) + return; + + if (pmd_trans_cont(*pmd)) { + for (i = 0; i < CONT_PMDS; i++, pmd++) + pmd_clear(pmd); + } else + pmd_clear(pmd); + + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pmd)); +} + +/** + * apt_dissolve_pud() - clear and flush huge PUD entry + * @kvm: pointer to kvm structure. + * @addr: IPA + * @pud: pud pointer for IPA + * + * Function clears a PUD entry, flushes TLBs. + */ +static void apt_dissolve_pud(struct kvm *kvm, phys_addr_t addr, pud_t *pudp) +{ + if (!pud_huge(*pudp)) + return; + + pud_clear(pudp); + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pudp)); +} + +static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, + int min, int max) +{ + void *page; + + BUG_ON(max > KVM_NR_MEM_OBJS); + if (cache->nobjs >= min) + return 0; + while (cache->nobjs < max) { + page = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return -ENOMEM; + cache->objects[cache->nobjs++] = page; + } + return 0; +} + +static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) +{ + while (mc->nobjs) + free_page((unsigned long)mc->objects[--mc->nobjs]); +} + +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) +{ + mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); +} + +static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) +{ + void *p; + + BUG_ON(!mc || !mc->nobjs); + p = mc->objects[--mc->nobjs]; + return p; +} + +static void unmap_apt_ptes(struct kvm *kvm, pmd_t *pmd, + phys_addr_t addr, phys_addr_t end) +{ + pte_t *pte, *start_pte; + struct page *ptr_page; + + start_pte = pte = pte_offset_kernel(pmd, addr); + do { + if (!pte_none(*pte)) { + /* Do we need WRITE_ONCE(pte, 0)? */ + set_pte(pte, __pte(0)); + put_page(virt_to_page(pte)); + } + } while (pte++, addr += PAGE_SIZE, addr != end); + + ptr_page = virt_to_page(start_pte); + if (page_count(ptr_page) == 1) { + pte_t *pte_table = pte_offset_kernel(pmd, 0); + + pmd_clear(pmd); + free_page((unsigned long)pte_table); + put_page(virt_to_page(pmd)); + } +} + +static void unmap_apt_pmds(struct kvm *kvm, pud_t *pud, + phys_addr_t addr, phys_addr_t end) +{ + phys_addr_t next; + pmd_t *pmd, *start_pmd; + struct page *ptr_page; + int i; + + start_pmd = pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (!pmd_none(*pmd)) { + if (pmd_trans_huge(*pmd)) { + if (pmd_trans_cont(*pmd)) { + for (i = 0; i < CONT_PMDS; i++, pmd++) + pmd_clear(pmd); + } else + pmd_clear(pmd); + /* Do we need flush tlb???? edited by lff */ + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pmd)); + } else { + unmap_apt_ptes(kvm, pmd, addr, next); + } + } + } while (pmd++, addr = next, addr != end); + + ptr_page = virt_to_page(start_pmd); + if (page_count(ptr_page) == 1) { + pmd_t *pmd_table __maybe_unused = pmd_offset(pud, 0UL); + + pud_clear(pud); + free_page((unsigned long)pmd_table); + put_page(virt_to_page(pud)); + } +} + +static void unmap_apt_puds(struct kvm *kvm, p4d_t *p4d, + phys_addr_t addr, phys_addr_t end) +{ + phys_addr_t next; + pud_t *pud, *start_pud; + struct page *ptr_page; + + start_pud = pud = pud_offset(p4d, addr); + do { + next = pud_addr_end(addr, end); + if (!pud_none(*pud)) { + if (pud_huge(*pud)) { + pud_clear(pud); + /* Do we need flush tlb???? edited by lff */ + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pud)); + } else { + unmap_apt_pmds(kvm, pud, addr, next); + } + } + } while (pud++, addr = next, addr != end); + + ptr_page = virt_to_page(start_pud); + if (page_count(ptr_page) == 1) { + pud_t *pud_table __maybe_unused = pud_offset(p4d, 0UL); + + p4d_clear(p4d); + kvm_flush_remote_tlbs(kvm); + free_page((unsigned long)pud_table); + put_page(virt_to_page(p4d)); + } +} + +/** + * unmap_apt_range -- Clear addtional page table entries to unmap a range + * @kvm: The VM pointer + * @start: The intermediate physical base address of the range to unmap + * @size: The size of the area to unmap + * + * Clear a range of apt mappings, lowering the various ref-counts. Must + * be called while holding mmu_lock (unless for freeing the apt pgd before + * destroying the VM), otherwise another faulting VCPU may come in and mess + * with things behind our backs. + */ +static void unmap_apt_range(struct kvm *kvm, phys_addr_t start, u64 size) +{ + pgd_t *pgd; + p4d_t *p4d; + phys_addr_t addr = start, end = start + size; + phys_addr_t next; + + assert_spin_locked(&kvm->mmu_lock); + WARN_ON(size & ~PAGE_MASK); + + pgd = kvm->arch.pgd + pgd_index(addr); + p4d = p4d_offset(pgd, addr); + do { + /* + * Make sure the page table is still active, as another thread + * could have possibly freed the page table, while we released + * the lock. + */ + if (!READ_ONCE(kvm->arch.pgd)) + break; + next = p4d_addr_end(addr, end); + if (!p4d_none(*p4d)) + unmap_apt_puds(kvm, p4d, addr, next); + /* + * If the range is too large, release the kvm->mmu_lock + * to prevent starvation and lockup detector warnings. + */ + if (next != end) + cond_resched_lock(&kvm->mmu_lock); + } while (pgd++, addr = next, addr != end); +} + +static void apt_unmap_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) +{ + hva_t hva = memslot->userspace_addr; + phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT; + phys_addr_t size = PAGE_SIZE * memslot->npages; + hva_t reg_end = hva + size; + + /* + * A memory region could potentially cover multiple VMAs, and any holes + * between them, so iterate over all of them to find out if we should + * unmap any of them. + * + * +--------------------------------------------+ + * +---------------+----------------+ +----------------+ + * | : VMA 1 | VMA 2 | | VMA 3 : | + * +---------------+----------------+ +----------------+ + * | memory region | + * +--------------------------------------------+ + */ + do { + struct vm_area_struct *vma = find_vma(current->mm, hva); + hva_t vm_start, vm_end; + + if (!vma || vma->vm_start >= reg_end) + break; + + /* + * Take the intersection of this VMA with the memory region + */ + vm_start = max(hva, vma->vm_start); + vm_end = min(reg_end, vma->vm_end); + + if (!(vma->vm_flags & VM_PFNMAP)) { + gpa_t gpa = addr + (vm_start - memslot->userspace_addr); + + unmap_apt_range(kvm, gpa, vm_end - vm_start); + } + hva = vm_end; + } while (hva < reg_end); +} + +/** + * apt_unmap_vm - Unmap Additional Stage RAM mappings + * @kvm: The struct kvm pointer + * + * Go through the memregions and unmap any reguler RAM + * backing memory already mapped to the VM. + */ +void apt_unmap_vm(struct kvm *kvm) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *memslot; + int idx; + + idx = srcu_read_lock(&kvm->srcu); + down_read(¤t->mm->mmap_lock); + spin_lock(&kvm->mmu_lock); + + slots = kvm_memslots(kvm); + kvm_for_each_memslot(memslot, slots) + apt_unmap_memslot(kvm, memslot); + spin_unlock(&kvm->mmu_lock); + up_read(¤t->mm->mmap_lock); + srcu_read_unlock(&kvm->srcu, idx); +} + +static pud_t *apt_get_pud(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr) +{ + p4d_t *p4d; + pud_t *pud; + + pgd += pgd_index(addr); + if (pgd_none(*pgd)) { + /* Not used on SW64 yet */ + VM_BUG_ON(pgd); + return NULL; + } + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) { + if (!cache) + return NULL; + pud = mmu_memory_cache_alloc(cache); + p4d_populate(NULL, p4d, pud); + get_page(virt_to_page(p4d)); + } + return pud_offset(p4d, addr); +} + +static pmd_t *apt_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, unsigned long sz) +{ + pud_t *pud; + pmd_t *pmd; + + pud = apt_get_pud(kvm->arch.pgd, cache, addr); + if (!pud || pud_huge(*pud)) + return NULL; + + if (pud_none(*pud)) { + if (!cache) + return NULL; + pmd = mmu_memory_cache_alloc(cache); + pud_populate(NULL, pud, pmd); + get_page(virt_to_page(pud)); + } + if (sz == CONT_PMD_SIZE) + addr &= CONT_PMD_MASK; + + return pmd_offset(pud, addr); +} + +static bool kvm_is_write_fault(unsigned long access_type) +{ + if (access_type == AF_WRITE_ACCESS_TYPE) + return true; + + return false; +} + +static bool kvm_is_exec_fault(unsigned long access_type) +{ + if (access_type == AF_EXEC_ACCESS_TYPE) + return true; + + return false; +} +/** + * apt_wp_ptes - write protect PMD range + * @pmd: pointer to pmd entry + * @addr: range start address + * @end: range end address + */ +static void apt_wp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end) +{ + pte_t *pte; + + pte = pte_offset_kernel(pmd, addr); + do { + if (!pte_none(*pte)) { + if (!kvm_aptpte_readonly(pte)) + kvm_set_aptpte_readonly(pte); + } + } while (pte++, addr += PAGE_SIZE, addr != end); +} + +/** + * apt_wp_pmds - write protect PUD range + * @pud: pointer to pud entry + * @addr: range start address + * @end: range end address + */ +static void apt_wp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end) +{ + pmd_t *pmd; + phys_addr_t next; + + pmd = pmd_offset(pud, addr); + + do { + next = pmd_addr_end(addr, end); + if (!pmd_none(*pmd)) { + if (pmd_trans_huge(*pmd)) { + if (!kvm_aptpmd_readonly(pmd)) + kvm_set_aptpmd_readonly(pmd); + } else { + apt_wp_ptes(pmd, addr, next); + } + } + } while (pmd++, addr = next, addr != end); +} + +/** + * apt_wp_puds - write protect PGD range + * @pgd: pointer to pgd entry + * @addr: range start address + * @end: range end address + * + * Process PUD entries, for a huge PUD we cause a panic. + */ +static void apt_wp_puds(p4d_t *p4d, phys_addr_t addr, phys_addr_t end) +{ + pud_t *pud; + phys_addr_t next; + + pud = pud_offset(p4d, addr); + do { + next = pud_addr_end(addr, end); + if (!pud_none(*pud)) { + if (pud_huge(*pud)) { + if (!kvm_aptpud_readonly(pud)) + kvm_set_aptpud_readonly(pud); + } else { + /* TODO:PUD not supported, revisit later if supported */ +// BUG_ON(pud_trans_huge(*pud)); + apt_wp_pmds(pud, addr, next); + } + } + } while (pud++, addr = next, addr != end); +} + +/** + * apt_wp_range() - write protect apt memory region range + * @kvm: The KVM pointer + * @addr: Start address of range + * @end: End address of range + */ +static void apt_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) +{ + pgd_t *pgd; + p4d_t *p4d; + phys_addr_t next; + + pgd = kvm->arch.pgd + pgd_index(addr); + p4d = p4d_offset(pgd, addr); + + do { + /* + * Release kvm_mmu_lock periodically if the memory region is + * large. Otherwise, we may see kernel panics with + * CONFIG_DETECT_HUNG_TASK, CONFIG_LOCKUP_DETECTOR, + * CONFIG_LOCKDEP. Additionally, holding the lock too long + * will also starve other vCPUs. We have to also make sure + * that the page tables are not freed while we released + * the lock. + */ + cond_resched_lock(&kvm->mmu_lock); + if (!READ_ONCE(kvm->arch.pgd)) + break; + next = p4d_addr_end(addr, end); + if (p4d_present(*p4d)) + apt_wp_puds(p4d, addr, next); + } while (p4d++, addr = next, addr != end); +} + +/** + * kvm_mmu_wp_memory_region() - write protect apt entries for memory slot + * @kvm: The KVM pointer + * @slot: The memory slot to write protect + * + * Called to start logging dirty pages after memory region + * KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns + * all present PMD and PTEs are write protected in the memory region. + * Afterwards read of dirty page log can be called. + * + * Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired, + * serializing operations for VM memory regions. + */ +void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot) +{ + struct kvm_memslots *slots = kvm_memslots(kvm); + struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); + phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; + phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; + + spin_lock(&kvm->mmu_lock); + apt_wp_range(kvm, start, end); + spin_unlock(&kvm->mmu_lock); + kvm_flush_remote_tlbs(kvm); +} + +void kvm_mark_migration(struct kvm *kvm, int mark) +{ + struct kvm_vcpu *vcpu; + unsigned long cpu; + + kvm_for_each_vcpu(cpu, vcpu, kvm) + vcpu->arch.migration_mark = mark; +} + +void kvm_arch_commit_memory_region(struct kvm *kvm, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + /* + * At this point memslot has been committed and there is an + * allocated dirty_bitmap[], dirty pages will be tracked while the + * memory slot is write protected. + */ + if (change == KVM_MR_FLAGS_ONLY && (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES) && + new->flags & KVM_MEM_LOG_DIRTY_PAGES)) { + kvm_mark_migration(kvm, 1); + kvm_mmu_wp_memory_region(kvm, new->id); + } + /* If dirty logging has been stopped, do nothing for now. */ + if ((change != KVM_MR_DELETE) + && (old->flags & KVM_MEM_LOG_DIRTY_PAGES) + && (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))) { + kvm_mark_migration(kvm, 0); + return; + } +} + +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) +{ + gpa_t gpa = slot->base_gfn << PAGE_SHIFT; + phys_addr_t size = slot->npages << PAGE_SHIFT; + + spin_lock(&kvm->mmu_lock); +// flush_apt_tlbs(kvm); + unmap_apt_range(kvm, gpa, size); + spin_unlock(&kvm->mmu_lock); +} + +/** + * kvm_alloc_addtional_stage_pgd - allocate level-1 table for addtional stage translation. + * @kvm: The KVM struct pointer for the VM. + * + * Allocates only the addtional stage HW PGD level table(s) (can support full + * 48-bit input addresses). Clears the allocated pages. + * + * Note we don't need locking here as this is only called when the VM is + * created, which can only be done once. + */ +int kvm_alloc_addtional_stage_pgd(struct kvm *kvm) +{ + pgd_t *pgd; + + if (kvm->arch.pgd != NULL) { + kvm_err("kvm_arch already initialized?\n"); + return -EINVAL; + } + + /* Allocate the HW PGD, making sure that each page gets its own refcount */ + pgd = alloc_pages_exact(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO); + if (!pgd) + return -ENOMEM; + + kvm->arch.pgd = pgd; + return 0; +} + +/** + * kvm_free_apt_pgd - free all apt tables + * @kvm: The KVM struct pointer for the VM. + * + * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all + * underlying level-2 and level-3 tables before freeing the actual level-1 table + * and setting the struct pointer to NULL. + */ +void kvm_free_apt_pgd(struct kvm *kvm) +{ + void *pgd = NULL; + + spin_lock(&kvm->mmu_lock); + if (kvm->arch.pgd) { + unmap_apt_range(kvm, 0, KVM_PHYS_SIZE); + pgd = READ_ONCE(kvm->arch.pgd); + kvm->arch.pgd = NULL; + } + spin_unlock(&kvm->mmu_lock); + + /* Free the HW pgd, one page at a time */ + if (pgd) + free_pages_exact(pgd, PAGE_SIZE); +} + +void kvm_arch_flush_shadow_all(struct kvm *kvm) +{ + kvm_free_apt_pgd(kvm); +} + +static void kvm_send_hwpoison_signal(unsigned long address, + struct vm_area_struct *vma) +{ + kernel_siginfo_t info; + + clear_siginfo(&info); + info.si_signo = SIGBUS; + info.si_errno = 0; + info.si_code = BUS_MCEERR_AR; + info.si_addr = (void __user *)address; + + if (is_vm_hugetlb_page(vma)) + info.si_addr_lsb = huge_page_shift(hstate_vma(vma)); + else + info.si_addr_lsb = PAGE_SHIFT; + + send_sig_info(SIGBUS, &info, current); +} + +static bool fault_supports_apt_huge_mapping(struct kvm_memory_slot *memslot, + unsigned long hva, + unsigned long map_size) +{ + gpa_t gpa_start; + hva_t uaddr_start, uaddr_end; + size_t size; + + /* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */ + if (map_size == PAGE_SIZE) + return true; + + size = memslot->npages * PAGE_SIZE; + + gpa_start = memslot->base_gfn << PAGE_SHIFT; + + uaddr_start = memslot->userspace_addr; + uaddr_end = uaddr_start + size; + + /* + * Pages belonging to memslots that don't have the same alignment + * within a PMD/PUD for userspace and IPA cannot be mapped with stage-2 + * PMD/PUD entries, because we'll end up mapping the wrong pages. + * + * Consider a layout like the following: + * + * memslot->userspace_addr: + * +-----+--------------------+--------------------+---+ + * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz| + * +-----+--------------------+--------------------+---+ + * + * memslot->base_gfn << PAGE_SHIFT: + * +---+--------------------+--------------------+-----+ + * |abc|def Stage-2 block | Stage-2 block |tvxyz| + * +---+--------------------+--------------------+-----+ + * + * If we create those stage-2 blocks, we'll end up with this incorrect + * mapping: + * d -> f + * e -> g + * f -> h + */ + if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1))) + return false; + + /* + * Next, let's make sure we're not trying to map anything not covered + * by the memslot. This means we have to prohibit block size mappings + * for the beginning and end of a non-block aligned and non-block sized + * memory slot (illustrated by the head and tail parts of the + * userspace view above containing pages 'abcde' and 'xyz', + * respectively). + * + * Note that it doesn't matter if we do the check using the + * userspace_addr or the base_gfn, as both are equally aligned (per + * the check above) and equally sized. + */ + return (hva & ~(map_size - 1)) >= uaddr_start && + (hva & ~(map_size - 1)) + map_size <= uaddr_end; +} + +/* + * apt_get_leaf_entry - walk the stage2 VM page tables and return + * true if a valid and present leaf-entry is found. A pointer to the + * leaf-entry is returned in the appropriate level variable - pudpp, + * pmdpp, ptepp. + */ +static bool apt_get_leaf_entry(struct kvm *kvm, phys_addr_t addr, + pud_t **pudpp, pmd_t **pmdpp, pte_t **ptepp) +{ + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + + *pudpp = NULL; + *pmdpp = NULL; + *ptepp = NULL; + + pudp = apt_get_pud(kvm->arch.pgd, NULL, addr); + if (!pudp || pud_none(*pudp) || !pud_present(*pudp)) + return false; + + if (pud_huge(*pudp)) { + *pudpp = pudp; + return true; + } + + pmdp = pmd_offset(pudp, addr); + if (!pmdp || pmd_none(*pmdp) || !pmd_present(*pmdp)) + return false; + + if (pmd_trans_huge(*pmdp)) { + *pmdpp = pmdp; + return true; + } + + ptep = pte_offset_kernel(pmdp, addr); + if (!ptep || pte_none(*ptep) || !pte_present(*ptep)) + return false; + + *ptepp = ptep; + return true; +} + +static bool apt_is_exec(struct kvm *kvm, phys_addr_t addr) +{ + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + bool found; + + found = apt_get_leaf_entry(kvm, addr, &pudp, &pmdp, &ptep); + if (!found) + return false; + + if (pudp) + return kvm_pud_exec(pudp); + else if (pmdp) + return kvm_pmd_exec(pmdp); + else + return kvm_pte_exec(ptep); +} + +static int apt_set_pte_fast(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, const pte_t *new_pte, + unsigned long flags) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte, old_pte; + bool logging_active = flags & KVM_APT_FLAG_LOGGING_ACTIVE; + int inv_level = ((read_csr(CSR_AS_INFO)) >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; + unsigned long inv_hpa = read_csr(CSR_AS_INFO) & AF_ENTRY_ADDR_MASK; + + VM_BUG_ON(logging_active && !cache); + + if (inv_level == 1) { + pud = (pud_t *)(inv_hpa | PAGE_OFFSET); + goto find_pud; + } else if (inv_level == 2) { + pmd = (pmd_t *)(inv_hpa | PAGE_OFFSET); + goto find_pmd; + } else if (inv_level == 3) { + pte = (pte_t *)(inv_hpa | PAGE_OFFSET); + goto find_pte; + } + + /* Create addtional page table mapping - Levels 0 and 1 */ + pud = apt_get_pud(kvm->arch.pgd, cache, addr); + if (!pud) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PUD, then continue + * on to allocate page. + */ + if (logging_active) + apt_dissolve_pud(kvm, addr, pud); + +find_pud: + if (pud_none(*pud)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pmd = mmu_memory_cache_alloc(cache); + pud_populate(NULL, pud, pmd); + get_page(virt_to_page(pud)); + } + + pmd = pmd_offset(pud, addr); + if (!pmd) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PMD, then continue on to + * allocate page. + */ + if (logging_active) + apt_dissolve_pmd(kvm, addr, pmd); + +find_pmd: + /* Create stage-2 page mappings - Level 2 */ + if (pmd_none(*pmd)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pte = mmu_memory_cache_alloc(cache); + pmd_populate_kernel(NULL, pmd, pte); + get_page(virt_to_page(pmd)); + } + + pte = pte_offset_kernel(pmd, addr); + +find_pte: + /* Create 2nd stage page table mapping - Level 3 */ + old_pte = *pte; + + /* new pte should be readonly? */ +// *new_pte = pte_wrprotect(*new_pte); + + if (pte_present(old_pte)) { + /* Skip page table update if there is no change */ + if (pte_val(old_pte) == pte_val(*new_pte)) + return 0; + + /* Do we need WRITE_ONCE(pte, 0)? */ + set_pte(pte, __pte(0)); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pte)); + } + + /* Do we need WRITE_ONCE(pte, new_pte)? */ + set_pte(pte, *new_pte); + return 0; +} + +static int apt_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, const pte_t *new_pte, + unsigned long flags) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte, old_pte; + bool logging_active = flags & KVM_APT_FLAG_LOGGING_ACTIVE; + + VM_BUG_ON(logging_active && !cache); + + /* Create addtional page table mapping - Levels 0 and 1 */ + pud = apt_get_pud(kvm->arch.pgd, cache, addr); + if (!pud) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PUD, then continue + * on to allocate page. + */ + if (logging_active) + apt_dissolve_pud(kvm, addr, pud); + + if (pud_none(*pud)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pmd = mmu_memory_cache_alloc(cache); + pud_populate(NULL, pud, pmd); + get_page(virt_to_page(pud)); + } + + pmd = pmd_offset(pud, addr); + if (!pmd) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PMD, then continue on to + * allocate page. + */ + if (logging_active) + apt_dissolve_pmd(kvm, addr, pmd); + + /* Create stage-2 page mappings - Level 2 */ + if (pmd_none(*pmd)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pte = mmu_memory_cache_alloc(cache); + pmd_populate_kernel(NULL, pmd, pte); + get_page(virt_to_page(pmd)); + } + + pte = pte_offset_kernel(pmd, addr); + + /* Create 2nd stage page table mapping - Level 3 */ + old_pte = *pte; + + /* new pte should be readonly? */ +// *new_pte = pte_wrprotect(*new_pte); + + if (pte_present(old_pte)) { + /* Skip page table update if there is no change */ + if (pte_val(old_pte) == pte_val(*new_pte)) + return 0; + + /* Do we need WRITE_ONCE(pte, 0)? */ + set_pte(pte, __pte(0)); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pte)); + } + + /* Do we need WRITE_ONCE(pte, new_pte)? */ + set_pte(pte, *new_pte); + return 0; +} + + + +static int apt_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache + *cache, phys_addr_t addr, const pmd_t *new_pmd, unsigned long sz) +{ + pmd_t *pmd, old_pmd, *ori_pmd; + int i; +retry: + pmd = apt_get_pmd(kvm, cache, addr, sz); + VM_BUG_ON(!pmd); + ori_pmd = pmd; + old_pmd = *pmd; + if (pmd_present(old_pmd)) { + /* + * If we already have PTE level mapping for this block, + * we must unmap it to avoid inconsistent TLB state and + * leaking the table page. We could end up in this situation + * if the memory slot was marked for dirty logging and was + * reverted, leaving PTE level mappings for the pages accessed + * during the period. So, unmap the PTE level mapping for this + * block and retry, as we could have released the upper level + * table in the process. + * + * Normal THP split/merge follows mmu_notifier callbacks and do + * get handled accordingly. + */ + if (!pmd_trans_huge(old_pmd)) { + unmap_apt_range(kvm, addr & PMD_MASK, PMD_SIZE); + goto retry; + } + /* + * Multiple vcpus faulting on the same PMD entry, can + * lead to them sequentially updating the PMD with the + * same value. Following the break-before-make + * (pmd_clear() followed by tlb_flush()) process can + * hinder forward progress due to refaults generated + * on missing translations. + * + * Skip updating the page table if the entry is + * unchanged. + */ + if (pmd_val(old_pmd) == pmd_val(*new_pmd)) + return 0; + + /* + * Mapping in huge pages should only happen through a + * fault. If a page is merged into a transparent huge + * page, the individual subpages of that huge page + * should be unmapped through MMU notifiers before we + * get here. + * + * Merging of CompoundPages is not supported; they + * should become splitting first, unmapped, merged, + * and mapped back in on-demand. + */ + VM_BUG_ON(pmd_pfn(old_pmd) != pmd_pfn(*new_pmd)); + + if (sz == CONT_PMD_SIZE) { + for (i = 0; i < CONT_PMDS; i++, pmd++) + pmd_clear(pmd); + } else + pmd_clear(pmd); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pmd)); + } + + /* Do we need WRITE_ONCE(pmd, new_pmd)? */ + if (sz == CONT_PMD_SIZE) { + for (i = 0; i < CONT_PMDS; i++, ori_pmd++) + set_pmd(ori_pmd, *new_pmd); + } else + set_pmd(pmd, *new_pmd); + return 0; +} + +static int apt_set_pud_huge(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, const pud_t *new_pudp) +{ + pud_t *pudp, old_pud; + +retry: + pudp = apt_get_pud(kvm->arch.pgd, cache, addr); + VM_BUG_ON(!pudp); + + old_pud = *pudp; + + /* + * A large number of vcpus faulting on the same stage 2 entry, + * can lead to a refault due to the stage2_pud_clear()/tlb_flush(). + * Skip updating the page tables if there is no change. + */ + if (pud_val(old_pud) == pud_val(*new_pudp)) + return 0; + + if (pud_present(old_pud)) { + /* + * If we already have table level mapping for this block, unmap + * the range for this block and retry. + */ + if (!pud_huge(old_pud)) { + unmap_apt_range(kvm, addr & PUD_MASK, PUD_SIZE); + goto retry; + } + +// WARN_ON_ONCE(kvm_pud_pfn(old_pud) != kvm_pud_pfn(*new_pudp)); + pud_clear(pudp); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pudp)); + } + + set_pud(pudp, *new_pudp); + return 0; +} + +static unsigned long +transparent_hugepage_adjust(struct kvm_memory_slot *memslot, + unsigned long hva, kvm_pfn_t *pfnp, + phys_addr_t *gpap) +{ + kvm_pfn_t pfn = *pfnp; + struct page *page = pfn_to_page(pfn); + + /* + * Make sure the adjustment is done only for THP pages. Also make + * sure that the HVA and IPA are sufficiently aligned and that the + * block map is contained within the memslot. + */ + if (!PageHuge(page) && PageTransCompoundMap(page) && + fault_supports_apt_huge_mapping(memslot, hva, PMD_SIZE)) { + /* + * The address we faulted on is backed by a transparent huge + * page. However, because we map the compound huge page and + * not the individual tail page, we need to transfer the + * refcount to the head page. We have to be careful that the + * THP doesn't start to split while we are adjusting the + * refcounts. + * + * We are sure this doesn't happen, because mmu_notifier_retry + * was successful and we are holding the mmu_lock, so if this + * THP is trying to split, it will be blocked in the mmu + * notifier before touching any of the pages, specifically + * before being able to call __split_huge_page_refcount(). + * + * We can therefore safely transfer the refcount from PG_tail + * to PG_head and switch the pfn from a tail page to the head + * page accordingly. + */ + *gpap &= PMD_MASK; + kvm_release_pfn_clean(pfn); + pfn &= ~(PTRS_PER_PMD - 1); + kvm_get_pfn(pfn); + *pfnp = pfn; + return PMD_SIZE; + } + + return PAGE_SIZE; +} + +static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_gpa, + struct kvm_memory_slot *memslot, unsigned long hva, + unsigned long fault_status) +{ + int ret; + bool write_fault, exec_fault, writable, force_pte = false; + unsigned long mmu_seq; + gfn_t gfn = fault_gpa >> PAGE_SHIFT; + struct kvm *kvm = vcpu->kvm; + struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; + struct vm_area_struct *vma; + kvm_pfn_t pfn; + pgprot_t mem_type = PAGE_READONLY; + bool logging_active = memslot_is_logging(memslot); + unsigned long vma_pagesize, flags = 0; + unsigned long as_info, access_type; + unsigned int vma_shift; + + as_info = read_csr(CSR_AS_INFO); + access_type = (as_info >> AF_ACCESS_TYPE_SHIFT) & AF_ACCESS_TYPE_MASK; + write_fault = kvm_is_write_fault(access_type); + exec_fault = kvm_is_exec_fault(access_type); + VM_BUG_ON(write_fault && exec_fault); + + if (fault_status == AF_STATUS_FOR) { + kvm_err("Unexpected APT read permission error\n"); + return -EFAULT; + } + + /* Let's check if we will get back a huge page backed by hugetlbfs */ + down_read(¤t->mm->mmap_lock); + vma = find_vma_intersection(current->mm, hva, hva + 1); + if (unlikely(!vma)) { + kvm_err("Failed to find VMA for hva 0x%lx\n", hva); + up_read(¤t->mm->mmap_lock); + return -EFAULT; + } + + if (is_vm_hugetlb_page(vma)) + vma_shift = huge_page_shift(hstate_vma(vma)); + else + vma_shift = PAGE_SHIFT; + + vma_pagesize = 1ULL << vma_shift; + if (logging_active || (vma->vm_flags & VM_PFNMAP) || + !fault_supports_apt_huge_mapping(memslot, hva, vma_pagesize)) { + force_pte = true; + vma_pagesize = PAGE_SIZE; + } + + if (vma_pagesize == PMD_SIZE || vma_pagesize == CONT_PMD_SIZE || vma_pagesize == PUD_SIZE) + gfn = (fault_gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; + up_read(¤t->mm->mmap_lock); + /* We need minimum second+third level pages */ + ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES, + KVM_NR_MEM_OBJS); + if (ret) + return ret; + + mmu_seq = vcpu->kvm->mmu_notifier_seq; + /* + * Ensure the read of mmu_notifier_seq happens before we call + * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk + * the page we just got a reference to gets unmapped before we have a + * chance to grab the mmu_lock, which ensure that if the page gets + * unmapped afterwards, the call to kvm_unmap_hva will take it away + * from us again properly. This smp_rmb() interacts with the smp_wmb() + * in kvm_mmu_notifier_invalidate_. + */ + smp_rmb(); + + pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable); + if (pfn == KVM_PFN_ERR_HWPOISON) { + kvm_send_hwpoison_signal(hva, vma); + return 0; + } + if (is_error_noslot_pfn(pfn)) + return -EFAULT; + + if (logging_active) { + /* + * Faults on pages in a memslot with logging enabled + * should not be mapped with huge pages (it introduces churn + * and performance degradation), so force a pte mapping. + */ + flags |= KVM_APT_FLAG_LOGGING_ACTIVE; + + /* + * Only actually map the page as writable if this was a write + * fault. + */ + if (!write_fault) + writable = false; + } + + spin_lock(&kvm->mmu_lock); + if (mmu_notifier_retry(kvm, mmu_seq)) + goto out_unlock; + + /* + * If we are not forced to use page mapping, check if we are + * backed by a THP and thus use block mapping if possible. + */ + if (vma_pagesize == PAGE_SIZE && !force_pte) { + vma_pagesize = transparent_hugepage_adjust(memslot, hva, + &pfn, &fault_gpa); + } + + if (vma_pagesize == PUD_SIZE) { + pud_t new_pud = pfn_pud(pfn, mem_type); + + new_pud = pud_mkhuge(new_pud); + + if (writable) { + new_pud = kvm_pud_mkwrite(new_pud); + kvm_set_pfn_dirty(pfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pud = kvm_pud_mkexec(new_pud); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pud = kvm_pud_mkexec(new_pud); + } + + ret = apt_set_pud_huge(kvm, memcache, fault_gpa, &new_pud); + } else if (vma_pagesize == CONT_PMD_SIZE) { + pmd_t new_pmd = pfn_pmd(pfn, mem_type); + + new_pmd = pmd_mkhuge(new_pmd); + new_pmd = pmd_mkcont(new_pmd); + + if (writable) { + new_pmd = kvm_pmd_mkwrite(new_pmd); + kvm_set_pfn_dirty(pfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pmd = kvm_pmd_mkexec(new_pmd); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pmd = kvm_pmd_mkexec(new_pmd); + } + + ret = apt_set_pmd_huge(kvm, memcache, fault_gpa, &new_pmd, vma_pagesize); + } else if (vma_pagesize == PMD_SIZE) { + pmd_t new_pmd = pfn_pmd(pfn, mem_type); + + new_pmd = pmd_mkhuge(new_pmd); + + if (writable) { + new_pmd = kvm_pmd_mkwrite(new_pmd); + kvm_set_pfn_dirty(pfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pmd = kvm_pmd_mkexec(new_pmd); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pmd = kvm_pmd_mkexec(new_pmd); + } + + ret = apt_set_pmd_huge(kvm, memcache, fault_gpa, &new_pmd, vma_pagesize); + } else { + pte_t new_pte = pfn_pte(pfn, mem_type); + + if (writable) { + new_pte = kvm_pte_mkwrite(new_pte); + kvm_set_pfn_dirty(pfn); + mark_page_dirty(kvm, gfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pte = kvm_pte_mkexec(new_pte); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pte = kvm_pte_mkexec(new_pte); + } + + ret = apt_set_pte_fast(kvm, memcache, fault_gpa, &new_pte, flags); + if (!ret) + goto out_unlock; + } + +out_unlock: + spin_unlock(&kvm->mmu_lock); + kvm_set_pfn_accessed(pfn); + kvm_release_pfn_clean(pfn); + return ret; +} + +/** + * kvm_handle_guest_abort - handles all 2nd stage aborts + * @vcpu: the VCPU pointer + * @run: the kvm_run structure + * + * Any abort that gets to the host is almost guaranteed to be caused by a + * missing second stage translation table entry, which can mean that either the + * guest simply needs more memory and we must allocate an appropriate page or it + * can mean that the guest tried to access I/O memory, which is emulated by user + * space. The distinction is based on the IPA causing the fault and whether this + * memory region has been registered as standard RAM by user space. + */ +#ifdef CONFIG_SUBARCH_C4 +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + unsigned long as_info; /* the value of CSR: AS_INFO */ + unsigned int access_type, inv_level; + unsigned int fault_status; + unsigned long fault_entry_addr; + phys_addr_t fault_gpa; + struct kvm_memory_slot *memslot; + unsigned long hva; + bool write_fault, writable; + gfn_t gfn; + + int ret, idx; + + as_info = read_csr(CSR_AS_INFO); + access_type = (as_info >> AF_ACCESS_TYPE_SHIFT) & AF_ACCESS_TYPE_MASK; + inv_level = (as_info >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; + fault_status = (as_info >> AF_FAULT_STATUS_SHIFT) & AF_FAULT_STATUS_MASK; + fault_entry_addr = (as_info & AF_ENTRY_ADDR_MASK) >> 3; + + fault_gpa = read_csr(CSR_EXC_GPA); + idx = srcu_read_lock(&vcpu->kvm->srcu); + + gfn = fault_gpa >> PAGE_SHIFT; + memslot = gfn_to_memslot(vcpu->kvm, gfn); + hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); + + write_fault = kvm_is_write_fault(access_type); + + /* The memory slot for IO doesn't register in memory region + * with kvm, if hva == KVM_HVA_ERR_BAD, the gpa used for MMIO + * needs emulation. + */ + + if (hva == KVM_HVA_ERR_BAD) { + ret = io_mem_abort(vcpu, run, NULL); + goto out_unlock; + } + /* Userspace should not be able to register out-of-bounds IPAs */ + VM_BUG_ON(fault_gpa >= KVM_PHYS_SIZE); + + ret = user_mem_abort(vcpu, fault_gpa, memslot, hva, fault_status); + if (ret == 0) + ret = 1; +out_unlock: + srcu_read_unlock(&vcpu->kvm->srcu, idx); + return ret; +} +#endif +static int handle_hva_to_gpa(struct kvm *kvm, unsigned long start, unsigned long end, + int (*handler)(struct kvm *kvm, gpa_t gpa, u64 size, void *data), + void *data) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *memslot; + int ret = 0; + + slots = kvm_memslots(kvm); + + /* we only care about the pages that the guest sees */ + kvm_for_each_memslot(memslot, slots) { + unsigned long hva_start, hva_end; + gfn_t gpa; + + hva_start = max(start, memslot->userspace_addr); + hva_end = min(end, memslot->userspace_addr + + (memslot->npages << PAGE_SHIFT)); + if (hva_start >= hva_end) + continue; + + gpa = hva_to_gfn_memslot(hva_start, memslot) << PAGE_SHIFT; + ret |= handler(kvm, gpa, (u64)(hva_end - hva_start), data); + } + + return ret; +} + +static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + unmap_apt_range(kvm, gpa, size); + return 0; +} + +int kvm_unmap_hva_range(struct kvm *kvm, + unsigned long start, unsigned long end, bool blockable) +{ + if (!kvm->arch.pgd) + return 0; + + handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL); + return 1; +} + +static int apt_ptep_test_and_clear_young(pte_t *pte) +{ + if (pte_young(*pte)) { + *pte = pte_mkold(*pte); + return 1; + } + return 0; +} + +static int apt_pmdp_test_and_clear_young(pmd_t *pmd) +{ + return apt_ptep_test_and_clear_young((pte_t *)pmd); +} + +static int apt_pudp_test_and_clear_young(pud_t *pud) +{ + return apt_ptep_test_and_clear_young((pte_t *)pud); +} + +static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); + if (!apt_get_leaf_entry(kvm, gpa, &pud, &pmd, &pte)) + return 0; + + if (pud) + return apt_pudp_test_and_clear_young(pud); + else if (pmd) + return apt_pmdp_test_and_clear_young(pmd); + else + return apt_ptep_test_and_clear_young(pte); +} + +static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); + if (!apt_get_leaf_entry(kvm, gpa, &pud, &pmd, &pte)) + return 0; + + if (pud) + return apt_pudp_test_and_clear_young(pud); + else if (pmd) + return apt_pmdp_test_and_clear_young(pmd); + else + return apt_ptep_test_and_clear_young(pte); +} + +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) +{ + if (!kvm->arch.pgd) + return 0; + + return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL); +} + +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) +{ + if (!kvm->arch.pgd) + return 0; + return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL); +} + +static int kvm_set_apte_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + pte_t *pte = (pte_t *)data; + + WARN_ON(size != PAGE_SIZE); + + apt_set_pte(kvm, NULL, gpa, pte, 0); + return 0; +} + +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) +{ + unsigned long end = hva + PAGE_SIZE; + pte_t apt_pte; + + if (!kvm->arch.pgd) + return 0; + + apt_pte = pte_wrprotect(pte); + handle_hva_to_gpa(kvm, hva, end, &kvm_set_apte_handler, &apt_pte); + return 0; +} + +/** + * kvm_mmu_write_protect_pt_masked() - write protect dirty pages + * @kvm: The KVM pointer + * @slot: The memory slot associated with mask + * @gfn_offset: The gfn offset in memory slot + * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory + * slot to be write protected + * + * Walks bits set in mask write protects the associated pte's. Caller must + * acquire kvm_mmu_lock. + */ +static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask) +{ + phys_addr_t base_gfn = slot->base_gfn + gfn_offset; + phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; + phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; + + apt_wp_range(kvm, start, end); +} + +/* + * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected + * dirty pages. + * + * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to + * enable dirty logging for them. + */ +void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask) +{ + kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); +} diff --git a/arch/sw_64/kvm/perf.c b/arch/sw_64/kvm/perf.c new file mode 100644 index 000000000000..730dd1feeccf --- /dev/null +++ b/arch/sw_64/kvm/perf.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Performance events support for KVM. + */ + +#include +#include + +#include + + +bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) +{ + return (vcpu->arch.regs.ps & 8) != 0; +} + +unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.regs.pc; +} + + + +static inline bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu) +{ + return IS_ENABLED(CONFIG_GUEST_PERF_EVENTS) && !!vcpu; +} diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c new file mode 100644 index 000000000000..f6bfb2452938 --- /dev/null +++ b/arch/sw_64/kvm/sw64.c @@ -0,0 +1,592 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include "trace.h" +#include "irq.h" + +bool set_msi_flag; + + + +static unsigned long get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) +{ + unsigned long vpn = last_vpn(cpu); + unsigned long next = vpn + 1; + + if ((vpn & VPN_MASK) >= VPN_MASK) { + kvm_flush_tlb_all(); + next = (vpn & ~VPN_MASK) + VPN_FIRST_VERSION + 1; /* bypass 0 */ + } + last_vpn(cpu) = next; + return next; +} + +int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) +{ + set_bit(number, (vcpu->arch.irqs_pending)); + kvm_vcpu_kick(vcpu); + return 0; +} + +int kvm_arch_check_processor_compat(void *opaque) +{ + return 0; +} + +int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + unsigned int vcid; + unsigned int vcpu_idx; + struct kvm_vcpu *vcpu = NULL; + int irq = e->msi.data & 0xff; + + vcid = (e->msi.address_lo & VT_MSIX_ADDR_DEST_ID_MASK) >> VT_MSIX_ADDR_DEST_ID_SHIFT; + vcpu_idx = vcid & 0x1f; + vcpu = kvm_get_vcpu(kvm, vcpu_idx); + + if (!vcpu) + return -EINVAL; + + return vcpu_interrupt_line(vcpu, irq, true); +} + +void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) +{ + unsigned long vpn; + unsigned long vpnc; + long cpu = smp_processor_id(); + + vpn = last_vpn(cpu); + vpnc = vcpu->arch.vpnc[cpu]; + + if ((vpnc ^ vpn) & ~VPN_MASK) { + /* vpnc and cpu vpn not in the same version, get new vpnc and vpn */ + vpnc = get_new_vpn_context(vcpu, cpu); + vcpu->arch.vpnc[cpu] = vpnc; + } + + vpn = vpnc & VPN_MASK; + + /* Always update vpn */ + /* Just setup vcb, hardware CSR will be changed later in HMcode */ + kvm_sw64_update_vpn(vcpu, vpn); + + /* + * If vcpu migrate to a new physical cpu, the new physical cpu may keep + * old tlb entries for this vcpu's vpn, upn in the old tlb entries and + * current vcpu's upn may not in the same version. + * For now, we don't know the vcpu's upn version and the current version. + * If we keep track of the vcpu's upn version, the TLB-flush could be less. + * To be safe and correct, flush all tlb entries of current vpn for now. + */ + + if (vcpu->arch.pcpu_id != cpu) { + tbivpn(0, 0, vpn); + vcpu->arch.pcpu_id = cpu; + vcpu->cpu = cpu; + } +} + +void check_vcpu_requests(struct kvm_vcpu *vcpu) +{ + unsigned long vpn; + long cpu = smp_processor_id(); + + if (kvm_request_pending(vcpu)) { + if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { + vpn = vcpu->arch.vpnc[cpu] & VPN_MASK; + tbivpn(0, 0, vpn); + } + } +} + + +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return ((!bitmap_empty(vcpu->arch.irqs_pending, SWVM_IRQS) || !vcpu->arch.halted) + && !vcpu->arch.power_off); +} + +int kvm_arch_hardware_enable(void) +{ + return 0; +} + +void kvm_arch_hardware_unsetup(void) +{ +} + +bool kvm_arch_has_vcpu_debugfs(void) +{ + return false; +} + +int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu) +{ + return 0; +} + +int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; +} + +int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) +{ + int r = 0; + + switch (ext) { + case KVM_CAP_IRQCHIP: + case KVM_CAP_IOEVENTFD: + case KVM_CAP_SYNC_MMU: + r = 1; + break; + case KVM_CAP_NR_VCPUS: + case KVM_CAP_MAX_VCPUS: + r = KVM_MAX_VCPUS; + break; + default: + r = 0; + } + + return r; +} + +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ +} + +int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) +{ + return test_bit(SW64_KVM_IRQ_TIMER, vcpu->arch.irqs_pending); +} + +int kvm_arch_hardware_setup(void *opaque) +{ + return 0; +} + +int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) +{ + if (type) + return -EINVAL; + + return kvm_sw64_init_vm(kvm); +} + +void kvm_arch_destroy_vm(struct kvm *kvm) +{ + return kvm_sw64_destroy_vm(kvm); +} + +long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + return -EINVAL; +} + +int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned long npages) +{ + return 0; +} + +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) +{ + kvm_mmu_free_memory_caches(vcpu); + hrtimer_cancel(&vcpu->arch.hrt); +} + +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) +{ + kvm_arch_vcpu_free(vcpu); +} + +int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) +{ + /* Set up the timer for Guest */ + pr_info("vcpu: [%d], regs addr = %#lx, vcpucb = %#lx\n", vcpu->vcpu_id, + (unsigned long)&vcpu->arch.regs, (unsigned long)&vcpu->arch.vcb); + vcpu->arch.vtimer_freq = cpuid(GET_CPU_FREQ, 0) * 1000UL * 1000UL; + hrtimer_init(&vcpu->arch.hrt, CLOCK_REALTIME, HRTIMER_MODE_ABS); + vcpu->arch.hrt.function = clockdev_fn; + vcpu->arch.tsk = current; + + vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; + vcpu->arch.vcb.vcpu_irq_disabled = 1; + vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ + + return 0; +} + +int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) +{ + return 0; +} + +int kvm_set_routing_entry(struct kvm *kvm, + struct kvm_kernel_irq_routing_entry *e, + const struct kvm_irq_routing_entry *ue) +{ + int r = -EINVAL; + + switch (ue->type) { + case KVM_IRQ_ROUTING_MSI: + e->set = kvm_set_msi; + e->msi.address_lo = ue->u.msi.address_lo; + e->msi.address_hi = ue->u.msi.address_hi; + e->msi.data = ue->u.msi.data; + e->msi.flags = ue->flags; + e->msi.devid = ue->u.msi.devid; + set_msi_flag = true; + break; + default: + goto out; + } + r = 0; +out: + return r; +} + +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr) +{ + return -EINVAL; /* not implemented yet */ +} + +int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) +{ + return 0; +} + + +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + vcpu->cpu = cpu; +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + /* + * The arch-generic KVM code expects the cpu field of a vcpu to be -1 + * if the vcpu is no longer assigned to a cpu. This is used for the + * optimized make_all_cpus_request path. + */ + vcpu->cpu = -1; +} + +int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + return -ENOIOCTLCMD; +} + +int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + return -ENOIOCTLCMD; +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + memcpy(&(vcpu->arch.regs), regs, sizeof(struct kvm_regs)); + return 0; +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + memcpy(regs, &(vcpu->arch.regs), sizeof(struct kvm_regs)); + return 0; +} + +int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, + struct kvm_guest_debug *dbg) +{ + return 0; +} + + +/* + * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on + * proper exit to userspace. + */ +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + struct vcpucb *vcb = &(vcpu->arch.vcb); + struct hcall_args hargs; + int irq, ret; + bool more; + sigset_t sigsaved; + + /* Set guest vcb */ + /* vpn will update later when vcpu is running */ + vcpu_set_numa_affinity(vcpu); +#ifdef CONFIG_PERF_EVENTS + vcpu_load(vcpu); +#endif + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); + + if (run->exit_reason == KVM_EXIT_MMIO) + kvm_handle_mmio_return(vcpu, run); + + run->exit_reason = KVM_EXIT_UNKNOWN; + ret = 1; + while (ret > 0) { + /* Check conditions before entering the guest */ + cond_resched(); + + preempt_disable(); + local_irq_disable(); + + if (signal_pending(current)) { + ret = -EINTR; + run->exit_reason = KVM_EXIT_INTR; + } + + if (ret <= 0) { + local_irq_enable(); + preempt_enable(); + continue; + } + + memset(&hargs, 0, sizeof(hargs)); + + clear_vcpu_irq(vcpu); + + if (vcpu->arch.restart == 1) { + /* handle reset vCPU */ + vcpu->arch.regs.pc = GUEST_RESET_PC; + vcpu->arch.restart = 0; + } + + irq = interrupt_pending(vcpu, &more); + if (irq < SWVM_IRQS) + try_deliver_interrupt(vcpu, irq, more); + + vcpu->arch.halted = 0; + + sw64_kvm_switch_vpn(vcpu); + check_vcpu_requests(vcpu); + guest_enter_irqoff(); + + /* update aptp before the guest runs */ + update_aptp((unsigned long)vcpu->kvm->arch.pgd); + + /* Enter the guest */ + trace_kvm_sw64_entry(vcpu->vcpu_id, vcpu->arch.regs.pc); + vcpu->mode = IN_GUEST_MODE; + + ret = __sw64_vcpu_run(__pa(vcb), &(vcpu->arch.regs), &hargs); + + /* Back from guest */ + vcpu->mode = OUTSIDE_GUEST_MODE; + + local_irq_enable(); + guest_exit_irqoff(); + + trace_kvm_sw64_exit(ret, vcpu->arch.regs.pc); + + preempt_enable(); + + /* ret = 0 indicate interrupt in guest mode, ret > 0 indicate hcall */ + ret = handle_exit(vcpu, run, ret, &hargs); + } + + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &sigsaved, NULL); + +#ifdef CONFIG_PERF_EVENTS + vcpu_put(vcpu); +#endif + + return ret; +} + +long kvm_arch_vcpu_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + int r; + + switch (ioctl) { + case KVM_SW64_VCPU_INIT: + r = kvm_sw64_vcpu_reset(vcpu); + break; + case KVM_SW64_GET_VCB: + r = kvm_sw64_get_vcb(filp, arg); + break; + case KVM_SW64_SET_VCB: + r = kvm_sw64_set_vcb(filp, arg); + break; + default: + r = -EINVAL; + } + + return r; +} + +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + struct kvm *kvm __maybe_unused = filp->private_data; + long r; + + switch (ioctl) { + case KVM_CREATE_IRQCHIP: { + struct kvm_irq_routing_entry routing; + + r = -EINVAL; + memset(&routing, 0, sizeof(routing)); + r = kvm_set_irq_routing(kvm, &routing, 0, 0); + break; + } + default: + r = -ENOIOCTLCMD; + } + return r; +} + +int kvm_arch_init(void *opaque) +{ + return 0; +} + +void kvm_arch_exit(void) +{ +} + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -ENOIOCTLCMD; +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -ENOIOCTLCMD; +} + +void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) +{ +} + +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + return -ENOIOCTLCMD; +} + +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + return -ENOIOCTLCMD; +} + +vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) +{ + return VM_FAULT_SIGBUS; +} + +void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) +{ + /* Let implementation handle TLB/GVA invalidation */ + kvm_arch_flush_shadow_memslot(kvm, memslot); +} + +int kvm_dev_ioctl_check_extension(long ext) +{ + int r; + + switch (ext) { + case KVM_CAP_IOEVENTFD: + r = 1; + break; + case KVM_CAP_NR_VCPUS: + case KVM_CAP_MAX_VCPUS: + r = KVM_MAX_VCPUS; + break; + default: + r = 0; + } + + return r; +} + +void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type) +{ + struct kvm_vcpu *target_vcpu = kvm_get_vcpu(vcpu->kvm, target_vcpuid); + + if (type == II_RESET) + target_vcpu->arch.restart = 1; + + if (target_vcpu != NULL) + vcpu_interrupt_line(target_vcpu, 1, 1); +} + +int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, + bool line_status) +{ + u32 irq = irq_level->irq; + unsigned int irq_num; + struct kvm_vcpu *vcpu = NULL; + bool level = irq_level->level; + + irq_num = irq; + /* target core for Intx is core0 */ + vcpu = kvm_get_vcpu(kvm, 0); + if (!vcpu) + return -EINVAL; + + return vcpu_interrupt_line(vcpu, irq_num, level); +} + + +const struct _kvm_stats_desc kvm_vm_stats_desc[] = { + KVM_GENERIC_VM_STATS() +}; + +const struct kvm_stats_header kvm_vm_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vm_stats_desc), +}; + +const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { + KVM_GENERIC_VCPU_STATS(), +}; + +const struct kvm_stats_header kvm_vcpu_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vcpu_stats_desc), +}; + + + +bool kvm_arch_irqchip_in_kernel(struct kvm *kvm) +{ + return irqchip_in_kernel(kvm); +} diff --git a/arch/sw_64/kvm/trace.h b/arch/sw_64/kvm/trace.h new file mode 100644 index 000000000000..2611df3d3fa5 --- /dev/null +++ b/arch/sw_64/kvm/trace.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(_SW64_KVM_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _SW64_KVM_TRACE_H + +#include + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm + +/* + * Tracepoint for guest mode entry. + */ +TRACE_EVENT(kvm_sw64_entry, + TP_PROTO(unsigned int vcpu_id, unsigned int vcpu_pc), + TP_ARGS(vcpu_id, vcpu_pc), + + TP_STRUCT__entry( + __field(unsigned int, vcpu_id) + __field(unsigned int, vcpu_pc) + ), + + TP_fast_assign( + __entry->vcpu_id = vcpu_id; + __entry->vcpu_pc = vcpu_pc; + ), + + TP_printk("VCPU %u: PC: 0x%08x", __entry->vcpu_id, __entry->vcpu_pc) +); + +/* + * Tracepoint for guest mode exit. + */ + +TRACE_EVENT(kvm_sw64_exit, + TP_PROTO(unsigned int exit_reason, unsigned long vcpu_pc), + TP_ARGS(exit_reason, vcpu_pc), + + TP_STRUCT__entry( + __field(unsigned int, exit_reason) + __field(unsigned long, vcpu_pc) + ), + + TP_fast_assign( + __entry->exit_reason = exit_reason; + __entry->vcpu_pc = vcpu_pc; + ), + + TP_printk("exit_reason: 0x%04x (%11s), PC: 0x%08lx", + __entry->exit_reason, + __print_symbolic(__entry->exit_reason, kvm_sw64_exception_type), + __entry->vcpu_pc) +); + +#endif /* _SW64_KVM_TRACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +/* This part must be outside protection */ +#include diff --git a/arch/sw_64/kvm/vmem.c b/arch/sw_64/kvm/vmem.c new file mode 100644 index 000000000000..688449b65fa5 --- /dev/null +++ b/arch/sw_64/kvm/vmem.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +static bool addr_in_pool(struct gen_pool *pool, + unsigned long start, size_t size) +{ + bool found = false; + unsigned long end = start + size - 1; + struct gen_pool_chunk *chunk; + + rcu_read_lock(); + list_for_each_entry_rcu(chunk, &(pool)->chunks, next_chunk) { + if (start >= chunk->start_addr && start <= chunk->end_addr) { + if (end <= chunk->end_addr) { + found = true; + break; + } + } + } + rcu_read_unlock(); + return found; +} + +static int vmem_vm_insert_page(struct vm_area_struct *vma) +{ + unsigned long addr, uaddr; + struct page *vmem_page; + struct vmem_info *info; + size_t size; + int ret; + + info = vma->vm_private_data; + addr = info->start; + size = info->size; + uaddr = vma->vm_start; + + vm_flags_init(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP); + vmem_page = pfn_to_page(addr >> PAGE_SHIFT); + do { + ret = vm_insert_page(vma, uaddr, vmem_page); + if (ret < 0) { + pr_info("vm_insert_page failed: %d\n", ret); + return ret; + } + vmem_page++; + uaddr += PAGE_SIZE; + size -= PAGE_SIZE; + } while (size > 0); + + return 0; +} + +static void vmem_vm_open(struct vm_area_struct *vma) +{ + struct vmem_info *info = vma->vm_private_data; + + atomic_inc(&info->refcnt); +} + +static void vmem_vm_close(struct vm_area_struct *vma) +{ + unsigned long addr; + size_t size; + struct vmem_info *info; + + info = vma->vm_private_data; + addr = info->start; + size = round_up(info->size, 8 << 20); + + if (atomic_dec_and_test(&info->refcnt)) { + if (sw64_kvm_pool && addr_in_pool(sw64_kvm_pool, addr, size)) { + pr_info("gen pool free addr: %#lx, size: %#lx\n", + addr, size); + gen_pool_free(sw64_kvm_pool, addr, size); + } + kfree(info); + } +} + +const struct vm_operations_struct vmem_vm_ops = { + .open = vmem_vm_open, + .close = vmem_vm_close, +}; +EXPORT_SYMBOL_GPL(vmem_vm_ops); + +static int vmem_open(struct inode *inode, struct file *flip) +{ + flip->private_data = NULL; + return 0; +} + +static loff_t vmem_llseek(struct file *filp, loff_t offset, int whence) +{ + loff_t newpos = 256UL << 30; + return newpos; +} + +static int vmem_release(struct inode *inode, struct file *flip) +{ + return 0; +} + +static int vmem_mmap(struct file *flip, struct vm_area_struct *vma) +{ + unsigned long addr; + static struct vmem_info *info; + size_t size = vma->vm_end - vma->vm_start; + int ret; + + if (!(vma->vm_flags & VM_SHARED)) { + pr_err("%s: mapping must be shared\n", __func__); + return -EINVAL; + } + + if (!sw64_kvm_pool) + return -ENOMEM; + + if (flip->private_data == NULL) { + addr = gen_pool_alloc(sw64_kvm_pool, round_up(size, 8 << 20)); + if (!addr) + return -ENOMEM; + + info = kzalloc(sizeof(struct vmem_info), GFP_KERNEL); + pr_info("guest phys addr=%#lx, size=%#lx\n", addr, size); + info->start = addr; + info->size = size; + flip->private_data = (void *)info; + } else { + info = flip->private_data; + addr = info->start; + } + + vma->vm_private_data = (void *)info; + vma->vm_ops = &vmem_vm_ops; + vma->vm_ops->open(vma); + + /*to do if size bigger than vm_mem_size*/ + pr_info("sw64_vmem: vm_start=%#lx, size= %#lx\n", vma->vm_start, size); + + vmem_vm_insert_page(vma); + if (ret < 0) + return ret; + + return 0; +} + +static const struct file_operations vmem_fops = { + .owner = THIS_MODULE, + .open = vmem_open, + .llseek = vmem_llseek, + .release = vmem_release, + .mmap = vmem_mmap, +}; + +static struct miscdevice vmem_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "sw64_vmem", + .fops = &vmem_fops, +}; + +int __init vmem_init(void) +{ + int err; + + err = misc_register(&vmem_dev); + if (err != 0) { + pr_err("Could not register sw64_vmem device\n"); + return err; + } + return 0; +} + +void vmem_exit(void) +{ + misc_deregister(&vmem_dev); +} -- Gitee From faea08bceb0815b4713b1774550f3f676a64bfd5 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:35 +0800 Subject: [PATCH 035/524] sw64: add stacktrace support Add stacktrace support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/stacktrace.h | 72 ++++++++ arch/sw_64/kernel/stacktrace.c | 247 ++++++++++++++++++++++++++++ 2 files changed, 319 insertions(+) create mode 100644 arch/sw_64/include/asm/stacktrace.h create mode 100644 arch/sw_64/kernel/stacktrace.c diff --git a/arch/sw_64/include/asm/stacktrace.h b/arch/sw_64/include/asm/stacktrace.h new file mode 100644 index 000000000000..958c9892fd6d --- /dev/null +++ b/arch/sw_64/include/asm/stacktrace.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SW64_STACKTRACE_H +#define _ASM_SW64_STACKTRACE_H + +#include +#include +#include +#include +#include + +struct stackframe { + unsigned long pc; + unsigned long fp; +}; + +enum stack_type { + STACK_TYPE_UNKNOWN, + STACK_TYPE_TASK, +}; + +struct stack_info { + unsigned long low; + unsigned long high; + enum stack_type type; +}; + +/* The form of the top of the frame on the stack */ +struct stack_frame { + unsigned long return_address; + struct stack_frame *next_frame; +}; + +extern int unwind_frame(struct task_struct *tsk, struct stackframe *frame); +extern void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, + int (*fn)(unsigned long, void *), void *data); + +static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp, + struct stack_info *info) +{ + unsigned long low = (unsigned long)task_stack_page(tsk); + unsigned long high = low + THREAD_SIZE; + + if (sp < low || sp >= high) + return false; + + if (info) { + info->low = low; + info->high = high; + info->type = STACK_TYPE_TASK; + } + + return true; +} + +/* + * We can only safely access per-cpu stacks from current in a non-preemptible + * context. + */ +static inline bool on_accessible_stack(struct task_struct *tsk, + unsigned long sp, + struct stack_info *info) +{ + if (on_task_stack(tsk, sp, info)) + return true; + if (tsk != current || preemptible()) + return false; + + return false; +} + +#endif /* _ASM_SW64_STACKTRACE_H */ diff --git a/arch/sw_64/kernel/stacktrace.c b/arch/sw_64/kernel/stacktrace.c new file mode 100644 index 000000000000..ff00506d5b82 --- /dev/null +++ b/arch/sw_64/kernel/stacktrace.c @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Stack trace management functions + * + * Copyright (C) 2018 snyh + */ +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * sw_64 PCS assigns the frame pointer to r15. + * + * A simple function prologue looks like this: + * ldi sp,-xx(sp) + * stl ra,0(sp) + * stl fp,8(sp) + * mov sp,fp + * + * A simple function epilogue looks like this: + * mov fp,sp + * ldl ra,0(sp) + * ldl fp,8(sp) + * ldi sp,+xx(sp) + */ + +#ifdef CONFIG_FRAME_POINTER + +int unwind_frame(struct task_struct *tsk, struct stackframe *frame) +{ + unsigned long fp = frame->fp; + + if (fp & 0x7) + return -EINVAL; + + if (!tsk) + tsk = current; + + if (!on_accessible_stack(tsk, fp, NULL)) + return -EINVAL; + + frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp)); + frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 8)); + + /* + * Frames created upon entry from user have NULL FP and PC values, so + * don't bother reporting these. Frames created by __noreturn functions + * might have a valid FP even if PC is bogus, so only terminate where + * both are NULL. + */ + if (!frame->fp && !frame->pc) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL_GPL(unwind_frame); + +void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, + int (*fn)(unsigned long, void *), void *data) +{ + unsigned long pc, fp; + + struct stackframe frame; + + if (regs) { + unsigned long offset; + + pc = regs->pc; + fp = regs->regs[15]; + if (kallsyms_lookup_size_offset(pc, NULL, &offset) + && offset < 16) { + /* call stack has not been setup + * store pc first then loop from ra + */ + if (fn(pc, data)) + return; + pc = regs->regs[26]; + } + } else if (tsk == current || tsk == NULL) { + fp = (unsigned long)__builtin_frame_address(0); + pc = (unsigned long)walk_stackframe; + } else { + fp = tsk->thread.s[6]; + pc = tsk->thread.ra; + } + + if (!__kernel_text_address(pc) || fn(pc, data)) + return; + + frame.pc = pc; + frame.fp = fp; + while (1) { + int ret; + + ret = unwind_frame(tsk, &frame); + if (ret < 0) + break; + + if (fn(frame.pc, data)) + break; + } +} +EXPORT_SYMBOL_GPL(walk_stackframe); + +#else /* !CONFIG_FRAME_POINTER */ +void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, + int (*fn)(unsigned long, void *), void *data) +{ + unsigned long *ksp; + unsigned long sp, pc; + + if (regs) { + sp = (unsigned long)(regs+1); + pc = regs->pc; + } else if (tsk == current || tsk == NULL) { + register unsigned long current_sp __asm__ ("$30"); + sp = current_sp; + pc = (unsigned long)walk_stackframe; + } else { + sp = tsk->thread.sp; + pc = tsk->thread.ra; + } + + ksp = (unsigned long *)sp; + + while (!kstack_end(ksp)) { + if (__kernel_text_address(pc) && fn(pc, data)) + break; + pc = *ksp++; + } +} +EXPORT_SYMBOL_GPL(walk_stackframe); + +#endif/* CONFIG_FRAME_POINTER */ + +static int print_address_trace(unsigned long pc, void *data) +{ + print_ip_sym((const char *)data, pc); + return 0; +} + +void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) +{ + pr_info("Trace:\n"); + walk_stackframe(task, NULL, print_address_trace, (void *)loglvl); +} + +#ifdef CONFIG_STACKTRACE +/* + * Save stack-backtrace addresses into a stack_trace buffer. + */ +struct stack_trace_data { + struct stack_trace *trace; + unsigned int nosched; +}; + +int save_trace(unsigned long pc, void *d) +{ + struct stack_trace_data *data = d; + struct stack_trace *trace = data->trace; + + if (data->nosched && in_sched_functions(pc)) + return 0; + if (trace->skip > 0) { + trace->skip--; + return 0; + } + + trace->entries[trace->nr_entries++] = pc; + return (trace->nr_entries >= trace->max_entries); +} + +void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) +{ + struct stack_trace_data data; + + data.trace = trace; + data.nosched = 0; + + walk_stackframe(current, regs, save_trace, &data); + + if (trace->nr_entries < trace->max_entries) + trace->entries[trace->nr_entries++] = ULONG_MAX; +} + +static void __save_stack_trace(struct task_struct *tsk, + struct stack_trace *trace, unsigned int nosched) +{ + struct stack_trace_data data; + + data.trace = trace; + data.nosched = nosched; + + walk_stackframe(tsk, NULL, save_trace, &data); + + if (trace->nr_entries < trace->max_entries) + trace->entries[trace->nr_entries++] = ULONG_MAX; +} + +void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) +{ + __save_stack_trace(tsk, trace, 1); +} +EXPORT_SYMBOL_GPL(save_stack_trace_tsk); + +void save_stack_trace(struct stack_trace *trace) +{ + __save_stack_trace(current, trace, 0); +} +EXPORT_SYMBOL_GPL(save_stack_trace); +#endif + +static int save_pc(unsigned long pc, void *data) +{ + unsigned long *p = data; + *p = 0; + + if (!in_sched_functions(pc)) + *p = pc; + + return *p; +} + +unsigned long __get_wchan(struct task_struct *tsk) +{ + unsigned long pc; + + if (!tsk || tsk == current || task_is_running(tsk)) + return 0; + walk_stackframe(tsk, NULL, save_pc, &pc); + + return pc; +} + +#ifdef CONFIG_HAVE_RELIABLE_STACKTRACE +int save_stack_trace_tsk_reliable(struct task_struct *tsk, + struct stack_trace *trace) +{ + return 0; +} +#endif -- Gitee From 9c97173688353c941726dda0c1b77ac6c79fcc5f Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:31 +0800 Subject: [PATCH 036/524] sw64: add qspinlock support Add qspinlock support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/spinlock.h | 24 ++++++++++++++++++++++++ arch/sw_64/include/asm/spinlock_types.h | 8 ++++++++ 2 files changed, 32 insertions(+) create mode 100644 arch/sw_64/include/asm/spinlock.h create mode 100644 arch/sw_64/include/asm/spinlock_types.h diff --git a/arch/sw_64/include/asm/spinlock.h b/arch/sw_64/include/asm/spinlock.h new file mode 100644 index 000000000000..64358f32cd9a --- /dev/null +++ b/arch/sw_64/include/asm/spinlock.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef _ASM_SW64_SPINLOCK_H +#define _ASM_SW64_SPINLOCK_H + +#include +#include + +/* See include/linux/spinlock.h */ +#define smp_mb__after_spinlock() smp_mb() + +#endif /* _ASM_SW64_SPINLOCK_H */ diff --git a/arch/sw_64/include/asm/spinlock_types.h b/arch/sw_64/include/asm/spinlock_types.h new file mode 100644 index 000000000000..62e554e4f48c --- /dev/null +++ b/arch/sw_64/include/asm/spinlock_types.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SPINLOCK_TYPES_H +#define _ASM_SW64_SPINLOCK_TYPES_H + +#include +#include + +#endif /* _ASM_SW64_SPINLOCK_TYPES_H */ -- Gitee From 2a4872de61b784762cd5e08ebf5ec92b11809552 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:30 +0800 Subject: [PATCH 037/524] sw64: add perf events support Add basic perf events support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/perf_event.h | 16 + arch/sw_64/include/asm/pmc.h | 55 ++ arch/sw_64/include/uapi/asm/perf_regs.h | 41 ++ arch/sw_64/kernel/perf_event.c | 787 ++++++++++++++++++++++++ arch/sw_64/kernel/perf_regs.c | 33 + 5 files changed, 932 insertions(+) create mode 100644 arch/sw_64/include/asm/perf_event.h create mode 100644 arch/sw_64/include/asm/pmc.h create mode 100644 arch/sw_64/include/uapi/asm/perf_regs.h create mode 100644 arch/sw_64/kernel/perf_event.c create mode 100644 arch/sw_64/kernel/perf_regs.c diff --git a/arch/sw_64/include/asm/perf_event.h b/arch/sw_64/include/asm/perf_event.h new file mode 100644 index 000000000000..dc55a361babd --- /dev/null +++ b/arch/sw_64/include/asm/perf_event.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_PERF_EVENT_H +#define _ASM_SW64_PERF_EVENT_H + +#include +#include + +#ifdef CONFIG_PERF_EVENTS +struct pt_regs; +extern unsigned long perf_instruction_pointer(struct pt_regs *regs); +extern unsigned long perf_misc_flags(struct pt_regs *regs); +#define perf_misc_flags(regs) perf_misc_flags(regs) +#define perf_arch_bpf_user_pt_regs(regs) ®s->user_regs +#endif + +#endif /* _ASM_SW64_PERF_EVENT_H */ diff --git a/arch/sw_64/include/asm/pmc.h b/arch/sw_64/include/asm/pmc.h new file mode 100644 index 000000000000..d5672dd940a7 --- /dev/null +++ b/arch/sw_64/include/asm/pmc.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Definitions for use with the sw64 PMC interface. + */ + +#ifndef _ASM_SW64_PMC_H +#define _ASM_SW64_PMC_H + +#define PMC_PC0 0 +#define PMC_PC1 1 + +/* Following commands are implemented on all CPUs */ +#define PMC_CMD_DISABLE 0 +#define PMC_CMD_ENABLE 1 +#define PMC_CMD_EVENT_BASE 2 +#define PMC_CMD_PM 4 +#define PMC_CMD_READ 5 +#define PMC_CMD_READ_CLEAR 6 +#define PMC_CMD_WRITE_BASE 7 + +#define PMC_DISABLE_BASE 1 + +#define PMC_ENABLE_BASE 1 + +#define PC0_RAW_BASE 0x0 +#define PC1_RAW_BASE 0x100 +#define PC0_MAX 0xF +#define PC1_MAX 0x3D + +#define SW64_PERFCTRL_KM 2 +#define SW64_PERFCTRL_UM 3 +#define SW64_PERFCTRL_AM 4 + +/* pc0 events */ +#define PC0_INSTRUCTIONS 0x0 +#define PC0_BRANCH_INSTRUCTIONS 0x3 +#define PC0_CPU_CYCLES 0x8 +#define PC0_ITB_READ 0x9 +#define PC0_DTB_READ 0xA +#define PC0_ICACHE_READ 0xB +#define PC0_DCACHE_READ 0xC +#define PC0_SCACHE_REFERENCES 0xD + +/* pc1 events */ +#define PC1_BRANCH_MISSES 0xB +#define PC1_SCACHE_MISSES 0x10 +#define PC1_ICACHE_READ_MISSES 0x16 +#define PC1_ITB_MISSES 0x17 +#define PC1_DTB_SINGLE_MISSES 0x30 +#define PC1_DCACHE_MISSES 0x32 + +#define MAX_HWEVENTS 2 +#define PMC_COUNT_MASK ((1UL << 58) - 1) + +#endif /* _ASM_SW64_PMC_H */ diff --git a/arch/sw_64/include/uapi/asm/perf_regs.h b/arch/sw_64/include/uapi/asm/perf_regs.h new file mode 100644 index 000000000000..871ad4663d1d --- /dev/null +++ b/arch/sw_64/include/uapi/asm/perf_regs.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ + +#ifndef _UAPI_ASM_SW64_PERF_REGS_H +#define _UAPI_ASM_SW64_PERF_REGS_H + +enum perf_event_sw64_regs { + PERF_REG_SW64_R0, + PERF_REG_SW64_R1, + PERF_REG_SW64_R2, + PERF_REG_SW64_R3, + PERF_REG_SW64_R4, + PERF_REG_SW64_R5, + PERF_REG_SW64_R6, + PERF_REG_SW64_R7, + PERF_REG_SW64_R8, + PERF_REG_SW64_R9, + PERF_REG_SW64_R10, + PERF_REG_SW64_R11, + PERF_REG_SW64_R12, + PERF_REG_SW64_R13, + PERF_REG_SW64_R14, + PERF_REG_SW64_R15, + PERF_REG_SW64_R16, + PERF_REG_SW64_R17, + PERF_REG_SW64_R18, + PERF_REG_SW64_R19, + PERF_REG_SW64_R20, + PERF_REG_SW64_R21, + PERF_REG_SW64_R22, + PERF_REG_SW64_R23, + PERF_REG_SW64_R24, + PERF_REG_SW64_R25, + PERF_REG_SW64_R26, + PERF_REG_SW64_R27, + PERF_REG_SW64_R28, + PERF_REG_SW64_GP, + PERF_REG_SW64_SP, + PERF_REG_SW64_PC, + PERF_REG_SW64_MAX, +}; +#endif /* _UAPI_ASM_SW64_PERF_REGS_H */ diff --git a/arch/sw_64/kernel/perf_event.c b/arch/sw_64/kernel/perf_event.c new file mode 100644 index 000000000000..83bb051be9de --- /dev/null +++ b/arch/sw_64/kernel/perf_event.c @@ -0,0 +1,787 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Performance events support for SW64 platforms. + * + * This code is based upon riscv and sparc perf event code. + */ + +#include +#include + +/* For tracking PMCs and the hw events they monitor on each CPU. */ +struct cpu_hw_events { + /* + * Set the bit (indexed by the counter number) when the counter + * is used for an event. + */ + unsigned long used_mask[BITS_TO_LONGS(MAX_HWEVENTS)]; + /* Array of events current scheduled on this cpu. */ + struct perf_event *event[MAX_HWEVENTS]; +}; + +DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); + +struct sw64_perf_event { + /* pmu index */ + int counter; + /* events selector */ + int event; +}; + +/* + * A structure to hold the description of the PMCs available on a particular + * type of SW64 CPU. + */ +struct sw64_pmu_t { + /* generic hw/cache events table */ + const struct sw64_perf_event *hw_events; + const struct sw64_perf_event (*cache_events)[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX]; + + /* method used to map hw/cache events */ + const struct sw64_perf_event *(*map_hw_event)(u64 config); + const struct sw64_perf_event *(*map_cache_event)(u64 config); + + /* The number of entries in the hw_event_map */ + int max_events; + + /* The number of counters on this pmu */ + int num_pmcs; + + /* + * All PMC counters reside in the IBOX register PCTR. This is the + * LSB of the counter. + */ + int pmc_count_shift[MAX_HWEVENTS]; + + /* + * The mask that isolates the PMC bits when the LSB of the counter + * is shifted to bit 0. + */ + unsigned long pmc_count_mask; + + /* The maximum period the PMC can count. */ + unsigned long pmc_max_period; + + /* + * The maximum value that may be written to the counter due to + * hardware restrictions is pmc_max_period - pmc_left. + */ + long pmc_left; + + /* Subroutine for checking validity of a raw event for this PMU. */ + bool (*raw_event_valid)(u64 config); +}; + +/* + * The SW64 PMU description currently in operation. This is set during + * the boot process to the specific CPU of the machine. + */ +static const struct sw64_pmu_t *sw64_pmu; + +/* + * SW64 PMC event types + * + * There is no one-to-one mapping of the possible hw event types to the + * actual codes that are used to program the PMCs hence we introduce our + * own hw event type identifiers. + */ +#define SW64_OP_UNSUP {-1, -1} + +/* Mapping of the hw event types to the perf tool interface */ +static const struct sw64_perf_event core3_hw_event_map[] = { + [PERF_COUNT_HW_CPU_CYCLES] = {PMC_PC0, PC0_CPU_CYCLES}, + [PERF_COUNT_HW_INSTRUCTIONS] = {PMC_PC0, PC0_INSTRUCTIONS}, + [PERF_COUNT_HW_CACHE_REFERENCES] = {PMC_PC0, PC0_SCACHE_REFERENCES}, + [PERF_COUNT_HW_CACHE_MISSES] = {PMC_PC1, PC1_SCACHE_MISSES}, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = {PMC_PC0, PC0_BRANCH_INSTRUCTIONS}, + [PERF_COUNT_HW_BRANCH_MISSES] = {PMC_PC1, PC1_BRANCH_MISSES}, +}; + +/* Mapping of the hw cache event types to the perf tool interface */ +#define C(x) PERF_COUNT_HW_CACHE_##x +static const struct sw64_perf_event core3_cache_event_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_DCACHE_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_DCACHE_MISSES} + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_ICACHE_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_ICACHE_READ_MISSES}, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + }, + [C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_DTB_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_DTB_SINGLE_MISSES}, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_ITB_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_ITB_MISSES}, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUP, + [C(RESULT_MISS)] = SW64_OP_UNSUP, + }, + }, + +}; + +static const struct sw64_perf_event *core3_map_hw_event(u64 config) +{ + return &sw64_pmu->hw_events[config]; +} + +static const struct sw64_perf_event *core3_map_cache_event(u64 config) +{ + unsigned int cache_type, cache_op, cache_result; + const struct sw64_perf_event *perf_event; + + cache_type = (config >> 0) & 0xff; + if (cache_type >= PERF_COUNT_HW_CACHE_MAX) + return ERR_PTR(-EINVAL); + + cache_op = (config >> 8) & 0xff; + if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) + return ERR_PTR(-EINVAL); + + cache_result = (config >> 16) & 0xff; + if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + return ERR_PTR(-EINVAL); + + perf_event = &((*sw64_pmu->cache_events)[cache_type][cache_op][cache_result]); + if (perf_event->counter == -1) /* SW64_OP_UNSUP */ + return ERR_PTR(-ENOENT); + + return perf_event; +} + +/* + * r0xx for counter0, r1yy for counter1. + * According to the datasheet, 00 <= xx <= 0F, 00 <= yy <= 3D + */ +static bool core3_raw_event_valid(u64 config) +{ + if ((config >= PC0_RAW_BASE && config <= (PC0_RAW_BASE + PC0_MAX)) || + (config >= PC1_RAW_BASE && config <= (PC1_RAW_BASE + PC1_MAX))) + return true; + + pr_info("sw64 pmu: invalid raw event config %#llx\n", config); + return false; +} + +static const struct sw64_pmu_t core3_pmu = { + .max_events = ARRAY_SIZE(core3_hw_event_map), + .hw_events = core3_hw_event_map, + .map_hw_event = core3_map_hw_event, + .cache_events = &core3_cache_event_map, + .map_cache_event = core3_map_cache_event, + .num_pmcs = MAX_HWEVENTS, + .pmc_count_mask = PMC_COUNT_MASK, + .pmc_max_period = PMC_COUNT_MASK, + .pmc_left = 4, + .raw_event_valid = core3_raw_event_valid, +}; + +/* + * Low-level functions: reading/writing counters + */ +static void sw64_write_pmc(int idx, unsigned long val) +{ + wrperfmon(PMC_CMD_WRITE_BASE + idx, val); +} + +static unsigned long sw64_read_pmc(int idx) +{ + return wrperfmon(PMC_CMD_READ, idx); +} + +/* Set a new period to sample over */ +static int sw64_perf_event_set_period(struct perf_event *event, + struct hw_perf_event *hwc, int idx) +{ + long left = local64_read(&hwc->period_left); + long period = hwc->sample_period; + int overflow = 0; + unsigned long value; + + if (unlikely(left <= -period)) { + left = period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + overflow = 1; + } + + if (unlikely(left <= 0)) { + left += period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + overflow = 1; + } + + if (left > (long)sw64_pmu->pmc_max_period) + left = sw64_pmu->pmc_max_period; + + value = sw64_pmu->pmc_max_period - left; + local64_set(&hwc->prev_count, value); + sw64_write_pmc(idx, value); + + perf_event_update_userpage(event); + + return overflow; +} + +/* + * Calculates the count (the 'delta') since the last time the PMC was read. + * + * As the PMCs' full period can easily be exceeded within the perf system + * sampling period we cannot use any high order bits as a guard bit in the + * PMCs to detect overflow as is done by other architectures. The code here + * calculates the delta on the basis that there is no overflow when ovf is + * zero. The value passed via ovf by the interrupt handler corrects for + * overflow. + * + * This can be racey on rare occasions -- a call to this routine can occur + * with an overflowed counter just before the PMI service routine is called. + * The check for delta negative hopefully always rectifies this situation. + */ +static unsigned long sw64_perf_event_update(struct perf_event *event, + struct hw_perf_event *hwc, int idx, long ovf) +{ + long prev_raw_count, new_raw_count; + long delta; + +again: + prev_raw_count = local64_read(&hwc->prev_count); + new_raw_count = sw64_read_pmc(idx); + + if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count) != prev_raw_count) + goto again; + + delta = (new_raw_count - (prev_raw_count & sw64_pmu->pmc_count_mask)) + ovf; + + /* It is possible on very rare occasions that the PMC has overflowed + * but the interrupt is yet to come. Detect and fix this situation. + */ + if (unlikely(delta < 0)) + delta += sw64_pmu->pmc_max_period + 1; + + local64_add(delta, &event->count); + local64_sub(delta, &hwc->period_left); + + return new_raw_count; +} + +/* + * State transition functions: + * + * add()/del() & start()/stop() + * + */ + +/* + * pmu->start: start the event. + */ +static void sw64_pmu_start(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) + return; + + if (flags & PERF_EF_RELOAD) { + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); + sw64_perf_event_set_period(event, hwc, hwc->idx); + } + + hwc->state = 0; + + /* counting in selected modes, for both counters */ + wrperfmon(PMC_CMD_PM, hwc->config_base); + wrperfmon(PMC_CMD_EVENT_BASE + hwc->idx, hwc->event_base); + wrperfmon(PMC_CMD_ENABLE, PMC_ENABLE_BASE + hwc->idx); +} + +/* + * pmu->stop: stop the counter + */ +static void sw64_pmu_stop(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!(hwc->state & PERF_HES_STOPPED)) { + wrperfmon(PMC_CMD_DISABLE, PMC_DISABLE_BASE + hwc->idx); + hwc->state |= PERF_HES_STOPPED; + barrier(); + } + + if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { + sw64_perf_event_update(event, hwc, hwc->idx, 0); + hwc->state |= PERF_HES_UPTODATE; + } +} + +/* + * pmu->add: add the event to PMU. + */ +static int sw64_pmu_add(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int err = 0; + unsigned long irq_flags; + + local_irq_save(irq_flags); + + if (__test_and_set_bit(hwc->idx, cpuc->used_mask)) { + err = -ENOSPC; + goto out; + } + + cpuc->event[hwc->idx] = event; + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + if (flags & PERF_EF_START) + sw64_pmu_start(event, PERF_EF_RELOAD); + + /* Propagate our changes to the userspace mapping. */ + perf_event_update_userpage(event); + +out: + local_irq_restore(irq_flags); + + return err; +} + +/* + * pmu->del: delete the event from PMU. + */ +static void sw64_pmu_del(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + unsigned long irq_flags; + + local_irq_save(irq_flags); + + sw64_pmu_stop(event, PERF_EF_UPDATE); + cpuc->event[hwc->idx] = NULL; + __clear_bit(event->hw.idx, cpuc->used_mask); + + /* Absorb the final count and turn off the event. */ + perf_event_update_userpage(event); + + local_irq_restore(irq_flags); +} + +/* + * pmu->read: read and update the counter + */ +static void sw64_pmu_read(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + sw64_perf_event_update(event, hwc, hwc->idx, 0); +} + +static bool supported_cpu(void) +{ + return true; +} + +static void hw_perf_event_destroy(struct perf_event *event) +{ + /* Nothing to be done! */ +} + +static int __hw_perf_event_init(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + struct hw_perf_event *hwc = &event->hw; + const struct sw64_perf_event *event_type; + + + /* + * SW64 does not have per-counter usr/os/guest/host bits, + * we can distinguish exclude_user and exclude_kernel by + * sample mode. + */ + if (event->attr.exclude_hv || event->attr.exclude_idle || + event->attr.exclude_host || event->attr.exclude_guest) + return -EINVAL; + + /* + * SW64 does not support precise ip feature, and system hang when + * detecting precise_ip by perf_event_attr__set_max_precise_ip + * in userspace + */ + if (attr->precise_ip != 0) + return -EOPNOTSUPP; + + /* SW64 has fixed counter for given event type */ + if (attr->type == PERF_TYPE_HARDWARE) { + if (attr->config >= sw64_pmu->max_events) + return -EINVAL; + event_type = sw64_pmu->map_hw_event(attr->config); + hwc->idx = event_type->counter; + hwc->event_base = event_type->event; + } else if (attr->type == PERF_TYPE_HW_CACHE) { + event_type = sw64_pmu->map_cache_event(attr->config); + if (IS_ERR(event_type)) /* */ + return PTR_ERR(event_type); + hwc->idx = event_type->counter; + hwc->event_base = event_type->event; + } else { /* PERF_TYPE_RAW */ + if (!sw64_pmu->raw_event_valid(attr->config)) + return -EINVAL; + hwc->idx = attr->config >> 8; /* counter selector */ + hwc->event_base = attr->config & 0xff; /* event selector */ + } + + hwc->config_base = SW64_PERFCTRL_AM; + + if (attr->exclude_user) + hwc->config_base = SW64_PERFCTRL_KM; + if (attr->exclude_kernel) + hwc->config_base = SW64_PERFCTRL_UM; + + hwc->config = attr->config; + + if (!is_sampling_event(event)) + pr_debug("not sampling event\n"); + + event->destroy = hw_perf_event_destroy; + + if (!hwc->sample_period) { + hwc->sample_period = sw64_pmu->pmc_max_period; + hwc->last_period = hwc->sample_period; + local64_set(&hwc->period_left, hwc->sample_period); + } + + return 0; +} + +/* + * Main entry point to initialise a HW performance event. + */ +static int sw64_pmu_event_init(struct perf_event *event) +{ + int err; + + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + + switch (event->attr.type) { + case PERF_TYPE_RAW: + case PERF_TYPE_HARDWARE: + case PERF_TYPE_HW_CACHE: + break; + default: + return -ENOENT; + } + + if (!sw64_pmu) + return -ENODEV; + + /* Do the real initialisation work. */ + err = __hw_perf_event_init(event); + + return err; +} + +static struct pmu pmu = { + .name = "core3-base", + .capabilities = PERF_PMU_CAP_NO_NMI, + .event_init = sw64_pmu_event_init, + .add = sw64_pmu_add, + .del = sw64_pmu_del, + .start = sw64_pmu_start, + .stop = sw64_pmu_stop, + .read = sw64_pmu_read, +}; + +void perf_event_print_debug(void) +{ + unsigned long flags; + unsigned long pcr0, pcr1; + int cpu; + + if (!supported_cpu()) + return; + + local_irq_save(flags); + + cpu = smp_processor_id(); + + pcr0 = wrperfmon(PMC_CMD_READ, PMC_PC0); + pcr1 = wrperfmon(PMC_CMD_READ, PMC_PC1); + + pr_info("CPU#%d: PCTR0[%lx] PCTR1[%lx]\n", cpu, pcr0, pcr1); + + local_irq_restore(flags); +} + +static void sw64_perf_event_irq_handler(unsigned long idx, + struct pt_regs *regs) +{ + struct cpu_hw_events *cpuc; + struct perf_sample_data data; + struct perf_event *event; + struct hw_perf_event *hwc; + + __this_cpu_inc(irq_pmi_count); + cpuc = this_cpu_ptr(&cpu_hw_events); + + event = cpuc->event[idx]; + + if (unlikely(!event)) { + irq_err_count++; + return; + } + + hwc = &event->hw; + sw64_perf_event_update(event, hwc, idx, sw64_pmu->pmc_max_period + 1); + perf_sample_data_init(&data, 0, hwc->last_period); + + if (sw64_perf_event_set_period(event, hwc, idx)) { + if (perf_event_overflow(event, &data, regs)) { + /* Interrupts coming too quickly; "throttle" the + * counter, i.e., disable it for a little while. + */ + sw64_pmu_stop(event, 0); + } + } +} + +bool valid_utext_addr(unsigned long addr) +{ + return addr >= current->mm->start_code && addr <= current->mm->end_code; +} + +bool valid_dy_addr(unsigned long addr) +{ + bool ret = false; + struct vm_area_struct *vma; + struct mm_struct *mm = current->mm; + + if (addr > TASK_SIZE || addr < TASK_UNMAPPED_BASE) + return ret; + vma = find_vma(mm, addr); + if (vma && vma->vm_start <= addr && (vma->vm_flags & VM_EXEC)) + ret = true; + return ret; +} + +#ifdef CONFIG_FRAME_POINTER +void perf_callchain_user(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + + struct stack_frame frame; + unsigned long __user *fp; + int err; + + perf_callchain_store(entry, regs->pc); + + fp = (unsigned long __user *)regs->regs[15]; + + while (entry->nr < entry->max_stack && (unsigned long)fp < current->mm->start_stack) { + if (!access_ok(fp, sizeof(frame))) + break; + + pagefault_disable(); + err = __copy_from_user_inatomic(&frame, fp, sizeof(frame)); + pagefault_enable(); + + if (err) + break; + + if (valid_utext_addr(frame.return_address) || valid_dy_addr(frame.return_address)) + perf_callchain_store(entry, frame.return_address); + fp = (void __user *)frame.next_frame; + } +} +#else /* !CONFIG_FRAME_POINTER */ +void perf_callchain_user(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + unsigned long usp = rdusp(); + unsigned long user_addr; + int err; + + perf_callchain_store(entry, regs->pc); + + while (entry->nr < entry->max_stack && usp < current->mm->start_stack) { + if (!access_ok((const void __user *)usp, 8)) + break; + + pagefault_disable(); + err = __get_user(user_addr, (unsigned long *)usp); + pagefault_enable(); + + if (err) + break; + + if (valid_utext_addr(user_addr) || valid_dy_addr(user_addr)) + perf_callchain_store(entry, user_addr); + usp = usp + 8; + } +} +#endif/* CONFIG_FRAME_POINTER */ + +/* + * Gets called by walk_stackframe() for every stackframe. This will be called + * whist unwinding the stackframe and is like a subroutine return so we use + * the PC. + */ +static int callchain_trace(unsigned long pc, void *data) +{ + struct perf_callchain_entry_ctx *entry = data; + + perf_callchain_store(entry, pc); + return 0; +} + +void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + walk_stackframe(NULL, regs, callchain_trace, entry); +} + +/* + * Gets the perf_instruction_pointer and perf_misc_flags for guest os. + */ + +unsigned long perf_instruction_pointer(struct pt_regs *regs) +{ + if (perf_guest_state()) + return perf_guest_get_ip(); + + return instruction_pointer(regs); +} + +unsigned long perf_misc_flags(struct pt_regs *regs) +{ + unsigned int guest_state = perf_guest_state(); + int misc = 0; + + if (guest_state) { + if (guest_state & PERF_GUEST_USER) + misc |= PERF_RECORD_MISC_GUEST_USER; + else + misc |= PERF_RECORD_MISC_GUEST_KERNEL; + } else { + if (user_mode(regs)) + misc |= PERF_RECORD_MISC_USER; + else + misc |= PERF_RECORD_MISC_KERNEL; + } + + return misc; +} + +/* + * Init call to initialise performance events at kernel startup. + */ +int __init init_hw_perf_events(void) +{ + if (!supported_cpu()) { + pr_info("Performance events: Unsupported CPU type!\n"); + return 0; + } + + pr_info("Performance events: Supported CPU type!\n"); + + /* Override performance counter IRQ vector */ + + perf_irq = sw64_perf_event_irq_handler; + + /* And set up PMU specification */ + sw64_pmu = &core3_pmu; + + perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); + + return 0; +} +early_initcall(init_hw_perf_events); diff --git a/arch/sw_64/kernel/perf_regs.c b/arch/sw_64/kernel/perf_regs.c new file mode 100644 index 000000000000..b036f213936b --- /dev/null +++ b/arch/sw_64/kernel/perf_regs.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +u64 perf_reg_value(struct pt_regs *regs, int idx) +{ + if (WARN_ON_ONCE((u32)idx >= PERF_REG_SW64_MAX)) + return 0; + + return ((unsigned long *)regs)[idx]; +} + +#define REG_RESERVED (~((1ULL << PERF_REG_SW64_MAX) - 1)) + +int perf_reg_validate(u64 mask) +{ + if (!mask || mask & REG_RESERVED) + return -EINVAL; + return 0; +} + +u64 perf_reg_abi(struct task_struct *task) +{ + return PERF_SAMPLE_REGS_ABI_64; +} + +void perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs) +{ + regs_user->regs = task_pt_regs(current); + regs_user->abi = perf_reg_abi(current); +} -- Gitee From 504ccd65aa315019c8382178564658ecd440d66f Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:17 +0800 Subject: [PATCH 038/524] sw64: add kexec support Add kexec support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kexec.h | 82 ++++++++++++ arch/sw_64/kernel/machine_kexec.c | 209 ++++++++++++++++++++++++++++++ 2 files changed, 291 insertions(+) create mode 100644 arch/sw_64/include/asm/kexec.h create mode 100644 arch/sw_64/kernel/machine_kexec.c diff --git a/arch/sw_64/include/asm/kexec.h b/arch/sw_64/include/asm/kexec.h new file mode 100644 index 000000000000..25e0d8da84f8 --- /dev/null +++ b/arch/sw_64/include/asm/kexec.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KEXEC_H +#define _ASM_SW64_KEXEC_H + +#ifdef CONFIG_KEXEC + +/* Maximum physical address we can use pages from */ +#define KEXEC_SOURCE_MEMORY_LIMIT (-1UL) +/* Maximum address we can reach in physical address mode */ +#define KEXEC_DESTINATION_MEMORY_LIMIT (-1UL) +/* Maximum address we can use for the control code buffer */ +#define KEXEC_CONTROL_MEMORY_LIMIT (-1UL) + +#define KEXEC_CONTROL_PAGE_SIZE 8192 + +#define KEXEC_ARCH KEXEC_ARCH_SW64 + +#define KEXEC_SW64_ATAGS_OFFSET 0x1000 +#define KEXEC_SW64_ZIMAGE_OFFSET 0x8000 + +#ifndef __ASSEMBLY__ + +/** + * crash_setup_regs() - save registers for the panic kernel + * @newregs: registers are saved here + * @oldregs: registers to be saved (may be %NULL) + * + * Function copies machine registers from @oldregs to @newregs. If @oldregs is + * %NULL then current registers are stored there. + */ +static inline void crash_setup_regs(struct pt_regs *newregs, + struct pt_regs *oldregs) +{ + if (oldregs) { + memcpy(newregs, oldregs, sizeof(*newregs)); + } else { + __asm__ __volatile__ ("stl $0, %0" : "=m" (newregs->regs[0])); + __asm__ __volatile__ ("stl $1, %0" : "=m" (newregs->regs[1])); + __asm__ __volatile__ ("stl $2, %0" : "=m" (newregs->regs[2])); + __asm__ __volatile__ ("stl $3, %0" : "=m" (newregs->regs[3])); + __asm__ __volatile__ ("stl $4, %0" : "=m" (newregs->regs[4])); + __asm__ __volatile__ ("stl $5, %0" : "=m" (newregs->regs[5])); + __asm__ __volatile__ ("stl $6, %0" : "=m" (newregs->regs[6])); + __asm__ __volatile__ ("stl $7, %0" : "=m" (newregs->regs[7])); + __asm__ __volatile__ ("stl $8, %0" : "=m" (newregs->regs[8])); + __asm__ __volatile__ ("stl $9, %0" : "=m" (newregs->regs[9])); + __asm__ __volatile__ ("stl $10, %0" : "=m" (newregs->regs[10])); + __asm__ __volatile__ ("stl $11, %0" : "=m" (newregs->regs[11])); + __asm__ __volatile__ ("stl $12, %0" : "=m" (newregs->regs[12])); + __asm__ __volatile__ ("stl $13, %0" : "=m" (newregs->regs[13])); + __asm__ __volatile__ ("stl $14, %0" : "=m" (newregs->regs[14])); + __asm__ __volatile__ ("stl $15, %0" : "=m" (newregs->regs[15])); + __asm__ __volatile__ ("stl $16, %0" : "=m" (newregs->regs[16])); + __asm__ __volatile__ ("stl $17, %0" : "=m" (newregs->regs[17])); + __asm__ __volatile__ ("stl $18, %0" : "=m" (newregs->regs[18])); + __asm__ __volatile__ ("stl $19, %0" : "=m" (newregs->regs[19])); + __asm__ __volatile__ ("stl $20, %0" : "=m" (newregs->regs[20])); + __asm__ __volatile__ ("stl $21, %0" : "=m" (newregs->regs[21])); + __asm__ __volatile__ ("stl $22, %0" : "=m" (newregs->regs[22])); + __asm__ __volatile__ ("stl $23, %0" : "=m" (newregs->regs[23])); + __asm__ __volatile__ ("stl $24, %0" : "=m" (newregs->regs[24])); + __asm__ __volatile__ ("stl $25, %0" : "=m" (newregs->regs[25])); + __asm__ __volatile__ ("stl $26, %0" : "=m" (newregs->regs[26])); + __asm__ __volatile__ ("stl $27, %0" : "=m" (newregs->regs[27])); + __asm__ __volatile__ ("stl $28, %0" : "=m" (newregs->regs[28])); + __asm__ __volatile__ ("stl $29, %0" : "=m" (newregs->regs[29])); + __asm__ __volatile__ ("stl $30, %0" : "=m" (newregs->regs[30])); + newregs->pc = (unsigned long)current_text_addr(); + } +} + +/* Function pointer to optional machine-specific reinitialization */ +extern void (*kexec_reinit)(void); + +#endif /* __ASSEMBLY__ */ + +struct kimage; +extern unsigned long kexec_args[4]; + +#endif /* CONFIG_KEXEC */ + +#endif /* _ASM_SW64_KEXEC_H */ diff --git a/arch/sw_64/kernel/machine_kexec.c b/arch/sw_64/kernel/machine_kexec.c new file mode 100644 index 000000000000..950998476cda --- /dev/null +++ b/arch/sw_64/kernel/machine_kexec.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * machine_kexec.c for kexec + * + * This source code is licensed under the GNU General Public License, + * Version 2. See the file COPYING for more details. + */ +#include +#include +#include +#include +#include + +#include + +extern void *kexec_control_page; +extern const unsigned char relocate_new_kernel[]; +extern const size_t relocate_new_kernel_size; + +extern unsigned long kexec_start_address; +extern unsigned long kexec_indirection_page; + +static atomic_t waiting_for_crash_ipi; + +#ifdef CONFIG_SMP +extern struct smp_rcb_struct *smp_rcb; + +/* + * Wait for relocation code is prepared and send + * secondary CPUs to spin until kernel is relocated. + */ +static void kexec_smp_down(void *ignored) +{ + int cpu = smp_processor_id(); + + local_irq_disable(); + while (READ_ONCE(smp_rcb->ready) != 0) + mdelay(1); + set_cpu_online(cpu, false); + reset_cpu(cpu); +} +#endif + +int machine_kexec_prepare(struct kimage *kimage) +{ + return 0; +} + +void machine_kexec_cleanup(struct kimage *kimage) +{ +} + +void machine_shutdown(void) +{ +#ifdef CONFIG_SMP + WRITE_ONCE(smp_rcb->ready, 0); + smp_call_function(kexec_smp_down, NULL, 0); + smp_wmb(); + while (num_online_cpus() > 1) { + cpu_relax(); + mdelay(1); + } +#endif +} + +#ifdef CONFIG_SMP +static void machine_crash_nonpanic_core(void *unused) +{ + int cpu; + struct pt_regs regs; + + cpu = smp_processor_id(); + + local_irq_disable(); + crash_setup_regs(®s, NULL); + pr_debug("CPU %u will stop doing anything useful since another CPU has crashed\n", cpu); + crash_save_cpu(®s, cpu); + flush_cache_all(); + + set_cpu_online(cpu, false); + atomic_dec(&waiting_for_crash_ipi); + while (READ_ONCE(smp_rcb->ready) != 0) + mdelay(1); + if (cpu != 0) + reset_cpu(cpu); + else + machine_kexec(kexec_crash_image); +} +#else +static inline void machine_crash_nonpanic_core(void *unused) { } +#endif + +static void machine_kexec_mask_interrupts(void) +{ + unsigned int i; + struct irq_desc *desc; + + for_each_irq_desc(i, desc) { + struct irq_chip *chip; + + chip = irq_desc_get_chip(desc); + if (!chip) + continue; + + if (chip->irq_eoi && irqd_irq_inprogress(&desc->irq_data)) + chip->irq_eoi(&desc->irq_data); + + if (chip->irq_mask) + chip->irq_mask(&desc->irq_data); + + if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data)) + chip->irq_disable(&desc->irq_data); + } +} + +void machine_crash_shutdown(struct pt_regs *regs) +{ + int cpu; + unsigned long msecs; + + cpu = smp_processor_id(); + local_irq_disable(); + kernel_restart_prepare(NULL); + atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1); + smp_call_function(machine_crash_nonpanic_core, NULL, false); + msecs = 1000; /* Wait at most a second for the other cpus to stop */ + while ((atomic_read(&waiting_for_crash_ipi) > 0) && msecs) { + mdelay(1); + msecs--; + } + if (atomic_read(&waiting_for_crash_ipi) > 0) + pr_warn("Non-crashing CPUs did not react to IPI\n"); + + crash_save_cpu(regs, cpu); + machine_kexec_mask_interrupts(); + pr_info("Loading crashdump kernel...\n"); +#ifdef CONFIG_SMP + WRITE_ONCE(smp_rcb->ready, 0); + if (cpu != 0) + reset_cpu(cpu); +#endif +} + +#define phys_to_ktext(pa) (__START_KERNEL_map + (pa)) + +typedef void (*noretfun_t)(void) __noreturn; + +void machine_kexec(struct kimage *image) +{ + void *reboot_code_buffer; + unsigned long entry; + unsigned long *ptr; + struct boot_params *params = sunway_boot_params; + + + reboot_code_buffer = kexec_control_page; + pr_info("reboot_code_buffer = %px\n", reboot_code_buffer); + kexec_start_address = phys_to_ktext(image->start); + pr_info("kexec_start_address = %#lx\n", kexec_start_address); + if (image->type == KEXEC_TYPE_DEFAULT) + kexec_indirection_page = + (unsigned long) phys_to_virt(image->head & PAGE_MASK); + else + kexec_indirection_page = (unsigned long)&image->head; + + pr_info("kexec_indirection_page = %#lx, image->head=%#lx\n", + kexec_indirection_page, image->head); + + params->cmdline = kexec_start_address - COMMAND_LINE_OFF; + params->initrd_start = *(__u64 *)(kexec_start_address - INITRD_START_OFF); + params->initrd_size = *(__u64 *)(kexec_start_address - INITRD_SIZE_OFF); + + pr_info("initrd_start = %#llx, initrd_size = %#llx\n" + "dtb_start = %#llx, efi_systab = %#llx\n" + "efi_memmap = %#llx, efi_memmap_size = %#llx\n" + "efi_memdesc_size = %#llx, efi_memdesc_version = %#llx\n" + "cmdline = %#llx\n", + params->initrd_start, params->initrd_size, + params->dtb_start, params->efi_systab, + params->efi_memmap, params->efi_memmap_size, + params->efi_memdesc_size, params->efi_memdesc_version, + params->cmdline); + + memcpy(reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size); + + /* + * The generic kexec code builds a page list with physical + * addresses. they are directly accessible through KSEG0 (or + * CKSEG0 or XPHYS if on 64bit system), hence the + * phys_to_virt() call. + */ + for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE); + ptr = (entry & IND_INDIRECTION) ? + phys_to_virt(entry & PAGE_MASK) : ptr + 1) { + if (*ptr & IND_SOURCE || *ptr & IND_INDIRECTION || + *ptr & IND_DESTINATION) + *ptr = (unsigned long) phys_to_virt(*ptr); + } + + /* + * we do not want to be bothered. + */ + local_irq_disable(); + + pr_info("Will call new kernel at %08lx\n", image->start); + pr_info("Bye ...\n"); + smp_wmb(); + ((noretfun_t) reboot_code_buffer)(); +} -- Gitee From 920b303f20000b94e46d5f6727e74f7864256883 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:16 +0800 Subject: [PATCH 039/524] sw64: add kdump support Add kdump support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/crash_dump.c | 56 ++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 arch/sw_64/kernel/crash_dump.c diff --git a/arch/sw_64/kernel/crash_dump.c b/arch/sw_64/kernel/crash_dump.c new file mode 100644 index 000000000000..4484673823b8 --- /dev/null +++ b/arch/sw_64/kernel/crash_dump.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arch/sw_64/kernel/crash_dump.c + * + * Copyright (C) 2019 JN + * Author: He Sheng + * + * This code is taken from arch/x86/kernel/crash_dump_64.c + * Created by: Hariprasad Nellitheertha (hari@in.ibm.com) + * Copyright (C) IBM Corporation, 2004. All rights reserved + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + +/** + * copy_oldmem_page() - copy one page from old kernel memory + * @pfn: page frame number to be copied + * @buf: buffer where the copied page is placed + * @csize: number of bytes to copy + * @offset: offset in bytes into the page + * @userbuf: if set, @buf is int he user address space + * + * This function copies one page from old kernel memory into buffer pointed by + * @buf. If @buf is in userspace, set @userbuf to %1. Returns number of bytes + * copied or negative error in case of failure. + */ +ssize_t copy_oldmem_page(unsigned long pfn, char *buf, + size_t csize, unsigned long offset, + int userbuf) +{ + void *vaddr; + + if (!csize) + return 0; + + vaddr = ioremap(__pfn_to_phys(pfn), PAGE_SIZE); + if (!vaddr) + return -ENOMEM; + + if (userbuf) { + if (copy_to_user(buf, vaddr + offset, csize)) { + iounmap(vaddr); + return -EFAULT; + } + } else { + memcpy(buf, vaddr + offset, csize); + } + + iounmap(vaddr); + return csize; +} -- Gitee From d79fe25fff2f655a9730ccf3af5394772d4c5d8b Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:03 +0800 Subject: [PATCH 040/524] sw64: add eBPF JIT support Add eBPF JIT support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/bpf_perf_event.h | 9 + arch/sw_64/net/Makefile | 5 + arch/sw_64/net/bpf_jit.h | 368 +++++ arch/sw_64/net/bpf_jit_comp.c | 1455 ++++++++++++++++++ 4 files changed, 1837 insertions(+) create mode 100644 arch/sw_64/include/uapi/asm/bpf_perf_event.h create mode 100644 arch/sw_64/net/Makefile create mode 100644 arch/sw_64/net/bpf_jit.h create mode 100644 arch/sw_64/net/bpf_jit_comp.c diff --git a/arch/sw_64/include/uapi/asm/bpf_perf_event.h b/arch/sw_64/include/uapi/asm/bpf_perf_event.h new file mode 100644 index 000000000000..52f6f1e555f1 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/bpf_perf_event.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_BPF_PERF_EVENT_H +#define _UAPI_ASM_SW64_BPF_PERF_EVENT_H + +#include + +typedef struct user_pt_regs bpf_user_pt_regs_t; + +#endif /* _UAPI_ASM_SW64_BPF_PERF_EVENT_H */ diff --git a/arch/sw_64/net/Makefile b/arch/sw_64/net/Makefile new file mode 100644 index 000000000000..d4663b4bf509 --- /dev/null +++ b/arch/sw_64/net/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Arch-specific network modules +# +obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o diff --git a/arch/sw_64/net/bpf_jit.h b/arch/sw_64/net/bpf_jit.h new file mode 100644 index 000000000000..929036d8ea6b --- /dev/null +++ b/arch/sw_64/net/bpf_jit.h @@ -0,0 +1,368 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BPF JIT compiler for SW64 + * + * Copyright (C) Mao Minkai + * Author: Mao Minkai + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef _SW64_NET_BPF_JIT_H +#define _SW64_NET_BPF_JIT_H + +/* SW64 instruction field shift */ +#define SW64_BPF_OPCODE_OFFSET 26 +#define SW64_BPF_RA_OFFSET 21 +#define SW64_BPF_RB_OFFSET 16 +#define SW64_BPF_SIMPLE_ALU_IMM_OFFSET 13 +#define SW64_BPF_SIMPLE_ALU_FUNC_OFFSET 5 +#define SW64_BPF_SIMPLE_ALU_RC_OFFSET 0 +#define SW64_BPF_LS_FUNC_OFFSET 12 + +/* SW64 instruction opcodes */ +#define SW64_BPF_OPCODE_CALL 0x01 +#define SW64_BPF_OPCODE_RET 0x02 +#define SW64_BPF_OPCODE_JMP 0x03 +#define SW64_BPF_OPCODE_BR 0x04 +#define SW64_BPF_OPCODE_BSR 0x05 +#define SW64_BPF_OPCODE_MISC 0x06 +#define SW64_BPF_OPCODE_LOCK 0x08 +#define SW64_BPF_OPCODE_ALU_REG 0x10 +#define SW64_BPF_OPCODE_ALU_IMM 0x12 +#define SW64_BPF_OPCODE_LDBU 0x20 +#define SW64_BPF_OPCODE_LDHU 0x21 +#define SW64_BPF_OPCODE_LDW 0x22 +#define SW64_BPF_OPCODE_LDL 0x23 +#define SW64_BPF_OPCODE_STB 0x28 +#define SW64_BPF_OPCODE_STH 0x29 +#define SW64_BPF_OPCODE_STW 0x2A +#define SW64_BPF_OPCODE_STL 0x2B +#define SW64_BPF_OPCODE_BEQ 0x30 +#define SW64_BPF_OPCODE_BNE 0x31 +#define SW64_BPF_OPCODE_BLT 0x32 +#define SW64_BPF_OPCODE_BLE 0x33 +#define SW64_BPF_OPCODE_BGT 0x34 +#define SW64_BPF_OPCODE_BGE 0x35 +#define SW64_BPF_OPCODE_BLBC 0x36 +#define SW64_BPF_OPCODE_BLBS 0x37 +#define SW64_BPF_OPCODE_LDI 0x3E +#define SW64_BPF_OPCODE_LDIH 0x3F + +/* SW64 MISC instructions function codes */ +#define SW64_BPF_FUNC_MISC_RD_F 0x1000 +#define SW64_BPF_FUNC_MISC_WR_F 0x1020 + +/* SW64 LOCK instructions function codes */ +#define SW64_BPF_FUNC_LOCK_LLDW 0x0 +#define SW64_BPF_FUNC_LOCK_LLDL 0x1 +#define SW64_BPF_FUNC_LOCK_LSTW 0x8 +#define SW64_BPF_FUNC_LOCK_LSTL 0x9 + +/* SW64 ALU instructions function codes */ +#define SW64_BPF_FUNC_ALU_ADDW 0x00 +#define SW64_BPF_FUNC_ALU_SUBW 0x01 +#define SW64_BPF_FUNC_ALU_ADDL 0x08 +#define SW64_BPF_FUNC_ALU_SUBL 0x09 +#define SW64_BPF_FUNC_ALU_MULW 0x10 +#define SW64_BPF_FUNC_ALU_MULL 0x18 +#define SW64_BPF_FUNC_ALU_CMPEQ 0x28 +#define SW64_BPF_FUNC_ALU_CMPLT 0x29 +#define SW64_BPF_FUNC_ALU_CMPLE 0x2A +#define SW64_BPF_FUNC_ALU_CMPULT 0x2B +#define SW64_BPF_FUNC_ALU_CMPULE 0x2C +#define SW64_BPF_FUNC_ALU_AND 0x38 +#define SW64_BPF_FUNC_ALU_BIC 0x39 +#define SW64_BPF_FUNC_ALU_BIS 0x3A +#define SW64_BPF_FUNC_ALU_ORNOT 0x3B +#define SW64_BPF_FUNC_ALU_XOR 0x3C +#define SW64_BPF_FUNC_ALU_EQV 0x3D +#define SW64_BPF_FUNC_ALU_SLL 0x48 +#define SW64_BPF_FUNC_ALU_SRL 0x49 +#define SW64_BPF_FUNC_ALU_SRA 0x4A +#define SW64_BPF_FUNC_ALU_ZAP 0x68 +#define SW64_BPF_FUNC_ALU_ZAPNOT 0x69 +#define SW64_BPF_FUNC_ALU_SEXTB 0x6A +#define SW64_BPF_FUNC_ALU_SEXTH 0x6B + +/* special instuction used in jit_fill_hole() */ +#define SW64_BPF_ILLEGAL_INSN (0x1ff00000) /* pri_ret/b $31 */ + +enum sw64_bpf_registers { + SW64_BPF_REG_V0 = 0, /* keep return value */ + SW64_BPF_REG_T0 = 1, + SW64_BPF_REG_T1 = 2, + SW64_BPF_REG_T2 = 3, + SW64_BPF_REG_T3 = 4, + SW64_BPF_REG_T4 = 5, + SW64_BPF_REG_T5 = 6, + SW64_BPF_REG_T6 = 7, + SW64_BPF_REG_T7 = 8, + SW64_BPF_REG_S0 = 9, /* callee saved */ + SW64_BPF_REG_S1 = 10, /* callee saved */ + SW64_BPF_REG_S2 = 11, /* callee saved */ + SW64_BPF_REG_S3 = 12, /* callee saved */ + SW64_BPF_REG_S4 = 13, /* callee saved */ + SW64_BPF_REG_S5 = 14, /* callee saved */ + SW64_BPF_REG_S6 = 15, /* callee saved */ + SW64_BPF_REG_FP = 15, /* frame pointer if necessary */ + SW64_BPF_REG_A0 = 16, /* argument 0 */ + SW64_BPF_REG_A1 = 17, /* argument 1 */ + SW64_BPF_REG_A2 = 18, /* argument 2 */ + SW64_BPF_REG_A3 = 19, /* argument 3 */ + SW64_BPF_REG_A4 = 20, /* argument 4 */ + SW64_BPF_REG_A5 = 21, /* argument 5 */ + SW64_BPF_REG_T8 = 22, + SW64_BPF_REG_T9 = 23, + SW64_BPF_REG_T10 = 24, + SW64_BPF_REG_T11 = 25, + SW64_BPF_REG_RA = 26, /* callee saved, keep retuen address */ + SW64_BPF_REG_T12 = 27, + SW64_BPF_REG_PV = 27, + SW64_BPF_REG_AT = 28, /* reserved by assembler */ + SW64_BPF_REG_GP = 29, /* global pointer */ + SW64_BPF_REG_SP = 30, /* callee saved, stack pointer */ + SW64_BPF_REG_ZR = 31 /* read 0 */ +}; + +/* SW64 load and store instructions */ +#define SW64_BPF_LDBU(dst, rb, offset16) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDBU, dst, rb, offset16) +#define SW64_BPF_LDHU(dst, rb, offset16) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDHU, dst, rb, offset16) +#define SW64_BPF_LDW(dst, rb, offset16) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDW, dst, rb, offset16) +#define SW64_BPF_LDL(dst, rb, offset16) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDL, dst, rb, offset16) +#define SW64_BPF_STB(src, rb, offset16) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_STB, src, rb, offset16) +#define SW64_BPF_STH(src, rb, offset16) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_STH, src, rb, offset16) +#define SW64_BPF_STW(src, rb, offset16) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_STW, src, rb, offset16) +#define SW64_BPF_STL(src, rb, offset16) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_STL, src, rb, offset16) +#define SW64_BPF_LDI(dst, rb, imm16) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDI, dst, rb, imm16) +#define SW64_BPF_LDIH(dst, rb, imm16) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_LDIH, dst, rb, imm16) + +/* SW64 lock instructions */ +#define SW64_BPF_LLDW(ra, rb, offset16) \ + sw64_bpf_gen_format_ls_func(SW64_BPF_OPCODE_LOCK, \ + ra, rb, offset16, SW64_BPF_FUNC_LOCK_LLDW) +#define SW64_BPF_LLDL(ra, rb, offset16) \ + sw64_bpf_gen_format_ls_func(SW64_BPF_OPCODE_LOCK, \ + ra, rb, offset16, SW64_BPF_FUNC_LOCK_LLDL) +#define SW64_BPF_LSTW(ra, rb, offset16) \ + sw64_bpf_gen_format_ls_func(SW64_BPF_OPCODE_LOCK, \ + ra, rb, offset16, SW64_BPF_FUNC_LOCK_LSTW) +#define SW64_BPF_LSTL(ra, rb, offset16) \ + sw64_bpf_gen_format_ls_func(SW64_BPF_OPCODE_LOCK, \ + ra, rb, offset16, SW64_BPF_FUNC_LOCK_LSTL) +#define SW64_BPF_RD_F(ra) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_MISC, \ + ra, SW64_BPF_REG_ZR, SW64_BPF_FUNC_MISC_RD_F) +#define SW64_BPF_WR_F(ra) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_MISC, \ + ra, SW64_BPF_REG_ZR, SW64_BPF_FUNC_MISC_WR_F) + +/* SW64 ALU instructions REG format */ +#define SW64_BPF_ADDW_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_ADDW) +#define SW64_BPF_ADDL_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_ADDL) +#define SW64_BPF_SUBW_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_SUBW) +#define SW64_BPF_SUBL_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_SUBL) +#define SW64_BPF_MULW_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_MULW) +#define SW64_BPF_MULL_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_MULL) +#define SW64_BPF_ZAP_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_ZAP) +#define SW64_BPF_ZAPNOT_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_ZAPNOT) +#define SW64_BPF_SEXTB_REG(rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + SW64_BPF_REG_ZR, rb, dst, SW64_BPF_FUNC_ALU_SEXTB) +#define SW64_BPF_SEXTH_REG(rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + SW64_BPF_REG_ZR, rb, dst, SW64_BPF_FUNC_ALU_SEXTH) + +/* SW64 ALU instructions IMM format */ +#define SW64_BPF_ADDW_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_ADDW) +#define SW64_BPF_ADDL_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_ADDL) +#define SW64_BPF_SUBW_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_SUBW) +#define SW64_BPF_SUBL_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_SUBL) +#define SW64_BPF_MULW_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_MULW) +#define SW64_BPF_MULL_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_MULL) +#define SW64_BPF_ZAP_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_ZAP) +#define SW64_BPF_ZAPNOT_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_ZAPNOT) +#define SW64_BPF_SEXTB_IMM(imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + SW64_BPF_REG_ZR, imm8, dst, SW64_BPF_FUNC_ALU_SEXTB) +#define SW64_BPF_SEXTH_IMM(imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + SW64_BPF_REG_ZR, imm8, dst, SW64_BPF_FUNC_ALU_SEXTH) + +/* SW64 bit shift instructions REG format */ +#define SW64_BPF_SLL_REG(src, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + src, rb, dst, SW64_BPF_FUNC_ALU_SLL) +#define SW64_BPF_SRL_REG(src, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + src, rb, dst, SW64_BPF_FUNC_ALU_SRL) +#define SW64_BPF_SRA_REG(src, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + src, rb, dst, SW64_BPF_FUNC_ALU_SRA) + +/* SW64 bit shift instructions IMM format */ +#define SW64_BPF_SLL_IMM(src, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + src, imm8, dst, SW64_BPF_FUNC_ALU_SLL) +#define SW64_BPF_SRL_IMM(src, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + src, imm8, dst, SW64_BPF_FUNC_ALU_SRL) +#define SW64_BPF_SRA_IMM(src, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + src, imm8, dst, SW64_BPF_FUNC_ALU_SRA) + +/* SW64 control instructions */ +#define SW64_BPF_CALL(ra, rb) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_CALL, ra, rb, 0) +#define SW64_BPF_RET(rb) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_RET, SW64_BPF_REG_ZR, rb, 0) +#define SW64_BPF_JMP(ra, rb) \ + sw64_bpf_gen_format_ls(SW64_BPF_OPCODE_JMP, ra, rb, 0) +#define SW64_BPF_BR(ra, offset) \ + sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BR, ra, offset) +#define SW64_BPF_BSR(ra, offset) \ + sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BSR, ra, offset) +#define SW64_BPF_BEQ(ra, offset) \ + sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BEQ, ra, offset) +#define SW64_BPF_BNE(ra, offset) \ + sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BNE, ra, offset) +#define SW64_BPF_BLT(ra, offset) \ + sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BLT, ra, offset) +#define SW64_BPF_BLE(ra, offset) \ + sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BLE, ra, offset) +#define SW64_BPF_BGT(ra, offset) \ + sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BGT, ra, offset) +#define SW64_BPF_BGE(ra, offset) \ + sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BGE, ra, offset) +#define SW64_BPF_BLBC(ra, offset) \ + sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BLBC, ra, offset) +#define SW64_BPF_BLBS(ra, offset) \ + sw64_bpf_gen_format_br(SW64_BPF_OPCODE_BLBS, ra, offset) + +/* SW64 bit logic instructions REG format */ +#define SW64_BPF_AND_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_AND) +#define SW64_BPF_ANDNOT_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_BIC) +#define SW64_BPF_BIS_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_BIS) +#define SW64_BPF_ORNOT_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_ORNOT) +#define SW64_BPF_XOR_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_XOR) +#define SW64_BPF_EQV_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_EQV) + +/* SW64 bit logic instructions IMM format */ +#define SW64_BPF_AND_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_AND) +#define SW64_BPF_ANDNOT_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_BIC) +#define SW64_BPF_BIS_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_BIS) +#define SW64_BPF_ORNOT_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_ORNOT) +#define SW64_BPF_XOR_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_XOR) +#define SW64_BPF_EQV_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_EQV) + +/* SW64 compare instructions REG format */ +#define SW64_BPF_CMPEQ_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_CMPEQ) +#define SW64_BPF_CMPLT_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_CMPLT) +#define SW64_BPF_CMPLE_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_CMPLE) +#define SW64_BPF_CMPULT_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_CMPULT) +#define SW64_BPF_CMPULE_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_CMPULE) + +/* SW64 compare instructions imm format */ +#define SW64_BPF_CMPEQ_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_CMPEQ) +#define SW64_BPF_CMPLT_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_CMPLT) +#define SW64_BPF_CMPLE_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_CMPLE) +#define SW64_BPF_CMPULT_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_CMPULT) +#define SW64_BPF_CMPULE_IMM(ra, imm8, dst) \ + sw64_bpf_gen_format_simple_alu_imm(SW64_BPF_OPCODE_ALU_IMM, \ + ra, imm8, dst, SW64_BPF_FUNC_ALU_CMPULE) + +#endif /* _SW64_NET_BPF_JIT_H */ diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c new file mode 100644 index 000000000000..31202dd0f9cf --- /dev/null +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -0,0 +1,1455 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * BPF JIT compiler for SW64 + * + * Copyright (C) Mao Minkai + * Author: Mao Minkai + * + * This file is taken from arch/arm64/net/bpf_jit_comp.c + * Copyright (C) 2014-2016 Zi Shen Lim + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#include + +#include "bpf_jit.h" + +#define TCALL_CNT (MAX_BPF_JIT_REG + 0) + +static const int bpf2sw64[] = { + /* return value from in-kernel function, and exit value from eBPF */ + [BPF_REG_0] = SW64_BPF_REG_V0, + /* arguments from eBPF program to in-kernel function */ + [BPF_REG_1] = SW64_BPF_REG_A0, + [BPF_REG_2] = SW64_BPF_REG_A1, + [BPF_REG_3] = SW64_BPF_REG_A2, + [BPF_REG_4] = SW64_BPF_REG_A3, + [BPF_REG_5] = SW64_BPF_REG_A4, + /* callee saved registers that in-kernel function will preserve */ + [BPF_REG_6] = SW64_BPF_REG_S0, + [BPF_REG_7] = SW64_BPF_REG_S1, + [BPF_REG_8] = SW64_BPF_REG_S2, + [BPF_REG_9] = SW64_BPF_REG_S3, + /* read-only frame pointer to access stack */ + [BPF_REG_FP] = SW64_BPF_REG_FP, + /* tail_call_cnt */ + [TCALL_CNT] = SW64_BPF_REG_S4, + /* temporary register for blinding constants */ + [BPF_REG_AX] = SW64_BPF_REG_T11, +}; + +struct jit_ctx { + const struct bpf_prog *prog; + int idx; // JITed instruction index + int current_tmp_reg; + int epilogue_offset; + int *insn_offset; // [bpf_insn_idx] = jited_insn_idx + int exentry_idx; + u32 *image; // JITed instruction + u32 stack_size; +}; + +struct sw64_jit_data { + struct bpf_binary_header *header; + u8 *image; // bpf instruction + struct jit_ctx ctx; +}; + +static inline u32 sw64_bpf_gen_format_br(int opcode, enum sw64_bpf_registers ra, u32 disp) +{ + opcode = opcode << SW64_BPF_OPCODE_OFFSET; + ra = ra << SW64_BPF_RA_OFFSET; + return opcode | ra | (disp & 0x1fffff); +} + +static inline u32 sw64_bpf_gen_format_ls(int opcode, enum sw64_bpf_registers ra, + enum sw64_bpf_registers rb, u16 disp) +{ + opcode = opcode << SW64_BPF_OPCODE_OFFSET; + ra = ra << SW64_BPF_RA_OFFSET; + rb = rb << SW64_BPF_RB_OFFSET; + return opcode | ra | rb | (disp & 0xffff); +} + +static inline u32 sw64_bpf_gen_format_ls_func(int opcode, enum sw64_bpf_registers ra, + enum sw64_bpf_registers rb, u16 disp, int function) +{ + opcode = opcode << SW64_BPF_OPCODE_OFFSET; + ra = ra << SW64_BPF_RA_OFFSET; + rb = rb << SW64_BPF_RB_OFFSET; + function = function << SW64_BPF_LS_FUNC_OFFSET; + return opcode | ra | rb | function | (disp & 0xfff); +} + +static inline u32 sw64_bpf_gen_format_simple_alu_reg(int opcode, enum sw64_bpf_registers ra, + enum sw64_bpf_registers rb, enum sw64_bpf_registers rc, int function) +{ + opcode = opcode << SW64_BPF_OPCODE_OFFSET; + ra = ra << SW64_BPF_RA_OFFSET; + rb = rb << SW64_BPF_RB_OFFSET; + rc = rc << SW64_BPF_SIMPLE_ALU_RC_OFFSET; + function = function << SW64_BPF_SIMPLE_ALU_FUNC_OFFSET; + return opcode | ra | rb | function | rc; +} + +static inline u32 sw64_bpf_gen_format_simple_alu_imm(int opcode, enum sw64_bpf_registers ra, + u32 imm, enum sw64_bpf_registers rc, int function) +{ + opcode = opcode << SW64_BPF_OPCODE_OFFSET; + ra = ra << SW64_BPF_RA_OFFSET; + imm = (imm & 0xff) << SW64_BPF_SIMPLE_ALU_IMM_OFFSET; + rc = rc << SW64_BPF_SIMPLE_ALU_RC_OFFSET; + function = function << SW64_BPF_SIMPLE_ALU_FUNC_OFFSET; + return opcode | ra | imm | function | rc; +} + +static inline void emit(const u32 insn, struct jit_ctx *ctx) +{ + if (ctx->image != NULL) + ctx->image[ctx->idx] = insn; + + ctx->idx++; +} + +static inline int get_tmp_reg(struct jit_ctx *ctx) +{ + ctx->current_tmp_reg++; + /* Do not use 22-25. Should be more than enough. */ + if (unlikely(ctx->current_tmp_reg == 8)) { + pr_err("eBPF JIT %s[%d]: not enough temporary registers!\n", + current->comm, current->pid); + return -1; + } + return ctx->current_tmp_reg; +} + +static inline void put_tmp_reg(struct jit_ctx *ctx) +{ + ctx->current_tmp_reg--; + if (ctx->current_tmp_reg == 21) + ctx->current_tmp_reg = 7; +} + +static void emit_sw64_ldu32(const int dst, const u32 imm, struct jit_ctx *ctx) +{ + u16 imm_tmp; + u8 reg_tmp = get_tmp_reg(ctx); + + if (!imm) { + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx); + put_tmp_reg(ctx); + return; + } + + if (imm <= S16_MAX) { + emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); + put_tmp_reg(ctx); + return; + } + + if (imm >= U32_MAX - S16_MAX) { + emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + put_tmp_reg(ctx); + return; + } + + imm_tmp = (imm >> 30) & 3; + emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm_tmp), ctx); + if (imm_tmp) + emit(SW64_BPF_SLL_IMM(dst, 30, dst), ctx); + + imm_tmp = (imm >> 15) & 0x7fff; + if (imm_tmp) { + emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); + emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx); + emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); + } + + imm_tmp = imm & 0x7fff; + if (imm_tmp) + emit(SW64_BPF_LDI(dst, dst, imm_tmp), ctx); + + put_tmp_reg(ctx); +} + +static void emit_sw64_lds32(const int dst, const s32 imm, struct jit_ctx *ctx) +{ + s16 hi = imm >> 16; + s16 lo = imm & 0xffff; + u8 reg_tmp = get_tmp_reg(ctx); + + if (!imm) { + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx); + put_tmp_reg(ctx); + return; + } + + if (imm >= S16_MIN && imm <= S16_MAX) { + emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); + put_tmp_reg(ctx); + return; + } + + emit(SW64_BPF_LDIH(dst, SW64_BPF_REG_ZR, hi), ctx); + if (lo & 0x8000) { // sign bit is 1 + lo = lo & 0x7fff; + emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, 1), ctx); + emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx); + emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); + if (lo) + emit(SW64_BPF_LDI(dst, dst, lo), ctx); + } else { // sign bit is 0 + if (lo) + emit(SW64_BPF_LDI(dst, dst, lo), ctx); + } + + put_tmp_reg(ctx); +} + +static void emit_sw64_ldu64(const int dst, const u64 imm, struct jit_ctx *ctx) +{ + u16 imm_tmp; + u8 reg_tmp = get_tmp_reg(ctx); + + if (!imm) { + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx); + put_tmp_reg(ctx); + return; + } + + if (imm <= U32_MAX) { + put_tmp_reg(ctx); + return emit_sw64_ldu32(dst, (u32)imm, ctx); + } + + if (imm >= (U64_MAX - S16_MAX) || imm <= S16_MAX) { + emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); + put_tmp_reg(ctx); + return; + } + + imm_tmp = (imm >> 60) & 0xf; + emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm_tmp), ctx); + if (imm_tmp) + emit(SW64_BPF_SLL_IMM(dst, 60, dst), ctx); + + imm_tmp = (imm >> 45) & 0x7fff; + if (imm_tmp) { + emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); + emit(SW64_BPF_SLL_IMM(reg_tmp, 45, reg_tmp), ctx); + emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); + } + + imm_tmp = (imm >> 30) & 0x7fff; + if (imm_tmp) { + emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); + emit(SW64_BPF_SLL_IMM(reg_tmp, 30, reg_tmp), ctx); + emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); + } + + imm_tmp = (imm >> 15) & 0x7fff; + if (imm_tmp) { + emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); + emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx); + emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); + } + + imm_tmp = imm & 0x7fff; + if (imm_tmp) + emit(SW64_BPF_LDI(dst, dst, imm_tmp), ctx); + + put_tmp_reg(ctx); +} + +/* Do not change!!! See arch/sw_64/lib/divide.S for more detail */ +#define REG(x) "$"str(x) +#define str(x) #x +#define DIV_RET_ADDR 23 +#define DIVIDEND 24 +#define DIVISOR 25 +#define RESULT 27 + +#include +static void emit_sw64_divmod(const int dst, const int src, struct jit_ctx *ctx, u8 code) +{ + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, dst, DIVIDEND), ctx); + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, src, DIVISOR), ctx); + switch (BPF_CLASS(code)) { + case BPF_ALU: + switch (BPF_OP(code)) { + case BPF_DIV: + emit_sw64_ldu64(SW64_BPF_REG_PV, (u64)__divwu, ctx); + break; + case BPF_MOD: + emit_sw64_ldu64(SW64_BPF_REG_PV, (u64)__remwu, ctx); + break; + } + emit(SW64_BPF_CALL(DIV_RET_ADDR, SW64_BPF_REG_PV), ctx); + emit(SW64_BPF_ZAP_IMM(RESULT, 0xf0, dst), ctx); + break; + case BPF_ALU64: + switch (BPF_OP(code)) { + case BPF_DIV: + emit_sw64_ldu64(SW64_BPF_REG_PV, (u64)__divlu, ctx); + break; + case BPF_MOD: + emit_sw64_ldu64(SW64_BPF_REG_PV, (u64)__remlu, ctx); + break; + } + emit(SW64_BPF_CALL(DIV_RET_ADDR, SW64_BPF_REG_PV), ctx); + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, RESULT, dst), ctx); + break; + } +} + +#undef REG +#undef str +#undef DIVIDEND +#undef DIVISOR +#undef RESULT + +/* STX XADD: lock *(u32 *)(dst + off) += src */ +static void emit_sw64_xadd32(const int src, int dst, s16 off, struct jit_ctx *ctx) +{ + int atomic_start; + int atomic_end; + u8 tmp1 = get_tmp_reg(ctx); + u8 tmp2 = get_tmp_reg(ctx); + u8 tmp3 = get_tmp_reg(ctx); + + if (off < -0x800 || off > 0x7ff) { + emit(SW64_BPF_LDI(tmp1, dst, off), ctx); + dst = tmp1; + off = 0; + } + + atomic_start = ctx->idx; + emit(SW64_BPF_LLDW(tmp2, dst, off), ctx); + emit(SW64_BPF_LDI(tmp3, SW64_BPF_REG_ZR, 1), ctx); + emit(SW64_BPF_WR_F(tmp3), ctx); + emit(SW64_BPF_ADDW_REG(tmp2, src, tmp2), ctx); + if (ctx->idx & 1) + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, SW64_BPF_REG_ZR), ctx); + emit(SW64_BPF_LSTW(tmp2, dst, off), ctx); + emit(SW64_BPF_RD_F(tmp3), ctx); + atomic_end = ctx->idx; + emit(SW64_BPF_BEQ(tmp3, atomic_start - atomic_end - 1), ctx); + + put_tmp_reg(ctx); + put_tmp_reg(ctx); + put_tmp_reg(ctx); +} + +/* STX XADD: lock *(u64 *)(dst + off) += src */ +static void emit_sw64_xadd64(const int src, int dst, s16 off, struct jit_ctx *ctx) +{ + int atomic_start; + int atomic_end; + u8 tmp1 = get_tmp_reg(ctx); + u8 tmp2 = get_tmp_reg(ctx); + u8 tmp3 = get_tmp_reg(ctx); + + if (off < -0x800 || off > 0x7ff) { + emit(SW64_BPF_LDI(tmp1, dst, off), ctx); + dst = tmp1; + off = 0; + } + + atomic_start = ctx->idx; + emit(SW64_BPF_LLDL(tmp2, dst, off), ctx); + emit(SW64_BPF_LDI(tmp3, SW64_BPF_REG_ZR, 1), ctx); + emit(SW64_BPF_WR_F(tmp3), ctx); + emit(SW64_BPF_ADDL_REG(tmp2, src, tmp2), ctx); + if (ctx->idx & 1) + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, SW64_BPF_REG_ZR), ctx); + emit(SW64_BPF_LSTL(tmp2, dst, off), ctx); + emit(SW64_BPF_RD_F(tmp3), ctx); + atomic_end = ctx->idx; + emit(SW64_BPF_BEQ(tmp3, atomic_start - atomic_end - 1), ctx); + + put_tmp_reg(ctx); + put_tmp_reg(ctx); + put_tmp_reg(ctx); +} + +static void emit_sw64_htobe16(const int dst, struct jit_ctx *ctx) +{ + u8 tmp = get_tmp_reg(ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x2, tmp), ctx); + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x1, dst), ctx); + emit(SW64_BPF_SRL_IMM(tmp, 8, tmp), ctx); + emit(SW64_BPF_SLL_IMM(dst, 8, dst), ctx); + emit(SW64_BPF_BIS_REG(dst, tmp, dst), ctx); + + put_tmp_reg(ctx); +} + +static void emit_sw64_htobe32(const int dst, struct jit_ctx *ctx) +{ + u8 tmp1 = get_tmp_reg(ctx); + u8 tmp2 = get_tmp_reg(ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x8, tmp1), ctx); + emit(SW64_BPF_SRL_IMM(tmp1, 24, tmp2), ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x4, tmp1), ctx); + emit(SW64_BPF_SRL_IMM(tmp1, 8, tmp1), ctx); + emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x2, tmp1), ctx); + emit(SW64_BPF_SLL_IMM(tmp1, 8, tmp1), ctx); + emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x1, dst), ctx); + emit(SW64_BPF_SLL_IMM(dst, 24, dst), ctx); + emit(SW64_BPF_BIS_REG(dst, tmp2, dst), ctx); + + put_tmp_reg(ctx); + put_tmp_reg(ctx); +} + +static void emit_sw64_htobe64(const int dst, struct jit_ctx *ctx) +{ + u8 tmp1 = get_tmp_reg(ctx); + u8 tmp2 = get_tmp_reg(ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x80, tmp1), ctx); + emit(SW64_BPF_SRL_IMM(tmp1, 56, tmp2), ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x40, tmp1), ctx); + emit(SW64_BPF_SRL_IMM(tmp1, 40, tmp1), ctx); + emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x20, tmp1), ctx); + emit(SW64_BPF_SRL_IMM(tmp1, 24, tmp1), ctx); + emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x10, tmp1), ctx); + emit(SW64_BPF_SRL_IMM(tmp1, 8, tmp1), ctx); + emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x08, tmp1), ctx); + emit(SW64_BPF_SLL_IMM(tmp1, 8, tmp1), ctx); + emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x04, tmp1), ctx); + emit(SW64_BPF_SLL_IMM(tmp1, 24, tmp1), ctx); + emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x02, tmp1), ctx); + emit(SW64_BPF_SLL_IMM(tmp1, 40, tmp1), ctx); + emit(SW64_BPF_BIS_REG(tmp2, tmp1, tmp2), ctx); + + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x01, dst), ctx); + emit(SW64_BPF_SLL_IMM(dst, 56, dst), ctx); + emit(SW64_BPF_BIS_REG(dst, tmp2, dst), ctx); + + put_tmp_reg(ctx); + put_tmp_reg(ctx); +} + +static void jit_fill_hole(void *area, unsigned int size) +{ + unsigned long c = SW64_BPF_ILLEGAL_INSN; + + c |= c << 32; + __constant_c_memset(area, c, size); +} + +static int offset_to_epilogue(const struct jit_ctx *ctx); +static int bpf2sw64_offset(int bpf_idx, s32 off, const struct jit_ctx *ctx) +{ + int from = ctx->insn_offset[bpf_idx + 1]; + int to = ctx->insn_offset[bpf_idx + 1 + off]; + + if (ctx->image == NULL) + return 0; + + return to - from; +} + +static int offset_to_epilogue(const struct jit_ctx *ctx) +{ + if (ctx->image == NULL) + return 0; + + return ctx->epilogue_offset - ctx->idx; +} + +/* For tail call, jump to set up function call stack */ +#define PROLOGUE_OFFSET 11 + +static void build_prologue(struct jit_ctx *ctx, bool was_classic) +{ + const u8 r6 = bpf2sw64[BPF_REG_6]; + const u8 r7 = bpf2sw64[BPF_REG_7]; + const u8 r8 = bpf2sw64[BPF_REG_8]; + const u8 r9 = bpf2sw64[BPF_REG_9]; + const u8 fp = bpf2sw64[BPF_REG_FP]; + const u8 tcc = bpf2sw64[TCALL_CNT]; + + /* Save callee-saved registers */ + emit(SW64_BPF_LDI(SW64_BPF_REG_SP, SW64_BPF_REG_SP, -64), ctx); + emit(SW64_BPF_STL(SW64_BPF_REG_RA, SW64_BPF_REG_SP, 0), ctx); + emit(SW64_BPF_STL(fp, SW64_BPF_REG_SP, 8), ctx); + emit(SW64_BPF_STL(r6, SW64_BPF_REG_SP, 16), ctx); + emit(SW64_BPF_STL(r7, SW64_BPF_REG_SP, 24), ctx); + emit(SW64_BPF_STL(r8, SW64_BPF_REG_SP, 32), ctx); + emit(SW64_BPF_STL(r9, SW64_BPF_REG_SP, 40), ctx); + emit(SW64_BPF_STL(tcc, SW64_BPF_REG_SP, 48), ctx); + emit(SW64_BPF_STL(SW64_BPF_REG_GP, SW64_BPF_REG_SP, 56), ctx); + + /* Set up BPF prog stack base register */ + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_SP, fp), ctx); + if (!was_classic) + /* Initialize tail_call_cnt */ + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, tcc), ctx); + + /* Set up function call stack */ + ctx->stack_size = (ctx->prog->aux->stack_depth + 15) & (~15); + emit(SW64_BPF_LDI(SW64_BPF_REG_SP, SW64_BPF_REG_SP, -ctx->stack_size), ctx); +} + +static void build_epilogue(struct jit_ctx *ctx) +{ + const u8 r6 = bpf2sw64[BPF_REG_6]; + const u8 r7 = bpf2sw64[BPF_REG_7]; + const u8 r8 = bpf2sw64[BPF_REG_8]; + const u8 r9 = bpf2sw64[BPF_REG_9]; + const u8 fp = bpf2sw64[BPF_REG_FP]; + const u8 tcc = bpf2sw64[TCALL_CNT]; + + /* Destroy function call stack */ + emit(SW64_BPF_LDI(SW64_BPF_REG_SP, SW64_BPF_REG_SP, ctx->stack_size), ctx); + + /* Restore callee-saved registers */ + emit(SW64_BPF_LDL(SW64_BPF_REG_RA, SW64_BPF_REG_SP, 0), ctx); + emit(SW64_BPF_LDL(fp, SW64_BPF_REG_SP, 8), ctx); + emit(SW64_BPF_LDL(r6, SW64_BPF_REG_SP, 16), ctx); + emit(SW64_BPF_LDL(r7, SW64_BPF_REG_SP, 24), ctx); + emit(SW64_BPF_LDL(r8, SW64_BPF_REG_SP, 32), ctx); + emit(SW64_BPF_LDL(r9, SW64_BPF_REG_SP, 40), ctx); + emit(SW64_BPF_LDL(tcc, SW64_BPF_REG_SP, 48), ctx); + emit(SW64_BPF_LDL(SW64_BPF_REG_GP, SW64_BPF_REG_SP, 56), ctx); + emit(SW64_BPF_LDI(SW64_BPF_REG_SP, SW64_BPF_REG_SP, 64), ctx); + + /* Return */ + emit(SW64_BPF_RET(SW64_BPF_REG_RA), ctx); +} + +static int emit_bpf_tail_call(struct jit_ctx *ctx) +{ + /* bpf_tail_call(void *ctx, struct bpf_map *prog_array_map, u32 index) */ + const u8 r2 = bpf2sw64[BPF_REG_2]; /* struct bpf_array *array */ + const u8 r3 = bpf2sw64[BPF_REG_3]; /* u32 index */ + + const u8 tmp = get_tmp_reg(ctx); + const u8 prg = get_tmp_reg(ctx); + const u8 tcc = bpf2sw64[TCALL_CNT]; + u64 offset; + static int out_idx; +#define out_offset (ctx->image ? (out_idx - ctx->idx - 1) : 0) + + /* if (index >= array->map.max_entries) + * goto out; + */ + offset = offsetof(struct bpf_array, map.max_entries); + emit_sw64_ldu64(tmp, offset, ctx); + emit(SW64_BPF_ADDL_REG(r2, tmp, tmp), ctx); /* tmp = r2 + tmp = &map.max_entries */ + emit(SW64_BPF_LDW(tmp, tmp, 0), ctx); /* tmp = *tmp = map.max_entries */ + emit(SW64_BPF_ZAP_IMM(tmp, 0xf0, tmp), ctx); /* map.max_entries is u32 */ + emit(SW64_BPF_ZAP_IMM(r3, 0xf0, r3), ctx); /* index is u32 */ + emit(SW64_BPF_CMPULE_REG(tmp, r3, tmp), ctx); + emit(SW64_BPF_BNE(tmp, out_offset), ctx); + + /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) + * goto out; + * tail_call_cnt++; + */ + emit_sw64_ldu64(tmp, MAX_TAIL_CALL_CNT, ctx); + emit(SW64_BPF_CMPULT_REG(tmp, tcc, tmp), ctx); + emit(SW64_BPF_BNE(tmp, out_offset), ctx); + emit(SW64_BPF_ADDL_IMM(tcc, 1, tcc), ctx); + + /* prog = array->ptrs[index]; + * if (prog == NULL) + * goto out; + */ + offset = offsetof(struct bpf_array, ptrs); + emit_sw64_ldu64(tmp, offset, ctx); + emit(SW64_BPF_ADDL_REG(r2, tmp, tmp), ctx); /* tmp = r2 + tmp = &ptrs[0] */ + emit(SW64_BPF_SLL_IMM(r3, 3, prg), ctx); /* prg = r3 * 8, each entry is a pointer */ + emit(SW64_BPF_ADDL_REG(tmp, prg, prg), ctx); /* prg = tmp + prg = &ptrs[index] */ + emit(SW64_BPF_LDL(prg, prg, 0), ctx); /* prg = *prg = ptrs[index] = prog */ + emit(SW64_BPF_BEQ(prg, out_offset), ctx); + + /* goto *(prog->bpf_func + prologue_offset); */ + offset = offsetof(struct bpf_prog, bpf_func); + emit_sw64_ldu64(tmp, offset, ctx); + emit(SW64_BPF_ADDL_REG(prg, tmp, tmp), ctx); /* tmp = prg + tmp = &bpf_func */ + emit(SW64_BPF_LDL(tmp, tmp, 0), ctx); /* tmp = *tmp = bpf_func */ + emit(SW64_BPF_BEQ(tmp, out_offset), ctx); + emit(SW64_BPF_LDI(tmp, tmp, sizeof(u32) * PROLOGUE_OFFSET), ctx); + emit(SW64_BPF_LDI(SW64_BPF_REG_SP, SW64_BPF_REG_SP, ctx->stack_size), ctx); + emit(SW64_BPF_JMP(SW64_BPF_REG_ZR, tmp), ctx); + + put_tmp_reg(ctx); + put_tmp_reg(ctx); + + /* out */ + if (ctx->image == NULL) + out_idx = ctx->idx; + if (ctx->image != NULL && out_idx <= 0) + return -1; +#undef out_offset + return 0; +} + +/* For accesses to BTF pointers, add an entry to the exception table */ +static int add_exception_handler(const struct bpf_insn *insn, + struct jit_ctx *ctx, + int dst_reg) +{ + off_t offset; + unsigned long pc; + struct exception_table_entry *ex; + + if (!ctx->image) + /* First pass */ + return 0; + + if (!ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM) + return 0; + + if (WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries)) + return -EINVAL; + + ex = &ctx->prog->aux->extable[ctx->exentry_idx]; + pc = (unsigned long)&ctx->image[ctx->idx - 1]; + + offset = (long)&ex->insn - pc; + ex->insn = offset; + + ex->fixup.bits.nextinsn = sizeof(u32); + ex->fixup.bits.valreg = dst_reg; + ex->fixup.bits.errreg = SW64_BPF_REG_ZR; + + ctx->exentry_idx++; + return 0; +} + +/* JITs an eBPF instruction. + * Returns: + * 0 - successfully JITed an 8-byte eBPF instruction. + * >0 - successfully JITed a 16-byte eBPF instruction. + * <0 - failed to JIT. + */ +static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) +{ + const u8 code = insn->code; + u8 dst = bpf2sw64[insn->dst_reg]; + u8 src = bpf2sw64[insn->src_reg]; + const u8 tmp1 __maybe_unused = get_tmp_reg(ctx); + const u8 tmp2 __maybe_unused = get_tmp_reg(ctx); + const s16 off = insn->off; + const s32 imm = insn->imm; + const int bpf_idx = insn - ctx->prog->insnsi; + s32 jmp_offset; + u64 func; + struct bpf_insn insn1; + u64 imm64; + int ret; + + switch (code) { + case BPF_ALU | BPF_MOV | BPF_X: + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_MOV | BPF_X: + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, src, dst), ctx); + break; + case BPF_ALU | BPF_ADD | BPF_X: + emit(SW64_BPF_ADDW_REG(dst, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_ADD | BPF_X: + emit(SW64_BPF_ADDL_REG(dst, src, dst), ctx); + break; + case BPF_ALU | BPF_SUB | BPF_X: + emit(SW64_BPF_SUBW_REG(dst, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_SUB | BPF_X: + emit(SW64_BPF_SUBL_REG(dst, src, dst), ctx); + break; + case BPF_ALU | BPF_MUL | BPF_X: + emit(SW64_BPF_MULW_REG(dst, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_MUL | BPF_X: + emit(SW64_BPF_MULL_REG(dst, src, dst), ctx); + break; + case BPF_ALU | BPF_DIV | BPF_X: + emit_sw64_divmod(dst, src, ctx, code); + break; + case BPF_ALU64 | BPF_DIV | BPF_X: + emit_sw64_divmod(dst, src, ctx, code); + break; + case BPF_ALU | BPF_MOD | BPF_X: + emit_sw64_divmod(dst, src, ctx, code); + break; + case BPF_ALU64 | BPF_MOD | BPF_X: + emit_sw64_divmod(dst, src, ctx, code); + break; + case BPF_ALU | BPF_LSH | BPF_X: + emit(SW64_BPF_SLL_REG(dst, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_LSH | BPF_X: + emit(SW64_BPF_SLL_REG(dst, src, dst), ctx); + break; + case BPF_ALU | BPF_RSH | BPF_X: + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + fallthrough; + case BPF_ALU64 | BPF_RSH | BPF_X: + emit(SW64_BPF_SRL_REG(dst, src, dst), ctx); + break; + case BPF_ALU | BPF_ARSH | BPF_X: + emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, dst, dst), ctx); + emit(SW64_BPF_SRA_REG(dst, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_ARSH | BPF_X: + emit(SW64_BPF_SRA_REG(dst, src, dst), ctx); + break; + case BPF_ALU | BPF_AND | BPF_X: + emit(SW64_BPF_AND_REG(dst, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_AND | BPF_X: + emit(SW64_BPF_AND_REG(dst, src, dst), ctx); + break; + case BPF_ALU | BPF_OR | BPF_X: + emit(SW64_BPF_BIS_REG(dst, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_OR | BPF_X: + emit(SW64_BPF_BIS_REG(dst, src, dst), ctx); + break; + case BPF_ALU | BPF_XOR | BPF_X: + emit(SW64_BPF_XOR_REG(dst, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_XOR | BPF_X: + emit(SW64_BPF_XOR_REG(dst, src, dst), ctx); + break; + case BPF_ALU | BPF_NEG: + emit(SW64_BPF_SUBW_REG(SW64_BPF_REG_ZR, dst, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_NEG: + emit(SW64_BPF_SUBL_REG(SW64_BPF_REG_ZR, dst, dst), ctx); + break; + case BPF_ALU | BPF_END | BPF_TO_LE: + switch (imm) { + case 16: + emit(SW64_BPF_ZAPNOT_IMM(dst, 0x3, dst), ctx); + break; + case 32: + emit(SW64_BPF_ZAPNOT_IMM(dst, 0xf, dst), ctx); + break; + case 64: + break; + default: + pr_err("eBPF JIT %s[%d]: BPF_TO_LE unknown size\n", + current->comm, current->pid); + return -EINVAL; + } + break; + case BPF_ALU | BPF_END | BPF_TO_BE: + switch (imm) { + case 16: + emit_sw64_htobe16(dst, ctx); + break; + case 32: + emit_sw64_htobe32(dst, ctx); + break; + case 64: + emit_sw64_htobe64(dst, ctx); + break; + default: + pr_err("eBPF JIT %s[%d]: BPF_TO_BE unknown size\n", + current->comm, current->pid); + return -EINVAL; + } + break; + + case BPF_ALU | BPF_MOV | BPF_K: + if (imm >= S16_MIN && imm <= S16_MAX) + emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); + else + emit_sw64_ldu32(dst, imm, ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_MOV | BPF_K: + if (imm >= S16_MIN && imm <= S16_MAX) + emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); + else + emit_sw64_lds32(dst, imm, ctx); + break; + case BPF_ALU | BPF_ADD | BPF_K: + if (imm >= S16_MIN && imm <= S16_MAX) { + emit(SW64_BPF_LDI(dst, dst, imm), ctx); + } else { + emit_sw64_ldu32(tmp1, imm, ctx); + emit(SW64_BPF_ADDW_REG(dst, tmp1, dst), ctx); + } + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_ADD | BPF_K: + if (imm >= S16_MIN && imm <= S16_MAX) { + emit(SW64_BPF_LDI(dst, dst, imm), ctx); + } else { + emit_sw64_lds32(tmp1, imm, ctx); + emit(SW64_BPF_ADDL_REG(dst, tmp1, dst), ctx); + } + break; + case BPF_ALU | BPF_SUB | BPF_K: + if (imm >= -S16_MAX && imm <= -S16_MIN) { + emit(SW64_BPF_LDI(dst, dst, -imm), ctx); + } else { + emit_sw64_ldu32(tmp1, imm, ctx); + emit(SW64_BPF_SUBL_REG(dst, tmp1, dst), ctx); + } + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_SUB | BPF_K: + if (imm >= -S16_MAX && imm <= -S16_MIN) { + emit(SW64_BPF_LDI(dst, dst, -imm), ctx); + } else { + emit_sw64_lds32(tmp1, imm, ctx); + emit(SW64_BPF_SUBL_REG(dst, tmp1, dst), ctx); + } + break; + case BPF_ALU | BPF_MUL | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_MULL_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_ldu32(tmp1, imm, ctx); + emit(SW64_BPF_MULL_REG(dst, tmp1, dst), ctx); + } + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_MUL | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_MULL_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_lds32(tmp1, imm, ctx); + emit(SW64_BPF_MULL_REG(dst, tmp1, dst), ctx); + } + break; + case BPF_ALU | BPF_DIV | BPF_K: + emit_sw64_ldu32(tmp1, imm, ctx); + emit_sw64_divmod(dst, tmp1, ctx, code); + break; + case BPF_ALU64 | BPF_DIV | BPF_K: + emit_sw64_lds32(tmp1, imm, ctx); + emit_sw64_divmod(dst, tmp1, ctx, code); + break; + case BPF_ALU | BPF_MOD | BPF_K: + emit_sw64_ldu32(tmp1, imm, ctx); + emit_sw64_divmod(dst, tmp1, ctx, code); + break; + case BPF_ALU64 | BPF_MOD | BPF_K: + emit_sw64_lds32(tmp1, imm, ctx); + emit_sw64_divmod(dst, tmp1, ctx, code); + break; + case BPF_ALU | BPF_LSH | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_SLL_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_ldu32(tmp1, imm, ctx); + emit(SW64_BPF_SLL_REG(dst, tmp1, dst), ctx); + } + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_LSH | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_SLL_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_lds32(tmp1, imm, ctx); + emit(SW64_BPF_SLL_REG(dst, tmp1, dst), ctx); + } + break; + case BPF_ALU | BPF_RSH | BPF_K: + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_SRL_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_ldu32(tmp1, imm, ctx); + emit(SW64_BPF_SRL_REG(dst, tmp1, dst), ctx); + } + break; + case BPF_ALU64 | BPF_RSH | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_SRL_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_lds32(tmp1, imm, ctx); + emit(SW64_BPF_SRL_REG(dst, tmp1, dst), ctx); + } + break; + case BPF_ALU | BPF_ARSH | BPF_K: + emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, dst, dst), ctx); + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_SRA_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_ldu32(tmp1, imm, ctx); + emit(SW64_BPF_SRA_REG(dst, tmp1, dst), ctx); + } + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_ARSH | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_SRA_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_lds32(tmp1, imm, ctx); + emit(SW64_BPF_SRA_REG(dst, tmp1, dst), ctx); + } + break; + case BPF_ALU | BPF_AND | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_AND_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_ldu32(tmp1, imm, ctx); + emit(SW64_BPF_AND_REG(dst, tmp1, dst), ctx); + } + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_AND | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_AND_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_lds32(tmp1, imm, ctx); + emit(SW64_BPF_AND_REG(dst, tmp1, dst), ctx); + } + break; + case BPF_ALU | BPF_OR | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_BIS_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_ldu32(tmp1, imm, ctx); + emit(SW64_BPF_BIS_REG(dst, tmp1, dst), ctx); + } + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_OR | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_BIS_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_lds32(tmp1, imm, ctx); + emit(SW64_BPF_BIS_REG(dst, tmp1, dst), ctx); + } + break; + case BPF_ALU | BPF_XOR | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_XOR_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_ldu32(tmp1, imm, ctx); + emit(SW64_BPF_XOR_REG(dst, tmp1, dst), ctx); + } + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_ALU64 | BPF_XOR | BPF_K: + if (imm >= 0 && imm <= U8_MAX) { + emit(SW64_BPF_XOR_IMM(dst, imm, dst), ctx); + } else { + emit_sw64_lds32(tmp1, imm, ctx); + emit(SW64_BPF_XOR_REG(dst, tmp1, dst), ctx); + } + break; + + case BPF_JMP | BPF_JA: + jmp_offset = bpf2sw64_offset(bpf_idx, off, ctx); + if (jmp_offset >= -0x100000 && jmp_offset <= 0xfffff) { + emit(SW64_BPF_BR(SW64_BPF_REG_ZR, jmp_offset), ctx); + } else { + pr_err("eBPF JIT %s[%d]: BPF_JMP out of range, %d instructions\n", + current->comm, current->pid, jmp_offset); + return -EINVAL; + } + break; + + case BPF_JMP32 | BPF_JEQ | BPF_X: + case BPF_JMP32 | BPF_JGT | BPF_X: + case BPF_JMP32 | BPF_JLT | BPF_X: + case BPF_JMP32 | BPF_JGE | BPF_X: + case BPF_JMP32 | BPF_JLE | BPF_X: + case BPF_JMP32 | BPF_JNE | BPF_X: + case BPF_JMP32 | BPF_JSGT | BPF_X: + case BPF_JMP32 | BPF_JSLT | BPF_X: + case BPF_JMP32 | BPF_JSGE | BPF_X: + case BPF_JMP32 | BPF_JSLE | BPF_X: + case BPF_JMP32 | BPF_JSET | BPF_X: + emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, src, tmp1), ctx); + src = tmp1; + emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, dst, tmp2), ctx); + dst = tmp2; + fallthrough; + case BPF_JMP | BPF_JEQ | BPF_X: + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JLT | BPF_X: + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JLE | BPF_X: + case BPF_JMP | BPF_JNE | BPF_X: + case BPF_JMP | BPF_JSGT | BPF_X: + case BPF_JMP | BPF_JSLT | BPF_X: + case BPF_JMP | BPF_JSGE | BPF_X: + case BPF_JMP | BPF_JSLE | BPF_X: + case BPF_JMP | BPF_JSET | BPF_X: + switch (BPF_OP(code)) { + case BPF_JEQ: + emit(SW64_BPF_CMPEQ_REG(dst, src, tmp1), ctx); + break; + case BPF_JGT: + emit(SW64_BPF_CMPULT_REG(src, dst, tmp1), ctx); + break; + case BPF_JLT: + emit(SW64_BPF_CMPULT_REG(dst, src, tmp1), ctx); + break; + case BPF_JGE: + emit(SW64_BPF_CMPULE_REG(src, dst, tmp1), ctx); + break; + case BPF_JLE: + emit(SW64_BPF_CMPULE_REG(dst, src, tmp1), ctx); + break; + case BPF_JNE: + emit(SW64_BPF_CMPEQ_REG(dst, src, tmp1), ctx); + emit(SW64_BPF_XOR_IMM(tmp1, 1, tmp1), ctx); + break; + case BPF_JSGT: + emit(SW64_BPF_CMPLT_REG(src, dst, tmp1), ctx); + break; + case BPF_JSLT: + emit(SW64_BPF_CMPLT_REG(dst, src, tmp1), ctx); + break; + case BPF_JSGE: + emit(SW64_BPF_CMPLE_REG(src, dst, tmp1), ctx); + break; + case BPF_JSLE: + emit(SW64_BPF_CMPLE_REG(dst, src, tmp1), ctx); + break; + case BPF_JSET: + emit(SW64_BPF_AND_REG(dst, src, tmp1), ctx); + break; + } + jmp_offset = bpf2sw64_offset(bpf_idx, off, ctx); + if (jmp_offset >= -0x100000 && jmp_offset <= 0xfffff) { + emit(SW64_BPF_BNE(tmp1, jmp_offset), ctx); + } else { + pr_err("eBPF JIT %s[%d]: BPF_JMP out of range, %d instructions\n", + current->comm, current->pid, jmp_offset); + return -EINVAL; + } + break; + + case BPF_JMP32 | BPF_JEQ | BPF_K: + case BPF_JMP32 | BPF_JGT | BPF_K: + case BPF_JMP32 | BPF_JLT | BPF_K: + case BPF_JMP32 | BPF_JGE | BPF_K: + case BPF_JMP32 | BPF_JLE | BPF_K: + case BPF_JMP32 | BPF_JNE | BPF_K: + case BPF_JMP32 | BPF_JSGT | BPF_K: + case BPF_JMP32 | BPF_JSLT | BPF_K: + case BPF_JMP32 | BPF_JSGE | BPF_K: + case BPF_JMP32 | BPF_JSLE | BPF_K: + case BPF_JMP32 | BPF_JSET | BPF_K: + emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, dst, tmp2), ctx); + dst = tmp2; + fallthrough; + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP | BPF_JLT | BPF_K: + case BPF_JMP | BPF_JGE | BPF_K: + case BPF_JMP | BPF_JLE | BPF_K: + case BPF_JMP | BPF_JNE | BPF_K: + case BPF_JMP | BPF_JSGT | BPF_K: + case BPF_JMP | BPF_JSLT | BPF_K: + case BPF_JMP | BPF_JSGE | BPF_K: + case BPF_JMP | BPF_JSLE | BPF_K: + case BPF_JMP | BPF_JSET | BPF_K: + emit_sw64_lds32(tmp1, imm, ctx); + switch (BPF_OP(code)) { + case BPF_JEQ: + emit(SW64_BPF_CMPEQ_REG(dst, tmp1, tmp2), ctx); + break; + case BPF_JGT: + emit(SW64_BPF_CMPULT_REG(tmp1, dst, tmp2), ctx); + break; + case BPF_JLT: + emit(SW64_BPF_CMPULT_REG(dst, tmp1, tmp2), ctx); + break; + case BPF_JGE: + emit(SW64_BPF_CMPULE_REG(tmp1, dst, tmp2), ctx); + break; + case BPF_JLE: + emit(SW64_BPF_CMPULE_REG(dst, tmp1, tmp2), ctx); + break; + case BPF_JNE: + emit(SW64_BPF_CMPEQ_REG(dst, tmp1, tmp2), ctx); + emit(SW64_BPF_XOR_IMM(tmp2, 1, tmp2), ctx); + break; + case BPF_JSGT: + emit(SW64_BPF_CMPLT_REG(tmp1, dst, tmp2), ctx); + break; + case BPF_JSLT: + emit(SW64_BPF_CMPLT_REG(dst, tmp1, tmp2), ctx); + break; + case BPF_JSGE: + emit(SW64_BPF_CMPLE_REG(tmp1, dst, tmp2), ctx); + break; + case BPF_JSLE: + emit(SW64_BPF_CMPLE_REG(dst, tmp1, tmp2), ctx); + break; + case BPF_JSET: + emit(SW64_BPF_AND_REG(dst, tmp1, tmp2), ctx); + break; + } + jmp_offset = bpf2sw64_offset(bpf_idx, off, ctx); + if (jmp_offset >= -0x100000 && jmp_offset <= 0xfffff) { + emit(SW64_BPF_BNE(tmp2, jmp_offset), ctx); + } else { + pr_err("eBPF JIT %s[%d]: BPF_JMP out of range, %d instructions\n", + current->comm, current->pid, jmp_offset); + return -EINVAL; + } + break; + + case BPF_JMP | BPF_CALL: + func = (u64)__bpf_call_base + imm; + if ((func & ~(KERNEL_IMAGE_SIZE - 1)) != __START_KERNEL_map) + /* calling bpf program, switch to vmalloc addr */ + func = (func & U32_MAX) | VMALLOC_START; + emit_sw64_ldu64(SW64_BPF_REG_PV, func, ctx); + emit(SW64_BPF_CALL(SW64_BPF_REG_RA, SW64_BPF_REG_PV), ctx); + break; + + case BPF_JMP | BPF_TAIL_CALL: + if (emit_bpf_tail_call(ctx)) + return -EFAULT; + break; + + case BPF_JMP | BPF_EXIT: + // if this is the last bpf instruction, skip to epilogue + if (bpf_idx == ctx->prog->len - 1) + break; + jmp_offset = offset_to_epilogue(ctx) - 1; + // epilogue is always at the end, must jump forward + if (jmp_offset >= -1 && jmp_offset <= 0xfffff) { + if (ctx->image && !jmp_offset) + // if this is the last jited instruction, generate nop + emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, SW64_BPF_REG_ZR), ctx); + else + emit(SW64_BPF_BR(SW64_BPF_REG_ZR, jmp_offset), ctx); + } else { + pr_err("eBPF JIT %s[%d]: BPF_EXIT out of range, %d instructions\n", + current->comm, current->pid, jmp_offset); + return -EINVAL; + } + break; + + case BPF_LD | BPF_IMM | BPF_DW: + insn1 = insn[1]; + imm64 = ((u64)insn1.imm << 32) | (u32)imm; + emit_sw64_ldu64(dst, imm64, ctx); + put_tmp_reg(ctx); + put_tmp_reg(ctx); + return 1; + + /* LDX: dst = *(size *)(src + off) */ + case BPF_LDX | BPF_MEM | BPF_W: + case BPF_LDX | BPF_MEM | BPF_H: + case BPF_LDX | BPF_MEM | BPF_B: + case BPF_LDX | BPF_MEM | BPF_DW: + case BPF_LDX | BPF_PROBE_MEM | BPF_DW: + case BPF_LDX | BPF_PROBE_MEM | BPF_W: + case BPF_LDX | BPF_PROBE_MEM | BPF_H: + case BPF_LDX | BPF_PROBE_MEM | BPF_B: + switch (BPF_SIZE(code)) { + case BPF_W: + emit(SW64_BPF_LDW(dst, src, off), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + break; + case BPF_H: + emit(SW64_BPF_LDHU(dst, src, off), ctx); + break; + case BPF_B: + emit(SW64_BPF_LDBU(dst, src, off), ctx); + break; + case BPF_DW: + emit(SW64_BPF_LDL(dst, src, off), ctx); + break; + } + + ret = add_exception_handler(insn, ctx, dst); + if (ret) + return ret; + break; + + /* ST: *(size *)(dst + off) = imm */ + case BPF_ST | BPF_MEM | BPF_W: + case BPF_ST | BPF_MEM | BPF_H: + case BPF_ST | BPF_MEM | BPF_B: + case BPF_ST | BPF_MEM | BPF_DW: + /* Load imm to a register then store it */ + emit_sw64_lds32(tmp1, imm, ctx); + switch (BPF_SIZE(code)) { + case BPF_W: + emit(SW64_BPF_STW(tmp1, dst, off), ctx); + break; + case BPF_H: + emit(SW64_BPF_STH(tmp1, dst, off), ctx); + break; + case BPF_B: + emit(SW64_BPF_STB(tmp1, dst, off), ctx); + break; + case BPF_DW: + emit(SW64_BPF_STL(tmp1, dst, off), ctx); + break; + } + break; + + /* STX: *(size *)(dst + off) = src */ + case BPF_STX | BPF_MEM | BPF_W: + emit(SW64_BPF_STW(src, dst, off), ctx); + break; + case BPF_STX | BPF_MEM | BPF_H: + emit(SW64_BPF_STH(src, dst, off), ctx); + break; + case BPF_STX | BPF_MEM | BPF_B: + emit(SW64_BPF_STB(src, dst, off), ctx); + break; + case BPF_STX | BPF_MEM | BPF_DW: + emit(SW64_BPF_STL(src, dst, off), ctx); + break; + + /* STX XADD: lock *(u32 *)(dst + off) += src */ + case BPF_STX | BPF_XADD | BPF_W: + emit_sw64_xadd32(src, dst, off, ctx); + break; + /* STX XADD: lock *(u64 *)(dst + off) += src */ + case BPF_STX | BPF_XADD | BPF_DW: + emit_sw64_xadd64(src, dst, off, ctx); + break; + + default: + pr_err("eBPF JIT %s[%d]: unknown opcode 0x%02x\n", + current->comm, current->pid, code); + return -EINVAL; + } + + put_tmp_reg(ctx); + put_tmp_reg(ctx); + return 0; +} + +static int build_body(struct jit_ctx *ctx) +{ + const struct bpf_prog *prog = ctx->prog; + int i; + + for (i = 0; i < prog->len; i++) { + const struct bpf_insn *insn = &prog->insnsi[i]; + int ret; + + if (ctx->image == NULL) + ctx->insn_offset[i] = ctx->idx; + ret = build_insn(insn, ctx); + if (ret < 0) + return ret; + while (ret > 0) { + i++; + if (ctx->image == NULL) + ctx->insn_offset[i] = ctx->insn_offset[i - 1]; + ret--; + } + } + + return 0; +} + +static int validate_code(struct jit_ctx *ctx) +{ + int i; + + for (i = 0; i < ctx->idx; i++) { + if (ctx->image[i] == SW64_BPF_ILLEGAL_INSN) + return -1; + } + + if (WARN_ON_ONCE(ctx->exentry_idx != ctx->prog->aux->num_exentries)) + return -1; + + return 0; +} + +static inline void bpf_flush_icache(void *start, void *end) +{ + flush_icache_range((unsigned long)start, (unsigned long)end); +} + +struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +{ + struct bpf_prog *tmp, *orig_prog = prog; + struct bpf_binary_header *header; + struct sw64_jit_data *jit_data; + bool was_classic = bpf_prog_was_classic(prog); + bool tmp_blinded = false; + bool extra_pass = false; + struct jit_ctx ctx; + int image_size, prog_size, extable_size; + u8 *image_ptr; + + if (!prog->jit_requested) + return orig_prog; + + tmp = bpf_jit_blind_constants(prog); + /* If blinding was requested and we failed during blinding, + * we must fall back to the interpreter. + */ + if (IS_ERR(tmp)) + return orig_prog; + if (tmp != prog) { + tmp_blinded = true; + prog = tmp; + } + + jit_data = prog->aux->jit_data; + if (!jit_data) { + jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); + if (!jit_data) { + prog = orig_prog; + goto out; + } + prog->aux->jit_data = jit_data; + } + if (jit_data->ctx.insn_offset) { + ctx = jit_data->ctx; + image_ptr = jit_data->image; + header = jit_data->header; + extra_pass = true; + prog_size = sizeof(u32) * ctx.idx; + goto skip_init_ctx; + } + memset(&ctx, 0, sizeof(ctx)); + ctx.prog = prog; + + ctx.insn_offset = kcalloc(prog->len + 1, sizeof(int), GFP_KERNEL); + if (ctx.insn_offset == NULL) { + prog = orig_prog; + goto out_off; + } + + /* 1. Initial fake pass to compute ctx->idx. */ + + /* Fake pass to fill in ctx->offset. */ + build_prologue(&ctx, was_classic); + + if (build_body(&ctx)) { + prog = orig_prog; + goto out_off; + } + + ctx.insn_offset[prog->len] = ctx.epilogue_offset = ctx.idx; + build_epilogue(&ctx); + + extable_size = prog->aux->num_exentries * + sizeof(struct exception_table_entry); + + /* Now we know the actual image size. */ + /* And we need extra 8 bytes for lock instructions alignment */ + prog_size = sizeof(u32) * ctx.idx + 8; + image_size = prog_size + extable_size; + header = bpf_jit_binary_alloc(image_size, &image_ptr, + sizeof(u32), jit_fill_hole); + if (header == NULL) { + prog = orig_prog; + goto out_off; + } + + /* 2. Now, the actual pass. */ + + /* lock instructions need 8-byte alignment */ + ctx.image = (u32 *)(((unsigned long)image_ptr + 7) & (~7)); + if (extable_size) + prog->aux->extable = (void *)image_ptr + prog_size; +skip_init_ctx: + ctx.idx = 0; + ctx.exentry_idx = 0; + + build_prologue(&ctx, was_classic); + + if (build_body(&ctx)) { + bpf_jit_binary_free(header); + prog = orig_prog; + goto out_off; + } + + build_epilogue(&ctx); + + /* 3. Extra pass to validate JITed code. */ + if (validate_code(&ctx)) { + bpf_jit_binary_free(header); + prog = orig_prog; + goto out_off; + } + + /* And we're done. */ + if (bpf_jit_enable > 1) + bpf_jit_dump(prog->len, prog_size, 2, ctx.image); + + bpf_flush_icache(header, ctx.image + ctx.idx); + + if (!prog->is_func || extra_pass) { + bpf_jit_binary_lock_ro(header); + } else { + jit_data->ctx = ctx; + jit_data->image = image_ptr; + jit_data->header = header; + } + prog->bpf_func = (void *)ctx.image; + prog->jited = 1; + prog->jited_len = prog_size; + if (ctx.current_tmp_reg) { + pr_err("eBPF JIT %s[%d]: unreleased temporary regsters %d\n", + current->comm, current->pid, ctx.current_tmp_reg); + } + + if (!prog->is_func || extra_pass) { +out_off: + kfree(ctx.insn_offset); + kfree(jit_data); + prog->aux->jit_data = NULL; + } +out: + if (tmp_blinded) + bpf_jit_prog_release_other(prog, prog == orig_prog ? + tmp : orig_prog); + return prog; +} -- Gitee From f3a1f348cb634f35df6e728cc10a93af89eb30ad Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:36 +0800 Subject: [PATCH 041/524] sw64: add suspend support Add suspend support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/suspend.h | 50 ++++++++++++++++ arch/sw_64/kernel/suspend.c | 57 ++++++++++++++++++ arch/sw_64/kernel/suspend_asm.S | 99 ++++++++++++++++++++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 arch/sw_64/include/asm/suspend.h create mode 100644 arch/sw_64/kernel/suspend.c create mode 100644 arch/sw_64/kernel/suspend_asm.S diff --git a/arch/sw_64/include/asm/suspend.h b/arch/sw_64/include/asm/suspend.h new file mode 100644 index 000000000000..833e27f9d5e1 --- /dev/null +++ b/arch/sw_64/include/asm/suspend.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_SUSPEND_H +#define _ASM_SW64_SUSPEND_H + +#include +#include +#include +#define SOFTINF_SLEEP_MAGIC 0x0123456789ABCDEFUL + +#ifdef CONFIG_HIBERNATION +#include +#include +#endif + +struct callee_saved_regs { + unsigned long r9; + unsigned long r10; + unsigned long r11; + unsigned long r12; + unsigned long r13; + unsigned long r14; + unsigned long r15; + unsigned long ra; +}; + +struct callee_saved_fpregs { + unsigned long f2[4]; + unsigned long f3[4]; + unsigned long f4[4]; + unsigned long f5[4]; + unsigned long f6[4]; + unsigned long f7[4]; + unsigned long f8[4]; + unsigned long f9[4]; +} __aligned(32); /* 256 bits aligned for simd */ + +struct processor_state { + struct callee_saved_regs regs; + struct callee_saved_fpregs fpregs; + unsigned long fpcr; + unsigned long ktp; +#ifdef CONFIG_HIBERNATION + unsigned long sp; + struct vcpucb vcb; +#endif +}; + +extern void sw64_suspend_deep_sleep(struct processor_state *state); +extern const struct platform_suspend_ops native_suspend_ops; +#endif /* _ASM_SW64_SUSPEND_H */ diff --git a/arch/sw_64/kernel/suspend.c b/arch/sw_64/kernel/suspend.c new file mode 100644 index 000000000000..27a240e66149 --- /dev/null +++ b/arch/sw_64/kernel/suspend.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include + +struct processor_state suspend_state; + +static int native_suspend_state_valid(suspend_state_t pm_state) +{ + switch (pm_state) { + case PM_SUSPEND_ON: + case PM_SUSPEND_STANDBY: + case PM_SUSPEND_MEM: + return 1; + default: + return 0; + } +} + +void disable_local_timer(void) +{ + wrtimer(0); +} + +extern struct pci_controller *hose_head; + +/* + * Boot Core will enter suspend stat here. + */ +void sw64_suspend_enter(void) +{ + /* boot processor will go to deep sleep mode from here + * After wake up boot processor, pc will go here + */ + disable_local_timer(); + current_thread_info()->pcb.tp = rtid(); + + sw64_suspend_deep_sleep(&suspend_state); + wrtp(current_thread_info()->pcb.tp); + + disable_local_timer(); +} + +static int native_suspend_enter(suspend_state_t state) +{ + if (is_in_guest()) + return 0; + /* processor specific suspend */ + sw64_suspend_enter(); + return 0; +} + +const struct platform_suspend_ops native_suspend_ops = { + .valid = native_suspend_state_valid, + .enter = native_suspend_enter, +}; diff --git a/arch/sw_64/kernel/suspend_asm.S b/arch/sw_64/kernel/suspend_asm.S new file mode 100644 index 000000000000..34ee349515a7 --- /dev/null +++ b/arch/sw_64/kernel/suspend_asm.S @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include + + .text + .set noat +ENTRY(sw64_suspend_deep_sleep) + /* a0 $16 will be the address of suspend_state */ + ldi $1, PSTATE_REGS($16) + stl $9, CALLEE_R9($1) + stl $10, CALLEE_R10($1) + stl $11, CALLEE_R11($1) + stl $12, CALLEE_R12($1) + stl $13, CALLEE_R13($1) + stl $14, CALLEE_R14($1) + stl $15, CALLEE_R15($1) + stl $26, CALLEE_RA($1) + /* SIMD-FP */ + ldi $1, PSTATE_FPREGS($16) + vstd $f2, CALLEE_F2($1) + vstd $f3, CALLEE_F3($1) + vstd $f4, CALLEE_F4($1) + vstd $f5, CALLEE_F5($1) + vstd $f6, CALLEE_F6($1) + vstd $f7, CALLEE_F7($1) + vstd $f8, CALLEE_F8($1) + vstd $f9, CALLEE_F9($1) + rfpcr $f0 + fstd $f0, PSTATE_FPCR($16) + stl $8, PSTATE_KTP($16) + + /* save the address of suspend_state to $18 */ + mov $16, $18 + + /* + * Now will Go to Deep Sleep + * HMcode should save pc, gp, ps, r16, r17, r18 + */ + + sys_call HMC_sleepen + sys_call HMC_whami + bis $0, $0, $16 + ldi $17, 0x2($31) + sys_call HMC_sendii + + /* wait for a while to receive interrupt */ + ldi $16, 0x1($31) + sll $16, 24, $16 +$subloop: + subl $16, 1, $16 + bis $16, $16, $16 + bis $16, $16, $16 + bne $16, $subloop + + + ldl $8, PSTATE_KTP($18) + ldi $1, PSTATE_REGS($18) + ldl $9, CALLEE_R9($1) + ldl $10, CALLEE_R10($1) + ldl $11, CALLEE_R11($1) + ldl $12, CALLEE_R12($1) + ldl $13, CALLEE_R13($1) + ldl $14, CALLEE_R14($1) + ldl $15, CALLEE_R15($1) + ldl $26, CALLEE_RA($1) + /* SIMD-FP */ + fldd $f0, PSTATE_FPCR($18) + wfpcr $f0 + fimovd $f0, $2 + and $2, 0x3, $2 + beq $2, $suspend_setfpec_0 + subl $2, 0x1, $2 + beq $2, $suspend_setfpec_1 + subl $2, 0x1, $2 + beq $2, $suspend_setfpec_2 + setfpec3 + br $suspend_setfpec_over +$suspend_setfpec_0: + setfpec0 + br $suspend_setfpec_over +$suspend_setfpec_1: + setfpec1 + br $suspend_setfpec_over +$suspend_setfpec_2: + setfpec2 +$suspend_setfpec_over: + ldi $1, PSTATE_FPREGS($18) + vldd $f2, CALLEE_F2($1) + vldd $f3, CALLEE_F3($1) + vldd $f4, CALLEE_F4($1) + vldd $f5, CALLEE_F5($1) + vldd $f6, CALLEE_F6($1) + vldd $f7, CALLEE_F7($1) + vldd $f8, CALLEE_F8($1) + vldd $f9, CALLEE_F9($1) + ret +END(sw64_suspend_deep_sleep) -- Gitee From 63d3c42d9d9d7aca0ac99e011673ed8ce4b1d057 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:13 +0800 Subject: [PATCH 042/524] sw64: add hibernation support Add hibernation support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/hibernate.c | 79 +++++++++++++++++++ arch/sw_64/kernel/hibernate_asm.S | 122 ++++++++++++++++++++++++++++++ arch/sw_64/kernel/pm.c | 18 +++++ 3 files changed, 219 insertions(+) create mode 100644 arch/sw_64/kernel/hibernate.c create mode 100644 arch/sw_64/kernel/hibernate_asm.S create mode 100644 arch/sw_64/kernel/pm.c diff --git a/arch/sw_64/kernel/hibernate.c b/arch/sw_64/kernel/hibernate.c new file mode 100644 index 000000000000..644ea8504313 --- /dev/null +++ b/arch/sw_64/kernel/hibernate.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +struct processor_state hibernate_state; +/* Defined in hibernate_asm.S */ +extern int restore_image(void); + +void save_processor_state(void) +{ + struct vcpucb *vcb = &(hibernate_state.vcb); + + vcb->ksp = rdksp(); + vcb->usp = rdusp(); + vcb->soft_tid = rtid(); + vcb->ptbr = rdptbr(); +} + +void restore_processor_state(void) +{ + struct vcpucb *vcb = &(hibernate_state.vcb); + + wrksp(vcb->ksp); + wrusp(vcb->usp); + wrtp(vcb->soft_tid); + wrptbr(vcb->ptbr); + sflush(); + tbiv(); +} + +int swsusp_arch_resume(void) +{ + restore_image(); + return 0; +} +/* References to section boundaries */ +extern const void __nosave_begin, __nosave_end; +int pfn_is_nosave(unsigned long pfn) +{ + unsigned long nosave_begin_pfn = PFN_DOWN(__pa(&__nosave_begin)); + unsigned long nosave_end_pfn = PFN_UP(__pa(&__nosave_end)); + + return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn); +} + +struct restore_data_record { + unsigned long magic; +}; + +#define RESTORE_MAGIC 0x0123456789ABCDEFUL + +/** + * arch_hibernation_header_save - populate the architecture specific part + * of a hibernation image header + * @addr: address to save the data at + */ +int arch_hibernation_header_save(void *addr, unsigned int max_size) +{ + struct restore_data_record *rdr = addr; + + if (max_size < sizeof(struct restore_data_record)) + return -EOVERFLOW; + rdr->magic = RESTORE_MAGIC; + return 0; +} + +/** + * arch_hibernation_header_restore - read the architecture specific data + * from the hibernation image header + * @addr: address to read the data from + */ +int arch_hibernation_header_restore(void *addr) +{ + struct restore_data_record *rdr = addr; + + return (rdr->magic == RESTORE_MAGIC) ? 0 : -EINVAL; +} diff --git a/arch/sw_64/kernel/hibernate_asm.S b/arch/sw_64/kernel/hibernate_asm.S new file mode 100644 index 000000000000..ff997cd76c5a --- /dev/null +++ b/arch/sw_64/kernel/hibernate_asm.S @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include + + .text + .set noat +ENTRY(swsusp_arch_suspend) + ldi $16, hibernate_state + ldi $1, PSTATE_REGS($16) + stl $9, CALLEE_R9($1) + stl $10, CALLEE_R10($1) + stl $11, CALLEE_R11($1) + stl $12, CALLEE_R12($1) + stl $13, CALLEE_R13($1) + stl $14, CALLEE_R14($1) + stl $15, CALLEE_R15($1) + stl $26, CALLEE_RA($1) + /* SIMD-FP */ + ldi $1, PSTATE_FPREGS($16) + vstd $f2, CALLEE_F2($1) + vstd $f3, CALLEE_F3($1) + vstd $f4, CALLEE_F4($1) + vstd $f5, CALLEE_F5($1) + vstd $f6, CALLEE_F6($1) + vstd $f7, CALLEE_F7($1) + vstd $f8, CALLEE_F8($1) + vstd $f9, CALLEE_F9($1) + rfpcr $f0 + fstd $f0, PSTATE_FPCR($16) + + stl $8, PSTATE_KTP($16) + stl sp, PSTATE_SP($16) + call swsusp_save + ldi $16, hibernate_state + ldi $1, PSTATE_REGS($16) + ldl $26, CALLEE_RA($1) + + /* save current_thread_info()->pcbb */ + ret +END(swsusp_arch_suspend) + +ENTRY(restore_image) + /* prepare to copy image data to their original locations */ + ldi t0, restore_pblist + ldl t0, 0(t0) +$loop: + beq t0, $done + + /* get addresses from the pbe and copy the page */ + ldl t1, PBE_ADDR(t0) /* source */ + ldl t2, PBE_ORIG_ADDR(t0) /* destination */ + ldi t3, PAGE_SIZE + addl t1, t3, t3 +$cpyloop: + ldl t8, 0(t1) + stl t8, 0(t2) + addl t1, 8, t1 + addl t2, 8, t2 + cmpeq t1, t3, t4 + beq t4, $cpyloop + + /* progress to the next pbe */ + ldl t0, PBE_NEXT(t0) + bne t0, $loop +$done: + + /* tell the hibernation core that we've just restored the memory */ + ldi $0, in_suspend + stl $31, 0($0) + + ldi $16, hibernate_state + ldi $1, PSTATE_REGS($16) + + ldl $9, CALLEE_R9($1) + ldl $10, CALLEE_R10($1) + ldl $11, CALLEE_R11($1) + ldl $12, CALLEE_R12($1) + ldl $13, CALLEE_R13($1) + ldl $14, CALLEE_R14($1) + ldl $15, CALLEE_R15($1) + ldl $26, CALLEE_RA($1) + /* SIMD-FP */ + fldd $f0, PSTATE_FPCR($16) + wfpcr $f0 + fimovd $f0, $2 + and $2, 0x3, $2 + beq $2, $hibernate_setfpec_0 + subl $2, 0x1, $2 + beq $2, $hibernate_setfpec_1 + subl $2, 0x1, $2 + beq $2, $hibernate_setfpec_2 + setfpec3 + br $hibernate_setfpec_over +$hibernate_setfpec_0: + setfpec0 + br $hibernate_setfpec_over +$hibernate_setfpec_1: + setfpec1 + br $hibernate_setfpec_over +$hibernate_setfpec_2: + setfpec2 +$hibernate_setfpec_over: + ldi $1, PSTATE_FPREGS($16) + vldd $f2, CALLEE_F2($1) + vldd $f3, CALLEE_F3($1) + vldd $f4, CALLEE_F4($1) + vldd $f5, CALLEE_F5($1) + vldd $f6, CALLEE_F6($1) + vldd $f7, CALLEE_F7($1) + vldd $f8, CALLEE_F8($1) + vldd $f9, CALLEE_F9($1) + + ldl sp, PSTATE_SP($16) + ldl $8, PSTATE_KTP($16) + sys_call HMC_wrktp + + ldi $0, 0($31) + + ret +END(restore_image) diff --git a/arch/sw_64/kernel/pm.c b/arch/sw_64/kernel/pm.c new file mode 100644 index 000000000000..f0a35e5d0486 --- /dev/null +++ b/arch/sw_64/kernel/pm.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include + +struct syscore_ops io_syscore_ops; + +static int __init sw64_pm_init(void) +{ +#ifdef CONFIG_SUSPEND + suspend_set_ops(&native_suspend_ops); +#endif + register_syscore_ops(&io_syscore_ops); + + return 0; +} +device_initcall(sw64_pm_init); -- Gitee From bc101791d9f803fc437242883419387bd2fda8ee Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:09 +0800 Subject: [PATCH 043/524] sw64: add ftrace support Add ftrace support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/ftrace.h | 44 ++++ arch/sw_64/include/asm/livepatch.h | 22 ++ arch/sw_64/kernel/entry-ftrace.S | 326 +++++++++++++++++++++++++++++ arch/sw_64/kernel/ftrace.c | 176 ++++++++++++++++ 4 files changed, 568 insertions(+) create mode 100644 arch/sw_64/include/asm/ftrace.h create mode 100644 arch/sw_64/include/asm/livepatch.h create mode 100644 arch/sw_64/kernel/entry-ftrace.S create mode 100644 arch/sw_64/kernel/ftrace.c diff --git a/arch/sw_64/include/asm/ftrace.h b/arch/sw_64/include/asm/ftrace.h new file mode 100644 index 000000000000..7ed6e3c06a33 --- /dev/null +++ b/arch/sw_64/include/asm/ftrace.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arch/sw_64/include/asm/ftrace.h + * + * Copyright (C) 2019, serveros, linyue + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef _ASM_SW64_FTRACE_H +#define _ASM_SW64_FTRACE_H + +#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_INSN_SIZE 20 /* 5 * SW64_INSN_SIZE */ +#define MCOUNT_LDGP_SIZE 8 /* 2 * SW64_INSN_SIZE */ + +#define ARCH_SUPPORTS_FTRACE_OPS 1 + +#ifndef __ASSEMBLY__ +#include +#include + + +extern void _mcount(unsigned long); + +struct dyn_arch_ftrace { + /* No extra data needed for sw64 */ +}; + +extern unsigned long ftrace_graph_call; + + +static inline unsigned long ftrace_call_adjust(unsigned long addr) +{ + /* + * addr is the address of the mcount call instruction. + * recordmcount does the necessary offset calculation. + */ + return addr; +} + +#endif /* ifndef __ASSEMBLY__ */ +#endif /* _ASM_SW64_FTRACE_H */ diff --git a/arch/sw_64/include/asm/livepatch.h b/arch/sw_64/include/asm/livepatch.h new file mode 100644 index 000000000000..1feec0f6be76 --- /dev/null +++ b/arch/sw_64/include/asm/livepatch.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * livepatch.h - sw64-specific Kernel Live Patching Core + */ + +#ifndef _ASM_SW64_LIVEPATCH_H +#define _ASM_SW64_LIVEPATCH_H + +#include + +static inline int klp_check_compiler_support(void) +{ + return 0; +} + +static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip) +{ + regs->regs[27] = ip; + regs->regs[28] = ip; +} + +#endif /* _ASM_SW64_LIVEPATCH_H */ diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S new file mode 100644 index 000000000000..73e8e043fc9d --- /dev/null +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -0,0 +1,326 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arch/sw_64/kernel/entry-ftrace.S + * + * Author: linyue + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ +#include +#include +#include + + .text + .set noat + .align 4 + +#define FTRACE_SP_OFF 0x50 + .macro mcount_enter + subl $sp, FTRACE_SP_OFF, $sp + stl $16, 0($sp) + stl $17, 0x8($sp) + stl $18, 0x10($sp) + stl $26, 0x18($sp) +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + stl $9, 0x20($sp) +#endif + stl $28, 0x28($sp) + stl $29, 0x30($sp) + stl $19, 0x38($sp) + stl $20, 0x40($sp) + stl $21, 0x48($sp) + .endm + + .macro mcount_end + ldl $16, 0($sp) + ldl $17, 0x8($sp) + ldl $18, 0x10($sp) + ldl $26, 0x18($sp) +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + ldl $9, 0x20($sp) +#endif + ldl $28, 0x28($sp) + ldl $29, 0x30($sp) + ldl $19, 0x38($sp) + ldl $20, 0x40($sp) + ldl $21, 0x48($sp) + addl $sp, FTRACE_SP_OFF, $sp + .endm + + .macro RESTORE_GRAPH_ARGS + ldi $16, 0x18($sp) /* &ra */ + bis $31, $9, $17 /* pc */ + #ifdef HAVE_FUNCTION_GRAPH_FP_TEST + bis $31, $15, $18 /* fp */ + #endif + .endm + + .macro SAVE_PT_REGS + ldi $sp, -PT_REGS_SIZE($sp) + stl $0, PT_REGS_R0($sp) + stl $1, PT_REGS_R1($sp) + stl $2, PT_REGS_R2($sp) + stl $3, PT_REGS_R3($sp) + stl $4, PT_REGS_R4($sp) + stl $5, PT_REGS_R5($sp) + stl $6, PT_REGS_R6($sp) + stl $7, PT_REGS_R7($sp) + stl $8, PT_REGS_R8($sp) + stl $9, PT_REGS_R9($sp) + stl $10, PT_REGS_R10($sp) + stl $11, PT_REGS_R11($sp) + stl $12, PT_REGS_R12($sp) + stl $13, PT_REGS_R13($sp) + stl $14, PT_REGS_R14($sp) + stl $15, PT_REGS_R15($sp) + stl $16, PT_REGS_R16($sp) + stl $17, PT_REGS_R17($sp) + stl $18, PT_REGS_R18($sp) + stl $19, PT_REGS_R19($sp) + stl $20, PT_REGS_R20($sp) + stl $21, PT_REGS_R21($sp) + stl $22, PT_REGS_R22($sp) + stl $23, PT_REGS_R23($sp) + stl $24, PT_REGS_R24($sp) + stl $25, PT_REGS_R25($sp) + stl $26, PT_REGS_R26($sp) + stl $27, PT_REGS_R27($sp) + stl $28, PT_REGS_R28($sp) + stl $29, PT_REGS_GP($sp) + ldi $0, PT_REGS_SIZE($sp) + stl $0, PT_REGS_SP($sp) + .endm + + .macro RESTORE_PT_REGS + ldl $0, PT_REGS_R0($sp) + ldl $1, PT_REGS_R1($sp) + ldl $2, PT_REGS_R2($sp) + ldl $3, PT_REGS_R3($sp) + ldl $4, PT_REGS_R4($sp) + ldl $5, PT_REGS_R5($sp) + ldl $6, PT_REGS_R6($sp) + ldl $7, PT_REGS_R7($sp) + ldl $8, PT_REGS_R8($sp) + ldl $9, PT_REGS_R9($sp) + ldl $10, PT_REGS_R10($sp) + ldl $11, PT_REGS_R11($sp) + ldl $12, PT_REGS_R12($sp) + ldl $13, PT_REGS_R13($sp) + ldl $14, PT_REGS_R14($sp) + ldl $15, PT_REGS_R15($sp) + ldl $16, PT_REGS_R16($sp) + ldl $17, PT_REGS_R17($sp) + ldl $18, PT_REGS_R18($sp) + ldl $19, PT_REGS_R19($sp) + ldl $20, PT_REGS_R20($sp) + ldl $21, PT_REGS_R21($sp) + ldl $22, PT_REGS_R22($sp) + ldl $23, PT_REGS_R23($sp) + ldl $24, PT_REGS_R24($sp) + ldl $25, PT_REGS_R25($sp) + ldl $26, PT_REGS_R26($sp) + ldl $27, PT_REGS_R27($sp) + ldl $28, PT_REGS_R28($sp) + ldl $29, PT_REGS_GP($sp) + ldi $sp, PT_REGS_SIZE($sp) + .endm + + .macro RESTORE_GRAPH_REG_ARGS + ldi $16, PT_REGS_R26($sp) + bis $31, $9, $17 +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + bis $31, $15, $18 +#endif + .endm + + /* save return value regs*/ + .macro save_return_regs + subl $sp, 0x8, $sp + stl $0, 0x0($sp) + .endm + + /* restore return value regs*/ + .macro restore_return_regs + ldl $0, 0x0($sp) + addl $sp, 0x8, $sp + .endm + + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * void ftrace_graph_caller(void) + * + * Called from ftrace_caller() or ftrace_regs_caller() when function_graph + * tracer is selected. + * This function prepare_ftrace_return() fakes ra's value on the call + * stack in order to intercept instrumented function's return path and + * run return_to_handler() later on its exit. + */ + +ENTRY(ftrace_graph_caller) + ldgp $29, 0($27) + ldi $sp, -16($sp) + stl $26, 0($sp) + stl $15, 8($sp) + bis $31, $sp, $15 + + ldi $27, prepare_ftrace_return +ftrace_graph_call: + .global ftrace_graph_call + /* + * Calling ftrace_enable/disable_ftrace_graph_caller would overwrite + * the nop below. + */ + nop /* nop, or call prepare_ftrace_return() */ + + ldl $26, 0($sp) + ldl $15, 8($sp) + ldi $sp, 16($sp) + ret $31, ($26), 1 +ENDPROC(ftrace_graph_caller) + +/* + * void return_to_handler(void) + * + * Run ftrace_return_to_handler() before going back to parent. + * @fp is checked against the value passed by ftrace_graph_caller() + * only when HAVE_FUNCTION_GRAPH_FP_TEST is enabled. + * + * It is run by "ret" instruction which does not modify $27, so it + * has to recaculate $27 before ldgp. + */ +ENTRY(return_to_handler) + br $27, 1f +1: ldgp $29, 0($27) + save_return_regs + bis $31, $15, $16 /* parent's fp */ + ldi $27, ftrace_return_to_handler + call $26, ($27) + bis $31, $0, $26 + restore_return_regs + ret $31, ($26), 1 +END(return_to_handler) + +#endif + +#ifdef CONFIG_DYNAMIC_FTRACE + .global _mcount + .ent _mcount +_mcount: + ret $31, ($28), 1 + .end _mcount + + + .global ftrace_caller + .ent ftrace_caller +ftrace_caller: + mcount_enter + br $27, 1f +1: ldgp $29, 0($27) + + subl $28, MCOUNT_INSN_SIZE, $16 + bis $26, $31, $17 + ldl $18, function_trace_op + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + /* + * the graph tracer (specifically, prepare_ftrace_return) needs these + * arguments but for now the function tracer occupies the regs, so we + * save them in callee-saved regs to recover later. + */ + bis $31, $16, $9 +#endif + ldi $4, current_tracer + ldl $27, 0($4) + + .global ftrace_call +ftrace_call: /* tracer(pc, ra); */ + nop + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + RESTORE_GRAPH_ARGS + call ftrace_graph_caller +#endif + mcount_end + ret $31, ($28), 1 + .end ftrace_caller +#else /* !CONFIG_DYNAMIC_FTRACE */ + + .global _mcount + .ent _mcount +_mcount: + mcount_enter + br $27, 1f +1: ldgp $29, 0($27) + + ldl $27, ftrace_trace_function // if (ftrace_trace_function + ldi $5, ftrace_stub // != ftrace_stub) + cmpeq $27, $5, $6 // + bne $6, skip_ftrace + + subl $28, MCOUNT_INSN_SIZE, $16 // function's pc +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + bis $31, $16, $9 +#endif + bis $26, $31, $17 // function's ra (parent's pc) + call $26, ($27) // (*ftrace_trace_function)(pc, ra); + +skip_ftrace: +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + ldl $4, ftrace_graph_return // if ((ftrace_graph_return + cmpeq $4, $5, $6 // != ftrace_stub) + beq $6, 2f + ldl $4, ftrace_graph_entry // || (ftrace_graph_entry + ldi $5, ftrace_graph_entry_stub // != ftrace_graph_entry_stub)) + cmpeq $4, $5, $6 + bne $6, 3f +2: RESTORE_GRAPH_ARGS + call ftrace_graph_caller // ftrace_graph_caller(); +#endif +3: mcount_end + ret $31, ($28), 1 + .end _mcount + +#endif /* CONFIG_DYNAMIC_FTRACE */ + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + .global ftrace_regs_caller + .ent ftrace_regs_caller +ftrace_regs_caller: + SAVE_PT_REGS + br $27, 1f +1: ldgp $29, 0($27) + + subl $28, MCOUNT_INSN_SIZE, $16 + bis $26, $31, $17 + ldi $4, function_trace_op + ldl $18, 0($4) + mov $sp, $19 + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + bis $31, $16, $9 +#endif + ldi $4, current_tracer + ldl $27, 0($4) + + .global ftrace_regs_call +ftrace_regs_call: + nop + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + RESTORE_GRAPH_REG_ARGS + call ftrace_graph_caller +#endif + RESTORE_PT_REGS + ret $31, ($28), 1 + .end ftrace_regs_caller +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ + + .global ftrace_stub + .ent ftrace_stub +ftrace_stub: + ret $31, ($26), 1 + .end ftrace_stub diff --git a/arch/sw_64/kernel/ftrace.c b/arch/sw_64/kernel/ftrace.c new file mode 100644 index 000000000000..fb25ffe3dbda --- /dev/null +++ b/arch/sw_64/kernel/ftrace.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Based on arch/arm64/kernel/ftrace.c + * + * Copyright (C) 2019 os kernel team + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include + +#include + +#ifdef CONFIG_FUNCTION_TRACER +EXPORT_SYMBOL(_mcount); +#endif + +#ifdef CONFIG_DYNAMIC_FTRACE + +#define TI_FTRACE_ADDR (offsetof(struct thread_info, dyn_ftrace_addr)) +#define TI_FTRACE_REGS_ADDR \ + (offsetof(struct thread_info, dyn_ftrace_regs_addr)) + +unsigned long current_tracer = (unsigned long)ftrace_stub; + +/* + * Replace a single instruction, which may be a branch or NOP. + */ +static int ftrace_modify_code(unsigned long pc, u32 new) +{ + if (sw64_insn_write((void *)pc, new)) + return -EPERM; + return 0; +} + +/* + * Replace tracer function in ftrace_caller() + */ +int ftrace_update_ftrace_func(ftrace_func_t func) +{ + unsigned long pc; + u32 new; + int ret; + + current_tracer = (unsigned long)func; + pc = (unsigned long)&ftrace_call; + new = SW64_CALL(R26, R27, 0); + ret = ftrace_modify_code(pc, new); + + if (!ret) { + pc = (unsigned long)&ftrace_regs_call; + new = SW64_CALL(R26, R27, 0); + ret = ftrace_modify_code(pc, new); + } + + return ret; +} + +/* + * Turn on the call to ftrace_caller() in instrumented function + */ +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned int insn[3]; + unsigned long pc = rec->ip + MCOUNT_LDGP_SIZE; + unsigned long offset; + + if (addr == FTRACE_ADDR) + offset = TI_FTRACE_ADDR; + else + offset = TI_FTRACE_REGS_ADDR; + + insn[0] = SW64_NOP; + /* ldl r28,(ftrace_addr_offset)(r8) */ + insn[1] = (0x23U << 26) | (28U << 21) | (8U << 16) | offset; + insn[2] = SW64_CALL(R28, R28, 0); + + /* replace the 3 mcount instructions at once */ + return copy_to_kernel_nofault((void *)pc, insn, 3 * SW64_INSN_SIZE); +} + +/* + * Turn off the call to ftrace_caller() in instrumented function + */ +int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, + unsigned long addr) +{ + unsigned long pc = rec->ip + MCOUNT_LDGP_SIZE; + unsigned int insn[3] = {SW64_NOP, SW64_NOP, SW64_NOP}; + + return copy_to_kernel_nofault((void *)pc, insn, 3 * SW64_INSN_SIZE); +} + +void arch_ftrace_update_code(int command) +{ + ftrace_modify_all_code(command); +} + +int __init ftrace_dyn_arch_init(void) +{ + struct thread_info *ti = task_thread_info(&init_task); + + ti->dyn_ftrace_addr = FTRACE_ADDR; + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + ti->dyn_ftrace_regs_addr = FTRACE_REGS_ADDR; +#endif + return 0; +} +#endif /* CONFIG_DYNAMIC_FTRACE */ + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS +int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, + unsigned long addr) +{ + return 0; +} +#endif + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * function_graph tracer expects ftrace_return_to_handler() to be called + * on the way back to parent. For this purpose, this function is called + * in _mcount() or ftrace_caller() to replace return address (*parent) on + * the call stack to return_to_handler. + * + * Note that @frame_pointer is used only for sanity check later. + */ +void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, + unsigned long frame_pointer) +{ + unsigned long return_hooker = (unsigned long)&return_to_handler; + unsigned long old; + + if (unlikely(atomic_read(¤t->tracing_graph_pause))) + return; + + /* + * Note: + * No protection against faulting at *parent, which may be seen + * on other archs. It's unlikely on AArch64. + */ + old = *parent; + + if (!function_graph_enter(old, self_addr, frame_pointer, NULL)) + *parent = return_hooker; +} + +#ifdef CONFIG_DYNAMIC_FTRACE +/* + * Turn on/off the call to ftrace_graph_caller() in ftrace_caller() + * depending on @enable. + */ +static int ftrace_modify_graph_caller(bool enable) +{ + unsigned long pc = (unsigned long)&ftrace_graph_call; + u32 new = SW64_NOP; + + if (enable) + new = SW64_CALL(R26, R27, 0); + return ftrace_modify_code(pc, new); +} + +int ftrace_enable_ftrace_graph_caller(void) +{ + return ftrace_modify_graph_caller(true); +} + +int ftrace_disable_ftrace_graph_caller(void) +{ + return ftrace_modify_graph_caller(false); +} +#endif /* CONFIG_DYNAMIC_FTRACE */ +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ -- Gitee From 80f9afd08359fb2becb628788e81ad12f250ed9e Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:32 +0800 Subject: [PATCH 044/524] sw64: add kernel relocation support Add kernel relocation support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/relocate.c | 284 +++++++++++++ arch/sw_64/kernel/relocate_kernel.S | 176 ++++++++ arch/sw_64/tools/.gitignore | 2 + arch/sw_64/tools/Makefile | 8 + arch/sw_64/tools/relocs.c | 635 ++++++++++++++++++++++++++++ arch/sw_64/tools/relocs.h | 72 ++++ arch/sw_64/tools/relocs_main.c | 86 ++++ 7 files changed, 1263 insertions(+) create mode 100644 arch/sw_64/kernel/relocate.c create mode 100644 arch/sw_64/kernel/relocate_kernel.S create mode 100644 arch/sw_64/tools/.gitignore create mode 100644 arch/sw_64/tools/Makefile create mode 100644 arch/sw_64/tools/relocs.c create mode 100644 arch/sw_64/tools/relocs.h create mode 100644 arch/sw_64/tools/relocs_main.c diff --git a/arch/sw_64/kernel/relocate.c b/arch/sw_64/kernel/relocate.c new file mode 100644 index 000000000000..ebdf7d894805 --- /dev/null +++ b/arch/sw_64/kernel/relocate.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for kernel relocation at boot time. + * + * Based on arch/mips/kernel/relocate.c + * + * Copyright (C) 2019 He Sheng + * Authors: He Sheng (hesheng05@gmail.com) + */ +#include +#include +#include + +#include + +#define KTEXT_MAX 0xffffffffa0000000UL +#define RELOCATED(x) ((void *)((unsigned long)x + offset)) + +extern unsigned long _got_start[]; +extern unsigned long _got_end[]; +extern char pre_start_kernel[]; + +extern unsigned int _relocation_start[]; /* End kernel image / start relocation table */ +extern unsigned int _relocation_end[]; /* End relocation table */ + +extern unsigned long __start___ex_table; /* Start exception table */ +extern unsigned long __stop___ex_table; /* End exception table */ +extern union thread_union init_thread_union; + +/* + * This function may be defined for a platform to perform any post-relocation + * fixup necessary. + * Return non-zero to abort relocation + */ +int __weak plat_post_relocation(long offset) +{ + return 0; +} + +static int __init apply_r_sw64_refquad(unsigned long *loc_orig, unsigned long *loc_new, unsigned int offset) +{ + *(unsigned long *)loc_new += offset; + + return 0; +} + +static int (*reloc_handlers_rel[]) (unsigned long *, unsigned long *, unsigned int) __initdata = { + [R_SW64_REFQUAD] = apply_r_sw64_refquad, +}; + +int __init do_relocations(void *kbase_old, void *kbase_new, unsigned int offset) +{ + unsigned int *r; + unsigned long *loc_orig; + unsigned long *loc_new; + int type; + int res; + + for (r = _relocation_start; r < _relocation_end; r++) { + /* Sentinel for last relocation */ + if (*r == 0) + break; + + type = (*r >> 24) & 0xff; + loc_orig = kbase_old + ((*r & 0x00ffffff) << 2); + loc_new = RELOCATED(loc_orig); + + if (reloc_handlers_rel[type] == NULL) { + /* Unsupported relocation */ + pr_err("Unhandled relocation type %d at 0x%pK\n", + type, loc_orig); + return -ENOEXEC; + } + + res = reloc_handlers_rel[type](loc_orig, loc_new, offset); + if (res) + return res; + } + + return 0; +} + +static int __init relocate_got(unsigned int offset) +{ + unsigned long *got_start, *got_end, *e; + + got_start = RELOCATED(&_got_start); + got_end = RELOCATED(&_got_end); + + for (e = got_start; e < got_end; e++) + *e += offset; + + return 0; +} + +#ifdef CONFIG_RANDOMIZE_BASE + +static inline __init unsigned long rotate_xor(unsigned long hash, + const void *area, size_t size) +{ + size_t i; + unsigned long start, *ptr; + /* Make sure start is 8 byte aligned */ + start = ALIGN((unsigned long)area, 8); + size -= (start - (unsigned long)area); + ptr = (unsigned long *) start; + for (i = 0; i < size / sizeof(hash); i++) { + /* Rotate by odd number of bits and XOR. */ + hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7); + hash ^= ptr[i]; + } + return hash; +} + +static inline __init unsigned long get_random_boot(void) +{ + unsigned long entropy = random_get_entropy(); + unsigned long hash = 0; + + /* Attempt to create a simple but unpredictable starting entropy. */ + hash = rotate_xor(hash, linux_banner, strlen(linux_banner)); + + /* Add in any runtime entropy we can get */ + hash = rotate_xor(hash, &entropy, sizeof(entropy)); + + return hash; +} + +static inline __init bool kaslr_disabled(void) +{ + char *str; + + str = strstr(COMMAND_LINE, "nokaslr"); + if (str == COMMAND_LINE || (str > COMMAND_LINE && *(str - 1) == ' ')) + return true; + + return false; +} + +static unsigned long __init determine_relocation_offset(void) +{ + /* Choose a new address for the kernel */ + unsigned long kernel_length; + unsigned long offset; + + if (kaslr_disabled()) + return 0; + + kernel_length = (unsigned long)_end - (unsigned long)(&_text); + + /* TODO: offset is 64K align. maybe 8KB align is okay. */ + offset = get_random_boot() << 16; + offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1); + if (offset < kernel_length) + offset += ALIGN(kernel_length, 0x10000); + + /* + * TODO:new location should not overlaps initrd, dtb, acpi + * tables, etc. + */ + + if ((KTEXT_MAX - (unsigned long)_end) < offset) + offset = 0; + + return offset; +} + +#else + +static inline unsigned long __init determine_relocation_offset(void) +{ + /* + * Choose a new address for the kernel + * For now we'll hard code the destination offset. + */ + return 0; +} + +#endif + +static inline int __init relocation_offset_valid(unsigned long offset) +{ + unsigned long loc_new = (unsigned long)_text + offset; + + if (loc_new & 0x0000ffff) { + /* Inappropriately aligned new location */ + return 0; + } + if (loc_new < (unsigned long)&_end) { + /* New location overlaps original kernel */ + return 0; + } + return 1; +} + +unsigned int __init relocate_kernel(void) +{ + void *loc_new; + unsigned long kernel_length; + unsigned long bss_length; + unsigned int offset = 0; + int res = 1; + + kernel_length = (unsigned long)(&_relocation_start) - (long)(&_text); + bss_length = (unsigned long)&__bss_stop - (long)&__bss_start; + + offset = determine_relocation_offset(); + /* Reset the command line now so we don't end up with a duplicate */ + + /* Sanity check relocation address */ + if (offset && relocation_offset_valid(offset)) { + + loc_new = RELOCATED(&_text); + /* Copy the kernel to it's new location */ + memcpy(loc_new, &_text, kernel_length); + + /* Perform relocations on the new kernel */ + res = do_relocations(&_text, loc_new, offset); + if (res < 0) + goto out; + + res = relocate_got(offset); + if (res < 0) + goto out; + + /* + * The original .bss has already been cleared, and + * some variables such as command line parameters + * stored to it so make a copy in the new location. + */ + memcpy(RELOCATED(&__bss_start), &__bss_start, bss_length); + + /* + * Last chance for the platform to abort relocation. + * This may also be used by the platform to perform any + * initialisation required now that the new kernel is + * resident in memory and ready to be executed. + */ + if (plat_post_relocation(offset)) + goto out; + + /* Return the new kernel's offset */ + return offset; + } +out: + return 0; +} + +/* + * Show relocation information on panic. + */ +void show_kernel_relocation(const char *level) +{ + unsigned long offset; + + offset = __pa_symbol(_text) - __pa_symbol(_TEXT_START); + + if (IS_ENABLED(CONFIG_RELOCATABLE) && offset > 0) { + printk(level); + pr_cont("Kernel relocated by 0x%pK\n", (void *)offset); + pr_cont(" .text @ 0x%pK\n", _text); + pr_cont(" .data @ 0x%pK\n", _sdata); + pr_cont(" .bss @ 0x%pK\n", __bss_start); + } +} + +static int kernel_location_notifier_fn(struct notifier_block *self, + unsigned long v, void *p) +{ + show_kernel_relocation(KERN_EMERG); + return NOTIFY_DONE; +} + +static struct notifier_block kernel_location_notifier = { + .notifier_call = kernel_location_notifier_fn +}; + +static int __init register_kernel_offset_dumper(void) +{ + atomic_notifier_chain_register(&panic_notifier_list, + &kernel_location_notifier); + return 0; +} +device_initcall(register_kernel_offset_dumper); diff --git a/arch/sw_64/kernel/relocate_kernel.S b/arch/sw_64/kernel/relocate_kernel.S new file mode 100644 index 000000000000..f1a160636212 --- /dev/null +++ b/arch/sw_64/kernel/relocate_kernel.S @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * relocate_kernel.S for kexec + * Created by Jul 2 2019 + * + * This source code is licensed under the GNU General Public License, + * Version 2. See the file COPYING for more details. + */ + +#include +#include + + .align 3 + .globl relocate_new_kernel + .ent relocate_new_kernel + +relocate_new_kernel: + .prologue 0 + ldl a0, arg0 + ldl a1, arg1 + ldl a2, arg2 + ldl a3, arg3 + + ldl s0, kexec_indirection_page + ldl s1, kexec_start_address + +process_entry: + ldl s2, 0(s0) + addl s0, 8, s0 + + /* + * In case of a kdump/crash kernel, the indirection page is not + * populated as the kernel is directly copied to a reserved location + */ + beq s2, done + + /* destination page */ + and s2, 0x1, s3 + beq s3, 1f + bic s2, 0x1, s4/* store destination addr in s4 */ + br $31, process_entry + +1: + /* indirection page, update s0*/ + and s2, 0x2, s3 + beq s3, 1f + bic s2, 0x2, s0 + br $31, process_entry + +1: + /* done page */ + and s2, 0x4, s3 + beq s3, 1f + br $31, done +1: + /* source page */ + and s2, 0x8, s3 + beq s3, process_entry + bic s2, 0x8, s2 + ldi s6, 0x1 + sll s6, (PAGE_SHIFT - 3), s6 + +copy_word: + /* copy page word by word */ + ldl s5, 0(s2) + stl s5, 0(s4) + addl s4, 8, s4 + addl s2, 8, s2 + subl s6, 1, s6 + beq s6, process_entry + br $31, copy_word + br $31, process_entry + +done: +#ifdef CONFIG_CRASH_SMP /* unsupported now!!!! */ + /* kexec_flag reset is signal to other CPUs what kernel + * was moved to it's location. Note - we need relocated address + * of kexec_flag. + */ + + br ra, 1f +1: mov ra, t1 + ldi t2, 1b + ldi t0, kexec_flag + subl t0, t2, t0 + addl t1, t0, t0 + stl zero, 0(t0) +#endif + memb + jmp ra, (s1) + .end relocate_new_kernel + .size relocate_new_kernel, .-relocate_new_kernel + +#ifdef CONFIG_CRASH_SMP + /* + * Other CPUs should wait until code is relocated and + * then start at entry (?) point. + */ + .align 3 + .globl kexec_smp_wait + .ent kexec_smp_wait +kexec_smp_wait: + ldl a0, s_arg0 + ldl a1, s_arg1 + ldl a2, s_arg2 + ldl a3, s_arg3 + ldl s1, kexec_start_address + + /* Non-relocated address works for args and kexec_start_address (old + * kernel is not overwritten). But we need relocated address of + * kexec_flag. + */ + + bsr ra, 1f +1: mov ra, t1 + ldi t2, 1b + ldi t0, kexec_flag + subl t0, t2, t0 + addl t1, t0, t0 + +1: stl s0, 0(t0) + bne s0, 1b + memb + jmp ra, (s1) + .end kexec_smp_wait + .size kexec_smp_wait, .-kexec_smp_wait +#endif + + .align 3 + + /* All parameters to new kernel are passed in registers a0-a3. + * kexec_args[0..3] are uses to prepare register values. + */ + +kexec_args: + .globl kexec_args +arg0: .quad 0x0 +arg1: .quad 0x0 +arg2: .quad 0x0 +arg3: .quad 0x0 + .size kexec_args, 8*4 + +#ifdef CONFIG_CRASH_SMP + /* + * Secondary CPUs may have different kernel parameters in + * their registers a0-a3. secondary_kexec_args[0..3] are used + * to prepare register values. + */ +secondary_kexec_args: + .globl secondary_kexec_args +s_arg0: .quad 0x0 +s_arg1: .quad 0x0 +s_arg2: .quad 0x0 +s_arg3: .quad 0x0 + .size secondary_kexec_args, 8*4 + +kexec_flag: + .quad 0x1 +#endif + +kexec_start_address: + .globl kexec_start_address + .quad 0x0 + .size kexec_start_address, 8 + +kexec_indirection_page: + .globl kexec_indirection_page + .quad 0 + .size kexec_indirection_page, 8 + +relocate_new_kernel_end: + +relocate_new_kernel_size: + .global relocate_new_kernel_size + .quad relocate_new_kernel_end - relocate_new_kernel + .size relocate_new_kernel_size, 8 diff --git a/arch/sw_64/tools/.gitignore b/arch/sw_64/tools/.gitignore new file mode 100644 index 000000000000..f73e86272b76 --- /dev/null +++ b/arch/sw_64/tools/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +relocs diff --git a/arch/sw_64/tools/Makefile b/arch/sw_64/tools/Makefile new file mode 100644 index 000000000000..66f55b035e22 --- /dev/null +++ b/arch/sw_64/tools/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 + +hostprogs += relocs +relocs-objs += relocs.o +relocs-objs += relocs_main.o +PHONY += relocs +relocs: $(obj)/relocs + @: diff --git a/arch/sw_64/tools/relocs.c b/arch/sw_64/tools/relocs.c new file mode 100644 index 000000000000..ec0ed422a836 --- /dev/null +++ b/arch/sw_64/tools/relocs.c @@ -0,0 +1,635 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "relocs.h" + +#define ELF_BITS 64 + +#define ELF_MACHINE EM_SW64 +#define ELF_MACHINE_NAME "SW64" +#define SHT_REL_TYPE SHT_RELA +#define Elf_Rel Elf64_Rela + +#define ELF_CLASS ELFCLASS64 +#define ELF_R_SYM(val) ELF64_R_SYM(val) +#define ELF_R_TYPE(val) ELF64_R_TYPE(val) +#define ELF_ST_TYPE(o) ELF64_ST_TYPE(o) +#define ELF_ST_BIND(o) ELF64_ST_BIND(o) +#define ELF_ST_VISIBILITY(o) ELF64_ST_VISIBILITY(o) + +#define ElfW(type) _ElfW(ELF_BITS, type) +#define _ElfW(bits, type) __ElfW(bits, type) +#define __ElfW(bits, type) Elf##bits##_##type + +#define Elf_Addr ElfW(Addr) +#define Elf_Ehdr ElfW(Ehdr) +#define Elf_Phdr ElfW(Phdr) +#define Elf_Shdr ElfW(Shdr) +#define Elf_Sym ElfW(Sym) + +static Elf_Ehdr ehdr; + +struct relocs { + uint32_t *offset; + unsigned long count; + unsigned long size; +}; + +static struct relocs relocs; + +struct section { + Elf_Shdr shdr; + struct section *link; + Elf_Sym *symtab; + Elf_Rel *reltab; + char *strtab; + long shdr_offset; +}; +static struct section *secs; + +static const char * const regex_sym_kernel = { +/* Symbols matching these regex's should never be relocated */ + "^(__crc_)", +}; + +static regex_t sym_regex_c; + +static int regex_skip_reloc(const char *sym_name) +{ + return !regexec(&sym_regex_c, sym_name, 0, NULL, 0); +} + +static void regex_init(void) +{ + char errbuf[128]; + int err; + + err = regcomp(&sym_regex_c, regex_sym_kernel, + REG_EXTENDED|REG_NOSUB); + + if (err) { + regerror(err, &sym_regex_c, errbuf, sizeof(errbuf)); + die("%s", errbuf); + } +} + +static const char *rel_type(unsigned int type) +{ + static const char * const type_name[] = { +#define REL_TYPE(X)[X] = #X + REL_TYPE(R_SW64_NONE), + REL_TYPE(R_SW64_REFQUAD), + REL_TYPE(R_SW64_LITERAL), + REL_TYPE(R_SW64_LITUSE), + REL_TYPE(R_SW64_GPDISP), + REL_TYPE(R_SW64_BRADDR), + REL_TYPE(R_SW64_HINT), + REL_TYPE(R_SW64_SREL32), + REL_TYPE(R_SW64_GPRELHIGH), + REL_TYPE(R_SW64_GPRELLOW), +#undef REL_TYPE + }; + const char *name = "unknown type rel type name"; + + if (type < ARRAY_SIZE(type_name) && type_name[type]) + name = type_name[type]; + return name; +} + +static const char *sec_name(unsigned int shndx) +{ + const char *sec_strtab; + const char *name; + + sec_strtab = secs[ehdr.e_shstrndx].strtab; + if (shndx < ehdr.e_shnum) + name = sec_strtab + secs[shndx].shdr.sh_name; + else if (shndx == SHN_ABS) + name = "ABSOLUTE"; + else if (shndx == SHN_COMMON) + name = "COMMON"; + else + name = ""; + return name; +} + +static struct section *sec_lookup(const char *secname) +{ + int i; + + for (i = 0; i < ehdr.e_shnum; i++) + if (strcmp(secname, sec_name(i)) == 0) + return &secs[i]; + + return NULL; +} + +static const char *sym_name(const char *sym_strtab, Elf_Sym *sym) +{ + const char *name; + + if (sym->st_name) + name = sym_strtab + sym->st_name; + else + name = sec_name(sym->st_shndx); + return name; +} + +#define le16_to_cpu(val) (val) +#define le32_to_cpu(val) (val) +#define le64_to_cpu(val) (val) + +#define cpu_to_le16(val) (val) +#define cpu_to_le32(val) (val) +#define cpu_to_le64(val) (val) + +static uint16_t elf16_to_cpu(uint16_t val) +{ + return le16_to_cpu(val); +} + +static uint32_t elf32_to_cpu(uint32_t val) +{ + return le32_to_cpu(val); +} + +static uint32_t cpu_to_elf32(uint32_t val) +{ + return cpu_to_le32(val); +} + +#define elf_half_to_cpu(x) elf16_to_cpu(x) +#define elf_word_to_cpu(x) elf32_to_cpu(x) + +#if ELF_BITS == 64 +static uint64_t elf64_to_cpu(uint64_t val) +{ + return le64_to_cpu(val); +} +#define elf_addr_to_cpu(x) elf64_to_cpu(x) +#define elf_off_to_cpu(x) elf64_to_cpu(x) +#define elf_xword_to_cpu(x) elf64_to_cpu(x) +#else +#define elf_addr_to_cpu(x) elf32_to_cpu(x) +#define elf_off_to_cpu(x) elf32_to_cpu(x) +#define elf_xword_to_cpu(x) elf32_to_cpu(x) +#endif + +static void read_ehdr(FILE *fp) +{ + if (fread(&ehdr, sizeof(ehdr), 1, fp) != 1) + die("Cannot read ELF header: %s\n", strerror(errno)); + + if (memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0) + die("No ELF magic\n"); + + if (ehdr.e_ident[EI_CLASS] != ELF_CLASS) + die("Not a %d bit executable\n", ELF_BITS); + + if ((ehdr.e_ident[EI_DATA] != ELFDATA2LSB) && + (ehdr.e_ident[EI_DATA] != ELFDATA2MSB)) + die("Unknown ELF Endianness\n"); + + if (ehdr.e_ident[EI_VERSION] != EV_CURRENT) + die("Unknown ELF version\n"); + + /* Convert the fields to native endian */ + ehdr.e_type = elf_half_to_cpu(ehdr.e_type); + ehdr.e_machine = elf_half_to_cpu(ehdr.e_machine); + ehdr.e_version = elf_word_to_cpu(ehdr.e_version); + ehdr.e_entry = elf_addr_to_cpu(ehdr.e_entry); + ehdr.e_phoff = elf_off_to_cpu(ehdr.e_phoff); + ehdr.e_shoff = elf_off_to_cpu(ehdr.e_shoff); + ehdr.e_flags = elf_word_to_cpu(ehdr.e_flags); + ehdr.e_ehsize = elf_half_to_cpu(ehdr.e_ehsize); + ehdr.e_phentsize = elf_half_to_cpu(ehdr.e_phentsize); + ehdr.e_phnum = elf_half_to_cpu(ehdr.e_phnum); + ehdr.e_shentsize = elf_half_to_cpu(ehdr.e_shentsize); + ehdr.e_shnum = elf_half_to_cpu(ehdr.e_shnum); + ehdr.e_shstrndx = elf_half_to_cpu(ehdr.e_shstrndx); + + if ((ehdr.e_type != ET_EXEC) && (ehdr.e_type != ET_DYN)) + die("Unsupported ELF header type\n"); + + if (ehdr.e_machine != ELF_MACHINE) + die("Not for %s\n", ELF_MACHINE_NAME); + + if (ehdr.e_version != EV_CURRENT) + die("Unknown ELF version\n"); + + if (ehdr.e_ehsize != sizeof(Elf_Ehdr)) + die("Bad Elf header size\n"); + + if (ehdr.e_phentsize != sizeof(Elf_Phdr)) + die("Bad program header entry\n"); + + if (ehdr.e_shentsize != sizeof(Elf_Shdr)) + die("Bad section header entry\n"); + + if (ehdr.e_shstrndx >= ehdr.e_shnum) + die("String table index out of bounds\n"); +} + +static void read_shdrs(FILE *fp) +{ + int i; + Elf_Shdr shdr; + + secs = calloc(ehdr.e_shnum, sizeof(struct section)); + if (!secs) + die("Unable to allocate %d section headers\n", ehdr.e_shnum); + + if (fseek(fp, ehdr.e_shoff, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", ehdr.e_shoff, strerror(errno)); + + for (i = 0; i < ehdr.e_shnum; i++) { + struct section *sec = &secs[i]; + + sec->shdr_offset = ftell(fp); + if (fread(&shdr, sizeof(shdr), 1, fp) != 1) + die("Cannot read ELF section headers %d/%d: %s\n", + i, ehdr.e_shnum, strerror(errno)); + sec->shdr.sh_name = elf_word_to_cpu(shdr.sh_name); + sec->shdr.sh_type = elf_word_to_cpu(shdr.sh_type); + sec->shdr.sh_flags = elf_xword_to_cpu(shdr.sh_flags); + sec->shdr.sh_addr = elf_addr_to_cpu(shdr.sh_addr); + sec->shdr.sh_offset = elf_off_to_cpu(shdr.sh_offset); + sec->shdr.sh_size = elf_xword_to_cpu(shdr.sh_size); + sec->shdr.sh_link = elf_word_to_cpu(shdr.sh_link); + sec->shdr.sh_info = elf_word_to_cpu(shdr.sh_info); + sec->shdr.sh_addralign = elf_xword_to_cpu(shdr.sh_addralign); + sec->shdr.sh_entsize = elf_xword_to_cpu(shdr.sh_entsize); + if (sec->shdr.sh_link < ehdr.e_shnum) + sec->link = &secs[sec->shdr.sh_link]; + } +} + +static void read_strtabs(FILE *fp) +{ + int i; + + for (i = 0; i < ehdr.e_shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_STRTAB) + continue; + + sec->strtab = malloc(sec->shdr.sh_size); + if (!sec->strtab) + die("malloc of %d bytes for strtab failed\n", + sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", + sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->strtab, 1, sec->shdr.sh_size, fp) != + sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + } +} + +static void read_symtabs(FILE *fp) +{ + int i, j; + + for (i = 0; i < ehdr.e_shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_SYMTAB) + continue; + + sec->symtab = malloc(sec->shdr.sh_size); + if (!sec->symtab) + die("malloc of %d bytes for symtab failed\n", + sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", + sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->symtab, 1, sec->shdr.sh_size, fp) != + sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + + for (j = 0; j < sec->shdr.sh_size/sizeof(Elf_Sym); j++) { + Elf_Sym *sym = &sec->symtab[j]; + + sym->st_name = elf_word_to_cpu(sym->st_name); + sym->st_value = elf_addr_to_cpu(sym->st_value); + sym->st_size = elf_xword_to_cpu(sym->st_size); + sym->st_shndx = elf_half_to_cpu(sym->st_shndx); + } + } +} + +static void read_relocs(FILE *fp) +{ + static unsigned long base; + int i, j; + + if (!base) { + struct section *sec = sec_lookup(".text"); + + if (!sec) + die("Could not find .text section\n"); + + base = sec->shdr.sh_addr; + } + + for (i = 0; i < ehdr.e_shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_REL_TYPE) + continue; + + sec->reltab = malloc(sec->shdr.sh_size); + if (!sec->reltab) + die("malloc of %d bytes for relocs failed\n", + sec->shdr.sh_size); + + if (fseek(fp, sec->shdr.sh_offset, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", + sec->shdr.sh_offset, strerror(errno)); + + if (fread(sec->reltab, 1, sec->shdr.sh_size, fp) != + sec->shdr.sh_size) + die("Cannot read symbol table: %s\n", strerror(errno)); + + for (j = 0; j < sec->shdr.sh_size/sizeof(Elf_Rel); j++) { + Elf_Rel *rel = &sec->reltab[j]; + + rel->r_offset = elf_addr_to_cpu(rel->r_offset); + /* Set offset into kernel image */ + rel->r_offset -= base; + /* Convert SW64 RELA format - only the symbol + * index needs converting to native endianness + */ + rel->r_info = elf_xword_to_cpu(rel->r_info); +#if (SHT_REL_TYPE == SHT_RELA) + rel->r_addend = elf_xword_to_cpu(rel->r_addend); +#endif + } + } +} + +static void remove_relocs(FILE *fp) +{ + int i; + Elf_Shdr shdr; + + for (i = 0; i < ehdr.e_shnum; i++) { + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_REL_TYPE) + continue; + + if (fseek(fp, sec->shdr_offset, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", + sec->shdr_offset, strerror(errno)); + + if (fread(&shdr, sizeof(shdr), 1, fp) != 1) + die("Cannot read ELF section headers %d/%d: %s\n", + i, ehdr.e_shnum, strerror(errno)); + + /* Set relocation section size to 0, effectively removing it. + * This is necessary due to lack of support for relocations + * in objcopy when creating 32bit elf from 64bit elf. + */ + shdr.sh_size = 0; + + if (fseek(fp, sec->shdr_offset, SEEK_SET) < 0) + die("Seek to %d failed: %s\n", + sec->shdr_offset, strerror(errno)); + + if (fwrite(&shdr, sizeof(shdr), 1, fp) != 1) + die("Cannot write ELF section headers %d/%d: %s\n", + i, ehdr.e_shnum, strerror(errno)); + } +} + +static void add_reloc(struct relocs *r, uint32_t offset, unsigned int type) +{ + /* Relocation representation in binary table: + * |76543210|76543210|76543210|76543210| + * | Type | offset from _text >> 2 | + */ + offset >>= 2; + if (offset > 0x00FFFFFF) + die("Kernel image exceeds maximum size for relocation!\n"); + + offset = (offset & 0x00FFFFFF) | ((type & 0xFF) << 24); + + if (r->count == r->size) { + unsigned long newsize = r->size + 50000; + void *mem = realloc(r->offset, newsize * sizeof(r->offset[0])); + + if (!mem) + die("realloc failed\n"); + + r->offset = mem; + r->size = newsize; + } + r->offset[r->count++] = offset; +} + +static void walk_relocs(int (*process)(struct section *sec, Elf_Rel *rel, + Elf_Sym *sym, const char *symname)) +{ + int i; + + /* Walk through the relocations */ + for (i = 0; i < ehdr.e_shnum; i++) { + char *sym_strtab; + Elf_Sym *sh_symtab; + struct section *sec_applies, *sec_symtab; + int j; + struct section *sec = &secs[i]; + + if (sec->shdr.sh_type != SHT_REL_TYPE) + continue; + sec_symtab = sec->link; + sec_applies = &secs[sec->shdr.sh_info]; + if (!(sec_applies->shdr.sh_flags & SHF_ALLOC)) + continue; + + sh_symtab = sec_symtab->symtab; + sym_strtab = sec_symtab->link->strtab; + for (j = 0; j < sec->shdr.sh_size/sizeof(Elf_Rel); j++) { + Elf_Rel *rel = &sec->reltab[j]; + Elf_Sym *sym = &sh_symtab[ELF_R_SYM(rel->r_info)]; + const char *symname = sym_name(sym_strtab, sym); + + process(sec, rel, sym, symname); + } + } +} + +static int do_reloc(struct section *sec, Elf_Rel *rel, Elf_Sym *sym, + const char *symname) +{ + unsigned int r_type = ELF_R_TYPE(rel->r_info); + unsigned int bind = ELF_ST_BIND(sym->st_info); + + if ((bind == STB_WEAK) && (sym->st_value == 0)) { + /* Don't relocate weak symbols without a target */ + return 0; + } + + if (regex_skip_reloc(symname)) + return 0; + + switch (r_type) { + case R_SW64_NONE: + case R_SW64_LITERAL: /* relocated by GOT */ + case R_SW64_LITUSE: + case R_SW64_GPDISP: + case R_SW64_BRADDR: + case R_SW64_HINT: + case R_SW64_SREL32: + case R_SW64_GPRELHIGH: + case R_SW64_GPRELLOW: + case R_SW64_LITERAL_GOT: + /* + * NONE can be ignored and PC relative relocations don't + * need to be adjusted. + */ + break; + + case R_SW64_REFQUAD: + add_reloc(&relocs, rel->r_offset, r_type); + break; + + default: + die("Unsupported relocation type: %s (%d)\n", + rel_type(r_type), r_type); + break; + } + + return 0; +} + +static int write_reloc_as_bin(uint32_t v, FILE *f) +{ + unsigned char buf[4]; + + v = cpu_to_elf32(v); + + memcpy(buf, &v, sizeof(uint32_t)); + return fwrite(buf, 1, 4, f); +} + +static int write_reloc_as_text(uint32_t v, FILE *f) +{ + int res; + + res = fprintf(f, "\t.long 0x%08"PRIx32"\n", v); + if (res < 0) + return res; + else + return sizeof(uint32_t); +} + +static void emit_relocs(int as_text, int as_bin, FILE *outf) +{ + int i; + int (*write_reloc)(uint32_t, FILE *) = write_reloc_as_bin; + int size = 0; + int size_reserved; + struct section *sec_reloc; + + sec_reloc = sec_lookup(".data.reloc"); + if (!sec_reloc) + die("Could not find relocation section\n"); + + size_reserved = sec_reloc->shdr.sh_size; + /* Collect up the relocations */ + walk_relocs(do_reloc); + + /* Print the relocations */ + if (as_text) { + /* Print the relocations in a form suitable that + * gas will like. + */ + printf(".section \".data.reloc\",\"a\"\n"); + printf(".balign 8\n"); + /* Output text to stdout */ + write_reloc = write_reloc_as_text; + outf = stdout; + } else if (as_bin) { + /* Output raw binary to stdout */ + outf = stdout; + } else { + /* + * Seek to offset of the relocation section. + * Each relocation is then written into the + * vmlinux kernel image. + */ + if (fseek(outf, sec_reloc->shdr.sh_offset, SEEK_SET) < 0) { + die("Seek to %d failed: %s\n", + sec_reloc->shdr.sh_offset, strerror(errno)); + } + } + + for (i = 0; i < relocs.count; i++) + size += write_reloc(relocs.offset[i], outf); + + /* Print a stop, but only if we've actually written some relocs */ + if (size) + size += write_reloc(0, outf); + + if (size > size_reserved) + /* + * Die, but suggest a value for CONFIG_RELOCATION_TABLE_SIZE + * which will fix this problem and allow a bit of headroom + * if more kernel features are enabled + */ + die("Relocations overflow available space!\n" + "Please adjust CONFIG_RELOCATION_TABLE_SIZE " + "to at least 0x%08x\n", (size + 0x1000) & ~0xFFF); +} + +/* + * As an aid to debugging problems with different linkers + * print summary information about the relocs. + * Since different linkers tend to emit the sections in + * different orders we use the section names in the output. + */ +static int do_reloc_info(struct section *sec, Elf_Rel *rel, ElfW(Sym) * sym, + const char *symname) +{ + printf("%16s 0x%x %16s %40s %16s\n", + sec_name(sec->shdr.sh_info), + (unsigned int)rel->r_offset, + rel_type(ELF_R_TYPE(rel->r_info)), + symname, + sec_name(sym->st_shndx)); + return 0; +} + +static void print_reloc_info(void) +{ + printf("%16s %10s %16s %40s %16s\n", + "reloc section", + "offset", + "reloc type", + "symbol", + "symbol section"); + walk_relocs(do_reloc_info); +} + +void process(FILE *fp, int as_text, int as_bin, + int show_reloc_info, int keep_relocs) +{ + regex_init(); + read_ehdr(fp); + read_shdrs(fp); + read_strtabs(fp); + read_symtabs(fp); + read_relocs(fp); + if (show_reloc_info) { + print_reloc_info(); + return; + } + emit_relocs(as_text, as_bin, fp); + if (!keep_relocs) + remove_relocs(fp); +} diff --git a/arch/sw_64/tools/relocs.h b/arch/sw_64/tools/relocs.h new file mode 100644 index 000000000000..17c7e31113a0 --- /dev/null +++ b/arch/sw_64/tools/relocs.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SW64_TOOLS_RELOCS_H +#define _SW64_TOOLS_RELOCS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define USE_BSD +#include +#include + +#define EM_SW64 0x9916 +/* + * SW64 ELF relocation types + */ +#define R_SW64_NONE 0 /* No reloc */ +#define R_SW64_REFLONG 1 /* Direct 32 bit */ +#define R_SW64_REFQUAD 2 /* Direct 64 bit */ +#define R_SW64_GPREL32 3 /* GP relative 32 bit */ +#define R_SW64_LITERAL 4 /* GP relative 16 bit w/optimization */ +#define R_SW64_LITUSE 5 /* Optimization hint for LITERAL */ +#define R_SW64_GPDISP 6 /* Add displacement to GP */ +#define R_SW64_BRADDR 7 /* PC+4 relative 23 bit shifted */ +#define R_SW64_HINT 8 /* PC+4 relative 16 bit shifted */ +#define R_SW64_SREL16 9 /* PC relative 16 bit */ +#define R_SW64_SREL32 10 /* PC relative 32 bit */ +#define R_SW64_SREL64 11 /* PC relative 64 bit */ +#define R_SW64_GPRELHIGH 17 /* GP relative 32 bit, high 16 bits */ +#define R_SW64_GPRELLOW 18 /* GP relative 32 bit, low 16 bits */ +#define R_SW64_GPREL16 19 /* GP relative 16 bit */ +#define R_SW64_COPY 24 /* Copy symbol at runtime */ +#define R_SW64_GLOB_DAT 25 /* Create GOT entry */ +#define R_SW64_JMP_SLOT 26 /* Create PLT entry */ +#define R_SW64_RELATIVE 27 /* Adjust by program base */ +#define R_SW64_BRSGP 28 +#define R_SW64_TLSGD 29 +#define R_SW64_TLS_LDM 30 +#define R_SW64_DTPMOD64 31 +#define R_SW64_GOTDTPREL 32 +#define R_SW64_DTPREL64 33 +#define R_SW64_DTPRELHI 34 +#define R_SW64_DTPRELLO 35 +#define R_SW64_DTPREL16 36 +#define R_SW64_GOTTPREL 37 +#define R_SW64_TPREL64 38 +#define R_SW64_TPRELHI 39 +#define R_SW64_TPRELLO 40 +#define R_SW64_TPREL16 41 +#define R_SW64_LITERAL_GOT 43 /* GP relative */ + +void die(char *fmt, ...); + +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + +enum symtype { + S_ABS, + S_REL, + S_SEG, + S_LIN, + S_NSYMTYPES +}; + +void process(FILE *fp, int as_text, int as_bin, + int show_reloc_info, int keep_relocs); +#endif /* _SW64_TOOLS_RELOCS_H */ diff --git a/arch/sw_64/tools/relocs_main.c b/arch/sw_64/tools/relocs_main.c new file mode 100644 index 000000000000..30a830a070db --- /dev/null +++ b/arch/sw_64/tools/relocs_main.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "relocs.h" + +void die(char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + exit(1); +} + +static void usage(void) +{ + die("relocs [--reloc-info|--text|--bin|--keep] vmlinux\n"); +} + +int main(int argc, char **argv) +{ + int show_reloc_info, as_text, as_bin, keep_relocs; + const char *fname; + FILE *fp; + int i; + unsigned char e_ident[EI_NIDENT]; + + show_reloc_info = 0; + as_text = 0; + as_bin = 0; + keep_relocs = 0; + fname = NULL; + for (i = 1; i < argc; i++) { + char *arg = argv[i]; + + if (*arg == '-') { + if (strcmp(arg, "--reloc-info") == 0) { + show_reloc_info = 1; + continue; + } + if (strcmp(arg, "--text") == 0) { + as_text = 1; + continue; + } + if (strcmp(arg, "--bin") == 0) { + as_bin = 1; + continue; + } + if (strcmp(arg, "--keep") == 0) { + keep_relocs = 1; + continue; + } + } else if (!fname) { + fname = arg; + continue; + } + usage(); + } + if (!fname) + usage(); + + fp = fopen(fname, "r+"); + if (!fp) + die("Cannot open %s: %s\n", fname, strerror(errno)); + + if (fread(&e_ident, 1, EI_NIDENT, fp) != EI_NIDENT) + die("Cannot read %s: %s", fname, strerror(errno)); + + rewind(fp); + if (e_ident[EI_CLASS] == ELFCLASS64) + process(fp, as_text, as_bin, show_reloc_info, keep_relocs); + else + die("Unsupport ELF class on SW64: %s", fname); + //process_32(fp, as_text, as_bin, show_reloc_info, keep_relocs); + fclose(fp); + return 0; +} -- Gitee From d40da53ec93dd8cfc2e14b509403b68444869064 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:18 +0800 Subject: [PATCH 045/524] sw64: add kprobe support Add kprobe support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kprobes.h | 76 +++++ arch/sw_64/kernel/insn.c | 110 ++++++++ arch/sw_64/kernel/kprobes/Makefile | 3 + arch/sw_64/kernel/kprobes/common.h | 9 + arch/sw_64/kernel/kprobes/decode-insn.c | 101 +++++++ arch/sw_64/kernel/kprobes/kprobes-ftrace.c | 48 ++++ arch/sw_64/kernel/kprobes/kprobes.c | 309 +++++++++++++++++++++ 7 files changed, 656 insertions(+) create mode 100644 arch/sw_64/include/asm/kprobes.h create mode 100644 arch/sw_64/kernel/insn.c create mode 100644 arch/sw_64/kernel/kprobes/Makefile create mode 100644 arch/sw_64/kernel/kprobes/common.h create mode 100644 arch/sw_64/kernel/kprobes/decode-insn.c create mode 100644 arch/sw_64/kernel/kprobes/kprobes-ftrace.c create mode 100644 arch/sw_64/kernel/kprobes/kprobes.c diff --git a/arch/sw_64/include/asm/kprobes.h b/arch/sw_64/include/asm/kprobes.h new file mode 100644 index 000000000000..0c7be8109ed2 --- /dev/null +++ b/arch/sw_64/include/asm/kprobes.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Kernel Probes (KProbes) + * Based on arch/mips/include/asm/kprobes.h + */ + +#ifndef _ASM_SW64_KPROBES_H +#define _ASM_SW64_KPROBES_H + +#include + +#define BREAK_KPROBE 0x40ffffff +#define BREAK_KPROBE_SS 0x40fffeff + +#ifdef CONFIG_KPROBES +#include +#include + +#include +#include + +#define __ARCH_WANT_KPROBES_INSN_SLOT + +struct kprobe; +struct pt_regs; + +typedef u32 kprobe_opcode_t; + +#define MAX_INSN_SIZE 2 + +#define flush_insn_slot(p) \ +do { \ + if (p->addr) \ + flush_icache_range((unsigned long)p->addr, \ + (unsigned long)p->addr + \ + (MAX_INSN_SIZE * sizeof(kprobe_opcode_t))); \ +} while (0) + + +#define kretprobe_blacklist_size 0 + +void arch_remove_kprobe(struct kprobe *p); + +/* Architecture specific copy of original instruction*/ +struct arch_specific_insn { + /* copy of the original instruction */ + kprobe_opcode_t *insn; + /* + * Set in kprobes code, initially to 0. If the instruction can be + * eumulated, this is set to 1, if not, to -1. + */ + int boostable; +}; + +struct prev_kprobe { + struct kprobe *kp; + unsigned long status; +}; + +#define SKIP_DELAYSLOT 0x0001 + +/* per-cpu kprobe control block */ +struct kprobe_ctlblk { + unsigned long kprobe_status; + /* Per-thread fields, used while emulating branches */ + unsigned long flags; + unsigned long target_pc; + struct prev_kprobe prev_kprobe; +}; +extern int kprobe_handler(struct pt_regs *regs); +extern int post_kprobe_handler(struct pt_regs *regs); +extern int kprobe_fault_handler(struct pt_regs *regs, unsigned long mmcsr); + + +#endif /* CONFIG_KPROBES */ +#endif /* _ASM_SW64_KPROBES_H */ diff --git a/arch/sw_64/kernel/insn.c b/arch/sw_64/kernel/insn.c new file mode 100644 index 000000000000..281578e1bfc0 --- /dev/null +++ b/arch/sw_64/kernel/insn.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019, serveros, linyue + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include + +//static DEFINE_RAW_SPINLOCK(patch_lock); + +int __kprobes sw64_insn_read(void *addr, u32 *insnp) +{ + int ret; + __le32 val; + + ret = copy_from_kernel_nofault(&val, addr, SW64_INSN_SIZE); + if (!ret) + *insnp = le32_to_cpu(val); + + return ret; +} + +static int __kprobes __sw64_insn_write(void *addr, __le32 insn) +{ + void *waddr = addr; + int ret; + + //raw_spin_lock_irqsave(&patch_lock, flags); + + ret = copy_to_kernel_nofault(waddr, &insn, SW64_INSN_SIZE); + + //raw_spin_unlock_irqrestore(&patch_lock, flags); + + return ret; +} + +static int __kprobes __sw64_insn_double_write(void *addr, __le64 insn) +{ + void *waddr = addr; + //unsigned long flags = 0; + int ret; + + //raw_spin_lock_irqsave(&patch_lock, flags); + + ret = copy_to_kernel_nofault(waddr, &insn, 2 * SW64_INSN_SIZE); + + //raw_spin_unlock_irqrestore(&patch_lock, flags); + + return ret; +} + +int __kprobes sw64_insn_write(void *addr, u32 insn) +{ + u32 *tp = addr; + /* SW64 instructions must be word aligned */ + if ((uintptr_t)tp & 0x3) + return -EINVAL; + return __sw64_insn_write(addr, cpu_to_le32(insn)); +} + +int __kprobes sw64_insn_double_write(void *addr, u64 insn) +{ + u32 *tp = addr; + /* SW64 instructions must be word aligned */ + if ((uintptr_t)tp & 0x3) + return -EINVAL; + return __sw64_insn_double_write(addr, cpu_to_le64(insn)); +} +unsigned int __kprobes sw64_insn_nop(void) +{ + return SW64_BIS(R31, R31, R31); +} + +unsigned int __kprobes sw64_insn_call(unsigned int ra, unsigned int rb) +{ + return SW64_CALL(ra, rb, 0); +} + +unsigned int __kprobes sw64_insn_sys_call(unsigned int num) +{ + return SW64_SYS_CALL(num); +} + +/* 'pc' is the address of br instruction, not the +4 PC. 'new_pc' is the target address. */ +unsigned int __kprobes sw64_insn_br(unsigned int ra, unsigned long pc, unsigned long new_pc) +{ + int offset = new_pc - pc; + unsigned int disp, minus = 0x1fffff; + + if (!(offset <= BR_MAX_DISP && offset >= -BR_MAX_DISP)) + return -1; + if (offset > 0) + disp = (offset - 4) / 4; + else + disp = ~(-offset / 4) & minus; + + return SW64_BR(ra, disp); + +} diff --git a/arch/sw_64/kernel/kprobes/Makefile b/arch/sw_64/kernel/kprobes/Makefile new file mode 100644 index 000000000000..110ba2bf7752 --- /dev/null +++ b/arch/sw_64/kernel/kprobes/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_KPROBES) += kprobes.o decode-insn.o +obj-$(CONFIG_KPROBES_ON_FTRACE) += kprobes-ftrace.o diff --git a/arch/sw_64/kernel/kprobes/common.h b/arch/sw_64/kernel/kprobes/common.h new file mode 100644 index 000000000000..de10058f0376 --- /dev/null +++ b/arch/sw_64/kernel/kprobes/common.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SW64_KERNEL_KPROBES_COMMON_H +#define _SW64_KERNEL_KPROBES_COMMON_H + + +extern bool sw64_insn_can_kprobe(kprobe_opcode_t *addr); + + +#endif /* _SW64_KERNEL_KPROBES_COMMON_H */ diff --git a/arch/sw_64/kernel/kprobes/decode-insn.c b/arch/sw_64/kernel/kprobes/decode-insn.c new file mode 100644 index 000000000000..91c31111f2b7 --- /dev/null +++ b/arch/sw_64/kernel/kprobes/decode-insn.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Based on arch/arm64/kernel/probes/decode-insn.c + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include + +#include "common.h" + +static bool __kprobes sw64_insn_is_steppable(u32 insn) +{ + /* + * Branch instructions will write a new value into the PC which is + * likely to be relative to the XOL address and therefore invalid. + * Deliberate generation of an exception during stepping is also not + * currently safe. Lastly, MSR instructions can do any number of nasty + * things we can't handle during single-stepping. + */ + if (sw64_insn_is_sys_call_b(insn) || + sw64_insn_is_sys_call(insn) || + sw64_insn_is_call(insn) || + sw64_insn_is_ret(insn) || + sw64_insn_is_jmp(insn) || + sw64_insn_is_br(insn) || + sw64_insn_is_bsr(insn) || + sw64_insn_is_memb(insn) || + sw64_insn_is_imemb(insn) || + sw64_insn_is_rtc(insn) || + sw64_insn_is_lldl(insn) || + sw64_insn_is_lldw(insn) || + sw64_insn_is_beq(insn) || + sw64_insn_is_bne(insn) || + sw64_insn_is_blt(insn) || + sw64_insn_is_ble(insn) || + sw64_insn_is_bgt(insn) || + sw64_insn_is_bge(insn) || + sw64_insn_is_blbc(insn) || + sw64_insn_is_blbs(insn) || + sw64_insn_is_fbeq(insn) || + sw64_insn_is_fbne(insn) || + sw64_insn_is_fblt(insn) || + sw64_insn_is_fble(insn) || + sw64_insn_is_fbgt(insn) || + sw64_insn_is_fbge(insn)) + return false; + + return true; +} + + +#ifdef CONFIG_KPROBES +// lldl rd_f +static bool __kprobes is_probed_between_atomic(kprobe_opcode_t *addr) +{ + int count = 0; + unsigned long size = 0, offset = 0; + kprobe_opcode_t *scan_start = NULL; + + if (kallsyms_lookup_size_offset((unsigned long)addr, &size, &offset)) + scan_start = addr - (offset / sizeof(kprobe_opcode_t)); + + while (scan_start < addr) { + if (sw64_insn_is_lldl(le32_to_cpu(*scan_start)) || + sw64_insn_is_lldw(le32_to_cpu(*scan_start))) + count++; + if (sw64_insn_is_rd_f(le32_to_cpu(*scan_start))) + count--; + scan_start++; + } + if (count) + return false; + + return true; +} + +bool __kprobes sw64_insn_can_kprobe(kprobe_opcode_t *addr) +{ + u32 insn = le32_to_cpu(*addr); + + if (!sw64_insn_is_steppable(insn)) { + pr_warn("addr is not steppable\n"); + return false; + } +#ifdef CONFIG_SUBARCH_C3B + if (!is_probed_between_atomic(addr)) { + pr_warn("addr between atomic can't probe\n"); + return false; + } +#endif + return true; +} +#endif diff --git a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c new file mode 100644 index 000000000000..89d7dba9dc25 --- /dev/null +++ b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Dynamic Ftrace based Kprobes Optimization + */ + +#include +#include +#include +#include +#include + +/* Ftrace callback handler for kprobes */ +void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *ops, struct pt_regs *regs) +{ + struct kprobe *p; + struct kprobe_ctlblk *kcb; + + p = get_kprobe((kprobe_opcode_t *)ip); + if (unlikely(!p) || kprobe_disabled(p)) + return; + + kcb = get_kprobe_ctlblk(); + if (kprobe_running()) { + kprobes_inc_nmissed_count(p); + } else { + regs->regs[28] -= MCOUNT_INSN_SIZE; + + __this_cpu_write(current_kprobe, p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + if (!p->pre_handler || !p->pre_handler(p, regs)) { + regs->regs[28] += MCOUNT_INSN_SIZE; + if (unlikely(p->post_handler)) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + p->post_handler(p, regs, 0); + } + } + __this_cpu_write(current_kprobe, NULL); + } +} +NOKPROBE_SYMBOL(kprobe_ftrace_handler); + +int arch_prepare_kprobe_ftrace(struct kprobe *p) +{ + p->ainsn.insn = NULL; + p->ainsn.boostable = -1; + return 0; +} diff --git a/arch/sw_64/kernel/kprobes/kprobes.c b/arch/sw_64/kernel/kprobes/kprobes.c new file mode 100644 index 000000000000..024ce7d99e61 --- /dev/null +++ b/arch/sw_64/kernel/kprobes/kprobes.c @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Kernel Probes (KProbes) + * arch/sw_64/kernel/kprobes.c + */ + +#include +#include +#include + +#include "common.h" + +static u32 breakpoint_insn = BREAK_KPROBE; +static u32 breakpoint2_insn = BREAK_KPROBE_SS; + +int post_kprobe_handler(struct pt_regs *regs); + +DEFINE_PER_CPU(struct kprobe *, current_kprobe); +DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); + +int __kprobes arch_prepare_kprobe(struct kprobe *p) +{ + int ret = 0; + extern char __start_rodata[]; + extern char __end_rodata[]; + unsigned long probe_addr = (unsigned long)p->addr; + + if (probe_addr & 0x3) + return -EINVAL; + + if (!sw64_insn_can_kprobe(p->addr)) + return -EINVAL; + /* copy instruction */ + p->opcode = le32_to_cpu(*p->addr); + + + if (probe_addr >= (unsigned long) __start_rodata && + probe_addr <= (unsigned long) __end_rodata) + return -EINVAL; + + + /* insn: must be on special executable page on mips. */ + p->ainsn.insn = get_insn_slot(); + if (!p->ainsn.insn) { + ret = -ENOMEM; + goto out; + } + /* + * In the kprobe->ainsn.insn[] array we store the original + * instruction at index zero and a break trap instruction at + * index one. + */ + p->ainsn.insn[0] = p->opcode; + p->ainsn.insn[1] = breakpoint2_insn; +out: + return ret; +} + +void __kprobes arch_arm_kprobe(struct kprobe *p) +{ + sw64_insn_write(p->addr, breakpoint_insn); + flush_insn_slot(p); +} + +void __kprobes arch_disarm_kprobe(struct kprobe *p) +{ + sw64_insn_write(p->addr, p->opcode); + flush_insn_slot(p); +} + +void __kprobes arch_remove_kprobe(struct kprobe *p) +{ + if (p->ainsn.insn) { + free_insn_slot(p->ainsn.insn, 0); + p->ainsn.insn = NULL; + } +} + +static void save_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + kcb->prev_kprobe.kp = kprobe_running(); + kcb->prev_kprobe.status = kcb->kprobe_status; +} + +static void restore_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); + kcb->kprobe_status = kcb->prev_kprobe.status; +} + +static void __kprobes set_current_kprobe(struct kprobe *p) +{ + __this_cpu_write(current_kprobe, p); +} + + +static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb, int reenter) +{ + if (reenter) { + save_previous_kprobe(kcb); + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_REENTER; + } else { + kcb->kprobe_status = KPROBE_HIT_SS; + } + + /* insn simulation */ + kcb->target_pc = regs->pc; + regs->pc = (unsigned long)&p->ainsn.insn[0]; +} + +static int __kprobes reenter_kprobe(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + switch (kcb->kprobe_status) { + case KPROBE_HIT_SSDONE: + case KPROBE_HIT_ACTIVE: + kprobes_inc_nmissed_count(p); + setup_singlestep(p, regs, kcb, 1); + break; + case KPROBE_HIT_SS: + case KPROBE_REENTER: + pr_warn("Unrecoverable kprobe detected.\n"); + dump_kprobe(p); + BUG(); + break; + default: + WARN_ON(1); + return 0; + } + return 1; +} + +int __kprobes kprobe_handler(struct pt_regs *regs) +{ + struct kprobe *p; + struct kprobe_ctlblk *kcb; + unsigned long addr = instruction_pointer(regs); + + if (user_mode(regs)) + return 0; + /* + * We don't want to be preempted for the entire + * duration of kprobe processing + */ + preempt_disable(); + kcb = get_kprobe_ctlblk(); + p = get_kprobe((kprobe_opcode_t *)(addr - 4)); + + if (p) { + if (kprobe_running()) { + if (reenter_kprobe(p, regs, kcb)) + return 1; + } else { + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + + /* + * If we have no pre-handler or it returned 0, we + * continue with normal processing. If we have a + * pre-handler and it returned non-zero, that means + * user handler setup registers to exit to another + * instruction, we must skip the single stepping. + */ + if (!p->pre_handler || !p->pre_handler(p, regs)) + setup_singlestep(p, regs, kcb, 0); + else + reset_current_kprobe(); + return 1; + } + } + return 0; + +} +int __kprobes post_kprobe_handler(struct pt_regs *regs) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if (!cur) + return 0; + + if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + cur->post_handler(cur, regs, 0); + } + + // resume_execution(cur, regs, kcb); + regs->pc = kcb->target_pc; + + + /* Restore back the original saved kprobes variables and continue. */ + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + goto out; + } + reset_current_kprobe(); +out: + preempt_enable_no_resched(); + + return 1; +} + +int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned long mmcsr) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if (kcb->kprobe_status & KPROBE_HIT_SS) { + regs->pc = kcb->target_pc; + + reset_current_kprobe(); + preempt_enable_no_resched(); + } + return 0; +} + +/* + * Wrapper routine for handling exceptions. + */ +int __kprobes kprobe_exceptions_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + + struct die_args *args = (struct die_args *)data; + int ret = NOTIFY_DONE; + + switch (val) { + case DIE_BREAK: + if (kprobe_handler(args->regs)) + ret = NOTIFY_STOP; + break; + case DIE_SSTEPBP: + if (post_kprobe_handler(args->regs)) + ret = NOTIFY_STOP; + break; + default: + break; + } + return ret; +} +/* + * Function return probe trampoline: + * - init_kprobes() establishes a probepoint here + * - When the probed function returns, this probe causes the + * handlers to fire + */ +static void __used kretprobe_trampoline_holder(void) +{ + asm volatile( + /* Keep the assembler from reordering and placing JR here. */ + ".set noreorder\n\t" + "nop\n\t" + ".global __kretprobe_trampoline\n" + "__kretprobe_trampoline:\n\t" + "nop\n\t" + : : : "memory"); +} + +void __kretprobe_trampoline(void); + +void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, + struct pt_regs *regs) +{ + ri->ret_addr = (kprobe_opcode_t *) regs->regs[26]; + ri->fp = NULL; + + /* Replace the return addr with trampoline addr */ + regs->regs[26] = (unsigned long)__kretprobe_trampoline; +} + +/* + * Called when the probe at kretprobe trampoline is hit + */ +static int __kprobes trampoline_probe_handler(struct kprobe *p, + struct pt_regs *regs) +{ + unsigned long orig_ret_address; + + orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); + instruction_pointer(regs) = orig_ret_address; + regs->regs[26] = orig_ret_address; + + /* + * By returning a non-zero value, we are telling + * kprobe_handler() that we don't want the post_handler + * to run (and have re-enabled preemption) + */ + return 1; +} + +int __kprobes arch_trampoline_kprobe(struct kprobe *p) +{ + if (p->addr == (kprobe_opcode_t *)__kretprobe_trampoline) + return 1; + + return 0; +} + +static struct kprobe trampoline_p = { + .addr = (kprobe_opcode_t *)__kretprobe_trampoline, + .pre_handler = trampoline_probe_handler +}; + +int __init arch_init_kprobes(void) +{ + return register_kprobe(&trampoline_p); +} -- Gitee From 7a9a9d0d707037ffd09f4df6e1628e47dd1b9383 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:38 +0800 Subject: [PATCH 046/524] sw64: add uprobe support Add uprobe support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uprobes.h | 45 ++++++++ arch/sw_64/kernel/uprobes.c | 182 +++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 arch/sw_64/include/asm/uprobes.h create mode 100644 arch/sw_64/kernel/uprobes.c diff --git a/arch/sw_64/include/asm/uprobes.h b/arch/sw_64/include/asm/uprobes.h new file mode 100644 index 000000000000..fcd2026c3622 --- /dev/null +++ b/arch/sw_64/include/asm/uprobes.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#ifndef _ASM_SW64_UPROBES_H +#define _ASM_SW64_UPROBES_H + +#include +#include +#include + +/* + * We want this to be defined as union sw64_instruction but that makes the + * generic code blow up. + */ +typedef u32 uprobe_opcode_t; + +#define MAX_UINSN_BYTES SW64_INSN_SIZE +#define UPROBE_XOL_SLOT_BYTES SW64_INSN_SIZE + +#define UPROBE_BRK_UPROBE 0x000d000d /* break 13 */ +#define UPROBE_BRK_UPROBE_XOL 0x000e000d /* break 14 */ + +#define UPROBE_SWBP_INSN UPROBE_BRK_UPROBE +#define UPROBE_SWBP_INSN_SIZE MAX_UINSN_BYTES + +struct arch_uprobe { + u32 insn; + u32 ixol[2]; +}; + +struct arch_uprobe_task { + unsigned long saved_trap_nr; +}; + +#ifdef CONFIG_UPROBES +void sw64_fix_uretprobe(struct pt_regs *regs, unsigned long exc_pc); +#else +static inline void +sw64_fix_uretprobe(struct pt_regs *regs, unsigned long exc_pc) {} +#endif + +#endif /* _ASM_SW64_UPROBES_H */ diff --git a/arch/sw_64/kernel/uprobes.c b/arch/sw_64/kernel/uprobes.c new file mode 100644 index 000000000000..928312d62cfd --- /dev/null +++ b/arch/sw_64/kernel/uprobes.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +/** + * arch_uprobe_analyze_insn - instruction analysis including validity and fixups. + * @mm: the probed address space. + * @arch_uprobe: the probepoint information. + * @addr: virtual address at which to install the probepoint + * Return 0 on success or a -ve number on error. + */ +int arch_uprobe_analyze_insn(struct arch_uprobe *aup, + struct mm_struct *mm, unsigned long addr) +{ + u32 inst; + + if (addr & 0x03) + return -EINVAL; + + inst = aup->insn; + + aup->ixol[0] = aup->insn; + aup->ixol[1] = UPROBE_BRK_UPROBE_XOL; /* NOP */ + + return 0; +} + +void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, + void *src, unsigned long len) +{ + unsigned long kaddr, kstart; + + /* Initialize the slot */ + kaddr = (unsigned long)kmap_local_page(page); + kstart = kaddr + (vaddr & ~PAGE_MASK); + memcpy((void *)kstart, src, len); + flush_icache_range(kstart, kstart + len); + kunmap_local((void *)kaddr); +} + +/* + * arch_uprobe_pre_xol - prepare to execute out of line. + * @auprobe: the probepoint information. + * @regs: reflects the saved user state of current task. + */ +int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + /* Instruction points to execute ol */ + instruction_pointer_set(regs, utask->xol_vaddr); + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *aup, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + /* Instruction points to execute next to breakpoint address */ + instruction_pointer_set(regs, utask->vaddr + 4); + + return 0; +} + +/* + * If xol insn itself traps and generates a signal(Say, + * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped + * instruction jumps back to its own address. It is assumed that anything + * like do_page_fault/do_trap/etc sets thread.trap_nr != -1. + * + * arch_uprobe_pre_xol/arch_uprobe_post_xol save/restore thread.trap_nr, + * arch_uprobe_xol_was_trapped() simply checks that ->trap_nr is not equal to + * UPROBE_TRAP_NR == -1 set by arch_uprobe_pre_xol(). + */ +bool arch_uprobe_xol_was_trapped(struct task_struct *tsk) +{ + return false; +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + struct die_args *args = data; + struct pt_regs *regs = args->regs; + + /* regs == NULL is a kernel bug */ + if (WARN_ON(!regs)) + return NOTIFY_DONE; + + /* We are only interested in userspace traps */ + if (!user_mode(regs)) + return NOTIFY_DONE; + + switch (val) { + case DIE_UPROBE: + if (uprobe_pre_sstep_notifier(regs)) + return NOTIFY_STOP; + break; + case DIE_UPROBE_XOL: + if (uprobe_post_sstep_notifier(regs)) + return NOTIFY_STOP; + default: + break; + } + + return 0; +} + +/* + * This function gets called when XOL instruction either gets trapped or + * the thread has a fatal signal. Reset the instruction pointer to its + * probed address for the potential restart or for post mortem analysis. + */ +void arch_uprobe_abort_xol(struct arch_uprobe *aup, + struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + instruction_pointer_set(regs, utask->vaddr); +} + +unsigned long arch_uretprobe_hijack_return_addr( + unsigned long trampoline_vaddr, struct pt_regs *regs) +{ + unsigned long ra; + + ra = regs->regs[26]; + + /* Replace the return address with the trampoline address */ + regs->regs[26] = trampoline_vaddr; + + return ra; +} + +/* + * See if the instruction can be emulated. + * Returns true if instruction was emulated, false otherwise. + * + * For now we always emulate so this function just returns 0. + */ +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + return 0; +} + +/* + * struct xol_area and get_trampoline_vaddr() are copied from + * kernel/events/uprobes.c to avoid modifying arch-independent + * code. + */ +struct xol_area { + wait_queue_head_t wq; + atomic_t slot_count; + unsigned long *bitmap; + struct vm_special_mapping xol_mapping; + struct page *pages[2]; + unsigned long vaddr; +}; + +static unsigned long get_trampoline_vaddr(void) +{ + struct xol_area *area; + unsigned long trampoline_vaddr = -1; + + area = READ_ONCE(current->mm->uprobes_state.xol_area); + if (area) + trampoline_vaddr = area->vaddr; + + return trampoline_vaddr; +} + +void sw64_fix_uretprobe(struct pt_regs *regs, unsigned long exc_pc) +{ + /* + * regs->pc has been changed to orig_ret_vaddr in handle_trampoline(). + */ + if (exc_pc == get_trampoline_vaddr()) + regs->regs[26] = regs->pc; +} -- Gitee From bf33cc51e99df6acc80808943da7fd96efa27c62 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:16 +0800 Subject: [PATCH 047/524] sw64: add jump_label support Add jump_label support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/jump_label.h | 50 +++++++++++++++++++++++++++++ arch/sw_64/kernel/jump_label.c | 32 ++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 arch/sw_64/include/asm/jump_label.h create mode 100644 arch/sw_64/kernel/jump_label.c diff --git a/arch/sw_64/include/asm/jump_label.h b/arch/sw_64/include/asm/jump_label.h new file mode 100644 index 000000000000..32fbf7573b20 --- /dev/null +++ b/arch/sw_64/include/asm/jump_label.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SW64_JUMP_LABEL_H +#define _ASM_SW64_JUMP_LABEL_H + +#ifndef __ASSEMBLY__ + +#include +#include + +#define JUMP_LABEL_NOP_SIZE SW64_INSN_SIZE + +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) +{ + asm_volatile_goto("1: nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 3\n\t" + ".quad 1b, %l[l_yes], %0\n\t" + ".popsection\n\t" + : : "i"(&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1: br %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 3\n\t" + ".quad 1b, %l[l_yes], %0\n\t" + ".popsection\n\t" + : : "i"(&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +typedef u64 jump_label_t; + +struct jump_entry { + jump_label_t code; + jump_label_t target; + jump_label_t key; +}; + +#endif /* __ASSEMBLY__ */ +#endif /* _ASM_SW64_JUMP_LABEL_H */ diff --git a/arch/sw_64/kernel/jump_label.c b/arch/sw_64/kernel/jump_label.c new file mode 100644 index 000000000000..f3bc40370e4d --- /dev/null +++ b/arch/sw_64/kernel/jump_label.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include +#include + +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) +{ + u32 *insnp = (u32 *)entry->code; + u32 insn; + + if (type == JUMP_LABEL_JMP) { + insn = sw64_insn_br(R31, (entry->code), entry->target); + BUG_ON(insn == -1); + } else { + insn = sw64_insn_nop(); + } + + *insnp = insn; + + flush_icache_range(entry->code, entry->code + SW64_INSN_SIZE); +} + +void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type) +{ + /* + * no need to rewrite NOP + */ +} -- Gitee From adef427fe40eab08dabfcd236ac4250bd0baf575 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:17 +0800 Subject: [PATCH 048/524] sw64: add kgdb support Add kgdb support for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kgdb.h | 68 ++++++++++ arch/sw_64/kernel/kgdb.c | 233 ++++++++++++++++++++++++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 arch/sw_64/include/asm/kgdb.h create mode 100644 arch/sw_64/kernel/kgdb.c diff --git a/arch/sw_64/include/asm/kgdb.h b/arch/sw_64/include/asm/kgdb.h new file mode 100644 index 000000000000..a00a45ce767c --- /dev/null +++ b/arch/sw_64/include/asm/kgdb.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sw64 KGDB support + * + * Based on arch/arm64/include/kgdb.h + * + * Copyright (C) Xia Bin + * Author: Xia Bin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#ifndef _ASM_SW64_KGDB_H +#define _ASM_SW64_KGDB_H + +#include +#include + +#ifndef __ASSEMBLY__ + + +#define GDB_ADJUSTS_BREAK_OFFSET +#define BREAK_INSTR_SIZE 4 +#define CACHE_FLUSH_IS_SAFE 0 + +static inline void arch_kgdb_breakpoint(void) +{ + __asm__ __volatile__("sys_call %0" : : "i"(HMC_bpt)); +} + +void sw64_task_to_gdb_regs(struct task_struct *task, unsigned long *regs); + +extern void kgdb_handle_bus_error(void); +extern int kgdb_fault_expected; +extern unsigned long get_reg(struct task_struct *task, unsigned long regno); + +#endif /* !__ASSEMBLY__ */ + +/* + * general purpose registers size in bytes. + */ +#define DBG_MAX_REG_NUM (67) + +/* + * Size of I/O buffer for gdb packet. + * considering to hold all register contents, size is set + */ + +#define BUFMAX 4096 + +/* + * Number of bytes required for gdb_regs buffer. + * _GP_REGS: 8 bytes, _FP_REGS: 16 bytes and _EXTRA_REGS: 4 bytes each + * GDB fails to connect for size beyond this with error + * "'g' packet reply is too long" + */ +#define NUMREGBYTES (DBG_MAX_REG_NUM * 8) + +#endif /* _ASM_SW64_KGDB_H */ diff --git a/arch/sw_64/kernel/kgdb.c b/arch/sw_64/kernel/kgdb.c new file mode 100644 index 000000000000..833f72a1577c --- /dev/null +++ b/arch/sw_64/kernel/kgdb.c @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sw64 KGDB support + * + * Based on arch/arm64/kernel/kgdb.c + * + * Copyright (C) Xia Bin + * Author: Xia Bin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include + +struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { + { "r0", 8, offsetof(struct pt_regs, regs[0])}, + { "r1", 8, offsetof(struct pt_regs, regs[1])}, + { "r2", 8, offsetof(struct pt_regs, regs[2])}, + { "r3", 8, offsetof(struct pt_regs, regs[3])}, + { "r4", 8, offsetof(struct pt_regs, regs[4])}, + { "r5", 8, offsetof(struct pt_regs, regs[5])}, + { "r6", 8, offsetof(struct pt_regs, regs[6])}, + { "r7", 8, offsetof(struct pt_regs, regs[7])}, + { "r8", 8, offsetof(struct pt_regs, regs[8])}, + + { "r9", 8, offsetof(struct pt_regs, regs[9])}, + { "r10", 8, offsetof(struct pt_regs, regs[10])}, + { "r11", 8, offsetof(struct pt_regs, regs[11])}, + { "r12", 8, offsetof(struct pt_regs, regs[12])}, + { "r13", 8, offsetof(struct pt_regs, regs[13])}, + { "r14", 8, offsetof(struct pt_regs, regs[14])}, + { "r15", 8, offsetof(struct pt_regs, regs[15])}, + + { "r16", 8, offsetof(struct pt_regs, regs[16])}, + { "r17", 8, offsetof(struct pt_regs, regs[17])}, + { "r18", 8, offsetof(struct pt_regs, regs[18])}, + + { "r19", 8, offsetof(struct pt_regs, regs[19])}, + { "r20", 8, offsetof(struct pt_regs, regs[20])}, + { "r21", 8, offsetof(struct pt_regs, regs[21])}, + { "r22", 8, offsetof(struct pt_regs, regs[22])}, + { "r23", 8, offsetof(struct pt_regs, regs[23])}, + { "r24", 8, offsetof(struct pt_regs, regs[24])}, + { "r25", 8, offsetof(struct pt_regs, regs[25])}, + { "r26", 8, offsetof(struct pt_regs, regs[26])}, + { "r27", 8, offsetof(struct pt_regs, regs[27])}, + { "at", 8, offsetof(struct pt_regs, regs[28])}, + { "gp", 8, offsetof(struct pt_regs, regs[29])}, + { "sp", 8, offsetof(struct pt_regs, regs[30])}, + { "zero", 8, -1 }, + + { "f0", 8, -1 }, + { "f1", 8, -1 }, + { "f2", 8, -1 }, + { "f3", 8, -1 }, + { "f4", 8, -1 }, + { "f5", 8, -1 }, + { "f6", 8, -1 }, + { "f7", 8, -1 }, + { "f8", 8, -1 }, + { "f9", 8, -1 }, + { "f10", 8, -1 }, + { "f11", 8, -1 }, + { "f12", 8, -1 }, + { "f13", 8, -1 }, + { "f14", 8, -1 }, + { "f15", 8, -1 }, + { "f16", 8, -1 }, + { "f17", 8, -1 }, + { "f18", 8, -1 }, + { "f19", 8, -1 }, + { "f20", 8, -1 }, + { "f21", 8, -1 }, + { "f22", 8, -1 }, + { "f23", 8, -1 }, + { "f24", 8, -1 }, + { "f25", 8, -1 }, + { "f26", 8, -1 }, + { "f27", 8, -1 }, + { "f28", 8, -1 }, + { "f29", 8, -1 }, + { "f30", 8, -1 }, + { "fpcr", 8, -1 }, + + { "pc", 8, offsetof(struct pt_regs, pc)}, + { "", 8, -1 }, + { "tp", 8, -1}, +}; + +char *dbg_get_reg(int regno, void *mem, struct pt_regs *regs) +{ + if (regno >= DBG_MAX_REG_NUM || regno < 0) + return NULL; + + if (dbg_reg_def[regno].offset != -1) + memcpy(mem, (void *)regs + dbg_reg_def[regno].offset, + dbg_reg_def[regno].size); + else + memset(mem, 0, dbg_reg_def[regno].size); + return dbg_reg_def[regno].name; +} + +int dbg_set_reg(int regno, void *mem, struct pt_regs *regs) +{ + if (regno >= DBG_MAX_REG_NUM || regno < 0) + return -EINVAL; + + if (dbg_reg_def[regno].offset != -1) + memcpy((void *)regs + dbg_reg_def[regno].offset, mem, + dbg_reg_def[regno].size); + return 0; +} + +void +sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task) +{ + int i; + /* Initialize to zero */ + memset((char *)gdb_regs, 0, NUMREGBYTES); + for (i = 0; i < DBG_MAX_REG_NUM; i++) + gdb_regs[i] = get_reg(task, i); +} + +void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long pc) +{ + pr_info("BEFORE SET PC WITH %lx\n", pc); + instruction_pointer(regs) = pc; + pr_info("AFTER SET PC IS %lx\n", instruction_pointer(regs)); +} + +void kgdb_call_nmi_hook(void *ignored) +{ + kgdb_nmicallback(raw_smp_processor_id(), NULL); +} + +void kgdb_roundup_cpus(void) +{ + local_irq_enable(); + smp_call_function(kgdb_call_nmi_hook, NULL, 0); + local_irq_disable(); +} + +int kgdb_arch_handle_exception(int exception_vector, int signo, + int err_code, char *remcom_in_buffer, + char *remcom_out_buffer, + struct pt_regs *linux_regs) +{ + char *ptr; + unsigned long address = -1; + + switch (remcom_in_buffer[0]) { + case 'c': + ptr = &remcom_in_buffer[1]; + if (kgdb_hex2long(&ptr, &address)) + kgdb_arch_set_pc(linux_regs, address); + return 0; + } + return -1; +} + +static int __kgdb_notify(struct die_args *args, unsigned long cmd) +{ + struct pt_regs *regs = args->regs; + + /* Userspace events, ignore. */ + if (user_mode(regs)) + return NOTIFY_DONE; + + if (kgdb_handle_exception(1, args->signr, cmd, regs)) + return NOTIFY_DONE; + + return NOTIFY_STOP; +} + +static int +kgdb_notify(struct notifier_block *self, unsigned long cmd, void *ptr) +{ + unsigned long flags; + int ret; + + local_irq_save(flags); + ret = __kgdb_notify(ptr, cmd); + local_irq_restore(flags); + + return ret; +} + +static struct notifier_block kgdb_notifier = { + .notifier_call = kgdb_notify, +}; + +/* + * kgdb_arch_init - Perform any architecture specific initalization. + * This function will handle the initalization of any architecture + * specific callbacks. + */ +int kgdb_arch_init(void) +{ + int ret = register_die_notifier(&kgdb_notifier); + + if (ret != 0) + return ret; + return 0; +} + +/* + * kgdb_arch_exit - Perform any architecture specific uninitalization. + * This function will handle the uninitalization of any architecture + * specific callbacks, for dynamic registration and unregistration. + */ +void kgdb_arch_exit(void) +{ + unregister_die_notifier(&kgdb_notifier); +} + +/* + * sw64 instructions are always in LE. + * Break instruction is encoded in LE format + */ +const struct kgdb_arch arch_kgdb_ops = { + .gdb_bpt_instr = {0x80, 00, 00, 00} +}; -- Gitee From 1f49c9ddc5b5ff5f2c9ba43235d1cf42f4f363cc Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:30 +0800 Subject: [PATCH 049/524] sw64: add dynamic frequency scaling support Add dynamic frequency scaling support for SW64 based xuelang platform. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpufreq.h | 66 ++++++++++++ arch/sw_64/platform/Makefile | 2 + arch/sw_64/platform/cpufreq_xuelang.c | 140 ++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 arch/sw_64/include/asm/cpufreq.h create mode 100644 arch/sw_64/platform/Makefile create mode 100644 arch/sw_64/platform/cpufreq_xuelang.c diff --git a/arch/sw_64/include/asm/cpufreq.h b/arch/sw_64/include/asm/cpufreq.h new file mode 100644 index 000000000000..cf47f1fc6866 --- /dev/null +++ b/arch/sw_64/include/asm/cpufreq.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SW64_CPUFREQ_H +#define _ASM_SW64_CPUFREQ_H + +#include +#include +#include +#include +#include + +struct clk; + +extern char curruent_policy[CPUFREQ_NAME_LEN]; + +struct clk_ops { + void (*init)(struct clk *clk); + void (*enable)(struct clk *clk); + void (*disable)(struct clk *clk); + void (*recalc)(struct clk *clk); + int (*set_rate)(struct clk *clk, unsigned long rate, int algo_id); + long (*round_rate)(struct clk *clk, unsigned long rate); +}; + +struct clk { + struct list_head node; + const char *name; + int id; + struct module *owner; + + struct clk *parent; + const struct clk_ops *ops; + + struct kref kref; + + unsigned long rate; + unsigned long flags; +}; + +#define CLK_ALWAYS_ENABLED (1 << 0) +#define CLK_RATE_PROPAGATES (1 << 1) + +#define CLK_PRT 0x1UL +#define CORE_CLK0_V (0x1UL << 1) +#define CORE_CLK0_R (0x1UL << 2) +#define CORE_CLK2_V (0x1UL << 15) +#define CORE_CLK2_R (0x1UL << 16) + +#define CLK_LV1_SEL_PRT 0x1UL +#define CLK_LV1_SEL_MUXA (0x1UL << 2) +#define CLK_LV1_SEL_MUXB (0x1UL << 3) + +#define CORE_PLL0_CFG_SHIFT 4 +#define CORE_PLL2_CFG_SHIFT 18 + +extern struct cpufreq_frequency_table freq_table[]; + +int clk_init(void); +void sw64_set_rate(unsigned int index); + +struct clk *sw64_clk_get(struct device *dev, const char *id); + +void sw64_update_clockevents(unsigned long cpu, u32 freq); + +unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy); +#endif /* _ASM_SW64_CPUFREQ_H */ diff --git a/arch/sw_64/platform/Makefile b/arch/sw_64/platform/Makefile new file mode 100644 index 000000000000..4c0edceb4a2c --- /dev/null +++ b/arch/sw_64/platform/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_PLATFORM_XUELANG) += cpufreq_xuelang.o diff --git a/arch/sw_64/platform/cpufreq_xuelang.c b/arch/sw_64/platform/cpufreq_xuelang.c new file mode 100644 index 000000000000..1259e58dc874 --- /dev/null +++ b/arch/sw_64/platform/cpufreq_xuelang.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include +#include +#include + +/* Minimum CLK support */ +enum { + DC_0, DC_1, DC_2, DC_3, DC_4, DC_5, DC_6, DC_7, DC_8, + DC_9, DC_10, DC_11, DC_12, DC_13, DC_14, DC_15, DC_RESV +}; + +struct cpufreq_frequency_table freq_table[] = { + {0, 200, CPUFREQ_ENTRY_INVALID}, + {0, DC_1, CPUFREQ_ENTRY_INVALID}, + {0, DC_2, 0}, + {0, DC_3, 0}, + {0, DC_4, 0}, + {0, DC_5, 0}, + {0, DC_6, 0}, + {0, DC_7, 0}, + {0, DC_8, 0}, + {0, DC_9, 0}, + {0, DC_10, 0}, + {0, DC_11, 0}, + {0, DC_12, 0}, + {0, DC_13, 0}, + {0, DC_14, 0}, + {0, DC_15, 0}, + {-1, DC_RESV, CPUFREQ_TABLE_END}, +}; + + +static struct platform_device sw64_cpufreq_device = { + .name = "sw64_cpufreq", + .id = -1, +}; + +static int __init sw64_cpufreq_init(void) +{ + int i; + unsigned char external_clk; + unsigned long max_rate, freq_off; + + max_rate = get_cpu_freq() / 1000; + + external_clk = *((unsigned char *)__va(MB_EXTCLK)); + + if (external_clk == 240) + freq_off = 60000; + else + freq_off = 50000; + + /* clock table init */ + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (i == 1) + freq_table[i].driver_data = freq_off * 24; + if (i == 2) + freq_table[i].frequency = freq_off * 36; + if (i > 2) + freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); + + if (freq_table[i].frequency == max_rate) + freq_table[i + 1].frequency = CPUFREQ_TABLE_END; + } + + return platform_device_register(&sw64_cpufreq_device); +} +arch_initcall(sw64_cpufreq_init); + +char curruent_policy[CPUFREQ_NAME_LEN]; + +static struct clk cpu_clk = { + .name = "cpu_clk", + .flags = CLK_ALWAYS_ENABLED | CLK_RATE_PROPAGATES, + .rate = 2400000000, +}; + +struct clk *sw64_clk_get(struct device *dev, const char *id) +{ + return &cpu_clk; +} +EXPORT_SYMBOL(sw64_clk_get); + +unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy) +{ + int i; + u64 val; + struct cpufreq_frequency_table *ft = policy->freq_table; + + val = sw64_io_read(0, CLK_CTL) >> CORE_PLL2_CFG_SHIFT; + + for (i = 0; ft[i].frequency != CPUFREQ_TABLE_END; i++) { + if (val == i) + return ft[i].frequency; + } + return 0; +} +EXPORT_SYMBOL(__sw64_cpufreq_get); + +void sw64_set_rate(unsigned int index) +{ + unsigned int i, val; + int cpu_num; + + cpu_num = sw64_chip->get_cpu_num(); + + for (i = 0; i < cpu_num; i++) { + sw64_io_write(i, CLK_CTL, CORE_CLK2_R | CORE_CLK2_V | CLK_PRT); + val = sw64_io_read(i, CLK_CTL); + + sw64_io_write(i, CLK_CTL, val | index << CORE_PLL2_CFG_SHIFT); + + udelay(1); + + sw64_io_write(i, CLK_CTL, CORE_CLK2_V | CLK_PRT + | index << CORE_PLL2_CFG_SHIFT); + val = sw64_io_read(i, CLK_CTL); + + /* LV1 select PLL1/PLL2 */ + sw64_io_write(i, CLU_LV1_SEL, CLK_LV1_SEL_MUXA | CLK_LV1_SEL_PRT); + + /* Set CLK_CTL PLL0 */ + sw64_io_write(i, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V); + + sw64_io_write(i, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V + | index << CORE_PLL0_CFG_SHIFT); + + udelay(1); + + sw64_io_write(i, CLK_CTL, val | CORE_CLK0_V + | index << CORE_PLL0_CFG_SHIFT); + + /* LV1 select PLL0/PLL1 */ + sw64_io_write(i, CLU_LV1_SEL, CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PRT); + } +} +EXPORT_SYMBOL_GPL(sw64_set_rate); -- Gitee From 803cde919bf2df120dfdffbe6c4acb43aed7fd28 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:20:06 +0800 Subject: [PATCH 050/524] sw64: add dynamic turning on/off cores support Add dynamic turning on/off cores support for SW64 based xuelang platform. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cputime.h | 9 + arch/sw_64/kernel/cpuautoplug.c | 485 +++++++++++++++++++++++++++++++ 2 files changed, 494 insertions(+) create mode 100644 arch/sw_64/include/asm/cputime.h create mode 100644 arch/sw_64/kernel/cpuautoplug.c diff --git a/arch/sw_64/include/asm/cputime.h b/arch/sw_64/include/asm/cputime.h new file mode 100644 index 000000000000..cdd46b05e228 --- /dev/null +++ b/arch/sw_64/include/asm/cputime.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CPUTIME_H +#define _ASM_SW64_CPUTIME_H + +typedef u64 __nocast cputime64_t; + +#define jiffies64_to_cputime64(__jif) ((__force cputime64_t)(__jif)) + +#endif /* _ASM_SW64_CPUTIME_H */ diff --git a/arch/sw_64/kernel/cpuautoplug.c b/arch/sw_64/kernel/cpuautoplug.c new file mode 100644 index 000000000000..a7571a77a72c --- /dev/null +++ b/arch/sw_64/kernel/cpuautoplug.c @@ -0,0 +1,485 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +int autoplug_enabled; +int autoplug_verbose; +int autoplug_adjusting; + +DEFINE_PER_CPU(int, cpu_adjusting); + +struct cpu_autoplug_info { + cputime64_t prev_idle; + cputime64_t prev_wall; + struct delayed_work work; + unsigned int sampling_rate; + int maxcpus; /* max cpus for autoplug */ + int mincpus; /* min cpus for autoplug */ + int dec_reqs; /* continuous core-decreasing requests */ + int inc_reqs; /* continuous core-increasing requests */ +}; + +struct cpu_autoplug_info ap_info; + +static ssize_t enabled_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", autoplug_enabled); +} + + +static ssize_t enabled_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + char val[5]; + int n; + + memcpy(val, buf, count); + n = kstrtol(val, 0, 0); + + if (n > 1 || n < 0) + return -EINVAL; + + autoplug_enabled = n; + + return count; +} + +static ssize_t verbose_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", autoplug_verbose); +} + +static ssize_t verbose_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + char val[5]; + int n; + + memcpy(val, buf, count); + n = kstrtol(val, 0, 0); + + if (n > 1 || n < 0) + return -EINVAL; + + autoplug_verbose = n; + + return count; +} + +static ssize_t maxcpus_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", ap_info.maxcpus); +} + +static ssize_t maxcpus_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + char val[5]; + int n; + + memcpy(val, buf, count); + n = kstrtol(val, 0, 0); + + if (n > num_possible_cpus() || n < ap_info.mincpus) + return -EINVAL; + + ap_info.maxcpus = n; + + return count; +} + +static ssize_t mincpus_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", ap_info.mincpus); +} + +static ssize_t mincpus_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + char val[5]; + int n; + + memcpy(val, buf, count); + n = kstrtol(val, 0, 0); + + if (n > ap_info.maxcpus || n < 1) + return -EINVAL; + + ap_info.mincpus = n; + + return count; +} + +static ssize_t sampling_rate_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", ap_info.sampling_rate); +} + +#define SAMPLING_RATE_MAX 1000 +#define SAMPLING_RATE_MIN 600 + +static ssize_t sampling_rate_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + char val[6]; + int n; + + memcpy(val, buf, count); + n = kstrtol(val, 0, 0); + + if (n > SAMPLING_RATE_MAX || n < SAMPLING_RATE_MIN) + return -EINVAL; + + ap_info.sampling_rate = n; + + return count; +} + +static ssize_t available_value_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "enabled: 0-1\nverbose: 0-1\nmaxcpus:" + "1-%d\nmincpus: 1-%d\nsampling_rate: %d-%d\n", + num_possible_cpus(), num_possible_cpus(), + SAMPLING_RATE_MIN, SAMPLING_RATE_MAX); +} + +static DEVICE_ATTR_RW(enabled); +static DEVICE_ATTR_RW(verbose); +static DEVICE_ATTR_RW(maxcpus); +static DEVICE_ATTR_RW(mincpus); +static DEVICE_ATTR_RW(sampling_rate); +static DEVICE_ATTR_RO(available_value); + +static struct attribute *cpuclass_default_attrs[] = { + &dev_attr_enabled.attr, + &dev_attr_verbose.attr, + &dev_attr_maxcpus.attr, + &dev_attr_mincpus.attr, + &dev_attr_sampling_rate.attr, + &dev_attr_available_value.attr, + NULL +}; + +static struct attribute_group cpuclass_attr_group = { + .attrs = cpuclass_default_attrs, + .name = "cpuautoplug", +}; + +static int __init setup_autoplug(char *str) +{ + if (!strcmp(str, "off")) + autoplug_enabled = 0; + else if (!strcmp(str, "on")) + autoplug_enabled = 1; + else + return 0; + return 1; +} + +__setup("autoplug=", setup_autoplug); + +static cputime64_t calc_busy_time(unsigned int cpu) +{ + cputime64_t busy_time; + + busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; + busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; + busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; + busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; + busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; + busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; + busy_time += 1; + + return busy_time; +} + +static inline cputime64_t get_idle_time_jiffy(cputime64_t *wall) +{ + unsigned int cpu; + cputime64_t idle_time = 0; + cputime64_t cur_wall_time; + cputime64_t busy_time; + + cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); + + for_each_online_cpu(cpu) { + busy_time = calc_busy_time(cpu); + + idle_time += cur_wall_time - busy_time; + } + + if (wall) + *wall = (cputime64_t)jiffies_to_usecs(cur_wall_time); + + return (cputime64_t)jiffies_to_usecs(idle_time); +} + +static inline cputime64_t get_idle_time(cputime64_t *wall) +{ + unsigned int cpu; + u64 idle_time = 0; + + for_each_online_cpu(cpu) { + idle_time += get_cpu_idle_time_us(cpu, wall); + if (idle_time == -1ULL) + return get_idle_time_jiffy(wall); + } + + return idle_time; +} + +static cputime64_t get_min_busy_time(cputime64_t arr[], int size) +{ + int i, min_cpu_idx; + cputime64_t min_time = arr[0]; + + for (i = 0; i < size; i++) { + if (arr[i] > 0 && arr[i] < min_time) { + min_time = arr[i]; + min_cpu_idx = i; + } + } + + return min_cpu_idx; +} + +static int find_min_busy_cpu(void) +{ + int nr_all_cpus = num_possible_cpus(); + unsigned int cpus, target_cpu; + cputime64_t busy_time; + cputime64_t b_time[NR_CPUS]; + + memset(b_time, 0, sizeof(b_time)); + for_each_online_cpu(cpus) { + busy_time = calc_busy_time(cpus); + b_time[cpus] = busy_time; + } + target_cpu = get_min_busy_time(b_time, nr_all_cpus); + return target_cpu; +} + +static void increase_cores(int cur_cpus) +{ + struct device *dev; + + if (cur_cpus == ap_info.maxcpus) + return; + + cur_cpus = cpumask_next_zero(0, cpu_online_mask); + + dev = get_cpu_device(cur_cpus); + + per_cpu(cpu_adjusting, dev->id) = 1; + lock_device_hotplug(); + cpu_device_up(dev); + pr_info("The target_cpu is %d, After cpu_up, the cpu_num is %d\n", + dev->id, num_online_cpus()); + get_cpu_device(dev->id)->offline = false; + unlock_device_hotplug(); + per_cpu(cpu_adjusting, dev->id) = 0; +} + +static void decrease_cores(int cur_cpus) +{ + struct device *dev; + + if (cur_cpus == ap_info.mincpus) + return; + + cur_cpus = find_min_busy_cpu(); + + dev = get_cpu_device(cur_cpus); + + if (dev->id > 0) { + per_cpu(cpu_adjusting, dev->id) = -1; + lock_device_hotplug(); + cpu_device_down(dev); + pr_info("The target_cpu is %d. After cpu_down, the cpu_num is %d\n", + cur_cpus, num_online_cpus()); + get_cpu_device(dev->id)->offline = true; + unlock_device_hotplug(); + per_cpu(cpu_adjusting, dev->id) = 0; + } +} + +#define INC_THRESHOLD 80 +#define DEC_THRESHOLD 40 + +static void do_autoplug_timer(struct work_struct *work) +{ + cputime64_t cur_wall_time = 0, cur_idle_time; + unsigned long idle_time, wall_time; + int delay, load; + int nr_cur_cpus = num_online_cpus(); + int nr_all_cpus = num_possible_cpus(); + int inc_req = 1, dec_req = 2; + struct cpufreq_policy *policy = cpufreq_cpu_get_raw(smp_processor_id()); + + if (!policy || IS_ERR(policy->clk)) { + pr_err("%s: No %s associated to cpu: %d\n", + __func__, policy ? "clk" : "policy", 0); + return; + } + + ap_info.maxcpus = + setup_max_cpus > nr_cpu_ids ? nr_cpu_ids : setup_max_cpus; + ap_info.mincpus = ap_info.maxcpus / 4; + + if (strcmp(policy->governor->name, "performance") == 0) { + ap_info.mincpus = ap_info.maxcpus; + } else if (strcmp(policy->governor->name, "powersave") == 0) { + ap_info.maxcpus = ap_info.mincpus; + } else if (strcmp(policy->governor->name, "ondemand") == 0) { + ap_info.sampling_rate = 500; + inc_req = 0; + dec_req = 2; + } else if (strcmp(policy->governor->name, "conservative") == 0) { + inc_req = 1; + dec_req = 3; + ap_info.sampling_rate = 1000; /* 1s */ + } + + BUG_ON(smp_processor_id() != 0); + delay = msecs_to_jiffies(ap_info.sampling_rate); + if (!autoplug_enabled || system_state != SYSTEM_RUNNING) + goto out; + + autoplug_adjusting = 1; + + if (nr_cur_cpus > ap_info.maxcpus) { + decrease_cores(nr_cur_cpus); + autoplug_adjusting = 0; + goto out; + } + if (nr_cur_cpus < ap_info.mincpus) { + increase_cores(nr_cur_cpus); + autoplug_adjusting = 0; + goto out; + } + + cur_idle_time = get_idle_time(&cur_wall_time); + if (cur_wall_time == 0) + cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); + + wall_time = (unsigned int)(cur_wall_time - ap_info.prev_wall); + ap_info.prev_wall = cur_wall_time; + + idle_time = (unsigned int)(cur_idle_time - ap_info.prev_idle); + idle_time += wall_time * (nr_all_cpus - nr_cur_cpus); + ap_info.prev_wall = cur_idle_time; + + if (unlikely(!wall_time || wall_time * nr_all_cpus < idle_time)) { + autoplug_adjusting = 0; + goto out; + } + + load = 100 * (wall_time * nr_all_cpus - idle_time) / wall_time; + + if (load < (nr_cur_cpus - 1) * 100 - DEC_THRESHOLD) { + ap_info.inc_reqs = 0; + if (ap_info.dec_reqs < dec_req) + ap_info.dec_reqs++; + else { + ap_info.dec_reqs = 0; + decrease_cores(nr_cur_cpus); + } + } else { + ap_info.dec_reqs = 0; + if (load > (nr_cur_cpus - 1) * 100 + INC_THRESHOLD) { + if (ap_info.inc_reqs < inc_req) + ap_info.inc_reqs++; + else { + ap_info.inc_reqs = 0; + increase_cores(nr_cur_cpus); + } + } + } + + autoplug_adjusting = 0; +out: + schedule_delayed_work_on(0, &ap_info.work, delay); +} + +static struct platform_device_id platform_device_ids[] = { + { + .name = "sw64_cpuautoplug", + }, + {} +}; + +MODULE_DEVICE_TABLE(platform, platform_device_ids); + +static struct platform_driver platform_driver = { + .driver = { + .name = "sw64_cpuautoplug", + .owner = THIS_MODULE, + }, + .id_table = platform_device_ids, +}; + +static int __init cpuautoplug_init(void) +{ + int i, ret, delay; + + ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, + &cpuclass_attr_group); + if (ret) + return ret; + + ret = platform_driver_register(&platform_driver); + if (ret) + return ret; + + pr_info("cpuautoplug: SW64 CPU autoplug driver.\n"); + + ap_info.maxcpus = + setup_max_cpus > nr_cpu_ids ? nr_cpu_ids : setup_max_cpus; + ap_info.mincpus = ap_info.maxcpus / 4; + ap_info.dec_reqs = 0; + ap_info.inc_reqs = 0; + ap_info.sampling_rate = 720; /* 720ms */ + if (setup_max_cpus == 0) { /* boot with npsmp */ + ap_info.maxcpus = 1; + autoplug_enabled = 0; + } + if (setup_max_cpus > num_possible_cpus()) + ap_info.maxcpus = num_possible_cpus(); + + pr_info("mincpu = %d, maxcpu = %d, autoplug_enabled = %d, rate = %d\n", + ap_info.mincpus, ap_info.maxcpus, autoplug_enabled, + ap_info.sampling_rate); + + for_each_possible_cpu(i) + per_cpu(cpu_adjusting, i) = 0; + delay = msecs_to_jiffies(ap_info.sampling_rate * 24); + INIT_DEFERRABLE_WORK(&ap_info.work, do_autoplug_timer); + schedule_delayed_work_on(0, &ap_info.work, delay); + + if (!autoplug_enabled) + cancel_delayed_work_sync(&ap_info.work); + + return ret; +} + +late_initcall(cpuautoplug_init); -- Gitee From 83e0e5fbfd55245482dc7ed11923056dae6615e1 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:38:32 +0800 Subject: [PATCH 051/524] sw64: fix build support Modify scripts for SW64 build support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- scripts/package/buildtar | 3 +++ scripts/package/mkdebian | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/package/buildtar b/scripts/package/buildtar index 65b4ea502962..93158943a4f7 100755 --- a/scripts/package/buildtar +++ b/scripts/package/buildtar @@ -64,6 +64,9 @@ case "${ARCH}" in alpha) [ -f "${objtree}/arch/alpha/boot/vmlinux.gz" ] && cp -v -- "${objtree}/arch/alpha/boot/vmlinux.gz" "${tmpdir}/boot/vmlinuz-${KERNELRELEASE}" ;; + sw_64) + [ -f "${objtree}/arch/sw_64/boot/vmlinux.bin" ] && cp -v -- "${objtree}/arch/sw_64/boot/vmlinux.bin" "${tmpdir}/boot/vmlinux-bin-${KERNELRELEASE}" + ;; parisc*) [ -f "${KBUILD_IMAGE}" ] && cp -v -- "${KBUILD_IMAGE}" "${tmpdir}/boot/vmlinux-${KERNELRELEASE}" [ -f "${objtree}/lifimage" ] && cp -v -- "${objtree}/lifimage" "${tmpdir}/boot/lifimage-${KERNELRELEASE}" diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian index c1a36da85e84..6808a98cecd7 100755 --- a/scripts/package/mkdebian +++ b/scripts/package/mkdebian @@ -26,7 +26,7 @@ set_debarch() { # Attempt to find the correct Debian architecture case "$UTS_MACHINE" in - i386|alpha|m68k|riscv*) + i386|alpha|m68k|riscv*|sw_64) debarch="$UTS_MACHINE" ;; x86_64) debarch=amd64 ;; -- Gitee From e545e2f3ea4d185402019282d1cec23c552cc02b Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:36:01 +0800 Subject: [PATCH 052/524] sw64: fix ELF support Modify generic headers for SW64 ELF support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- include/uapi/linux/elf-em.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/uapi/linux/elf-em.h b/include/uapi/linux/elf-em.h index ef38c2bc5ab7..32458706a403 100644 --- a/include/uapi/linux/elf-em.h +++ b/include/uapi/linux/elf-em.h @@ -59,6 +59,7 @@ * up with a final number. */ #define EM_ALPHA 0x9026 +#define EM_SW64 0x9916 /* Bogus old m32r magic number, used by old tools. */ #define EM_CYGNUS_M32R 0x9041 -- Gitee From 0e16d8f3e882227b23aec05395af2d38e768d656 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:37:09 +0800 Subject: [PATCH 053/524] sw64: fix rrk support Modify generic routines for SW64 rrk support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- kernel/printk/printk.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 51c43e0f9b29..efb927e1b34a 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2202,10 +2202,17 @@ int vprintk_store(int facility, int level, u16 text_len; int ret = 0; u64 ts_nsec; +#ifdef CONFIG_SW64_RRK + extern int sw64_printk(const char *fmt, va_list args); +#endif if (!printk_enter_irqsave(recursion_ptr, irqflags)) return 0; +#ifdef CONFIG_SW64_RRK + sw64_printk(fmt, args); +#endif + /* * Since the duration of printk() can vary depending on the message * and state of the ringbuffer, grab the timestamp now so that it is -- Gitee From 5bf0f5ca1f1fb9f6cdacda97efb4a0886ab2fe9e Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:30:40 +0800 Subject: [PATCH 054/524] sw64: fix ACPI support Modify generic headers for SW64 ACPI support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- include/acpi/pdc_sw64.h | 34 ++++++++++++++++++++++++++++++++++ include/linux/acpi.h | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 include/acpi/pdc_sw64.h diff --git a/include/acpi/pdc_sw64.h b/include/acpi/pdc_sw64.h new file mode 100644 index 000000000000..4724f10e8c6a --- /dev/null +++ b/include/acpi/pdc_sw64.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_PDC_SW64_H +#define _ASM_PDC_SW64_H + +#define ACPI_PDC_P_FFH (0x0001) +#define ACPI_PDC_C_C1_HALT (0x0002) +#define ACPI_PDC_T_FFH (0x0004) +#define ACPI_PDC_SMP_C1PT (0x0008) +#define ACPI_PDC_SMP_C2C3 (0x0010) +#define ACPI_PDC_SMP_P_SWCOORD (0x0020) +#define ACPI_PDC_SMP_C_SWCOORD (0x0040) +#define ACPI_PDC_SMP_T_SWCOORD (0x0080) +#define ACPI_PDC_C_C1_FFH (0x0100) +#define ACPI_PDC_C_C2C3_FFH (0x0200) +#define ACPI_PDC_SMP_P_HWCOORD (0x0800) + +#define ACPI_PDC_EST_CAPABILITY_SMP (ACPI_PDC_SMP_C1PT | \ + ACPI_PDC_C_C1_HALT | \ + ACPI_PDC_P_FFH) + +#define ACPI_PDC_EST_CAPABILITY_SWSMP (ACPI_PDC_SMP_C1PT | \ + ACPI_PDC_C_C1_HALT | \ + ACPI_PDC_SMP_P_SWCOORD | \ + ACPI_PDC_SMP_P_HWCOORD | \ + ACPI_PDC_P_FFH) + +#define ACPI_PDC_C_CAPABILITY_SMP (ACPI_PDC_SMP_C2C3 | \ + ACPI_PDC_SMP_C1PT | \ + ACPI_PDC_C_C1_HALT | \ + ACPI_PDC_C_C1_FFH | \ + ACPI_PDC_C_C2C3_FFH) + +#endif /* _ASM_PDC_SW64_H */ diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 7c6f4006389d..fd820045a21b 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -259,7 +259,7 @@ void acpi_table_print_madt_entry (struct acpi_subtable_header *madt); /* the following numa functions are architecture-dependent */ void acpi_numa_slit_init (struct acpi_table_slit *slit); -#if defined(CONFIG_X86) || defined(CONFIG_IA64) || defined(CONFIG_LOONGARCH) +#if defined(CONFIG_X86) || defined(CONFIG_IA64) || defined(CONFIG_LOONGARCH) || defined(CONFIG_SW64) void acpi_numa_processor_affinity_init (struct acpi_srat_cpu_affinity *pa); #else static inline void -- Gitee From 6aff0cc8b062593be869ce32081c42864ed6d2a4 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:33:32 +0800 Subject: [PATCH 055/524] sw64: fix module support Modify generic headers for SW64 kernel module support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- include/linux/moduleparam.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h index 962cd41a2cb5..c401b4e975cc 100644 --- a/include/linux/moduleparam.h +++ b/include/linux/moduleparam.h @@ -276,7 +276,7 @@ struct kparam_array read-only sections (which is part of respective UNIX ABI on these platforms). So 'const' makes no sense and even causes compile failures with some compilers. */ -#if defined(CONFIG_ALPHA) || defined(CONFIG_IA64) || defined(CONFIG_PPC64) +#if defined(CONFIG_ALPHA) || defined(CONFIG_IA64) || defined(CONFIG_PPC64) || defined(CONFIG_SW64) #define __moduleparam_const #else #define __moduleparam_const const -- Gitee From 79f16b26d702aeac509bb4609f98e083e6b2e70d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:32:24 +0800 Subject: [PATCH 056/524] sw64: fix KVM support Modify generic headers and routines for SW64 KVM support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- include/linux/kvm_host.h | 18 ++++++ include/uapi/linux/kvm.h | 5 ++ virt/kvm/kvm_main.c | 126 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 2328e04a75cf..4388cfc5bb74 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1776,6 +1776,9 @@ static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa) enum kvm_stat_kind { KVM_STAT_VM, KVM_STAT_VCPU, +#ifdef CONFIG_SW64 + KVM_STAT_DFX_SW64, /* Detail For vcpu stat EXtension */ +#endif }; struct kvm_stat_data { @@ -1905,6 +1908,21 @@ struct _kvm_stats_desc { HALT_POLL_HIST_COUNT), \ STATS_DESC_IBOOLEAN(VCPU_GENERIC, blocking) +#ifdef CONFIG_SW64 +enum dfx_sw64_stat_kind { + DFX_SW64_STAT_U64, + DFX_SW64_STAT_CPUTIME, +}; + +/* Detail For vcpu stat EXtension debugfs item */ +struct dfx_sw64_kvm_stats_debugfs_item { + const char *name; + int offset; + enum dfx_sw64_stat_kind dfx_kind; + struct dentry *dentry; +}; +extern struct dfx_sw64_kvm_stats_debugfs_item dfx_sw64_debugfs_entries[]; +#endif extern struct dentry *kvm_debugfs_dir; ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 02bd39b2367d..23fae6de3caa 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1598,6 +1598,11 @@ struct kvm_s390_ucas_mapping { /* Available with KVM_CAP_COUNTER_OFFSET */ #define KVM_ARM_SET_COUNTER_OFFSET _IOW(KVMIO, 0xb5, struct kvm_arm_counter_offset) +/* ioctl for SW vcpu init*/ +#define KVM_SW64_VCPU_INIT _IO(KVMIO, 0xba) +#define KVM_SW64_GET_VCB _IO(KVMIO, 0xbc) +#define KVM_SW64_SET_VCB _IO(KVMIO, 0xbd) + /* ioctl for vm fd */ #define KVM_CREATE_DEVICE _IOWR(KVMIO, 0xe0, struct kvm_create_device) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index d039d778a7d5..3f13223bcfc3 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -154,6 +154,11 @@ static unsigned long long kvm_active_vms; static DEFINE_PER_CPU(cpumask_var_t, cpu_kick_mask); +#ifdef CONFIG_SW64 +#define DFX_SW64_MAX_VCPU 1024 +#define DFX_SW64_MAX_VCPU_STAT_SIZE 1024 +#endif + __weak void kvm_arch_guest_memory_reclaimed(struct kvm *kvm) { } @@ -4182,6 +4187,9 @@ static long kvm_vcpu_ioctl(struct file *filp, if (oldpid) synchronize_rcu(); put_pid(oldpid); +#ifdef CONFIG_SW64 + vcpu->stat.pid = current->pid; +#endif } r = kvm_arch_vcpu_ioctl_run(vcpu); trace_kvm_userspace_exit(vcpu->run->exit_reason, r); @@ -5807,6 +5815,10 @@ static int kvm_stat_data_get(void *data, u64 *val) r = kvm_get_stat_per_vcpu(stat_data->kvm, stat_data->desc->desc.offset, val); break; +#ifdef CONFIG_SW64 + case KVM_STAT_DFX_SW64: + break; +#endif } return r; @@ -5829,6 +5841,10 @@ static int kvm_stat_data_clear(void *data, u64 val) r = kvm_clear_stat_per_vcpu(stat_data->kvm, stat_data->desc->desc.offset); break; +#ifdef CONFIG_SW64 + case KVM_STAT_DFX_SW64: + break; +#endif } return r; @@ -5923,6 +5939,116 @@ DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear, "%llu\n"); DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_readonly_fops, vcpu_stat_get, NULL, "%llu\n"); +#ifdef CONFIG_SW64 +void __weak kvm_arch_vcpu_stat_reset(struct kvm_vcpu_stat *vcpu_stat) +{ +} + +/* + * copy of seq_buf_alloc of kernel, kernel not export it + */ +static void *dfx_sw64_seq_buf_alloc(unsigned long size) +{ + return kvmalloc(size, GFP_KERNEL_ACCOUNT); +} + +static void dfx_sw64_seq_buf_free(const void *buf) +{ + kvfree(buf); +} + +static int dfx_sw64_seq_buf_alloc_vcpu(struct seq_file *p, int vcpu_nr) +{ + char *buf; + size_t size; + + size = (vcpu_nr + 1) * DFX_SW64_MAX_VCPU_STAT_SIZE; + buf = dfx_sw64_seq_buf_alloc(size); + if (!buf) + return -ENOMEM; + if (p->buf) + dfx_sw64_seq_buf_free(p->buf); + p->buf = buf; + p->size = size; + return 0; +} + +static int __dfx_sw64_vcpu_stats_get(struct seq_file *p, void *v) +{ + struct kvm *kvm; + struct kvm_vcpu *vcpu; + struct kvm_vcpu_stat *vcpu_stats; + struct dfx_sw64_kvm_stats_debugfs_item *dp; + int vcpu_nr = 0; + int index = 0; + unsigned long i; + + mutex_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) + kvm_for_each_vcpu(i, vcpu, kvm) { + vcpu_nr++; + } + mutex_unlock(&kvm_lock); + vcpu_nr = min(vcpu_nr, DFX_SW64_MAX_VCPU); + if (!vcpu_nr) { + seq_putc(p, '\n'); + return 0; + } + + if (dfx_sw64_seq_buf_alloc_vcpu(p, vcpu_nr)) + return -ENOMEM; + + vcpu_stats = vmalloc(vcpu_nr * sizeof(struct kvm_vcpu_stat)); + if (!vcpu_stats) + return -ENOMEM; + + mutex_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) { + kvm_for_each_vcpu(i, vcpu, kvm) { + if (index >= vcpu_nr) + break; + memcpy(vcpu_stats + index, &(vcpu->stat), + sizeof(struct kvm_vcpu_stat)); + kvm_arch_vcpu_stat_reset(&vcpu->stat); + ++index; + } + } + mutex_unlock(&kvm_lock); + for (i = 0; i < vcpu_nr; i++) { + for (dp = dfx_sw64_debugfs_entries; dp->name; ++dp) { + switch (dp->dfx_kind) { + case DFX_SW64_STAT_U64: + seq_put_decimal_ull(p, " ", + *(u64 *)((void *)&vcpu_stats[i] + dp->offset)); + break; + case DFX_SW64_STAT_CPUTIME: + pr_warn("DFX_SW64_STAT_CPUTIME not supported currently!"); + break; + default: + pr_warn("Bad dfx_sw64_kind in dfx_debugfs_entries!"); + break; + } + } + seq_putc(p, '\n'); + } + + vfree(vcpu_stats); + return 0; +} + +static int dfx_sw64_vcpu_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, __dfx_sw64_vcpu_stats_get, NULL); +} + +static const struct file_operations dfx_sw64_stat_fops = { + .open = dfx_sw64_vcpu_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) { struct kobj_uevent_env *env; -- Gitee From 6f3c1e99b95bedc6a42e348d133f088f972bb8b9 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:34:00 +0800 Subject: [PATCH 057/524] sw64: fix PCI support Modify generic headers for SW64 PCI support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- include/linux/pci-ecam.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/pci-ecam.h b/include/linux/pci-ecam.h index 6b1301e2498e..863e572202e2 100644 --- a/include/linux/pci-ecam.h +++ b/include/linux/pci-ecam.h @@ -88,6 +88,7 @@ extern const struct pci_ecam_ops xgene_v2_pcie_ecam_ops; /* APM X-Gene PCIe v2.x extern const struct pci_ecam_ops al_pcie_ops; /* Amazon Annapurna Labs PCIe */ extern const struct pci_ecam_ops tegra194_pcie_ops; /* Tegra194 PCIe */ extern const struct pci_ecam_ops loongson_pci_ecam_ops; /* Loongson PCIe */ +extern const struct pci_ecam_ops sw64_pci_ecam_ops; /* SW64 PCIe */ #endif #if IS_ENABLED(CONFIG_PCI_HOST_COMMON) -- Gitee From ee5b4a866577c27ea5f317cf91020c24ff9f38e6 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:36:22 +0800 Subject: [PATCH 058/524] sw64: fix kexec support Modify generic headers for SW64 kexec support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- include/uapi/linux/kexec.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/uapi/linux/kexec.h b/include/uapi/linux/kexec.h index 01766dd839b0..3be3e81c67ae 100644 --- a/include/uapi/linux/kexec.h +++ b/include/uapi/linux/kexec.h @@ -45,6 +45,7 @@ #define KEXEC_ARCH_AARCH64 (183 << 16) #define KEXEC_ARCH_RISCV (243 << 16) #define KEXEC_ARCH_LOONGARCH (258 << 16) +#define KEXEC_ARCH_SW64 (0x9916UL << 16) /* The artificial cap on the number of segments passed to kexec_load. */ #define KEXEC_SEGMENT_MAX 16 -- Gitee From d04008cd7137bdfb05e6e5acbf655098c75b4687 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:36:41 +0800 Subject: [PATCH 059/524] sw64: fix audit support Modify generic headers for SW64 audit support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- include/uapi/linux/audit.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h index d676ed2b246e..f428015e85de 100644 --- a/include/uapi/linux/audit.h +++ b/include/uapi/linux/audit.h @@ -441,6 +441,7 @@ enum { #define AUDIT_ARCH_XTENSA (EM_XTENSA) #define AUDIT_ARCH_LOONGARCH32 (EM_LOONGARCH|__AUDIT_ARCH_LE) #define AUDIT_ARCH_LOONGARCH64 (EM_LOONGARCH|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE) +#define AUDIT_ARCH_SW64 (EM_SW64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE) #define AUDIT_PERM_EXEC 1 #define AUDIT_PERM_WRITE 2 -- Gitee From a2277ba9b935bafcf5dd9d3ad93903fe600c4e83 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:38:04 +0800 Subject: [PATCH 060/524] sw64: fix ftrace support Modify scripts for SW64 ftrace support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- scripts/recordmcount.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index 3e4f54799cc0..31a0243df35b 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c @@ -52,6 +52,12 @@ #define R_AARCH64_CALL26 283 +#ifndef EM_SW64 +#define EM_SW64 0x9916 +#define R_SW64_NONE 0 +#define R_SW64_REFQUAD 2 /* Direct 64 bit */ +#endif + static int fd_map; /* File descriptor for file being modified. */ static int mmap_failed; /* Boolean flag. */ static char gpfx; /* prefix for global symbol name (sometimes '_') */ @@ -326,6 +332,16 @@ static int make_nop_arm64(void *map, size_t const offset) return 0; } +static unsigned char ideal_nop4_sw64[4] = {0x5f, 0x07, 0xff, 0x43}; + +static int make_nop_sw64(void *map, size_t const offset) +{ + /* Convert to nop */ + ulseek(offset, SEEK_SET); + uwrite(ideal_nop, 4); + return 0; +} + static int write_file(const char *fname) { char tmp_file[strlen(fname) + 4]; @@ -475,6 +491,21 @@ static int LARCH64_is_fake_mcount(Elf64_Rel const *rp) return 1; } +#define SW64_FAKEMCOUNT_OFFSET 4 + +static int sw64_is_fake_mcount(Elf64_Rel const *rp) +{ + static Elf64_Addr old_r_offset = ~(Elf64_Addr)0; + Elf64_Addr current_r_offset = _w(rp->r_offset); + int is_fake; + + is_fake = (old_r_offset != ~(Elf64_Addr)0) && + (current_r_offset - old_r_offset == SW64_FAKEMCOUNT_OFFSET); + old_r_offset = current_r_offset; + + return is_fake; +} + /* 64-bit EM_MIPS has weird ELF64_Rela.r_info. * http://techpubs.sgi.com/library/manuals/4000/007-4658-001/pdf/007-4658-001.pdf * We interpret Table 29 Relocation Operation (Elf64_Rel, Elf64_Rela) [p.40] @@ -597,6 +628,14 @@ static int do_file(char const *const fname) case EM_S390: /* reltype: e_class */ break; case EM_SH: reltype = R_SH_DIR32; gpfx = 0; break; case EM_SPARCV9: reltype = R_SPARC_64; break; + case EM_SW64: + reltype = R_SW64_REFQUAD; + make_nop = make_nop_sw64; + rel_type_nop = R_SW64_NONE; + ideal_nop = ideal_nop4_sw64; + mcount_adjust_64 = -12; + is_fake_mcount64 = sw64_is_fake_mcount; + break; case EM_X86_64: make_nop = make_nop_x86; ideal_nop = ideal_nop5_x86_64; -- Gitee From 08daadd8969a69a46256bfe15f81515e78bc7761 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:23:00 +0800 Subject: [PATCH 061/524] tools: add basic sw64 support Add common headers and routines for SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/arch/sw_64/include/asm/barrier.h | 9 ++ .../arch/sw_64/include/uapi/asm/bitsperlong.h | 9 ++ tools/arch/sw_64/include/uapi/asm/errno.h | 128 ++++++++++++++++++ tools/arch/sw_64/include/uapi/asm/mman.h | 46 +++++++ tools/arch/sw_64/include/uapi/asm/perf_regs.h | 41 ++++++ tools/build/feature/test-libunwind-sw_64.c | 27 ++++ 6 files changed, 260 insertions(+) create mode 100644 tools/arch/sw_64/include/asm/barrier.h create mode 100644 tools/arch/sw_64/include/uapi/asm/bitsperlong.h create mode 100644 tools/arch/sw_64/include/uapi/asm/errno.h create mode 100644 tools/arch/sw_64/include/uapi/asm/mman.h create mode 100644 tools/arch/sw_64/include/uapi/asm/perf_regs.h create mode 100644 tools/build/feature/test-libunwind-sw_64.c diff --git a/tools/arch/sw_64/include/asm/barrier.h b/tools/arch/sw_64/include/asm/barrier.h new file mode 100644 index 000000000000..bc4aeffeb681 --- /dev/null +++ b/tools/arch/sw_64/include/asm/barrier.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _TOOLS_LINUX_ASM_SW64_BARRIER_H +#define _TOOLS_LINUX_ASM_SW64_BARRIER_H + +#define mb() __asm__ __volatile__("mb" : : : "memory") +#define rmb() __asm__ __volatile__("mb" : : : "memory") +#define wmb() __asm__ __volatile__("mb" : : : "memory") + +#endif /* _TOOLS_LINUX_ASM_SW64_BARRIER_H */ diff --git a/tools/arch/sw_64/include/uapi/asm/bitsperlong.h b/tools/arch/sw_64/include/uapi/asm/bitsperlong.h new file mode 100644 index 000000000000..f6a510c28233 --- /dev/null +++ b/tools/arch/sw_64/include/uapi/asm/bitsperlong.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef __ASM_SW64_BITSPERLONG_H +#define __ASM_SW64_BITSPERLONG_H + +#define __BITS_PER_LONG 64 + +#include + +#endif /* __ASM_SW64_BITSPERLONG_H */ diff --git a/tools/arch/sw_64/include/uapi/asm/errno.h b/tools/arch/sw_64/include/uapi/asm/errno.h new file mode 100644 index 000000000000..2a43a943581a --- /dev/null +++ b/tools/arch/sw_64/include/uapi/asm/errno.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SW64_ERRNO_H +#define _SW64_ERRNO_H + +#include + +#undef EAGAIN /* 11 in errno-base.h */ + +#define EDEADLK 11 /* Resource deadlock would occur */ + +#define EAGAIN 35 /* Try again */ +#define EWOULDBLOCK EAGAIN /* Operation would block */ +#define EINPROGRESS 36 /* Operation now in progress */ +#define EALREADY 37 /* Operation already in progress */ +#define ENOTSOCK 38 /* Socket operation on non-socket */ +#define EDESTADDRREQ 39 /* Destination address required */ +#define EMSGSIZE 40 /* Message too long */ +#define EPROTOTYPE 41 /* Protocol wrong type for socket */ +#define ENOPROTOOPT 42 /* Protocol not available */ +#define EPROTONOSUPPORT 43 /* Protocol not supported */ +#define ESOCKTNOSUPPORT 44 /* Socket type not supported */ +#define EOPNOTSUPP 45 /* Operation not supported on transport endpoint */ +#define EPFNOSUPPORT 46 /* Protocol family not supported */ +#define EAFNOSUPPORT 47 /* Address family not supported by protocol */ +#define EADDRINUSE 48 /* Address already in use */ +#define EADDRNOTAVAIL 49 /* Cannot assign requested address */ +#define ENETDOWN 50 /* Network is down */ +#define ENETUNREACH 51 /* Network is unreachable */ +#define ENETRESET 52 /* Network dropped connection because of reset */ +#define ECONNABORTED 53 /* Software caused connection abort */ +#define ECONNRESET 54 /* Connection reset by peer */ +#define ENOBUFS 55 /* No buffer space available */ +#define EISCONN 56 /* Transport endpoint is already connected */ +#define ENOTCONN 57 /* Transport endpoint is not connected */ +#define ESHUTDOWN 58 /* Cannot send after transport endpoint shutdown */ +#define ETOOMANYREFS 59 /* Too many references: cannot splice */ +#define ETIMEDOUT 60 /* Connection timed out */ +#define ECONNREFUSED 61 /* Connection refused */ +#define ELOOP 62 /* Too many symbolic links encountered */ +#define ENAMETOOLONG 63 /* File name too long */ +#define EHOSTDOWN 64 /* Host is down */ +#define EHOSTUNREACH 65 /* No route to host */ +#define ENOTEMPTY 66 /* Directory not empty */ + +#define EUSERS 68 /* Too many users */ +#define EDQUOT 69 /* Quota exceeded */ +#define ESTALE 70 /* Stale file handle */ +#define EREMOTE 71 /* Object is remote */ + +#define ENOLCK 77 /* No record locks available */ +#define ENOSYS 78 /* Function not implemented */ + +#define ENOMSG 80 /* No message of desired type */ +#define EIDRM 81 /* Identifier removed */ +#define ENOSR 82 /* Out of streams resources */ +#define ETIME 83 /* Timer expired */ +#define EBADMSG 84 /* Not a data message */ +#define EPROTO 85 /* Protocol error */ +#define ENODATA 86 /* No data available */ +#define ENOSTR 87 /* Device not a stream */ + +#define ENOPKG 92 /* Package not installed */ + +#define EILSEQ 116 /* Illegal byte sequence */ + +/* The following are just random noise.. */ +#define ECHRNG 88 /* Channel number out of range */ +#define EL2NSYNC 89 /* Level 2 not synchronized */ +#define EL3HLT 90 /* Level 3 halted */ +#define EL3RST 91 /* Level 3 reset */ + +#define ELNRNG 93 /* Link number out of range */ +#define EUNATCH 94 /* Protocol driver not attached */ +#define ENOCSI 95 /* No CSI structure available */ +#define EL2HLT 96 /* Level 2 halted */ +#define EBADE 97 /* Invalid exchange */ +#define EBADR 98 /* Invalid request descriptor */ +#define EXFULL 99 /* Exchange full */ +#define ENOANO 100 /* No anode */ +#define EBADRQC 101 /* Invalid request code */ +#define EBADSLT 102 /* Invalid slot */ + +#define EDEADLOCK EDEADLK + +#define EBFONT 104 /* Bad font file format */ +#define ENONET 105 /* Machine is not on the network */ +#define ENOLINK 106 /* Link has been severed */ +#define EADV 107 /* Advertise error */ +#define ESRMNT 108 /* Srmount error */ +#define ECOMM 109 /* Communication error on send */ +#define EMULTIHOP 110 /* Multihop attempted */ +#define EDOTDOT 111 /* RFS specific error */ +#define EOVERFLOW 112 /* Value too large for defined data type */ +#define ENOTUNIQ 113 /* Name not unique on network */ +#define EBADFD 114 /* File descriptor in bad state */ +#define EREMCHG 115 /* Remote address changed */ + +#define EUCLEAN 117 /* Structure needs cleaning */ +#define ENOTNAM 118 /* Not a XENIX named type file */ +#define ENAVAIL 119 /* No XENIX semaphores available */ +#define EISNAM 120 /* Is a named type file */ +#define EREMOTEIO 121 /* Remote I/O error */ + +#define ELIBACC 122 /* Can not access a needed shared library */ +#define ELIBBAD 123 /* Accessing a corrupted shared library */ +#define ELIBSCN 124 /* .lib section in a.out corrupted */ +#define ELIBMAX 125 /* Attempting to link in too many shared libraries */ +#define ELIBEXEC 126 /* Cannot exec a shared library directly */ +#define ERESTART 127 /* Interrupted system call should be restarted */ +#define ESTRPIPE 128 /* Streams pipe error */ + +#define ENOMEDIUM 129 /* No medium found */ +#define EMEDIUMTYPE 130 /* Wrong medium type */ +#define ECANCELED 131 /* Operation Cancelled */ +#define ENOKEY 132 /* Required key not available */ +#define EKEYEXPIRED 133 /* Key has expired */ +#define EKEYREVOKED 134 /* Key has been revoked */ +#define EKEYREJECTED 135 /* Key was rejected by service */ + +/* for robust mutexes */ +#define EOWNERDEAD 136 /* Owner died */ +#define ENOTRECOVERABLE 137 /* State not recoverable */ + +#define ERFKILL 138 /* Operation not possible due to RF-kill */ + +#define EHWPOISON 139 /* Memory page has hardware error */ + +#endif /* _SW64_ERRNO_H */ diff --git a/tools/arch/sw_64/include/uapi/asm/mman.h b/tools/arch/sw_64/include/uapi/asm/mman.h new file mode 100644 index 000000000000..a9603c93a34b --- /dev/null +++ b/tools/arch/sw_64/include/uapi/asm/mman.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef TOOLS_ARCH_SW64_UAPI_ASM_MMAN_FIX_H +#define TOOLS_ARCH_SW64_UAPI_ASM_MMAN_FIX_H +#define MADV_DODUMP 17 +#define MADV_DOFORK 11 +#define MADV_DONTDUMP 16 +#define MADV_DONTFORK 10 +#define MADV_DONTNEED 6 +#define MADV_FREE 8 +#define MADV_HUGEPAGE 14 +#define MADV_MERGEABLE 12 +#define MADV_NOHUGEPAGE 15 +#define MADV_NORMAL 0 +#define MADV_RANDOM 1 +#define MADV_REMOVE 9 +#define MADV_SEQUENTIAL 2 +#define MADV_UNMERGEABLE 13 +#define MADV_WILLNEED 3 +#define MAP_ANONYMOUS 0x10 +#define MAP_DENYWRITE 0x02000 +#define MAP_EXECUTABLE 0x04000 +#define MAP_FILE 0 +#define MAP_FIXED 0x100 +#define MAP_GROWSDOWN 0x01000 +#define MAP_HUGETLB 0x100000 +#define MAP_LOCKED 0x08000 +#define MAP_NONBLOCK 0x40000 +#define MAP_NORESERVE 0x10000 +#define MAP_POPULATE 0x20000 +#define MAP_STACK 0x80000 +#define PROT_EXEC 0x4 +#define PROT_GROWSDOWN 0x01000000 +#define PROT_GROWSUP 0x02000000 +#define PROT_NONE 0x0 +#define PROT_READ 0x1 +#define PROT_SEM 0x8 +#define PROT_WRITE 0x2 +/* MADV_HWPOISON is undefined on alpha, fix it for perf */ +#define MADV_HWPOISON 100 +/* MADV_SOFT_OFFLINE is undefined on alpha, fix it for perf */ +#define MADV_SOFT_OFFLINE 101 +/* MAP_32BIT is undefined on alpha, fix it for perf */ +#define MAP_32BIT 0 +/* MAP_UNINITIALIZED is undefined on alpha, fix it for perf */ +#define MAP_UNINITIALIZED 0 +#endif /* TOOLS_ARCH_SW64_UAPI_ASM_MMAN_FIX_H */ diff --git a/tools/arch/sw_64/include/uapi/asm/perf_regs.h b/tools/arch/sw_64/include/uapi/asm/perf_regs.h new file mode 100644 index 000000000000..892be5261026 --- /dev/null +++ b/tools/arch/sw_64/include/uapi/asm/perf_regs.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ + +#ifndef _ASM_SW64_PERF_REGS_H +#define _ASM_SW64_PERF_REGS_H + +enum perf_event_sw64_regs { + PERF_REG_SW64_R0, + PERF_REG_SW64_R1, + PERF_REG_SW64_R2, + PERF_REG_SW64_R3, + PERF_REG_SW64_R4, + PERF_REG_SW64_R5, + PERF_REG_SW64_R6, + PERF_REG_SW64_R7, + PERF_REG_SW64_R8, + PERF_REG_SW64_R9, + PERF_REG_SW64_R10, + PERF_REG_SW64_R11, + PERF_REG_SW64_R12, + PERF_REG_SW64_R13, + PERF_REG_SW64_R14, + PERF_REG_SW64_R15, + PERF_REG_SW64_R16, + PERF_REG_SW64_R17, + PERF_REG_SW64_R18, + PERF_REG_SW64_R19, + PERF_REG_SW64_R20, + PERF_REG_SW64_R21, + PERF_REG_SW64_R22, + PERF_REG_SW64_R23, + PERF_REG_SW64_R24, + PERF_REG_SW64_R25, + PERF_REG_SW64_R26, + PERF_REG_SW64_R27, + PERF_REG_SW64_R28, + PERF_REG_SW64_GP, + PERF_REG_SW64_SP, + PERF_REG_SW64_PC, + PERF_REG_SW64_MAX, +}; +#endif /* _ASM_SW64_PERF_REGS_H */ diff --git a/tools/build/feature/test-libunwind-sw_64.c b/tools/build/feature/test-libunwind-sw_64.c new file mode 100644 index 000000000000..274948b961f4 --- /dev/null +++ b/tools/build/feature/test-libunwind-sw_64.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); + +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) + +static unw_accessors_t accessors; + +int main(void) +{ + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, 0); + if (addr_space) + return 0; + + unw_init_remote(NULL, addr_space, NULL); + dwarf_search_unwind_table(addr_space, 0, NULL, NULL, 0, NULL); + + return 0; +} -- Gitee From bd4451f026f394e07e76bc094929a38bad940dcc Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:28:12 +0800 Subject: [PATCH 062/524] tools: fix basic sw64 support Modify generic headers and Makefiles for SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/build/Makefile.feature | 1 + tools/build/feature/Makefile | 9 +++++++++ tools/include/uapi/asm/bitsperlong.h | 2 ++ tools/include/uapi/asm/errno.h | 2 ++ 4 files changed, 14 insertions(+) diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index 934e2777a2db..fb290a90b263 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -54,6 +54,7 @@ FEATURE_TESTS_BASIC := \ libtracefs \ libcrypto \ libunwind \ + libunwind-sw_64 \ pthread-attr-setaffinity-np \ pthread-barrier \ reallocarray \ diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index dad79ede4e0a..cb57e46cec4b 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 include ../../scripts/Makefile.include +ARCH ?= $(shell uname -m) FILES= \ test-all.bin \ test-backtrace.bin \ @@ -45,6 +46,7 @@ FILES= \ test-libunwind-x86_64.bin \ test-libunwind-arm.bin \ test-libunwind-aarch64.bin \ + test-libunwind-sw_64.bin \ test-libunwind-debug-frame-arm.bin \ test-libunwind-debug-frame-aarch64.bin \ test-pthread-attr-setaffinity-np.bin \ @@ -86,7 +88,11 @@ all: $(FILES) __BUILD = $(CC) $(CFLAGS) -MD -Wall -Werror -o $@ $(patsubst %.bin,%.c,$(@F)) $(LDFLAGS) BUILD = $(__BUILD) > $(@:.bin=.make.output) 2>&1 +ifeq ($(ARCH),sw_64) + BUILD_BFD = $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl -liberty -lz +else BUILD_BFD = $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl +endif BUILD_ALL = $(BUILD) -fstack-protector-all -O2 -D_FORTIFY_SOURCE=2 -ldw -lelf -lnuma -lelf -lslang $(FLAGS_PERL_EMBED) $(FLAGS_PYTHON_EMBED) -DPACKAGE='"perf"' -lbfd -ldl -lz -llzma -lzstd -lcap __BUILDXX = $(CXX) $(CXXFLAGS) -MD -Wall -Werror -o $@ $(patsubst %.bin,%.cpp,$(@F)) $(LDFLAGS) @@ -189,6 +195,9 @@ $(OUTPUT)test-libunwind-arm.bin: $(OUTPUT)test-libunwind-aarch64.bin: $(BUILD) -lelf -lunwind-aarch64 +$(OUTPUT)test-libunwind-sw_64.bin: + $(BUILD) -lelf -lunwind-sw_64 + $(OUTPUT)test-libunwind-debug-frame-arm.bin: $(BUILD) -lelf -lunwind-arm diff --git a/tools/include/uapi/asm/bitsperlong.h b/tools/include/uapi/asm/bitsperlong.h index c65267afc341..036e2fc92d1a 100644 --- a/tools/include/uapi/asm/bitsperlong.h +++ b/tools/include/uapi/asm/bitsperlong.h @@ -13,6 +13,8 @@ #include "../../../arch/ia64/include/uapi/asm/bitsperlong.h" #elif defined(__alpha__) #include "../../../arch/alpha/include/uapi/asm/bitsperlong.h" +#elif defined(__sw_64__) +#include "../../../arch/sw_64/include/uapi/asm/bitsperlong.h" #else #include #endif diff --git a/tools/include/uapi/asm/errno.h b/tools/include/uapi/asm/errno.h index 869379f91fe4..bcfa3d742933 100644 --- a/tools/include/uapi/asm/errno.h +++ b/tools/include/uapi/asm/errno.h @@ -11,6 +11,8 @@ #include "../../../arch/mips/include/uapi/asm/errno.h" #elif defined(__hppa__) #include "../../../arch/parisc/include/uapi/asm/errno.h" +#elif defined(__sw_64__) +#include "../../../arch/sw_64/include/uapi/asm/errno.h" #else #include #endif -- Gitee From 04b95fb0256ab3417c257c7457e47b59d7910677 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:27:19 +0800 Subject: [PATCH 063/524] perf: add sw64 support Add Build, Makefiles, common headers and routines for SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/perf/arch/sw_64/Build | 2 + tools/perf/arch/sw_64/Makefile | 4 + tools/perf/arch/sw_64/include/arch-tests.h | 12 +++ tools/perf/arch/sw_64/include/perf_regs.h | 92 ++++++++++++++++++ tools/perf/arch/sw_64/tests/Build | 3 + tools/perf/arch/sw_64/tests/arch-tests.c | 16 ++++ tools/perf/arch/sw_64/tests/dwarf-unwind.c | 63 +++++++++++++ tools/perf/arch/sw_64/tests/regs_load.S | 47 ++++++++++ tools/perf/arch/sw_64/util/Build | 4 + tools/perf/arch/sw_64/util/dwarf-regs.c | 94 +++++++++++++++++++ tools/perf/arch/sw_64/util/perf_regs.c | 6 ++ tools/perf/arch/sw_64/util/unwind-libdw.c | 60 ++++++++++++ tools/perf/arch/sw_64/util/unwind-libunwind.c | 84 +++++++++++++++++ tools/perf/util/libunwind/sw64.c | 33 +++++++ 14 files changed, 520 insertions(+) create mode 100644 tools/perf/arch/sw_64/Build create mode 100644 tools/perf/arch/sw_64/Makefile create mode 100644 tools/perf/arch/sw_64/include/arch-tests.h create mode 100644 tools/perf/arch/sw_64/include/perf_regs.h create mode 100644 tools/perf/arch/sw_64/tests/Build create mode 100644 tools/perf/arch/sw_64/tests/arch-tests.c create mode 100644 tools/perf/arch/sw_64/tests/dwarf-unwind.c create mode 100644 tools/perf/arch/sw_64/tests/regs_load.S create mode 100644 tools/perf/arch/sw_64/util/Build create mode 100644 tools/perf/arch/sw_64/util/dwarf-regs.c create mode 100644 tools/perf/arch/sw_64/util/perf_regs.c create mode 100644 tools/perf/arch/sw_64/util/unwind-libdw.c create mode 100644 tools/perf/arch/sw_64/util/unwind-libunwind.c create mode 100644 tools/perf/util/libunwind/sw64.c diff --git a/tools/perf/arch/sw_64/Build b/tools/perf/arch/sw_64/Build new file mode 100644 index 000000000000..36222e64bbf7 --- /dev/null +++ b/tools/perf/arch/sw_64/Build @@ -0,0 +1,2 @@ +perf-y += util/ +perf-$(CONFIG_DWARF_UNWIND) += tests/ diff --git a/tools/perf/arch/sw_64/Makefile b/tools/perf/arch/sw_64/Makefile new file mode 100644 index 000000000000..1aa9dd772489 --- /dev/null +++ b/tools/perf/arch/sw_64/Makefile @@ -0,0 +1,4 @@ +ifndef NO_DWARF +PERF_HAVE_DWARF_REGS := 1 +endif +PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET := 1 diff --git a/tools/perf/arch/sw_64/include/arch-tests.h b/tools/perf/arch/sw_64/include/arch-tests.h new file mode 100644 index 000000000000..90ec4c8cb880 --- /dev/null +++ b/tools/perf/arch/sw_64/include/arch-tests.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_TESTS_H +#define ARCH_TESTS_H + +#ifdef HAVE_DWARF_UNWIND_SUPPORT +struct thread; +struct perf_sample; +#endif + +extern struct test arch_tests[]; + +#endif diff --git a/tools/perf/arch/sw_64/include/perf_regs.h b/tools/perf/arch/sw_64/include/perf_regs.h new file mode 100644 index 000000000000..e0c1b15375b5 --- /dev/null +++ b/tools/perf/arch/sw_64/include/perf_regs.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_PERF_REGS_H +#define ARCH_PERF_REGS_H + +#include +#include +#include + +void perf_regs_load(u64 *regs); + +#define PERF_REGS_MASK ((1ULL << PERF_REG_SW64_MAX) - 1) +#define PERF_REGS_MAX PERF_REG_SW64_MAX +#define PERF_SAMPLE_REGS_ABI PERF_SAMPLE_REGS_ABI_64 + +#define PERF_REG_IP PERF_REG_SW64_PC +#define PERF_REG_SP PERF_REG_SW64_SP + +static inline const char *perf_reg_name(int id) +{ + switch (id) { + case PERF_REG_SW64_R0: + return "r0"; + case PERF_REG_SW64_R1: + return "r1"; + case PERF_REG_SW64_R2: + return "r2"; + case PERF_REG_SW64_R3: + return "r3"; + case PERF_REG_SW64_R4: + return "r4"; + case PERF_REG_SW64_R5: + return "r5"; + case PERF_REG_SW64_R6: + return "r6"; + case PERF_REG_SW64_R7: + return "r7"; + case PERF_REG_SW64_R8: + return "r8"; + case PERF_REG_SW64_R9: + return "r9"; + case PERF_REG_SW64_R10: + return "r10"; + case PERF_REG_SW64_R11: + return "r11"; + case PERF_REG_SW64_R12: + return "r12"; + case PERF_REG_SW64_R13: + return "r13"; + case PERF_REG_SW64_R14: + return "r14"; + case PERF_REG_SW64_R15: + return "r15"; + case PERF_REG_SW64_R16: + return "r16"; + case PERF_REG_SW64_R17: + return "r17"; + case PERF_REG_SW64_R18: + return "r18"; + case PERF_REG_SW64_R19: + return "r19"; + case PERF_REG_SW64_R20: + return "r20"; + case PERF_REG_SW64_R21: + return "r21"; + case PERF_REG_SW64_R22: + return "r22"; + case PERF_REG_SW64_R23: + return "r23"; + case PERF_REG_SW64_R24: + return "r24"; + case PERF_REG_SW64_R25: + return "r25"; + case PERF_REG_SW64_R26: + return "r26"; + case PERF_REG_SW64_R27: + return "r27"; + case PERF_REG_SW64_R28: + return "r28"; + case PERF_REG_SW64_GP: + return "gp"; + case PERF_REG_SW64_SP: + return "sp"; + case PERF_REG_SW64_PC: + return "pc"; + default: + return NULL; + } + + return NULL; +} + +#endif /* ARCH_PERF_REGS_H */ diff --git a/tools/perf/arch/sw_64/tests/Build b/tools/perf/arch/sw_64/tests/Build new file mode 100644 index 000000000000..b8a38eadfb35 --- /dev/null +++ b/tools/perf/arch/sw_64/tests/Build @@ -0,0 +1,3 @@ +perf-y += regs_load.o +perf-y += dwarf-unwind.o +perf-y += arch-tests.o diff --git a/tools/perf/arch/sw_64/tests/arch-tests.c b/tools/perf/arch/sw_64/tests/arch-tests.c new file mode 100644 index 000000000000..5b1543c98022 --- /dev/null +++ b/tools/perf/arch/sw_64/tests/arch-tests.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "tests/tests.h" +#include "arch-tests.h" + +struct test arch_tests[] = { +#ifdef HAVE_DWARF_UNWIND_SUPPORT + { + .desc = "DWARF unwind", + .func = test__dwarf_unwind, + }, +#endif + { + .func = NULL, + }, +}; diff --git a/tools/perf/arch/sw_64/tests/dwarf-unwind.c b/tools/perf/arch/sw_64/tests/dwarf-unwind.c new file mode 100644 index 000000000000..cd7047b7a546 --- /dev/null +++ b/tools/perf/arch/sw_64/tests/dwarf-unwind.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "perf_regs.h" +#include "thread.h" +#include "map.h" +#include "maps.h" +#include "event.h" +#include "debug.h" +#include "tests/tests.h" + +#define STACK_SIZE 8192 + +static int sample_ustack(struct perf_sample *sample, + struct thread *thread, u64 *regs) +{ + struct stack_dump *stack = &sample->user_stack; + struct map *map; + unsigned long sp; + u64 stack_size, *buf; + + buf = malloc(STACK_SIZE); + if (!buf) { + printf("failed to allocate sample uregs data\n"); + return -1; + } + + sp = (unsigned long) regs[PERF_REG_SW64_SP]; + + map = maps__find(thread->maps, (u64)sp); + if (!map) { + printf("failed to get stack map\n"); + free(buf); + return -1; + } + + stack_size = map->end - sp; + stack_size = stack_size > STACK_SIZE ? STACK_SIZE : stack_size; + + memcpy(buf, (void *) sp, stack_size); + stack->data = (char *) buf; + stack->size = stack_size; + return 0; +} + +int test__arch_unwind_sample(struct perf_sample *sample, + struct thread *thread) +{ + struct regs_dump *regs = &sample->user_regs; + u64 *buf; + + buf = calloc(1, sizeof(u64) * PERF_REGS_MAX); + if (!buf) { + printf("failed to allocate sample uregs data\n"); + return -1; + } + + perf_regs_load(buf); + regs->abi = PERF_SAMPLE_REGS_ABI; + regs->regs = buf; + regs->mask = PERF_REGS_MASK; + + return sample_ustack(sample, thread, buf); +} diff --git a/tools/perf/arch/sw_64/tests/regs_load.S b/tools/perf/arch/sw_64/tests/regs_load.S new file mode 100644 index 000000000000..8c5aabc2c6fb --- /dev/null +++ b/tools/perf/arch/sw_64/tests/regs_load.S @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include + +.text +.set noat +.type perf_regs_load,%function +#define STL_REG(r) stl $r, (8 * r)($16) +#define LDL_REG(r) ldl $r, (8 * r)($16) +#define SP (8 * 30) +#define PC (8 * 31) +SYM_FUNC_START(perf_regs_load) + STL_REG(0) + STL_REG(1) + STL_REG(2) + STL_REG(3) + STL_REG(4) + STL_REG(5) + STL_REG(6) + STL_REG(7) + STL_REG(8) + STL_REG(9) + STL_REG(10) + STL_REG(11) + STL_REG(12) + STL_REG(13) + STL_REG(14) + STL_REG(15) + STL_REG(16) + STL_REG(17) + STL_REG(18) + STL_REG(19) + STL_REG(20) + STL_REG(21) + STL_REG(22) + STL_REG(23) + STL_REG(24) + STL_REG(25) + STL_REG(26) + STL_REG(27) + STL_REG(28) + STL_REG(29) + mov $30, $17 + stl $17, (SP)($16) + stl $26, (PC)($16) + LDL_REG(17) + ret +SYM_FUNC_END(perf_regs_load) diff --git a/tools/perf/arch/sw_64/util/Build b/tools/perf/arch/sw_64/util/Build new file mode 100644 index 000000000000..39f459b636a0 --- /dev/null +++ b/tools/perf/arch/sw_64/util/Build @@ -0,0 +1,4 @@ +perf-y += perf_regs.o +perf-$(CONFIG_DWARF) += dwarf-regs.o +perf-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o +perf-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o diff --git a/tools/perf/arch/sw_64/util/dwarf-regs.c b/tools/perf/arch/sw_64/util/dwarf-regs.c new file mode 100644 index 000000000000..11c1ee5444da --- /dev/null +++ b/tools/perf/arch/sw_64/util/dwarf-regs.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Mapping of DWARF debug register numbers into register names. + * + * Copyright (C) 2010 Will Deacon, ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include /* for struct user_pt_regs */ +#include +#include "util.h" + +struct pt_regs_dwarfnum { + const char *name; + unsigned int dwarfnum; +}; + +#define REG_DWARFNUM_NAME(r, num) {.name = r, .dwarfnum = num} +#define GPR_DWARFNUM_NAME(num) \ + {.name = __stringify(%x##num), .dwarfnum = num} +#define REG_DWARFNUM_END {.name = NULL, .dwarfnum = 0} +#define DWARFNUM2OFFSET(index) \ + (index * sizeof((struct user_pt_regs *)0)->regs[0]) + +static const struct pt_regs_dwarfnum regdwarfnum_table[] = { + GPR_DWARFNUM_NAME(0), + GPR_DWARFNUM_NAME(1), + GPR_DWARFNUM_NAME(2), + GPR_DWARFNUM_NAME(3), + GPR_DWARFNUM_NAME(4), + GPR_DWARFNUM_NAME(5), + GPR_DWARFNUM_NAME(6), + GPR_DWARFNUM_NAME(7), + GPR_DWARFNUM_NAME(8), + GPR_DWARFNUM_NAME(9), + GPR_DWARFNUM_NAME(10), + GPR_DWARFNUM_NAME(11), + GPR_DWARFNUM_NAME(12), + GPR_DWARFNUM_NAME(13), + GPR_DWARFNUM_NAME(14), + GPR_DWARFNUM_NAME(15), + REG_DWARFNUM_NAME("%fp", 15), + GPR_DWARFNUM_NAME(16), + GPR_DWARFNUM_NAME(17), + GPR_DWARFNUM_NAME(18), + GPR_DWARFNUM_NAME(19), + GPR_DWARFNUM_NAME(20), + GPR_DWARFNUM_NAME(21), + GPR_DWARFNUM_NAME(22), + GPR_DWARFNUM_NAME(23), + GPR_DWARFNUM_NAME(24), + GPR_DWARFNUM_NAME(25), + GPR_DWARFNUM_NAME(26), + GPR_DWARFNUM_NAME(27), + GPR_DWARFNUM_NAME(28), + REG_DWARFNUM_NAME("%gp", 29), + REG_DWARFNUM_NAME("%sp", 30), + REG_DWARFNUM_END, +}; + +/** + * get_arch_regstr() - lookup register name from it's DWARF register number + * @n: the DWARF register number + * + * get_arch_regstr() returns the name of the register in struct + * regdwarfnum_table from it's DWARF register number. If the register is not + * found in the table, this returns NULL; + */ +const char *get_arch_regstr(unsigned int n) +{ + const struct pt_regs_dwarfnum *roff; + + for (roff = regdwarfnum_table; roff->name != NULL; roff++) + if (roff->dwarfnum == n) + return roff->name; + return NULL; +} + +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_dwarfnum *roff; + + for (roff = regdwarfnum_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return DWARFNUM2OFFSET(roff->dwarfnum); + return -EINVAL; +} diff --git a/tools/perf/arch/sw_64/util/perf_regs.c b/tools/perf/arch/sw_64/util/perf_regs.c new file mode 100644 index 000000000000..2833e101a7c6 --- /dev/null +++ b/tools/perf/arch/sw_64/util/perf_regs.c @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "../../../util/perf_regs.h" + +const struct sample_reg sample_reg_masks[] = { + SMPL_REG_END +}; diff --git a/tools/perf/arch/sw_64/util/unwind-libdw.c b/tools/perf/arch/sw_64/util/unwind-libdw.c new file mode 100644 index 000000000000..3e2b6acc40ac --- /dev/null +++ b/tools/perf/arch/sw_64/util/unwind-libdw.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "../../util/unwind-libdw.h" +#include "../../util/perf_regs.h" +#include "../../util/event.h" + +bool libdw__arch_set_initial_registers(Dwfl_Thread *thread, void *arg) +{ + struct unwind_info *ui = arg; + struct regs_dump *user_regs = &ui->sample->user_regs; + Dwarf_Word dwarf_regs[PERF_REG_SW64_MAX], dwarf_pc; + +#define REG(r) ({ \ + Dwarf_Word val = 0; \ + perf_reg_value(&val, user_regs, PERF_REG_SW64_##r); \ + val; \ +}) + + dwarf_regs[0] = REG(R0); + dwarf_regs[1] = REG(R1); + dwarf_regs[2] = REG(R2); + dwarf_regs[3] = REG(R3); + dwarf_regs[4] = REG(R4); + dwarf_regs[5] = REG(R5); + dwarf_regs[6] = REG(R6); + dwarf_regs[7] = REG(R7); + dwarf_regs[8] = REG(R8); + dwarf_regs[9] = REG(R9); + dwarf_regs[10] = REG(R10); + dwarf_regs[11] = REG(R11); + dwarf_regs[12] = REG(R12); + dwarf_regs[13] = REG(R13); + dwarf_regs[14] = REG(R14); + dwarf_regs[15] = REG(R15); + dwarf_regs[16] = REG(R16); + dwarf_regs[17] = REG(R17); + dwarf_regs[18] = REG(R18); + dwarf_regs[19] = REG(R19); + dwarf_regs[20] = REG(R20); + dwarf_regs[21] = REG(R21); + dwarf_regs[22] = REG(R22); + dwarf_regs[23] = REG(R23); + dwarf_regs[24] = REG(R24); + dwarf_regs[25] = REG(R25); + dwarf_regs[26] = REG(R26); + dwarf_regs[27] = REG(R27); + dwarf_regs[28] = REG(R28); + dwarf_regs[29] = REG(R29); + dwarf_regs[30] = REG(R30); + dwarf_regs[31] = REG(R31); + + if (!dwfl_thread_state_registers(thread, 0, PERF_REG_SW64_MAX, + dwarf_regs)) + return false; + + dwarf_pc = REG(PC); + dwfl_thread_state_register_pc(thread, dwarf_pc); + + return true; +} diff --git a/tools/perf/arch/sw_64/util/unwind-libunwind.c b/tools/perf/arch/sw_64/util/unwind-libunwind.c new file mode 100644 index 000000000000..134e3c2280d2 --- /dev/null +++ b/tools/perf/arch/sw_64/util/unwind-libunwind.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#ifndef REMOTE_UNWIND_LIBUNWIND +#include +#include "perf_regs.h" +#include "../../util/unwind.h" +#include "../../util/debug.h" +#endif + +int LIBUNWIND__ARCH_REG_ID(int regnum) +{ + switch (regnum) { + case UNW_SW_64_R0: + return PERF_REG_SW64_R0; + case UNW_SW_64_R1: + return PERF_REG_SW64_R1; + case UNW_SW_64_R2: + return PERF_REG_SW64_R2; + case UNW_SW_64_R3: + return PERF_REG_SW64_R3; + case UNW_SW_64_R4: + return PERF_REG_SW64_R4; + case UNW_SW_64_R5: + return PERF_REG_SW64_R5; + case UNW_SW_64_R6: + return PERF_REG_SW64_R6; + case UNW_SW_64_R7: + return PERF_REG_SW64_R7; + case UNW_SW_64_R8: + return PERF_REG_SW64_R8; + case UNW_SW_64_R9: + return PERF_REG_SW64_R9; + case UNW_SW_64_R10: + return PERF_REG_SW64_R10; + case UNW_SW_64_R11: + return PERF_REG_SW64_R11; + case UNW_SW_64_R12: + return PERF_REG_SW64_R12; + case UNW_SW_64_R13: + return PERF_REG_SW64_R13; + case UNW_SW_64_R14: + return PERF_REG_SW64_R14; + case UNW_SW_64_R15: + return PERF_REG_SW64_R15; + case UNW_SW_64_R16: + return PERF_REG_SW64_R16; + case UNW_SW_64_R17: + return PERF_REG_SW64_R17; + case UNW_SW_64_R18: + return PERF_REG_SW64_R18; + case UNW_SW_64_R19: + return PERF_REG_SW64_R19; + case UNW_SW_64_R20: + return PERF_REG_SW64_R20; + case UNW_SW_64_R21: + return PERF_REG_SW64_R21; + case UNW_SW_64_R22: + return PERF_REG_SW64_R22; + case UNW_SW_64_R23: + return PERF_REG_SW64_R23; + case UNW_SW_64_R24: + return PERF_REG_SW64_R24; + case UNW_SW_64_R25: + return PERF_REG_SW64_R25; + case UNW_SW_64_R26: + return PERF_REG_SW64_R26; + case UNW_SW_64_R27: + return PERF_REG_SW64_R27; + case UNW_SW_64_R28: + return PERF_REG_SW64_R28; + case UNW_SW_64_R29: + return PERF_REG_SW64_GP; + case UNW_SW_64_R30: + return PERF_REG_SW64_SP; + case UNW_SW_64_PC: + return PERF_REG_SW64_PC; + default: + pr_err("unwind: invalid reg id %d\n", regnum); + return -EINVAL; + } + + return -EINVAL; +} diff --git a/tools/perf/util/libunwind/sw64.c b/tools/perf/util/libunwind/sw64.c new file mode 100644 index 000000000000..12452bf2ab8b --- /dev/null +++ b/tools/perf/util/libunwind/sw64.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file setups defines to compile arch specific binary from the + * generic one. + * + * The function 'LIBUNWIND__ARCH_REG_ID' name is set according to arch + * name and the defination of this function is included directly from + * 'arch/arm64/util/unwind-libunwind.c', to make sure that this function + * is defined no matter what arch the host is. + * + * Finally, the arch specific unwind methods are exported which will + * be assigned to each arm64 thread. + */ + +#define REMOTE_UNWIND_LIBUNWIND + +/* Define arch specific functions & regs for libunwind, should be + * defined before including "unwind.h" + */ +#define LIBUNWIND__ARCH_REG_ID(regnum) libunwind__sw_64_reg_id(regnum) +#define LIBUNWIND__ARCH_REG_IP PERF_REG_SW64_PC +#define LIBUNWIND__ARCH_REG_SP PERF_REG_SW64_SP + +#include "unwind.h" +#include "debug.h" +#include "libunwind-sw_64.h" +#include <../../../arch/sw_64/include/uapi/asm/perf_regs.h> +#include "../../arch/sw_64/util/unwind-libunwind.c" + +#include "util/unwind-libunwind-local.c" + +struct unwind_libunwind_ops * +sw64_unwind_libunwind_ops = &_unwind_libunwind_ops; -- Gitee From 8c0dfa0caef50fec552dea533f25f42bd8ec0005 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:27:37 +0800 Subject: [PATCH 064/524] perf: fix sw64 support Modify generic Build, Makefiles and routines for SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/perf/Makefile.config | 13 +++++++++++++ tools/perf/tests/Build | 2 +- tools/perf/util/Build | 1 + tools/perf/util/annotate.c | 3 +++ tools/perf/util/env.c | 2 ++ tools/perf/util/unwind-libunwind.c | 4 ++++ 6 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 9da9f878f50f..1c2dae9f0152 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -94,6 +94,12 @@ ifeq ($(SRCARCH),csky) NO_PERF_REGS := 0 endif +ifeq ($(SRCARCH),sw_64) + NO_PERF_REGS := 0 + CFLAGS += -mieee + LIBUNWIND_LIBS = -lunwind -lunwind-sw_64 +endif + ifeq ($(ARCH),s390) NO_PERF_REGS := 0 CFLAGS += -fPIC -I$(OUTPUT)arch/s390/include/generated @@ -642,6 +648,13 @@ ifndef NO_LIBUNWIND CFLAGS += -DNO_LIBUNWIND_DEBUG_FRAME_AARCH64 endif endif + ifeq ($(feature-libunwind-sw_64), 1) + $(call detected,CONFIG_LIBUNWIND_SW64) + CFLAGS += -DHAVE_LIBUNWIND_SW_64_SUPPORT + LDFLAGS += -lunwind-sw_64 + EXTLIBS_LIBUNWIND += -lunwind-sw_64 + have_libunwind = 1 + endif ifneq ($(feature-libunwind), 1) msg := $(warning No libunwind found. Please install libunwind-dev[el] >= 1.1 and/or set LIBUNWIND_DIR); diff --git a/tools/perf/tests/Build b/tools/perf/tests/Build index 2b45ffa462a6..29c065768a8b 100644 --- a/tools/perf/tests/Build +++ b/tools/perf/tests/Build @@ -68,7 +68,7 @@ perf-y += event_groups.o perf-y += symbols.o perf-y += util.o -ifeq ($(SRCARCH),$(filter $(SRCARCH),x86 arm arm64 powerpc)) +ifeq ($(SRCARCH),$(filter $(SRCARCH),x86 arm arm64 powerpc sw_64)) perf-$(CONFIG_DWARF_UNWIND) += dwarf-unwind.o endif diff --git a/tools/perf/util/Build b/tools/perf/util/Build index 6d657c9927f7..89a051732e87 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -199,6 +199,7 @@ perf-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind-local.o perf-$(CONFIG_LIBUNWIND) += unwind-libunwind.o perf-$(CONFIG_LIBUNWIND_X86) += libunwind/x86_32.o perf-$(CONFIG_LIBUNWIND_AARCH64) += libunwind/arm64.o +perf-$(CONFIG_LIBUNWIND_SW64) += libunwind/sw64.o ifeq ($(CONFIG_LIBTRACEEVENT),y) perf-$(CONFIG_LIBBABELTRACE) += data-convert-bt.o diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index 6dfe11cbf30e..51625af5f85d 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -183,6 +183,9 @@ static struct arch architectures[] = { .comment_char = '#', }, }, + { + .name = "sw_64", + }, { .name = "x86", .init = x86__annotate_init, diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c index cea15c568602..3004e50043b0 100644 --- a/tools/perf/util/env.c +++ b/tools/perf/util/env.c @@ -441,6 +441,8 @@ static const char *normalize_arch(char *arch) return "arm64"; if (!strncmp(arch, "arm", 3) || !strcmp(arch, "sa110")) return "arm"; + if (!strncmp(arch, "sw_64", 5)) + return "sw_64"; if (!strncmp(arch, "s390", 4)) return "s390"; if (!strncmp(arch, "parisc", 6)) diff --git a/tools/perf/util/unwind-libunwind.c b/tools/perf/util/unwind-libunwind.c index 76cd63de80a8..c2e84a827e33 100644 --- a/tools/perf/util/unwind-libunwind.c +++ b/tools/perf/util/unwind-libunwind.c @@ -11,6 +11,7 @@ struct unwind_libunwind_ops __weak *local_unwind_libunwind_ops; struct unwind_libunwind_ops __weak *x86_32_unwind_libunwind_ops; struct unwind_libunwind_ops __weak *arm64_unwind_libunwind_ops; +struct unwind_libunwind_ops __weak *sw64_unwind_libunwind_ops; static void unwind__register_ops(struct maps *maps, struct unwind_libunwind_ops *ops) { @@ -53,6 +54,9 @@ int unwind__prepare_access(struct maps *maps, struct map *map, bool *initialized } else if (!strcmp(arch, "arm64") || !strcmp(arch, "arm")) { if (dso_type == DSO__TYPE_64BIT) ops = arm64_unwind_libunwind_ops; + } else if (!strcmp(arch, "sw_64")) { + if (dso_type == DSO__TYPE_64BIT) + ops = sw64_unwind_libunwind_ops; } if (!ops) { -- Gitee From 8080dd6515a1d6f413f65cdd20871b3704760108 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Jan 2024 16:29:33 +0800 Subject: [PATCH 065/524] selftests: fix sw64 support Modify generic routines for SW64 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- .../ftrace/test.d/kprobe/kprobe_args_string.tc | 3 +++ .../ftrace/test.d/kprobe/kprobe_args_syntax.tc | 4 ++++ .../testing/selftests/mm/virtual_address_range.c | 5 +++++ .../testing/selftests/seccomp/seccomp_benchmark.c | 5 +++++ tools/testing/selftests/seccomp/seccomp_bpf.c | 15 ++++++++++++++- 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc index 4f72c2875f6b..dbc76ca50ab5 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc @@ -31,6 +31,9 @@ mips*) loongarch*) ARG1=%r4 ;; +sw_64) + ARG1=%r16 +;; *) echo "Please implement other architecture here" exit_untested diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc index 1df61e13a812..8de38fb00bae 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc @@ -44,6 +44,10 @@ loongarch*) GOODREG=%r4 BADREG=%r12 ;; +sw_64) + GOODREG=%r16 + BADREG=%ps +;; *) echo "Please implement other architecture here" exit_untested diff --git a/tools/testing/selftests/mm/virtual_address_range.c b/tools/testing/selftests/mm/virtual_address_range.c index bae0ceaf95b1..76efbd5637cb 100644 --- a/tools/testing/selftests/mm/virtual_address_range.c +++ b/tools/testing/selftests/mm/virtual_address_range.c @@ -54,6 +54,11 @@ #define HIGH_ADDR_SHIFT 49 #define NR_CHUNKS_LOW NR_CHUNKS_256TB #define NR_CHUNKS_HIGH NR_CHUNKS_3840TB +#elif defined __sw_64__ +#define HIGH_ADDR_MARK ADDR_MARK_128TB * 32UL +#define HIGH_ADDR_SHIFT 53 +#define NR_CHUNKS_LOW NR_CHUNKS_128TB * 32UL +#define NR_CHUNKS_HIGH 0 #else #define HIGH_ADDR_MARK ADDR_MARK_128TB #define HIGH_ADDR_SHIFT 48 diff --git a/tools/testing/selftests/seccomp/seccomp_benchmark.c b/tools/testing/selftests/seccomp/seccomp_benchmark.c index 5b5c9d558dee..7004099ce11b 100644 --- a/tools/testing/selftests/seccomp/seccomp_benchmark.c +++ b/tools/testing/selftests/seccomp/seccomp_benchmark.c @@ -20,6 +20,11 @@ #include "../kselftest.h" +#ifdef __sw_64__ +#define __NR_getpid 174 +#define __NR_getppid 175 +#endif + unsigned long long timing(clockid_t clk_id, unsigned long long samples) { struct timespec start, finish; diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index 15325ca35f1e..06a3d10a9766 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -66,6 +66,11 @@ # define PR_SET_PTRACER 0x59616d61 #endif +#ifdef __sw_64__ +#define __NR_getpid 174 +#define __NR_getppid 175 +#endif + #ifndef PR_SET_NO_NEW_PRIVS #define PR_SET_NO_NEW_PRIVS 38 #define PR_GET_NO_NEW_PRIVS 39 @@ -142,6 +147,8 @@ struct seccomp_data { # define __NR_seccomp 372 # elif defined(__mc68000__) # define __NR_seccomp 380 +# elif defined(__sw_64__) +# define __NR_seccomp 514 # else # warning "seccomp syscall number unknown for this architecture" # define __NR_seccomp 0xffff @@ -1850,6 +1857,12 @@ TEST_F(TRACE_poke, getpid_runs_normally) # define ARCH_REGS struct user_regs_struct # define SYSCALL_NUM(_regs) (_regs).orig_d0 # define SYSCALL_RET(_regs) (_regs).d0 +#elif defined(__sw_64__) +# define ARCH_REGS struct user_pt_regs +# define SYSCALL_NUM(_regs) (_regs).regs[0] +# define SYSCALL_RET(_regs) (_regs).regs[0] +# define SYSCALL_RET_SET(_regs, _val) \ + TH_LOG("Can't modify syscall return on this architecture") #else # error "Do not know how to find your architecture's registers and syscalls" #endif @@ -1914,7 +1927,7 @@ const bool ptrace_entry_set_syscall_ret = * Use PTRACE_GETREGS and PTRACE_SETREGS when available. This is useful for * architectures without HAVE_ARCH_TRACEHOOK (e.g. User-mode Linux). */ -#if defined(__x86_64__) || defined(__i386__) || defined(__mips__) || defined(__mc68000__) +#if defined(__x86_64__) || defined(__i386__) || defined(__mips__) || defined(__mc68000__) || defined(__sw_64__) # define ARCH_GETREGS(_regs) ptrace(PTRACE_GETREGS, tracee, 0, &(_regs)) # define ARCH_SETREGS(_regs) ptrace(PTRACE_SETREGS, tracee, 0, &(_regs)) #else -- Gitee From 4dd45861b9d441ccf9225a1d1a90471731270659 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:55:15 +0800 Subject: [PATCH 066/524] drivers: acpi: add sw64 support Add acpi drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/acpi/acpi_apd.c | 19 ++++++++++++++++++- drivers/acpi/numa/Kconfig | 2 +- drivers/acpi/numa/srat.c | 2 +- drivers/acpi/pci_mcfg.c | 26 ++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index 80f945cbec8a..791f4b234e02 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -40,7 +40,8 @@ struct apd_private_data { const struct apd_device_desc *dev_desc; }; -#if defined(CONFIG_X86_AMD_PLATFORM_DEVICE) || defined(CONFIG_ARM64) +#if defined(CONFIG_X86_AMD_PLATFORM_DEVICE) || \ +defined(CONFIG_ARM64) || defined(CONFIG_SW64) #define APD_ADDR(desc) ((unsigned long)&desc) static int acpi_apd_setup(struct apd_private_data *pdata) @@ -178,6 +179,18 @@ static const struct apd_device_desc hip08_spi_desc = { }; #endif /* CONFIG_ARM64 */ +#ifdef CONFIG_SW64 +static const struct apd_device_desc sunway_i2c_desc = { + .setup = acpi_apd_setup, + .fixed_clk_rate = 25000000, +}; + +static const struct apd_device_desc sunway_spi_desc = { + .setup = acpi_apd_setup, + .fixed_clk_rate = 25000000, +}; +#endif + #endif /* @@ -246,6 +259,10 @@ static const struct acpi_device_id acpi_apd_device_ids[] = { { "HISI02A3", APD_ADDR(hip08_lite_i2c_desc) }, { "HISI0173", APD_ADDR(hip08_spi_desc) }, { "NXP0001", APD_ADDR(nxp_i2c_desc) }, +#endif +#ifdef CONFIG_SW64 + { "HISI02A1", APD_ADDR(sunway_i2c_desc) }, + { "HISI0173", APD_ADDR(sunway_spi_desc) }, #endif { } }; diff --git a/drivers/acpi/numa/Kconfig b/drivers/acpi/numa/Kconfig index 39b1f34c21df..67d1f40bfa9f 100644 --- a/drivers/acpi/numa/Kconfig +++ b/drivers/acpi/numa/Kconfig @@ -2,7 +2,7 @@ config ACPI_NUMA bool "NUMA support" depends on NUMA - depends on (X86 || IA64 || ARM64 || LOONGARCH) + depends on (X86 || IA64 || ARM64 || LOONGARCH || SW64) default y if IA64 || ARM64 config ACPI_HMAT diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index a44c0761fd1c..8ed90017a56d 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -211,7 +211,7 @@ __weak int __init numa_fill_memblks(u64 start, u64 end) return NUMA_NO_MEMBLK; } -#if defined(CONFIG_X86) || defined(CONFIG_ARM64) || defined(CONFIG_LOONGARCH) +#if defined(CONFIG_X86) || defined(CONFIG_ARM64) || defined(CONFIG_LOONGARCH) || defined(CONFIG_SW64) /* * Callback for SLIT parsing. pxm_to_node() returns NUMA_NO_NODE for * I/O localities since SRAT does not list them. I/O localities are diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c index f378af5bd220..98bbb01981c5 100644 --- a/drivers/acpi/pci_mcfg.c +++ b/drivers/acpi/pci_mcfg.c @@ -195,6 +195,32 @@ static struct mcfg_fixup mcfg_quirks[] = { LOONGSON_ECAM_MCFG("LOONGSON", 7), #endif /* LOONGARCH */ + +#ifdef CONFIG_SW64 +#define _SW64_ECAM_QUIRK(rev, seg) \ + { "SUNWAY", "SUNWAY. ", rev, seg, MCFG_BUS_ANY, &sw64_pci_ecam_ops } +#define SW64_ECAM_QUIRK(rev, node) _SW64_ECAM_QUIRK(rev, node * 8 + 0),\ + _SW64_ECAM_QUIRK(rev, node * 8 + 1),\ + _SW64_ECAM_QUIRK(rev, node * 8 + 2),\ + _SW64_ECAM_QUIRK(rev, node * 8 + 3),\ + _SW64_ECAM_QUIRK(rev, node * 8 + 4),\ + _SW64_ECAM_QUIRK(rev, node * 8 + 5),\ + _SW64_ECAM_QUIRK(rev, node * 8 + 6),\ + _SW64_ECAM_QUIRK(rev, node * 8 + 7) + + /** + * According to the address space of sw64, up to 8 nodes supported + * with a maximum of 8 pcie controllers per node + */ + SW64_ECAM_QUIRK(1, 0x00), + SW64_ECAM_QUIRK(1, 0x01), + SW64_ECAM_QUIRK(1, 0x02), + SW64_ECAM_QUIRK(1, 0x03), + SW64_ECAM_QUIRK(1, 0x04), + SW64_ECAM_QUIRK(1, 0x05), + SW64_ECAM_QUIRK(1, 0x06), + SW64_ECAM_QUIRK(1, 0x07), +#endif /* SW64 */ }; static char mcfg_oem_id[ACPI_OEM_ID_SIZE]; -- Gitee From f28302be4da1b2b3581afaf48df9c2be58d57905 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:42:24 +0800 Subject: [PATCH 067/524] drivers: clocksource: add sw64 support Add clocksource driver for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/clocksource/Kconfig | 4 + drivers/clocksource/Makefile | 1 + drivers/clocksource/timer-sw64.c | 411 +++++++++++++++++++++++++++++++ 3 files changed, 416 insertions(+) create mode 100644 drivers/clocksource/timer-sw64.c diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 8208a3d89563..fd459065efce 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -733,4 +733,8 @@ config GOLDFISH_TIMER help Support for the timer/counter of goldfish-rtc +config SW64_TIMER + bool + depends on SW64 || COMPILE_TEST + endmenu diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index 368c3461dab8..b9ef4c79915e 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -89,3 +89,4 @@ obj-$(CONFIG_MSC313E_TIMER) += timer-msc313e.o obj-$(CONFIG_GOLDFISH_TIMER) += timer-goldfish.o obj-$(CONFIG_GXP_TIMER) += timer-gxp.o obj-$(CONFIG_CLKSRC_LOONGSON1_PWM) += timer-loongson1-pwm.o +obj-$(CONFIG_SW64_TIMER) += timer-sw64.o diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c new file mode 100644 index 000000000000..a124b6d8fed9 --- /dev/null +++ b/drivers/clocksource/timer-sw64.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define SHTCLK_RATE_KHZ 25000 +#define SHTCLK_RATE (SHTCLK_RATE_KHZ * 1000) + +#if defined(CONFIG_SUBARCH_C4) +static u64 read_longtime(struct clocksource *cs) +{ + return read_csr(CSR_SHTCLOCK); +} + +static struct clocksource clocksource_longtime = { + .name = "longtime", + .rating = 100, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_longtime, +}; + +static u64 notrace read_sched_clock(void) +{ + return read_csr(CSR_SHTCLOCK); +} + +void __init sw64_setup_clocksource(void) +{ + clocksource_register_khz(&clocksource_longtime, SHTCLK_RATE_KHZ); + sched_clock_register(read_sched_clock, BITS_PER_LONG, SHTCLK_RATE); +} + +void __init setup_sched_clock(void) { } +#elif defined(CONFIG_SUBARCH_C3B) +#ifdef CONFIG_SMP +static u64 read_longtime(struct clocksource *cs) +{ + unsigned long node; + + node = __this_cpu_read(hard_node_id); + return __io_read_longtime(node); +} + +static int longtime_enable(struct clocksource *cs) +{ + switch (cpu_desc.model) { + case CPU_SW3231: + sw64_io_write(0, GPIO_SWPORTA_DR, 0); + sw64_io_write(0, GPIO_SWPORTA_DDR, 0xff); + break; + case CPU_SW831: + __io_write_longtime_start_en(0, 0x1); + break; + default: + break; + } + + return 0; +} + +static struct clocksource clocksource_longtime = { + .name = "longtime", + .rating = 100, + .enable = longtime_enable, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_longtime, +}; + +static u64 read_vtime(struct clocksource *cs) +{ + unsigned long vtime_addr; + + vtime_addr = IO_BASE | LONG_TIME; + return rdio64(vtime_addr); +} + +static int vtime_enable(struct clocksource *cs) +{ + return 0; +} + +static struct clocksource clocksource_vtime = { + .name = "vtime", + .rating = 100, + .enable = vtime_enable, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_vtime, +}; +#else /* !SMP */ +static u64 read_tc(struct clocksource *cs) +{ + return rdtc(); +} + +static struct clocksource clocksource_tc = { + .name = "tc", + .rating = 300, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 22, + .mult = 0, /* To be filled in */ + .read = read_tc, +}; +#endif /* SMP */ + +#define DEFAULT_MCLK 25 /* Mhz */ + +void __init sw64_setup_clocksource(void) +{ + unsigned int mclk = *((unsigned char *)__va(MB_MCLK)); + + if (!mclk) + mclk = DEFAULT_MCLK; + +#ifdef CONFIG_SMP + if (is_in_host()) + clocksource_register_khz(&clocksource_longtime, mclk * 1000); + else + clocksource_register_khz(&clocksource_vtime, DEFAULT_MCLK * 1000); +#else + clocksource_register_hz(&clocksource_tc, get_cpu_freq()); + pr_info("Setup clocksource TC, mult = %d\n", clocksource_tc.mult); +#endif +} + +DECLARE_PER_CPU(u64, tc_offset); +static u64 sc_start, sc_shift, sc_multi; +DEFINE_STATIC_KEY_FALSE(use_tc_as_sched_clock); + +static int __init sched_clock_setup(char *opt) +{ + if (!opt) + return -EINVAL; + + if (!strncmp(opt, "on", 2)) { + static_branch_enable(&use_tc_as_sched_clock); + pr_info("Using TC instead of jiffies as source of sched_clock()\n"); + } + + return 0; +} +early_param("tc_sched_clock", sched_clock_setup); + +static void __init calibrate_sched_clock(void) +{ + sc_start = rdtc(); +} + +void __init setup_sched_clock(void) +{ + unsigned long step; + + sc_shift = 7; + step = 1UL << sc_shift; + sc_multi = step * NSEC_PER_SEC / get_cpu_freq(); + calibrate_sched_clock(); + + pr_info("sched_clock: sc_multi=%llu, sc_shift=%llu\n", sc_multi, sc_shift); +} + +#ifdef CONFIG_GENERIC_SCHED_CLOCK +static u64 notrace read_sched_clock(void) +{ + return (rdtc() - sc_start) >> sc_shift; +} + +void __init sw64_sched_clock_init(void) +{ + sched_clock_register(sched_clock_read, BITS_PER_LONG, get_cpu_freq() >> sc_shift); +} +#else /* !CONFIG_GENERIC_SCHED_CLOCK */ +/* + * scheduler clock - returns current time in nanoseconds. + */ +unsigned long long notrace sched_clock(void) +{ + if (static_branch_likely(&use_tc_as_sched_clock)) + return ((rdtc() - sc_start + __this_cpu_read(tc_offset)) >> sc_shift) * sc_multi; + else + return (jiffies - INITIAL_JIFFIES) * (NSEC_PER_SEC / HZ); +} + +#ifdef CONFIG_DEBUG_FS +static ssize_t sched_clock_status_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + char buf[2]; + + if (static_key_enabled(&use_tc_as_sched_clock)) + buf[0] = 'Y'; + else + buf[0] = 'N'; + buf[1] = '\n'; + return simple_read_from_buffer(user_buf, count, ppos, buf, 2); +} + +static ssize_t sched_clock_status_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + int r; + bool bv; + bool val = static_key_enabled(&use_tc_as_sched_clock); + + r = kstrtobool_from_user(user_buf, count, &bv); + if (!r) { + if (val != bv) { + if (bv) { + static_branch_enable(&use_tc_as_sched_clock); + pr_info("source of sched_clock() switched from jiffies to TC\n"); + } else { + static_branch_disable(&use_tc_as_sched_clock); + pr_info("source of sched_clock() switched from TC to jiffies\n"); + } + } else { + if (val) + pr_info("source of sched_clock() unchanged (using TC)\n"); + else + pr_info("source of sched_clock() unchanged (using jiffies)\n"); + } + } + + return count; +} + +static const struct file_operations sched_clock_status_fops = { + .read = sched_clock_status_read, + .write = sched_clock_status_write, + .open = nonseekable_open, + .llseek = no_llseek, +}; + +static int __init sched_clock_debug_init(void) +{ + struct dentry *sched_clock_status; + + if (!sw64_debugfs_dir) + return -ENODEV; + + sched_clock_status = debugfs_create_file("tc_sched_clock", + 0644, sw64_debugfs_dir, NULL, + &sched_clock_status_fops); + + if (!sched_clock_status) + return -ENOMEM; + + return 0; +} +late_initcall(sched_clock_debug_init); +#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_GENERIC_SCHED_CLOCK */ + +#endif + + + +static int timer_next_event(unsigned long delta, + struct clock_event_device *evt); +static int timer_set_shutdown(struct clock_event_device *evt); +static int timer_set_oneshot(struct clock_event_device *evt); + +/* + * The local apic timer can be used for any function which is CPU local. + */ +static struct clock_event_device timer_clockevent = { + .name = "timer", + .features = CLOCK_EVT_FEAT_ONESHOT, + .shift = 20, + .mult = 0, + .set_state_shutdown = timer_set_shutdown, + .set_state_oneshot = timer_set_oneshot, + .set_next_event = timer_next_event, + .rating = 300, + .irq = -1, +}; + +static int vtimer_next_event(unsigned long delta, + struct clock_event_device *evt) +{ + hcall(HCALL_SET_CLOCKEVENT, delta, 0, 0); + return 0; +} + +static int vtimer_shutdown(struct clock_event_device *evt) +{ + hcall(HCALL_SET_CLOCKEVENT, 0, 0, 0); + return 0; +} + +static int vtimer_set_oneshot(struct clock_event_device *evt) +{ + return 0; +} +static struct clock_event_device vtimer_clockevent = { + .name = "vtimer", + .features = CLOCK_EVT_FEAT_ONESHOT, + .shift = 20, + .mult = 0, + .set_state_shutdown = vtimer_shutdown, + .set_state_oneshot = vtimer_set_oneshot, + .set_next_event = vtimer_next_event, + .rating = 300, + .irq = -1, +}; + +static DEFINE_PER_CPU(struct clock_event_device, timer_events); + +/* + * Program the next event, relative to now + */ +static int timer_next_event(unsigned long delta, + struct clock_event_device *evt) +{ + wrtimer(delta); + return 0; +} + +static int timer_set_shutdown(struct clock_event_device *evt) +{ + wrtimer(0); + return 0; +} + +static int timer_set_oneshot(struct clock_event_device *evt) +{ + /* + * SW-TIMER support CLOCK_EVT_MODE_ONESHOT only, and automatically. + * unlike PIT and HPET, which support ONESHOT or PERIODIC by setting PIT_MOD or HPET_Tn_CFG + * so, nothing to do here ... + */ + return 0; +} + +void sw64_update_clockevents(unsigned long cpu, u32 freq) +{ + struct clock_event_device *swevt = &per_cpu(timer_events, cpu); + + if (cpu == smp_processor_id()) + clockevents_update_freq(swevt, freq); + else { + clockevents_calc_mult_shift(swevt, freq, 4); + swevt->min_delta_ns = clockevent_delta2ns(swevt->min_delta_ticks, swevt); + swevt->max_delta_ns = clockevent_delta2ns(swevt->max_delta_ticks, swevt); + } +} + +/* + * Setup the local timer for this CPU. Copy the initialized values + * of the boot CPU and register the clock event in the framework. + */ +void sw64_setup_timer(void) +{ + unsigned long min_delta; + int cpu = smp_processor_id(); + struct clock_event_device *swevt = &per_cpu(timer_events, cpu); + + /* min_delta ticks => 100ns */ + min_delta = get_cpu_freq()/1000/1000/10; + + if (is_in_guest()) { + memcpy(swevt, &vtimer_clockevent, sizeof(*swevt)); + /* + * CUIWEI: This value is very important. + * If it's too small, the timer will timeout when the IER + * haven't been opened. + */ + min_delta *= 4; + } else { + memcpy(swevt, &timer_clockevent, sizeof(*swevt)); + } + swevt->cpumask = cpumask_of(cpu); + swevt->set_state_shutdown(swevt); + clockevents_config_and_register(swevt, get_cpu_freq(), min_delta, ULONG_MAX); +} + +void sw64_timer_interrupt(void) +{ + struct clock_event_device *evt = this_cpu_ptr(&timer_events); + + irq_enter(); + if (!evt->event_handler) { + pr_warn("Spurious local timer interrupt on cpu %d\n", + smp_processor_id()); + timer_set_shutdown(evt); + return; + } + + inc_irq_stat(timer_irqs_event); + + evt->event_handler(evt); + + irq_exit(); +} -- Gitee From 6f1e469fd99ea14c43e244216c15b36b5d614750 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:42:49 +0800 Subject: [PATCH 068/524] drivers: cpufreq: add sw64 support Add cpufreq drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/cpufreq/Kconfig | 23 ++++ drivers/cpufreq/Makefile | 2 + drivers/cpufreq/sw64_cpufreq.c | 175 +++++++++++++++++++++++++ drivers/cpufreq/sw64_cpufreq_debugfs.c | 101 ++++++++++++++ 4 files changed, 301 insertions(+) create mode 100644 drivers/cpufreq/sw64_cpufreq.c create mode 100644 drivers/cpufreq/sw64_cpufreq_debugfs.c diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 8cbe5c4ddca3..23b2e3b2d8c6 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -314,6 +314,29 @@ config SH_CPU_FREQ If unsure, say N. endif +if SW64 +config SW64_CPUFREQ + bool "SW64 CPU Frequency interface" + depends on UNCORE_XUELANG + default y + help + This adds the CPUFreq driver for SW64 processor which supports + software configurable cpu frequency. + + For details, take a look at . + + If unsure, say N. + +config SW64_CPUFREQ_DEBUGFS + bool "SW64 CPU Frequency debugfs interface" + depends on SW64_CPUFREQ && DEBUG_FS + default y + help + Turns on the DebugFS interface for CPU Frequency. + + If you don't know what to do here, say N. +endif + config QORIQ_CPUFREQ tristate "CPU frequency scaling driver for Freescale QorIQ SoCs" depends on OF && COMMON_CLK diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 076ea3ac1b56..f9c1c9012ce7 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -108,3 +108,5 @@ obj-$(CONFIG_LOONGSON3_ACPI_CPUFREQ) += loongson3-acpi-cpufreq.o obj-$(CONFIG_SH_CPU_FREQ) += sh-cpufreq.o obj-$(CONFIG_SPARC_US2E_CPUFREQ) += sparc-us2e-cpufreq.o obj-$(CONFIG_SPARC_US3_CPUFREQ) += sparc-us3-cpufreq.o +obj-$(CONFIG_SW64_CPUFREQ) += sw64_cpufreq.o +obj-$(CONFIG_SW64_CPUFREQ_DEBUGFS) += sw64_cpufreq_debugfs.o diff --git a/drivers/cpufreq/sw64_cpufreq.c b/drivers/cpufreq/sw64_cpufreq.c new file mode 100644 index 000000000000..f4bf5f3cc550 --- /dev/null +++ b/drivers/cpufreq/sw64_cpufreq.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/arch/sw/kernel/setup.c + * + * Copyright (C) 1995 Linus Torvalds + */ + +/* + * Cpufreq driver for the sw64 processors + * + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include /* set_cpus_allowed() */ +#include +#include +#include + +#include +#include +#include + +static uint nowait; + +static struct clk *cpuclk; + + +static int sw64_cpu_freq_notifier(struct notifier_block *nb, + unsigned long val, void *data); + +static struct notifier_block sw64_cpufreq_notifier_block = { + .notifier_call = sw64_cpu_freq_notifier +}; + +static int sw64_cpu_freq_notifier(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freqs = (struct cpufreq_freqs *)data; + unsigned long cpu = freqs->policy->cpu; + + if (val == CPUFREQ_POSTCHANGE) + sw64_update_clockevents(cpu, freqs->new * 1000); + + return 0; +} + +static unsigned int sw64_cpufreq_get(unsigned int cpu) +{ + struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); + + if (!policy || IS_ERR(policy->clk)) { + pr_err("%s: No %s associated to cpu: %d\n", + __func__, policy ? "clk" : "policy", cpu); + return 0; + } + + return __sw64_cpufreq_get(policy); +} + +/* + * Here we notify other drivers of the proposed change and the final change. + */ +static int sw64_cpufreq_target(struct cpufreq_policy *policy, + unsigned int index) +{ + unsigned int cpu = policy->cpu; + + if (!cpu_online(cpu)) + return -ENODEV; + + /* setting the cpu frequency */ + sw64_set_rate(index); + update_cpu_freq(freq_table[index].frequency); + + return 0; +} + +static int sw64_cpufreq_cpu_init(struct cpufreq_policy *policy) +{ + cpuclk = sw64_clk_get(NULL, "cpu_clk"); + if (IS_ERR(cpuclk)) { + pr_err("couldn't get CPU clk\n"); + return PTR_ERR(cpuclk); + } + + policy->clk = cpuclk; + + cpufreq_generic_init(policy, freq_table, 0); + + return 0; +} + +static int sw64_cpufreq_verify(struct cpufreq_policy_data *policy) +{ + return cpufreq_frequency_table_verify(policy, freq_table); +} + +static int sw64_cpufreq_exit(struct cpufreq_policy *policy) +{ + return 0; +} + +static struct freq_attr *sw64_table_attr[] = { + &cpufreq_freq_attr_scaling_available_freqs, NULL, +}; + +static struct cpufreq_driver sw64_cpufreq_driver = { + .name = "sw64", + .init = sw64_cpufreq_cpu_init, + .verify = sw64_cpufreq_verify, + .target_index = sw64_cpufreq_target, + .get = sw64_cpufreq_get, + .exit = sw64_cpufreq_exit, + .attr = sw64_table_attr, +}; + +static const struct platform_device_id platform_device_ids[] = { + { + .name = "sw64_cpufreq", + }, + {} +}; + +MODULE_DEVICE_TABLE(platform, platform_device_ids); + +static struct platform_driver platform_driver = { + .driver = { + .name = "sw64_cpufreq", + }, + .id_table = platform_device_ids, +}; + + +static int __init cpufreq_init(void) +{ + int ret; + + if (is_in_guest()) { + pr_warn("Now sw_64 CPUFreq does not support virtual machines\n"); + return -ENODEV; + } + + /* Register platform stuff */ + ret = platform_driver_register(&platform_driver); + if (ret) + return ret; + + pr_info("SW-64 CPU frequency driver\n"); + + cpufreq_register_notifier(&sw64_cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + + return cpufreq_register_driver(&sw64_cpufreq_driver); +} + +static void __exit cpufreq_exit(void) +{ + cpufreq_unregister_driver(&sw64_cpufreq_driver); + cpufreq_unregister_notifier(&sw64_cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + + platform_driver_unregister(&platform_driver); +} + +module_init(cpufreq_init); +module_exit(cpufreq_exit); + +module_param(nowait, uint, 0644); +MODULE_PARM_DESC(nowait, "Disable SW-64 specific wait"); + +MODULE_DESCRIPTION("cpufreq driver for sw64"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/sw64_cpufreq_debugfs.c b/drivers/cpufreq/sw64_cpufreq_debugfs.c new file mode 100644 index 000000000000..bb4ae26bc22b --- /dev/null +++ b/drivers/cpufreq/sw64_cpufreq_debugfs.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#include +#include +#include + +static int cpufreq_show(struct seq_file *m, void *v) +{ + int i; + u64 val; + int freq; + + val = sw64_io_read(0, CLK_CTL); + val = val >> CORE_PLL2_CFG_SHIFT; + + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (freq_table[i].frequency != CPUFREQ_ENTRY_INVALID) + freq = freq_table[i].frequency; + else + freq = freq_table[i].driver_data; + + if (val == i) + seq_printf(m, "[%d] ", freq); + else + seq_printf(m, "%d ", freq); + } + seq_puts(m, "\n"); + + return 0; +} + +static int cpufreq_open(struct inode *inode, struct file *file) +{ + return single_open(file, cpufreq_show, NULL); +} + +static ssize_t cpufreq_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + char buf[5]; + size_t size; + int cf, i, err, index, freq; + + size = min(sizeof(buf) - 1, len); + if (copy_from_user(buf, user_buf, size)) + return -EFAULT; + buf[size] = '\0'; + + err = kstrtoint(buf, 10, &cf); + if (err) + return err; + + index = -1; + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (freq_table[i].frequency != CPUFREQ_ENTRY_INVALID) + freq = freq_table[i].frequency; + else + freq = freq_table[i].driver_data; + + if (cf == freq) { + index = i; + break; + } + } + + if (index < 0) + return -EINVAL; + + sw64_set_rate(index); + update_cpu_freq(freq); + return len; +} + +static const struct file_operations set_cpufreq_fops = { + .open = cpufreq_open, + .read = seq_read, + .write = cpufreq_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __init cpufreq_debugfs_init(void) +{ + struct dentry *cpufreq_entry; + + if (!sw64_debugfs_dir) + return -ENODEV; + + cpufreq_entry = debugfs_create_file("cpufreq", 0600, + sw64_debugfs_dir, NULL, + &set_cpufreq_fops); + if (!cpufreq_entry) + return -ENOMEM; + + return 0; +} +late_initcall(cpufreq_debugfs_init); -- Gitee From 119342591d556f6064164e7bae1db9fa63b60197 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:47:32 +0800 Subject: [PATCH 069/524] drivers: efi: add sw64 support Add efi drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/firmware/efi/Kconfig | 2 +- drivers/firmware/efi/Makefile | 2 + drivers/firmware/efi/efi.c | 2 +- drivers/firmware/efi/sunway-init.c | 221 ++++++++++++++++++++++++++ drivers/firmware/efi/sunway-runtime.c | 85 ++++++++++ 5 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 drivers/firmware/efi/sunway-init.c create mode 100644 drivers/firmware/efi/sunway-runtime.c diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig index 231f1c70d1db..138491a4b494 100644 --- a/drivers/firmware/efi/Kconfig +++ b/drivers/firmware/efi/Kconfig @@ -224,7 +224,7 @@ config EFI_DISABLE_PCI_DMA config EFI_EARLYCON def_bool y - depends on SERIAL_EARLYCON && !ARM && !IA64 + depends on SERIAL_EARLYCON && !ARM && !IA64 && !SW64 select FONT_SUPPORT select ARCH_USE_MEMREMAP_PROT diff --git a/drivers/firmware/efi/Makefile b/drivers/firmware/efi/Makefile index b4528af86517..7c1b924e8ea3 100644 --- a/drivers/firmware/efi/Makefile +++ b/drivers/firmware/efi/Makefile @@ -35,6 +35,8 @@ obj-$(CONFIG_SYSFB) += sysfb_efi.o arm-obj-$(CONFIG_EFI) := efi-init.o arm-runtime.o obj-$(CONFIG_ARM) += $(arm-obj-y) obj-$(CONFIG_ARM64) += $(arm-obj-y) +sw64-obj-$(CONFIG_EFI) := sunway-init.o sunway-runtime.o +obj-$(CONFIG_SW64) += $(sw64-obj-y) riscv-obj-$(CONFIG_EFI) := efi-init.o riscv-runtime.o obj-$(CONFIG_RISCV) += $(riscv-obj-y) #obj-$(CONFIG_LOONGARCH) += efi-init.o diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 1ab161e00c86..610ed21d5bdb 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -809,7 +809,7 @@ int __init efi_systab_check_header(const efi_table_hdr_t *systab_hdr) return 0; } -#ifndef CONFIG_IA64 +#if !defined(CONFIG_IA64) && !defined(CONFIG_SW64) static const efi_char16_t *__init map_fw_vendor(unsigned long fw_vendor, size_t size) { diff --git a/drivers/firmware/efi/sunway-init.c b/drivers/firmware/efi/sunway-init.c new file mode 100644 index 000000000000..870abc2f5afe --- /dev/null +++ b/drivers/firmware/efi/sunway-init.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Extensible Firmware Interface + * + * Based on Extensible Firmware Interface Specification version 2.4 + * + * Copyright (C) 2013 - 2015 Linaro Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#define pr_fmt(fmt) "efi: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +unsigned long entSuspend; +unsigned long bios_version; + +static int __init is_memory(efi_memory_desc_t *md) +{ + if (md->attribute & (EFI_MEMORY_WB|EFI_MEMORY_WT|EFI_MEMORY_WC)) + return 1; + return 0; +} +static efi_config_table_type_t arch_tables[] __initdata = { + {SMBIOS3_TABLE_GUID, NULL, ""}, + {SLEEP_ENTRY_GUID, &entSuspend, "SLEEP ENTRY"}, + {BIOS_VERSION_GUID, &bios_version, "BIOS VERSION"}, + {}, +}; + +static int __init uefi_init(u64 efi_system_table) +{ + efi_char16_t *c16; + efi_config_table_t *config_tables; + efi_system_table_t *systab; + size_t table_size; + char vendor[100] = "unknown"; + int i, retval; + + systab = early_memremap(efi_system_table, + sizeof(efi_system_table_t)); + if (systab == NULL) { + pr_warn("Unable to map EFI system table.\n"); + return -ENOMEM; + } + + set_bit(EFI_BOOT, &efi.flags); + if (IS_ENABLED(CONFIG_64BIT)) + set_bit(EFI_64BIT, &efi.flags); + + /* + * Verify the EFI Table + */ + if (systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) { + pr_err("System table signature incorrect\n"); + retval = -EINVAL; + goto out; + } + if ((systab->hdr.revision >> 16) < 2) + pr_warn("Warning: EFI system table version %d.%02d, expected 2.00 or greater\n", + systab->hdr.revision >> 16, + systab->hdr.revision & 0xffff); + + efi.runtime = systab->runtime; + efi.runtime_version = systab->hdr.revision; + + /* Show what we know for posterity */ + c16 = early_memremap(systab->fw_vendor, + sizeof(vendor) * sizeof(efi_char16_t)); + if (c16) { + for (i = 0; i < (int) sizeof(vendor) - 1 && *c16; ++i) + vendor[i] = c16[i]; + vendor[i] = '\0'; + early_memunmap(c16, sizeof(vendor) * sizeof(efi_char16_t)); + } + + pr_info("EFI v%u.%.02u by %s\n", + systab->hdr.revision >> 16, + systab->hdr.revision & 0xffff, vendor); + + table_size = sizeof(efi_config_table_64_t) * systab->nr_tables; + config_tables = early_memremap(systab->tables, table_size); + if (config_tables == NULL) { + pr_warn("Unable to map EFI config table array.\n"); + retval = -ENOMEM; + goto out; + } + + retval = efi_config_parse_tables(config_tables, systab->nr_tables, + arch_tables); + + early_memunmap(config_tables, table_size); +out: + early_memunmap(systab, sizeof(efi_system_table_t)); + + if (!bios_version) + retval = -EINVAL; + + return retval; +} + +/* + * Return true for regions that can be used as System RAM. + */ +static __init int is_usable_memory(efi_memory_desc_t *md) +{ + switch (md->type) { + case EFI_LOADER_CODE: + case EFI_LOADER_DATA: + case EFI_ACPI_RECLAIM_MEMORY: + case EFI_BOOT_SERVICES_CODE: + case EFI_BOOT_SERVICES_DATA: + case EFI_CONVENTIONAL_MEMORY: + case EFI_PERSISTENT_MEMORY: + /* + * According to the spec, these regions are no longer reserved + * after calling ExitBootServices(). However, we can only use + * them as System RAM if they can be mapped writeback cacheable. + */ + return (md->attribute & EFI_MEMORY_WB); + default: + break; + } + return false; +} + +static __init void reserve_regions(void) +{ + efi_memory_desc_t *md; + u64 paddr, npages, size; + + if (efi_enabled(EFI_DBG)) + pr_info("Processing EFI memory map:\n"); + + for_each_efi_memory_desc(md) { + paddr = md->phys_addr; + npages = md->num_pages; + + if (efi_enabled(EFI_DBG)) { + char buf[64]; + + pr_info(" 0x%012llx-0x%012llx %s\n", + paddr, paddr + (npages << EFI_PAGE_SHIFT) - 1, + efi_md_typeattr_format(buf, sizeof(buf), md)); + } + + memrange_efi_to_native(&paddr, &npages); + size = npages << PAGE_SHIFT; + + if (is_memory(md)) { + early_init_dt_add_memory_arch(paddr, size); + + if (!is_usable_memory(md)) + memblock_mark_nomap(paddr, size); + + /* keep ACPI reclaim memory intact for kexec etc. */ + if (md->type == EFI_ACPI_RECLAIM_MEMORY) + memblock_reserve(paddr, size); + } + } +} + +void __init efi_init(void) +{ + struct efi_memory_map_data data; + u64 efi_system_table; + + if (sunway_boot_params->efi_systab == 0) { + pr_info("System Table is not exist, disabling EFI.\n"); + return; + } + + /* Grab UEFI information placed in struct boot_params by stub */ + efi_system_table = sunway_boot_params->efi_systab; + if (!efi_system_table) + return; + + data.desc_version = sunway_boot_params->efi_memdesc_version; + data.desc_size = sunway_boot_params->efi_memdesc_size; + data.size = sunway_boot_params->efi_memmap_size; + data.phys_map = sunway_boot_params->efi_memmap; + + if (efi_memmap_init_early(&data) < 0) { + /* + * If we are booting via UEFI, the UEFI memory map is the only + * description of memory we have, so there is little point in + * proceeding if we cannot access it. + */ + panic("Unable to map EFI memory map.\n"); + } + + WARN(efi.memmap.desc_version != 1, + "Unexpected EFI_MEMORY_DESCRIPTOR version %ld", + efi.memmap.desc_version); + + if (uefi_init(efi_system_table) < 0) { + efi_memmap_unmap(); + return; + } + + reserve_regions(); + + memblock_reserve(sunway_boot_params->efi_memmap & PAGE_MASK, + PAGE_ALIGN(sunway_boot_params->efi_memmap_size + + (sunway_boot_params->efi_memmap & ~PAGE_MASK))); + +} diff --git a/drivers/firmware/efi/sunway-runtime.c b/drivers/firmware/efi/sunway-runtime.c new file mode 100644 index 000000000000..6bd96cff7d5d --- /dev/null +++ b/drivers/firmware/efi/sunway-runtime.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Extensible Firmware Interface + * + * Based on Extensible Firmware Interface Specification version 2.4 + * + * Copyright (C) 2013, 2014 Linaro Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * Enable the UEFI Runtime Services if all prerequisites are in place, i.e., + * non-early mapping of the UEFI system table and virtual mappings for all + * EFI_MEMORY_RUNTIME regions. + */ +static int __init sunway_enable_runtime_services(void) +{ + u64 mapsize; + + if (!efi_enabled(EFI_BOOT)) { + pr_info("EFI services will not be available.\n"); + return 0; + } + + efi_memmap_unmap(); + + mapsize = efi.memmap.desc_size * efi.memmap.nr_map; + + if (efi_memmap_init_late(efi.memmap.phys_map, mapsize)) { + pr_err("Failed to remap EFI memory map\n"); + return 0; + } + + if (efi_runtime_disabled()) { + pr_info("EFI runtime services will be disabled.\n"); + return 0; + } + + if (efi_enabled(EFI_RUNTIME_SERVICES)) { + pr_info("EFI runtime services access via paravirt.\n"); + return 0; + } + + /* Set up runtime services function pointers */ + efi_native_runtime_setup(); + set_bit(EFI_RUNTIME_SERVICES, &efi.flags); + + return 0; +} +early_initcall(sunway_enable_runtime_services); + + +static int __init sunway_dmi_init(void) +{ + /* + * On SW64, DMI depends on UEFI, and dmi_scan_machine() needs to + * be called early because dmi_id_init(), which is an arch_initcall + * itself, depends on dmi_scan_machine() having been called already. + */ + dmi_setup(); + return 0; +} +core_initcall(sunway_dmi_init); -- Gitee From 443e9ea693050095a6495548263f816a826a7d72 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:43:45 +0800 Subject: [PATCH 070/524] drivers: gpio: add sw64 support Add gpio drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/gpio/Kconfig | 9 + drivers/gpio/Makefile | 1 + drivers/gpio/gpio-sunway.c | 861 ++++++++++++++++++++++ include/linux/platform_data/gpio-sunway.h | 33 + 4 files changed, 904 insertions(+) create mode 100644 drivers/gpio/gpio-sunway.c create mode 100644 include/linux/platform_data/gpio-sunway.h diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index ebd4e113dc26..509f42e6ab6a 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -246,6 +246,15 @@ config GPIO_DWAPB Say Y or M here to build support for the Synopsys DesignWare APB GPIO block. +config GPIO_SUNWAY + tristate "Sunway gpio driver" + depends on SW64 + select GPIO_GENERIC + select GENERIC_IRQ_CHIP + help + Say Y or M here to build support for the Sunway + GPIO block. + config GPIO_EIC_SPRD tristate "Spreadtrum EIC support" depends on ARCH_SPRD || COMPILE_TEST diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index eb73b5d633eb..e44a700ec7d3 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -195,3 +195,4 @@ obj-$(CONFIG_GPIO_XTENSA) += gpio-xtensa.o obj-$(CONFIG_GPIO_ZEVIO) += gpio-zevio.o obj-$(CONFIG_GPIO_ZYNQ) += gpio-zynq.o obj-$(CONFIG_GPIO_ZYNQMP_MODEPIN) += gpio-zynqmp-modepin.o +obj-$(CONFIG_GPIO_SUNWAY) += gpio-sunway.o diff --git a/drivers/gpio/gpio-sunway.c b/drivers/gpio/gpio-sunway.c new file mode 100644 index 000000000000..b9c6848317db --- /dev/null +++ b/drivers/gpio/gpio-sunway.c @@ -0,0 +1,861 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2011 Jamie Iles + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * All enquiries to support@picochip.com + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gpiolib.h" +#include "gpiolib-acpi.h" + + +#define GPIO_SWPORTA_DR (0x00UL<<7) +#define GPIO_SWPORTA_DDR (0X04UL<<7) +#define GPIO_SWPORTB_DR (0X0CUL<<7) +#define GPIO_SWPORTB_DDR (0X10UL<<7) +#define GPIO_SWPORTC_DR (0x18UL<<7) +#define GPIO_SWPORTC_DDR (0x1cUL<<7) +#define GPIO_SWPORTD_DR (0x24UL<<7) +#define GPIO_SWPORTD_DDR (0x28UL<<7) +#define GPIO_INTEN (0x30UL<<7) +#define GPIO_INTMASK (0x34UL<<7) +#define GPIO_INTTYPE_LEVEL (0x38UL<<7) +#define GPIO_INT_POLARITY (0x3cUL<<7) +#define GPIO_INTSTATUS (0x40UL<<7) +#define GPIO_PORTA_DEBOUNCE (0x48UL<<7) +#define GPIO_PORTA_EOI (0x4cUL<<7) +#define GPIO_EXT_PORTA (0x50UL<<7) +#define GPIO_EXT_PORTB (0x54UL<<7) +#define GPIO_EXT_PORTC (0x58UL<<7) +#define GPIO_EXT_PORTD (0x5cUL<<7) + +#define DWAPB_MAX_PORTS 4 +#define GPIO_EXT_PORT_STRIDE 0x04 /* register stride 32 bits */ +#define GPIO_SWPORT_DR_STRIDE 0x0c /* register stride 3*32 bits */ +#define GPIO_SWPORT_DDR_STRIDE 0x0c /* register stride 3*32 bits */ + +#define GPIO_REG_OFFSET_V2 1 + +#define GPIO_INTMASK_V2 0x44 +#define GPIO_INTTYPE_LEVEL_V2 0x34 +#define GPIO_INT_POLARITY_V2 0x38 +#define GPIO_INTSTATUS_V2 0x3c +#define GPIO_PORTA_EOI_V2 0x40 + +struct sunway_gpio; + +#ifdef CONFIG_PM_SLEEP +/* Store GPIO context across system-wide suspend/resume transitions */ +struct sunway_context { + u32 data; + u32 dir; + u32 ext; + u32 int_en; + u32 int_mask; + u32 int_type; + u32 int_pol; + u32 int_deb; + u32 wake_en; +}; +#endif + +struct sunway_gpio_port { + struct gpio_chip gc; + bool is_registered; + struct sunway_gpio *gpio; +#ifdef CONFIG_PM_SLEEP + struct sunway_context *ctx; +#endif + unsigned int idx; +}; + +struct sunway_gpio { + struct device *dev; + void __iomem *regs; + struct sunway_gpio_port *ports; + unsigned int nr_ports; + struct irq_domain *domain; + unsigned int flags; + struct reset_control *rst; + struct clk *clk; +}; + +static inline u32 gpio_reg_v2_convert(unsigned int offset) +{ + switch (offset) { + case GPIO_INTMASK: + return GPIO_INTMASK_V2; + case GPIO_INTTYPE_LEVEL: + return GPIO_INTTYPE_LEVEL_V2; + case GPIO_INT_POLARITY: + return GPIO_INT_POLARITY_V2; + case GPIO_INTSTATUS: + return GPIO_INTSTATUS_V2; + case GPIO_PORTA_EOI: + return GPIO_PORTA_EOI_V2; + } + + return offset; +} + +static inline u32 gpio_reg_convert(struct sunway_gpio *gpio, unsigned int offset) +{ + if (gpio->flags & GPIO_REG_OFFSET_V2) + return gpio_reg_v2_convert(offset); + + return offset; +} + +static inline u32 sunway_read(struct sunway_gpio *gpio, unsigned int offset) +{ + struct gpio_chip *gc = &gpio->ports[0].gc; + void __iomem *reg_base = gpio->regs; + + return gc->read_reg(reg_base + gpio_reg_convert(gpio, offset)); +} + +static inline void sunway_write(struct sunway_gpio *gpio, unsigned int offset, + u32 val) +{ + struct gpio_chip *gc = &gpio->ports[0].gc; + void __iomem *reg_base = gpio->regs; + + gc->write_reg(reg_base + gpio_reg_convert(gpio, offset), val); +} + +static int sunway_gpio_to_irq(struct gpio_chip *gc, unsigned int offset) +{ + struct sunway_gpio_port *port = gpiochip_get_data(gc); + struct sunway_gpio *gpio = port->gpio; + + return irq_find_mapping(gpio->domain, offset); +} + +static struct sunway_gpio_port *sunway_offs_to_port(struct sunway_gpio *gpio, unsigned int offs) +{ + struct sunway_gpio_port *port; + int i; + + for (i = 0; i < gpio->nr_ports; i++) { + port = &gpio->ports[i]; + if (port->idx == offs / 32) + return port; + } + + return NULL; +} + +static void sunway_toggle_trigger(struct sunway_gpio *gpio, unsigned int offs) +{ + struct sunway_gpio_port *port = sunway_offs_to_port(gpio, offs); + struct gpio_chip *gc; + u32 pol; + int val; + + if (!port) + return; + gc = &port->gc; + + pol = sunway_read(gpio, GPIO_INT_POLARITY); + /* Just read the current value right out of the data register */ + val = gc->get(gc, offs % 32); + if (val) + pol &= ~BIT(offs); + else + pol |= BIT(offs); + + sunway_write(gpio, GPIO_INT_POLARITY, pol); +} + +static u32 sunway_do_irq(struct sunway_gpio *gpio) +{ + u32 irq_status = sunway_read(gpio, GPIO_INTSTATUS); + u32 ret = irq_status; + + while (irq_status) { + int hwirq = fls(irq_status) - 1; + int gpio_irq = irq_find_mapping(gpio->domain, hwirq); + + generic_handle_irq(gpio_irq); + irq_status &= ~BIT(hwirq); + + if ((irq_get_trigger_type(gpio_irq) & IRQ_TYPE_SENSE_MASK) + == IRQ_TYPE_EDGE_BOTH) + sunway_toggle_trigger(gpio, hwirq); + } + + return ret; +} + +static void sunway_irq_handler(struct irq_desc *desc) +{ + struct sunway_gpio *gpio = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + + sunway_do_irq(gpio); + + if (chip->irq_eoi) + chip->irq_eoi(irq_desc_get_irq_data(desc)); +} + +static void sunway_irq_enable(struct irq_data *d) +{ + struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d); + struct sunway_gpio *gpio = igc->private; + struct gpio_chip *gc = &gpio->ports[0].gc; + unsigned long flags; + u32 val; + + spin_lock_irqsave(&gc->bgpio_lock, flags); + val = sunway_read(gpio, GPIO_INTEN); + val |= BIT(d->hwirq); + sunway_write(gpio, GPIO_INTEN, val); + spin_unlock_irqrestore(&gc->bgpio_lock, flags); +} + +static void sunway_irq_disable(struct irq_data *d) +{ + struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d); + struct sunway_gpio *gpio = igc->private; + struct gpio_chip *gc = &gpio->ports[0].gc; + unsigned long flags; + u32 val; + + spin_lock_irqsave(&gc->bgpio_lock, flags); + val = sunway_read(gpio, GPIO_INTEN); + val &= ~BIT(d->hwirq); + sunway_write(gpio, GPIO_INTEN, val); + spin_unlock_irqrestore(&gc->bgpio_lock, flags); +} + +static int sunway_irq_reqres(struct irq_data *d) +{ + struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d); + struct sunway_gpio *gpio = igc->private; + struct gpio_chip *gc = &gpio->ports[0].gc; + int ret; + + ret = gpiochip_lock_as_irq(gc, irqd_to_hwirq(d)); + if (ret) { + dev_err(gpio->dev, "unable to lock HW IRQ %lu for IRQ\n", + irqd_to_hwirq(d)); + return ret; + } + return 0; +} + +static void sunway_irq_relres(struct irq_data *d) +{ + struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d); + struct sunway_gpio *gpio = igc->private; + struct gpio_chip *gc = &gpio->ports[0].gc; + + gpiochip_unlock_as_irq(gc, irqd_to_hwirq(d)); +} + +static int sunway_irq_set_type(struct irq_data *d, u32 type) +{ + struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d); + struct sunway_gpio *gpio = igc->private; + struct gpio_chip *gc = &gpio->ports[0].gc; + int bit = d->hwirq; + unsigned long level, polarity, flags; + + if (type & ~(IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING | + IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW)) + return -EINVAL; + + spin_lock_irqsave(&gc->bgpio_lock, flags); + level = sunway_read(gpio, GPIO_INTTYPE_LEVEL); + polarity = sunway_read(gpio, GPIO_INT_POLARITY); + + switch (type) { + case IRQ_TYPE_EDGE_BOTH: + level |= BIT(bit); + sunway_toggle_trigger(gpio, bit); + break; + case IRQ_TYPE_EDGE_RISING: + level |= BIT(bit); + polarity |= BIT(bit); + break; + case IRQ_TYPE_EDGE_FALLING: + level |= BIT(bit); + polarity &= ~BIT(bit); + break; + case IRQ_TYPE_LEVEL_HIGH: + level &= ~BIT(bit); + polarity |= BIT(bit); + break; + case IRQ_TYPE_LEVEL_LOW: + level &= ~BIT(bit); + polarity &= ~BIT(bit); + break; + } + + irq_setup_alt_chip(d, type); + + sunway_write(gpio, GPIO_INTTYPE_LEVEL, level); + if (type != IRQ_TYPE_EDGE_BOTH) + sunway_write(gpio, GPIO_INT_POLARITY, polarity); + spin_unlock_irqrestore(&gc->bgpio_lock, flags); + + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int sunway_irq_set_wake(struct irq_data *d, unsigned int enable) +{ + struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d); + struct sunway_gpio *gpio = igc->private; + struct sunway_context *ctx = gpio->ports[0].ctx; + + if (enable) + ctx->wake_en |= BIT(d->hwirq); + else + ctx->wake_en &= ~BIT(d->hwirq); + + return 0; +} +#endif + +static int sunway_gpio_set_debounce(struct gpio_chip *gc, + unsigned int offset, unsigned int debounce) +{ + struct sunway_gpio_port *port = gpiochip_get_data(gc); + struct sunway_gpio *gpio = port->gpio; + unsigned long flags, val_deb; + unsigned long mask = BIT(offset); + + spin_lock_irqsave(&gc->bgpio_lock, flags); + + val_deb = sunway_read(gpio, GPIO_PORTA_DEBOUNCE); + if (debounce) + sunway_write(gpio, GPIO_PORTA_DEBOUNCE, val_deb | mask); + else + sunway_write(gpio, GPIO_PORTA_DEBOUNCE, val_deb & ~mask); + + spin_unlock_irqrestore(&gc->bgpio_lock, flags); + + return 0; +} + +static int sunway_gpio_set_config(struct gpio_chip *gc, unsigned int offset, + unsigned long config) +{ + u32 debounce; + + if (pinconf_to_config_param(config) != PIN_CONFIG_INPUT_DEBOUNCE) + return -ENOTSUPP; + + debounce = pinconf_to_config_argument(config); + return sunway_gpio_set_debounce(gc, offset, debounce); +} + +static irqreturn_t sunway_irq_handler_mfd(int irq, void *dev_id) +{ + u32 worked; + struct sunway_gpio *gpio = dev_id; + + worked = sunway_do_irq(gpio); + + return worked ? IRQ_HANDLED : IRQ_NONE; +} + +static void sunway_configure_irqs(struct sunway_gpio *gpio, + struct sunway_gpio_port *port, + struct sunway_port_property *pp) +{ + struct gpio_chip *gc = &port->gc; + struct fwnode_handle *fwnode = pp->fwnode; + struct irq_chip_generic *irq_gc = NULL; + unsigned int hwirq, ngpio = gc->ngpio; + struct irq_chip_type *ct; + int err, i; + + gpio->domain = irq_domain_create_linear(fwnode, ngpio, + &irq_generic_chip_ops, gpio); + if (!gpio->domain) + return; + + err = irq_alloc_domain_generic_chips(gpio->domain, ngpio, 2, + "gpio-dwapb", handle_level_irq, + IRQ_NOREQUEST, 0, + IRQ_GC_INIT_NESTED_LOCK); + if (err) { + dev_info(gpio->dev, "irq_alloc_domain_generic_chips failed\n"); + irq_domain_remove(gpio->domain); + gpio->domain = NULL; + return; + } + + irq_gc = irq_get_domain_generic_chip(gpio->domain, 0); + if (!irq_gc) { + irq_domain_remove(gpio->domain); + gpio->domain = NULL; + return; + } + + irq_gc->reg_base = gpio->regs; + irq_gc->private = gpio; + + for (i = 0; i < 2; i++) { + ct = &irq_gc->chip_types[i]; + ct->chip.irq_ack = irq_gc_ack_set_bit; + ct->chip.irq_mask = irq_gc_mask_set_bit; + ct->chip.irq_unmask = irq_gc_mask_clr_bit; + ct->chip.irq_set_type = sunway_irq_set_type; + ct->chip.irq_enable = sunway_irq_enable; + ct->chip.irq_disable = sunway_irq_disable; + ct->chip.irq_request_resources = sunway_irq_reqres; + ct->chip.irq_release_resources = sunway_irq_relres; +#ifdef CONFIG_PM_SLEEP + ct->chip.irq_set_wake = sunway_irq_set_wake; +#endif + ct->regs.ack = gpio_reg_convert(gpio, GPIO_PORTA_EOI); + ct->regs.mask = gpio_reg_convert(gpio, GPIO_INTMASK); + ct->type = IRQ_TYPE_LEVEL_MASK; + } + + irq_gc->chip_types[0].type = IRQ_TYPE_LEVEL_MASK; + irq_gc->chip_types[1].type = IRQ_TYPE_EDGE_BOTH; + irq_gc->chip_types[1].handler = handle_edge_irq; + + if (!pp->irq_shared) { + int i; + + for (i = 0; i < pp->ngpio; i++) { + if (pp->irq[i] >= 0) + irq_set_chained_handler_and_data(pp->irq[i], + sunway_irq_handler, gpio); + } + } else { + /* + * Request a shared IRQ since where MFD would have devices + * using the same irq pin + */ + err = devm_request_irq(gpio->dev, pp->irq[0], + sunway_irq_handler_mfd, + IRQF_SHARED, "gpio-dwapb-mfd", gpio); + if (err) { + dev_err(gpio->dev, "error requesting IRQ\n"); + irq_domain_remove(gpio->domain); + gpio->domain = NULL; + return; + } + } + + for (hwirq = 0 ; hwirq < ngpio ; hwirq++) + irq_create_mapping(gpio->domain, hwirq); + + port->gc.to_irq = sunway_gpio_to_irq; +} + +static void sunway_irq_teardown(struct sunway_gpio *gpio) +{ + struct sunway_gpio_port *port = &gpio->ports[0]; + struct gpio_chip *gc = &port->gc; + unsigned int ngpio = gc->ngpio; + irq_hw_number_t hwirq; + + if (!gpio->domain) + return; + + for (hwirq = 0 ; hwirq < ngpio ; hwirq++) + irq_dispose_mapping(irq_find_mapping(gpio->domain, hwirq)); + + irq_domain_remove(gpio->domain); + gpio->domain = NULL; +} + +static int sunway_gpio_add_port(struct sunway_gpio *gpio, + struct sunway_port_property *pp, + unsigned int offs) +{ + struct sunway_gpio_port *port; + void __iomem *dat, *set, *dirout; + int err; + + port = &gpio->ports[offs]; + port->gpio = gpio; + port->idx = pp->idx; + +#ifdef CONFIG_PM_SLEEP + port->ctx = devm_kzalloc(gpio->dev, sizeof(*port->ctx), GFP_KERNEL); + if (!port->ctx) + return -ENOMEM; +#endif + + dat = gpio->regs + GPIO_EXT_PORTA + (pp->idx * GPIO_EXT_PORT_STRIDE); + set = gpio->regs + GPIO_SWPORTA_DR + (pp->idx * GPIO_SWPORT_DR_STRIDE); + dirout = gpio->regs + GPIO_SWPORTA_DDR + + (pp->idx * GPIO_SWPORT_DDR_STRIDE); + + /* This registers 32 GPIO lines per port */ + err = bgpio_init(&port->gc, gpio->dev, 4, dat, set, NULL, dirout, + NULL, 0); + if (err) { + dev_err(gpio->dev, "failed to init gpio chip for port%d\n", + port->idx); + return err; + } + +#ifdef CONFIG_OF_GPIO + port->gc.of_node = to_of_node(pp->fwnode); +#endif + port->gc.ngpio = pp->ngpio; + port->gc.base = pp->gpio_base; + + /* Only port A support debounce */ + if (pp->idx == 0) + port->gc.set_config = sunway_gpio_set_config; + + if (pp->has_irq) + sunway_configure_irqs(gpio, port, pp); + + err = gpiochip_add_data(&port->gc, port); + if (err) + dev_err(gpio->dev, "failed to register gpiochip for port%d\n", + port->idx); + else + port->is_registered = true; + + /* Add GPIO-signaled ACPI event support */ + if (pp->has_irq) + acpi_gpiochip_request_interrupts(&port->gc); + + return err; +} + +static void sunway_gpio_unregister(struct sunway_gpio *gpio) +{ + unsigned int m; + + for (m = 0; m < gpio->nr_ports; ++m) + if (gpio->ports[m].is_registered) + gpiochip_remove(&gpio->ports[m].gc); +} + +static struct sunway_platform_data * +sunway_gpio_get_pdata(struct device *dev) +{ + struct fwnode_handle *fwnode; + struct sunway_platform_data *pdata; + struct sunway_port_property *pp; + int nports; + int i, j; + + nports = device_get_child_node_count(dev); + if (nports == 0) + return ERR_PTR(-ENODEV); + + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return ERR_PTR(-ENOMEM); + + pdata->properties = devm_kcalloc(dev, nports, sizeof(*pp), GFP_KERNEL); + if (!pdata->properties) + return ERR_PTR(-ENOMEM); + + pdata->nports = nports; + + i = 0; + device_for_each_child_node(dev, fwnode) { + struct device_node *np = NULL; + + pp = &pdata->properties[i++]; + pp->fwnode = fwnode; + + if (fwnode_property_read_u32(fwnode, "reg", &pp->idx) || + pp->idx >= DWAPB_MAX_PORTS) { + dev_err(dev, + "missing/invalid port index for port%d\n", i); + fwnode_handle_put(fwnode); + return ERR_PTR(-EINVAL); + } + + if (fwnode_property_read_u32(fwnode, "snps,nr-gpios", + &pp->ngpio)) { + dev_info(dev, + "failed to get number of gpios for port%d\n", + i); + pp->ngpio = 32; + } + + pp->irq_shared = false; + pp->gpio_base = -1; + + /* + * Only port A can provide interrupts in all configurations of + * the IP. + */ + if (pp->idx != 0) + continue; + + if (dev->of_node && fwnode_property_read_bool(fwnode, + "interrupt-controller")) { + np = to_of_node(fwnode); + } + + for (j = 0; j < pp->ngpio; j++) { + pp->irq[j] = -ENXIO; + + if (np) + pp->irq[j] = of_irq_get(np, j); + else if (has_acpi_companion(dev)) + pp->irq[j] = platform_get_irq(to_platform_device(dev), j); + + if (pp->irq[j] >= 0) + pp->has_irq = true; + } + + if (!pp->has_irq) + dev_warn(dev, "no irq for port%d\n", pp->idx); + } + + return pdata; +} + +static const struct of_device_id sunway_of_match[] = { + { .compatible = "snps,sw-gpio", .data = (void *)0}, + { .compatible = "apm,xgene-gpio-v2", .data = (void *)GPIO_REG_OFFSET_V2}, + { /* Sentinel */ } +}; +MODULE_DEVICE_TABLE(of, sunway_of_match); + +static const struct acpi_device_id sunway_acpi_match[] = { + {"HISI0181", 0}, + {"APMC0D07", 0}, + {"APMC0D81", GPIO_REG_OFFSET_V2}, + { } +}; +MODULE_DEVICE_TABLE(acpi, sunway_acpi_match); + +static int sunway_gpio_probe(struct platform_device *pdev) +{ + unsigned int i; + struct resource *res; + struct sunway_gpio *gpio; + int err; + struct device *dev = &pdev->dev; + struct sunway_platform_data *pdata = dev_get_platdata(dev); + + if (!pdata) { + pdata = sunway_gpio_get_pdata(dev); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + } + + if (!pdata->nports) + return -ENODEV; + + gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL); + if (!gpio) + return -ENOMEM; + + gpio->dev = &pdev->dev; + gpio->nr_ports = pdata->nports; + + gpio->rst = devm_reset_control_get_optional_shared(dev, NULL); + if (IS_ERR(gpio->rst)) + return PTR_ERR(gpio->rst); + + reset_control_deassert(gpio->rst); + + gpio->ports = devm_kcalloc(&pdev->dev, gpio->nr_ports, + sizeof(*gpio->ports), GFP_KERNEL); + if (!gpio->ports) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + gpio->regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(gpio->regs)) + return PTR_ERR(gpio->regs); + + /* Optional bus clock */ + gpio->clk = devm_clk_get(&pdev->dev, "bus"); + if (!IS_ERR(gpio->clk)) { + err = clk_prepare_enable(gpio->clk); + if (err) { + dev_info(&pdev->dev, "Cannot enable clock\n"); + return err; + } + } + + gpio->flags = 0; + if (dev->of_node) { + gpio->flags = (uintptr_t)of_device_get_match_data(dev); + } else if (has_acpi_companion(dev)) { + const struct acpi_device_id *acpi_id; + + acpi_id = acpi_match_device(sunway_acpi_match, dev); + if (acpi_id) { + if (acpi_id->driver_data) + gpio->flags = acpi_id->driver_data; + } + } + + for (i = 0; i < gpio->nr_ports; i++) { + err = sunway_gpio_add_port(gpio, &pdata->properties[i], i); + if (err) + goto out_unregister; + } + platform_set_drvdata(pdev, gpio); + + return 0; + +out_unregister: + sunway_gpio_unregister(gpio); + sunway_irq_teardown(gpio); + clk_disable_unprepare(gpio->clk); + + return err; +} + +static int sunway_gpio_remove(struct platform_device *pdev) +{ + struct sunway_gpio *gpio = platform_get_drvdata(pdev); + + sunway_gpio_unregister(gpio); + sunway_irq_teardown(gpio); + reset_control_assert(gpio->rst); + clk_disable_unprepare(gpio->clk); + + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int sunway_gpio_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sunway_gpio *gpio = platform_get_drvdata(pdev); + struct gpio_chip *gc = &gpio->ports[0].gc; + unsigned long flags; + int i; + + spin_lock_irqsave(&gc->bgpio_lock, flags); + for (i = 0; i < gpio->nr_ports; i++) { + unsigned int offset; + unsigned int idx = gpio->ports[i].idx; + struct sunway_context *ctx = gpio->ports[i].ctx; + + BUG_ON(!ctx); + + offset = GPIO_SWPORTA_DDR + idx * GPIO_SWPORT_DDR_STRIDE; + ctx->dir = sunway_read(gpio, offset); + + offset = GPIO_SWPORTA_DR + idx * GPIO_SWPORT_DR_STRIDE; + ctx->data = sunway_read(gpio, offset); + + offset = GPIO_EXT_PORTA + idx * GPIO_EXT_PORT_STRIDE; + ctx->ext = sunway_read(gpio, offset); + + /* Only port A can provide interrupts */ + if (idx == 0) { + ctx->int_mask = sunway_read(gpio, GPIO_INTMASK); + ctx->int_en = sunway_read(gpio, GPIO_INTEN); + ctx->int_pol = sunway_read(gpio, GPIO_INT_POLARITY); + ctx->int_type = sunway_read(gpio, GPIO_INTTYPE_LEVEL); + ctx->int_deb = sunway_read(gpio, GPIO_PORTA_DEBOUNCE); + + /* Mask out interrupts */ + sunway_write(gpio, GPIO_INTMASK, + 0xffffffff & ~ctx->wake_en); + } + } + spin_unlock_irqrestore(&gc->bgpio_lock, flags); + + clk_disable_unprepare(gpio->clk); + + return 0; +} + +static int sunway_gpio_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sunway_gpio *gpio = platform_get_drvdata(pdev); + struct gpio_chip *gc = &gpio->ports[0].gc; + unsigned long flags; + int i; + + if (!IS_ERR(gpio->clk)) + clk_prepare_enable(gpio->clk); + + spin_lock_irqsave(&gc->bgpio_lock, flags); + for (i = 0; i < gpio->nr_ports; i++) { + unsigned int offset; + unsigned int idx = gpio->ports[i].idx; + struct sunway_context *ctx = gpio->ports[i].ctx; + + BUG_ON(!ctx); + + offset = GPIO_SWPORTA_DR + idx * GPIO_SWPORT_DR_STRIDE; + sunway_write(gpio, offset, ctx->data); + + offset = GPIO_SWPORTA_DDR + idx * GPIO_SWPORT_DDR_STRIDE; + sunway_write(gpio, offset, ctx->dir); + + offset = GPIO_EXT_PORTA + idx * GPIO_EXT_PORT_STRIDE; + sunway_write(gpio, offset, ctx->ext); + + /* Only port A can provide interrupts */ + if (idx == 0) { + sunway_write(gpio, GPIO_INTTYPE_LEVEL, ctx->int_type); + sunway_write(gpio, GPIO_INT_POLARITY, ctx->int_pol); + sunway_write(gpio, GPIO_PORTA_DEBOUNCE, ctx->int_deb); + sunway_write(gpio, GPIO_INTEN, ctx->int_en); + sunway_write(gpio, GPIO_INTMASK, ctx->int_mask); + + /* Clear out spurious interrupts */ + sunway_write(gpio, GPIO_PORTA_EOI, 0xffffffff); + } + } + spin_unlock_irqrestore(&gc->bgpio_lock, flags); + + return 0; +} +#endif + +static SIMPLE_DEV_PM_OPS(sunway_gpio_pm_ops, sunway_gpio_suspend, + sunway_gpio_resume); + +static struct platform_driver sunway_gpio_driver = { + .driver = { + .name = "gpio-sunway", + .pm = &sunway_gpio_pm_ops, + .of_match_table = of_match_ptr(sunway_of_match), + .acpi_match_table = ACPI_PTR(sunway_acpi_match), + }, + .probe = sunway_gpio_probe, + .remove = sunway_gpio_remove, +}; + +module_platform_driver(sunway_gpio_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Jamie Iles"); +MODULE_DESCRIPTION("Sunway GPIO driver"); diff --git a/include/linux/platform_data/gpio-sunway.h b/include/linux/platform_data/gpio-sunway.h new file mode 100644 index 000000000000..58b1bddeb409 --- /dev/null +++ b/include/linux/platform_data/gpio-sunway.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(c) 2014 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef GPIO_SUNWAY_H +#define GPIO_SUNWAY_H + +struct sunway_port_property { + struct fwnode_handle *fwnode; + unsigned int idx; + unsigned int ngpio; + unsigned int gpio_base; + int irq[32]; + bool has_irq; + bool irq_shared; +}; + +struct sunway_platform_data { + struct sunway_port_property *properties; + unsigned int nports; +}; + +#endif -- Gitee From a2d382da519069951a70682008e7f382f56f5f32 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:55:35 +0800 Subject: [PATCH 071/524] drivers: hwmon: add sw64 support Add hwmon drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/hwmon/Kconfig | 10 ++ drivers/hwmon/Makefile | 1 + drivers/hwmon/sw64_pvt.c | 224 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 235 insertions(+) create mode 100644 drivers/hwmon/sw64_pvt.c diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index d6c5eead770a..ea41c7e24c6e 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -38,6 +38,16 @@ config HWMON_DEBUG_CHIP comment "Native drivers" +config SENSORS_PVT + tristate "SW64 PVT monitor" + depends on SW64 + help + If you say yes here you get support for the voltage + sensor inside your CPU. + + This driver can also be built as a module. If so, the module + will be called PVT. + config SENSORS_ABITUGURU tristate "Abit uGuru (rev 1 & 2)" depends on X86 && DMI diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index cab312e74d3c..f7da084cfc46 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -221,6 +221,7 @@ obj-$(CONFIG_SENSORS_W83L786NG) += w83l786ng.o obj-$(CONFIG_SENSORS_WM831X) += wm831x-hwmon.o obj-$(CONFIG_SENSORS_WM8350) += wm8350-hwmon.o obj-$(CONFIG_SENSORS_XGENE) += xgene-hwmon.o +obj-$(CONFIG_SENSORS_PVT) += sw64_pvt.o obj-$(CONFIG_SENSORS_OCC) += occ/ obj-$(CONFIG_SENSORS_PECI) += peci/ diff --git a/drivers/hwmon/sw64_pvt.c b/drivers/hwmon/sw64_pvt.c new file mode 100644 index 000000000000..aedc29d44077 --- /dev/null +++ b/drivers/hwmon/sw64_pvt.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * PVT device driver. + * + * Part of lm_sensors, Linux kernel modules + * for hardware monitoring in sunway. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define PVT_VSYS 0 +#define PVT0_CTRL 0x7c00 +#define PVT02SPBU_DATA_OUT (0x1 << 26) +#define PVT_READ 0xc000 +#define PVT_WADDR 0xc800 +#define PVT_WDATA 0xcc00 + +/* The PVT registers */ +#define PVT_SAFECTRL 0x0 +#define CLK_SEL 0x1 +#define PVT_RUN 0x2 +#define PVT_CONFIG 0x3 +#define PVT_WAIT_TIME 0x4 +#define TS_ALARM_HVALUE_L 0x5 +#define TS_ALARM_HVALUE_H 0x6 +#define TS_ALARM_LVALUE_L 0x7 +#define TS_ALARM_LVALUE_H 0x8 +#define TS_ALARM_TIMES 0x9 +#define TRIMG 0xa +#define TRIMO 0xb +#define VS_ALARM_HVALUE_L 0xc +#define VS_ALARM_HVALUE_H 0xd +#define VS_ALARM_LVALUE_L 0xe +#define VS_ALARM_LVALUE_H 0xf +#define VS_ALARM_TIMES 0x10 +#define PVT_ALARM_CLEAR 0x11 +#define PVT_ALARM_MASK 0x12 +#define PVT_DATA_OUT_L 0x13 +#define PVT_DATA_OUT_H 0x14 +#define PVT_STATE_INFO 0x15 +#define PVT_ALARM_INFO 0x16 +#define COFFICIENT 71 +#define FIXEDVAL 45598 + +#define vol_algorithm(m, n) (((((m >> 16) & 0x3) * 0x100) +\ + ((n >> 16) & 0xff)) * COFFICIENT + FIXEDVAL) + + +struct pvt_hwmon { + struct pvt *pvt; + void __iomem *base; +}; + +static const char * const input_names[] = { + [PVT_VSYS] = "voltage", +}; + +static inline void pvt_write_reg(struct pvt_hwmon *pvtvol, u64 a, + u64 b, unsigned int offset) +{ + writel(a | b, pvtvol->base + offset); +} + +static inline u64 pvt_read_reg(struct pvt_hwmon *pvtvol, unsigned int offset) +{ + u64 value; + + value = readl(pvtvol->base + offset); + return value; +} + +void pvt_configure(struct pvt_hwmon *pvtvol, u64 value, u64 reg) +{ + pvt_write_reg(pvtvol, PVT_WDATA, value, PVT0_CTRL); + pvt_write_reg(pvtvol, PVT_WADDR, reg, PVT0_CTRL); +} + +static inline u64 pvt_read_vol(struct pvt_hwmon *pvtvol, u64 data, + u64 reg, unsigned int offset) +{ + unsigned int value; + + pvt_write_reg(pvtvol, data, reg, offset); + msleep(100); + value = pvt_read_reg(pvtvol, offset); + return value; +} + +static int pvt_get_vol(struct pvt_hwmon *pvtvol) +{ + unsigned long long data_h, data_l; + + pvt_configure(pvtvol, 0x1, PVT_SAFECTRL); + + /* configure PVT mode */ + pvt_configure(pvtvol, 0x3, PVT_CONFIG); + + /* PVT monitor enable */ + pvt_configure(pvtvol, 0x1, PVT_RUN); + + /* get the upper 2 bits of the PVT voltage */ + data_h = pvt_read_vol(pvtvol, PVT_READ, PVT_DATA_OUT_H, PVT0_CTRL); + if ((data_h & PVT02SPBU_DATA_OUT) == 0) { + pr_err("error: the Voltage_h is error\n"); + return false; + } + + /* get the lower 8 bits of the PVT voltage */ + data_l = pvt_read_vol(pvtvol, PVT_READ, PVT_DATA_OUT_L, PVT0_CTRL); + if ((data_l & PVT02SPBU_DATA_OUT) == 0) { + pr_err("error: the Voltage_l is error\n"); + return false; + } + + return vol_algorithm(data_h, data_l); +} + +static ssize_t pvt_read(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct pvt_hwmon *pvtvol = dev_get_drvdata(dev); + unsigned long long pvt_vol; + + pvt_vol = pvt_get_vol(pvtvol); + return sprintf(buf, "%lld\n", (pvt_vol / 100)); +} + +static ssize_t show_label(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + return sprintf(buf, "%s\n", + input_names[to_sensor_dev_attr(devattr)->index]); +} + +static SENSOR_DEVICE_ATTR(in0_input, 0444, pvt_read, NULL, + PVT_VSYS); +static SENSOR_DEVICE_ATTR(in0_label, 0444, show_label, NULL, + PVT_VSYS); + +static struct attribute *pvt_attrs[] = { + &sensor_dev_attr_in0_input.dev_attr.attr, + &sensor_dev_attr_in0_label.dev_attr.attr, + NULL +}; + +ATTRIBUTE_GROUPS(pvt); + +static int pvt_vol_plat_probe(struct platform_device *pdev) +{ + struct resource *res; + struct pvt_hwmon *pvtvol; + struct device *hwmon_dev; + unsigned long long value; + struct device *dev = &pdev->dev; + + pvtvol = devm_kzalloc(&pdev->dev, sizeof(*pvtvol), GFP_KERNEL); + if (!pvtvol) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + goto err; + + pvtvol->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(pvtvol->base)) + return PTR_ERR(pvtvol->base); + + platform_set_drvdata(pdev, pvtvol); + hwmon_dev = devm_hwmon_device_register_with_groups(dev, "pvt", + pvtvol, pvt_groups); + + if (IS_ERR(hwmon_dev)) + return PTR_ERR(hwmon_dev); + + value = pvt_get_vol(pvtvol); + if (!value) { + dev_info(&pdev->dev, "pvt_vol get failed\n"); + return false; + } + + return 0; + +err: + dev_err(&pdev->dev, "no PVT resource\n"); + return -ENXIO; +} + +#ifdef CONFIG_OF +static const struct of_device_id pvt_vol_of_match[] = { + { .compatible = "sw64,pvt-vol", }, + {}, +}; +MODULE_DEVICE_TABLE(of, pvt_vol_of_match); +#endif + +static struct platform_driver pvt_vol_driver = { + .probe = pvt_vol_plat_probe, + .driver = { + .name = "pvt-sw64", + .of_match_table = of_match_ptr(pvt_vol_of_match), + }, +}; + +static int __init pvt_vol_init_driver(void) +{ + return platform_driver_register(&pvt_vol_driver); +} +subsys_initcall(pvt_vol_init_driver); + +static void __exit pvt_vol_exit_driver(void) +{ + platform_driver_unregister(&pvt_vol_driver); +} +module_exit(pvt_vol_exit_driver); + +MODULE_AUTHOR("Wang Yingying "); +MODULE_DESCRIPTION("pvt controller"); +MODULE_LICENSE("GPL"); -- Gitee From 3d1e614682dbf1857ee4c2559651805c2b2a898a Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:54:01 +0800 Subject: [PATCH 072/524] drivers: i2c: add sw64 support Add i2c drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/i2c/busses/Kconfig | 10 + drivers/i2c/busses/Makefile | 1 + drivers/i2c/busses/i2c-designware-common.c | 14 + drivers/i2c/busses/i2c-designware-core.h | 3 +- drivers/i2c/busses/i2c-designware-platdrv.c | 5 + drivers/i2c/busses/i2c-sunway.c | 405 ++++++++++++++++++++ 6 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 drivers/i2c/busses/i2c-sunway.c diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index e99cf4d971c2..60fdda4f2154 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -348,6 +348,16 @@ config I2C_ZHAOXIN This driver can also be built as a module. If so, the module will be called i2c-zhaoxin. +config I2C_SUNWAY + tristate "Sunway i2c lib" + depends on SW64 + help + If you say yes to this option, support will be included for the + Sunway Soc I2C interface on SW64 platform. + + This driver can also be built as a module. If so, the module + will be called i2c-sunway. + if ACPI comment "ACPI drivers" diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index f8c8a3554427..738519b0a9cb 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -30,6 +30,7 @@ obj-$(CONFIG_I2C_SIS96X) += i2c-sis96x.o obj-$(CONFIG_I2C_VIA) += i2c-via.o obj-$(CONFIG_I2C_VIAPRO) += i2c-viapro.o obj-$(CONFIG_I2C_ZHAOXIN) += i2c-zhaoxin.o +obj-$(CONFIG_I2C_SUNWAY) += i2c-sunway.o # Mac SMBus host controller drivers obj-$(CONFIG_I2C_HYDRA) += i2c-hydra.o diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c index 79e083aab08e..3f8fd593ea17 100644 --- a/drivers/i2c/busses/i2c-designware-common.c +++ b/drivers/i2c/busses/i2c-designware-common.c @@ -63,6 +63,11 @@ static int dw_reg_read(void *context, unsigned int reg, unsigned int *val) { struct dw_i2c_dev *dev = context; +#ifdef CONFIG_SW64 + if ((dev->flags & MODEL_MASK) == MODEL_SUNWAY) + reg = reg << 7; +#endif + *val = readl(dev->base + reg); return 0; @@ -72,6 +77,11 @@ static int dw_reg_write(void *context, unsigned int reg, unsigned int val) { struct dw_i2c_dev *dev = context; +#ifdef CONFIG_SW64 + if ((dev->flags & MODEL_MASK) == MODEL_SUNWAY) + reg = reg << 7; +#endif + writel(val, dev->base + reg); return 0; @@ -149,6 +159,10 @@ int i2c_dw_init_regmap(struct dw_i2c_dev *dev) return ret; reg = readl(dev->base + DW_IC_COMP_TYPE); +#ifdef CONFIG_SW64 + if ((dev->flags & MODEL_MASK) == MODEL_SUNWAY) + reg = readl(dev->base + (DW_IC_COMP_TYPE << 7)); +#endif i2c_dw_release_lock(dev); if ((dev->flags & MODEL_MASK) == MODEL_AMD_NAVI_GPU) diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index e93870a0f9a4..a7faeaea1163 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -309,7 +309,8 @@ struct dw_i2c_dev { #define MODEL_BAIKAL_BT1 BIT(9) #define MODEL_AMD_NAVI_GPU BIT(10) #define MODEL_WANGXUN_SP BIT(11) -#define MODEL_MASK GENMASK(11, 8) +#define MODEL_SUNWAY BIT(12) +#define MODEL_MASK GENMASK(12, 8) /* * Enable UCSI interrupt by writing 0xd at register diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index f3245a685630..5d7ee3430f0a 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -150,9 +150,14 @@ static int dw_i2c_of_configure(struct platform_device *pdev) } static const struct of_device_id dw_i2c_of_match[] = { +#ifdef CONFIG_SW64 + { .compatible = "snps,designware-i2c", .data = (void *)MODEL_SUNWAY }, +#else { .compatible = "snps,designware-i2c", }, +#endif { .compatible = "mscc,ocelot-i2c", .data = (void *)MODEL_MSCC_OCELOT }, { .compatible = "baikal,bt1-sys-i2c", .data = (void *)MODEL_BAIKAL_BT1 }, + { .compatible = "sunway,suntai-i2c", .data = (void *)MODEL_SUNWAY }, {}, }; MODULE_DEVICE_TABLE(of, dw_i2c_of_match); diff --git a/drivers/i2c/busses/i2c-sunway.c b/drivers/i2c/busses/i2c-sunway.c new file mode 100644 index 000000000000..cc7268c6a2da --- /dev/null +++ b/drivers/i2c/busses/i2c-sunway.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 WXIAT Platform Software + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + * + * The drivers in this file are synchronous/blocking. In addition, + * use poll mode to read/write slave devices on the I2C bus instead + * of the interrupt mode. + */ + +#include +#include +#include +#include + +#include + +#define CPLD_BUSNR 2 + +#define IC_CLK_KHZ 25000 + +/* I2C register definitions */ +#define DW_IC_CON 0x0 +#define DW_IC_STATUS 0x3800 +#define DW_IC_DATA_CMD 0x0800 +#define DW_IC_TAR 0x00200 +#define DW_IC_ENABLE 0x3600 +#define DW_IC_CMD 0x0100 +#define DW_IC_STOP 0x0200 +#define DW_IC_SDA_HOLD 0x3e00 +#define DW_IC_SDA_SETUP 0x4a00 +#define DW_IC_SS_SCL_HCNT 0x0a00 +#define DW_IC_SS_SCL_LCNT 0x0c00 +#define DW_IC_FS_SCL_HCNT 0x0e00 +#define DW_IC_FS_SCL_LCNT 0x1000 +#define DW_IC_TX_TL 0x1e00 +#define DW_IC_RX_TL 0x1c00 +#define DW_IC_INTR_MASK 0x1800 + +#define MAX_RETRY 10000000 + +#define DW_IC_STATUS_ACTIVITY 0x1 +#define DW_IC_STATUS_TFNF 0x2 +#define DW_IC_STATUS_TFE 0x4 +#define DW_IC_STATUS_RFNE 0x8 +#define DW_IC_STATUS_RFF 0x10 + +#define DW_IC_CON_MASTER 0x1 +#define DW_IC_CON_SPEED_STD 0x2 +#define DW_IC_CON_SPEED_FAST 0x4 +#define DW_IC_CON_10BITADDR_MASTER 0x10 +#define DW_IC_CON_RESTART_EN 0x20 +#define DW_IC_CON_SLAVE_DISABLE 0x40 + +#define INTEL_MID_STD_CFG (DW_IC_CON_MASTER | \ + DW_IC_CON_SLAVE_DISABLE | \ + DW_IC_CON_RESTART_EN) + +#define DW_IC_INTR_RX_UNDER 0x001 +#define DW_IC_INTR_RX_OVER 0x002 +#define DW_IC_INTR_RX_FULL 0x004 +#define DW_IC_INTR_TX_OVER 0x008 +#define DW_IC_INTR_TX_EMPTY 0x010 +#define DW_IC_INTR_RD_REQ 0x020 +#define DW_IC_INTR_TX_ABRT 0x040 +#define DW_IC_INTR_RX_DONE 0x080 +#define DW_IC_INTR_ACTIVITY 0x100 +#define DW_IC_INTR_STOP_DET 0x200 +#define DW_IC_INTR_START_DET 0x400 +#define DW_IC_INTR_GEN_CALL 0x800 + +#define DW_IC_INTR_DEFAULT_MASK (DW_IC_INTR_RX_FULL | \ + DW_IC_INTR_TX_EMPTY | \ + DW_IC_INTR_TX_ABRT | \ + DW_IC_INTR_STOP_DET) + +enum i2c_bus_operation { + I2C_BUS_READ, + I2C_BUS_WRITE, +}; + +static void __iomem *m_i2c_base_address; + +/* + * This function get I2Cx controller base address + * + * @param i2c_controller_index Bus Number of I2C controller. + * @return I2C BAR. + */ +void __iomem *get_i2c_bar_addr(uint8_t i2c_controller_index) +{ + switch (i2c_controller_index) { + case 0: + return __va(IO_BASE | IIC0_BASE); + case 1: + return __va(IO_BASE | IIC1_BASE); + case 2: + return __va(IO_BASE | IIC2_BASE); + default: + return NULL; + } +} + +static inline void write_cpu_i2c_controller(uint64_t offset, uint32_t data) +{ + writel(data, m_i2c_base_address + offset); +} + +static inline uint32_t read_cpu_i2c_controller(uint64_t offset) +{ + return readl(m_i2c_base_address + offset); +} + +static int poll_for_status_set0(uint16_t status_bit) +{ + uint64_t retry = 0; + uint32_t temp = read_cpu_i2c_controller(DW_IC_STATUS); + + temp = read_cpu_i2c_controller(DW_IC_STATUS); + + while (retry < MAX_RETRY) { + if (read_cpu_i2c_controller(DW_IC_STATUS) & status_bit) + break; + retry++; + } + + if (retry == MAX_RETRY) + return -ETIME; + + return 0; +} + +static uint32_t i2c_dw_scl_lcnt(uint32_t ic_clk, uint32_t t_low, + uint32_t tf, uint32_t offset) +{ + /* + * Conditional expression: + * + * IC_[FS]S_SCL_LCNT + 1 >= IC_CLK * (t_low + tf) + * + * DW I2C core starts counting the SCL CNTs for the LOW period + * of the SCL clock (t_low) as soon as it pulls the SCL line. + * In order to meet the t_low timing spec, we need to take into + * account the fall time of SCL signal (tf). Default tf value + * should be 0.3 us, for safety. + */ + return ((ic_clk * (t_low + tf) + 500000) / 1000000) - 1 + offset; +} + +static uint32_t i2c_dw_scl_hcnt(uint32_t ic_clk, uint32_t t_symbol, + uint32_t tf, uint32_t cond, uint32_t offset) +{ + /* + * DesignWare I2C core doesn't seem to have solid strategy to meet + * the tHD;STA timing spec. Configuring _HCNT based on tHIGH spec + * will result in violation of the tHD;STA spec. + */ + if (cond) + /* + * Conditional expression: + * + * IC_[FS]S_SCL_HCNT + (1+4+3) >= IC_CLK * tHIGH + * + * This is based on the DW manuals, and represents an ideal + * configuration. The resulting I2C bus speed will be faster + * than any of the others. + * + * If your hardware is free from tHD;STA issue, try this one. + */ + return (ic_clk * t_symbol + 500000) / 1000000 - 8 + offset; + /* + * Conditional expression: + * + * IC_[FS]S_SCL_HCNT + 3 >= IC_CLK * (tHD;STA + tf) + * + * This is just experimental rule; the tHD;STA period turned + * out to be proportinal to (_HCNT + 3). With this setting, + * we could meet both tHIGH and tHD;STA timing specs. + * + * If unsure, you'd better to take this alternative. + * + * The reason why we need to take into account "tf" here, + * is the same as described in i2c_dw_scl_lcnt(). + */ + return (ic_clk * (t_symbol + tf) + 500000) / 1000000 - 3 + offset; +} + +static int wait_for_cpu_i2c_bus_busy(void) +{ + uint64_t retry = 0; + uint32_t status = 0; + + do { + retry++; + status = !!(read_cpu_i2c_controller(DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY); + } while ((retry < MAX_RETRY) && status); + + if (retry == MAX_RETRY) + return -ETIME; + + return 0; +} + +static int i2c_read(uint8_t reg_offset, uint8_t *buffer, uint32_t length) +{ + int status; + uint32_t i; + + status = poll_for_status_set0(DW_IC_STATUS_TFE); + if (status) + return status; + + write_cpu_i2c_controller(DW_IC_DATA_CMD, reg_offset); + + for (i = 0; i < length; i++) { + if (i == length - 1) + write_cpu_i2c_controller(DW_IC_DATA_CMD, DW_IC_CMD | DW_IC_STOP); + else + write_cpu_i2c_controller(DW_IC_DATA_CMD, DW_IC_CMD); + + if (poll_for_status_set0(DW_IC_STATUS_RFNE) == 0) + buffer[i] = readb(m_i2c_base_address + DW_IC_DATA_CMD); + else + pr_err("Read timeout line %d.\n", __LINE__); + } + + return 0; +} + +static int i2c_write(uint8_t reg_offset, uint8_t *buffer, uint32_t length) +{ + int status; + uint32_t i; + + /* Data transfer, poll till transmit ready bit is set */ + status = poll_for_status_set0(DW_IC_STATUS_TFE); + if (status) { + pr_err("In i2c-lib.c, line %d.\n", __LINE__); + return status; + } + + write_cpu_i2c_controller(DW_IC_DATA_CMD, reg_offset); + + for (i = 0; i < length; i++) { + if (poll_for_status_set0(DW_IC_STATUS_TFNF) == 0) { + if (i == length - 1) + write_cpu_i2c_controller(DW_IC_DATA_CMD, buffer[i] | DW_IC_STOP); + else + write_cpu_i2c_controller(DW_IC_DATA_CMD, buffer[i]); + } else { + pr_err("Write timeout %d.\n", __LINE__); + } + } + + mdelay(200); + status = poll_for_status_set0(DW_IC_STATUS_TFE); + if (status) { + pr_err("In i2c-lib.c, line %d.\n", __LINE__); + return status; + } + + return 0; +} + +/* Initialize I2c controller */ +void init_cpu_i2c_controller(void) +{ + uint32_t h_cnt; + uint32_t l_cnt; + uint32_t input_ic_clk_rate = IC_CLK_KHZ; /* by unit KHz ie. 25MHz */ + uint32_t sda_falling_time = 300; + uint32_t scl_falling_time = 300; + + /* + * The I2C protocol specification requires 300ns of hold time on the + * SDA signal (tHD;DAT) in standard and fast speed modes, and a hold + * time long enough to bridge the undefined part between logic 1 and + * logic 0 of the falling edge of SCL in high speed mode. + */ + uint32_t sda_hold_time = 432; + uint32_t sda_hold = 0; + + /* Firstly disable the controller. */ + pr_debug("Initialize CPU I2C controller\n"); + + write_cpu_i2c_controller(DW_IC_ENABLE, 0); + + sda_hold = (input_ic_clk_rate * sda_hold_time + 500000) / 1000000; + write_cpu_i2c_controller(DW_IC_SDA_HOLD, sda_hold); + + /* Set standard and fast speed deviders for high/low periods. */ + /* Standard-mode */ + h_cnt = i2c_dw_scl_hcnt(input_ic_clk_rate, 4000, sda_falling_time, 0, 0); + l_cnt = i2c_dw_scl_lcnt(input_ic_clk_rate, 4700, scl_falling_time, 0); + + write_cpu_i2c_controller(DW_IC_SS_SCL_HCNT, h_cnt); + write_cpu_i2c_controller(DW_IC_SS_SCL_LCNT, l_cnt); + + pr_debug("Standard-mode HCNT=%x, LCNT=%x\n", h_cnt, l_cnt); + + /* Fast-mode */ + h_cnt = i2c_dw_scl_hcnt(input_ic_clk_rate, 600, sda_falling_time, 0, 0); + l_cnt = i2c_dw_scl_lcnt(input_ic_clk_rate, 1300, scl_falling_time, 0); + + write_cpu_i2c_controller(DW_IC_FS_SCL_HCNT, h_cnt); + write_cpu_i2c_controller(DW_IC_FS_SCL_LCNT, l_cnt); + + pr_debug("Fast-mode HCNT=%x, LCNT=%d\n\n", h_cnt, l_cnt); + + /* Configure Tx/Rx FIFO threshold levels, since we will be working + * in polling mode set both thresholds to their minimum + */ + write_cpu_i2c_controller(DW_IC_TX_TL, 0); + write_cpu_i2c_controller(DW_IC_RX_TL, 0); + write_cpu_i2c_controller(DW_IC_INTR_MASK, DW_IC_INTR_DEFAULT_MASK); + + /* Configure the i2c master */ + write_cpu_i2c_controller(DW_IC_CON, + INTEL_MID_STD_CFG | DW_IC_CON_SPEED_STD); + +} + +/* + * This function enables I2C controllers. + * + * @param i2c_controller_index Bus Number of I2C controllers. + */ +void enable_i2c_controller(uint8_t i2c_controller_index) +{ + m_i2c_base_address = get_i2c_bar_addr(i2c_controller_index); + init_cpu_i2c_controller(); +} + +/* + * Write/Read data from I2C device. + * + * @i2c_controller_index: i2c bus number + * @slave_address: slave address + * @operation: to read or write + * @length: number of bytes + * @reg_offset: register offset + * @buffer: in/out buffer + */ +int i2c_bus_rw(uint8_t i2c_controller_index, uint8_t slave_address, + enum i2c_bus_operation operation, uint32_t length, + uint8_t reg_offset, void *buffer) +{ + uint8_t *byte_buffer = buffer; + int status = 0; + uint32_t databuffer, temp; + + m_i2c_base_address = get_i2c_bar_addr(i2c_controller_index); + status = wait_for_cpu_i2c_bus_busy(); + if (status) { + pr_err("%d\n", __LINE__); + return status; + } + + mdelay(1000); + + /* Set the slave address. */ + write_cpu_i2c_controller(DW_IC_ENABLE, 0x0); /* Disable controller */ + databuffer = read_cpu_i2c_controller(DW_IC_CON); + databuffer &= ~DW_IC_CON_10BITADDR_MASTER; + write_cpu_i2c_controller(DW_IC_CON, databuffer); + + /* Fill the target addr. */ + write_cpu_i2c_controller(DW_IC_TAR, slave_address); + + temp = read_cpu_i2c_controller(DW_IC_TAR); + + /* Configure Tx/Rx FIFO threshold levels. */ + write_cpu_i2c_controller(DW_IC_ENABLE, 0x1); /* Enable the adapter */ + write_cpu_i2c_controller(DW_IC_INTR_MASK, DW_IC_INTR_DEFAULT_MASK); + + if (operation == I2C_BUS_READ) + status = i2c_read(reg_offset, byte_buffer, length); + else if (operation == I2C_BUS_WRITE) + status = i2c_write(reg_offset, byte_buffer, length); + + /* Disable controller */ + write_cpu_i2c_controller(DW_IC_ENABLE, 0x0); + + return status; +} + +void disable_i2c_controller(uint8_t i2c_controller_index) +{ + m_i2c_base_address = get_i2c_bar_addr(i2c_controller_index); + + /* Disable controller */ + write_cpu_i2c_controller(DW_IC_ENABLE, 0x0); + m_i2c_base_address = 0; +} + +void cpld_write(uint8_t slave_addr, uint8_t reg, uint8_t data) +{ + enable_i2c_controller(CPLD_BUSNR); + i2c_bus_rw(CPLD_BUSNR, slave_addr, I2C_BUS_WRITE, sizeof(uint8_t), reg, &data); + disable_i2c_controller(CPLD_BUSNR); +} -- Gitee From f7c2beb8611d8087ab8c24b42a76eb8f0ebbdbaf Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:49:04 +0800 Subject: [PATCH 073/524] drivers: iommu: add sw64 support Add iommu drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/Kconfig | 1 + drivers/iommu/Makefile | 2 +- drivers/iommu/sw64/Kconfig | 21 + drivers/iommu/sw64/Makefile | 3 + drivers/iommu/sw64/iommu.c | 1277 +++++++++++++++++++++ drivers/iommu/sw64/iommu_v2.c | 1780 +++++++++++++++++++++++++++++ drivers/iommu/sw64/sunway_iommu.h | 79 ++ 7 files changed, 3162 insertions(+), 1 deletion(-) create mode 100644 drivers/iommu/sw64/Kconfig create mode 100644 drivers/iommu/sw64/Makefile create mode 100644 drivers/iommu/sw64/iommu.c create mode 100644 drivers/iommu/sw64/iommu_v2.c create mode 100644 drivers/iommu/sw64/sunway_iommu.h diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 1003e3fbe074..6a13ad0332f2 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -191,6 +191,7 @@ config MSM_IOMMU source "drivers/iommu/amd/Kconfig" source "drivers/iommu/intel/Kconfig" source "drivers/iommu/iommufd/Kconfig" +source "drivers/iommu/sw64/Kconfig" config IRQ_REMAP bool "Support for Interrupt Remapping" diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 3f4592bbd7f9..724a56c2976a 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += amd/ intel/ arm/ iommufd/ +obj-y += amd/ intel/ arm/ iommufd/ sw64/ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o diff --git a/drivers/iommu/sw64/Kconfig b/drivers/iommu/sw64/Kconfig new file mode 100644 index 000000000000..3a6a1e994f31 --- /dev/null +++ b/drivers/iommu/sw64/Kconfig @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +# SW64 IOMMU SUPPORT +config SUNWAY_IOMMU + bool "Sunway IOMMU Support" + select IOMMU_API + select IOMMU_IOVA + select IOMMU_DMA + depends on SW64 && PCI && SUBARCH_C3B + help + Support for IOMMU on SW64 platform. It can enable or bypass specific device by + adding boot param "iommu_enable" and "iommu.passthrough". + +# SW64 IOMMU V2 SUPPORT +config SUNWAY_IOMMU_V2 + bool "Sunway IOMMU V2 Support" + select IOMMU_API + select IOMMU_IOVA + depends on SW64 && PCI && SUBARCH_C4 + help + Support for IOMMU V2 on SW64 platform. It can enable or bypass specific device by + adding boot param "iommu_enable" and "iommu.passthrough". diff --git a/drivers/iommu/sw64/Makefile b/drivers/iommu/sw64/Makefile new file mode 100644 index 000000000000..e61b343490aa --- /dev/null +++ b/drivers/iommu/sw64/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_SUNWAY_IOMMU) += iommu.o +obj-$(CONFIG_SUNWAY_IOMMU_V2) += iommu_v2.o diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c new file mode 100644 index 000000000000..32b18f726fd9 --- /dev/null +++ b/drivers/iommu/sw64/iommu.c @@ -0,0 +1,1277 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * iommu.c: Generic sw64 IOMMU support + * + * This is designed and tested for 3231. If there are no changes in hardware + * in later chips, then it should work just as well. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "sunway_iommu.h" + +#define MAX_DOMAIN_NUM 65536 +#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) +#define SW64_DMA_LIMIT (0xe0000000 - 1) +#define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) + +#define SW64_IOMMU_LEVEL1_OFFSET 0x1ff +#define SW64_IOMMU_LEVEL2_OFFSET 0x3ff + +#define SW64_IOMMU_GRN_8K ((0UL) << 4) /* page size as 8KB */ +#define SW64_IOMMU_GRN_8M ((0x2UL) << 4) /* page size as 8MB */ +#define SW64_IOMMU_PGSIZES (((1ULL) << PAGE_SHIFT) | ((1ULL) << PAGE_8M_SHIFT)) + +#define IDENTMAP_ALL ((1U) << 0) +#define DMA_MASK64 ((1U) << 1) + +/* IOMMU Exceptional Status */ +enum exceptype { + DTE_LEVEL1 = 0x0, + DTE_LEVEL2, + PTE_LEVEL1, + PTE_LEVEL2, + UNAUTHORIZED_ACCESS, + ILLEGAL_RESPONSE, + DTE_LEVEL1_VAL, + DTE_LEVEL2_VAL, + PTE_LEVEL1_VAL, + PTE_LEVEL2_VAL, +}; + +u64 iommu_enable_cmd; /* default IOMMU boot param: 0 */ + +unsigned long *sunway_iommu_domain_bitmap; + +static DEFINE_SPINLOCK(domain_bitmap_lock); +static DEFINE_SPINLOCK(sunway_iommu_device_table_lock); +spinlock_t sunway_domain_lock; + +static LLIST_HEAD(dev_data_list); +LIST_HEAD(sunway_domain_list); + +struct dma_domain { + struct sunway_iommu_domain sdomain; + struct iova_domain iovad; +}; +const struct iommu_ops sunway_iommu_ops; + +static int iommu_identity_mapping; + +/* flush helpers */ +static void piu_flush_all(struct pci_controller *hose) +{ + write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); + write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); + write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); +} + +void dev_flush_dtlb(struct sunway_iommu_domain *sdomain, + struct sunway_iommu_dev *sdev_data) +{ + struct pci_controller *hose; + int devid; + + list_for_each_entry(sdev_data, &sdomain->dev_list, list) { + hose = pci_bus_to_pci_controller(sdev_data->pdev->bus); + devid = sdev_data->devid; + + write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, devid); + } +} + +void flush_pcache_by_addr(struct sunway_iommu_domain *sdomain, + unsigned long flush_addr) +{ + struct pci_controller *hose; + struct sunway_iommu_dev *sdev_data; + + list_for_each_entry(sdev_data, &sdomain->dev_list, list) { + hose = pci_bus_to_pci_controller(sdev_data->pdev->bus); + + flush_addr = __pa(flush_addr); + write_piu_ior0(hose->node, hose->index, + PCACHE_FLUSHPADDR, flush_addr); + } +} + +void flush_ptlb_by_addr(struct sunway_iommu_domain *sdomain, + unsigned long flush_addr) +{ + struct pci_controller *hose; + struct pci_dev *pdev; + struct sunway_iommu_dev *sdev_data; + + list_for_each_entry(sdev_data, &sdomain->dev_list, list) { + pdev = sdev_data->pdev; + hose = pci_bus_to_pci_controller(pdev->bus); + + flush_addr = (pdev->bus->number << 8) + | pdev->devfn | (flush_addr << 16); + write_piu_ior0(hose->node, hose->index, + PTLB_FLUSHVADDR, flush_addr); + } +} + +/* domain helpers */ +static struct sunway_iommu_domain *to_sunway_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct sunway_iommu_domain, domain); +} + +static struct dma_domain *to_dma_domain(struct sunway_iommu_domain *sdomain) +{ + return container_of(sdomain, struct dma_domain, sdomain); +} + +static void add_domain_to_list(struct sunway_iommu_domain *sdomain) +{ + unsigned long flags; + + spin_lock_irqsave(&sunway_domain_lock, flags); + list_add(&sdomain->list, &sunway_domain_list); + spin_unlock_irqrestore(&sunway_domain_lock, flags); +} + +static void del_domain_from_list(struct sunway_iommu_domain *sdomain) +{ + unsigned long flags; + + spin_lock_irqsave(&sunway_domain_lock, flags); + list_del(&sdomain->list); + spin_unlock_irqrestore(&sunway_domain_lock, flags); +} + +static void free_pagetable(struct sunway_iommu_domain *sdomain) +{ + unsigned long pde; + unsigned long *pde_ptr; + int i, pdes_one_page; + + pde_ptr = sdomain->pt_root; + if (!pde_ptr) + return; + + pdes_one_page = PAGE_SIZE/sizeof(pde); + for (i = 0; i < pdes_one_page; i++, pde_ptr++) { + pde = *pde_ptr; + if ((pde & SW64_IOMMU_ENTRY_VALID) == 0) + continue; + + pde &= ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK; + pde |= PAGE_OFFSET; + free_page(pde); + } + + free_page((unsigned long)sdomain->pt_root); +} + +static void domain_id_free(int id) +{ + spin_lock(&domain_bitmap_lock); + if (id > 0) + __clear_bit(id, sunway_iommu_domain_bitmap); + spin_unlock(&domain_bitmap_lock); +} + +static void dma_domain_free(struct dma_domain *dma_dom) +{ + if (!dma_dom) + return; + + del_domain_from_list(&dma_dom->sdomain); + put_iova_domain(&dma_dom->iovad); + free_pagetable(&dma_dom->sdomain); + if (dma_dom->sdomain.id) + domain_id_free(dma_dom->sdomain.id); + + kfree(dma_dom); +} + +static void sunway_domain_free(struct sunway_iommu_domain *sdomain) +{ + if (!sdomain) + return; + + del_domain_from_list(sdomain); + if (sdomain->id) + domain_id_free(sdomain->id); + + kfree(sdomain); +} + +static u16 sunway_domain_id_alloc(void) +{ + int id; + + spin_lock(&domain_bitmap_lock); + id = find_first_zero_bit(sunway_iommu_domain_bitmap, MAX_DOMAIN_NUM); + if (id > 0 && id < MAX_DOMAIN_NUM) + __set_bit(id, sunway_iommu_domain_bitmap); + else + id = 0; + spin_unlock(&domain_bitmap_lock); + + return id; +} + +static int sunway_domain_init(struct sunway_iommu_domain *sdomain) +{ + spin_lock_init(&sdomain->lock); + mutex_init(&sdomain->api_lock); + sdomain->id = sunway_domain_id_alloc(); + if (!sdomain->id) + return -ENOMEM; + INIT_LIST_HEAD(&sdomain->dev_list); + + return 1; +} + +static struct sunway_iommu_domain *sunway_domain_alloc(void) +{ + struct sunway_iommu_domain *sdomain; + + sdomain = kzalloc(sizeof(struct sunway_iommu_domain), GFP_KERNEL); + if (!sdomain) + return NULL; + + if (!sunway_domain_init(sdomain)) { + kfree(sdomain); + return NULL; + } + + add_domain_to_list(sdomain); + return sdomain; +} + +static struct dma_domain *dma_domain_alloc(void) +{ + struct dma_domain *dma_dom; + struct page; + + dma_dom = kzalloc(sizeof(struct dma_domain), GFP_KERNEL); + if (!dma_dom) + return NULL; + + sunway_domain_init(&dma_dom->sdomain); + dma_dom->sdomain.type = IOMMU_DOMAIN_DMA; + + dma_dom->sdomain.pt_root = (unsigned long *)get_zeroed_page(GFP_KERNEL); + if (dma_dom->sdomain.pt_root == NULL) { + pr_err("Allocating a new sdomain pt_root failed!\n"); + dma_domain_free(dma_dom); + return NULL; + } + + add_domain_to_list(&dma_dom->sdomain); + + return dma_dom; +} + +static void device_flush_all(struct sunway_iommu_dev *sdata) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(sdata->pdev->bus); + + if (hose == NULL) + return; + + write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, sdata->devid); + write_piu_ior0(hose->node, hose->index, PTLB_FLUSHDEV, sdata->devid); + write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHDEV, sdata->devid); +} + +/* iommu_ops device attach/unattach helpers */ +static void +set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu *iommu; + struct pci_dev *pdev; + struct page *page; + unsigned long *dte_l1, *dte_l2; + unsigned long dte_l1_val, dte_l2_base, dte_l2_val; + + pdev = sdev->pdev; + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + return; + + sdev->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); + iommu = sdev->iommu; + dte_l1 = iommu->iommu_dtbr + (pdev->bus->number); + dte_l1_val = *dte_l1; + + if (!dte_l1_val) { + /* Alloc a new level-2 device table page */ + page = alloc_pages_node(iommu->node, __GFP_ZERO, + get_order(PAGE_SIZE)); + if (!page) { + pr_err("Allocating a new level-2 device table page failed.\n"); + return; + } + + dte_l2_base = (unsigned long)page_address(page); + dte_l1_val = (__pa(dte_l2_base) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; + *dte_l1 = dte_l1_val; + } + + dte_l2 = __va(dte_l1_val & ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + (pdev->devfn << 3); + dte_l2_val = (__pa(sdomain->pt_root) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; + if (iommu_identity_mapping) { + dte_l2_val |= 0x1; + sdev->passthrough = IDENTMAP_ALL; + } + *dte_l2 = dte_l2_val; + + device_flush_all(sdev); +} + +static void +do_attach(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomain) +{ + sdev_data->domain = sdomain; + list_add(&sdev_data->list, &sdomain->dev_list); + + sdomain->dev_cnt++; + set_dte_entry(sdev_data, sdomain); + + pr_debug("iommu: device %d add to domain: %d\n", + sdev_data->devid, sdomain->id); +} + +static void do_detach(struct sunway_iommu_dev *sdev_data) +{ + struct sunway_iommu_domain *sdomain = sdev_data->domain; + + sdev_data->domain = NULL; + list_del(&sdev_data->list); + device_flush_all(sdev_data); + + sdomain->dev_cnt--; + pr_debug("iommu: device %d detached from domain %d\n", + sdev_data->devid, sdomain->id); +} + +static int +__attach_device(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomain) +{ + int ret; + + spin_lock(&sdomain->lock); + ret = -EBUSY; + if (sdev_data->domain != NULL) + goto out_unlock; + + do_attach(sdev_data, sdomain); + ret = 0; + +out_unlock: + spin_unlock(&sdomain->lock); + return ret; +} + +static void __detach_device(struct sunway_iommu_dev *sunway_dev_data) +{ + struct sunway_iommu_domain *domain; + + domain = sunway_dev_data->domain; + + spin_lock(&domain->lock); + do_detach(sunway_dev_data); + spin_unlock(&domain->lock); +} + +static int attach_device(struct device *dev, struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu_dev *sdev; + unsigned long flags; + int ret; + + sdev = dev_iommu_priv_get(dev); + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + ret = __attach_device(sdev, sdomain); + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); + + return ret; +} + +static void detach_device(struct device *dev) +{ + struct sunway_iommu_domain *sunway_domain; + struct sunway_iommu_dev *sdev_data; + unsigned long flags; + + sdev_data = dev_iommu_priv_get(dev); + sunway_domain = sdev_data->domain; + + if (WARN_ON(!sdev_data->domain)) + return; + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + __detach_device(sdev_data); + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); + + if (!dev_is_pci(dev)) + return; +} + +static struct sunway_iommu_dev *search_dev_data(u16 devid) +{ + struct sunway_iommu_dev *sdev_data; + struct llist_node *node; + + if (llist_empty(&dev_data_list)) + return NULL; + + node = dev_data_list.first; + llist_for_each_entry(sdev_data, node, dev_data_list) { + if (sdev_data->devid == devid) + return sdev_data; + } + + return NULL; +} + +/********************************************************************** + * + * Following functions describe IOMMU init ops + * + **********************************************************************/ + +static struct sunway_iommu *sunway_iommu_early_init(struct pci_controller *hose) +{ + struct sunway_iommu *iommu; + struct page *page; + unsigned long base; + + hose->pci_iommu = kzalloc(sizeof(struct sunway_iommu), GFP_KERNEL); + if (!hose->pci_iommu) + return 0; + + iommu = hose->pci_iommu; + spin_lock_init(&iommu->dt_lock); + + iommu->node = hose->node; + if (!node_online(hose->node)) + iommu->node = -1; + + page = alloc_pages_node(iommu->node, __GFP_ZERO, get_order(PAGE_SIZE)); + if (!page) { + pr_err("Allocating a new iommu_dtbr page failed.\n"); + kfree(hose->pci_iommu); + return NULL; + } + + iommu->iommu_dtbr = page_address(page); + + iommu->hose_pt = hose; + iommu->index = hose->index; + + iommu->enabled = true; + + base = __pa(iommu->iommu_dtbr) & PAGE_MASK; + write_piu_ior0(hose->node, hose->index, DTBASEADDR, base); + + return iommu; +} + +unsigned long fetch_dte(struct sunway_iommu *iommu, unsigned long devid, + enum exceptype type) +{ + unsigned long *dte_l1, *dte_l2; + unsigned long dte_l1_val, dte_l2_val; + + if (!iommu) + return 0; + dte_l1 = iommu->iommu_dtbr + (devid >> 8); + if (type == DTE_LEVEL1) + return (unsigned long)dte_l1; + + dte_l1_val = *dte_l1; + if (type == DTE_LEVEL1_VAL) + return dte_l1_val; + + dte_l1_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + dte_l1_val |= PAGE_OFFSET; + dte_l2 = (unsigned long *)(dte_l1_val + ((devid & 0xff) << 3)); + if (type == DTE_LEVEL2) + return (unsigned long)dte_l2; + + dte_l2_val = *dte_l2; + if (type == DTE_LEVEL2_VAL) + return dte_l2_val; + + return dte_l2_val; +} + +unsigned long fetch_pte(struct sunway_iommu_domain *sdomain, dma_addr_t iova, + enum exceptype type) +{ + unsigned long iova_pfn, pte_l1_val, pte_l2_val; + unsigned long *pte_l1, *pte_l2; + unsigned long pte_root; + unsigned long offset; + + if (!sdomain) + return -EINVAL; + + pte_root = __pa(sdomain->pt_root) & PAGE_MASK; + iova_pfn = iova >> PAGE_SHIFT; + pte_root = ((pte_root) & (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK)); + pte_root |= PAGE_OFFSET; + offset = ((iova_pfn >> 10) & SW64_IOMMU_LEVEL1_OFFSET) << 3; + pte_l1 = (unsigned long *)(pte_root + offset); + if (type == PTE_LEVEL1) + return (unsigned long)pte_l1; + + pte_l1_val = *pte_l1; + if (type == PTE_LEVEL1_VAL) + return pte_l1_val; + + pte_l1_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + pte_l1_val |= PAGE_OFFSET; + offset = (iova_pfn & SW64_IOMMU_LEVEL2_OFFSET) << 3; + pte_l2 = (unsigned long *)(pte_l1_val + offset); + + if (type == PTE_LEVEL2) + return (unsigned long)pte_l2; + + pte_l2_val = *pte_l2; + if (type == PTE_LEVEL2_VAL) + return pte_l2_val; + + return pte_l2_val; +} + +/* IOMMU Interrupt handle */ +irqreturn_t iommu_interrupt(int irq, void *dev) +{ + struct pci_controller *hose = (struct pci_controller *)dev; + struct sunway_iommu_domain *sdomain; + struct sunway_iommu_dev *sdev; + unsigned long iommu_status; + unsigned long type; + unsigned long devid, dva; + + iommu_status = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); + if (!(iommu_status >> 63)) + return IRQ_NONE; + + type = (iommu_status >> 59) & 0x7; + devid = (iommu_status >> 37) & 0xffff; + dva = iommu_status & 0xffffffff; + pr_info("%s, iommu_status = %#lx, devid %#lx, dva %#lx, ", + __func__, iommu_status, devid, dva); + + sdev = search_dev_data(devid); + if (sdev == NULL) { + pr_info("no such dev!!!\n"); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + + return IRQ_HANDLED; + } + + sdomain = sdev->domain; + switch (type) { + case DTE_LEVEL1: + pr_info("invalid level1 dte, addr:%#lx, val:%#lx\n", + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1), + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1_VAL)); + break; + case DTE_LEVEL2: + pr_info("invalid level2 dte, addr:%#lx, val:%#lx\n", + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2), + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2_VAL)); + break; + case PTE_LEVEL1: + pr_info("invalid level1 pte, addr: %#lx, val:%#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL1), + fetch_pte(sdomain, dva, PTE_LEVEL1_VAL)); + break; + case PTE_LEVEL2: + pr_info("invalid level2 pte, addr: %#lx, val: %#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL2), + fetch_pte(sdomain, dva, PTE_LEVEL2_VAL)); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + break; + + case UNAUTHORIZED_ACCESS: + pr_info("unauthorized access\n"); + break; + case ILLEGAL_RESPONSE: + pr_info("illegal response\n"); + break; + default: + pr_info("unknown error\n"); + break; + } + + return IRQ_HANDLED; +} + +struct irqaction iommu_irqaction = { + .handler = iommu_interrupt, + .flags = IRQF_SHARED | IRQF_NO_THREAD, + .name = "sunway_iommu", +}; + +void sunway_enable_iommu_func(struct pci_controller *hose) +{ + unsigned int iommu_irq, err; + unsigned long iommu_conf, iommu_ctrl; + + iommu_irq = hose->int_irq; + pr_debug("%s node %ld rc %ld iommu_irq %d\n", + __func__, hose->node, hose->index, iommu_irq); + err = request_irq(iommu_irq, iommu_interrupt, + IRQF_SHARED, "sunway_iommu", hose); + if (err < 0) + pr_info("sw iommu request irq failed!\n"); + + iommu_ctrl = (1UL << 63) | (0x100UL << 10); + write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_CTRL, iommu_ctrl); + iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + iommu_conf = iommu_conf | (0x3 << 7); + write_piu_ior0(hose->node, hose->index, PIUCONFIG0, iommu_conf); + write_piu_ior0(hose->node, hose->index, TIMEOUT_CONFIG, 0xf); + iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + pr_debug("SW arch configure node %ld hose-%ld iommu_conf = %#lx\n", + hose->node, hose->index, iommu_conf); +} + +static bool is_iommu_enable(struct pci_controller *hose) +{ + u64 rc_mask = 0x1; + + rc_mask <<= (8 * hose->node + hose->index); + if (iommu_enable_cmd & rc_mask) + return true; + + return false; +} + +/* iommu cpu syscore ops */ +static int iommu_cpu_suspend(void) +{ + return 0; +} + +static void iommu_cpu_resume(void) +{ + +} + +struct syscore_ops iommu_cpu_syscore_ops = { + .suspend = iommu_cpu_suspend, + .resume = iommu_cpu_resume, +}; + +static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type); + +static int sunway_iommu_init(void) +{ + struct pci_controller *hose; + struct sunway_iommu *iommu; + int ret; + int iommu_index = 0; + + sunway_iommu_domain_bitmap = + (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(MAX_DOMAIN_NUM / 8)); + if (sunway_iommu_domain_bitmap == NULL) + return 0; + __set_bit(0, sunway_iommu_domain_bitmap); + + /* Do the loop */ + for (hose = hose_head; hose; hose = hose->next) { + if (!is_iommu_enable(hose)) { + hose->iommu_enable = false; + continue; + } + + iommu = sunway_iommu_early_init(hose); + if (!iommu) { + pr_err("Allocating sunway_iommu failed\n"); + hose->iommu_enable = false; + continue; + } + + iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", + iommu_index); + iommu_index++; + sunway_enable_iommu_func(hose); + hose->iommu_enable = true; + + iommu_device_register(&iommu->iommu, &sunway_iommu_ops, NULL); + } + + ret = iova_cache_get(); + if (ret) + return ret; + + for (hose = hose_head; hose; hose = hose->next) + if (hose->iommu_enable) + piu_flush_all(hose); + + register_syscore_ops(&iommu_cpu_syscore_ops); + + return 1; +} +device_initcall(sunway_iommu_init); + +/******************************************************************************* + * + * DMA OPS Functions + * + ******************************************************************************/ + +struct sunway_iommu *get_first_iommu_from_domain(struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu *iommu; + struct sunway_iommu_dev *entry; + + entry = list_first_entry(&sdomain->dev_list, struct sunway_iommu_dev, list); + iommu = entry->iommu; + + return iommu; +} + +static unsigned long +sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, + unsigned long iova, unsigned long page_size) +{ + unsigned long *pte_l2, unmapped; + + pr_debug("%s iova %#lx, page_size %#lx\n", __func__, iova, page_size); + BUG_ON(!is_power_of_2(page_size)); + + unmapped = 0; + while (unmapped < page_size) { + pte_l2 = (unsigned long *)fetch_pte(sunway_domain, iova, PTE_LEVEL2); + *pte_l2 = 0; + + flush_pcache_by_addr(sunway_domain, (unsigned long)pte_l2); + flush_ptlb_by_addr(sunway_domain, (iova >> PAGE_SHIFT)); + + iova += PAGE_SIZE; + unmapped += PAGE_SIZE; + } + + return unmapped; +} + +int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, + unsigned long bus_addr, unsigned long paddr, + size_t page_size) +{ + /* + * pde: page table entry + * pte: level 2 page table entry + * pte_root: page table root + */ + struct page *page; + struct sunway_iommu *iommu; + unsigned long pde, pte, iova_pfn; + unsigned long pdebaseaddr; + u64 *ptebasecond, ptebaseaddr; + u64 pte_root = (__pa(sunway_domain->pt_root) & PAGE_MASK); + + iova_pfn = (unsigned long)(bus_addr >> PAGE_SHIFT); + + pdebaseaddr = ((iova_pfn >> 10) & SW64_IOMMU_LEVEL1_OFFSET) << 3; + pdebaseaddr += ((pte_root) & (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK)) + + PAGE_OFFSET; + + pde = *(unsigned long *)pdebaseaddr; + if (pde) { + ptebaseaddr = (pde & (~SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + PAGE_OFFSET; + ptebaseaddr += (iova_pfn & SW64_IOMMU_LEVEL2_OFFSET) << 3; + + goto direct_map; + } + + iommu = get_first_iommu_from_domain(sunway_domain); + if (!iommu) + return -1; + page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0); + if (!page) { + pr_err("Allocating pages failed.\n"); + return -1; + } + + ptebasecond = page_address(page); + pde = (__pa(ptebasecond) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; + + /* + * If pde exists, no need to allocate a new page. + * Atomic compare and exchange, compare the value the pointer points to + * with 0UL. If identical, store pde where the pointer points to, return + * 0UL. Otherwise, return the value the pointer points to. + */ + if (cmpxchg64((volatile u64 *)pdebaseaddr, 0ULL, pde)) { + ptebaseaddr = ((*(volatile u64 *)pdebaseaddr) + & (~SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + PAGE_OFFSET; + ptebaseaddr += (iova_pfn & SW64_IOMMU_LEVEL2_OFFSET) << 3; + free_page((unsigned long)ptebasecond); + } else { + flush_pcache_by_addr(sunway_domain, pdebaseaddr); + ptebaseaddr = (unsigned long)ptebasecond + + ((iova_pfn & SW64_IOMMU_LEVEL2_OFFSET) << 3); + } + +direct_map: + /* case 8K */ + if (page_size == (1UL << PAGE_SHIFT)) { + if (*(volatile u64 *)ptebaseaddr) { + pr_err("IOVA 4G overlap. IOVA is %#lx.\n", bus_addr); + return -EFAULT; + } + + pte = (paddr & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID + | SW64_IOMMU_GRN_8K | SW64_IOMMU_ENABLE; + *(volatile u64 *)ptebaseaddr = pte; + flush_pcache_by_addr(sunway_domain, ptebaseaddr); + /* case 8M */ + } else if (page_size == (1UL << PAGE_8M_SHIFT)) { + unsigned long *ptr; + int i, ptes_one_page, ptes_one_cache; + + ptr = (unsigned long *)ptebaseaddr; + ptes_one_page = PAGE_SIZE/sizeof(pte); + ptes_one_cache = L1_CACHE_BYTES/sizeof(pte); + + pte = (paddr & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID + | SW64_IOMMU_GRN_8M | SW64_IOMMU_ENABLE; + + for (i = 0; i < ptes_one_page; i++) { + if (*ptr) { + pr_err("IOVA 4G overlap. IOVA is %#lx.\n", bus_addr); + return -EFAULT; + } + + *ptr = pte; + + /* just do once flush per cache line */ + if (i % ptes_one_cache == (ptes_one_cache - 1)) + flush_pcache_by_addr(sunway_domain, (unsigned long)ptr); + ptr++; + } + } +#ifdef CONFIG_SW64_GUEST + flush_ptlb_by_addr(sunway_domain, pfn | SW64_IOMMU_MAP_FLAG); +#endif + return 0; +} + +/********************************************************************** + * + * IOMMU OPS Functions + * + **********************************************************************/ + +static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + sdomain = sunway_domain_alloc(); + if (!sdomain) { + pr_err("Allocating sunway_domain failed!\n"); + return NULL; + } + + sdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); + if (!sdomain->pt_root) { + pr_err("Allocating pt_root failed!\n"); + sunway_domain_free(sdomain); + return NULL; + } + + sdomain->domain.geometry.aperture_start = 0ULL; + sdomain->domain.geometry.aperture_end = (~0ULL); + sdomain->domain.geometry.force_aperture = true; + sdomain->type = IOMMU_DOMAIN_UNMANAGED; + break; + + case IOMMU_DOMAIN_DMA: + dma_dom = dma_domain_alloc(); + if (!dma_dom) { + pr_err("Failed to alloc dma domain!\n"); + return NULL; + } + + sdomain = &dma_dom->sdomain; + break; + + case IOMMU_DOMAIN_IDENTITY: + sdomain = sunway_domain_alloc(); + if (!sdomain) + return NULL; + + sdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); + if (!sdomain->pt_root) { + pr_err("Allocating pt_root failed!\n"); + sunway_domain_free(sdomain); + return NULL; + } + + sdomain->type = IOMMU_DOMAIN_IDENTITY; + iommu_identity_mapping = 1; + break; + + default: + return NULL; + } + + return &sdomain->domain; +} + +static void clean_domain(struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu_dev *entry; + unsigned long flags; + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + + while (!list_empty(&sdomain->dev_list)) { + entry = list_first_entry(&sdomain->dev_list, + struct sunway_iommu_dev, list); + + __detach_device(entry); + } + + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); +} + +static void sunway_iommu_domain_free(struct iommu_domain *dom) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + + sdomain = to_sunway_domain(dom); + + if (sdomain->dev_cnt > 0) + clean_domain(sdomain); + + if (!dom) + return; + + switch (dom->type) { + case IOMMU_DOMAIN_DMA: + dma_dom = to_dma_domain(sdomain); + dma_domain_free(dma_dom); + break; + + default: + free_pagetable(sdomain); + sunway_domain_free(sdomain); + break; + } + +} + +static int sunway_iommu_attach_device(struct iommu_domain *dom, struct device *dev) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + struct sunway_iommu_dev *sdev_data; + struct pci_dev *pdev; + struct pci_controller *hose; + int ret; + + if (!dev_is_pci(dev)) + return -ENODEV; + + pdev = to_pci_dev(dev); + if (!pdev) + return -EINVAL; + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose) + return -EINVAL; + + if (!hose->iommu_enable) + return -EINVAL; + + sdev_data = dev_iommu_priv_get(dev); + if (!sdev_data) + return -EINVAL; + + if (sdev_data->domain) + detach_device(dev); + + ret = attach_device(dev, sdomain); + + return ret; +} + +static phys_addr_t +sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + unsigned long paddr, grn; + + if (iova >= SW64_BAR_ADDRESS) + return iova; + + paddr = fetch_pte(sdomain, iova, PTE_LEVEL2_VAL); + + if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) + return 0; + + paddr &= ~SW64_IOMMU_ENTRY_VALID; + grn = paddr & SW64_PTE_GRN_MASK; /* get page granularity */ + paddr &= PAGE_MASK; + + switch (grn) { + case SW64_IOMMU_GRN_8M: + paddr += (iova & ~HPAGE_MASK); + break; + case SW64_IOMMU_GRN_8K: + default: + paddr += (iova & ~PAGE_MASK); + break; + } + + return paddr; +} + +static int +sunway_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, + phys_addr_t paddr, size_t page_size, size_t pgcount, + int iommu_prot, gfp_t gfp, size_t *mapped) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + size_t size = pgcount << PAGE_SHIFT; + int ret; + + /* + * As VFIO cannot distinguish between normal DMA request + * and pci device BAR, check should be introduced manually + * to avoid VFIO trying to map pci config space. + */ + if (iova >= SW64_BAR_ADDRESS) + return 0; + + mutex_lock(&sdomain->api_lock); + while (pgcount--) { + ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size); + if (ret) { + pr_info("Failed to map page from IOVA %lx.\n", iova); + return ret; + } + iova += page_size; + paddr += page_size; + } + mutex_unlock(&sdomain->api_lock); + + if (!ret && mapped) + *mapped = size; + + return ret; +} + +static size_t +sunway_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova, + size_t page_size, size_t pgcount, + struct iommu_iotlb_gather *gather) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + size_t unmap_size; + size_t total_unmap = 0; + + if (iova >= SW64_BAR_ADDRESS) + return page_size; + + mutex_lock(&sdomain->api_lock); + while (pgcount--) { + unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); + iova += page_size; + total_unmap += page_size; + } + mutex_unlock(&sdomain->api_lock); + + return total_unmap; +} + +static struct iommu_group *sunway_iommu_device_group(struct device *dev) +{ + return generic_device_group(dev); +} + +static int iommu_init_device(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + struct sunway_iommu *iommu; + struct pci_dev *pdev; + struct pci_controller *hose; + + if (dev_iommu_priv_get(dev)) + return 0; + + sdev = kzalloc(sizeof(struct sunway_iommu_dev), GFP_KERNEL); + if (!sdev) + return -ENOMEM; + + pdev = to_pci_dev(dev); + hose = pci_bus_to_pci_controller(pdev->bus); + iommu = hose->pci_iommu; + llist_add(&sdev->dev_data_list, &dev_data_list); + sdev->pdev = pdev; + sdev->iommu = iommu; + + dev_iommu_priv_set(dev, sdev); + + return 0; +} + +static void iommu_uninit_device(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + + sdev = dev_iommu_priv_get(dev); + if (!sdev) + return; + + if (sdev->domain) + detach_device(dev); + + dev_iommu_priv_set(dev, NULL); +} + +static void sunway_iommu_release_device(struct device *dev) +{ + struct pci_dev *pdev; + struct pci_controller *hose; + + pdev = to_pci_dev(dev); + if (!pdev) + return; + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose->iommu_enable) + return; + + iommu_uninit_device(dev); +} + +static struct iommu_device *sunway_iommu_probe_device(struct device *dev) +{ + struct pci_dev *pdev; + struct pci_controller *hose; + struct sunway_iommu *iommu; + int ret; + + if (!dev_is_pci(dev)) + return ERR_PTR(-ENODEV); + + pdev = to_pci_dev(dev); + if (!pdev) + return ERR_PTR(-ENODEV); + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose) + return ERR_PTR(-ENODEV); + + if (!hose->iommu_enable) + return ERR_PTR(-ENODEV); + + if (dev_iommu_priv_get(dev)) + return &iommu->iommu; + + ret = iommu_init_device(dev); + if (ret) + return ERR_PTR(ret); + + iommu = hose->pci_iommu; + + return &iommu->iommu; +} + +static int sunway_iommu_def_domain_type(struct device *dev) +{ + if (dev_is_pci(dev)) { + if (iommu_identity_mapping) + return IOMMU_DOMAIN_IDENTITY; + } + + return 0; +} + +static bool sunway_iommu_capable(struct device *dev, enum iommu_cap cap) +{ + return false; +} + +static void sunway_iommu_probe_finalize(struct device *dev) +{ + set_dma_ops(dev, NULL); + iommu_setup_dma_ops(dev, 0, SW64_DMA_LIMIT); +} + +const struct iommu_ops sunway_iommu_ops = { + .capable = sunway_iommu_capable, + .domain_alloc = sunway_iommu_domain_alloc, + .probe_device = sunway_iommu_probe_device, + .probe_finalize = sunway_iommu_probe_finalize, + .release_device = sunway_iommu_release_device, + .device_group = sunway_iommu_device_group, + .pgsize_bitmap = SW64_IOMMU_PGSIZES, + .def_domain_type = sunway_iommu_def_domain_type, + .default_domain_ops = &(const struct iommu_domain_ops) { + .attach_dev = sunway_iommu_attach_device, + .map_pages = sunway_iommu_map_pages, + .unmap_pages = sunway_iommu_unmap_pages, + .iova_to_phys = sunway_iommu_iova_to_phys, + .free = sunway_iommu_domain_free, + } +}; + +/***************************************************************************** + * + * Boot param handle + * Each bit of iommu_enable bitmap represents an rc enable, and every 8 bits + * represents one cpu node. For example, iommu_enable=0x0100 means enabling + * rc0 for cpu node 1. + * + *****************************************************************************/ +static int __init iommu_enable_setup(char *str) +{ + int ret; + unsigned long rc_bitmap = 0xffffffffUL; + + ret = kstrtoul(str, 16, &rc_bitmap); + iommu_enable_cmd = rc_bitmap; + + return ret; +} +early_param("iommu_enable", iommu_enable_setup); diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c new file mode 100644 index 000000000000..f3e19e524210 --- /dev/null +++ b/drivers/iommu/sw64/iommu_v2.c @@ -0,0 +1,1780 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * iommu.c: Generic sw64 IOMMU support + * + * This is designed and tested for 3231. If there are no changes in hardware + * in later chips, then it should work just as well. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sunway_iommu.h" + +#define MAX_DOMAIN_NUM 65536 +#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) +#define SW64_32BIT_DMA_LIMIT (0xe0000000 - 1) +#define SW64_64BIT_DMA_LIMIT ((1UL << 41) - 1) +#define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) + +#define SW64_IOMMU_PGSIZES (((1ULL) << PAGE_SHIFT) \ + | ((1ULL) << PAGE_8M_SHIFT) \ + | ((1ULL) << PAGE_512M_SHIFT) \ + | ((1ULL) << PAGE_8G_SHIFT)) + +#define IDENTMAP_ALL ((1U) << 0) +#define DMA_MASK64 ((1U) << 1) + +#define PTE_VALID 0x8000000000000000UL +#define LAST_STAGE 0x100UL +#define PTE_GRN_8M 0x10UL +#define PTE_GRN_512M 0x20UL +#define PTE_GRN_8G 0x30UL +#define PTE_WRITEE 0x2UL +#define PTE_READE 0x1UL +#define PTE_RWE 0x3UL +#define PTE_FLAGS_MASK 0x8000000000000133UL +#define PAGE_8G_OFFSET_MASK ((1UL << PAGE_8G_SHIFT) - 1) +#define PAGE_512M_OFFSET_MASK ((1UL << PAGE_512M_SHIFT) - 1) +#define PAGE_8M_OFFSET_MASK ((1UL << PAGE_8M_SHIFT) - 1) + +/* IOMMU Exceptional Status */ +enum exceptype { + DTE_LEVEL1 = 0x0, + DTE_LEVEL2, + PTE_LEVEL1, + PTE_LEVEL2, + PTE_LEVEL3, + UNAUTHORIZED_ACCESS, + ILLEGAL_RESPONSE, + DTE_LEVEL1_VAL, + DTE_LEVEL2_VAL, + PTE_LEVEL1_VAL, + PTE_LEVEL2_VAL, + PTE_LEVEL3_VAL, +}; + +u64 iommu_enable_cmd; /* default IOMMU boot param: 0 */ + +unsigned long *sunway_iommu_domain_bitmap; + +static DEFINE_SPINLOCK(domain_bitmap_lock); +static DEFINE_SPINLOCK(sunway_iommu_device_table_lock); +spinlock_t sunway_domain_lock; + +static LLIST_HEAD(dev_data_list); +LIST_HEAD(sunway_domain_list); + +struct dma_domain { + struct sunway_iommu_domain sdomain; + struct iova_domain iovad; +}; +const struct iommu_ops sunway_iommu_ops; +static const struct dma_map_ops sunway_dma_ops; + + +/* flush helpers */ +static void piu_flush_all(struct pci_controller *hose) +{ + write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); + write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); + write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); +} + +void flush_pcache_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) +{ + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + + list_for_each_entry(sdev, &sdomain->dev_list, list) { + hose = pci_bus_to_pci_controller(sdev->pdev->bus); + + flush_addr = __pa(flush_addr); + /* Set memory bar here */ + mb(); + write_piu_ior0(hose->node, hose->index, + PCACHE_FLUSHPADDR, flush_addr); + } +} + +void flush_ptlb_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) +{ + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + struct pci_dev *pdev; + + list_for_each_entry(sdev, &sdomain->dev_list, list) { + pdev = sdev->pdev; + hose = pci_bus_to_pci_controller(pdev->bus); + + flush_addr = (pdev->bus->number << 8) + | pdev->devfn | (flush_addr << 16); + write_piu_ior0(hose->node, hose->index, + PTLB_FLUSHVADDR, flush_addr); + } +} + +/* domain helpers */ +static struct sunway_iommu_domain *to_sunway_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct sunway_iommu_domain, domain); +} + +static struct dma_domain *to_dma_domain(struct sunway_iommu_domain *sdomain) +{ + return container_of(sdomain, struct dma_domain, sdomain); +} + +static void add_domain_to_list(struct sunway_iommu_domain *sdomain) +{ + unsigned long flags; + + spin_lock_irqsave(&sunway_domain_lock, flags); + list_add(&sdomain->list, &sunway_domain_list); + spin_unlock_irqrestore(&sunway_domain_lock, flags); +} + +static void del_domain_from_list(struct sunway_iommu_domain *sdomain) +{ + unsigned long flags; + + spin_lock_irqsave(&sunway_domain_lock, flags); + list_del(&sdomain->list); + spin_unlock_irqrestore(&sunway_domain_lock, flags); +} + +static void free_pagetable(struct sunway_iommu_domain *sdomain) +{ + unsigned long *l2_pte, *l3_pte; + unsigned long l2_pte_val, l3_pte_val; + int l2_index, l3_index, ptes_one_page; + + l2_pte = sdomain->pt_root; + if (!l2_pte) + return; + + ptes_one_page = PAGE_SIZE/sizeof(unsigned long); + for (l2_index = 0; l2_index < ptes_one_page; l2_index++, l2_pte++) { + l2_pte_val = *l2_pte; + if ((l2_pte_val & SW64_IOMMU_ENTRY_VALID) == 0) + continue; + + l2_pte_val &= ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK; + l2_pte_val |= PAGE_OFFSET; + l3_pte = (unsigned long *)l2_pte_val; + for (l3_index = 0; l3_index < ptes_one_page; l3_index++, l3_pte++) { + l3_pte_val = *l3_pte; + if ((l3_pte_val & SW64_IOMMU_ENTRY_VALID) == 0) + continue; + + l3_pte_val &= ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK; + l3_pte_val |= PAGE_OFFSET; + free_page(l3_pte_val); + } + free_page(l2_pte_val); + } + + free_page((unsigned long)sdomain->pt_root); +} + +static void domain_id_free(int id) +{ + spin_lock(&domain_bitmap_lock); + if (id > 0) + __clear_bit(id, sunway_iommu_domain_bitmap); + spin_unlock(&domain_bitmap_lock); +} + +static void dma_domain_free(struct dma_domain *dma_dom) +{ + if (!dma_dom) + return; + + del_domain_from_list(&dma_dom->sdomain); + put_iova_domain(&dma_dom->iovad); + free_pagetable(&dma_dom->sdomain); + if (dma_dom->sdomain.id) + domain_id_free(dma_dom->sdomain.id); + + kfree(dma_dom); +} + +static void sunway_domain_free(struct sunway_iommu_domain *sdomain) +{ + if (!sdomain) + return; + + del_domain_from_list(sdomain); + if (sdomain->id) + domain_id_free(sdomain->id); + + kfree(sdomain); +} + +static u16 sunway_domain_id_alloc(void) +{ + int id; + + spin_lock(&domain_bitmap_lock); + id = find_first_zero_bit(sunway_iommu_domain_bitmap, MAX_DOMAIN_NUM); + if (id > 0 && id < MAX_DOMAIN_NUM) + __set_bit(id, sunway_iommu_domain_bitmap); + else + id = 0; + spin_unlock(&domain_bitmap_lock); + + return id; +} + +static int sunway_domain_init(struct sunway_iommu_domain *sdomain) +{ + spin_lock_init(&sdomain->lock); + mutex_init(&sdomain->api_lock); + sdomain->id = sunway_domain_id_alloc(); + if (!sdomain->id) + return -ENOMEM; + INIT_LIST_HEAD(&sdomain->dev_list); + + return 1; +} + +static struct sunway_iommu_domain *sunway_domain_alloc(void) +{ + struct sunway_iommu_domain *sdomain; + + sdomain = kzalloc(sizeof(struct sunway_iommu_domain), GFP_KERNEL); + if (!sdomain) + return NULL; + + if (!sunway_domain_init(sdomain)) { + kfree(sdomain); + return NULL; + } + + add_domain_to_list(sdomain); + return sdomain; +} + +static struct dma_domain *dma_domain_alloc(void) +{ + struct dma_domain *dma_dom; + struct page; + + dma_dom = kzalloc(sizeof(struct dma_domain), GFP_KERNEL); + if (!dma_dom) + return NULL; + + sunway_domain_init(&dma_dom->sdomain); + dma_dom->sdomain.type = IOMMU_DOMAIN_DMA; + init_iova_domain(&dma_dom->iovad, PAGE_SIZE, IOVA_PFN(SW64_DMA_START)); + reserve_iova(&dma_dom->iovad, (0xe0000000UL >> PAGE_SHIFT), (0x100000000UL >> PAGE_SHIFT)); + + add_domain_to_list(&dma_dom->sdomain); + + return dma_dom; +} + +static void device_flush_all(struct sunway_iommu_dev *sdata) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(sdata->pdev->bus); + + if (hose == NULL) + return; + + write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, sdata->devid); + write_piu_ior0(hose->node, hose->index, PTLB_FLUSHDEV, sdata->devid); + write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHDEV, sdata->devid); +} + +/* iommu_ops device attach/unattach helpers */ +static void +set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu *iommu; + struct pci_dev *pdev; + struct page *dt_page, *pt_page; + unsigned long *dte_l1, *dte_l2; + unsigned long dte_l1_val, dte_l2_base, dte_l2_val; + + pdev = sdev->pdev; + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + return; + + sdev->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); + iommu = sdev->iommu; + dte_l1 = iommu->iommu_dtbr + (pdev->bus->number); + dte_l1_val = *dte_l1; + + if (!dte_l1_val) { + /* Alloc a new level-2 device table page */ + dt_page = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, + get_order(PAGE_SIZE)); + if (!dt_page) { + pr_err("Allocating a new level-2 device table page failed.\n"); + return; + } + + dte_l2_base = (unsigned long)page_address(dt_page); + dte_l1_val = (__pa(dte_l2_base) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; + *dte_l1 = dte_l1_val; + } + + if (!sdomain->pt_root) { + pt_page = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, 0); + if (!pt_page) { + pr_err("Allocating pt_root failed!\n"); + return; + } + + sdomain->pt_root = page_address(pt_page); + } + + dte_l2 = __va(dte_l1_val & ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + (pdev->devfn << 3); + dte_l2_val = (__pa(sdomain->pt_root) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; + if (sdomain->type == IOMMU_DOMAIN_IDENTITY) { + dte_l2_val |= 0x1; + sdev->passthrough = IDENTMAP_ALL; + } + *dte_l2 = dte_l2_val; + device_flush_all(sdev); +} + +static void +do_attach(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomain) +{ + sdev_data->domain = sdomain; + list_add(&sdev_data->list, &sdomain->dev_list); + + sdomain->dev_cnt++; + set_dte_entry(sdev_data, sdomain); + + pr_debug("iommu: device %d add to domain: %d\n", + sdev_data->devid, sdomain->id); +} + +static void do_detach(struct sunway_iommu_dev *sdev_data) +{ + struct sunway_iommu_domain *sdomain = sdev_data->domain; + + sdev_data->domain = NULL; + list_del(&sdev_data->list); + device_flush_all(sdev_data); + + sdomain->dev_cnt--; + pr_debug("iommu: device %d detached from domain %d\n", + sdev_data->devid, sdomain->id); +} + +static int +__attach_device(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomain) +{ + int ret; + + spin_lock(&sdomain->lock); + ret = -EBUSY; + if (sdev_data->domain != NULL) + goto out_unlock; + + do_attach(sdev_data, sdomain); + ret = 0; + +out_unlock: + spin_unlock(&sdomain->lock); + return ret; +} + +static void __detach_device(struct sunway_iommu_dev *sunway_dev_data) +{ + struct sunway_iommu_domain *domain; + + domain = sunway_dev_data->domain; + + spin_lock(&domain->lock); + do_detach(sunway_dev_data); + spin_unlock(&domain->lock); +} + +static int attach_device(struct device *dev, struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu_dev *sdev; + unsigned long flags; + int ret; + + sdev = dev_iommu_priv_get(dev); + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + ret = __attach_device(sdev, sdomain); + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); + + return ret; +} + +static void detach_device(struct device *dev) +{ + struct sunway_iommu_domain *sunway_domain; + struct sunway_iommu_dev *sdev; + unsigned long flags; + + sdev = dev_iommu_priv_get(dev); + sunway_domain = sdev->domain; + + if (WARN_ON(!sdev->domain)) + return; + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + __detach_device(sdev); + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); + + if (!dev_is_pci(dev)) + return; +} + +static struct sunway_iommu_dev *search_dev_data(u16 devid) +{ + struct sunway_iommu_dev *sdev_data; + struct llist_node *node; + + if (llist_empty(&dev_data_list)) + return NULL; + + node = dev_data_list.first; + llist_for_each_entry(sdev_data, node, dev_data_list) { + if (sdev_data->devid == devid) + return sdev_data; + } + + return NULL; +} + +/* dma_ops helpers*/ +static struct sunway_iommu_domain *get_sunway_domain(struct device *dev) +{ + struct sunway_iommu_domain *sdomain; + struct iommu_domain *domain; + struct pci_dev *pdev; + struct sunway_iommu_dev *sdev; + + pdev = to_pci_dev(dev); + if (!pdev) + return ERR_PTR(-ENODEV); + + sdev = dev_iommu_priv_get(dev); + sdomain = sdev->domain; + if (sdomain == NULL) { + domain = iommu_get_domain_for_dev(dev); + sdomain = to_sunway_domain(domain); + attach_device(dev, sdomain); + } + + if (sdomain == NULL) + return ERR_PTR(-EBUSY); + + return sdomain; +} + +/********************************************************************** + * + * Following functions describe IOMMU init ops + * + **********************************************************************/ + +static struct sunway_iommu *sunway_iommu_early_init(struct pci_controller *hose) +{ + struct sunway_iommu *iommu; + struct page *page; + unsigned long base; + + hose->pci_iommu = kzalloc(sizeof(struct sunway_iommu), GFP_KERNEL); + if (!hose->pci_iommu) + return 0; + + iommu = hose->pci_iommu; + spin_lock_init(&iommu->dt_lock); + + iommu->node = hose->node; + if (!node_online(hose->node)) + iommu->node = -1; + + page = alloc_pages_node(iommu->node, __GFP_ZERO, get_order(PAGE_SIZE)); + if (!page) { + pr_err("Allocating a new iommu_dtbr page failed.\n"); + kfree(hose->pci_iommu); + return NULL; + } + iommu->iommu_dtbr = page_address(page); + + iommu->hose_pt = hose; + iommu->index = hose->index; + + iommu->enabled = true; + + base = __pa(iommu->iommu_dtbr) & PAGE_MASK; + write_piu_ior0(hose->node, hose->index, DTBASEADDR, base); + + return iommu; +} + +unsigned long fetch_dte(struct sunway_iommu *iommu, unsigned long devid, + enum exceptype type) +{ + unsigned long *dte_l1, *dte_l2; + unsigned long dte_l1_val, dte_l2_val; + + if (!iommu) + return 0; + dte_l1 = iommu->iommu_dtbr + (devid >> 8); + if (type == DTE_LEVEL1) + return (unsigned long)dte_l1; + + dte_l1_val = *dte_l1; + if (type == DTE_LEVEL1_VAL) + return dte_l1_val; + + dte_l1_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + dte_l1_val |= PAGE_OFFSET; + dte_l2 = (unsigned long *)(dte_l1_val + ((devid & 0xff) << 3)); + if (type == DTE_LEVEL2) + return (unsigned long)dte_l2; + + dte_l2_val = *dte_l2; + if (type == DTE_LEVEL2_VAL) + return dte_l2_val; + + return dte_l2_val; +} + +unsigned long fetch_pte(struct sunway_iommu_domain *sdomain, dma_addr_t iova, + enum exceptype type) +{ + unsigned long iova_pfn; + unsigned long pte_l1_val, pte_l2_val, pte_l3_val; + unsigned long *pte_l1, *pte_l2, *pte_l3; + unsigned long pte_root; + unsigned long offset; + + if (!sdomain) + return -EINVAL; + + pte_root = __pa(sdomain->pt_root) & PAGE_MASK; + iova_pfn = iova >> PAGE_SHIFT; + pte_root = ((pte_root) & (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK)); + pte_root |= PAGE_OFFSET; + offset = ((iova_pfn >> 20) & SW64_IOMMU_LEVEL1_OFFSET) << 3; + pte_l1 = (unsigned long *)(pte_root + offset); + if (type == PTE_LEVEL1) + return (unsigned long)pte_l1; + + pte_l1_val = *pte_l1; + if (type == PTE_LEVEL1_VAL) + return pte_l1_val; + + pte_l1_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + pte_l1_val |= PAGE_OFFSET; + offset = ((iova_pfn >> 10) & SW64_IOMMU_LEVEL2_OFFSET) << 3; + pte_l2 = (unsigned long *)(pte_l1_val + offset); + + if (type == PTE_LEVEL2) + return (unsigned long)pte_l2; + + pte_l2_val = *pte_l2; + if (type == PTE_LEVEL2_VAL) + return pte_l2_val; + + pte_l2_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + pte_l2_val |= PAGE_OFFSET; + offset = (iova_pfn & SW64_IOMMU_LEVEL3_OFFSET) << 3; + pte_l3 = (unsigned long *)(pte_l2_val + offset); + if (type == PTE_LEVEL3) + return (unsigned long)pte_l3; + + pte_l3_val = *pte_l3; + if (type == PTE_LEVEL3_VAL) + return pte_l3_val; + + return pte_l3_val; +} + +/* IOMMU Interrupt handle */ +irqreturn_t iommu_interrupt(int irq, void *dev) +{ + struct pci_controller *hose = (struct pci_controller *)dev; + struct sunway_iommu_domain *sdomain; + struct sunway_iommu_dev *sdev; + unsigned long iommu_status; + unsigned long type; + unsigned long devid, dva; + + iommu_status = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); + if (!(iommu_status >> 63)) + return IRQ_NONE; + + type = (iommu_status >> 58) & 0xf; + devid = (iommu_status >> 36) & 0xffff; + dva = ((iommu_status & 0xffffffff) >> 3) << 13; + pr_info("%s, iommu_status = %#lx, devid %#lx, dva %#lx, ", + __func__, iommu_status, devid, dva); + + sdev = search_dev_data(devid); + if (sdev == NULL) { + pr_info("no such dev!!!\n"); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + + return IRQ_HANDLED; + } + + sdomain = sdev->domain; + switch (type) { + case DTE_LEVEL1: + pr_info("invalid level1 dte, addr:%#lx, val:%#lx\n", + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1), + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1_VAL)); + break; + case DTE_LEVEL2: + pr_info("invalid level2 dte, addr:%#lx, val:%#lx\n", + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2), + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2_VAL)); + break; + case PTE_LEVEL1: + pr_info("invalid level1 pte, addr: %#lx, val:%#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL1), + fetch_pte(sdomain, dva, PTE_LEVEL1_VAL)); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + break; + case PTE_LEVEL2: + pr_info("invalid level2 pte, addr: %#lx, val: %#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL2), + fetch_pte(sdomain, dva, PTE_LEVEL2_VAL)); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + break; + + case PTE_LEVEL3: + pr_info("invalid level3 pte, addr: %#lx, val: %#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL3), + fetch_pte(sdomain, dva, PTE_LEVEL3_VAL)); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + break; + default: + pr_info("iommu exception type %ld\n", type); + break; + } + + return IRQ_HANDLED; +} + +struct irqaction iommu_irqaction = { + .handler = iommu_interrupt, + .flags = IRQF_SHARED | IRQF_NO_THREAD, + .name = "sunway_iommu", +}; + +void sunway_enable_iommu_func(struct pci_controller *hose) +{ + unsigned int iommu_irq, err; + unsigned long iommu_conf, iommu_ctrl; + + iommu_irq = hose->int_irq; + pr_debug("%s node %ld rc %ld iommu_irq %d\n", + __func__, hose->node, hose->index, iommu_irq); + err = request_irq(iommu_irq, iommu_interrupt, + IRQF_SHARED, "sunway_iommu", hose); + if (err < 0) + pr_info("sw iommu request irq failed!\n"); + + iommu_ctrl = (1UL << 63) | (0x100UL << 10); + write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_CTRL, iommu_ctrl); + iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + iommu_conf = iommu_conf | (0x3 << 7); + write_piu_ior0(hose->node, hose->index, PIUCONFIG0, iommu_conf); + write_piu_ior0(hose->node, hose->index, TIMEOUT_CONFIG, 0xf); + iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + pr_debug("SW arch configure node %ld hose-%ld iommu_conf = %#lx\n", + hose->node, hose->index, iommu_conf); +} + +static bool is_iommu_enable(struct pci_controller *hose) +{ + u64 rc_mask = 0x1; + + rc_mask <<= (8 * hose->node + hose->index); + if (iommu_enable_cmd & rc_mask) + return true; + + return false; +} + +/* iommu cpu syscore ops */ +static int iommu_cpu_suspend(void) +{ + return 0; +} + +static void iommu_cpu_resume(void) +{ + +} + +struct syscore_ops iommu_cpu_syscore_ops = { + .suspend = iommu_cpu_suspend, + .resume = iommu_cpu_resume, +}; + +static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type); + +static int sunway_iommu_init(void) +{ + struct pci_controller *hose; + struct sunway_iommu *iommu; + int ret; + int iommu_index = 0; + + sunway_iommu_domain_bitmap = + (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(MAX_DOMAIN_NUM / 8)); + if (sunway_iommu_domain_bitmap == NULL) + return 0; + __set_bit(0, sunway_iommu_domain_bitmap); + + /* Do the loop */ + for (hose = hose_head; hose; hose = hose->next) { + if (!is_iommu_enable(hose)) { + hose->iommu_enable = false; + continue; + } + + iommu = sunway_iommu_early_init(hose); + if (!iommu) { + pr_err("Allocating sunway_iommu failed\n"); + hose->iommu_enable = false; + continue; + } + + iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", + iommu_index); + iommu_device_set_ops(&iommu->iommu, &sunway_iommu_ops); + iommu_device_register(&iommu->iommu); + iommu_index++; + sunway_enable_iommu_func(hose); + hose->iommu_enable = true; + } + + ret = iova_cache_get(); + if (ret) + return ret; + + ret = bus_set_iommu(&pci_bus_type, &sunway_iommu_ops); + if (ret) + return ret; + + for (hose = hose_head; hose; hose = hose->next) + if (hose->iommu_enable) + piu_flush_all(hose); + + register_syscore_ops(&iommu_cpu_syscore_ops); + + return 1; +} +subsys_initcall_sync(sunway_iommu_init); + +/******************************************************************************* + * + * DMA OPS Functions + * + ******************************************************************************/ + +struct sunway_iommu *get_first_iommu_from_domain(struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu *iommu; + struct sunway_iommu_dev *entry; + + entry = list_first_entry(&sdomain->dev_list, struct sunway_iommu_dev, list); + iommu = entry->iommu; + + return iommu; +} + +static unsigned long +sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, + unsigned long iova, unsigned long page_size) +{ + unsigned long offset, iova_pfn; + unsigned long *pte_base, *pte; + unsigned long grn; + int level, current_level; + int tmp = 1; + + pr_debug("%s iova %#lx, page_size %#lx\n", __func__, iova, page_size); + BUG_ON(!is_power_of_2(page_size)); + + switch (page_size) { + case (1UL << 33): + level = 1; + grn = PTE_GRN_8G; + break; + case (1UL << 29): + level = 2; + grn = PTE_GRN_512M; + break; + case (1UL << 23): + level = 2; + grn = PTE_GRN_8M; + break; + default: + level = 3; + break; + } + + pte_base = sunway_domain->pt_root; + iova_pfn = iova >> PAGE_SHIFT; + offset = (iova_pfn >> 20) & 0x1ff; + current_level = 1; + while (current_level <= level) { + pte = &pte_base[offset]; + if (current_level == level) { + if (grn == PTE_GRN_512M) { + int i; + + for (i = 0; i < 64; i++) { + *(pte + i) = 0; + flush_pcache_by_addr(sunway_domain, (unsigned long)pte); + } + + } else { + *pte = 0; + flush_pcache_by_addr(sunway_domain, (unsigned long)pte); + } + flush_ptlb_by_addr(sunway_domain, (iova >> PAGE_SHIFT)); + break; + } + + pte_base = (unsigned long *)((*pte & (~PTE_FLAGS_MASK)) | PAGE_OFFSET); + offset = (iova_pfn >> (tmp--) * 10) & 0x3ff; + current_level++; + } + + return page_size; +} + +int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, + unsigned long bus_addr, unsigned long paddr, + size_t page_size) +{ + struct page *page; + struct sunway_iommu *iommu; + unsigned long iova_pfn, pte_val; + unsigned long *pte_base, *pte; + unsigned long offset, grn = 0; + int level = 0, current_level; + int tmp = 1; + + iommu = get_first_iommu_from_domain(sunway_domain); + if (!iommu) + return -1; + iova_pfn = bus_addr >> PAGE_SHIFT; + pte_base = sunway_domain->pt_root; + + switch (page_size) { + case (1UL << 33): + level = 1; + grn = PTE_GRN_8G; + break; + case (1UL << 29): + level = 2; + grn = PTE_GRN_512M; + break; + case (1UL << 23): + grn = PTE_GRN_8M; + level = 2; + break; + default: + level = 3; + break; + } + + offset = (iova_pfn >> 20) & 0x1ff; + current_level = 1; + while (current_level <= level) { + pte = &pte_base[offset]; + + if (!(*pte) || (current_level == level)) { + pte_val = PTE_VALID | PTE_RWE | grn; + if (current_level == level) { + *(volatile u64 *)(pte) = 0; + pte_val |= ((paddr & PAGE_MASK) | LAST_STAGE); + } else { + page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0); + if (!page) { + pr_err("Allocating level%d page table pages failed.\n", (level + 1)); + return -ENOMEM; + } + + pte_val |= (page_to_phys(page) & PAGE_MASK); + } + + if ((grn == PTE_GRN_512M) && (current_level == 2)) { + int i; + + for (i = 0; i < 64; i++) { + cmpxchg64((volatile u64 *)(pte + i), 0UL, pte_val); + flush_pcache_by_addr(sunway_domain, (unsigned long)(pte + i)); + } + } else { + if (cmpxchg64((volatile u64 *)pte, 0UL, pte_val)) + free_page((unsigned long)page_address(page)); + else + flush_pcache_by_addr(sunway_domain, (unsigned long)pte); + } + } + + pte_base = (unsigned long *)__va((*pte) & (~PTE_FLAGS_MASK)); + offset = (iova_pfn >> (tmp--) * 10) & 0x3ff; + current_level++; + } + + return 0; +} + +static unsigned long +sunway_alloc_iova(struct dma_domain *dma_dom, unsigned long pages, struct pci_dev *pdev) +{ + struct device *dev; + unsigned long pfn = 0; + + pages = __roundup_pow_of_two(pages); + dev = &(pdev->dev); + if (min(dev->coherent_dma_mask, *dev->dma_mask) == DMA_BIT_MASK(32)) { + pfn = alloc_iova_fast(&dma_dom->iovad, pages, + IOVA_PFN(SW64_32BIT_DMA_LIMIT), true); + } else { + /* IOVA boundary should be 16M ~ 3.5G */ + pfn = alloc_iova_fast(&dma_dom->iovad, pages, + IOVA_PFN(SW64_64BIT_DMA_LIMIT), true); + } + + return (pfn << PAGE_SHIFT); +} + +static void sunway_free_iova(struct dma_domain *dma_dom, + unsigned long address, unsigned long pages) +{ + pages = __roundup_pow_of_two(pages); + address >>= PAGE_SHIFT; + + free_iova_fast(&dma_dom->iovad, address, pages); +} + +static dma_addr_t +__sunway_map_single(struct dma_domain *dma_dom, + struct pci_dev *pdev, phys_addr_t paddr, size_t size) +{ + dma_addr_t ret, address, start; + unsigned long npages, i; + + npages = iommu_num_pages(paddr, size, PAGE_SIZE); + + address = sunway_alloc_iova(dma_dom, npages, pdev); + if (!address) + return 0; + + start = address; + for (i = 0; i < npages; ++i) { + ret = sunway_iommu_map_page(&dma_dom->sdomain, start, + paddr, PAGE_SIZE); + if (ret) { + pr_info("error when map page.\n"); + goto out_unmap; + } + + start += PAGE_SIZE; + paddr += PAGE_SIZE; + } + + address += paddr & ~PAGE_MASK; + return address; + +out_unmap: + for (--i; i >= 0; --i) { + start -= PAGE_SIZE; + sunway_iommu_unmap_page(&dma_dom->sdomain, start, PAGE_SIZE); + } + + sunway_free_iova(dma_dom, address, npages); + return 0; +} + +static dma_addr_t +pci_iommu_map_single(struct pci_dev *pdev, + struct dma_domain *dma_dom, void *cpu_addr, size_t size) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(pdev->bus); + unsigned long paddr; + + if (hose == NULL) { + pr_err("%s: hose does not exist!\n", __func__); + return 0; + } + + paddr = __sunway_map_single(dma_dom, pdev, __pa(cpu_addr), size); + + pr_debug("pci_alloc_consistent: %zx -> [%px,%lx] from %ps\n", + size, cpu_addr, paddr, __builtin_return_address(0)); + + return paddr; +} + +static void *sunway_alloc_coherent(struct device *dev, + size_t size, + dma_addr_t *dma_addr, gfp_t gfp, + unsigned long attrs) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_controller *hose; + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct sunway_iommu_dev *sdev; + struct page *page; + void *cpu_addr; + + if (!pdev) + return NULL; + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose) + return NULL; + + gfp &= ~GFP_DMA; + +try_again: + page = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, get_order(size)); + if (!page) { + pr_err("Allocating pages failed.\n"); + return NULL; + } + + cpu_addr = page_address(page); + if (!cpu_addr) { + pr_info + ("pci_alloc_consistent: get_free_pages failed from %ps\n", + __builtin_return_address(0)); + + return NULL; + } + + *dma_addr = __pa(cpu_addr); + if (!(hose->iommu_enable)) + return cpu_addr; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough & DMA_MASK64) + return cpu_addr; + else if (sdev->passthrough) { + if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { + sdev->passthrough |= DMA_MASK64; + return cpu_addr; + } + + __free_pages(page, get_order(size)); + set_dma_ops(dev, get_arch_dma_ops(dev->bus)); + return dev->dma_ops->alloc(dev, size, dma_addr, gfp, attrs); + } + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + *dma_addr = pci_iommu_map_single(pdev, dma_dom, cpu_addr, size); + if (*dma_addr == 0) { + free_pages((unsigned long)cpu_addr, get_order(size)); + if (gfp & GFP_DMA) + return NULL; + + gfp |= GFP_DMA; + goto try_again; + } + + return cpu_addr; +} + +static void +__sunway_unmap_single(struct dma_domain *dma_dom, dma_addr_t dma_addr, size_t size) +{ + dma_addr_t start; + unsigned long npages; + int i; + + npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); + dma_addr &= PAGE_MASK; + start = dma_addr; + + for (i = 0; i < npages; i++) { + sunway_iommu_unmap_page(&dma_dom->sdomain, start, PAGE_SIZE); + start += PAGE_SIZE; + } + + sunway_free_iova(dma_dom, dma_addr, npages); + pr_debug("pci_free_consistent: %zx -> [%llx] from %ps\n", + size, dma_addr, __builtin_return_address(0)); + +} + +static void +sunway_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_addr, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + + if (!pdev) + goto out_unmap; + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose || !(hose->iommu_enable)) + goto out_unmap; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough) + goto out_unmap; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + __sunway_unmap_single(dma_dom, dma_addr, size); + goto out_free; + +out_unmap: + pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); + +out_free: + pr_debug("sunway_free_consistent: [%llx,%zx] from %ps\n", + dma_addr, size, __builtin_return_address(0)); + + free_pages((unsigned long)vaddr, get_order(size)); +} + +static dma_addr_t +sunway_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, unsigned long attrs) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + phys_addr_t paddr = page_to_phys(page) + offset; + + if (!pdev) + return 0; + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose || !(hose->iommu_enable)) + return paddr; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough & DMA_MASK64) + return paddr; + else if (sdev->passthrough) { + if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { + sdev->passthrough |= DMA_MASK64; + return paddr; + } + + set_dma_ops(dev, get_arch_dma_ops(dev->bus)); + return dev->dma_ops->map_page(dev, page, offset, size, dir, attrs); + } + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + return pci_iommu_map_single(pdev, dma_dom, + (char *)page_address(page) + offset, size); +} + +static void +sunway_unmap_page(struct device *dev, dma_addr_t dma_addr, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct pci_dev *pdev; + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + + pdev = to_pci_dev(dev); + if (!pdev) + return; + + hose = pci_bus_to_pci_controller(pdev->bus); + if (hose == NULL) + return; + + if (!hose->iommu_enable) + return; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough) + return; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + __sunway_unmap_single(dma_dom, dma_addr, size); +} + +#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) +static int +sunway_map_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom = NULL; + struct scatterlist *sg; + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + int i, out_nents = 0; + + if (dir == PCI_DMA_NONE) + BUG(); + + if (!pdev) + return 0; + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose) + return 0; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + for_each_sg(sgl, sg, nents, i) { + BUG_ON(!sg_page(sg)); + + sg_dma_address(sg) = __pa(SG_ENT_VIRT_ADDRESS(sg)); + if (!(hose->iommu_enable)) + goto check; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough & DMA_MASK64) + goto check; + else if (sdev->passthrough) { + if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { + sdev->passthrough |= DMA_MASK64; + goto check; + } + + set_dma_ops(dev, get_arch_dma_ops(dev->bus)); + return dev->dma_ops->map_sg(dev, sgl, nents, dir, attrs); + } + + sg_dma_address(sg) = + pci_iommu_map_single(pdev, dma_dom, + SG_ENT_VIRT_ADDRESS(sg), sg->length); +check: + if (sg_dma_address(sg) == 0) + goto error; + + sg_dma_len(sg) = sg->length; + out_nents++; + } + + return nents; + +error: + pr_warn("pci_map_sg failed:"); + pr_warn("could not allocate dma page tables\n"); + + if (out_nents) + pci_unmap_sg(pdev, sgl, out_nents, dir); + return 0; +} + +static void +sunway_unmap_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct scatterlist *sg; + struct pci_dev *pdev; + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + dma_addr_t dma_addr; + long size; + int j; + + pdev = to_pci_dev(dev); + if (!pdev) + return; + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose->iommu_enable) + return; + + sdev = dev_iommu_priv_get(dev); + if (sdev->passthrough) + return; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + for_each_sg(sgl, sg, nents, j) { + dma_addr = sg->dma_address; + size = sg->dma_length; + if (!size) + break; + + __sunway_unmap_single(dma_dom, dma_addr, size); + } +} + +static const struct dma_map_ops sunway_dma_ops = { + .alloc = sunway_alloc_coherent, + .free = sunway_free_coherent, + .map_sg = sunway_map_sg, + .unmap_sg = sunway_unmap_sg, + .map_page = sunway_map_page, + .unmap_page = sunway_unmap_page, + .dma_supported = dma_direct_supported, +}; + +/********************************************************************** + * + * IOMMU OPS Functions + * + **********************************************************************/ + +static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + sdomain = sunway_domain_alloc(); + if (!sdomain) { + pr_err("Allocating sunway_domain failed!\n"); + return NULL; + } + + sdomain->domain.geometry.aperture_start = 0UL; + sdomain->domain.geometry.aperture_end = ~0ULL; + sdomain->domain.geometry.force_aperture = true; + sdomain->type = IOMMU_DOMAIN_UNMANAGED; + break; + + case IOMMU_DOMAIN_DMA: + dma_dom = dma_domain_alloc(); + if (!dma_dom) { + pr_err("Failed to alloc dma domain!\n"); + return NULL; + } + + sdomain = &dma_dom->sdomain; + break; + + case IOMMU_DOMAIN_IDENTITY: + sdomain = sunway_domain_alloc(); + if (!sdomain) + return NULL; + + sdomain->type = IOMMU_DOMAIN_IDENTITY; + break; + + default: + return NULL; + } + + return &sdomain->domain; +} + +static void clean_domain(struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu_dev *entry; + unsigned long flags; + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + + while (!list_empty(&sdomain->dev_list)) { + entry = list_first_entry(&sdomain->dev_list, + struct sunway_iommu_dev, list); + + BUG_ON(!entry->domain); + __detach_device(entry); + } + + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); +} + +static void sunway_iommu_domain_free(struct iommu_domain *dom) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + + sdomain = to_sunway_domain(dom); + + if (sdomain->dev_cnt > 0) + clean_domain(sdomain); + + BUG_ON(sdomain->dev_cnt != 0); + + if (!dom) + return; + + switch (dom->type) { + case IOMMU_DOMAIN_DMA: + dma_dom = to_dma_domain(sdomain); + dma_domain_free(dma_dom); + break; + + default: + free_pagetable(sdomain); + sunway_domain_free(sdomain); + break; + } + +} + +static int sunway_iommu_attach_device(struct iommu_domain *dom, struct device *dev) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + struct sunway_iommu_dev *sdev; + struct pci_dev *pdev; + struct pci_controller *hose; + int ret; + + pdev = to_pci_dev(dev); + if (!pdev) + return -EINVAL; + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose) + return -EINVAL; + + if (!hose->iommu_enable) + return -EINVAL; + + sdev = dev_iommu_priv_get(dev); + if (!sdev) + return -EINVAL; + + if (sdev->domain) + detach_device(dev); + + ret = attach_device(dev, sdomain); + + return ret; +} + +static void sunway_iommu_detach_device(struct iommu_domain *dom, struct device *dev) +{ + struct sunway_iommu_dev *sdev; + struct pci_dev *pdev = to_pci_dev(dev); + + if (!pdev) + return; + + sdev = dev_iommu_priv_get(dev); + if (sdev->domain != NULL) + detach_device(dev); +} + +static phys_addr_t +sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + unsigned long paddr, grn; + unsigned long is_last; + + if (iova > SW64_BAR_ADDRESS) + return iova; + + paddr = fetch_pte(sdomain, iova, PTE_LEVEL1_VAL); + if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) + return 0; + + is_last = paddr & SW64_PTE_LAST_MASK; + grn = paddr & SW64_PTE_GRN_MASK; + if (is_last) { + if (grn == PTE_GRN_8G) { + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_8G_OFFSET_MASK; + return paddr; + } + + return 0; + } + + paddr = fetch_pte(sdomain, iova, PTE_LEVEL2_VAL); + if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) + return 0; + + is_last = paddr & SW64_PTE_LAST_MASK; + grn = paddr & SW64_PTE_GRN_MASK; + if (is_last) { + if (grn == PTE_GRN_512M) { + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_512M_OFFSET_MASK; + return paddr; + } + + if (grn == PTE_GRN_8M) { + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_8M_OFFSET_MASK; + return paddr; + } + + return 0; + } + + paddr = fetch_pte(sdomain, iova, PTE_LEVEL3_VAL); + if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) + return 0; + + grn = paddr & SW64_PTE_GRN_MASK; + if (grn != 0) + return 0; + + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_MASK; + return paddr; +} + +static int +sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, + phys_addr_t paddr, size_t page_size, int iommu_prot, gfp_t gfp) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + int ret; + + /* + * As VFIO cannot distinguish between normal DMA request + * and pci device BAR, check should be introduced manually + * to avoid VFIO trying to map pci config space. + */ + if (iova > SW64_BAR_ADDRESS) + return 0; + + mutex_lock(&sdomain->api_lock); + ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size); + mutex_unlock(&sdomain->api_lock); + + return ret; +} + +static size_t +sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, + size_t page_size, + struct iommu_iotlb_gather *gather) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + size_t unmap_size; + + if (iova > SW64_BAR_ADDRESS) + return page_size; + + mutex_lock(&sdomain->api_lock); + unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); + mutex_unlock(&sdomain->api_lock); + + return unmap_size; +} + +static struct iommu_group *sunway_iommu_device_group(struct device *dev) +{ + return pci_device_group(dev); +} + +static void iommu_uninit_device(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + + sdev = dev_iommu_priv_get(dev); + if (!sdev) + return; + + if (sdev->domain) + detach_device(dev); + + dev_iommu_priv_set(dev, NULL); +} + +static void sunway_iommu_release_device(struct device *dev) +{ + struct pci_dev *pdev; + struct pci_controller *hose; + + pdev = to_pci_dev(dev); + if (!pdev) + return; + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose->iommu_enable) + return; + + iommu_uninit_device(dev); +} + +static int iommu_init_device(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + struct sunway_iommu *iommu; + struct pci_dev *pdev; + struct pci_controller *hose; + + if (dev_iommu_priv_get(dev)) + return 0; + + sdev = kzalloc(sizeof(struct sunway_iommu_dev), GFP_KERNEL); + if (!sdev) + return -ENOMEM; + + pdev = to_pci_dev(dev); + hose = pci_bus_to_pci_controller(pdev->bus); + iommu = hose->pci_iommu; + llist_add(&sdev->dev_data_list, &dev_data_list); + sdev->pdev = pdev; + sdev->iommu = iommu; + + dev_iommu_priv_set(dev, sdev); + + return 0; +} + +static struct iommu_device *sunway_iommu_probe_device(struct device *dev) +{ + struct pci_dev *pdev; + struct pci_controller *hose; + struct sunway_iommu *iommu; + int ret; + + pdev = to_pci_dev(dev); + if (!pdev) + return ERR_PTR(-ENODEV); + + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + return ERR_PTR(-ENODEV); + + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) + return ERR_PTR(-ENODEV); + + hose = pci_bus_to_pci_controller(pdev->bus); + if (!hose) + return ERR_PTR(-ENODEV); + + if (!hose->iommu_enable) + return ERR_PTR(-ENODEV); + + if (dev_iommu_priv_get(dev)) { + iommu = hose->pci_iommu; + return &iommu->iommu; + } + + ret = iommu_init_device(dev); + if (ret) + return ERR_PTR(ret); + + iommu = hose->pci_iommu; + + return &iommu->iommu; +} + +static int sunway_iommu_def_domain_type(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + + sdev = dev_iommu_priv_get(dev); + if (sdev->domain) + return 0; + + return sdev->domain->type; +} + +static bool sunway_iommu_capable(enum iommu_cap cap) +{ + switch (cap) { + case IOMMU_CAP_INTR_REMAP: + return true; + default: + return false; + } +} + +static void sunway_iommu_probe_finalize(struct device *dev) +{ + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (domain) + set_dma_ops(dev, &sunway_dma_ops); +} + +const struct iommu_ops sunway_iommu_ops = { + .capable = sunway_iommu_capable, + .domain_alloc = sunway_iommu_domain_alloc, + .domain_free = sunway_iommu_domain_free, + .attach_dev = sunway_iommu_attach_device, + .detach_dev = sunway_iommu_detach_device, + .probe_device = sunway_iommu_probe_device, + .probe_finalize = sunway_iommu_probe_finalize, + .release_device = sunway_iommu_release_device, + .map = sunway_iommu_map, + .unmap = sunway_iommu_unmap, + .iova_to_phys = sunway_iommu_iova_to_phys, + .device_group = sunway_iommu_device_group, + .pgsize_bitmap = SW64_IOMMU_PGSIZES, + .def_domain_type = sunway_iommu_def_domain_type, +}; + +/***************************************************************************** + * + * Boot param handle + * Each bit of iommu_enable bitmap represents an rc enable, and every 8 bits + * represents one cpu node. For example, iommu_enable=0x0100 means enabling + * rc0 for cpu node 1. + * + *****************************************************************************/ +static int __init iommu_enable_setup(char *str) +{ + int ret; + unsigned long rc_bitmap = 0xffffffffUL; + + ret = kstrtoul(str, 16, &rc_bitmap); + iommu_enable_cmd = rc_bitmap; + + return ret; +} +__setup("iommu_enable=", iommu_enable_setup); diff --git a/drivers/iommu/sw64/sunway_iommu.h b/drivers/iommu/sw64/sunway_iommu.h new file mode 100644 index 000000000000..94a155001d1b --- /dev/null +++ b/drivers/iommu/sw64/sunway_iommu.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file contains declarations and inline functions for interfacing + * with the PCI initialization routines. + */ +#include +#include +#include +#include + +struct sunway_iommu_bypass_id { + unsigned int vendor; + unsigned int device; +}; + +struct sunway_iommu { + int index; + bool enabled; + unsigned long *iommu_dtbr; + spinlock_t dt_lock; /* Device Table Lock */ + int node; /* NUMA node */ + + struct pci_controller *hose_pt; + struct iommu_device iommu; /* IOMMU core code handle */ +}; + +struct sunway_iommu_dev { + struct list_head list; /* For domain->dev_list */ + struct llist_node dev_data_list; /* Global device list */ + u16 devid; + int alias; + unsigned int passthrough; + struct sunway_iommu *iommu; + struct pci_dev *pdev; + + spinlock_t lock; /* Lock the page table mainly */ + struct sunway_iommu_domain *domain; /* Domain device is bound to */ +}; + +struct sunway_iommu_domain { + unsigned int type; + spinlock_t lock; + struct mutex api_lock; + u16 id; /* Domain ID */ + struct list_head list; /* For list of all SW domains */ + struct list_head dev_list; /* List of devices in this domain */ + struct iommu_domain domain; /* IOMMU domain handle */ + unsigned long *pt_root; /* Page Table root */ + unsigned int dev_cnt; /* Number of devices in this domain */ +}; + +struct sw64dev_table_entry { + u64 data; +}; + +struct sunway_iommu_group { + struct pci_dev *dev; + struct iommu_group *group; +}; + +#define SW64_IOMMU_ENTRY_VALID ((1UL) << 63) +#define SW64_PTE_LAST_MASK ((1UL) << 8) /*last stage valid*/ +#define SW64_DMA_START 0x1000000 +#define SW64_PTE_GRN_MASK ((0x3UL) << 4) +#define PAGE_8M_SHIFT 23 +#define PAGE_512M_SHIFT 29 +#define PAGE_8G_SHIFT 33 +#define SW64_IOMMU_ENABLE 3 +#define SW64_IOMMU_DISABLE 0 +#define SW64_IOMMU_LEVEL1_OFFSET 0x1ff +#define SW64_IOMMU_LEVEL2_OFFSET 0x3ff +#define SW64_IOMMU_LEVEL3_OFFSET 0x3ff +#define SW64_IOMMU_BYPASS 0x1 +#define SW64_IOMMU_MAP_FLAG ((0x1UL) << 20) + +#define PAGE_SHIFT_IOMMU 18 +#define PAGE_SIZE_IOMMU (_AC(1, UL) << PAGE_SHIFT_IOMMU) + +#define PCACHE_FLUSHPADDR_MASK 0xffffffffff80UL -- Gitee From 06800836a86776ae442d57bf721907842b2bc915 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:48:45 +0800 Subject: [PATCH 074/524] drivers: irqchip: add sw64 support Add irqchip drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/Kconfig | 33 ++ drivers/irqchip/Makefile | 11 + drivers/irqchip/irq-sunway-cpu.c | 213 ++++++++++++ drivers/irqchip/irq-sunway-msi-v2.c | 512 ++++++++++++++++++++++++++++ drivers/irqchip/irq-sunway-msi-vt.c | 280 +++++++++++++++ drivers/irqchip/irq-sunway-msi.c | 472 +++++++++++++++++++++++++ drivers/irqchip/irq-sw64-intc-v2.c | 89 +++++ drivers/irqchip/irq-sw64-lpc-intc.c | 137 ++++++++ 8 files changed, 1747 insertions(+) create mode 100644 drivers/irqchip/irq-sunway-cpu.c create mode 100644 drivers/irqchip/irq-sunway-msi-v2.c create mode 100644 drivers/irqchip/irq-sunway-msi-vt.c create mode 100644 drivers/irqchip/irq-sunway-msi.c create mode 100644 drivers/irqchip/irq-sw64-intc-v2.c create mode 100644 drivers/irqchip/irq-sw64-lpc-intc.c diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 5b85f1bc94b7..027df575d57f 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -11,6 +11,39 @@ config ARM_GIC select IRQ_DOMAIN_HIERARCHY select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP +config SW64_INTC_V2 + bool "SW64 Interrupt Controller V2" + depends on UNCORE_XUELANG + default y + select GENERIC_IRQ_CHIP + select IRQ_DOMAIN + help + This enables support for the INTC chip found in SW CHIP3 systems. + The INTC controls devices interrupts and connects them to each + core's local interrupt controller. + +config SW64_LPC_INTC + bool "SW64 cpu builtin LPC Interrupt Controller" + depends on SW64_INTC_V2 + help + Say yes here to add support for the SW64 cpu builtin LPC + IRQ controller. + +config SW64_IRQ_CPU + bool + depends on SW64 + default y + +config SW64_IRQ_MSI + bool + depends on SW64 && PCI_MSI + default y + +config SW64_IRQ_MSI_VT + bool + depends on SW64_IRQ_MSI + default y + config ARM_GIC_PM bool depends on PM diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 9184ba8ce53d..787206e166fc 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -27,6 +27,17 @@ obj-$(CONFIG_SUN6I_R_INTC) += irq-sun6i-r.o obj-$(CONFIG_SUNXI_NMI_INTC) += irq-sunxi-nmi.o obj-$(CONFIG_ARCH_SPEAR3XX) += spear-shirq.o obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o +obj-$(CONFIG_SW64_INTC_V2) += irq-sw64-intc-v2.o +obj-$(CONFIG_SW64_LPC_INTC) += irq-sw64-lpc-intc.o +obj-$(CONFIG_SW64_IRQ_CPU) += irq-sunway-cpu.o + +ifeq ($(CONFIG_UNCORE_XUELANG),y) +obj-$(CONFIG_SW64_IRQ_MSI) += irq-sunway-msi.o +else +obj-$(CONFIG_SW64_IRQ_MSI) += irq-sunway-msi-v2.o +endif + +obj-$(CONFIG_SW64_IRQ_MSI_VT) += irq-sunway-msi-vt.o obj-$(CONFIG_ARM_GIC_PM) += irq-gic-pm.o obj-$(CONFIG_ARCH_REALVIEW) += irq-gic-realview.o obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c new file mode 100644 index 000000000000..ff7455c0f3ec --- /dev/null +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include +#include +#include + +static void handle_intx(unsigned int offset) +{ + struct pci_controller *hose; + unsigned long value; + + hose = hose_head; + for (hose = hose_head; hose; hose = hose->next) { + value = read_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7)); + if (value >> 63) { + value = value & (~(1UL << 62)); + write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); + handle_irq(hose->int_irq); + value = value | (1UL << 62); + write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); + } + + if (IS_ENABLED(CONFIG_PCIE_PME)) { + value = read_piu_ior0(hose->node, hose->index, PMEINTCONFIG); + if (value >> 63) { + handle_irq(hose->service_irq); + write_piu_ior0(hose->node, hose->index, PMEINTCONFIG, value); + } + } + + if (IS_ENABLED(CONFIG_PCIEAER)) { + value = read_piu_ior0(hose->node, hose->index, AERERRINTCONFIG); + if (value >> 63) { + handle_irq(hose->service_irq); + write_piu_ior0(hose->node, hose->index, AERERRINTCONFIG, value); + } + } + + if (hose->iommu_enable) { + value = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); + if (value >> 63) + handle_irq(hose->int_irq); + } + } +} + +static void handle_device_interrupt(unsigned long irq_info) +{ + unsigned int i; + + if (is_guest_or_emul()) { + handle_irq(irq_info); + return; + } + + for (i = 0; i < 4; i++) { + if ((irq_info >> i) & 0x1) + handle_intx(i); + } +} + +/* Performance counter hook. A module can override this to do something useful. */ +static void dummy_perf(unsigned long vector, struct pt_regs *regs) +{ + irq_err_count++; + pr_crit("Performance counter interrupt!\n"); +} + +void (*perf_irq)(unsigned long vector, struct pt_regs *regs) = dummy_perf; +EXPORT_SYMBOL(perf_irq); + +static void handle_fault_int(void) +{ + int node; + unsigned long value; + + node = __this_cpu_read(hard_node_id); + pr_info("enter fault int, si_fault_stat = %#lx\n", + sw64_io_read(node, SI_FAULT_STAT)); + sw64_io_write(node, SI_FAULT_INT_EN, 0); + sw64_io_write(node, DLI_RLTD_FAULT_INTEN, 0); +#if defined(CONFIG_UNCORE_XUELANG) + value = 0; +#elif defined(CONFIG_UNCORE_JUNZHANG) + value = sw64_io_read(node, FAULT_INT_CONFIG); + value |= (1 << 8); +#endif + __io_write_fault_int_en(node, value); +} + +static void handle_mt_int(void) +{ + pr_info("enter mt int\n"); +} + +static void handle_nmi_int(void) +{ + pr_info("enter nmi int\n"); +} + +static void handle_dev_int(struct pt_regs *regs) +{ + unsigned long config_val, val, stat; + int node = 0; + unsigned int hwirq; + + config_val = sw64_io_read(node, DEV_INT_CONFIG); + val = config_val & (~(1UL << 8)); + sw64_io_write(node, DEV_INT_CONFIG, val); + stat = sw64_io_read(node, MCU_DVC_INT); + + while (stat) { + hwirq = ffs(stat) - 1; + generic_handle_domain_irq(NULL, hwirq); + stat &= ~(1UL << hwirq); + } + /*do handle irq */ + + sw64_io_write(node, DEV_INT_CONFIG, config_val); +} + +asmlinkage void do_entInt(unsigned long type, unsigned long vector, + unsigned long irq_arg, struct pt_regs *regs) +{ + struct pt_regs *old_regs; + extern char __idle_start[], __idle_end[]; + + if (is_guest_or_emul()) { + if ((type & 0xffff) > 15) { + vector = type; + if (vector == 16) + type = INT_INTx; + else + type = INT_MSI; + } + } + + /* restart idle routine if it is interrupted */ + if (regs->pc > (u64)__idle_start && regs->pc < (u64)__idle_end) + regs->pc = (u64)__idle_start; + + switch (type & 0xffff) { + case INT_MSI: + old_regs = set_irq_regs(regs); + handle_pci_msi_interrupt(type, vector, irq_arg); + set_irq_regs(old_regs); + return; + case INT_INTx: + old_regs = set_irq_regs(regs); + handle_device_interrupt(vector); + set_irq_regs(old_regs); + return; + + case INT_IPI: +#ifdef CONFIG_SMP + handle_ipi(regs); + return; +#else + irq_err_count++; + pr_crit("Interprocessor interrupt? You must be kidding!\n"); +#endif + break; + case INT_RTC: + old_regs = set_irq_regs(regs); + sw64_timer_interrupt(); + set_irq_regs(old_regs); + return; + case INT_VT_SERIAL: + old_regs = set_irq_regs(regs); + handle_irq(type); + set_irq_regs(old_regs); + return; + case INT_VT_HOTPLUG: + old_regs = set_irq_regs(regs); + handle_irq(type); + set_irq_regs(old_regs); + return; + case INT_PC0: + perf_irq(PMC_PC0, regs); + return; + case INT_PC1: + perf_irq(PMC_PC1, regs); + return; + case INT_DEV: + old_regs = set_irq_regs(regs); + handle_dev_int(regs); + set_irq_regs(old_regs); + return; + case INT_FAULT: + old_regs = set_irq_regs(regs); + handle_fault_int(); + set_irq_regs(old_regs); + return; + case INT_MT: + old_regs = set_irq_regs(regs); + handle_mt_int(); + set_irq_regs(old_regs); + return; + case INT_NMI: + old_regs = set_irq_regs(regs); + handle_nmi_int(); + set_irq_regs(old_regs); + return; + default: + pr_crit("Hardware intr %ld %lx? uh?\n", type, vector); + } + pr_crit("PC = %016lx PS = %04lx\n", regs->pc, regs->ps); +} +EXPORT_SYMBOL(do_entInt); diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c new file mode 100644 index 000000000000..36790dfedb33 --- /dev/null +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +#include +#include + +static struct irq_domain *msi_default_domain; +static DEFINE_RAW_SPINLOCK(vector_lock); +DEFINE_PER_CPU(vector_irq_t, vector_irq) = { + [0 ... PERCPU_MSI_IRQS - 1] = 0, +}; + +static struct sw64_msi_chip_data *alloc_sw_msi_chip_data(struct irq_data *irq_data) +{ + struct sw64_msi_chip_data *data; + int node; + + node = irq_data_get_node(irq_data); + data = kzalloc_node(sizeof(*data), GFP_KERNEL, node); + if (!data) + return NULL; + spin_lock_init(&data->cdata_lock); + return data; +} + +static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct sw64_msi_chip_data *chip_data; + int rcid; + + chip_data = irq_data_get_irq_chip_data(data->parent_data); + rcid = cpu_to_rcid(chip_data->dst_cpu); + msg->address_hi = MSI_ADDR_BASE_HI; + msg->address_lo = + (unsigned int)chip_data->msiaddr | + (rcid_to_msicid(rcid) << MSI_ADDR_DEST_ID_SHIFT); + msg->data = chip_data->vector; +} + +bool find_free_cpu_vector(const struct cpumask *search_mask, + int *found_cpu, int *found_vector) +{ + int vector, max_vector, cpu; + bool find_once_global = false; + + cpu = cpumask_first(search_mask); +try_again: + if (is_guest_or_emul()) { + vector = IRQ_PENDING_MSI_VECTORS_SHIFT; + max_vector = SWVM_IRQS; + } else { + vector = 0; + max_vector = 256; + } + for (; vector < max_vector; vector++) { + while (per_cpu(vector_irq, cpu)[vector]) { + cpu = cpumask_next(cpu, search_mask); + if (cpu >= nr_cpu_ids) { + if (vector == 255) { + if (find_once_global) { + pr_warn("No global free vector\n"); + return false; + } + pr_warn("No local free vector\n"); + search_mask = cpu_online_mask; + cpu = cpumask_first(search_mask); + find_once_global = true; + goto try_again; + } + cpu = cpumask_first(search_mask); + break; + } + } + if (!per_cpu(vector_irq, cpu)[vector]) + break; + } + + *found_cpu = cpu; + *found_vector = vector; + return true; +} + +static bool find_free_cpu_vectors(const struct cpumask *search_mask, int *found_cpu, int *found_vector, unsigned int nr_irqs) +{ + int i, vector, cpu; + bool found = false, find_once_global = false; + + cpu = cpumask_first(search_mask); +try_again: + for (vector = 0; vector < 256; vector++) { + for (i = 0; i < nr_irqs; i++) + if (per_cpu(vector_irq, cpu)[vector + i]) + break; + + if (i == nr_irqs) { + found = true; + *found_cpu = cpu; + *found_vector = vector; + return found; + } + + vector += i; + } + + cpu = cpumask_next(cpu, search_mask); + if (cpu < nr_cpu_ids) + goto try_again; + else { + if (find_once_global) { + pr_warn("No global free vectors\n"); + return found; + } + pr_warn("No local free vectors\n"); + search_mask = cpu_online_mask; + cpu = cpumask_first(search_mask); + find_once_global = true; + goto try_again; + } +} + +static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, bool force) +{ + struct sw64_msi_chip_data *cdata; + struct irq_data *irqd; + struct msi_desc *entry; + struct cpumask searchmask; + unsigned long flags; + int vector, cpu; + int i; + struct msi_msg msg; + + /* Is this valid ? */ + if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) + return -EINVAL; + + irqd = irq_domain_get_irq_data(msi_default_domain->parent, d->irq); + /* Don't do anything if the interrupt isn't started */ + if (!irqd_is_started(irqd)) + return IRQ_SET_MASK_OK; + + cdata = irqd->chip_data; + if (!cdata) + return -ENOMEM; + + /* + * If existing target cpu is already in the new mask and is online + * then do nothing. + */ + if (cpu_online(cdata->dst_cpu) && cpumask_test_cpu(cdata->dst_cpu, cpumask)) + return IRQ_SET_MASK_OK; + + raw_spin_lock_irqsave(&vector_lock, flags); + + cpumask_and(&searchmask, cpumask, cpu_online_mask); + if (cdata->multi_msi > 1) { + if (!find_free_cpu_vectors(&searchmask, &cpu, + &vector, cdata->multi_msi)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); + return -ENOSPC; + } + } else { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); + return -ENOSPC; + } + } + + /* update new setting */ + entry = irq_get_msi_desc(irqd->irq); + spin_lock(&cdata->cdata_lock); + for (i = 0; i < cdata->multi_msi; i++) + per_cpu(vector_irq, cpu)[vector + i] = entry->irq + i; + BUG_ON(irq_chip_compose_msi_msg(irqd, &msg)); + __pci_write_msi_msg(entry, &msg); + cdata->prev_vector = cdata->vector; + cdata->prev_cpu = cdata->dst_cpu; + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->move_in_progress = true; + spin_unlock(&cdata->cdata_lock); + cpumask_copy(irq_data_get_affinity_mask(irqd), &searchmask); + + raw_spin_unlock_irqrestore(&vector_lock, flags); + + return 0; +} + +static void chip_irq_ack(struct irq_data *data) +{ +} + +static struct irq_chip pci_msi_controller = { + .name = "PCI-MSI", + .irq_unmask = pci_msi_unmask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_ack = chip_irq_ack, + .irq_compose_msi_msg = irq_msi_compose_msg, + .flags = IRQCHIP_SKIP_SET_WAKE, + .irq_set_affinity = sw64_set_affinity, +}; + +static int __assign_irq_vector(int virq, unsigned int nr_irqs, + struct irq_domain *domain, enum irq_alloc_type type) +{ + struct irq_data *irq_data; + const struct cpumask *mask; + struct cpumask searchmask; + struct sw64_msi_chip_data *cdata; + int node; + int i, vector, cpu; + unsigned long msiaddr; + + if (unlikely((nr_irqs > 1) && (!is_power_of_2(nr_irqs)))) + nr_irqs = __roundup_pow_of_two(nr_irqs); + + irq_data = irq_domain_get_irq_data(domain, virq); + BUG_ON(!irq_data); + irq_data->chip = &pci_msi_controller; + + if (irqd_affinity_is_managed(irq_data)) { + mask = irq_data_get_affinity_mask(irq_data); + cpumask_and(&searchmask, mask, cpu_online_mask); + } else { + node = irq_data_get_node(irq_data); + cpumask_copy(&searchmask, cpumask_of_node(node)); + } + + if (cpumask_first(&searchmask) >= nr_cpu_ids) + cpumask_copy(&searchmask, cpu_online_mask); + + if (type == IRQ_ALLOC_TYPE_MSI && nr_irqs > 1) { + if (!find_free_cpu_vectors(&searchmask, &cpu, + &vector, nr_irqs)) + return -ENOSPC; + + cdata = alloc_sw_msi_chip_data(irq_data); + if (!cdata) { + pr_warn("error alloc irq chip data\n"); + return -ENOMEM; + } + + for (i = 0; i < nr_irqs; i++) { + per_cpu(vector_irq, cpu)[vector + i] = virq + i; + + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &pci_msi_controller; + } + + irq_data->chip_data = cdata; + } + + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->msiaddr = MSIX_MSG_ADDR; + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + cdata->multi_msi = nr_irqs; + cdata->move_in_progress = false; + } else { + for (i = 0; i < nr_irqs; i++) { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + return -ENOSPC; + + per_cpu(vector_irq, cpu)[vector] = virq + i; + + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &pci_msi_controller; + } + + cdata = alloc_sw_msi_chip_data(irq_data); + if (!cdata) { + pr_warn("error alloc irq chip data\n"); + return -ENOMEM; + } + + irq_data->chip_data = cdata; + + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->msiaddr = MSIX_MSG_ADDR; + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + cdata->multi_msi = 1; + cdata->move_in_progress = false; + } + } + return 0; +} + +static int assign_irq_vector(int irq, unsigned int nr_irqs, + struct irq_domain *domain, enum irq_alloc_type type) +{ + int err; + unsigned long flags; + + raw_spin_lock_irqsave(&vector_lock, flags); + err = __assign_irq_vector(irq, nr_irqs, domain, type); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return err; +} + +static void sw64_vector_free_irqs(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + int i, j; + struct irq_data *irq_data; + unsigned long flags; + unsigned int multi_msi; + + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + if (irq_data && irq_data->chip_data) { + struct sw64_msi_chip_data *cdata; + + raw_spin_lock_irqsave(&vector_lock, flags); + cdata = irq_data->chip_data; + irq_domain_reset_irq_data(irq_data); + multi_msi = cdata->multi_msi; + for (j = 0; j < multi_msi; j++) + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector + j] = 0; + kfree(cdata); + raw_spin_unlock_irqrestore(&vector_lock, flags); + if (multi_msi > 1) + break; + } + } +} + +static void sw64_irq_free_descs(unsigned int virq, unsigned int nr_irqs) +{ + if (is_guest_or_emul()) { + vt_sw64_vector_free_irqs(virq, nr_irqs); + return irq_free_descs(virq, nr_irqs); + } + + return irq_domain_free_irqs(virq, nr_irqs); +} + +void arch_teardown_msi_irqs(struct pci_dev *dev) +{ + struct msi_desc *desc; + int i; + + for_each_pci_msi_entry(desc, dev) { + if (desc->irq) { + for (i = 0; i < desc->nvec_used; i++) + sw64_irq_free_descs(desc->irq + i, 1); + desc->irq = 0; + } + } +} + +static int sw64_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + int err; + struct irq_alloc_info *info = arg; + enum irq_alloc_type msi_type; + + if (arg == NULL) + return -ENODEV; + msi_type = info->type; + err = assign_irq_vector(virq, nr_irqs, domain, msi_type); + if (err) + goto error; + return 0; +error: + sw64_vector_free_irqs(domain, virq, nr_irqs); + return err; +} + +static int pci_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *arg) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct msi_desc *desc = first_pci_msi_entry(pdev); + + memset(arg, 0, sizeof(*arg)); + arg->msi_dev = pdev; + if (desc->msi_attrib.is_msix) + arg->type = IRQ_ALLOC_TYPE_MSIX; + else + arg->type = IRQ_ALLOC_TYPE_MSI; + return 0; +} + +static struct msi_domain_ops pci_msi_domain_ops = { + .msi_prepare = pci_msi_prepare, +}; + +static struct msi_domain_info pci_msi_domain_info = { + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, + .ops = &pci_msi_domain_ops, + .chip = &pci_msi_controller, + .handler = handle_edge_irq, + .handler_name = "edge", +}; + +static int sw64_irq_map(struct irq_domain *d, unsigned int virq, irq_hw_number_t hw) +{ + irq_set_chip_and_handler(virq, &sw64_irq_chip, handle_level_irq); + irq_set_status_flags(virq, IRQ_LEVEL); + return 0; +} + +const struct irq_domain_ops sw64_msi_domain_ops = { + .map = sw64_irq_map, + .alloc = sw64_vector_alloc_irqs, + .free = sw64_vector_free_irqs, +}; + +int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) +{ + int err; + + if (is_guest_or_emul()) + return sw64_setup_vt_msi_irqs(pdev, nvec, type); + + if (!msi_default_domain) + return -EIO; + err = msi_domain_alloc_irqs(msi_default_domain, &pdev->dev, nvec); + return err; +} + +void arch_init_msi_domain(struct irq_domain *parent) +{ + struct irq_domain *sw64_irq_domain; + + if (is_guest_or_emul()) + return; + + sw64_irq_domain = irq_domain_add_tree(NULL, &sw64_msi_domain_ops, NULL); + BUG_ON(sw64_irq_domain == NULL); + irq_set_default_host(sw64_irq_domain); + msi_default_domain = pci_msi_create_irq_domain(NULL, + &pci_msi_domain_info, sw64_irq_domain); + if (!msi_default_domain) + pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n"); +} + +static void irq_move_complete(struct sw64_msi_chip_data *cdata, int cpu, int vector) +{ + if (likely(!cdata->move_in_progress)) + return; + if (cdata->dst_cpu == cpu) { + if (vector >= cdata->vector && + vector < cdata->vector + cdata->multi_msi) { + int i; + + raw_spin_lock(&vector_lock); + cdata->move_in_progress = false; + for (i = 0; i < cdata->multi_msi; i++) + per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector + i] = 0; + raw_spin_unlock(&vector_lock); + } + } +} + +void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned long pci_msi1_addr) +{ + int i, irq, msi_index = 0; + int cpu, vector_index = 0; + unsigned long int_pci_msi[3]; + unsigned long *ptr; + struct irq_data *irq_data; + struct sw64_msi_chip_data *cdata; + + if (is_guest_or_emul()) { + cpu = smp_processor_id(); + irq = per_cpu(vector_irq, cpu)[vector]; + handle_irq(irq); + return; + } + + ptr = (unsigned long *)pci_msi1_addr; + int_pci_msi[0] = *ptr; + int_pci_msi[1] = *(ptr + 1); + int_pci_msi[2] = *(ptr + 2); + + cpu = smp_processor_id(); + + for (i = 0; i < 4; i++) { + vector_index = i * 64; + while (vector != 0) { + int irq = 0; + + msi_index = find_next_bit(&vector, 64, msi_index); + if (msi_index == 64) { + msi_index = 0; + continue; + } + + irq = per_cpu(vector_irq, cpu)[vector_index + msi_index]; + irq_data = irq_domain_get_irq_data(msi_default_domain->parent, irq); + cdata = irq_data_get_irq_chip_data(irq_data); + spin_lock(&cdata->cdata_lock); + irq_move_complete(cdata, cpu, vector_index + msi_index); + spin_unlock(&cdata->cdata_lock); + handle_irq(irq); + + vector = vector & (~(1UL << msi_index)); + } + + vector = int_pci_msi[i % 3]; + } +} diff --git a/drivers/irqchip/irq-sunway-msi-vt.c b/drivers/irqchip/irq-sunway-msi-vt.c new file mode 100644 index 000000000000..df8c7d72671b --- /dev/null +++ b/drivers/irqchip/irq-sunway-msi-vt.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +static DEFINE_RAW_SPINLOCK(vector_lock); + +static void __vt_irq_msi_compose_msg(struct sw64_msi_chip_data *cdata, + struct msi_msg *msg) +{ + msg->address_hi = (u32)(VT_MSIX_MSG_ADDR >> 32); + msg->address_lo = (u32)(VT_MSIX_MSG_ADDR & 0xffffffff) + | VT_MSIX_ADDR_DEST_ID(cdata->dst_cpu); + msg->data = cdata->vector; +} + +static void vt_irq_msi_compose_msg(struct irq_data *irqd, struct msi_msg *msg) +{ + struct sw64_msi_chip_data *cdata; + + cdata = irqd->chip_data; + __vt_irq_msi_compose_msg(cdata, msg); +} + +static void vt_irq_msi_update_msg(struct irq_data *irqd, + struct sw64_msi_chip_data *cdata) +{ + struct msi_msg msg[2] = { [1] = { }, }; + + __vt_irq_msi_compose_msg(cdata, msg); + pci_write_msi_msg(irqd->irq, msg); +} + +static int +vt_set_affinity(struct irq_data *irqd, const struct cpumask *cpumask, + bool force) +{ + struct sw64_msi_chip_data *cdata; + struct cpumask searchmask; + int cpu, vector; + + /* Is this valid ? */ + if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) + return -EINVAL; + + if (!irqd_is_started(irqd)) + return IRQ_SET_MASK_OK; + + cdata = irqd->chip_data; + if (!cdata) + return -ENOMEM; + + /* + * If existing target coreid is already in the new mask, + * and is online then do nothing. + */ + if (cpu_online(cdata->dst_cpu) && cpumask_test_cpu(cdata->dst_cpu, cpumask)) + return IRQ_SET_MASK_OK; + + cpumask_and(&searchmask, cpumask, cpu_online_mask); + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + return -ENOSPC; + + per_cpu(vector_irq, cpu)[vector] = irqd->irq; + spin_lock(&cdata->cdata_lock); + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->prev_cpu = cdata->dst_cpu; + cdata->prev_vector = cdata->vector; + cdata->move_in_progress = true; + spin_unlock(&cdata->cdata_lock); + cpumask_copy((struct cpumask *)irq_data_get_affinity_mask(irqd), &searchmask); + vt_irq_msi_update_msg(irqd, irqd->chip_data); + + return 0; +} + +static struct irq_chip vt_pci_msi_controller = { + .name = "PCI-MSI", + .irq_unmask = pci_msi_unmask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_ack = sw64_irq_noop, + .irq_compose_msi_msg = vt_irq_msi_compose_msg, + .irq_set_affinity = vt_set_affinity, +}; + +int chip_setup_vt_msix_irq(struct pci_dev *dev, struct msi_desc *desc) +{ + int virq, val_node = 0; + struct irq_data *irq_data; + struct sw64_msi_chip_data *cdata; + struct pci_controller *hose = pci_bus_to_pci_controller(dev->bus); + unsigned long flags, node, rc_index; + const struct cpumask *mask; + + struct cpumask searchmask; + int cpu, vector; + + node = hose->node; + rc_index = hose->index; + mask = cpumask_of_node(node); + + raw_spin_lock_irqsave(&vector_lock, flags); + /* Find unused msi config reg in PIU-IOR0 */ + if (!node_online(node)) + val_node = next_node_in(node, node_online_map); + else + val_node = node; + + virq = irq_alloc_descs_from(NR_IRQS_LEGACY, desc->nvec_used, val_node); + if (virq < 0) { + pr_err("Failed to allocate IRQ(base 16, count %d)\n", desc->nvec_used); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return virq; + } + + irq_data = irq_get_irq_data(virq); + + if (irqd_affinity_is_managed(irq_data)) { + mask = irq_data_get_affinity_mask(irq_data); + cpumask_and(&searchmask, mask, cpu_online_mask); + } else { + node = irq_data_get_node(irq_data); + cpumask_copy(&searchmask, cpumask_of_node(node)); + } + if (cpumask_first(&searchmask) >= nr_cpu_ids) + cpumask_copy(&searchmask, cpu_online_mask); + + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + return -ENOSPC; + + cdata = kzalloc(sizeof(*cdata), GFP_KERNEL); + if (!cdata) + return -ENOMEM; + + per_cpu(vector_irq, cpu)[vector] = virq; + + irq_set_msi_desc(virq, desc); + irq_set_chip_and_handler_name(virq, &vt_pci_msi_controller, + handle_edge_irq, "edge"); + + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->rc_index = hose->index; + cdata->rc_node = hose->node; + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + + irq_data->chip_data = cdata; + + vt_irq_msi_update_msg(irq_data, irq_data->chip_data); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return 0; +} +EXPORT_SYMBOL(chip_setup_vt_msix_irq); + +int chip_setup_vt_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + struct msi_desc *desc; + struct pci_controller *hose = pci_bus_to_pci_controller(dev->bus); + struct irq_data *irq_data; + struct sw64_msi_chip_data *cdata; + unsigned long node, rc_index; + int virq = -1, val_node = 0; + unsigned long flags; + + const struct cpumask *mask; + struct cpumask searchmask; + int i, vector, cpu; + + if (type == PCI_CAP_ID_MSI && nvec > 32) + return 1; + + node = hose->node; + rc_index = hose->index; + raw_spin_lock_irqsave(&vector_lock, flags); + msi_for_each_desc(desc, &(dev->dev), MSI_DESC_ALL) { + /* Find unused msi config reg in PIU-IOR0 */ + if (!node_online(node)) + val_node = next_node_in(node, node_online_map); + else + val_node = node; + virq = irq_alloc_descs_from(NR_IRQS_LEGACY, desc->nvec_used, val_node); + if (virq < 0) { + pr_err("Failed to allocate IRQ(base 16, count %d)\n", desc->nvec_used); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return virq; + } + + irq_data = irq_get_irq_data(virq); + if (irqd_affinity_is_managed(irq_data)) { + mask = irq_data_get_affinity_mask(irq_data); + cpumask_and(&searchmask, mask, cpu_online_mask); + } else { + node = irq_data_get_node(irq_data); + cpumask_copy(&searchmask, cpumask_of_node(node)); + } + if (cpumask_first(&searchmask) >= nr_cpu_ids) + cpumask_copy(&searchmask, cpu_online_mask); + + for (i = 0; i < desc->nvec_used; i++) { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + return -ENOSPC; + + cdata = kzalloc(sizeof(*cdata), GFP_KERNEL); + if (!cdata) + return -ENOMEM; + + per_cpu(vector_irq, cpu)[vector] = virq + i; + irq_set_msi_desc_off(virq, i, desc); + irq_set_chip_and_handler_name(virq + i, &vt_pci_msi_controller, handle_edge_irq, "edge"); + irq_data = irq_get_irq_data(virq + i); + + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->rc_index = hose->index; + cdata->rc_node = hose->node; + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + + irq_data->chip_data = cdata; + + vt_irq_msi_update_msg(irq_data, irq_data->chip_data); + } + } + + raw_spin_unlock_irqrestore(&vector_lock, flags); + return 0; +} +EXPORT_SYMBOL(chip_setup_vt_msi_irqs); + +void vt_sw64_vector_free_irqs(unsigned int virq, unsigned int nr_irqs) +{ + int i; + unsigned long flags; + struct irq_data *irq_data; + struct sw64_msi_chip_data *cdata; + + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_get_irq_data(virq + i); + if (irq_data && irq_data->chip_data) { + raw_spin_lock_irqsave(&vector_lock, flags); + cdata = irq_data->chip_data; + irq_data->hwirq = 0; + irq_data->chip = &no_irq_chip; + irq_data->chip_data = NULL; + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; + kfree(cdata); + raw_spin_unlock_irqrestore(&vector_lock, flags); + } + } +} + +int __arch_setup_vt_msix_irqs(struct pci_dev *dev, int nvec, int type) +{ + struct msi_desc *entry; + int ret; + + msi_for_each_desc(entry, &dev->dev, MSI_DESC_ALL) { + ret = chip_setup_vt_msix_irq(dev, entry); + if (ret) + return ret; + } + + return 0; +} + +int sw64_setup_vt_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + int ret = 0; + + if (type == PCI_CAP_ID_MSI) + ret = chip_setup_vt_msi_irqs(dev, nvec, type); + else if (type == PCI_CAP_ID_MSIX) + ret = __arch_setup_vt_msix_irqs(dev, nvec, type); + else + pr_info("SW arch do not identify ID:%d\n", type); + + return ret; +} diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c new file mode 100644 index 000000000000..060aa96711b7 --- /dev/null +++ b/drivers/irqchip/irq-sunway-msi.c @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#include +#include + +static struct irq_domain *msi_default_domain; +static DEFINE_RAW_SPINLOCK(vector_lock); +DEFINE_PER_CPU(vector_irq_t, vector_irq) = { + [0 ... PERCPU_MSI_IRQS - 1] = 0, +}; + +static struct sw64_msi_chip_data *alloc_sw_msi_chip_data(struct irq_data *irq_data) +{ + struct sw64_msi_chip_data *data; + int node; + + node = irq_data_get_node(irq_data); + data = kzalloc_node(sizeof(*data), GFP_KERNEL, node); + if (!data) + return NULL; + spin_lock_init(&data->cdata_lock); + return data; +} + +static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct sw64_msi_chip_data *chip_data; + + chip_data = irq_data_get_irq_chip_data(data->parent_data); + + msg->address_hi = MSI_ADDR_BASE_HI; + msg->address_lo = MSI_ADDR_BASE_LO; + msg->data = chip_data->msi_config_index; +} + +bool find_free_cpu_vector(const struct cpumask *search_mask, + int *found_cpu, int *found_vector) +{ + int vector, max_vector, cpu; + bool find_once_global = false; + + cpu = cpumask_first(search_mask); +try_again: + if (is_guest_or_emul()) { + vector = IRQ_PENDING_MSI_VECTORS_SHIFT; + max_vector = SWVM_IRQS; + } else { + vector = 0; + max_vector = 256; + } + for (; vector < max_vector; vector++) { + while (per_cpu(vector_irq, cpu)[vector]) { + cpu = cpumask_next(cpu, search_mask); + if (cpu >= nr_cpu_ids) { + if (vector == 255) { + if (find_once_global) { + pr_warn("No global free vector\n"); + return false; + } + pr_warn("No local free vector\n"); + search_mask = cpu_online_mask; + cpu = cpumask_first(search_mask); + find_once_global = true; + goto try_again; + } + cpu = cpumask_first(search_mask); + break; + } + } + if (!per_cpu(vector_irq, cpu)[vector]) + break; + } + + *found_cpu = cpu; + *found_vector = vector; + return true; +} + +static unsigned long set_piu_msi_config(struct pci_controller *hose, int cpu, + int msiconf_index, int vector) +{ + unsigned int reg; + unsigned long msi_config; + int phy_cpu; + + msi_config = (1UL << 62) | ((unsigned long)vector << 10); + phy_cpu = cpu_to_rcid(cpu); + msi_config |= ((phy_cpu >> 5) << 6) | (phy_cpu & 0x1f); + reg = MSICONFIG0 + (unsigned long)(msiconf_index << 7); + write_piu_ior0(hose->node, hose->index, reg, msi_config); + msi_config = read_piu_ior0(hose->node, hose->index, reg); + set_bit(msiconf_index, hose->piu_msiconfig); + + return msi_config; +} + +static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, bool force) +{ + struct sw64_msi_chip_data *cdata; + struct pci_controller *hose; + struct pci_dev *pdev; + struct irq_data *irqd; + struct msi_desc *entry; + struct cpumask searchmask; + unsigned long flags, msi_config; + int vector, cpu; + + /* Is this valid ? */ + if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) + return -EINVAL; + + irqd = irq_domain_get_irq_data(msi_default_domain->parent, d->irq); + /* Don't do anything if the interrupt isn't started */ + if (!irqd_is_started(irqd)) + return IRQ_SET_MASK_OK; + + cdata = irqd->chip_data; + if (!cdata) + return -ENOMEM; + + /* + * If existing target cpu is already in the new mask and is online + * then do nothing. + */ + if (cpu_online(cdata->dst_cpu) && cpumask_test_cpu(cdata->dst_cpu, cpumask)) + return IRQ_SET_MASK_OK; + + raw_spin_lock_irqsave(&vector_lock, flags); + + cpumask_and(&searchmask, cpumask, cpu_online_mask); + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); + return -ENOSPC; + } + + /* update new setting */ + entry = irq_get_msi_desc(irqd->irq); + pdev = (struct pci_dev *)msi_desc_to_pci_dev(entry); + hose = pci_bus_to_pci_controller(pdev->bus); + spin_lock(&cdata->cdata_lock); + per_cpu(vector_irq, cpu)[vector] = irqd->irq; + msi_config = set_piu_msi_config(hose, cpu, cdata->msi_config_index, vector); + cdata->prev_vector = cdata->vector; + cdata->prev_cpu = cdata->dst_cpu; + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->msi_config = msi_config; + cdata->move_in_progress = true; + spin_unlock(&cdata->cdata_lock); + cpumask_copy((struct cpumask *)irq_data_get_affinity_mask(irqd), &searchmask); + + raw_spin_unlock_irqrestore(&vector_lock, flags); + + return 0; +} + +static void chip_irq_ack(struct irq_data *data) +{ +} + +static struct irq_chip pci_msi_controller = { + .name = "PCI-MSI", + .irq_unmask = pci_msi_unmask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_ack = chip_irq_ack, + .irq_compose_msi_msg = irq_msi_compose_msg, + .flags = IRQCHIP_SKIP_SET_WAKE, + .irq_set_affinity = sw64_set_affinity, +}; + +static int __assign_irq_vector(int virq, unsigned int nr_irqs, + struct irq_domain *domain, struct pci_controller *hose) +{ + struct irq_data *irq_data; + const struct cpumask *mask; + struct cpumask searchmask; + struct sw64_msi_chip_data *cdata; + int msiconf_index, node; + int i, vector, cpu; + unsigned long msi_config; + int start_index; + + if (unlikely((nr_irqs > 1) && (!is_power_of_2(nr_irqs)))) + nr_irqs = __roundup_pow_of_two(nr_irqs); + + msiconf_index = bitmap_find_next_zero_area(hose->piu_msiconfig, 256, 0, + nr_irqs, nr_irqs - 1); + + if (msiconf_index >= 256) { + pr_warn("No free msi on PIU!\n"); + return -ENOSPC; + } + + start_index = msiconf_index; + irq_data = irq_domain_get_irq_data(domain, virq); + BUG_ON(!irq_data); + irq_data->chip = &pci_msi_controller; + + if (irqd_affinity_is_managed(irq_data)) { + mask = irq_data_get_affinity_mask(irq_data); + cpumask_and(&searchmask, mask, cpu_online_mask); + } else { + node = irq_data_get_node(irq_data); + cpumask_copy(&searchmask, cpumask_of_node(node)); + } + + if (cpumask_first(&searchmask) >= nr_cpu_ids) + cpumask_copy(&searchmask, cpu_online_mask); + + for (i = 0; i < nr_irqs; i++) { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + return -ENOSPC; + + per_cpu(vector_irq, cpu)[vector] = virq + i; + + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &pci_msi_controller; + } + + cdata = alloc_sw_msi_chip_data(irq_data); + if (!cdata) { + pr_warn("error alloc irq chip data\n"); + return -ENOMEM; + } + + irq_data->chip_data = cdata; + msiconf_index = start_index + i; + msi_config = set_piu_msi_config(hose, cpu, msiconf_index, vector); + + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->rc_index = hose->index; + cdata->rc_node = hose->node; + cdata->msi_config = msi_config; + cdata->msi_config_index = msiconf_index; + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + cdata->move_in_progress = false; + } + return 0; +} + +static int assign_irq_vector(int irq, unsigned int nr_irqs, + struct irq_domain *domain, struct pci_controller *hose) +{ + int err; + unsigned long flags; + + raw_spin_lock_irqsave(&vector_lock, flags); + err = __assign_irq_vector(irq, nr_irqs, domain, hose); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return err; +} + +static void sw64_vector_free_irqs(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + int i; + struct irq_data *irq_data; + struct pci_dev *pdev; + unsigned long flags; + + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + if (irq_data && irq_data->chip_data) { + struct sw64_msi_chip_data *cdata; + struct msi_desc *entry; + struct pci_controller *hose; + + raw_spin_lock_irqsave(&vector_lock, flags); + cdata = irq_data->chip_data; + entry = irq_get_msi_desc(virq + i); + if (entry) { + pdev = (struct pci_dev *)msi_desc_to_pci_dev(entry); + hose = pci_bus_to_pci_controller(pdev->bus); + clear_bit(cdata->msi_config_index, hose->piu_msiconfig); + } + irq_domain_reset_irq_data(irq_data); + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; + kfree(cdata); + raw_spin_unlock_irqrestore(&vector_lock, flags); + } + } +} + +static void sw64_irq_free_descs(unsigned int virq, unsigned int nr_irqs) +{ + if (is_guest_or_emul()) { + vt_sw64_vector_free_irqs(virq, nr_irqs); + return irq_free_descs(virq, nr_irqs); + } + + return irq_domain_free_irqs(virq, nr_irqs); +} + +void arch_teardown_msi_irqs(struct pci_dev *dev) +{ + struct msi_desc *desc; + int i; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + if (desc->irq) { + for (i = 0; i < desc->nvec_used; i++) + sw64_irq_free_descs(desc->irq + i, 1); + desc->irq = 0; + } + } +} + +static int sw64_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + int err; + struct irq_alloc_info *info = arg; + struct pci_controller *hose; + + if (arg == NULL) + return -ENODEV; + + hose = pci_bus_to_pci_controller(info->msi_dev->bus); + err = assign_irq_vector(virq, nr_irqs, domain, hose); + if (err) + goto error; + return 0; +error: + sw64_vector_free_irqs(domain, virq, nr_irqs); + return err; +} + +static int pci_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *arg) +{ + struct pci_dev *pdev = to_pci_dev(dev); + + memset(arg, 0, sizeof(*arg)); + arg->msi_dev = pdev; + if (pdev->msix_enabled) + arg->type = IRQ_ALLOC_TYPE_MSIX; + else + arg->type = IRQ_ALLOC_TYPE_MSI; + return 0; +} + +static struct msi_domain_ops pci_msi_domain_ops = { + .msi_prepare = pci_msi_prepare, +}; + +static struct msi_domain_info pci_msi_domain_info = { + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, + .ops = &pci_msi_domain_ops, + .chip = &pci_msi_controller, + .handler = handle_edge_irq, + .handler_name = "edge", +}; + +static int sw64_irq_map(struct irq_domain *d, unsigned int virq, irq_hw_number_t hw) +{ + irq_set_chip_and_handler(virq, &sw64_irq_chip, handle_level_irq); + irq_set_status_flags(virq, IRQ_LEVEL); + return 0; +} + +const struct irq_domain_ops sw64_msi_domain_ops = { + .map = sw64_irq_map, + .alloc = sw64_vector_alloc_irqs, + .free = sw64_vector_free_irqs, +}; + +int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) +{ + int err; + + if (is_guest_or_emul()) + return sw64_setup_vt_msi_irqs(pdev, nvec, type); + + if (!msi_default_domain) + return -EIO; + + err = msi_domain_alloc_irqs_all_locked(&pdev->dev, MSI_DEFAULT_DOMAIN, nvec); + return err; +} + +void arch_init_msi_domain(struct irq_domain *parent) +{ + struct irq_domain *sw64_irq_domain; + + if (is_guest_or_emul()) + return; + + sw64_irq_domain = irq_domain_add_tree(NULL, &sw64_msi_domain_ops, NULL); + BUG_ON(sw64_irq_domain == NULL); + irq_set_default_host(sw64_irq_domain); + msi_default_domain = pci_msi_create_irq_domain(NULL, + &pci_msi_domain_info, sw64_irq_domain); + if (!msi_default_domain) + pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n"); +} + +int pcibios_device_add(struct pci_dev *dev) +{ + if (msi_default_domain) + dev_set_msi_domain(&dev->dev, msi_default_domain); + return 0; +} + +static void irq_move_complete(struct sw64_msi_chip_data *cdata, int cpu, int vector) +{ + if (likely(!cdata->move_in_progress)) + return; + if (vector == cdata->vector && cdata->dst_cpu == cpu) { + raw_spin_lock(&vector_lock); + cdata->move_in_progress = 0; + per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector] = 0; + raw_spin_unlock(&vector_lock); + } +} + +void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned long pci_msi1_addr) +{ + int i, irq, piu_index, msi_index = 0; + int cpu, vector_index = 0; + unsigned long value = 0; + unsigned long int_pci_msi[3]; + unsigned long *ptr; + struct irq_data *irq_data; + struct sw64_msi_chip_data *cdata; + + if (is_guest_or_emul()) { + cpu = smp_processor_id(); + irq = per_cpu(vector_irq, cpu)[vector]; + handle_irq(irq); + return; + } + + ptr = (unsigned long *)pci_msi1_addr; + int_pci_msi[0] = *ptr; + int_pci_msi[1] = *(ptr + 1); + int_pci_msi[2] = *(ptr + 2); + + cpu = smp_processor_id(); + + for (i = 0; i < 4; i++) { + vector_index = i * 64; + while (vector != 0) { + msi_index = find_next_bit(&vector, 64, msi_index); + if (msi_index == 64) { + msi_index = 0; + continue; + } + + irq = per_cpu(vector_irq, cpu)[vector_index + msi_index]; + irq_data = irq_domain_get_irq_data(msi_default_domain->parent, irq); + cdata = irq_data_get_irq_chip_data(irq_data); + spin_lock(&cdata->cdata_lock); + irq_move_complete(cdata, cpu, vector_index + msi_index); + piu_index = cdata->msi_config_index; + value = cdata->msi_config | (1UL << 63); + write_piu_ior0(cdata->rc_node, cdata->rc_index, MSICONFIG0 + (piu_index << 7), value); + spin_unlock(&cdata->cdata_lock); + handle_irq(irq); + + vector = vector & (~(1UL << msi_index)); + } + + vector = int_pci_msi[i % 3]; + } +} diff --git a/drivers/irqchip/irq-sw64-intc-v2.c b/drivers/irqchip/irq-sw64-intc-v2.c new file mode 100644 index 000000000000..bc2c8ef3ed2f --- /dev/null +++ b/drivers/irqchip/irq-sw64-intc-v2.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +static void fake_irq_mask(struct irq_data *data) +{ +} + +static void fake_irq_unmask(struct irq_data *data) +{ +} + +static struct irq_chip onchip_intc = { + .name = "SW fake Intc", + .irq_mask = fake_irq_mask, + .irq_unmask = fake_irq_unmask, +}; + +static int sw64_intc_domain_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hw) +{ + + irq_set_chip_and_handler(irq, &onchip_intc, handle_level_irq); + irq_set_status_flags(irq, IRQ_LEVEL); + return 0; +} + +static const struct irq_domain_ops sw64_intc_domain_ops = { + .xlate = irq_domain_xlate_onecell, + .map = sw64_intc_domain_map, +}; + +#ifdef CONFIG_OF +static struct irq_domain *root_domain; + +static int __init +init_onchip_IRQ(struct device_node *intc, struct device_node *parent) +{ + + int node = 0; + int hwirq = 0, nirq = 8; + + if (parent) + panic("DeviceTree incore intc not a root irq controller\n"); + + root_domain = irq_domain_add_linear(intc, 8, + &sw64_intc_domain_ops, NULL); + + if (!root_domain) + panic("root irq domain not avail\n"); + + /* with this we don't need to export root_domain */ + irq_set_default_host(root_domain); + + for (hwirq = 0 ; hwirq < nirq ; hwirq++) + irq_create_mapping(root_domain, hwirq); + + /*enable MCU_DVC_INT_EN*/ + sw64_io_write(node, MCU_DVC_INT_EN, 0xff); + + return 0; +} + +IRQCHIP_DECLARE(sw64_intc, "sw64,sw6_irq_controller", init_onchip_IRQ); + +static int __init +init_onchip_vt_IRQ(struct device_node *intc, struct device_node *parent) +{ + if (parent) + panic("DeviceTree incore intc not a root irq controller\n"); + + root_domain = irq_domain_add_legacy(intc, 16, 0, 0, + &sw64_intc_domain_ops, NULL); + + if (!root_domain) + panic("root irq domain not avail\n"); + + /* with this we don't need to export root_domain */ + irq_set_default_host(root_domain); + + return 0; +} + +IRQCHIP_DECLARE(sw64_vt_intc, "sw64,sw6_irq_vt_controller", init_onchip_vt_IRQ); +#endif diff --git a/drivers/irqchip/irq-sw64-lpc-intc.c b/drivers/irqchip/irq-sw64-lpc-intc.c new file mode 100644 index 000000000000..1cbf87478242 --- /dev/null +++ b/drivers/irqchip/irq-sw64-lpc-intc.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#define LPC_NR_IRQS 16 +#define LPC_IRQ 0x4 +#define LPC_IRQ_MASK 0x8 + +struct lpc_intc_data { + struct irq_domain *domain; + struct irq_chip_generic *gc; +}; + +static void lpc_irq_mask_ack(struct irq_data *data) +{ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(data); + struct irq_chip_type *ct = irq_data_get_chip_type(data); + unsigned int mask = data->mask; + + irq_gc_lock(gc); + *ct->mask_cache |= mask; + irq_reg_writel(gc, *ct->mask_cache, ct->regs.mask); + irq_reg_writel(gc, mask, ct->regs.ack); + irq_gc_unlock(gc); +} + +static void lpc_irq_handler(struct irq_desc *desc) +{ + struct lpc_intc_data *b = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned int irq; + u32 status; + + chained_irq_enter(chip, desc); + + status = irq_reg_readl(b->gc, LPC_IRQ); + + if (status == 0) { + raw_spin_lock(&desc->lock); + handle_bad_irq(desc); + raw_spin_unlock(&desc->lock); + goto out; + } + + while (status) { + irq = __ffs(status); + status &= ~BIT(irq); + generic_handle_irq(irq_find_mapping(b->domain, irq)); + } + +out: + chained_irq_exit(chip, desc); +} + +static int __init lpc_intc_of_init(struct device_node *np, + struct device_node *parent) +{ + unsigned int set = IRQ_NOPROBE | IRQ_LEVEL; + struct lpc_intc_data *data; + struct irq_chip_type *ct; + int parent_irq, ret; + void __iomem *base; + int hwirq = 0; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + base = of_iomap(np, 0); + if (!base) { + pr_err("failed to remap lpc intc registers\n"); + ret = -ENOMEM; + goto out_free; + } + + parent_irq = irq_of_parse_and_map(np, 0); + if (!parent_irq) { + pr_err("failed to find parent interrupt\n"); + ret = -EINVAL; + goto out_unmap; + } + + data->domain = irq_domain_add_linear(np, LPC_NR_IRQS, + &irq_generic_chip_ops, NULL); + if (!data->domain) { + ret = -ENOMEM; + goto out_unmap; + } + + /* Allocate a single Generic IRQ chip for this node */ + ret = irq_alloc_domain_generic_chips(data->domain, 16, 1, np->name, + handle_level_irq, 0, set, + IRQ_GC_INIT_MASK_CACHE); + if (ret) { + pr_err("failed to allocate generic irq chip\n"); + goto out_free_domain; + } + + /* Set the IRQ chaining logic */ + irq_set_chained_handler_and_data(parent_irq, + lpc_irq_handler, data); + + data->gc = irq_get_domain_generic_chip(data->domain, 0); + data->gc->reg_base = base; + data->gc->private = data; + + ct = data->gc->chip_types; + + ct->regs.ack = LPC_IRQ; + ct->regs.mask = LPC_IRQ_MASK; + ct->chip.irq_mask = irq_gc_mask_set_bit; + ct->chip.irq_unmask = irq_gc_mask_clr_bit; + ct->chip.irq_ack = irq_gc_ack_set_bit; + ct->chip.irq_mask_ack = lpc_irq_mask_ack; + + for (hwirq = 0 ; hwirq < 16 ; hwirq++) + irq_create_mapping(data->domain, hwirq); + + /* Enable LPC interrupts */ + writel(0xffffebdd, base + LPC_IRQ_MASK); + + return 0; + +out_free_domain: + irq_domain_remove(data->domain); +out_unmap: + iounmap(base); +out_free: + kfree(data); + return ret; +} +IRQCHIP_DECLARE(sw_lpc_intc, "sw64,lpc_intc", lpc_intc_of_init); -- Gitee From df2b9b5e6b5cb83b384899a2675ce87d2ed0b1a9 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:53:23 +0800 Subject: [PATCH 075/524] drivers: mfd: add sw64 support Add mfd drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/mfd/Kconfig | 16 ++ drivers/mfd/Makefile | 3 + drivers/mfd/lpc_sunway_chip3.c | 325 +++++++++++++++++++++++++++++++++ drivers/mfd/sunway_ast2400.c | 223 ++++++++++++++++++++++ 4 files changed, 567 insertions(+) create mode 100644 drivers/mfd/lpc_sunway_chip3.c create mode 100644 drivers/mfd/sunway_ast2400.c diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 68d71b4b55bd..6b653487d954 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -604,6 +604,22 @@ config LPC_SCH LPC bridge function of the Intel SCH provides support for System Management Bus and General Purpose I/O. +config LPC_CHIP3 + tristate "CHIP3 LPC" + depends on UNCORE_XUELANG + select MFD_CORE + help + LPC bridge function of the chip3 provides support for + System Management Bus and General Purpose I/O. + +config SUNWAY_SUPERIO_AST2400 + tristate "SUNWAY SUPERIO AST2400" + depends on SW64 + select MFD_CORE + help + Nuvoton AST2400 Super I/O chip platform driver written + for SUNWAY LPC controller. + config INTEL_SOC_PMIC bool "Support for Crystal Cove PMIC" depends on HAS_IOMEM && I2C=y && GPIOLIB && COMMON_CLK diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile index db1ba39de3b5..50b42df268ea 100644 --- a/drivers/mfd/Makefile +++ b/drivers/mfd/Makefile @@ -269,6 +269,9 @@ obj-$(CONFIG_MFD_KHADAS_MCU) += khadas-mcu.o obj-$(CONFIG_MFD_ACER_A500_EC) += acer-ec-a500.o obj-$(CONFIG_MFD_QCOM_PM8008) += qcom-pm8008.o +obj-$(CONFIG_LPC_CHIP3) += lpc_sunway_chip3.o +obj-$(CONFIG_SUNWAY_SUPERIO_AST2400) += sunway_ast2400.o + obj-$(CONFIG_SGI_MFD_IOC3) += ioc3.o obj-$(CONFIG_MFD_SIMPLE_MFD_I2C) += simple-mfd-i2c.o obj-$(CONFIG_MFD_SMPRO) += smpro-core.o diff --git a/drivers/mfd/lpc_sunway_chip3.c b/drivers/mfd/lpc_sunway_chip3.c new file mode 100644 index 000000000000..1bcf40d6a6f7 --- /dev/null +++ b/drivers/mfd/lpc_sunway_chip3.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * lpc_sunway_chip3.c - LPC interface for SUNWAY CHIP3 + * + * LPC bridge function contains many other functional units, + * such as Interrupt controllers, Timers, Power Management, + * System Management, GPIO, RTC, and LPC Configuration + * Registers. + * + * Copyright (c) 2014 JN + * Author: Weiqiang Su + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum features { + LPC_USE_MSI = (1 << 0), + LPC_USE_INTX = (1 << 1), +}; + +enum { + LPC_HST_BAR = 0, + LPC_MEM_BAR = 2, + LPC_FWH_BAR = 4, +}; + +enum { + LPC_CTL = 0x0, + LPC_IRQ = 0x4, + LPC_IRQ_MASK = 0x8, + LPC_STAT = 0xc, + LPC_ERR_INF = 0x10, + LPC_MEM_HADDR = 0x14, + LPC_FWH_IDSEL_R1 = 0x18, + LPC_FWH_IDSEL_R2 = 0x1c, + LPC_FWH_IDSEL_R3 = 0x20, + LPC_FWH_IDSEL_R4 = 0x24, + LPC_FWH_IDSEL_R5 = 0x28, + LPC_FWH_DEC_EN1 = 0x2c, + LPC_FWH_DEC_EN2 = 0x30, + LPC_DMA_CTL = 0x34, + LPC_CH_STAT = 0x38, + LPC_CH0_ADDR = 0x3c, + LPC_CH1_ADDR = 0x40, + LPC_CH2_ADDR = 0x44, + LPC_CH3_ADDR = 0x48, + LPC_CH0_LENG = 0x4c, + LPC_CH1_LENG = 0x50, + LPC_CH2_LENG = 0x54, + LPC_CH3_LENG = 0x58, + LPC_CH0_MODE = 0x5c, + LPC_CH1_MODE = 0x60, + LPC_CH2_MODE = 0x64, + LPC_CH3_MODE = 0x68, + LPC_CH_MASK = 0x6c, + LPC_DMA_SWRST = 0x70, +}; + +struct lpc_chip3_adapter { + void __iomem *hst_regs; + struct device *dev; + int irq; + unsigned int features; +}; + +static struct resource superio_chip3_resources[] = { + { + .flags = IORESOURCE_IO, + } +}; + +static struct resource mem_flash_resource = { + .flags = IORESOURCE_MEM, +}; + +static struct resource fw_flash_resource = { + .flags = IORESOURCE_MEM, +}; + +static struct physmap_flash_data mem_flash_data = { + .width = 1, +}; + +static struct physmap_flash_data fw_flash_data = { + .width = 1, +}; + +static struct mfd_cell lpc_chip3_cells[] = { + { + .name = "sunway_superio_ast2400", + .id = 0, + .num_resources = ARRAY_SIZE(superio_chip3_resources), + .resources = superio_chip3_resources, + }, + { + .name = "chip3-flash", + .id = 0, + .num_resources = 1, + .resources = &mem_flash_resource, + .platform_data = &mem_flash_data, + .pdata_size = sizeof(mem_flash_data), + }, + { + .name = "chip3_fwh-flash", + .id = 0, + .num_resources = 1, + .resources = &fw_flash_resource, + .platform_data = &fw_flash_data, + .pdata_size = sizeof(fw_flash_data), + } +}; + +static inline void lpc_writel(void *address, int reg_base, int value) +{ + unsigned long addr = (unsigned long)address + reg_base; + + writel(value, (void *)addr); +} + +static inline int lpc_readl(void *address, int reg_base) +{ + unsigned long addr = (unsigned long)address + reg_base; + int value = readl((void *)addr); + + return value; +} + +static void lpc_enable(struct lpc_chip3_adapter *lpc_adapter) +{ + unsigned int value; + + value = lpc_readl(lpc_adapter->hst_regs, LPC_CTL); + value |= 0x1600; + + /* LPC host enable */ + lpc_writel(lpc_adapter->hst_regs, LPC_CTL, value); +} + +static void lpc_mem_flash_init(struct platform_device *pdev, + struct lpc_chip3_adapter *lpc_adapter) +{ + mem_flash_resource.start = + (((unsigned long)(lpc_adapter->hst_regs) & (~(0xfUL << 28))) | (0x2UL << 28)); + mem_flash_resource.end = mem_flash_resource.start + SZ_256M - 1; + + writel(0x1f, lpc_adapter->hst_regs + LPC_MEM_HADDR); +} + +static void lpc_fw_flash_init(struct platform_device *pdev, + struct lpc_chip3_adapter *lpc_adapter) +{ + fw_flash_resource.start = + (((unsigned long)(lpc_adapter->hst_regs) & (~(0xfUL << 28))) | (0x3UL << 28)); + fw_flash_resource.end = fw_flash_resource.start + SZ_256M - 1; + + writel(0xff0f, lpc_adapter->hst_regs + LPC_FWH_DEC_EN1); + writel(0xffff11ff, lpc_adapter->hst_regs + LPC_FWH_IDSEL_R5); + writel(0xffffffff, lpc_adapter->hst_regs + LPC_FWH_IDSEL_R4); + writel(0xffffffff, lpc_adapter->hst_regs + LPC_FWH_IDSEL_R3); + writel(0xffffffff, lpc_adapter->hst_regs + LPC_FWH_IDSEL_R2); + writel(0xffffffff, lpc_adapter->hst_regs + LPC_FWH_IDSEL_R1); + +} + +static int lpc_chip3_probe(struct platform_device *pdev) +{ + int ret; + struct lpc_chip3_adapter *lpc_adapter; + struct resource *mem; + + lpc_adapter = kzalloc(sizeof(*lpc_adapter), GFP_KERNEL); + if (lpc_adapter == NULL) { + dev_err(&pdev->dev, "%s kzalloc failed !\n", __func__); + return -ENOMEM; + } + + platform_set_drvdata(pdev, lpc_adapter); + /* Get basic io resource and map it */ + mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!mem) { + dev_err(&pdev->dev, "no mem resource?\n"); + return -EINVAL; + } + + lpc_adapter->hst_regs = devm_ioremap_resource(&pdev->dev, mem); + if (IS_ERR(lpc_adapter->hst_regs)) { + dev_err(&pdev->dev, "lpc region map failed\n"); + return PTR_ERR(lpc_adapter->hst_regs); + } + + lpc_adapter->dev = &pdev->dev; + lpc_adapter->features = 0; + + lpc_enable(lpc_adapter); + + lpc_mem_flash_init(pdev, lpc_adapter); + lpc_fw_flash_init(pdev, lpc_adapter); + + ret = mfd_add_devices(&pdev->dev, 0, + lpc_chip3_cells, ARRAY_SIZE(lpc_chip3_cells), + NULL, 0, NULL); + if (ret) + goto out_dev; + + dev_info(lpc_adapter->dev, "probe succeed !\n"); + + return ret; + +out_dev: + dev_info(lpc_adapter->dev, "probe failed !\n"); + + mfd_remove_devices(&pdev->dev); + kfree(lpc_adapter); + + return ret; +} + +static int lpc_chip3_remove(struct platform_device *pdev) +{ + struct lpc_chip3_adapter *lpc_adapter = platform_get_drvdata(pdev); + + mfd_remove_devices(&pdev->dev); + iounmap(lpc_adapter->hst_regs); + kfree(lpc_adapter); + + return 0; +} + +static const struct of_device_id chip3_lpc_of_match[] = { + {.compatible = "sunway,chip3_lpc",}, + { /* end of table */ } +}; + +MODULE_DEVICE_TABLE(of, chip3_lpc_of_match); + +#ifdef CONFIG_PM_SLEEP +unsigned int lpc_irq_ctrl_value; +unsigned int lpc_irq_irq_value; +unsigned int lpc_irq_mask_value; + +/** + * chip3_lpc_platform_suspend - Suspend an chip3_lpc-platform device + * @dev: the platform device to suspend + * + * This function stores the lpc controller register values and + * restores them when the machine wakes up. + */ +int chip3_lpc_platform_suspend(struct device *dev) +{ + struct lpc_chip3_adapter *lpc_adapter = dev_get_drvdata(dev); + + lpc_irq_ctrl_value = lpc_readl(lpc_adapter->hst_regs, LPC_CTL); + lpc_irq_irq_value = lpc_readl(lpc_adapter->hst_regs, LPC_IRQ); + lpc_irq_mask_value = lpc_readl(lpc_adapter->hst_regs, LPC_IRQ_MASK); + + return 0; +} + +/** + * chip3_lpc_platform_resume - Resume an chip3_lpc-platform device + * @dev: the platform device to resume + * + * This function restores the register value before the suspend. + */ +int chip3_lpc_platform_resume(struct device *dev) +{ + struct lpc_chip3_adapter *lpc_adapter = dev_get_drvdata(dev); + + lpc_writel(lpc_adapter->hst_regs, LPC_CTL, lpc_irq_ctrl_value); + lpc_writel(lpc_adapter->hst_regs, LPC_IRQ, lpc_irq_irq_value); + lpc_writel(lpc_adapter->hst_regs, LPC_IRQ_MASK, lpc_irq_mask_value); + + return 0; +} +static SIMPLE_DEV_PM_OPS(chip3_lpc_pm_ops, chip3_lpc_platform_suspend, + chip3_lpc_platform_resume); +#endif + + +static struct platform_driver chip3_lpc_platform_driver = { + .driver = { + .name = "chip3_lpc", + .of_match_table = chip3_lpc_of_match, +#ifdef CONFIG_PM_SLEEP + .pm = &chip3_lpc_pm_ops, +#endif + }, + .remove = lpc_chip3_remove, +}; + +static int __init chip3_lpc_drvinit(void) +{ + return platform_driver_probe(&chip3_lpc_platform_driver, + lpc_chip3_probe); +} + +/* + * lpc controller init configure before serial drivers; + * The lpc & ast2400 should be initialized much before + * the serial initialized functions are called. + */ +subsys_initcall_sync(chip3_lpc_drvinit); + +static void __exit chip3_lpc_drvexit(void) +{ + platform_driver_unregister(&chip3_lpc_platform_driver); +} + +module_exit(chip3_lpc_drvexit); + +MODULE_AUTHOR("Weiqiang Su "); +MODULE_DESCRIPTION("LPC Interface for CHIP3"); +MODULE_LICENSE("GPL"); diff --git a/drivers/mfd/sunway_ast2400.c b/drivers/mfd/sunway_ast2400.c new file mode 100644 index 000000000000..fbea07813643 --- /dev/null +++ b/drivers/mfd/sunway_ast2400.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/drivers/mfd/sunway_ast2400.c + * + * Copyright (C) 20014 - 2015 JN + * Author: Weiqiang Su + * + * Nuvoton AST2400 Super I/O chip platform driver written for + * SUNWAY LPC controller. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static int superio_uart0_irq; +static int superio_uart1_irq; +static void pnp_enable(device_t dev) +{ + pnp_enter_conf_mode(dev); + pnp_set_logical_device(dev); + pnp_set_enable(dev, 1); + pnp_exit_conf_mode(dev); +} + +const struct pnp_mode_ops pnp_conf_mode_8787_aa = { + .enter_conf_mode = pnp_enter_conf_mode_a5a5, + .exit_conf_mode = pnp_exit_conf_mode_aa, +}; + +static struct device_operations ops = { + .enable = pnp_enable, + .ops_pnp_mode = &pnp_conf_mode_8787_aa, +}; + +static struct pnp_info pnp_dev_info[] = { + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_FDC}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_PP }, + { true, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_SP1}, + { true, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_SP2}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_KBC}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_CIR}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_ACPI}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_HWM_FPLED}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_VID}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_CIRWKUP}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO_PP_OD}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_SVID}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_DSLP}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIOA_LDN}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_WDT1}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIOBASE}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO0}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO1}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO2}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO3}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO4}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO5}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO6}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO7}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO8}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIO9}, + { false, {SUPERIO_PNP_PORT, 0, &ops}, AST2400_GPIOA}, +}; + +static void superio_com1_init(struct pnp_device *device) +{ + pnp_enter_conf_mode(device); + pnp_set_logical_device(device); + pnp_set_enable(device, 1); + + pnp_write_config(device, 0x60, 0x3); + pnp_write_config(device, 0x61, 0xf8); + + pnp_write_config(device, 0x70, superio_uart0_irq); + pnp_write_config(device, 0x71, 0x1); + + pnp_write_config(device, 0xf0, 0x0); + + pnp_exit_conf_mode(device); +} + +static void superio_com2_init(struct pnp_device *device) +{ + pnp_enter_conf_mode(device); + pnp_set_logical_device(device); + pnp_set_enable(device, 1); + + pnp_write_config(device, 0x60, 0x2); + pnp_write_config(device, 0x61, 0xf8); + + pnp_write_config(device, 0x70, superio_uart1_irq); + pnp_write_config(device, 0x71, 0x1); + + pnp_write_config(device, 0xf0, 0x0); + + pnp_exit_conf_mode(device); +} + +static void pnp_enable_devices(superio_device_t superio_device, + struct device_operations *ops, + unsigned int functions, struct pnp_info *info) +{ + int i = 0; + struct pnp_info *each_info; + struct pnp_device *each_device; + + /* Setup the ops and resources on the newly allocated devices. */ + for (i = 0; i < functions; i++) { + each_info = info + i; + each_device = &each_info->pnp_device; + + /* Skip logical devices this Super I/O doesn't enable. */ + if (each_info->enabled == false) + continue; + + each_device->device = each_info->function; + each_device->ops = ops; + each_device->port = superio_device->superio_ast2400_efir; + + switch (each_device->device) { + case AST2400_SP1: + each_device->ops->init = superio_com1_init; + break; + case AST2400_SP2: + each_device->ops->init = superio_com2_init; + break; + } + + if (each_device->ops->init) + each_device->ops->init(each_device); + } +} + +static void superio_enable_devices(superio_device_t superio_device) +{ + pnp_enable_devices(superio_device, &ops, + ARRAY_SIZE(pnp_dev_info), pnp_dev_info); +} + +static int superio_ast2400_probe(struct platform_device *pdev) +{ + int err = 0; + superio_device_t superio_device; + struct resource *res; + resource_size_t physaddr = 0; + + /* allocate space for device info */ + superio_device = kzalloc(sizeof(struct superio_ast2400_device), GFP_KERNEL); + if (superio_device == NULL) { + err = -ENOMEM; + return err; + } + + res = platform_get_resource(pdev, IORESOURCE_IO, 1); + if (res) { + physaddr = res->start; + dev_info(&pdev->dev, "request memory region %pR\n", res); + } + + superio_device->dev = &pdev->dev; + superio_device->enabled = 1; + superio_device->superio_ast2400_efir = physaddr + SUPERIO_PNP_PORT; + superio_device->superio_ast2400_efdr = physaddr + SUPERIO_PNP_PORT + 1; + superio_uart0_irq = platform_get_irq_byname(pdev, "uart0_irq"); + superio_uart1_irq = platform_get_irq_byname(pdev, "uart1_irq"); + + superio_enable_devices(superio_device); + + platform_set_drvdata(pdev, superio_device); + + dev_info(superio_device->dev, "probe succeed !\n"); + + return 0; +} + +static int superio_ast2400_remove(struct platform_device *pdev) +{ + superio_device_t superio_device = platform_get_drvdata(pdev); + + platform_set_drvdata(pdev, NULL); + + kfree(superio_device); + + return 0; +} + +static struct platform_driver superio_nuvoton_ast2400_driver = { + .probe = superio_ast2400_probe, + .remove = superio_ast2400_remove, + .driver = { + .name = "sunway_superio_ast2400" + }, +}; + +static int __init superio_nuvoton_ast2400_init(void) +{ + return platform_driver_register(&superio_nuvoton_ast2400_driver); +} + +subsys_initcall_sync(superio_nuvoton_ast2400_init); + +static void __exit superio_nuvoton_ast2400_exit(void) +{ + platform_driver_unregister(&superio_nuvoton_ast2400_driver); +} + +module_exit(superio_nuvoton_ast2400_exit); + +MODULE_DESCRIPTION("NUVOTON AST2400 Super I/O DRIVER"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:" DRIVER_NAME); +MODULE_AUTHOR("Weiqiang Su"); -- Gitee From d39cc2654d6c41e027d45f478f83511b1573f10a Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:55:48 +0800 Subject: [PATCH 076/524] drivers: misc: add sw64 support Add misc drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/misc/Kconfig | 8 ++ drivers/misc/Makefile | 1 + drivers/misc/kgdbts.c | 3 +- drivers/misc/sunway-ged.c | 253 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 drivers/misc/sunway-ged.c diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index cadd4a820c03..1e9def44eb09 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -376,6 +376,14 @@ config HMC6352 This driver provides support for the Honeywell HMC6352 compass, providing configuration and heading data via sysfs. +config SUNWAY_GED + tristate "sunway generic device driver for memhotplug" + depends on SW64 + depends on MEMORY_HOTPLUG + help + This driver provides support for sunway generic device driver for + memhotplug, providing configuration and heading data via sysfs. + config DS1682 tristate "Dallas DS1682 Total Elapsed Time Recorder with Alarm" depends on I2C diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index f2a4d1ff65d4..ccf5456e1d88 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -35,6 +35,7 @@ obj-$(CONFIG_SENSORS_TSL2550) += tsl2550.o obj-$(CONFIG_DS1682) += ds1682.o obj-$(CONFIG_C2PORT) += c2port/ obj-$(CONFIG_HMC6352) += hmc6352.o +obj-$(CONFIG_SUNWAY_GED) += sunway-ged.o obj-y += eeprom/ obj-y += cb710/ obj-$(CONFIG_VMWARE_BALLOON) += vmw_balloon.o diff --git a/drivers/misc/kgdbts.c b/drivers/misc/kgdbts.c index 88b91ad8e541..6cd73f2a487f 100644 --- a/drivers/misc/kgdbts.c +++ b/drivers/misc/kgdbts.c @@ -130,7 +130,8 @@ static int hw_break_val2; static int cont_instead_of_sstep; static unsigned long cont_thread_id; static unsigned long sstep_thread_id; -#if defined(CONFIG_ARM) || defined(CONFIG_MIPS) || defined(CONFIG_SPARC) +#if defined(CONFIG_ARM) || defined(CONFIG_MIPS) || defined(CONFIG_SPARC) \ + || defined(CONFIG_SW64) static int arch_needs_sstep_emulation = 1; #else static int arch_needs_sstep_emulation; diff --git a/drivers/misc/sunway-ged.c b/drivers/misc/sunway-ged.c new file mode 100644 index 000000000000..b4e4ca315852 --- /dev/null +++ b/drivers/misc/sunway-ged.c @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Generic Event Device for ACPI. */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define OFFSET_START_ADDR 0 +#define OFFSET_LENGTH 8 +#define OFFSET_STATUS 16 +#define OFFSET_SLOT 24 + +/* Memory hotplug event */ +#define SUNWAY_MEMHOTPLUG_ADD 0x1 +#define SUNWAY_MEMHOTPLUG_REMOVE 0x2 + +struct sunway_memory_device { + struct sunway_ged_device *device; + unsigned int state; /* State of the memory device */ + struct list_head list; + + u64 start_addr; /* Memory Range start physical addr */ + u64 length; /* Memory Range length */ + u64 slot; /* Memory Range slot */ + unsigned int enabled:1; +}; + +struct sunway_ged_device { + struct device *dev; + void __iomem *membase; + void *driver_data; + spinlock_t lock; + struct list_head dev_list; +}; + +static int sunway_memory_enable_device(struct sunway_memory_device *mem_device) +{ + int num_enabled = 0; + int result = 0; + + if (mem_device->enabled) { /* just sanity check...*/ + num_enabled++; + goto out; + } + + /* + * If the memory block size is zero, please ignore it. + * Don't try to do the following memory hotplug flowchart. + */ + if (!mem_device->length) + goto out; + + lock_device_hotplug(); + /* suppose node = 0, fix me! */ + result = __add_memory(0, mem_device->start_addr, mem_device->length); + unlock_device_hotplug(); + /* + * If the memory block has been used by the kernel, add_memory() + * returns -EEXIST. If add_memory() returns the other error, it + * means that this memory block is not used by the kernel. + */ + if (result && result != -EEXIST) + goto out; + + mem_device->enabled = 1; + + /* + * Add num_enable even if add_memory() returns -EEXIST, so the + * device is bound to this driver. + */ + num_enabled++; +out: + if (!num_enabled) { + dev_err(mem_device->device->dev, "add_memory failed\n"); + return -EINVAL; + } + + return 0; +} + +static int sunway_memory_get_meminfo(struct sunway_memory_device *mem_device) +{ + struct sunway_ged_device *geddev; + + if (!mem_device) + return -EINVAL; + + if (mem_device->enabled) + return 0; + + geddev = mem_device->device; + + mem_device->start_addr = readq(geddev->membase + OFFSET_START_ADDR); + mem_device->length = readq(geddev->membase + OFFSET_LENGTH); + + return 0; +} + +static void sunway_memory_device_remove(struct sunway_ged_device *device) +{ + struct sunway_memory_device *mem_dev, *n; + unsigned long start_addr, length, slot; + + if (!device) + return; + + start_addr = readq(device->membase + OFFSET_START_ADDR); + length = readq(device->membase + OFFSET_LENGTH); + slot = readq(device->membase + OFFSET_SLOT); + + list_for_each_entry_safe(mem_dev, n, &device->dev_list, list) { + if (!mem_dev->enabled) + continue; + + if ((start_addr == mem_dev->start_addr) && + (length == mem_dev->length)) { + /* suppose node = 0, fix me! */ + remove_memory(0, start_addr, length); + list_del(&mem_dev->list); + kfree(mem_dev); + } + } + + writeq(slot, device->membase + OFFSET_SLOT); +} + +static int sunway_memory_device_add(struct sunway_ged_device *device) +{ + struct sunway_memory_device *mem_device; + int result; + + if (!device) + return -EINVAL; + + mem_device = kzalloc(sizeof(struct sunway_memory_device), GFP_KERNEL); + if (!mem_device) + return -ENOMEM; + + INIT_LIST_HEAD(&mem_device->list); + mem_device->device = device; + + /* Get the range from the IO */ + mem_device->start_addr = readq(device->membase + OFFSET_START_ADDR); + mem_device->length = readq(device->membase + OFFSET_LENGTH); + mem_device->slot = readq(device->membase + OFFSET_SLOT); + + result = sunway_memory_enable_device(mem_device); + if (result) { + dev_err(device->dev, "sunway_memory_enable_device() error\n"); + sunway_memory_device_remove(device); + + return result; + } + + list_add_tail(&mem_device->list, &device->dev_list); + dev_dbg(device->dev, "Memory device configured\n"); + + hcall(HCALL_MEMHOTPLUG, mem_device->start_addr, 0, 0); + + return 1; +} + +static irqreturn_t sunwayged_ist(int irq, void *data) +{ + struct sunway_ged_device *sunwayged_dev = data; + unsigned int status; + + status = readl(sunwayged_dev->membase + OFFSET_STATUS); + + /* through IO status to add or remove memory device */ + if (status & SUNWAY_MEMHOTPLUG_ADD) + sunway_memory_device_add(sunwayged_dev); + + if (status & SUNWAY_MEMHOTPLUG_REMOVE) + sunway_memory_device_remove(sunwayged_dev); + + return IRQ_HANDLED; +} + +static irqreturn_t sunwayged_irq_handler(int irq, void *data) +{ + return IRQ_WAKE_THREAD; +} + +static int sunwayged_probe(struct platform_device *pdev) +{ + struct resource *regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); + int irq = platform_get_irq(pdev, 0); + struct sunway_ged_device *geddev; + struct device *dev; + int irqflags; + + if (!regs) { + dev_err(dev, "no registers defined\n"); + return -EINVAL; + } + + geddev = devm_kzalloc(&pdev->dev, sizeof(*geddev), GFP_KERNEL); + if (!geddev) + return -ENOMEM; + + spin_lock_init(&geddev->lock); + geddev->membase = devm_ioremap(&pdev->dev, + regs->start, resource_size(regs)); + if (!geddev->membase) + return -ENOMEM; + + INIT_LIST_HEAD(&geddev->dev_list); + geddev->dev = &pdev->dev; + irqflags = IRQF_SHARED; + + if (request_threaded_irq(irq, sunwayged_irq_handler, sunwayged_ist, + irqflags, "SUNWAY:Ged", geddev)) { + dev_err(dev, "failed to setup event handler for irq %u\n", irq); + + return -EINVAL; + } + + platform_set_drvdata(pdev, geddev); + + return 0; +} + +static int sunwayged_remove(struct platform_device *pdev) +{ + return 0; +} + +static const struct of_device_id sunwayged_of_match[] = { + {.compatible = "sw6,sunway-ged", }, + { } +}; +MODULE_DEVICE_TABLE(of, sunwayged_of_match); + +static struct platform_driver sunwayged_platform_driver = { + .driver = { + .name = "sunway-ged", + .of_match_table = sunwayged_of_match, + }, + .probe = sunwayged_probe, + .remove = sunwayged_remove, +}; +module_platform_driver(sunwayged_platform_driver); + +MODULE_AUTHOR("Lu Feifei"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Sunway ged driver"); -- Gitee From 9a1aacb890a7acb3eec3b3447a5e80ece86e0bfd Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:44:12 +0800 Subject: [PATCH 077/524] drivers: pci: add sw64 support Add common pci and pci controller drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/controller/Kconfig | 4 + drivers/pci/controller/Makefile | 1 + drivers/pci/controller/pci-sunway.c | 805 ++++++++++++++++++++++++++++ drivers/pci/pci.c | 4 + drivers/pci/probe.c | 1 + drivers/pci/quirks.c | 2 + 6 files changed, 817 insertions(+) create mode 100644 drivers/pci/controller/pci-sunway.c diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index c0c3f2824990..2a2a3ccd66ad 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -342,6 +342,10 @@ config PCIE_XILINX_CPM Say 'Y' here if you want kernel support for the Xilinx Versal CPM host bridge. +config PCI_SW64 + bool + depends on SW64 && PCI + source "drivers/pci/controller/cadence/Kconfig" source "drivers/pci/controller/dwc/Kconfig" source "drivers/pci/controller/mobiveil/Kconfig" diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 37c8663de7fe..9d161c053bc4 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -39,6 +39,7 @@ obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o obj-$(CONFIG_PCIE_HISI_ERR) += pcie-hisi-error.o obj-$(CONFIG_PCIE_APPLE) += pcie-apple.o obj-$(CONFIG_PCIE_MT7621) += pcie-mt7621.o +obj-$(CONFIG_PCI_SW64) += pci-sunway.o # pcie-hisi.o quirks are needed even without CONFIG_PCIE_DW obj-y += dwc/ diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c new file mode 100644 index 000000000000..036994ffde38 --- /dev/null +++ b/drivers/pci/controller/pci-sunway.c @@ -0,0 +1,805 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#include + +void set_devint_wken(int node) +{ + unsigned long val; + + /* enable INTD wakeup */ + val = 0x80; + sw64_io_write(node, DEVINT_WKEN, val); + sw64_io_write(node, DEVINTWK_INTEN, val); +} + +#ifdef CONFIG_UNCORE_JUNZHANG +void set_adr_int(int node) +{ + sw64_io_write(node, ADR_INT_CONFIG, (0x0 << 16 | 0x3f)); + sw64_io_write(node, ADR_CTL, 0xc); +} +#endif + +void set_pcieport_service_irq(int node, int index) +{ + if (IS_ENABLED(CONFIG_PCIE_PME)) + write_piu_ior0(node, index, PMEINTCONFIG, PME_ENABLE_INTD_CORE0); + + if (IS_ENABLED(CONFIG_PCIEAER)) + write_piu_ior0(node, index, AERERRINTCONFIG, AER_ENABLE_INTD_CORE0); +} + +int chip_pcie_configure(struct pci_controller *hose) +{ + struct pci_dev *dev; + struct pci_bus *bus, *top; + struct list_head *next; + unsigned int max_read_size, smallest_max_payload; + int max_payloadsize; + unsigned long rc_index, node; + unsigned long piuconfig0, value; + unsigned int pcie_caps_offset; + unsigned int rc_conf_value; + u16 devctl, new_values; + bool rc_ari_disabled = false, found = false; + unsigned char bus_max_num; + + node = hose->node; + rc_index = hose->index; + smallest_max_payload = read_rc_conf(node, rc_index, RC_EXP_DEVCAP); + smallest_max_payload &= PCI_EXP_DEVCAP_PAYLOAD; + bus_max_num = hose->busn_space->start; + + top = hose->bus; + bus = top; + next = top->devices.next; + + for (;;) { + if (next == &bus->devices) { + /* end of this bus, go up or finish */ + if (bus == top) + break; + + next = bus->self->bus_list.next; + bus = bus->self->bus; + continue; + } + dev = list_entry(next, struct pci_dev, bus_list); + if (dev->subordinate) { + /* this is a pci-pci bridge, do its devices next */ + next = dev->subordinate->devices.next; + bus = dev->subordinate; + } else + next = dev->bus_list.next; + + if (!found) { + if (pci_is_root_bus(dev->bus)) { + if (list_empty(&dev->subordinate->devices)) + rc_ari_disabled = true; + } else { + if (!pci_ari_enabled(dev->bus)) { + rc_ari_disabled = true; + found = true; + } + } + } + + if (bus->busn_res.end > bus_max_num) + bus_max_num = bus->busn_res.end; + + /* Query device PCIe capability register */ + pcie_caps_offset = dev->pcie_cap; + if (pcie_caps_offset == 0) + continue; + max_payloadsize = dev->pcie_mpss; + if (max_payloadsize < smallest_max_payload) + smallest_max_payload = max_payloadsize; + } + + if (rc_ari_disabled) { + rc_conf_value = read_rc_conf(node, rc_index, RC_EXP_DEVCTL2); + rc_conf_value &= ~PCI_EXP_DEVCTL2_ARI; + write_rc_conf(node, rc_index, RC_EXP_DEVCTL2, rc_conf_value); + } else { + rc_conf_value = read_rc_conf(node, rc_index, RC_EXP_DEVCTL2); + rc_conf_value |= PCI_EXP_DEVCTL2_ARI; + write_rc_conf(node, rc_index, RC_EXP_DEVCTL2, rc_conf_value); + } + + rc_conf_value = read_rc_conf(node, rc_index, RC_EXP_DEVCAP); + rc_conf_value &= PCI_EXP_DEVCAP_PAYLOAD; + max_payloadsize = rc_conf_value; + if (max_payloadsize < smallest_max_payload) + smallest_max_payload = max_payloadsize; + + max_read_size = 0x2; /* Limit to 512B */ + value = read_rc_conf(node, rc_index, RC_EXP_DEVCTL); + value &= ~(PCI_EXP_DEVCTL_PAYLOAD | PCI_EXP_DEVCTL_READRQ); + value |= (max_read_size << 12) | (smallest_max_payload << 5); + write_rc_conf(node, rc_index, RC_EXP_DEVCTL, value); + new_values = (max_read_size << 12) | (smallest_max_payload << 5); + + piuconfig0 = read_piu_ior0(node, rc_index, PIUCONFIG0); + piuconfig0 &= ~(0x7fUL << 9); + if (smallest_max_payload == 0x2) { + piuconfig0 |= (0x20UL << 9); + write_piu_ior0(node, rc_index, PIUCONFIG0, piuconfig0); + } else { + piuconfig0 |= (0x40UL << 9); + write_piu_ior0(node, rc_index, PIUCONFIG0, piuconfig0); + } + + pr_info("Node%ld RC%ld MPSS %luB, MRRS %luB, Piuconfig0 %#lx, ARI %s\n", + node, rc_index, (1UL << smallest_max_payload) << 7, + (1UL << max_read_size) << 7, piuconfig0, + rc_ari_disabled ? "disabled" : "enabled"); + + /* Now, set the max_payload_size for all devices to that value. */ + bus = top; + next = top->devices.next; + for (;;) { + if (next == &bus->devices) { + /* end of this bus, go up or finish */ + if (bus == top) + break; + next = bus->self->bus_list.next; + bus = bus->self->bus; + continue; + } + dev = list_entry(next, struct pci_dev, bus_list); + if (dev->subordinate) { + /* this is a pci-pci bridge, do its devices next */ + next = dev->subordinate->devices.next; + bus = dev->subordinate; + } else + next = dev->bus_list.next; + + pcie_caps_offset = dev->pcie_cap; + if (pcie_caps_offset == 0) + continue; + + pci_read_config_word(dev, pcie_caps_offset + PCI_EXP_DEVCTL, &devctl); + devctl &= ~(PCI_EXP_DEVCTL_PAYLOAD | PCI_EXP_DEVCTL_READRQ); + devctl |= new_values; + pci_write_config_word(dev, pcie_caps_offset + PCI_EXP_DEVCTL, devctl); + } + + return bus_max_num; +} + +static int check_pci_linkup(unsigned long node, unsigned long index) +{ + unsigned long rc_debug; + + if (is_guest_or_emul()) { + if (node == 0 && index == 0) + return 0; + else + return 1; + } else { + rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); + } + + return !(rc_debug == 0x111); +} + +static void set_rc_piu(unsigned long node, unsigned long index) +{ + unsigned int i __maybe_unused; + unsigned int value; + u32 rc_misc_ctrl; + + if (is_guest_or_emul()) + return; + + /* configure RC, set PCI-E root controller */ + write_rc_conf(node, index, RC_COMMAND, 0x00100007); + write_rc_conf(node, index, RC_PORT_LINK_CTL, 0x1f0020); + write_rc_conf(node, index, RC_EXP_DEVCTL, 0x2850); + write_rc_conf(node, index, RC_EXP_DEVCTL2, 0x6); + write_rc_conf(node, index, RC_ORDER_RULE_CTL, 0x0100); + + /* enable DBI_RO_WR_EN */ + rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); + write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); + + /* fix up DEVICE_ID_VENDOR_ID register */ + value = (PCI_DEVICE_ID_SW64_ROOT_BRIDGE << 16) | PCI_VENDOR_ID_JN; + write_rc_conf(node, index, RC_VENDOR_ID, value); + + /* set PCI-E root class code */ + value = read_rc_conf(node, index, RC_REVISION_ID); + write_rc_conf(node, index, RC_REVISION_ID, (PCI_CLASS_BRIDGE_HOST << 16) | value); + + /* disable DBI_RO_WR_EN */ + write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); + + write_rc_conf(node, index, RC_PRIMARY_BUS, 0xffffff); + write_piu_ior0(node, index, PIUCONFIG0, PIUCONFIG0_INIT_VAL); + + write_piu_ior1(node, index, PIUCONFIG1, 0x2); + write_piu_ior1(node, index, ERRENABLE, -1); + + /* set DMA offset value PCITODMA_OFFSET */ + write_piu_ior0(node, index, EPDMABAR, PCITODMA_OFFSET); + if (IS_ENABLED(CONFIG_PCI_MSI)) { + write_piu_ior0(node, index, MSIADDR, MSIX_MSG_ADDR); +#ifdef CONFIG_UNCORE_XUELANG + for (i = 0; i < 256; i++) + write_piu_ior0(node, index, MSICONFIG0 + (i << 7), 0); +#endif + } +} + +static void set_intx(unsigned long node, unsigned long index, + unsigned long int_conf) +{ + if (is_guest_or_emul()) + return; + +#if defined(CONFIG_UNCORE_XUELANG) + write_piu_ior0(node, index, INTACONFIG, int_conf | (0x8UL << 10)); + write_piu_ior0(node, index, INTBCONFIG, int_conf | (0x4UL << 10)); + write_piu_ior0(node, index, INTCCONFIG, int_conf | (0x2UL << 10)); + write_piu_ior0(node, index, INTDCONFIG, int_conf | (0x1UL << 10)); +#elif defined(CONFIG_UNCORE_JUNZHANG) + write_piu_ior0(node, index, INTACONFIG, int_conf | (0x1UL << 10)); + write_piu_ior0(node, index, INTBCONFIG, int_conf | (0x2UL << 10)); + write_piu_ior0(node, index, INTCCONFIG, int_conf | (0x4UL << 10)); + write_piu_ior0(node, index, INTDCONFIG, int_conf | (0x8UL << 10)); +#endif +} + +static unsigned long get_rc_enable(unsigned long node) +{ + unsigned long rc_enable; + + if (is_guest_or_emul()) + return 1; + + rc_enable = sw64_io_read(node, IO_START); + + return rc_enable; +} + +static int map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(dev->bus); + + if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) + return hose->service_irq; + else + return hose->int_irq; +} + +static void hose_init(struct pci_controller *hose) +{ + unsigned long pci_io_base; + + hose->sparse_mem_base = 0; + hose->sparse_io_base = 0; + pci_io_base = IO_BASE | (hose->node << IO_NODE_SHIFT) + | PCI_BASE | (hose->index << IO_RC_SHIFT); + + hose->dense_mem_base = pci_io_base; + hose->dense_io_base = pci_io_base | PCI_LEGACY_IO; + hose->ep_config_space_base = __va(pci_io_base | PCI_EP_CFG); + hose->rc_config_space_base = __va(pci_io_base | PCI_RC_CFG); + + hose->mem_space->start = pci_io_base + PCI_32BIT_MEMIO; + hose->mem_space->end = hose->mem_space->start + PCI_32BIT_MEMIO_SIZE - 1; + hose->mem_space->name = "pci memory space"; + hose->mem_space->flags = IORESOURCE_MEM; + + if (request_resource(&iomem_resource, hose->mem_space) < 0) + pr_err("Failed to request MEM on hose %ld\n", hose->index); + hose->pre_mem_space->start = pci_io_base | PCI_64BIT_MEMIO; + hose->pre_mem_space->end = hose->pre_mem_space->start + PCI_64BIT_MEMIO_SIZE - 1; + hose->pre_mem_space->name = "pci pre mem space"; + hose->pre_mem_space->flags = IORESOURCE_MEM | IORESOURCE_PREFETCH | IORESOURCE_MEM_64; + + if (request_resource(&iomem_resource, hose->pre_mem_space) < 0) + pr_err("Failed to request 64bit MEM on hose %ld\n", hose->index); + hose->io_space->start = pci_io_base | PCI_LEGACY_IO; + hose->io_space->end = hose->io_space->start + PCI_LEGACY_IO_SIZE - 1; + hose->io_space->name = "pci io space"; + hose->io_space->flags = IORESOURCE_IO; + + if (request_resource(&ioport_resource, hose->io_space) < 0) + pr_err("Failed to request IO on hose %ld\n", hose->index); + hose->busn_space->name = "PCI busn"; + hose->busn_space->start = 0xff; + hose->busn_space->end = 0xff; + hose->busn_space->flags = IORESOURCE_BUS; + hose->first_busno = hose->self_busno = hose->busn_space->start; + hose->last_busno = hose->busn_space->end; + + if (is_in_host()) { + if (IS_ENABLED(CONFIG_PCI_MSI)) + memset(hose->piu_msiconfig, 0, 256/8); + } +}; + +static struct sw64_pci_init_ops chip_pci_init_ops = { + .map_irq = map_irq, + .get_rc_enable = get_rc_enable, + .hose_init = hose_init, + .set_rc_piu = set_rc_piu, + .check_pci_linkup = check_pci_linkup, + .set_intx = set_intx, +}; + +void __init setup_chip_pci_ops(void) +{ + sw64_chip_init->pci_init = chip_pci_init_ops; +} + +static unsigned long rc_linkup; +static struct pci_controller *head, **tail = &head; + +static void pci_mark_rc_linkup(unsigned long node, unsigned long index) +{ + set_bit(node * 8 + index, &rc_linkup); +} + +static int pci_get_rc_linkup(unsigned long node, unsigned long index) +{ + return test_bit(node * 8 + index, &rc_linkup); +} + +/** + * Link the specified pci controller to list + */ +extern struct pci_controller *hose_head; +static void pci_link_controller(struct pci_controller *hose) +{ + if (unlikely(!hose)) + return; + + *tail = hose; + tail = &hose->next; + + if (!hose_head) + hose_head = head; +} + +struct pci_controller *bus_num_to_pci_controller(unsigned long bus_num) +{ + struct pci_controller *hose; + + for (hose = head; hose; hose = hose->next) { + if (bus_num >= hose->first_busno && bus_num <= hose->last_busno) + return hose; + } + + return NULL; +} + +struct pci_controller *pci_bus_to_pci_controller(const struct pci_bus *bus) +{ + struct pci_config_window *cfg = NULL; + + if (unlikely(!bus)) + return NULL; + + if (acpi_disabled) + return (struct pci_controller *)(bus->sysdata); + + cfg = (struct pci_config_window *)bus->sysdata; + return (struct pci_controller *)(cfg->priv); +} + +/** + * PCIe Root Complex read config space operations + */ +static int sw64_pcie_read_rc_cfg(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + u32 data; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + void __iomem *cfg_iobase = hose->rc_config_space_base; + + if (IS_ENABLED(CONFIG_PCI_DEBUG)) + pr_debug("rc read addr:%px bus %d, devfn %#x, where %#x size=%d\t", + cfg_iobase + ((where & ~3) << 5), bus->number, devfn, where, size); + + if ((uintptr_t)where & (size - 1)) { + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + /** + * Workaround for sw6a chipset due to only support scan with devfn = 0, + * while sw6b does not have this limit. + */ + if (unlikely(devfn > 0)) { + *val = ~0; + return PCIBIOS_DEVICE_NOT_FOUND; + } + + data = readl(cfg_iobase + ((where & ~3) << 5)); + + switch (size) { + case 1: + *val = (data >> (8 * (where & 0x3))) & 0xff; + break; + case 2: + *val = (data >> (8 * (where & 0x2))) & 0xffff; + break; + default: + *val = data; + break; + } + + if (IS_ENABLED(CONFIG_PCI_DEBUG)) + pr_debug("*val %#x\n ", *val); + + return PCIBIOS_SUCCESSFUL; +} + +/** + * PCIe Root Complex write config space operations + */ +int sw64_pcie_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + u32 data; + u32 shift = 8 * (where & 3); + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + void __iomem *cfg_iobase = (void *)hose->rc_config_space_base; + + if ((uintptr_t)where & (size - 1)) + return PCIBIOS_BAD_REGISTER_NUMBER; + + switch (size) { + case 1: + data = readl(cfg_iobase + ((where & ~3) << 5)); + data &= ~(0xff << shift); + data |= (val & 0xff) << shift; + break; + case 2: + data = readl(cfg_iobase + ((where & ~3) << 5)); + data &= ~(0xffff << shift); + data |= (val & 0xffff) << shift; + break; + default: + data = val; + break; + } + + if (IS_ENABLED(CONFIG_PCI_DEBUG)) + pr_debug("rc write addr:%px bus %d, devfn %#x, where %#x *val %#x size %d\n", + cfg_iobase + ((where & ~3) << 5), bus->number, devfn, where, val, size); + + writel(data, cfg_iobase + ((where & ~3) << 5)); + + return PCIBIOS_SUCCESSFUL; +} + +/** + * sw64_pcie_valid_device - check if a valid device is present on bus + * @bus : PCI bus structure + * @devfn: device/function + * + * @return: 'true' on success and 'false' if invalid device is found + */ +static bool sw64_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + if (is_in_host()) { + /* Only one device down on each root complex */ + if (bus->number == hose->self_busno && devfn > 0) + return false; + } + + return true; +} + +/** + * sw64_pcie_config_read - read val from config space of PCI host controller or device + * @bus : PCI bus structure + * @devfn: device/function + * @where: offset from base + * @size : size of val + * @val[out]: the value read from PCI host controller or device + * + * @return: Whether read operation success + */ +static int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + int ret = PCIBIOS_DEVICE_NOT_FOUND; + + if (is_guest_or_emul()) + return pci_generic_config_read(bus, devfn, where, size, val); + + hose->self_busno = hose->busn_space->start; + + if (unlikely(bus->number == hose->self_busno)) { + ret = sw64_pcie_read_rc_cfg(bus, devfn, where, size, val); + } else { + if (pci_get_rc_linkup(hose->node, hose->index)) + ret = pci_generic_config_read(bus, devfn, where, size, val); + else + return ret; + } + return ret; +} + +/** + * sw64_pcie_config_write - write val to config space of PCI host controller or device + * @bus : PCI bus structure + * @devfn: device/function + * @where: offset from base + * @size : size of val + * @val : the value write to PCI host controller or device + * + * @return: Whether write operation success + */ +static int sw64_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + if (is_guest_or_emul()) + return pci_generic_config_write(bus, devfn, where, size, val); + + hose->self_busno = hose->busn_space->start; + + if (unlikely(bus->number == hose->self_busno)) + return sw64_pcie_write_rc_cfg(bus, devfn, where, size, val); + else + return pci_generic_config_write(bus, devfn, where, size, val); +} + +/** + * sw64_pcie_map_bus - get configuration base address + * @bus : PCI bus structure + * @devfn: device/function + * @where: offset from base + * + * @return: base address of the configuration space needed to be + * accessed. + */ +static void __iomem *sw64_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + void __iomem *cfg_iobase; + unsigned long relbus; + + if (!sw64_pcie_valid_device(bus, devfn)) + return NULL; + + /** + * ECAM of Sunway PCI host controller is slightly + * different from the standrad: + * [31:24]: bus number + * [23:19]: device number + * [18:16]: function number + * [15:12]: reserved + * [11:8] : extended config space registers + * [7:2] : legacy config space registers + */ + relbus = (bus->number << 24) | (devfn << 16) | where; + + cfg_iobase = hose->ep_config_space_base + relbus; + + if (IS_ENABLED(CONFIG_PCI_DEBUG)) + pr_debug("addr:%px bus %d, devfn %d, where %d\n", + cfg_iobase, bus->number, devfn, where); + return cfg_iobase; +} + +#ifdef CONFIG_ACPI +int sw64_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + return map_irq(dev, slot, pin); +} + +static void setup_intx_irqs(struct pci_controller *hose) +{ + unsigned long int_conf, node, val_node; + unsigned long index, irq; + int rcid; + + node = hose->node; + index = hose->index; + + if (!node_online(node)) + val_node = next_node_in(node, node_online_map); + else + val_node = node; + irq = irq_alloc_descs_from(NR_IRQS_LEGACY, 2, val_node); + WARN_ON(irq < 0); + irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_level_irq); + irq_set_status_flags(irq, IRQ_LEVEL); + hose->int_irq = irq; + irq_set_chip_and_handler(irq + 1, &dummy_irq_chip, handle_level_irq); + hose->service_irq = irq + 1; + rcid = cpu_to_rcid(0); + + pr_info_once("INTx are directed to node %d core %d.\n", + ((rcid >> 6) & 0x3), (rcid & 0x1f)); + int_conf = 1UL << 62 | rcid; /* rebase all intx on the first logical cpu */ + + set_intx(node, index, int_conf); + + set_pcieport_service_irq(node, index); +} + +static int sw64_pci_prepare_controller(struct pci_controller *hose, + struct acpi_device *adev) +{ + unsigned long long index, node; + unsigned long long rc_config_base_addr; + unsigned long long pci_io_base_addr; + unsigned long long ep_io_base_addr; + acpi_status rc; + + /* Get node from ACPI namespace */ + node = acpi_get_node(adev->handle); + if (node == NUMA_NO_NODE) { + dev_err(&adev->dev, "unable to get node ID\n"); + return -EEXIST; + } + + /* Get index from ACPI namespace */ + rc = acpi_evaluate_integer(adev->handle, "INDX", NULL, &index); + if (rc != AE_OK) { + dev_err(&adev->dev, "unable to retrieve INDX\n"); + return -EEXIST; + } + + /** + * Get Root Complex config space base address. + * + * For sw64, Root Complex config space base addr is different + * from Endpoint config space base address. Use MCFG table to + * pass Endpoint config space base address, and define Root Complex + * config space base address("RCCB") separately in the ACPI namespace. + */ + rc = acpi_evaluate_integer(adev->handle, "RCCB", NULL, &rc_config_base_addr); + if (rc != AE_OK) { + dev_err(&adev->dev, "unable to retrieve RCCB\n"); + return -EEXIST; + } + + /* Get Root Complex I/O space base addr from ACPI namespace */ + rc = acpi_evaluate_integer(adev->handle, "RCIO", NULL, &pci_io_base_addr); + if (rc != AE_OK) { + dev_err(&adev->dev, "unable to retrieve RCIO\n"); + return -EEXIST; + } + + /* Get Endpoint I/O space base addr from ACPI namespace */ + rc = acpi_evaluate_integer(adev->handle, "EPIO", NULL, &ep_io_base_addr); + if (rc != AE_OK) { + dev_err(&adev->dev, "unable to retrieve EPIO\n"); + return -EEXIST; + } + + hose->iommu_enable = false; + hose->index = index; + hose->node = node; + + hose->sparse_mem_base = 0; + hose->sparse_io_base = 0; + hose->dense_mem_base = pci_io_base_addr; + hose->dense_io_base = ep_io_base_addr; + + hose->rc_config_space_base = __va(rc_config_base_addr); + + hose->first_busno = 0xff; + hose->last_busno = 0xff; + hose->self_busno = 0xff; + + hose->need_domain_info = 0; + +#if IS_ENABLED(CONFIG_PCI_MSI) + if (is_in_host()) + memset(hose->piu_msiconfig, 0, 256 / 8); /* 256 bits bitmap */ +#endif + + /** + * There are two prerequisites for Root Complex + * of Sunway to work: + * 1. Root Complex enable + * 2. Root Complex link up + */ + set_rc_piu(hose->node, hose->index); + if (check_pci_linkup(hose->node, hose->index)) { + /** + * Root Complex link up failed. + * This usually means that no device on the slot. + */ + dev_info(&adev->dev, ": failed to link up\n", + hose->node, hose->index); + } else { + pci_mark_rc_linkup(hose->node, hose->index); + dev_info(&adev->dev, ": successfully link up\n", + hose->node, hose->index); + } + + setup_intx_irqs(hose); + + pci_link_controller(hose); + + return 0; +} + +/** + * Use the info from ACPI to init pci_controller + */ +static int sw64_pci_ecam_init(struct pci_config_window *cfg) +{ + struct pci_controller *hose = NULL; + struct device *dev = cfg->parent; + struct acpi_device *adev = to_acpi_device(dev); + phys_addr_t mcfg_addr; + int ret; + + /** + * First, check whether Root Complex is enabled. + * If Root Complex disabled, there's no need to continue. + * + * In ACPI namespace, we use _STA method to indicate + * whether Root Complex is enabled. + * + * The _STA has been checked when creating acpi_device. + * Double check here to get the latest hardware status. + */ + ret = acpi_bus_get_status(adev); + if (ret) { + dev_err(dev, "unable to retrieve _STA\n"); + return ret; + } + + if (!adev->status.present) { + dev_err(dev, "RC is not enabled\n"); + return -ENODEV; + } + + hose = kzalloc(sizeof(*hose), GFP_KERNEL); + if (!hose) { + dev_err(dev, "out of memory when alloc mem for pci_controller\n"); + return -ENOMEM; + } + + /* Get Endpoint config space base address from MCFG table */ + mcfg_addr = cfg->res.start - (cfg->busr.start << cfg->ops->bus_shift); + + /** + * "__va(mcfg_addr)" is equal to "cfg->win", so we can also use + * "hose->ep_config_space_base = cfg->win" here + */ + hose->ep_config_space_base = __va(mcfg_addr); + + /* Init pci_controller */ + ret = sw64_pci_prepare_controller(hose, adev); + if (ret) { + kfree(hose); + dev_err(&adev->dev, "failed to init pci controller\n"); + return ret; + } + + cfg->priv = (void *)hose; + + return 0; +} + +const struct pci_ecam_ops sw64_pci_ecam_ops = { + .bus_shift = 24, + .init = sw64_pci_ecam_init, + .pci_ops = { + .map_bus = sw64_pcie_map_bus, + .read = sw64_pcie_config_read, + .write = sw64_pcie_config_write, + } +}; +#endif diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 2fb570346ef6..1a9593042506 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4858,7 +4858,11 @@ int pcie_flr(struct pci_dev *dev) * 100ms, but may silently discard requests while the FLR is in * progress. Wait 100ms before trying to access the device. */ +#ifdef CONFIG_SW64 + msleep(1000); +#else msleep(100); +#endif return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS); } diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 0b4fb962067e..18fe32880862 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -145,6 +145,7 @@ static inline unsigned long decode_bar(struct pci_dev *dev, u32 bar) flags = bar & ~PCI_BASE_ADDRESS_MEM_MASK; flags |= IORESOURCE_MEM; + if (flags & PCI_BASE_ADDRESS_MEM_PREFETCH) flags |= IORESOURCE_PREFETCH; diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 7c71743b7254..9ae9829ba2c9 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -4599,6 +4599,7 @@ DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_AMD, 0x1a01, PCI_CLASS_NOT_DEFINED, DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_AMD, 0x1a02, PCI_CLASS_NOT_DEFINED, 8, quirk_relaxedordering_disable); +#ifndef CONFIG_SW64 /* * Per PCIe r3.0, sec 2.2.9, "Completion headers must supply the same * values for the Attribute as were supplied in the header of the @@ -4655,6 +4656,7 @@ static void quirk_chelsio_T5_disable_root_port_attributes(struct pci_dev *pdev) } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID, quirk_chelsio_T5_disable_root_port_attributes); +#endif /* * pci_acs_ctrl_enabled - compare desired ACS controls with those provided -- Gitee From 33486c61d382cbeb3c7ab8dbb0232df520ceec76 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:48:27 +0800 Subject: [PATCH 078/524] drivers: platform: add sw64 support Add platform drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/platform/Makefile | 1 + drivers/platform/sw64/Makefile | 2 + drivers/platform/sw64/legacy_xuelang.c | 63 ++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 drivers/platform/sw64/Makefile create mode 100644 drivers/platform/sw64/legacy_xuelang.c diff --git a/drivers/platform/Makefile b/drivers/platform/Makefile index 41640172975a..8296d4c41eb7 100644 --- a/drivers/platform/Makefile +++ b/drivers/platform/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_X86) += x86/ obj-$(CONFIG_LOONGARCH) += loongarch/ +obj-$(CONFIG_SW64) += sw64/ obj-$(CONFIG_MELLANOX_PLATFORM) += mellanox/ obj-$(CONFIG_MIPS) += mips/ obj-$(CONFIG_OLPC_EC) += olpc/ diff --git a/drivers/platform/sw64/Makefile b/drivers/platform/sw64/Makefile new file mode 100644 index 000000000000..28922224fb17 --- /dev/null +++ b/drivers/platform/sw64/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_PLATFORM_XUELANG) += legacy_xuelang.o diff --git a/drivers/platform/sw64/legacy_xuelang.c b/drivers/platform/sw64/legacy_xuelang.c new file mode 100644 index 000000000000..8a63d9edf9f2 --- /dev/null +++ b/drivers/platform/sw64/legacy_xuelang.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include +#include + +static void vt_mode_kill_arch(int mode) +{ + hcall(HCALL_SET_CLOCKEVENT, 0, 0, 0); + + switch (mode) { + case LINUX_REBOOT_CMD_RESTART: + hcall(HCALL_RESTART, 0, 0, 0); + mb(); + break; + case LINUX_REBOOT_CMD_HALT: + case LINUX_REBOOT_CMD_POWER_OFF: + hcall(HCALL_SHUTDOWN, 0, 0, 0); + mb(); + break; + default: + break; + } +} + +void sw64_halt(void) +{ + if (is_in_host()) + cpld_write(0x64, 0x00, 0xf0); + else + vt_mode_kill_arch(LINUX_REBOOT_CMD_HALT); +} + +void sw64_poweroff(void) +{ + if (is_in_host()) + cpld_write(0x64, 0x00, 0xf0); + else + vt_mode_kill_arch(LINUX_REBOOT_CMD_POWER_OFF); +} + +void sw64_restart(void) +{ + if (is_in_host()) { + fix_jm585_reset(); + cpld_write(0x64, 0x00, 0xc3); + } else + vt_mode_kill_arch(LINUX_REBOOT_CMD_RESTART); +} + +static int sw64_reset_init(void) +{ +#ifdef CONFIG_EFI + if (BIOS_SUPPORT_RESET_CLALLBACK((void *)bios_version)) + return 0; +#endif + pm_restart = sw64_restart; + pm_power_off = sw64_poweroff; + pm_halt = sw64_halt; + return 0; +} +subsys_initcall(sw64_reset_init); -- Gitee From 8db1ef2f26b1cca434d5516cca347d1607d14490 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:46:49 +0800 Subject: [PATCH 079/524] drivers: qemu_fw_cfg: add sw64 support Add qemu_fw_cfg drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/firmware/Kconfig | 2 +- drivers/firmware/qemu_fw_cfg.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index 8ecffdce94b1..11e8d19658aa 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -154,7 +154,7 @@ config RASPBERRYPI_FIRMWARE config FW_CFG_SYSFS tristate "QEMU fw_cfg device support in sysfs" - depends on SYSFS && (ARM || ARM64 || PARISC || PPC_PMAC || SPARC || X86) + depends on SYSFS && (ARM || ARM64 || PARISC || PPC_PMAC || SPARC || X86 || SW64) depends on HAS_IOPORT_MAP default n help diff --git a/drivers/firmware/qemu_fw_cfg.c b/drivers/firmware/qemu_fw_cfg.c index a69399a6b7c0..f4fea1ec3201 100644 --- a/drivers/firmware/qemu_fw_cfg.c +++ b/drivers/firmware/qemu_fw_cfg.c @@ -211,7 +211,7 @@ static void fw_cfg_io_cleanup(void) /* arch-specific ctrl & data register offsets are not available in ACPI, DT */ #if !(defined(FW_CFG_CTRL_OFF) && defined(FW_CFG_DATA_OFF)) -# if (defined(CONFIG_ARM) || defined(CONFIG_ARM64)) +# if (defined(CONFIG_ARM) || defined(CONFIG_ARM64) || defined(CONFIG_SW64)) # define FW_CFG_CTRL_OFF 0x08 # define FW_CFG_DATA_OFF 0x00 # define FW_CFG_DMA_OFF 0x10 -- Gitee From 6b1734f3a33a830f01668dd9fdd0a0ba62df62e6 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:54:17 +0800 Subject: [PATCH 080/524] drivers: rtc: add sw64 rtc support Add virtual rtc drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/rtc/Kconfig | 7 +++ drivers/rtc/Makefile | 5 ++ drivers/rtc/rtc-sw64-guest.c | 54 +++++++++++++++++++ drivers/rtc/rtc-sw64-virt-platform.c | 25 +++++++++ drivers/rtc/rtc-sw64-virt.c | 77 ++++++++++++++++++++++++++++ 5 files changed, 168 insertions(+) create mode 100644 drivers/rtc/rtc-sw64-guest.c create mode 100644 drivers/rtc/rtc-sw64-virt-platform.c create mode 100644 drivers/rtc/rtc-sw64-virt.c diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 92f46a6312c2..6f270577df86 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -984,6 +984,13 @@ config RTC_DRV_ALPHA Direct support for the real-time clock found on every Alpha system, specifically MC146818 compatibles. If in doubt, say Y. +config RTC_DRV_SW64_VIRT + bool "SW64 Hypervisor based RTC" + depends on SW64 + default y + help + Get support for the Hypervisor based RTC on SW64 systems. + config RTC_DRV_DS1216 tristate "Dallas DS1216" depends on SNI_RM diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile index fd209883ee2e..7711f79787ac 100644 --- a/drivers/rtc/Makefile +++ b/drivers/rtc/Makefile @@ -10,6 +10,10 @@ obj-$(CONFIG_RTC_CLASS) += rtc-core.o obj-$(CONFIG_RTC_MC146818_LIB) += rtc-mc146818-lib.o rtc-core-y := class.o interface.o +ifdef CONFIG_RTC_DRV_SW64_VIRT +rtc-core-y += rtc-sw64-virt-platform.o +endif + rtc-core-$(CONFIG_RTC_NVMEM) += nvmem.o rtc-core-$(CONFIG_RTC_INTF_DEV) += dev.o rtc-core-$(CONFIG_RTC_INTF_PROC) += proc.o @@ -168,6 +172,7 @@ obj-$(CONFIG_RTC_DRV_ST_LPC) += rtc-st-lpc.o obj-$(CONFIG_RTC_DRV_STM32) += rtc-stm32.o obj-$(CONFIG_RTC_DRV_STMP) += rtc-stmp3xxx.o obj-$(CONFIG_RTC_DRV_SUN4V) += rtc-sun4v.o +obj-$(CONFIG_RTC_DRV_SW64_VIRT) += rtc-sw64-virt.o obj-$(CONFIG_RTC_DRV_SUN6I) += rtc-sun6i.o obj-$(CONFIG_RTC_DRV_SUNPLUS) += rtc-sunplus.o obj-$(CONFIG_RTC_DRV_SUNXI) += rtc-sunxi.o diff --git a/drivers/rtc/rtc-sw64-guest.c b/drivers/rtc/rtc-sw64-guest.c new file mode 100644 index 000000000000..5d86ce20a1fb --- /dev/null +++ b/drivers/rtc/rtc-sw64-guest.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Lu Feifei + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#define RTC_IO_ADDR (0x804910000000ULL) + +static int sw_guest_read_time(struct device *dev, struct rtc_time *tm) +{ + unsigned long *ioaddr; + + ioaddr = ioremap(RTC_IO_ADDR, sizeof(long)); + rtc_time64_to_tm(*ioaddr, tm); + return 0; +} + +static const struct rtc_class_ops rtc_sw_guest_ops = { + .read_time = sw_guest_read_time, +}; + +static int __init rtc_sw_guest_probe(struct platform_device *pdev) +{ + struct rtc_device *rtc; + + rtc = devm_rtc_device_register(&pdev->dev, "sw_guest", + &rtc_sw_guest_ops, THIS_MODULE); + if (IS_ERR(rtc)) + return PTR_ERR(rtc); + + platform_set_drvdata(pdev, rtc); + return 0; +} + +static struct platform_driver rtc_sw_guest_driver = { + .driver = { + .name = "rtc_sw_guest", + }, +}; + +module_platform_driver_probe(rtc_sw_guest_driver, rtc_sw_guest_probe); + +MODULE_AUTHOR("Lu Feifei "); +MODULE_DESCRIPTION("SW GUEST RTC driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:rtc_sw_guest"); diff --git a/drivers/rtc/rtc-sw64-virt-platform.c b/drivers/rtc/rtc-sw64-virt-platform.c new file mode 100644 index 000000000000..3db9ff2f0e64 --- /dev/null +++ b/drivers/rtc/rtc-sw64-virt-platform.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +static struct platform_device rtc_sw64_virt_device = { + .name = "rtc_sw64_virt", + .id = -1, +}; + +static int __init rtc_sw64_virt_init(void) +{ + if (is_in_host()) + return 0; + + if (platform_device_register(&rtc_sw64_virt_device) < 0) + pr_err("unable to register rtc device...\n"); + /* not necessarily an error */ + return 0; +} +module_init(rtc_sw64_virt_init); diff --git a/drivers/rtc/rtc-sw64-virt.c b/drivers/rtc/rtc-sw64-virt.c new file mode 100644 index 000000000000..23c93d7ddbae --- /dev/null +++ b/drivers/rtc/rtc-sw64-virt.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* rtc-sw64-virt.c: Hypervisor based RTC for SW64 systems. + * + * Copyright (C) 2021 Lu Feifei + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#define RTC_IO_ADDR (0x804910000000ULL) +unsigned long vtime_old, vtime_new; + +static int sw64_virt_read_time(struct device *dev, struct rtc_time *tm) +{ + unsigned long *ioaddr; + unsigned long vtime_now; + long vtime_offset; + + ioaddr = ioremap(RTC_IO_ADDR, sizeof(long)); + if (!vtime_new) { + rtc_time64_to_tm(*ioaddr, tm); + } else { + vtime_now = *ioaddr; + vtime_offset = vtime_new - vtime_old; + vtime_now += vtime_offset; + rtc_time64_to_tm(vtime_now, tm); + } + return 0; +} + +static int sw64_virt_set_time(struct device *dev, struct rtc_time *tm) +{ + unsigned long *ioaddr; + + ioaddr = ioremap(RTC_IO_ADDR, sizeof(long)); + vtime_old = *ioaddr; + + vtime_new = rtc_tm_to_time64(tm); + return 0; +} + +static const struct rtc_class_ops rtc_sw64_virt_ops = { + .read_time = sw64_virt_read_time, + .set_time = sw64_virt_set_time, +}; + +static int __init rtc_sw64_virt_probe(struct platform_device *pdev) +{ + struct rtc_device *rtc; + + rtc = devm_rtc_device_register(&pdev->dev, "sw64_virt", + &rtc_sw64_virt_ops, THIS_MODULE); + if (IS_ERR(rtc)) + return PTR_ERR(rtc); + + platform_set_drvdata(pdev, rtc); + return 0; +} + +static struct platform_driver rtc_sw64_virt_driver = { + .driver = { + .name = "rtc_sw64_virt", + }, +}; + +module_platform_driver_probe(rtc_sw64_virt_driver, rtc_sw64_virt_probe); + +MODULE_AUTHOR("Lu Feifei "); +MODULE_DESCRIPTION("Sunway virtual RTC driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:rtc_sw64_virt"); -- Gitee From ee4727a66d245ec0b2092d7bf153cf60aa4c17ee Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:56:09 +0800 Subject: [PATCH 081/524] drivers: scsi: add sw64 support Add scsi drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/scsi/megaraid/megaraid_sas_fusion.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c index 2a820ed04c1e..72377ca3cd64 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fusion.c +++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c @@ -3592,6 +3592,14 @@ complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex, d_val.u.high != cpu_to_le32(UINT_MAX)) { smid = le16_to_cpu(reply_desc->SMID); +#ifdef CONFIG_SUBARCH_C3B + if (smid == 0xffff) { + smid = d_val.u.low >> 16; + if (smid == 0xffff) + break; + } +#endif + cmd_fusion = fusion->cmd_list[smid - 1]; scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *) cmd_fusion->io_request; -- Gitee From ea34b45efb76b8708751e94126556ea34477a827 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:53:34 +0800 Subject: [PATCH 082/524] drivers: spi: add sw64 support Add spi drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/spi/Kconfig | 6 + drivers/spi/Makefile | 1 + drivers/spi/spi-chip3-mmio.c | 147 +++++++++++++ drivers/spi/spi-chip3.c | 404 +++++++++++++++++++++++++++++++++++ drivers/spi/spi-chip3.h | 245 +++++++++++++++++++++ 5 files changed, 803 insertions(+) create mode 100644 drivers/spi/spi-chip3-mmio.c create mode 100644 drivers/spi/spi-chip3.c create mode 100644 drivers/spi/spi-chip3.h diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index 3ce0fd5df8e9..60826b7ed21e 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -1179,6 +1179,12 @@ config SPI_AMD # # Add new SPI master controllers in alphabetical order above this line # +config SPI_CHIP3 + tristate "Memory-mapped io interface driver for SUNWAY CHIP3 SPI core" + depends on UNCORE_XUELANG + help + general driver for SPI controller core from DesignWare + comment "SPI Multiplexer support" diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index 6af54842b9fa..26bf16fcf890 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -48,6 +48,7 @@ spi-dw-y := spi-dw-core.o spi-dw-$(CONFIG_SPI_DW_DMA) += spi-dw-dma.o obj-$(CONFIG_SPI_DW_BT1) += spi-dw-bt1.o obj-$(CONFIG_SPI_DW_MMIO) += spi-dw-mmio.o +obj-$(CONFIG_SPI_CHIP3) += spi-chip3.o spi-chip3-mmio.o obj-$(CONFIG_SPI_DW_PCI) += spi-dw-pci.o obj-$(CONFIG_SPI_EP93XX) += spi-ep93xx.o obj-$(CONFIG_SPI_FALCON) += spi-falcon.o diff --git a/drivers/spi/spi-chip3-mmio.c b/drivers/spi/spi-chip3-mmio.c new file mode 100644 index 000000000000..a907f13d4ae5 --- /dev/null +++ b/drivers/spi/spi-chip3-mmio.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Memory-mapped interface driver for SUNWAY CHIP3 SPI Core + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-chip3.h" + +#define DRIVER_NAME "sunway_chip3_spi" + +struct chip3_spi_mmio { + struct chip3_spi dws; + struct clk *clk; + void *priv; +}; + +static int chip3_spi_mmio_probe(struct platform_device *pdev) +{ + int (*init_func)(struct platform_device *pdev, + struct chip3_spi_mmio *dwsmmio); + struct chip3_spi_mmio *dwsmmio; + struct chip3_spi *dws; + struct resource *mem; + int ret; + int num_cs; + + dwsmmio = devm_kzalloc(&pdev->dev, sizeof(struct chip3_spi_mmio), + GFP_KERNEL); + if (!dwsmmio) + return -ENOMEM; + + dws = &dwsmmio->dws; + + /* Get basic io resource and map it */ + mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + dws->regs = devm_ioremap_resource(&pdev->dev, mem); + if (IS_ERR(dws->regs)) { + dev_err(&pdev->dev, "SPI region map failed\n"); + return PTR_ERR(dws->regs); + } + + dws->irq = platform_get_irq(pdev, 0); + if (dws->irq < 0) { + dev_err(&pdev->dev, "no irq resource?\n"); + return dws->irq; /* -ENXIO */ + } + + dwsmmio->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(dwsmmio->clk)) + return PTR_ERR(dwsmmio->clk); + ret = clk_prepare_enable(dwsmmio->clk); + if (ret) + return ret; + + dws->bus_num = pdev->id; + dws->max_freq = clk_get_rate(dwsmmio->clk); + + device_property_read_u32(&pdev->dev, "reg-io-width", + &dws->reg_io_width); + + num_cs = 4; + device_property_read_u32(&pdev->dev, "num-cs", &num_cs); + dws->num_cs = num_cs; + + if (pdev->dev.of_node) { + int i; + + for (i = 0; i < dws->num_cs; i++) { + int cs_gpio = of_get_named_gpio(pdev->dev.of_node, + "cs-gpios", i); + + if (cs_gpio == -EPROBE_DEFER) { + ret = cs_gpio; + goto out; + } + + if (gpio_is_valid(cs_gpio)) { + ret = devm_gpio_request(&pdev->dev, cs_gpio, + dev_name(&pdev->dev)); + if (ret) + goto out; + } + } + } + + init_func = device_get_match_data(&pdev->dev); + if (init_func) { + ret = init_func(pdev, dwsmmio); + if (ret) + goto out; + } + + ret = chip3_spi_add_host(&pdev->dev, dws); + if (ret) + goto out; + + platform_set_drvdata(pdev, dwsmmio); + + return 0; +out: + clk_disable_unprepare(dwsmmio->clk); + return ret; +} + +static int chip3_spi_mmio_remove(struct platform_device *pdev) +{ + struct chip3_spi_mmio *dwsmmio = platform_get_drvdata(pdev); + + chip3_spi_remove_host(&dwsmmio->dws); + clk_disable_unprepare(dwsmmio->clk); + + return 0; +} + +static const struct of_device_id chip3_spi_mmio_of_match[] = { + { .compatible = "sunway,chip3-spi", }, + { /* end of table */} +}; +MODULE_DEVICE_TABLE(of, chip3_spi_mmio_of_match); + +static struct platform_driver chip3_spi_mmio_driver = { + .probe = chip3_spi_mmio_probe, + .remove = chip3_spi_mmio_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = chip3_spi_mmio_of_match, + }, +}; +module_platform_driver(chip3_spi_mmio_driver); + +MODULE_AUTHOR("Platform@wxiat.com"); +MODULE_DESCRIPTION("Memory-mapped I/O interface driver for Sunway CHIP3"); +MODULE_LICENSE("GPL"); diff --git a/drivers/spi/spi-chip3.c b/drivers/spi/spi-chip3.c new file mode 100644 index 000000000000..8186c84eca8c --- /dev/null +++ b/drivers/spi/spi-chip3.c @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SUNWAY CHIP3 SPI core controller driver + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-chip3.h" + +/* Slave spi_dev related */ +struct chip_data { + u8 tmode; /* TR/TO/RO/EEPROM */ + u8 type; /* SPI/SSP/MicroWire */ + + u8 poll_mode; /* 1 means use poll mode */ + + u16 clk_div; /* baud rate divider */ + u32 speed_hz; /* baud rate */ + void (*cs_control)(u32 command); +}; + +static void chip3_spi_handle_err(struct spi_controller *master, + struct spi_message *msg) +{ + struct chip3_spi *dws = spi_controller_get_devdata(master); + + spi_reset_chip(dws); +} + +static size_t chip3_spi_max_length(struct spi_device *spi) +{ + struct chip3_spi *dws = spi_controller_get_devdata(spi->master); + + return dws->fifo_len; +} + +static int chip3_spi_transfer_one_message(struct spi_controller *master, + struct spi_message *m) +{ + struct chip3_spi *dws = spi_controller_get_devdata(master); + struct spi_transfer *t = NULL; + u16 clk_div; + u32 freq; + u32 speed_hz; + u32 status; + u32 len = 0; + int ret = 0; + int i = 0; + + spi_enable_chip(dws, 0); + + /* Handle per transfer options for bpw and speed. */ + freq = clamp(m->spi->max_speed_hz, 0U, dws->max_freq); + clk_div = (DIV_ROUND_UP(dws->max_freq, freq) + 1) & 0xfffe; + speed_hz = dws->max_freq / clk_div; + + if (dws->current_freq != speed_hz) { + spi_set_clk(dws, clk_div); + dws->current_freq = speed_hz; + } + + dws->n_bytes = 1; + + /* For poll mode just disable all interrupts */ + spi_mask_intr(dws, 0xff); + + chip3_writel(dws, CHIP3_SPI_CTRL0, SPI_TRANSMIT_RECEIVE); + + spi_enable_chip(dws, 1); + + list_for_each_entry(t, &m->transfers, transfer_list) { + len += t->len; + /* Judge if data is overflow */ + if (len > dws->fifo_len) { + pr_err("SPI transfer overflow.\n"); + m->actual_length = 0; + m->status = -EIO; + ret = -EIO; + goto way_out; + } + + if (t->tx_buf) + memcpy(&dws->buf[len], t->tx_buf, t->len); + else + memset(&dws->buf[len], 0, t->len); + } + + chip3_writel(dws, CHIP3_SPI_SER, 0x0); + for (i = 0; i < len; i++) + chip3_writel(dws, CHIP3_SPI_DR, dws->buf[i]); + chip3_writel(dws, CHIP3_SPI_SER, BIT(m->spi->chip_select)); + + do { + status = chip3_readl(dws, CHIP3_SPI_SR); + } while (status & SR_BUSY); + + list_for_each_entry(t, &m->transfers, transfer_list) { + if (t->rx_buf) { + for (i = 0; i < t->len; i++, t->rx_buf += 1) + *(u8 *)t->rx_buf = chip3_readl(dws, CHIP3_SPI_DR); + } else { + for (i = 0; i < t->len; i++) + chip3_readl(dws, CHIP3_SPI_DR); + } + } + + m->actual_length = len; + m->status = 0; + spi_finalize_current_message(master); + +way_out: + return ret; +} + +static int chip3_spi_adjust_mem_op_size(struct spi_mem *mem, + struct spi_mem_op *op) +{ + struct chip3_spi *dws = spi_controller_get_devdata(mem->spi->controller); + size_t len; + + len = sizeof(op->cmd.opcode) + op->addr.nbytes + op->dummy.nbytes; + + op->data.nbytes = min((size_t)op->data.nbytes, (dws->fifo_len - len)); + if (!op->data.nbytes) + return -EINVAL; + + return 0; +} + +static int chip3_spi_init_mem_buf(struct chip3_spi *dws, + const struct spi_mem_op *op) +{ + int ret = 0; + int i, j, len; + + /* Calculate the total length of the transfer. */ + len = sizeof(op->cmd.opcode) + op->addr.nbytes + op->dummy.nbytes; + + /* Judge if data is overflow */ + if (len + op->data.nbytes > dws->fifo_len) { + ret = -EIO; + goto way_out; + } + + /* + * Collect the operation code, address and dummy bytes into the single + * buffer. If it's a transfer with data to be sent, also copy it into + * the single buffer. + */ + for (i = 0; i < sizeof(op->cmd.opcode); i++) + dws->buf[i] = op->cmd.opcode; + for (j = 0; j < op->addr.nbytes; i++, j++) + dws->buf[i] = op->addr.val >> (8 * (op->addr.nbytes - i)); + for (j = 0; j < op->dummy.nbytes; i++, j++) + dws->buf[i] = 0xff; + + if (op->data.dir == SPI_MEM_DATA_OUT) { + memcpy(&dws->buf[i], op->data.buf.out, op->data.nbytes); + len += op->data.nbytes; + } + + dws->tx_len = len; + + if (op->data.dir == SPI_MEM_DATA_IN) { + dws->rx = op->data.buf.in; + dws->rx_len = op->data.nbytes; + } else { + dws->rx = NULL; + dws->rx_len = 0; + } + +way_out: + return ret; +} + +static int chip3_spi_exec_mem_op(struct spi_mem *mem, + const struct spi_mem_op *op) +{ + struct chip3_spi *dws = spi_controller_get_devdata(mem->spi->controller); + u16 clk_div; + int ret = 0; + int i; + unsigned short value; + u32 freq; + u32 speed_hz; + + ret = chip3_spi_init_mem_buf(dws, op); + if (ret) + return ret; + + spi_enable_chip(dws, 0); + + /* Handle per transfer options for bpw and speed. */ + freq = clamp(mem->spi->max_speed_hz, 0U, dws->max_freq); + clk_div = (DIV_ROUND_UP(dws->max_freq, freq) + 1) & 0xfffe; + speed_hz = dws->max_freq / clk_div; + + if (dws->current_freq != speed_hz) { + spi_set_clk(dws, clk_div); + dws->current_freq = speed_hz; + } + + dws->n_bytes = 1; + + /* For poll mode just disable all interrupts */ + spi_mask_intr(dws, 0xff); + + if ((dws->tx_len != 0) && (dws->rx_len != 0)) { + chip3_writel(dws, CHIP3_SPI_CTRL0, SPI_EEPROM_READ); + chip3_writel(dws, CHIP3_SPI_CTRL1, (dws->rx_len - 1)); + } else { + chip3_writel(dws, CHIP3_SPI_CTRL0, SPI_TRANSMIT_ONLY); + } + + spi_enable_chip(dws, 1); + + chip3_writel(dws, CHIP3_SPI_SER, 0x0); + for (i = 0; i < dws->tx_len; i++) + chip3_writel(dws, CHIP3_SPI_DR, dws->buf[i]); + chip3_writel(dws, CHIP3_SPI_SER, BIT(mem->spi->chip_select)); + + value = chip3_readl(dws, CHIP3_SPI_SR); + while (value & SR_BUSY) + value = chip3_readl(dws, CHIP3_SPI_SR); + + for (i = 0; i < dws->rx_len; dws->rx += dws->n_bytes, i++) + *(u8 *)dws->rx = chip3_readl(dws, CHIP3_SPI_DR); + + return ret; +} + +/* This may be called twice for each spi dev */ +static int chip3_spi_setup(struct spi_device *spi) +{ + struct chip3_spi_chip *chip_info = NULL; + struct chip_data *chip; + u32 poll_mode = 0; + struct device_node *np = spi->dev.of_node; + + /* Only alloc on first setup */ + chip = spi_get_ctldata(spi); + if (!chip) { + chip = kzalloc(sizeof(struct chip_data), GFP_KERNEL); + if (!chip) + return -ENOMEM; + spi_set_ctldata(spi, chip); + } + + /* + * Protocol drivers may change the chip settings, so... + * if chip_info exists, use it + */ + chip_info = spi->controller_data; + + /* chip_info doesn't always exist */ + if (chip_info) { + if (chip_info->cs_control) + chip->cs_control = chip_info->cs_control; + + chip->poll_mode = chip_info->poll_mode; + chip->type = chip_info->type; + } else { + if (np) { + of_property_read_u32(np, "poll_mode", &poll_mode); + chip->poll_mode = poll_mode; + } + + } + + chip->tmode = SPI_TMOD_TR; + return 0; +} + +static void chip3_spi_cleanup(struct spi_device *spi) +{ + struct chip_data *chip = spi_get_ctldata(spi); + + kfree(chip); + spi_set_ctldata(spi, NULL); +} + +/* Restart the controller, disable all interrupts, clean rx fifo */ +static void spi_hw_init(struct device *dev, struct chip3_spi *dws) +{ + spi_reset_chip(dws); + + /* + * Try to detect the FIFO depth if not set by interface driver, + * the depth could be from 2 to 256 from HW spec + */ + if (!dws->fifo_len) { + u32 fifo; + + for (fifo = 1; fifo < 256; fifo++) { + chip3_writel(dws, CHIP3_SPI_TXFLTR, fifo); + if (fifo != chip3_readl(dws, CHIP3_SPI_TXFLTR)) + break; + } + chip3_writel(dws, CHIP3_SPI_TXFLTR, 0); + + dws->fifo_len = (fifo == 1) ? 0 : fifo; + dev_info(dev, "Detected FIFO size: %u bytes\n", dws->fifo_len); + } +} + +static const struct spi_controller_mem_ops chip3_mem_ops = { + .adjust_op_size = chip3_spi_adjust_mem_op_size, + .exec_op = chip3_spi_exec_mem_op, +}; + + +int chip3_spi_add_host(struct device *dev, struct chip3_spi *dws) +{ + struct spi_controller *master; + int ret; + + BUG_ON(dws == NULL); + + master = spi_alloc_master(dev, 0); + if (!master) + return -ENOMEM; + + dws->master = master; + dws->type = SSI_MOTO_SPI; + + spi_controller_set_devdata(master, dws); + + master->mode_bits = SPI_CPOL | SPI_CPHA; + master->bits_per_word_mask = SPI_BPW_MASK(8) | SPI_BPW_MASK(16); + master->bus_num = dws->bus_num; + master->num_chipselect = dws->num_cs; + master->setup = chip3_spi_setup; + master->cleanup = chip3_spi_cleanup; + master->transfer_one_message = chip3_spi_transfer_one_message; + master->handle_err = chip3_spi_handle_err; + master->max_speed_hz = dws->max_freq; + master->dev.of_node = dev->of_node; + master->flags = SPI_CONTROLLER_GPIO_SS; + master->max_transfer_size = chip3_spi_max_length; + master->max_message_size = chip3_spi_max_length; + + master->mem_ops = &chip3_mem_ops; + + /* Basic HW init */ + spi_hw_init(dev, dws); + + ret = devm_spi_register_controller(dev, master); + if (ret) { + dev_err(&master->dev, "problem registering spi master\n"); + spi_enable_chip(dws, 0); + free_irq(dws->irq, master); + } + + return 0; +} +EXPORT_SYMBOL_GPL(chip3_spi_add_host); + +void chip3_spi_remove_host(struct chip3_spi *dws) +{ + spi_shutdown_chip(dws); + + free_irq(dws->irq, dws->master); +} +EXPORT_SYMBOL_GPL(chip3_spi_remove_host); + +int chip3_spi_suspend_host(struct chip3_spi *dws) +{ + int ret; + + ret = spi_controller_suspend(dws->master); + if (ret) + return ret; + + spi_shutdown_chip(dws); + return 0; +} +EXPORT_SYMBOL_GPL(chip3_spi_suspend_host); + +int chip3_spi_resume_host(struct chip3_spi *dws) +{ + int ret; + + spi_hw_init(&dws->master->dev, dws); + ret = spi_controller_resume(dws->master); + if (ret) + dev_err(&dws->master->dev, "fail to start queue (%d)\n", ret); + return ret; +} +EXPORT_SYMBOL_GPL(chip3_spi_resume_host); + +MODULE_AUTHOR("Platform@wxiat.com"); +MODULE_DESCRIPTION("Driver for Sunway CHIP3 SPI controller core"); +MODULE_LICENSE("GPL"); diff --git a/drivers/spi/spi-chip3.h b/drivers/spi/spi-chip3.h new file mode 100644 index 000000000000..88e49a9091a5 --- /dev/null +++ b/drivers/spi/spi-chip3.h @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef CHIP3_SPI_HEADER_H +#define CHIP3_SPI_HEADER_H + +#include +#include +#include +#include + +/* Register offsets */ +#define CHIP3_SPI_CTRL0 (0x00<<7) +#define CHIP3_SPI_CTRL1 (0x04<<7) +#define CHIP3_SPI_SSIENR (0x08<<7) +#define CHIP3_SPI_MWCR (0x0c<<7) +#define CHIP3_SPI_SER (0x10<<7) +#define CHIP3_SPI_BAUDR (0x14<<7) +#define CHIP3_SPI_TXFLTR (0x18<<7) +#define CHIP3_SPI_RXFLTR (0x1c<<7) +#define CHIP3_SPI_TXFLR (0x20<<7) +#define CHIP3_SPI_RXFLR (0x24<<7) +#define CHIP3_SPI_SR (0x28<<7) +#define CHIP3_SPI_IMR (0x2c<<7) +#define CHIP3_SPI_ISR (0x30<<7) +#define CHIP3_SPI_RISR (0x34<<7) +#define CHIP3_SPI_TXOICR (0x38<<7) +#define CHIP3_SPI_RXOICR (0x3c<<7) +#define CHIP3_SPI_RXUICR (0x40<<7) +#define CHIP3_SPI_MSTICR (0x44<<7) +#define CHIP3_SPI_ICR (0x48<<7) +#define CHIP3_SPI_DMACR (0x4c<<7) +#define CHIP3_SPI_DMATDLR (0x50<<7) +#define CHIP3_SPI_DMARDLR (0x54<<7) +#define CHIP3_SPI_IDR (0x58<<7) +#define CHIP3_SPI_VERSION (0x5c<<7) +#define CHIP3_SPI_DR (0x60<<7) + +/* Bit fields in CTRLR0 */ +#define SPI_DFS_OFFSET 0 + +#define SPI_FRF_OFFSET 4 +#define SPI_FRF_SPI 0x0 +#define SPI_FRF_SSP 0x1 +#define SPI_FRF_MICROWIRE 0x2 +#define SPI_FRF_RESV 0x3 + +#define SPI_MODE_OFFSET 6 +#define SPI_SCPH_OFFSET 6 +#define SPI_SCOL_OFFSET 7 + +#define SPI_TMOD_OFFSET 8 +#define SPI_TMOD_MASK (0x3 << SPI_TMOD_OFFSET) +#define SPI_TMOD_TR 0x0 /* xmit & recv */ +#define SPI_TMOD_TO 0x1 /* xmit only */ +#define SPI_TMOD_RO 0x2 /* recv only */ +#define SPI_TMOD_EPROMREAD 0x3 /* eeprom read mode */ + +#define SPI_SLVOE_OFFSET 10 +#define SPI_SRL_OFFSET 11 +#define SPI_CFS_OFFSET 12 + +/* Bit fields in SR, 7 bits */ +#define SR_MASK 0x7f /* cover 7 bits */ +#define SR_BUSY (1 << 0) +#define SR_TF_NOT_FULL (1 << 1) +#define SR_TF_EMPT (1 << 2) +#define SR_RF_NOT_EMPT (1 << 3) +#define SR_RF_FULL (1 << 4) +#define SR_TX_ERR (1 << 5) +#define SR_DCOL (1 << 6) + +/* Bit fields in ISR, IMR, RISR, 7 bits */ +#define SPI_INT_TXEI (1 << 0) +#define SPI_INT_TXOI (1 << 1) +#define SPI_INT_RXUI (1 << 2) +#define SPI_INT_RXOI (1 << 3) +#define SPI_INT_RXFI (1 << 4) +#define SPI_INT_MSTI (1 << 5) + +/* Bit fields in DMACR */ +#define SPI_DMA_RDMAE (1 << 0) +#define SPI_DMA_TDMAE (1 << 1) + +/* TX RX interrupt level threshold, max can be 256 */ +#define SPI_INT_THRESHOLD 32 + +/* The depth of the FIFO buffer is 256, so the max transfer length is 256. */ +#define MAX_LEN 256 + +/* The mode of spi controller. */ +#define SPI_TRANSMIT_RECEIVE 0x0c7 +#define SPI_EEPROM_READ 0x3c7 +#define SPI_TRANSMIT_ONLY 0x1c7 + +enum chip3_ssi_type { + SSI_MOTO_SPI = 0, + SSI_TI_SSP, + SSI_NS_MICROWIRE, +}; + +struct chip3_spi; + +struct chip3_spi { + struct spi_controller *master; + enum chip3_ssi_type type; + + void __iomem *regs; + unsigned long paddr; + int irq; + u32 fifo_len; /* depth of the FIFO buffer */ + u32 max_freq; /* max bus freq supported */ + + u32 reg_io_width; /* DR I/O width in bytes */ + u16 bus_num; + u16 num_cs; /* supported slave numbers */ + void (*set_cs)(struct spi_device *spi, bool enable); + + /* Current message transfer state info */ + size_t len; + void *tx; + unsigned int tx_len; + void *rx; + unsigned int rx_len; + u8 n_bytes; /* current is a 1/2 bytes op */ + u32 current_freq; /* frequency in hz */ + + u8 buf[MAX_LEN]; + + /* Bus interface info */ + void *priv; +#ifdef CONFIG_DEBUG_FS + struct dentry *debugfs; +#endif +}; + +static inline u32 chip3_readl(struct chip3_spi *dws, u32 offset) +{ + return __raw_readl(dws->regs + offset); +} + +static inline u16 chip3_readw(struct chip3_spi *dws, u32 offset) +{ + return __raw_readw(dws->regs + offset); +} + +static inline void chip3_writel(struct chip3_spi *dws, u32 offset, u32 val) +{ + __raw_writel(val, dws->regs + offset); +} + +static inline void chip3_writew(struct chip3_spi *dws, u32 offset, u16 val) +{ + __raw_writew(val, dws->regs + offset); +} + +static inline u32 chip3_read_io_reg(struct chip3_spi *dws, u32 offset) +{ + switch (dws->reg_io_width) { + case 2: + return chip3_readw(dws, offset); + case 4: + default: + return chip3_readl(dws, offset); + } +} + +static inline void chip3_write_io_reg(struct chip3_spi *dws, u32 offset, u32 val) +{ + switch (dws->reg_io_width) { + case 2: + chip3_writew(dws, offset, val); + break; + case 4: + default: + chip3_writel(dws, offset, val); + break; + } +} + +static inline void spi_enable_chip(struct chip3_spi *dws, int enable) +{ + chip3_writel(dws, CHIP3_SPI_SSIENR, (enable ? 1 : 0)); +} + +static inline void spi_set_clk(struct chip3_spi *dws, u16 div) +{ + chip3_writel(dws, CHIP3_SPI_BAUDR, div); +} + +/* Disable IRQ bits */ +static inline void spi_mask_intr(struct chip3_spi *dws, u32 mask) +{ + u32 new_mask; + + new_mask = chip3_readl(dws, CHIP3_SPI_IMR) & ~mask; + chip3_writel(dws, CHIP3_SPI_IMR, new_mask); +} + +/* Enable IRQ bits */ +static inline void spi_umask_intr(struct chip3_spi *dws, u32 mask) +{ + u32 new_mask; + + new_mask = chip3_readl(dws, CHIP3_SPI_IMR) | mask; + chip3_writel(dws, CHIP3_SPI_IMR, new_mask); +} + +/* + * This does disable the SPI controller, interrupts, and re-enable the + * controller back. Transmit and receive FIFO buffers are cleared when the + * device is disabled. + */ +static inline void spi_reset_chip(struct chip3_spi *dws) +{ + spi_enable_chip(dws, 0); + spi_mask_intr(dws, 0xff); + spi_enable_chip(dws, 1); +} + +static inline void spi_shutdown_chip(struct chip3_spi *dws) +{ + spi_enable_chip(dws, 0); + spi_set_clk(dws, 0); +} + +/* + * Each SPI slave device to work with chip3_api controller should + * has such a structure claiming its working mode (poll or PIO/DMA), + * which can be save in the "controller_data" member of the + * struct spi_device. + */ +struct chip3_spi_chip { + u8 poll_mode; /* 1 for controller polling mode */ + u8 type; /* SPI/SSP/MicroWire */ + u8 chip_select; + void (*cs_control)(u32 command); +}; + +extern int chip3_spi_add_host(struct device *dev, struct chip3_spi *dws); +extern void chip3_spi_remove_host(struct chip3_spi *dws); +extern int chip3_spi_suspend_host(struct chip3_spi *dws); +extern int chip3_spi_resume_host(struct chip3_spi *dws); + +/* platform related setup */ +extern int chip3_spi_mid_init(struct chip3_spi *dws); /* Intel MID platforms */ +#endif /* CHIP3_SPI_HEADER_H */ -- Gitee From fb0b6c99e123176e6da7d0656642cc57baa3c02a Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:53:47 +0800 Subject: [PATCH 083/524] drivers: tty: add sw64 support Add tty drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/tty/serial/8250/8250_sunway.c | 786 ++++++++++++++++++++++++++ drivers/tty/serial/8250/Kconfig | 7 + drivers/tty/serial/8250/Makefile | 1 + 3 files changed, 794 insertions(+) create mode 100644 drivers/tty/serial/8250/8250_sunway.c diff --git a/drivers/tty/serial/8250/8250_sunway.c b/drivers/tty/serial/8250/8250_sunway.c new file mode 100644 index 000000000000..9e3db232c832 --- /dev/null +++ b/drivers/tty/serial/8250/8250_sunway.c @@ -0,0 +1,786 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Synopsys SUNWAY 8250 driver. + * + * Copyright 2011 Picochip, Jamie Iles. + * Copyright 2013 Intel Corporation + * + * The Synopsys SUNWAY 8250 has an extra feature whereby it detects if the + * LCR is written whilst busy. If it is, then a busy detect interrupt is + * raised, the LCR needs to be rewritten and the uart status register read. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "8250.h" + +/* Offsets for the DesignWare specific registers */ +#define SUNWAY_UART_USR 0x1f /* UART Status Register */ +#define SUNWAY_UART_DLF 0xc0 /* Divisor Latch Fraction Register */ +#define SUNWAY_UART_CPR 0xf4 /* Component Parameter Register */ +#define SUNWAY_UART_UCV 0xf8 /* UART Component Version */ + +/* Component Parameter Register bits */ +#define SUNWAY_UART_CPR_ABP_DATA_WIDTH (3 << 0) +#define SUNWAY_UART_CPR_AFCE_MODE (1 << 4) +#define SUNWAY_UART_CPR_THRE_MODE (1 << 5) +#define SUNWAY_UART_CPR_SIR_MODE (1 << 6) +#define SUNWAY_UART_CPR_SIR_LP_MODE (1 << 7) +#define SUNWAY_UART_CPR_ADDITIONAL_FEATURES (1 << 8) +#define SUNWAY_UART_CPR_FIFO_ACCESS (1 << 9) +#define SUNWAY_UART_CPR_FIFO_STAT (1 << 10) +#define SUNWAY_UART_CPR_SHADOW (1 << 11) +#define SUNWAY_UART_CPR_ENCODED_PARMS (1 << 12) +#define SUNWAY_UART_CPR_DMA_EXTRA (1 << 13) +#define SUNWAY_UART_CPR_FIFO_MODE (0xff << 16) +/* Helper for fifo size calculation */ +#define SUNWAY_UART_CPR_FIFO_SIZE(a) (((a >> 16) & 0xff) * 16) + +/* DesignWare specific register fields */ +#define SUNWAY_UART_MCR_SIRE BIT(6) + +struct sunway8250_data { + u8 usr_reg; + u8 dlf_size; + int line; + int msr_mask_on; + int msr_mask_off; + struct clk *clk; + struct clk *pclk; + struct reset_control *rst; + struct uart_8250_dma dma; + + unsigned int skip_autocfg:1; + unsigned int uart_16550_compatible:1; +}; + +static inline u32 sunway8250_readl_ext(struct uart_port *p, int offset) +{ + if (p->iotype == UPIO_MEM32BE) + return ioread32be(p->membase + offset); + return readl(p->membase + offset); +} + +static inline void sunway8250_writel_ext(struct uart_port *p, int offset, u32 reg) +{ + if (p->iotype == UPIO_MEM32BE) + iowrite32be(reg, p->membase + offset); + else + writel(reg, p->membase + offset); +} + +static inline int sunway8250_modify_msr(struct uart_port *p, int offset, int value) +{ + struct sunway8250_data *d = p->private_data; + + /* Override any modem control signals if needed */ + if (offset == UART_MSR) { + value |= d->msr_mask_on; + value &= ~d->msr_mask_off; + } + + return value; +} + +static void sunway8250_force_idle(struct uart_port *p) +{ + struct uart_8250_port *up = up_to_u8250p(p); + + serial8250_clear_and_reinit_fifos(up); + (void)p->serial_in(p, UART_RX); +} + +static void sunway8250_check_lcr(struct uart_port *p, int value) +{ + void __iomem *offset = p->membase + (UART_LCR << p->regshift); + int tries = 1000; + + /* Make sure LCR write wasn't ignored */ + while (tries--) { + unsigned int lcr = p->serial_in(p, UART_LCR); + + if ((value & ~UART_LCR_SPAR) == (lcr & ~UART_LCR_SPAR)) + return; + + sunway8250_force_idle(p); + +#ifdef CONFIG_64BIT + if (p->type == PORT_OCTEON) { + __raw_writeq(value & 0xff, offset); + continue; + } +#endif + if (p->iotype == UPIO_MEM32) + writel(value, offset); + else if (p->iotype == UPIO_MEM32BE) + iowrite32be(value, offset); + else + writeb(value, offset); + } + /* + * FIXME: this deadlocks if port->lock is already held + * dev_err(p->dev, "Couldn't set LCR to %d\n", value); + */ +} + +/* Returns once the transmitter is empty or we run out of retries */ +static void sunway8250_tx_wait_empty(struct uart_port *p) +{ + unsigned int tries = 20000; + unsigned int delay_threshold = tries - 1000; + unsigned int lsr; + + while (tries--) { + lsr = readb(p->membase + (UART_LSR << p->regshift)); + if (lsr & UART_LSR_TEMT) + break; + + /* + * The device is first given a chance to empty without delay, + * to avoid slowdowns at high bitrates. If after 1000 tries + * the buffer has still not emptied, allow more time for low- + * speed links. + */ + if (tries < delay_threshold) + udelay(1); + } +} + +static void sunway8250_serial_out38x(struct uart_port *p, int offset, int value) +{ + struct sunway8250_data *d = p->private_data; + + /* Allow the TX to drain before we reconfigure */ + if (offset == UART_LCR) + sunway8250_tx_wait_empty(p); + + writeb(value, p->membase + (offset << p->regshift)); + + if (offset == UART_LCR && !d->uart_16550_compatible) + sunway8250_check_lcr(p, value); +} + + +static void sunway8250_serial_out(struct uart_port *p, int offset, int value) +{ + struct sunway8250_data *d = p->private_data; + + writeb(value, p->membase + (offset << p->regshift)); + + if (offset == UART_LCR && !d->uart_16550_compatible) + sunway8250_check_lcr(p, value); +} + +static unsigned int sunway8250_serial_in(struct uart_port *p, int offset) +{ + unsigned int value = readb(p->membase + (offset << p->regshift)); + + return sunway8250_modify_msr(p, offset, value); +} + +#ifdef CONFIG_64BIT +static unsigned int sunway8250_serial_inq(struct uart_port *p, int offset) +{ + unsigned int value; + + value = (u8)__raw_readq(p->membase + (offset << p->regshift)); + + return sunway8250_modify_msr(p, offset, value); +} + +static void sunway8250_serial_outq(struct uart_port *p, int offset, int value) +{ + struct sunway8250_data *d = p->private_data; + + value &= 0xff; + __raw_writeq(value, p->membase + (offset << p->regshift)); + /* Read back to ensure register write ordering. */ + __raw_readq(p->membase + (UART_LCR << p->regshift)); + + if (offset == UART_LCR && !d->uart_16550_compatible) + sunway8250_check_lcr(p, value); +} +#endif /* CONFIG_64BIT */ + +static void sunway8250_serial_out32(struct uart_port *p, int offset, int value) +{ + struct sunway8250_data *d = p->private_data; + + writel(value, p->membase + (offset << p->regshift)); + + if (offset == UART_LCR && !d->uart_16550_compatible) + sunway8250_check_lcr(p, value); +} + +static unsigned int sunway8250_serial_in32(struct uart_port *p, int offset) +{ + unsigned int value = readl(p->membase + (offset << p->regshift)); + + return sunway8250_modify_msr(p, offset, value); +} + +static void sunway8250_serial_out32be(struct uart_port *p, int offset, int value) +{ + struct sunway8250_data *d = p->private_data; + + iowrite32be(value, p->membase + (offset << p->regshift)); + + if (offset == UART_LCR && !d->uart_16550_compatible) + sunway8250_check_lcr(p, value); +} + +static unsigned int sunway8250_serial_in32be(struct uart_port *p, int offset) +{ + unsigned int value = ioread32be(p->membase + (offset << p->regshift)); + + return sunway8250_modify_msr(p, offset, value); +} + + +static int sunway8250_handle_irq(struct uart_port *p) +{ + struct uart_8250_port *up = up_to_u8250p(p); + struct sunway8250_data *d = p->private_data; + unsigned int iir = p->serial_in(p, UART_IIR); + unsigned int status; + unsigned long flags; + + /* + * There are ways to get Designware-based UARTs into a state where + * they are asserting UART_IIR_RX_TIMEOUT but there is no actual + * data available. If we see such a case then we'll do a bogus + * read. If we don't do this then the "RX TIMEOUT" interrupt will + * fire forever. + * + * This problem has only been observed so far when not in DMA mode + * so we limit the workaround only to non-DMA mode. + */ + if (!up->dma && ((iir & 0x3f) == UART_IIR_RX_TIMEOUT)) { + spin_lock_irqsave(&p->lock, flags); + status = p->serial_in(p, UART_LSR); + + if (!(status & (UART_LSR_DR | UART_LSR_BI))) + (void) p->serial_in(p, UART_RX); + + spin_unlock_irqrestore(&p->lock, flags); + } + + if (serial8250_handle_irq(p, iir)) + return 1; + + if ((iir & UART_IIR_BUSY) == UART_IIR_BUSY) { + /* Clear the USR */ + (void)p->serial_in(p, d->usr_reg); + + return 1; + } + + return 0; +} + +static void +sunway8250_do_pm(struct uart_port *port, unsigned int state, unsigned int old) +{ + if (!state) + pm_runtime_get_sync(port->dev); + + serial8250_do_pm(port, state, old); + + if (state) + pm_runtime_put_sync_suspend(port->dev); +} + +static void sunway8250_set_termios(struct uart_port *p, struct ktermios *termios, + const struct ktermios *old) +{ + unsigned int baud = tty_termios_baud_rate(termios); + struct sunway8250_data *d = p->private_data; + long rate; + int ret; + + if (IS_ERR(d->clk)) + goto out; + + clk_disable_unprepare(d->clk); + rate = clk_round_rate(d->clk, baud * 16); + if (rate < 0) + ret = rate; + else if (rate == 0) + ret = -ENOENT; + else + ret = clk_set_rate(d->clk, rate); + clk_prepare_enable(d->clk); + + if (!ret) + p->uartclk = rate; + +out: + p->status &= ~UPSTAT_AUTOCTS; + if (termios->c_cflag & CRTSCTS) + p->status |= UPSTAT_AUTOCTS; + + serial8250_do_set_termios(p, termios, old); +} + +static void sunway8250_set_ldisc(struct uart_port *p, struct ktermios *termios) +{ + struct uart_8250_port *up = up_to_u8250p(p); + unsigned int mcr = p->serial_in(p, UART_MCR); + + if (up->capabilities & UART_CAP_IRDA) { + if (termios->c_line == N_IRDA) + mcr |= SUNWAY_UART_MCR_SIRE; + else + mcr &= ~SUNWAY_UART_MCR_SIRE; + + p->serial_out(p, UART_MCR, mcr); + } + serial8250_do_set_ldisc(p, termios); +} + +/* + * sunway8250_fallback_dma_filter will prevent the UART from getting just any free + * channel on platforms that have DMA engines, but don't have any channels + * assigned to the UART. + * + * REVISIT: This is a work around for limitation in the DMA Engine API. Once the + * core problem is fixed, this function is no longer needed. + */ +static bool sunway8250_fallback_dma_filter(struct dma_chan *chan, void *param) +{ + return false; +} + +static bool sunway8250_idma_filter(struct dma_chan *chan, void *param) +{ + return param == chan->device->dev; +} + +/* + * divisor = div(I) + div(F) + * "I" means integer, "F" means fractional + * quot = div(I) = clk / (16 * baud) + * frac = div(F) * 2^dlf_size + * + * let rem = clk % (16 * baud) + * we have: div(F) * (16 * baud) = rem + * so frac = 2^dlf_size * rem / (16 * baud) = (rem << dlf_size) / (16 * baud) + */ +static unsigned int sunway8250_get_divisor(struct uart_port *p, + unsigned int baud, + unsigned int *frac) +{ + unsigned int quot, rem, base_baud = baud * 16; + struct sunway8250_data *d = p->private_data; + + quot = p->uartclk / base_baud; + rem = p->uartclk % base_baud; + *frac = DIV_ROUND_CLOSEST(rem << d->dlf_size, base_baud); + + return quot; +} + +static void sunway8250_set_divisor(struct uart_port *p, unsigned int baud, + unsigned int quot, unsigned int quot_frac) +{ + sunway8250_writel_ext(p, SUNWAY_UART_DLF, quot_frac); + serial8250_do_set_divisor(p, baud, quot, quot_frac); +} + +static void sunway8250_quirks(struct uart_port *p, struct sunway8250_data *data) +{ + if (p->dev->of_node) { + struct device_node *np = p->dev->of_node; + int id; + + /* get index of serial line, if found in DT aliases */ + id = of_alias_get_id(np, "serial"); + if (id >= 0) + p->line = id; +#ifdef CONFIG_64BIT + if (of_device_is_compatible(np, "cavium,octeon-3860-uart")) { + p->serial_in = sunway8250_serial_inq; + p->serial_out = sunway8250_serial_outq; + p->flags = UPF_SKIP_TEST | UPF_SHARE_IRQ | UPF_FIXED_TYPE; + p->type = PORT_OCTEON; + data->usr_reg = 0x27; + data->skip_autocfg = true; + } +#endif + if (of_device_is_big_endian(p->dev->of_node)) { + p->iotype = UPIO_MEM32BE; + p->serial_in = sunway8250_serial_in32be; + p->serial_out = sunway8250_serial_out32be; + } + if (of_device_is_compatible(np, "marvell,armada-38x-uart")) + p->serial_out = sunway8250_serial_out38x; + + } else if (acpi_dev_present("APMC0D08", NULL, -1)) { + p->iotype = UPIO_MEM32; + p->regshift = 2; + p->serial_in = sunway8250_serial_in32; + data->uart_16550_compatible = true; + } + + /* Platforms with iDMA 64-bit */ + if (platform_get_resource_byname(to_platform_device(p->dev), + IORESOURCE_MEM, "lpss_priv")) { + data->dma.rx_param = p->dev->parent; + data->dma.tx_param = p->dev->parent; + data->dma.fn = sunway8250_idma_filter; + } +} + +static void sunway8250_setup_port(struct uart_port *p) +{ + struct uart_8250_port *up = up_to_u8250p(p); + u32 reg; + + /* + * If the Component Version Register returns zero, we know that + * ADDITIONAL_FEATURES are not enabled. No need to go any further. + */ + reg = sunway8250_readl_ext(p, SUNWAY_UART_UCV); + if (!reg) + return; + + dev_dbg(p->dev, "Designware UART version %c.%c%c\n", + (reg >> 24) & 0xff, (reg >> 16) & 0xff, (reg >> 8) & 0xff); + + sunway8250_writel_ext(p, SUNWAY_UART_DLF, ~0U); + reg = sunway8250_readl_ext(p, SUNWAY_UART_DLF); + sunway8250_writel_ext(p, SUNWAY_UART_DLF, 0); + + if (reg) { + struct sunway8250_data *d = p->private_data; + + d->dlf_size = fls(reg); + p->get_divisor = sunway8250_get_divisor; + p->set_divisor = sunway8250_set_divisor; + } + + reg = sunway8250_readl_ext(p, SUNWAY_UART_CPR); + if (!reg) + return; + + /* Select the type based on fifo */ + if (reg & SUNWAY_UART_CPR_FIFO_MODE) { + p->type = PORT_16550A; + p->flags |= UPF_FIXED_TYPE; + p->fifosize = SUNWAY_UART_CPR_FIFO_SIZE(reg); + up->capabilities = UART_CAP_FIFO; + } + + if (reg & SUNWAY_UART_CPR_AFCE_MODE) + up->capabilities |= UART_CAP_AFE; + + if (reg & SUNWAY_UART_CPR_SIR_MODE) + up->capabilities |= UART_CAP_IRDA; +} + +static int sunway8250_probe(struct platform_device *pdev) +{ + struct uart_8250_port uart = {}; + struct resource *regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); + int irq = platform_get_irq(pdev, 0); + struct uart_port *p = &uart.port; + struct device *dev = &pdev->dev; + struct sunway8250_data *data; + int err; + u32 val; + + if (!regs) { + dev_err(dev, "no registers defined\n"); + return -EINVAL; + } + + if (irq < 0) { + if (irq != -EPROBE_DEFER) + dev_err(dev, "cannot get irq\n"); + irq = 0; // Set serial poll mode + } + + spin_lock_init(&p->lock); + p->mapbase = regs->start; + p->irq = irq; + p->handle_irq = sunway8250_handle_irq; + p->pm = sunway8250_do_pm; + p->type = PORT_8250; + p->flags = UPF_SHARE_IRQ | UPF_FIXED_PORT; + p->dev = dev; + p->iotype = UPIO_MEM; + p->serial_in = sunway8250_serial_in; + p->serial_out = sunway8250_serial_out; + p->set_ldisc = sunway8250_set_ldisc; + p->set_termios = sunway8250_set_termios; + + p->membase = devm_ioremap(dev, regs->start, resource_size(regs)); + if (!p->membase) + return -ENOMEM; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->dma.fn = sunway8250_fallback_dma_filter; + data->usr_reg = SUNWAY_UART_USR; + p->private_data = data; + + data->uart_16550_compatible = device_property_read_bool(dev, + "snps,uart-16550-compatible"); + + err = device_property_read_u32(dev, "reg-shift", &val); + if (!err) + p->regshift = val; + + err = device_property_read_u32(dev, "reg-io-width", &val); + if (!err && val == 4) { + p->iotype = UPIO_MEM32; + p->serial_in = sunway8250_serial_in32; + p->serial_out = sunway8250_serial_out32; + } + + if (device_property_read_bool(dev, "dcd-override")) { + /* Always report DCD as active */ + data->msr_mask_on |= UART_MSR_DCD; + data->msr_mask_off |= UART_MSR_DDCD; + } + + if (device_property_read_bool(dev, "dsr-override")) { + /* Always report DSR as active */ + data->msr_mask_on |= UART_MSR_DSR; + data->msr_mask_off |= UART_MSR_DDSR; + } + + if (device_property_read_bool(dev, "cts-override")) { + /* Always report CTS as active */ + data->msr_mask_on |= UART_MSR_CTS; + data->msr_mask_off |= UART_MSR_DCTS; + } + + if (device_property_read_bool(dev, "ri-override")) { + /* Always report Ring indicator as inactive */ + data->msr_mask_off |= UART_MSR_RI; + data->msr_mask_off |= UART_MSR_TERI; + } + + /* Always ask for fixed clock rate from a property. */ + device_property_read_u32(dev, "clock-frequency", &p->uartclk); + + /* If there is separate baudclk, get the rate from it. */ + data->clk = devm_clk_get(dev, "baudclk"); + if (IS_ERR(data->clk) && PTR_ERR(data->clk) != -EPROBE_DEFER) + data->clk = devm_clk_get(dev, NULL); + if (IS_ERR(data->clk) && PTR_ERR(data->clk) == -EPROBE_DEFER) + return -EPROBE_DEFER; + if (!IS_ERR_OR_NULL(data->clk)) { + err = clk_prepare_enable(data->clk); + if (err) + dev_warn(dev, "could not enable optional baudclk: %d\n", + err); + else + p->uartclk = clk_get_rate(data->clk); + } + + /* If no clock rate is defined, fail. */ + if (!p->uartclk) { + dev_err(dev, "clock rate not defined\n"); + err = -EINVAL; + goto err_clk; + } + + data->pclk = devm_clk_get(dev, "apb_pclk"); + if (IS_ERR(data->pclk) && PTR_ERR(data->pclk) == -EPROBE_DEFER) { + err = -EPROBE_DEFER; + goto err_clk; + } + if (!IS_ERR(data->pclk)) { + err = clk_prepare_enable(data->pclk); + if (err) { + dev_err(dev, "could not enable apb_pclk\n"); + goto err_clk; + } + } + + data->rst = devm_reset_control_get_optional_exclusive(dev, NULL); + if (IS_ERR(data->rst)) { + err = PTR_ERR(data->rst); + goto err_pclk; + } + reset_control_deassert(data->rst); + + sunway8250_quirks(p, data); + + /* If the Busy Functionality is not implemented, don't handle it */ + if (data->uart_16550_compatible) + p->handle_irq = NULL; + + if (!data->skip_autocfg) + sunway8250_setup_port(p); + + /* If we have a valid fifosize, try hooking up DMA */ + if (p->fifosize) { + data->dma.rxconf.src_maxburst = p->fifosize / 4; + data->dma.txconf.dst_maxburst = p->fifosize / 4; + uart.dma = &data->dma; + } + + data->line = serial8250_register_8250_port(&uart); + if (data->line < 0) { + err = data->line; + goto err_reset; + } + + platform_set_drvdata(pdev, data); + + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + + return 0; + +err_reset: + reset_control_assert(data->rst); + +err_pclk: + if (!IS_ERR(data->pclk)) + clk_disable_unprepare(data->pclk); + +err_clk: + if (!IS_ERR(data->clk)) + clk_disable_unprepare(data->clk); + + return err; +} + +static int sunway8250_remove(struct platform_device *pdev) +{ + struct sunway8250_data *data = platform_get_drvdata(pdev); + + pm_runtime_get_sync(&pdev->dev); + + serial8250_unregister_port(data->line); + + reset_control_assert(data->rst); + + if (!IS_ERR(data->pclk)) + clk_disable_unprepare(data->pclk); + + if (!IS_ERR(data->clk)) + clk_disable_unprepare(data->clk); + + pm_runtime_disable(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int sunway8250_suspend(struct device *dev) +{ + struct sunway8250_data *data = dev_get_drvdata(dev); + + serial8250_suspend_port(data->line); + + return 0; +} + +static int sunway8250_resume(struct device *dev) +{ + struct sunway8250_data *data = dev_get_drvdata(dev); + + serial8250_resume_port(data->line); + + return 0; +} +#endif /* CONFIG_PM_SLEEP */ + +#ifdef CONFIG_PM +static int sunway8250_runtime_suspend(struct device *dev) +{ + struct sunway8250_data *data = dev_get_drvdata(dev); + + if (!IS_ERR(data->clk)) + clk_disable_unprepare(data->clk); + + if (!IS_ERR(data->pclk)) + clk_disable_unprepare(data->pclk); + + return 0; +} + +static int sunway8250_runtime_resume(struct device *dev) +{ + struct sunway8250_data *data = dev_get_drvdata(dev); + + if (!IS_ERR(data->pclk)) + clk_prepare_enable(data->pclk); + + if (!IS_ERR(data->clk)) + clk_prepare_enable(data->clk); + + return 0; +} +#endif + +static const struct dev_pm_ops sunway8250_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(sunway8250_suspend, sunway8250_resume) + SET_RUNTIME_PM_OPS(sunway8250_runtime_suspend, sunway8250_runtime_resume, NULL) +}; + +static const struct of_device_id sunway8250_of_match[] = { + { .compatible = "sw6,sunway-apb-uart" }, + { .compatible = "cavium,octeon-3860-uart" }, + { .compatible = "marvell,armada-38x-uart" }, + { .compatible = "renesas,rzn1-uart" }, + { /* Sentinel */ } +}; +MODULE_DEVICE_TABLE(of, sunway8250_of_match); + +static const struct acpi_device_id sunway8250_acpi_match[] = { + { "INT33C4", 0 }, + { "INT33C5", 0 }, + { "INT3434", 0 }, + { "INT3435", 0 }, + { "80860F0A", 0 }, + { "8086228A", 0 }, + { "APMC0D08", 0}, + { "AMD0020", 0 }, + { "AMDI0020", 0 }, + { "BRCM2032", 0 }, + { "HISI0031", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, sunway8250_acpi_match); + +static struct platform_driver sunway8250_platform_driver = { + .driver = { + .name = "sunway-apb-uart", + .pm = &sunway8250_pm_ops, + .of_match_table = sunway8250_of_match, + .acpi_match_table = ACPI_PTR(sunway8250_acpi_match), + }, + .probe = sunway8250_probe, + .remove = sunway8250_remove, +}; + +module_platform_driver(sunway8250_platform_driver); + +MODULE_AUTHOR("Jamie Iles"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Synopsys DesignWare 8250 serial port driver"); +MODULE_ALIAS("platform:sunway-apb-uart"); diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig index ee17cf5c44c6..e8edd9388d76 100644 --- a/drivers/tty/serial/8250/Kconfig +++ b/drivers/tty/serial/8250/Kconfig @@ -407,6 +407,13 @@ config SERIAL_8250_DW Selecting this option will enable handling of the extra features present in the Synopsys DesignWare APB UART. +config SERIAL_8250_SUNWAY + tristate "Support for SW6B Builtin Synopsys DesignWare 8250 quirks" + depends on SERIAL_8250 && SW64 + help + Selecting this option will enable handling of the extra features + present in the Synopsys DesignWare APB UART of SW6. + config SERIAL_8250_EM tristate "Support for Emma Mobile integrated serial port" depends on SERIAL_8250 && HAVE_CLK diff --git a/drivers/tty/serial/8250/Makefile b/drivers/tty/serial/8250/Makefile index 628b75be312e..8186ea891405 100644 --- a/drivers/tty/serial/8250/Makefile +++ b/drivers/tty/serial/8250/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_SERIAL_8250_FSL) += 8250_fsl.o obj-$(CONFIG_SERIAL_8250_MEN_MCB) += 8250_men_mcb.o obj-$(CONFIG_SERIAL_8250_DFL) += 8250_dfl.o obj-$(CONFIG_SERIAL_8250_DW) += 8250_dw.o +obj-$(CONFIG_SERIAL_8250_SUNWAY) += 8250_sunway.o obj-$(CONFIG_SERIAL_8250_EM) += 8250_em.o obj-$(CONFIG_SERIAL_8250_IOC3) += 8250_ioc3.o obj-$(CONFIG_SERIAL_8250_OMAP) += 8250_omap.o -- Gitee From 64a3a5ce028b276327a93876bac1b4587374a93d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:55:58 +0800 Subject: [PATCH 084/524] drivers: usb: add sw64 support Add usb drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/usb/host/pci-quirks.c | 127 ++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c index b96d9062a083..80e5ca7b940f 100644 --- a/drivers/usb/host/pci-quirks.c +++ b/drivers/usb/host/pci-quirks.c @@ -1292,3 +1292,130 @@ static void quirk_usb_early_handoff(struct pci_dev *pdev) } DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_SERIAL_USB, 8, quirk_usb_early_handoff); + +#ifdef CONFIG_SW64 +#include +#define XHCI_STS_FATAL (1 << 2) +#define XHCI_STS_EINT (1 << 3) +#define XHCI_STS_PORT (1 << 4) +#define XHCI_STS_SRE (1 << 10) +#define STS_RW1C_BITS (XHCI_STS_FATAL | XHCI_STS_EINT | XHCI_STS_PORT | XHCI_STS_SRE) + +static void +fixup_usb_xhci_reset(struct pci_dev *dev) +{ + void __iomem *op_reg_base; + int timeout; + u32 xhci_command; + u32 tmp, val; + void __iomem *base; + struct pci_controller *hose = pci_bus_to_pci_controller(dev->bus); + unsigned long offset; + int ext_cap_offset; + int retries = 3; + + pci_read_config_dword(dev, PCI_COMMAND, &tmp); + tmp |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + pci_write_config_dword(dev, PCI_COMMAND, tmp); + + pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, &tmp); + if (tmp & PCI_BASE_ADDRESS_MEM_TYPE_MASK) { + pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, &val); + offset = (unsigned long)(val) << 32 | (tmp & (~0xf)); + } else + offset = (unsigned long)(tmp & (~0xf)); + + if (offset == 0) + return; + + base = (void *)__va(SW64_PCI_IO_BASE(hose->node, hose->index) | offset); + + ext_cap_offset = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_LEGACY); + if (!ext_cap_offset) + goto hc_init; + + val = readl(base + ext_cap_offset); + + if ((dev->vendor == PCI_VENDOR_ID_TI && dev->device == 0x8241) || + (dev->vendor == PCI_VENDOR_ID_RENESAS + && dev->device == 0x0014)) { + val = (val | XHCI_HC_OS_OWNED) & ~XHCI_HC_BIOS_OWNED; + writel(val, base + ext_cap_offset); + } + + if (val & XHCI_HC_BIOS_OWNED) { + writel(val | XHCI_HC_OS_OWNED, base + ext_cap_offset); + + timeout = handshake(base + ext_cap_offset, XHCI_HC_BIOS_OWNED, + 0, 1000000, 10); + if (timeout) { + pr_err("xHCI BIOS handoff failed (BIOS bug ?) %08x\n", val); + writel(val & ~XHCI_HC_BIOS_OWNED, base + ext_cap_offset); + } + } + + val = readl(base + ext_cap_offset + XHCI_LEGACY_CONTROL_OFFSET); + val &= XHCI_LEGACY_DISABLE_SMI; + val |= XHCI_LEGACY_SMI_EVENTS; + writel(val, base + ext_cap_offset + XHCI_LEGACY_CONTROL_OFFSET); + +hc_init: + if (dev->vendor == PCI_VENDOR_ID_INTEL) + usb_enable_intel_xhci_ports(dev); + + op_reg_base = base + XHCI_HC_LENGTH(readl(base)); + + timeout = handshake(op_reg_base + XHCI_STS_OFFSET, XHCI_STS_CNR, 0, + 5000000, 10); + if (timeout) { + val = readl(op_reg_base + XHCI_STS_OFFSET); + pr_err("xHCI HW not ready after 5 sec (HC bug?) status = 0x%x\n", val); + } + + xhci_command = readl(op_reg_base + XHCI_CMD_OFFSET); + xhci_command |= 0x2; + writel(xhci_command, op_reg_base + XHCI_CMD_OFFSET); + + timeout = handshake(op_reg_base + XHCI_CMD_OFFSET, + 0x2, 0, 10 * 1000 * 1000, 125); + if (timeout) + pr_err("xHCI BIOS handoff time out\n"); + +retry: + val = readl(op_reg_base + XHCI_STS_OFFSET); + val |= STS_RW1C_BITS; + writel(val, op_reg_base + XHCI_STS_OFFSET); + val = readl(op_reg_base + XHCI_STS_OFFSET); + + if ((val & STS_RW1C_BITS) && retries--) { + pr_err("clear USB Status Register (status = %#x) failed, retry\n", val); + goto retry; + } + + val = readl(op_reg_base + XHCI_CMD_OFFSET); + val &= ~(XHCI_CMD_RUN | XHCI_IRQS); + writel(val, op_reg_base + XHCI_CMD_OFFSET); + timeout = handshake(op_reg_base + XHCI_STS_OFFSET, XHCI_STS_HALT, 1, + XHCI_MAX_HALT_USEC, 125); + if (timeout) { + val = readl(op_reg_base + XHCI_STS_OFFSET); + pr_err("xHCI HW did not halt within %d usec status = 0x%x\n", + XHCI_MAX_HALT_USEC, val); + } + + xhci_command = readl(op_reg_base + XHCI_CMD_OFFSET); + xhci_command |= 0x2; + writel(xhci_command, op_reg_base + XHCI_CMD_OFFSET); + + timeout = handshake(op_reg_base + XHCI_CMD_OFFSET, + 0x2, 0, 10 * 1000 * 1000, 125); + if (timeout) + pr_err("xHCI BIOS handoff time out\n"); + + pci_read_config_dword(dev, PCI_COMMAND, &tmp); + tmp &= ~(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + pci_write_config_dword(dev, PCI_COMMAND, tmp); +} +DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_ANY_ID, PCI_ANY_ID, + PCI_CLASS_SERIAL_USB_XHCI, 0, fixup_usb_xhci_reset); +#endif -- Gitee From 00a01a93cc8f7f96686aee129eb8fd111650d3e1 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 18 Jan 2024 16:56:19 +0800 Subject: [PATCH 085/524] drivers: vfio: add sw64 support Add vfio drivers for SW64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/vfio/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index f7103ae71202..a88c09330fb3 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -39,7 +39,7 @@ config VFIO_GROUP config VFIO_CONTAINER bool "Support for the VFIO container /dev/vfio/vfio" - select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64 || LOONGARCH) + select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64 || LOONGARCH || SW64) depends on VFIO_GROUP default y help -- Gitee From 5713732cb28ce438dcd59074f4c7f22f966c0aec Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Tue, 16 Apr 2024 08:51:51 +0800 Subject: [PATCH 086/524] sw64: add arch-specific SYM_END definition Redefine SYM_END to make low version compiler work. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/linkage.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/sw_64/include/asm/linkage.h b/arch/sw_64/include/asm/linkage.h index 85b279f6211e..b0703086e902 100644 --- a/arch/sw_64/include/asm/linkage.h +++ b/arch/sw_64/include/asm/linkage.h @@ -6,4 +6,8 @@ #define SYSCALL_ALIAS(alias, name) \ asm (#alias " = " #name "\n\t.globl " #alias) +#define SYM_END(name, sym_type) \ + .type name sym_type ASM_NL \ + .size name, .-name + #endif /* _ASM_SW64_LINKAGE_H */ -- Gitee From 0f64423414543d73d4de2bb251db2fec4d0e140c Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Thu, 16 Nov 2023 11:08:55 +0800 Subject: [PATCH 087/524] sw64: remove useless current_policy This variable is no longer needed and can be removed. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpufreq.h | 2 -- arch/sw_64/platform/cpufreq_xuelang.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/arch/sw_64/include/asm/cpufreq.h b/arch/sw_64/include/asm/cpufreq.h index cf47f1fc6866..e6f1d16e5620 100644 --- a/arch/sw_64/include/asm/cpufreq.h +++ b/arch/sw_64/include/asm/cpufreq.h @@ -11,8 +11,6 @@ struct clk; -extern char curruent_policy[CPUFREQ_NAME_LEN]; - struct clk_ops { void (*init)(struct clk *clk); void (*enable)(struct clk *clk); diff --git a/arch/sw_64/platform/cpufreq_xuelang.c b/arch/sw_64/platform/cpufreq_xuelang.c index 1259e58dc874..e639fdc34fb9 100644 --- a/arch/sw_64/platform/cpufreq_xuelang.c +++ b/arch/sw_64/platform/cpufreq_xuelang.c @@ -70,8 +70,6 @@ static int __init sw64_cpufreq_init(void) } arch_initcall(sw64_cpufreq_init); -char curruent_policy[CPUFREQ_NAME_LEN]; - static struct clk cpu_clk = { .name = "cpu_clk", .flags = CLK_ALWAYS_ENABLED | CLK_RATE_PROPAGATES, -- Gitee From ad1588437033c57bf64873fc963197bba993b58c Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 15 Nov 2023 14:12:45 +0800 Subject: [PATCH 088/524] sw64: extern vdso_start and vdso_end as char[] Extern vdso_start and vdso_end as char[] to avoid gcc warning '__builtin_memcmp_eq' specified bound 4 exceeds source size 1 [-Wstringop-overread] | if (memcmp(&vdso_start, "\177ELF", 4)) { Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/vdso.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/kernel/vdso.c b/arch/sw_64/kernel/vdso.c index b4126cbaa4bd..15f1f244a82f 100644 --- a/arch/sw_64/kernel/vdso.c +++ b/arch/sw_64/kernel/vdso.c @@ -21,7 +21,7 @@ #include -extern char vdso_start, vdso_end; +extern char vdso_start[], vdso_end[]; static unsigned long vdso_pages; static struct page **vdso_pagelist; @@ -40,14 +40,14 @@ static int __init vdso_init(void) { int i; - if (memcmp(&vdso_start, "\177ELF", 4)) { + if (memcmp(vdso_start, "\177ELF", 4)) { pr_err("vDSO is not a valid ELF object!\n"); return -EINVAL; } - vdso_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT; + vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n", - vdso_pages + 1, vdso_pages, &vdso_start, 1L, vdso_data); + vdso_pages + 1, vdso_pages, vdso_start, 1L, vdso_data); /* Allocate the vDSO pagelist, plus a page for the data. */ vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *), @@ -60,7 +60,7 @@ static int __init vdso_init(void) /* Grab the vDSO code pages. */ for (i = 0; i < vdso_pages; i++) - vdso_pagelist[i + 1] = virt_to_page(&vdso_start + i * PAGE_SIZE); + vdso_pagelist[i + 1] = virt_to_page(vdso_start + i * PAGE_SIZE); /* Populate the special mapping structures */ vdso_spec[0] = (struct vm_special_mapping) { -- Gitee From c3e8be912a65798fa87a701b8d90fdf55fe7967a Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Mon, 20 Nov 2023 09:22:27 +0800 Subject: [PATCH 089/524] sw64: fix hugepage support Add missing pud_write() and more hugepage granularity support. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/page.h | 2 +- arch/sw_64/include/asm/pgtable.h | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/page.h b/arch/sw_64/include/asm/page.h index 68b4f2fc1b48..3a95151b67f8 100644 --- a/arch/sw_64/include/asm/page.h +++ b/arch/sw_64/include/asm/page.h @@ -15,7 +15,7 @@ #define HPAGE_MASK (~(HPAGE_SIZE - 1)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) -#define HUGE_MAX_HSTATE 2 +#define HUGE_MAX_HSTATE 3 #ifdef __KERNEL__ #ifndef __ASSEMBLY__ diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 0b1f825eb74c..a7741a67e687 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -233,6 +233,18 @@ static inline void set_pte(pte_t *ptep, pte_t pteval) } } +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pteval) +{ + set_pte(ptep, pteval); +} + +#define pud_write pud_write +static inline int pud_write(pud_t pud) +{ + return !(pud_val(pud) & _PAGE_FOW); +} + static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) { pte_t pte; -- Gitee From 7569f59b4cbe517cd5e7e82818aa1576ba87cfe7 Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Mon, 20 Nov 2023 16:35:37 +0800 Subject: [PATCH 090/524] sw64: perf: hide hardware performance events in guest os Hardware performance events are not supported in guest os, and software performance events are used by default. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/perf_event.c | 57 +++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/arch/sw_64/kernel/perf_event.c b/arch/sw_64/kernel/perf_event.c index 83bb051be9de..855f4246433c 100644 --- a/arch/sw_64/kernel/perf_event.c +++ b/arch/sw_64/kernel/perf_event.c @@ -729,6 +729,38 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, walk_stackframe(NULL, regs, callchain_trace, entry); } +/* + * Init call to initialise performance events at kernel startup. + */ +int __init init_hw_perf_events(void) +{ + pr_info("Performance Events: "); + + if (!supported_cpu()) { + pr_cont("Unsupported CPU type!\n"); + return 0; + } + + if (is_in_guest()) { + pr_cont("No PMU driver, software events only.\n"); + return 0; + } + + pr_cont("Supported CPU type!\n"); + + /* Override performance counter IRQ vector */ + + perf_irq = sw64_perf_event_irq_handler; + + /* And set up PMU specification */ + sw64_pmu = &core3_pmu; + + perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); + + return 0; +} +early_initcall(init_hw_perf_events); + /* * Gets the perf_instruction_pointer and perf_misc_flags for guest os. */ @@ -760,28 +792,3 @@ unsigned long perf_misc_flags(struct pt_regs *regs) return misc; } - -/* - * Init call to initialise performance events at kernel startup. - */ -int __init init_hw_perf_events(void) -{ - if (!supported_cpu()) { - pr_info("Performance events: Unsupported CPU type!\n"); - return 0; - } - - pr_info("Performance events: Supported CPU type!\n"); - - /* Override performance counter IRQ vector */ - - perf_irq = sw64_perf_event_irq_handler; - - /* And set up PMU specification */ - sw64_pmu = &core3_pmu; - - perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); - - return 0; -} -early_initcall(init_hw_perf_events); -- Gitee From a04cd29f30f604626ee727f9621066ecd81ebb87 Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Mon, 20 Nov 2023 14:14:42 +0800 Subject: [PATCH 091/524] sw64: uapi: fix a compile error for headers_install The following error was reported in uapi/asm/fpu.h when make headers_install: - error: arch/sw_64/include/uapi/asm/fpu.h: - leak CONFIG_SUBARCH_C3B to user-space Fix it based on the error information. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/fpu.h | 8 -------- arch/sw_64/kernel/process.c | 4 ++++ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/arch/sw_64/include/uapi/asm/fpu.h b/arch/sw_64/include/uapi/asm/fpu.h index 8945816c542b..7a1514c978b9 100644 --- a/arch/sw_64/include/uapi/asm/fpu.h +++ b/arch/sw_64/include/uapi/asm/fpu.h @@ -6,11 +6,7 @@ * SW-64 floating-point control register defines: */ #define FPCR_DNOD (1UL << 47) /* denorm INV trap disable */ -#ifdef CONFIG_SUBARCH_C3B -#define FPCR_DNZ (1UL << 48) /* denorms to zero */ -#else #define FPCR_DNOE (1UL << 48) /* hardware denormal support */ -#endif #define FPCR_INVD (1UL << 49) /* invalid op disable (opt.) */ #define FPCR_DZED (1UL << 50) /* division by zero disable (opt.) */ #define FPCR_OVFD (1UL << 51) /* overflow disable (optional) */ @@ -34,11 +30,7 @@ #define FPCR_MASK 0xffff800000000000L -#ifdef CONFIG_SUBARCH_C3B #define FPCR_INIT FPCR_DYN_NORMAL -#else -#define FPCR_INIT (FPCR_DYN_NORMAL | FPCR_DNOE) -#endif /* status bit coming from hardware fpcr . definde by fire3 */ #define FPCR_STATUS_INV0 (1UL << 52) diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c index fa58a0de4368..4f3a3a8b4123 100644 --- a/arch/sw_64/kernel/process.c +++ b/arch/sw_64/kernel/process.c @@ -35,7 +35,11 @@ flush_thread(void) * with respect to the FPU. This is all exceptions disabled. */ current_thread_info()->ieee_state = 0; +#ifdef CONFIG_SUBARCH_C3B wrfpcr(FPCR_INIT | ieee_swcr_to_fpcr(0)); +#else + wrfpcr(FPCR_INIT | FPCR_DNOE | ieee_swcr_to_fpcr(0)); +#endif /* Clean slate for TLS. */ current_thread_info()->pcb.tp = 0; -- Gitee From 9aade47173330d0664377a69a927e6514b57e541 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 4 Dec 2023 13:41:08 +0800 Subject: [PATCH 092/524] sw64: pci: export some pci functions Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/controller/pci-sunway.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 036994ffde38..e9314e4cb86e 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -377,6 +377,7 @@ struct pci_controller *bus_num_to_pci_controller(unsigned long bus_num) return NULL; } +EXPORT_SYMBOL(bus_num_to_pci_controller); struct pci_controller *pci_bus_to_pci_controller(const struct pci_bus *bus) { @@ -391,6 +392,7 @@ struct pci_controller *pci_bus_to_pci_controller(const struct pci_bus *bus) cfg = (struct pci_config_window *)bus->sysdata; return (struct pci_controller *)(cfg->priv); } +EXPORT_SYMBOL(pci_bus_to_pci_controller); /** * PCIe Root Complex read config space operations -- Gitee From ff09bbd71339cc6f8d49170fdea19884933709f2 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 4 Dec 2023 15:55:49 +0800 Subject: [PATCH 093/524] sw64: numa: fix compile error when CONFIG_ACPI_NUMA=n Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/numa.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/mm/numa.c b/arch/sw_64/mm/numa.c index fcf1f97a7840..e46372f3ac77 100644 --- a/arch/sw_64/mm/numa.c +++ b/arch/sw_64/mm/numa.c @@ -344,10 +344,30 @@ static int __init manual_numa_init(void) return 0; } +#ifdef CONFIG_ACPI_NUMA +static int __init sw64_acpi_numa_init(void) +{ + int ret; + + ret = acpi_numa_init(); + if (ret) { + pr_info("Failed to initialise from firmware\n"); + return ret; + } + + return srat_disabled() ? -EINVAL : 0; +} +#else +static int __init sw64_acpi_numa_init(void) +{ + return -EOPNOTSUPP; +} +#endif + void __init sw64_numa_init(void) { if (!numa_off) { - if (!acpi_disabled && !numa_init(acpi_numa_init)) + if (!acpi_disabled && !numa_init(sw64_acpi_numa_init)) return; if (acpi_disabled && !numa_init(of_numa_init)) return; -- Gitee From a78861c567da6a0577e6913e460533ad74329bdc Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Mon, 20 Nov 2023 15:45:02 +0800 Subject: [PATCH 094/524] sw64: perf: do all event checks in sw64_pmu_event_init() It used to check perf events in two functions. To make the code clear, merge these checks into sw64_pmu_event_init(). BTW, configure PMC with `hwc->config` instead of `hwc->event_base`, and this will makes no functional changes intended. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/perf_event.c | 101 ++++++++++++++++----------------- 1 file changed, 48 insertions(+), 53 deletions(-) diff --git a/arch/sw_64/kernel/perf_event.c b/arch/sw_64/kernel/perf_event.c index 855f4246433c..3c6a8283b1e6 100644 --- a/arch/sw_64/kernel/perf_event.c +++ b/arch/sw_64/kernel/perf_event.c @@ -378,7 +378,7 @@ static void sw64_pmu_start(struct perf_event *event, int flags) /* counting in selected modes, for both counters */ wrperfmon(PMC_CMD_PM, hwc->config_base); - wrperfmon(PMC_CMD_EVENT_BASE + hwc->idx, hwc->event_base); + wrperfmon(PMC_CMD_EVENT_BASE + hwc->idx, hwc->config); wrperfmon(PMC_CMD_ENABLE, PMC_ENABLE_BASE + hwc->idx); } @@ -474,49 +474,10 @@ static void hw_perf_event_destroy(struct perf_event *event) /* Nothing to be done! */ } -static int __hw_perf_event_init(struct perf_event *event) +static void __hw_perf_event_init(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; struct hw_perf_event *hwc = &event->hw; - const struct sw64_perf_event *event_type; - - - /* - * SW64 does not have per-counter usr/os/guest/host bits, - * we can distinguish exclude_user and exclude_kernel by - * sample mode. - */ - if (event->attr.exclude_hv || event->attr.exclude_idle || - event->attr.exclude_host || event->attr.exclude_guest) - return -EINVAL; - - /* - * SW64 does not support precise ip feature, and system hang when - * detecting precise_ip by perf_event_attr__set_max_precise_ip - * in userspace - */ - if (attr->precise_ip != 0) - return -EOPNOTSUPP; - - /* SW64 has fixed counter for given event type */ - if (attr->type == PERF_TYPE_HARDWARE) { - if (attr->config >= sw64_pmu->max_events) - return -EINVAL; - event_type = sw64_pmu->map_hw_event(attr->config); - hwc->idx = event_type->counter; - hwc->event_base = event_type->event; - } else if (attr->type == PERF_TYPE_HW_CACHE) { - event_type = sw64_pmu->map_cache_event(attr->config); - if (IS_ERR(event_type)) /* */ - return PTR_ERR(event_type); - hwc->idx = event_type->counter; - hwc->event_base = event_type->event; - } else { /* PERF_TYPE_RAW */ - if (!sw64_pmu->raw_event_valid(attr->config)) - return -EINVAL; - hwc->idx = attr->config >> 8; /* counter selector */ - hwc->event_base = attr->config & 0xff; /* event selector */ - } hwc->config_base = SW64_PERFCTRL_AM; @@ -525,8 +486,6 @@ static int __hw_perf_event_init(struct perf_event *event) if (attr->exclude_kernel) hwc->config_base = SW64_PERFCTRL_UM; - hwc->config = attr->config; - if (!is_sampling_event(event)) pr_debug("not sampling event\n"); @@ -537,8 +496,6 @@ static int __hw_perf_event_init(struct perf_event *event) hwc->last_period = hwc->sample_period; local64_set(&hwc->period_left, hwc->sample_period); } - - return 0; } /* @@ -546,28 +503,66 @@ static int __hw_perf_event_init(struct perf_event *event) */ static int sw64_pmu_event_init(struct perf_event *event) { - int err; + struct perf_event_attr *attr = &event->attr; + struct hw_perf_event *hwc = &event->hw; + const struct sw64_perf_event *event_type; + + if (!sw64_pmu) + return -ENODEV; /* does not support taken branch sampling */ if (has_branch_stack(event)) return -EOPNOTSUPP; - switch (event->attr.type) { - case PERF_TYPE_RAW: + /* + * SW64 does not have per-counter usr/os/guest/host bits, + * we can distinguish exclude_user and exclude_kernel by + * sample mode. + */ + if (attr->exclude_hv || attr->exclude_idle || + attr->exclude_host || attr->exclude_guest) + return -EINVAL; + + if (attr->exclude_user && attr->exclude_kernel) + return -EOPNOTSUPP; + /* + * SW64 does not support precise ip feature, and system hang when + * detecting precise_ip by perf_event_attr__set_max_precise_ip + * in userspace + */ + if (attr->precise_ip != 0) + return -EOPNOTSUPP; + + /* SW64 has fixed counter for given event type */ + switch (attr->type) { case PERF_TYPE_HARDWARE: + if (attr->config >= sw64_pmu->max_events) + return -EINVAL; + event_type = sw64_pmu->map_hw_event(attr->config); + hwc->idx = event_type->counter; + hwc->config = event_type->event; + break; case PERF_TYPE_HW_CACHE: + event_type = sw64_pmu->map_cache_event(attr->config); + if (IS_ERR(event_type)) + return PTR_ERR(event_type); + hwc->idx = event_type->counter; + hwc->config = event_type->event; + break; + case PERF_TYPE_RAW: + if (!sw64_pmu->raw_event_valid(attr->config)) + return -EINVAL; + hwc->idx = attr->config >> 8; /* counter selector */ + hwc->config = attr->config & 0xff; /* event selector */ break; default: return -ENOENT; } - if (!sw64_pmu) - return -ENODEV; - /* Do the real initialisation work. */ - err = __hw_perf_event_init(event); + __hw_perf_event_init(event); - return err; + return 0; } static struct pmu pmu = { -- Gitee From 26cdca75221252777af4cb0c105e10eb799da2a5 Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Tue, 5 Dec 2023 09:17:26 +0800 Subject: [PATCH 095/524] sw64: fix lockdep error Initialize a dynamically allocated struct bin_attribute so we can make lockdep happy. This is a new requirement for attributes and initially this is only needed when lockdep is enabled. Lockdep gives a nice error when your attribute is added to sysfs if you don't have this. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/pci/pci-sysfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/pci/pci-sysfs.c b/arch/sw_64/pci/pci-sysfs.c index 5b52a534fa80..fd097d52b16a 100644 --- a/arch/sw_64/pci/pci-sysfs.c +++ b/arch/sw_64/pci/pci-sysfs.c @@ -200,6 +200,7 @@ static int pci_create_attr(struct pci_dev *pdev, int num) return -ENOMEM; attr_name = (char *)(attr + res_count); + sysfs_bin_attr_init(attr); pdev->res_attr[num] = attr; retval = pci_create_one_attr(pdev, num, attr_name, suffix, attr, sparse_base); -- Gitee From 025d62d0e667629742832bf97d1fa01407c83b20 Mon Sep 17 00:00:00 2001 From: Zhi Tongze Date: Tue, 21 Nov 2023 14:58:49 +0800 Subject: [PATCH 096/524] sw64: kgdb: add single-step debug support kgdb single-step debug relies on gdb tool, and unexpected exceptions will occur if single-step debug is interrupted. This patch adds kgdb single-step debugging to solve this problem. This patch also adds support for detach and kill commands. Signed-off-by: Zhi Tongze Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/kgdb.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/sw_64/kernel/kgdb.c b/arch/sw_64/kernel/kgdb.c index 833f72a1577c..a9b2db81c494 100644 --- a/arch/sw_64/kernel/kgdb.c +++ b/arch/sw_64/kernel/kgdb.c @@ -160,11 +160,21 @@ int kgdb_arch_handle_exception(int exception_vector, int signo, unsigned long address = -1; switch (remcom_in_buffer[0]) { + case 'D': + case 'k': case 'c': ptr = &remcom_in_buffer[1]; if (kgdb_hex2long(&ptr, &address)) kgdb_arch_set_pc(linux_regs, address); return 0; + case 's': + ptr = &remcom_in_buffer[1]; + atomic_set(&kgdb_cpu_doing_single_step, -1); + if (kgdb_hex2long(&ptr, &address)) + kgdb_arch_set_pc(linux_regs, address); + atomic_set(&kgdb_cpu_doing_single_step, raw_smp_processor_id()); + kgdb_single_step = 1; + return 0; } return -1; } -- Gitee From dd410a54b2e875d7b7c3168872ab88613937d7f5 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Sun, 19 Nov 2023 18:38:19 +0800 Subject: [PATCH 097/524] sw64: use rvpcr() to setup run mode Use rvpcr() to read CSR__VPCR instead of MMSIZE to get the system mode. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hmcall.h | 1 + arch/sw_64/include/asm/hw_init.h | 1 + arch/sw_64/kernel/setup.c | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/hmcall.h b/arch/sw_64/include/asm/hmcall.h index e3bac3016740..0bcd901f671d 100644 --- a/arch/sw_64/include/asm/hmcall.h +++ b/arch/sw_64/include/asm/hmcall.h @@ -159,6 +159,7 @@ __CALL_HMC_VOID(wrktp); #define save_ktp() wrktp() __CALL_HMC_R0(rdps, unsigned long); +__CALL_HMC_R0(rvpcr, unsigned long); __CALL_HMC_R0(rdusp, unsigned long); __CALL_HMC_W1(wrusp, unsigned long); diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h index 2078c66d1c4f..4710f15a9a56 100644 --- a/arch/sw_64/include/asm/hw_init.h +++ b/arch/sw_64/include/asm/hw_init.h @@ -7,6 +7,7 @@ #include #define MMSIZE __va(0x2040) +#define VPCR_SHIFT 44 /* * Descriptor for a cache diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 0c1ddb9b46d7..412710b17a80 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -674,7 +674,7 @@ static void __init setup_cpu_info(void) static void __init setup_run_mode(void) { - if (*(unsigned long *)MMSIZE) { + if (rvpcr() >> VPCR_SHIFT) { static_branch_disable(&run_mode_host_key); if (*(unsigned long *)MMSIZE & EMUL_FLAG) { pr_info("run mode: emul\n"); -- Gitee From f8abc2d86ab6e300603faaa176d9d9959855cd63 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Wed, 27 Dec 2023 01:10:08 +0000 Subject: [PATCH 098/524] sw64: iommu: fix the bug when vfio_iommu unpin pages When the page granularity is 8K, iova_to_phys return the wrong page address, so fix it. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu_v2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index f3e19e524210..4a8b576c2530 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -1569,7 +1569,7 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) return 0; paddr &= ~PTE_FLAGS_MASK; - paddr += iova & PAGE_MASK; + paddr += iova & ~PAGE_MASK; return paddr; } -- Gitee From 855c2f2e3a88428f646588c630e568ba62d50137 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 8 Dec 2023 17:22:31 +0800 Subject: [PATCH 099/524] sw64: pci: synchronize the OEM table ID of MCFG with BIOS Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/acpi/pci_mcfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c index 98bbb01981c5..667eb2d65432 100644 --- a/drivers/acpi/pci_mcfg.c +++ b/drivers/acpi/pci_mcfg.c @@ -198,7 +198,7 @@ static struct mcfg_fixup mcfg_quirks[] = { #ifdef CONFIG_SW64 #define _SW64_ECAM_QUIRK(rev, seg) \ - { "SUNWAY", "SUNWAY. ", rev, seg, MCFG_BUS_ANY, &sw64_pci_ecam_ops } + { "SUNWAY", "MCFG", rev, seg, MCFG_BUS_ANY, &sw64_pci_ecam_ops } #define SW64_ECAM_QUIRK(rev, node) _SW64_ECAM_QUIRK(rev, node * 8 + 0),\ _SW64_ECAM_QUIRK(rev, node * 8 + 1),\ _SW64_ECAM_QUIRK(rev, node * 8 + 2),\ -- Gitee From 0f6a440ae8fa9adc48b370355ab648e6bf1081ee Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 6 Dec 2023 14:18:51 +0800 Subject: [PATCH 100/524] sw64: smp: modify function smp_rcb_init() to receive base address We can pass the base address of smp_rcb via firmware instead of hardcoding it. For the convenience of receiving the base address passed by the firmware, this commit modify the function smp_rcb_init(). Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/smp.h | 2 +- arch/sw_64/kernel/smp.c | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h index 3a2fcf62b30c..103f16ca64a7 100644 --- a/arch/sw_64/include/asm/smp.h +++ b/arch/sw_64/include/asm/smp.h @@ -43,7 +43,7 @@ struct smp_rcb_struct { }; #define INIT_SMP_RCB ((struct smp_rcb_struct *) __va(0x820000UL)) - +extern void __init smp_rcb_init(struct smp_rcb_struct *smp_rcb_base_addr); #ifdef GENERATING_ASM_OFFSETS #define raw_smp_processor_id() (0) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 6d1aab4be1c0..8df4a200a422 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -20,7 +20,7 @@ #include "proto.h" -struct smp_rcb_struct *smp_rcb; +struct smp_rcb_struct *smp_rcb = NULL; extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; @@ -167,9 +167,12 @@ static void __init process_nr_cpu_ids(void) nr_cpu_ids = num_possible_cpus(); } -void __init smp_rcb_init(void) +void __init smp_rcb_init(struct smp_rcb_struct *smp_rcb_base_addr) { - smp_rcb = INIT_SMP_RCB; + if (smp_rcb != NULL) + return; + + smp_rcb = smp_rcb_base_addr; memset(smp_rcb, 0, sizeof(struct smp_rcb_struct)); /* Setup SMP_RCB fields that uses to activate secondary CPU */ smp_rcb->restart_entry = __smp_callin; @@ -203,7 +206,7 @@ void __init setup_smp(void) pr_info("Detected %u possible CPU(s), %u CPU(s) are present\n", nr_cpu_ids, num_present_cpus()); - smp_rcb_init(); + smp_rcb_init(INIT_SMP_RCB); } /* * Called by smp_init prepare the secondaries -- Gitee From abd461033f6d028c031fdb171786723f1cabb24f Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 6 Dec 2023 15:39:01 +0800 Subject: [PATCH 101/524] sw64: smp: add new structure to record rcid information The SW CINTC in ACPI MADT table of SW64 include some core related information. To record this information, a new structure named rcid_information has been added. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/smp.h | 22 ++++++++++++ arch/sw_64/kernel/smp.c | 68 ++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h index 103f16ca64a7..590d4391bdea 100644 --- a/arch/sw_64/include/asm/smp.h +++ b/arch/sw_64/include/asm/smp.h @@ -58,6 +58,11 @@ extern int __cpu_to_rcid[NR_CPUS]; #define cpu_to_rcid(cpu) __cpu_to_rcid[cpu] #define cpu_physical_id(cpu) __cpu_to_rcid[cpu] +static inline void set_rcid_map(unsigned int logical, int rcid) +{ + __cpu_to_rcid[logical] = rcid; +} + extern unsigned long tidle_pcb[NR_CPUS]; extern void arch_send_call_function_single_ipi(int cpu); extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); @@ -67,6 +72,23 @@ int __cpu_disable(void); void __cpu_die(unsigned int cpu); #endif /* CONFIG_HOTPLUG_CPU */ +struct rcid_infomation { + unsigned long thread_bits : 8; /* which thread */ + unsigned long thread_shift : 8; + unsigned long core_bits : 8; /* which core */ + unsigned long core_shift : 8; + unsigned long domain_bits : 8; /* which node */ + unsigned long domain_shift : 8; + unsigned long initialized : 1; +}; + +extern struct rcid_infomation rcid_info; +extern void rcid_infomation_init(int core_version); + +extern int get_core_id_from_rcid(int rcid); +extern int get_thread_id_from_rcid(int rcid); +extern int get_domain_id_from_rcid(int rcid); + #else /* CONFIG_SMP */ #define hard_smp_processor_id() 0 #define smp_call_function_on_cpu(func, info, wait, cpu) ({ 0; }) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 8df4a200a422..3e0d6a980e52 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -45,9 +45,18 @@ enum ipi_message_type { int smp_num_cpus = 1; /* Number that came online. */ EXPORT_SYMBOL(smp_num_cpus); +struct rcid_infomation rcid_info = { 0 }; + #define send_sleep_interrupt(cpu) send_ipi((cpu), II_SLEEP) #define send_wakeup_interrupt(cpu) send_ipi((cpu), II_WAKE) +enum core_version { + CORE_VERSION_NONE = 0, + CORE_VERSION_C3B = 1, + CORE_VERSION_C4 = 2, + CORE_VERSION_RESERVED = 3 /* 3 and greater are reserved */ +}; + /* * Where secondaries begin a life of C. */ @@ -208,6 +217,65 @@ void __init setup_smp(void) smp_rcb_init(INIT_SMP_RCB); } + +void rcid_infomation_init(int core_version) +{ + if (rcid_info.initialized) + return; + + switch (core_version) { + case CORE_VERSION_C3B: + rcid_info.thread_bits = 1; + rcid_info.thread_shift = 31; + rcid_info.core_bits = 5; + rcid_info.core_shift = 0; + rcid_info.domain_bits = 2; + rcid_info.domain_shift = 5; + break; + case CORE_VERSION_C4: + rcid_info.thread_bits = 1; + rcid_info.thread_shift = 8; + rcid_info.core_bits = 6; + rcid_info.core_shift = 0; + rcid_info.domain_bits = 2; + rcid_info.domain_shift = 12; + break; + default: + rcid_info.initialized = 0; + return; + } + + rcid_info.initialized = 1; +} + +static int get_rcid_field(int rcid, unsigned int shift, unsigned int bits) +{ + unsigned int h, l; + + if (WARN_ON_ONCE(!rcid_info.initialized)) + return -1; + + h = shift + bits - 1; + l = shift; + + return (rcid & GENMASK(h, l)) >> shift; +} + +int get_core_id_from_rcid(int rcid) +{ + return get_rcid_field(rcid, rcid_info.core_shift, rcid_info.core_bits); +} + +int get_thread_id_from_rcid(int rcid) +{ + return get_rcid_field(rcid, rcid_info.thread_shift, rcid_info.thread_bits); +} + +int get_domain_id_from_rcid(int rcid) +{ + return get_rcid_field(rcid, rcid_info.domain_shift, rcid_info.domain_bits); +} + /* * Called by smp_init prepare the secondaries */ -- Gitee From a7b6c34a4d05e3ff1b1cb25739fd33052d6d4213 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 6 Dec 2023 16:40:55 +0800 Subject: [PATCH 102/524] sw64: acpi: add ACPI-style structures for SW CINTC Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- include/acpi/actbl2.h | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/include/acpi/actbl2.h b/include/acpi/actbl2.h index 8104c262bbae..986cf4a32f65 100644 --- a/include/acpi/actbl2.h +++ b/include/acpi/actbl2.h @@ -899,6 +899,18 @@ enum acpi_madt_type { ACPI_MADT_TYPE_RESERVED = 28, /* 28 to 0x7F are reserved */ ACPI_MADT_TYPE_OEM_RESERVED = 0x80, /* 0x80 to 0xFF are reserved for OEM use */ ACPI_MADT_TYPE_PHYTIUM_2500 = 128 + +#ifdef CONFIG_SW64 + , + /** + * Add for SW64 + * Due to the fact that the interrupt controller structures + * of SW64 have not yet been added to the ACPI specification, + * interrupt controller structure types reserved for OEM are + * used(start from 0x80). + */ + ACPI_MADT_TYPE_SW_CINTC = 0x80, +#endif }; /* @@ -1326,6 +1338,30 @@ struct acpi_madt_oem_data { ACPI_FLEX_ARRAY(u8, oem_data); }; +#ifdef CONFIG_SW64 +/* 0x80: Sunway Core Interrupt Controller (To be added to ACPI spec) */ + +struct acpi_madt_sw_cintc { + struct acpi_subtable_header header; + u8 version; + u8 reserved0; + u32 flags; + u32 reserved1; + u32 hardware_id; + u32 uid; + u64 boot_flag_address; +}; + +/* Values for version field above */ + +enum acpi_madt_sw_cintc_version { + ACPI_MADT_SW_CINTC_VERSION_NONE = 0, + ACPI_MADT_SW_CINTC_VERSION_V1 = 1, + ACPI_MADT_SW_CINTC_VERSION_V2 = 2, + ACPI_MADT_SW_CINTC_VERSION_RESERVED = 3 /* 3 and greater are reserved */ +}; +#endif + /* * Common flags fields for MADT subtables */ -- Gitee From 479bd236fc75230a244dae29d630310c3a122170 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 6 Dec 2023 17:20:08 +0800 Subject: [PATCH 103/524] sw64: acpi: support MADT entry print for SW CINTC Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/acpi/tables.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 8ab0a82b4da4..4848e21a84b8 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -230,6 +230,17 @@ void acpi_table_print_madt_entry(struct acpi_subtable_header *header) } break; +#ifdef CONFIG_SW64 + case ACPI_MADT_TYPE_SW_CINTC: + { + struct acpi_madt_sw_cintc *p = + (struct acpi_madt_sw_cintc *)header; + pr_debug("SW CINTC (version[%u] flags[0x%x] hardware_id[0x%x])\n", + p->version, p->flags, p->hardware_id); + } + break; +#endif + default: pr_warn("Found unsupported MADT entry (type = 0x%x)\n", header->type); -- Gitee From 1596c2fb46c591fb3573de2fe784ab8ee4bbf235 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 6 Dec 2023 17:22:36 +0800 Subject: [PATCH 104/524] sw64: acpi: parse SW CINTC for SMP initialization We create SW CINTC in MADT table for SW64. This type of table entry can carry some core related information, which can help SMP initialization and reduce hard coding in the kernel of SW64. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/smp.h | 24 +++-- arch/sw_64/kernel/acpi.c | 183 +++++++++++++++++++++++++++++++++-- 2 files changed, 193 insertions(+), 14 deletions(-) diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h index 590d4391bdea..0ca636b29dde 100644 --- a/arch/sw_64/include/asm/smp.h +++ b/arch/sw_64/include/asm/smp.h @@ -28,12 +28,6 @@ read_vpcr(void) return __r0; } -#ifdef CONFIG_SMP -/* SMP initialization hook for setup_arch */ -void __init setup_smp(void); - -#include - /* smp reset control block */ struct smp_rcb_struct { void (*restart_entry)(unsigned long args); @@ -42,6 +36,12 @@ struct smp_rcb_struct { unsigned long init_done; }; +#ifdef CONFIG_SMP +/* SMP initialization hook for setup_arch */ +void __init setup_smp(void); + +#include + #define INIT_SMP_RCB ((struct smp_rcb_struct *) __va(0x820000UL)) extern void __init smp_rcb_init(struct smp_rcb_struct *smp_rcb_base_addr); @@ -96,6 +96,18 @@ extern int get_domain_id_from_rcid(int rcid); extern int __cpu_to_rcid[NR_CPUS]; #define cpu_to_rcid(cpu) __cpu_to_rcid[0] #define cpu_physical_id(cpu) __cpu_to_rcid[0] + +static inline void smp_rcb_init(struct smp_rcb_struct *smp_rcb_base_addr) { } + +static inline void set_rcid_map(unsigned int logical, int rcid) +{ + __cpu_to_rcid[0] = 0; +} + +static inline void rcid_infomation_init(int core_version) { } +static inline int get_core_id_from_rcid(int rcid) { return 0; } +static inline int get_thread_id_from_rcid(int rcid) { return 0; } +static inline int get_domain_id_from_rcid(int rcid) { return 0; } #endif /* CONFIG_SMP */ #define NO_PROC_ID (-1) diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 9779d4bdea0d..4c1243c95c95 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -21,10 +22,24 @@ EXPORT_SYMBOL(acpi_pci_disabled); static bool param_acpi_on __initdata; static bool param_acpi_off __initdata; +static unsigned int possible_cores = 1; /* number of possible cores(at least boot core) */ +static unsigned int present_cores = 1; /* number of present cores(at least boot core) */ +static unsigned int disabled_cores; /* number of disabled cores */ + int acpi_strict; u64 arch_acpi_wakeup_start; u64 acpi_saved_sp_s3; +#define SW_CINTC_FLAG_ENABLED ACPI_MADT_ENABLED /* 0x1 */ +#define SW_CINTC_FLAG_ONLINE_CAPABLE 0x2 /* hotplug capable */ +#define SW_CINTC_FLAG_HT_CAPABLE 0x4 /* hyper thread capable */ +#define SW_CINTC_FLAG_HT_ENABLED 0x8 /* hyper thread enabled */ + +#define is_core_enabled(flags) ((flags) & SW_CINTC_FLAG_ENABLED) +#define is_core_online_capable(flags) ((flags) & SW_CINTC_FLAG_ONLINE_CAPABLE) +#define is_core_ht_capable(flags) ((flags) & SW_CINTC_FLAG_HT_CAPABLE) +#define is_core_ht_enabled(flags) ((flags) & SW_CINTC_FLAG_HT_ENABLED) + #define MAX_LOCAL_APIC 256 #define PREFIX "ACPI: " @@ -271,7 +286,7 @@ int acpi_unmap_cpu(int cpu) set_cpuid_to_node(cpu, NUMA_NO_NODE); #endif set_cpu_present(cpu, false); - num_processors--; + present_cores--; pr_info("cpu%d hot remove!\n", cpu); @@ -280,6 +295,143 @@ int acpi_unmap_cpu(int cpu) EXPORT_SYMBOL(acpi_unmap_cpu); #endif /* CONFIG_ACPI_HOTPLUG_CPU */ +static bool __init is_rcid_duplicate(int rcid) +{ + unsigned int i; + + for (i = 0; (i < possible_cores) && (i < NR_CPUS); i++) { + if (cpu_to_rcid(i) == rcid) + return true; + } + + return false; +} + +static int __init +setup_rcid_and_core_mask(struct acpi_madt_sw_cintc *sw_cintc) +{ + unsigned int logical_core_id; + int rcid = sw_cintc->hardware_id; + + /** + * The initial value of nr_cpu_ids is NR_CPUS, which + * represents the maximum number of cores in the system. + */ + if (possible_cores >= nr_cpu_ids) { + pr_err(PREFIX "Max core num [%u] reached, core [0x%x] ignored." + " Please check the firmware or CONFIG_NR_CPUS!\n", + nr_cpu_ids, rcid); + return -ENODEV; + } + + /* The rcid of each core is unique */ + if (is_rcid_duplicate(rcid)) { + pr_err(PREFIX "Duplicate core [0x%x] in MADT." + " Please check the firmware!\n", rcid); + return -EINVAL; + } + + /* We can never disable the boot core, whose rcid is 0 */ + if ((rcid == 0) && !is_core_enabled(sw_cintc->flags)) { + pr_err(PREFIX "Boot core disabled in MADT." + " Please check the firmware!\n"); + return -EINVAL; + } + + /* Online capable makes core possible */ + if (!is_core_enabled(sw_cintc->flags) && \ + !is_core_online_capable(sw_cintc->flags)) { + disabled_cores++; + return 0; + } + + rcid_infomation_init(sw_cintc->version); + + /* The logical core ID of the boot core must be 0 */ + if (rcid == 0) + logical_core_id = 0; + else + logical_core_id = possible_cores++; + + set_rcid_map(logical_core_id, rcid); + set_cpu_possible(logical_core_id, true); + store_cpu_data(logical_core_id); + + /** + * Whether the core will finally be online + * depends on two conditions: + * 1. core is enabled via firmware + * 2. core is not disabled by cmdline param(offline) + */ + if (is_core_enabled(sw_cintc->flags) && \ + !cpumask_test_cpu(logical_core_id, &cpu_offline)) { + set_cpu_present(logical_core_id, true); + if (logical_core_id != 0) + present_cores++; + } + + return 0; +} + +static int __init acpi_parse_sw_cintc(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_sw_cintc *sw_cintc = NULL; + struct smp_rcb_struct *smp_rcb_base_addr = NULL; + int ret; + + sw_cintc = (struct acpi_madt_sw_cintc *)header; + if (BAD_MADT_ENTRY(sw_cintc, end)) { + pr_err(PREFIX "SW CINTC entry error." + " Please check the firmware!\n"); + return -EINVAL; + } + + acpi_table_print_madt_entry(&header->common); + + ret = setup_rcid_and_core_mask(sw_cintc); + if (ret) + return ret; + + /** + * We use smp_rcb to help SMP boot. Its base + * address is hold in the MADT entry of SW CINTC. + */ + smp_rcb_base_addr = __va(sw_cintc->boot_flag_address); + smp_rcb_init(smp_rcb_base_addr); + + return 0; +} + +static int __init acpi_process_madt_sw_cintc(void) +{ + int logical_core, ret; + + /* Clean the map from logical core ID to physical core ID */ + for (logical_core = 0; logical_core < NR_CPUS; ++logical_core) + set_rcid_map(logical_core, -1); + + /* Clean core mask */ + init_cpu_possible(cpu_none_mask); + init_cpu_present(cpu_none_mask); + + /* Parse SW CINTC entries one by one */ + ret = acpi_table_parse_madt(ACPI_MADT_TYPE_SW_CINTC, + acpi_parse_sw_cintc, 0); + if (ret < 0) + return ret; + +#if NR_CPUS > 1 + /* It's time to update nr_cpu_ids */ + nr_cpu_ids = possible_cores; +#endif + + pr_info(PREFIX "Detected %u possible CPU(s), %u CPU(s) are present\n", + possible_cores, present_cores); + + return 0; +} + void __init acpi_boot_table_init(void) { /** @@ -293,12 +445,27 @@ void __init acpi_boot_table_init(void) /* * If acpi_disabled, bail out */ - if (!acpi_disabled) { - pr_warn("Currently, ACPI is an experimental feature!\n"); - if (acpi_table_init()) { - pr_err("Failed to init ACPI tables\n"); - disable_acpi(); - } else - pr_info("Successfully parsed ACPI table\n"); + if (acpi_disabled) + return; + + pr_warn("Currently, ACPI is an experimental feature!\n"); + if (acpi_table_init()) { + pr_err("Failed to init ACPI tables\n"); + disable_acpi(); + return; + } else + pr_info("Successfully parsed ACPI table\n"); + + /** + * Process SW64 Core Interrupt Controller(SW CINTC) in MADT table. + * No initialization of the interrupt controller here, mainly used + * to establish the mapping from logical core IDs to physical core + * IDs and set cpu mask. + */ + if (acpi_process_madt_sw_cintc()) { + /* May be fatal error in MADT table */ + pr_err("Failed to parse SW CINTC\n"); + disable_acpi(); + return; } } -- Gitee From bbaa463b098ef90aa22ccc5c3591a3141ebe538a Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 8 Dec 2023 17:12:39 +0800 Subject: [PATCH 105/524] sw64: smp: suppress function setup_smp() when ACPI enabled When ACPI enabled, we setup rcid and cpu mask via SW CINTC entry in ACPI MADT table. At this point, function setup_smp() is no longer needed. This function is only used to keep compatibility with legacy codes. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 412710b17a80..87324cd2268b 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -839,11 +839,13 @@ setup_arch(char **cmdline_p) /* Parse the ACPI tables for possible boot-time configuration */ acpi_boot_table_init(); + if (acpi_disabled) { #ifdef CONFIG_SMP - setup_smp(); + setup_smp(); #else - store_cpu_data(0); + store_cpu_data(0); #endif + } sw64_numa_init(); -- Gitee From 7863645774c5de1d8e553fea3da0e9bbfd77b591 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 27 Dec 2023 17:03:25 +0800 Subject: [PATCH 106/524] sw64: pci: fix compile error when CONFIG_ACPI=n Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/controller/pci-sunway.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index e9314e4cb86e..32e850f63a32 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -598,12 +598,12 @@ static void __iomem *sw64_pcie_map_bus(struct pci_bus *bus, return cfg_iobase; } -#ifdef CONFIG_ACPI int sw64_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) { return map_irq(dev, slot, pin); } +#ifdef CONFIG_ACPI static void setup_intx_irqs(struct pci_controller *hose) { unsigned long int_conf, node, val_node; -- Gitee From 650c9794a3ba3ffc7d027092bb2c96b67cf145ea Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Wed, 27 Sep 2023 13:47:48 +0800 Subject: [PATCH 107/524] USB: xHCI: fix resume issue of some ZHAOXIN hosts On ZHAOXIN ZX-100 and ZX-200 project, xHCI can't work normally after resume from system Sx state. To fix this issue, when resume from system Sx state, reinitialize xHCI instead of restore. So, Add XHCI_RESET_ON_RESUME quirk for ZX-100 or ZX-200 to fix issue of resuming from system Sx state. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/usb/host/xhci-pci.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index 22efa4090979..7cdb1ce5104f 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -500,6 +500,13 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) xhci->quirks |= XHCI_RESET_ON_RESUME; xhci->quirks |= XHCI_ZERO_64B_REGS; } + + if (pdev->vendor == PCI_VENDOR_ID_ZHAOXIN) { + if (pdev->device == 0x9202 || + pdev->device == 0x9203) + xhci->quirks |= XHCI_RESET_ON_RESUME; + } + if (pdev->vendor == PCI_VENDOR_ID_VIA) xhci->quirks |= XHCI_RESET_ON_RESUME; -- Gitee From ea42a6b04dce0325dc826467fdb42341add182cd Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Tue, 26 Dec 2023 17:04:43 +0800 Subject: [PATCH 108/524] sw64: fix macro definition compilation error Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/csr.h | 34 +++++--- arch/sw_64/kernel/irq_sw64.c | 8 +- arch/sw_64/kernel/match.c | 46 +++++----- arch/sw_64/kernel/ptrace.c | 114 ++++++++++++------------- arch/sw_64/kernel/smp.c | 8 +- arch/sw_64/kernel/vdso/vgettimeofday.c | 2 +- arch/sw_64/kvm/emulate.c | 4 +- arch/sw_64/kvm/kvm_core4.c | 6 +- arch/sw_64/kvm/mmio.c | 2 +- arch/sw_64/kvm/mmu.c | 10 +-- drivers/clocksource/timer-sw64.c | 4 +- 11 files changed, 122 insertions(+), 116 deletions(-) diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h index 0610384208a4..dc400069a771 100644 --- a/arch/sw_64/include/asm/csr.h +++ b/arch/sw_64/include/asm/csr.h @@ -61,30 +61,36 @@ #ifdef CONFIG_HAVE_CSRRW -#define read_csr(x) \ - ({ unsigned long __val; \ - __asm__ __volatile__("csrr %0,%1" : "=r"(__val) : "i"(x)); \ - __val; }) - -#define write_csr(x, y) \ - ({ __asm__ __volatile__("csrw %0,%1" ::"r"(x), "i"(y)); }) +#ifndef __ASSEMBLY__ +static inline unsigned long sw64_read_csr(unsigned long x) +{ + unsigned long __val; + __asm__ __volatile__("csrr %0,%1" : "=r"(__val) : "i"(x)); + return __val; +} -#define write_csr_imb(x, y) \ - ({ __asm__ __volatile__("csrw %0,%1; imemb" ::"r"(x), "i"(y)); }) +static inline void sw64_write_csr(unsigned long x, unsigned long y) +{ + __asm__ __volatile__("csrw %0,%1" ::"r"(x), "i"(y)); +} +static inline void sw64_write_csr_imb(unsigned long x, unsigned long y) +{ + __asm__ __volatile__("csrw %0,%1; imemb" ::"r"(x), "i"(y)); +} -#ifndef __ASSEMBLY__ #include static inline void update_ptbr_sys(unsigned long ptbr) { imemb(); - write_csr_imb(ptbr, CSR_PTBR_SYS); + sw64_write_csr_imb(ptbr, CSR_PTBR_SYS); } + #endif #else -#define read_csr(x) (0) -#define write_csr(x, y) do { } while (0) -#define write_csr_imb(x, y) do { } while (0) +#define sw64_read_csr(x) (0) +#define sw64_write_csr(x, y) do { } while (0) +#define sw64_write_csr_imb(x, y) do { } while (0) #ifndef __ASSEMBLY__ static inline void update_ptbr_sys(unsigned long ptbr) diff --git a/arch/sw_64/kernel/irq_sw64.c b/arch/sw_64/kernel/irq_sw64.c index 989d55ee1b1b..03aef463ec05 100644 --- a/arch/sw_64/kernel/irq_sw64.c +++ b/arch/sw_64/kernel/irq_sw64.c @@ -17,10 +17,10 @@ init_IRQ(void) * (as is the case with RAWHIDE, at least). */ if (is_in_host()) { - write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); - write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); - write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); - write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); } wrent(entInt, 0); diff --git a/arch/sw_64/kernel/match.c b/arch/sw_64/kernel/match.c index 3926391270da..9ecd5d838846 100644 --- a/arch/sw_64/kernel/match.c +++ b/arch/sw_64/kernel/match.c @@ -97,14 +97,14 @@ write_da_match(void *i) { unsigned long dc_ctl; - write_csr(da_match_cf1, CSR_DA_MATCH); - write_csr(da_match_cf2, CSR_DA_MASK); - dc_ctl = read_csr(CSR_DC_CTLP); + sw64_write_csr(da_match_cf1, CSR_DA_MATCH); + sw64_write_csr(da_match_cf2, CSR_DA_MASK); + dc_ctl = sw64_read_csr(CSR_DC_CTLP); dc_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH)); dc_ctl |= da_match_cf3; - write_csr(dc_ctl, CSR_DC_CTLP); + sw64_write_csr(dc_ctl, CSR_DC_CTLP); } static void @@ -112,13 +112,13 @@ write_dv_match(void *i) { unsigned long dc_ctl; - write_csr(dv_match_cf1, CSR_DV_MATCH); - write_csr(dv_match_cf2, CSR_DV_MASK); - dc_ctl = read_csr(CSR_DC_CTLP); + sw64_write_csr(dv_match_cf1, CSR_DV_MATCH); + sw64_write_csr(dv_match_cf2, CSR_DV_MASK); + dc_ctl = sw64_read_csr(CSR_DC_CTLP); dc_ctl &= ~((0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH)); dc_ctl |= ((0x1UL << DV_MATCH_EN_S) | dv_match_cf3); - write_csr(dc_ctl, CSR_DC_CTLP); + sw64_write_csr(dc_ctl, CSR_DC_CTLP); } static void @@ -126,26 +126,26 @@ write_dav_match(void *i) { unsigned long dc_ctl; - write_csr(dav_match_cf1, CSR_DA_MATCH); - write_csr(dav_match_cf2, CSR_DA_MASK); - write_csr(dav_match_cf3, CSR_DV_MATCH); - write_csr(dav_match_cf4, CSR_DV_MASK); - dc_ctl = read_csr(CSR_DC_CTLP); + sw64_write_csr(dav_match_cf1, CSR_DA_MATCH); + sw64_write_csr(dav_match_cf2, CSR_DA_MASK); + sw64_write_csr(dav_match_cf3, CSR_DV_MATCH); + sw64_write_csr(dav_match_cf4, CSR_DV_MASK); + dc_ctl = sw64_read_csr(CSR_DC_CTLP); dc_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH)); dc_ctl |= ((0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S) | dav_match_cf5); - write_csr(dc_ctl, CSR_DC_CTLP); + sw64_write_csr(dc_ctl, CSR_DC_CTLP); } static void write_ia_match(void *i) { ia_match_cf1 |= (0x1UL << IA_MATCH_EN_S); - write_csr_imb(ia_match_cf1, CSR_IA_MATCH); - write_csr_imb(ia_match_cf2, CSR_IA_MASK); - write_csr(((0x3ffUL << 18) | ia_match_cf3), CSR_IA_VPNMATCH); - write_csr(((0x3ffUL << 18) | ia_match_cf4), CSR_IA_UPNMATCH); + sw64_write_csr_imb(ia_match_cf1, CSR_IA_MATCH); + sw64_write_csr_imb(ia_match_cf2, CSR_IA_MASK); + sw64_write_csr(((0x3ffUL << 18) | ia_match_cf3), CSR_IA_VPNMATCH); + sw64_write_csr(((0x3ffUL << 18) | ia_match_cf4), CSR_IA_UPNMATCH); } static void @@ -153,12 +153,12 @@ write_iv_match(void *i) { unsigned long ia_match_tmp; - ia_match_tmp = read_csr(CSR_IA_MATCH); + ia_match_tmp = sw64_read_csr(CSR_IA_MATCH); ia_match_tmp &= ~(0x1UL << IV_PM_EN_S); ia_match_tmp |= ((((iv_match_cf2 >> IV_PM_EN_S) & 0x1) << IV_PM_EN_S) | (iv_match_cf2 & 0x3) | (0x1UL << IV_MATCH_EN_S)); - write_csr_imb(iv_match_cf1, CSR_IV_MATCH); - write_csr_imb(ia_match_tmp, CSR_IA_MATCH); + sw64_write_csr_imb(iv_match_cf1, CSR_IV_MATCH); + sw64_write_csr_imb(ia_match_tmp, CSR_IA_MATCH); } static void @@ -166,8 +166,8 @@ write_ida_match(void *i) { ida_match_cf1 |= (0x1UL << IDA_MATCH_EN_S); - write_csr(ida_match_cf1, CSR_IDA_MATCH); - write_csr(ida_match_cf2, CSR_IDA_MASK); + sw64_write_csr(ida_match_cf1, CSR_IDA_MATCH); + sw64_write_csr(ida_match_cf2, CSR_IDA_MASK); } static ssize_t da_match_set(struct file *file, const char __user *user_buf, diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index 070e27ee2567..0af3d35163f9 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -545,49 +545,49 @@ int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_r if (!(current->ptrace & PT_PTRACED)) { pr_notice(" pid %d %s not be ptraced, return\n", current->pid, current->comm); if (mmcsr == MMCSR__DA_MATCH) { - match_ctl = read_csr(CSR_DC_CTLP); + match_ctl = sw64_read_csr(CSR_DC_CTLP); match_ctl &= ~(0x3UL << DA_MATCH_EN_S); - write_csr(match_ctl, CSR_DC_CTLP); - write_csr(0, CSR_DA_MATCH); // clear da_match + sw64_write_csr(match_ctl, CSR_DC_CTLP); + sw64_write_csr(0, CSR_DA_MATCH); // clear da_match task_thread_info(current)->pcb.match_ctl &= ~0x1; task_thread_info(current)->pcb.da_match = 0; } if (mmcsr == MMCSR__DV_MATCH) { - match_ctl = read_csr(CSR_DC_CTLP); + match_ctl = sw64_read_csr(CSR_DC_CTLP); match_ctl &= ~(0x1UL << DV_MATCH_EN_S); - write_csr(match_ctl, CSR_DC_CTLP); - write_csr(0, CSR_DV_MATCH); // clear dv_match + sw64_write_csr(match_ctl, CSR_DC_CTLP); + sw64_write_csr(0, CSR_DV_MATCH); // clear dv_match task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 1); task_thread_info(current)->pcb.dv_match = 0; } if (mmcsr == MMCSR__DAV_MATCH) { - match_ctl = read_csr(CSR_DC_CTLP); + match_ctl = sw64_read_csr(CSR_DC_CTLP); match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); - write_csr(match_ctl, CSR_DC_CTLP); - write_csr(0, CSR_DA_MATCH); // clear da_match - write_csr(0, CSR_DV_MATCH); // clear dv_match + sw64_write_csr(match_ctl, CSR_DC_CTLP); + sw64_write_csr(0, CSR_DA_MATCH); // clear da_match + sw64_write_csr(0, CSR_DV_MATCH); // clear dv_match task_thread_info(current)->pcb.match_ctl &= ~(0x1 | (0x1 << 1) | (0x1 << 2)); task_thread_info(current)->pcb.da_match = 0; task_thread_info(current)->pcb.dv_match = 0; } if (mmcsr == MMCSR__IA_MATCH) { - ia_match = read_csr(CSR_IA_MATCH); + ia_match = sw64_read_csr(CSR_IA_MATCH); ia_match &= ~((0x1UL << IA_MATCH_EN_S) | (0x7ffffffffffffUL << 2)); - write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + sw64_write_csr(ia_match, CSR_IA_MATCH); // clear ia_match task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 3); task_thread_info(current)->pcb.ia_match = 0; } if (mmcsr == MMCSR__IV_MATCH) { - ia_match = read_csr(CSR_IA_MATCH); + ia_match = sw64_read_csr(CSR_IA_MATCH); ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); - write_csr(ia_match, CSR_IA_MATCH); // clear ia_match - write_csr(0, CSR_IV_MATCH); // clear iv_match + sw64_write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + sw64_write_csr(0, CSR_IV_MATCH); // clear iv_match task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 4); task_thread_info(current)->pcb.ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); task_thread_info(current)->pcb.iv_match = 0; } if (mmcsr == MMCSR__IDA_MATCH) { - write_csr(0, CSR_IDA_MATCH); // clear ida_match + sw64_write_csr(0, CSR_IDA_MATCH); // clear ida_match task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 5); task_thread_info(current)->pcb.ida_match = 0; } @@ -602,54 +602,54 @@ int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_r if (mmcsr == MMCSR__DA_MATCH) { info.si_errno = 1; - match_ctl = read_csr(CSR_DC_CTLP); + match_ctl = sw64_read_csr(CSR_DC_CTLP); match_ctl &= ~(0x3UL << DA_MATCH_EN_S); - write_csr(match_ctl, CSR_DC_CTLP); - write_csr(0, CSR_DA_MATCH); // clear da_match + sw64_write_csr(match_ctl, CSR_DC_CTLP); + sw64_write_csr(0, CSR_DA_MATCH); // clear da_match task_thread_info(current)->pcb.match_ctl &= ~0x1; task_thread_info(current)->pcb.da_match = 0; } if (mmcsr == MMCSR__DV_MATCH) { info.si_errno = 2; - match_ctl = read_csr(CSR_DC_CTLP); + match_ctl = sw64_read_csr(CSR_DC_CTLP); match_ctl &= ~(0x1UL << DV_MATCH_EN_S); - write_csr(match_ctl, CSR_DC_CTLP); - write_csr(0, CSR_DV_MATCH); // clear dv_match + sw64_write_csr(match_ctl, CSR_DC_CTLP); + sw64_write_csr(0, CSR_DV_MATCH); // clear dv_match task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 1); task_thread_info(current)->pcb.dv_match = 0; } if (mmcsr == MMCSR__DAV_MATCH) { info.si_errno = 3; - match_ctl = read_csr(CSR_DC_CTLP); + match_ctl = sw64_read_csr(CSR_DC_CTLP); match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); - write_csr(match_ctl, CSR_DC_CTLP); - write_csr(0, CSR_DA_MATCH); // clear da_match - write_csr(0, CSR_DV_MATCH); // clear dv_match + sw64_write_csr(match_ctl, CSR_DC_CTLP); + sw64_write_csr(0, CSR_DA_MATCH); // clear da_match + sw64_write_csr(0, CSR_DV_MATCH); // clear dv_match task_thread_info(current)->pcb.match_ctl &= ~(0x1 | (0x1 << 1) | (0x1 << 2)); task_thread_info(current)->pcb.da_match = 0; task_thread_info(current)->pcb.dv_match = 0; } if (mmcsr == MMCSR__IA_MATCH) { info.si_errno = 4; - ia_match = read_csr(CSR_IA_MATCH); + ia_match = sw64_read_csr(CSR_IA_MATCH); ia_match &= ~((0x1UL << IA_MATCH_EN_S) | (0x7ffffffffffffUL << 2)); - write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + sw64_write_csr(ia_match, CSR_IA_MATCH); // clear ia_match task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 3); task_thread_info(current)->pcb.ia_match = 0; } if (mmcsr == MMCSR__IV_MATCH) { info.si_errno = 5; - ia_match = read_csr(CSR_IA_MATCH); + ia_match = sw64_read_csr(CSR_IA_MATCH); ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); - write_csr(ia_match, CSR_IA_MATCH); // clear ia_match - write_csr(0, CSR_IV_MATCH); // clear iv_match + sw64_write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + sw64_write_csr(0, CSR_IV_MATCH); // clear iv_match task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 4); task_thread_info(current)->pcb.ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); task_thread_info(current)->pcb.iv_match = 0; } if (mmcsr == MMCSR__IDA_MATCH) { info.si_errno = 6; - write_csr(0, CSR_IDA_MATCH); // clear ida_match + sw64_write_csr(0, CSR_IDA_MATCH); // clear ida_match task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 5); task_thread_info(current)->pcb.ida_match = 0; } @@ -691,39 +691,39 @@ void restore_da_match_after_sched(void) pr_info("Restroe MATCH status, pid: %d\n", current->pid); if (pcb->match_ctl & DA_MATCH) { - write_csr(pcb->da_match, CSR_DA_MATCH); - write_csr(pcb->da_mask, CSR_DA_MASK); + sw64_write_csr(pcb->da_match, CSR_DA_MATCH); + sw64_write_csr(pcb->da_mask, CSR_DA_MASK); match_ctl_mode = (pcb->match_ctl >> 8) & 0x3; - match_ctl = read_csr(CSR_DC_CTLP); + match_ctl = sw64_read_csr(CSR_DC_CTLP); match_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); match_ctl |= (match_ctl_mode << DA_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH); - write_csr(match_ctl, CSR_DC_CTLP); + sw64_write_csr(match_ctl, CSR_DC_CTLP); pr_info("da_match:%#lx da_mask:%#lx match_ctl:%#lx\n", pcb->da_match, pcb->da_mask, match_ctl); } if (pcb->match_ctl & DV_MATCH) { - write_csr(pcb->dv_match, CSR_DV_MATCH); - write_csr(pcb->dv_mask, CSR_DV_MASK); - match_ctl = read_csr(CSR_DC_CTLP); + sw64_write_csr(pcb->dv_match, CSR_DV_MATCH); + sw64_write_csr(pcb->dv_mask, CSR_DV_MASK); + match_ctl = sw64_read_csr(CSR_DC_CTLP); match_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); match_ctl |= (0x1UL << DV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH); - write_csr(match_ctl, CSR_DC_CTLP); + sw64_write_csr(match_ctl, CSR_DC_CTLP); pr_info("dv_match:%#lx dv_mask:%#lx match_ctl:%#lx\n", pcb->dv_match, pcb->dv_mask, match_ctl); } if (pcb->match_ctl & DAV_MATCH) { - write_csr(pcb->da_match, CSR_DA_MATCH); - write_csr(pcb->da_mask, CSR_DA_MASK); - write_csr(pcb->dv_match, CSR_DV_MATCH); - write_csr(pcb->dv_mask, CSR_DV_MASK); - write_csr(0xfffffffff, CSR_DA_MATCH_MODE); + sw64_write_csr(pcb->da_match, CSR_DA_MATCH); + sw64_write_csr(pcb->da_mask, CSR_DA_MASK); + sw64_write_csr(pcb->dv_match, CSR_DV_MATCH); + sw64_write_csr(pcb->dv_mask, CSR_DV_MASK); + sw64_write_csr(0xfffffffff, CSR_DA_MATCH_MODE); match_ctl_mode = (pcb->match_ctl >> 8) & 0x3; - match_ctl = read_csr(CSR_DC_CTLP); + match_ctl = sw64_read_csr(CSR_DC_CTLP); match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); match_ctl |= (match_ctl_mode << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH); - write_csr(match_ctl, CSR_DC_CTLP); + sw64_write_csr(match_ctl, CSR_DC_CTLP); pr_info("da_match:%#lx da_mask:%#lx dv_match:%#lx dv_mask:%#lx match_ctl:%#lx\n", pcb->da_match, pcb->da_mask, pcb->dv_match, pcb->dv_mask, match_ctl); } @@ -731,27 +731,27 @@ void restore_da_match_after_sched(void) if (pcb->match_ctl & IA_MATCH) { pcb->ia_match |= (0x1UL << IA_MATCH_EN_S) | 0x3; pcb->ia_mask |= 0x3; - write_csr(pcb->ia_match, CSR_IA_MATCH); - write_csr(pcb->ia_mask, CSR_IA_MASK); - vpn = read_csr(CSR_VPCR) >> 44; + sw64_write_csr(pcb->ia_match, CSR_IA_MATCH); + sw64_write_csr(pcb->ia_mask, CSR_IA_MASK); + vpn = sw64_read_csr(CSR_VPCR) >> 44; vpn &= 0x3ff; - upn = read_csr(CSR_UPCR); + upn = sw64_read_csr(CSR_UPCR); upn &= 0x3ff; - write_csr(((0x3ff << 18) | vpn), CSR_IA_VPNMATCH); - write_csr(((0x3ff << 18) | upn), CSR_IA_UPNMATCH); + sw64_write_csr(((0x3ff << 18) | vpn), CSR_IA_VPNMATCH); + sw64_write_csr(((0x3ff << 18) | upn), CSR_IA_UPNMATCH); pr_info("ia_match:%#lx ia_mask:%#lx\n", pcb->ia_match, pcb->ia_mask); } if (pcb->match_ctl & IV_MATCH) { pcb->ia_match |= (0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S) | 0x3; - write_csr(pcb->ia_match, CSR_IA_MATCH); - write_csr(pcb->iv_match, CSR_IV_MATCH); + sw64_write_csr(pcb->ia_match, CSR_IA_MATCH); + sw64_write_csr(pcb->iv_match, CSR_IV_MATCH); pr_info("ia_match:%#lx iv_match:%#lx\n", pcb->ia_match, pcb->iv_match); } if (pcb->match_ctl & IDA_MATCH) { pcb->ida_match |= (0x1UL << IDA_MATCH_EN_S) | 0x3; pcb->ida_mask |= 0x3; - write_csr(pcb->ida_match, CSR_IDA_MATCH); - write_csr(pcb->ida_mask, CSR_IDA_MASK); + sw64_write_csr(pcb->ida_match, CSR_IDA_MATCH); + sw64_write_csr(pcb->ida_mask, CSR_IDA_MASK); pr_info("ida_match:%#lx ida_mask:%#lx\n", pcb->ida_match, pcb->ida_mask); } } diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 3e0d6a980e52..42450119ea7d 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -81,10 +81,10 @@ void smp_callin(void) /* Set interrupt vector. */ if (is_in_host()) { - write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); - write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); - write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); - write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); } wrent(entInt, 0); diff --git a/arch/sw_64/kernel/vdso/vgettimeofday.c b/arch/sw_64/kernel/vdso/vgettimeofday.c index 0aa16e988e88..07e023d3d64b 100644 --- a/arch/sw_64/kernel/vdso/vgettimeofday.c +++ b/arch/sw_64/kernel/vdso/vgettimeofday.c @@ -88,7 +88,7 @@ static __always_inline u64 read_longtime(void) #elif defined(CONFIG_SUBARCH_C4) static __always_inline u64 read_longtime(void) { - return read_csr(CSR_SHTCLOCK); + return sw64_read_csr(CSR_SHTCLOCK); } #endif diff --git a/arch/sw_64/kvm/emulate.c b/arch/sw_64/kvm/emulate.c index fc37461b97a0..66e6335d9cce 100644 --- a/arch/sw_64/kvm/emulate.c +++ b/arch/sw_64/kvm/emulate.c @@ -19,8 +19,8 @@ void sw64_decode(struct kvm_vcpu *vcpu, unsigned int insn, struct kvm_run *run) #elif defined(CONFIG_SUBARCH_C4) unsigned long ds_stat, exc_sum; - ds_stat = read_csr(CSR_DS_STAT); - exc_sum = read_csr(CSR_EXC_SUM); + ds_stat = sw64_read_csr(CSR_DS_STAT); + exc_sum = sw64_read_csr(CSR_EXC_SUM); opc = (ds_stat >> 4) & 0x3f; ra = (exc_sum >> 8) & 0x1f; diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c index 08d28a365a3b..3c52b0d7776c 100644 --- a/arch/sw_64/kvm/kvm_core4.c +++ b/arch/sw_64/kvm/kvm_core4.c @@ -25,7 +25,7 @@ static unsigned long shtclock_offset; void update_aptp(unsigned long pgd) { imemb(); - write_csr_imb(pgd, CSR_APTP); + sw64_write_csr_imb(pgd, CSR_APTP); } void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) @@ -69,7 +69,7 @@ long kvm_sw64_get_vcb(struct file *filp, unsigned long arg) struct kvm_vcpu *vcpu = filp->private_data; if (vcpu->arch.migration_mark) - vcpu->arch.shtclock = read_csr(CSR_SHTCLOCK) + vcpu->arch.shtclock = sw64_read_csr(CSR_SHTCLOCK) + vcpu->arch.vcb.shtclock_offset; if (copy_to_user((void __user *)arg, &(vcpu->arch.vcb), sizeof(struct vcpucb))) return -EINVAL; @@ -88,7 +88,7 @@ long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) if (vcpu->arch.migration_mark) { /* synchronize the longtime of source and destination */ if (vcpu->arch.vcb.soft_cid == 0) - shtclock_offset = vcpu->arch.shtclock - read_csr(CSR_SHTCLOCK); + shtclock_offset = vcpu->arch.shtclock - sw64_read_csr(CSR_SHTCLOCK); vcpu->arch.vcb.shtclock_offset = shtclock_offset; set_timer(vcpu, 200000000); vcpu->arch.migration_mark = 0; diff --git a/arch/sw_64/kvm/mmio.c b/arch/sw_64/kvm/mmio.c index 21ad89722f9a..0bde3406eb89 100644 --- a/arch/sw_64/kvm/mmio.c +++ b/arch/sw_64/kvm/mmio.c @@ -68,7 +68,7 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, run->mmio.phys_addr = hargs->arg1 & 0xfffffffffffffUL; sw64_decode(vcpu, hargs->arg2, run); #elif defined(CONFIG_SUBARCH_C4) - run->mmio.phys_addr = read_csr(CSR_DVA) & 0xfffffffffffffUL; + run->mmio.phys_addr = sw64_read_csr(CSR_DVA) & 0xfffffffffffffUL; sw64_decode(vcpu, 0, run); #endif if (run->mmio.is_write) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 573d32b171d6..054856c5a499 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -790,8 +790,8 @@ static int apt_set_pte_fast(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, pmd_t *pmd; pte_t *pte, old_pte; bool logging_active = flags & KVM_APT_FLAG_LOGGING_ACTIVE; - int inv_level = ((read_csr(CSR_AS_INFO)) >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; - unsigned long inv_hpa = read_csr(CSR_AS_INFO) & AF_ENTRY_ADDR_MASK; + int inv_level = ((sw64_read_csr(CSR_AS_INFO)) >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; + unsigned long inv_hpa = sw64_read_csr(CSR_AS_INFO) & AF_ENTRY_ADDR_MASK; VM_BUG_ON(logging_active && !cache); @@ -1148,7 +1148,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_gpa, unsigned long as_info, access_type; unsigned int vma_shift; - as_info = read_csr(CSR_AS_INFO); + as_info = sw64_read_csr(CSR_AS_INFO); access_type = (as_info >> AF_ACCESS_TYPE_SHIFT) & AF_ACCESS_TYPE_MASK; write_fault = kvm_is_write_fault(access_type); exec_fault = kvm_is_exec_fault(access_type); @@ -1352,13 +1352,13 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) int ret, idx; - as_info = read_csr(CSR_AS_INFO); + as_info = sw64_read_csr(CSR_AS_INFO); access_type = (as_info >> AF_ACCESS_TYPE_SHIFT) & AF_ACCESS_TYPE_MASK; inv_level = (as_info >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; fault_status = (as_info >> AF_FAULT_STATUS_SHIFT) & AF_FAULT_STATUS_MASK; fault_entry_addr = (as_info & AF_ENTRY_ADDR_MASK) >> 3; - fault_gpa = read_csr(CSR_EXC_GPA); + fault_gpa = sw64_read_csr(CSR_EXC_GPA); idx = srcu_read_lock(&vcpu->kvm->srcu); gfn = fault_gpa >> PAGE_SHIFT; diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index a124b6d8fed9..615e8a2ed63a 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -19,7 +19,7 @@ #if defined(CONFIG_SUBARCH_C4) static u64 read_longtime(struct clocksource *cs) { - return read_csr(CSR_SHTCLOCK); + return sw64_read_csr(CSR_SHTCLOCK); } static struct clocksource clocksource_longtime = { @@ -34,7 +34,7 @@ static struct clocksource clocksource_longtime = { static u64 notrace read_sched_clock(void) { - return read_csr(CSR_SHTCLOCK); + return sw64_read_csr(CSR_SHTCLOCK); } void __init sw64_setup_clocksource(void) -- Gitee From 767c31273b97de8b29c67e5f4f65f3a554a6fb18 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Mon, 11 Dec 2023 07:15:22 +0000 Subject: [PATCH 109/524] sw64: refactor device interrupt domain Refactor device interrupt domain code to support sw64 interrupt controller redesign. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hw_irq.h | 2 + arch/sw_64/include/asm/uncore_io_junzhang.h | 2 + arch/sw_64/include/asm/uncore_io_xuelang.h | 2 + drivers/gpio/gpio-sunway.c | 1 + drivers/irqchip/Kconfig | 15 +- drivers/irqchip/Makefile | 4 +- drivers/irqchip/irq-sunway-cpu.c | 5 +- ...-sw64-lpc-intc.c => irq-sunway-lpc-intc.c} | 153 ++++++---- drivers/irqchip/irq-sunway-pintc.c | 287 ++++++++++++++++++ drivers/irqchip/irq-sw64-intc-v2.c | 89 ------ 10 files changed, 396 insertions(+), 164 deletions(-) rename drivers/irqchip/{irq-sw64-lpc-intc.c => irq-sunway-lpc-intc.c} (34%) create mode 100644 drivers/irqchip/irq-sunway-pintc.c delete mode 100644 drivers/irqchip/irq-sw64-intc-v2.c diff --git a/arch/sw_64/include/asm/hw_irq.h b/arch/sw_64/include/asm/hw_irq.h index 3cfc725f7517..e0f8ee91c43a 100644 --- a/arch/sw_64/include/asm/hw_irq.h +++ b/arch/sw_64/include/asm/hw_irq.h @@ -9,6 +9,8 @@ DECLARE_PER_CPU(unsigned long, irq_pmi_count); #define ACTUAL_NR_IRQS NR_IRQS +extern struct irq_domain *mcu_irq_domain; + #ifdef CONFIG_PCI_MSI typedef unsigned int vector_irq_t[PERCPU_MSI_IRQS]; DECLARE_PER_CPU(vector_irq_t, vector_irq); diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index 37cfe1fd6807..ac37d186a29f 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -70,6 +70,8 @@ #define PIUCONFIG0_INIT_VAL 0x38016 +#define DEV_INT_TARGET(phy_cpu) ((((phy_cpu) >> 5) << 7) | ((phy_cpu) & 0x3f)) + /*-----------------------addr-----------------------*/ /* INTPU REG */ enum { diff --git a/arch/sw_64/include/asm/uncore_io_xuelang.h b/arch/sw_64/include/asm/uncore_io_xuelang.h index aeaadec5be16..873f7e86f83b 100644 --- a/arch/sw_64/include/asm/uncore_io_xuelang.h +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -74,6 +74,8 @@ #define PIUCONFIG0_INIT_VAL 0x38056 +#define DEV_INT_TARGET(phy_cpu) ((((phy_cpu) >> 5) << 6) | ((phy_cpu) & 0x1f)) + /*-----------------------addr-----------------------*/ /* CAB0 REG */ enum { diff --git a/drivers/gpio/gpio-sunway.c b/drivers/gpio/gpio-sunway.c index b9c6848317db..11581dded64f 100644 --- a/drivers/gpio/gpio-sunway.c +++ b/drivers/gpio/gpio-sunway.c @@ -425,6 +425,7 @@ static void sunway_configure_irqs(struct sunway_gpio *gpio, for (i = 0; i < 2; i++) { ct = &irq_gc->chip_types[i]; + ct->chip.name = "GPIO-INT"; ct->chip.irq_ack = irq_gc_ack_set_bit; ct->chip.irq_mask = irq_gc_mask_set_bit; ct->chip.irq_unmask = irq_gc_mask_clr_bit; diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 027df575d57f..9810922df2fd 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -11,20 +11,21 @@ config ARM_GIC select IRQ_DOMAIN_HIERARCHY select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP -config SW64_INTC_V2 - bool "SW64 Interrupt Controller V2" - depends on UNCORE_XUELANG +config SW64_PINTC + bool "SW64 Interrupt Controller" + depends on SW64 default y select GENERIC_IRQ_CHIP select IRQ_DOMAIN help - This enables support for the INTC chip found in SW CHIP3 systems. - The INTC controls devices interrupts and connects them to each - core's local interrupt controller. + This enables support for the pINTC chip found in SW systems. + The PINTC receives all platform devices' interrupts and passes + them to certain core's local interrupt controller for further + interrupt handling. config SW64_LPC_INTC bool "SW64 cpu builtin LPC Interrupt Controller" - depends on SW64_INTC_V2 + depends on SW64_PINTC help Say yes here to add support for the SW64 cpu builtin LPC IRQ controller. diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 787206e166fc..6b90d7ce01f4 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -27,8 +27,8 @@ obj-$(CONFIG_SUN6I_R_INTC) += irq-sun6i-r.o obj-$(CONFIG_SUNXI_NMI_INTC) += irq-sunxi-nmi.o obj-$(CONFIG_ARCH_SPEAR3XX) += spear-shirq.o obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o -obj-$(CONFIG_SW64_INTC_V2) += irq-sw64-intc-v2.o -obj-$(CONFIG_SW64_LPC_INTC) += irq-sw64-lpc-intc.o +obj-$(CONFIG_SW64_PINTC) += irq-sunway-pintc.o +obj-$(CONFIG_SW64_LPC_INTC) += irq-sunway-lpc-intc.o obj-$(CONFIG_SW64_IRQ_CPU) += irq-sunway-cpu.o ifeq ($(CONFIG_UNCORE_XUELANG),y) diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index ff7455c0f3ec..09addf73e132 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -115,10 +115,9 @@ static void handle_dev_int(struct pt_regs *regs) while (stat) { hwirq = ffs(stat) - 1; - generic_handle_domain_irq(NULL, hwirq); + generic_handle_domain_irq(mcu_irq_domain, hwirq); stat &= ~(1UL << hwirq); } - /*do handle irq */ sw64_io_write(node, DEV_INT_CONFIG, config_val); } @@ -186,9 +185,7 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, perf_irq(PMC_PC1, regs); return; case INT_DEV: - old_regs = set_irq_regs(regs); handle_dev_int(regs); - set_irq_regs(old_regs); return; case INT_FAULT: old_regs = set_irq_regs(regs); diff --git a/drivers/irqchip/irq-sw64-lpc-intc.c b/drivers/irqchip/irq-sunway-lpc-intc.c similarity index 34% rename from drivers/irqchip/irq-sw64-lpc-intc.c rename to drivers/irqchip/irq-sunway-lpc-intc.c index 1cbf87478242..b594ddb6eb24 100644 --- a/drivers/irqchip/irq-sw64-lpc-intc.c +++ b/drivers/irqchip/irq-sunway-lpc-intc.c @@ -11,65 +11,125 @@ #define LPC_IRQ 0x4 #define LPC_IRQ_MASK 0x8 -struct lpc_intc_data { - struct irq_domain *domain; - struct irq_chip_generic *gc; -}; +static DEFINE_RAW_SPINLOCK(lpc_lock); + +static int parent_irq; + +static unsigned int cached_irq_mask = 0xffffffff; +static void lpc_irq_mask(struct irq_data *irq_data) +{ + void __iomem *base = irq_data->domain->host_data; + unsigned long flags; + u32 mask = 1 << (irq_data->irq); + + raw_spin_lock_irqsave(&lpc_lock, flags); + cached_irq_mask |= mask; + writel(cached_irq_mask, base + LPC_IRQ_MASK); + raw_spin_unlock_irqrestore(&lpc_lock, flags); +} + +static void lpc_irq_unmask(struct irq_data *irq_data) +{ + void __iomem *base = irq_data->domain->host_data; + unsigned long flags; + u32 mask = 1 << (irq_data->irq); + + raw_spin_lock_irqsave(&lpc_lock, flags); + cached_irq_mask &= ~mask; + writel(cached_irq_mask, base + LPC_IRQ_MASK); + raw_spin_unlock_irqrestore(&lpc_lock, flags); +} -static void lpc_irq_mask_ack(struct irq_data *data) +static void lpc_irq_mask_ack(struct irq_data *irq_data) { - struct irq_chip_generic *gc = irq_data_get_irq_chip_data(data); - struct irq_chip_type *ct = irq_data_get_chip_type(data); - unsigned int mask = data->mask; - - irq_gc_lock(gc); - *ct->mask_cache |= mask; - irq_reg_writel(gc, *ct->mask_cache, ct->regs.mask); - irq_reg_writel(gc, mask, ct->regs.ack); - irq_gc_unlock(gc); + void __iomem *base = irq_data->domain->host_data; + unsigned long flags; + u32 mask = 1 << (irq_data->irq); + + raw_spin_lock_irqsave(&lpc_lock, flags); + cached_irq_mask |= mask; + writel(cached_irq_mask, base + LPC_IRQ_MASK); + writel(mask, base + LPC_IRQ); + raw_spin_unlock_irqrestore(&lpc_lock, flags); } +static struct irq_chip sw64_lpc_chip = { + .name = "LPC-INT", + .irq_mask = lpc_irq_mask, + .irq_unmask = lpc_irq_unmask, + .irq_mask_ack = lpc_irq_mask_ack, + .irq_set_affinity = irq_chip_set_affinity_parent, +}; + static void lpc_irq_handler(struct irq_desc *desc) { - struct lpc_intc_data *b = irq_desc_get_handler_data(desc); + struct irq_domain *domain = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); + void __iomem *base = domain->host_data; unsigned int irq; u32 status; chained_irq_enter(chip, desc); - status = irq_reg_readl(b->gc, LPC_IRQ); + status = readl(base + LPC_IRQ); if (status == 0) { - raw_spin_lock(&desc->lock); handle_bad_irq(desc); - raw_spin_unlock(&desc->lock); goto out; } while (status) { irq = __ffs(status); status &= ~BIT(irq); - generic_handle_irq(irq_find_mapping(b->domain, irq)); + generic_handle_irq(irq_find_mapping(domain, irq)); } out: chained_irq_exit(chip, desc); } +static int sw64_lpc_domain_map(struct irq_domain *d, unsigned int virq, + irq_hw_number_t hw) +{ + struct irq_data *irq_data, *parent_data; + + irq_data = irq_domain_get_irq_data(d, virq); + parent_data = irq_get_irq_data(parent_irq); + if (!parent_data) { + pr_warn("Failed to get lpc parent irq data!\n"); + return -EFAULT; + } + + irq_data->parent_data = parent_data; + + irq_set_chip_and_handler(virq, &sw64_lpc_chip, handle_level_irq); + irq_set_probe(virq); + irq_set_status_flags(virq, IRQ_LEVEL); + + return 0; +} + +static const struct irq_domain_ops sw64_lpc_domain_ops = { + .map = sw64_lpc_domain_map, + .xlate = irq_domain_xlate_onecell, +}; + +struct device_node *sw_lpc_intc_node; +EXPORT_SYMBOL(sw_lpc_intc_node); + static int __init lpc_intc_of_init(struct device_node *np, struct device_node *parent) { - unsigned int set = IRQ_NOPROBE | IRQ_LEVEL; - struct lpc_intc_data *data; - struct irq_chip_type *ct; - int parent_irq, ret; + struct irq_domain *lpc_domain; + int ret; void __iomem *base; - int hwirq = 0; - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; + sw_lpc_intc_node = np; + + if (!parent) { + pr_err("no parent intc found\n"); + return -ENXIO; + } base = of_iomap(np, 0); if (!base) { @@ -85,53 +145,22 @@ static int __init lpc_intc_of_init(struct device_node *np, goto out_unmap; } - data->domain = irq_domain_add_linear(np, LPC_NR_IRQS, - &irq_generic_chip_ops, NULL); - if (!data->domain) { + lpc_domain = irq_domain_add_legacy(np, LPC_NR_IRQS, + 0, 0, &sw64_lpc_domain_ops, base); + if (!lpc_domain) { ret = -ENOMEM; goto out_unmap; } - /* Allocate a single Generic IRQ chip for this node */ - ret = irq_alloc_domain_generic_chips(data->domain, 16, 1, np->name, - handle_level_irq, 0, set, - IRQ_GC_INIT_MASK_CACHE); - if (ret) { - pr_err("failed to allocate generic irq chip\n"); - goto out_free_domain; - } - /* Set the IRQ chaining logic */ irq_set_chained_handler_and_data(parent_irq, - lpc_irq_handler, data); - - data->gc = irq_get_domain_generic_chip(data->domain, 0); - data->gc->reg_base = base; - data->gc->private = data; - - ct = data->gc->chip_types; - - ct->regs.ack = LPC_IRQ; - ct->regs.mask = LPC_IRQ_MASK; - ct->chip.irq_mask = irq_gc_mask_set_bit; - ct->chip.irq_unmask = irq_gc_mask_clr_bit; - ct->chip.irq_ack = irq_gc_ack_set_bit; - ct->chip.irq_mask_ack = lpc_irq_mask_ack; - - for (hwirq = 0 ; hwirq < 16 ; hwirq++) - irq_create_mapping(data->domain, hwirq); - - /* Enable LPC interrupts */ - writel(0xffffebdd, base + LPC_IRQ_MASK); + lpc_irq_handler, lpc_domain); return 0; -out_free_domain: - irq_domain_remove(data->domain); out_unmap: iounmap(base); out_free: - kfree(data); return ret; } IRQCHIP_DECLARE(sw_lpc_intc, "sw64,lpc_intc", lpc_intc_of_init); diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c new file mode 100644 index 000000000000..8d789fd255c8 --- /dev/null +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +/* + * Multi-node platform device implementation hasn't been thought through yet, + * which means how to obtain CPU node is ambiguous here. It is highly likely + * that this will be passed through ACPI or DTS. Leave node with 0 as default + * for now and wait for platform guys to check this later. + */ +#define DEFAULT_CPU_NODE 0 +static int cpu_node = DEFAULT_CPU_NODE; + +struct devint_chipdata { + int node; +}; + +static DEFINE_RAW_SPINLOCK(devint_lock); +static void lock_dev_lock(void) +{ + raw_spin_lock(&devint_lock); +} + +static void unlock_dev_lock(void) +{ + raw_spin_unlock(&devint_lock); +} + +static void mcu_irq_mask(struct irq_data *data) +{ + struct devint_chipdata *chip_data = data->chip_data; + unsigned int mask; + int hwirq = data->hwirq; + int node; + + node = chip_data->node; + + mask = sw64_io_read(node, MCU_DVC_INT_EN); + mask &= ~(0x1UL << hwirq); + sw64_io_write(node, MCU_DVC_INT_EN, mask); +} + +static void mcu_irq_unmask(struct irq_data *data) +{ + struct devint_chipdata *chip_data = data->chip_data; + unsigned int mask; + int hwirq = data->hwirq; + int node; + + node = chip_data->node; + + mask = sw64_io_read(node, MCU_DVC_INT_EN); + mask |= (0x1UL << hwirq); + sw64_io_write(node, MCU_DVC_INT_EN, mask); +} + +static void mcu_irq_enable(struct irq_data *irq_data) +{ + struct devint_chipdata *data = irq_data->chip_data; + unsigned long devint_conf; + int node; + + node = data->node; + + devint_conf = sw64_io_read(node, DEV_INT_CONFIG); + devint_conf |= (1UL << 8); + sw64_io_write(node, DEV_INT_CONFIG, devint_conf); + mcu_irq_unmask(irq_data); +} + +static void mcu_irq_disable(struct irq_data *irq_data) +{ + struct devint_chipdata *data = irq_data->chip_data; + unsigned long devint_conf; + int node; + + node = data->node; + + devint_conf = sw64_io_read(node, DEV_INT_CONFIG); + devint_conf &= ~(1UL << 8); + sw64_io_write(node, DEV_INT_CONFIG, devint_conf); + mcu_irq_mask(irq_data); +} + +static int __assign_mcu_irq_config(int node, cpumask_t *targets) +{ + unsigned long dev_int_tar, val; + unsigned int cpu; + int phy_cpu; + + for_each_cpu(cpu, targets) { + /* + * Hardware requires dev ints be redirected to on-node + * cores only. Thus, we remove all off-node cpu in the + * target mask. + */ + if (cpu_to_node(cpu) != node) + cpumask_clear_cpu(cpu, targets); + } + + /* Use the last one in valid cpus to avoid core 0. */ + cpu = cpumask_last(targets); + if (cpu >= nr_cpu_ids) + return -EPERM; + + phy_cpu = cpu_to_rcid(cpu); + + val = sw64_io_read(node, DEV_INT_CONFIG); + dev_int_tar = DEV_INT_TARGET(phy_cpu); + val &= 0xffff; + val |= dev_int_tar << 16; + sw64_io_write(node, DEV_INT_CONFIG, val); + + return 0; +} + +static int assign_mcu_irq_config(int node, cpumask_t *targets) +{ + int ret; + + lock_dev_lock(); + ret = __assign_mcu_irq_config(node, targets); + unlock_dev_lock(); + + return ret; +} + +static int mcu_irq_set_affinity(struct irq_data *irq_data, + const struct cpumask *dest, bool force) +{ + struct devint_chipdata *chip_data = irq_data->chip_data; + cpumask_t targets; + int node, ret = 0; + + if (cpumask_any_and(dest, cpu_online_mask) >= nr_cpu_ids) + return -EINVAL; + + cpumask_and(&targets, dest, cpu_online_mask); + + node = chip_data->node; + + mcu_irq_disable(irq_data); + ret = assign_mcu_irq_config(node, &targets); + mcu_irq_enable(irq_data); + + return ret; +} + +static struct irq_chip onchip_intc = { + .name = "MCU-INT", + .irq_enable = mcu_irq_enable, + .irq_disable = mcu_irq_disable, + .irq_mask = mcu_irq_mask, + .irq_unmask = mcu_irq_unmask, + .irq_set_affinity = mcu_irq_set_affinity, +}; + +static struct devint_chipdata * +alloc_sw64_devint_chip_data(struct irq_data *irq_data) +{ + struct devint_chipdata *chip_data; + int node; + + node = irq_data_get_node(irq_data); + chip_data = kzalloc_node(sizeof(*chip_data), GFP_KERNEL, node); + if (!chip_data) + return NULL; + + return chip_data; +} + +static void sw64_intc_free_irqs(struct irq_domain *irq_domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *irq_data; + struct devint_chipdata *chip_data; + int i = 0; + + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_domain_get_irq_data(irq_domain, virq + i); + if (irq_data && irq_data->chip_data) { + lock_dev_lock(); + chip_data = irq_data->chip_data; + irq_domain_reset_irq_data(irq_data); + kfree(chip_data); + unlock_dev_lock(); + } + } +} + +static int sw64_intc_alloc_irqs(struct irq_domain *irq_domain, + unsigned int virq, + unsigned int nr_irqs, + void *arg) +{ + struct irq_data *irq_data; + struct devint_chipdata *chip_data; + struct irq_fwspec *fwspec = arg; + int default_node = cpu_node, i = 0, hwirq; + + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_domain_get_irq_data(irq_domain, virq + i); + hwirq = fwspec->param[0]; + irq_data->hwirq = hwirq; + + chip_data = alloc_sw64_devint_chip_data(irq_data); + if (!chip_data) + goto out_free; + + chip_data->node = default_node; + irq_data->chip_data = chip_data; + irq_set_chip_and_handler(virq, &onchip_intc, handle_level_irq); + irq_set_status_flags(virq, IRQ_LEVEL); + } + + return 0; + +out_free: + sw64_intc_free_irqs(irq_domain, virq, nr_irqs); + return -ENOMEM; +} + +static const struct irq_domain_ops sw64_intc_domain_ops = { + .xlate = irq_domain_xlate_onecell, + .alloc = sw64_intc_alloc_irqs, + .free = sw64_intc_free_irqs, +}; + +struct irq_domain *mcu_irq_domain; +EXPORT_SYMBOL(mcu_irq_domain); + +#ifdef CONFIG_OF +static int __init +init_mcu_IRQ(struct device_node *intc, struct device_node *parent) +{ + if (parent) { + pr_warn("DeviceTree incore intc not a root irq controller\n"); + return -ENODEV; + } + + mcu_irq_domain = irq_domain_add_linear(intc, 8, + &sw64_intc_domain_ops, NULL); + + if (!mcu_irq_domain) { + pr_warn("root irq domain not avail\n"); + return -ENODEV; + } + + /* with this we don't need to export root_domain */ + irq_set_default_host(mcu_irq_domain); + + /* mask all interrupts for now */ + sw64_io_write(cpu_node, MCU_DVC_INT_EN, 0x0); + + return 0; +} + +IRQCHIP_DECLARE(sw64_intc, "sw64,sw6_irq_controller", init_mcu_IRQ); + +static int __init +init_mcu_vt_IRQ(struct device_node *intc, struct device_node *parent) +{ + if (parent) { + pr_warn("DeviceTree incore intc not a root irq controller\n"); + return -ENODEV; + } + + mcu_irq_domain = irq_domain_add_legacy(intc, 16, 0, 0, + &sw64_intc_domain_ops, NULL); + + if (!mcu_irq_domain) { + pr_warn("root irq domain not avail\n"); + return -ENODEV; + } + + /* with this we don't need to export root_domain */ + irq_set_default_host(mcu_irq_domain); + + return 0; +} + +IRQCHIP_DECLARE(sw64_vt_intc, "sw64,sw6_irq_vt_controller", init_mcu_vt_IRQ); +#endif diff --git a/drivers/irqchip/irq-sw64-intc-v2.c b/drivers/irqchip/irq-sw64-intc-v2.c deleted file mode 100644 index bc2c8ef3ed2f..000000000000 --- a/drivers/irqchip/irq-sw64-intc-v2.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include -#include - -static void fake_irq_mask(struct irq_data *data) -{ -} - -static void fake_irq_unmask(struct irq_data *data) -{ -} - -static struct irq_chip onchip_intc = { - .name = "SW fake Intc", - .irq_mask = fake_irq_mask, - .irq_unmask = fake_irq_unmask, -}; - -static int sw64_intc_domain_map(struct irq_domain *d, unsigned int irq, - irq_hw_number_t hw) -{ - - irq_set_chip_and_handler(irq, &onchip_intc, handle_level_irq); - irq_set_status_flags(irq, IRQ_LEVEL); - return 0; -} - -static const struct irq_domain_ops sw64_intc_domain_ops = { - .xlate = irq_domain_xlate_onecell, - .map = sw64_intc_domain_map, -}; - -#ifdef CONFIG_OF -static struct irq_domain *root_domain; - -static int __init -init_onchip_IRQ(struct device_node *intc, struct device_node *parent) -{ - - int node = 0; - int hwirq = 0, nirq = 8; - - if (parent) - panic("DeviceTree incore intc not a root irq controller\n"); - - root_domain = irq_domain_add_linear(intc, 8, - &sw64_intc_domain_ops, NULL); - - if (!root_domain) - panic("root irq domain not avail\n"); - - /* with this we don't need to export root_domain */ - irq_set_default_host(root_domain); - - for (hwirq = 0 ; hwirq < nirq ; hwirq++) - irq_create_mapping(root_domain, hwirq); - - /*enable MCU_DVC_INT_EN*/ - sw64_io_write(node, MCU_DVC_INT_EN, 0xff); - - return 0; -} - -IRQCHIP_DECLARE(sw64_intc, "sw64,sw6_irq_controller", init_onchip_IRQ); - -static int __init -init_onchip_vt_IRQ(struct device_node *intc, struct device_node *parent) -{ - if (parent) - panic("DeviceTree incore intc not a root irq controller\n"); - - root_domain = irq_domain_add_legacy(intc, 16, 0, 0, - &sw64_intc_domain_ops, NULL); - - if (!root_domain) - panic("root irq domain not avail\n"); - - /* with this we don't need to export root_domain */ - irq_set_default_host(root_domain); - - return 0; -} - -IRQCHIP_DECLARE(sw64_vt_intc, "sw64,sw6_irq_vt_controller", init_onchip_vt_IRQ); -#endif -- Gitee From 31eeb74886ea546abf56c15a8d4801dd01c86e4a Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Fri, 5 Jan 2024 14:24:31 +0800 Subject: [PATCH 110/524] sw64: fix rd_f compatibility issue for C4 To maintain binary compatibility with existing programs, this patch replaces `rd_f` instruction when the specific exception is caught. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/traps.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index a30e18ad1f00..133ad318edb6 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -193,6 +193,32 @@ void simd_emulate(unsigned int inst, unsigned long va) } } +static int try_fix_rd_f(unsigned int inst, struct pt_regs *regs) +{ + int copied; + unsigned int prev_inst, new_inst; + unsigned int ra, prev_ra; + + /* not rd_f */ + if ((inst & 0xfc00ffffU) != 0x18001000) + return -1; + + get_user(prev_inst, (__u32 *)(regs->pc - 8)); + if ((prev_inst & 0xfc00e000U) == 0x20008000) { /* lstw/lstl */ + ra = (inst >> 21) & 0x1f; + prev_ra = (prev_inst >> 21) & 0x1f; + /* ldi ra, 0(prev_ra) */ + new_inst = (0x3e << 26) | (ra << 21) | (prev_ra << 16); + copied = access_process_vm(current, regs->pc - 4, &new_inst, + sizeof(unsigned int), FOLL_FORCE | FOLL_WRITE); + if (copied != sizeof(unsigned int)) + return -1; + regs->pc -= 4; + return 0; + } + return -1; +} + /* * BPT/GENTRAP/OPDEC make regs->pc = exc_pc + 4. debugger should * do something necessary to handle it correctly. @@ -294,6 +320,8 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) return; case IF_OPDEC: + if (try_fix_rd_f(inst, regs) == 0) + return; switch (inst) { #ifdef CONFIG_KPROBES case BREAK_KPROBE: -- Gitee From 4343eb9b28bf5e5f6ffbd96c73f0d64dec292add Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 17 Jan 2024 17:01:55 +0800 Subject: [PATCH 111/524] sw64: acpi: fix coding style and typos Make the coding style conform to linux style without any functional changes. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 3 +- arch/sw_64/include/asm/smp.h | 8 +- arch/sw_64/kernel/acpi.c | 47 ++++++------ arch/sw_64/kernel/smp.c | 47 ++++++------ arch/sw_64/pci/acpi.c | 112 +++++++++++++++------------- arch/sw_64/pci/pci.c | 33 ++++---- drivers/pci/controller/pci-sunway.c | 56 +++++++++----- 7 files changed, 168 insertions(+), 138 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 21bfcef21c5f..689a65e63517 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -91,7 +91,8 @@ extern void __init setup_chip_pci_ops(void); #define setup_chip_pci_ops() do { } while (0) #endif -extern struct pci_controller *pci_bus_to_pci_controller(const struct pci_bus *bus); +extern struct pci_controller * +pci_bus_to_pci_controller(const struct pci_bus *bus); extern struct pci_controller *bus_num_to_pci_controller(unsigned long bus_num); extern void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge); diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h index 0ca636b29dde..097f22b72c25 100644 --- a/arch/sw_64/include/asm/smp.h +++ b/arch/sw_64/include/asm/smp.h @@ -72,7 +72,7 @@ int __cpu_disable(void); void __cpu_die(unsigned int cpu); #endif /* CONFIG_HOTPLUG_CPU */ -struct rcid_infomation { +struct rcid_information { unsigned long thread_bits : 8; /* which thread */ unsigned long thread_shift : 8; unsigned long core_bits : 8; /* which core */ @@ -82,8 +82,8 @@ struct rcid_infomation { unsigned long initialized : 1; }; -extern struct rcid_infomation rcid_info; -extern void rcid_infomation_init(int core_version); +extern struct rcid_information rcid_info; +extern void rcid_information_init(int core_version); extern int get_core_id_from_rcid(int rcid); extern int get_thread_id_from_rcid(int rcid); @@ -104,7 +104,7 @@ static inline void set_rcid_map(unsigned int logical, int rcid) __cpu_to_rcid[0] = 0; } -static inline void rcid_infomation_init(int core_version) { } +static inline void rcid_information_init(int core_version) { } static inline int get_core_id_from_rcid(int rcid) { return 0; } static inline int get_thread_id_from_rcid(int rcid) { return 0; } static inline int get_domain_id_from_rcid(int rcid) { return 0; } diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 4c1243c95c95..20c6e851254b 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -173,10 +173,9 @@ static int rcid_to_cpu(int physical_id) { int i; - for (i = 0; i < NR_CPUS; ++i) { + for (i = 0; i < ARRAY_SIZE(__cpu_to_rcid); ++i) if (__cpu_to_rcid[i] == physical_id) return i; - } /* physical id not found */ return -1; @@ -212,21 +211,24 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) } if (pa->apic_id >= CONFIG_NR_CPUS) { - pr_err("SRAT: PXM %u -> CPU 0x%02x -> Node %u skipped apicid that is too big\n", pxm, pa->apic_id, node); + pr_err("SRAT: PXM %u -> CPU 0x%02x -> Node %u skipped apicid that is too big\n", + pxm, pa->apic_id, node); return; } /* Record the mapping from logical core id to node id */ cpu = rcid_to_cpu(pa->apic_id); if (cpu < 0) { - pr_err("SRAT: Can not find the logical id for physical Core 0x%02x\n", pa->apic_id); + pr_err("SRAT: Can not find the logical id for physical Core 0x%02x\n", + pa->apic_id); return; } early_map_cpu_to_node(cpu, node); node_set(node, numa_nodes_parsed); - pr_info("SRAT: PXM %u -> CPU 0x%02x -> Node %u\n", pxm, pa->apic_id, node); + pr_info("SRAT: PXM %u -> CPU 0x%02x -> Node %u\n", + pxm, pa->apic_id, node); } #ifdef CONFIG_MEMORY_HOTPLUG @@ -297,9 +299,9 @@ EXPORT_SYMBOL(acpi_unmap_cpu); static bool __init is_rcid_duplicate(int rcid) { - unsigned int i; + int i; - for (i = 0; (i < possible_cores) && (i < NR_CPUS); i++) { + for_each_possible_cpu(i) { if (cpu_to_rcid(i) == rcid) return true; } @@ -318,34 +320,31 @@ setup_rcid_and_core_mask(struct acpi_madt_sw_cintc *sw_cintc) * represents the maximum number of cores in the system. */ if (possible_cores >= nr_cpu_ids) { - pr_err(PREFIX "Max core num [%u] reached, core [0x%x] ignored." - " Please check the firmware or CONFIG_NR_CPUS!\n", + pr_err(PREFIX "Max core num [%u] reached, core [0x%x] ignored\n", nr_cpu_ids, rcid); return -ENODEV; } /* The rcid of each core is unique */ if (is_rcid_duplicate(rcid)) { - pr_err(PREFIX "Duplicate core [0x%x] in MADT." - " Please check the firmware!\n", rcid); + pr_err(PREFIX "Duplicate core [0x%x] in MADT\n", rcid); return -EINVAL; } /* We can never disable the boot core, whose rcid is 0 */ if ((rcid == 0) && !is_core_enabled(sw_cintc->flags)) { - pr_err(PREFIX "Boot core disabled in MADT." - " Please check the firmware!\n"); + pr_err(PREFIX "Boot core disabled in MADT\n"); return -EINVAL; } /* Online capable makes core possible */ - if (!is_core_enabled(sw_cintc->flags) && \ + if (!is_core_enabled(sw_cintc->flags) && !is_core_online_capable(sw_cintc->flags)) { disabled_cores++; return 0; } - rcid_infomation_init(sw_cintc->version); + rcid_information_init(sw_cintc->version); /* The logical core ID of the boot core must be 0 */ if (rcid == 0) @@ -363,8 +362,8 @@ setup_rcid_and_core_mask(struct acpi_madt_sw_cintc *sw_cintc) * 1. core is enabled via firmware * 2. core is not disabled by cmdline param(offline) */ - if (is_core_enabled(sw_cintc->flags) && \ - !cpumask_test_cpu(logical_core_id, &cpu_offline)) { + if (is_core_enabled(sw_cintc->flags) && + !cpumask_test_cpu(logical_core_id, &cpu_offline)) { set_cpu_present(logical_core_id, true); if (logical_core_id != 0) present_cores++; @@ -382,8 +381,7 @@ static int __init acpi_parse_sw_cintc(union acpi_subtable_headers *header, sw_cintc = (struct acpi_madt_sw_cintc *)header; if (BAD_MADT_ENTRY(sw_cintc, end)) { - pr_err(PREFIX "SW CINTC entry error." - " Please check the firmware!\n"); + pr_err(PREFIX "SW CINTC entry error\n"); return -EINVAL; } @@ -405,11 +403,11 @@ static int __init acpi_parse_sw_cintc(union acpi_subtable_headers *header, static int __init acpi_process_madt_sw_cintc(void) { - int logical_core, ret; + int i, ret; /* Clean the map from logical core ID to physical core ID */ - for (logical_core = 0; logical_core < NR_CPUS; ++logical_core) - set_rcid_map(logical_core, -1); + for (i = 0; i < ARRAY_SIZE(__cpu_to_rcid); ++i) + set_rcid_map(i, -1); /* Clean core mask */ init_cpu_possible(cpu_none_mask); @@ -453,8 +451,9 @@ void __init acpi_boot_table_init(void) pr_err("Failed to init ACPI tables\n"); disable_acpi(); return; - } else - pr_info("Successfully parsed ACPI table\n"); + } + + pr_info("Successfully parsed ACPI table\n"); /** * Process SW64 Core Interrupt Controller(SW CINTC) in MADT table. diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 42450119ea7d..c198e0510996 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -1,7 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -/* - * linux/arch/sw_64/kernel/smp.c - */ #include #include @@ -20,7 +17,7 @@ #include "proto.h" -struct smp_rcb_struct *smp_rcb = NULL; +struct smp_rcb_struct *smp_rcb; extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; @@ -45,7 +42,7 @@ enum ipi_message_type { int smp_num_cpus = 1; /* Number that came online. */ EXPORT_SYMBOL(smp_num_cpus); -struct rcid_infomation rcid_info = { 0 }; +struct rcid_information rcid_info = { 0 }; #define send_sleep_interrupt(cpu) send_ipi((cpu), II_SLEEP) #define send_wakeup_interrupt(cpu) send_ipi((cpu), II_WAKE) @@ -218,31 +215,31 @@ void __init setup_smp(void) smp_rcb_init(INIT_SMP_RCB); } -void rcid_infomation_init(int core_version) +void rcid_information_init(int core_version) { if (rcid_info.initialized) return; switch (core_version) { - case CORE_VERSION_C3B: - rcid_info.thread_bits = 1; - rcid_info.thread_shift = 31; - rcid_info.core_bits = 5; - rcid_info.core_shift = 0; - rcid_info.domain_bits = 2; - rcid_info.domain_shift = 5; - break; - case CORE_VERSION_C4: - rcid_info.thread_bits = 1; - rcid_info.thread_shift = 8; - rcid_info.core_bits = 6; - rcid_info.core_shift = 0; - rcid_info.domain_bits = 2; - rcid_info.domain_shift = 12; - break; - default: - rcid_info.initialized = 0; - return; + case CORE_VERSION_C3B: + rcid_info.thread_bits = 1; + rcid_info.thread_shift = 31; + rcid_info.core_bits = 5; + rcid_info.core_shift = 0; + rcid_info.domain_bits = 2; + rcid_info.domain_shift = 5; + break; + case CORE_VERSION_C4: + rcid_info.thread_bits = 1; + rcid_info.thread_shift = 8; + rcid_info.core_bits = 6; + rcid_info.core_shift = 0; + rcid_info.domain_bits = 2; + rcid_info.domain_shift = 12; + break; + default: + rcid_info.initialized = 0; + return; } rcid_info.initialized = 1; diff --git a/arch/sw_64/pci/acpi.c b/arch/sw_64/pci/acpi.c index 1353994320b3..2f446116e69e 100644 --- a/arch/sw_64/pci/acpi.c +++ b/arch/sw_64/pci/acpi.c @@ -42,6 +42,7 @@ pci_acpi_setup_ecam_mapping(struct acpi_pci_root *root) struct resource *bus_res = &root->secondary; struct resource cfg_res; struct acpi_device *adev = NULL; + resource_size_t bus_res_size; int ret = 0, bus_shift = 0; u16 seg = root->segment; @@ -53,11 +54,12 @@ pci_acpi_setup_ecam_mapping(struct acpi_pci_root *root) /** * Do the quirk of bus shift here, since we can not - * know the ECAM addr in MCFG table when fill mcfg_quirks + * get the ECAM addr when fill mcfg_quirks. */ bus_shift = ecam_ops->bus_shift; cfg_res.start = root->mcfg_addr + (bus_res->start << bus_shift); - cfg_res.end = cfg_res.start + ((resource_size(bus_res)) << bus_shift) - 1; + bus_res_size = resource_size(bus_res); + cfg_res.end = cfg_res.start + (bus_res_size << bus_shift) - 1; cfg_res.flags = IORESOURCE_MEM; /** @@ -87,66 +89,66 @@ static int pci_acpi_prepare_root_resources(struct acpi_pci_root_info *ci) { int status = 0; acpi_status rc; - unsigned long long mem_space_base = 0; + unsigned long long memh = 0; struct resource_entry *entry = NULL, *tmp = NULL; struct acpi_device *device = ci->bridge; + /** + * To distinguish between mem and pre_mem, firmware + * only pass the lower 32bits of mem via acpi and + * use vendor specific "MEMH" to record the upper + * 32 bits of mem. + * + * Get the upper 32 bits here. + */ + rc = acpi_evaluate_integer(ci->bridge->handle, "MEMH", NULL, &memh); + if (rc != AE_OK) { + dev_err(&device->dev, "unable to retrieve MEMH\n"); + return -EEXIST; + } + /** * Get host bridge resources via _CRS method, the return value * is the num of resource parsed. */ status = acpi_pci_probe_root_resources(ci); - if (status > 0) { + if (status <= 0) { /** - * To distinguish between mem and pre_mem, firmware only pass the - * lower 32bits of mem via acpi and use vendor specific "MEMH" to - * record the upper 32 bits of mem. - * - * Get the upper 32 bits here. + * If not successfully parse resources, destroy + * resources which have been parsed. */ - rc = acpi_evaluate_integer(ci->bridge->handle, - "MEMH", NULL, &mem_space_base); - if (rc != AE_OK) { - dev_err(&device->dev, "unable to retrieve MEMH\n"); - return -EEXIST; - } - resource_list_for_each_entry_safe(entry, tmp, &ci->resources) { - if (entry->res->flags & IORESOURCE_MEM) { - if (!(entry->res->end & 0xFFFFFFFF00000000ULL)) { - /* Patch the mem resource with upper 32 bits */ - entry->res->start |= (mem_space_base << 32); - entry->res->end |= (mem_space_base << 32); - } else { - /** - * Add PREFETCH and MEM_64 flags for pre_mem, - * so that we can distinguish between mem and - * pre_mem. - */ - entry->res->flags |= IORESOURCE_PREFETCH; - entry->res->flags |= IORESOURCE_MEM_64; - } - } - - dev_dbg(&device->dev, - "host bridge resource: 0x%llx-0x%llx flags [0x%lx]\n", - entry->res->start, entry->res->end, entry->res->flags); + dev_info(&device->dev, + "host bridge resource(ignored): %pR\n", + entry->res); + resource_list_destroy_entry(entry); } - return status; + + return 0; } - /** - * If not successfully parse resources, destroy - * resources which have been parsed. - */ resource_list_for_each_entry_safe(entry, tmp, &ci->resources) { - dev_info(&device->dev, - "host bridge resource(ignored): 0x%llx-0x%llx flags [0x%lx]\n", - entry->res->start, entry->res->end, entry->res->flags); - resource_list_destroy_entry(entry); + if (entry->res->flags & IORESOURCE_MEM) { + if (!(entry->res->end & 0xFFFFFFFF00000000ULL)) { + /* Patch mem res with upper 32 bits */ + entry->res->start |= (memh << 32); + entry->res->end |= (memh << 32); + } else { + /** + * Add PREFETCH and MEM_64 flags for + * pre_mem, so that we can distinguish + * between mem and pre_mem. + */ + entry->res->flags |= IORESOURCE_PREFETCH; + entry->res->flags |= IORESOURCE_MEM_64; + } + } + + dev_dbg(&device->dev, + "host bridge resource: %pR\n", entry->res); } - return 0; + return status; } /** @@ -179,21 +181,26 @@ struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root) bus = pci_find_bus(domain, busnum); if (bus) { - memcpy(bus->sysdata, pci_ri->cfg, sizeof(struct pci_config_window)); + memcpy(bus->sysdata, pci_ri->cfg, + sizeof(struct pci_config_window)); kfree(pci_ri->cfg); kfree(pci_ri); kfree(root_ops); } else { - bus = acpi_pci_root_create(root, root_ops, &pci_ri->info, pci_ri->cfg); + bus = acpi_pci_root_create(root, root_ops, + &pci_ri->info, pci_ri->cfg); /** - * No need to do kfree here, because acpi_pci_root_create will free - * mem alloced when it cannot create pci_bus. + * No need to do kfree here, because acpi_pci_root_create + * will free mem alloced when it cannot create pci_bus. */ if (!bus) return NULL; - /* Some quirks for pci controller of Sunway after scanning Root Complex */ + /** + * Some quirks for pci controller of Sunway + * after scanning Root Complex + */ sw64_pci_root_bridge_scan_finish_up(pci_find_host_bridge(bus)); pci_bus_size_bridges(bus); @@ -227,7 +234,10 @@ int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge) ACPI_COMPANION_SET(&bridge->dev, adev); set_dev_node(bus_dev, hose->node); - /* Some quirks for pci controller of Sunway before scanning Root Complex */ + /** + * Some quirks for pci controller of Sunway + * before scanning Root Complex + */ sw64_pci_root_bridge_prepare(bridge); } diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index 3db9816e19f1..87930a62595a 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -130,7 +130,8 @@ void pcibios_fixup_bus(struct pci_bus *bus) * Provide information on locations of various I/O regions in physical * memory. Do this on a per-card basis so that we choose the right hose. */ -asmlinkage long sys_pciconfig_iobase(long which, unsigned long bus, unsigned long dfn) +asmlinkage long +sys_pciconfig_iobase(long which, unsigned long bus, unsigned long dfn) { struct pci_controller *hose; @@ -172,7 +173,8 @@ void __init reserve_mem_for_pci(void) return; } - pr_info("reserved pages for pcie memory space %lx:%lx\n", base >> PAGE_SHIFT, + pr_info("reserved pages for pcie memory space %lx:%lx\n", + base >> PAGE_SHIFT, (base + PCI_32BIT_MEMIO_SIZE) >> PAGE_SHIFT); } @@ -184,7 +186,8 @@ static void quirk_isa_bridge(struct pci_dev *dev) { dev->class = PCI_CLASS_BRIDGE_ISA << 8; } -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82378, quirk_isa_bridge); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82378, + quirk_isa_bridge); /* * Early fix up the Root Complex settings @@ -251,8 +254,10 @@ static void enable_sw_dca(struct pci_dev *dev) continue; else { dca_conf = (1UL << 63) | (dev->bus->number << 8) | dev->devfn; - pr_info("dca device index %d, dca_conf = %#lx\n", i, dca_conf); - write_piu_ior1(node, rc_index, DEVICEID0 + (i << 7), dca_conf); + pr_info("dca device index %d, dca_conf = %#lx\n", + i, dca_conf); + write_piu_ior1(node, rc_index, + DEVICEID0 + (i << 7), dca_conf); break; } } @@ -324,7 +329,8 @@ void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge) * So, need to update bus num of pci host bridge here. */ bridge->busnr = last_bus; - dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus), last_bus); + dev_set_name(&bridge->dev, "pci%04x:%02x", + pci_domain_nr(bus), last_bus); /** * At this point, pci_bus has been created and use old @@ -343,7 +349,8 @@ void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge) pci_add_flags(PCI_REASSIGN_ALL_BUS); } -static void sw64_pci_root_bridge_reserve_legacy_io(struct pci_host_bridge *bridge) +static void +sw64_pci_root_bridge_reserve_legacy_io(struct pci_host_bridge *bridge) { struct pci_bus *bus = bridge->bus; struct resource_entry *entry = NULL; @@ -354,10 +361,8 @@ static void sw64_pci_root_bridge_reserve_legacy_io(struct pci_host_bridge *bridg continue; res = kzalloc(sizeof(struct resource), GFP_KERNEL); - if (res == NULL) { - pr_err("alloc resource for legacy io out of mem\n"); + if (WARN_ON(!res)) return; - } res->name = "legacy io"; res->flags = IORESOURCE_IO; @@ -426,11 +431,11 @@ void sw64_pci_root_bridge_scan_finish_up(struct pci_host_bridge *bridge) * * > echo 1 > /sys/bus/pci/rescan * - * Unexpected errors may occur on the endpoint devices due to the re-assign - * bus numbers of upstream bridges. + * Unexpected errors may occur on the endpoint devices due to the + * re-assign bus numbers of upstream bridges. * - * To work around this problem, the flag PCI_REASSIGN_ALL_BUS is set before - * scanning Root Complex and cleared after scanning Root Complex. + * To work around this problem, the flag PCI_REASSIGN_ALL_BUS is set + * before scanning Root Complex and cleared after scanning Root Complex. */ pci_clear_flags(PCI_REASSIGN_ALL_BUS); } diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 32e850f63a32..4e6a12765bf3 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -2,6 +2,7 @@ #include #include #include +#include #include @@ -26,10 +27,12 @@ void set_adr_int(int node) void set_pcieport_service_irq(int node, int index) { if (IS_ENABLED(CONFIG_PCIE_PME)) - write_piu_ior0(node, index, PMEINTCONFIG, PME_ENABLE_INTD_CORE0); + write_piu_ior0(node, index, + PMEINTCONFIG, PME_ENABLE_INTD_CORE0); if (IS_ENABLED(CONFIG_PCIEAER)) - write_piu_ior0(node, index, AERERRINTCONFIG, AER_ENABLE_INTD_CORE0); + write_piu_ior0(node, index, + AERERRINTCONFIG, AER_ENABLE_INTD_CORE0); } int chip_pcie_configure(struct pci_controller *hose) @@ -161,10 +164,12 @@ int chip_pcie_configure(struct pci_controller *hose) if (pcie_caps_offset == 0) continue; - pci_read_config_word(dev, pcie_caps_offset + PCI_EXP_DEVCTL, &devctl); + pci_read_config_word(dev, + pcie_caps_offset + PCI_EXP_DEVCTL, &devctl); devctl &= ~(PCI_EXP_DEVCTL_PAYLOAD | PCI_EXP_DEVCTL_READRQ); devctl |= new_values; - pci_write_config_word(dev, pcie_caps_offset + PCI_EXP_DEVCTL, devctl); + pci_write_config_word(dev, + pcie_caps_offset + PCI_EXP_DEVCTL, devctl); } return bus_max_num; @@ -212,7 +217,8 @@ static void set_rc_piu(unsigned long node, unsigned long index) /* set PCI-E root class code */ value = read_rc_conf(node, index, RC_REVISION_ID); - write_rc_conf(node, index, RC_REVISION_ID, (PCI_CLASS_BRIDGE_HOST << 16) | value); + write_rc_conf(node, index, RC_REVISION_ID, + (PCI_CLASS_BRIDGE_HOST << 16) | value); /* disable DBI_RO_WR_EN */ write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); @@ -228,8 +234,8 @@ static void set_rc_piu(unsigned long node, unsigned long index) if (IS_ENABLED(CONFIG_PCI_MSI)) { write_piu_ior0(node, index, MSIADDR, MSIX_MSG_ADDR); #ifdef CONFIG_UNCORE_XUELANG - for (i = 0; i < 256; i++) - write_piu_ior0(node, index, MSICONFIG0 + (i << 7), 0); + for (i = 0; i < 256; i++) + write_piu_ior0(node, index, MSICONFIG0 + (i << 7), 0); #endif } } @@ -302,7 +308,8 @@ static void hose_init(struct pci_controller *hose) hose->pre_mem_space->flags = IORESOURCE_MEM | IORESOURCE_PREFETCH | IORESOURCE_MEM_64; if (request_resource(&iomem_resource, hose->pre_mem_space) < 0) - pr_err("Failed to request 64bit MEM on hose %ld\n", hose->index); + pr_err("Failed to request 64bit MEM on hose %ld\n", + hose->index); hose->io_space->start = pci_io_base | PCI_LEGACY_IO; hose->io_space->end = hose->io_space->start + PCI_LEGACY_IO_SIZE - 1; hose->io_space->name = "pci io space"; @@ -406,7 +413,9 @@ static int sw64_pcie_read_rc_cfg(struct pci_bus *bus, unsigned int devfn, if (IS_ENABLED(CONFIG_PCI_DEBUG)) pr_debug("rc read addr:%px bus %d, devfn %#x, where %#x size=%d\t", - cfg_iobase + ((where & ~3) << 5), bus->number, devfn, where, size); + cfg_iobase + ((where & ~3) << 5), + bus->number, + devfn, where, size); if ((uintptr_t)where & (size - 1)) { *val = 0; @@ -474,7 +483,9 @@ int sw64_pcie_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, if (IS_ENABLED(CONFIG_PCI_DEBUG)) pr_debug("rc write addr:%px bus %d, devfn %#x, where %#x *val %#x size %d\n", - cfg_iobase + ((where & ~3) << 5), bus->number, devfn, where, val, size); + cfg_iobase + ((where & ~3) << 5), + bus->number, + devfn, where, val, size); writel(data, cfg_iobase + ((where & ~3) << 5)); @@ -482,7 +493,9 @@ int sw64_pcie_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, } /** - * sw64_pcie_valid_device - check if a valid device is present on bus + * sw64_pcie_valid_device - check if a valid device is present + * on bus + * * @bus : PCI bus structure * @devfn: device/function * @@ -502,7 +515,9 @@ static bool sw64_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) } /** - * sw64_pcie_config_read - read val from config space of PCI host controller or device + * sw64_pcie_config_read - read val from config space of + * PCI host controller or device + * * @bus : PCI bus structure * @devfn: device/function * @where: offset from base @@ -534,7 +549,9 @@ static int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, } /** - * sw64_pcie_config_write - write val to config space of PCI host controller or device + * sw64_pcie_config_write - write val to config space of PCI + * host controller or device + * * @bus : PCI bus structure * @devfn: device/function * @where: offset from base @@ -666,21 +683,24 @@ static int sw64_pci_prepare_controller(struct pci_controller *hose, * pass Endpoint config space base address, and define Root Complex * config space base address("RCCB") separately in the ACPI namespace. */ - rc = acpi_evaluate_integer(adev->handle, "RCCB", NULL, &rc_config_base_addr); + rc = acpi_evaluate_integer(adev->handle, + "RCCB", NULL, &rc_config_base_addr); if (rc != AE_OK) { dev_err(&adev->dev, "unable to retrieve RCCB\n"); return -EEXIST; } /* Get Root Complex I/O space base addr from ACPI namespace */ - rc = acpi_evaluate_integer(adev->handle, "RCIO", NULL, &pci_io_base_addr); + rc = acpi_evaluate_integer(adev->handle, + "RCIO", NULL, &pci_io_base_addr); if (rc != AE_OK) { dev_err(&adev->dev, "unable to retrieve RCIO\n"); return -EEXIST; } /* Get Endpoint I/O space base addr from ACPI namespace */ - rc = acpi_evaluate_integer(adev->handle, "EPIO", NULL, &ep_io_base_addr); + rc = acpi_evaluate_integer(adev->handle, + "EPIO", NULL, &ep_io_base_addr); if (rc != AE_OK) { dev_err(&adev->dev, "unable to retrieve EPIO\n"); return -EEXIST; @@ -768,10 +788,8 @@ static int sw64_pci_ecam_init(struct pci_config_window *cfg) } hose = kzalloc(sizeof(*hose), GFP_KERNEL); - if (!hose) { - dev_err(dev, "out of memory when alloc mem for pci_controller\n"); + if (!hose) return -ENOMEM; - } /* Get Endpoint config space base address from MCFG table */ mcfg_addr = cfg->res.start - (cfg->busr.start << cfg->ops->bus_shift); -- Gitee From 01222e61dc96c7312582cad224d15484334732a5 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 10 Jan 2024 16:45:32 +0800 Subject: [PATCH 112/524] sw64: dts: fix properties for spi controller device node Fix #address-cells and #size-cells. Remove unused properties and related code in driver. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/chip3.dts | 24 ++++++++---------------- drivers/spi/spi-chip3-mmio.c | 6 ------ 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/arch/sw_64/boot/dts/chip3.dts b/arch/sw_64/boot/dts/chip3.dts index 082506393ac9..502903040bbd 100644 --- a/arch/sw_64/boot/dts/chip3.dts +++ b/arch/sw_64/boot/dts/chip3.dts @@ -127,24 +127,20 @@ pvt: pvt@0x8030 { }; spi: spi@0x8032 { - #address-cells = <2>; - #size-cells = <2>; + #address-cells = <1>; + #size-cells = <0>; compatible = "sunway,chip3-spi"; reg = <0x8032 0x0 0x0 0x8000>; clocks = <&spiclk>; - interrupt-parent=<&intc>; - interrupts = <4>; + poll_mode = <1>; /* poll_mode:1 interrupt mode: 0 */ + reg-io-width = <2>; status = "okay"; flash@0 { - compatible = "winbond,w25q32dw", "jedec,spi-flash"; + compatible = "winbond,w25q32dw", "jedec,spi-nor"; spi-max-frequency = <25000000>; m25p,fast-read; - spi-cpha; - spi-cpol; - poll_mode = <1>; /* poll_mode:1 interrupt mode: 0 */ - reg-io-width = <2>; - reg = <0 0 0 0 >; /* 0: flash chip selected bit */ + reg = <0>; /* 0: flash chip selected bit */ partitions { compatible = "fixed-partitions"; @@ -159,14 +155,10 @@ partition@0 { }; flash@1 { - compatible = "winbond,w25q32dw", "jedec,spi-flash"; + compatible = "winbond,w25q32dw", "jedec,spi-nor"; spi-max-frequency = <25000000>; m25p,fast-read; - spi-cpha; - spi-cpol; - poll_mode = <1>; /* poll_mode:1 interrupt mode: 0 */ - reg-io-width = <2>; - reg = <1 0 0 0 >; /* 1: flash chip selected bit */ + reg = <1>; /* 1: flash chip selected bit */ partitions { compatible = "fixed-partitions"; diff --git a/drivers/spi/spi-chip3-mmio.c b/drivers/spi/spi-chip3-mmio.c index a907f13d4ae5..5e366457ddc8 100644 --- a/drivers/spi/spi-chip3-mmio.c +++ b/drivers/spi/spi-chip3-mmio.c @@ -53,12 +53,6 @@ static int chip3_spi_mmio_probe(struct platform_device *pdev) return PTR_ERR(dws->regs); } - dws->irq = platform_get_irq(pdev, 0); - if (dws->irq < 0) { - dev_err(&pdev->dev, "no irq resource?\n"); - return dws->irq; /* -ENXIO */ - } - dwsmmio->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(dwsmmio->clk)) return PTR_ERR(dwsmmio->clk); -- Gitee From 63c0f114b4fb613271a5da6b451df345f7e92025 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 15 Jan 2024 09:27:28 +0800 Subject: [PATCH 113/524] sw64: dts: add chosen node It is better to have a chosen node in the device tree. If not present, the newer kernel will give a warning. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/chip3.dts | 11 +++++++---- arch/sw_64/boot/dts/chip_vt.dts | 11 +++++++---- arch/sw_64/boot/dts/empty.dts | 11 +++++++---- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/arch/sw_64/boot/dts/chip3.dts b/arch/sw_64/boot/dts/chip3.dts index 502903040bbd..587f9427cbf6 100644 --- a/arch/sw_64/boot/dts/chip3.dts +++ b/arch/sw_64/boot/dts/chip3.dts @@ -5,10 +5,13 @@ /dts-v1/; / { - compatible = "sunway,chip3"; - model = "chip3"; - #address-cells = <2>; - #size-cells = <2>; + compatible = "sunway,chip3"; + model = "chip3"; + #address-cells = <2>; + #size-cells = <2>; + + chosen { + }; soc { compatible = "simple-bus"; diff --git a/arch/sw_64/boot/dts/chip_vt.dts b/arch/sw_64/boot/dts/chip_vt.dts index f26285367f98..0c7227755b1c 100644 --- a/arch/sw_64/boot/dts/chip_vt.dts +++ b/arch/sw_64/boot/dts/chip_vt.dts @@ -5,10 +5,13 @@ /dts-v1/; / { - compatible = "sunway,chip3"; - model = "chip3"; - #address-cells = <2>; - #size-cells = <2>; + compatible = "sunway,chip3"; + model = "chip3"; + #address-cells = <2>; + #size-cells = <2>; + + chosen { + }; soc { compatible = "simple-bus"; diff --git a/arch/sw_64/boot/dts/empty.dts b/arch/sw_64/boot/dts/empty.dts index f8fe34e29641..367f75d43107 100644 --- a/arch/sw_64/boot/dts/empty.dts +++ b/arch/sw_64/boot/dts/empty.dts @@ -5,10 +5,13 @@ /dts-v1/; / { - compatible = "sunway,chip3"; - model = "chip3"; - #address-cells = <2>; - #size-cells = <2>; + compatible = "sunway,chip3"; + model = "chip3"; + #address-cells = <2>; + #size-cells = <2>; + + chosen { + }; soc { }; -- Gitee From 7570fc19538e525da1b3e5d87ef5b207b6e90bee Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 15 Jan 2024 09:28:41 +0800 Subject: [PATCH 114/524] sw64: dts: fix interrupt related properties for GPIO The GPIO controller can send two interrupts to the parent interrupt controller. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/chip3.dts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/boot/dts/chip3.dts b/arch/sw_64/boot/dts/chip3.dts index 587f9427cbf6..0a1ef5305351 100644 --- a/arch/sw_64/boot/dts/chip3.dts +++ b/arch/sw_64/boot/dts/chip3.dts @@ -226,10 +226,9 @@ porta: gpio-contraller@0 { reg = <0 0 0 0>; interrupt-controller; #interrupt-cells = <2>; - interrupt-parent=<&intc>; - interrupts = <0>; - }; + interrupts-extended = <&intc 0>, <&intc 1>; }; + }; }; }; -- Gitee From e4b8d29e5e922a3cc050be7781dbf0785ff3470c Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 15 Jan 2024 11:14:21 +0800 Subject: [PATCH 115/524] sw64: dts: improve properties for LPC-INTC Add some SW64 specific properties: sw64,node : interrupt controller on which node sw64,irq-num : number of interrupts supported sw64,ver : hardware version It should be noted that the LPC-INTC device node will no longer maintain backward compatibility after this commit. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/chip3.dts | 3 ++ drivers/irqchip/irq-sunway-lpc-intc.c | 40 ++++++++++++++++++++------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/arch/sw_64/boot/dts/chip3.dts b/arch/sw_64/boot/dts/chip3.dts index 0a1ef5305351..c60c84074dfc 100644 --- a/arch/sw_64/boot/dts/chip3.dts +++ b/arch/sw_64/boot/dts/chip3.dts @@ -45,6 +45,9 @@ lpc_intc: interrupt-controller@0x8037 { compatible = "sw64,lpc_intc"; reg = <0x8037 0x40000000 0x0 0x8000>; interrupt-controller; + sw64,node = <0>; + sw64,irq-num = <16>; + sw64,ver = <1>; #interrupt-cells = <1>; interrupt-parent = <&intc>; interrupts = <2>; diff --git a/drivers/irqchip/irq-sunway-lpc-intc.c b/drivers/irqchip/irq-sunway-lpc-intc.c index b594ddb6eb24..bc69d1647aed 100644 --- a/drivers/irqchip/irq-sunway-lpc-intc.c +++ b/drivers/irqchip/irq-sunway-lpc-intc.c @@ -7,7 +7,8 @@ #include #include -#define LPC_NR_IRQS 16 +#define PREFIX "LPC-INTC: " + #define LPC_IRQ 0x4 #define LPC_IRQ_MASK 0x8 @@ -122,36 +123,56 @@ static int __init lpc_intc_of_init(struct device_node *np, { struct irq_domain *lpc_domain; int ret; + u32 nr_irqs, node, version; void __iomem *base; + if (WARN_ON(!np || !parent)) + return -ENODEV; + sw_lpc_intc_node = np; - if (!parent) { - pr_err("no parent intc found\n"); - return -ENXIO; + ret = of_property_read_u32(np, "sw64,node", &node); + if (ret) { + pr_err(PREFIX "\"sw64,node\" not found\n"); + return -EINVAL; + } + + ret = of_property_read_u32(np, "sw64,irq-num", &nr_irqs); + if (ret) { + pr_err(PREFIX "\"sw64,irq-num\" not found\n"); + return -EINVAL; + } + + ret = of_property_read_u32(np, "sw64,ver", &version); + if (ret) { + pr_err(PREFIX "\"sw64,ver\" not found\n"); + return -EINVAL; } base = of_iomap(np, 0); if (!base) { - pr_err("failed to remap lpc intc registers\n"); - ret = -ENOMEM; - goto out_free; + pr_err(PREFIX "failed to remap lpc intc registers\n"); + return -ENXIO; } parent_irq = irq_of_parse_and_map(np, 0); if (!parent_irq) { - pr_err("failed to find parent interrupt\n"); + pr_err(PREFIX "failed to find parent interrupt\n"); ret = -EINVAL; goto out_unmap; } - lpc_domain = irq_domain_add_legacy(np, LPC_NR_IRQS, + lpc_domain = irq_domain_add_legacy(np, nr_irqs, 0, 0, &sw64_lpc_domain_ops, base); if (!lpc_domain) { + pr_err(PREFIX "failed to create irq domain\n"); ret = -ENOMEM; goto out_unmap; } + pr_info(PREFIX "version [%u] on node [%u] initialized\n", + version, node); + /* Set the IRQ chaining logic */ irq_set_chained_handler_and_data(parent_irq, lpc_irq_handler, lpc_domain); @@ -160,7 +181,6 @@ static int __init lpc_intc_of_init(struct device_node *np, out_unmap: iounmap(base); -out_free: return ret; } IRQCHIP_DECLARE(sw_lpc_intc, "sw64,lpc_intc", lpc_intc_of_init); -- Gitee From a95d605142958b0efacb83a270e3c50f5753e1b4 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 16 Jan 2024 10:16:37 +0800 Subject: [PATCH 116/524] sw64: dts: improve properties for PINTC Firstly, add some SW64 specific properties: sw64,node : interrupt controller on which node sw64,irq-num : number of interrupts supported sw64,ver : hardware version Secondly, the "reg" property is added to pass the base address of PINTC registers, decoupling the driver code and hardware. Thirdly, some refactoring is done, making each PINTC chip a chip data instead of each irq a chip data. It should be noted that the PINTC device node will no longer maintain backward compatibility after this commit. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/chip3.dts | 21 +- arch/sw_64/boot/dts/chip_vt.dts | 13 +- arch/sw_64/include/asm/uncore_io_junzhang.h | 2 - arch/sw_64/include/asm/uncore_io_xuelang.h | 2 - drivers/irqchip/irq-sunway-pintc.c | 306 +++++++++++--------- 5 files changed, 193 insertions(+), 151 deletions(-) diff --git a/arch/sw_64/boot/dts/chip3.dts b/arch/sw_64/boot/dts/chip3.dts index c60c84074dfc..0e9ad8f5d464 100644 --- a/arch/sw_64/boot/dts/chip3.dts +++ b/arch/sw_64/boot/dts/chip3.dts @@ -35,9 +35,14 @@ spiclk: spiclk { }; - intc: interrupt-controller { - compatible = "sw64,sw6_irq_controller"; + pintc: interrupt-controller { + compatible = "sw64,pintc"; interrupt-controller; + sw64,node = <0>; + sw64,irq-num = <8>; + sw64,ver = <1>; + reg = <0x802a 0x0 0x0 0x1680>, + <0x8030 0x0 0x0 0x8f00>; #interrupt-cells = <1>; }; @@ -49,7 +54,7 @@ lpc_intc: interrupt-controller@0x8037 { sw64,irq-num = <16>; sw64,ver = <1>; #interrupt-cells = <1>; - interrupt-parent = <&intc>; + interrupt-parent = <&pintc>; interrupts = <2>; }; @@ -58,7 +63,7 @@ uart: serial0@8033 { #size-cells = <2>; compatible = "sw6,sunway-apb-uart"; reg = <0x8033 0x0 0x0 0x1000>; - interrupt-parent=<&intc>; + interrupt-parent=<&pintc>; interrupts = <3>; reg-shift = <9>; reg-io-width = <4>; @@ -85,7 +90,7 @@ i2c0@0x8031 { reg = <0x8031 0x0 0x0 0x8000>; clock-frequency = <100000>; clocks = <&i2cclk>; - interrupt-parent=<&intc>; + interrupt-parent = <&pintc>; interrupts = <5>; status = "okay"; }; @@ -97,7 +102,7 @@ i2c1@0x8034 { reg = <0x8034 0x0 0x0 0x8000>; clock-frequency = <100000>; clocks = <&i2cclk>; - interrupt-parent=<&intc>; + interrupt-parent = <&pintc>; interrupts = <6>; status = "okay"; }; @@ -109,7 +114,7 @@ i2c2@0x8035 { reg = <0x8035 0x0 0x0 0x8000>; clock-frequency = <100000>; clocks = <&i2cclk>; - interrupt-parent=<&intc>; + interrupt-parent = <&pintc>; interrupts = <7>; status = "okay"; @@ -229,7 +234,7 @@ porta: gpio-contraller@0 { reg = <0 0 0 0>; interrupt-controller; #interrupt-cells = <2>; - interrupts-extended = <&intc 0>, <&intc 1>; + interrupts-extended = <&pintc 0>, <&pintc 1>; }; }; diff --git a/arch/sw_64/boot/dts/chip_vt.dts b/arch/sw_64/boot/dts/chip_vt.dts index 0c7227755b1c..9889fcae7a5c 100644 --- a/arch/sw_64/boot/dts/chip_vt.dts +++ b/arch/sw_64/boot/dts/chip_vt.dts @@ -19,9 +19,12 @@ soc { #size-cells = <2>; ranges; - intc: interrupt-controller{ - compatible = "sw64,sw6_irq_vt_controller"; + pintc: interrupt-controller { + compatible = "sw64,pintc_vt"; interrupt-controller; + sw64,node = <0>; + sw64,irq-num = <16>; + sw64,ver = <1>; #interrupt-cells = <1>; }; @@ -30,25 +33,27 @@ uart: serial0@8801 { #size-cells = <2>; compatible = "ns16550a"; reg = <0x8801 0x3f8 0x0 0x10>; - interrupt-parent=<&intc>; + interrupt-parent = <&pintc>; interrupts = <12>; reg-shift = <0>; reg-io-width = <1>; clock-frequency = <24000000>; status = "okay"; }; + misc: misc0@8036 { #address-cells = <2>; #size-cells = <2>; compatible = "sw6,sunway-ged"; reg = <0x8036 0x0 0x0 0x20>; - interrupt-parent=<&intc>; + interrupt-parent = <&pintc>; interrupts = <13>; reg-shift = <0>; reg-io-width = <8>; clock-frequency = <24000000>; status = "okay"; }; + fw_cfg: fw_cfg@8049 { dma-coherent; reg = <0x8049 0x20000000 0x0 0x18>; diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index ac37d186a29f..37cfe1fd6807 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -70,8 +70,6 @@ #define PIUCONFIG0_INIT_VAL 0x38016 -#define DEV_INT_TARGET(phy_cpu) ((((phy_cpu) >> 5) << 7) | ((phy_cpu) & 0x3f)) - /*-----------------------addr-----------------------*/ /* INTPU REG */ enum { diff --git a/arch/sw_64/include/asm/uncore_io_xuelang.h b/arch/sw_64/include/asm/uncore_io_xuelang.h index 873f7e86f83b..aeaadec5be16 100644 --- a/arch/sw_64/include/asm/uncore_io_xuelang.h +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -74,8 +74,6 @@ #define PIUCONFIG0_INIT_VAL 0x38056 -#define DEV_INT_TARGET(phy_cpu) ((((phy_cpu) >> 5) << 6) | ((phy_cpu) & 0x1f)) - /*-----------------------addr-----------------------*/ /* CAB0 REG */ enum { diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index 8d789fd255c8..d65c788e354e 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -4,93 +4,107 @@ #include #include #include -#include - -/* - * Multi-node platform device implementation hasn't been thought through yet, - * which means how to obtain CPU node is ambiguous here. It is highly likely - * that this will be passed through ACPI or DTS. Leave node with 0 as default - * for now and wait for platform guys to check this later. - */ -#define DEFAULT_CPU_NODE 0 -static int cpu_node = DEFAULT_CPU_NODE; - -struct devint_chipdata { - int node; +#include +#include + +#define PREFIX "PINTC: " + +#define OFFSET_MCU_DVC_INT_EN 0x3080UL + +#define OFFSET_DEV_INT_CONFIG 0x480UL + +struct pintc_chip_data { + bool vt; /* virtual pintc */ + u32 node; /* node ID */ + u32 version; /* PINTC version */ + void __iomem *pintc_base; /* INTPU base address */ + void __iomem *mcu_base; /* MCU/SPBU base address */ }; -static DEFINE_RAW_SPINLOCK(devint_lock); +static DEFINE_RAW_SPINLOCK(pintc_lock); static void lock_dev_lock(void) { - raw_spin_lock(&devint_lock); + raw_spin_lock(&pintc_lock); } static void unlock_dev_lock(void) { - raw_spin_unlock(&devint_lock); + raw_spin_unlock(&pintc_lock); } static void mcu_irq_mask(struct irq_data *data) { - struct devint_chipdata *chip_data = data->chip_data; - unsigned int mask; + struct pintc_chip_data *chip_data = data->chip_data; + unsigned long mask; int hwirq = data->hwirq; - int node; - node = chip_data->node; - - mask = sw64_io_read(node, MCU_DVC_INT_EN); + mask = readq(chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); mask &= ~(0x1UL << hwirq); - sw64_io_write(node, MCU_DVC_INT_EN, mask); + writeq(mask, chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); } static void mcu_irq_unmask(struct irq_data *data) { - struct devint_chipdata *chip_data = data->chip_data; - unsigned int mask; + struct pintc_chip_data *chip_data = data->chip_data; + unsigned long mask; int hwirq = data->hwirq; - int node; - - node = chip_data->node; - mask = sw64_io_read(node, MCU_DVC_INT_EN); + mask = readq(chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); mask |= (0x1UL << hwirq); - sw64_io_write(node, MCU_DVC_INT_EN, mask); + writeq(mask, chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); } static void mcu_irq_enable(struct irq_data *irq_data) { - struct devint_chipdata *data = irq_data->chip_data; + struct pintc_chip_data *chip_data = irq_data->chip_data; unsigned long devint_conf; - int node; - node = data->node; - - devint_conf = sw64_io_read(node, DEV_INT_CONFIG); + devint_conf = readq(chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); devint_conf |= (1UL << 8); - sw64_io_write(node, DEV_INT_CONFIG, devint_conf); + writeq(devint_conf, chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); mcu_irq_unmask(irq_data); } static void mcu_irq_disable(struct irq_data *irq_data) { - struct devint_chipdata *data = irq_data->chip_data; + struct pintc_chip_data *chip_data = irq_data->chip_data; unsigned long devint_conf; - int node; - - node = data->node; - devint_conf = sw64_io_read(node, DEV_INT_CONFIG); + devint_conf = readq(chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); devint_conf &= ~(1UL << 8); - sw64_io_write(node, DEV_INT_CONFIG, devint_conf); + writeq(devint_conf, chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); mcu_irq_mask(irq_data); } -static int __assign_mcu_irq_config(int node, cpumask_t *targets) +static unsigned long make_mcu_int_target(u32 version, int rcid) +{ + int node, core, thread; + unsigned long target = 0; + + thread = rcid_to_thread_id(rcid); + core = rcid_to_core_id(rcid); + node = rcid_to_domain_id(rcid); + + switch (version) { + case 0x1: /* PINTC v1 */ + target = core | (thread << 5) | (node << 6); + break; + case 0x2: /* PINTC v2 */ + target = core | (thread << 6) | (node << 7); + break; + default: + break; + } + + return target; +} + +static int __assign_mcu_irq_config(const struct pintc_chip_data *chip_data, + cpumask_t *targets) { unsigned long dev_int_tar, val; unsigned int cpu; - int phy_cpu; + int rcid; for_each_cpu(cpu, targets) { /* @@ -98,7 +112,7 @@ static int __assign_mcu_irq_config(int node, cpumask_t *targets) * cores only. Thus, we remove all off-node cpu in the * target mask. */ - if (cpu_to_node(cpu) != node) + if (cpu_to_node(cpu) != chip_data->node) cpumask_clear_cpu(cpu, targets); } @@ -107,23 +121,24 @@ static int __assign_mcu_irq_config(int node, cpumask_t *targets) if (cpu >= nr_cpu_ids) return -EPERM; - phy_cpu = cpu_to_rcid(cpu); + rcid = cpu_to_rcid(cpu); - val = sw64_io_read(node, DEV_INT_CONFIG); - dev_int_tar = DEV_INT_TARGET(phy_cpu); + val = readq(chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); + dev_int_tar = make_mcu_int_target(chip_data->version, rcid); val &= 0xffff; val |= dev_int_tar << 16; - sw64_io_write(node, DEV_INT_CONFIG, val); + writeq(val, chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); return 0; } -static int assign_mcu_irq_config(int node, cpumask_t *targets) +static int assign_mcu_irq_config(const struct pintc_chip_data *chip_data, + cpumask_t *targets) { int ret; lock_dev_lock(); - ret = __assign_mcu_irq_config(node, targets); + ret = __assign_mcu_irq_config(chip_data, targets); unlock_dev_lock(); return ret; @@ -132,25 +147,23 @@ static int assign_mcu_irq_config(int node, cpumask_t *targets) static int mcu_irq_set_affinity(struct irq_data *irq_data, const struct cpumask *dest, bool force) { - struct devint_chipdata *chip_data = irq_data->chip_data; + struct pintc_chip_data *chip_data = irq_data->chip_data; cpumask_t targets; - int node, ret = 0; + int ret = 0; if (cpumask_any_and(dest, cpu_online_mask) >= nr_cpu_ids) return -EINVAL; cpumask_and(&targets, dest, cpu_online_mask); - node = chip_data->node; - mcu_irq_disable(irq_data); - ret = assign_mcu_irq_config(node, &targets); + ret = assign_mcu_irq_config(chip_data, &targets); mcu_irq_enable(irq_data); return ret; } -static struct irq_chip onchip_intc = { +static struct irq_chip pintc_mcu_chip = { .name = "MCU-INT", .irq_enable = mcu_irq_enable, .irq_disable = mcu_irq_disable, @@ -159,75 +172,40 @@ static struct irq_chip onchip_intc = { .irq_set_affinity = mcu_irq_set_affinity, }; -static struct devint_chipdata * -alloc_sw64_devint_chip_data(struct irq_data *irq_data) -{ - struct devint_chipdata *chip_data; - int node; - - node = irq_data_get_node(irq_data); - chip_data = kzalloc_node(sizeof(*chip_data), GFP_KERNEL, node); - if (!chip_data) - return NULL; - - return chip_data; -} - -static void sw64_intc_free_irqs(struct irq_domain *irq_domain, +static void pintc_mcu_free_irqs(struct irq_domain *irq_domain, unsigned int virq, unsigned int nr_irqs) { - struct irq_data *irq_data; - struct devint_chipdata *chip_data; int i = 0; - for (i = 0; i < nr_irqs; i++) { - irq_data = irq_domain_get_irq_data(irq_domain, virq + i); - if (irq_data && irq_data->chip_data) { - lock_dev_lock(); - chip_data = irq_data->chip_data; - irq_domain_reset_irq_data(irq_data); - kfree(chip_data); - unlock_dev_lock(); - } - } + irq_domain_free_irqs_top(irq_domain, virq, nr_irqs); + + for (i = 0; i < nr_irqs; i++) + irq_clear_status_flags(virq + i, IRQ_LEVEL); } -static int sw64_intc_alloc_irqs(struct irq_domain *irq_domain, +static int pintc_mcu_alloc_irqs(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *arg) { - struct irq_data *irq_data; - struct devint_chipdata *chip_data; struct irq_fwspec *fwspec = arg; - int default_node = cpu_node, i = 0, hwirq; + irq_hw_number_t hwirq = fwspec->param[0]; + int i; for (i = 0; i < nr_irqs; i++) { - irq_data = irq_domain_get_irq_data(irq_domain, virq + i); - hwirq = fwspec->param[0]; - irq_data->hwirq = hwirq; - - chip_data = alloc_sw64_devint_chip_data(irq_data); - if (!chip_data) - goto out_free; - - chip_data->node = default_node; - irq_data->chip_data = chip_data; - irq_set_chip_and_handler(virq, &onchip_intc, handle_level_irq); - irq_set_status_flags(virq, IRQ_LEVEL); + irq_domain_set_info(domain, virq + i, hwirq + i, + &pintc_mcu_chip, domain->host_data, + handle_level_irq, NULL, NULL); + irq_set_status_flags(virq + i, IRQ_LEVEL); } return 0; - -out_free: - sw64_intc_free_irqs(irq_domain, virq, nr_irqs); - return -ENOMEM; } -static const struct irq_domain_ops sw64_intc_domain_ops = { +static const struct irq_domain_ops pintc_mcu_domain_ops = { .xlate = irq_domain_xlate_onecell, - .alloc = sw64_intc_alloc_irqs, - .free = sw64_intc_free_irqs, + .alloc = pintc_mcu_alloc_irqs, + .free = pintc_mcu_free_irqs, }; struct irq_domain *mcu_irq_domain; @@ -235,53 +213,111 @@ EXPORT_SYMBOL(mcu_irq_domain); #ifdef CONFIG_OF static int __init -init_mcu_IRQ(struct device_node *intc, struct device_node *parent) +pintc_of_init_common(struct device_node *pintc, + struct device_node *parent, bool vt) { - if (parent) { - pr_warn("DeviceTree incore intc not a root irq controller\n"); + int ret; + u32 nr_irqs, node, version; + void __iomem *pintc_base; + void __iomem *mcu_base; + struct pintc_chip_data *chip_data; + + if (WARN_ON(!pintc)) return -ENODEV; - } - mcu_irq_domain = irq_domain_add_linear(intc, 8, - &sw64_intc_domain_ops, NULL); + if (vt && parent) { + pr_err(PREFIX "virtual pintc has no parent controller\n"); + return -EINVAL; + } - if (!mcu_irq_domain) { - pr_warn("root irq domain not avail\n"); - return -ENODEV; + ret = of_property_read_u32(pintc, "sw64,node", &node); + if (ret) { + pr_err(PREFIX "\"sw64,node\" not found\n"); + return -EINVAL; } - /* with this we don't need to export root_domain */ - irq_set_default_host(mcu_irq_domain); + ret = of_property_read_u32(pintc, "sw64,irq-num", &nr_irqs); + if (ret) { + pr_err(PREFIX "\"sw64,irq-num\" not found\n"); + return -EINVAL; + } - /* mask all interrupts for now */ - sw64_io_write(cpu_node, MCU_DVC_INT_EN, 0x0); + ret = of_property_read_u32(pintc, "sw64,ver", &version); + if (ret) { + pr_err(PREFIX "\"sw64,ver\" not found\n"); + return -EINVAL; + } - return 0; -} + pintc_base = of_iomap(pintc, 0); + if (!pintc_base) { + pr_err(PREFIX "failed to map pintc base address\n"); + return -ENXIO; + } -IRQCHIP_DECLARE(sw64_intc, "sw64,sw6_irq_controller", init_mcu_IRQ); + mcu_base = of_iomap(pintc, 1); + if (!mcu_base) { + pr_err(PREFIX "failed to map mcu base address\n"); + ret = -ENXIO; + goto out_unmap0; + } -static int __init -init_mcu_vt_IRQ(struct device_node *intc, struct device_node *parent) -{ - if (parent) { - pr_warn("DeviceTree incore intc not a root irq controller\n"); - return -ENODEV; + chip_data = kzalloc_node(sizeof(*chip_data), GFP_KERNEL, node); + if (!chip_data) { + ret = -ENOMEM; + goto out_unmap1; } - mcu_irq_domain = irq_domain_add_legacy(intc, 16, 0, 0, - &sw64_intc_domain_ops, NULL); + chip_data->vt = vt; + chip_data->node = node; + chip_data->version = version; + chip_data->pintc_base = pintc_base; + chip_data->mcu_base = mcu_base; + + if (vt) + mcu_irq_domain = irq_domain_add_legacy(pintc, nr_irqs, 0, 0, + &pintc_mcu_domain_ops, chip_data); + else { + mcu_irq_domain = irq_domain_add_linear(pintc, nr_irqs, + &pintc_mcu_domain_ops, chip_data); + /* mask all interrupts for now */ + writeq(0x0, mcu_base + OFFSET_MCU_DVC_INT_EN); + } if (!mcu_irq_domain) { - pr_warn("root irq domain not avail\n"); - return -ENODEV; + pr_err(PREFIX "failed to create irq domain\n"); + ret = -ENOMEM; + goto out_free_mem; } - /* with this we don't need to export root_domain */ + pr_info(PREFIX "version [%u] on node [%u] initialized\n", + version, node); + irq_set_default_host(mcu_irq_domain); return 0; + +out_free_mem: + kfree(chip_data); +out_unmap1: + iounmap(mcu_base); +out_unmap0: + iounmap(pintc_base); + return ret; +} + +static int __init +pintc_of_init(struct device_node *pintc, struct device_node *parent) +{ + return pintc_of_init_common(pintc, parent, false); +} + +IRQCHIP_DECLARE(sw64_pintc, "sw64,pintc", pintc_of_init); + +static int __init +pintc_vt_of_init(struct device_node *pintc, struct device_node *parent) +{ + return pintc_of_init_common(pintc, parent, true); } -IRQCHIP_DECLARE(sw64_vt_intc, "sw64,sw6_irq_vt_controller", init_mcu_vt_IRQ); +IRQCHIP_DECLARE(sw64_pintc_vt, "sw64,pintc_vt", pintc_vt_of_init); #endif -- Gitee From a14f103e3643a2ac2b2befc96e38ccfc2b767fd3 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 1 Feb 2024 15:35:31 +0800 Subject: [PATCH 117/524] sw64: irqchip: fix virtual PINTC not working properly Virtual PINTC is different from the physical one in the following aspects: - Virtual PINTC does not have the registers used in physical PINTC - The topology of virtual interrupt controllers containing virtual PINTC is a flat structure, which is different from the physical one Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-pintc.c | 37 ++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index d65c788e354e..41c52f9fb09b 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -19,6 +19,7 @@ struct pintc_chip_data { u32 version; /* PINTC version */ void __iomem *pintc_base; /* INTPU base address */ void __iomem *mcu_base; /* MCU/SPBU base address */ + struct irq_chip *mcu_chip; }; static DEFINE_RAW_SPINLOCK(pintc_lock); @@ -172,6 +173,10 @@ static struct irq_chip pintc_mcu_chip = { .irq_set_affinity = mcu_irq_set_affinity, }; +static struct irq_chip pintc_mcu_vt_chip = { + .name = "VMCU-INT", +}; + static void pintc_mcu_free_irqs(struct irq_domain *irq_domain, unsigned int virq, unsigned int nr_irqs) { @@ -183,6 +188,19 @@ static void pintc_mcu_free_irqs(struct irq_domain *irq_domain, irq_clear_status_flags(virq + i, IRQ_LEVEL); } +static int pintc_mcu_map_irq(struct irq_domain *domain, + unsigned int virq, irq_hw_number_t hwirq) +{ + struct pintc_chip_data *chip_data = domain->host_data; + + irq_domain_set_info(domain, virq, hwirq, + chip_data->mcu_chip, chip_data, + handle_level_irq, NULL, NULL); + irq_set_status_flags(virq, IRQ_LEVEL); + + return 0; +} + static int pintc_mcu_alloc_irqs(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, @@ -192,17 +210,14 @@ static int pintc_mcu_alloc_irqs(struct irq_domain *domain, irq_hw_number_t hwirq = fwspec->param[0]; int i; - for (i = 0; i < nr_irqs; i++) { - irq_domain_set_info(domain, virq + i, hwirq + i, - &pintc_mcu_chip, domain->host_data, - handle_level_irq, NULL, NULL); - irq_set_status_flags(virq + i, IRQ_LEVEL); - } + for (i = 0; i < nr_irqs; i++) + pintc_mcu_map_irq(domain, virq + i, hwirq + i); return 0; } static const struct irq_domain_ops pintc_mcu_domain_ops = { + .map = pintc_mcu_map_irq, .xlate = irq_domain_xlate_onecell, .alloc = pintc_mcu_alloc_irqs, .free = pintc_mcu_free_irqs, @@ -249,13 +264,13 @@ pintc_of_init_common(struct device_node *pintc, } pintc_base = of_iomap(pintc, 0); - if (!pintc_base) { + if (!vt && !pintc_base) { pr_err(PREFIX "failed to map pintc base address\n"); return -ENXIO; } mcu_base = of_iomap(pintc, 1); - if (!mcu_base) { + if (!vt && !mcu_base) { pr_err(PREFIX "failed to map mcu base address\n"); ret = -ENXIO; goto out_unmap0; @@ -273,10 +288,12 @@ pintc_of_init_common(struct device_node *pintc, chip_data->pintc_base = pintc_base; chip_data->mcu_base = mcu_base; - if (vt) + if (vt) { + chip_data->mcu_chip = &pintc_mcu_vt_chip; mcu_irq_domain = irq_domain_add_legacy(pintc, nr_irqs, 0, 0, &pintc_mcu_domain_ops, chip_data); - else { + } else { + chip_data->mcu_chip = &pintc_mcu_chip; mcu_irq_domain = irq_domain_add_linear(pintc, nr_irqs, &pintc_mcu_domain_ops, chip_data); /* mask all interrupts for now */ -- Gitee From 44ce5e9e43c005213a0454606d0dd6853880ab9f Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Fri, 2 Feb 2024 18:02:05 +0000 Subject: [PATCH 118/524] sw64: pci: change pci transaction ordering rule settings Due to the risk of commit da4d8d735527 ("sw_64: pci: change pci transaction ordering rule settings"), running ltpstress on SW8A virtual machine cause IOMMU to report an error that the AHCI device level-3 page table is invalid. The AHCI driver reads the IO register to learn that the command has completed and then releases the dma address corresponding to the command. However, since Completion is permitted to pass a Posted Request, IO response has come back but the DMA write is not actually finished. To solve this problem, on SW8A platform, we change PCI transaction ordering settings by disabling CPL_PASS_P bit in RC's ORDER_RULE_CTRL register to ensure that a Completion must not pass a Posted Request. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/controller/pci-sunway.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 4e6a12765bf3..7cd612bb82bb 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -205,7 +205,9 @@ static void set_rc_piu(unsigned long node, unsigned long index) write_rc_conf(node, index, RC_PORT_LINK_CTL, 0x1f0020); write_rc_conf(node, index, RC_EXP_DEVCTL, 0x2850); write_rc_conf(node, index, RC_EXP_DEVCTL2, 0x6); +#ifdef CONFIG_UNCORE_XUELANG write_rc_conf(node, index, RC_ORDER_RULE_CTL, 0x0100); +#endif /* enable DBI_RO_WR_EN */ rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); -- Gitee From 01929064a41a0ba5d934568d58a6e7e7283ba7fc Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 6 Feb 2024 14:28:23 +0000 Subject: [PATCH 119/524] sw64: msi: fix bugs in multi-msi interrupts allocation and release First, according to the definition of the PCI bus specification (Rev. 4.0 Version 1.0, 7.7.1.5 "Message Data Register for MSI"), the first vector assigned to multi-msi interrupts must be aligned to the number of allocation. Second, since each irq_data of multi-msi interrupts corresponds to the same chip_data, it can only be released when all multi-msi interrupts have been released. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-msi-v2.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c index 36790dfedb33..698c5c86408a 100644 --- a/drivers/irqchip/irq-sunway-msi-v2.c +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -90,7 +90,7 @@ static bool find_free_cpu_vectors(const struct cpumask *search_mask, int *found_ cpu = cpumask_first(search_mask); try_again: - for (vector = 0; vector < 256; vector++) { + for (vector = 0; vector < 256; vector += nr_irqs) { for (i = 0; i < nr_irqs; i++) if (per_cpu(vector_irq, cpu)[vector + i]) break; @@ -101,8 +101,6 @@ static bool find_free_cpu_vectors(const struct cpumask *search_mask, int *found_ *found_vector = vector; return found; } - - vector += i; } cpu = cpumask_next(cpu, search_mask); @@ -211,7 +209,6 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, struct sw64_msi_chip_data *cdata; int node; int i, vector, cpu; - unsigned long msiaddr; if (unlikely((nr_irqs > 1) && (!is_power_of_2(nr_irqs)))) nr_irqs = __roundup_pow_of_two(nr_irqs); @@ -307,10 +304,9 @@ static int assign_irq_vector(int irq, unsigned int nr_irqs, static void sw64_vector_free_irqs(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { - int i, j; + int i; struct irq_data *irq_data; unsigned long flags; - unsigned int multi_msi; for (i = 0; i < nr_irqs; i++) { irq_data = irq_domain_get_irq_data(domain, virq + i); @@ -320,13 +316,15 @@ static void sw64_vector_free_irqs(struct irq_domain *domain, raw_spin_lock_irqsave(&vector_lock, flags); cdata = irq_data->chip_data; irq_domain_reset_irq_data(irq_data); - multi_msi = cdata->multi_msi; - for (j = 0; j < multi_msi; j++) - per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector + j] = 0; - kfree(cdata); + cdata->multi_msi--; + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; + + if (cdata->multi_msi) + cdata->vector++; + + if (cdata->multi_msi == 0) + kfree(cdata); raw_spin_unlock_irqrestore(&vector_lock, flags); - if (multi_msi > 1) - break; } } } -- Gitee From c13c8cdd009bea31453353056ae361c5fa6641ce Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 20 Feb 2024 14:31:53 +0800 Subject: [PATCH 120/524] sw64: improve sw64_printk() Skip printk headers before writing into sw64_printk_buf, so we won't have unprintable characters in it. Make sw64_printk() void since the return value is never used. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/dup_print.c | 34 ++++++++++++++++++++-------------- kernel/printk/printk.c | 2 +- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/arch/sw_64/kernel/dup_print.c b/arch/sw_64/kernel/dup_print.c index 439ac75feb01..eef31527dc13 100644 --- a/arch/sw_64/kernel/dup_print.c +++ b/arch/sw_64/kernel/dup_print.c @@ -15,33 +15,39 @@ static DEFINE_SPINLOCK(printk_lock); unsigned long sw64_printk_offset; #define PRINTK_SIZE 0x100000UL -int sw64_printk(const char *fmt, va_list args) +void sw64_printk(const char *fmt, va_list args) { char *sw64_printk_buf; - int printed_len = 0; unsigned long flags; + char textbuf[1024]; + const char *text; + size_t text_len; spin_lock_irqsave(&printk_lock, flags); sw64_printk_buf = (char *)(KERNEL_PRINTK_BUFF_BASE + sw64_printk_offset); - if (sw64_printk_offset >= (PRINTK_SIZE-1024)) { //printk wrapped + text_len = vscnprintf(textbuf, sizeof(textbuf), fmt, args); + text = printk_skip_headers(textbuf); + text_len -= text - textbuf; + + if (sw64_printk_offset >= (PRINTK_SIZE - 1024)) { sw64_printk_offset = 0; sw64_printk_buf = (char *)(KERNEL_PRINTK_BUFF_BASE + sw64_printk_offset); memset(sw64_printk_buf, 0, PRINTK_SIZE); - printed_len += vscnprintf(sw64_printk_buf, 1024, fmt, args); - } else { - printed_len += vscnprintf(sw64_printk_buf, 1024, fmt, args); - if (is_in_emul()) { - void __iomem *addr = __va(QEMU_PRINTF_BUFF_BASE); - u64 data = ((u64)sw64_printk_buf & 0xffffffffUL) - | ((u64)printed_len << 32); - *(u64 *)addr = data; - } } - sw64_printk_offset += printed_len; + + memcpy(sw64_printk_buf, text, text_len); + sw64_printk_offset += text_len; + + if (is_in_emul()) { + void __iomem *addr = __va(QEMU_PRINTF_BUFF_BASE); + u64 data = ((u64)sw64_printk_buf & 0xffffffffUL) + | ((u64)text_len << 32); + *(u64 *)addr = data; + } + spin_unlock_irqrestore(&printk_lock, flags); - return printed_len; } #endif diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index efb927e1b34a..a63883b046ef 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2203,7 +2203,7 @@ int vprintk_store(int facility, int level, int ret = 0; u64 ts_nsec; #ifdef CONFIG_SW64_RRK - extern int sw64_printk(const char *fmt, va_list args); + extern void sw64_printk(const char *fmt, va_list args); #endif if (!printk_enter_irqsave(recursion_ptr, irqflags)) -- Gitee From b8b3c863d6db57f88f711b5c4956da6c57e7a0f9 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 20 Feb 2024 16:10:20 +0800 Subject: [PATCH 121/524] sw64: add a new kernel parameter to override mclk Add a new cmdline parameter to override the mclk we have got. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/clocksource/timer-sw64.c | 40 +++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index 615e8a2ed63a..a2f4caced0a3 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -13,8 +13,7 @@ #include #include -#define SHTCLK_RATE_KHZ 25000 -#define SHTCLK_RATE (SHTCLK_RATE_KHZ * 1000) +#define DEFAULT_MCLK 25000 /* Khz */ #if defined(CONFIG_SUBARCH_C4) static u64 read_longtime(struct clocksource *cs) @@ -37,10 +36,31 @@ static u64 notrace read_sched_clock(void) return sw64_read_csr(CSR_SHTCLOCK); } +static int override_mclk_khz; +static int __init setup_mclk(char *str) +{ + get_option(&str, &override_mclk_khz); + + return 0; +} +early_param("mclk_khz", setup_mclk); + void __init sw64_setup_clocksource(void) { - clocksource_register_khz(&clocksource_longtime, SHTCLK_RATE_KHZ); - sched_clock_register(read_sched_clock, BITS_PER_LONG, SHTCLK_RATE); + unsigned long mclk_khz = *((unsigned char *)__va(MB_MCLK)) * 1000; + + if (override_mclk_khz) { + mclk_khz = override_mclk_khz; + pr_info("Override mclk by cmdline.\n"); + } + + if (!mclk_khz) + mclk_khz = DEFAULT_MCLK; + + pr_info("mclk: %ldKhz\n", mclk_khz); + + clocksource_register_khz(&clocksource_longtime, mclk_khz); + sched_clock_register(read_sched_clock, BITS_PER_LONG, mclk_khz * 1000); } void __init setup_sched_clock(void) { } @@ -122,20 +142,18 @@ static struct clocksource clocksource_tc = { }; #endif /* SMP */ -#define DEFAULT_MCLK 25 /* Mhz */ - void __init sw64_setup_clocksource(void) { - unsigned int mclk = *((unsigned char *)__va(MB_MCLK)); + unsigned long mclk_khz = *((unsigned char *)__va(MB_MCLK)) * 1000; - if (!mclk) - mclk = DEFAULT_MCLK; + if (!mclk_khz) + mclk_khz = DEFAULT_MCLK; #ifdef CONFIG_SMP if (is_in_host()) - clocksource_register_khz(&clocksource_longtime, mclk * 1000); + clocksource_register_khz(&clocksource_longtime, mclk_khz); else - clocksource_register_khz(&clocksource_vtime, DEFAULT_MCLK * 1000); + clocksource_register_khz(&clocksource_vtime, mclk_khz); #else clocksource_register_hz(&clocksource_tc, get_cpu_freq()); pr_info("Setup clocksource TC, mult = %d\n", clocksource_tc.mult); -- Gitee From 0f4fb0243846698e52f3887de961fbc3bd7d5605 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Mon, 15 Jan 2024 17:09:59 +0800 Subject: [PATCH 122/524] sw64: add four-socket system support Allow a maximum of 512 cpus to support 4-socket servers. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 7 ++++--- arch/sw_64/configs/junzhang_defconfig | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 0fd1be7195cc..bc85bebde3be 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -396,12 +396,13 @@ config SCHED_SMT places. If unsure say N here. config NR_CPUS - int "Maximum number of CPUs (2-256)" - range 2 256 + int "Maximum number of CPUs (2-512)" + range 2 512 depends on SMP default "64" if UNCORE_XUELANG + default "512" if UNCORE_JUNZHANG help - SW6 support can handle a maximum of 256 CPUs. + SW64 support can handle a maximum of 512 CPUs. config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" diff --git a/arch/sw_64/configs/junzhang_defconfig b/arch/sw_64/configs/junzhang_defconfig index 4f25770ca193..0459d08fd40d 100644 --- a/arch/sw_64/configs/junzhang_defconfig +++ b/arch/sw_64/configs/junzhang_defconfig @@ -28,7 +28,7 @@ CONFIG_DEBUG_PERF_USE_VMALLOC=y CONFIG_SUBARCH_C4=y CONFIG_SMP=y CONFIG_SCHED_SMT=y -CONFIG_NR_CPUS=64 +CONFIG_NR_CPUS=512 CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_NUMA=y CONFIG_HZ=100 -- Gitee From 7ebb4ef635aedf5b251185181525d37b536e7f79 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 12 Dec 2023 16:27:59 +0800 Subject: [PATCH 123/524] sw64: enable hardware denorm by default Enable hardware denormalized number processing by default to improve performance. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/fpu.h | 2 ++ arch/sw_64/kernel/process.c | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/include/uapi/asm/fpu.h b/arch/sw_64/include/uapi/asm/fpu.h index 7a1514c978b9..7f88cfb9399d 100644 --- a/arch/sw_64/include/uapi/asm/fpu.h +++ b/arch/sw_64/include/uapi/asm/fpu.h @@ -6,6 +6,7 @@ * SW-64 floating-point control register defines: */ #define FPCR_DNOD (1UL << 47) /* denorm INV trap disable */ +#define FPCR_DNZ (1UL << 48) /* denorms to zero */ #define FPCR_DNOE (1UL << 48) /* hardware denormal support */ #define FPCR_INVD (1UL << 49) /* invalid op disable (opt.) */ #define FPCR_DZED (1UL << 50) /* division by zero disable (opt.) */ @@ -96,6 +97,7 @@ /* Denorm and Underflow flushing */ #define IEEE_MAP_DMZ (1UL << 12) /* Map denorm inputs to zero */ +#define IEEE_HARD_DM (1UL << 12) /* Hardware denorm processing */ #define IEEE_MAP_UMZ (1UL << 13) /* Map underflowed outputs to zero */ #define IEEE_MAP_MASK (IEEE_MAP_DMZ | IEEE_MAP_UMZ) diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c index 4f3a3a8b4123..cd371a71cfca 100644 --- a/arch/sw_64/kernel/process.c +++ b/arch/sw_64/kernel/process.c @@ -34,12 +34,13 @@ flush_thread(void) /* Arrange for each exec'ed process to start off with a clean slate * with respect to the FPU. This is all exceptions disabled. */ - current_thread_info()->ieee_state = 0; -#ifdef CONFIG_SUBARCH_C3B - wrfpcr(FPCR_INIT | ieee_swcr_to_fpcr(0)); -#else - wrfpcr(FPCR_INIT | FPCR_DNOE | ieee_swcr_to_fpcr(0)); + unsigned int *ieee_state = ¤t_thread_info()->ieee_state; + + *ieee_state = 0; +#ifndef CONFIG_SUBARCH_C3B + *ieee_state |= IEEE_HARD_DM; #endif + wrfpcr(FPCR_INIT | ieee_swcr_to_fpcr(*ieee_state)); /* Clean slate for TLS. */ current_thread_info()->pcb.tp = 0; -- Gitee From 059c77c20f3d78b6153af53feaafd5098d7410ab Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 14 Dec 2023 11:29:34 +0800 Subject: [PATCH 124/524] sw64: rewrite fpcr and swcr conversion Rewrite the conversion between fpcr and swcr. Add C4 new bits support and fix SIMD support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/fpu.h | 167 ++++++++++++++++++++- arch/sw_64/include/asm/sfp-machine.h | 5 + arch/sw_64/include/uapi/asm/fpu.h | 215 ++++++++++++++++++--------- arch/sw_64/kernel/sys_sw64.c | 6 +- arch/sw_64/kernel/traps.c | 42 +++++- arch/sw_64/math-emu/math.c | 90 ++++++----- 6 files changed, 395 insertions(+), 130 deletions(-) diff --git a/arch/sw_64/include/asm/fpu.h b/arch/sw_64/include/asm/fpu.h index a0b0ff5af168..1d5e9b0efecc 100644 --- a/arch/sw_64/include/asm/fpu.h +++ b/arch/sw_64/include/asm/fpu.h @@ -3,8 +3,85 @@ #define _ASM_SW64_FPU_H #include + +#define EXC_SUM_SWC (1UL << 0) + +#define EXC_SUM_DZE_INT (1UL << 39) + +#define EXC_SUM_INV0 (1UL << 1) +#define EXC_SUM_DZE0 (1UL << 2) +#define EXC_SUM_OVF0 (1UL << 3) +#define EXC_SUM_UNF0 (1UL << 4) +#define EXC_SUM_INE0 (1UL << 5) +#define EXC_SUM_OVI0 (1UL << 6) +#define EXC_SUM_DNO0 (1UL << 40) + +#define EXC_SUM_INV1 (1UL << 15) +#define EXC_SUM_DZE1 (1UL << 16) +#define EXC_SUM_OVF1 (1UL << 17) +#define EXC_SUM_UNF1 (1UL << 18) +#define EXC_SUM_INE1 (1UL << 19) +#define EXC_SUM_OVI1 (1UL << 20) +#define EXC_SUM_DNO1 (1UL << 41) + +#define EXC_SUM_INV2 (1UL << 21) +#define EXC_SUM_DZE2 (1UL << 22) +#define EXC_SUM_OVF2 (1UL << 23) +#define EXC_SUM_UNF2 (1UL << 24) +#define EXC_SUM_INE2 (1UL << 25) +#define EXC_SUM_OVI2 (1UL << 26) +#define EXC_SUM_DNO2 (1UL << 42) + +#define EXC_SUM_INV3 (1UL << 27) +#define EXC_SUM_DZE3 (1UL << 28) +#define EXC_SUM_OVF3 (1UL << 29) +#define EXC_SUM_UNF3 (1UL << 30) +#define EXC_SUM_INE3 (1UL << 31) +#define EXC_SUM_OVI3 (1UL << 32) +#define EXC_SUM_DNO3 (1UL << 43) + +#define EXC_SUM_FP_STATUS0 (EXC_SUM_INV0 | EXC_SUM_DZE0 | \ + EXC_SUM_OVF0 | EXC_SUM_UNF0 | \ + EXC_SUM_INE0 | EXC_SUM_OVI0 | \ + EXC_SUM_DNO0) + +#define EXC_SUM_FP_STATUS1 (EXC_SUM_INV1 | EXC_SUM_DZE1 | \ + EXC_SUM_OVF1 | EXC_SUM_UNF1 | \ + EXC_SUM_INE1 | EXC_SUM_OVI1 | \ + EXC_SUM_DNO1) + +#define EXC_SUM_FP_STATUS2 (EXC_SUM_INV2 | EXC_SUM_DZE2 | \ + EXC_SUM_OVF2 | EXC_SUM_UNF2 | \ + EXC_SUM_INE2 | EXC_SUM_OVI2 | \ + EXC_SUM_DNO2) + +#define EXC_SUM_FP_STATUS3 (EXC_SUM_INV3 | EXC_SUM_DZE3 | \ + EXC_SUM_OVF3 | EXC_SUM_UNF3 | \ + EXC_SUM_INE3 | EXC_SUM_OVI3 | \ + EXC_SUM_DNO3) + +#define EXC_SUM_FP_STATUS_ALL (EXC_SUM_FP_STATUS0 | EXC_SUM_FP_STATUS1 | \ + EXC_SUM_FP_STATUS2 | EXC_SUM_FP_STATUS3) + +#define EXC_SUM_INV (EXC_SUM_INV0 | EXC_SUM_INV1 | \ + EXC_SUM_INV2 | EXC_SUM_INV3) +#define EXC_SUM_DZE (EXC_SUM_DZE0 | EXC_SUM_DZE1 | \ + EXC_SUM_DZE2 | EXC_SUM_DZE3) +#define EXC_SUM_OVF (EXC_SUM_OVF0 | EXC_SUM_OVF1 | \ + EXC_SUM_OVF2 | EXC_SUM_OVF3) +#define EXC_SUM_UNF (EXC_SUM_UNF0 | EXC_SUM_UNF1 | \ + EXC_SUM_UNF2 | EXC_SUM_UNF3) +#define EXC_SUM_INE (EXC_SUM_INE0 | EXC_SUM_INE1 | \ + EXC_SUM_INE2 | EXC_SUM_INE3) +#define EXC_SUM_OVI (EXC_SUM_OVI0 | EXC_SUM_OVI1 | \ + EXC_SUM_OVI2 | EXC_SUM_OVI3) +#define EXC_SUM_DNO (EXC_SUM_DNO0 | EXC_SUM_DNO1 | \ + EXC_SUM_DNO2 | EXC_SUM_DNO3) + #ifdef __KERNEL__ +#include + /* * The following two functions don't need trapb/excb instructions * around the mf_fpcr/mt_fpcr instructions because (a) the kernel @@ -62,15 +139,93 @@ swcr_update_status(unsigned long swcr, unsigned long fpcr) * SW64 implements most of the bits in hardware. Collect * the acrued exception bits from the real fpcr. */ - swcr &= ~(IEEE_STATUS_MASK0 | IEEE_STATUS_MASK1 - | IEEE_STATUS_MASK2 | IEEE_STATUS_MASK3); - swcr |= (fpcr >> 35) & IEEE_STATUS_MASK0; - swcr |= (fpcr >> 13) & IEEE_STATUS_MASK1; - swcr |= (fpcr << 14) & IEEE_STATUS_MASK2; - swcr |= (fpcr << 36) & IEEE_STATUS_MASK3; + swcr &= ~IEEE_STATUS_MASK_ALL; + swcr |= ieee_status_fpcr_to_swcr(fpcr & FPCR_STATUS_MASK_ALL); return swcr; } +static inline unsigned long +swcr_status_to_fex(unsigned long swcr_status, int part) +{ + unsigned long fex = 0; + + if (part < -1 || part > 3) { + pr_warn("%s: invalid part index, counting all parts\n", __func__); + part = -1; + } + + if (part == -1 || part == 0) { + fex |= (swcr_status & (IEEE_STATUS_INV0 | IEEE_STATUS_DZE0 | + IEEE_STATUS_OVF0 | IEEE_STATUS_UNF0 | + IEEE_STATUS_INE0 | IEEE_STATUS_DNO0)) >> + (17 - 1); + fex |= fex & IEEE_STATUS_OVI0 ? FP_EX_OVERINT : 0; + } + + if (part == -1 || part == 1) { + fex |= (swcr_status & (IEEE_STATUS_INV1 | IEEE_STATUS_DZE1 | + IEEE_STATUS_OVF1 | IEEE_STATUS_UNF1 | + IEEE_STATUS_INE1 | IEEE_STATUS_DNO1)) >> + (23 - 1); + fex |= fex & IEEE_STATUS_OVI1 ? FP_EX_OVERINT : 0; + } + + if (part == -1 || part == 2) { + fex |= (swcr_status & (IEEE_STATUS_INV2 | IEEE_STATUS_DZE2 | + IEEE_STATUS_OVF2 | IEEE_STATUS_UNF2 | + IEEE_STATUS_INE2 | IEEE_STATUS_DNO2)) >> + (34 - 1); + fex |= fex & IEEE_STATUS_OVI2 ? FP_EX_OVERINT : 0; + } + + if (part == -1 || part == 3) { + fex |= (swcr_status & (IEEE_STATUS_INV3 | IEEE_STATUS_DZE3 | + IEEE_STATUS_OVF3 | IEEE_STATUS_UNF3 | + IEEE_STATUS_INE3 | IEEE_STATUS_DNO3)) >> + (40 - 1); + fex |= fex & IEEE_STATUS_OVI3 ? FP_EX_OVERINT : 0; + } + + return fex; +} + +static inline unsigned long +fex_to_swcr_status(unsigned long fex, int part) +{ + unsigned long swcr_status = 0; + + switch (part) { + case 0: + swcr_status |= (fex & (FP_EX_INVALID | FP_EX_OVERFLOW | + FP_EX_UNDERFLOW | FP_EX_DIVZERO | + FP_EX_INEXACT | FP_EX_DENORM)) << (17 - 1); + swcr_status |= fex & FP_EX_OVERINT ? IEEE_STATUS_OVI0 : 0; + break; + case 1: + swcr_status |= (fex & (FP_EX_INVALID | FP_EX_OVERFLOW | + FP_EX_UNDERFLOW | FP_EX_DIVZERO | + FP_EX_INEXACT | FP_EX_DENORM)) << (23 - 1); + swcr_status |= fex & FP_EX_OVERINT ? IEEE_STATUS_OVI1 : 0; + break; + case 2: + swcr_status |= (fex & (FP_EX_INVALID | FP_EX_OVERFLOW | + FP_EX_UNDERFLOW | FP_EX_DIVZERO | + FP_EX_INEXACT | FP_EX_DENORM)) << (34 - 1); + swcr_status |= fex & FP_EX_OVERINT ? IEEE_STATUS_OVI2 : 0; + break; + case 3: + swcr_status |= (fex & (FP_EX_INVALID | FP_EX_OVERFLOW | + FP_EX_UNDERFLOW | FP_EX_DIVZERO | + FP_EX_INEXACT | FP_EX_DENORM)) << (40 - 1); + swcr_status |= fex & FP_EX_OVERINT ? IEEE_STATUS_OVI3 : 0; + break; + default: + pr_err("%s: invalid part index\n", __func__); + } + + return swcr_status; +} + extern unsigned long sw64_read_fp_reg(unsigned long reg); extern void sw64_write_fp_reg(unsigned long reg, unsigned long val); extern unsigned long sw64_read_fp_reg_s(unsigned long reg); diff --git a/arch/sw_64/include/asm/sfp-machine.h b/arch/sw_64/include/asm/sfp-machine.h index 156bebc9c515..b96938727d70 100644 --- a/arch/sw_64/include/asm/sfp-machine.h +++ b/arch/sw_64/include/asm/sfp-machine.h @@ -60,8 +60,13 @@ do { \ #define FP_EX_DIVZERO IEEE_TRAP_ENABLE_DZE #define FP_EX_INEXACT IEEE_TRAP_ENABLE_INE #define FP_EX_DENORM IEEE_TRAP_ENABLE_DNO +#define FP_EX_OVERINT IEEE_TRAP_ENABLE_OVI +#if defined(CONFIG_SUBARCH_C3A) || defined(CONFIG_SUBARCH_C3B) #define FP_DENORM_ZERO (swcr & IEEE_MAP_DMZ) +#elif defined(CONFIG_SUBARCH_C4) +#define FP_DENORM_ZERO 1 +#endif /* We write the results always */ #define FP_INHIBIT_RESULTS 0 diff --git a/arch/sw_64/include/uapi/asm/fpu.h b/arch/sw_64/include/uapi/asm/fpu.h index 7f88cfb9399d..5b2acc55aa57 100644 --- a/arch/sw_64/include/uapi/asm/fpu.h +++ b/arch/sw_64/include/uapi/asm/fpu.h @@ -5,18 +5,14 @@ /* * SW-64 floating-point control register defines: */ +#define FPCR_EXUN (1UL << 44) /* exact denorm result underflow */ +#define FPCR_OVID (1UL << 45) /* integer overflow disable */ #define FPCR_DNOD (1UL << 47) /* denorm INV trap disable */ #define FPCR_DNZ (1UL << 48) /* denorms to zero */ #define FPCR_DNOE (1UL << 48) /* hardware denormal support */ #define FPCR_INVD (1UL << 49) /* invalid op disable (opt.) */ #define FPCR_DZED (1UL << 50) /* division by zero disable (opt.) */ #define FPCR_OVFD (1UL << 51) /* overflow disable (optional) */ -#define FPCR_INV (1UL << 52) /* invalid operation */ -#define FPCR_DZE (1UL << 53) /* division by zero */ -#define FPCR_OVF (1UL << 54) /* overflow */ -#define FPCR_UNF (1UL << 55) /* underflow */ -#define FPCR_INE (1UL << 56) /* inexact */ -#define FPCR_IOV (1UL << 57) /* integer overflow */ #define FPCR_UNDZ (1UL << 60) /* underflow to zero (opt.) */ #define FPCR_UNFD (1UL << 61) /* underflow disable (opt.) */ #define FPCR_INED (1UL << 62) /* inexact disable (opt.) */ @@ -34,6 +30,7 @@ #define FPCR_INIT FPCR_DYN_NORMAL /* status bit coming from hardware fpcr . definde by fire3 */ +#define FPCR_STATUS_DNO0 (1UL << 46) #define FPCR_STATUS_INV0 (1UL << 52) #define FPCR_STATUS_DZE0 (1UL << 53) #define FPCR_STATUS_OVF0 (1UL << 54) @@ -47,6 +44,7 @@ #define FPCR_STATUS_UNF1 (1UL << 39) #define FPCR_STATUS_INE1 (1UL << 40) #define FPCR_STATUS_OVI1 (1UL << 41) +#define FPCR_STATUS_DNO1 (1UL << 42) #define FPCR_STATUS_INV2 (1UL << 20) #define FPCR_STATUS_DZE2 (1UL << 21) @@ -54,6 +52,7 @@ #define FPCR_STATUS_UNF2 (1UL << 23) #define FPCR_STATUS_INE2 (1UL << 24) #define FPCR_STATUS_OVI2 (1UL << 25) +#define FPCR_STATUS_DNO2 (1UL << 26) #define FPCR_STATUS_INV3 (1UL << 4) #define FPCR_STATUS_DZE3 (1UL << 5) @@ -61,22 +60,30 @@ #define FPCR_STATUS_UNF3 (1UL << 7) #define FPCR_STATUS_INE3 (1UL << 8) #define FPCR_STATUS_OVI3 (1UL << 9) +#define FPCR_STATUS_DNO3 (1UL << 10) -#define FPCR_STATUS_MASK0 (FPCR_STATUS_INV0 | FPCR_STATUS_DZE0 | \ +#define FPCR_STATUS_MASK0 (FPCR_STATUS_INV0 | FPCR_STATUS_DZE0 | \ FPCR_STATUS_OVF0 | FPCR_STATUS_UNF0 | \ - FPCR_STATUS_INE0 | FPCR_STATUS_OVI0) + FPCR_STATUS_INE0 | FPCR_STATUS_OVI0 | \ + FPCR_STATUS_DNO0) -#define FPCR_STATUS_MASK1 (FPCR_STATUS_INV1 | FPCR_STATUS_DZE1 | \ +#define FPCR_STATUS_MASK1 (FPCR_STATUS_INV1 | FPCR_STATUS_DZE1 | \ FPCR_STATUS_OVF1 | FPCR_STATUS_UNF1 | \ - FPCR_STATUS_INE1 | FPCR_STATUS_OVI1) + FPCR_STATUS_INE1 | FPCR_STATUS_OVI1 | \ + FPCR_STATUS_DNO1) -#define FPCR_STATUS_MASK2 (FPCR_STATUS_INV2 | FPCR_STATUS_DZE2 | \ +#define FPCR_STATUS_MASK2 (FPCR_STATUS_INV2 | FPCR_STATUS_DZE2 | \ FPCR_STATUS_OVF2 | FPCR_STATUS_UNF2 | \ - FPCR_STATUS_INE2 | FPCR_STATUS_OVI2) + FPCR_STATUS_INE2 | FPCR_STATUS_OVI2 | \ + FPCR_STATUS_DNO2) -#define FPCR_STATUS_MASK3 (FPCR_STATUS_INV3 | FPCR_STATUS_DZE3 | \ +#define FPCR_STATUS_MASK3 (FPCR_STATUS_INV3 | FPCR_STATUS_DZE3 | \ FPCR_STATUS_OVF3 | FPCR_STATUS_UNF3 | \ - FPCR_STATUS_INE3 | FPCR_STATUS_OVI3) + FPCR_STATUS_INE3 | FPCR_STATUS_OVI3 | \ + FPCR_STATUS_DNO3) + +#define FPCR_STATUS_MASK_ALL (FPCR_STATUS_MASK0 | FPCR_STATUS_MASK1 |\ + FPCR_STATUS_MASK2 | FPCR_STATUS_MASK3) /* @@ -91,38 +98,25 @@ #define IEEE_TRAP_ENABLE_UNF (1UL << 4) /* underflow */ #define IEEE_TRAP_ENABLE_INE (1UL << 5) /* inexact */ #define IEEE_TRAP_ENABLE_DNO (1UL << 6) /* denorm */ +#define IEEE_TRAP_ENABLE_OVI (1UL << 7) /* integer overflow */ #define IEEE_TRAP_ENABLE_MASK (IEEE_TRAP_ENABLE_INV | IEEE_TRAP_ENABLE_DZE |\ IEEE_TRAP_ENABLE_OVF | IEEE_TRAP_ENABLE_UNF |\ - IEEE_TRAP_ENABLE_INE | IEEE_TRAP_ENABLE_DNO) + IEEE_TRAP_ENABLE_INE | IEEE_TRAP_ENABLE_DNO |\ + IEEE_TRAP_ENABLE_OVI) + +#define IEEE_CTL_EXUN (1UL << 10) /* exact denorm result underflow */ /* Denorm and Underflow flushing */ #define IEEE_MAP_DMZ (1UL << 12) /* Map denorm inputs to zero */ #define IEEE_HARD_DM (1UL << 12) /* Hardware denorm processing */ #define IEEE_MAP_UMZ (1UL << 13) /* Map underflowed outputs to zero */ -#define IEEE_MAP_MASK (IEEE_MAP_DMZ | IEEE_MAP_UMZ) +#define IEEE_MAP_MASK (IEEE_HARD_DM | IEEE_MAP_UMZ) /* status bits coming from fpcr: */ -#define IEEE_STATUS_INV (1UL << 17) -#define IEEE_STATUS_DZE (1UL << 18) -#define IEEE_STATUS_OVF (1UL << 19) -#define IEEE_STATUS_UNF (1UL << 20) -#define IEEE_STATUS_INE (1UL << 21) -#define IEEE_STATUS_DNO (1UL << 22) - - -#define IEEE_STATUS_MASK (IEEE_STATUS_INV | IEEE_STATUS_DZE | \ - IEEE_STATUS_OVF | IEEE_STATUS_UNF | \ - IEEE_STATUS_INE | IEEE_STATUS_DNO) - -#define IEEE_SW_MASK (IEEE_TRAP_ENABLE_MASK | \ - IEEE_STATUS_MASK | IEEE_MAP_MASK) - #define IEEE_CURRENT_RM_SHIFT 32 #define IEEE_CURRENT_RM_MASK (3UL << IEEE_CURRENT_RM_SHIFT) -#define IEEE_STATUS_TO_EXCSUM_SHIFT 16 - #define IEEE_INHERIT (1UL << 63) /* inherit on thread create? */ /* ieee_state expand to surport simd added by fire3 */ @@ -133,11 +127,11 @@ #define IEEE_STATUS_UNF0 (1UL << 20) #define IEEE_STATUS_INE0 (1UL << 21) #define IEEE_STATUS_DNO0 (1UL << 22) +#define IEEE_STATUS_OVI0 (1UL << 46) #define IEEE_STATUS_MASK0 (IEEE_STATUS_INV0 | IEEE_STATUS_DZE0 | \ IEEE_STATUS_OVF0 | IEEE_STATUS_UNF0 | \ - IEEE_STATUS_INE0 | IEEE_STATUS_DNO0) - -#define IEEE_STATUS0_TO_EXCSUM_SHIFT 16 + IEEE_STATUS_INE0 | IEEE_STATUS_DNO0 | \ + IEEE_STATUS_OVI0) #define IEEE_STATUS_INV1 (1UL << 23) #define IEEE_STATUS_DZE1 (1UL << 24) @@ -145,11 +139,11 @@ #define IEEE_STATUS_UNF1 (1UL << 26) #define IEEE_STATUS_INE1 (1UL << 27) #define IEEE_STATUS_DNO1 (1UL << 28) +#define IEEE_STATUS_OVI1 (1UL << 47) #define IEEE_STATUS_MASK1 (IEEE_STATUS_INV1 | IEEE_STATUS_DZE1 | \ IEEE_STATUS_OVF1 | IEEE_STATUS_UNF1 | \ - IEEE_STATUS_INE1 | IEEE_STATUS_DNO1) - -#define IEEE_STATUS1_TO_EXCSUM_SHIFT 22 + IEEE_STATUS_INE1 | IEEE_STATUS_DNO1 | \ + IEEE_STATUS_OVI1) #define IEEE_STATUS_INV2 (1UL << 34) #define IEEE_STATUS_DZE2 (1UL << 35) @@ -157,11 +151,11 @@ #define IEEE_STATUS_UNF2 (1UL << 37) #define IEEE_STATUS_INE2 (1UL << 38) #define IEEE_STATUS_DNO2 (1UL << 39) +#define IEEE_STATUS_OVI2 (1UL << 48) #define IEEE_STATUS_MASK2 (IEEE_STATUS_INV2 | IEEE_STATUS_DZE2 | \ IEEE_STATUS_OVF2 | IEEE_STATUS_UNF2 | \ - IEEE_STATUS_INE2 | IEEE_STATUS_DNO2) - -#define IEEE_STATUS2_TO_EXCSUM_SHIFT 33 + IEEE_STATUS_INE2 | IEEE_STATUS_DNO2 | \ + IEEE_STATUS_OVI2) #define IEEE_STATUS_INV3 (1UL << 40) #define IEEE_STATUS_DZE3 (1UL << 41) @@ -169,12 +163,18 @@ #define IEEE_STATUS_UNF3 (1UL << 43) #define IEEE_STATUS_INE3 (1UL << 44) #define IEEE_STATUS_DNO3 (1UL << 45) +#define IEEE_STATUS_OVI3 (1UL << 49) #define IEEE_STATUS_MASK3 (IEEE_STATUS_INV3 | IEEE_STATUS_DZE3 | \ IEEE_STATUS_OVF3 | IEEE_STATUS_UNF3 | \ - IEEE_STATUS_INE3 | IEEE_STATUS_DNO3) + IEEE_STATUS_INE3 | IEEE_STATUS_DNO3 | \ + IEEE_STATUS_OVI3) + +#define IEEE_STATUS_MASK_ALL (IEEE_STATUS_MASK0 | IEEE_STATUS_MASK1 |\ + IEEE_STATUS_MASK2 | IEEE_STATUS_MASK3) -#define IEEE_STATUS3_TO_EXCSUM_SHIFT 39 +#define IEEE_CTL_MASK (IEEE_TRAP_ENABLE_MASK | IEEE_MAP_MASK) +#define IEEE_SW_MASK (IEEE_STATUS_MASK_ALL | IEEE_CTL_MASK) /* * Convert the software IEEE trap enable and status bits into the @@ -185,43 +185,122 @@ * receive my thanks for making all the not-implemented fpcr bits * RAZ forcing us to use system calls to read/write this value. */ + +static inline unsigned long +ieee_status_swcr_to_fpcr(unsigned long sw_status) +{ + unsigned long fp_status = 0; + + fp_status |= (sw_status & (IEEE_STATUS_INV0 | IEEE_STATUS_DZE0 | + IEEE_STATUS_OVF0 | IEEE_STATUS_UNF0 | + IEEE_STATUS_INE0)) << (52 - 17); + + fp_status |= (sw_status & (IEEE_STATUS_INV1 | IEEE_STATUS_DZE1 | + IEEE_STATUS_OVF1 | IEEE_STATUS_UNF1 | + IEEE_STATUS_INE1)) << (36 - 23); + + fp_status |= (sw_status & (IEEE_STATUS_INV2 | IEEE_STATUS_DZE2 | + IEEE_STATUS_OVF2 | IEEE_STATUS_UNF2 | + IEEE_STATUS_INE2)) >> (34 - 20); + + fp_status |= (sw_status & (IEEE_STATUS_INV3 | IEEE_STATUS_DZE3 | + IEEE_STATUS_OVF3 | IEEE_STATUS_UNF3 | + IEEE_STATUS_INE3)) >> (40 - 4); + + fp_status |= sw_status & IEEE_STATUS_OVI0 ? FPCR_STATUS_OVI0 : 0; + fp_status |= sw_status & IEEE_STATUS_DNO0 ? FPCR_STATUS_DNO0 : 0; + + fp_status |= sw_status & IEEE_STATUS_OVI1 ? FPCR_STATUS_OVI1 : 0; + fp_status |= sw_status & IEEE_STATUS_DNO1 ? FPCR_STATUS_DNO1 : 0; + + fp_status |= sw_status & IEEE_STATUS_OVI2 ? FPCR_STATUS_OVI2 : 0; + fp_status |= sw_status & IEEE_STATUS_DNO2 ? FPCR_STATUS_DNO2 : 0; + + fp_status |= sw_status & IEEE_STATUS_OVI3 ? FPCR_STATUS_OVI3 : 0; + fp_status |= sw_status & IEEE_STATUS_DNO3 ? FPCR_STATUS_DNO3 : 0; + + return fp_status; +} + static inline unsigned long ieee_swcr_to_fpcr(unsigned long sw) { unsigned long fp; - fp = (sw & IEEE_STATUS_MASK0) << 35; - fp |= (sw & IEEE_STATUS_MASK1) << 13; - fp |= (sw & IEEE_STATUS_MASK2) >> 14; - fp |= (sw & IEEE_STATUS_MASK3) >> 36; - - fp |= (sw & IEEE_MAP_DMZ) << 36; - fp |= (sw & IEEE_STATUS_MASK0 ? FPCR_SUM : 0); - fp |= (sw & IEEE_STATUS_MASK1 ? FPCR_SUM : 0); - fp |= (sw & IEEE_STATUS_MASK2 ? FPCR_SUM : 0); - fp |= (sw & IEEE_STATUS_MASK3 ? FPCR_SUM : 0); - fp |= (~sw & (IEEE_TRAP_ENABLE_INV - | IEEE_TRAP_ENABLE_DZE - | IEEE_TRAP_ENABLE_OVF)) << 48; - fp |= (~sw & (IEEE_TRAP_ENABLE_UNF | IEEE_TRAP_ENABLE_INE)) << 57; - fp |= (sw & IEEE_MAP_UMZ ? FPCR_UNDZ | FPCR_UNFD : 0); - fp |= (~sw & IEEE_TRAP_ENABLE_DNO) << 41; + fp = ieee_status_swcr_to_fpcr(sw & IEEE_STATUS_MASK_ALL); + + fp |= sw & IEEE_STATUS_MASK_ALL ? FPCR_SUM : 0; + + fp |= sw & IEEE_CTL_EXUN ? FPCR_EXUN : 0; + fp |= sw & IEEE_HARD_DM ? FPCR_DNOE : 0; + fp |= sw & IEEE_MAP_UMZ ? FPCR_UNDZ : 0; + + fp |= sw & IEEE_TRAP_ENABLE_INV ? 0 : FPCR_INVD; + fp |= sw & IEEE_TRAP_ENABLE_DZE ? 0 : FPCR_DZED; + fp |= sw & IEEE_TRAP_ENABLE_OVF ? 0 : FPCR_OVFD; + fp |= sw & IEEE_TRAP_ENABLE_UNF ? 0 : FPCR_UNFD; + fp |= sw & IEEE_TRAP_ENABLE_INE ? 0 : FPCR_INED; + fp |= sw & IEEE_TRAP_ENABLE_DNO ? 0 : FPCR_DNOD; + fp |= sw & IEEE_TRAP_ENABLE_OVI ? 0 : FPCR_OVID; + return fp; } +static inline unsigned long +ieee_status_fpcr_to_swcr(unsigned long fp_status) +{ + unsigned long sw_status = 0; + + sw_status |= (fp_status & (FPCR_STATUS_INV0 | FPCR_STATUS_DZE0 | + FPCR_STATUS_OVF0 | FPCR_STATUS_UNF0 | + FPCR_STATUS_INE0)) >> (52 - 17); + + sw_status |= (fp_status & (FPCR_STATUS_INV1 | FPCR_STATUS_DZE1 | + FPCR_STATUS_OVF1 | FPCR_STATUS_UNF1 | + FPCR_STATUS_INE1)) >> (36 - 23); + + sw_status |= (fp_status & (FPCR_STATUS_INV2 | FPCR_STATUS_DZE2 | + FPCR_STATUS_OVF2 | FPCR_STATUS_UNF2 | + FPCR_STATUS_INE2)) << (34 - 20); + + sw_status |= (fp_status & (FPCR_STATUS_INV3 | FPCR_STATUS_DZE3 | + FPCR_STATUS_OVF3 | FPCR_STATUS_UNF3 | + FPCR_STATUS_INE3)) << (40 - 4); + + sw_status |= fp_status & FPCR_STATUS_OVI0 ? IEEE_STATUS_OVI0 : 0; + sw_status |= fp_status & FPCR_STATUS_DNO0 ? IEEE_STATUS_DNO0 : 0; + + sw_status |= fp_status & FPCR_STATUS_OVI1 ? IEEE_STATUS_OVI1 : 0; + sw_status |= fp_status & FPCR_STATUS_DNO1 ? IEEE_STATUS_DNO1 : 0; + + sw_status |= fp_status & FPCR_STATUS_OVI2 ? IEEE_STATUS_OVI2 : 0; + sw_status |= fp_status & FPCR_STATUS_DNO2 ? IEEE_STATUS_DNO2 : 0; + + sw_status |= fp_status & FPCR_STATUS_OVI3 ? IEEE_STATUS_OVI3 : 0; + sw_status |= fp_status & FPCR_STATUS_DNO3 ? IEEE_STATUS_DNO3 : 0; + + return sw_status; +} + static inline unsigned long ieee_fpcr_to_swcr(unsigned long fp) { unsigned long sw; - sw = (fp >> 35) & IEEE_STATUS_MASK; - sw |= (fp >> 36) & IEEE_MAP_DMZ; - sw |= (~fp >> 48) & (IEEE_TRAP_ENABLE_INV - | IEEE_TRAP_ENABLE_DZE - | IEEE_TRAP_ENABLE_OVF); - sw |= (~fp >> 57) & (IEEE_TRAP_ENABLE_UNF | IEEE_TRAP_ENABLE_INE); - sw |= (fp >> 47) & IEEE_MAP_UMZ; - sw |= (~fp >> 41) & IEEE_TRAP_ENABLE_DNO; + sw = ieee_status_fpcr_to_swcr(fp & FPCR_STATUS_MASK_ALL); + + sw |= fp & FPCR_EXUN ? IEEE_CTL_EXUN : 0; + sw |= fp & FPCR_DNOE ? IEEE_HARD_DM : 0; + sw |= fp & FPCR_UNDZ ? IEEE_MAP_UMZ : 0; + + sw |= fp & FPCR_INVD ? 0 : IEEE_TRAP_ENABLE_INV; + sw |= fp & FPCR_DZED ? 0 : IEEE_TRAP_ENABLE_DZE; + sw |= fp & FPCR_OVFD ? 0 : IEEE_TRAP_ENABLE_OVF; + sw |= fp & FPCR_UNFD ? 0 : IEEE_TRAP_ENABLE_UNF; + sw |= fp & FPCR_INED ? 0 : IEEE_TRAP_ENABLE_INE; + sw |= fp & FPCR_DNOD ? 0 : IEEE_TRAP_ENABLE_DNO; + sw |= fp & FPCR_OVID ? 0 : IEEE_TRAP_ENABLE_OVI; + return sw; } #endif /* _UAPI_ASM_SW64_FPU_H */ diff --git a/arch/sw_64/kernel/sys_sw64.c b/arch/sw_64/kernel/sys_sw64.c index d0198aef554d..46e6197dce76 100644 --- a/arch/sw_64/kernel/sys_sw64.c +++ b/arch/sw_64/kernel/sys_sw64.c @@ -62,7 +62,7 @@ SYSCALL_DEFINE5(setsysinfo, unsigned long, op, void __user *, buffer, if (get_user(exc, (unsigned long __user *)buffer)) return -EFAULT; state = ¤t_thread_info()->ieee_state; - exc &= IEEE_STATUS_MASK; + exc &= IEEE_STATUS_MASK_ALL; /* Update softare trap enable bits. */ swcr = (*state & IEEE_SW_MASK) | exc; @@ -76,7 +76,7 @@ SYSCALL_DEFINE5(setsysinfo, unsigned long, op, void __user *, buffer, /* If any exceptions set by this call, and are unmasked, * send a signal. Old exceptions are not signaled. */ - fex = (exc >> IEEE_STATUS_TO_EXCSUM_SHIFT) & swcr; + fex = swcr_status_to_fex(exc, -1) & swcr; if (fex) { int si_code = FPE_FLTUNK; @@ -92,6 +92,8 @@ SYSCALL_DEFINE5(setsysinfo, unsigned long, op, void __user *, buffer, si_code = FPE_FLTDIV; if (fex & IEEE_TRAP_ENABLE_INV) si_code = FPE_FLTINV; + if (fex & IEEE_TRAP_ENABLE_OVI) + si_code = FPE_INTOVF; send_sig_fault(SIGFPE, si_code, (void __user *)NULL, current); } diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 133ad318edb6..295e8e3e73a6 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -143,17 +143,47 @@ EXPORT_SYMBOL_GPL(sw64_fp_emul_imprecise); long (*sw64_fp_emul)(unsigned long pc) = (void *)dummy_emul; EXPORT_SYMBOL_GPL(sw64_fp_emul); #else -long sw64_fp_emul_imprecise(struct pt_regs *regs, unsigned long writemask); -long sw64_fp_emul(unsigned long pc); +extern long sw64_fp_emul_imprecise(struct pt_regs *regs, unsigned long writemask); +extern long sw64_fp_emul(unsigned long pc); #endif asmlinkage void -do_entArith(unsigned long summary, unsigned long write_mask, +do_entArith(unsigned long exc_sum, unsigned long write_mask, struct pt_regs *regs) { long si_code = FPE_FLTINV; - if (summary & 1) { +#ifndef CONFIG_SUBARCH_C3B + /* integer divide by zero */ + if (exc_sum & EXC_SUM_DZE_INT) + si_code = FPE_INTDIV; + /* integer overflow */ + else if (exc_sum & EXC_SUM_OVI) + si_code = FPE_INTOVF; + /* floating point invalid operation */ + else if (exc_sum & EXC_SUM_INV) + si_code = FPE_FLTINV; + /* floating point divide by zero */ + else if (exc_sum & EXC_SUM_DZE) + si_code = FPE_FLTDIV; + /* floating point overflow */ + else if (exc_sum & EXC_SUM_OVF) + si_code = FPE_FLTOVF; + /* floating point underflow */ + else if (exc_sum & EXC_SUM_UNF) + si_code = FPE_FLTUND; + /* floating point inexact result */ + else if (exc_sum & EXC_SUM_INE) + si_code = FPE_FLTRES; + /* denormalized operand */ + else if (exc_sum & EXC_SUM_DNO) + si_code = FPE_FLTUND; + /* undiagnosed floating-point exception */ + else + si_code = FPE_FLTUNK; +#endif + + if ((exc_sum & EXC_SUM_FP_STATUS_ALL) && (exc_sum & EXC_SUM_SWC)) { /* Software-completion summary bit is set, so try to * emulate the instruction. If the processor supports * precise exceptions, we don't have to search. @@ -166,10 +196,6 @@ do_entArith(unsigned long summary, unsigned long write_mask, if (!user_mode(regs)) die("Arithmetic fault", regs, 0); - /*summary<39> means integer divide by zero in C4.*/ - if ((summary >> 39) & 1) - si_code = FPE_INTDIV; - force_sig_fault(SIGFPE, si_code, (void __user *)regs->pc); } diff --git a/arch/sw_64/math-emu/math.c b/arch/sw_64/math-emu/math.c index b578752f0730..b0c2fbca70b1 100644 --- a/arch/sw_64/math-emu/math.c +++ b/arch/sw_64/math-emu/math.c @@ -22,8 +22,6 @@ * This is for sw64 */ -#define IEEE_E_STATUS_MASK IEEE_STATUS_MASK -#define IEEE_E_STATUS_TO_EXCSUM_SHIFT 0 #define SW64_FP_DENOMAL 1 /* A denormal data */ #define SW64_FP_NORMAL 0 /* A denormal data */ #define SW64_FP_NAN 2 @@ -593,9 +591,9 @@ long sw64_fp_emul(unsigned long pc) done: if (_fex) { /* Record exceptions in software control word. */ - swcr |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + swcr |= fex_to_swcr_status(_fex, 0); current_thread_info()->ieee_state - |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(_fex, 0); /* Update hardware control register. */ fpcr &= (~FPCR_MASK | FPCR_DYN_MASK); @@ -852,9 +850,9 @@ long simd_cmp_emul_d(unsigned long pc) swcr_p0 = swcr_p1 = swcr_p2 = swcr_p3 = swcr; /* manage fpcr_p0 */ if (fex_p0) { - swcr_p0 |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + swcr_p0 |= fex_to_swcr_status(fex_p0, 0); current_thread_info()->ieee_state - |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p0, 0); /* Update hardware control register. */ fpcr_p0 = fpcr; @@ -863,9 +861,9 @@ long simd_cmp_emul_d(unsigned long pc) } if (fex_p1) { - swcr_p1 |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + swcr_p1 |= fex_to_swcr_status(fex_p1, 1); current_thread_info()->ieee_state - |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p1, 1); /* Update hardware control register. */ fpcr_p1 = fpcr; @@ -874,9 +872,9 @@ long simd_cmp_emul_d(unsigned long pc) } if (fex_p2) { - swcr_p2 |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + swcr_p2 |= fex_to_swcr_status(fex_p2, 2); current_thread_info()->ieee_state - |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p2, 2); /* Update hardware control register. */ fpcr_p2 = fpcr; @@ -885,9 +883,9 @@ long simd_cmp_emul_d(unsigned long pc) } if (fex_p3) { - swcr_p3 |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + swcr_p3 |= fex_to_swcr_status(fex_p3, 3); current_thread_info()->ieee_state - |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p3, 3); /* Update hardware control register. */ fpcr_p3 = fpcr; @@ -1119,9 +1117,9 @@ long simd_fp_emul_d(unsigned long pc) swcr_p0 = swcr_p1 = swcr_p2 = swcr_p3 = swcr; /* manage fpcr_p0 */ if (fex_p0) { - swcr_p0 |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + swcr_p0 |= fex_to_swcr_status(fex_p0, 0); current_thread_info()->ieee_state - |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p0, 0); /* Update hardware control register. */ fpcr_p0 = fpcr; @@ -1130,9 +1128,9 @@ long simd_fp_emul_d(unsigned long pc) } if (fex_p1) { - swcr_p1 |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + swcr_p1 |= fex_to_swcr_status(fex_p1, 1); current_thread_info()->ieee_state - |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p1, 1); /* Update hardware control register. */ fpcr_p1 = fpcr; @@ -1141,9 +1139,9 @@ long simd_fp_emul_d(unsigned long pc) } if (fex_p2) { - swcr_p2 |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + swcr_p2 |= fex_to_swcr_status(fex_p2, 2); current_thread_info()->ieee_state - |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p2, 2); /* Update hardware control register. */ fpcr_p2 = fpcr; @@ -1152,9 +1150,9 @@ long simd_fp_emul_d(unsigned long pc) } if (fex_p3) { - swcr_p3 |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + swcr_p3 |= fex_to_swcr_status(fex_p3, 3); current_thread_info()->ieee_state - |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p3, 3); /* Update hardware control register. */ fpcr_p3 = fpcr; @@ -1385,9 +1383,9 @@ long simd_fp_emul_s(unsigned long pc) swcr_p0 = swcr_p1 = swcr_p2 = swcr_p3 = swcr; /* manage fpcr_p0 */ if (fex_p0) { - swcr_p0 |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + swcr_p0 |= fex_to_swcr_status(fex_p0, 0); current_thread_info()->ieee_state - |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p0, 0); /* Update hardware control register. */ fpcr_p0 = fpcr; @@ -1397,9 +1395,9 @@ long simd_fp_emul_s(unsigned long pc) } if (fex_p1) { - swcr_p1 |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + swcr_p1 |= fex_to_swcr_status(fex_p1, 1); current_thread_info()->ieee_state - |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p1, 1); /* Update hardware control register. */ fpcr_p1 = fpcr; @@ -1409,9 +1407,9 @@ long simd_fp_emul_s(unsigned long pc) } if (fex_p2) { - swcr_p2 |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + swcr_p2 |= fex_to_swcr_status(fex_p2, 2); current_thread_info()->ieee_state - |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p2, 2); /* Update hardware control register. */ fpcr_p2 = fpcr; @@ -1421,9 +1419,9 @@ long simd_fp_emul_s(unsigned long pc) } if (fex_p3) { - swcr_p3 |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + swcr_p3 |= fex_to_swcr_status(fex_p3, 3); current_thread_info()->ieee_state - |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p3, 3); /* Update hardware control register. */ fpcr_p3 = fpcr; @@ -1598,9 +1596,9 @@ long mul_add_fp_emul(unsigned long pc) pr_debug("vd = %#lx\n", vd); if (_fex) { /* Record exceptions in software control word. */ - swcr |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + swcr |= fex_to_swcr_status(_fex, 0); current_thread_info()->ieee_state - |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(_fex, 0); /* Update hardware control register. */ fpcr &= (~FPCR_MASK | FPCR_DYN_MASK); @@ -1843,9 +1841,9 @@ long simd_mul_add_fp_emul_s(unsigned long pc) swcr_p0 = swcr_p1 = swcr_p2 = swcr_p3 = swcr; /* manage fpcr_p0 */ if (fex_p0) { - swcr_p0 |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + swcr_p0 |= fex_to_swcr_status(fex_p0, 0); current_thread_info()->ieee_state - |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p0, 0); /* Update hardware control register. */ fpcr_p0 = fpcr; @@ -1854,9 +1852,9 @@ long simd_mul_add_fp_emul_s(unsigned long pc) } if (fex_p1) { - swcr_p1 |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + swcr_p1 |= fex_to_swcr_status(fex_p1, 1); current_thread_info()->ieee_state - |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p1, 1); /* Update hardware control register. */ fpcr_p1 = fpcr; @@ -1865,9 +1863,9 @@ long simd_mul_add_fp_emul_s(unsigned long pc) } if (fex_p2) { - swcr_p2 |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + swcr_p2 |= fex_to_swcr_status(fex_p2, 2); current_thread_info()->ieee_state - |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p2, 2); /* Update hardware control register. */ fpcr_p2 = fpcr; @@ -1876,9 +1874,9 @@ long simd_mul_add_fp_emul_s(unsigned long pc) } if (fex_p3) { - swcr_p3 |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + swcr_p3 |= fex_to_swcr_status(fex_p3, 3); current_thread_info()->ieee_state - |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p3, 3); /* Update hardware control register. */ fpcr_p3 = fpcr; @@ -2131,9 +2129,9 @@ long simd_mul_add_fp_emul_d(unsigned long pc) swcr_p0 = swcr_p1 = swcr_p2 = swcr_p3 = swcr; /* manage fpcr_p0 */ if (fex_p0) { - swcr_p0 |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + swcr_p0 |= fex_to_swcr_status(fex_p0, 0); current_thread_info()->ieee_state - |= (fex_p0 << IEEE_STATUS0_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p0, 0); /* Update hardware control register. */ fpcr_p0 = fpcr; @@ -2142,9 +2140,9 @@ long simd_mul_add_fp_emul_d(unsigned long pc) } if (fex_p1) { - swcr_p1 |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + swcr_p1 |= fex_to_swcr_status(fex_p1, 1); current_thread_info()->ieee_state - |= (fex_p1 << IEEE_STATUS1_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p1, 1); /* Update hardware control register. */ fpcr_p1 = fpcr; @@ -2153,9 +2151,9 @@ long simd_mul_add_fp_emul_d(unsigned long pc) } if (fex_p2) { - swcr_p2 |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + swcr_p2 |= fex_to_swcr_status(fex_p2, 2); current_thread_info()->ieee_state - |= (fex_p2 << IEEE_STATUS2_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p2, 2); /* Update hardware control register. */ fpcr_p2 = fpcr; @@ -2164,9 +2162,9 @@ long simd_mul_add_fp_emul_d(unsigned long pc) } if (fex_p3) { - swcr_p3 |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + swcr_p3 |= fex_to_swcr_status(fex_p3, 3); current_thread_info()->ieee_state - |= (fex_p3 << IEEE_STATUS3_TO_EXCSUM_SHIFT); + |= fex_to_swcr_status(fex_p3, 3); /* Update hardware control register. */ fpcr_p3 = fpcr; -- Gitee From 8e55d9ba23000b29eb7e6c1cc87782219e894789 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 2 Feb 2024 17:12:13 +0800 Subject: [PATCH 125/524] sw64: use a new wrap_asid hmcall when asid wraps Use a new wrap_asid hmcall which loads the new asid and ptbr before tbivp(). Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hmcall.h | 10 ++++++++++ arch/sw_64/include/asm/mmu_context.h | 7 +++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/include/asm/hmcall.h b/arch/sw_64/include/asm/hmcall.h index 0bcd901f671d..41c656470ca4 100644 --- a/arch/sw_64/include/asm/hmcall.h +++ b/arch/sw_64/include/asm/hmcall.h @@ -20,6 +20,7 @@ #define HMC_rdhtctl 0x0D #define HMC_wrksp 0x0E #define HMC_mtinten 0x0F +#define HMC_wrap_asid 0x10 #define HMC_load_mm 0x11 #define HMC_tbisasid 0x14 #define HMC_tbivpn 0x19 @@ -231,6 +232,15 @@ __CALL_HMC_W1(wrtp, unsigned long); /* Invalidate all user TLB with current UPN and VPN */ #define tbiu() __tbi(4, /* no second argument */) +#ifndef CONFIG_SUBARCH_C3B +__CALL_HMC_W2(wrap_asid, unsigned long, unsigned long); +#else +static inline void wrap_asid(unsigned long asid, unsigned long ptbr) +{ + tbivp(); +} +#endif + #endif /* !__ASSEMBLY__ */ #endif /* __KERNEL__ */ diff --git a/arch/sw_64/include/asm/mmu_context.h b/arch/sw_64/include/asm/mmu_context.h index 420ad5f745be..1ff7df9539c5 100644 --- a/arch/sw_64/include/asm/mmu_context.h +++ b/arch/sw_64/include/asm/mmu_context.h @@ -41,10 +41,13 @@ static inline bool asid_valid(struct mm_struct *mm, unsigned int cpu) static inline void __get_new_mm_context(struct mm_struct *mm, long cpu) { + unsigned long ptbr; unsigned long asid = last_asid(cpu); - if (!(++asid & ASID_MASK)) - tbivp(); + if (!(++asid & ASID_MASK)) { + ptbr = virt_to_pfn(mm->pgd); + wrap_asid(asid, ptbr); + } mm->context.asid[cpu] = last_asid(cpu) = asid; } -- Gitee From 319a2036b227eb8706087018361b28a0d3534b48 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 19 Mar 2024 13:49:57 +0800 Subject: [PATCH 126/524] sw64: fix kconfig selections Fix kconfig selections to avoid linker error when using junzhang_defconfig. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + drivers/irqchip/Kconfig | 2 ++ 2 files changed, 3 insertions(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index bc85bebde3be..2639987c44bf 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -243,6 +243,7 @@ config PLATFORM_JUNZHANG depends on UNCORE_JUNZHANG select SPARSE_IRQ select SYS_HAS_EARLY_PRINTK + select I2C_SUNWAY if I2C help Sunway board chipset for C4 diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 9810922df2fd..35037f1e8d7a 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -17,6 +17,8 @@ config SW64_PINTC default y select GENERIC_IRQ_CHIP select IRQ_DOMAIN + select IRQ_DOMAIN_HIERARCHY + select SW64_LPC_INTC help This enables support for the pINTC chip found in SW systems. The PINTC receives all platform devices' interrupts and passes -- Gitee From 5fdad08bfafb9de7bcb73620fe6ce5bd8f09212a Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Thu, 7 Mar 2024 13:50:10 +0800 Subject: [PATCH 127/524] sw64: fix compilation warning Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/kprobes/decode-insn.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/kernel/kprobes/decode-insn.c b/arch/sw_64/kernel/kprobes/decode-insn.c index 91c31111f2b7..0bbd03407013 100644 --- a/arch/sw_64/kernel/kprobes/decode-insn.c +++ b/arch/sw_64/kernel/kprobes/decode-insn.c @@ -58,6 +58,7 @@ static bool __kprobes sw64_insn_is_steppable(u32 insn) #ifdef CONFIG_KPROBES +#ifdef CONFIG_SUBARCH_C3B // lldl rd_f static bool __kprobes is_probed_between_atomic(kprobe_opcode_t *addr) { @@ -81,6 +82,7 @@ static bool __kprobes is_probed_between_atomic(kprobe_opcode_t *addr) return true; } +#endif bool __kprobes sw64_insn_can_kprobe(kprobe_opcode_t *addr) { -- Gitee From 8a39f7610edaf17e93fb01fece5f01d0d5f90eb5 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 20 Dec 2023 19:45:03 +0800 Subject: [PATCH 128/524] sw64: acpi: add ACPI-style structures for SW PINTC, MSIC and LPC-INTC SW PINTC : Sunway Platform Interrupt Controller SW MSIC : Sunway MSI Controller SW LPC-INTC : Sunway LPC Interrupt Controller Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- include/acpi/actbl2.h | 87 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/include/acpi/actbl2.h b/include/acpi/actbl2.h index 986cf4a32f65..76ffed60a79a 100644 --- a/include/acpi/actbl2.h +++ b/include/acpi/actbl2.h @@ -910,6 +910,9 @@ enum acpi_madt_type { * used(start from 0x80). */ ACPI_MADT_TYPE_SW_CINTC = 0x80, + ACPI_MADT_TYPE_SW_PINTC = 0x81, + ACPI_MADT_TYPE_SW_MSIC = 0x82, + ACPI_MADT_TYPE_SW_LPC_INTC = 0x83 #endif }; @@ -1360,6 +1363,90 @@ enum acpi_madt_sw_cintc_version { ACPI_MADT_SW_CINTC_VERSION_V2 = 2, ACPI_MADT_SW_CINTC_VERSION_RESERVED = 3 /* 3 and greater are reserved */ }; + +/* 0x81: Sunway Platform Interrupt Controller (To be added to ACPI spec) */ + +struct acpi_madt_sw_sub_pintc { + u8 type; + u8 reserved[3]; + u32 hardware_id; + u64 address; + u32 size; + u32 gsi_base; + u32 gsi_count; + u32 cascade_vector; +}; + +struct acpi_madt_sw_pintc { + struct acpi_subtable_header header; + u8 version; + u8 reserved; + u32 flags; + u32 node; + u64 address; + u32 size; + u32 sub_num; + struct acpi_madt_sw_sub_pintc sub[]; +}; + +/* Values for version field above */ + +enum acpi_madt_sw_pintc_version { + ACPI_MADT_SW_PINTC_VERSION_NONE = 0, + ACPI_MADT_SW_PINTC_VERSION_V1 = 1, + ACPI_MADT_SW_PINTC_VERSION_V2 = 2, + ACPI_MADT_SW_PINTC_VERSION_RESERVED = 3 /* 3 and greater are reserved */ +}; + +/* 0x82: Sunway MSI Controller (To be added to ACPI spec) */ + +struct acpi_madt_sw_msic { + struct acpi_subtable_header header; + u8 version; + u8 reserved0; + u32 hardware_id; + u32 flags; + u64 address; + u32 size; + u32 cascade_vector; + u32 node; + u32 rc; + u32 num; + u32 reserved1[4]; +}; + +/* Values for version field above */ + +enum acpi_madt_sw_msic_version { + ACPI_MADT_SW_MSIC_VERSION_NONE = 0, + ACPI_MADT_SW_MSIC_VERSION_V1 = 1, + ACPI_MADT_SW_MSIC_VERSION_V2 = 2, + ACPI_MADT_SW_MSIC_VERSION_RESERVED = 3 /* 3 and greater are reserved */ +}; + +/* 0x83: Sunway LPC Interrupt Controller (To be added to ACPI spec) */ + +struct acpi_madt_sw_lpc_intc { + struct acpi_subtable_header header; + u8 version; + u8 reserved; + u32 hardware_id; + u32 flags; + u64 address; + u32 size; + u32 node; + u32 cascade_vector; + u32 gsi_base; + u32 gsi_count; +}; + +/* Values for version field above */ + +enum acpi_madt_sw_lpc_intc_version { + ACPI_MADT_SW_LPC_INTC_VERSION_NONE = 0, + ACPI_MADT_SW_LPC_INTC_VERSION_V1 = 1, + ACPI_MADT_SW_LPC_INTC_VERSION_RESERVED = 2 /* 2 and greater are reserved */ +}; #endif /* -- Gitee From b691a36645bd8d4bb74df95f7c77ff3afb3aa789 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 17 Jan 2024 19:16:33 +0800 Subject: [PATCH 129/524] sw64: acpi: support MADT entry print for SW PINTC, MSIC and LPC-INTC After this commit, we can print MADT entries of Sunway via the function acpi_table_print_madt_entry(). Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/acpi/tables.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 4848e21a84b8..b27bc9ee5115 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -239,6 +239,33 @@ void acpi_table_print_madt_entry(struct acpi_subtable_header *header) p->version, p->flags, p->hardware_id); } break; + + case ACPI_MADT_TYPE_SW_PINTC: + { + struct acpi_madt_sw_pintc *p = + (struct acpi_madt_sw_pintc *)header; + pr_info("SW PINTC (version[%u] flags[0x%x] address[0x%llx])\n", + p->version, p->flags, p->address); + } + break; + + case ACPI_MADT_TYPE_SW_MSIC: + { + struct acpi_madt_sw_msic *p = + (struct acpi_madt_sw_msic *)header; + pr_info("SW MSIC (version[%u] flags[0x%x] hardware_id[0x%x])\n", + p->version, p->flags, p->hardware_id); + } + break; + + case ACPI_MADT_TYPE_SW_LPC_INTC: + { + struct acpi_madt_sw_lpc_intc *p = + (struct acpi_madt_sw_lpc_intc *)header; + pr_info("SW LPC INTC (version[%u] flags[0x%x] hardware_id[0x%x])\n", + p->version, p->flags, p->hardware_id); + } + break; #endif default: -- Gitee From f6727548ae7f804c99fee97222bc91d7914e1eb0 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 21 Dec 2023 16:50:05 +0800 Subject: [PATCH 130/524] sw64: irqchip: support ACPI-based interrupt controller initialization Update driver to support initialization of PINTC and LPC-INTC in ACPI environment, enabling devices to use interrupts. This commit also fixes compatibility issue with old dts description for PINTC and LPC-INTC. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/chip3.dts | 2 +- arch/sw_64/boot/dts/chip_vt.dts | 2 +- arch/sw_64/include/asm/irq.h | 16 ++ arch/sw_64/kernel/acpi.c | 4 - drivers/acpi/bus.c | 5 + drivers/irqchip/irq-sunway-cpu.c | 213 ++++++++++++++++ drivers/irqchip/irq-sunway-lpc-intc.c | 163 ++++++++++-- drivers/irqchip/irq-sunway-msi-v2.c | 35 +++ drivers/irqchip/irq-sunway-msi.c | 35 +++ drivers/irqchip/irq-sunway-pintc.c | 347 +++++++++++++++++++++++--- include/acpi/actbl2.h | 6 +- include/linux/acpi.h | 3 + 12 files changed, 767 insertions(+), 64 deletions(-) diff --git a/arch/sw_64/boot/dts/chip3.dts b/arch/sw_64/boot/dts/chip3.dts index 0e9ad8f5d464..920519566c58 100644 --- a/arch/sw_64/boot/dts/chip3.dts +++ b/arch/sw_64/boot/dts/chip3.dts @@ -36,7 +36,7 @@ spiclk: spiclk { }; pintc: interrupt-controller { - compatible = "sw64,pintc"; + compatible = "sw64,pintc", "sw64,sw6_irq_controller"; interrupt-controller; sw64,node = <0>; sw64,irq-num = <8>; diff --git a/arch/sw_64/boot/dts/chip_vt.dts b/arch/sw_64/boot/dts/chip_vt.dts index 9889fcae7a5c..5f2ec68cb1d0 100644 --- a/arch/sw_64/boot/dts/chip_vt.dts +++ b/arch/sw_64/boot/dts/chip_vt.dts @@ -20,7 +20,7 @@ soc { ranges; pintc: interrupt-controller { - compatible = "sw64,pintc_vt"; + compatible = "sw64,pintc_vt", "sw64,sw6_irq_vt_controller"; interrupt-controller; sw64,node = <0>; sw64,irq-num = <16>; diff --git a/arch/sw_64/include/asm/irq.h b/arch/sw_64/include/asm/irq.h index b3ac4105c29e..e393162d928b 100644 --- a/arch/sw_64/include/asm/irq.h +++ b/arch/sw_64/include/asm/irq.h @@ -28,4 +28,20 @@ extern void (*perf_irq)(unsigned long vector, struct pt_regs *regs); extern void fixup_irqs(void); extern void sw64_timer_interrupt(void); +struct irq_domain; +struct fwnode_handle; + +struct acpi_madt_sw_pintc; +struct acpi_madt_sw_msic; +struct acpi_madt_sw_lpc_intc; + +extern int __init sw64_add_gsi_domain_map(u32 gsi_base, u32 gsi_count, + struct fwnode_handle *handle); +extern int __init pintc_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_pintc *pintc); +extern int __init msic_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_msic *msic); +extern int __init lpc_intc_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_lpc_intc *lpc_intc); + #endif /* _ASM_SW64_IRQ_H */ diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 20c6e851254b..56805db4431e 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -32,13 +32,9 @@ u64 acpi_saved_sp_s3; #define SW_CINTC_FLAG_ENABLED ACPI_MADT_ENABLED /* 0x1 */ #define SW_CINTC_FLAG_ONLINE_CAPABLE 0x2 /* hotplug capable */ -#define SW_CINTC_FLAG_HT_CAPABLE 0x4 /* hyper thread capable */ -#define SW_CINTC_FLAG_HT_ENABLED 0x8 /* hyper thread enabled */ #define is_core_enabled(flags) ((flags) & SW_CINTC_FLAG_ENABLED) #define is_core_online_capable(flags) ((flags) & SW_CINTC_FLAG_ONLINE_CAPABLE) -#define is_core_ht_capable(flags) ((flags) & SW_CINTC_FLAG_HT_CAPABLE) -#define is_core_ht_enabled(flags) ((flags) & SW_CINTC_FLAG_HT_ENABLED) #define MAX_LOCAL_APIC 256 diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index 645464f02363..2fcd0f87568b 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -1158,6 +1158,11 @@ static int __init acpi_bus_init_irq(void) case ACPI_IRQ_MODEL_LPIC: message = "LPIC"; break; +#ifdef CONFIG_SW64 + case ACPI_IRQ_MODEL_SWPIC: + message = "SWPIC"; + break; +#endif default: pr_info("Unknown interrupt routing model\n"); return -ENODEV; diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 09addf73e132..20cfa4f2b999 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -2,12 +2,43 @@ #include #include +#include +#include #include #include #include #include +/** + * The topology of interrupt controllers of SW64 is as follows: + * + * +-----------------------------------------------------------+ + * | | + * | +-------------+ | + * | | Core | | + * | +-------------+ | + * | | | + * | +----------------------+ | + * | | SW CINTC | | + * | +----------------------+ | + * | ______| |______ | + * | | | | + * | +-----------+ +--------------+ | + * | | SW MSIC | | SW PINTC | | + * | +-----------+ +--------------+ | + * | | | + * | +--------------+ | + * | | SW LPC INTC | | + * | +--------------+ | + * | | + * +-----------------------------------------------------------+ + */ + +#define PREFIX "CINTC: " + +struct fwnode_handle *cintc_handle; + static void handle_intx(unsigned int offset) { struct pci_controller *hose; @@ -208,3 +239,185 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, pr_crit("PC = %016lx PS = %04lx\n", regs->pc, regs->ps); } EXPORT_SYMBOL(do_entInt); + +#ifdef CONFIG_ACPI +#define SW_CINTC_FLAG_VIRTUAL 0x4 /* virtual CINTC */ + +#define is_core_virtual(flags) ((flags) & SW_CINTC_FLAG_VIRTUAL) + +struct gsi_domain_map { + u32 gsi_base; + u32 gsi_count; + struct fwnode_handle *handle; + struct gsi_domain_map *next; +}; + +static struct gsi_domain_map *gsi_domain_map_list; + +int __init sw64_add_gsi_domain_map(u32 gsi_base, u32 gsi_count, + struct fwnode_handle *handle) +{ + struct gsi_domain_map *map; + + if (WARN_ON(!handle)) + return -EINVAL; + + map = kzalloc(sizeof(struct gsi_domain_map), GFP_KERNEL); + if (!map) + return -ENOMEM; + + map->gsi_base = gsi_base; + map->gsi_count = gsi_count; + map->handle = handle; + + map->next = gsi_domain_map_list; + gsi_domain_map_list = map; + + return 0; +} + +/** + * The starting GSI num occupied by different domains are: + * + * SW CINTC on Node(x) : 0 + (512 * x) + * SW PINTC on Node(x) : 64 + (512 * x) + * SW LPC-INTC on Node(x) : 256 + (512 * x) + */ +static struct fwnode_handle *sw64_gsi_to_domain_id(u32 gsi) +{ + struct gsi_domain_map *map; + u32 base, limit; + + for (map = gsi_domain_map_list; map; map = map->next) { + base = map->gsi_base; + limit = map->gsi_base + map->gsi_count; + + if ((gsi >= base) && (gsi < limit)) + return map->handle; + } + + return NULL; +} + +static int __init pintc_parse_madt(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_sw_pintc *pintc; + + pintc = (struct acpi_madt_sw_pintc *)header; + + /* Not yet supported */ + if (pintc->node > 0) { + pr_warn(PREFIX "PINTC and LPC-INTC on node x(x > 0) are not supported\n"); + return 0; + } + + if ((pintc->version == ACPI_MADT_SW_PINTC_VERSION_NONE) || + (pintc->version >= ACPI_MADT_SW_PINTC_VERSION_RESERVED)) { + pr_err(PREFIX "invalid PINTC version\n"); + return -EINVAL; + } + + return pintc_acpi_init(NULL, pintc); +} + +#ifdef CONFIG_SW64_IRQ_MSI +static int __init msic_parse_madt(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_sw_msic *msic; + + msic = (struct acpi_madt_sw_msic *)header; + if ((msic->version == ACPI_MADT_SW_MSIC_VERSION_NONE) || + (msic->version >= ACPI_MADT_SW_MSIC_VERSION_RESERVED)) { + pr_err(PREFIX "invalid MSIC version\n"); + return -EINVAL; + } + + return msic_acpi_init(NULL, msic); +} +#endif + +static bool __init +acpi_check_sw_cintc_entry(struct acpi_subtable_header *header, + struct acpi_probe_entry *ape) +{ + struct acpi_madt_sw_cintc *cintc; + + cintc = (struct acpi_madt_sw_cintc *)header; + if (cintc->version != ape->driver_data) + return false; + + return true; +} + +static __init int cintc_acpi_init(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_sw_cintc *cintc; + bool virtual; + + /** + * There are more than one MADT entry of SW CINTC in + * multi-core system, but the initialization here only + * needs to be performed once per node. + */ + if (cintc_handle) + return 0; + + cintc = (struct acpi_madt_sw_cintc *)header; + virtual = is_core_virtual(cintc->flags); + pr_info(PREFIX "version [%u] (%s) found\n", cintc->version, + virtual ? "virtual" : "physical"); + + /** + * Currently, no irq_domain created for SW CINTC. The + * handle only used to avoid multiple initializations. + * Apart from this, there is no other meaning. + * + * Maybe we will create irq_domain for SW CINTC in the + * future to optimize the code. + */ + cintc_handle = irq_domain_alloc_named_fwnode("CINTC"); + if (!cintc_handle) { + pr_err(PREFIX "failed to alloc fwnode\n"); + return -ENOMEM; + } + + acpi_set_irq_model(ACPI_IRQ_MODEL_SWPIC, sw64_gsi_to_domain_id); + + /* Init SW PINTC */ + acpi_table_parse_madt(ACPI_MADT_TYPE_SW_PINTC, + pintc_parse_madt, 0); + +#ifdef CONFIG_SW64_IRQ_MSI + /* Init SW MSIC */ + acpi_table_parse_madt(ACPI_MADT_TYPE_SW_MSIC, + msic_parse_madt, 0); +#endif + + /** + * After initializing MSIC, it's time to enable MSI interrupts + * for boot core. For other SMP cores, if present, this + * initialization is performed during SMP startup. + */ + if (!virtual) { + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); + } + + return 0; +} + +IRQCHIP_ACPI_DECLARE(cintc_v1, ACPI_MADT_TYPE_SW_CINTC, + acpi_check_sw_cintc_entry, + ACPI_MADT_SW_CINTC_VERSION_V1, + cintc_acpi_init); + +IRQCHIP_ACPI_DECLARE(cintc_v2, ACPI_MADT_TYPE_SW_CINTC, + acpi_check_sw_cintc_entry, + ACPI_MADT_SW_CINTC_VERSION_V2, + cintc_acpi_init); +#endif diff --git a/drivers/irqchip/irq-sunway-lpc-intc.c b/drivers/irqchip/irq-sunway-lpc-intc.c index bc69d1647aed..80442dae812f 100644 --- a/drivers/irqchip/irq-sunway-lpc-intc.c +++ b/drivers/irqchip/irq-sunway-lpc-intc.c @@ -12,6 +12,8 @@ #define LPC_IRQ 0x4 #define LPC_IRQ_MASK 0x8 +#define SW_LPC_INTC_GSI_BASE 256 + static DEFINE_RAW_SPINLOCK(lpc_lock); static int parent_irq; @@ -110,18 +112,70 @@ static int sw64_lpc_domain_map(struct irq_domain *d, unsigned int virq, return 0; } +static int lpc_intc_translate(struct irq_domain *domain, + struct irq_fwspec *fwspec, + unsigned long *hwirq, + unsigned int *type) +{ + if (WARN_ON(fwspec->param_count < 1)) + return -EINVAL; + + /* Device tree */ + if (is_of_node(fwspec->fwnode)) { + *hwirq = fwspec->param[0]; + *type = IRQ_TYPE_NONE; + return 0; + } + + /* ACPI */ + if (is_fwnode_irqchip(fwspec->fwnode)) { + if (WARN_ON(fwspec->param[0] < SW_LPC_INTC_GSI_BASE)) + return -EINVAL; + *hwirq = fwspec->param[0] - SW_LPC_INTC_GSI_BASE; + *type = IRQ_TYPE_NONE; + return 0; + } + + return -EINVAL; +} + static const struct irq_domain_ops sw64_lpc_domain_ops = { .map = sw64_lpc_domain_map, - .xlate = irq_domain_xlate_onecell, + .translate = lpc_intc_translate, }; +static struct irq_domain *lpc_irq_domain; + +static int __init lpc_intc_init(struct fwnode_handle *handle, + unsigned int irqnr, int parent_irq, void __iomem *base_addr) +{ + /** + * The current kernel does not support "irq_domain_create_legacy", + * we have to call "__irq_domain_add" directly. + */ + lpc_irq_domain = __irq_domain_add(handle, irqnr, irqnr, + 0, &sw64_lpc_domain_ops, base_addr); + if (!lpc_irq_domain) { + pr_info(PREFIX "failed to create irq domain\n"); + return -ENOMEM; + } + + irq_domain_associate_many(lpc_irq_domain, 0, 0, irqnr); + + /* Set the IRQ chaining logic */ + irq_set_chained_handler_and_data(parent_irq, + lpc_irq_handler, lpc_irq_domain); + + return 0; +} + +#ifdef CONFIG_OF struct device_node *sw_lpc_intc_node; EXPORT_SYMBOL(sw_lpc_intc_node); static int __init lpc_intc_of_init(struct device_node *np, struct device_node *parent) { - struct irq_domain *lpc_domain; int ret; u32 nr_irqs, node, version; void __iomem *base; @@ -133,20 +187,23 @@ static int __init lpc_intc_of_init(struct device_node *np, ret = of_property_read_u32(np, "sw64,node", &node); if (ret) { - pr_err(PREFIX "\"sw64,node\" not found\n"); - return -EINVAL; + node = 0; + pr_warn(PREFIX "\"sw64,node\" fallback to %u\n", + node); } ret = of_property_read_u32(np, "sw64,irq-num", &nr_irqs); if (ret) { - pr_err(PREFIX "\"sw64,irq-num\" not found\n"); - return -EINVAL; + nr_irqs = 16; + pr_warn(PREFIX "\"sw64,irq-num\" fallback to %u\n", + nr_irqs); } ret = of_property_read_u32(np, "sw64,ver", &version); if (ret) { - pr_err(PREFIX "\"sw64,ver\" not found\n"); - return -EINVAL; + version = 1; + pr_warn(PREFIX "\"sw64,ver\" fallback to %u\n", + version); } base = of_iomap(np, 0); @@ -162,21 +219,13 @@ static int __init lpc_intc_of_init(struct device_node *np, goto out_unmap; } - lpc_domain = irq_domain_add_legacy(np, nr_irqs, - 0, 0, &sw64_lpc_domain_ops, base); - if (!lpc_domain) { - pr_err(PREFIX "failed to create irq domain\n"); - ret = -ENOMEM; + ret = lpc_intc_init(of_node_to_fwnode(np), nr_irqs, parent_irq, base); + if (ret) goto out_unmap; - } pr_info(PREFIX "version [%u] on node [%u] initialized\n", version, node); - /* Set the IRQ chaining logic */ - irq_set_chained_handler_and_data(parent_irq, - lpc_irq_handler, lpc_domain); - return 0; out_unmap: @@ -184,3 +233,81 @@ static int __init lpc_intc_of_init(struct device_node *np, return ret; } IRQCHIP_DECLARE(sw_lpc_intc, "sw64,lpc_intc", lpc_intc_of_init); +#endif + +#ifdef CONFIG_ACPI +#define SW_LPC_INTC_FLAG_ENABLED ACPI_MADT_ENABLED /* 0x1 */ + +#define is_lpc_intc_enabled(flags) ((flags) & SW_LPC_INTC_FLAG_ENABLED) + +int __init lpc_intc_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_lpc_intc *lpc_intc) +{ + struct fwnode_handle *handle; + struct irq_fwspec fwspec; + void __iomem *base_addr; + int ret; + bool enabled; + + enabled = is_lpc_intc_enabled(lpc_intc->flags); + pr_info(PREFIX "version [%u] on node [%u] %s\n", + lpc_intc->version, lpc_intc->node, + enabled ? "found" : "disabled"); + if (!enabled) + return 0; + + if (lpc_intc->gsi_base != SW_LPC_INTC_GSI_BASE) { + pr_err(PREFIX "invalid GSI\n"); + return -EINVAL; + } + + handle = irq_domain_alloc_named_id_fwnode("LPC-INTC", lpc_intc->node); + if (!handle) { + pr_err(PREFIX "failed to alloc fwnode\n"); + return -ENOMEM; + } + + fwspec.fwnode = parent->fwnode; + fwspec.param[0] = lpc_intc->cascade_vector; + fwspec.param_count = 1; + + parent_irq = irq_create_fwspec_mapping(&fwspec); + if (parent_irq <= 0) { + pr_err(PREFIX "failed to map parent irq\n"); + ret = -EINVAL; + goto out_acpi_free_fwnode; + } + + base_addr = ioremap(lpc_intc->address, lpc_intc->size); + if (!base_addr) { + pr_err(PREFIX "failed to map base address\n"); + ret = -ENXIO; + goto out_acpi_free_fwnode; + } + + ret = lpc_intc_init(handle, lpc_intc->gsi_count, + parent_irq, base_addr); + if (ret) + goto out_acpi_unmap; + + ret = sw64_add_gsi_domain_map(lpc_intc->gsi_base, + lpc_intc->gsi_count, handle); + if (ret) { + pr_info(PREFIX "failed to add GSI map\n"); + goto out_acpi_free_lpc_domain; + } + + pr_info(PREFIX "version [%u] on node [%u] initialized\n", + lpc_intc->version, lpc_intc->node); + + return 0; + +out_acpi_free_lpc_domain: + irq_domain_remove(lpc_irq_domain); +out_acpi_unmap: + iounmap(base_addr); +out_acpi_free_fwnode: + irq_domain_free_fwnode(handle); + return ret; +} +#endif diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c index 698c5c86408a..6f5c921fc520 100644 --- a/drivers/irqchip/irq-sunway-msi-v2.c +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -3,10 +3,13 @@ #include #include #include +#include #include #include +#define PREFIX "MSIC: " + static struct irq_domain *msi_default_domain; static DEFINE_RAW_SPINLOCK(vector_lock); DEFINE_PER_CPU(vector_irq_t, vector_irq) = { @@ -508,3 +511,35 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned vector = int_pci_msi[i % 3]; } } + +MODULE_LICENSE("GPL v2"); + +#ifdef CONFIG_ACPI +#define SW_MSIC_FLAG_ENABLED ACPI_MADT_ENABLED /* 0x1 */ +#define SW_MSIC_FLAG_VIRTUAL 0x2 /* virtual MSIC */ + +#define is_msic_enabled(flags) ((flags) & SW_MSIC_FLAG_ENABLED) +#define is_msic_virtual(flags) ((flags) & SW_MSIC_FLAG_VIRTUAL) + +int __init msic_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_msic *msic) +{ + bool enabled, virtual; + + enabled = is_msic_enabled(msic->flags); + virtual = is_msic_virtual(msic->flags); + + pr_info(PREFIX "version [%u] on node [%u] Root Complex [%u] (%s) %s\n", + msic->version, msic->node, msic->rc, + virtual ? "virtual" : "physical", + enabled ? "found" : "disabled"); + + if (!enabled) + return 0; + + if (!msi_default_domain) + arch_init_msi_domain(parent); + + return 0; +} +#endif diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index 060aa96711b7..c231e504464b 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -2,10 +2,13 @@ #include #include #include +#include #include #include +#define PREFIX "MSIC: " + static struct irq_domain *msi_default_domain; static DEFINE_RAW_SPINLOCK(vector_lock); DEFINE_PER_CPU(vector_irq_t, vector_irq) = { @@ -470,3 +473,35 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned vector = int_pci_msi[i % 3]; } } + +MODULE_LICENSE("GPL v2"); + +#ifdef CONFIG_ACPI +#define SW_MSIC_FLAG_ENABLED ACPI_MADT_ENABLED /* 0x1 */ +#define SW_MSIC_FLAG_VIRTUAL 0x2 /* virtual MSIC */ + +#define is_msic_enabled(flags) ((flags) & SW_MSIC_FLAG_ENABLED) +#define is_msic_virtual(flags) ((flags) & SW_MSIC_FLAG_VIRTUAL) + +int __init msic_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_msic *msic) +{ + bool enabled, virtual; + + enabled = is_msic_enabled(msic->flags); + virtual = is_msic_virtual(msic->flags); + + pr_info(PREFIX "version [%u] on node [%u] Root Complex [%u] (%s) %s\n", + msic->version, msic->node, msic->rc, + virtual ? "virtual" : "physical", + enabled ? "found" : "disabled"); + + if (!enabled) + return 0; + + if (!msi_default_domain) + arch_init_msi_domain(parent); + + return 0; +} +#endif diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index 41c52f9fb09b..fa3af8c1c585 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -7,12 +7,54 @@ #include #include +/** + * Currently, Peripheral interrupt control logic of Sunway is mainly + * distributed on the device side, which are hardware entities + * corresponding to SW sub PINTC structures. + * + * At the same time, there are some interrupt configuration registers + * concentrated in INTPU, which is hardware entity corresponding to + * SW PINTC, excluding SW sub PINTC structures. + * + * The topology of SW PINTC(physical) is as follows: + * + * +----------------------------------------------------------------+ + * | | + * | +--------------------------------+ | + * | | SW CINTC | | + * | +--------------------------------+ | + * | | | | | | + * | _________| __| |__ |______ | + * | | | | | | + * | +-------|---------------|--------------|------------|------+ | + * | | | | | | | | + * | | +----------+ +--------+ +--------+ +--------+ | | + * | | | MCU | | MT | | ADR | | ...... | | | + * | | +----------+ +--------+ +--------+ +--------+ | | + * | | | SW PINTC | | + * | +-------|--------------------------------------------------+ | + * | | | + * | +--------------+ | + * | | SW LPC INTC | | + * | +--------------+ | + * | | + * +----------------------------------------------------------------+ + */ + #define PREFIX "PINTC: " #define OFFSET_MCU_DVC_INT_EN 0x3080UL #define OFFSET_DEV_INT_CONFIG 0x480UL +#define SW_PINTC_MCU_GSI_BASE 64 + +#define INTPU_BASE_V1 0x802a00000000 +#define INTPU_SIZE_V1 0x1680 + +#define MCU_BASE_V1 0x803000000000 +#define MCU_SIZE_V1 0x8f00 + struct pintc_chip_data { bool vt; /* virtual pintc */ u32 node; /* node ID */ @@ -20,6 +62,7 @@ struct pintc_chip_data { void __iomem *pintc_base; /* INTPU base address */ void __iomem *mcu_base; /* MCU/SPBU base address */ struct irq_chip *mcu_chip; + u32 mcu_irq_num; }; static DEFINE_RAW_SPINLOCK(pintc_lock); @@ -177,6 +220,33 @@ static struct irq_chip pintc_mcu_vt_chip = { .name = "VMCU-INT", }; +static int pintc_mcu_translate(struct irq_domain *domain, + struct irq_fwspec *fwspec, + unsigned long *hwirq, + unsigned int *type) +{ + if (WARN_ON(fwspec->param_count < 1)) + return -EINVAL; + + /* Device tree */ + if (is_of_node(fwspec->fwnode)) { + *hwirq = fwspec->param[0]; + *type = IRQ_TYPE_NONE; + return 0; + } + + /* ACPI */ + if (is_fwnode_irqchip(fwspec->fwnode)) { + if (WARN_ON(fwspec->param[0] < SW_PINTC_MCU_GSI_BASE)) + return -EINVAL; + *hwirq = fwspec->param[0] - SW_PINTC_MCU_GSI_BASE; + *type = IRQ_TYPE_NONE; + return 0; + } + + return -EINVAL; +} + static void pintc_mcu_free_irqs(struct irq_domain *irq_domain, unsigned int virq, unsigned int nr_irqs) { @@ -207,8 +277,13 @@ static int pintc_mcu_alloc_irqs(struct irq_domain *domain, void *arg) { struct irq_fwspec *fwspec = arg; - irq_hw_number_t hwirq = fwspec->param[0]; - int i; + irq_hw_number_t hwirq; + unsigned int type; + int i, ret; + + ret = pintc_mcu_translate(domain, fwspec, &hwirq, &type); + if (ret) + return ret; for (i = 0; i < nr_irqs; i++) pintc_mcu_map_irq(domain, virq + i, hwirq + i); @@ -218,7 +293,7 @@ static int pintc_mcu_alloc_irqs(struct irq_domain *domain, static const struct irq_domain_ops pintc_mcu_domain_ops = { .map = pintc_mcu_map_irq, - .xlate = irq_domain_xlate_onecell, + .translate = pintc_mcu_translate, .alloc = pintc_mcu_alloc_irqs, .free = pintc_mcu_free_irqs, }; @@ -226,6 +301,44 @@ static const struct irq_domain_ops pintc_mcu_domain_ops = { struct irq_domain *mcu_irq_domain; EXPORT_SYMBOL(mcu_irq_domain); +static int __init pintc_init_mcu(struct pintc_chip_data *chip_data, + struct fwnode_handle *handle) +{ + unsigned int mcu_irq_num = chip_data->mcu_irq_num; + + if (chip_data->vt) { + chip_data->mcu_chip = &pintc_mcu_vt_chip; + + /** + * The current kernel does not support the API + * "irq_domain_create_legacy", we have to call + * "__irq_domain_add" directly. + */ + mcu_irq_domain = __irq_domain_add(handle, mcu_irq_num, + mcu_irq_num, 0, &pintc_mcu_domain_ops, + chip_data); + if (mcu_irq_domain) + irq_domain_associate_many(mcu_irq_domain, + 0, 0, mcu_irq_num); + } else { + chip_data->mcu_chip = &pintc_mcu_chip; + mcu_irq_domain = irq_domain_create_linear(handle, mcu_irq_num, + &pintc_mcu_domain_ops, chip_data); + /* Mask all interrupts for now */ + writeq(0x0, chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); + } + + if (!mcu_irq_domain) { + pr_err(PREFIX "failed to create MCU irq domain\n"); + return -ENOMEM; + } + + pr_info(PREFIX "MCU sub controller [%u:%u] initialized\n", + chip_data->version, chip_data->node); + + return 0; +} + #ifdef CONFIG_OF static int __init pintc_of_init_common(struct device_node *pintc, @@ -247,39 +360,43 @@ pintc_of_init_common(struct device_node *pintc, ret = of_property_read_u32(pintc, "sw64,node", &node); if (ret) { - pr_err(PREFIX "\"sw64,node\" not found\n"); - return -EINVAL; + node = 0; + pr_warn(PREFIX "\"sw64,node\" fallback to %u\n", + node); } ret = of_property_read_u32(pintc, "sw64,irq-num", &nr_irqs); if (ret) { - pr_err(PREFIX "\"sw64,irq-num\" not found\n"); - return -EINVAL; + nr_irqs = vt ? 16 : 8; + pr_warn(PREFIX "\"sw64,irq-num\" fallback to %u\n", + nr_irqs); } ret = of_property_read_u32(pintc, "sw64,ver", &version); if (ret) { - pr_err(PREFIX "\"sw64,ver\" not found\n"); - return -EINVAL; + version = 1; + pr_warn(PREFIX "\"sw64,ver\" fallback to %u\n", + version); } pintc_base = of_iomap(pintc, 0); if (!vt && !pintc_base) { - pr_err(PREFIX "failed to map pintc base address\n"); - return -ENXIO; + pintc_base = ioremap(INTPU_BASE_V1, INTPU_SIZE_V1); + pr_warn(PREFIX "pintc base address fallback to 0x%lx\n", + INTPU_BASE_V1); } mcu_base = of_iomap(pintc, 1); if (!vt && !mcu_base) { - pr_err(PREFIX "failed to map mcu base address\n"); - ret = -ENXIO; - goto out_unmap0; + mcu_base = ioremap(MCU_BASE_V1, MCU_SIZE_V1); + pr_warn(PREFIX "mcu base address fallback to 0x%lx\n", + MCU_BASE_V1); } chip_data = kzalloc_node(sizeof(*chip_data), GFP_KERNEL, node); if (!chip_data) { ret = -ENOMEM; - goto out_unmap1; + goto out_unmap; } chip_data->vt = vt; @@ -287,37 +404,18 @@ pintc_of_init_common(struct device_node *pintc, chip_data->version = version; chip_data->pintc_base = pintc_base; chip_data->mcu_base = mcu_base; + chip_data->mcu_irq_num = nr_irqs; - if (vt) { - chip_data->mcu_chip = &pintc_mcu_vt_chip; - mcu_irq_domain = irq_domain_add_legacy(pintc, nr_irqs, 0, 0, - &pintc_mcu_domain_ops, chip_data); - } else { - chip_data->mcu_chip = &pintc_mcu_chip; - mcu_irq_domain = irq_domain_add_linear(pintc, nr_irqs, - &pintc_mcu_domain_ops, chip_data); - /* mask all interrupts for now */ - writeq(0x0, mcu_base + OFFSET_MCU_DVC_INT_EN); - } - - if (!mcu_irq_domain) { - pr_err(PREFIX "failed to create irq domain\n"); - ret = -ENOMEM; + ret = pintc_init_mcu(chip_data, of_node_to_fwnode(pintc)); + if (ret) goto out_free_mem; - } - - pr_info(PREFIX "version [%u] on node [%u] initialized\n", - version, node); - - irq_set_default_host(mcu_irq_domain); return 0; out_free_mem: kfree(chip_data); -out_unmap1: +out_unmap: iounmap(mcu_base); -out_unmap0: iounmap(pintc_base); return ret; } @@ -329,6 +427,7 @@ pintc_of_init(struct device_node *pintc, struct device_node *parent) } IRQCHIP_DECLARE(sw64_pintc, "sw64,pintc", pintc_of_init); +IRQCHIP_DECLARE(sw64_pintc_legacy, "sw64,sw6_irq_controller", pintc_of_init); static int __init pintc_vt_of_init(struct device_node *pintc, struct device_node *parent) @@ -337,4 +436,176 @@ pintc_vt_of_init(struct device_node *pintc, struct device_node *parent) } IRQCHIP_DECLARE(sw64_pintc_vt, "sw64,pintc_vt", pintc_vt_of_init); +IRQCHIP_DECLARE(sw64_pintc_vt_legacy, "sw64,sw6_irq_vt_controller", pintc_vt_of_init); +#endif + +#ifdef CONFIG_ACPI +#define SW_PINTC_FLAG_ENABLED ACPI_MADT_ENABLED /* 0x1 */ +#define SW_PINTC_FLAG_VIRTUAL 0x2 /* virtual PINTC */ + +#define is_pintc_enabled(flags) ((flags) & SW_PINTC_FLAG_ENABLED) +#define is_pintc_virtual(flags) ((flags) & SW_PINTC_FLAG_VIRTUAL) + +/* Physical sub interrupt controllers */ +enum sw_pintc_sub_type { + SW_PINTC_SUB_TYPE_MCU = 0x00, + SW_PINTC_SUB_TYPE_MT = 0x01, + SW_PINTC_SUB_TYPE_FAULT = 0x02, + SW_PINTC_SUB_TYPE_NMI = 0x03, + SW_PINTC_SUB_TYPE_S3 = 0x04, + SW_PINTC_SUB_TYPE_ADR = 0x05, + SW_PINTC_SUB_TYPE_COUNT +}; + +static int __init lpc_intc_parse_madt(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_sw_lpc_intc *lpc_intc; + + lpc_intc = (struct acpi_madt_sw_lpc_intc *)header; + + /* Not yet supported */ + if (lpc_intc->node > 0) + return 0; + + if ((lpc_intc->version == ACPI_MADT_SW_LPC_INTC_VERSION_NONE) || + (lpc_intc->version >= ACPI_MADT_SW_LPC_INTC_VERSION_RESERVED)) { + pr_err(PREFIX "invalid LPC-INTC version\n"); + return -EINVAL; + } + + return lpc_intc_acpi_init(mcu_irq_domain, lpc_intc); +} + +static bool __init +pintc_sub_type_check(const struct acpi_madt_sw_pintc *pintc) +{ + int i, count = 0; + + for (i = 0; i < pintc->sub_num; ++i) { + if (pintc->sub[i].type >= SW_PINTC_SUB_TYPE_COUNT) + count++; + } + + return count; +} + +static int __init pintc_acpi_init_mcu(struct pintc_chip_data *chip_data, + struct acpi_madt_sw_sub_pintc *mcu) +{ + struct fwnode_handle *handle; + int ret; + + if (!mcu->status) { + pr_info(PREFIX "MCU sub controller [%u:%u] disabled\n", + chip_data->version, chip_data->node); + return 0; + } + + if (mcu->gsi_base != SW_PINTC_MCU_GSI_BASE) { + pr_err(PREFIX "invalid MCU GSI\n"); + return -EINVAL; + } + + handle = irq_domain_alloc_named_id_fwnode("PINTC-MCU", chip_data->node); + if (!handle) { + pr_err(PREFIX "failed to alloc fwnode\n"); + return -ENOMEM; + } + + chip_data->mcu_irq_num = mcu->gsi_count; + + chip_data->mcu_base = ioremap(mcu->address, mcu->size); + if (!chip_data->mcu_base) { + pr_err(PREFIX "failed to map mcu base address\n"); + ret = -ENXIO; + goto out_acpi_free_fwnode; + } + + ret = pintc_init_mcu(chip_data, handle); + if (ret) + goto out_acpi_unmap_mcu; + + ret = sw64_add_gsi_domain_map(mcu->gsi_base, mcu->gsi_count, handle); + if (ret) { + pr_info(PREFIX "failed to add GSI map\n"); + goto out_acpi_free_mcu_domain; + } + + return 0; + +out_acpi_free_mcu_domain: + irq_domain_remove(mcu_irq_domain); +out_acpi_unmap_mcu: + iounmap(chip_data->mcu_base); +out_acpi_free_fwnode: + irq_domain_free_fwnode(handle); + return ret; +} + +int __init pintc_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_pintc *pintc) +{ + struct pintc_chip_data *chip_data; + int ret, i; + bool enabled, virtual; + + enabled = is_pintc_enabled(pintc->flags); + virtual = is_pintc_virtual(pintc->flags); + + pr_info(PREFIX "version [%u] on node [%u] (%s) %s\n", + pintc->version, pintc->node, + virtual ? "virtual" : "physical", + enabled ? "found" : "disabled"); + + if (!enabled) + return 0; + + if (pintc_sub_type_check(pintc)) { + pr_err(PREFIX "invalid sub type\n"); + return -EINVAL; + } + + chip_data = kzalloc_node(sizeof(*chip_data), GFP_KERNEL, + pintc->node); + if (!chip_data) + return -ENOMEM; + + /** + * The topology of interrupt controllers in Qemu is + * different from physical environment. We need to + * distinguish between them. + */ + chip_data->vt = virtual; + + chip_data->node = pintc->node; + chip_data->version = pintc->version; + + chip_data->pintc_base = ioremap(pintc->address, pintc->size); + if (!chip_data->pintc_base) { + pr_err(PREFIX "failed to map pintc base address\n"); + ret = -ENXIO; + goto out_acpi_free_chip_data; + } + + for (i = 0; i < pintc->sub_num; ++i) { + switch (pintc->sub[i].type) { + case SW_PINTC_SUB_TYPE_MCU: + pintc_acpi_init_mcu(chip_data, &pintc->sub[i]); + break; + default: + break; + } + } + + /* Init SW LPC INTC */ + acpi_table_parse_madt(ACPI_MADT_TYPE_SW_LPC_INTC, + lpc_intc_parse_madt, 0); + + return 0; + +out_acpi_free_chip_data: + kfree(chip_data); + return ret; +} #endif diff --git a/include/acpi/actbl2.h b/include/acpi/actbl2.h index 76ffed60a79a..ba0aef9d0646 100644 --- a/include/acpi/actbl2.h +++ b/include/acpi/actbl2.h @@ -1368,7 +1368,8 @@ enum acpi_madt_sw_cintc_version { struct acpi_madt_sw_sub_pintc { u8 type; - u8 reserved[3]; + u8 status; + u16 reserved; u32 hardware_id; u64 address; u32 size; @@ -1412,7 +1413,8 @@ struct acpi_madt_sw_msic { u32 node; u32 rc; u32 num; - u32 reserved1[4]; + u64 message_address; + u32 reserved1[2]; }; /* Values for version field above */ diff --git a/include/linux/acpi.h b/include/linux/acpi.h index fd820045a21b..f8ce8d6ee5cc 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -95,6 +95,9 @@ enum acpi_irq_model_id { ACPI_IRQ_MODEL_PLATFORM, ACPI_IRQ_MODEL_GIC, ACPI_IRQ_MODEL_LPIC, +#ifdef CONFIG_SW64 + ACPI_IRQ_MODEL_SWPIC, +#endif ACPI_IRQ_MODEL_COUNT }; -- Gitee From 05ddc61a74331014c0c5e321ed809ba3dc48dd26 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 20 Dec 2023 19:12:17 +0800 Subject: [PATCH 131/524] sw64: acpi: enable generic GSI Use common implementation of GSI provided by kernel instead of the architecture specific implementation. In addition, this commit also remove some useless code. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/kernel/acpi.c | 67 +--------------------------------------- 2 files changed, 2 insertions(+), 66 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 2639987c44bf..02e4c672c8b8 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -3,6 +3,7 @@ config SW64 bool default y select ACPI + select ACPI_GENERIC_GSI if ACPI select ACPI_MCFG if (ACPI && PCI) select ACPI_REDUCED_HARDWARE_ONLY select ARCH_ATOMIC diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 56805db4431e..f2b2a4d5ce67 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -39,11 +39,7 @@ u64 acpi_saved_sp_s3; #define MAX_LOCAL_APIC 256 #define PREFIX "ACPI: " -/* - * The default interrupt routing model is PIC (8259). This gets - * overridden if IOAPICs are enumerated (below). - */ -enum acpi_irq_model_id acpi_irq_model = ACPI_IRQ_MODEL_IOSAPIC; + void __iomem *__init __acpi_map_table(unsigned long phys, unsigned long size) { if (!phys || !size) @@ -58,17 +54,6 @@ void __init __acpi_unmap_table(void __iomem *map, unsigned long size) early_iounmap(map, size); } -/* - * Following __acpi_xx functions should be implemented for sepecific cpu. - */ -int acpi_gsi_to_irq(u32 gsi, unsigned int *irqp) -{ - if (irqp != NULL) - *irqp = acpi_register_gsi(NULL, gsi, -1, -1); - - return 0; -} -EXPORT_SYMBOL_GPL(acpi_gsi_to_irq); int acpi_isa_irq_to_gsi(unsigned int isa_irq, u32 *gsi) { @@ -80,56 +65,6 @@ int acpi_isa_irq_to_gsi(unsigned int isa_irq, u32 *gsi) int (*acpi_suspend_lowlevel)(void); -/* - * success: return IRQ number (>=0) - * failure: return < 0 - */ -static struct irq_domain *irq_default_domain; -int acpi_register_gsi(struct device *dev, u32 gsi, int trigger, int polarity) -{ - u32 irq; - - irq = irq_find_mapping(irq_default_domain, gsi); - - return irq; -} -EXPORT_SYMBOL_GPL(acpi_register_gsi); - -void acpi_unregister_gsi(u32 gsi) -{ - -} -EXPORT_SYMBOL_GPL(acpi_unregister_gsi); - -/* - * ACPI based hotplug support for CPU - */ -#ifdef CONFIG_ACPI_HOTPLUG_CPU -/* wrapper to silence section mismatch warning */ -int __ref acpi_map_lsapic(acpi_handle handle, int physid, int *pcpu) -{ - return 0; -} -EXPORT_SYMBOL(acpi_map_lsapic); - -int acpi_unmap_lsapic(int cpu) -{ - return 0; -} -EXPORT_SYMBOL(acpi_unmap_lsapic); -#endif /* CONFIG_ACPI_HOTPLUG_CPU */ - -u8 acpi_checksum(u8 *table, u32 length) -{ - u8 ret = 0; - - while (length--) { - ret += *table; - table++; - } - return -ret; -} - static int __init parse_acpi(char *arg) { if (!arg) -- Gitee From be5f607609b4590cec2a30e5c39b5b9ff8e73d13 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 21 Feb 2024 09:49:32 +0800 Subject: [PATCH 132/524] sw64: irq: remove unnecessary initialization It is unnecessary to set irqchip and handle for all irqs during early initialization. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/Makefile | 2 +- arch/sw_64/kernel/irq.c | 41 ++++++++++++++++++ arch/sw_64/kernel/irq_sw64.c | 84 ------------------------------------ 3 files changed, 42 insertions(+), 85 deletions(-) delete mode 100644 arch/sw_64/kernel/irq_sw64.c diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index abf27ad19a94..5ac06882ebc0 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -14,7 +14,7 @@ CFLAGS_REMOVE_printk.o = -pg endif obj-y := entry.o fpu.o traps.o process.o sys_sw64.o irq.o \ - irq_sw64.o signal.o setup.o ptrace.o time.o \ + signal.o setup.o ptrace.o time.o \ systbls.o dup_print.o chip_setup.o \ insn.o early_init.o topology.o cacheinfo.o \ vdso.o vdso/ hmcall.o stacktrace.o idle.o reset.o \ diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c index 126fe2f70495..58ee5131e2a9 100644 --- a/arch/sw_64/kernel/irq.c +++ b/arch/sw_64/kernel/irq.c @@ -14,8 +14,11 @@ #include #include #include +#include #include +#include + volatile unsigned long irq_err_count; DEFINE_PER_CPU(unsigned long, irq_pmi_count); DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); @@ -106,3 +109,41 @@ void fixup_irqs(void) irq_migrate_all_off_this_cpu(); } #endif + +void __init init_IRQ(void) +{ + /* + * Just in case the platform init_irq() causes interrupts/mchecks + * (as is the case with RAWHIDE, at least). + */ + if (is_in_host()) { + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); + sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); + } + + wrent(entInt, 0); + + sw64_init_irq(); + irqchip_init(); +} + +void __weak arch_init_msi_domain(struct irq_domain *parent) {} + +int __init arch_early_irq_init(void) +{ + arch_init_msi_domain(NULL); + + return 0; +} + +int __init arch_probe_nr_irqs(void) +{ + return NR_IRQS_LEGACY; +} + +struct irq_chip sw64_irq_chip = { + .name = "SW64_DUMMY" +}; +EXPORT_SYMBOL(sw64_irq_chip); diff --git a/arch/sw_64/kernel/irq_sw64.c b/arch/sw_64/kernel/irq_sw64.c deleted file mode 100644 index 03aef463ec05..000000000000 --- a/arch/sw_64/kernel/irq_sw64.c +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * SW64 specific irq code. - */ - -#include -#include - -#include -#include - -void __init -init_IRQ(void) -{ - /* - * Just in case the platform init_irq() causes interrupts/mchecks - * (as is the case with RAWHIDE, at least). - */ - if (is_in_host()) { - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); - } - - wrent(entInt, 0); - - sw64_init_irq(); - irqchip_init(); -} - -DEFINE_SPINLOCK(irq_lock); - -static void -__enable_irq(struct irq_data *d) -{ -} - -static void -__disable_irq(struct irq_data *d) -{ -} - -static unsigned int -__startup_irq(struct irq_data *d) -{ - __enable_irq(d); - return 0; -} - -static void -__mask_and_ack_irq(struct irq_data *d) -{ - spin_lock(&irq_lock); - __disable_irq(d); - spin_unlock(&irq_lock); -} - -struct irq_chip sw64_irq_chip = { - .name = "SW64_NODE", - .irq_startup = __startup_irq, - .irq_unmask = __enable_irq, - .irq_mask = __disable_irq, - .irq_mask_ack = __mask_and_ack_irq, -}; - -void __weak arch_init_msi_domain(struct irq_domain *parent) {} - -int __init arch_early_irq_init(void) -{ - int i; - - for (i = 0; i < NR_IRQS; ++i) { - irq_set_chip_and_handler(i, &sw64_irq_chip, handle_level_irq); - irq_set_status_flags(i, IRQ_LEVEL); - } - arch_init_msi_domain(NULL); - return 0; -} - -int __init arch_probe_nr_irqs(void) -{ - return NR_IRQS_LEGACY; -} -- Gitee From 03420978efe5f8cbdd1980ca8367b9861779a199 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 11 Jan 2024 13:43:01 +0800 Subject: [PATCH 133/524] sw64: pci: remove duplicate pcie operations Use pcie operations implemented in pci-sunway.c and remove the duplicate one. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 10 ++ arch/sw_64/pci/pci-legacy.c | 192 +++------------------------- drivers/pci/controller/pci-sunway.c | 17 ++- 3 files changed, 40 insertions(+), 179 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 689a65e63517..07c447479efa 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -99,6 +99,16 @@ extern void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge); extern void sw64_pci_root_bridge_scan_finish_up(struct pci_host_bridge *bridge); extern int sw64_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); +extern void __iomem *sw64_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where); +extern int sw64_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val); +extern int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val); + +extern void pci_mark_rc_linkup(unsigned long node, unsigned long index); +extern int pci_get_rc_linkup(unsigned long node, unsigned long index); + #ifdef CONFIG_PCI_DOMAINS static inline int pci_proc_domain(struct pci_bus *bus) { diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index 2a44463db0a4..bfaa08bf72c0 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -5,11 +5,10 @@ #include #include +#include #include #include -unsigned long rc_linkup; - /* * The PCI controller list. */ @@ -196,175 +195,10 @@ static void __init pcibios_reserve_legacy_regions(struct pci_bus *bus) return; } -/* PCIe RC operations */ -int sw6_pcie_read_rc_cfg(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 *val) -{ - u32 data; - struct pci_controller *hose = pci_bus_to_pci_controller(bus); - void __iomem *cfg_iobase = hose->rc_config_space_base; - - if (IS_ENABLED(CONFIG_PCI_DEBUG)) - pr_debug("rc read addr:%px bus %d, devfn %#x, where %#x size=%d\t", - cfg_iobase + ((where & ~3) << 5), bus->number, devfn, where, size); - - if ((uintptr_t)where & (size - 1)) { - *val = 0; - return PCIBIOS_BAD_REGISTER_NUMBER; - } - - if (unlikely(devfn > 0)) { - *val = ~0; - return PCIBIOS_DEVICE_NOT_FOUND; - } - - data = readl(cfg_iobase + ((where & ~3) << 5)); - - switch (size) { - case 1: - *val = (data >> (8 * (where & 0x3))) & 0xff; - break; - case 2: - *val = (data >> (8 * (where & 0x2))) & 0xffff; - break; - default: - *val = data; - break; - } - - if (IS_ENABLED(CONFIG_PCI_DEBUG)) - pr_debug("*val %#x\n ", *val); - - return PCIBIOS_SUCCESSFUL; -} - -int sw6_pcie_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 val) -{ - u32 data; - u32 shift = 8 * (where & 3); - struct pci_controller *hose = pci_bus_to_pci_controller(bus); - void __iomem *cfg_iobase = (void *)hose->rc_config_space_base; - - if ((uintptr_t)where & (size - 1)) - return PCIBIOS_BAD_REGISTER_NUMBER; - - switch (size) { - case 1: - data = readl(cfg_iobase + ((where & ~3) << 5)); - data &= ~(0xff << shift); - data |= (val & 0xff) << shift; - break; - case 2: - data = readl(cfg_iobase + ((where & ~3) << 5)); - data &= ~(0xffff << shift); - data |= (val & 0xffff) << shift; - break; - default: - data = val; - break; - } - - if (IS_ENABLED(CONFIG_PCI_DEBUG)) - pr_debug("rc write addr:%px bus %d, devfn %#x, where %#x *val %#x size %d\n", - cfg_iobase + ((where & ~3) << 5), bus->number, devfn, where, val, size); - - writel(data, cfg_iobase + ((where & ~3) << 5)); - - return PCIBIOS_SUCCESSFUL; -} - -int sw6_pcie_config_read(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 *val) -{ - struct pci_controller *hose = pci_bus_to_pci_controller(bus); - int ret = PCIBIOS_DEVICE_NOT_FOUND; - - if (is_guest_or_emul()) - return pci_generic_config_read(bus, devfn, where, size, val); - - hose->self_busno = hose->busn_space->start; - - if (unlikely(bus->number == hose->self_busno)) { - ret = sw6_pcie_read_rc_cfg(bus, devfn, where, size, val); - } else { - if (test_bit(hose->node * 8 + hose->index, &rc_linkup)) - ret = pci_generic_config_read(bus, devfn, where, size, val); - else - return ret; - } - return ret; -} - -int sw6_pcie_config_write(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 val) -{ - struct pci_controller *hose = pci_bus_to_pci_controller(bus); - - if (is_guest_or_emul()) - return pci_generic_config_write(bus, devfn, where, size, val); - - hose->self_busno = hose->busn_space->start; - - if (unlikely(bus->number == hose->self_busno)) - return sw6_pcie_write_rc_cfg(bus, devfn, where, size, val); - else - return pci_generic_config_write(bus, devfn, where, size, val); -} - -/* - *sw6_pcie_valid_device - Check if a valid device is present on bus - *@bus: PCI Bus structure - *@devfn: device/function - * - *Return: 'true' on success and 'false' if invalid device is found - */ -static bool sw6_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) -{ - struct pci_controller *hose = pci_bus_to_pci_controller(bus); - - if (is_in_host()) { - /* Only one device down on each root complex */ - if (bus->number == hose->self_busno && devfn > 0) - return false; - } - - return true; -} - -/* - *sw6_pcie_map_bus - Get configuration base - *@bus: PCI Bus structure - *@devfn: Device/function - *@where: Offset from base - * - *Return: Base address of the configuration space needed to be - *accessed. - */ -static void __iomem *sw6_pcie_map_bus(struct pci_bus *bus, - unsigned int devfn, int where) -{ - struct pci_controller *hose = pci_bus_to_pci_controller(bus); - void __iomem *cfg_iobase; - unsigned long relbus; - - if (!sw6_pcie_valid_device(bus, devfn)) - return NULL; - - relbus = (bus->number << 24) | (devfn << 16) | where; - - cfg_iobase = hose->ep_config_space_base + relbus; - - if (IS_ENABLED(CONFIG_PCI_DEBUG)) - pr_debug("addr:%px bus %d, devfn %d, where %d\n", - cfg_iobase, bus->number, devfn, where); - return cfg_iobase; -} - struct pci_ops sw64_pci_ops = { - .map_bus = sw6_pcie_map_bus, - .read = sw6_pcie_config_read, - .write = sw6_pcie_config_write, + .map_bus = sw64_pcie_map_bus, + .read = sw64_pcie_config_read, + .write = sw64_pcie_config_write, }; int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) @@ -399,13 +233,25 @@ sw64_init_host(unsigned long node, unsigned long index) ret = sw64_chip_init->pci_init.check_pci_linkup(node, index); if (ret == 0) { /* Root Complex downstream port is link up */ - set_bit(node * 8 + index, &rc_linkup); //8-bit per node + pci_mark_rc_linkup(node, index); // 8-bit per node } } void __weak set_devint_wken(int node) {} void __weak set_adr_int(int node) {} +static bool __init is_any_rc_linkup_one_node(unsigned long node) +{ + int i; + + for (i = 0; i < 8; ++i) { + if (pci_get_rc_linkup(node, i)) + return true; + } + + return false; +} + void __init sw64_init_arch(void) { if (IS_ENABLED(CONFIG_PCI)) { @@ -437,11 +283,11 @@ void __init sw64_init_arch(void) if ((rc_enable >> i) & 0x1) sw64_init_host(node, i); } - if ((rc_linkup >> node * 8) & 0xff) { + if (is_any_rc_linkup_one_node(node)) { memset(msg, 0, 64); sprintf(msg, "Node %ld: RC [ ", node); for (i = 0; i < MAX_NR_RCS; i++) { - if ((rc_linkup >> (i + node * 8)) & 1) { + if (pci_get_rc_linkup(node, i)) { memset(id, 0, 8); sprintf(id, "%d ", i); strcat(msg, id); diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 7cd612bb82bb..46a75ef1a3e9 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -349,15 +349,17 @@ void __init setup_chip_pci_ops(void) static unsigned long rc_linkup; static struct pci_controller *head, **tail = &head; -static void pci_mark_rc_linkup(unsigned long node, unsigned long index) +void pci_mark_rc_linkup(unsigned long node, unsigned long index) { set_bit(node * 8 + index, &rc_linkup); } +EXPORT_SYMBOL(pci_mark_rc_linkup); -static int pci_get_rc_linkup(unsigned long node, unsigned long index) +int pci_get_rc_linkup(unsigned long node, unsigned long index) { return test_bit(node * 8 + index, &rc_linkup); } +EXPORT_SYMBOL(pci_get_rc_linkup); /** * Link the specified pci controller to list @@ -456,7 +458,7 @@ static int sw64_pcie_read_rc_cfg(struct pci_bus *bus, unsigned int devfn, /** * PCIe Root Complex write config space operations */ -int sw64_pcie_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, +static int sw64_pcie_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { u32 data; @@ -528,7 +530,7 @@ static bool sw64_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) * * @return: Whether read operation success */ -static int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, +int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { struct pci_controller *hose = pci_bus_to_pci_controller(bus); @@ -549,6 +551,7 @@ static int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, } return ret; } +EXPORT_SYMBOL(sw64_pcie_config_read); /** * sw64_pcie_config_write - write val to config space of PCI @@ -562,7 +565,7 @@ static int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, * * @return: Whether write operation success */ -static int sw64_pcie_config_write(struct pci_bus *bus, unsigned int devfn, +int sw64_pcie_config_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { struct pci_controller *hose = pci_bus_to_pci_controller(bus); @@ -577,6 +580,7 @@ static int sw64_pcie_config_write(struct pci_bus *bus, unsigned int devfn, else return pci_generic_config_write(bus, devfn, where, size, val); } +EXPORT_SYMBOL(sw64_pcie_config_write); /** * sw64_pcie_map_bus - get configuration base address @@ -587,7 +591,7 @@ static int sw64_pcie_config_write(struct pci_bus *bus, unsigned int devfn, * @return: base address of the configuration space needed to be * accessed. */ -static void __iomem *sw64_pcie_map_bus(struct pci_bus *bus, +void __iomem *sw64_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, int where) { struct pci_controller *hose = pci_bus_to_pci_controller(bus); @@ -616,6 +620,7 @@ static void __iomem *sw64_pcie_map_bus(struct pci_bus *bus, cfg_iobase, bus->number, devfn, where); return cfg_iobase; } +EXPORT_SYMBOL(sw64_pcie_map_bus); int sw64_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) { -- Gitee From ff01b22c667107a5b8db9051df751de3764a0966 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 26 Feb 2024 08:31:19 +0800 Subject: [PATCH 134/524] sw64: acpi: fix missed upper bits of rcid for junzhang When initializing CPU affinity, only lower 8 bits of rcid are recorded. Because 8 bits are sufficient for xuelang's rcid. However, the rcid for junzhang needs more bits. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/acpi.c | 36 +++++++++++++++++------------------- include/linux/acpi.h | 2 +- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index f2b2a4d5ce67..af80ae304948 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -114,26 +114,24 @@ static int rcid_to_cpu(int physical_id) /* Callback for Proximity Domain -> CPUID mapping */ void __init -acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) +acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) { int pxm, node; - int cpu; // logical core id + int cpu; // logical core id + int rcid; // physical core id if (srat_disabled()) return; - if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) { + + if (pa->header.length != sizeof(struct acpi_srat_x2apic_cpu_affinity)) { bad_srat(); return; } + if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0) return; - pxm = pa->proximity_domain_lo; - if (acpi_srat_revision >= 2) { - pxm |= (pa->proximity_domain_hi[0] << 8); - pxm |= (pa->proximity_domain_hi[1] << 16); - pxm |= (pa->proximity_domain_hi[2] << 24); - } + pxm = pa->proximity_domain; node = acpi_map_pxm_to_node(pxm); if (node < 0) { pr_err("SRAT: Too many proximity domains %x\n", pxm); @@ -141,25 +139,25 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) return; } - if (pa->apic_id >= CONFIG_NR_CPUS) { - pr_err("SRAT: PXM %u -> CPU 0x%02x -> Node %u skipped apicid that is too big\n", - pxm, pa->apic_id, node); - return; - } + /** + * In fact, SW64 does not support local X2_APIC and just + * uses this structure to pass CPU affinity information. + */ + rcid = pa->apic_id; /* Record the mapping from logical core id to node id */ - cpu = rcid_to_cpu(pa->apic_id); + cpu = rcid_to_cpu(rcid); if (cpu < 0) { - pr_err("SRAT: Can not find the logical id for physical Core 0x%02x\n", - pa->apic_id); + pr_err("SRAT: Can not find the logical id for physical Core 0x%04x\n", + rcid); return; } early_map_cpu_to_node(cpu, node); node_set(node, numa_nodes_parsed); - pr_info("SRAT: PXM %u -> CPU 0x%02x -> Node %u\n", - pxm, pa->apic_id, node); + pr_debug("SRAT: PXM %u -> CPU 0x%04x -> Node %u\n", + pxm, rcid, node); } #ifdef CONFIG_MEMORY_HOTPLUG diff --git a/include/linux/acpi.h b/include/linux/acpi.h index f8ce8d6ee5cc..faf84970c639 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -262,7 +262,7 @@ void acpi_table_print_madt_entry (struct acpi_subtable_header *madt); /* the following numa functions are architecture-dependent */ void acpi_numa_slit_init (struct acpi_table_slit *slit); -#if defined(CONFIG_X86) || defined(CONFIG_IA64) || defined(CONFIG_LOONGARCH) || defined(CONFIG_SW64) +#if defined(CONFIG_X86) || defined(CONFIG_IA64) || defined(CONFIG_LOONGARCH) void acpi_numa_processor_affinity_init (struct acpi_srat_cpu_affinity *pa); #else static inline void -- Gitee From 1f2185954c512afea676b03b7721ec5a05bf4890 Mon Sep 17 00:00:00 2001 From: Zhi Tongze Date: Tue, 12 Mar 2024 09:52:54 +0800 Subject: [PATCH 135/524] sw64: add unaligned access support for C4 new instructions C4 has introduced some new load/store instructions which may cause unaligned access. This patch adds unaligned exception handler for these instructions. Signed-off-by: Zhi Tongze Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/traps.c | 431 +++++++++++++++++++++++++++++++++++++- 1 file changed, 428 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 295e8e3e73a6..1405e42f27b4 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -388,7 +388,8 @@ asmlinkage void do_entUna(void *va, unsigned long opcode, unsigned long reg, struct pt_regs *regs) { - long error; + long error, disp; + unsigned int insn, fncode, rb; unsigned long tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; unsigned long pc = regs->pc - 4; @@ -399,6 +400,188 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, */ switch (opcode) { + case 0x1e: + insn = *(unsigned int *)pc; + fncode = (insn >> 12) & 0xf; + rb = (insn >> 16) & 0x1f; + disp = insn & 0xfff; + + disp = (disp << 52) >> 52; /* sext */ + + switch (fncode) { + case 0x1: /* ldhu_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 1(%3)\n" + " extlh %1, %3, %1\n" + " exthh %2, %3, %2\n" + "3:\n" + ".section __ex_table,\"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto got_exception; + regs->regs[reg] = tmp1 | tmp2; + regs->regs[rb] = regs->regs[rb] + disp; + return; + + case 0x2: /* ldw_a */ + __asm__ __volatile__( + "1: ldl_u %1,0(%3)\n" + "2: ldl_u %2,3(%3)\n" + " extlw %1,%3,%1\n" + " exthw %2,%3,%2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + regs->regs[reg] = (int)(tmp1 | tmp2); + regs->regs[rb] = regs->regs[rb] + disp; + return; + + case 0x3: /* ldl_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + regs->regs[reg] = tmp1 | tmp2; + regs->regs[rb] = regs->regs[rb] + disp; + return; + + case 0x7: /* sth_a */ + __asm__ __volatile__( + " zap %6, 2, %1\n" + " srl %6, 8, %2\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %2, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %1, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(regs->regs[reg]), "0"(0)); + + if (error) + goto got_exception; + regs->regs[rb] = regs->regs[rb] + disp; + return; + + case 0x8: /* stw_a */ + __asm__ __volatile__( + " zapnot %6, 0x1, %1\n" + " srl %6, 8, %2\n" + " zapnot %2, 0x1,%2\n" + " srl %6, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %6, 24, %4\n" + " zapnot %4, 0x1, %4\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3: stb %3, 0x2(%5)\n" + "4: stb %4, 0x3(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi $31, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(regs->regs[reg]), "0"(0)); + + if (error) + goto got_exception; + regs->regs[rb] = regs->regs[rb] + disp; + return; + + case 0x9: /* stl_a */ + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(regs->regs[reg]), "0"(0)); + + if (error) + goto got_exception; + regs->regs[rb] = regs->regs[rb] + disp; + return; + } case 0x21: __asm__ __volatile__( "1: ldl_u %1, 0(%3)\n" @@ -614,6 +797,11 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, 1L << 0x21 | 1L << 0x29 | /* ldhu sth */ \ 1L << 0x20 | 1L << 0x28) /* ldbu stb */ +#define FN_INT_MASK (1L << 0x0 | 1L << 0x6 | /* ldbu_a stb_a */ \ + 1L << 0x1 | 1L << 0x7 | /* ldhu_a sth_a */ \ + 1L << 0x2 | 1L << 0x8 | /* ldw_a stw_a */ \ + 1L << 0x3 | 1L << 0x9) /* ldl_a stl_a */ + asmlinkage void do_entUnaUser(void __user *va, unsigned long opcode, unsigned long reg, struct pt_regs *regs) @@ -628,7 +816,9 @@ do_entUnaUser(void __user *va, unsigned long opcode, long error; unsigned long tmp, tmp5, tmp6, tmp7, tmp8, vb; unsigned long fp[4]; - unsigned long instr, instr_op, value; + unsigned long instr, instr_op, value, fncode; + unsigned int rb = -1U; + long disp; #ifdef CONFIG_DEBUG_FS /* @@ -673,7 +863,11 @@ do_entUnaUser(void __user *va, unsigned long opcode, if ((unsigned long)va >= TASK_SIZE) goto give_sigsegv; - if ((1L << opcode) & OP_INT_MASK) { + get_user(instr, (__u32 *)(regs->pc - 4)); + fncode = (instr >> 12) & 0xf; + + if (((1L << opcode) & OP_INT_MASK) || + ((opcode == 0x1e) && ((1L << fncode) & FN_INT_MASK))) { /* it's an integer load/store */ if (reg < 31) { reg_addr = ®s->regs[reg]; @@ -1287,6 +1481,237 @@ do_entUnaUser(void __user *va, unsigned long opcode, } } switch (opcode) { + case 0x1e: + rb = (instr >> 16) & 0x1f; + disp = instr & 0xfff; + disp = (disp << 52) >> 52; + + switch (fncode) { + case 0x1: /* ldhu_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 1(%3)\n" + " extlh %1, %3, %1\n" + " exthh %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = tmp1 | tmp2; + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0x2: /* ldw_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = (int)(tmp1 | tmp2); + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0x3: /* ldl_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = tmp1 | tmp2; + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0x4: /* flds_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + sw64_write_fp_reg_s(reg, tmp1 | tmp2); + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0x5: /* fldd_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + sw64_write_fp_reg(reg, tmp1 | tmp2); + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0x7: /* sth_a */ + __asm__ __volatile__( + " zap %6, 2, %1\n" + " srl %6, 8, %2\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %2, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %1, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0xa: /* fsts_a */ + fake_reg = sw64_read_fp_reg_s(reg); + regs->regs[rb] = regs->regs[rb] + disp; + break; + /* fallthrough; */ + case 0x8: /* stw_a */ + __asm__ __volatile__( + " zapnot %6, 0x1, %1\n" + " srl %6, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %6, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %6, 24, %4\n" + " zapnot %4, 0x1, %4\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3: stb %3, 0x2(%5)\n" + "4: stb %4, 0x3(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi $31, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0xb: /* fstd_a */ + fake_reg = sw64_read_fp_reg(reg); + regs->regs[rb] = regs->regs[rb] + disp; + break; + /* fallthrough; */ + case 0x9: /* stl_a */ + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + regs->regs[rb] = regs->regs[rb] + disp; + break; + } + break; + case 0x21: /* ldhu */ __asm__ __volatile__( "1: ldl_u %1, 0(%3)\n" -- Gitee From 06a1d155efe01ce08615b7aa7a4e3b35d33013b7 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Thu, 14 Mar 2024 17:31:28 +0000 Subject: [PATCH 136/524] sw64: msi: remove redundant interrupt enables Each cpu has enabled its msi interrupt in firmware, so remove the enables in kernel. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/irq.c | 7 ------- arch/sw_64/kernel/smp.c | 6 ------ drivers/irqchip/irq-sunway-cpu.c | 12 ------------ 3 files changed, 25 deletions(-) diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c index 58ee5131e2a9..ac3d089eb656 100644 --- a/arch/sw_64/kernel/irq.c +++ b/arch/sw_64/kernel/irq.c @@ -116,13 +116,6 @@ void __init init_IRQ(void) * Just in case the platform init_irq() causes interrupts/mchecks * (as is the case with RAWHIDE, at least). */ - if (is_in_host()) { - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); - } - wrent(entInt, 0); sw64_init_irq(); diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index c198e0510996..82222ed63f0a 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -77,12 +77,6 @@ void smp_callin(void) trap_init(); /* Set interrupt vector. */ - if (is_in_host()) { - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); - } wrent(entInt, 0); /* Get our local ticker going. */ diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 20cfa4f2b999..3f5151b9b31d 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -396,18 +396,6 @@ static __init int cintc_acpi_init(union acpi_subtable_headers *header, msic_parse_madt, 0); #endif - /** - * After initializing MSIC, it's time to enable MSI interrupts - * for boot core. For other SMP cores, if present, this - * initialization is performed during SMP startup. - */ - if (!virtual) { - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); - sw64_write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); - } - return 0; } -- Gitee From d77e8553cf8f3eb731b69c08614d6f1b4436a45c Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 15 Mar 2024 14:44:35 +0800 Subject: [PATCH 137/524] sw64: dynamically switch between lib implementation Use jump_label to dynamically switch between lib with hardware unaligned memory access handling and software unaligned memory access handling. With software unaligned memory access handling, the lib itself will check the alignment of memory address and make sure it's aligned with the memory access operation to improve performance because the software handling is too slow. With hardware unaligned memory access handling, the above check is removed because hardware handling is fast enough. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hw_init.h | 3 + arch/sw_64/kernel/head.S | 1 + arch/sw_64/kernel/setup.c | 3 + arch/sw_64/lib/Makefile | 12 +-- ...-clear_user.S => deep-clear_user-hw_una.S} | 17 ++-- arch/sw_64/lib/deep-clear_user-sw_una.S | 47 +++++++++++ arch/sw_64/lib/deep-copy_template.S | 14 ++-- arch/sw_64/lib/deep-copy_template_c4.S | 10 +-- arch/sw_64/lib/deep-copy_user-hw_una.S | 40 ++++++++++ ...ep-copy_user.S => deep-copy_user-sw_una.S} | 21 ++--- .../{deep-memcpy.S => deep-memcpy-hw_una.S} | 19 ++--- arch/sw_64/lib/deep-memcpy-sw_una.S | 15 ++++ .../{deep-memset.S => deep-memset-hw_una.S} | 59 ++------------ arch/sw_64/lib/deep-memset-sw_una.S | 54 +++++++++++++ arch/sw_64/lib/deep-set_template.S | 12 +-- arch/sw_64/lib/deep-set_template_c4.S | 10 +-- arch/sw_64/lib/memmove.S | 7 +- arch/sw_64/lib/string.c | 77 +++++++++++++++++++ arch/sw_64/lib/uaccess.c | 29 +++++++ 19 files changed, 330 insertions(+), 120 deletions(-) rename arch/sw_64/lib/{deep-clear_user.S => deep-clear_user-hw_una.S} (78%) create mode 100644 arch/sw_64/lib/deep-clear_user-sw_una.S create mode 100644 arch/sw_64/lib/deep-copy_user-hw_una.S rename arch/sw_64/lib/{deep-copy_user.S => deep-copy_user-sw_una.S} (68%) rename arch/sw_64/lib/{deep-memcpy.S => deep-memcpy-hw_una.S} (40%) create mode 100644 arch/sw_64/lib/deep-memcpy-sw_una.S rename arch/sw_64/lib/{deep-memset.S => deep-memset-hw_una.S} (50%) create mode 100644 arch/sw_64/lib/deep-memset-sw_una.S create mode 100644 arch/sw_64/lib/string.c create mode 100644 arch/sw_64/lib/uaccess.c diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h index 4710f15a9a56..d1d2c71bc344 100644 --- a/arch/sw_64/include/asm/hw_init.h +++ b/arch/sw_64/include/asm/hw_init.h @@ -3,6 +3,7 @@ #define _ASM_SW64_HW_INIT_H #include #include +#include #include @@ -98,6 +99,8 @@ DECLARE_STATIC_KEY_TRUE(run_mode_host_key); DECLARE_STATIC_KEY_FALSE(run_mode_guest_key); DECLARE_STATIC_KEY_FALSE(run_mode_emul_key); +DECLARE_STATIC_KEY_FALSE(core_hw_una_enabled); + #define is_in_host() static_branch_likely(&run_mode_host_key) #define is_in_guest() static_branch_unlikely(&run_mode_guest_key) #define is_in_emul() static_branch_unlikely(&run_mode_emul_key) diff --git a/arch/sw_64/kernel/head.S b/arch/sw_64/kernel/head.S index fd0fbfbcf5b6..f3d27ea9e30e 100644 --- a/arch/sw_64/kernel/head.S +++ b/arch/sw_64/kernel/head.S @@ -33,6 +33,7 @@ __start: subl $18, $16, $18 mov $31, $17 call $26, __constant_c_memset + ldgp $29, 0($26) #ifdef CONFIG_RELOCATABLE ldi $30, -8($30) stl $29, 0($30) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 87324cd2268b..4bf49300d09e 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -108,6 +108,9 @@ EXPORT_SYMBOL(cpu_data); DEFINE_STATIC_KEY_TRUE(run_mode_host_key); DEFINE_STATIC_KEY_FALSE(run_mode_guest_key); DEFINE_STATIC_KEY_FALSE(run_mode_emul_key); + +DEFINE_STATIC_KEY_FALSE(core_hw_una_enabled); + struct cpu_desc_t cpu_desc; struct socket_desc_t socket_desc[MAX_NUMSOCKETS]; int memmap_nr; diff --git a/arch/sw_64/lib/Makefile b/arch/sw_64/lib/Makefile index e6455bb51139..2028d61dd195 100644 --- a/arch/sw_64/lib/Makefile +++ b/arch/sw_64/lib/Makefile @@ -15,25 +15,27 @@ lib-y = __divlu.o __remlu.o __divwu.o __remwu.o \ strcpy.o \ strncpy.o \ fls.o \ - csum_ipv6_magic.o + csum_ipv6_magic.o \ + string.o \ + uaccess.o lib-clear_page-y := clear_page.o lib-clear_page-$(CONFIG_DEEP_CLEAR_PAGE) := deep-clear_page.o lib-clear_user-y := clear_user.o -lib-clear_user-$(CONFIG_DEEP_CLEAR_USER) := deep-clear_user.o +lib-clear_user-$(CONFIG_DEEP_CLEAR_USER) := deep-clear_user-hw_una.o deep-clear_user-sw_una.o lib-copy_page-y := copy_page.o lib-copy_page-$(CONFIG_DEEP_COPY_PAGE) := deep-copy_page.o lib-copy_user-y := copy_user.o -lib-copy_user-$(CONFIG_DEEP_COPY_USER) := deep-copy_user.o +lib-copy_user-$(CONFIG_DEEP_COPY_USER) := deep-copy_user-hw_una.o deep-copy_user-sw_una.o lib-memcpy-y := memcpy.o -lib-memcpy-$(CONFIG_DEEP_MEMCPY) := deep-memcpy.o +lib-memcpy-$(CONFIG_DEEP_MEMCPY) := deep-memcpy-hw_una.o deep-memcpy-sw_una.o lib-memset-y := memset.o -lib-memset-$(CONFIG_DEEP_MEMSET) := deep-memset.o +lib-memset-$(CONFIG_DEEP_MEMSET) := deep-memset-hw_una.o deep-memset-sw_una.o lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o diff --git a/arch/sw_64/lib/deep-clear_user.S b/arch/sw_64/lib/deep-clear_user-hw_una.S similarity index 78% rename from arch/sw_64/lib/deep-clear_user.S rename to arch/sw_64/lib/deep-clear_user-hw_una.S index c81418ed99a2..81c04dd0f056 100644 --- a/arch/sw_64/lib/deep-clear_user.S +++ b/arch/sw_64/lib/deep-clear_user-hw_una.S @@ -16,7 +16,7 @@ 99: x,##y; \ .section __ex_table,"a"; \ .long 99b - .; \ - ldi $31, $out-99b($31); \ + ldi $31, 255f-99b($31); \ .previous /* @@ -27,19 +27,15 @@ * $18: bytes left to copy * */ - .globl __clear_user - .ent __clear_user -__clear_user: + .globl ____clear_user_hw_una + .ent ____clear_user_hw_una +____clear_user_hw_una: .prologue 0 bis $31, $31, $7 mov $17, $18 bis $31, $31, $17 -#if defined(CONFIG_SUBARCH_C3B) -#include "deep-set_template.S" -#elif defined(CONFIG_SUBARCH_C4) #include "deep-set_template_c4.S" -#endif -$out: +255: bis $31, $18, $0 beq $7, $return @@ -48,5 +44,4 @@ $restore_simd: $return: ret - .end __clear_user - EXPORT_SYMBOL(__clear_user) + .end ____clear_user_hw_una diff --git a/arch/sw_64/lib/deep-clear_user-sw_una.S b/arch/sw_64/lib/deep-clear_user-sw_una.S new file mode 100644 index 000000000000..e48e5be9efac --- /dev/null +++ b/arch/sw_64/lib/deep-clear_user-sw_una.S @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Contributed by Mao Minkai + * + * Zero user space, handling exceptions as we go. + * + * We have to make sure that $0 is always up-to-date and contains the + * right "bytes left to zero" value (and that it is updated only _after_ + * a successful copy). There is also some rather minor exception setup + * stuff. + * + */ +#include +/* Allow an exception for an insn; exit if we get one. */ +#define FIXUP_LDST(x,y...) \ + 99: x,##y; \ + .section __ex_table,"a"; \ + .long 99b - .; \ + ldi $31, 255f-99b($31); \ + .previous + +/* + * $7: SIMD status + * 0: not in simd loop + * 1: in simd loop + * 2: in simd_u loop + * $18: bytes left to copy + * + */ + .globl ____clear_user_sw_una + .ent ____clear_user_sw_una +____clear_user_sw_una: + .prologue 0 + bis $31, $31, $7 + mov $17, $18 + bis $31, $31, $17 +#include "deep-set_template.S" +255: + bis $31, $18, $0 + beq $7, $return + +$restore_simd: + RESTORE_SIMD_REGS + +$return: + ret + .end ____clear_user_sw_una diff --git a/arch/sw_64/lib/deep-copy_template.S b/arch/sw_64/lib/deep-copy_template.S index 7705eb3f36d4..fbb5912e1213 100644 --- a/arch/sw_64/lib/deep-copy_template.S +++ b/arch/sw_64/lib/deep-copy_template.S @@ -55,7 +55,7 @@ ldi $sp, 0xc0($sp); \ bis $31, $31, $7 - ble $18, $out + ble $18, 255f and $16, 7, $1 beq $1, $dest_aligned_8 @@ -65,7 +65,7 @@ $byte_loop_head: subl $18, 1, $18 addl $17, 1, $17 addl $16, 1, $16 - ble $18, $out + ble $18, 255f and $16, 7, $1 bne $1, $byte_loop_head @@ -140,7 +140,7 @@ $no_more_simd: RESTORE_SIMD_REGS $simd_end: - ble $18, $out + ble $18, 255f cmplt $18, 16, $1 bne $1, $quad_loop_end bne $4, $prep_quad_u_loop_tail @@ -158,7 +158,7 @@ $quad_loop_tail: beq $1, $quad_loop_tail $quad_loop_end: - ble $18, $out + ble $18, 255f cmplt $18, 8, $1 bne $1, $byte_loop_tail bne $4, $move_one_quad_u @@ -169,7 +169,7 @@ $move_one_quad: subl $18, 8, $18 addl $17, 8, $17 addl $16, 8, $16 - ble $18, $out + ble $18, 255f .align 3 $byte_loop_tail: @@ -179,7 +179,7 @@ $byte_loop_tail: addl $17, 1, $17 addl $16, 1, $16 bgt $18, $byte_loop_tail - br $31, $out + br $31, 255f /* misaligned src and dst */ $quad_u_loop_head: @@ -297,5 +297,5 @@ $move_one_quad_u: subl $18, 8, $18 addl $17, 8, $17 addl $16, 8, $16 - ble $18, $out + ble $18, 255f br $31, $byte_loop_tail diff --git a/arch/sw_64/lib/deep-copy_template_c4.S b/arch/sw_64/lib/deep-copy_template_c4.S index e0740874dfa3..b89ea4a75db3 100644 --- a/arch/sw_64/lib/deep-copy_template_c4.S +++ b/arch/sw_64/lib/deep-copy_template_c4.S @@ -29,7 +29,7 @@ bis $31, $31, $7 - ble $18, $out + ble $18, 255f cmplt $18, 8, $1 bne $1, $byte_loop_tail @@ -68,7 +68,7 @@ $no_more_simd: RESTORE_SIMD_REGS $simd_end: - ble $18, $out + ble $18, 255f cmplt $18, 16, $1 bne $1, $quad_loop_end @@ -85,7 +85,7 @@ $quad_loop_tail: beq $1, $quad_loop_tail $quad_loop_end: - ble $18, $out + ble $18, 255f cmplt $18, 8, $1 bne $1, $byte_loop_tail @@ -95,7 +95,7 @@ $move_one_quad: subl $18, 8, $18 addl $17, 8, $17 addl $16, 8, $16 - ble $18, $out + ble $18, 255f .align 3 $byte_loop_tail: @@ -105,4 +105,4 @@ $byte_loop_tail: addl $17, 1, $17 addl $16, 1, $16 bgt $18, $byte_loop_tail - br $31, $out + br $31, 255f diff --git a/arch/sw_64/lib/deep-copy_user-hw_una.S b/arch/sw_64/lib/deep-copy_user-hw_una.S new file mode 100644 index 000000000000..b411a7ed52c2 --- /dev/null +++ b/arch/sw_64/lib/deep-copy_user-hw_una.S @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Allow an exception for an insn; exit if we get one. */ +#define FIXUP_LDST(x, y) \ + 99: x, y; \ + .section __ex_table, "a"; \ + .long 99b - .; \ + ldi $31, 255f-99b($31); \ + .previous + +/* + * $7: SIMD status for C3B + * 0: not in simd loop + * 1: in simd loop + * 2: in simd_u loop + * $7: SIMD status for C4 + * 0: not in simd loop + * 1: in simd and simd_u loop + * $18: bytes left to copy + * + */ + .globl ____copy_user_hw_una + .ent ____copy_user_hw_una +____copy_user_hw_una: + .prologue 0 + .set noreorder + bis $31, $31, $7 +#include "deep-copy_template_c4.S" +255: + bis $31, $18, $0 + beq $7, $return + subl $7, 1, $7 + beq $7, $restore_simd + +$restore_simd: + RESTORE_SIMD_REGS + +$return: + ret + .end ____copy_user_hw_una diff --git a/arch/sw_64/lib/deep-copy_user.S b/arch/sw_64/lib/deep-copy_user-sw_una.S similarity index 68% rename from arch/sw_64/lib/deep-copy_user.S rename to arch/sw_64/lib/deep-copy_user-sw_una.S index b79f8f3f0f4a..a359aee65583 100644 --- a/arch/sw_64/lib/deep-copy_user.S +++ b/arch/sw_64/lib/deep-copy_user-sw_una.S @@ -1,13 +1,11 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#include - /* Allow an exception for an insn; exit if we get one. */ #define FIXUP_LDST(x, y) \ 99: x, y; \ .section __ex_table, "a"; \ .long 99b - .; \ - ldi $31, $out-99b($31); \ + ldi $31, 255f-99b($31); \ .previous /* @@ -21,33 +19,26 @@ * $18: bytes left to copy * */ - .globl __copy_user - .ent __copy_user -__copy_user: + .globl ____copy_user_sw_una + .ent ____copy_user_sw_una +____copy_user_sw_una: .prologue 0 .set noreorder bis $31, $31, $7 -#if defined(CONFIG_SUBARCH_C3B) #include "deep-copy_template.S" -#elif defined(CONFIG_SUBARCH_C4) -#include "deep-copy_template_c4.S" -#endif -$out: +255: bis $31, $18, $0 beq $7, $return subl $7, 1, $7 beq $7, $restore_simd -#if defined(CONFIG_SUBARCH_C3B) $restore_simd_u: RESTORE_SIMD_U_REGS br $31, $return -#endif $restore_simd: RESTORE_SIMD_REGS $return: ret - .end __copy_user - EXPORT_SYMBOL(__copy_user) + .end ____copy_user_sw_una diff --git a/arch/sw_64/lib/deep-memcpy.S b/arch/sw_64/lib/deep-memcpy-hw_una.S similarity index 40% rename from arch/sw_64/lib/deep-memcpy.S rename to arch/sw_64/lib/deep-memcpy-hw_una.S index 78a6bd85cf01..ccdfaed4b8ab 100644 --- a/arch/sw_64/lib/deep-memcpy.S +++ b/arch/sw_64/lib/deep-memcpy-hw_una.S @@ -1,24 +1,15 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#include - #define FIXUP_LDST(x, y) \ x, y - .globl memcpy - .ent memcpy -memcpy: + .globl ____memcpy_hw_una + .ent ____memcpy_hw_una +____memcpy_hw_una: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 -#if defined(CONFIG_SUBARCH_C3B) -#include "deep-copy_template.S" -#elif defined(CONFIG_SUBARCH_C4) #include "deep-copy_template_c4.S" -#endif -$out: +255: ret - .end memcpy - EXPORT_SYMBOL(memcpy) -__memcpy = memcpy -.globl __memcpy + .end ____memcpy_hw_una diff --git a/arch/sw_64/lib/deep-memcpy-sw_una.S b/arch/sw_64/lib/deep-memcpy-sw_una.S new file mode 100644 index 000000000000..75aa10a42b52 --- /dev/null +++ b/arch/sw_64/lib/deep-memcpy-sw_una.S @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#define FIXUP_LDST(x, y) \ + x, y + + .globl ____memcpy_sw_una + .ent ____memcpy_sw_una +____memcpy_sw_una: + .frame $30, 0, $26, 0 + .prologue 0 + mov $16, $0 +#include "deep-copy_template.S" +255: + ret + .end ____memcpy_sw_una diff --git a/arch/sw_64/lib/deep-memset.S b/arch/sw_64/lib/deep-memset-hw_una.S similarity index 50% rename from arch/sw_64/lib/deep-memset.S rename to arch/sw_64/lib/deep-memset-hw_una.S index c6b5355beec6..dff6b2c2003c 100644 --- a/arch/sw_64/lib/deep-memset.S +++ b/arch/sw_64/lib/deep-memset-hw_una.S @@ -25,7 +25,6 @@ * */ -#include #include #define FIXUP_LDST(x, y) \ @@ -35,63 +34,21 @@ .set noreorder .text .align 4 - .globl memset - .globl __memset - .globl ___memset - .globl __memsetw - .globl __constant_c_memset - .ent ___memset -___memset: + + .globl ____constant_c_memset_hw_una + .ent ____constant_c_memset_hw_una .frame $30, 0, $26, 0 .prologue 0 - +____constant_c_memset_hw_una: + bis $31, $31, $7 + bis $31, $16, $0 #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -/* expand 1 byte data to 8 bytes */ - and $17, 0xff, $17 - sll $17, 8, $4 - bis $17, $4, $17 - sll $17, 16, $4 - bis $17, $4, $17 - sll $17, 32, $4 - bis $17, $4, $17 - -__constant_c_memset: - bis $31, $31, $7 - bis $31, $16, $0 -#if defined(CONFIG_SUBARCH_C3B) -#include "deep-set_template.S" -#elif defined(CONFIG_SUBARCH_C4) #include "deep-set_template_c4.S" -#endif -$out: +255: #ifdef CONFIG_SUBARCH_C4 csrw $6, CSR_WR_FREGS #endif ret - - .end ___memset - EXPORT_SYMBOL(___memset) - - .align 5 - .ent __memsetw -__memsetw: - .prologue 0 - - inslh $17, 0, $1 - inslh $17, 2, $2 - inslh $17, 4, $3 - bis $1, $2, $1 - inslh $17, 6, $4 - bis $1, $3, $1 - bis $1, $4, $17 - br $31, __constant_c_memset - - .end __memsetw - EXPORT_SYMBOL(__memsetw) - -memset = ___memset -EXPORT_SYMBOL(memset) -__memset = ___memset -EXPORT_SYMBOL(__memset) + .end ____constant_c_memset_hw_una diff --git a/arch/sw_64/lib/deep-memset-sw_una.S b/arch/sw_64/lib/deep-memset-sw_una.S new file mode 100644 index 000000000000..b354b36dbbc9 --- /dev/null +++ b/arch/sw_64/lib/deep-memset-sw_una.S @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Optimized memset() for SW64 with SIMD instructions + * + * Copyright (C) Mao Minkai + * Author: Mao Minkai + * + * Fill SIZE bytes pointed to by SRC with CHAR. + * + * Input: + * $16: SRC, clobbered + * $17: CHAR, clobbered + * $18: SIZE, clobbered + * + * Output: + * $0: SRC + * + * Temporaries: + * $1: unaligned parts of addr (0 means aligned addr), tmp data + * $2: tmp data + * $3: tmp data + * $4: tmp data + * $5: compare result + * $f10: 32 bytes data (manually saved) + * + */ + +#include + +#define FIXUP_LDST(x, y) \ + x, y + + .set noat + .set noreorder + .text + .align 4 + + .globl ____constant_c_memset_sw_una + .ent ____constant_c_memset_sw_una + .frame $30, 0, $26, 0 + .prologue 0 +____constant_c_memset_sw_una: + bis $31, $31, $7 + bis $31, $16, $0 +#ifdef CONFIG_SUBARCH_C4 + csrr $6, CSR_WR_FREGS +#endif +#include "deep-set_template.S" +255: +#ifdef CONFIG_SUBARCH_C4 + csrw $6, CSR_WR_FREGS +#endif + ret + .end ____constant_c_memset_sw_una diff --git a/arch/sw_64/lib/deep-set_template.S b/arch/sw_64/lib/deep-set_template.S index f9073d638468..d581e2eb35d7 100644 --- a/arch/sw_64/lib/deep-set_template.S +++ b/arch/sw_64/lib/deep-set_template.S @@ -27,7 +27,7 @@ ldi $sp, 0x40($sp); \ bis $31, $31, $7 - ble $18, $out + ble $18, 255f and $16, 7, $1 beq $1, $dest_aligned_8 @@ -36,7 +36,7 @@ $byte_loop_head: FIXUP_LDST( stb $17, 0($16) ) subl $18, 1, $18 addl $16, 1, $16 - ble $18, $out + ble $18, 255f and $16, 7, $1 bne $1, $byte_loop_head @@ -100,7 +100,7 @@ $no_more_simd: RESTORE_SIMD_REGS $simd_end: - ble $18, $out + ble $18, 255f cmplt $18, 16, $1 bne $1, $quad_loop_end @@ -114,7 +114,7 @@ $quad_loop_tail: beq $1, $quad_loop_tail $quad_loop_end: - ble $18, $out + ble $18, 255f cmplt $18, 8, $1 bne $1, $byte_loop_tail @@ -122,7 +122,7 @@ $move_one_quad: FIXUP_LDST( stl $17, 0($16) ) subl $18, 8, $18 addl $16, 8, $16 - ble $18, $out + ble $18, 255f .align 3 $byte_loop_tail: @@ -130,4 +130,4 @@ $byte_loop_tail: subl $18, 1, $18 addl $16, 1, $16 bgt $18, $byte_loop_tail - br $31, $out + br $31, 255f diff --git a/arch/sw_64/lib/deep-set_template_c4.S b/arch/sw_64/lib/deep-set_template_c4.S index 2b1bcab8fec9..8a59d2733318 100644 --- a/arch/sw_64/lib/deep-set_template_c4.S +++ b/arch/sw_64/lib/deep-set_template_c4.S @@ -24,7 +24,7 @@ ldi $sp, 0x40($sp); \ bis $31, $31, $7 - ble $18, $out + ble $18, 255f cmplt $18, 8, $1 bne $1, $byte_loop_tail @@ -60,7 +60,7 @@ $no_more_simd: RESTORE_SIMD_REGS $simd_end: - ble $18, $out + ble $18, 255f cmplt $18, 16, $1 bne $1, $quad_loop_end @@ -74,7 +74,7 @@ $quad_loop_tail: beq $1, $quad_loop_tail $quad_loop_end: - ble $18, $out + ble $18, 255f cmplt $18, 8, $1 bne $1, $byte_loop_tail @@ -82,7 +82,7 @@ $move_one_quad: FIXUP_LDST( stl $17, 0($16) ) subl $18, 8, $18 addl $16, 8, $16 - ble $18, $out + ble $18, 255f .align 3 $byte_loop_tail: @@ -90,4 +90,4 @@ $byte_loop_tail: subl $18, 1, $18 addl $16, 1, $16 bgt $18, $byte_loop_tail - br $31, $out + br $31, 255f diff --git a/arch/sw_64/lib/memmove.S b/arch/sw_64/lib/memmove.S index 3e34fcd5b217..02c44d21bdb6 100644 --- a/arch/sw_64/lib/memmove.S +++ b/arch/sw_64/lib/memmove.S @@ -25,8 +25,13 @@ memmove: bis $1, $2, $1 mov $16, $0 xor $16, $17, $2 - bne $1, memcpy # samegp + beq $1, $memmove_cont + ldi $27, memcpy + jmp $31, ($27) + /* should never reach here + * memcpy should return to caller directly */ +$memmove_cont: and $2, 7, $2 # Test for src/dest co-alignment. and $16, 7, $1 cmpule $16, $17, $3 diff --git a/arch/sw_64/lib/string.c b/arch/sw_64/lib/string.c new file mode 100644 index 000000000000..4d10e0639bb8 --- /dev/null +++ b/arch/sw_64/lib/string.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include + +extern void *____memcpy_hw_una(void *dest, const void *src, size_t n); +extern void *____memcpy_sw_una(void *dest, const void *src, size_t n); + +static inline void *____memcpy(void *dest, const void *src, size_t n) +{ + if (static_branch_likely(&core_hw_una_enabled)) + return ____memcpy_hw_una(dest, src, n); + else + return ____memcpy_sw_una(dest, src, n); +} + +void *memcpy(void *dest, const void *src, size_t n) +{ + return ____memcpy(dest, src, n); +} +EXPORT_SYMBOL(memcpy); + +/* For backward compatibility with modules. Unused otherwise. */ +void *__memcpy(void *dest, const void *src, size_t n) +{ + return ____memcpy(dest, src, n); +} +EXPORT_SYMBOL(__memcpy); + +extern void *____constant_c_memset_hw_una(void *s, unsigned long c, size_t n); +extern void *____constant_c_memset_sw_una(void *s, unsigned long c, size_t n); + +static inline void *____constant_c_memset(void *s, unsigned long c, size_t n) +{ + if (static_branch_likely(&core_hw_una_enabled)) + return ____constant_c_memset_hw_una(s, c, n); + else + return ____constant_c_memset_sw_una(s, c, n); +} + +void *__constant_c_memset(void *s, unsigned long c, size_t n) +{ + return ____constant_c_memset(s, c, n); +} + +void *___memset(void *s, int c, size_t n) +{ + unsigned long c_ul = (c & 0xff) * 0x0101010101010101UL; + + return ____constant_c_memset(s, c_ul, n); +} +EXPORT_SYMBOL(___memset); + +void *__memset(void *s, int c, size_t n) +{ + unsigned long c_ul = (c & 0xff) * 0x0101010101010101UL; + + return ____constant_c_memset(s, c_ul, n); +} +EXPORT_SYMBOL(__memset); + +void *memset(void *s, int c, size_t n) +{ + unsigned long c_ul = (c & 0xff) * 0x0101010101010101UL; + + return ____constant_c_memset(s, c_ul, n); +} +EXPORT_SYMBOL(memset); + +void *__memsetw(void *dest, unsigned short c, size_t count) +{ + unsigned long c_ul = (c & 0xffff) * 0x0001000100010001UL; + + return ____constant_c_memset(dest, c_ul, count); +} +EXPORT_SYMBOL(__memsetw); diff --git a/arch/sw_64/lib/uaccess.c b/arch/sw_64/lib/uaccess.c new file mode 100644 index 000000000000..56d3075aedbb --- /dev/null +++ b/arch/sw_64/lib/uaccess.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include + +extern long ____copy_user_hw_una(void *to, const void *from, long len); +extern long ____copy_user_sw_una(void *to, const void *from, long len); + +long __copy_user(void *to, const void *from, long len) +{ + if (static_branch_likely(&core_hw_una_enabled)) + return ____copy_user_hw_una(to, from, len); + else + return ____copy_user_sw_una(to, from, len); +} +EXPORT_SYMBOL(__copy_user); + +extern long ____clear_user_hw_una(void __user *to, long len); +extern long ____clear_user_sw_una(void __user *to, long len); + +long __clear_user(void __user *to, long len) +{ + if (static_branch_likely(&core_hw_una_enabled)) + return ____clear_user_hw_una(to, len); + else + return ____clear_user_sw_una(to, len); +} +EXPORT_SYMBOL(__clear_user); -- Gitee From 2d6e9b25ff2a0c0e38484c6228359fa619347786 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 19 Mar 2024 15:53:07 +0000 Subject: [PATCH 138/524] sw64: irq: update the initial value of nr_irqs nr_irqs is initialized to NR_IRQS, making the /proc/stat file too large. So let arch update it's initial value to NR_IRQS_LEGACY, and then it grows with interrupt request. Along with that we redefine ACTUAL_NR_IRQS to nr_irqs for semantic correctness. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hw_irq.h | 2 +- arch/sw_64/kernel/irq.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/hw_irq.h b/arch/sw_64/include/asm/hw_irq.h index e0f8ee91c43a..59418a5fb1d3 100644 --- a/arch/sw_64/include/asm/hw_irq.h +++ b/arch/sw_64/include/asm/hw_irq.h @@ -7,7 +7,7 @@ extern volatile unsigned long irq_err_count; DECLARE_PER_CPU(unsigned long, irq_pmi_count); -#define ACTUAL_NR_IRQS NR_IRQS +#define ACTUAL_NR_IRQS nr_irqs extern struct irq_domain *mcu_irq_domain; diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c index ac3d089eb656..bdc92b40c4c0 100644 --- a/arch/sw_64/kernel/irq.c +++ b/arch/sw_64/kernel/irq.c @@ -133,6 +133,7 @@ int __init arch_early_irq_init(void) int __init arch_probe_nr_irqs(void) { + nr_irqs = NR_IRQS_LEGACY; return NR_IRQS_LEGACY; } -- Gitee From 3fe63c794bc55fee578c097f0fdccc68a83ae46f Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Tue, 19 Mar 2024 10:01:15 +0800 Subject: [PATCH 139/524] sw64: introduce new entry framework for C4 The pt_regs used to be partly handled by firmware, which limits the extension of os because of compatibility. This patch introduces new entry framework for Core4. Firmware is responsible for preparing the necessary parameters when exception or interrupt occurs and saving them to the specified CSRs. struct pt_regs is accessed by the kernel only and ksp is saved to struct pcb_struct instead of struct vcpucb. For guest os, since SAVE_ALL may be interrupted by host IRQ, we have to reserve enough space in vcpucb to save these CSRs. Due to a flaw of hardware, when there are consecutive write and read on a CSR in a recent period, the read may be issued before write in some cases, leading to a wrong result. It has to read the CSR twice to avoid this bug. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/csr.h | 9 + arch/sw_64/include/asm/hmcall.h | 2 - arch/sw_64/include/asm/kvm.h | 142 ++++++++++ arch/sw_64/include/asm/ptrace.h | 6 +- arch/sw_64/include/asm/thread_info.h | 1 + arch/sw_64/include/asm/vcpu.h | 13 +- arch/sw_64/include/uapi/asm/kvm.h | 87 ------ arch/sw_64/kernel/Makefile | 6 +- arch/sw_64/kernel/asm-offsets.c | 18 +- arch/sw_64/kernel/early_init.c | 1 - arch/sw_64/kernel/{entry.S => entry_c3.S} | 15 +- arch/sw_64/kernel/entry_c4.S | 330 ++++++++++++++++++++++ arch/sw_64/kernel/head.S | 12 +- arch/sw_64/kernel/hibernate_asm.S | 10 +- arch/sw_64/kernel/smp.c | 4 - arch/sw_64/kvm/Makefile | 14 +- arch/sw_64/kvm/{entry.S => entry_core3.S} | 0 arch/sw_64/kvm/entry_core4.S | 275 ++++++++++++++++++ 18 files changed, 817 insertions(+), 128 deletions(-) create mode 100644 arch/sw_64/include/asm/kvm.h rename arch/sw_64/kernel/{entry.S => entry_c3.S} (96%) create mode 100644 arch/sw_64/kernel/entry_c4.S rename arch/sw_64/kvm/{entry.S => entry_core3.S} (100%) create mode 100644 arch/sw_64/kvm/entry_core4.S diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h index dc400069a771..8f4f3d142db4 100644 --- a/arch/sw_64/include/asm/csr.h +++ b/arch/sw_64/include/asm/csr.h @@ -46,6 +46,15 @@ #define CSR_DV_MASK 0x57 #define CSR_IDA_MATCH 0xc5 #define CSR_IDA_MASK 0xc6 +#define CSR_BASE_KREGS 0xe0 +#define CSR_PS 0xe8 +#define CSR_PC 0xe9 +#define CSR_EARG0 0xea +#define CSR_EARG1 0xeb +#define CSR_EARG2 0xec +#define CSR_SCRATCH 0xed +#define CSR_SP 0xee +#define CSR_KTP 0xef #define DA_MATCH_EN_S 4 #define DV_MATCH_EN_S 6 diff --git a/arch/sw_64/include/asm/hmcall.h b/arch/sw_64/include/asm/hmcall.h index 41c656470ca4..71328c4eb332 100644 --- a/arch/sw_64/include/asm/hmcall.h +++ b/arch/sw_64/include/asm/hmcall.h @@ -155,9 +155,7 @@ __CALL_HMC_VOID(sleepen); __CALL_HMC_VOID(mtinten); __CALL_HMC_VOID(rdktp); -#define restore_ktp() rdktp() __CALL_HMC_VOID(wrktp); -#define save_ktp() wrktp() __CALL_HMC_R0(rdps, unsigned long); __CALL_HMC_R0(rvpcr, unsigned long); diff --git a/arch/sw_64/include/asm/kvm.h b/arch/sw_64/include/asm/kvm.h new file mode 100644 index 000000000000..f6e3022efcfe --- /dev/null +++ b/arch/sw_64/include/asm/kvm.h @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_H +#define _ASM_SW64_KVM_H +#include + +/* + * KVM SW specific structures and definitions. + */ +#define SWVM_IRQS 256 +#define IRQ_PENDING_INTX_SHIFT 16 +#define IRQ_PENDING_MSI_VECTORS_SHIFT 17 + +#define SWVM_NUM_NUMA_MEMBANKS 1 + +/* + * for KVM_GET_REGS and KVM_SET_REGS + */ +#ifdef CONFIG_SUBARCH_C3B +struct kvm_regs { + unsigned long r0; + unsigned long r1; + unsigned long r2; + unsigned long r3; + + unsigned long r4; + unsigned long r5; + unsigned long r6; + unsigned long r7; + + unsigned long r8; + unsigned long r9; + unsigned long r10; + unsigned long r11; + + unsigned long r12; + unsigned long r13; + unsigned long r14; + unsigned long r15; + + unsigned long r19; + unsigned long r20; + unsigned long r21; + unsigned long r22; + + unsigned long r23; + unsigned long r24; + unsigned long r25; + unsigned long r26; + + unsigned long r27; + unsigned long r28; + unsigned long __padding0; + unsigned long fpcr; + + unsigned long fp[124]; + /* These are saved by HMcode: */ + unsigned long ps; + unsigned long pc; + unsigned long gp; + unsigned long r16; + unsigned long r17; + unsigned long r18; +}; + +#elif CONFIG_SUBARCH_C4 +struct kvm_regs { + unsigned long r0; + unsigned long r1; + unsigned long r2; + unsigned long r3; + + unsigned long r4; + unsigned long r5; + unsigned long r6; + unsigned long r7; + + unsigned long r8; + unsigned long r9; + unsigned long r10; + unsigned long r11; + + unsigned long r12; + unsigned long r13; + unsigned long r14; + unsigned long r15; + + unsigned long r16; + unsigned long r17; + unsigned long r18; + unsigned long r19; + + unsigned long r20; + unsigned long r21; + unsigned long r22; + unsigned long r23; + + unsigned long r24; + unsigned long r25; + unsigned long r26; + unsigned long r27; + + unsigned long r28; + unsigned long gp; + unsigned long sp; + unsigned long fpcr; + + unsigned long fp[124]; + unsigned long ps; + unsigned long pc; +}; +#endif + +/* + * return stack for __sw64_vcpu_run + */ +struct vcpu_run_ret_stack { + unsigned long ra; + unsigned long r0; +}; + +struct host_int_args { + unsigned long r18; + unsigned long r17; + unsigned long r16; +}; + +struct hcall_args { + unsigned long arg0, arg1, arg2; +}; + +struct swvm_mem_bank { + unsigned long guest_phys_addr; + unsigned long host_phys_addr; + unsigned long host_addr; + unsigned long size; +}; + +struct swvm_mem { + struct swvm_mem_bank membank[SWVM_NUM_NUMA_MEMBANKS]; +}; + +#endif /* _ASM_SW64_KVM_H */ diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h index 964f4fc730f2..aa8b269a7ef4 100644 --- a/arch/sw_64/include/asm/ptrace.h +++ b/arch/sw_64/include/asm/ptrace.h @@ -31,9 +31,9 @@ struct pt_regs { unsigned long hm_ps; unsigned long hm_pc; unsigned long hm_gp; - unsigned long hm_r16; - unsigned long hm_r17; - unsigned long hm_r18; + unsigned long earg0; + unsigned long earg1; + unsigned long earg2; }; #define arch_has_single_step() (1) diff --git a/arch/sw_64/include/asm/thread_info.h b/arch/sw_64/include/asm/thread_info.h index 4f3b837e2e90..17a93bd8baa4 100644 --- a/arch/sw_64/include/asm/thread_info.h +++ b/arch/sw_64/include/asm/thread_info.h @@ -14,6 +14,7 @@ typedef struct { struct pcb_struct { + unsigned long ksp; unsigned long tp; unsigned long da_match, da_mask; unsigned long dv_match, dv_mask; diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h index c4e3caacbc70..a68e8ff56af5 100644 --- a/arch/sw_64/include/asm/vcpu.h +++ b/arch/sw_64/include/asm/vcpu.h @@ -55,7 +55,7 @@ struct vcpucb { unsigned long reserved[3]; }; -#else +#elif CONFIG_SUBARCH_C4 struct vcpucb { unsigned long ktp; @@ -98,7 +98,16 @@ struct vcpucb { unsigned long ipaddr; unsigned long vcpu_pc_save; unsigned long shtclock_offset; - unsigned long reserved[8]; + unsigned long migration_mark; + unsigned long shtclock; + unsigned long csr_pc; + unsigned long csr_ps; + unsigned long csr_sp; + unsigned long csr_ktp; + unsigned long csr_earg0; + unsigned long csr_earg1; + unsigned long csr_earg2; + unsigned long csr_scratch; }; #endif diff --git a/arch/sw_64/include/uapi/asm/kvm.h b/arch/sw_64/include/uapi/asm/kvm.h index 2253475deaa5..2ae0cc74bb2b 100644 --- a/arch/sw_64/include/uapi/asm/kvm.h +++ b/arch/sw_64/include/uapi/asm/kvm.h @@ -16,75 +16,9 @@ enum SW64_KVM_IRQ { SW64_KVM_IRQ_MOUSE = 30, }; -#define SWVM_VM_TYPE_DEFAULT 0 -#define SWVM_VM_TYPE_PHYVCPU 1 #define __KVM_HAVE_IRQ_LINE -#define SWVM_NUM_NUMA_MEMBANKS 1 #define KVM_NR_IRQCHIPS 1 -/* - * for KVM_GET_REGS and KVM_SET_REGS - */ -struct kvm_regs { - unsigned long r0; - unsigned long r1; - unsigned long r2; - unsigned long r3; - - unsigned long r4; - unsigned long r5; - unsigned long r6; - unsigned long r7; - - unsigned long r8; - unsigned long r9; - unsigned long r10; - unsigned long r11; - - unsigned long r12; - unsigned long r13; - unsigned long r14; - unsigned long r15; - - unsigned long r19; - unsigned long r20; - unsigned long r21; - unsigned long r22; - - unsigned long r23; - unsigned long r24; - unsigned long r25; - unsigned long r26; - - unsigned long r27; - unsigned long r28; - unsigned long __padding0; - unsigned long fpcr; - - unsigned long fp[124]; - /* These are saved by HMcode: */ - unsigned long ps; - unsigned long pc; - unsigned long gp; - unsigned long r16; - unsigned long r17; - unsigned long r18; -}; - - -/* - * return stack for __sw64_vcpu_run - */ -struct vcpu_run_ret_stack { - unsigned long ra; - unsigned long r0; -}; - -struct host_int_args { - unsigned long r18; - unsigned long r17; - unsigned long r16; -}; /* * for KVM_GET_FPU and KVM_SET_FPU @@ -92,15 +26,6 @@ struct host_int_args { struct kvm_fpu { }; -struct hcall_args { - unsigned long arg0, arg1, arg2; -}; - -struct phyvcpu_hcall_args { - unsigned long call; - struct hcall_args args; -}; - struct kvm_debug_exit_arch { unsigned long epc; }; @@ -116,16 +41,4 @@ struct kvm_sync_regs { /* dummy definition */ struct kvm_sregs { }; - -struct swvm_mem_bank { - unsigned long guest_phys_addr; - unsigned long host_phys_addr; - unsigned long host_addr; - unsigned long size; -}; - -struct swvm_mem { - struct swvm_mem_bank membank[SWVM_NUM_NUMA_MEMBANKS]; -}; - #endif /* _UAPI_ASM_SW64_KVM_H */ diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 5ac06882ebc0..57124b8546ec 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -13,14 +13,16 @@ CFLAGS_REMOVE_insn.o = -pg CFLAGS_REMOVE_printk.o = -pg endif -obj-y := entry.o fpu.o traps.o process.o sys_sw64.o irq.o \ +obj-y := fpu.o traps.o process.o sys_sw64.o irq.o \ signal.o setup.o ptrace.o time.o \ systbls.o dup_print.o chip_setup.o \ insn.o early_init.o topology.o cacheinfo.o \ vdso.o vdso/ hmcall.o stacktrace.o idle.o reset.o \ head.o termios.o -obj-$(CONFIG_SUBARCH_C3B) += tc.o +obj-$(CONFIG_SUBARCH_C3B) += entry_c3.o tc.o +obj-$(CONFIG_SUBARCH_C4) += entry_c4.o + obj-$(CONFIG_ACPI) += acpi.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_MODULES) += module.o diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c index 41310a8a7af1..2fb2d807ae3f 100644 --- a/arch/sw_64/kernel/asm-offsets.c +++ b/arch/sw_64/kernel/asm-offsets.c @@ -21,6 +21,7 @@ void foo(void) { DEFINE(ASM_THREAD_SIZE, THREAD_SIZE); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); + OFFSET(TI_PCB_KSP, thread_info, pcb.ksp); BLANK(); DEFINE(TASK_BLOCKED, offsetof(struct task_struct, blocked)); @@ -106,9 +107,9 @@ void foo(void) DEFINE(PT_REGS_HM_PS, offsetof(struct pt_regs, hm_ps)); DEFINE(PT_REGS_HM_PC, offsetof(struct pt_regs, hm_pc)); DEFINE(PT_REGS_HM_GP, offsetof(struct pt_regs, hm_gp)); - DEFINE(PT_REGS_HM_R16, offsetof(struct pt_regs, hm_r16)); - DEFINE(PT_REGS_HM_R17, offsetof(struct pt_regs, hm_r17)); - DEFINE(PT_REGS_HM_R18, offsetof(struct pt_regs, hm_r18)); + DEFINE(PT_REGS_EARG0, offsetof(struct pt_regs, earg0)); + DEFINE(PT_REGS_EARG1, offsetof(struct pt_regs, earg1)); + DEFINE(PT_REGS_EARG2, offsetof(struct pt_regs, earg2)); BLANK(); DEFINE(KVM_REGS_SIZE, sizeof(struct kvm_regs)); @@ -128,6 +129,9 @@ void foo(void) DEFINE(KVM_REGS_R13, offsetof(struct kvm_regs, r13)); DEFINE(KVM_REGS_R14, offsetof(struct kvm_regs, r14)); DEFINE(KVM_REGS_R15, offsetof(struct kvm_regs, r15)); + DEFINE(KVM_REGS_R16, offsetof(struct kvm_regs, r16)); + DEFINE(KVM_REGS_R17, offsetof(struct kvm_regs, r17)); + DEFINE(KVM_REGS_R18, offsetof(struct kvm_regs, r18)); DEFINE(KVM_REGS_R19, offsetof(struct kvm_regs, r19)); DEFINE(KVM_REGS_R20, offsetof(struct kvm_regs, r20)); DEFINE(KVM_REGS_R21, offsetof(struct kvm_regs, r21)); @@ -138,6 +142,7 @@ void foo(void) DEFINE(KVM_REGS_R26, offsetof(struct kvm_regs, r26)); DEFINE(KVM_REGS_R27, offsetof(struct kvm_regs, r27)); DEFINE(KVM_REGS_R28, offsetof(struct kvm_regs, r28)); + DEFINE(KVM_REGS_GP, offsetof(struct kvm_regs, gp)); DEFINE(KVM_REGS_FPCR, offsetof(struct kvm_regs, fpcr)); DEFINE(KVM_REGS_F0, offsetof(struct kvm_regs, fp[0 * 4])); DEFINE(KVM_REGS_F1, offsetof(struct kvm_regs, fp[1 * 4])); @@ -172,10 +177,9 @@ void foo(void) DEFINE(KVM_REGS_F30, offsetof(struct kvm_regs, fp[30 * 4])); DEFINE(KVM_REGS_PS, offsetof(struct kvm_regs, ps)); DEFINE(KVM_REGS_PC, offsetof(struct kvm_regs, pc)); - DEFINE(KVM_REGS_GP, offsetof(struct kvm_regs, gp)); - DEFINE(KVM_REGS_R16, offsetof(struct kvm_regs, r16)); - DEFINE(KVM_REGS_R17, offsetof(struct kvm_regs, r17)); - DEFINE(KVM_REGS_R18, offsetof(struct kvm_regs, r18)); +#ifdef CONFIG_SUBARCH_C4 + DEFINE(KVM_REGS_SP, offsetof(struct kvm_regs, sp)); +#endif BLANK(); DEFINE(VCPU_RET_SIZE, sizeof(struct vcpu_run_ret_stack)); diff --git a/arch/sw_64/kernel/early_init.c b/arch/sw_64/kernel/early_init.c index 2ec7a3e99443..3d7b9c4325a9 100644 --- a/arch/sw_64/kernel/early_init.c +++ b/arch/sw_64/kernel/early_init.c @@ -6,6 +6,5 @@ asmlinkage __visible void __init sw64_start_kernel(void) { fixup_hmcall(); - save_ktp(); start_kernel(); } diff --git a/arch/sw_64/kernel/entry.S b/arch/sw_64/kernel/entry_c3.S similarity index 96% rename from arch/sw_64/kernel/entry.S rename to arch/sw_64/kernel/entry_c3.S index 59c2ff4eb915..a7183817671b 100644 --- a/arch/sw_64/kernel/entry.S +++ b/arch/sw_64/kernel/entry_c3.S @@ -49,9 +49,9 @@ stl $25, PT_REGS_R25($sp) stl $26, PT_REGS_R26($sp) stl $27, PT_REGS_R27($sp) - ldl $1, PT_REGS_HM_R16($sp) - ldl $2, PT_REGS_HM_R17($sp) - ldl $3, PT_REGS_HM_R18($sp) + ldl $1, PT_REGS_EARG0($sp) + ldl $2, PT_REGS_EARG1($sp) + ldl $3, PT_REGS_EARG2($sp) ldl $4, PT_REGS_HM_GP($sp) ldl $5, PT_REGS_HM_PC($sp) ldl $6, PT_REGS_HM_PS($sp) @@ -83,9 +83,9 @@ ldl $4, PT_REGS_GP($sp) ldl $5, PT_REGS_PC($sp) ldl $6, PT_REGS_PS($sp) - stl $1, PT_REGS_HM_R16($sp) - stl $2, PT_REGS_HM_R17($sp) - stl $3, PT_REGS_HM_R18($sp) + stl $1, PT_REGS_EARG0($sp) + stl $2, PT_REGS_EARG1($sp) + stl $3, PT_REGS_EARG2($sp) stl $4, PT_REGS_HM_GP($sp) stl $5, PT_REGS_HM_PC($sp) stl $6, PT_REGS_HM_PS($sp) @@ -214,10 +214,9 @@ entSys: .globl ret_from_sys_call .ent ret_from_sys_call ret_from_sys_call: -#ifdef CONFIG_SUBARCH_C3B fillcs 0($sp) /* prefetch */ fillcs 128($sp) /* prefetch */ -#endif + br $27, 1f 1: ldgp $29, 0($27) /* Make sure need_resched and sigpending don't change between diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S new file mode 100644 index 000000000000..1209e627a7f1 --- /dev/null +++ b/arch/sw_64/kernel/entry_c4.S @@ -0,0 +1,330 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Kernel entry-points. + */ + +#include +#include +#include +#include +#include +#include +#include + + .text + .set noat + + .macro SAVE_ALL + csrw $sp, CSR_SP + csrw $1, CSR_SCRATCH + csrr $1, CSR_PS + and $1, 8, $1 + beq $1, 1f + csrr $1, CSR_KTP + ldl $sp, TI_PCB_KSP($1) +1: + /* Due to a flaw in the CSR design, in some cases the + * read CSR instruction is sent before the write instruction, + * so it needs to be read twice to ensure that the correct + * CSR value is read. Don`t delete it! + */ + csrr $1, CSR_SCRATCH + csrr $1, CSR_SCRATCH + ldi $sp, -PT_REGS_SIZE($sp) + stl $0, PT_REGS_R0($sp) + stl $1, PT_REGS_R1($sp) + stl $2, PT_REGS_R2($sp) + stl $3, PT_REGS_R3($sp) + stl $4, PT_REGS_R4($sp) + stl $5, PT_REGS_R5($sp) + stl $6, PT_REGS_R6($sp) + stl $7, PT_REGS_R7($sp) + stl $8, PT_REGS_R8($sp) + stl $9, PT_REGS_R9($sp) + stl $10, PT_REGS_R10($sp) + stl $11, PT_REGS_R11($sp) + stl $12, PT_REGS_R12($sp) + stl $13, PT_REGS_R13($sp) + stl $14, PT_REGS_R14($sp) + stl $15, PT_REGS_R15($sp) + stl $16, PT_REGS_R16($sp) + stl $17, PT_REGS_R17($sp) + stl $18, PT_REGS_R18($sp) + stl $19, PT_REGS_R19($sp) + stl $20, PT_REGS_R20($sp) + stl $21, PT_REGS_R21($sp) + stl $22, PT_REGS_R22($sp) + stl $23, PT_REGS_R23($sp) + stl $24, PT_REGS_R24($sp) + stl $25, PT_REGS_R25($sp) + stl $26, PT_REGS_R26($sp) + stl $27, PT_REGS_R27($sp) + stl $28, PT_REGS_R28($sp) + stl $29, PT_REGS_GP($sp) + + csrr $1, CSR_SP + csrr $1, CSR_SP + csrr $2, CSR_PS + csrr $3, CSR_PC + csrr $16, CSR_EARG0 + csrr $17, CSR_EARG1 + csrr $18, CSR_EARG2 + stl $16, PT_REGS_EARG0($sp) + stl $17, PT_REGS_EARG1($sp) + stl $18, PT_REGS_EARG2($sp) + + stl $1, PT_REGS_SP($sp) + stl $2, PT_REGS_PS($sp) + stl $3, PT_REGS_PC($sp) + ldi $1, NO_SYSCALL + stl $1, PT_REGS_ORIG_R0($sp) + csrr $8, CSR_KTP + .endm + + .macro RESTORE_ALL + ldi $16, 7 + sys_call HMC_swpipl + + ldl $1, PT_REGS_PS($sp) + ldl $2, PT_REGS_PC($sp) + csrw $1, CSR_PS + csrw $2, CSR_PC + and $1, 0x8, $1 + beq $1, 1f + ldi $16,PT_REGS_SIZE($sp) + stl $16, TI_PCB_KSP($8) +1: + ldl $0, PT_REGS_R0($sp) + ldl $1, PT_REGS_R1($sp) + ldl $2, PT_REGS_R2($sp) + ldl $3, PT_REGS_R3($sp) + ldl $4, PT_REGS_R4($sp) + ldl $5, PT_REGS_R5($sp) + ldl $6, PT_REGS_R6($sp) + ldl $7, PT_REGS_R7($sp) + ldl $8, PT_REGS_R8($sp) + ldl $9, PT_REGS_R9($sp) + ldl $10, PT_REGS_R10($sp) + ldl $11, PT_REGS_R11($sp) + ldl $12, PT_REGS_R12($sp) + ldl $13, PT_REGS_R13($sp) + ldl $14, PT_REGS_R14($sp) + ldl $15, PT_REGS_R15($sp) + ldl $16, PT_REGS_R16($sp) + ldl $17, PT_REGS_R17($sp) + ldl $18, PT_REGS_R18($sp) + ldl $19, PT_REGS_R19($sp) + ldl $20, PT_REGS_R20($sp) + ldl $21, PT_REGS_R21($sp) + ldl $22, PT_REGS_R22($sp) + ldl $23, PT_REGS_R23($sp) + ldl $24, PT_REGS_R24($sp) + ldl $25, PT_REGS_R25($sp) + ldl $26, PT_REGS_R26($sp) + ldl $27, PT_REGS_R27($sp) + ldl $28, PT_REGS_R28($sp) + ldl $29, PT_REGS_GP($sp) + ldl $sp, PT_REGS_SP($sp) + .endm + + .macro RESTORE_IRQ + csrr $16, CSR_PS + sys_call HMC_swpipl + ldl $16, PT_REGS_EARG0($sp) + .endm +/* + * Non-syscall kernel entry points. + */ + + .align 4 + .globl entInt + .ent entInt +entInt: + SAVE_ALL + br $27, 1f +1: ldgp $29, 0($27) + mov $sp, $19 + call $26, do_entInt + br ret_from_sys_call + .end entInt + + .align 4 + .globl entArith + .ent entArith +entArith: + SAVE_ALL + br $27, 1f +1: ldgp $29, 0($27) + RESTORE_IRQ + mov $sp, $18 + call $26, do_entArith + br ret_from_sys_call + .end entArith + + .align 4 + .globl entMM + .ent entMM +entMM: + SAVE_ALL + br $27, 1f +1: ldgp $29, 0($27) + RESTORE_IRQ + mov $sp, $19 + call $26, do_page_fault + br ret_from_sys_call + .end entMM + + .align 4 + .globl entIF + .ent entIF +entIF: + SAVE_ALL + br $27, 1f +1: ldgp $29, 0($27) + RESTORE_IRQ + mov $sp, $18 + call $26, do_entIF + br ret_from_sys_call + .end entIF + +/* + * Handle unalignment exception. + * We don't handle the "gp" register correctly, but if we fault on a + * gp-register unaligned load/store, something is _very_ wrong in the + * kernel anyway. + */ + .align 4 + .globl entUna + .ent entUna +entUna: + SAVE_ALL + br $27, 1f +1: ldgp $29, 0($27) + RESTORE_IRQ + mov $sp, $19 + ldl $0, PT_REGS_PS($sp) + and $0, 8, $0 /* user mode ? */ + beq $0, 1f + call $26, do_entUnaUser /* return to ret_from_syscall */ + br ret_from_sys_call +1: ldl $9, PT_REGS_GP($sp) + call $26, do_entUna + stl $9, PT_REGS_GP($sp) + RESTORE_ALL + sys_call HMC_rti + .end entUna + +/* + * The system call entry point is special. Most importantly, it looks + * like a function call to userspace as far as clobbered registers. We + * do preserve the argument registers (for syscall restarts) and $26 + * (for leaf syscall functions). + * + * So much for theory. We don't take advantage of this yet. + * + * Note that a0-a2 are not saved by HMcode as with the other entry points. + */ + + .align 4 + .globl entSys + .ent entSys +entSys: + SAVE_ALL + br $27, 1f +1: ldgp $29, 0($27) + RESTORE_IRQ + mov $sp, $16 + call $26, do_entSys + br ret_from_sys_call + .end entSys + + .align 4 + .globl ret_from_sys_call + .ent ret_from_sys_call +ret_from_sys_call: + br $27, 1f +1: ldgp $29, 0($27) + /* Make sure need_resched and sigpending don't change between + sampling and the rti. */ + ldi $16, 7 + sys_call HMC_swpipl + ldl $0, PT_REGS_PS($sp) + and $0, 8, $0 + beq $0, restore_all +ret_to_user: + ldw $17, TI_FLAGS($8) + and $17, _TIF_WORK_MASK, $2 + beq $2, restore_all + mov $sp, $16 + call $26, do_notify_resume +restore_all: + RESTORE_ALL + sys_call HMC_rti + .end ret_from_sys_call + +/* + * Integer register context switch + * The callee-saved registers must be saved and restored. + * + * a0: previous task_struct (must be preserved across the switch) + * a1: next task_struct + * + * The value of a0 must be preserved by this function, as that's how + * arguments are passed to schedule_tail. + */ + .align 4 + .globl __switch_to + .ent __switch_to +__switch_to: + .prologue 0 + /* Save context into prev->thread */ + stl $26, TASK_THREAD_RA($16) + stl $30, TASK_THREAD_SP($16) + stl $9, TASK_THREAD_S0($16) + stl $10, TASK_THREAD_S1($16) + stl $11, TASK_THREAD_S2($16) + stl $12, TASK_THREAD_S3($16) + stl $13, TASK_THREAD_S4($16) + stl $14, TASK_THREAD_S5($16) + stl $15, TASK_THREAD_S6($16) + /* Restore context from next->thread */ + ldl $26, TASK_THREAD_RA($17) + ldl $30, TASK_THREAD_SP($17) + ldl $9, TASK_THREAD_S0($17) + ldl $10, TASK_THREAD_S1($17) + ldl $11, TASK_THREAD_S2($17) + ldl $12, TASK_THREAD_S3($17) + ldl $13, TASK_THREAD_S4($17) + ldl $14, TASK_THREAD_S5($17) + ldl $15, TASK_THREAD_S6($17) + mov $17, $8 + csrw $8, CSR_KTP + mov $16, $0 + ret + .end __switch_to + +/* + * New processes begin life here. + */ + + .globl ret_from_fork + .align 4 + .ent ret_from_fork +ret_from_fork: + call $26, schedule_tail + br ret_from_sys_call + .end ret_from_fork + +/* + * ... and new kernel threads - here + */ + .align 4 + .globl ret_from_kernel_thread + .ent ret_from_kernel_thread +ret_from_kernel_thread: + call $26, schedule_tail + mov $9, $27 + mov $10, $16 + call $26, ($9) + br ret_to_user + .end ret_from_kernel_thread diff --git a/arch/sw_64/kernel/head.S b/arch/sw_64/kernel/head.S index f3d27ea9e30e..bfeb6859ecc2 100644 --- a/arch/sw_64/kernel/head.S +++ b/arch/sw_64/kernel/head.S @@ -10,6 +10,15 @@ #include #include #include +#include + + .macro SAVE_KTP +#ifdef CONFIG_SUBARCH_C3B + sys_call HMC_wrktp +#else + csrw $8, CSR_KTP +#endif + .endm __HEAD .globl _stext @@ -23,6 +32,7 @@ __start: 1: ldgp $29, 0($27) /* We need to get current_task_info loaded up... */ ldi $8, init_task + SAVE_KTP ldl $30, TASK_STACK($8) /* ... and find our stack ... */ ldi $30, ASM_THREAD_SIZE($30) @@ -88,7 +98,7 @@ __smp_callin: ldi $2, idle_task_pointer s8addl $0, $2, $2 ldl $8, 0($2) # Get ksp of idle thread - sys_call HMC_wrktp + SAVE_KTP ldl $30, TASK_STACK($8) ldi $30, ASM_THREAD_SIZE($30) diff --git a/arch/sw_64/kernel/hibernate_asm.S b/arch/sw_64/kernel/hibernate_asm.S index ff997cd76c5a..af0372879bb9 100644 --- a/arch/sw_64/kernel/hibernate_asm.S +++ b/arch/sw_64/kernel/hibernate_asm.S @@ -4,6 +4,13 @@ #include #include + .macro SAVE_KTP +#ifdef CONFIG_SUBARCH_C3B + sys_call HMC_wrktp +#else + csrw $8, CSR_KTP + .endm +#endif .text .set noat ENTRY(swsusp_arch_suspend) @@ -114,8 +121,7 @@ $hibernate_setfpec_over: ldl sp, PSTATE_SP($16) ldl $8, PSTATE_KTP($16) - sys_call HMC_wrktp - + SAVE_KTP ldi $0, 0($31) ret diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 82222ed63f0a..9fda2f95d0cb 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -69,10 +69,6 @@ void smp_callin(void) } set_cpu_online(cpuid, true); - /* clear ksp, usp */ - wrksp(0); - wrusp(0); - /* Set trap vectors. */ trap_init(); diff --git a/arch/sw_64/kvm/Makefile b/arch/sw_64/kvm/Makefile index 8111014c5cca..8f594155f352 100644 --- a/arch/sw_64/kvm/Makefile +++ b/arch/sw_64/kvm/Makefile @@ -9,12 +9,8 @@ include $(srctree)/virt/kvm/Makefile.kvm obj-$(CONFIG_KVM) += kvm.o -kvm-y += sw64.o -kvm-y += entry.o -kvm-y += emulate.o -kvm-y += mmio.o -kvm-y += kvm_timer.o -kvm-y += handle_exit.o -kvm-y += perf.o -kvm-$(CONFIG_SUBARCH_C3B) += kvm_core3.o kvm_cma.o -kvm-$(CONFIG_SUBARCH_C4) += kvm_core4.o mmu.o +kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o \ + sw64.o emulate.o mmio.o kvm_timer.o handle_exit.o perf.o + +kvm-$(CONFIG_SUBARCH_C3B) += kvm_core3.o entry_core3.o +kvm-$(CONFIG_SUBARCH_C4) += kvm_core4.o mmu.o entry_core4.o diff --git a/arch/sw_64/kvm/entry.S b/arch/sw_64/kvm/entry_core3.S similarity index 100% rename from arch/sw_64/kvm/entry.S rename to arch/sw_64/kvm/entry_core3.S diff --git a/arch/sw_64/kvm/entry_core4.S b/arch/sw_64/kvm/entry_core4.S new file mode 100644 index 000000000000..fb6757425f76 --- /dev/null +++ b/arch/sw_64/kvm/entry_core4.S @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 WXIAT + * Author: Wang Yuanheng + */ + .text +#include +#include +#include +#include +#include + + .set noat + +/* + * r16: physical address of guest kvm_vcpu.arch.vcb + * r17: pointer to guest kvm_vcpu.arch.kvm_regs + * r18: pointer to hcall args + */ +ENTRY(__sw64_vcpu_run) + /* save host fpregs */ + rfpcr $f0 + fstd $f0, TASK_THREAD_FPCR($8) + vstd $f2, TASK_THREAD_F2($8) + vstd $f3, TASK_THREAD_F3($8) + vstd $f4, TASK_THREAD_F4($8) + vstd $f5, TASK_THREAD_F5($8) + vstd $f6, TASK_THREAD_F6($8) + vstd $f7, TASK_THREAD_F7($8) + vstd $f8, TASK_THREAD_F8($8) + vstd $f9, TASK_THREAD_F9($8) + + ldi sp, -VCPU_RET_SIZE(sp) + /* save host pt_regs to current kernel stack */ + ldi sp, -PT_REGS_SIZE(sp) + stl $9, PT_REGS_R9(sp) + stl $8, PT_REGS_R8(sp) + stl $10, PT_REGS_R10(sp) + stl $11, PT_REGS_R11(sp) + stl $12, PT_REGS_R12(sp) + stl $13, PT_REGS_R13(sp) + stl $14, PT_REGS_R14(sp) + stl $15, PT_REGS_R15(sp) + stl $26, PT_REGS_R26(sp) + stl $29, PT_REGS_GP(sp) + csrw $16, CSR_EARG0 + csrw $17, CSR_EARG1 + csrw $18, CSR_EARG2 + + /* restore guest switch stack from guest kvm_regs struct */ + ldl $0, KVM_REGS_R0($17) + ldl $1, KVM_REGS_R1($17) + /* restore $2 later */ + ldl $3, KVM_REGS_R3($17) + ldl $4, KVM_REGS_R4($17) + ldl $5, KVM_REGS_R5($17) + ldl $6, KVM_REGS_R6($17) + ldl $7, KVM_REGS_R7($17) + ldl $8, KVM_REGS_R8($17) + ldl $9, KVM_REGS_R9($17) + ldl $10, KVM_REGS_R10($17) + ldl $11, KVM_REGS_R11($17) + ldl $12, KVM_REGS_R12($17) + ldl $13, KVM_REGS_R13($17) + ldl $14, KVM_REGS_R14($17) + ldl $15, KVM_REGS_R15($17) + ldl $16, KVM_REGS_R16($17) + ldl $18, KVM_REGS_R18($17) + ldl $19, KVM_REGS_R19($17) + ldl $20, KVM_REGS_R20($17) + ldl $21, KVM_REGS_R21($17) + ldl $22, KVM_REGS_R22($17) + ldl $23, KVM_REGS_R23($17) + ldl $24, KVM_REGS_R24($17) + ldl $25, KVM_REGS_R25($17) + ldl $26, KVM_REGS_R26($17) + ldl $27, KVM_REGS_R27($17) + ldl $28, KVM_REGS_R28($17) + ldl $29, KVM_REGS_GP($17) + + fldd $f0, KVM_REGS_FPCR($17) + wfpcr $f0 + fimovd $f0, $2 + and $2, 0x3, $2 + beq $2, $g_setfpec_0 + subl $2, 0x1, $2 + beq $2, $g_setfpec_1 + subl $2, 0x1, $2 + beq $2, $g_setfpec_2 + setfpec3 + br $g_setfpec_over +$g_setfpec_0: + setfpec0 + br $g_setfpec_over +$g_setfpec_1: + setfpec1 + br $g_setfpec_over +$g_setfpec_2: + setfpec2 +$g_setfpec_over: + ldl $2, KVM_REGS_R2($17) + vldd $f0, KVM_REGS_F0($17) + vldd $f1, KVM_REGS_F1($17) + vldd $f2, KVM_REGS_F2($17) + vldd $f3, KVM_REGS_F3($17) + vldd $f4, KVM_REGS_F4($17) + vldd $f5, KVM_REGS_F5($17) + vldd $f6, KVM_REGS_F6($17) + vldd $f7, KVM_REGS_F7($17) + vldd $f8, KVM_REGS_F8($17) + vldd $f9, KVM_REGS_F9($17) + vldd $f10, KVM_REGS_F10($17) + vldd $f11, KVM_REGS_F11($17) + vldd $f12, KVM_REGS_F12($17) + vldd $f13, KVM_REGS_F13($17) + vldd $f14, KVM_REGS_F14($17) + vldd $f15, KVM_REGS_F15($17) + vldd $f16, KVM_REGS_F16($17) + vldd $f17, KVM_REGS_F17($17) + vldd $f18, KVM_REGS_F18($17) + vldd $f19, KVM_REGS_F19($17) + vldd $f20, KVM_REGS_F20($17) + vldd $f21, KVM_REGS_F21($17) + vldd $f22, KVM_REGS_F22($17) + vldd $f23, KVM_REGS_F23($17) + vldd $f24, KVM_REGS_F24($17) + vldd $f25, KVM_REGS_F25($17) + vldd $f26, KVM_REGS_F26($17) + vldd $f27, KVM_REGS_F27($17) + vldd $f28, KVM_REGS_F28($17) + vldd $f29, KVM_REGS_F29($17) + vldd $f30, KVM_REGS_F30($17) + csrw $17, CSR_BASE_KREGS + ldl $17, KVM_REGS_R17($17) + + /* enter guest now */ + sys_call 0x31 + /* exit guest now */ + csrw $17, CSR_SCRATCH + csrr $17, CSR_BASE_KREGS /* r17: base of kvm_regs */ + + vstd $f0, KVM_REGS_F0($17) + vstd $f1, KVM_REGS_F1($17) + vstd $f2, KVM_REGS_F2($17) + vstd $f3, KVM_REGS_F3($17) + vstd $f4, KVM_REGS_F4($17) + vstd $f5, KVM_REGS_F5($17) + vstd $f6, KVM_REGS_F6($17) + vstd $f7, KVM_REGS_F7($17) + vstd $f8, KVM_REGS_F8($17) + vstd $f9, KVM_REGS_F9($17) + vstd $f10, KVM_REGS_F10($17) + vstd $f11, KVM_REGS_F11($17) + vstd $f12, KVM_REGS_F12($17) + vstd $f13, KVM_REGS_F13($17) + vstd $f14, KVM_REGS_F14($17) + vstd $f15, KVM_REGS_F15($17) + vstd $f16, KVM_REGS_F16($17) + vstd $f17, KVM_REGS_F17($17) + vstd $f18, KVM_REGS_F18($17) + vstd $f19, KVM_REGS_F19($17) + vstd $f20, KVM_REGS_F20($17) + vstd $f21, KVM_REGS_F21($17) + vstd $f22, KVM_REGS_F22($17) + vstd $f23, KVM_REGS_F23($17) + vstd $f24, KVM_REGS_F24($17) + vstd $f25, KVM_REGS_F25($17) + vstd $f26, KVM_REGS_F26($17) + vstd $f27, KVM_REGS_F27($17) + vstd $f28, KVM_REGS_F28($17) + vstd $f29, KVM_REGS_F29($17) + vstd $f30, KVM_REGS_F30($17) + + rfpcr $f0 + fstd $f0, KVM_REGS_FPCR($17) + + /* don't save r0 Hmcode have saved r0 for us */ + stl $1, KVM_REGS_R1($17) + stl $2, KVM_REGS_R2($17) + stl $3, KVM_REGS_R3($17) + stl $4, KVM_REGS_R4($17) + stl $5, KVM_REGS_R5($17) + stl $6, KVM_REGS_R6($17) + stl $7, KVM_REGS_R7($17) + stl $8, KVM_REGS_R8($17) + stl $9, KVM_REGS_R9($17) + stl $10, KVM_REGS_R10($17) + stl $11, KVM_REGS_R11($17) + stl $12, KVM_REGS_R12($17) + stl $13, KVM_REGS_R13($17) + stl $14, KVM_REGS_R14($17) + stl $15, KVM_REGS_R15($17) + stl $19, KVM_REGS_R19($17) + stl $20, KVM_REGS_R20($17) + stl $21, KVM_REGS_R21($17) + stl $22, KVM_REGS_R22($17) + stl $23, KVM_REGS_R23($17) + stl $24, KVM_REGS_R24($17) + stl $25, KVM_REGS_R25($17) + stl $26, KVM_REGS_R26($17) + stl $27, KVM_REGS_R27($17) + stl $28, KVM_REGS_R28($17) + stl $29, KVM_REGS_GP($17) + stl $16, KVM_REGS_R16($17) + stl $18, KVM_REGS_R18($17) + imemb + csrr $9, CSR_SCRATCH + stl $9, KVM_REGS_R17($17) + + /* restore host regs from host sp */ + csrr sp, CSR_SP + ldl $8, PT_REGS_R8(sp) + ldl $9, PT_REGS_R9(sp) + ldl $10, PT_REGS_R10(sp) + ldl $11, PT_REGS_R11(sp) + ldl $12, PT_REGS_R12(sp) + ldl $13, PT_REGS_R13(sp) + ldl $14, PT_REGS_R14(sp) + ldl $15, PT_REGS_R15(sp) + ldl $26, PT_REGS_R26(sp) + ldl $29, PT_REGS_GP(sp) + csrr $2, CSR_PS + stl $2, PT_REGS_PS(sp) + ldi sp, PT_REGS_SIZE(sp) + + /* restore host fpregs */ + fldd $f0, TASK_THREAD_FPCR($8) + wfpcr $f0 + fimovd $f0, $2 + and $2, 0x3, $2 + beq $2, $setfpec_0 + subl $2, 0x1, $2 + beq $2, $setfpec_1 + subl $2, 0x1, $2 + beq $2, $setfpec_2 + setfpec3 + br $setfpec_over +$setfpec_0: + setfpec0 + br $setfpec_over +$setfpec_1: + setfpec1 + br $setfpec_over +$setfpec_2: + setfpec2 +$setfpec_over: + vldd $f2, TASK_THREAD_F2($8) + vldd $f3, TASK_THREAD_F3($8) + vldd $f4, TASK_THREAD_F4($8) + vldd $f5, TASK_THREAD_F5($8) + vldd $f6, TASK_THREAD_F6($8) + vldd $f7, TASK_THREAD_F7($8) + vldd $f8, TASK_THREAD_F8($8) + vldd $f9, TASK_THREAD_F9($8) + + /* if $0 > 0, handle hcall */ + bgt $0, $ret_to + + stl $26, VCPU_RET_RA(sp) + stl $0, VCPU_RET_R0(sp) + + /* Hmcode will setup in */ + /* restore $16 $17 $18, do interrupt trick */ + csrr $16, CSR_EARG0 + csrr $17, CSR_EARG1 + csrr $18, CSR_EARG2 + + ldi $19, -PT_REGS_SIZE(sp) + call $26, do_entInt + ldl $26, VCPU_RET_RA(sp) + ldl $0, VCPU_RET_R0(sp) +$ret_to: + /* ret($0) indicate hcall number */ + ldi sp, VCPU_RET_SIZE(sp) /* pop stack */ + ret -- Gitee From 72f2fa1073242e85925dfadacec189046b69dd89 Mon Sep 17 00:00:00 2001 From: Zhi Tongze Date: Thu, 21 Mar 2024 15:40:46 +0800 Subject: [PATCH 140/524] sw64: improve cpu bring up method It used to notify firmware through a shared flag that secondary cpu has been prepared, and firmware continuously read this flag to check whether secondary cpu is ready. This will continuously occupy cpu resources, resulting in aonther thread on the same physical core being unable to utilize processor resources. This patch makes it halt directly and wait for NMII to wake up. Signed-off-by: Zhi Tongze Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 9fda2f95d0cb..9cb7c64b3afe 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -21,7 +21,6 @@ struct smp_rcb_struct *smp_rcb; extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; -int smp_booted; void *idle_task_pointer[NR_CPUS]; @@ -304,10 +303,8 @@ int vt_cpu_up(unsigned int cpu, struct task_struct *tidle) wmb(); smp_rcb->ready = 0; - if (smp_booted) { - /* irq must be disabled before reset vCPU */ - reset_cpu(cpu); - } + /* irq must be disabled before reset vCPU */ + reset_cpu(cpu); smp_boot_one_cpu(cpu, tidle); return cpu_online(cpu) ? 0 : -EIO; @@ -328,23 +325,19 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) /* send wake up signal */ send_wakeup_interrupt(cpu); /* send reset signal */ - if (smp_booted) { - if (is_in_host()) { - reset_cpu(cpu); - } else { - while (1) - cpu_relax(); - } + if (is_in_host()) { + reset_cpu(cpu); + } else { + while (1) + cpu_relax(); } smp_boot_one_cpu(cpu, tidle); #ifdef CONFIG_SUBARCH_C3B if (static_branch_likely(&use_tc_as_sched_clock)) { - if (smp_booted) { - tc_sync_clear(); - smp_call_function_single(cpu, tc_sync_ready, NULL, 0); - tc_sync_set(); - } + tc_sync_clear(); + smp_call_function_single(cpu, tc_sync_ready, NULL, 0); + tc_sync_set(); } #endif @@ -353,7 +346,6 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) void __init smp_cpus_done(unsigned int max_cpus) { - smp_booted = 1; pr_info("SMP: Total of %d processors activated.\n", num_online_cpus()); } -- Gitee From 6fb0aed6c2de32f42c141acdeda1120ac6451fb6 Mon Sep 17 00:00:00 2001 From: Xu Chenjiao Date: Tue, 26 Mar 2024 10:58:55 +0000 Subject: [PATCH 141/524] sw64: lpc: work around hardware flaws For lpc host of C4: - Legacy IO address can not be accessed normally, and we borrow memory IO address to access legacy IO. - It does not support interrupt, so the IPMI driver have to use poll mode instead of interrupt mode. Force the kipmi daemon to be enabled with "ipmi_si.force_kipmid=1" in the kernel command line. Signed-off-by: Xu Chenjiao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 - drivers/mfd/Kconfig | 8 ++++---- drivers/mfd/Makefile | 2 +- drivers/mfd/{lpc_sunway_chip3.c => lpc_sunway.c} | 0 drivers/mfd/sunway_ast2400.c | 4 ++-- 5 files changed, 7 insertions(+), 8 deletions(-) rename drivers/mfd/{lpc_sunway_chip3.c => lpc_sunway.c} (100%) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 02e4c672c8b8..79be887a5a0c 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -234,7 +234,6 @@ config PLATFORM_XUELANG depends on UNCORE_XUELANG select SPARSE_IRQ select SYS_HAS_EARLY_PRINTK - select SW64_INTC_V2 select I2C_SUNWAY if I2C help Sunway board chipset for C3B diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 6b653487d954..96633e8d4a9c 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -604,12 +604,12 @@ config LPC_SCH LPC bridge function of the Intel SCH provides support for System Management Bus and General Purpose I/O. -config LPC_CHIP3 - tristate "CHIP3 LPC" - depends on UNCORE_XUELANG +config LPC_SUNWAY + tristate "SUNWAY LPC" + depends on SW64 select MFD_CORE help - LPC bridge function of the chip3 provides support for + LPC bridge function of the Sunway provides support for System Management Bus and General Purpose I/O. config SUNWAY_SUPERIO_AST2400 diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile index 50b42df268ea..95867f732d2e 100644 --- a/drivers/mfd/Makefile +++ b/drivers/mfd/Makefile @@ -269,7 +269,7 @@ obj-$(CONFIG_MFD_KHADAS_MCU) += khadas-mcu.o obj-$(CONFIG_MFD_ACER_A500_EC) += acer-ec-a500.o obj-$(CONFIG_MFD_QCOM_PM8008) += qcom-pm8008.o -obj-$(CONFIG_LPC_CHIP3) += lpc_sunway_chip3.o +obj-$(CONFIG_LPC_SUNWAY) += lpc_sunway.o obj-$(CONFIG_SUNWAY_SUPERIO_AST2400) += sunway_ast2400.o obj-$(CONFIG_SGI_MFD_IOC3) += ioc3.o diff --git a/drivers/mfd/lpc_sunway_chip3.c b/drivers/mfd/lpc_sunway.c similarity index 100% rename from drivers/mfd/lpc_sunway_chip3.c rename to drivers/mfd/lpc_sunway.c diff --git a/drivers/mfd/sunway_ast2400.c b/drivers/mfd/sunway_ast2400.c index fbea07813643..ea981d9d0215 100644 --- a/drivers/mfd/sunway_ast2400.c +++ b/drivers/mfd/sunway_ast2400.c @@ -79,7 +79,7 @@ static void superio_com1_init(struct pnp_device *device) pnp_set_logical_device(device); pnp_set_enable(device, 1); - pnp_write_config(device, 0x60, 0x3); + pnp_write_config(device, 0x60, 0x2); pnp_write_config(device, 0x61, 0xf8); pnp_write_config(device, 0x70, superio_uart0_irq); @@ -96,7 +96,7 @@ static void superio_com2_init(struct pnp_device *device) pnp_set_logical_device(device); pnp_set_enable(device, 1); - pnp_write_config(device, 0x60, 0x2); + pnp_write_config(device, 0x60, 0x3); pnp_write_config(device, 0x61, 0xf8); pnp_write_config(device, 0x70, superio_uart1_irq); -- Gitee From 7b9792e750427dcc6885f6a0fd8c2b80cd35de53 Mon Sep 17 00:00:00 2001 From: Xu Chenjiao Date: Tue, 26 Mar 2024 11:37:16 +0000 Subject: [PATCH 142/524] sw64: dts: add Junzhang Add Device Tree for Sunway Junzhang SoC. Signed-off-by: Xu Chenjiao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/junzhang.dts | 250 +++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 arch/sw_64/boot/dts/junzhang.dts diff --git a/arch/sw_64/boot/dts/junzhang.dts b/arch/sw_64/boot/dts/junzhang.dts new file mode 100644 index 000000000000..4e8cd655c798 --- /dev/null +++ b/arch/sw_64/boot/dts/junzhang.dts @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Default device tree; + */ + +/dts-v1/; +/ { + compatible = "sunway,junzhang"; + model = "junzhang"; + #address-cells = <2>; + #size-cells = <2>; + + chosen { + }; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + clocks { + i2cclk: i2cclk { + compatible = "fixed-clock"; + clock-frequency = <12500000>; + #clock-cells = <0>; + clock-output-names = "i2cclk_12.5mhz"; + }; + spiclk: spiclk { + compatible = "fixed-clock"; + clock-frequency = <12500000>; + #clock-cells = <0>; + clock-output-names = "spiclk_12.5mhz"; + }; + + }; + + pintc: interrupt-controller { + compatible = "sw64,pintc"; + interrupt-controller; + sw64,node = <0>; + sw64,irq-num = <8>; + sw64,ver = <2>; + reg = <0x803a 0x0 0x0 0x1900>, + <0x8030 0x0 0x0 0xe000>; + #interrupt-cells = <1>; + }; + + lpc_intc: interrupt-controller@0x8037 { + compatible = "sw64,lpc_intc"; + reg = <0x8037 0x40000000 0x0 0x8000>; + interrupt-controller; + sw64,node = <0>; + sw64,irq-num = <16>; + sw64,ver = <1>; + #interrupt-cells = <1>; + interrupt-parent = <&pintc>; + interrupts = <2>; + }; + + uart: serial0@8033 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sw6,sunway-apb-uart"; + reg = <0x8033 0x0 0x0 0x1000>; + interrupt-parent=<&pintc>; + interrupts = <3>; + reg-shift = <9>; + reg-io-width = <4>; + clock-frequency = <24000000>; + status = "okay"; + }; + + serial1@9033 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sw6,sunway-apb-uart"; + reg = <0x9033 0x0 0x0 0x1000>; + reg-shift = <9>; + reg-io-width = <4>; + clock-frequency = <24000000>; + status = "okay"; + }; + + + i2c0@0x8031 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "snps,designware-i2c"; + reg = <0x8031 0x0 0x0 0x8000>; + clock-frequency = <100000>; + clocks = <&i2cclk>; + interrupt-parent=<&pintc>; + interrupts = <5>; + status = "okay"; + }; + + i2c1@0x8034 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,designware-i2c"; + reg = <0x8034 0x0 0x0 0x8000>; + clock-frequency = <100000>; + clocks = <&i2cclk>; + interrupt-parent=<&pintc>; + interrupts = <6>; + status = "okay"; + }; + + i2c2@0x8035 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,designware-i2c"; + reg = <0x8035 0x0 0x0 0x8000>; + clock-frequency = <100000>; + clocks = <&i2cclk>; + interrupt-parent=<&pintc>; + interrupts = <7>; + status = "okay"; + + rtc: pcf8523@68 { + compatible = "nxp,pcf8523"; + reg = <0x68>; + }; + + lm75: at30tse752a@48 { + compatible = "microchip,tcn75"; + reg = <0x48>; + }; + }; + + pvt: pvt@0x8030 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sw64,pvt-vol"; + reg = <0x8030 0x0 0x0 0x7c00>; + status = "okay"; + }; + + spi: spi@0x8032 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "sunway,chip-spi"; + reg = <0x8032 0x0 0x0 0x8000>; + clocks = <&spiclk>; + poll_mode = <1>; /* poll_mode:1 interrupt mode: 0 */ + reg-io-width = <2>; + status = "okay"; + + flash@0 { + compatible = "winbond,w25q32dw", "jedec,spi-nor"; + spi-max-frequency = <25000000>; + m25p,fast-read; + reg = <0>; /* 0: flash chip selected bit */ + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "test"; + reg = <0 0x400000>; + }; + }; + }; + + flash@1 { + compatible = "winbond,w25q32dw", "jedec,spi-nor"; + spi-max-frequency = <25000000>; + m25p,fast-read; + reg = <1>; /* 1: flash chip selected bit */ + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "test"; + reg = <0 0x400000>; + }; + }; + }; + }; + + lpc: lpc@0x8037 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sunway,chip3_lpc"; + ranges; + reg = <0x8037 0x40000000 0x0 0x8000>; + status = "okay"; + }; + + ipmi-kcs@0x8037 { + #address-cells = <2>; + #size-cells = <2>; + device_type = "ipmi"; + compatible = "ipmi-kcs"; + reg = <0x8037 0x20ca2000 0x0 0x10>; + reg-size = <1>; + reg-spacing = <4096>; + reg-shift = <0>; + status = "disabled"; + }; + + ipmi-bt@0x8037 { + #address-cells = <2>; + #size-cells = <2>; + device_type = "ipmi"; + compatible = "ipmi-bt"; + reg = <0x8037 0x200e4000 0x0 0x10>; + reg-size = <1>; + reg-spacing = <4096>; + reg-shift = <0>; + status = "disabled"; + }; + + suart1: suart1@0x3f8 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "ns16550a"; + clock-frequency = <1843200>; + reg = <0x8037 0x203f8000 0x0 0x10>; + reg-shift = <12>; + reg-io-width = <1>; + status = "disabled"; + }; + + gpio: gpio@8036 { + #address-cells = <2>; + #size-cells = <2>; + compatible = "snps,sw-gpio"; + reg = <0x8036 0x0 0x0 0x8000>; + status = "okay"; + + porta: gpio-contraller@0 { + compatible = "snps,dw-apb-gpio-port"; + gpio-controller; + #gpio-cells = <2>; + snps,nr-gpios = <8>; + reg = <0 0 0 0>; + interrupt-controller; + #interrupt-cells = <2>; + interrupts-extended = <&pintc 0>, <&pintc 1>; + }; + }; + }; +}; -- Gitee From 6d02ca9e3afac213422be2da521bb95033fe460b Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 22 Mar 2024 11:26:29 +0800 Subject: [PATCH 143/524] sw64: fix syscall_fallback() for vdso gettimeofday Get $0 and $19 from inline assembly output to avoid compiler clobbering these two registers before we use them. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/vdso/vgettimeofday.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/arch/sw_64/kernel/vdso/vgettimeofday.c b/arch/sw_64/kernel/vdso/vgettimeofday.c index 07e023d3d64b..69dcab66cb10 100644 --- a/arch/sw_64/kernel/vdso/vgettimeofday.c +++ b/arch/sw_64/kernel/vdso/vgettimeofday.c @@ -21,19 +21,22 @@ static __always_inline int syscall_fallback(clockid_t clkid, struct timespec64 *ts) { - register int r0 asm("$0"); - register unsigned long r19 asm("$19"); + long retval; + long error; asm volatile( - " mov %0, $16\n" - " mov %1, $17\n" - " ldi $0, %2\n" - " sys_call %3\n" - :: "r"(clkid), "r"(ts), "i"(__NR_clock_gettime), "i"(HMC_callsys) + " mov %2, $16\n" + " mov %3, $17\n" + " ldi $0, %4\n" + " sys_call %5\n" + " mov $0, %0\n" + " mov $19, %1" + : "=r"(retval), "=r"(error) + : "r"(clkid), "r"(ts), "i"(__NR_clock_gettime), "i"(HMC_callsys) : "$0", "$16", "$17", "$19"); - if (unlikely(r19)) - return -r0; + if (unlikely(error)) + return -retval; else - return r0; + return retval; } static __always_inline int do_realtime_coarse(struct timespec64 *ts, -- Gitee From c9de19c358c87656c97f526bbf7683bf86842f3f Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 26 Mar 2024 16:40:06 +0800 Subject: [PATCH 144/524] sw64: enable CONFIG_FRAME_POINTER by default for junzhang The new compiler for junzhang platform has fixed stackframe issue, so enable CONFIG_FRAME_POINTER by default. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/configs/junzhang_defconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/configs/junzhang_defconfig b/arch/sw_64/configs/junzhang_defconfig index 0459d08fd40d..9baf68f8e0ac 100644 --- a/arch/sw_64/configs/junzhang_defconfig +++ b/arch/sw_64/configs/junzhang_defconfig @@ -662,6 +662,6 @@ CONFIG_CRYPTO_DEFLATE=y CONFIG_CRYPTO_LZO=y # CONFIG_CRYPTO_HW is not set CONFIG_CONSOLE_LOGLEVEL_QUIET=7 -# CONFIG_FRAME_POINTER is not set +# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_SCHEDSTATS=y # CONFIG_RCU_TRACE is not set -- Gitee From 9f21ee905ba265cbbdc9ebe288a5821ea64eadb3 Mon Sep 17 00:00:00 2001 From: Zhi Tongze Date: Tue, 26 Mar 2024 17:01:18 +0800 Subject: [PATCH 145/524] sw64: employ trick on regs.pc for specific exception In order to ensure compatibility with firmware, this patch implements special handing on regs.pc for specific exception entries. Signed-off-by: Zhi Tongze Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry_c4.S | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S index 1209e627a7f1..097f4279e69b 100644 --- a/arch/sw_64/kernel/entry_c4.S +++ b/arch/sw_64/kernel/entry_c4.S @@ -153,6 +153,9 @@ entInt: .ent entArith entArith: SAVE_ALL + ldl $1, PT_REGS_PC($sp) + ldi $1, 4($1) + stl $1, PT_REGS_PC($sp) br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ @@ -179,6 +182,9 @@ entMM: .ent entIF entIF: SAVE_ALL + ldl $1, PT_REGS_PC($sp) + ldi $1, 4($1) + stl $1, PT_REGS_PC($sp) br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ @@ -198,6 +204,9 @@ entIF: .ent entUna entUna: SAVE_ALL + ldl $1, PT_REGS_PC($sp) + ldi $1, 4($1) + stl $1, PT_REGS_PC($sp) br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ @@ -230,6 +239,9 @@ entUna: .ent entSys entSys: SAVE_ALL + ldl $1, PT_REGS_PC($sp) + ldi $1, 4($1) + stl $1, PT_REGS_PC($sp) br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ -- Gitee From 348896b1f694877dfbf59d0a26c93ead977cb6e3 Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Mon, 25 Mar 2024 15:33:21 +0800 Subject: [PATCH 146/524] sw64: work around suspend for C4 This patch introduces a light weight sleep mechanism to work around suspend for Core4. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/sw64_init.h | 7 ++++ arch/sw_64/include/asm/uncore_io_junzhang.h | 3 ++ arch/sw_64/kernel/smp.c | 40 +++++++++++++++++++++ arch/sw_64/kernel/suspend.c | 10 ++++++ drivers/irqchip/irq-sunway-cpu.c | 16 +++++++++ 5 files changed, 76 insertions(+) diff --git a/arch/sw_64/include/asm/sw64_init.h b/arch/sw_64/include/asm/sw64_init.h index 86ddd2cb65f8..546be1a35250 100644 --- a/arch/sw_64/include/asm/sw64_init.h +++ b/arch/sw_64/include/asm/sw64_init.h @@ -43,6 +43,13 @@ extern struct sw64_chip_ops *sw64_chip; extern struct sw64_chip_init_ops *sw64_chip_init; #ifdef CONFIG_PM extern struct syscore_ops io_syscore_ops; + +#define PME_CLEAR 0 +#define PME_WFW 1 /* wait for wake */ +#define PME_PENDING 2 + +extern int pme_state; + #endif DECLARE_PER_CPU(unsigned long, hard_node_id); diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index 37cfe1fd6807..c0f759bbe740 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -116,6 +116,9 @@ enum { SI_FAULT_INT_EN = SPBU_BASE | 0x3200UL, ADR_CTL = SPBU_BASE | 0x3600UL, MC_ONLINE = SPBU_BASE | 0x3780UL, + CLK_CTL = SPBU_BASE | 0x3b80UL, + CLU_LV2_SELH = SPBU_BASE | 0x3a00UL, + CLU_LV2_SELL = SPBU_BASE | 0x3b00UL, PIU_TOP0_CONFIG = SPBU_BASE | 0x4c80UL, PIU_TOP1_CONFIG = SPBU_BASE | 0x4d00UL, SOFT_INFO0 = SPBU_BASE | 0xa000UL, diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 9cb7c64b3afe..cdb691692125 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -60,12 +60,26 @@ void smp_callin(void) { int cpuid = smp_processor_id(); +#ifdef CONFIG_SUBARCH_C4 + /* LV2 select PLL1 */ + int i, cpu_num; + + cpu_num = sw64_chip->get_cpu_num(); + + for (i = 0; i < cpu_num; i++) { + sw64_io_write(i, CLU_LV2_SELH, -1UL); + sw64_io_write(i, CLU_LV2_SELL, -1UL); + udelay(1000); + } +#endif + local_irq_disable(); if (cpu_online(cpuid)) { pr_err("??, cpu 0x%x already present??\n", cpuid); BUG(); } + set_cpu_online(cpuid, true); /* Set trap vectors. */ @@ -322,8 +336,10 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) wmb(); smp_rcb->ready = 0; +#ifdef CONFIG_SUBARCH_C3B /* send wake up signal */ send_wakeup_interrupt(cpu); +#endif /* send reset signal */ if (is_in_host()) { reset_cpu(cpu); @@ -602,6 +618,22 @@ void __cpu_die(unsigned int cpu) void arch_cpu_idle_dead(void) { +#ifdef CONFIG_SUBARCH_C4 + /* LV2 select PLL0 */ + int cpuid = smp_processor_id(); + int core_id = rcid_to_core_id(cpu_to_rcid(cpuid)); + int node_id = rcid_to_domain_id(cpu_to_rcid(cpuid)); + unsigned long value; + + if (core_id > 31) { + value = 1UL << (2 * (core_id - 32)); + sw64_io_write(node_id, CLU_LV2_SELH, value); + } else { + value = 1UL << (2 * core_id); + sw64_io_write(node_id, CLU_LV2_SELL, value); + } +#endif + idle_task_exit(); mb(); __this_cpu_write(cpu_state, CPU_DEAD); @@ -616,10 +648,18 @@ void arch_cpu_idle_dead(void) } #ifdef CONFIG_SUSPEND + +#ifdef CONFIG_SUBARCH_C3B sleepen(); send_sleep_interrupt(smp_processor_id()); while (1) asm("nop"); +#else + asm volatile("halt"); + while (1) + asm("nop"); +#endif + #else asm volatile("memb"); asm volatile("halt"); diff --git a/arch/sw_64/kernel/suspend.c b/arch/sw_64/kernel/suspend.c index 27a240e66149..97e5478a678f 100644 --- a/arch/sw_64/kernel/suspend.c +++ b/arch/sw_64/kernel/suspend.c @@ -4,6 +4,8 @@ #include #include +#define PME_EN 0x2 + struct processor_state suspend_state; static int native_suspend_state_valid(suspend_state_t pm_state) @@ -36,7 +38,15 @@ void sw64_suspend_enter(void) disable_local_timer(); current_thread_info()->pcb.tp = rtid(); +#ifdef CONFIG_SUBARCH_C3B sw64_suspend_deep_sleep(&suspend_state); +#else + pme_state = PME_WFW; + sw64_write_csr_imb(PME_EN, CSR_INT_EN); + asm("halt"); + local_irq_disable(); +#endif + wrtp(current_thread_info()->pcb.tp); disable_local_timer(); diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 3f5151b9b31d..2f4880036a2c 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -153,12 +153,28 @@ static void handle_dev_int(struct pt_regs *regs) sw64_io_write(node, DEV_INT_CONFIG, config_val); } +int pme_state; + asmlinkage void do_entInt(unsigned long type, unsigned long vector, unsigned long irq_arg, struct pt_regs *regs) { struct pt_regs *old_regs; extern char __idle_start[], __idle_end[]; +#ifdef CONFIG_SUBARCH_C4 + if (pme_state == PME_WFW) { + pme_state = PME_PENDING; + return; + } + + if (pme_state == PME_PENDING) { + old_regs = set_irq_regs(regs); + handle_device_interrupt(vector); + set_irq_regs(old_regs); + pme_state = PME_CLEAR; + } +#endif + if (is_guest_or_emul()) { if ((type & 0xffff) > 15) { vector = type; -- Gitee From a54abbdef8825b6b5b0c1d102572418b6a1b2d7b Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Mar 2024 17:02:47 +0800 Subject: [PATCH 147/524] sw64: store topology and numa info by each cpu itself Let the boot cpu calculate and store the topology and numa info of a non-boot cpu after it is booted could cause some data issue. Now the non-boot cpu will calculate and store its own topology and numa info during initialization process. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index cdb691692125..f3ac8a899976 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -102,6 +102,8 @@ void smp_callin(void) per_cpu(cpu_state, cpuid) = CPU_ONLINE; per_cpu(hard_node_id, cpuid) = rcid_to_domain_id(cpu_to_rcid(cpuid)); + store_cpu_topology(cpuid); + numa_add_cpu(cpuid); /* Must have completely accurate bogos. */ local_irq_enable(); @@ -149,8 +151,6 @@ static int secondary_cpu_start(int cpuid, struct task_struct *idle) return -1; started: - store_cpu_topology(cpuid); - numa_add_cpu(cpuid); return 0; } -- Gitee From 421298679c32de8c974b69de099ca2d6ef77175d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Mar 2024 20:45:28 +0800 Subject: [PATCH 148/524] sw64: choose lib at compile time Compile both SISD and SIMD version of memcpy/memset like libs and let the compiler optimize out the unused version based on kernel config. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/lib/Makefile | 12 +++------ arch/sw_64/lib/clear_user.S | 10 +++----- arch/sw_64/lib/copy_user.S | 10 +++----- arch/sw_64/lib/memcpy.S | 15 ++++------- arch/sw_64/lib/memset.S | 50 +++---------------------------------- arch/sw_64/lib/string.c | 8 ++++++ arch/sw_64/lib/uaccess.c | 8 ++++++ 7 files changed, 37 insertions(+), 76 deletions(-) diff --git a/arch/sw_64/lib/Makefile b/arch/sw_64/lib/Makefile index 2028d61dd195..6e428f2dcd05 100644 --- a/arch/sw_64/lib/Makefile +++ b/arch/sw_64/lib/Makefile @@ -22,20 +22,16 @@ lib-y = __divlu.o __remlu.o __divwu.o __remwu.o \ lib-clear_page-y := clear_page.o lib-clear_page-$(CONFIG_DEEP_CLEAR_PAGE) := deep-clear_page.o -lib-clear_user-y := clear_user.o -lib-clear_user-$(CONFIG_DEEP_CLEAR_USER) := deep-clear_user-hw_una.o deep-clear_user-sw_una.o +lib-clear_user-y := clear_user.o deep-clear_user-hw_una.o deep-clear_user-sw_una.o lib-copy_page-y := copy_page.o lib-copy_page-$(CONFIG_DEEP_COPY_PAGE) := deep-copy_page.o -lib-copy_user-y := copy_user.o -lib-copy_user-$(CONFIG_DEEP_COPY_USER) := deep-copy_user-hw_una.o deep-copy_user-sw_una.o +lib-copy_user-y := copy_user.o deep-copy_user-hw_una.o deep-copy_user-sw_una.o -lib-memcpy-y := memcpy.o -lib-memcpy-$(CONFIG_DEEP_MEMCPY) := deep-memcpy-hw_una.o deep-memcpy-sw_una.o +lib-memcpy-y := memcpy.o deep-memcpy-hw_una.o deep-memcpy-sw_una.o -lib-memset-y := memset.o -lib-memset-$(CONFIG_DEEP_MEMSET) := deep-memset-hw_una.o deep-memset-sw_una.o +lib-memset-y := memset.o deep-memset-hw_una.o deep-memset-sw_una.o lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o diff --git a/arch/sw_64/lib/clear_user.S b/arch/sw_64/lib/clear_user.S index 5ac77fc8ca0d..e006e0e6ceab 100644 --- a/arch/sw_64/lib/clear_user.S +++ b/arch/sw_64/lib/clear_user.S @@ -10,7 +10,6 @@ * stuff. * */ -#include /* Allow an exception for an insn; exit if we get one. */ #define EX(x,y...) \ 99: x,##y; \ @@ -23,11 +22,11 @@ .set noreorder .align 4 - .globl __clear_user - .ent __clear_user + .globl ____clear_user_sisd + .ent ____clear_user_sisd .frame $30, 0, $26 .prologue 0 -__clear_user: +____clear_user_sisd: and $17, $17, $0 and $16, 7, $4 beq $0, $zerolength @@ -98,5 +97,4 @@ $tail: clr $0 ret $31, ($26), 1 - .end __clear_user - EXPORT_SYMBOL(__clear_user) + .end ____clear_user_sisd diff --git a/arch/sw_64/lib/copy_user.S b/arch/sw_64/lib/copy_user.S index 2c3dd0b5656c..ac1336641b9f 100644 --- a/arch/sw_64/lib/copy_user.S +++ b/arch/sw_64/lib/copy_user.S @@ -9,7 +9,6 @@ * only _after_ a successful copy). There is also some rather minor * exception setup stuff.. */ -#include /* Allow an exception for an insn; exit if we get one. */ #define EXI(x,y...) \ 99: x,##y; \ @@ -27,9 +26,9 @@ .set noat .align 4 - .globl __copy_user - .ent __copy_user -__copy_user: + .globl ____copy_user_sisd + .ent ____copy_user_sisd +____copy_user_sisd: .prologue 0 and $18, $18, $0 and $16, 7, $3 @@ -102,5 +101,4 @@ $exitin: $exitout: ret $31, ($26), 1 - .end __copy_user - EXPORT_SYMBOL(__copy_user) + .end ____copy_user_sisd diff --git a/arch/sw_64/lib/memcpy.S b/arch/sw_64/lib/memcpy.S index 31c422b393ee..4a29ab6a6eab 100644 --- a/arch/sw_64/lib/memcpy.S +++ b/arch/sw_64/lib/memcpy.S @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Reasonably optimized memcpy() routine for the sw64 + * Reasonably optimized ____memcpy_sisd() routine for the sw64 * * - memory accessed as aligned quadwords only * - uses bcmpge to compare 8 bytes in parallel @@ -8,14 +8,13 @@ * Temp usage notes: * $1, $2, - scratch */ -#include .set noreorder .set noat .align 4 - .globl memcpy - .ent memcpy -memcpy: + .globl ____memcpy_sisd + .ent ____memcpy_sisd +____memcpy_sisd: .frame $30, 0, $26, 0 .prologue 0 @@ -194,8 +193,4 @@ $misalign_byte: $nomoredata: ret $31, ($26), 1 - .end memcpy - EXPORT_SYMBOL(memcpy) -/* For backwards module compatibility. */ -__memcpy = memcpy -.globl __memcpy + .end ____memcpy_sisd diff --git a/arch/sw_64/lib/memset.S b/arch/sw_64/lib/memset.S index dbc4d775c7ea..e5180adb4389 100644 --- a/arch/sw_64/lib/memset.S +++ b/arch/sw_64/lib/memset.S @@ -13,35 +13,16 @@ * hand, so they might well be incorrect, please do tell me about it..) */ -#include - .set noat .set noreorder .text - .globl memset - .globl __memset - .globl ___memset - .globl __memsetw - .globl __constant_c_memset + .globl ____constant_c_memset_sisd - .ent ___memset -.align 5 -___memset: + .ent ____constant_c_memset_sisd .frame $30, 0, $26, 0 .prologue 0 - - and $17, 255, $1 - inslb $17, 1, $17 - bis $17, $1, $17 - sll $17, 16, $1 - - bis $17, $1, $17 - sll $17, 32, $1 - bis $17, $1, $17 - ldl_u $31, 0($30) - .align 5 -__constant_c_memset: +____constant_c_memset_sisd: addl $18, $16, $6 bis $16, $16, $0 xor $16, $6, $1 @@ -127,27 +108,4 @@ within_one_quad_loop: end: ret $31, ($26), 1 - .end ___memset - EXPORT_SYMBOL(___memset) - - .align 5 - .ent __memsetw -__memsetw: - .prologue 0 - - inslh $17, 0, $1 - inslh $17, 2, $2 - inslh $17, 4, $3 - or $1, $2, $1 - inslh $17, 6, $4 - or $1, $3, $1 - or $1, $4, $17 - br __constant_c_memset - - .end __memsetw - EXPORT_SYMBOL(__memsetw) - -memset = ___memset -EXPORT_SYMBOL(memset) -__memset = ___memset -EXPORT_SYMBOL(__memset) + .end ____constant_c_memset_sisd diff --git a/arch/sw_64/lib/string.c b/arch/sw_64/lib/string.c index 4d10e0639bb8..2adca19a0fb3 100644 --- a/arch/sw_64/lib/string.c +++ b/arch/sw_64/lib/string.c @@ -4,11 +4,15 @@ #include +extern void *____memcpy_sisd(void *dest, const void *src, size_t n); extern void *____memcpy_hw_una(void *dest, const void *src, size_t n); extern void *____memcpy_sw_una(void *dest, const void *src, size_t n); static inline void *____memcpy(void *dest, const void *src, size_t n) { + if (!IS_ENABLED(CONFIG_DEEP_MEMCPY)) + return ____memcpy_sisd(dest, src, n); + if (static_branch_likely(&core_hw_una_enabled)) return ____memcpy_hw_una(dest, src, n); else @@ -28,11 +32,15 @@ void *__memcpy(void *dest, const void *src, size_t n) } EXPORT_SYMBOL(__memcpy); +extern void *____constant_c_memset_sisd(void *s, unsigned long c, size_t n); extern void *____constant_c_memset_hw_una(void *s, unsigned long c, size_t n); extern void *____constant_c_memset_sw_una(void *s, unsigned long c, size_t n); static inline void *____constant_c_memset(void *s, unsigned long c, size_t n) { + if (!IS_ENABLED(CONFIG_DEEP_MEMSET)) + return ____constant_c_memset_sisd(s, c, n); + if (static_branch_likely(&core_hw_una_enabled)) return ____constant_c_memset_hw_una(s, c, n); else diff --git a/arch/sw_64/lib/uaccess.c b/arch/sw_64/lib/uaccess.c index 56d3075aedbb..72bd66f5b48a 100644 --- a/arch/sw_64/lib/uaccess.c +++ b/arch/sw_64/lib/uaccess.c @@ -4,11 +4,15 @@ #include +extern long ____copy_user_sisd(void *to, const void *from, long len); extern long ____copy_user_hw_una(void *to, const void *from, long len); extern long ____copy_user_sw_una(void *to, const void *from, long len); long __copy_user(void *to, const void *from, long len) { + if (!IS_ENABLED(CONFIG_DEEP_COPY_USER)) + return ____copy_user_sisd(to, from, len); + if (static_branch_likely(&core_hw_una_enabled)) return ____copy_user_hw_una(to, from, len); else @@ -16,11 +20,15 @@ long __copy_user(void *to, const void *from, long len) } EXPORT_SYMBOL(__copy_user); +extern long ____clear_user_sisd(void __user *to, long len); extern long ____clear_user_hw_una(void __user *to, long len); extern long ____clear_user_sw_una(void __user *to, long len); long __clear_user(void __user *to, long len) { + if (!IS_ENABLED(CONFIG_DEEP_CLEAR_USER)) + return ____clear_user_sisd(to, len); + if (static_branch_likely(&core_hw_una_enabled)) return ____clear_user_hw_una(to, len); else -- Gitee From 9339c2af170234603947e3415d861904b87dfe0a Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Mar 2024 21:07:16 +0800 Subject: [PATCH 149/524] sw64: rename copy and set lib files Rename the memcpy/memset like lib files to help understand their differences. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/lib/Makefile | 8 ++++---- ...ear_user-sw_una.S => deep-clear_user-force_ali_addr.S} | 2 +- ...clear_user-hw_una.S => deep-clear_user-no_una_check.S} | 2 +- ...opy_template.S => deep-copy_template-force_ali_addr.S} | 0 ...py_template_c4.S => deep-copy_template-no_una_check.S} | 0 ...copy_user-sw_una.S => deep-copy_user-force_ali_addr.S} | 2 +- ...p-copy_user-hw_una.S => deep-copy_user-no_una_check.S} | 2 +- ...{deep-memcpy-sw_una.S => deep-memcpy-force_ali_addr.S} | 2 +- .../{deep-memcpy-hw_una.S => deep-memcpy-no_una_check.S} | 2 +- ...{deep-memset-sw_una.S => deep-memset-force_ali_addr.S} | 2 +- .../{deep-memset-hw_una.S => deep-memset-no_una_check.S} | 2 +- ...-set_template.S => deep-set_template-force_ali_addr.S} | 0 ...set_template_c4.S => deep-set_template-no_una_check.S} | 0 13 files changed, 12 insertions(+), 12 deletions(-) rename arch/sw_64/lib/{deep-clear_user-sw_una.S => deep-clear_user-force_ali_addr.S} (95%) rename arch/sw_64/lib/{deep-clear_user-hw_una.S => deep-clear_user-no_una_check.S} (95%) rename arch/sw_64/lib/{deep-copy_template.S => deep-copy_template-force_ali_addr.S} (100%) rename arch/sw_64/lib/{deep-copy_template_c4.S => deep-copy_template-no_una_check.S} (100%) rename arch/sw_64/lib/{deep-copy_user-sw_una.S => deep-copy_user-force_ali_addr.S} (94%) rename arch/sw_64/lib/{deep-copy_user-hw_una.S => deep-copy_user-no_una_check.S} (94%) rename arch/sw_64/lib/{deep-memcpy-sw_una.S => deep-memcpy-force_ali_addr.S} (82%) rename arch/sw_64/lib/{deep-memcpy-hw_una.S => deep-memcpy-no_una_check.S} (83%) rename arch/sw_64/lib/{deep-memset-sw_una.S => deep-memset-force_ali_addr.S} (95%) rename arch/sw_64/lib/{deep-memset-hw_una.S => deep-memset-no_una_check.S} (95%) rename arch/sw_64/lib/{deep-set_template.S => deep-set_template-force_ali_addr.S} (100%) rename arch/sw_64/lib/{deep-set_template_c4.S => deep-set_template-no_una_check.S} (100%) diff --git a/arch/sw_64/lib/Makefile b/arch/sw_64/lib/Makefile index 6e428f2dcd05..8d3bc1fdd604 100644 --- a/arch/sw_64/lib/Makefile +++ b/arch/sw_64/lib/Makefile @@ -22,16 +22,16 @@ lib-y = __divlu.o __remlu.o __divwu.o __remwu.o \ lib-clear_page-y := clear_page.o lib-clear_page-$(CONFIG_DEEP_CLEAR_PAGE) := deep-clear_page.o -lib-clear_user-y := clear_user.o deep-clear_user-hw_una.o deep-clear_user-sw_una.o +lib-clear_user-y := clear_user.o deep-clear_user-no_una_check.o deep-clear_user-force_ali_addr.o lib-copy_page-y := copy_page.o lib-copy_page-$(CONFIG_DEEP_COPY_PAGE) := deep-copy_page.o -lib-copy_user-y := copy_user.o deep-copy_user-hw_una.o deep-copy_user-sw_una.o +lib-copy_user-y := copy_user.o deep-copy_user-no_una_check.o deep-copy_user-force_ali_addr.o -lib-memcpy-y := memcpy.o deep-memcpy-hw_una.o deep-memcpy-sw_una.o +lib-memcpy-y := memcpy.o deep-memcpy-no_una_check.o deep-memcpy-force_ali_addr.o -lib-memset-y := memset.o deep-memset-hw_una.o deep-memset-sw_una.o +lib-memset-y := memset.o deep-memset-no_una_check.o deep-memset-force_ali_addr.o lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o diff --git a/arch/sw_64/lib/deep-clear_user-sw_una.S b/arch/sw_64/lib/deep-clear_user-force_ali_addr.S similarity index 95% rename from arch/sw_64/lib/deep-clear_user-sw_una.S rename to arch/sw_64/lib/deep-clear_user-force_ali_addr.S index e48e5be9efac..adca61aede83 100644 --- a/arch/sw_64/lib/deep-clear_user-sw_una.S +++ b/arch/sw_64/lib/deep-clear_user-force_ali_addr.S @@ -34,7 +34,7 @@ ____clear_user_sw_una: bis $31, $31, $7 mov $17, $18 bis $31, $31, $17 -#include "deep-set_template.S" +#include "deep-set_template-force_ali_addr.S" 255: bis $31, $18, $0 beq $7, $return diff --git a/arch/sw_64/lib/deep-clear_user-hw_una.S b/arch/sw_64/lib/deep-clear_user-no_una_check.S similarity index 95% rename from arch/sw_64/lib/deep-clear_user-hw_una.S rename to arch/sw_64/lib/deep-clear_user-no_una_check.S index 81c04dd0f056..bbc84c0776aa 100644 --- a/arch/sw_64/lib/deep-clear_user-hw_una.S +++ b/arch/sw_64/lib/deep-clear_user-no_una_check.S @@ -34,7 +34,7 @@ ____clear_user_hw_una: bis $31, $31, $7 mov $17, $18 bis $31, $31, $17 -#include "deep-set_template_c4.S" +#include "deep-set_template-no_una_check.S" 255: bis $31, $18, $0 beq $7, $return diff --git a/arch/sw_64/lib/deep-copy_template.S b/arch/sw_64/lib/deep-copy_template-force_ali_addr.S similarity index 100% rename from arch/sw_64/lib/deep-copy_template.S rename to arch/sw_64/lib/deep-copy_template-force_ali_addr.S diff --git a/arch/sw_64/lib/deep-copy_template_c4.S b/arch/sw_64/lib/deep-copy_template-no_una_check.S similarity index 100% rename from arch/sw_64/lib/deep-copy_template_c4.S rename to arch/sw_64/lib/deep-copy_template-no_una_check.S diff --git a/arch/sw_64/lib/deep-copy_user-sw_una.S b/arch/sw_64/lib/deep-copy_user-force_ali_addr.S similarity index 94% rename from arch/sw_64/lib/deep-copy_user-sw_una.S rename to arch/sw_64/lib/deep-copy_user-force_ali_addr.S index a359aee65583..f3c84c93d71b 100644 --- a/arch/sw_64/lib/deep-copy_user-sw_una.S +++ b/arch/sw_64/lib/deep-copy_user-force_ali_addr.S @@ -25,7 +25,7 @@ ____copy_user_sw_una: .prologue 0 .set noreorder bis $31, $31, $7 -#include "deep-copy_template.S" +#include "deep-copy_template-force_ali_addr.S" 255: bis $31, $18, $0 beq $7, $return diff --git a/arch/sw_64/lib/deep-copy_user-hw_una.S b/arch/sw_64/lib/deep-copy_user-no_una_check.S similarity index 94% rename from arch/sw_64/lib/deep-copy_user-hw_una.S rename to arch/sw_64/lib/deep-copy_user-no_una_check.S index b411a7ed52c2..c913d85cb469 100644 --- a/arch/sw_64/lib/deep-copy_user-hw_una.S +++ b/arch/sw_64/lib/deep-copy_user-no_una_check.S @@ -25,7 +25,7 @@ ____copy_user_hw_una: .prologue 0 .set noreorder bis $31, $31, $7 -#include "deep-copy_template_c4.S" +#include "deep-copy_template-no_una_check.S" 255: bis $31, $18, $0 beq $7, $return diff --git a/arch/sw_64/lib/deep-memcpy-sw_una.S b/arch/sw_64/lib/deep-memcpy-force_ali_addr.S similarity index 82% rename from arch/sw_64/lib/deep-memcpy-sw_una.S rename to arch/sw_64/lib/deep-memcpy-force_ali_addr.S index 75aa10a42b52..c5629b3b4e06 100644 --- a/arch/sw_64/lib/deep-memcpy-sw_una.S +++ b/arch/sw_64/lib/deep-memcpy-force_ali_addr.S @@ -9,7 +9,7 @@ ____memcpy_sw_una: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 -#include "deep-copy_template.S" +#include "deep-copy_template-force_ali_addr.S" 255: ret .end ____memcpy_sw_una diff --git a/arch/sw_64/lib/deep-memcpy-hw_una.S b/arch/sw_64/lib/deep-memcpy-no_una_check.S similarity index 83% rename from arch/sw_64/lib/deep-memcpy-hw_una.S rename to arch/sw_64/lib/deep-memcpy-no_una_check.S index ccdfaed4b8ab..da61247c7eb3 100644 --- a/arch/sw_64/lib/deep-memcpy-hw_una.S +++ b/arch/sw_64/lib/deep-memcpy-no_una_check.S @@ -9,7 +9,7 @@ ____memcpy_hw_una: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 -#include "deep-copy_template_c4.S" +#include "deep-copy_template-no_una_check.S" 255: ret .end ____memcpy_hw_una diff --git a/arch/sw_64/lib/deep-memset-sw_una.S b/arch/sw_64/lib/deep-memset-force_ali_addr.S similarity index 95% rename from arch/sw_64/lib/deep-memset-sw_una.S rename to arch/sw_64/lib/deep-memset-force_ali_addr.S index b354b36dbbc9..61e3ffdd0e5b 100644 --- a/arch/sw_64/lib/deep-memset-sw_una.S +++ b/arch/sw_64/lib/deep-memset-force_ali_addr.S @@ -45,7 +45,7 @@ ____constant_c_memset_sw_una: #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -#include "deep-set_template.S" +#include "deep-set_template-force_ali_addr.S" 255: #ifdef CONFIG_SUBARCH_C4 csrw $6, CSR_WR_FREGS diff --git a/arch/sw_64/lib/deep-memset-hw_una.S b/arch/sw_64/lib/deep-memset-no_una_check.S similarity index 95% rename from arch/sw_64/lib/deep-memset-hw_una.S rename to arch/sw_64/lib/deep-memset-no_una_check.S index dff6b2c2003c..aa43d7b7df08 100644 --- a/arch/sw_64/lib/deep-memset-hw_una.S +++ b/arch/sw_64/lib/deep-memset-no_una_check.S @@ -45,7 +45,7 @@ ____constant_c_memset_hw_una: #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -#include "deep-set_template_c4.S" +#include "deep-set_template-no_una_check.S" 255: #ifdef CONFIG_SUBARCH_C4 csrw $6, CSR_WR_FREGS diff --git a/arch/sw_64/lib/deep-set_template.S b/arch/sw_64/lib/deep-set_template-force_ali_addr.S similarity index 100% rename from arch/sw_64/lib/deep-set_template.S rename to arch/sw_64/lib/deep-set_template-force_ali_addr.S diff --git a/arch/sw_64/lib/deep-set_template_c4.S b/arch/sw_64/lib/deep-set_template-no_una_check.S similarity index 100% rename from arch/sw_64/lib/deep-set_template_c4.S rename to arch/sw_64/lib/deep-set_template-no_una_check.S -- Gitee From 63863aac035dcdbedd2956be0af21d6a67c9cb0c Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Mar 2024 21:16:10 +0800 Subject: [PATCH 150/524] sw64: rename copy and set lib functions Rename memcpy/memset like lib functions to help understand their differences. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/lib/deep-clear_user-force_ali_addr.S | 8 ++++---- arch/sw_64/lib/deep-clear_user-no_una_check.S | 8 ++++---- arch/sw_64/lib/deep-copy_user-force_ali_addr.S | 8 ++++---- arch/sw_64/lib/deep-copy_user-no_una_check.S | 8 ++++---- arch/sw_64/lib/deep-memcpy-force_ali_addr.S | 8 ++++---- arch/sw_64/lib/deep-memcpy-no_una_check.S | 8 ++++---- arch/sw_64/lib/deep-memset-force_ali_addr.S | 8 ++++---- arch/sw_64/lib/deep-memset-no_una_check.S | 8 ++++---- arch/sw_64/lib/string.c | 16 ++++++++-------- arch/sw_64/lib/uaccess.c | 16 ++++++++-------- 10 files changed, 48 insertions(+), 48 deletions(-) diff --git a/arch/sw_64/lib/deep-clear_user-force_ali_addr.S b/arch/sw_64/lib/deep-clear_user-force_ali_addr.S index adca61aede83..24b1f03b6a4d 100644 --- a/arch/sw_64/lib/deep-clear_user-force_ali_addr.S +++ b/arch/sw_64/lib/deep-clear_user-force_ali_addr.S @@ -27,9 +27,9 @@ * $18: bytes left to copy * */ - .globl ____clear_user_sw_una - .ent ____clear_user_sw_una -____clear_user_sw_una: + .globl ____clear_user_simd_force_ali_addr + .ent ____clear_user_simd_force_ali_addr +____clear_user_simd_force_ali_addr: .prologue 0 bis $31, $31, $7 mov $17, $18 @@ -44,4 +44,4 @@ $restore_simd: $return: ret - .end ____clear_user_sw_una + .end ____clear_user_simd_force_ali_addr diff --git a/arch/sw_64/lib/deep-clear_user-no_una_check.S b/arch/sw_64/lib/deep-clear_user-no_una_check.S index bbc84c0776aa..b6c23a72702b 100644 --- a/arch/sw_64/lib/deep-clear_user-no_una_check.S +++ b/arch/sw_64/lib/deep-clear_user-no_una_check.S @@ -27,9 +27,9 @@ * $18: bytes left to copy * */ - .globl ____clear_user_hw_una - .ent ____clear_user_hw_una -____clear_user_hw_una: + .globl ____clear_user_simd_no_una_check + .ent ____clear_user_simd_no_una_check +____clear_user_simd_no_una_check: .prologue 0 bis $31, $31, $7 mov $17, $18 @@ -44,4 +44,4 @@ $restore_simd: $return: ret - .end ____clear_user_hw_una + .end ____clear_user_simd_no_una_check diff --git a/arch/sw_64/lib/deep-copy_user-force_ali_addr.S b/arch/sw_64/lib/deep-copy_user-force_ali_addr.S index f3c84c93d71b..e04180acfc41 100644 --- a/arch/sw_64/lib/deep-copy_user-force_ali_addr.S +++ b/arch/sw_64/lib/deep-copy_user-force_ali_addr.S @@ -19,9 +19,9 @@ * $18: bytes left to copy * */ - .globl ____copy_user_sw_una - .ent ____copy_user_sw_una -____copy_user_sw_una: + .globl ____copy_user_simd_force_ali_addr + .ent ____copy_user_simd_force_ali_addr +____copy_user_simd_force_ali_addr: .prologue 0 .set noreorder bis $31, $31, $7 @@ -41,4 +41,4 @@ $restore_simd: $return: ret - .end ____copy_user_sw_una + .end ____copy_user_simd_force_ali_addr diff --git a/arch/sw_64/lib/deep-copy_user-no_una_check.S b/arch/sw_64/lib/deep-copy_user-no_una_check.S index c913d85cb469..ec3e0f095b2d 100644 --- a/arch/sw_64/lib/deep-copy_user-no_una_check.S +++ b/arch/sw_64/lib/deep-copy_user-no_una_check.S @@ -19,9 +19,9 @@ * $18: bytes left to copy * */ - .globl ____copy_user_hw_una - .ent ____copy_user_hw_una -____copy_user_hw_una: + .globl ____copy_user_simd_no_una_check + .ent ____copy_user_simd_no_una_check +____copy_user_simd_no_una_check: .prologue 0 .set noreorder bis $31, $31, $7 @@ -37,4 +37,4 @@ $restore_simd: $return: ret - .end ____copy_user_hw_una + .end ____copy_user_simd_no_una_check diff --git a/arch/sw_64/lib/deep-memcpy-force_ali_addr.S b/arch/sw_64/lib/deep-memcpy-force_ali_addr.S index c5629b3b4e06..3b9652fd7ac4 100644 --- a/arch/sw_64/lib/deep-memcpy-force_ali_addr.S +++ b/arch/sw_64/lib/deep-memcpy-force_ali_addr.S @@ -3,13 +3,13 @@ #define FIXUP_LDST(x, y) \ x, y - .globl ____memcpy_sw_una - .ent ____memcpy_sw_una -____memcpy_sw_una: + .globl ____memcpy_simd_force_ali_addr + .ent ____memcpy_simd_force_ali_addr +____memcpy_simd_force_ali_addr: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 #include "deep-copy_template-force_ali_addr.S" 255: ret - .end ____memcpy_sw_una + .end ____memcpy_simd_force_ali_addr diff --git a/arch/sw_64/lib/deep-memcpy-no_una_check.S b/arch/sw_64/lib/deep-memcpy-no_una_check.S index da61247c7eb3..47d0b683e45e 100644 --- a/arch/sw_64/lib/deep-memcpy-no_una_check.S +++ b/arch/sw_64/lib/deep-memcpy-no_una_check.S @@ -3,13 +3,13 @@ #define FIXUP_LDST(x, y) \ x, y - .globl ____memcpy_hw_una - .ent ____memcpy_hw_una -____memcpy_hw_una: + .globl ____memcpy_simd_no_una_check + .ent ____memcpy_simd_no_una_check +____memcpy_simd_no_una_check: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 #include "deep-copy_template-no_una_check.S" 255: ret - .end ____memcpy_hw_una + .end ____memcpy_simd_no_una_check diff --git a/arch/sw_64/lib/deep-memset-force_ali_addr.S b/arch/sw_64/lib/deep-memset-force_ali_addr.S index 61e3ffdd0e5b..cbb0130100dd 100644 --- a/arch/sw_64/lib/deep-memset-force_ali_addr.S +++ b/arch/sw_64/lib/deep-memset-force_ali_addr.S @@ -35,11 +35,11 @@ .text .align 4 - .globl ____constant_c_memset_sw_una - .ent ____constant_c_memset_sw_una + .globl ____constant_c_memset_simd_force_ali_addr + .ent ____constant_c_memset_simd_force_ali_addr .frame $30, 0, $26, 0 .prologue 0 -____constant_c_memset_sw_una: +____constant_c_memset_simd_force_ali_addr: bis $31, $31, $7 bis $31, $16, $0 #ifdef CONFIG_SUBARCH_C4 @@ -51,4 +51,4 @@ ____constant_c_memset_sw_una: csrw $6, CSR_WR_FREGS #endif ret - .end ____constant_c_memset_sw_una + .end ____constant_c_memset_simd_force_ali_addr diff --git a/arch/sw_64/lib/deep-memset-no_una_check.S b/arch/sw_64/lib/deep-memset-no_una_check.S index aa43d7b7df08..91525bd67790 100644 --- a/arch/sw_64/lib/deep-memset-no_una_check.S +++ b/arch/sw_64/lib/deep-memset-no_una_check.S @@ -35,11 +35,11 @@ .text .align 4 - .globl ____constant_c_memset_hw_una - .ent ____constant_c_memset_hw_una + .globl ____constant_c_memset_simd_no_una_check + .ent ____constant_c_memset_simd_no_una_check .frame $30, 0, $26, 0 .prologue 0 -____constant_c_memset_hw_una: +____constant_c_memset_simd_no_una_check: bis $31, $31, $7 bis $31, $16, $0 #ifdef CONFIG_SUBARCH_C4 @@ -51,4 +51,4 @@ ____constant_c_memset_hw_una: csrw $6, CSR_WR_FREGS #endif ret - .end ____constant_c_memset_hw_una + .end ____constant_c_memset_simd_no_una_check diff --git a/arch/sw_64/lib/string.c b/arch/sw_64/lib/string.c index 2adca19a0fb3..ef7439b3715b 100644 --- a/arch/sw_64/lib/string.c +++ b/arch/sw_64/lib/string.c @@ -5,8 +5,8 @@ #include extern void *____memcpy_sisd(void *dest, const void *src, size_t n); -extern void *____memcpy_hw_una(void *dest, const void *src, size_t n); -extern void *____memcpy_sw_una(void *dest, const void *src, size_t n); +extern void *____memcpy_simd_no_una_check(void *dest, const void *src, size_t n); +extern void *____memcpy_simd_force_ali_addr(void *dest, const void *src, size_t n); static inline void *____memcpy(void *dest, const void *src, size_t n) { @@ -14,9 +14,9 @@ static inline void *____memcpy(void *dest, const void *src, size_t n) return ____memcpy_sisd(dest, src, n); if (static_branch_likely(&core_hw_una_enabled)) - return ____memcpy_hw_una(dest, src, n); + return ____memcpy_simd_no_una_check(dest, src, n); else - return ____memcpy_sw_una(dest, src, n); + return ____memcpy_simd_force_ali_addr(dest, src, n); } void *memcpy(void *dest, const void *src, size_t n) @@ -33,8 +33,8 @@ void *__memcpy(void *dest, const void *src, size_t n) EXPORT_SYMBOL(__memcpy); extern void *____constant_c_memset_sisd(void *s, unsigned long c, size_t n); -extern void *____constant_c_memset_hw_una(void *s, unsigned long c, size_t n); -extern void *____constant_c_memset_sw_una(void *s, unsigned long c, size_t n); +extern void *____constant_c_memset_simd_no_una_check(void *s, unsigned long c, size_t n); +extern void *____constant_c_memset_simd_force_ali_addr(void *s, unsigned long c, size_t n); static inline void *____constant_c_memset(void *s, unsigned long c, size_t n) { @@ -42,9 +42,9 @@ static inline void *____constant_c_memset(void *s, unsigned long c, size_t n) return ____constant_c_memset_sisd(s, c, n); if (static_branch_likely(&core_hw_una_enabled)) - return ____constant_c_memset_hw_una(s, c, n); + return ____constant_c_memset_simd_no_una_check(s, c, n); else - return ____constant_c_memset_sw_una(s, c, n); + return ____constant_c_memset_simd_force_ali_addr(s, c, n); } void *__constant_c_memset(void *s, unsigned long c, size_t n) diff --git a/arch/sw_64/lib/uaccess.c b/arch/sw_64/lib/uaccess.c index 72bd66f5b48a..f8efa5ad853d 100644 --- a/arch/sw_64/lib/uaccess.c +++ b/arch/sw_64/lib/uaccess.c @@ -5,8 +5,8 @@ #include extern long ____copy_user_sisd(void *to, const void *from, long len); -extern long ____copy_user_hw_una(void *to, const void *from, long len); -extern long ____copy_user_sw_una(void *to, const void *from, long len); +extern long ____copy_user_simd_no_una_check(void *to, const void *from, long len); +extern long ____copy_user_simd_force_ali_addr(void *to, const void *from, long len); long __copy_user(void *to, const void *from, long len) { @@ -14,15 +14,15 @@ long __copy_user(void *to, const void *from, long len) return ____copy_user_sisd(to, from, len); if (static_branch_likely(&core_hw_una_enabled)) - return ____copy_user_hw_una(to, from, len); + return ____copy_user_simd_no_una_check(to, from, len); else - return ____copy_user_sw_una(to, from, len); + return ____copy_user_simd_force_ali_addr(to, from, len); } EXPORT_SYMBOL(__copy_user); extern long ____clear_user_sisd(void __user *to, long len); -extern long ____clear_user_hw_una(void __user *to, long len); -extern long ____clear_user_sw_una(void __user *to, long len); +extern long ____clear_user_simd_no_una_check(void __user *to, long len); +extern long ____clear_user_simd_force_ali_addr(void __user *to, long len); long __clear_user(void __user *to, long len) { @@ -30,8 +30,8 @@ long __clear_user(void __user *to, long len) return ____clear_user_sisd(to, len); if (static_branch_likely(&core_hw_una_enabled)) - return ____clear_user_hw_una(to, len); + return ____clear_user_simd_no_una_check(to, len); else - return ____clear_user_sw_una(to, len); + return ____clear_user_simd_force_ali_addr(to, len); } EXPORT_SYMBOL(__clear_user); -- Gitee From b2607067fc1cc7a479784b08be4f33fe4ac129ed Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Mar 2024 15:06:08 +0800 Subject: [PATCH 151/524] sw64: save and restore CSR_WR_FREGS for SIMD libs If the lib uses SIMD optimization, save CSR_WR_FREGS before the main part and restore it afterwards to avoid clobbering user SIMD state. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/lib/deep-clear_page.S | 7 +++++++ arch/sw_64/lib/deep-clear_user-force_ali_addr.S | 7 +++++++ arch/sw_64/lib/deep-clear_user-no_una_check.S | 7 +++++++ arch/sw_64/lib/deep-copy_user-force_ali_addr.S | 8 ++++++++ arch/sw_64/lib/deep-copy_user-no_una_check.S | 8 ++++++++ arch/sw_64/lib/deep-memcpy-force_ali_addr.S | 8 ++++++++ arch/sw_64/lib/deep-memcpy-no_una_check.S | 8 ++++++++ 7 files changed, 53 insertions(+) diff --git a/arch/sw_64/lib/deep-clear_page.S b/arch/sw_64/lib/deep-clear_page.S index 52a3db33fc17..0c7f163d781e 100644 --- a/arch/sw_64/lib/deep-clear_page.S +++ b/arch/sw_64/lib/deep-clear_page.S @@ -2,6 +2,7 @@ /* * Zero an entire page. */ +#include #include .text .align 4 @@ -37,6 +38,9 @@ clear_page: stl_nc $31,0x78($16) */ +#ifdef CONFIG_SUBARCH_C4 + csrr $7, CSR_WR_FREGS +#endif vstd_nc $f31, 0x0($16) vstd_nc $f31, 0x20($16) subl $0, 1, $0 @@ -47,6 +51,9 @@ clear_page: bne $0, 1b memb +#ifdef CONFIG_SUBARCH_C4 + csrw $7, CSR_WR_FREGS +#endif ret .end clear_page diff --git a/arch/sw_64/lib/deep-clear_user-force_ali_addr.S b/arch/sw_64/lib/deep-clear_user-force_ali_addr.S index 24b1f03b6a4d..78d989609e12 100644 --- a/arch/sw_64/lib/deep-clear_user-force_ali_addr.S +++ b/arch/sw_64/lib/deep-clear_user-force_ali_addr.S @@ -10,6 +10,7 @@ * stuff. * */ +#include #include /* Allow an exception for an insn; exit if we get one. */ #define FIXUP_LDST(x,y...) \ @@ -34,6 +35,9 @@ ____clear_user_simd_force_ali_addr: bis $31, $31, $7 mov $17, $18 bis $31, $31, $17 +#ifdef CONFIG_SUBARCH_C4 + csrr $6, CSR_WR_FREGS +#endif #include "deep-set_template-force_ali_addr.S" 255: bis $31, $18, $0 @@ -43,5 +47,8 @@ $restore_simd: RESTORE_SIMD_REGS $return: +#ifdef CONFIG_SUBARCH_C4 + csrw $6, CSR_WR_FREGS +#endif ret .end ____clear_user_simd_force_ali_addr diff --git a/arch/sw_64/lib/deep-clear_user-no_una_check.S b/arch/sw_64/lib/deep-clear_user-no_una_check.S index b6c23a72702b..da25c6be66fb 100644 --- a/arch/sw_64/lib/deep-clear_user-no_una_check.S +++ b/arch/sw_64/lib/deep-clear_user-no_una_check.S @@ -10,6 +10,7 @@ * stuff. * */ +#include #include /* Allow an exception for an insn; exit if we get one. */ #define FIXUP_LDST(x,y...) \ @@ -34,6 +35,9 @@ ____clear_user_simd_no_una_check: bis $31, $31, $7 mov $17, $18 bis $31, $31, $17 +#ifdef CONFIG_SUBARCH_C4 + csrr $6, CSR_WR_FREGS +#endif #include "deep-set_template-no_una_check.S" 255: bis $31, $18, $0 @@ -43,5 +47,8 @@ $restore_simd: RESTORE_SIMD_REGS $return: +#ifdef CONFIG_SUBARCH_C4 + csrw $6, CSR_WR_FREGS +#endif ret .end ____clear_user_simd_no_una_check diff --git a/arch/sw_64/lib/deep-copy_user-force_ali_addr.S b/arch/sw_64/lib/deep-copy_user-force_ali_addr.S index e04180acfc41..febb46f9b691 100644 --- a/arch/sw_64/lib/deep-copy_user-force_ali_addr.S +++ b/arch/sw_64/lib/deep-copy_user-force_ali_addr.S @@ -1,5 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include + /* Allow an exception for an insn; exit if we get one. */ #define FIXUP_LDST(x, y) \ 99: x, y; \ @@ -25,6 +27,9 @@ ____copy_user_simd_force_ali_addr: .prologue 0 .set noreorder bis $31, $31, $7 +#ifdef CONFIG_SUBARCH_C4 + csrr $6, CSR_WR_FREGS +#endif #include "deep-copy_template-force_ali_addr.S" 255: bis $31, $18, $0 @@ -40,5 +45,8 @@ $restore_simd: RESTORE_SIMD_REGS $return: +#ifdef CONFIG_SUBARCH_C4 + csrw $6, CSR_WR_FREGS +#endif ret .end ____copy_user_simd_force_ali_addr diff --git a/arch/sw_64/lib/deep-copy_user-no_una_check.S b/arch/sw_64/lib/deep-copy_user-no_una_check.S index ec3e0f095b2d..cadd5e0f31c1 100644 --- a/arch/sw_64/lib/deep-copy_user-no_una_check.S +++ b/arch/sw_64/lib/deep-copy_user-no_una_check.S @@ -1,5 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include + /* Allow an exception for an insn; exit if we get one. */ #define FIXUP_LDST(x, y) \ 99: x, y; \ @@ -25,6 +27,9 @@ ____copy_user_simd_no_una_check: .prologue 0 .set noreorder bis $31, $31, $7 +#ifdef CONFIG_SUBARCH_C4 + csrr $6, CSR_WR_FREGS +#endif #include "deep-copy_template-no_una_check.S" 255: bis $31, $18, $0 @@ -36,5 +41,8 @@ $restore_simd: RESTORE_SIMD_REGS $return: +#ifdef CONFIG_SUBARCH_C4 + csrw $6, CSR_WR_FREGS +#endif ret .end ____copy_user_simd_no_una_check diff --git a/arch/sw_64/lib/deep-memcpy-force_ali_addr.S b/arch/sw_64/lib/deep-memcpy-force_ali_addr.S index 3b9652fd7ac4..7e990bda936c 100644 --- a/arch/sw_64/lib/deep-memcpy-force_ali_addr.S +++ b/arch/sw_64/lib/deep-memcpy-force_ali_addr.S @@ -1,5 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include + #define FIXUP_LDST(x, y) \ x, y @@ -9,7 +11,13 @@ ____memcpy_simd_force_ali_addr: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 +#ifdef CONFIG_SUBARCH_C4 + csrr $6, CSR_WR_FREGS +#endif #include "deep-copy_template-force_ali_addr.S" 255: +#ifdef CONFIG_SUBARCH_C4 + csrw $6, CSR_WR_FREGS +#endif ret .end ____memcpy_simd_force_ali_addr diff --git a/arch/sw_64/lib/deep-memcpy-no_una_check.S b/arch/sw_64/lib/deep-memcpy-no_una_check.S index 47d0b683e45e..c7ded71b1ade 100644 --- a/arch/sw_64/lib/deep-memcpy-no_una_check.S +++ b/arch/sw_64/lib/deep-memcpy-no_una_check.S @@ -1,5 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include + #define FIXUP_LDST(x, y) \ x, y @@ -9,7 +11,13 @@ ____memcpy_simd_no_una_check: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 +#ifdef CONFIG_SUBARCH_C4 + csrr $6, CSR_WR_FREGS +#endif #include "deep-copy_template-no_una_check.S" 255: +#ifdef CONFIG_SUBARCH_C4 + csrw $6, CSR_WR_FREGS +#endif ret .end ____memcpy_simd_no_una_check -- Gitee From 5d492794d7ec3335587e25ce12b0b5c462d26817 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 27 Feb 2024 14:03:10 +0800 Subject: [PATCH 152/524] sw64: setup stackframe at kernel entry point Manually setup a stackframe at kernel entry point to produce a more accurate stacktrace. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/asm-offsets.c | 8 ++++++ arch/sw_64/kernel/entry_c3.S | 36 +++++++++++++++---------- arch/sw_64/kernel/entry_c4.S | 48 +++++++++++++++++++-------------- arch/sw_64/kernel/process.c | 2 +- 4 files changed, 59 insertions(+), 35 deletions(-) diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c index 2fb2d807ae3f..334f7f2e9550 100644 --- a/arch/sw_64/kernel/asm-offsets.c +++ b/arch/sw_64/kernel/asm-offsets.c @@ -241,4 +241,12 @@ void foo(void) BLANK(); DEFINE(RT_SIGFRAME_SIZE, sizeof(struct rt_sigframe)); OFFSET(RT_SIGFRAME_MCTX, rt_sigframe, uc.uc_mcontext); + BLANK(); +#ifdef CONFIG_FRAME_POINTER + DEFINE(STACKFRAME_SIZE, sizeof(struct stackframe)); + OFFSET(STACKFRAME_PC, stackframe, pc); + OFFSET(STACKFRAME_FP, stackframe, fp); +#else + DEFINE(STACKFRAME_SIZE, 0); +#endif } diff --git a/arch/sw_64/kernel/entry_c3.S b/arch/sw_64/kernel/entry_c3.S index a7183817671b..df8bff5d09b3 100644 --- a/arch/sw_64/kernel/entry_c3.S +++ b/arch/sw_64/kernel/entry_c3.S @@ -70,9 +70,17 @@ ldi $1, NO_SYSCALL stl $1, PT_REGS_ORIG_R0($sp) sys_call HMC_rdktp +#ifdef CONFIG_FRAME_POINTER + ldi $sp, -STACKFRAME_SIZE($sp) + stl $5, STACKFRAME_PC($sp) + stl $15, STACKFRAME_FP($sp) + mov $sp, $15 +#endif .endm .macro RESTORE_ALL + ldi $sp, STACKFRAME_SIZE($sp) + ldl $16, PT_REGS_SP($sp) /* skip wrusp if returning to kernel */ blt $16, 1f @@ -127,7 +135,7 @@ .ent entInt entInt: SAVE_ALL - mov $sp, $19 + ldi $19, STACKFRAME_SIZE($sp) call $26, do_entInt br ret_from_sys_call .end entInt @@ -137,7 +145,7 @@ entInt: .ent entArith entArith: SAVE_ALL - mov $sp, $18 + ldi $18, STACKFRAME_SIZE($sp) call $26, do_entArith br ret_from_sys_call .end entArith @@ -147,7 +155,7 @@ entArith: .ent entMM entMM: SAVE_ALL - mov $sp, $19 + ldi $19, STACKFRAME_SIZE($sp) call $26, do_page_fault br ret_from_sys_call .end entMM @@ -157,7 +165,7 @@ entMM: .ent entIF entIF: SAVE_ALL - mov $sp, $18 + ldi $18, STACKFRAME_SIZE($sp) call $26, do_entIF br ret_from_sys_call .end entIF @@ -173,15 +181,15 @@ entIF: .ent entUna entUna: SAVE_ALL - mov $sp, $19 - ldl $0, PT_REGS_PS($sp) + ldi $19, STACKFRAME_SIZE($sp) + ldl $0, (PT_REGS_PS + STACKFRAME_SIZE)($sp) and $0, 8, $0 /* user mode ? */ beq $0, 1f call $26, do_entUnaUser /* return to ret_from_syscall */ br ret_from_sys_call -1: ldl $9, PT_REGS_GP($sp) +1: ldl $9, (PT_REGS_GP + STACKFRAME_SIZE)($sp) call $26, do_entUna - stl $9, PT_REGS_GP($sp) + stl $9, (PT_REGS_GP + STACKFRAME_SIZE)($sp) RESTORE_ALL sys_call HMC_rti .end entUna @@ -202,10 +210,10 @@ entUna: .ent entSys entSys: SAVE_ALL - stl $16, PT_REGS_R16($sp) - stl $17, PT_REGS_R17($sp) - stl $18, PT_REGS_R18($sp) - mov $sp, $16 + stl $16, (PT_REGS_R16 + STACKFRAME_SIZE)($sp) + stl $17, (PT_REGS_R17 + STACKFRAME_SIZE)($sp) + stl $18, (PT_REGS_R18 + STACKFRAME_SIZE)($sp) + ldi $16, STACKFRAME_SIZE($sp) call $26, do_entSys br ret_from_sys_call .end entSys @@ -223,14 +231,14 @@ ret_from_sys_call: sampling and the rti. */ ldi $16, 7 sys_call HMC_swpipl - ldl $0, PT_REGS_PS($sp) + ldl $0, (PT_REGS_PS + STACKFRAME_SIZE)($sp) and $0, 8, $0 beq $0, restore_all ret_to_user: ldw $17, TI_FLAGS($8) and $17, _TIF_WORK_MASK, $2 beq $2, restore_all - mov $sp, $16 + ldi $16, STACKFRAME_SIZE($sp) call $26, do_notify_resume restore_all: RESTORE_ALL diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S index 097f4279e69b..c31143645b6f 100644 --- a/arch/sw_64/kernel/entry_c4.S +++ b/arch/sw_64/kernel/entry_c4.S @@ -79,12 +79,20 @@ ldi $1, NO_SYSCALL stl $1, PT_REGS_ORIG_R0($sp) csrr $8, CSR_KTP +#ifdef CONFIG_FRAME_POINTER + ldi $sp, -STACKFRAME_SIZE($sp) + stl $3, STACKFRAME_PC($sp) + stl $15, STACKFRAME_FP($sp) + mov $sp, $15 +#endif .endm .macro RESTORE_ALL ldi $16, 7 sys_call HMC_swpipl + ldi $sp, STACKFRAME_SIZE($sp) + ldl $1, PT_REGS_PS($sp) ldl $2, PT_REGS_PC($sp) csrw $1, CSR_PS @@ -130,7 +138,7 @@ .macro RESTORE_IRQ csrr $16, CSR_PS sys_call HMC_swpipl - ldl $16, PT_REGS_EARG0($sp) + ldl $16, (PT_REGS_EARG0 + STACKFRAME_SIZE)($sp) .endm /* * Non-syscall kernel entry points. @@ -143,7 +151,7 @@ entInt: SAVE_ALL br $27, 1f 1: ldgp $29, 0($27) - mov $sp, $19 + ldi $19, STACKFRAME_SIZE($sp) call $26, do_entInt br ret_from_sys_call .end entInt @@ -153,13 +161,13 @@ entInt: .ent entArith entArith: SAVE_ALL - ldl $1, PT_REGS_PC($sp) + ldl $1, (PT_REGS_PC + STACKFRAME_SIZE)($sp) ldi $1, 4($1) - stl $1, PT_REGS_PC($sp) + stl $1, (PT_REGS_PC + STACKFRAME_SIZE)($sp) br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ - mov $sp, $18 + ldi $18, STACKFRAME_SIZE($sp) call $26, do_entArith br ret_from_sys_call .end entArith @@ -172,7 +180,7 @@ entMM: br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ - mov $sp, $19 + ldi $19, STACKFRAME_SIZE($sp) call $26, do_page_fault br ret_from_sys_call .end entMM @@ -182,13 +190,13 @@ entMM: .ent entIF entIF: SAVE_ALL - ldl $1, PT_REGS_PC($sp) + ldl $1, (PT_REGS_PC + STACKFRAME_SIZE)($sp) ldi $1, 4($1) - stl $1, PT_REGS_PC($sp) + stl $1, (PT_REGS_PC + STACKFRAME_SIZE)($sp) br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ - mov $sp, $18 + ldi $18, STACKFRAME_SIZE($sp) call $26, do_entIF br ret_from_sys_call .end entIF @@ -204,21 +212,21 @@ entIF: .ent entUna entUna: SAVE_ALL - ldl $1, PT_REGS_PC($sp) + ldl $1, (PT_REGS_PC + STACKFRAME_SIZE)($sp) ldi $1, 4($1) - stl $1, PT_REGS_PC($sp) + stl $1, (PT_REGS_PC + STACKFRAME_SIZE)($sp) br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ - mov $sp, $19 - ldl $0, PT_REGS_PS($sp) + ldi $19, STACKFRAME_SIZE($sp) + ldl $0, (PT_REGS_PS + STACKFRAME_SIZE)($sp) and $0, 8, $0 /* user mode ? */ beq $0, 1f call $26, do_entUnaUser /* return to ret_from_syscall */ br ret_from_sys_call -1: ldl $9, PT_REGS_GP($sp) +1: ldl $9, (PT_REGS_GP + STACKFRAME_SIZE)($sp) call $26, do_entUna - stl $9, PT_REGS_GP($sp) + stl $9, (PT_REGS_GP + STACKFRAME_SIZE)($sp) RESTORE_ALL sys_call HMC_rti .end entUna @@ -239,13 +247,13 @@ entUna: .ent entSys entSys: SAVE_ALL - ldl $1, PT_REGS_PC($sp) + ldl $1, (PT_REGS_PC + STACKFRAME_SIZE)($sp) ldi $1, 4($1) - stl $1, PT_REGS_PC($sp) + stl $1, (PT_REGS_PC + STACKFRAME_SIZE)($sp) br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ - mov $sp, $16 + ldi $16, STACKFRAME_SIZE($sp) call $26, do_entSys br ret_from_sys_call .end entSys @@ -260,14 +268,14 @@ ret_from_sys_call: sampling and the rti. */ ldi $16, 7 sys_call HMC_swpipl - ldl $0, PT_REGS_PS($sp) + ldl $0, (PT_REGS_PS + STACKFRAME_SIZE)($sp) and $0, 8, $0 beq $0, restore_all ret_to_user: ldw $17, TI_FLAGS($8) and $17, _TIF_WORK_MASK, $2 beq $2, restore_all - mov $sp, $16 + ldi $16, STACKFRAME_SIZE($sp) call $26, do_notify_resume restore_all: RESTORE_ALL diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c index cd371a71cfca..ded9f780497a 100644 --- a/arch/sw_64/kernel/process.c +++ b/arch/sw_64/kernel/process.c @@ -78,7 +78,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) extern void ret_from_fork(void); extern void ret_from_kernel_thread(void); - p->thread.sp = (unsigned long) childregs; + p->thread.sp = (unsigned long) childregs - STACKFRAME_SIZE; if (unlikely(args->fn)) { /* kernel thread */ -- Gitee From be3a1ee548027394b90f02ad85345eb582130cff Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 27 Mar 2024 14:47:13 +0800 Subject: [PATCH 153/524] sw64: receive new boot params from firmware Starting from junzhang, firmware pass a magic number (0xDEED2024UL) via r16 and dtb base address via r17 when jumping to kernel. When kernel receives the magic number, it means that we can parse the required boot params from the chosen node in DTB. Otherwise, kernel will use legacy "struct boot_params" for backward compatibility. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/platform.h | 2 ++ arch/sw_64/kernel/head.S | 23 +++++++++++++++++++++++ arch/sw_64/kernel/setup.c | 10 ++++++++++ 3 files changed, 35 insertions(+) diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h index ad54cdc772e1..7374f2087f90 100644 --- a/arch/sw_64/include/asm/platform.h +++ b/arch/sw_64/include/asm/platform.h @@ -19,6 +19,8 @@ extern unsigned long bios_version; #endif extern struct boot_params *sunway_boot_params; +extern unsigned long sunway_boot_magic; +extern unsigned long sunway_dtb_address; extern void sw64_halt(void); extern void sw64_poweroff(void); diff --git a/arch/sw_64/kernel/head.S b/arch/sw_64/kernel/head.S index bfeb6859ecc2..db4af8f9366b 100644 --- a/arch/sw_64/kernel/head.S +++ b/arch/sw_64/kernel/head.S @@ -37,6 +37,16 @@ __start: /* ... and find our stack ... */ ldi $30, ASM_THREAD_SIZE($30) + /** + * Starting from Core4, Grub uses $16 to pass magic num + * and use $17 to pass DTB base address. + * + * So, we need to save $16 and $17 before clearing bss. + */ + ldi $30, -16($30) + stl $16, 8($30) + stl $17, 0($30) + /* ... and then we can clear bss data. */ ldi $16, __bss_start ldi $18, __bss_stop @@ -44,6 +54,18 @@ __start: mov $31, $17 call $26, __constant_c_memset ldgp $29, 0($26) + + /* Get magic num and DTB base address. */ + ldl $16, 8($30) + ldl $17, 0($30) + ldi $30, 16($30) + /* sunway_boot_magic = $16 */ + ldi $7, sunway_boot_magic + stl $16, 0($7) + /* sunway_dtb_address = $17 */ + ldi $7, sunway_dtb_address + stl $17, 0($7) + #ifdef CONFIG_RELOCATABLE ldi $30, -8($30) stl $29, 0($30) @@ -56,6 +78,7 @@ __start: /* Repoint the sp into the new kernel image */ addl $30, $0, $30 #endif + /* ... and then we can start the kernel. */ call $26, sw64_start_kernel sys_call HMC_halt diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 4bf49300d09e..43d180a4fcca 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -125,8 +125,18 @@ static char builtin_cmdline[COMMAND_LINE_SIZE] __initdata = CONFIG_CMDLINE; #endif /* boot_params */ +/** + * Keep sunway_boot_params for backward compatibility. All related code + * will be removed when kernel no longer support C3B(xuelang). + */ struct boot_params *sunway_boot_params = (struct boot_params *) (PARAM + 0x100); +unsigned long sunway_boot_magic; +EXPORT_SYMBOL(sunway_boot_magic); + +unsigned long sunway_dtb_address; +EXPORT_SYMBOL(sunway_dtb_address); + /* * The format of "screen_info" is strange, and due to early * i386-setup code. This is just enough to make the console -- Gitee From 94be689efb0fc625d70265c82904b562e1279a15 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 27 Mar 2024 14:52:38 +0800 Subject: [PATCH 154/524] sw64: apply new boot params from DTB chosen node Starting from junzhang, we use the chosen node in DTB to pass boot params, involving the following properties: - bootargs - linux,initrd-start - linux,initrd-end - linux,uefi-system-table - linux,uefi-mmap-start - linux,uefi-mmap-size - linux,uefi-mmap-desc-size - linux,uefi-mmap-desc-ver This commit will not break backward compatibility. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 81 +++++---- arch/sw_64/configs/junzhang_defconfig | 1 - arch/sw_64/configs/kata_guest_defconfig | 5 +- arch/sw_64/configs/kata_xuelang_defconfig | 1 - arch/sw_64/configs/xuelang_defconfig | 1 - arch/sw_64/include/asm/hw_init.h | 4 +- arch/sw_64/kernel/chip_setup.c | 2 +- arch/sw_64/kernel/machine_kexec.c | 203 +++++++++++++++++++-- arch/sw_64/kernel/setup.c | 212 +++++++++------------- arch/sw_64/mm/init.c | 80 +++++++- drivers/firmware/efi/sunway-init.c | 31 ++-- 11 files changed, 409 insertions(+), 212 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 79be887a5a0c..2753e1b0b058 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -103,12 +103,14 @@ config SW64 select HAVE_RELIABLE_STACKTRACE if STACKTRACE select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS + select IRQ_DOMAIN select IRQ_FORCED_THREADING select LOCK_MM_AND_FIND_VMA select MEMORY_HOTPLUG_SPARSE if MEMORY_HOTPLUG select MODULES_USE_ELF_RELA select NO_BOOTMEM - select OF_EARLY_FLATTREE if OF + select OF + select OF_EARLY_FLATTREE select OLD_SIGSUSPEND select PCI_DOMAINS_GENERIC if PCI select PCI_ECAM if (ACPI && PCI) @@ -508,13 +510,6 @@ endmenu menu "Boot options" -config USE_OF - bool "Flattened Device Tree support" - select OF - select IRQ_DOMAIN - help - Include support for flattened device tree machine descriptions. - config BUILTIN_DTB bool "Embed DTB in kernel image" depends on OF @@ -531,8 +526,10 @@ config BUILTIN_DTB_NAME config EFI bool "UEFI runtime support" + depends on OF select UCS2_STRING select EFI_RUNTIME_WRAPPERS + select EFI_PARAMS_FROM_FDT default y help This option provides support for runtime services provided @@ -558,48 +555,52 @@ config DMI firmware need to be enabled. This would require the DMI subsystem to be enabled much earlier than we do on ARM, which is non-trivial. -config CMDLINE_BOOL - bool "Built-in kernel command line" +config CMDLINE + string "Built-in kernel command line" + default "" help - Allow for specifying boot arguments to the kernel at - build time. On some systems (e.g. embedded ones), it is - necessary or convenient to provide some or all of the - kernel boot arguments with the kernel itself (that is, - to not rely on the boot loader to provide them.) + Allow for specifying boot arguments to the kernel at build + time. On some systems (e.g. embedded ones), it is necessary + or convenient to provide some or all of the kernel boot + arguments with the kernel itself (that is, to not rely on + firmware to provide them.) - To compile command line arguments into the kernel, - set this option to 'Y', then fill in the - boot arguments in CONFIG_CMDLINE. + Enter arguments here that should be compiled into the kernel + image. And choose how the kernel should use it later on. - Systems with fully functional boot loaders (i.e. non-embedded) - should leave this option set to 'N'. + In most cases, the command line (whether built-in or provided + by firmware) should specify the device for the root file system. -config CMDLINE - string "Built-in kernel command string" - depends on CMDLINE_BOOL - default "" +choice + prompt "Built-in kernel command line usage" if CMDLINE != "" + default CMDLINE_FROM_FIRMWARE help - Enter arguments here that should be compiled into the kernel - image and used at boot time. If the boot loader provides a - command line at boot time, it is appended to this string to - form the full kernel command line, when the system boots. + Choose how the kernel will handle the provided built-in kernel + command line. - However, you can use the CONFIG_CMDLINE_OVERRIDE option to - change this behavior. +config CMDLINE_FROM_FIRMWARE + bool "Use kernel command line provided by firmware" + help + Use kernel command line provided by firmware. If the firmware + doesn't provide any, the built-in kernel command line provided + in CMDLINE will be used. - In most cases, the command line (whether built-in or provided - by the boot loader) should specify the device for the root - file system. +config CMDLINE_EXTEND + bool "Extend kernel command line" + help + The built-in kernel command line will be appended to the command + line provided by firmware. This is useful in cases where the + command line provided by firmware is insufficient and you don't + want to or cannot modify them. -config CMDLINE_OVERRIDE - bool "Built-in command line overrides boot loader arguments" - depends on CMDLINE_BOOL +config CMDLINE_FORCE + bool "Always use built-in kernel command line" help - Set this option to 'Y' to have the kernel ignore the boot loader - command line, and use ONLY the built-in command line. + Always use built-in kernel command line, even if we get one + from firmware. This is useful in some cases. For example, + you need to work around broken boot loaders. - This is used to work around broken boot loaders. This should - be set to 'N' under normal conditions. +endchoice config FORCE_MAX_ZONEORDER int diff --git a/arch/sw_64/configs/junzhang_defconfig b/arch/sw_64/configs/junzhang_defconfig index 9baf68f8e0ac..62fccb2a8117 100644 --- a/arch/sw_64/configs/junzhang_defconfig +++ b/arch/sw_64/configs/junzhang_defconfig @@ -33,7 +33,6 @@ CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_NUMA=y CONFIG_HZ=100 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_USE_OF=y CONFIG_FIRMWARE_MEMMAP=y CONFIG_DMI_SYSFS=m CONFIG_ACPI_TAD=y diff --git a/arch/sw_64/configs/kata_guest_defconfig b/arch/sw_64/configs/kata_guest_defconfig index 8122155c1276..6a7c442b94a7 100644 --- a/arch/sw_64/configs/kata_guest_defconfig +++ b/arch/sw_64/configs/kata_guest_defconfig @@ -34,9 +34,8 @@ CONFIG_NUMA=y CONFIG_HZ=100 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=y -CONFIG_USE_OF=y -CONFIG_SW64_BUILTIN_DTB=y -CONFIG_SW64_BUILTIN_DTB_NAME="chip_vt" +CONFIG_BUILTIN_DTB=y +CONFIG_BUILTIN_DTB_NAME="chip_vt" CONFIG_FIRMWARE_MEMMAP=y CONFIG_DMI_SYSFS=m CONFIG_GOOGLE_FIRMWARE=y diff --git a/arch/sw_64/configs/kata_xuelang_defconfig b/arch/sw_64/configs/kata_xuelang_defconfig index f553f0e71dbf..c059bbb7a576 100644 --- a/arch/sw_64/configs/kata_xuelang_defconfig +++ b/arch/sw_64/configs/kata_xuelang_defconfig @@ -36,7 +36,6 @@ CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_NUMA=y CONFIG_HZ=100 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_USE_OF=y CONFIG_FIRMWARE_MEMMAP=y CONFIG_DMI_SYSFS=m # CONFIG_SUSPEND is not set diff --git a/arch/sw_64/configs/xuelang_defconfig b/arch/sw_64/configs/xuelang_defconfig index b1c0101d0089..3da2cffa2085 100644 --- a/arch/sw_64/configs/xuelang_defconfig +++ b/arch/sw_64/configs/xuelang_defconfig @@ -30,7 +30,6 @@ CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_NUMA=y CONFIG_HZ=100 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_USE_OF=y CONFIG_FIRMWARE_MEMMAP=y CONFIG_DMI_SYSFS=m CONFIG_ACPI_TAD=y diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h index d1d2c71bc344..ab0e2ee4c9a6 100644 --- a/arch/sw_64/include/asm/hw_init.h +++ b/arch/sw_64/include/asm/hw_init.h @@ -7,7 +7,7 @@ #include -#define MMSIZE __va(0x2040) +#define MM_SIZE __va(0x2040) #define VPCR_SHIFT 44 /* @@ -93,7 +93,7 @@ static inline void update_cpu_freq(unsigned long khz) } #define EMUL_FLAG (0x1UL << 63) -#define MMSIZE_MASK (EMUL_FLAG - 1) +#define MM_SIZE_MASK (EMUL_FLAG - 1) DECLARE_STATIC_KEY_TRUE(run_mode_host_key); DECLARE_STATIC_KEY_FALSE(run_mode_guest_key); diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index b8c359db2ef6..60373429a64e 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -21,7 +21,7 @@ static unsigned long __init get_node_mem(int nodeid) { if (is_guest_or_emul()) - return *(unsigned long *)MMSIZE & MMSIZE_MASK; + return *(unsigned long *)MM_SIZE & MM_SIZE_MASK; return __get_node_mem(nodeid); } diff --git a/arch/sw_64/kernel/machine_kexec.c b/arch/sw_64/kernel/machine_kexec.c index 950998476cda..d6758d48dc26 100644 --- a/arch/sw_64/kernel/machine_kexec.c +++ b/arch/sw_64/kernel/machine_kexec.c @@ -10,8 +10,13 @@ #include #include #include +#include +#include +#include +#include #include +#include extern void *kexec_control_page; extern const unsigned char relocate_new_kernel[]; @@ -143,15 +148,189 @@ void machine_crash_shutdown(struct pt_regs *regs) #define phys_to_ktext(pa) (__START_KERNEL_map + (pa)) -typedef void (*noretfun_t)(void) __noreturn; +typedef void (*noretfun_t)(unsigned long, unsigned long) __noreturn; + +/** + * Current kernel does not yet have a common implementation for + * this function. So, make an arch-specific one. + */ +static void *arch_kexec_alloc_and_setup_fdt(unsigned long initrd_start, + unsigned long initrd_size, const char *cmdline) +{ + void *fdt; + int ret, chosen_node; + size_t fdt_size; + + fdt_size = fdt_totalsize(initial_boot_params) + + (cmdline ? strlen(cmdline) : 0) + 0x1000; + fdt = kzalloc(fdt_size, GFP_KERNEL); + if (!fdt) + return NULL; + + ret = fdt_open_into(initial_boot_params, fdt, fdt_size); + if (ret < 0) { + pr_err("Error %d setting up the new device tree\n", ret); + goto out; + } + + chosen_node = fdt_path_offset(fdt, "/chosen"); + if (chosen_node < 0) { + pr_err("Failed to find chosen node\n"); + goto out; + } + + /* update initrd params */ + if (initrd_size) { + ret = fdt_setprop_u64(fdt, chosen_node, "linux,initrd-start", + initrd_start); + if (ret) + goto out; + + ret = fdt_setprop_u64(fdt, chosen_node, "linux,initrd-end", + initrd_start + initrd_size); + if (ret) + goto out; + } else { + ret = fdt_delprop(fdt, chosen_node, "linux,initrd-start"); + if (ret) + goto out; + + ret = fdt_delprop(fdt, chosen_node, "linux,initrd-end"); + if (ret) + goto out; + } + + /* update cmdline */ + if (cmdline) { + ret = fdt_setprop_string(fdt, chosen_node, "bootargs", cmdline); + if (ret) + goto out; + } else { + ret = fdt_delprop(fdt, chosen_node, "bootargs"); + if (ret) + goto out; + } + + return fdt; + +out: + kfree(fdt); + return NULL; +} + +#ifdef CONFIG_EFI +static int update_efi_properties(const struct boot_params *params) +{ + int chosen_node, ret; + void *dtb_start = (void *)params->dtb_start; + + if (!dtb_start) + return -EINVAL; + + chosen_node = fdt_path_offset(dtb_start, "/chosen"); + if (chosen_node < 0) + return -EINVAL; + + ret = fdt_setprop_u64(dtb_start, chosen_node, + "linux,uefi-system-table", + params->efi_systab); + if (ret) + return ret; + + ret = fdt_setprop_u64(dtb_start, chosen_node, + "linux,uefi-mmap-start", + params->efi_memmap); + if (ret) + return ret; + + ret = fdt_setprop_u64(dtb_start, chosen_node, + "linux,uefi-mmap-size", + params->efi_memmap_size); + if (ret) + return ret; + + ret = fdt_setprop_u64(dtb_start, chosen_node, + "linux,uefi-mmap-desc-size", + params->efi_memdesc_size); + if (ret) + return ret; + + ret = fdt_setprop_u64(dtb_start, chosen_node, + "linux,uefi-mmap-desc-ver", + params->efi_memdesc_version); + if (ret) + return ret; + + return 0; +} +#endif + +static void update_boot_params(void) +{ + struct boot_params params = { 0 }; + + /* Cmdline and initrd can be new */ + params.cmdline = kexec_start_address - COMMAND_LINE_OFF; + params.initrd_start = *(__u64 *)(kexec_start_address - INITRD_START_OFF); + params.initrd_size = *(__u64 *)(kexec_start_address - INITRD_SIZE_OFF); + + if (sunway_boot_magic != 0xDEED2024UL) { + sunway_boot_params->cmdline = params.cmdline; + sunway_boot_params->initrd_start = params.initrd_start; + sunway_boot_params->initrd_size = params.initrd_size; + + params.dtb_start = sunway_boot_params->dtb_start; + params.efi_systab = sunway_boot_params->efi_systab; + params.efi_memmap = sunway_boot_params->efi_memmap; + params.efi_memmap_size = sunway_boot_params->efi_memmap_size; + params.efi_memdesc_size = sunway_boot_params->efi_memdesc_size; + params.efi_memdesc_version = sunway_boot_params->efi_memdesc_version; + } else { + params.dtb_start = (unsigned long)arch_kexec_alloc_and_setup_fdt( + params.initrd_start, params.initrd_size, + (const char *)params.cmdline); + /* update dtb base address */ + sunway_dtb_address = params.dtb_start; + +#ifdef CONFIG_EFI + params.efi_systab = virt_to_phys((void *)efi.systab); + params.efi_memmap = efi.memmap.phys_map; + params.efi_memmap_size = efi.memmap.map_end - efi.memmap.map; + params.efi_memdesc_size = efi.memmap.desc_size; + params.efi_memdesc_version = efi.memmap.desc_version; + + /** + * If current kernel take built-in DTB, it's possible that + * there are no efi related properties in "chosen" node. So, + * update these properties here. + * + * Harmless for the following cases: + * 1. Current kernel take DTB from firmware + * 2. New kernel with CONFIG_EFI=n + * 3. New kernel take built-in DTB + */ + if (update_efi_properties(¶ms)) + pr_err("Note: failed to update efi properties\n"); +#endif + } + + pr_info("initrd_start = %#llx, initrd_size = %#llx\n" + "dtb_start = %#llx, efi_systab = %#llx\n" + "efi_memmap = %#llx, efi_memmap_size = %#llx\n" + "efi_memdesc_size = %#llx, efi_memdesc_version = %#llx\n" + "cmdline = %s\n", + params.initrd_start, params.initrd_size, + params.dtb_start, params.efi_systab, + params.efi_memmap, params.efi_memmap_size, + params.efi_memdesc_size, params.efi_memdesc_version, + (char *)params.cmdline); +} void machine_kexec(struct kimage *image) { void *reboot_code_buffer; unsigned long entry; unsigned long *ptr; - struct boot_params *params = sunway_boot_params; - reboot_code_buffer = kexec_control_page; pr_info("reboot_code_buffer = %px\n", reboot_code_buffer); @@ -166,20 +345,7 @@ void machine_kexec(struct kimage *image) pr_info("kexec_indirection_page = %#lx, image->head=%#lx\n", kexec_indirection_page, image->head); - params->cmdline = kexec_start_address - COMMAND_LINE_OFF; - params->initrd_start = *(__u64 *)(kexec_start_address - INITRD_START_OFF); - params->initrd_size = *(__u64 *)(kexec_start_address - INITRD_SIZE_OFF); - - pr_info("initrd_start = %#llx, initrd_size = %#llx\n" - "dtb_start = %#llx, efi_systab = %#llx\n" - "efi_memmap = %#llx, efi_memmap_size = %#llx\n" - "efi_memdesc_size = %#llx, efi_memdesc_version = %#llx\n" - "cmdline = %#llx\n", - params->initrd_start, params->initrd_size, - params->dtb_start, params->efi_systab, - params->efi_memmap, params->efi_memmap_size, - params->efi_memdesc_size, params->efi_memdesc_version, - params->cmdline); + update_boot_params(); memcpy(reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size); @@ -205,5 +371,6 @@ void machine_kexec(struct kimage *image) pr_info("Will call new kernel at %08lx\n", image->start); pr_info("Bye ...\n"); smp_wmb(); - ((noretfun_t) reboot_code_buffer)(); + ((noretfun_t) reboot_code_buffer)(sunway_boot_magic, + sunway_dtb_address); } diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 43d180a4fcca..eb4a37f1b86f 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -10,7 +10,6 @@ #include #include #include -#include #ifdef CONFIG_MAGIC_SYSRQ #include #include @@ -119,11 +118,6 @@ bool memblock_initialized; cpumask_t cpu_offline = CPU_MASK_NONE; -static char command_line[COMMAND_LINE_SIZE] __initdata; -#ifdef CONFIG_CMDLINE_BOOL -static char builtin_cmdline[COMMAND_LINE_SIZE] __initdata = CONFIG_CMDLINE; -#endif - /* boot_params */ /** * Keep sunway_boot_params for backward compatibility. All related code @@ -335,31 +329,6 @@ static int __init setup_cpuoffline(char *p) } early_param("cpuoffline", setup_cpuoffline); -#ifdef CONFIG_BLK_DEV_INITRD -static void * __init move_initrd(unsigned long mem_limit) -{ - void *start; - unsigned long size; - - size = initrd_end - initrd_start; - start = memblock_alloc_from(PAGE_ALIGN(size), PAGE_SIZE, 0); - if (!start || __pa(start) + size > mem_limit) { - initrd_start = initrd_end = 0; - return NULL; - } - memmove(start, (void *)initrd_start, size); - initrd_start = (unsigned long)start; - initrd_end = initrd_start + size; - pr_info("initrd moved to 0x%px\n", start); - return start; -} -#else -static void * __init move_initrd(unsigned long mem_limit) -{ - return NULL; -} -#endif - static bool __init memmap_range_valid(phys_addr_t base, phys_addr_t *size) { if (base > memblock_end_of_DRAM()) @@ -415,26 +384,9 @@ void __init process_memmap(void) } break; case memmap_initrd: - if ((base + size) > memblock_end_of_DRAM()) { - phys_addr_t old_base = base; - - base = (unsigned long) move_initrd(memblock_end_of_DRAM()); - if (!base) { - pr_err("initrd memmap region [mem %#018llx-%#018llx] extends beyond end of memory (%#018llx)\n", - old_base, old_base + size - 1, memblock_end_of_DRAM()); - break; - } - memmap_map[i].addr = base; - } - pr_info("initrd memmap region [mem %#018llx-%#018llx]\n", base, base + size - 1); - ret = memblock_reserve(base, size); - if (ret) - pr_err("reserve memmap region [mem %#018llx-%#018llx] failed\n", - base, base + size - 1); - break; case memmap_kvm: case memmap_crashkernel: - /* kvm and crashkernel are handled elsewhere, skip */ + /* initrd, kvm and crashkernel are handled elsewhere, skip */ break; case memmap_acpi: pr_err("ACPI memmap region is not supported.\n"); @@ -580,51 +532,103 @@ static int __init topology_init(void) } subsys_initcall(topology_init); -static void __init setup_machine_fdt(void) +static bool __init arch_dtb_verify(void *dt_virt, bool from_firmware) +{ + unsigned long dt_phys = __boot_pa(dt_virt); + + if (!phys_addr_valid(dt_phys)) { + pr_crit("Invalid physical DTB address 0x%lx\n", dt_phys); + return false; + } + + /* Only for non built-in DTB */ + if (from_firmware && + (dt_phys < virt_to_phys((void *)__bss_stop))) { + pr_crit("DTB has been corrupted by kernel image!\n"); + return false; + } + + return true; +} + +static void __init setup_firmware_fdt(void) { -#ifdef CONFIG_USE_OF void *dt_virt; const char *name; - /* Give a chance to select kernel builtin DTB firstly */ - if (IS_ENABLED(CONFIG_BUILTIN_DTB)) - dt_virt = (void *)__dtb_start; - else { + /** + * Use DTB provided by firmware for early initialization, regardless + * of whether a Built-in DTB configured or not. Since we need the + * boot params from firmware when using new method to pass boot params. + */ + if (sunway_boot_magic != 0xDEED2024UL) dt_virt = (void *)sunway_boot_params->dtb_start; - if (virt_to_phys(dt_virt) < virt_to_phys(__bss_stop)) { - pr_emerg("BUG: DTB has been corrupted by kernel image!\n"); - while (true) - cpu_relax(); - } + else { + pr_info("Parse boot params in DTB chosen node\n"); + dt_virt = (void *)sunway_dtb_address; } - if (!phys_addr_valid(__boot_pa(dt_virt)) || + if (!arch_dtb_verify(dt_virt, true) || !early_init_dt_scan(dt_virt)) { - pr_crit("\n" - "Error: invalid device tree blob at virtual address %px\n" - "The dtb must be 8-byte aligned and must not exceed 2 MB in size\n" - "\nPlease check your bootloader.", - dt_virt); - + pr_crit("Invalid DTB(from firmware) at virtual address 0x%lx\n", + (unsigned long)dt_virt); while (true) cpu_relax(); } name = of_flat_dt_get_machine_name(); - if (!name) - return; + if (name) + pr_info("Machine model: %s\n", name); - pr_info("Machine model: %s\n", name); + /** + * For C3B(xuelang), kernel command line always comes from + * "sunway_boot_params->cmdline". These code can be removed + * when no longer support C3B(xuelang). + */ + if (sunway_boot_magic != 0xDEED2024UL) { + if (!sunway_boot_params->cmdline) + sunway_boot_params->cmdline = (unsigned long)COMMAND_LINE; + strlcpy(boot_command_line, (char *)sunway_boot_params->cmdline, + COMMAND_LINE_SIZE); +#ifdef CONFIG_CMDLINE +#if defined(CONFIG_CMDLINE_EXTEND) + strlcat(boot_command_line, " ", COMMAND_LINE_SIZE); + strlcat(boot_command_line, CONFIG_CMDLINE, COMMAND_LINE_SIZE); +#elif defined(CONFIG_CMDLINE_FORCE) + strlcpy(boot_command_line, CONFIG_CMDLINE, COMMAND_LINE_SIZE); #else - pr_info("Kernel disable device tree support.\n"); - return; + /* No arguments from firmware, use kernel's built-in cmdline */ + if (!((char *)boot_command_line)[0]) + strlcpy(boot_command_line, CONFIG_CMDLINE, COMMAND_LINE_SIZE); #endif +#endif /* CONFIG_CMDLINE */ + } +} + +static void __init setup_builtin_fdt(void) +{ + void *dt_virt; + + dt_virt = (void *)__dtb_start; + if (!arch_dtb_verify(dt_virt, false) || + !early_init_dt_verify(dt_virt)) { + pr_crit("Invalid DTB(built-in) at virtual address 0x%lx\n", + (unsigned long)dt_virt); + while (true) + cpu_relax(); + } } -void __init device_tree_init(void) +static void __init device_tree_init(void) { - unflatten_and_copy_device_tree(); - sunway_boot_params->dtb_start = (__u64)initial_boot_params; + /** + * Built-in DTB is placed in init data, so we need + * to copy it. + */ + if (IS_ENABLED(CONFIG_BUILTIN_DTB)) + unflatten_and_copy_device_tree(); + else + unflatten_device_tree(); } static void __init setup_cpu_info(void) @@ -689,7 +693,7 @@ static void __init setup_run_mode(void) { if (rvpcr() >> VPCR_SHIFT) { static_branch_disable(&run_mode_host_key); - if (*(unsigned long *)MMSIZE & EMUL_FLAG) { + if (*(unsigned long *)MM_SIZE & EMUL_FLAG) { pr_info("run mode: emul\n"); static_branch_disable(&run_mode_guest_key); static_branch_enable(&run_mode_emul_key); @@ -721,26 +725,6 @@ static void __init setup_socket_info(void) } } -#ifdef CONFIG_BLK_DEV_INITRD -static void __init reserve_mem_for_initrd(void) -{ - int ret; - - initrd_start = sunway_boot_params->initrd_start; - if (initrd_start) { - initrd_start = __pa(initrd_start) + PAGE_OFFSET; - initrd_end = initrd_start + sunway_boot_params->initrd_size; - pr_info("Initial ramdisk at: 0x%px (%llu bytes)\n", - (void *)initrd_start, sunway_boot_params->initrd_size); - - ret = add_memmap_region(__pa(initrd_start), initrd_end - initrd_start, memmap_initrd); - if (ret) - pr_err("Add initrd area [mem %#018lx-%#018lx] to memmap region failed.\n", - __pa(initrd_start), __pa(initrd_end - 1)); - } -} -#endif /* CONFIG_BLK_DEV_INITRD */ - #ifdef CONFIG_SUBARCH_C3B #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) static int __init early_kvm_reserved_mem(char *p) @@ -787,7 +771,11 @@ setup_arch(char **cmdline_p) setup_sched_clock(); - setup_machine_fdt(); + /* Early initialization for device tree */ + setup_firmware_fdt(); + + /* Now we get the final boot_command_line */ + *cmdline_p = boot_command_line; /* Register a call for panic conditions. */ atomic_notifier_chain_register(&panic_notifier_list, @@ -795,28 +783,6 @@ setup_arch(char **cmdline_p) callback_init(); - /* command line */ - if (!sunway_boot_params->cmdline) - sunway_boot_params->cmdline = (unsigned long)COMMAND_LINE; - - strscpy(boot_command_line, (char *)sunway_boot_params->cmdline, COMMAND_LINE_SIZE); - -#if IS_ENABLED(CONFIG_CMDLINE_BOOL) -#if IS_ENABLED(CONFIG_CMDLINE_OVERRIDE) - strscpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE); - strscpy((char *)sunway_boot_params->cmdline, boot_command_line, COMMAND_LINE_SIZE); -#else - if (builtin_cmdline[0]) { - /* append builtin to boot loader cmdline */ - strlcat(boot_command_line, " ", COMMAND_LINE_SIZE); - strlcat(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE); - } -#endif /* CMDLINE_EXTEND */ -#endif - - strscpy(command_line, boot_command_line, COMMAND_LINE_SIZE); - *cmdline_p = command_line; - /* * Process command-line arguments. */ @@ -829,10 +795,6 @@ setup_arch(char **cmdline_p) reserve_mem_for_pci(); #endif -#ifdef CONFIG_BLK_DEV_INITRD - reserve_mem_for_initrd(); -#endif - sw64_memblock_init(); reserve_crashkernel(); @@ -846,6 +808,10 @@ setup_arch(char **cmdline_p) efi_init(); + /* After efi initialization, switch to Builtin-in DTB if configured */ + if (IS_ENABLED(CONFIG_BUILTIN_DTB)) + setup_builtin_fdt(); + /* Try to upgrade ACPI tables via initrd */ acpi_table_upgrade(); diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index ca761b602ab6..d33d5a83e5b1 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -11,9 +11,13 @@ #include #include #include +#include +#include +#include #include #include +#include struct mem_desc_t mem_desc; #ifndef CONFIG_NUMA @@ -157,6 +161,61 @@ void __init mem_detect(void) mem_desc.size = mem_desc.phys_size - NODE0_START; } +#ifdef CONFIG_BLK_DEV_INITRD +static void __init reserve_mem_for_initrd(void) +{ + phys_addr_t phys_initrd_start, initrd_size; + + /** + * Get initrd params from boot_params for backward + * compatibility. These code can be removed when + * no longer support C3B(xuelang). + */ + if (sunway_boot_magic != 0xDEED2024UL) { + initrd_start = sunway_boot_params->initrd_start; + if (initrd_start) { + /** + * It works regardless of whether the firmware + * passes a virtual address or a physical address. + * + * __boot_pa here is used for compatibility with + * old firmware. We can use __pa instead when no + * longer support C3B(xuelang). + */ + initrd_start = (unsigned long)__va(__boot_pa(initrd_start)); + initrd_end = initrd_start + sunway_boot_params->initrd_size; + } + } + + phys_initrd_start = __boot_pa(initrd_start); + initrd_size = initrd_end - initrd_start; + + if (!initrd_start || !initrd_size) { + pr_info("No initrd found\n"); + return; + } + + pr_info("Initial ramdisk at: 0x%lx(va)/0x%llx(pa) (%llu bytes)\n", + initrd_start, phys_initrd_start, initrd_size); + + /** + * Usually, it means that there is an error in the + * initrd params. We should check the firmware. + */ + if ((phys_initrd_start + initrd_size) > memblock_end_of_DRAM()) { + /* Disable initrd */ + initrd_start = 0; + initrd_end = 0; + pr_err("Initial ramdisk exceed DRAM limitation\n"); + return; + } + + /* Reserve initrd */ + memblock_add(phys_initrd_start, initrd_size); + memblock_reserve(phys_initrd_start, initrd_size); +} +#endif /* CONFIG_BLK_DEV_INITRD */ + void __init sw64_memblock_init(void) { memblock_add(mem_desc.base, mem_desc.size); @@ -173,14 +232,21 @@ void __init sw64_memblock_init(void) memblock_add(__pa_symbol(_text), _end - _text); memblock_reserve(__pa_symbol(_text), _end - _text); +#ifdef CONFIG_BLK_DEV_INITRD /* Make sure initrd is in memory range. */ - if (sunway_boot_params->initrd_start) { - phys_addr_t base = __boot_pa(sunway_boot_params->initrd_start); - phys_addr_t size = sunway_boot_params->initrd_size; - - memblock_add(base, size); - memblock_reserve(base, size); - } + reserve_mem_for_initrd(); +#endif + /** + * When using non built-in DTB, we need to reserve + * it but avoid using early_init_fdt_reserve_self() + * since __pa() does not work for some old firmware. + * + * We can use early_init_fdt_reserve_self() instead + * when kernel no longer support C3B. + */ + if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) + memblock_reserve(__boot_pa(initial_boot_params), + fdt_totalsize(initial_boot_params)); /* end of DRAM range may have been changed */ max_pfn = max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); diff --git a/drivers/firmware/efi/sunway-init.c b/drivers/firmware/efi/sunway-init.c index 870abc2f5afe..8557c181c28c 100644 --- a/drivers/firmware/efi/sunway-init.c +++ b/drivers/firmware/efi/sunway-init.c @@ -179,20 +179,23 @@ void __init efi_init(void) struct efi_memory_map_data data; u64 efi_system_table; - if (sunway_boot_params->efi_systab == 0) { - pr_info("System Table is not exist, disabling EFI.\n"); - return; + if (sunway_boot_magic != 0xDEED2024UL) { + /* Legacy way: UEFI information placed in struct boot_params */ + efi_system_table = sunway_boot_params->efi_systab; + + data.desc_version = sunway_boot_params->efi_memdesc_version; + data.desc_size = sunway_boot_params->efi_memdesc_size; + data.size = sunway_boot_params->efi_memmap_size; + data.phys_map = sunway_boot_params->efi_memmap; + } else { + /* UEFI information placed in FDT("chosen" node) */ + efi_system_table = efi_get_fdt_params(&data); } - /* Grab UEFI information placed in struct boot_params by stub */ - efi_system_table = sunway_boot_params->efi_systab; - if (!efi_system_table) + if (efi_system_table == 0) { + pr_info("System Table is not exist, disabling EFI.\n"); return; - - data.desc_version = sunway_boot_params->efi_memdesc_version; - data.desc_size = sunway_boot_params->efi_memdesc_size; - data.size = sunway_boot_params->efi_memmap_size; - data.phys_map = sunway_boot_params->efi_memmap; + } if (efi_memmap_init_early(&data) < 0) { /* @@ -214,8 +217,6 @@ void __init efi_init(void) reserve_regions(); - memblock_reserve(sunway_boot_params->efi_memmap & PAGE_MASK, - PAGE_ALIGN(sunway_boot_params->efi_memmap_size + - (sunway_boot_params->efi_memmap & ~PAGE_MASK))); - + memblock_reserve(data.phys_map & PAGE_MASK, + PAGE_ALIGN(data.size + (data.phys_map & ~PAGE_MASK))); } -- Gitee From 72517e19d0852eade4808592653201a901dbb48e Mon Sep 17 00:00:00 2001 From: Jing Li Date: Sat, 30 Mar 2024 13:57:54 +0800 Subject: [PATCH 155/524] sw64: improve the verification logic of DTB Provide better compatibility when using legacy boot_params. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index eb4a37f1b86f..ba3402d2319d 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -543,10 +543,8 @@ static bool __init arch_dtb_verify(void *dt_virt, bool from_firmware) /* Only for non built-in DTB */ if (from_firmware && - (dt_phys < virt_to_phys((void *)__bss_stop))) { - pr_crit("DTB has been corrupted by kernel image!\n"); - return false; - } + (dt_phys < virt_to_phys((void *)__bss_stop))) + pr_warn("DTB(from firmware) may have been corrupted by kernel image!\n"); return true; } @@ -556,14 +554,18 @@ static void __init setup_firmware_fdt(void) void *dt_virt; const char *name; - /** - * Use DTB provided by firmware for early initialization, regardless - * of whether a Built-in DTB configured or not. Since we need the - * boot params from firmware when using new method to pass boot params. - */ - if (sunway_boot_magic != 0xDEED2024UL) + if (sunway_boot_magic != 0xDEED2024UL) { + /* Bypass DTB from firmware if built-in DTB configured */ + if (IS_ENABLED(CONFIG_BUILTIN_DTB)) + goto cmd_handle; dt_virt = (void *)sunway_boot_params->dtb_start; - else { + } else { + /** + * Use DTB provided by firmware for early initialization, + * regardless of whether a Built-in DTB configured or not. + * Since we need the boot params from firmware when using + * new method to pass boot params. + */ pr_info("Parse boot params in DTB chosen node\n"); dt_virt = (void *)sunway_dtb_address; } @@ -572,14 +574,16 @@ static void __init setup_firmware_fdt(void) !early_init_dt_scan(dt_virt)) { pr_crit("Invalid DTB(from firmware) at virtual address 0x%lx\n", (unsigned long)dt_virt); + while (true) cpu_relax(); } name = of_flat_dt_get_machine_name(); if (name) - pr_info("Machine model: %s\n", name); + pr_info("DTB(from firmware): Machine model: %s\n", name); +cmd_handle: /** * For C3B(xuelang), kernel command line always comes from * "sunway_boot_params->cmdline". These code can be removed @@ -608,6 +612,7 @@ static void __init setup_firmware_fdt(void) static void __init setup_builtin_fdt(void) { void *dt_virt; + const char *name; dt_virt = (void *)__dtb_start; if (!arch_dtb_verify(dt_virt, false) || @@ -617,6 +622,13 @@ static void __init setup_builtin_fdt(void) while (true) cpu_relax(); } + + /* Parse {size,address}-cells */ + early_init_dt_scan_root(); + + name = of_flat_dt_get_machine_name(); + if (name) + pr_info("DTB(built-in): Machine model: %s\n", name); } static void __init device_tree_init(void) -- Gitee From 7a55cb05af50720d371abcb5ecadf2be7bfeed52 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 18 Mar 2024 14:49:25 +0800 Subject: [PATCH 156/524] sw64: irq: fix compile error when PINTC or LPC-INTC is not set Fix compile error when CONFIG_SW64_PINTC=n or CONFIG_SW64_LPC_INTC=n. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/irq.h | 23 +++++++++++++++++++++-- drivers/irqchip/irq-sunway-cpu.c | 7 +++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/include/asm/irq.h b/arch/sw_64/include/asm/irq.h index e393162d928b..6a4abfc9687b 100644 --- a/arch/sw_64/include/asm/irq.h +++ b/arch/sw_64/include/asm/irq.h @@ -37,11 +37,30 @@ struct acpi_madt_sw_lpc_intc; extern int __init sw64_add_gsi_domain_map(u32 gsi_base, u32 gsi_count, struct fwnode_handle *handle); -extern int __init pintc_acpi_init(struct irq_domain *parent, - struct acpi_madt_sw_pintc *pintc); + extern int __init msic_acpi_init(struct irq_domain *parent, struct acpi_madt_sw_msic *msic); + +#ifdef CONFIG_SW64_PINTC +extern int __init pintc_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_pintc *pintc); +#else +static inline int __init pintc_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_pintc *pintc) +{ + return 0; +} +#endif + +#ifdef CONFIG_SW64_LPC_INTC extern int __init lpc_intc_acpi_init(struct irq_domain *parent, struct acpi_madt_sw_lpc_intc *lpc_intc); +#else +static inline int __init lpc_intc_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_lpc_intc *lpc_intc) +{ + return 0; +} +#endif #endif /* _ASM_SW64_IRQ_H */ diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 2f4880036a2c..dca5020a789d 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -133,6 +133,7 @@ static void handle_nmi_int(void) pr_info("enter nmi int\n"); } +#ifdef CONFIG_SW64_PINTC static void handle_dev_int(struct pt_regs *regs) { unsigned long config_val, val, stat; @@ -152,6 +153,12 @@ static void handle_dev_int(struct pt_regs *regs) sw64_io_write(node, DEV_INT_CONFIG, config_val); } +#else +static void handle_dev_int(struct pt_regs *regs) +{ + pr_crit(PREFIX "the child controller PINTC is not configured!\n"); +} +#endif int pme_state; -- Gitee From b96e957b9eaee6011583c5d331618e6ac9eb5c6d Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Tue, 2 Apr 2024 10:56:37 +0800 Subject: [PATCH 157/524] sw64: fix secondary cpu bring up bug If smp_rcb->ready is set after secondary cpu receives reset signal, the secondary cpu may fail to wake up. Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index f3ac8a899976..a57d05fe700f 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -139,6 +139,9 @@ static int secondary_cpu_start(int cpuid, struct task_struct *idle) set_secondary_ready(cpuid); + /* send reset signal */ + reset_cpu(cpuid); + /* Wait 10 seconds for secondary cpu. */ timeout = jiffies + 10*HZ; while (time_before(jiffies, timeout)) { @@ -318,7 +321,6 @@ int vt_cpu_up(unsigned int cpu, struct task_struct *tidle) wmb(); smp_rcb->ready = 0; /* irq must be disabled before reset vCPU */ - reset_cpu(cpu); smp_boot_one_cpu(cpu, tidle); return cpu_online(cpu) ? 0 : -EIO; @@ -340,13 +342,6 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) /* send wake up signal */ send_wakeup_interrupt(cpu); #endif - /* send reset signal */ - if (is_in_host()) { - reset_cpu(cpu); - } else { - while (1) - cpu_relax(); - } smp_boot_one_cpu(cpu, tidle); #ifdef CONFIG_SUBARCH_C3B -- Gitee From b620b2aa52517eec6858ffd7045e7c8975296b2b Mon Sep 17 00:00:00 2001 From: He Sheng Date: Thu, 28 Mar 2024 10:31:56 +0800 Subject: [PATCH 158/524] sw64: rename static key and related simd libraries No functional change intended. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hw_init.h | 2 +- arch/sw_64/kernel/setup.c | 2 +- arch/sw_64/lib/Makefile | 12 +++++------ .../{deep-clear_page.S => clear_page_simd.S} | 0 ..._user-no_una_check.S => clear_user_simd.S} | 10 +++++----- ...rce_ali_addr.S => clear_user_simd_align.S} | 10 +++++----- .../{deep-copy_page.S => copy_page_simd.S} | 0 ..._ali_addr.S => copy_simd_align_template.S} | 0 ...te-no_una_check.S => copy_simd_template.S} | 0 ...y_user-no_una_check.S => copy_user_simd.S} | 10 +++++----- ...orce_ali_addr.S => copy_user_simd_align.S} | 10 +++++----- ...ep-memcpy-no_una_check.S => memcpy_simd.S} | 10 +++++----- ...y-force_ali_addr.S => memcpy_simd_align.S} | 10 +++++----- ...ep-memset-no_una_check.S => memset_simd.S} | 10 +++++----- ...t-force_ali_addr.S => memset_simd_align.S} | 10 +++++----- ...e_ali_addr.S => set_simd_align_template.S} | 2 +- ...ate-no_una_check.S => set_simd_template.S} | 0 arch/sw_64/lib/string.c | 20 +++++++++---------- arch/sw_64/lib/uaccess.c | 20 +++++++++---------- 19 files changed, 69 insertions(+), 69 deletions(-) rename arch/sw_64/lib/{deep-clear_page.S => clear_page_simd.S} (100%) rename arch/sw_64/lib/{deep-clear_user-no_una_check.S => clear_user_simd.S} (83%) rename arch/sw_64/lib/{deep-clear_user-force_ali_addr.S => clear_user_simd_align.S} (82%) rename arch/sw_64/lib/{deep-copy_page.S => copy_page_simd.S} (100%) rename arch/sw_64/lib/{deep-copy_template-force_ali_addr.S => copy_simd_align_template.S} (100%) rename arch/sw_64/lib/{deep-copy_template-no_una_check.S => copy_simd_template.S} (100%) rename arch/sw_64/lib/{deep-copy_user-no_una_check.S => copy_user_simd.S} (79%) rename arch/sw_64/lib/{deep-copy_user-force_ali_addr.S => copy_user_simd_align.S} (79%) rename arch/sw_64/lib/{deep-memcpy-no_una_check.S => memcpy_simd.S} (59%) rename arch/sw_64/lib/{deep-memcpy-force_ali_addr.S => memcpy_simd_align.S} (57%) rename arch/sw_64/lib/{deep-memset-no_una_check.S => memset_simd.S} (78%) rename arch/sw_64/lib/{deep-memset-force_ali_addr.S => memset_simd_align.S} (77%) rename arch/sw_64/lib/{deep-set_template-force_ali_addr.S => set_simd_align_template.S} (97%) rename arch/sw_64/lib/{deep-set_template-no_una_check.S => set_simd_template.S} (100%) diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h index ab0e2ee4c9a6..5af3336d5fd6 100644 --- a/arch/sw_64/include/asm/hw_init.h +++ b/arch/sw_64/include/asm/hw_init.h @@ -99,7 +99,7 @@ DECLARE_STATIC_KEY_TRUE(run_mode_host_key); DECLARE_STATIC_KEY_FALSE(run_mode_guest_key); DECLARE_STATIC_KEY_FALSE(run_mode_emul_key); -DECLARE_STATIC_KEY_FALSE(core_hw_una_enabled); +DECLARE_STATIC_KEY_FALSE(hw_una_enabled); #define is_in_host() static_branch_likely(&run_mode_host_key) #define is_in_guest() static_branch_unlikely(&run_mode_guest_key) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index ba3402d2319d..cb69314d7684 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -108,7 +108,7 @@ DEFINE_STATIC_KEY_TRUE(run_mode_host_key); DEFINE_STATIC_KEY_FALSE(run_mode_guest_key); DEFINE_STATIC_KEY_FALSE(run_mode_emul_key); -DEFINE_STATIC_KEY_FALSE(core_hw_una_enabled); +DEFINE_STATIC_KEY_FALSE(hw_una_enabled); struct cpu_desc_t cpu_desc; struct socket_desc_t socket_desc[MAX_NUMSOCKETS]; diff --git a/arch/sw_64/lib/Makefile b/arch/sw_64/lib/Makefile index 8d3bc1fdd604..a37d1ac0ca07 100644 --- a/arch/sw_64/lib/Makefile +++ b/arch/sw_64/lib/Makefile @@ -20,18 +20,18 @@ lib-y = __divlu.o __remlu.o __divwu.o __remwu.o \ uaccess.o lib-clear_page-y := clear_page.o -lib-clear_page-$(CONFIG_DEEP_CLEAR_PAGE) := deep-clear_page.o +lib-clear_page-$(CONFIG_DEEP_CLEAR_PAGE) := clear_page_simd.o -lib-clear_user-y := clear_user.o deep-clear_user-no_una_check.o deep-clear_user-force_ali_addr.o +lib-clear_user-y := clear_user.o clear_user_simd.o clear_user_simd_align.o lib-copy_page-y := copy_page.o -lib-copy_page-$(CONFIG_DEEP_COPY_PAGE) := deep-copy_page.o +lib-copy_page-$(CONFIG_DEEP_COPY_PAGE) := copy_page_simd.o -lib-copy_user-y := copy_user.o deep-copy_user-no_una_check.o deep-copy_user-force_ali_addr.o +lib-copy_user-y := copy_user.o copy_user_simd.o copy_user_simd_align.o -lib-memcpy-y := memcpy.o deep-memcpy-no_una_check.o deep-memcpy-force_ali_addr.o +lib-memcpy-y := memcpy.o memcpy_simd.o memcpy_simd_align.o -lib-memset-y := memset.o deep-memset-no_una_check.o deep-memset-force_ali_addr.o +lib-memset-y := memset.o memset_simd.o memset_simd_align.o lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o diff --git a/arch/sw_64/lib/deep-clear_page.S b/arch/sw_64/lib/clear_page_simd.S similarity index 100% rename from arch/sw_64/lib/deep-clear_page.S rename to arch/sw_64/lib/clear_page_simd.S diff --git a/arch/sw_64/lib/deep-clear_user-no_una_check.S b/arch/sw_64/lib/clear_user_simd.S similarity index 83% rename from arch/sw_64/lib/deep-clear_user-no_una_check.S rename to arch/sw_64/lib/clear_user_simd.S index da25c6be66fb..a8d6eebfc4f8 100644 --- a/arch/sw_64/lib/deep-clear_user-no_una_check.S +++ b/arch/sw_64/lib/clear_user_simd.S @@ -28,9 +28,9 @@ * $18: bytes left to copy * */ - .globl ____clear_user_simd_no_una_check - .ent ____clear_user_simd_no_una_check -____clear_user_simd_no_una_check: + .globl ____clear_user_simd + .ent ____clear_user_simd +____clear_user_simd: .prologue 0 bis $31, $31, $7 mov $17, $18 @@ -38,7 +38,7 @@ ____clear_user_simd_no_una_check: #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -#include "deep-set_template-no_una_check.S" +#include "set_simd_template.S" 255: bis $31, $18, $0 beq $7, $return @@ -51,4 +51,4 @@ $return: csrw $6, CSR_WR_FREGS #endif ret - .end ____clear_user_simd_no_una_check + .end ____clear_user_simd diff --git a/arch/sw_64/lib/deep-clear_user-force_ali_addr.S b/arch/sw_64/lib/clear_user_simd_align.S similarity index 82% rename from arch/sw_64/lib/deep-clear_user-force_ali_addr.S rename to arch/sw_64/lib/clear_user_simd_align.S index 78d989609e12..3c1cd06f742f 100644 --- a/arch/sw_64/lib/deep-clear_user-force_ali_addr.S +++ b/arch/sw_64/lib/clear_user_simd_align.S @@ -28,9 +28,9 @@ * $18: bytes left to copy * */ - .globl ____clear_user_simd_force_ali_addr - .ent ____clear_user_simd_force_ali_addr -____clear_user_simd_force_ali_addr: + .globl ____clear_user_simd_align + .ent ____clear_user_simd_align +____clear_user_simd_align: .prologue 0 bis $31, $31, $7 mov $17, $18 @@ -38,7 +38,7 @@ ____clear_user_simd_force_ali_addr: #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -#include "deep-set_template-force_ali_addr.S" +#include "set_simd_align_template.S" 255: bis $31, $18, $0 beq $7, $return @@ -51,4 +51,4 @@ $return: csrw $6, CSR_WR_FREGS #endif ret - .end ____clear_user_simd_force_ali_addr + .end ____clear_user_simd_align diff --git a/arch/sw_64/lib/deep-copy_page.S b/arch/sw_64/lib/copy_page_simd.S similarity index 100% rename from arch/sw_64/lib/deep-copy_page.S rename to arch/sw_64/lib/copy_page_simd.S diff --git a/arch/sw_64/lib/deep-copy_template-force_ali_addr.S b/arch/sw_64/lib/copy_simd_align_template.S similarity index 100% rename from arch/sw_64/lib/deep-copy_template-force_ali_addr.S rename to arch/sw_64/lib/copy_simd_align_template.S diff --git a/arch/sw_64/lib/deep-copy_template-no_una_check.S b/arch/sw_64/lib/copy_simd_template.S similarity index 100% rename from arch/sw_64/lib/deep-copy_template-no_una_check.S rename to arch/sw_64/lib/copy_simd_template.S diff --git a/arch/sw_64/lib/deep-copy_user-no_una_check.S b/arch/sw_64/lib/copy_user_simd.S similarity index 79% rename from arch/sw_64/lib/deep-copy_user-no_una_check.S rename to arch/sw_64/lib/copy_user_simd.S index cadd5e0f31c1..02082a2f2e1c 100644 --- a/arch/sw_64/lib/deep-copy_user-no_una_check.S +++ b/arch/sw_64/lib/copy_user_simd.S @@ -21,16 +21,16 @@ * $18: bytes left to copy * */ - .globl ____copy_user_simd_no_una_check - .ent ____copy_user_simd_no_una_check -____copy_user_simd_no_una_check: + .globl ____copy_user_simd + .ent ____copy_user_simd +____copy_user_simd: .prologue 0 .set noreorder bis $31, $31, $7 #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -#include "deep-copy_template-no_una_check.S" +#include "copy_simd_template.S" 255: bis $31, $18, $0 beq $7, $return @@ -45,4 +45,4 @@ $return: csrw $6, CSR_WR_FREGS #endif ret - .end ____copy_user_simd_no_una_check + .end ____copy_user_simd diff --git a/arch/sw_64/lib/deep-copy_user-force_ali_addr.S b/arch/sw_64/lib/copy_user_simd_align.S similarity index 79% rename from arch/sw_64/lib/deep-copy_user-force_ali_addr.S rename to arch/sw_64/lib/copy_user_simd_align.S index febb46f9b691..c459ca0f95ae 100644 --- a/arch/sw_64/lib/deep-copy_user-force_ali_addr.S +++ b/arch/sw_64/lib/copy_user_simd_align.S @@ -21,16 +21,16 @@ * $18: bytes left to copy * */ - .globl ____copy_user_simd_force_ali_addr - .ent ____copy_user_simd_force_ali_addr -____copy_user_simd_force_ali_addr: + .globl ____copy_user_simd_align + .ent ____copy_user_simd_align +____copy_user_simd_align: .prologue 0 .set noreorder bis $31, $31, $7 #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -#include "deep-copy_template-force_ali_addr.S" +#include "copy_simd_align_template.S" 255: bis $31, $18, $0 beq $7, $return @@ -49,4 +49,4 @@ $return: csrw $6, CSR_WR_FREGS #endif ret - .end ____copy_user_simd_force_ali_addr + .end ____copy_user_simd_align diff --git a/arch/sw_64/lib/deep-memcpy-no_una_check.S b/arch/sw_64/lib/memcpy_simd.S similarity index 59% rename from arch/sw_64/lib/deep-memcpy-no_una_check.S rename to arch/sw_64/lib/memcpy_simd.S index c7ded71b1ade..f42d8efb00b6 100644 --- a/arch/sw_64/lib/deep-memcpy-no_una_check.S +++ b/arch/sw_64/lib/memcpy_simd.S @@ -5,19 +5,19 @@ #define FIXUP_LDST(x, y) \ x, y - .globl ____memcpy_simd_no_una_check - .ent ____memcpy_simd_no_una_check -____memcpy_simd_no_una_check: + .globl ____memcpy_simd + .ent ____memcpy_simd +____memcpy_simd: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -#include "deep-copy_template-no_una_check.S" +#include "copy_simd_template.S" 255: #ifdef CONFIG_SUBARCH_C4 csrw $6, CSR_WR_FREGS #endif ret - .end ____memcpy_simd_no_una_check + .end ____memcpy_simd diff --git a/arch/sw_64/lib/deep-memcpy-force_ali_addr.S b/arch/sw_64/lib/memcpy_simd_align.S similarity index 57% rename from arch/sw_64/lib/deep-memcpy-force_ali_addr.S rename to arch/sw_64/lib/memcpy_simd_align.S index 7e990bda936c..6450c96a2406 100644 --- a/arch/sw_64/lib/deep-memcpy-force_ali_addr.S +++ b/arch/sw_64/lib/memcpy_simd_align.S @@ -5,19 +5,19 @@ #define FIXUP_LDST(x, y) \ x, y - .globl ____memcpy_simd_force_ali_addr - .ent ____memcpy_simd_force_ali_addr -____memcpy_simd_force_ali_addr: + .globl ____memcpy_simd_align + .ent ____memcpy_simd_align +____memcpy_simd_align: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -#include "deep-copy_template-force_ali_addr.S" +#include "copy_simd_align_template.S" 255: #ifdef CONFIG_SUBARCH_C4 csrw $6, CSR_WR_FREGS #endif ret - .end ____memcpy_simd_force_ali_addr + .end ____memcpy_simd_align diff --git a/arch/sw_64/lib/deep-memset-no_una_check.S b/arch/sw_64/lib/memset_simd.S similarity index 78% rename from arch/sw_64/lib/deep-memset-no_una_check.S rename to arch/sw_64/lib/memset_simd.S index 91525bd67790..03d301cccf7a 100644 --- a/arch/sw_64/lib/deep-memset-no_una_check.S +++ b/arch/sw_64/lib/memset_simd.S @@ -35,20 +35,20 @@ .text .align 4 - .globl ____constant_c_memset_simd_no_una_check - .ent ____constant_c_memset_simd_no_una_check + .globl ____constant_c_memset_simd + .ent ____constant_c_memset_simd .frame $30, 0, $26, 0 .prologue 0 -____constant_c_memset_simd_no_una_check: +____constant_c_memset_simd: bis $31, $31, $7 bis $31, $16, $0 #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -#include "deep-set_template-no_una_check.S" +#include "set_simd_template.S" 255: #ifdef CONFIG_SUBARCH_C4 csrw $6, CSR_WR_FREGS #endif ret - .end ____constant_c_memset_simd_no_una_check + .end ____constant_c_memset_simd diff --git a/arch/sw_64/lib/deep-memset-force_ali_addr.S b/arch/sw_64/lib/memset_simd_align.S similarity index 77% rename from arch/sw_64/lib/deep-memset-force_ali_addr.S rename to arch/sw_64/lib/memset_simd_align.S index cbb0130100dd..f5329cd6125e 100644 --- a/arch/sw_64/lib/deep-memset-force_ali_addr.S +++ b/arch/sw_64/lib/memset_simd_align.S @@ -35,20 +35,20 @@ .text .align 4 - .globl ____constant_c_memset_simd_force_ali_addr - .ent ____constant_c_memset_simd_force_ali_addr + .globl ____constant_c_memset_simd_align + .ent ____constant_c_memset_simd_align .frame $30, 0, $26, 0 .prologue 0 -____constant_c_memset_simd_force_ali_addr: +____constant_c_memset_simd_align: bis $31, $31, $7 bis $31, $16, $0 #ifdef CONFIG_SUBARCH_C4 csrr $6, CSR_WR_FREGS #endif -#include "deep-set_template-force_ali_addr.S" +#include "set_simd_align_template.S" 255: #ifdef CONFIG_SUBARCH_C4 csrw $6, CSR_WR_FREGS #endif ret - .end ____constant_c_memset_simd_force_ali_addr + .end ____constant_c_memset_simd_align diff --git a/arch/sw_64/lib/deep-set_template-force_ali_addr.S b/arch/sw_64/lib/set_simd_align_template.S similarity index 97% rename from arch/sw_64/lib/deep-set_template-force_ali_addr.S rename to arch/sw_64/lib/set_simd_align_template.S index d581e2eb35d7..fa77cacf684f 100644 --- a/arch/sw_64/lib/deep-set_template-force_ali_addr.S +++ b/arch/sw_64/lib/set_simd_align_template.S @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * template for memcpy and copy_user with SIMD + * template for memset and clear_user with SIMD * * $7: SIMD status * 0: not in simd loop diff --git a/arch/sw_64/lib/deep-set_template-no_una_check.S b/arch/sw_64/lib/set_simd_template.S similarity index 100% rename from arch/sw_64/lib/deep-set_template-no_una_check.S rename to arch/sw_64/lib/set_simd_template.S diff --git a/arch/sw_64/lib/string.c b/arch/sw_64/lib/string.c index ef7439b3715b..e1ff596d8667 100644 --- a/arch/sw_64/lib/string.c +++ b/arch/sw_64/lib/string.c @@ -5,18 +5,18 @@ #include extern void *____memcpy_sisd(void *dest, const void *src, size_t n); -extern void *____memcpy_simd_no_una_check(void *dest, const void *src, size_t n); -extern void *____memcpy_simd_force_ali_addr(void *dest, const void *src, size_t n); +extern void *____memcpy_simd(void *dest, const void *src, size_t n); +extern void *____memcpy_simd_align(void *dest, const void *src, size_t n); static inline void *____memcpy(void *dest, const void *src, size_t n) { if (!IS_ENABLED(CONFIG_DEEP_MEMCPY)) return ____memcpy_sisd(dest, src, n); - if (static_branch_likely(&core_hw_una_enabled)) - return ____memcpy_simd_no_una_check(dest, src, n); + if (static_branch_likely(&hw_una_enabled)) + return ____memcpy_simd(dest, src, n); else - return ____memcpy_simd_force_ali_addr(dest, src, n); + return ____memcpy_simd_align(dest, src, n); } void *memcpy(void *dest, const void *src, size_t n) @@ -33,18 +33,18 @@ void *__memcpy(void *dest, const void *src, size_t n) EXPORT_SYMBOL(__memcpy); extern void *____constant_c_memset_sisd(void *s, unsigned long c, size_t n); -extern void *____constant_c_memset_simd_no_una_check(void *s, unsigned long c, size_t n); -extern void *____constant_c_memset_simd_force_ali_addr(void *s, unsigned long c, size_t n); +extern void *____constant_c_memset_simd(void *s, unsigned long c, size_t n); +extern void *____constant_c_memset_simd_align(void *s, unsigned long c, size_t n); static inline void *____constant_c_memset(void *s, unsigned long c, size_t n) { if (!IS_ENABLED(CONFIG_DEEP_MEMSET)) return ____constant_c_memset_sisd(s, c, n); - if (static_branch_likely(&core_hw_una_enabled)) - return ____constant_c_memset_simd_no_una_check(s, c, n); + if (static_branch_likely(&hw_una_enabled)) + return ____constant_c_memset_simd(s, c, n); else - return ____constant_c_memset_simd_force_ali_addr(s, c, n); + return ____constant_c_memset_simd_align(s, c, n); } void *__constant_c_memset(void *s, unsigned long c, size_t n) diff --git a/arch/sw_64/lib/uaccess.c b/arch/sw_64/lib/uaccess.c index f8efa5ad853d..fd28379fdc22 100644 --- a/arch/sw_64/lib/uaccess.c +++ b/arch/sw_64/lib/uaccess.c @@ -5,33 +5,33 @@ #include extern long ____copy_user_sisd(void *to, const void *from, long len); -extern long ____copy_user_simd_no_una_check(void *to, const void *from, long len); -extern long ____copy_user_simd_force_ali_addr(void *to, const void *from, long len); +extern long ____copy_user_simd(void *to, const void *from, long len); +extern long ____copy_user_simd_align(void *to, const void *from, long len); long __copy_user(void *to, const void *from, long len) { if (!IS_ENABLED(CONFIG_DEEP_COPY_USER)) return ____copy_user_sisd(to, from, len); - if (static_branch_likely(&core_hw_una_enabled)) - return ____copy_user_simd_no_una_check(to, from, len); + if (static_branch_likely(&hw_una_enabled)) + return ____copy_user_simd(to, from, len); else - return ____copy_user_simd_force_ali_addr(to, from, len); + return ____copy_user_simd_align(to, from, len); } EXPORT_SYMBOL(__copy_user); extern long ____clear_user_sisd(void __user *to, long len); -extern long ____clear_user_simd_no_una_check(void __user *to, long len); -extern long ____clear_user_simd_force_ali_addr(void __user *to, long len); +extern long ____clear_user_simd(void __user *to, long len); +extern long ____clear_user_simd_align(void __user *to, long len); long __clear_user(void __user *to, long len) { if (!IS_ENABLED(CONFIG_DEEP_CLEAR_USER)) return ____clear_user_sisd(to, len); - if (static_branch_likely(&core_hw_una_enabled)) - return ____clear_user_simd_no_una_check(to, len); + if (static_branch_likely(&hw_una_enabled)) + return ____clear_user_simd(to, len); else - return ____clear_user_simd_force_ali_addr(to, len); + return ____clear_user_simd_align(to, len); } EXPORT_SYMBOL(__clear_user); -- Gitee From d7a655d5e951bdb11cd7272c0363b999861855fb Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Tue, 2 Apr 2024 16:59:29 +0800 Subject: [PATCH 159/524] sw64: redesign run mode setup method For C4, run mode is determined by the lower two bits of sunway_boot_magic passed in by firmware. Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index cb69314d7684..564fc74de2b3 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -701,9 +701,10 @@ static void __init setup_cpu_info(void) } } +#ifdef CONFIG_SUBARCH_C3B static void __init setup_run_mode(void) { - if (rvpcr() >> VPCR_SHIFT) { + if (*(unsigned long *)MMSIZE) { static_branch_disable(&run_mode_host_key); if (*(unsigned long *)MM_SIZE & EMUL_FLAG) { pr_info("run mode: emul\n"); @@ -722,6 +723,29 @@ static void __init setup_run_mode(void) static_branch_disable(&run_mode_emul_key); } } +#elif CONFIG_SUBARCH_C4 +static void __init setup_run_mode(void) +{ + if (rvpcr() >> VPCR_SHIFT) { + pr_info("run mode: guest\n"); + static_branch_disable(&run_mode_host_key); + static_branch_disable(&run_mode_emul_key); + static_branch_enable(&run_mode_guest_key); + } else if (sunway_boot_magic == 0xA2024) { + pr_info("run mode: emul\n"); + static_branch_disable(&run_mode_host_key); + static_branch_disable(&run_mode_guest_key); + static_branch_enable(&run_mode_emul_key); + sunway_boot_magic = 0xDEED2024; + } else { + pr_info("run mode: host\n"); + static_branch_disable(&run_mode_guest_key); + static_branch_disable(&run_mode_emul_key); + static_branch_enable(&run_mode_host_key); + } + +} +#endif static void __init setup_socket_info(void) { -- Gitee From 9bb1a98c1ac5dd86c9f2ec9fb300576cf6268b9e Mon Sep 17 00:00:00 2001 From: Zhou Xuemei Date: Mon, 8 Apr 2024 11:40:59 +0000 Subject: [PATCH 160/524] sw64: ahci: disable parallel bus scan On sw64 platform, if scan this host(type MARVELL 9230) in parallel, ahci ports reset failed as follows: ata4: link is slow to respond, please be patient (ready=0) ata8.00: qc timeout (cmd 0xa1) ata8.00: failed to IDENTIFY (I/O error, err_mask=0x4) ata3: link is slow to respond, please be patient (ready=0) ata8: SATA link up 1.5 Gbps (SStatus 113 SControl 300) ata4: COMRESET failed (errno=-16) ata3: COMRESET failed (errno=-16) ata4: link is slow to respond, please be patient (ready=0) ata3: link is slow to respond, please be patient (ready=0) ata8.00: qc timeout (cmd 0xa1) ata8.00: failed to IDENTIFY (I/O error, err_mask=0x4) ata8: SATA link up 1.5 Gbps (SStatus 113 SControl 300) ata4: COMRESET failed (errno=-16) ata3: COMRESET failed (errno=-16) ata3: link is slow to respond, please be patient (ready=0) ata4: link is slow to respond, please be patient (ready=0) ata8.00: qc timeout (cmd 0xa1) ata8.00: failed to IDENTIFY (I/O error, err_mask=0x4) ata8: SATA link up 1.5 Gbps (SStatus 113 SControl 300) ata4: COMRESET failed (errno=-16) ata4: limiting SATA link speed to 3.0 Gbps ata3: COMRESET failed (errno=-16) ata3: limiting SATA link speed to 3.0 Gbps ata4: COMRESET failed (errno=-16) ata4: reset failed, giving up ata3: COMRESET failed (errno=-16) ata3: reset failed, giving up To work around this problem, we disable it for synchronous scanning. Signed-off-by: Zhou Xuemei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/ata/ahci.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 98104d0b842b..7b61915a141e 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1989,10 +1989,12 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) } hpriv->irq = pci_irq_vector(pdev, 0); - if (!(hpriv->cap & HOST_CAP_SSS) || ahci_ignore_sss) - host->flags |= ATA_HOST_PARALLEL_SCAN; - else - dev_info(&pdev->dev, "SSS flag set, parallel bus scan disabled\n"); + if (!IS_ENABLED(CONFIG_SW64)) { + if (!(hpriv->cap & HOST_CAP_SSS) || ahci_ignore_sss) + host->flags |= ATA_HOST_PARALLEL_SCAN; + else + dev_info(&pdev->dev, "SSS flag set, parallel bus scan disabled\n"); + } if (!(hpriv->cap & HOST_CAP_PART)) host->flags |= ATA_HOST_NO_PART; -- Gitee From a4b18ebbe4e9edc3371d6732625f6e8b0b80af77 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 9 Apr 2024 19:45:57 +0800 Subject: [PATCH 161/524] sw64: acpi: disable ACPI for xuelang Enabling ACPI on xuelang platform requires the latest firmware, otherwise kernel may fail to boot. To avoid potential risks, disable ACPI for xuelang platform. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/acpi.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index af80ae304948..c50ae846ff62 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -361,6 +361,12 @@ static int __init acpi_process_madt_sw_cintc(void) void __init acpi_boot_table_init(void) { + if (IS_ENABLED(CONFIG_SUBARCH_C3B)) { + pr_info(PREFIX "Current platform does not support ACPI\n"); + disable_acpi(); + return; + } + /** * ACPI is disabled by default. * ACPI is only enabled when firmware passes ACPI table @@ -375,7 +381,6 @@ void __init acpi_boot_table_init(void) if (acpi_disabled) return; - pr_warn("Currently, ACPI is an experimental feature!\n"); if (acpi_table_init()) { pr_err("Failed to init ACPI tables\n"); disable_acpi(); -- Gitee From c350006b4317e2645aa7956032b50c8c48b6ef8f Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 9 Apr 2024 15:03:31 +0800 Subject: [PATCH 162/524] sw64: fix compile error of MMSIZE The MMSIZE macro has been renamed to MM_SIZE to avoid conflicts with MMSIZE in fdtparams.c. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 564fc74de2b3..ad1e90876d5e 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -704,7 +704,7 @@ static void __init setup_cpu_info(void) #ifdef CONFIG_SUBARCH_C3B static void __init setup_run_mode(void) { - if (*(unsigned long *)MMSIZE) { + if (*(unsigned long *)MM_SIZE) { static_branch_disable(&run_mode_host_key); if (*(unsigned long *)MM_SIZE & EMUL_FLAG) { pr_info("run mode: emul\n"); -- Gitee From 61662926a605e4d34d4bc37adf3aad1fedf24742 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 3 Apr 2024 09:30:21 +0800 Subject: [PATCH 163/524] sw64: get mclk and external clk from firmware Starting from junzhang, kernel will not directly access Motherboard Configuration Tables to get mclk and external clk. Instead, kernel get mclk and external clk from firmware. This commit will not break backward compatibility. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/setup.h | 16 ++++++-- arch/sw_64/kernel/setup.c | 66 ++++++++++++++++++++++++++++++++ drivers/clocksource/timer-sw64.c | 7 +++- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/include/asm/setup.h b/arch/sw_64/include/asm/setup.h index 2d557b349555..0a2edf9af3ca 100644 --- a/arch/sw_64/include/asm/setup.h +++ b/arch/sw_64/include/asm/setup.h @@ -38,14 +38,22 @@ #define INITRD_START_OFF (0x10000UL - 0xA100UL) #define INITRD_SIZE_OFF (0x10000UL - 0xA108UL) -/* Motherboard Configuration Tables */ -#define MB_CONFIG_START 0x908000 -#define MB_MCLK (MB_CONFIG_START + 0x1) -#define MB_EXTCLK (MB_CONFIG_START + 0x11) +/** + * Motherboard Configuration Tables + * + * Starting from junzhang, we will not directly access Motherboard + * Configuration Tables in kernel. These macros and related code can + * be removed when kernel no longer support C3B(xuelang). + */ +#define MB_CONFIG_START 0x908000 +#define MB_MCLK (MB_CONFIG_START + 0x1) +#define MB_EXTCLK (MB_CONFIG_START + 0x11) #ifndef __ASSEMBLY__ #include extern struct boot_params *sunway_boot_params; +extern u64 sunway_mclk_hz; +extern u64 sunway_extclk_hz; #endif #endif /* _ASM_SW64_SETUP_H */ diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index ad1e90876d5e..b154165dcb24 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -19,6 +19,7 @@ #endif #include #include +#include #include #include #include @@ -131,6 +132,9 @@ EXPORT_SYMBOL(sunway_boot_magic); unsigned long sunway_dtb_address; EXPORT_SYMBOL(sunway_dtb_address); +u64 sunway_mclk_hz; +u64 sunway_extclk_hz; + /* * The format of "screen_info" is strange, and due to early * i386-setup code. This is just enough to make the console @@ -549,6 +553,34 @@ static bool __init arch_dtb_verify(void *dt_virt, bool from_firmware) return true; } +static void early_parse_fdt_property(const void *fdt, const char *path, + const char *prop_name, u64 *property, int size) +{ + int node, prop_len; + const __be32 *prop; + + if (!path || !prop_name) + return; + + node = fdt_path_offset(fdt, path); + if (node < 0) { + pr_err("Failed to get node [%s]\n", path); + return; + } + + prop = fdt_getprop(initial_boot_params, node, prop_name, &prop_len); + if (!prop) { + pr_err("Failed to get property [%s]\n", prop_name); + return; + } + + if (prop_len != size) + pr_warn("Expect [%s] %d bytes, but %d bytes\n", + prop_name, size, prop_len); + + *property = of_read_number(prop, size / 4); +} + static void __init setup_firmware_fdt(void) { void *dt_virt; @@ -579,6 +611,18 @@ static void __init setup_firmware_fdt(void) cpu_relax(); } + if (sunway_boot_magic == 0xDEED2024UL) { + /* Parse MCLK(Hz) from firmware DTB */ + early_parse_fdt_property(dt_virt, "/soc/clocks/mclk", + "clock-frequency", &sunway_mclk_hz, sizeof(u32)); + pr_info("MCLK: %llu Hz\n", sunway_mclk_hz); + + /* Parse EXTCLK(Hz) from firmware DTB */ + early_parse_fdt_property(dt_virt, "/soc/clocks/extclk", + "clock-frequency", &sunway_extclk_hz, sizeof(u32)); + pr_info("EXTCLK: %llu Hz\n", sunway_extclk_hz); + } + name = of_flat_dt_get_machine_name(); if (name) pr_info("DTB(from firmware): Machine model: %s\n", name); @@ -1019,6 +1063,28 @@ static int __init debugfs_sw64(void) return 0; } arch_initcall(debugfs_sw64); + +static int __init debugfs_mclk_init(void) +{ + struct dentry *dir = sw64_debugfs_dir; + static u64 mclk_mhz, mclk_hz; + + if (!dir) + return -ENODEV; + + if (sunway_boot_magic != 0xDEED2024UL) { + mclk_mhz = *((unsigned char *)__va(MB_MCLK)); + mclk_hz = mclk_mhz * 1000000; + debugfs_create_u64("mclk", 0644, dir, &mclk_mhz); + debugfs_create_u64("mclk_hz", 0644, dir, &mclk_hz); + } else { + mclk_hz = sunway_mclk_hz; + debugfs_create_u64("mclk_hz", 0644, dir, &mclk_hz); + } + + return 0; +} +late_initcall(debugfs_mclk_init); #endif #ifdef CONFIG_OF diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index a2f4caced0a3..c70cf340a4b9 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -47,7 +47,12 @@ early_param("mclk_khz", setup_mclk); void __init sw64_setup_clocksource(void) { - unsigned long mclk_khz = *((unsigned char *)__va(MB_MCLK)) * 1000; + unsigned long mclk_khz; + + if (sunway_mclk_hz) + mclk_khz = sunway_mclk_hz / 1000; + else + mclk_khz = *((unsigned char *)__va(MB_MCLK)) * 1000; if (override_mclk_khz) { mclk_khz = override_mclk_khz; -- Gitee From b2a045edd5febeb4a6979b209210cdfe20d9a720 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 10 Apr 2024 10:29:28 +0800 Subject: [PATCH 164/524] sw64: pci: adjust the maximum number of RC per node Adjust the maximum number of RC per node from 6 to 12, since the upcoming hardware platform supports 12 RC per node. This commit also update the OEM table ID of MCFG. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 3 ++ arch/sw_64/include/asm/uncore_io_junzhang.h | 3 -- arch/sw_64/include/asm/uncore_io_xuelang.h | 3 -- arch/sw_64/pci/pci-legacy.c | 6 +-- drivers/acpi/pci_mcfg.c | 46 ++++++++++----------- drivers/pci/controller/pci-sunway.c | 13 ++++-- 6 files changed, 39 insertions(+), 35 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 07c447479efa..f39d07144af9 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -107,6 +107,7 @@ extern int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val); extern void pci_mark_rc_linkup(unsigned long node, unsigned long index); +extern void pci_clear_rc_linkup(unsigned long node, unsigned long index); extern int pci_get_rc_linkup(unsigned long node, unsigned long index); #ifdef CONFIG_PCI_DOMAINS @@ -171,4 +172,6 @@ extern int chip_pcie_configure(struct pci_controller *hose); #define PCITODMA_OFFSET 0x0 /*0 offset*/ +#define MAX_NR_RCS_PER_NODE 12 + #endif /* _ASM_SW64_PCI_H */ diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index c0f759bbe740..2f745fe35365 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -48,9 +48,6 @@ #define SW64_PCI0_BUS 0 #define PCI0_BUS SW64_PCI0_BUS -#define MAX_NR_NODES 0x2 -#define MAX_NR_RCS 0x6 - #define SPBU_BASE (0x3UL << 36) #define INTPU_BASE (0x3aUL << 32) #define IIC0_BASE (0x31UL << 32) diff --git a/arch/sw_64/include/asm/uncore_io_xuelang.h b/arch/sw_64/include/asm/uncore_io_xuelang.h index aeaadec5be16..695ce68dded9 100644 --- a/arch/sw_64/include/asm/uncore_io_xuelang.h +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -47,9 +47,6 @@ #define SW64_PCI0_BUS 0 #define PCI0_BUS SW64_PCI0_BUS -#define MAX_NR_NODES 0x2 -#define MAX_NR_RCS 0x6 - #define MCU_BASE (0x3UL << 36) #define CAB0_BASE (0x10UL << 32) #define INTPU_BASE (0x2aUL << 32) diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index bfaa08bf72c0..15ee626593e9 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -244,7 +244,7 @@ static bool __init is_any_rc_linkup_one_node(unsigned long node) { int i; - for (i = 0; i < 8; ++i) { + for (i = 0; i < MAX_NR_RCS_PER_NODE; ++i) { if (pci_get_rc_linkup(node, i)) return true; } @@ -279,14 +279,14 @@ void __init sw64_init_arch(void) pr_notice("PCIe is disabled on node %ld\n", node); continue; } - for (i = 0; i < MAX_NR_RCS; i++) { + for (i = 0; i < MAX_NR_RCS_PER_NODE; i++) { if ((rc_enable >> i) & 0x1) sw64_init_host(node, i); } if (is_any_rc_linkup_one_node(node)) { memset(msg, 0, 64); sprintf(msg, "Node %ld: RC [ ", node); - for (i = 0; i < MAX_NR_RCS; i++) { + for (i = 0; i < MAX_NR_RCS_PER_NODE; i++) { if (pci_get_rc_linkup(node, i)) { memset(id, 0, 8); sprintf(id, "%d ", i); diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c index 667eb2d65432..58d19e109f61 100644 --- a/drivers/acpi/pci_mcfg.c +++ b/drivers/acpi/pci_mcfg.c @@ -197,29 +197,29 @@ static struct mcfg_fixup mcfg_quirks[] = { #endif /* LOONGARCH */ #ifdef CONFIG_SW64 -#define _SW64_ECAM_QUIRK(rev, seg) \ - { "SUNWAY", "MCFG", rev, seg, MCFG_BUS_ANY, &sw64_pci_ecam_ops } -#define SW64_ECAM_QUIRK(rev, node) _SW64_ECAM_QUIRK(rev, node * 8 + 0),\ - _SW64_ECAM_QUIRK(rev, node * 8 + 1),\ - _SW64_ECAM_QUIRK(rev, node * 8 + 2),\ - _SW64_ECAM_QUIRK(rev, node * 8 + 3),\ - _SW64_ECAM_QUIRK(rev, node * 8 + 4),\ - _SW64_ECAM_QUIRK(rev, node * 8 + 5),\ - _SW64_ECAM_QUIRK(rev, node * 8 + 6),\ - _SW64_ECAM_QUIRK(rev, node * 8 + 7) - - /** - * According to the address space of sw64, up to 8 nodes supported - * with a maximum of 8 pcie controllers per node - */ - SW64_ECAM_QUIRK(1, 0x00), - SW64_ECAM_QUIRK(1, 0x01), - SW64_ECAM_QUIRK(1, 0x02), - SW64_ECAM_QUIRK(1, 0x03), - SW64_ECAM_QUIRK(1, 0x04), - SW64_ECAM_QUIRK(1, 0x05), - SW64_ECAM_QUIRK(1, 0x06), - SW64_ECAM_QUIRK(1, 0x07), +#define SW64_ECAM_QUIRK(table_id, rev, node, ops) \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 0), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 1), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 2), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 3), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 4), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 5), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 6), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 7), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 8), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 9), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 10), MCFG_BUS_ANY, ops }, \ + { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 11), MCFG_BUS_ANY, ops } \ + + /* up to 8 nodes for SW64 series */ + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x00, &sw64_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x01, &sw64_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x02, &sw64_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x03, &sw64_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x04, &sw64_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x05, &sw64_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x06, &sw64_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x07, &sw64_pci_ecam_ops), #endif /* SW64 */ }; diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 46a75ef1a3e9..2f2b5c19a414 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -346,18 +346,25 @@ void __init setup_chip_pci_ops(void) sw64_chip_init->pci_init = chip_pci_init_ops; } -static unsigned long rc_linkup; static struct pci_controller *head, **tail = &head; +static DECLARE_BITMAP(rc_linkup, (MAX_NUMNODES * MAX_NR_RCS_PER_NODE)); + void pci_mark_rc_linkup(unsigned long node, unsigned long index) { - set_bit(node * 8 + index, &rc_linkup); + set_bit(node * MAX_NR_RCS_PER_NODE + index, rc_linkup); } EXPORT_SYMBOL(pci_mark_rc_linkup); +void pci_clear_rc_linkup(unsigned long node, unsigned long index) +{ + clear_bit(node * MAX_NR_RCS_PER_NODE + index, rc_linkup); +} +EXPORT_SYMBOL(pci_clear_rc_linkup); + int pci_get_rc_linkup(unsigned long node, unsigned long index) { - return test_bit(node * 8 + index, &rc_linkup); + return test_bit(node * MAX_NR_RCS_PER_NODE + index, rc_linkup); } EXPORT_SYMBOL(pci_get_rc_linkup); -- Gitee From b31f2da6fd909795c3848854c39facc75907b260 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 12 Apr 2024 09:15:58 +0800 Subject: [PATCH 165/524] sw64: fix compile error of efi.systab The systab field has been removed from struct efi. To get systab, we can parse it from the DTB provided by firmware. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/platform.h | 2 ++ arch/sw_64/kernel/machine_kexec.c | 7 ++++--- arch/sw_64/kernel/setup.c | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h index 7374f2087f90..4f93d9532f9a 100644 --- a/arch/sw_64/include/asm/platform.h +++ b/arch/sw_64/include/asm/platform.h @@ -30,5 +30,7 @@ extern void (*pm_halt)(void); extern int i2c_set_adapter(void); extern void cpld_write(uint8_t slave_addr, uint8_t reg, uint8_t data); extern void fix_jm585_reset(void); +extern void early_parse_fdt_property(const void *fdt, const char *path, + const char *prop_name, u64 *property, int size); #endif /* _ASM_SW64_PLATFORM_H */ diff --git a/arch/sw_64/kernel/machine_kexec.c b/arch/sw_64/kernel/machine_kexec.c index d6758d48dc26..dc1b2b4f5949 100644 --- a/arch/sw_64/kernel/machine_kexec.c +++ b/arch/sw_64/kernel/machine_kexec.c @@ -289,11 +289,10 @@ static void update_boot_params(void) params.dtb_start = (unsigned long)arch_kexec_alloc_and_setup_fdt( params.initrd_start, params.initrd_size, (const char *)params.cmdline); - /* update dtb base address */ - sunway_dtb_address = params.dtb_start; #ifdef CONFIG_EFI - params.efi_systab = virt_to_phys((void *)efi.systab); + early_parse_fdt_property((void *)sunway_dtb_address, "/chosen", + "linux,uefi-system-table", ¶ms.efi_systab, sizeof(u64)); params.efi_memmap = efi.memmap.phys_map; params.efi_memmap_size = efi.memmap.map_end - efi.memmap.map; params.efi_memdesc_size = efi.memmap.desc_size; @@ -312,6 +311,8 @@ static void update_boot_params(void) if (update_efi_properties(¶ms)) pr_err("Note: failed to update efi properties\n"); #endif + /* update dtb base address */ + sunway_dtb_address = params.dtb_start; } pr_info("initrd_start = %#llx, initrd_size = %#llx\n" diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index b154165dcb24..6505fe0486aa 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -553,7 +553,7 @@ static bool __init arch_dtb_verify(void *dt_virt, bool from_firmware) return true; } -static void early_parse_fdt_property(const void *fdt, const char *path, +void early_parse_fdt_property(const void *fdt, const char *path, const char *prop_name, u64 *property, int size) { int node, prop_len; -- Gitee From 52b9106fadc49b54e36d9419661aef7b42f2b0f3 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Sun, 7 Apr 2024 09:50:09 +0800 Subject: [PATCH 166/524] sw64: fix the .endm in hibernate_asm.S This patch fixes compile error because of #ifdef guard mistake. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/hibernate_asm.S | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/hibernate_asm.S b/arch/sw_64/kernel/hibernate_asm.S index af0372879bb9..ebba57b8bd8b 100644 --- a/arch/sw_64/kernel/hibernate_asm.S +++ b/arch/sw_64/kernel/hibernate_asm.S @@ -9,8 +9,9 @@ sys_call HMC_wrktp #else csrw $8, CSR_KTP - .endm #endif + .endm + .text .set noat ENTRY(swsusp_arch_suspend) -- Gitee From cd9d72bf00eb45354592e8d467845b219ada211e Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 9 Apr 2024 17:07:31 +0800 Subject: [PATCH 167/524] sw64: define cpu_relax() as imb() Use imb() to implement cpu_relax() which pause the cpu for a very short period of time until all instructions in pipelines are committed. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/processor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/processor.h b/arch/sw_64/include/asm/processor.h index ec68fe6cc6f2..f66a77f233c9 100644 --- a/arch/sw_64/include/asm/processor.h +++ b/arch/sw_64/include/asm/processor.h @@ -65,7 +65,7 @@ unsigned long __get_wchan(struct task_struct *p); #define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[30]) -#define cpu_relax() barrier() +#define cpu_relax() imemb() #define ARCH_HAS_PREFETCH #define ARCH_HAS_PREFETCHW -- Gitee From 7714b07058084ae4266b7a69ddae7b7622e35d34 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Thu, 11 Apr 2024 18:31:08 +0800 Subject: [PATCH 168/524] sw64: add -fno-sw-unalign-byte to cflags for C4 This ensures that kernel does not throw an unalign exception. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index 84f0dca5e9f7..cfea96316c94 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -28,7 +28,7 @@ endif CHECKFLAGS += -D__sw__ cflags-y := -pipe -ffixed-8 -mno-fp-regs #-msmall-data ifeq ($(CONFIG_SUBARCH_C4),y) - cflags-y += -fsw-rev + cflags-y += -fsw-rev -fno-sw-unalign-byte endif cflags-y += $(call cc-option, -fno-jump-tables) -- Gitee From 9fe17becc0b4652f598b4f4b8067e10d871968e4 Mon Sep 17 00:00:00 2001 From: Xu Chenjiao Date: Tue, 16 Apr 2024 15:07:03 +0000 Subject: [PATCH 169/524] sw64: spi: update spi controller driver Update spi controller driver. It should be noted as follows: - The m25p80 driver has been removed in commit b35b9a10362d ("mtd: spi-nor: Move m25p80 code in spi-nor.c"). - The data structures used by spi memory driver have changed. Signed-off-by: Xu Chenjiao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/chip3.dts | 2 +- arch/sw_64/boot/dts/junzhang.dts | 4 +- drivers/spi/Kconfig | 21 +- drivers/spi/Makefile | 4 +- drivers/spi/spi-sunway-mmio.c | 144 ++++++++ drivers/spi/spi-sunway.c | 409 ++++++++++++++++++++++ drivers/spi/{spi-chip3.h => spi-sunway.h} | 162 +++++---- 7 files changed, 664 insertions(+), 82 deletions(-) create mode 100644 drivers/spi/spi-sunway-mmio.c create mode 100644 drivers/spi/spi-sunway.c rename drivers/spi/{spi-chip3.h => spi-sunway.h} (49%) diff --git a/arch/sw_64/boot/dts/chip3.dts b/arch/sw_64/boot/dts/chip3.dts index 920519566c58..4967e43dc005 100644 --- a/arch/sw_64/boot/dts/chip3.dts +++ b/arch/sw_64/boot/dts/chip3.dts @@ -140,7 +140,7 @@ pvt: pvt@0x8030 { spi: spi@0x8032 { #address-cells = <1>; #size-cells = <0>; - compatible = "sunway,chip3-spi"; + compatible = "sunway,chip-spi"; reg = <0x8032 0x0 0x0 0x8000>; clocks = <&spiclk>; poll_mode = <1>; /* poll_mode:1 interrupt mode: 0 */ diff --git a/arch/sw_64/boot/dts/junzhang.dts b/arch/sw_64/boot/dts/junzhang.dts index 4e8cd655c798..8712678a66e8 100644 --- a/arch/sw_64/boot/dts/junzhang.dts +++ b/arch/sw_64/boot/dts/junzhang.dts @@ -160,7 +160,7 @@ partitions { partition@0 { label = "test"; - reg = <0 0x400000>; + reg = <0 0x800000>; }; }; }; @@ -178,7 +178,7 @@ partitions { partition@0 { label = "test"; - reg = <0 0x400000>; + reg = <0 0x800000>; }; }; }; diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index 60826b7ed21e..2b9e1070b967 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -343,6 +343,21 @@ config SPI_DW_BT1_DIRMAP endif +config SPI_SUNWAY + tristate "Sunway SPI controller support" + depends on SW64 + imply SPI_MEM + help + general driver for Sunway SPI controller + +if SPI_SUNWAY + +config SPI_SUNWAY_MMIO + tristate "Memory-mapped io interface driver for Sunway SPI controller" + depends on HAS_IOMEM + +endif + config SPI_DLN2 tristate "Diolan DLN-2 USB SPI adapter" depends on MFD_DLN2 @@ -1179,12 +1194,6 @@ config SPI_AMD # # Add new SPI master controllers in alphabetical order above this line # -config SPI_CHIP3 - tristate "Memory-mapped io interface driver for SUNWAY CHIP3 SPI core" - depends on UNCORE_XUELANG - help - general driver for SPI controller core from DesignWare - comment "SPI Multiplexer support" diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index 26bf16fcf890..d6fdb887d97d 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -48,8 +48,10 @@ spi-dw-y := spi-dw-core.o spi-dw-$(CONFIG_SPI_DW_DMA) += spi-dw-dma.o obj-$(CONFIG_SPI_DW_BT1) += spi-dw-bt1.o obj-$(CONFIG_SPI_DW_MMIO) += spi-dw-mmio.o -obj-$(CONFIG_SPI_CHIP3) += spi-chip3.o spi-chip3-mmio.o obj-$(CONFIG_SPI_DW_PCI) += spi-dw-pci.o +obj-$(CONFIG_SPI_SUNWAY) += spi-sunway.o +obj-$(CONFIG_SPI_SUNWAY_MMIO) += spi-sunway-mmio.o +obj-$(CONFIG_SPI_EFM32) += spi-efm32.o obj-$(CONFIG_SPI_EP93XX) += spi-ep93xx.o obj-$(CONFIG_SPI_FALCON) += spi-falcon.o obj-$(CONFIG_SPI_FSI) += spi-fsi.o diff --git a/drivers/spi/spi-sunway-mmio.c b/drivers/spi/spi-sunway-mmio.c new file mode 100644 index 000000000000..b170207cd104 --- /dev/null +++ b/drivers/spi/spi-sunway-mmio.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Memory-mapped interface driver for SUNWAY CHIP SPI Core + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-sunway.h" + + +#define DRIVER_NAME "sunway_chip_spi" + +struct chip_spi_mmio { + struct spi_chip spi_chip; + struct clk *clk; + void *priv; +}; + +static int chip_spi_mmio_probe(struct platform_device *pdev) +{ + int (*init_func)(struct platform_device *pdev, + struct chip_spi_mmio *spimmio); + struct chip_spi_mmio *spimmio; + struct spi_chip *spi_chip; + struct resource *mem; + int ret; + int num_cs; + + spimmio = devm_kzalloc(&pdev->dev, sizeof(struct chip_spi_mmio), + GFP_KERNEL); + if (!spimmio) + return -ENOMEM; + + spi_chip = &spimmio->spi_chip; + + /* Get basic io resource and map it */ + mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + spi_chip->regs = devm_ioremap_resource(&pdev->dev, mem); + if (IS_ERR(spi_chip->regs)) { + dev_err(&pdev->dev, "SPI region map failed\n"); + return PTR_ERR(spi_chip->regs); + } + + spimmio->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(spimmio->clk)) + return PTR_ERR(spimmio->clk); + ret = clk_prepare_enable(spimmio->clk); + if (ret) + return ret; + + spi_chip->bus_num = pdev->id; + spi_chip->max_freq = clk_get_rate(spimmio->clk); + + device_property_read_u32(&pdev->dev, "reg-io-width", + &spi_chip->reg_io_width); + + num_cs = 4; + device_property_read_u32(&pdev->dev, "num-cs", &num_cs); + spi_chip->num_cs = num_cs; + + if (pdev->dev.of_node) { + int i; + + for (i = 0; i < spi_chip->num_cs; i++) { + int cs_gpio = of_get_named_gpio(pdev->dev.of_node, + "cs-gpios", i); + + if (cs_gpio == -EPROBE_DEFER) { + ret = cs_gpio; + goto out; + } + + if (gpio_is_valid(cs_gpio)) { + ret = devm_gpio_request(&pdev->dev, cs_gpio, + dev_name(&pdev->dev)); + if (ret) + goto out; + } + } + } + + init_func = device_get_match_data(&pdev->dev); + if (init_func) { + ret = init_func(pdev, spimmio); + if (ret) + goto out; + } + + spi_chip->flags = SPI_PLAT; + + ret = spi_chip_add_host(&pdev->dev, spi_chip); + if (ret) + goto out; + + platform_set_drvdata(pdev, spimmio); + + return 0; +out: + clk_disable_unprepare(spimmio->clk); + return ret; +} + +static int chip_spi_mmio_remove(struct platform_device *pdev) +{ + struct chip_spi_mmio *spimmio = platform_get_drvdata(pdev); + + spi_chip_remove_host(&spimmio->spi_chip); + clk_disable_unprepare(spimmio->clk); + + return 0; +} + +static const struct of_device_id chip_spi_mmio_of_match[] = { + { .compatible = "sunway,chip-spi",}, + { /* end of table */} +}; +MODULE_DEVICE_TABLE(of, chip_spi_mmio_of_match); + +static struct platform_driver chip_spi_mmio_driver = { + .probe = chip_spi_mmio_probe, + .remove = chip_spi_mmio_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = chip_spi_mmio_of_match, + }, +}; +module_platform_driver(chip_spi_mmio_driver); + +MODULE_AUTHOR("Platform@wiat.com"); +MODULE_DESCRIPTION("Memory-mapped I/O interface driver for Sunway CHIP"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/spi/spi-sunway.c b/drivers/spi/spi-sunway.c new file mode 100644 index 000000000000..6133173ea09a --- /dev/null +++ b/drivers/spi/spi-sunway.c @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SPI core controller driver + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-sunway.h" + +/* Slave spi_dev related */ +struct chip_data { + u8 tmode; /* TR/TO/RO/EEPROM */ + u8 type; /* SPI/SSP/MicroWire */ + + u8 poll_mode; /* 1 means use poll mode */ + + u16 clk_div; /* baud rate divider */ + u32 speed_hz; /* baud rate */ + void (*cs_control)(u32 command); +}; + +static void spi_chip_handle_err(struct spi_controller *master, + struct spi_message *msg) +{ + struct spi_chip *spi_chip = spi_controller_get_devdata(master); + + spi_reset_chip(spi_chip); +} + +static size_t spi_chip_max_length(struct spi_device *spi) +{ + struct spi_chip *spi_chip = spi_controller_get_devdata(spi->master); + + return spi_chip->fifo_len; +} + +static int spi_chip_transfer_one_message(struct spi_controller *master, + struct spi_message *m) +{ + struct spi_chip *spi_chip = spi_controller_get_devdata(master); + struct spi_transfer *t = NULL; + u16 clk_div; + u32 freq; + u32 speed_hz; + u32 status; + u32 len = 0; + int ret = 0; + int i = 0; + + spi_enable_chip(spi_chip, 0); + + /* Handle per transfer options for bpw and speed. */ + freq = clamp(m->spi->max_speed_hz, 0U, spi_chip->max_freq); + clk_div = (DIV_ROUND_UP(spi_chip->max_freq, freq) + 1) & 0xfffe; + speed_hz = spi_chip->max_freq / clk_div; + + if (spi_chip->current_freq != speed_hz) { + spi_set_clk(spi_chip, clk_div); + spi_chip->current_freq = speed_hz; + } + + spi_chip->n_bytes = 1; + + /* For poll mode just disable all interrupts */ + spi_mask_intr(spi_chip, 0xff); + + spi_writel(spi_chip, SPI_CHIP_CTRL0, SPI_TRANSMIT_RECEIVE); + + spi_enable_chip(spi_chip, 1); + + list_for_each_entry(t, &m->transfers, transfer_list) { + len += t->len; + /* Judge if data is overflow */ + if (len > spi_chip->fifo_len) { + pr_err("SPI transfer overflow.\n"); + m->actual_length = 0; + m->status = -EIO; + ret = -EIO; + goto way_out; + } + + if (t->tx_buf) + memcpy(&spi_chip->buf[len], t->tx_buf, t->len); + else + memset(&spi_chip->buf[len], 0, t->len); + } + + spi_writel(spi_chip, SPI_CHIP_SER, 0x0); + for (i = 0; i < len; i++) + spi_writel(spi_chip, SPI_CHIP_DR, spi_chip->buf[i]); + spi_writel(spi_chip, SPI_CHIP_SER, BIT(m->spi->chip_select)); + + do { + status = spi_readl(spi_chip, SPI_CHIP_SR); + } while (status & SR_BUSY); + + list_for_each_entry(t, &m->transfers, transfer_list) { + if (t->rx_buf) { + for (i = 0; i < t->len; i++, t->rx_buf += 1) + *(u8 *)t->rx_buf = spi_readl(spi_chip, + SPI_CHIP_DR); + } else { + for (i = 0; i < t->len; i++) + spi_readl(spi_chip, SPI_CHIP_DR); + } + } + + m->actual_length = len; + m->status = 0; + spi_finalize_current_message(master); + +way_out: + return ret; +} + +static int spi_chip_adjust_mem_op_size(struct spi_mem *mem, + struct spi_mem_op *op) +{ + struct spi_chip *spi_chip = spi_controller_get_devdata( + mem->spi->controller); + size_t len; + + len = sizeof(op->cmd.opcode) + op->addr.nbytes + op->dummy.nbytes; + + op->data.nbytes = min((size_t)op->data.nbytes, + (spi_chip->fifo_len - len)); + if (!op->data.nbytes) + return -EINVAL; + + return 0; +} + +static int spi_chip_init_mem_buf(struct spi_chip *spi_chip, + const struct spi_mem_op *op) +{ + int ret = 0; + int i, j, len; + + /* Calculate the total length of the transfer. */ + len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes; + + /* Judge if data is overflow */ + if (len + op->data.nbytes > spi_chip->fifo_len) { + ret = -EIO; + goto way_out; + } + + /* + * Collect the operation code, address and dummy bytes into the single + * buffer. If it's a transfer with data to be sent, also copy it into + * the single buffer. + */ + for (i = 0; i < op->cmd.nbytes; i++) + spi_chip->buf[i] = op->cmd.opcode; + for (j = 0; j < op->addr.nbytes; i++, j++) + spi_chip->buf[i] = op->addr.val >> (8 * (op->addr.nbytes - i)); + for (j = 0; j < op->dummy.nbytes; i++, j++) + spi_chip->buf[i] = 0xff; + + if (op->data.dir == SPI_MEM_DATA_OUT) { + memcpy(&spi_chip->buf[i], op->data.buf.out, op->data.nbytes); + len += op->data.nbytes; + } + + spi_chip->tx_len = len; + + if (op->data.dir == SPI_MEM_DATA_IN) { + spi_chip->rx = op->data.buf.in; + spi_chip->rx_len = op->data.nbytes; + } else { + spi_chip->rx = NULL; + spi_chip->rx_len = 0; + } + +way_out: + return ret; +} + +static int spi_chip_exec_mem_op(struct spi_mem *mem, + const struct spi_mem_op *op) +{ + struct spi_chip *spi_chip = spi_controller_get_devdata( + mem->spi->controller); + u16 clk_div; + int ret = 0; + int i; + unsigned short value; + u32 freq; + u32 speed_hz; + + ret = spi_chip_init_mem_buf(spi_chip, op); + if (ret) + return ret; + + spi_enable_chip(spi_chip, 0); + + /* Handle per transfer options for bpw and speed. */ + freq = clamp(mem->spi->max_speed_hz, 0U, spi_chip->max_freq); + clk_div = (DIV_ROUND_UP(spi_chip->max_freq, freq) + 1) & 0xfffe; + speed_hz = spi_chip->max_freq / clk_div; + + if (spi_chip->current_freq != speed_hz) { + spi_set_clk(spi_chip, clk_div); + spi_chip->current_freq = speed_hz; + } + + spi_chip->n_bytes = 1; + + /* For poll mode just disable all interrupts */ + spi_mask_intr(spi_chip, 0xff); + + if ((spi_chip->tx_len != 0) && (spi_chip->rx_len != 0)) { + spi_writel(spi_chip, SPI_CHIP_CTRL0, SPI_EEPROM_READ); + spi_writel(spi_chip, SPI_CHIP_CTRL1, (spi_chip->rx_len - 1)); + } else { + spi_writel(spi_chip, SPI_CHIP_CTRL0, SPI_TRANSMIT_ONLY); + } + + spi_enable_chip(spi_chip, 1); + + spi_writel(spi_chip, SPI_CHIP_SER, 0x0); + for (i = 0; i < spi_chip->tx_len; i++) + spi_writel(spi_chip, SPI_CHIP_DR, spi_chip->buf[i]); + spi_writel(spi_chip, SPI_CHIP_SER, BIT(mem->spi->chip_select)); + + value = spi_readl(spi_chip, SPI_CHIP_SR); + while (value & SR_BUSY) + value = spi_readl(spi_chip, SPI_CHIP_SR); + + for (i = 0; i < spi_chip->rx_len; spi_chip->rx += spi_chip->n_bytes, i++) + *(u8 *)spi_chip->rx = spi_readl(spi_chip, SPI_CHIP_DR); + + return ret; +} + +/* This may be called twice for each spi dev */ +static int spi_chip_setup(struct spi_device *spi) +{ + struct spi_chip_info *chip_info = NULL; + struct chip_data *chip; + u32 poll_mode = 0; + struct device_node *np = spi->dev.of_node; + + /* Only alloc on first setup */ + chip = spi_get_ctldata(spi); + if (!chip) { + chip = kzalloc(sizeof(struct chip_data), GFP_KERNEL); + if (!chip) + return -ENOMEM; + spi_set_ctldata(spi, chip); + } + + /* + * Protocol drivers may change the chip settings, so... + * if chip_info exists, use it + */ + chip_info = spi->controller_data; + + /* chip_info doesn't always exist */ + if (chip_info) { + if (chip_info->cs_control) + chip->cs_control = chip_info->cs_control; + + chip->poll_mode = chip_info->poll_mode; + chip->type = chip_info->type; + } else { + if (np) { + of_property_read_u32(np, "poll_mode", &poll_mode); + chip->poll_mode = poll_mode; + } + + } + + chip->tmode = SPI_TMOD_TR; + return 0; +} + +static void spi_chip_cleanup(struct spi_device *spi) +{ + struct chip_data *chip = spi_get_ctldata(spi); + + kfree(chip); + spi_set_ctldata(spi, NULL); +} + +/* Restart the controller, disable all interrupts, clean rx fifo */ +static void spi_hw_init(struct device *dev, struct spi_chip *spi_chip) +{ + spi_reset_chip(spi_chip); + + /* + * Try to detect the FIFO depth if not set by interface driver, + * the depth could be from 2 to 256 from HW spec + */ + if (!spi_chip->fifo_len) { + u32 fifo; + + for (fifo = 1; fifo < 256; fifo++) { + spi_writel(spi_chip, SPI_CHIP_TXFLTR, fifo); + if (fifo != spi_readl(spi_chip, SPI_CHIP_TXFLTR)) + break; + } + spi_writel(spi_chip, SPI_CHIP_TXFLTR, 0); + + spi_chip->fifo_len = (fifo == 1) ? 0 : fifo; + dev_info(dev, "Detected FIFO size: %u bytes\n", + spi_chip->fifo_len); + } +} + +static const struct spi_controller_mem_ops spi_mem_ops = { + .adjust_op_size = spi_chip_adjust_mem_op_size, + .exec_op = spi_chip_exec_mem_op, +}; + +int spi_chip_add_host(struct device *dev, struct spi_chip *spi_chip) +{ + struct spi_controller *master; + int ret; + + WARN_ON(spi_chip == NULL); + + master = spi_alloc_master(dev, 0); + if (!master) + return -ENOMEM; + + spi_chip->master = master; + spi_chip->type = SSI_MOTO_SPI; + + spi_controller_set_devdata(master, spi_chip); + + master->mode_bits = SPI_CPOL | SPI_CPHA; + master->bits_per_word_mask = SPI_BPW_MASK(8) | SPI_BPW_MASK(16); + master->bus_num = spi_chip->bus_num; + master->num_chipselect = spi_chip->num_cs; + master->setup = spi_chip_setup; + master->cleanup = spi_chip_cleanup; + master->transfer_one_message = spi_chip_transfer_one_message; + master->handle_err = spi_chip_handle_err; + master->max_speed_hz = spi_chip->max_freq; + master->dev.of_node = dev->of_node; + master->flags = SPI_MASTER_GPIO_SS; + master->max_transfer_size = spi_chip_max_length; + master->max_message_size = spi_chip_max_length; + + master->mem_ops = &spi_mem_ops; + + /* Basic HW init */ + spi_hw_init(dev, spi_chip); + + ret = devm_spi_register_controller(dev, master); + if (ret) { + dev_err(&master->dev, "problem registering spi master\n"); + spi_enable_chip(spi_chip, 0); + free_irq(spi_chip->irq, master); + } + + return 0; +} +EXPORT_SYMBOL_GPL(spi_chip_add_host); + +void spi_chip_remove_host(struct spi_chip *spi_chip) +{ + spi_shutdown_chip(spi_chip); + + free_irq(spi_chip->irq, spi_chip->master); +} +EXPORT_SYMBOL_GPL(spi_chip_remove_host); + +int spi_chip_suspend_host(struct spi_chip *spi_chip) +{ + int ret; + + ret = spi_controller_suspend(spi_chip->master); + if (ret) + return ret; + + spi_shutdown_chip(spi_chip); + return 0; +} +EXPORT_SYMBOL_GPL(spi_chip_suspend_host); + +int spi_chip_resume_host(struct spi_chip *spi_chip) +{ + int ret; + + spi_hw_init(&spi_chip->master->dev, spi_chip); + ret = spi_controller_resume(spi_chip->master); + if (ret) + dev_err(&spi_chip->master->dev, "fail to start queue (%d)\n", + ret); + return ret; +} +EXPORT_SYMBOL_GPL(spi_chip_resume_host); + +MODULE_AUTHOR("Platform@wiat.com"); +MODULE_DESCRIPTION("Driver for SPI controller core"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/spi/spi-chip3.h b/drivers/spi/spi-sunway.h similarity index 49% rename from drivers/spi/spi-chip3.h rename to drivers/spi/spi-sunway.h index 88e49a9091a5..5f44050001d3 100644 --- a/drivers/spi/spi-chip3.h +++ b/drivers/spi/spi-sunway.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef CHIP3_SPI_HEADER_H -#define CHIP3_SPI_HEADER_H +#ifndef SPI_CHIP_HEADER_H +#define SPI_CHIP_HEADER_H #include #include @@ -8,31 +8,31 @@ #include /* Register offsets */ -#define CHIP3_SPI_CTRL0 (0x00<<7) -#define CHIP3_SPI_CTRL1 (0x04<<7) -#define CHIP3_SPI_SSIENR (0x08<<7) -#define CHIP3_SPI_MWCR (0x0c<<7) -#define CHIP3_SPI_SER (0x10<<7) -#define CHIP3_SPI_BAUDR (0x14<<7) -#define CHIP3_SPI_TXFLTR (0x18<<7) -#define CHIP3_SPI_RXFLTR (0x1c<<7) -#define CHIP3_SPI_TXFLR (0x20<<7) -#define CHIP3_SPI_RXFLR (0x24<<7) -#define CHIP3_SPI_SR (0x28<<7) -#define CHIP3_SPI_IMR (0x2c<<7) -#define CHIP3_SPI_ISR (0x30<<7) -#define CHIP3_SPI_RISR (0x34<<7) -#define CHIP3_SPI_TXOICR (0x38<<7) -#define CHIP3_SPI_RXOICR (0x3c<<7) -#define CHIP3_SPI_RXUICR (0x40<<7) -#define CHIP3_SPI_MSTICR (0x44<<7) -#define CHIP3_SPI_ICR (0x48<<7) -#define CHIP3_SPI_DMACR (0x4c<<7) -#define CHIP3_SPI_DMATDLR (0x50<<7) -#define CHIP3_SPI_DMARDLR (0x54<<7) -#define CHIP3_SPI_IDR (0x58<<7) -#define CHIP3_SPI_VERSION (0x5c<<7) -#define CHIP3_SPI_DR (0x60<<7) +#define SPI_CHIP_CTRL0 0x00 +#define SPI_CHIP_CTRL1 0x04 +#define SPI_CHIP_SSIENR 0x08 +#define SPI_CHIP_MWCR 0x0c +#define SPI_CHIP_SER 0x10 +#define SPI_CHIP_BAUDR 0x14 +#define SPI_CHIP_TXFLTR 0x18 +#define SPI_CHIP_RXFLTR 0x1c +#define SPI_CHIP_TXFLR 0x20 +#define SPI_CHIP_RXFLR 0x24 +#define SPI_CHIP_SR 0x28 +#define SPI_CHIP_IMR 0x2c +#define SPI_CHIP_ISR 0x30 +#define SPI_CHIP_RISR 0x34 +#define SPI_CHIP_TXOICR 0x38 +#define SPI_CHIP_RXOICR 0x3c +#define SPI_CHIP_RXUICR 0x40 +#define SPI_CHIP_MSTICR 0x44 +#define SPI_CHIP_ICR 0x48 +#define SPI_CHIP_DMACR 0x4c +#define SPI_CHIP_DMATDLR 0x50 +#define SPI_CHIP_DMARDLR 0x54 +#define SPI_CHIP_IDR 0x58 +#define SPI_CHIP_VERSION 0x5c +#define SPI_CHIP_DR 0x60 /* Bit fields in CTRLR0 */ #define SPI_DFS_OFFSET 0 @@ -91,17 +91,21 @@ #define SPI_EEPROM_READ 0x3c7 #define SPI_TRANSMIT_ONLY 0x1c7 -enum chip3_ssi_type { +#define SPI_PLAT 0x1 +#define SPI_PCI 0x2 + +enum spi_ssi_type { SSI_MOTO_SPI = 0, SSI_TI_SSP, SSI_NS_MICROWIRE, }; -struct chip3_spi; -struct chip3_spi { +struct spi_chip; + +struct spi_chip { struct spi_controller *master; - enum chip3_ssi_type type; + enum spi_ssi_type type; void __iomem *regs; unsigned long paddr; @@ -122,6 +126,7 @@ struct chip3_spi { unsigned int rx_len; u8 n_bytes; /* current is a 1/2 bytes op */ u32 current_freq; /* frequency in hz */ + int flags; u8 buf[MAX_LEN]; @@ -132,76 +137,89 @@ struct chip3_spi { #endif }; -static inline u32 chip3_readl(struct chip3_spi *dws, u32 offset) +static inline u32 spi_readl(struct spi_chip *spi_chip, u32 offset) { - return __raw_readl(dws->regs + offset); + if (spi_chip->flags & SPI_PLAT) + offset <<= 7; + + return __raw_readl(spi_chip->regs + offset); } -static inline u16 chip3_readw(struct chip3_spi *dws, u32 offset) +static inline u16 spi_readw(struct spi_chip *spi_chip, u32 offset) { - return __raw_readw(dws->regs + offset); + if (spi_chip->flags & SPI_PLAT) + offset <<= 7; + + return __raw_readw(spi_chip->regs + offset); } -static inline void chip3_writel(struct chip3_spi *dws, u32 offset, u32 val) +static inline void spi_writel(struct spi_chip *spi_chip, u32 offset, u32 val) { - __raw_writel(val, dws->regs + offset); + if (spi_chip->flags & SPI_PLAT) + offset <<= 7; + + __raw_writel(val, spi_chip->regs + offset); } -static inline void chip3_writew(struct chip3_spi *dws, u32 offset, u16 val) +static inline void spi_writew(struct spi_chip *spi_chip, u32 offset, u16 val) { - __raw_writew(val, dws->regs + offset); + if (spi_chip->flags & SPI_PLAT) + offset <<= 7; + + __raw_writew(val, spi_chip->regs + offset); } -static inline u32 chip3_read_io_reg(struct chip3_spi *dws, u32 offset) +static inline u32 spi_read_io_reg(struct spi_chip *spi_chip, u32 offset) { - switch (dws->reg_io_width) { + switch (spi_chip->reg_io_width) { case 2: - return chip3_readw(dws, offset); + return spi_readw(spi_chip, offset); case 4: default: - return chip3_readl(dws, offset); + return spi_readl(spi_chip, offset); } } -static inline void chip3_write_io_reg(struct chip3_spi *dws, u32 offset, u32 val) +static inline void spi_write_io_reg(struct spi_chip *spi_chip, u32 offset, + u32 val) { - switch (dws->reg_io_width) { + switch (spi_chip->reg_io_width) { case 2: - chip3_writew(dws, offset, val); + spi_writew(spi_chip, offset, val); break; case 4: default: - chip3_writel(dws, offset, val); + spi_writel(spi_chip, offset, val); break; } } -static inline void spi_enable_chip(struct chip3_spi *dws, int enable) +static inline void spi_enable_chip(struct spi_chip *spi_chip, int enable) { - chip3_writel(dws, CHIP3_SPI_SSIENR, (enable ? 1 : 0)); + spi_writel(spi_chip, SPI_CHIP_SSIENR, (enable ? 1 : 0)); } -static inline void spi_set_clk(struct chip3_spi *dws, u16 div) +static inline void spi_set_clk(struct spi_chip *spi_chip, u16 div) { - chip3_writel(dws, CHIP3_SPI_BAUDR, div); + spi_writel(spi_chip, SPI_CHIP_BAUDR, div); } /* Disable IRQ bits */ -static inline void spi_mask_intr(struct chip3_spi *dws, u32 mask) +static inline void spi_mask_intr(struct spi_chip *spi_chip, u32 mask) { u32 new_mask; - new_mask = chip3_readl(dws, CHIP3_SPI_IMR) & ~mask; - chip3_writel(dws, CHIP3_SPI_IMR, new_mask); + new_mask = spi_readl(spi_chip, SPI_CHIP_IMR) & ~mask; + spi_writel(spi_chip, SPI_CHIP_IMR, new_mask); } /* Enable IRQ bits */ -static inline void spi_umask_intr(struct chip3_spi *dws, u32 mask) +static inline void spi_umask_intr(struct spi_chip *spi_chip, u32 mask) { u32 new_mask; - new_mask = chip3_readl(dws, CHIP3_SPI_IMR) | mask; - chip3_writel(dws, CHIP3_SPI_IMR, new_mask); + new_mask = spi_readl(spi_chip, SPI_CHIP_IMR) | mask; + spi_writel(spi_chip, SPI_CHIP_IMR, new_mask); } /* @@ -209,37 +227,37 @@ static inline void spi_umask_intr(struct chip3_spi *dws, u32 mask) * controller back. Transmit and receive FIFO buffers are cleared when the * device is disabled. */ -static inline void spi_reset_chip(struct chip3_spi *dws) +static inline void spi_reset_chip(struct spi_chip *spi_chip) { - spi_enable_chip(dws, 0); - spi_mask_intr(dws, 0xff); - spi_enable_chip(dws, 1); + spi_enable_chip(spi_chip, 0); + spi_mask_intr(spi_chip, 0xff); + spi_enable_chip(spi_chip, 1); } -static inline void spi_shutdown_chip(struct chip3_spi *dws) +static inline void spi_shutdown_chip(struct spi_chip *spi_chip) { - spi_enable_chip(dws, 0); - spi_set_clk(dws, 0); + spi_enable_chip(spi_chip, 0); + spi_set_clk(spi_chip, 0); } /* - * Each SPI slave device to work with chip3_api controller should + * Each SPI slave device to work with spi_api controller should * has such a structure claiming its working mode (poll or PIO/DMA), * which can be save in the "controller_data" member of the * struct spi_device. */ -struct chip3_spi_chip { +struct spi_chip_info { u8 poll_mode; /* 1 for controller polling mode */ u8 type; /* SPI/SSP/MicroWire */ u8 chip_select; void (*cs_control)(u32 command); }; -extern int chip3_spi_add_host(struct device *dev, struct chip3_spi *dws); -extern void chip3_spi_remove_host(struct chip3_spi *dws); -extern int chip3_spi_suspend_host(struct chip3_spi *dws); -extern int chip3_spi_resume_host(struct chip3_spi *dws); +extern int spi_chip_add_host(struct device *dev, struct spi_chip *spi_chip); +extern void spi_chip_remove_host(struct spi_chip *spi_chip); +extern int spi_chip_suspend_host(struct spi_chip *spi_chip); +extern int spi_chip_resume_host(struct spi_chip *spi_chip); /* platform related setup */ -extern int chip3_spi_mid_init(struct chip3_spi *dws); /* Intel MID platforms */ -#endif /* CHIP3_SPI_HEADER_H */ +extern int spi_ich_init(struct spi_chip *spi_chip); +#endif /* SPI_CHIP_HEADER_H */ -- Gitee From 12440b1570caebdeb31961eff9baed8c9d161b9b Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 19 Apr 2024 14:17:17 +0000 Subject: [PATCH 170/524] sw64: match platform device by acpi method Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/chip3.dts | 6 +++--- drivers/acpi/acpi_apd.c | 4 ++-- drivers/gpio/gpio-sunway.c | 5 +++++ drivers/hwmon/sw64_pvt.c | 11 +++++++++++ drivers/i2c/busses/i2c-designware-platdrv.c | 1 + drivers/mfd/lpc_sunway.c | 10 +++++++++- drivers/spi/spi-sunway-mmio.c | 12 ++++++++++++ drivers/tty/serial/8250/8250_sunway.c | 1 + 8 files changed, 44 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/boot/dts/chip3.dts b/arch/sw_64/boot/dts/chip3.dts index 4967e43dc005..fecb0f0f3be7 100644 --- a/arch/sw_64/boot/dts/chip3.dts +++ b/arch/sw_64/boot/dts/chip3.dts @@ -86,7 +86,7 @@ serial1@9033 { i2c0@0x8031 { #address-cells = <2>; #size-cells = <2>; - compatible = "snps,designware-i2c"; + compatible = "sunway,suntai-i2c", "snps,designware-i2c"; reg = <0x8031 0x0 0x0 0x8000>; clock-frequency = <100000>; clocks = <&i2cclk>; @@ -98,7 +98,7 @@ i2c0@0x8031 { i2c1@0x8034 { #address-cells = <1>; #size-cells = <0>; - compatible = "snps,designware-i2c"; + compatible = "sunway,suntai-i2c", "snps,designware-i2c"; reg = <0x8034 0x0 0x0 0x8000>; clock-frequency = <100000>; clocks = <&i2cclk>; @@ -110,7 +110,7 @@ i2c1@0x8034 { i2c2@0x8035 { #address-cells = <1>; #size-cells = <0>; - compatible = "snps,designware-i2c"; + compatible = "sunway,suntai-i2c", "snps,designware-i2c"; reg = <0x8035 0x0 0x0 0x8000>; clock-frequency = <100000>; clocks = <&i2cclk>; diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index 791f4b234e02..c5afed1eeb9c 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -261,8 +261,8 @@ static const struct acpi_device_id acpi_apd_device_ids[] = { { "NXP0001", APD_ADDR(nxp_i2c_desc) }, #endif #ifdef CONFIG_SW64 - { "HISI02A1", APD_ADDR(sunway_i2c_desc) }, - { "HISI0173", APD_ADDR(sunway_spi_desc) }, + { "SUNW0005", APD_ADDR(sunway_i2c_desc) }, + { "SUNW0008", APD_ADDR(sunway_spi_desc) }, #endif { } }; diff --git a/drivers/gpio/gpio-sunway.c b/drivers/gpio/gpio-sunway.c index 11581dded64f..ecb87c41dc65 100644 --- a/drivers/gpio/gpio-sunway.c +++ b/drivers/gpio/gpio-sunway.c @@ -648,13 +648,16 @@ static const struct of_device_id sunway_of_match[] = { }; MODULE_DEVICE_TABLE(of, sunway_of_match); +#ifdef CONFIG_ACPI static const struct acpi_device_id sunway_acpi_match[] = { {"HISI0181", 0}, {"APMC0D07", 0}, {"APMC0D81", GPIO_REG_OFFSET_V2}, + {"SUNW0002", 0}, { } }; MODULE_DEVICE_TABLE(acpi, sunway_acpi_match); +#endif static int sunway_gpio_probe(struct platform_device *pdev) { @@ -727,6 +730,8 @@ static int sunway_gpio_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, gpio); + def_info(&pdev->dev, "GPIO probe succeed\n"); + return 0; out_unregister: diff --git a/drivers/hwmon/sw64_pvt.c b/drivers/hwmon/sw64_pvt.c index aedc29d44077..78620f91ae52 100644 --- a/drivers/hwmon/sw64_pvt.c +++ b/drivers/hwmon/sw64_pvt.c @@ -184,6 +184,8 @@ static int pvt_vol_plat_probe(struct platform_device *pdev) return false; } + dev_info(&pdev->dev, "PVT probe succeed\n"); + return 0; err: @@ -199,11 +201,20 @@ static const struct of_device_id pvt_vol_of_match[] = { MODULE_DEVICE_TABLE(of, pvt_vol_of_match); #endif +#ifdef CONFIG_ACPI +static const struct acpi_device_id pvt_vol_acpi_match[] = { + { "SUNW0007", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, pvt_vol_acpi_match); +#endif + static struct platform_driver pvt_vol_driver = { .probe = pvt_vol_plat_probe, .driver = { .name = "pvt-sw64", .of_match_table = of_match_ptr(pvt_vol_of_match), + .acpi_match_table = ACPI_PTR(pvt_vol_acpi_match), }, }; diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index 5d7ee3430f0a..3ba563eee10d 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -57,6 +57,7 @@ static const struct acpi_device_id dw_i2c_acpi_match[] = { { "HISI02A2", 0 }, { "HISI02A3", 0 }, { "HYGO0010", ACCESS_INTR_MASK }, + { "SUNW0005", MODEL_SUNWAY }, { } }; MODULE_DEVICE_TABLE(acpi, dw_i2c_acpi_match); diff --git a/drivers/mfd/lpc_sunway.c b/drivers/mfd/lpc_sunway.c index 1bcf40d6a6f7..a70971e7912d 100644 --- a/drivers/mfd/lpc_sunway.c +++ b/drivers/mfd/lpc_sunway.c @@ -242,9 +242,16 @@ static const struct of_device_id chip3_lpc_of_match[] = { {.compatible = "sunway,chip3_lpc",}, { /* end of table */ } }; - MODULE_DEVICE_TABLE(of, chip3_lpc_of_match); +#ifdef CONFIG_ACPI +static const struct acpi_device_id chip3_lpc_acpi_match[] = { + { "SUNW0006", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, chip3_lpc_acpi_match); +#endif + #ifdef CONFIG_PM_SLEEP unsigned int lpc_irq_ctrl_value; unsigned int lpc_irq_irq_value; @@ -293,6 +300,7 @@ static struct platform_driver chip3_lpc_platform_driver = { .driver = { .name = "chip3_lpc", .of_match_table = chip3_lpc_of_match, + .acpi_match_table = ACPI_PTR(chip3_lpc_acpi_match), #ifdef CONFIG_PM_SLEEP .pm = &chip3_lpc_pm_ops, #endif diff --git a/drivers/spi/spi-sunway-mmio.c b/drivers/spi/spi-sunway-mmio.c index b170207cd104..7c8ba0a61ffb 100644 --- a/drivers/spi/spi-sunway-mmio.c +++ b/drivers/spi/spi-sunway-mmio.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "spi-sunway.h" @@ -107,6 +108,8 @@ static int chip_spi_mmio_probe(struct platform_device *pdev) platform_set_drvdata(pdev, spimmio); + dev_info(&pdev->dev, "SPI(MMIO) probe succeed\n"); + return 0; out: clk_disable_unprepare(spimmio->clk); @@ -129,12 +132,21 @@ static const struct of_device_id chip_spi_mmio_of_match[] = { }; MODULE_DEVICE_TABLE(of, chip_spi_mmio_of_match); +#ifdef CONFIG_ACPI +static const struct acpi_device_id chip_spi_mmio_acpi_match[] = { + { "SUNW0008", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, chip_spi_mmio_acpi_match); +#endif + static struct platform_driver chip_spi_mmio_driver = { .probe = chip_spi_mmio_probe, .remove = chip_spi_mmio_remove, .driver = { .name = DRIVER_NAME, .of_match_table = chip_spi_mmio_of_match, + .acpi_match_table = ACPI_PTR(chip_spi_mmio_acpi_match), }, }; module_platform_driver(chip_spi_mmio_driver); diff --git a/drivers/tty/serial/8250/8250_sunway.c b/drivers/tty/serial/8250/8250_sunway.c index 9e3db232c832..7ff5ff160725 100644 --- a/drivers/tty/serial/8250/8250_sunway.c +++ b/drivers/tty/serial/8250/8250_sunway.c @@ -763,6 +763,7 @@ static const struct acpi_device_id sunway8250_acpi_match[] = { { "AMDI0020", 0 }, { "BRCM2032", 0 }, { "HISI0031", 0 }, + { "SUNW0009", 0}, { }, }; MODULE_DEVICE_TABLE(acpi, sunway8250_acpi_match); -- Gitee From 010e70c0576b7a000c151256f3bc3d5c11781ac9 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 8 Apr 2024 10:33:56 +0800 Subject: [PATCH 171/524] sw64: force kernel stack to be 16 bytes aligned Make struct pt_regs and struct rt_sigframe 16 bytes aligned to make sure the kernel stack is always 16 bytes aligned. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/ptrace.h | 2 +- arch/sw_64/kernel/signal.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h index aa8b269a7ef4..44c9e2d1fe6a 100644 --- a/arch/sw_64/include/asm/ptrace.h +++ b/arch/sw_64/include/asm/ptrace.h @@ -34,7 +34,7 @@ struct pt_regs { unsigned long earg0; unsigned long earg1; unsigned long earg2; -}; +} __aligned(16); #define arch_has_single_step() (1) #define user_mode(regs) (((regs)->ps & 8) != 0) diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 496f33bb1c89..2155e6fa98c3 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -82,7 +82,7 @@ SYSCALL_DEFINE3(odd_sigaction, int, sig, struct rt_sigframe { struct siginfo info; struct ucontext uc; -}; +} __aligned(16); /* * If this changes, userland unwinders that Know Things about our signal -- Gitee From 36c81e70a47100ecca8b64d1cdba7362ab1e9079 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 9 Apr 2024 09:17:34 +0800 Subject: [PATCH 172/524] sw64: fix broken rdfpcr() The memory output is used again after the register output is written, so the register of the register output constraint cannot be reused to store the memory address. Add the necessary constraint modifier to avoid this register reuse. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/fpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/fpu.h b/arch/sw_64/include/asm/fpu.h index 1d5e9b0efecc..763d763c0477 100644 --- a/arch/sw_64/include/asm/fpu.h +++ b/arch/sw_64/include/asm/fpu.h @@ -100,7 +100,7 @@ rdfpcr(void) " rfpcr $f0\n\t" " fimovd $f0, %1\n\t" " vldd $f0, %0\n\t" - : "=m"(*fp), "=r"(ret)); + : "=m"(*fp), "=&r"(ret)); return ret; } -- Gitee From c17bd4a17087822df6fcf62a61a3f477816ce2a8 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 28 Feb 2024 09:38:22 +0800 Subject: [PATCH 173/524] sw64: add syscall pfh_ops Add a new syscall pfh_ops which allows system administrators to modify hardware cache prefetch configuration. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/csr.h | 4 ++ arch/sw_64/kernel/sys_sw64.c | 61 ++++++++++++++++++++++++++ arch/sw_64/kernel/syscalls/syscall.tbl | 2 +- 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h index 8f4f3d142db4..cdb9249eedd2 100644 --- a/arch/sw_64/include/asm/csr.h +++ b/arch/sw_64/include/asm/csr.h @@ -20,8 +20,12 @@ #define CSR_EXC_PC 0xe #define CSR_AS_INFO 0x3c #define CSR_DS_STAT 0x48 +#define CSR_PFH_CTL 0x4f #define CSR_SOFTCID 0xc9 #define CSR_DVA 0x54 +#define CSR_PFH_CNT 0x5c +#define CSR_BRRETC 0x5e +#define CSR_BRFAILC 0x5f #define CSR_PTBR_SYS 0x68 #define CSR_PTBR_USR 0x69 #define CSR_APTP 0x6a diff --git a/arch/sw_64/kernel/sys_sw64.c b/arch/sw_64/kernel/sys_sw64.c index 46e6197dce76..31e6a8d65cf9 100644 --- a/arch/sw_64/kernel/sys_sw64.c +++ b/arch/sw_64/kernel/sys_sw64.c @@ -151,3 +151,64 @@ SYSCALL_DEFINE0(sw64_pipe) } return res; } + +#ifdef CONFIG_SUBARCH_C4 + +struct pfh_val { + unsigned long pfh_ctl; + unsigned long pfh_cnt; +}; + +static void local_set_pfh(void *info) +{ + struct pfh_val *kbuf = info; + + if (kbuf->pfh_ctl) + sw64_write_csr(kbuf->pfh_ctl, CSR_PFH_CTL); + if (kbuf->pfh_cnt) + sw64_write_csr(kbuf->pfh_cnt, CSR_PFH_CNT); +} + +SYSCALL_DEFINE3(pfh_ops, unsigned long, op, + unsigned long __user *, pfh_ctl_p, + unsigned long __user *, pfh_cnt_p) +{ + struct pfh_val kbuf = {0, 0}; + long error = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (op) { // op != 0, set + if (pfh_ctl_p) + error |= get_user(kbuf.pfh_ctl, pfh_ctl_p); + if (pfh_cnt_p) + error |= get_user(kbuf.pfh_cnt, pfh_cnt_p); + + if (!error && (kbuf.pfh_ctl || kbuf.pfh_cnt)) { + smp_call_function(local_set_pfh, &kbuf, 1); + local_set_pfh(&kbuf); + } + } else { // op == 0, get + if (pfh_ctl_p) { + kbuf.pfh_ctl = sw64_read_csr(CSR_PFH_CTL); + error |= put_user(kbuf.pfh_ctl, pfh_ctl_p); + } + + if (pfh_cnt_p) { + kbuf.pfh_cnt = sw64_read_csr(CSR_PFH_CNT); + error |= put_user(kbuf.pfh_cnt, pfh_cnt_p); + } + } + + return error; +} + +#else + +SYSCALL_DEFINE0(pfh_ops) +{ + return -ENOSYS; +} + +#endif /* CONFIG_SUBARCH_C4 */ diff --git a/arch/sw_64/kernel/syscalls/syscall.tbl b/arch/sw_64/kernel/syscalls/syscall.tbl index fdf9e4cb03eb..3acd8b5ea7db 100644 --- a/arch/sw_64/kernel/syscalls/syscall.tbl +++ b/arch/sw_64/kernel/syscalls/syscall.tbl @@ -117,7 +117,7 @@ #107 is unused #108 is unused #109 is unused -#110 is unused +110 common pfh_ops sys_pfh_ops 111 common sigsuspend sys_sigsuspend #112 is unused 113 common recvmsg sys_recvmsg -- Gitee From 2ef16fcd4fc60f97a52a24be072ba1f7643b6f92 Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Mon, 15 Apr 2024 17:22:18 +0800 Subject: [PATCH 174/524] sw64: fix PTBR save/restore for hibernation For C4, both CSR:PTBR_USR and CSR:PTBR_SYS have to be saved/restored to make hibernation work. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/hibernate.c | 10 ++++++++++ arch/sw_64/kernel/hibernate_asm.S | 1 + 2 files changed, 11 insertions(+) diff --git a/arch/sw_64/kernel/hibernate.c b/arch/sw_64/kernel/hibernate.c index 644ea8504313..e84f93762f13 100644 --- a/arch/sw_64/kernel/hibernate.c +++ b/arch/sw_64/kernel/hibernate.c @@ -15,7 +15,12 @@ void save_processor_state(void) vcb->ksp = rdksp(); vcb->usp = rdusp(); vcb->soft_tid = rtid(); +#if defined(CONFIG_SUBARCH_C3B) vcb->ptbr = rdptbr(); +#elif defined(CONFIG_SUBARCH_C4) + vcb->ptbr_usr = sw64_read_csr(CSR_PTBR_USR); + vcb->ptbr_sys = sw64_read_csr(CSR_PTBR_SYS); +#endif } void restore_processor_state(void) @@ -25,7 +30,12 @@ void restore_processor_state(void) wrksp(vcb->ksp); wrusp(vcb->usp); wrtp(vcb->soft_tid); +#if defined(CONFIG_SUBARCH_C3B) wrptbr(vcb->ptbr); +#elif defined(CONFIG_SUBARCH_C4) + sw64_write_csr_imb(vcb->ptbr_usr, CSR_PTBR_USR); + sw64_write_csr_imb(vcb->ptbr_sys, CSR_PTBR_SYS); +#endif sflush(); tbiv(); } diff --git a/arch/sw_64/kernel/hibernate_asm.S b/arch/sw_64/kernel/hibernate_asm.S index ebba57b8bd8b..0655efc59a25 100644 --- a/arch/sw_64/kernel/hibernate_asm.S +++ b/arch/sw_64/kernel/hibernate_asm.S @@ -3,6 +3,7 @@ #include #include #include +#include .macro SAVE_KTP #ifdef CONFIG_SUBARCH_C3B -- Gitee From d9af02bb6458e674827817969d11c2b2695cc468 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Thu, 18 Apr 2024 10:34:54 +0800 Subject: [PATCH 175/524] sw64: fix save_ktp compatibility for C3 Commit 388c8e2bbf53 ("sw64: introduce new entry framework for C4") moves save_ktp() to head.S, and breaks compatibility with old firmware because hmcall_wrktp was not implemented. To fix it, save_ktp() should come after fixup_hmcall(). Fixes: 388c8e2bbf53 ("sw64: introduce new entry framework for C4") Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hmcall.h | 9 +++++++-- arch/sw_64/kernel/early_init.c | 1 + arch/sw_64/kernel/head.S | 1 - 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/include/asm/hmcall.h b/arch/sw_64/include/asm/hmcall.h index 71328c4eb332..c9509327a229 100644 --- a/arch/sw_64/include/asm/hmcall.h +++ b/arch/sw_64/include/asm/hmcall.h @@ -230,13 +230,18 @@ __CALL_HMC_W1(wrtp, unsigned long); /* Invalidate all user TLB with current UPN and VPN */ #define tbiu() __tbi(4, /* no second argument */) -#ifndef CONFIG_SUBARCH_C3B +#if defined(CONFIG_SUBARCH_C4) __CALL_HMC_W2(wrap_asid, unsigned long, unsigned long); -#else +static inline void save_ktp(void) +{ + __asm__ __volatile__("csrw $8, 0xef"); +} +#elif defined(CONFIG_SUBARCH_C3B) static inline void wrap_asid(unsigned long asid, unsigned long ptbr) { tbivp(); } +#define save_ktp() wrktp() #endif #endif /* !__ASSEMBLY__ */ diff --git a/arch/sw_64/kernel/early_init.c b/arch/sw_64/kernel/early_init.c index 3d7b9c4325a9..2ec7a3e99443 100644 --- a/arch/sw_64/kernel/early_init.c +++ b/arch/sw_64/kernel/early_init.c @@ -6,5 +6,6 @@ asmlinkage __visible void __init sw64_start_kernel(void) { fixup_hmcall(); + save_ktp(); start_kernel(); } diff --git a/arch/sw_64/kernel/head.S b/arch/sw_64/kernel/head.S index db4af8f9366b..15265fa4a0ea 100644 --- a/arch/sw_64/kernel/head.S +++ b/arch/sw_64/kernel/head.S @@ -32,7 +32,6 @@ __start: 1: ldgp $29, 0($27) /* We need to get current_task_info loaded up... */ ldi $8, init_task - SAVE_KTP ldl $30, TASK_STACK($8) /* ... and find our stack ... */ ldi $30, ASM_THREAD_SIZE($30) -- Gitee From d54eee89386f4c8574a1fd87074eb63bd2b179f7 Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Mon, 22 Apr 2024 14:51:36 +0800 Subject: [PATCH 176/524] sw64: fix compile warning of do_entUna() Fix the following warnings: arch/sw_64/kernel/traps.c: In function 'do_entUna': arch/sw_64/kernel/traps.c:411:3: warning: this statement may fall through [-Wimplicit-fallthrough=] switch (fncode) { ^~~~~~ arch/sw_64/kernel/traps.c:585:2: note: here case 0x21: ^~~~ Fixes: 678e75e7273f ("sw64: add unaligned access support for C4 new instructions") Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/traps.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 1405e42f27b4..8dd522d09860 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -582,6 +582,8 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, regs->regs[rb] = regs->regs[rb] + disp; return; } + return; + case 0x21: __asm__ __volatile__( "1: ldl_u %1, 0(%3)\n" -- Gitee From 994e9049266b0ffa73016a73f980fa147fa7835c Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 22 Apr 2024 19:58:47 +0800 Subject: [PATCH 177/524] sw64: pci: get register base address from firmware Kernel get all register base addresses of PCIe controller from firmware. In this way, even if these base addresses change in the future, related drivers in kernel do not need to be modified. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 2 + arch/sw_64/pci/acpi.c | 26 +++--- drivers/pci/controller/pci-sunway.c | 123 +++++++++++++--------------- 3 files changed, 72 insertions(+), 79 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index f39d07144af9..074b57908aad 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -51,6 +51,8 @@ struct pci_controller { /* This one's for the kernel only. It's in KSEG somewhere. */ void __iomem *ep_config_space_base; void __iomem *rc_config_space_base; + void __iomem *piu_ior0_base; + void __iomem *piu_ior1_base; unsigned long index; unsigned long node; diff --git a/arch/sw_64/pci/acpi.c b/arch/sw_64/pci/acpi.c index 2f446116e69e..6f03ea79131c 100644 --- a/arch/sw_64/pci/acpi.c +++ b/arch/sw_64/pci/acpi.c @@ -5,6 +5,8 @@ #include #include +#define UPPER_32_BITS_OF_U64(u64_val) ((u64_val) & 0xFFFFFFFF00000000ULL) + struct pci_root_info { struct acpi_pci_root_info info; struct pci_config_window *cfg; @@ -88,25 +90,25 @@ pci_acpi_setup_ecam_mapping(struct acpi_pci_root *root) static int pci_acpi_prepare_root_resources(struct acpi_pci_root_info *ci) { int status = 0; - acpi_status rc; - unsigned long long memh = 0; + u64 memh; struct resource_entry *entry = NULL, *tmp = NULL; struct acpi_device *device = ci->bridge; /** * To distinguish between mem and pre_mem, firmware - * only pass the lower 32bits of mem via acpi and - * use vendor specific "MEMH" to record the upper - * 32 bits of mem. + * only pass the lower 32bits of mem via _CRS method. * * Get the upper 32 bits here. */ - rc = acpi_evaluate_integer(ci->bridge->handle, "MEMH", NULL, &memh); - if (rc != AE_OK) { - dev_err(&device->dev, "unable to retrieve MEMH\n"); - return -EEXIST; + status = fwnode_property_read_u64_array(&device->fwnode, + "sw64,ep_mem_32_base", &memh, 1); + if (status) { + dev_err(&device->dev, "unable to retrieve \"sw64,ep_mem_32_base\"\n"); + return status; } + memh = UPPER_32_BITS_OF_U64(memh); + /** * Get host bridge resources via _CRS method, the return value * is the num of resource parsed. @@ -129,10 +131,10 @@ static int pci_acpi_prepare_root_resources(struct acpi_pci_root_info *ci) resource_list_for_each_entry_safe(entry, tmp, &ci->resources) { if (entry->res->flags & IORESOURCE_MEM) { - if (!(entry->res->end & 0xFFFFFFFF00000000ULL)) { + if (!UPPER_32_BITS_OF_U64(entry->res->end)) { /* Patch mem res with upper 32 bits */ - entry->res->start |= (memh << 32); - entry->res->end |= (memh << 32); + entry->res->start |= memh; + entry->res->end |= memh; } else { /** * Add PREFETCH and MEM_64 flags for diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 2f2b5c19a414..1a4bc27eb660 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -666,70 +666,61 @@ static void setup_intx_irqs(struct pci_controller *hose) set_pcieport_service_irq(node, index); } -static int sw64_pci_prepare_controller(struct pci_controller *hose, - struct acpi_device *adev) -{ - unsigned long long index, node; - unsigned long long rc_config_base_addr; - unsigned long long pci_io_base_addr; - unsigned long long ep_io_base_addr; - acpi_status rc; - - /* Get node from ACPI namespace */ - node = acpi_get_node(adev->handle); - if (node == NUMA_NO_NODE) { - dev_err(&adev->dev, "unable to get node ID\n"); - return -EEXIST; - } - - /* Get index from ACPI namespace */ - rc = acpi_evaluate_integer(adev->handle, "INDX", NULL, &index); - if (rc != AE_OK) { - dev_err(&adev->dev, "unable to retrieve INDX\n"); - return -EEXIST; - } - - /** - * Get Root Complex config space base address. - * - * For sw64, Root Complex config space base addr is different - * from Endpoint config space base address. Use MCFG table to - * pass Endpoint config space base address, and define Root Complex - * config space base address("RCCB") separately in the ACPI namespace. - */ - rc = acpi_evaluate_integer(adev->handle, - "RCCB", NULL, &rc_config_base_addr); - if (rc != AE_OK) { - dev_err(&adev->dev, "unable to retrieve RCCB\n"); - return -EEXIST; - } +enum pci_props { + PROP_RC_CONFIG_BASE = 0, + PROP_EP_CONFIG_BASE, + PROP_EP_MEM_32_BASE, + PROP_EP_MEM_64_BASE, + PROP_EP_IO_BASE, + PROP_PIU_IOR0_BASE, + PROP_PIU_IOR1_BASE, + PROP_RC_INDEX, + PROP_PCIE_IO_BASE, + PROP_NUM +}; - /* Get Root Complex I/O space base addr from ACPI namespace */ - rc = acpi_evaluate_integer(adev->handle, - "RCIO", NULL, &pci_io_base_addr); - if (rc != AE_OK) { - dev_err(&adev->dev, "unable to retrieve RCIO\n"); - return -EEXIST; - } +const char *prop_names[PROP_NUM] = { + "sw64,rc_config_base", + "sw64,ep_config_base", + "sw64,ep_mem_32_base", + "sw64,ep_mem_64_base", + "sw64,ep_io_base", + "sw64,piu_ior0_base", + "sw64,piu_ior1_base", + "sw64,rc_index", + "sw64,pcie_io_base" +}; - /* Get Endpoint I/O space base addr from ACPI namespace */ - rc = acpi_evaluate_integer(adev->handle, - "EPIO", NULL, &ep_io_base_addr); - if (rc != AE_OK) { - dev_err(&adev->dev, "unable to retrieve EPIO\n"); - return -EEXIST; +static int sw64_pci_prepare_controller(struct pci_controller *hose, + struct fwnode_handle *fwnode) +{ + u64 props[PROP_NUM]; + int i, ret; + + /* Get properties of Root Complex */ + for (i = 0; i < PROP_NUM; ++i) { + ret = fwnode_property_read_u64_array(fwnode, prop_names[i], + &props[i], 1); + if (ret) { + pr_err("unable to retrieve \"%s\"\n", + prop_names[i]); + return ret; + } } hose->iommu_enable = false; - hose->index = index; - hose->node = node; + + hose->index = props[PROP_RC_INDEX]; hose->sparse_mem_base = 0; hose->sparse_io_base = 0; - hose->dense_mem_base = pci_io_base_addr; - hose->dense_io_base = ep_io_base_addr; + hose->dense_mem_base = props[PROP_PCIE_IO_BASE]; + hose->dense_io_base = props[PROP_EP_IO_BASE]; - hose->rc_config_space_base = __va(rc_config_base_addr); + hose->rc_config_space_base = __va(props[PROP_RC_CONFIG_BASE]); + hose->ep_config_space_base = __va(props[PROP_EP_CONFIG_BASE]); + hose->piu_ior0_base = __va(props[PROP_PIU_IOR0_BASE]); + hose->piu_ior1_base = __va(props[PROP_PIU_IOR1_BASE]); hose->first_busno = 0xff; hose->last_busno = 0xff; @@ -754,11 +745,11 @@ static int sw64_pci_prepare_controller(struct pci_controller *hose, * Root Complex link up failed. * This usually means that no device on the slot. */ - dev_info(&adev->dev, ": failed to link up\n", + pr_info(": link down\n", hose->node, hose->index); } else { pci_mark_rc_linkup(hose->node, hose->index); - dev_info(&adev->dev, ": successfully link up\n", + pr_info(": successfully link up\n", hose->node, hose->index); } @@ -777,7 +768,6 @@ static int sw64_pci_ecam_init(struct pci_config_window *cfg) struct pci_controller *hose = NULL; struct device *dev = cfg->parent; struct acpi_device *adev = to_acpi_device(dev); - phys_addr_t mcfg_addr; int ret; /** @@ -805,17 +795,16 @@ static int sw64_pci_ecam_init(struct pci_config_window *cfg) if (!hose) return -ENOMEM; - /* Get Endpoint config space base address from MCFG table */ - mcfg_addr = cfg->res.start - (cfg->busr.start << cfg->ops->bus_shift); - - /** - * "__va(mcfg_addr)" is equal to "cfg->win", so we can also use - * "hose->ep_config_space_base = cfg->win" here - */ - hose->ep_config_space_base = __va(mcfg_addr); + /* Get node from ACPI namespace (_PXM) */ + hose->node = acpi_get_node(adev->handle); + if (hose->node == NUMA_NO_NODE) { + kfree(hose); + dev_err(&adev->dev, "unable to get node ID\n"); + return -EINVAL; + } /* Init pci_controller */ - ret = sw64_pci_prepare_controller(hose, adev); + ret = sw64_pci_prepare_controller(hose, &adev->fwnode); if (ret) { kfree(hose); dev_err(&adev->dev, "failed to init pci controller\n"); -- Gitee From 6604347a379b9102cf7e26c279efca402e97c8ab Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 24 Apr 2024 20:15:54 +0800 Subject: [PATCH 178/524] sw64: acpi: remove constraint that cid of boot core must be zero The current code has a constraint that cid of boot core must be zero. Actually, the boot core is the first online core, and its cid does not have to be zero. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/acpi.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index c50ae846ff62..d0e853659211 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -22,9 +22,9 @@ EXPORT_SYMBOL(acpi_pci_disabled); static bool param_acpi_on __initdata; static bool param_acpi_off __initdata; -static unsigned int possible_cores = 1; /* number of possible cores(at least boot core) */ -static unsigned int present_cores = 1; /* number of present cores(at least boot core) */ -static unsigned int disabled_cores; /* number of disabled cores */ +static unsigned int possible_cores; /* number of possible cores */ +static unsigned int present_cores; /* number of present cores */ +static unsigned int disabled_cores; /* number of disabled cores */ int acpi_strict; u64 arch_acpi_wakeup_start; @@ -260,12 +260,6 @@ setup_rcid_and_core_mask(struct acpi_madt_sw_cintc *sw_cintc) return -EINVAL; } - /* We can never disable the boot core, whose rcid is 0 */ - if ((rcid == 0) && !is_core_enabled(sw_cintc->flags)) { - pr_err(PREFIX "Boot core disabled in MADT\n"); - return -EINVAL; - } - /* Online capable makes core possible */ if (!is_core_enabled(sw_cintc->flags) && !is_core_online_capable(sw_cintc->flags)) { @@ -273,13 +267,9 @@ setup_rcid_and_core_mask(struct acpi_madt_sw_cintc *sw_cintc) return 0; } - rcid_information_init(sw_cintc->version); + logical_core_id = possible_cores++; - /* The logical core ID of the boot core must be 0 */ - if (rcid == 0) - logical_core_id = 0; - else - logical_core_id = possible_cores++; + rcid_information_init(sw_cintc->version); set_rcid_map(logical_core_id, rcid); set_cpu_possible(logical_core_id, true); @@ -294,8 +284,7 @@ setup_rcid_and_core_mask(struct acpi_madt_sw_cintc *sw_cintc) if (is_core_enabled(sw_cintc->flags) && !cpumask_test_cpu(logical_core_id, &cpu_offline)) { set_cpu_present(logical_core_id, true); - if (logical_core_id != 0) - present_cores++; + present_cores++; } return 0; -- Gitee From b22499a829015e3fc3f41662679da1042755f4c2 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 26 Apr 2024 16:55:50 +0000 Subject: [PATCH 179/524] sw64: distribute intx interrupts by hose CPU0 may encounter performance issue if there are too many intx interrupts erupted. We change the intx interrupts init strategy to improve this condition. Instead of sending all intx interrupts to CPU0 directly, these interrupts will be distributed to different cores by hose accordingly. Thus, the overall pressure on CPU0 is expected to be substantially reduced. We also refactor intx interrupts handling and integrate all intx related code into another file to make it look more clean and organized. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 5 + arch/sw_64/pci/pci-legacy.c | 41 ------ drivers/irqchip/Kconfig | 5 + drivers/irqchip/Makefile | 1 + drivers/irqchip/irq-sunway-pci-intx.c | 185 ++++++++++++++++++++++++++ drivers/pci/controller/pci-sunway.c | 30 ----- 6 files changed, 196 insertions(+), 71 deletions(-) create mode 100644 drivers/irqchip/irq-sunway-pci-intx.c diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 074b57908aad..1bd2123362f0 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -82,6 +82,7 @@ struct pci_controller { extern void __init sw64_init_pci(void); extern void __init sw64_device_interrupt(unsigned long vector); +extern void setup_intx_irqs(struct pci_controller *hose); extern void __init sw64_init_irq(void); extern void __init sw64_init_arch(void); extern struct pci_ops sw64_pci_ops; @@ -162,6 +163,10 @@ extern void pci_remove_resource_files(struct pci_dev *dev); extern void __init reserve_mem_for_pci(void); extern int chip_pcie_configure(struct pci_controller *hose); +#define PCI_INTX_ENABLE ((1UL) << 62) +#define PCI_INTX_DISABLE ~((1UL) << 62) +#define PCI_INTX_VALID (1UL << 63) + #define PCI_VENDOR_ID_JN 0x5656 #define PCI_DEVICE_ID_SW64_ROOT_BRIDGE 0x3231 #define PCI_DEVICE_ID_JN_PCIESW 0x1000 diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index 15ee626593e9..c968b3543dbe 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -304,47 +304,6 @@ void __init sw64_init_arch(void) void __weak set_pcieport_service_irq(int node, int index) {} -static void __init sw64_init_intx(struct pci_controller *hose) -{ - unsigned long int_conf, node, val_node; - unsigned long index, irq; - int rcid; - - node = hose->node; - index = hose->index; - - if (!node_online(node)) - val_node = next_node_in(node, node_online_map); - else - val_node = node; - irq = irq_alloc_descs_from(NR_IRQS_LEGACY, 2, val_node); - WARN_ON(irq < 0); - irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_level_irq); - irq_set_status_flags(irq, IRQ_LEVEL); - hose->int_irq = irq; - irq_set_chip_and_handler(irq + 1, &dummy_irq_chip, handle_level_irq); - hose->service_irq = irq + 1; - rcid = cpu_to_rcid(0); - - pr_info_once("INTx are directed to node %d core %d.\n", - ((rcid >> 6) & 0x3), (rcid & 0x1f)); - int_conf = 1UL << 62 | rcid; /* rebase all intx on the first logical cpu */ - if (sw64_chip_init->pci_init.set_intx) - sw64_chip_init->pci_init.set_intx(node, index, int_conf); - - set_pcieport_service_irq(node, index); -} - -void __init sw64_init_irq(void) -{ - struct pci_controller *hose; - - /* Scan all of the recorded PCI controllers. */ - hose = hose_head; - for (hose = hose_head; hose; hose = hose->next) - sw64_init_intx(hose); -} - void __init sw64_init_pci(void) { diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 35037f1e8d7a..21aeae9af675 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -37,6 +37,11 @@ config SW64_IRQ_CPU depends on SW64 default y +config SW64_PCI_INTX + bool + depends on SW64 && PCI + default y + config SW64_IRQ_MSI bool depends on SW64 && PCI_MSI diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 6b90d7ce01f4..cd085dde0d35 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_ARCH_SPEAR3XX) += spear-shirq.o obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o obj-$(CONFIG_SW64_PINTC) += irq-sunway-pintc.o obj-$(CONFIG_SW64_LPC_INTC) += irq-sunway-lpc-intc.o +obj-$(CONFIG_SW64_PCI_INTX) += irq-sunway-pci-intx.o obj-$(CONFIG_SW64_IRQ_CPU) += irq-sunway-cpu.o ifeq ($(CONFIG_UNCORE_XUELANG),y) diff --git a/drivers/irqchip/irq-sunway-pci-intx.c b/drivers/irqchip/irq-sunway-pci-intx.c new file mode 100644 index 000000000000..93ee2361e958 --- /dev/null +++ b/drivers/irqchip/irq-sunway-pci-intx.c @@ -0,0 +1,185 @@ +#include +#include +#include + +#include +#include +#include +#include +#include + +static DEFINE_RAW_SPINLOCK(legacy_lock); +static void lock_legacy_lock(void) +{ + raw_spin_lock(&legacy_lock); +} + +static void unlock_legacy_lock(void) +{ + raw_spin_unlock(&legacy_lock); +} + +static void set_intx(unsigned long node, unsigned long index, unsigned long intx_conf) +{ + if (is_guest_or_emul()) + return; + + write_piu_ior0(node, index, INTACONFIG, intx_conf | (0x8UL << 10)); + write_piu_ior0(node, index, INTBCONFIG, intx_conf | (0x4UL << 10)); + write_piu_ior0(node, index, INTCCONFIG, intx_conf | (0x2UL << 10)); + write_piu_ior0(node, index, INTDCONFIG, intx_conf | (0x1UL << 10)); +} + +static int __assign_piu_intx_config(struct pci_controller *hose, cpumask_t *targets) +{ + unsigned long intx_conf; + unsigned int cpu; + int node, index; + int phy_cpu; + + node = hose->node; + index = hose->index; + + /* Use the last cpu in valid cpus to avoid core 0. */ + cpu = cpumask_last(targets); + phy_cpu = cpu_to_rcid(cpu); + + intx_conf = ((phy_cpu >> 5) << 6) | (phy_cpu & 0x1f); + set_intx(node, index, intx_conf); + + return 0; +} + +static int assign_piu_intx_config(struct pci_controller *hose, cpumask_t *targets) +{ + int ret; + + lock_legacy_lock(); + ret = __assign_piu_intx_config(hose, targets); + unlock_legacy_lock(); + + return ret; +} + +static void intx_irq_enable(struct irq_data *irq_data) +{ + struct pci_controller *hose = irq_data->chip_data; + unsigned long intx_conf, node, index; + + if (is_guest_or_emul()) + return; + BUG_ON(!hose); + + node = hose->node; + index = hose->index; + + intx_conf = read_piu_ior0(node, index, INTACONFIG); + intx_conf |= PCI_INTX_ENABLE; + write_piu_ior0(node, index, INTACONFIG, intx_conf); + + intx_conf = read_piu_ior0(node, index, INTBCONFIG); + intx_conf |= PCI_INTX_ENABLE; + write_piu_ior0(node, index, INTBCONFIG, intx_conf); + + intx_conf = read_piu_ior0(node, index, INTCCONFIG); + intx_conf |= PCI_INTX_ENABLE; + write_piu_ior0(node, index, INTCCONFIG, intx_conf); + + intx_conf = read_piu_ior0(node, index, INTDCONFIG); + intx_conf |= PCI_INTX_ENABLE; + write_piu_ior0(node, index, INTDCONFIG, intx_conf); +} + +static void intx_irq_disable(struct irq_data *irq_data) +{ + struct pci_controller *hose = irq_data->chip_data; + unsigned long intx_conf, node, index; + + if (is_guest_or_emul()) + return; + + BUG_ON(!hose); + node = hose->node; + index = hose->index; + + intx_conf = read_piu_ior0(node, index, INTACONFIG); + intx_conf &= PCI_INTX_DISABLE; + write_piu_ior0(node, index, INTACONFIG, intx_conf); + + intx_conf = read_piu_ior0(node, index, INTBCONFIG); + intx_conf &= PCI_INTX_DISABLE; + write_piu_ior0(node, index, INTBCONFIG, intx_conf); + + intx_conf = read_piu_ior0(node, index, INTCCONFIG); + intx_conf &= PCI_INTX_DISABLE; + write_piu_ior0(node, index, INTCCONFIG, intx_conf); + + intx_conf = read_piu_ior0(node, index, INTDCONFIG); + intx_conf &= PCI_INTX_DISABLE; + write_piu_ior0(node, index, INTDCONFIG, intx_conf); +} + +static int intx_set_affinity(struct irq_data *irq_data, + const struct cpumask *dest, bool force) +{ + struct pci_controller *hose = irq_data->chip_data; + cpumask_t targets; + int ret = 0; + + if (cpumask_any_and(dest, cpu_online_mask) >= nr_cpu_ids) + return -EINVAL; + + cpumask_copy(&targets, dest); + + intx_irq_disable(irq_data); + ret = assign_piu_intx_config(hose, &targets); + intx_irq_enable(irq_data); + + return ret; +} + +static void noop(struct irq_data *d) {} + +static struct irq_chip sw64_intx_chip = { + .name = "PCI_INTX", + .irq_enable = intx_irq_enable, + .irq_disable = intx_irq_disable, + .irq_set_affinity = intx_set_affinity, + .irq_ack = noop, + .flags = IRQCHIP_SKIP_SET_WAKE, +}; + +void __weak set_pcieport_service_irq(int node, int index) {} + +void setup_intx_irqs(struct pci_controller *hose) +{ + unsigned long irq, node, index, val_node; + + node = hose->node; + index = hose->index; + + if (!node_online(node)) + val_node = next_node_in(node, node_online_map); + else + val_node = node; + + irq = irq_alloc_descs_from(NR_IRQS_LEGACY, 2, val_node); + WARN_ON(irq < 0); + irq_set_chip_and_handler(irq, &sw64_intx_chip, handle_level_irq); + irq_set_status_flags(irq, IRQ_LEVEL); + irq_set_chip_data(irq, hose); + hose->int_irq = irq; + irq_set_chip_and_handler(irq + 1, &dummy_irq_chip, handle_level_irq); + hose->service_irq = irq + 1; + + set_pcieport_service_irq(node, index); +} + +void __init sw64_init_irq(void) +{ + struct pci_controller *hose = hose_head; + + for (hose = hose_head; hose; hose = hose->next) + setup_intx_irqs(hose); +} + diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 1a4bc27eb660..420572592779 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -635,36 +635,6 @@ int sw64_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) } #ifdef CONFIG_ACPI -static void setup_intx_irqs(struct pci_controller *hose) -{ - unsigned long int_conf, node, val_node; - unsigned long index, irq; - int rcid; - - node = hose->node; - index = hose->index; - - if (!node_online(node)) - val_node = next_node_in(node, node_online_map); - else - val_node = node; - irq = irq_alloc_descs_from(NR_IRQS_LEGACY, 2, val_node); - WARN_ON(irq < 0); - irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_level_irq); - irq_set_status_flags(irq, IRQ_LEVEL); - hose->int_irq = irq; - irq_set_chip_and_handler(irq + 1, &dummy_irq_chip, handle_level_irq); - hose->service_irq = irq + 1; - rcid = cpu_to_rcid(0); - - pr_info_once("INTx are directed to node %d core %d.\n", - ((rcid >> 6) & 0x3), (rcid & 0x1f)); - int_conf = 1UL << 62 | rcid; /* rebase all intx on the first logical cpu */ - - set_intx(node, index, int_conf); - - set_pcieport_service_irq(node, index); -} enum pci_props { PROP_RC_CONFIG_BASE = 0, -- Gitee From d6f72efdf0779139f6b40b4c777b24c79cd786b9 Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Mon, 22 Apr 2024 14:51:36 +0800 Subject: [PATCH 180/524] sw64: fix compile warning of smp_callin() This patch fixes a mixed declaration in smp_callin(). Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index a57d05fe700f..465775cdb4fe 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -58,8 +58,7 @@ enum core_version { */ void smp_callin(void) { - int cpuid = smp_processor_id(); - + int cpuid; #ifdef CONFIG_SUBARCH_C4 /* LV2 select PLL1 */ int i, cpu_num; @@ -72,7 +71,7 @@ void smp_callin(void) udelay(1000); } #endif - + cpuid = smp_processor_id(); local_irq_disable(); if (cpu_online(cpuid)) { -- Gitee From 9ea15a0b0cf134a18d302983cc72b690201030e3 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Tue, 23 Apr 2024 09:15:51 +0800 Subject: [PATCH 181/524] sw64: remove redudant save_ktp Macro save_ktp are defined in both hmcall.h and head.S which is unnecessary. Remove the definition in head.S. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/head.S | 9 --------- arch/sw_64/kernel/smp.c | 1 + 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/arch/sw_64/kernel/head.S b/arch/sw_64/kernel/head.S index 15265fa4a0ea..03da6f80dd0d 100644 --- a/arch/sw_64/kernel/head.S +++ b/arch/sw_64/kernel/head.S @@ -12,14 +12,6 @@ #include #include - .macro SAVE_KTP -#ifdef CONFIG_SUBARCH_C3B - sys_call HMC_wrktp -#else - csrw $8, CSR_KTP -#endif - .endm - __HEAD .globl _stext .set noreorder @@ -120,7 +112,6 @@ __smp_callin: ldi $2, idle_task_pointer s8addl $0, $2, $2 ldl $8, 0($2) # Get ksp of idle thread - SAVE_KTP ldl $30, TASK_STACK($8) ldi $30, ASM_THREAD_SIZE($30) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 465775cdb4fe..4b5a97bdbeef 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -71,6 +71,7 @@ void smp_callin(void) udelay(1000); } #endif + save_ktp(); cpuid = smp_processor_id(); local_irq_disable(); -- Gitee From 339ba409c27d6820fdefd7ed4961c08fd95be098 Mon Sep 17 00:00:00 2001 From: He Sheng Date: Tue, 23 Apr 2024 15:05:45 +0800 Subject: [PATCH 182/524] sw64: wrap frequency workaround into standalone functions This will make the code easier to understand and maintain. Besides, frequency workaround is useless on guest and emulator, so nothing is done in that situation. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 61 +++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 4b5a97bdbeef..78d4082e1a58 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -53,25 +53,53 @@ enum core_version { CORE_VERSION_RESERVED = 3 /* 3 and greater are reserved */ }; -/* - * Where secondaries begin a life of C. - */ -void smp_callin(void) -{ - int cpuid; #ifdef CONFIG_SUBARCH_C4 - /* LV2 select PLL1 */ +static void upshift_freq(void) +{ int i, cpu_num; + if (is_guest_or_emul()) + return; cpu_num = sw64_chip->get_cpu_num(); - for (i = 0; i < cpu_num; i++) { sw64_io_write(i, CLU_LV2_SELH, -1UL); sw64_io_write(i, CLU_LV2_SELL, -1UL); udelay(1000); } +} + +static void downshift_freq(void) +{ + unsigned long value; + int cpuid, core_id, node_id; + + if (is_guest_or_emul()) + return; + cpuid = smp_processor_id(); + core_id = rcid_to_core_id(cpu_to_rcid(cpuid)); + node_id = rcid_to_domain_id(cpu_to_rcid(cpuid)); + + if (core_id > 31) { + value = 1UL << (2 * (core_id - 32)); + sw64_io_write(node_id, CLU_LV2_SELH, value); + } else { + value = 1UL << (2 * core_id); + sw64_io_write(node_id, CLU_LV2_SELL, value); + } +} +#else +static void upshift_freq(void) { } +static void downshift_freq(void) { } #endif + +/* + * Where secondaries begin a life of C. + */ +void smp_callin(void) +{ + int cpuid; save_ktp(); + upshift_freq(); cpuid = smp_processor_id(); local_irq_disable(); @@ -613,22 +641,7 @@ void __cpu_die(unsigned int cpu) void arch_cpu_idle_dead(void) { -#ifdef CONFIG_SUBARCH_C4 - /* LV2 select PLL0 */ - int cpuid = smp_processor_id(); - int core_id = rcid_to_core_id(cpu_to_rcid(cpuid)); - int node_id = rcid_to_domain_id(cpu_to_rcid(cpuid)); - unsigned long value; - - if (core_id > 31) { - value = 1UL << (2 * (core_id - 32)); - sw64_io_write(node_id, CLU_LV2_SELH, value); - } else { - value = 1UL << (2 * core_id); - sw64_io_write(node_id, CLU_LV2_SELL, value); - } -#endif - + downshift_freq(); idle_task_exit(); mb(); __this_cpu_write(cpu_state, CPU_DEAD); -- Gitee From f969760c2149207d2e789fdf70dbe7f11d7fb143 Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Fri, 26 Apr 2024 16:33:35 +0800 Subject: [PATCH 183/524] sw64: perf: fix event check order The perf test: Event times requires software events to enable exclude_hv, but sw64 does not support exclude_hv and directly returns -EINVAL during initialization check, result in failure in processing software events. This patch moves this check after the event type check to make the test pass. Fixes: af102b07f066 ("sw64: perf: do all event checks in sw64_pmu_event_init()") Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/perf_event.c | 18 +- arch/sw_64/kernel/perf_event_c4.c | 690 ++++++++++++++++++++++++++++++ 2 files changed, 699 insertions(+), 9 deletions(-) create mode 100644 arch/sw_64/kernel/perf_event_c4.c diff --git a/arch/sw_64/kernel/perf_event.c b/arch/sw_64/kernel/perf_event.c index 3c6a8283b1e6..7544814615a8 100644 --- a/arch/sw_64/kernel/perf_event.c +++ b/arch/sw_64/kernel/perf_event.c @@ -514,15 +514,6 @@ static int sw64_pmu_event_init(struct perf_event *event) if (has_branch_stack(event)) return -EOPNOTSUPP; - /* - * SW64 does not have per-counter usr/os/guest/host bits, - * we can distinguish exclude_user and exclude_kernel by - * sample mode. - */ - if (attr->exclude_hv || attr->exclude_idle || - attr->exclude_host || attr->exclude_guest) - return -EINVAL; - if (attr->exclude_user && attr->exclude_kernel) return -EOPNOTSUPP; /* @@ -559,6 +550,15 @@ static int sw64_pmu_event_init(struct perf_event *event) return -ENOENT; } + /* + * SW64 does not have per-counter usr/os/guest/host bits, + * we can distinguish exclude_user and exclude_kernel by + * sample mode. + */ + if (attr->exclude_hv || attr->exclude_idle || + attr->exclude_host || attr->exclude_guest) + return -EINVAL; + /* Do the real initialisation work. */ __hw_perf_event_init(event); diff --git a/arch/sw_64/kernel/perf_event_c4.c b/arch/sw_64/kernel/perf_event_c4.c new file mode 100644 index 000000000000..7a5e8deba0ee --- /dev/null +++ b/arch/sw_64/kernel/perf_event_c4.c @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Performance events support for SW64 platforms. + * + * This code is based upon riscv and sparc perf event code. + */ + +#include +#include +#include + +DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); + +/* + * A structure to hold the description of the PMCs available on a particular + * type of SW64 CPU. + */ +struct sw64_pmu_t { + /* generic hw/cache events table */ + const int (*cache_events)[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX]; + + /* method used to map hw/cache events */ + const int (*map_hw_event)(u64 config); + const int (*map_cache_event)(u64 config); + + /* The number of entries in the hw_event_map */ + int max_events; + + /* The number of counters on this pmu */ + int num_pmcs; + + /* + * The mask that isolates the PMC bits when the LSB of the counter + * is shifted to bit 0. + */ + unsigned long pmc_count_mask; + + /* The maximum period the PMC can count. */ + unsigned long pmc_max_period; + + /* + * The maximum value that may be written to the counter due to + * hardware restrictions is pmc_max_period - pmc_left. + */ + long pmc_left; + + /* Subroutine for checking validity of a raw event for this PMU. */ + bool (*raw_event_valid)(u64 config); +}; + +/* + * The SW64 PMU description currently in operation. This is set during + * the boot process to the specific CPU of the machine. + */ +static const struct sw64_pmu_t *sw64_pmu; + +/* + * SW64 PMC event types + * + * There is no one-to-one mapping of the possible hw event types to the + * actual codes that are used to program the PMCs hence we introduce our + * own hw event type identifiers. + */ +#define SW64_OP_UNSUPP (-EOPNOTSUPP) + +/* Mapping of the hw event types to the perf tool interface */ +static const int core4_hw_event_map[] = { + [PERF_COUNT_HW_CPU_CYCLES] = SW64_PMU_CYCLE, + [PERF_COUNT_HW_INSTRUCTIONS] = SW64_PMU_INSTRUCTIONS, + [PERF_COUNT_HW_CACHE_REFERENCES] = SW64_PMU_L2_REFERENCES, + [PERF_COUNT_HW_CACHE_MISSES] = SW64_PMU_L2_MISSES, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = SW64_PMU_BRANCH, + [PERF_COUNT_HW_BRANCH_MISSES] = SW64_PMU_BRANCH_MISSES, +}; + +/* Mapping of the hw cache event types to the perf tool interface */ +#define C(x) PERF_COUNT_HW_CACHE_##x +static const int core4_cache_event_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = SW64_L1D_CACHE, + [C(RESULT_MISS)] = SW64_L1D_CACHE_MISSES, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_L1D_CACHE, + [C(RESULT_MISS)] = SW64_L1D_CACHE_MISSES, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = SW64_L1I_CACHE, + [C(RESULT_MISS)] = SW64_L1I_CACHE_MISSES, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_L1I_CACHE, + [C(RESULT_MISS)] = SW64_L1I_CACHE_MISSES, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + }, + [C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = SW64_DTB, + [C(RESULT_MISS)] = SW64_DTB_MISSES, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_DTB, + [C(RESULT_MISS)] = SW64_DTB_MISSES, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, + }, + }, + +}; + +static const int core4_map_hw_event(u64 config) +{ + return core4_hw_event_map[config]; +} + +static const int core4_map_cache_event(u64 config) +{ + unsigned int cache_type, cache_op, cache_result; + int ev; + + cache_type = (config >> 0) & 0xff; + if (cache_type >= PERF_COUNT_HW_CACHE_MAX) + return -EINVAL; + + cache_op = (config >> 8) & 0xff; + if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) + return -EINVAL; + + + cache_result = (config >> 16) & 0xff; + if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + return -EINVAL; + + ev = (*sw64_pmu->cache_events)[cache_type][cache_op][cache_result]; + + return ev; +} + +/* + * rxyy for counterx. + * According to the datasheet, 0 <= x < 5, 00 <= yy <= 8D + */ +static bool core4_raw_event_valid(u64 config) +{ + int idx = config >> 8; + int event = config & 0xff; + + if (idx >= 0 && idx < MAX_HWEVENTS && + event >= PC_RAW_BASE && event <= (PC_RAW_BASE + PC_MAX)) + return true; + + pr_info("sw64 pmu: invalid raw event config %#llx\n", config); + return false; +} + +static const struct sw64_pmu_t core4_pmu = { + .max_events = ARRAY_SIZE(core4_hw_event_map), + .map_hw_event = core4_map_hw_event, + .cache_events = &core4_cache_event_map, + .map_cache_event = core4_map_cache_event, + .num_pmcs = MAX_HWEVENTS, + .pmc_count_mask = PMC_COUNT_MASK, + .pmc_max_period = PMC_COUNT_MASK, + .raw_event_valid = core4_raw_event_valid, +}; + +/* Set a new period to sample over */ +static int sw64_perf_event_set_period(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + long left = local64_read(&hwc->period_left); + long period = hwc->sample_period; + int overflow = 0, idx = hwc->idx; + long value; + + if (unlikely(left <= -period)) { + left = period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + overflow = 1; + } + + if (unlikely(left <= 0)) { + left += period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + overflow = 1; + } + + if (left > sw64_pmu->pmc_max_period) + left = sw64_pmu->pmc_max_period; + + value = sw64_pmu->pmc_max_period - left; + local64_set(&hwc->prev_count, value); + switch (hwc->config) { + case SW64_PMU_INSTRUCTIONS: + sw64_write_csr(value, CSR_RETIC); + break; + case SW64_PMU_BRANCH: + sw64_write_csr(value, CSR_BRRETC); + break; + case SW64_PMU_BRANCH_MISSES: + sw64_write_csr(value, CSR_BRFAILC); + break; + case SW64_L1I_CACHE: + sw64_write_csr(value, CSR_IACC); + break; + case SW64_L1I_CACHE_MISSES: + sw64_write_csr(value, CSR_IMISC); + break; + default: + wrperfmon(idx + PMC_CMD_WRITE_BASE, value); + } + + perf_event_update_userpage(event); + + return overflow; +} + +/* + * Calculates the count (the 'delta') since the last time the PMC was read. + * + * As the PMCs' full period can easily be exceeded within the perf system + * sampling period we cannot use any high order bits as a guard bit in the + * PMCs to detect overflow as is done by other architectures. The code here + * calculates the delta on the basis that there is no overflow when ovf is + * zero. The value passed via ovf by the interrupt handler corrects for + * overflow. + * + * This can be racey on rare occasions -- a call to this routine can occur + * with an overflowed counter just before the PMI service routine is called. + * The check for delta negative hopefully always rectifies this situation. + */ +static unsigned long sw64_perf_event_update(struct perf_event *event) +{ + long prev_raw_count, new_raw_count; + long delta; + struct hw_perf_event *hwc = &event->hw; + int idx = event->hw.idx; + +again: + prev_raw_count = local64_read(&hwc->prev_count); + switch (hwc->config) { + case SW64_PMU_INSTRUCTIONS: + new_raw_count = sw64_read_csr(CSR_RETIC); + break; + case SW64_PMU_BRANCH: + new_raw_count = sw64_read_csr(CSR_BRRETC); + break; + case SW64_PMU_BRANCH_MISSES: + new_raw_count = sw64_read_csr(CSR_BRFAILC); + break; + case SW64_L1I_CACHE: + new_raw_count = sw64_read_csr(CSR_IACC); + break; + case SW64_L1I_CACHE_MISSES: + new_raw_count = sw64_read_csr(CSR_IMISC); + break; + default: + new_raw_count = wrperfmon(idx + MAX_HWEVENTS, 0); + } + + if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count) != prev_raw_count) + goto again; + + delta = new_raw_count - prev_raw_count; + + local64_add(delta, &event->count); + local64_sub(delta, &hwc->period_left); + + return new_raw_count; +} + +/* + * State transition functions: + * + * add()/del() & start()/stop() + * + */ + +/* + * pmu->stop: stop the counter + */ +static void sw64_pmu_stop(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + unsigned long value; + + if (!(hwc->state & PERF_HES_STOPPED)) { + value = sw64_read_csr(CSR_IDR_PCCTL); + switch (hwc->config) { + case SW64_L1I_CACHE: + sw64_write_csr(value & ~IACC_EN, CSR_IDR_PCCTL); + break; + case SW64_L1I_CACHE_MISSES: + sw64_write_csr(value & ~IMISC_EN, CSR_IDR_PCCTL); + break; + case SW64_PMU_INSTRUCTIONS: + sw64_write_csr(value & ~RETIC_EN, CSR_IDR_PCCTL); + break; + case SW64_PMU_BRANCH: + sw64_write_csr(value & ~BRRETC_EN, CSR_IDR_PCCTL); + break; + case SW64_PMU_BRANCH_MISSES: + sw64_write_csr(value & ~BRFAILC_EN, CSR_IDR_PCCTL); + break; + default: + wrperfmon(PMC_CMD_DISABLE, idx); + } + cpuc->event[idx] = NULL; + event->hw.state |= PERF_HES_STOPPED; + barrier(); + } + + if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { + sw64_perf_event_update(event); + hwc->state |= PERF_HES_UPTODATE; + } +} + +/* + * pmu->start: start the event. + */ +static void sw64_pmu_start(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + unsigned long value; + + if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) + return; + + if (flags & PERF_EF_RELOAD) { + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); + sw64_perf_event_set_period(event); + } + + cpuc->event[idx] = event; + event->hw.state = 0; + value = sw64_read_csr(CSR_IDR_PCCTL); + + /* counting in selected modes, for both counters */ + switch (hwc->config) { + case SW64_L1I_CACHE: + sw64_write_csr(value | IACC_EN, CSR_IDR_PCCTL); + break; + case SW64_L1I_CACHE_MISSES: + sw64_write_csr(value | IMISC_EN, CSR_IDR_PCCTL); + break; + case SW64_PMU_INSTRUCTIONS: + sw64_write_csr(value | RETIC_EN, CSR_IDR_PCCTL); + break; + case SW64_PMU_BRANCH: + sw64_write_csr(value | BRRETC_EN, CSR_IDR_PCCTL); + break; + case SW64_PMU_BRANCH_MISSES: + sw64_write_csr(value | BRFAILC_EN, CSR_IDR_PCCTL); + break; + default: + wrperfmon(idx, hwc->config << PC_ALL_PM_SET | hwc->config_base); + wrperfmon(PMC_CMD_ENABLE, idx); + } + perf_event_update_userpage(event); +} + +/* + * pmu->add: add the event to PMU. + */ +static int sw64_pmu_add(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + int err = -EAGAIN; + + if (__test_and_set_bit(idx, cpuc->used_mask)) { + idx = find_first_zero_bit(cpuc->used_mask, sw64_pmu->num_pmcs); + if (idx == sw64_pmu->num_pmcs) + goto out; + + __set_bit(idx, cpuc->used_mask); + hwc->idx = idx; + } + + event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; + if (flags & PERF_EF_START) + sw64_pmu_start(event, PERF_EF_RELOAD); + + /* Propagate our changes to the userspace mapping. */ + perf_event_update_userpage(event); + err = 0; +out: + return err; +} + +/* + * pmu->del: delete the event from PMU. + */ +static void sw64_pmu_del(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + + if (cpuc->event[hwc->idx] != event) + return; + + sw64_pmu_stop(event, PERF_EF_UPDATE); + __clear_bit(event->hw.idx, cpuc->used_mask); + + /* Absorb the final count and turn off the event. */ + perf_event_update_userpage(event); +} + +/* + * pmu->read: read and update the counter + */ +static void sw64_pmu_read(struct perf_event *event) +{ + sw64_perf_event_update(event); +} + +static bool supported_cpu(void) +{ + return true; +} + +static void hw_perf_event_destroy(struct perf_event *event) +{ + /* Nothing to be done! */ +} + +static int __hw_perf_event_init(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + hwc->config_base = SW64_PERFCTRL_AM; + + if (!is_sampling_event(event)) + pr_debug("not sampling event\n"); + + event->destroy = hw_perf_event_destroy; + + if (!hwc->sample_period) { + hwc->sample_period = sw64_pmu->pmc_max_period; + hwc->last_period = hwc->sample_period; + local64_set(&hwc->period_left, hwc->sample_period); + } + + return 0; +} + +/* + * Main entry point to initialise a HW performance event. + */ +static int sw64_pmu_event_init(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + struct hw_perf_event *hwc = &event->hw; + int config = -1; + + if (!sw64_pmu) + return -ENODEV; + + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + + /* + * SW64 does not support precise ip feature, and system hang when + * detecting precise_ip by perf_event_attr__set_max_precise_ip + * in userspace + */ + if (attr->precise_ip != 0) + return -EOPNOTSUPP; + + /* SW64 has fixed counter for given event type */ + switch (attr->type) { + case PERF_TYPE_HARDWARE: + if (attr->config >= sw64_pmu->max_events) + return -EINVAL; + config = sw64_pmu->map_hw_event(attr->config); + break; + case PERF_TYPE_HW_CACHE: + config = sw64_pmu->map_cache_event(attr->config); + break; + case PERF_TYPE_RAW: + if (!sw64_pmu->raw_event_valid(attr->config)) + return -EINVAL; + hwc->idx = attr->config >> 8; /* counter selector */ + config = attr->config & 0xff; /* event selector */ + break; + default: + return -ENOENT; + } + + if (config < 0) + return config; + + /* + * SW64 does not have per-counter usr/os/guest/host bits + */ + if (attr->exclude_hv || attr->exclude_idle || + attr->exclude_host || attr->exclude_guest) + return -EINVAL; + + hwc->config = config; + /* Do the real initialisation work. */ + __hw_perf_event_init(event); + + return 0; +} + +static struct pmu pmu = { + .name = "core4-base", + .capabilities = PERF_PMU_CAP_NO_NMI, + .event_init = sw64_pmu_event_init, + .add = sw64_pmu_add, + .del = sw64_pmu_del, + .start = sw64_pmu_start, + .stop = sw64_pmu_stop, + .read = sw64_pmu_read, +}; + +void perf_event_print_debug(void) +{ + unsigned long flags; + unsigned long pcr0, pcr1, pcr2, pcr3, pcr4; + int cpu; + + if (!supported_cpu()) + return; + + local_irq_save(flags); + + cpu = smp_processor_id(); + + pcr0 = wrperfmon(PMC_CMD_READ_PC0, 0); + pcr1 = wrperfmon(PMC_CMD_READ_PC1, 0); + pcr2 = wrperfmon(PMC_CMD_READ_PC2, 0); + pcr3 = wrperfmon(PMC_CMD_READ_PC3, 0); + pcr4 = wrperfmon(PMC_CMD_READ_PC4, 0); + + pr_info("CPU#%d: PCTR0[%lx] PCTR1[%lx]\n", cpu, pcr0, pcr1); + pr_info("PCTR0[%lx] PCTR0[%lx] PCTR1[%lx]\n", pcr2, pcr3, pcr4); + + local_irq_restore(flags); +} + +static void sw64_perf_event_irq_handler(unsigned long perfmon_num, + struct pt_regs *regs) +{ + struct perf_sample_data data; + struct cpu_hw_events *cpuc; + int idx; + u64 val; + + cpuc = this_cpu_ptr(&cpu_hw_events); + + for (idx = 0; idx < sw64_pmu->num_pmcs; ++idx) { + struct perf_event *event = cpuc->event[idx]; + + /* Ignore if we don't have an event. */ + if (!event) + continue; + + val = sw64_perf_event_update(event); + /* + * We have a single interrupt for all counters. Check that + * each counter has overflowed before we process it. + */ + if (val & (1ULL << (64 - 1))) + continue; + + /* + * event overflow + */ + perf_sample_data_init(&data, 0, event->hw.last_period); + + if (!sw64_perf_event_set_period(event)) + continue; + + if (perf_event_overflow(event, &data, regs)) + sw64_pmu_stop(event, 0); + } + + +} + +/* + * Init call to initialise performance events at kernel startup. + */ +int __init init_hw_perf_events(void) +{ + if (!supported_cpu()) { + pr_info("Performance events: Unsupported CPU type!\n"); + return 0; + } + + pr_info("Performance events: Supported CPU type!\n"); + + /* Override performance counter IRQ vector */ + + perf_irq = sw64_perf_event_irq_handler; + + /* And set up PMU specification */ + sw64_pmu = &core4_pmu; + + perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); + + return 0; +} +early_initcall(init_hw_perf_events); -- Gitee From 36d139476798aabcc2fa60a57f09385da9b34c11 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Wed, 8 May 2024 15:48:21 +0800 Subject: [PATCH 184/524] sw64: change FORCE_MAX_ZONEORDER default value FORCE_MAX_ZONEORDER=16 is used for 256M hugepage on C3B. To support guest memory hotplug, change its default value to 11 for C4. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 2753e1b0b058..a7c378d679cc 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -604,7 +604,7 @@ endchoice config FORCE_MAX_ZONEORDER int - default "16" if (HUGETLB_PAGE) + default "16" if (HUGETLB_PAGE && SUBARCH_C3B) default "11" help The kernel memory allocator divides physically contiguous memory -- Gitee From 30576e1b16b6e4045e89842e49a5fda324d0d50d Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Wed, 8 May 2024 15:56:13 +0800 Subject: [PATCH 185/524] sw64: fix usage of __add_memory() in sunway-ged driver This driver has not been tested yet. Add the missing fourth argument mhp_flags which is MHP_NONE. Fixes: 51f31c3271e5 ("sw64: add memhotplug support for guest os") Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/misc/sunway-ged.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/misc/sunway-ged.c b/drivers/misc/sunway-ged.c index b4e4ca315852..5c397cb454c9 100644 --- a/drivers/misc/sunway-ged.c +++ b/drivers/misc/sunway-ged.c @@ -58,7 +58,7 @@ static int sunway_memory_enable_device(struct sunway_memory_device *mem_device) lock_device_hotplug(); /* suppose node = 0, fix me! */ - result = __add_memory(0, mem_device->start_addr, mem_device->length); + result = __add_memory(0, mem_device->start_addr, mem_device->length, MHP_NONE); unlock_device_hotplug(); /* * If the memory block has been used by the kernel, add_memory() -- Gitee From 256669771bcddf31329144c1323b4ea33fb566e2 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Wed, 8 May 2024 19:10:53 +0800 Subject: [PATCH 186/524] sw64: fix compile warning of handle_pci_msi_interrupt Fix the following warnings: drivers/irqchip/irq-sunway-msi-v2.c: In function 'handle_pci_msi_interrupt': drivers/irqchip/irq-sunway-msi-v2.c:461:9: warning: unused variable 'irq' [-Wunused-variable] int i, irq, msi_index = 0; ^~~ Fixes: f9d96be59d6c ("sw64: msi: split the msi file") Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-msi-v2.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c index 6f5c921fc520..2beacb1be39d 100644 --- a/drivers/irqchip/irq-sunway-msi-v2.c +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -489,8 +489,6 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned for (i = 0; i < 4; i++) { vector_index = i * 64; while (vector != 0) { - int irq = 0; - msi_index = find_next_bit(&vector, 64, msi_index); if (msi_index == 64) { msi_index = 0; -- Gitee From f6e2e0c097badf019ea07d926f47d945cc36ca2f Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Wed, 8 May 2024 14:29:07 +0000 Subject: [PATCH 187/524] sw64: fix typo in gpio-sunway Fix a typo in gpio-sunway.c, which is causing failed GPIO compilation. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/gpio/gpio-sunway.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/gpio-sunway.c b/drivers/gpio/gpio-sunway.c index ecb87c41dc65..0fd113ff6b11 100644 --- a/drivers/gpio/gpio-sunway.c +++ b/drivers/gpio/gpio-sunway.c @@ -730,7 +730,7 @@ static int sunway_gpio_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, gpio); - def_info(&pdev->dev, "GPIO probe succeed\n"); + dev_info(&pdev->dev, "GPIO probe succeed\n"); return 0; -- Gitee From a5e6c1f46dfeec18a8808427094df33a4853bcc6 Mon Sep 17 00:00:00 2001 From: Xu Chenjiao Date: Thu, 9 May 2024 09:43:19 +0000 Subject: [PATCH 188/524] sw64: lpc: fix compile error with CONFIG_SUNWAY_SUPERIO_AST2400=y This patch fixes undefined reference to DRIVER_NAME. Signed-off-by: Xu Chenjiao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/mfd/sunway_ast2400.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/mfd/sunway_ast2400.c b/drivers/mfd/sunway_ast2400.c index ea981d9d0215..4e1e3e3d21e2 100644 --- a/drivers/mfd/sunway_ast2400.c +++ b/drivers/mfd/sunway_ast2400.c @@ -22,6 +22,7 @@ #include #include +#define DRIVER_NAME "sunway_superio_ast2400" static int superio_uart0_irq; static int superio_uart1_irq; @@ -199,7 +200,7 @@ static struct platform_driver superio_nuvoton_ast2400_driver = { .probe = superio_ast2400_probe, .remove = superio_ast2400_remove, .driver = { - .name = "sunway_superio_ast2400" + .name = DRIVER_NAME }, }; -- Gitee From ebd84f5b9fabb630996c73fdc6d2953ef6c5021e Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 16 Apr 2024 10:30:15 +0800 Subject: [PATCH 189/524] sw64: mm: take memory information from firmware Currently, we directly detect the memory via I/O operations in kernel. Considering that it is the responsibility of the firmware to mask hardware differences, it is best for the firmware to detect memory and pass the memory information to kernel via a standard interface. The standard interface stated above mainly contains two aspects: 1. UEFI memory map If UEFI BIOS is used to boot kernel, the UEFI BIOS is responsible for detecting usable physical memory and passing it to kernel via UEFI memory map. 2. Memory node in device tree If UEFI BIOS is not used, the firmware in use is responsible for detecting usable physical memory and passing it to kernel via memory node in device tree. An example of a memory node is as follows: memory@0 { device_type = "memory"; reg = ; /* base address and size */ numa-node-id = <0>; /* memory of NUMA node 0 */ } This commit brings two changes: 1. Take memory information from firmware when receive the boot magic "0xDEED2024". 2. Some refactoring, moving memblock related initialization from setup.c to other files. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kexec.h | 10 ++ arch/sw_64/include/asm/memory.h | 1 - arch/sw_64/kernel/machine_kexec.c | 69 ++++++++- arch/sw_64/kernel/setup.c | 233 ++---------------------------- arch/sw_64/mm/init.c | 219 +++++++++++++++++++--------- 5 files changed, 241 insertions(+), 291 deletions(-) diff --git a/arch/sw_64/include/asm/kexec.h b/arch/sw_64/include/asm/kexec.h index 25e0d8da84f8..1a63899f9c11 100644 --- a/arch/sw_64/include/asm/kexec.h +++ b/arch/sw_64/include/asm/kexec.h @@ -72,11 +72,21 @@ static inline void crash_setup_regs(struct pt_regs *newregs, /* Function pointer to optional machine-specific reinitialization */ extern void (*kexec_reinit)(void); +extern void __init reserve_crashkernel(void); +extern void __init kexec_control_page_init(void); + #endif /* __ASSEMBLY__ */ struct kimage; extern unsigned long kexec_args[4]; +#else /* CONFIG_KEXEC */ + +#ifndef __ASSEMBLY__ +static inline void __init reserve_crashkernel(void) {} +static inline void __init kexec_control_page_init(void) {} +#endif + #endif /* CONFIG_KEXEC */ #endif /* _ASM_SW64_KEXEC_H */ diff --git a/arch/sw_64/include/asm/memory.h b/arch/sw_64/include/asm/memory.h index b2b7492ae477..65e5f8f79727 100644 --- a/arch/sw_64/include/asm/memory.h +++ b/arch/sw_64/include/asm/memory.h @@ -26,7 +26,6 @@ struct numa_node_desc_t { extern struct numa_node_desc_t numa_nodes_desc[]; void __init callback_init(void); -void __init mem_detect(void); void __init sw64_memblock_init(void); void __init zone_sizes_init(void); void __init sw64_numa_init(void); diff --git a/arch/sw_64/kernel/machine_kexec.c b/arch/sw_64/kernel/machine_kexec.c index dc1b2b4f5949..a85cca14444b 100644 --- a/arch/sw_64/kernel/machine_kexec.c +++ b/arch/sw_64/kernel/machine_kexec.c @@ -14,11 +14,11 @@ #include #include #include +#include #include #include -extern void *kexec_control_page; extern const unsigned char relocate_new_kernel[]; extern const size_t relocate_new_kernel_size; @@ -26,6 +26,7 @@ extern unsigned long kexec_start_address; extern unsigned long kexec_indirection_page; static atomic_t waiting_for_crash_ipi; +static void *kexec_control_page; #ifdef CONFIG_SMP extern struct smp_rcb_struct *smp_rcb; @@ -46,6 +47,72 @@ static void kexec_smp_down(void *ignored) } #endif +#define KTEXT_MAX KERNEL_IMAGE_SIZE + +void __init kexec_control_page_init(void) +{ + phys_addr_t addr; + + addr = memblock_phys_alloc_range(KEXEC_CONTROL_PAGE_SIZE, PAGE_SIZE, + 0, KTEXT_MAX); + kexec_control_page = (void *)(__START_KERNEL_map + addr); +} + +/* + * reserve_crashkernel() - reserves memory are for crash kernel + * + * This function reserves memory area given in "crashkernel=" kernel command + * line parameter. The memory reserved is used by a dump capture kernel when + * primary kernel is crashing. + */ +void __init reserve_crashkernel(void) +{ + unsigned long long crash_size, crash_base; + unsigned long long mem_size = memblock_phys_mem_size(); + int ret; + + ret = parse_crashkernel(boot_command_line, mem_size, + &crash_size, &crash_base); + if (ret || !crash_size) + return; + + if (!crash_size) { + pr_warn("size of crash kernel memory unspecified, no memory reserved for crash kernel\n"); + return; + } + if (!crash_base) { + pr_warn("base of crash kernel memory unspecified, no memory reserved for crash kernel\n"); + return; + } + + if (!memblock_is_region_memory(crash_base, crash_size)) + memblock_add(crash_base, crash_size); + + ret = memblock_reserve(crash_base, crash_size); + if (ret < 0) { + pr_warn("crashkernel reservation failed - memory is in use [mem %#018llx-%#018llx]\n", + crash_base, crash_base + crash_size - 1); + return; + } + + pr_info("Reserving %ldMB of memory at %ldMB for crashkernel (System RAM: %ldMB)\n", + (unsigned long)(crash_size >> 20), + (unsigned long)(crash_base >> 20), + (unsigned long)(mem_size >> 20)); + + ret = add_memmap_region(crash_base, crash_size, memmap_crashkernel); + if (ret) + pr_warn("Add crash kernel area [mem %#018llx-%#018llx] to memmap region failed.\n", + crash_base, crash_base + crash_size - 1); + + if (crash_base >= KERNEL_IMAGE_SIZE) + pr_warn("Crash base should be less than %#x\n", KERNEL_IMAGE_SIZE); + + crashk_res.start = crash_base; + crashk_res.end = crash_base + crash_size - 1; + insert_resource(&iomem_resource, &crashk_res); +} + int machine_kexec_prepare(struct kimage *kimage) { return 0; diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 6505fe0486aa..95162a3d073f 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -20,16 +20,15 @@ #include #include #include -#include #include #include #include -#include #include #include #include #include +#include #include "proto.h" @@ -46,19 +45,6 @@ EXPORT_SYMBOL(__cpu_to_rcid); DEFINE_PER_CPU(unsigned long, hard_node_id) = { 0 }; static DEFINE_PER_CPU(struct cpu, cpu_devices); -#ifdef CONFIG_SUBARCH_C3B -#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) -struct cma *sw64_kvm_cma; -EXPORT_SYMBOL(sw64_kvm_cma); - -static phys_addr_t kvm_mem_size; -static phys_addr_t kvm_mem_base; - -struct gen_pool *sw64_kvm_pool; -EXPORT_SYMBOL(sw64_kvm_pool); -#endif -#endif - static inline int phys_addr_valid(unsigned long addr) { /* @@ -159,79 +145,6 @@ void store_cpu_data(int cpu) cpu_data[cpu].last_asid = ASID_FIRST_VERSION; } -#ifdef CONFIG_KEXEC - -void *kexec_control_page; - -#define KTEXT_MAX KERNEL_IMAGE_SIZE - -static void __init kexec_control_page_init(void) -{ - phys_addr_t addr; - - addr = memblock_phys_alloc_range(KEXEC_CONTROL_PAGE_SIZE, PAGE_SIZE, - 0, KTEXT_MAX); - kexec_control_page = (void *)(__START_KERNEL_map + addr); -} - -/* - * reserve_crashkernel() - reserves memory are for crash kernel - * - * This function reserves memory area given in "crashkernel=" kernel command - * line parameter. The memory reserved is used by a dump capture kernel when - * primary kernel is crashing. - */ -static void __init reserve_crashkernel(void) -{ - unsigned long long crash_size, crash_base; - int ret; - - ret = parse_crashkernel(boot_command_line, mem_desc.size, - &crash_size, &crash_base); - if (ret || !crash_size) - return; - - if (!crash_size) { - pr_warn("size of crash kernel memory unspecified, no memory reserved for crash kernel\n"); - return; - } - if (!crash_base) { - pr_warn("base of crash kernel memory unspecified, no memory reserved for crash kernel\n"); - return; - } - - if (!memblock_is_region_memory(crash_base, crash_size)) - memblock_add(crash_base, crash_size); - - ret = memblock_reserve(crash_base, crash_size); - if (ret < 0) { - pr_warn("crashkernel reservation failed - memory is in use [mem %#018llx-%#018llx]\n", - crash_base, crash_base + crash_size - 1); - return; - } - - pr_info("Reserving %ldMB of memory at %ldMB for crashkernel (System RAM: %ldMB)\n", - (unsigned long)(crash_size >> 20), - (unsigned long)(crash_base >> 20), - (unsigned long)(mem_desc.size >> 20)); - - ret = add_memmap_region(crash_base, crash_size, memmap_crashkernel); - if (ret) - pr_warn("Add crash kernel area [mem %#018llx-%#018llx] to memmap region failed.\n", - crash_base, crash_base + crash_size - 1); - - if (crash_base >= KERNEL_IMAGE_SIZE) - pr_warn("Crash base should be less than %#x\n", KERNEL_IMAGE_SIZE); - - crashk_res.start = crash_base; - crashk_res.end = crash_base + crash_size - 1; - insert_resource(&iomem_resource, &crashk_res); -} -#else /* !defined(CONFIG_KEXEC) */ -static void __init reserve_crashkernel(void) {} -static void __init kexec_control_page_init(void) {} -#endif /* !defined(CONFIG_KEXEC) */ - /* * I/O resources inherited from PeeCees. Except for perhaps the * turbochannel SWs, everyone has these on some sort of SuperIO chip. @@ -488,26 +401,6 @@ subsys_initcall(request_standard_resources); extern void cpu_set_node(void); #endif -static void __init show_socket_mem_layout(void) -{ - int i; - phys_addr_t base, size, end; - - base = 0; - - pr_info("Socket memory layout:\n"); - for (i = 0; i < MAX_NUMSOCKETS; i++) { - if (socket_desc[i].is_online) { - size = socket_desc[i].socket_mem; - end = base + size - 1; - pr_info("Socket %d: [mem %#018llx-%#018llx], size %llu\n", - i, base, end, size); - base = end + 1; - } - } - pr_info("Reserved memory size for Socket 0: %#lx\n", NODE0_START); -} - int page_is_ram(unsigned long pfn) { pfn <<= PAGE_SHIFT; @@ -592,16 +485,14 @@ static void __init setup_firmware_fdt(void) goto cmd_handle; dt_virt = (void *)sunway_boot_params->dtb_start; } else { - /** - * Use DTB provided by firmware for early initialization, - * regardless of whether a Built-in DTB configured or not. - * Since we need the boot params from firmware when using - * new method to pass boot params. - */ + /* Use DTB provided by firmware for early initialization */ pr_info("Parse boot params in DTB chosen node\n"); dt_virt = (void *)sunway_dtb_address; } + /* reserve the DTB from firmware in case it is used later */ + memblock_reserve(__boot_pa(dt_virt), fdt_totalsize(dt_virt)); + if (!arch_dtb_verify(dt_virt, true) || !early_init_dt_scan(dt_virt)) { pr_crit("Invalid DTB(from firmware) at virtual address 0x%lx\n", @@ -791,45 +682,6 @@ static void __init setup_run_mode(void) } #endif -static void __init setup_socket_info(void) -{ - int i; - int numsockets = sw64_chip->get_cpu_num(); - - memset(socket_desc, 0, MAX_NUMSOCKETS * sizeof(struct socket_desc_t)); - - for (i = 0; i < numsockets; i++) { - socket_desc[i].is_online = 1; - if (sw64_chip_init->early_init.get_node_mem) - socket_desc[i].socket_mem = sw64_chip_init->early_init.get_node_mem(i); - } -} - -#ifdef CONFIG_SUBARCH_C3B -#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) -static int __init early_kvm_reserved_mem(char *p) -{ - if (!p) { - pr_err("Config string not provided\n"); - return -EINVAL; - } - - kvm_mem_size = memparse(p, &p); - if (*p != '@') - return -EINVAL; - kvm_mem_base = memparse(p + 1, &p); - return 0; -} -early_param("kvm_mem", early_kvm_reserved_mem); - -void __init sw64_kvm_reserve(void) -{ - kvm_cma_declare_contiguous(kvm_mem_base, kvm_mem_size, 0, - PAGE_SIZE, 0, "sw64_kvm_cma", &sw64_kvm_cma); -} -#endif -#endif - void __init setup_arch(char **cmdline_p) { @@ -843,8 +695,6 @@ setup_arch(char **cmdline_p) setup_cpu_info(); setup_run_mode(); setup_chip_ops(); - setup_socket_info(); - show_socket_mem_layout(); sw64_chip_init->early_init.setup_core_map(&core_start); if (is_guest_or_emul()) get_vt_smp_info(); @@ -868,30 +718,19 @@ setup_arch(char **cmdline_p) */ parse_early_param(); - /* Find our memory. */ - mem_detect(); - -#ifdef CONFIG_PCI - reserve_mem_for_pci(); -#endif - - sw64_memblock_init(); - - reserve_crashkernel(); - - /* Reserve large chunks of memory for use by CMA for KVM. */ -#ifdef CONFIG_SUBARCH_C3B -#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) - sw64_kvm_reserve(); -#endif -#endif - efi_init(); - /* After efi initialization, switch to Builtin-in DTB if configured */ + /** + * Switch to builtin-in DTB if configured. + * Must be placed after efi_init(), Since + * efi_init() may parse boot params from DTB + * provided by firmware. + */ if (IS_ENABLED(CONFIG_BUILTIN_DTB)) setup_builtin_fdt(); + sw64_memblock_init(); + /* Try to upgrade ACPI tables via initrd */ acpi_table_upgrade(); @@ -1096,49 +935,3 @@ static int __init sw64_of_init(void) core_initcall(sw64_of_init); #endif -#ifdef CONFIG_SUBARCH_C3B -#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) -static int __init sw64_kvm_pool_init(void) -{ - int status = 0; - unsigned long kvm_pool_virt; - struct page *base_page, *end_page, *p; - - if (!sw64_kvm_cma) - goto out; - - kvm_pool_virt = (unsigned long)kvm_mem_base; - - sw64_kvm_pool = gen_pool_create(PAGE_SHIFT, -1); - if (!sw64_kvm_pool) - goto out; - - status = gen_pool_add_virt(sw64_kvm_pool, kvm_pool_virt, kvm_mem_base, - kvm_mem_size, -1); - if (status < 0) { - pr_err("failed to add memory chunks to sw64 kvm pool\n"); - gen_pool_destroy(sw64_kvm_pool); - sw64_kvm_pool = NULL; - goto out; - } - gen_pool_set_algo(sw64_kvm_pool, gen_pool_best_fit, NULL); - - base_page = pfn_to_page(kvm_mem_base >> PAGE_SHIFT); - end_page = pfn_to_page((kvm_mem_base + kvm_mem_size - 1) >> PAGE_SHIFT); - - p = base_page; - while (p <= end_page && page_ref_count(p) == 0) { - set_page_count(p, 1); - page_mapcount_reset(p); - SetPageReserved(p); - p++; - } - - return status; - -out: - return -ENOMEM; -} -core_initcall_sync(sw64_kvm_pool_init); -#endif -#endif diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index d33d5a83e5b1..e35f90e239e2 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -14,10 +14,14 @@ #include #include #include +#include #include #include #include +#include +#include +#include struct mem_desc_t mem_desc; #ifndef CONFIG_NUMA @@ -138,7 +142,41 @@ void __init paging_init(void) { } -void __init mem_detect(void) +static void __init setup_socket_info(void) +{ + int i; + int numsockets = sw64_chip->get_cpu_num(); + + memset(socket_desc, 0, MAX_NUMSOCKETS * sizeof(struct socket_desc_t)); + + for (i = 0; i < numsockets; i++) { + socket_desc[i].is_online = 1; + if (sw64_chip_init->early_init.get_node_mem) + socket_desc[i].socket_mem = sw64_chip_init->early_init.get_node_mem(i); + } +} + +static void __init show_socket_mem_layout(void) +{ + int i; + phys_addr_t base, size, end; + + base = 0; + + pr_info("Socket memory layout:\n"); + for (i = 0; i < MAX_NUMSOCKETS; i++) { + if (socket_desc[i].is_online) { + size = socket_desc[i].socket_mem; + end = base + size - 1; + pr_info("Socket %d: [mem %#018llx-%#018llx], size %llu\n", + i, base, end, size); + base = end + 1; + } + } + pr_info("Reserved memory size for Socket 0: %#lx\n", NODE0_START); +} + +static void __init mem_detect(void) { int i; @@ -216,14 +254,110 @@ static void __init reserve_mem_for_initrd(void) } #endif /* CONFIG_BLK_DEV_INITRD */ +#ifdef CONFIG_SUBARCH_C3B +#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) +struct cma *sw64_kvm_cma; +EXPORT_SYMBOL(sw64_kvm_cma); + +static phys_addr_t kvm_mem_size; +static phys_addr_t kvm_mem_base; + +struct gen_pool *sw64_kvm_pool; +EXPORT_SYMBOL(sw64_kvm_pool); + +static int __init early_kvm_reserved_mem(char *p) +{ + if (!p) { + pr_err("Config string not provided\n"); + return -EINVAL; + } + + kvm_mem_size = memparse(p, &p); + if (*p != '@') + return -EINVAL; + kvm_mem_base = memparse(p + 1, &p); + return 0; +} +early_param("kvm_mem", early_kvm_reserved_mem); + +void __init sw64_kvm_reserve(void) +{ + kvm_cma_declare_contiguous(kvm_mem_base, kvm_mem_size, 0, + PAGE_SIZE, 0, "sw64_kvm_cma", &sw64_kvm_cma); +} + +static int __init sw64_kvm_pool_init(void) +{ + int status = 0; + unsigned long kvm_pool_virt; + struct page *base_page, *end_page, *p; + + if (!sw64_kvm_cma) + goto out; + + kvm_pool_virt = (unsigned long)kvm_mem_base; + + sw64_kvm_pool = gen_pool_create(PAGE_SHIFT, -1); + if (!sw64_kvm_pool) + goto out; + + status = gen_pool_add_virt(sw64_kvm_pool, kvm_pool_virt, kvm_mem_base, + kvm_mem_size, -1); + if (status < 0) { + pr_err("failed to add memory chunks to sw64 kvm pool\n"); + gen_pool_destroy(sw64_kvm_pool); + sw64_kvm_pool = NULL; + goto out; + } + gen_pool_set_algo(sw64_kvm_pool, gen_pool_best_fit, NULL); + + base_page = pfn_to_page(kvm_mem_base >> PAGE_SHIFT); + end_page = pfn_to_page((kvm_mem_base + kvm_mem_size - 1) >> PAGE_SHIFT); + + p = base_page; + while (p <= end_page && page_ref_count(p) == 0) { + set_page_count(p, 1); + page_mapcount_reset(p); + SetPageReserved(p); + p++; + } + + return status; + +out: + return -ENOMEM; +} +core_initcall_sync(sw64_kvm_pool_init); +#endif +#endif + void __init sw64_memblock_init(void) { - memblock_add(mem_desc.base, mem_desc.size); + /** + * Detect all memory on all nodes, used in the following + * cases: + * 1. Legacy memory detect + * 2. Legacy NUMA initialization + */ + setup_socket_info(); + show_socket_mem_layout(); + + if (sunway_boot_magic != 0xDEED2024UL) { + /* Find our usable memory */ + mem_detect(); + + /* Add usable memory */ + memblock_add(mem_desc.base, mem_desc.size); + } memblock_remove(1ULL << MAX_PHYSMEM_BITS, PHYS_ADDR_MAX); max_pfn = max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); +#ifdef CONFIG_PCI + reserve_mem_for_pci(); +#endif + memblock_allow_resize(); memblock_initialized = true; process_memmap(); @@ -236,17 +370,15 @@ void __init sw64_memblock_init(void) /* Make sure initrd is in memory range. */ reserve_mem_for_initrd(); #endif - /** - * When using non built-in DTB, we need to reserve - * it but avoid using early_init_fdt_reserve_self() - * since __pa() does not work for some old firmware. - * - * We can use early_init_fdt_reserve_self() instead - * when kernel no longer support C3B. - */ - if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) - memblock_reserve(__boot_pa(initial_boot_params), - fdt_totalsize(initial_boot_params)); + +#ifdef CONFIG_SUBARCH_C3B +#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) + /* Reserve large chunks of memory for use by CMA for KVM. */ + sw64_kvm_reserve(); +#endif +#endif + + reserve_crashkernel(); /* end of DRAM range may have been changed */ max_pfn = max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); @@ -255,12 +387,14 @@ void __init sw64_memblock_init(void) #ifndef CONFIG_NUMA void __init sw64_numa_init(void) { + phys_addr_t mem_base = memblock_start_of_DRAM(); + phys_addr_t mem_size = memblock_phys_mem_size(); const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES); u64 nd_pa; void *nd; int tnid; - memblock_set_node(mem_desc.base, mem_desc.size, &memblock.memory, 0); + memblock_set_node(mem_base, mem_size, &memblock.memory, 0); nd_pa = memblock_phys_alloc(nd_size, SMP_CACHE_BYTES); nd = __va(nd_pa); @@ -274,8 +408,8 @@ void __init sw64_numa_init(void) node_data[0] = nd; memset(NODE_DATA(0), 0, sizeof(pg_data_t)); NODE_DATA(0)->node_id = 0; - NODE_DATA(0)->node_start_pfn = mem_desc.base >> PAGE_SHIFT; - NODE_DATA(0)->node_spanned_pages = mem_desc.size >> PAGE_SHIFT; + NODE_DATA(0)->node_start_pfn = mem_base >> PAGE_SHIFT; + NODE_DATA(0)->node_spanned_pages = mem_size >> PAGE_SHIFT; node_set_online(0); } #endif /* CONFIG_NUMA */ @@ -304,59 +438,6 @@ void vmemmap_free(unsigned long start, unsigned long end, } #endif -#ifdef CONFIG_HAVE_MEMBLOCK -#ifndef MIN_MEMBLOCK_ADDR -#define MIN_MEMBLOCK_ADDR __pa(PAGE_OFFSET) -#endif -#ifndef MAX_MEMBLOCK_ADDR -#define MAX_MEMBLOCK_ADDR ((phys_addr_t)~0) -#endif -void __init early_init_dt_add_memory_arch(u64 base, u64 size) -{ - const u64 phys_offset = MIN_MEMBLOCK_ADDR; - - if (acpi_disabled) { - if (!PAGE_ALIGNED(base)) { - if (size < PAGE_SIZE - (base & ~PAGE_MASK)) { - pr_warn("Ignoring memory block 0x%llx - 0x%llx\n", - base, base + size); - return; - } - size -= PAGE_SIZE - (base & ~PAGE_MASK); - base = PAGE_ALIGN(base); - } - size &= PAGE_MASK; - - if (base > MAX_MEMBLOCK_ADDR) { - pr_warn("Ignoring memory block 0x%llx - 0x%llx\n", - base, base + size); - return; - } - - if (base + size - 1 > MAX_MEMBLOCK_ADDR) { - pr_warn("Ignoring memory range 0x%llx - 0x%llx\n", - ((u64)MAX_MEMBLOCK_ADDR) + 1, base + size); - size = MAX_MEMBLOCK_ADDR - base + 1; - } - - if (base + size < phys_offset) { - pr_warn("Ignoring memory block 0x%llx - 0x%llx\n", - base, base + size); - return; - } - - if (base < phys_offset) { - pr_warn("Ignoring memory range 0x%llx - 0x%llx\n", - base, phys_offset); - size -= phys_offset - base; - base = phys_offset; - } - memblock_add(base, size); - } else - return; -} -#endif - #ifdef CONFIG_MEMORY_HOTPLUG int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params) { -- Gitee From f89198417a3c9d9e9839f5ea919fb05004556b8a Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 26 Apr 2024 10:23:49 +0800 Subject: [PATCH 190/524] sw64: use generic page_is_ram() The generic page_is_ram() provides more rigorous check. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 95162a3d073f..aceaecb5bf8e 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -401,13 +401,6 @@ subsys_initcall(request_standard_resources); extern void cpu_set_node(void); #endif -int page_is_ram(unsigned long pfn) -{ - pfn <<= PAGE_SHIFT; - - return pfn >= mem_desc.base && pfn < (mem_desc.base + mem_desc.size); -} - static int __init topology_init(void) { int i, ret; -- Gitee From 7ff017038200f823b078bd9445d26caa8a656607 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 26 Apr 2024 14:48:25 +0800 Subject: [PATCH 191/524] sw64: mm: fix mem=xxx command line invalid in some cases The strategy of adding the intersection of [start, start + size) and usable physical memory is flawed. Since efi_init() adds memory without considering [start, start + size). We need to explicitly remove memory. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/init.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index e35f90e239e2..05c0b8350480 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -69,6 +69,13 @@ static int __init setup_mem_size(char *p) mem_start = start; mem_size_limit = size; + + if (mem_start < NODE0_START) { + mem_start = NODE0_START; + mem_size_limit -= min(mem_size_limit, + NODE0_START - mem_start); + } + return 0; } early_param("mem", setup_mem_size); @@ -186,17 +193,8 @@ static void __init mem_detect(void) mem_desc.phys_size += socket_desc[i].socket_mem; } - if (mem_start >= NODE0_START) { - mem_desc.base = mem_start; - } else { - mem_desc.base = NODE0_START; - mem_size_limit -= NODE0_START - mem_start; - } - - if (mem_size_limit && mem_size_limit < mem_desc.phys_size - NODE0_START) - mem_desc.size = mem_size_limit; - else - mem_desc.size = mem_desc.phys_size - NODE0_START; + mem_desc.base = NODE0_START; + mem_desc.size = mem_desc.phys_size - NODE0_START; } #ifdef CONFIG_BLK_DEV_INITRD @@ -380,6 +378,16 @@ void __init sw64_memblock_init(void) reserve_crashkernel(); + /* All memory has been added, it's time to handle memory limitation */ + if (mem_size_limit) { + memblock_remove(0, mem_start); + memblock_remove(mem_start + mem_size_limit, PHYS_ADDR_MAX); + if (sunway_boot_magic != 0xDEED2024UL) { + mem_desc.base = mem_start; + mem_desc.size = memblock_phys_mem_size(); + } + } + /* end of DRAM range may have been changed */ max_pfn = max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); } -- Gitee From e1a61201646c8e439c323ca0fd3265aa9fe3fe4b Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 30 Apr 2024 09:45:42 +0800 Subject: [PATCH 192/524] sw64: acpi: enable ACPI by default ACPI is enabled by default and can be disabled via the command line parameter "acpi=off". Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/acpi.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index d0e853659211..5be5d93018cc 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -12,11 +12,11 @@ #include #endif -int acpi_disabled = 1; +int acpi_disabled; EXPORT_SYMBOL(acpi_disabled); -int acpi_noirq = 1; /* skip ACPI IRQ initialization */ -int acpi_pci_disabled = 1; /* skip ACPI PCI scan and IRQ initialization */ +int acpi_noirq; /* skip ACPI IRQ initialization */ +int acpi_pci_disabled; /* skip ACPI PCI scan and IRQ initialization */ EXPORT_SYMBOL(acpi_pci_disabled); static bool param_acpi_on __initdata; @@ -357,12 +357,18 @@ void __init acpi_boot_table_init(void) } /** - * ACPI is disabled by default. - * ACPI is only enabled when firmware passes ACPI table - * and sets boot parameter "acpi=on". + * ACPI is enabled by default. + * + * ACPI is disabled only when firmware explicitly passes + * the boot cmdline "acpi=off". + * + * Note: If no valid ACPI table is found, it will eventually + * be disabled. */ if (param_acpi_on) enable_acpi(); + else if (param_acpi_off) + disable_acpi(); /* * If acpi_disabled, bail out -- Gitee From 9344e943cc53f05b1cdb8ccd0f9b8eb0317df8c2 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 10 May 2024 15:21:13 +0800 Subject: [PATCH 193/524] sw64: irq: support interrupt for virtual GPIO As a hardware-reduced ACPI platform, sw64 uses GPIO to signal ACPI events. This commit is used to support interrupt for virtual GPIO, specifically pin 0 of GPIO port A. We use the virtual pin 0 to signal power button, so that we can do soft power off based on ACPI in virtual environment. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/irq_impl.h | 1 + drivers/irqchip/irq-sunway-cpu.c | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/include/asm/irq_impl.h b/arch/sw_64/include/asm/irq_impl.h index 797af433a126..8040d22c53a5 100644 --- a/arch/sw_64/include/asm/irq_impl.h +++ b/arch/sw_64/include/asm/irq_impl.h @@ -33,6 +33,7 @@ enum sw64_irq_type { INT_FAULT = 10, INT_VT_SERIAL = 12, INT_VT_HOTPLUG = 13, + INT_VT_GPIOA_PIN0 = 15, INT_DEV = 17, INT_NMI = 18, INT_LEGACY = 31, diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index dca5020a789d..b8204fbdd6e7 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -223,11 +223,8 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, set_irq_regs(old_regs); return; case INT_VT_SERIAL: - old_regs = set_irq_regs(regs); - handle_irq(type); - set_irq_regs(old_regs); - return; case INT_VT_HOTPLUG: + case INT_VT_GPIOA_PIN0: old_regs = set_irq_regs(regs); handle_irq(type); set_irq_regs(old_regs); -- Gitee From 7c68002c0b66c045864202467d51328b8fec397d Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Sat, 11 May 2024 10:33:14 +0000 Subject: [PATCH 194/524] sw64: remove redundant intx code Remove redundant pci set_intx codes in pci_sunway.c. They have already been included as one of the irqchip behaviours. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/controller/pci-sunway.c | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 420572592779..36e87a639bed 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -242,25 +242,6 @@ static void set_rc_piu(unsigned long node, unsigned long index) } } -static void set_intx(unsigned long node, unsigned long index, - unsigned long int_conf) -{ - if (is_guest_or_emul()) - return; - -#if defined(CONFIG_UNCORE_XUELANG) - write_piu_ior0(node, index, INTACONFIG, int_conf | (0x8UL << 10)); - write_piu_ior0(node, index, INTBCONFIG, int_conf | (0x4UL << 10)); - write_piu_ior0(node, index, INTCCONFIG, int_conf | (0x2UL << 10)); - write_piu_ior0(node, index, INTDCONFIG, int_conf | (0x1UL << 10)); -#elif defined(CONFIG_UNCORE_JUNZHANG) - write_piu_ior0(node, index, INTACONFIG, int_conf | (0x1UL << 10)); - write_piu_ior0(node, index, INTBCONFIG, int_conf | (0x2UL << 10)); - write_piu_ior0(node, index, INTCCONFIG, int_conf | (0x4UL << 10)); - write_piu_ior0(node, index, INTDCONFIG, int_conf | (0x8UL << 10)); -#endif -} - static unsigned long get_rc_enable(unsigned long node) { unsigned long rc_enable; @@ -338,7 +319,6 @@ static struct sw64_pci_init_ops chip_pci_init_ops = { .hose_init = hose_init, .set_rc_piu = set_rc_piu, .check_pci_linkup = check_pci_linkup, - .set_intx = set_intx, }; void __init setup_chip_pci_ops(void) -- Gitee From 05ef74032c0db7a1cbce66465c5df117f0513f91 Mon Sep 17 00:00:00 2001 From: Xu Chenjiao Date: Mon, 13 May 2024 10:27:39 +0000 Subject: [PATCH 195/524] sw64: pciehp: add pcie hotplug driver support for C4 This driver is added to make hotplug work on C4, since PCIe hotplug is implemented on hardware. Signed-off-by: Xu Chenjiao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uncore_io_junzhang.h | 7 + arch/sw_64/pci/pci.c | 4 +- drivers/irqchip/irq-sunway-cpu.c | 9 + drivers/pci/controller/pci-sunway.c | 17 + drivers/pci/hotplug/Kconfig | 9 + drivers/pci/hotplug/Makefile | 6 + drivers/pci/hotplug/sunway_pciehp.h | 215 ++++ drivers/pci/hotplug/sunway_pciehp_core.c | 278 +++++ drivers/pci/hotplug/sunway_pciehp_ctrl.c | 671 +++++++++++ drivers/pci/hotplug/sunway_pciehp_hpc.c | 1152 +++++++++++++++++++ drivers/pci/hotplug/sunway_pciehp_pci.c | 113 ++ 11 files changed, 2479 insertions(+), 2 deletions(-) create mode 100644 drivers/pci/hotplug/sunway_pciehp.h create mode 100644 drivers/pci/hotplug/sunway_pciehp_core.c create mode 100644 drivers/pci/hotplug/sunway_pciehp_ctrl.c create mode 100644 drivers/pci/hotplug/sunway_pciehp_hpc.c create mode 100644 drivers/pci/hotplug/sunway_pciehp_pci.c diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index 2f745fe35365..8c14890e5c1d 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -64,6 +64,7 @@ #define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) #define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) +#define HP_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) #define PIUCONFIG0_INIT_VAL 0x38016 @@ -112,12 +113,15 @@ enum { SI_FAULT_STAT_EN = SPBU_BASE | 0x3180UL, SI_FAULT_INT_EN = SPBU_BASE | 0x3200UL, ADR_CTL = SPBU_BASE | 0x3600UL, + PIUH_CTRL = SPBU_BASE | 0x3680UL, MC_ONLINE = SPBU_BASE | 0x3780UL, CLK_CTL = SPBU_BASE | 0x3b80UL, CLU_LV2_SELH = SPBU_BASE | 0x3a00UL, CLU_LV2_SELL = SPBU_BASE | 0x3b00UL, PIU_TOP0_CONFIG = SPBU_BASE | 0x4c80UL, PIU_TOP1_CONFIG = SPBU_BASE | 0x4d00UL, + PIU_PHY_SRST_H = SPBU_BASE | 0x6280UL, + BUTTON_RST_N_PCIE0 = SPBU_BASE | 0x6a80UL, SOFT_INFO0 = SPBU_BASE | 0xa000UL, }; @@ -140,6 +144,8 @@ enum { PMEMSICONFIG = 0xa580UL, HPINTCONFIG = 0xa600UL, HPMSICONFIG = 0xa680UL, + HP_CTRL = 0xac80UL, + HP_WATCHOUT = 0xae00UL, DTBASEADDR = 0xb000UL, DTLB_FLUSHALL = 0xb080UL, DTLB_FLUSHDEV = 0xb100UL, @@ -165,6 +171,7 @@ enum { /* PIU IOR1 */ enum { PIUCONFIG1 = 0x0UL, + NEWLTSSMSTATE0 = 0x300UL, ERRENABLE = 0x880UL, RCDEBUGINF1 = 0xc80UL, DCACONTROL = 0x1a00UL, diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index 87930a62595a..77dbd2f4bf37 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -201,9 +201,9 @@ static void fixup_root_complex(struct pci_dev *dev) hose->self_busno = hose->busn_space->start; if (likely(bus->number == hose->self_busno)) { - if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) { + if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE_SUNWAY)) { /* Check Root Complex port again */ - dev->is_hotplug_bridge = 0; + dev->is_hotplug_bridge = 1; dev->current_state = PCI_D0; } diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index b8204fbdd6e7..250ddbd61cf4 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -71,6 +71,15 @@ static void handle_intx(unsigned int offset) } } + if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE_SUNWAY)) { + value = read_piu_ior0(hose->node, hose->index, HPINTCONFIG); + if (value >> 63) { + handle_irq(hose->service_irq); + write_piu_ior0(hose->node, hose->index, HPINTCONFIG, value); + } + + } + if (hose->iommu_enable) { value = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); if (value >> 63) diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 36e87a639bed..688793d1b136 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -33,6 +33,23 @@ void set_pcieport_service_irq(int node, int index) if (IS_ENABLED(CONFIG_PCIEAER)) write_piu_ior0(node, index, AERERRINTCONFIG, AER_ENABLE_INTD_CORE0); + +#ifdef CONFIG_UNCORE_JUNZHANG + if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE_SUNWAY)) + write_piu_ior0(node, index, + HPINTCONFIG, HP_ENABLE_INTD_CORE0); +#endif +} + +int pcibios_enable_device(struct pci_dev *dev, int bars) +{ + struct pci_bus *bus = dev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + if (!is_guest_or_emul() && unlikely(bus->number == hose->self_busno)) + return 0; + else + return pci_enable_resources(dev, bars); } int chip_pcie_configure(struct pci_controller *hose) diff --git a/drivers/pci/hotplug/Kconfig b/drivers/pci/hotplug/Kconfig index 48113b210cf9..16e38e1db976 100644 --- a/drivers/pci/hotplug/Kconfig +++ b/drivers/pci/hotplug/Kconfig @@ -114,6 +114,15 @@ config HOTPLUG_PCI_SHPC When in doubt, say N. +config HOTPLUG_PCI_PCIE_SUNWAY + bool "SUNWAY PCI Express Hotplug driver" + depends on SW64 && SUBARCH_C4 && !HOTPLUG_PCI_PCIE + help + Say Y here if you have a motherboard with a SUNWAY PCI Express Hotplug + controller. + + When in doubt, say N. + config HOTPLUG_PCI_POWERNV tristate "PowerPC PowerNV PCI Hotplug driver" depends on PPC_POWERNV && EEH diff --git a/drivers/pci/hotplug/Makefile b/drivers/pci/hotplug/Makefile index 5196983220df..e7193995acde 100644 --- a/drivers/pci/hotplug/Makefile +++ b/drivers/pci/hotplug/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_HOTPLUG_PCI_IBM) += ibmphp.o # generic support. obj-$(CONFIG_HOTPLUG_PCI_PCIE) += pciehp.o +obj-$(CONFIG_HOTPLUG_PCI_PCIE_SUNWAY) += sunway_pciehp.o obj-$(CONFIG_HOTPLUG_PCI_CPCI_ZT5550) += cpcihp_zt5550.o obj-$(CONFIG_HOTPLUG_PCI_CPCI_GENERIC) += cpcihp_generic.o obj-$(CONFIG_HOTPLUG_PCI_SHPC) += shpchp.o @@ -70,3 +71,8 @@ shpchp-objs := shpchp_core.o \ shpchp_pci.o \ shpchp_sysfs.o \ shpchp_hpc.o + +sunway_pciehp-objs := sunway_pciehp_core.o \ + sunway_pciehp_ctrl.o \ + sunway_pciehp_pci.o \ + sunway_pciehp_hpc.o diff --git a/drivers/pci/hotplug/sunway_pciehp.h b/drivers/pci/hotplug/sunway_pciehp.h new file mode 100644 index 000000000000..5ef5e745543b --- /dev/null +++ b/drivers/pci/hotplug/sunway_pciehp.h @@ -0,0 +1,215 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Sunway PCI Express Hot Plug Controller Driver + */ +#ifndef _SUNWAYPCIEHP_H +#define _SUNWAYPCIEHP_H + +#include +#include +#include +#include +#include /* signal_pending() */ +#include +#include +#include + +#include "../pcie/portdrv.h" + +#define MY_NAME "sunway_pciehp" + +extern bool sunway_pciehp_poll_mode; +extern int sunway_pciehp_poll_time; + +/* + * Set CONFIG_DYNAMIC_DEBUG=y and boot with 'dyndbg="file sunway_pciehp* +p"' to + * enable debug messages. + */ +#define ctrl_dbg(ctrl, format, arg...) \ + pci_dbg(ctrl->pci_dev, format, ## arg) +#define ctrl_err(ctrl, format, arg...) \ + pci_err(ctrl->pci_dev, format, ## arg) +#define ctrl_info(ctrl, format, arg...) \ + pci_info(ctrl->pci_dev, format, ## arg) +#define ctrl_warn(ctrl, format, arg...) \ + pci_warn(ctrl->pci_dev, format, ## arg) + +#define SLOT_NAME_SIZE 10 + +struct saved_piu_space { + unsigned long epdmabar; + unsigned long msiaddr; + unsigned long iommuexcpt_ctrl; + unsigned long dtbaseaddr; + unsigned long intaconfig; + unsigned long intbconfig; + unsigned long intcconfig; + unsigned long intdconfig; + unsigned long pmeintconfig; + unsigned long aererrintconfig; + unsigned long hpintconfig; + unsigned int state_saved:1; +}; + +/** + * struct controller - PCIe hotplug controller + * @slot_cap: cached copy of the Slot Capabilities register + * @slot_ctrl: cached copy of the Slot Control register + * @ctrl_lock: serializes writes to the Slot Control register + * @cmd_started: jiffies when the Slot Control register was last written; + * the next write is allowed 1 second later, absent a Command Completed + * interrupt (PCIe r4.0, sec 6.7.3.2) + * @cmd_busy: flag set on Slot Control register write, cleared by IRQ handler + * on reception of a Command Completed event + * @queue: wait queue to wake up on reception of a Command Completed event, + * used for synchronous writes to the Slot Control register + * @pending_events: used by the IRQ handler to save events retrieved from the + * Slot Status register for later consumption by the IRQ thread + * @notification_enabled: whether the IRQ was requested successfully + * @power_fault_detected: whether a power fault was detected by the hardware + * that has not yet been cleared by the user + * @poll_thread: thread to poll for slot events if no IRQ is available, + * enabled with pciehp_poll_mode module parameter + * @state: current state machine position + * @state_lock: protects reads and writes of @state; + * protects scheduling, execution and cancellation of @button_work + * @button_work: work item to turn the slot on or off after 5 seconds + * in response to an Attention Button press + * @hotplug_slot: structure registered with the PCI hotplug core + * @reset_lock: prevents access to the Data Link Layer Link Active bit in the + * Link Status register and to the Presence Detect State bit in the Slot + * Status register during a slot reset which may cause them to flap + * @ist_running: flag to keep user request waiting while IRQ thread is running + * @request_result: result of last user request submitted to the IRQ thread + * @requester: wait queue to wake up on completion of user request, + * used for synchronous slot enable/disable request via sysfs + * + * PCIe hotplug has a 1:1 relationship between controller and slot, hence + * unlike other drivers, the two aren't represented by separate structures. + */ +struct controller { + struct pci_dev *pci_dev; + + u32 slot_cap; /* capabilities and quirks */ + unsigned int inband_presence_disabled:1; + u16 slot_ctrl; /* control register access */ + struct mutex ctrl_lock; + unsigned long cmd_started; + unsigned int cmd_busy:1; + wait_queue_head_t queue; + + atomic_t pending_events; /* event handling */ + unsigned int notification_enabled:1; + unsigned int power_fault_detected; + struct task_struct *poll_thread; + + u8 state; /* state machine */ + struct mutex state_lock; + struct delayed_work button_work; + + struct hotplug_slot hotplug_slot; /* hotplug core interface */ + struct rw_semaphore reset_lock; + unsigned int ist_running; + int request_result; + wait_queue_head_t requester; + + struct saved_piu_space saved_piu; +}; + +/** + * DOC: Slot state + * + * @OFF_STATE: slot is powered off, no subordinate devices are enumerated + * @BLINKINGON_STATE: slot will be powered on after the 5 second delay, + * Power Indicator is blinking + * @BLINKINGOFF_STATE: slot will be powered off after the 5 second delay, + * Power Indicator is blinking + * @POWERON_STATE: slot is currently powering on + * @POWEROFF_STATE: slot is currently powering off + * @ON_STATE: slot is powered on, subordinate devices have been enumerated + */ +#define OFF_STATE 0 +#define BLINKINGON_STATE 1 +#define BLINKINGOFF_STATE 2 +#define POWERON_STATE 3 +#define POWEROFF_STATE 4 +#define ON_STATE 5 + +/** + * DOC: Flags to request an action from the IRQ thread + * + * These are stored together with events read from the Slot Status register, + * hence must be greater than its 16-bit width. + * + * %DISABLE_SLOT: Disable the slot in response to a user request via sysfs or + * an Attention Button press after the 5 second delay + * %RERUN_ISR: Used by the IRQ handler to inform the IRQ thread that the + * hotplug port was inaccessible when the interrupt occurred, requiring + * that the IRQ handler is rerun by the IRQ thread after it has made the + * hotplug port accessible by runtime resuming its parents to D0 + */ +#define DISABLE_SLOT (1 << 16) +#define RERUN_ISR (1 << 17) +#define SW64_POLL_DISABLE_SLOT (1 << 18) +#define SW64_POLL_ENABLE_SLOT (1 << 19) + +#define ATTN_BUTTN(ctrl) ((ctrl)->slot_cap & PCI_EXP_SLTCAP_ABP) +#define POWER_CTRL(ctrl) ((ctrl)->slot_cap & PCI_EXP_SLTCAP_PCP) +#define MRL_SENS(ctrl) ((ctrl)->slot_cap & PCI_EXP_SLTCAP_MRLSP) +#define ATTN_LED(ctrl) ((ctrl)->slot_cap & PCI_EXP_SLTCAP_AIP) +#define PWR_LED(ctrl) ((ctrl)->slot_cap & PCI_EXP_SLTCAP_PIP) +#define NO_CMD_CMPL(ctrl) ((ctrl)->slot_cap & PCI_EXP_SLTCAP_NCCS) +#define PSN(ctrl) (((ctrl)->slot_cap & PCI_EXP_SLTCAP_PSN) >> 19) + +#define HP_CTRL_FINISH 0x0 +#define HP_CTRL_INSERT 0x1 +#define HP_CTRL_REMOVE 0x2 + +void sunway_pciehp_request(struct controller *ctrl, int action); +void sunway_pciehp_handle_button_press(struct controller *ctrl); +void sunway_pciehp_handle_disable_request(struct controller *ctrl); +void sunway_pciehp_handle_presence_or_link_change(struct controller *ctrl, u32 events); +int sunway_pciehp_configure_device(struct controller *ctrl); +void sunway_pciehp_unconfigure_device(struct controller *ctrl, bool presence); +void sunway_pciehp_queue_pushbutton_work(struct work_struct *work); +struct controller *sunwayhpc_init(struct pci_dev *dev); +int sunway_pcie_init_notification(struct controller *ctrl); +void sunway_pcie_shutdown_notification(struct controller *ctrl); +void sunway_pcie_clear_hotplug_events(struct controller *ctrl); +int sunway_pciehp_power_on_slot(struct controller *ctrl); +void sunway_pciehp_power_off_slot(struct controller *ctrl); +void sunway_pciehp_get_power_status(struct controller *ctrl, u8 *status); + +#define INDICATOR_NOOP -1 /* Leave indicator unchanged */ +void sunway_pciehp_set_indicators(struct controller *ctrl, int pwr, int attn); + +void sunway_pciehp_get_latch_status(struct controller *ctrl, u8 *status); +int sunway_pciehp_query_power_fault(struct controller *ctrl); +int sunway_pciehp_card_present(struct controller *ctrl); +int sunway_pciehp_card_present_or_link_active(struct controller *ctrl); +int sunway_pciehp_check_link_status(struct controller *ctrl); +int sunway_pciehp_check_link_active(struct controller *ctrl); +void sunway_pciehp_release_ctrl(struct controller *ctrl); + +int sunway_pciehp_sysfs_enable_slot(struct hotplug_slot *hotplug_slot); +int sunway_pciehp_sysfs_disable_slot(struct hotplug_slot *hotplug_slot); +int sunway_pciehp_reset_slot(struct hotplug_slot *hotplug_slot, int probe); +int sunway_pciehp_get_attention_status(struct hotplug_slot *hotplug_slot, u8 *status); +int sunway_pciehp_set_raw_indicator_status(struct hotplug_slot *hotplug_slot, u8 status); +int sunway_pciehp_get_raw_indicator_status(struct hotplug_slot *hotplug_slot, u8 *status); + +int sunway_pciehp_link_enable(struct controller *ctrl); +int sunway_pciehp_link_disable(struct controller *ctrl); +void sunway_pciehp_restore_rc_piu(struct controller *ctrl); + +static inline const char *slot_name(struct controller *ctrl) +{ + return hotplug_slot_name(&ctrl->hotplug_slot); +} + +static inline struct controller *to_ctrl(struct hotplug_slot *hotplug_slot) +{ + return container_of(hotplug_slot, struct controller, hotplug_slot); +} + +#endif /* _PCIEHP_H */ diff --git a/drivers/pci/hotplug/sunway_pciehp_core.c b/drivers/pci/hotplug/sunway_pciehp_core.c new file mode 100644 index 000000000000..e79a074d2557 --- /dev/null +++ b/drivers/pci/hotplug/sunway_pciehp_core.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Sunway PCI Express Hot Plug Controller Driver + */ + +#define pr_fmt(fmt) "sunway_pciehp: " fmt +#define dev_fmt pr_fmt + +#include +#include +#include +#include +#include +#include + +#include "../pci.h" +#include "sunway_pciehp.h" + +/* Global variables */ +bool sunway_pciehp_poll_mode; +int sunway_pciehp_poll_time; + +#define DRIVER_VERSION "0.1" +#define DRIVER_DESC "Sunway PCI Express Hot Plug Controller Driver" + +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("GPL"); + +/* + * not really modular, but the easiest way to keep compat with existing + * bootargs behaviour is to continue using module_param here. + */ +module_param(sunway_pciehp_poll_mode, bool, 0644); +module_param(sunway_pciehp_poll_time, int, 0644); +MODULE_PARM_DESC(sunway_pciehp_poll_mode, "Using polling mechanism for hot-plug events or not"); +MODULE_PARM_DESC(sunway_pciehp_poll_time, "Polling mechanism frequency, in seconds"); + +#define PCIE_MODULE_NAME "sunway_pciehp" + +static int set_attention_status(struct hotplug_slot *slot, u8 value); +static int get_power_status(struct hotplug_slot *slot, u8 *value); +static int get_latch_status(struct hotplug_slot *slot, u8 *value); +static int get_adapter_status(struct hotplug_slot *slot, u8 *value); + +static int init_slot(struct controller *ctrl) +{ + struct hotplug_slot_ops *ops; + char name[SLOT_NAME_SIZE]; + int retval; + + /* Setup hotplug slot ops */ + ops = kzalloc(sizeof(*ops), GFP_KERNEL); + if (!ops) + return -ENOMEM; + + ops->enable_slot = sunway_pciehp_sysfs_enable_slot; + ops->disable_slot = sunway_pciehp_sysfs_disable_slot; + ops->get_power_status = get_power_status; + ops->get_adapter_status = get_adapter_status; + ops->reset_slot = sunway_pciehp_reset_slot; + if (MRL_SENS(ctrl)) + ops->get_latch_status = get_latch_status; + if (ATTN_LED(ctrl)) { + ops->get_attention_status = sunway_pciehp_get_attention_status; + ops->set_attention_status = set_attention_status; + } else if (ctrl->pci_dev->hotplug_user_indicators) { + ops->get_attention_status = sunway_pciehp_get_raw_indicator_status; + ops->set_attention_status = sunway_pciehp_set_raw_indicator_status; + } + + /* register this slot with the hotplug pci core */ + ctrl->hotplug_slot.ops = ops; + snprintf(name, SLOT_NAME_SIZE, "%u", PSN(ctrl)); + + retval = pci_hp_initialize(&ctrl->hotplug_slot, + ctrl->pci_dev->subordinate, 0, name); + if (retval) { + ctrl_err(ctrl, "pci_hp_initialize failed: error %d\n", retval); + kfree(ops); + } + return retval; +} + +static void cleanup_slot(struct controller *ctrl) +{ + struct hotplug_slot *hotplug_slot = &ctrl->hotplug_slot; + + pci_hp_destroy(hotplug_slot); + kfree(hotplug_slot->ops); +} + +/* + * set_attention_status - Turns the Attention Indicator on, off or blinking + */ +static int set_attention_status(struct hotplug_slot *hotplug_slot, u8 status) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + struct pci_dev *pdev = ctrl->pci_dev; + + if (status) + status <<= PCI_EXP_SLTCTL_ATTN_IND_SHIFT; + else + status = PCI_EXP_SLTCTL_ATTN_IND_OFF; + + pci_config_pm_runtime_get(pdev); + sunway_pciehp_set_indicators(ctrl, INDICATOR_NOOP, status); + pci_config_pm_runtime_put(pdev); + return 0; +} + +static int get_power_status(struct hotplug_slot *hotplug_slot, u8 *value) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + struct pci_dev *pdev = ctrl->pci_dev; + + pci_config_pm_runtime_get(pdev); + sunway_pciehp_get_power_status(ctrl, value); + pci_config_pm_runtime_put(pdev); + return 0; +} + +static int get_latch_status(struct hotplug_slot *hotplug_slot, u8 *value) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + struct pci_dev *pdev = ctrl->pci_dev; + + pci_config_pm_runtime_get(pdev); + sunway_pciehp_get_latch_status(ctrl, value); + pci_config_pm_runtime_put(pdev); + return 0; +} + +static int get_adapter_status(struct hotplug_slot *hotplug_slot, u8 *value) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + struct pci_dev *pdev = ctrl->pci_dev; + int ret; + + pci_config_pm_runtime_get(pdev); + ret = sunway_pciehp_card_present_or_link_active(ctrl); + pci_config_pm_runtime_put(pdev); + + if (ret < 0) + return ret; + + *value = ret; + return 0; +} + +/** + * sunway_pciehp_check_presence() - synthesize event if presence has changed + * + * On probe and resume, an explicit presence check is necessary to bring up an + * occupied slot or bring down an unoccupied slot. This can't be triggered by + * events in the Slot Status register, they may be stale and are therefore + * cleared. Secondly, sending an interrupt for "events that occur while + * interrupt generation is disabled [when] interrupt generation is subsequently + * enabled" is optional per PCIe r4.0, sec 6.7.3.4. + */ +static void sunway_pciehp_check_presence(struct controller *ctrl) +{ + int occupied; + + down_read(&ctrl->reset_lock); + mutex_lock(&ctrl->state_lock); + + occupied = sunway_pciehp_card_present_or_link_active(ctrl); + if ((occupied > 0 && (ctrl->state == OFF_STATE || + ctrl->state == BLINKINGON_STATE)) || + (!occupied && (ctrl->state == ON_STATE || + ctrl->state == BLINKINGOFF_STATE))) + sunway_pciehp_request(ctrl, PCI_EXP_SLTSTA_PDC); + + mutex_unlock(&ctrl->state_lock); + up_read(&ctrl->reset_lock); +} + +static int sunwayhp_init(struct pci_dev *dev) +{ + int rc; + struct controller *ctrl; + + if (!dev->subordinate) { + /* Can happen if we run out of bus numbers during probe */ + dev_err(&dev->dev, + "Hotplug bridge without secondary bus, ignoring\n"); + return -ENODEV; + } + + ctrl = sunwayhpc_init(dev); + if (!ctrl) { + dev_err(&dev->dev, "Controller initialization failed\n"); + return -ENODEV; + } + pci_set_drvdata(dev, ctrl); + + /* Setup the slot information structures */ + rc = init_slot(ctrl); + if (rc) { + if (rc == -EBUSY) + ctrl_warn(ctrl, "Slot already registered by another hotplug driver\n"); + else + ctrl_err(ctrl, "Slot initialization failed (%d)\n", rc); + goto err_out_release_ctlr; + } + + /* Enable events after we have setup the data structures */ + rc = sunway_pcie_init_notification(ctrl); + if (rc) { + ctrl_err(ctrl, "Notification initialization failed (%d)\n", rc); + goto err_out_free_ctrl_slot; + } + + /* Publish to user space */ + rc = pci_hp_add(&ctrl->hotplug_slot); + if (rc) { + ctrl_err(ctrl, "Publication to user space failed (%d)\n", rc); + goto err_out_shutdown_notification; + } + + sunway_pciehp_check_presence(ctrl); + + return 0; + +err_out_shutdown_notification: + sunway_pcie_shutdown_notification(ctrl); +err_out_free_ctrl_slot: + cleanup_slot(ctrl); +err_out_release_ctlr: + sunway_pciehp_release_ctrl(ctrl); + return -ENODEV; +} + +static void sunwayhp_remove(struct pci_dev *dev) +{ + struct controller *ctrl = pci_get_drvdata(dev); + + pci_hp_del(&ctrl->hotplug_slot); + sunway_pcie_shutdown_notification(ctrl); + cleanup_slot(ctrl); + sunway_pciehp_release_ctrl(ctrl); +} + +extern struct pci_controller *hose_head, **hose_tail; +static int __init sunway_pciehp_init(void) +{ + int retval; + struct pci_dev *pdev = NULL; + struct pci_controller *hose = NULL; + + pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); + + for (hose = hose_head; hose; hose = hose->next) { + pdev = pci_get_device(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, pdev); + + retval = sunwayhp_init(pdev); + } + + return retval; +} + +static void __exit sunway_pciehp_exit(void) +{ + struct pci_dev *pdev = NULL; + struct pci_controller *hose = NULL; + + pr_info(DRIVER_DESC " version: " DRIVER_VERSION " unloaded\n"); + + for (hose = hose_head; hose; hose = hose->next) { + pdev = pci_get_device(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, pdev); + + sunwayhp_remove(pdev); + } + +} + +module_init(sunway_pciehp_init); +module_exit(sunway_pciehp_exit); diff --git a/drivers/pci/hotplug/sunway_pciehp_ctrl.c b/drivers/pci/hotplug/sunway_pciehp_ctrl.c new file mode 100644 index 000000000000..c85ba8e684cc --- /dev/null +++ b/drivers/pci/hotplug/sunway_pciehp_ctrl.c @@ -0,0 +1,671 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Sunway PCI Express Hot Plug Controller Driver + */ + +#define dev_fmt(fmt) "sunway_pciehp: " fmt + +#include +#include +#include +#include +#include +#include + +#include "sunway_pciehp.h" +#include "../pci.h" + +/* The following routines constitute the bulk of the + * hotplug controller logic + */ + +#define SAFE_REMOVAL true +#define SURPRISE_REMOVAL false + +static void set_slot_off(struct controller *ctrl) +{ + /* + * Turn off slot, turn on attention indicator, turn off power + * indicator + */ + if (POWER_CTRL(ctrl)) { + sunway_pciehp_power_off_slot(ctrl); + + /* + * After turning power off, we must wait for at least 1 second + * before taking any action that relies on power having been + * removed from the slot/adapter. + */ + msleep(1000); + } + + sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF, + PCI_EXP_SLTCTL_ATTN_IND_ON); +} + +/** + * board_added - Called after a board has been added to the system. + * @ctrl: PCIe hotplug controller where board is added + * + * Turns power on for the board. + * Configures board. + */ +static int board_added(struct controller *ctrl) +{ + int retval = 0; + struct pci_bus *parent = ctrl->pci_dev->subordinate; + + if (POWER_CTRL(ctrl)) { + /* Power on slot */ + retval = sunway_pciehp_power_on_slot(ctrl); + if (retval) + return retval; + } + + sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_BLINK, + INDICATOR_NOOP); + + /* Check link training status */ + retval = sunway_pciehp_check_link_status(ctrl); + if (retval) + goto err_exit; + + /* Check for a power fault */ + if (ctrl->power_fault_detected || sunway_pciehp_query_power_fault(ctrl)) { + ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(ctrl)); + retval = -EIO; + goto err_exit; + } + + retval = sunway_pciehp_configure_device(ctrl); + if (retval) { + if (retval != -EEXIST) { + ctrl_err(ctrl, "Cannot add device at %04x:%02x:00\n", + pci_domain_nr(parent), parent->number); + goto err_exit; + } + } + + sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_ON, + PCI_EXP_SLTCTL_ATTN_IND_OFF); + return 0; + +err_exit: + set_slot_off(ctrl); + return retval; +} + +/** + * remove_board - Turn off slot and Power Indicator + * @ctrl: PCIe hotplug controller where board is being removed + * @safe_removal: whether the board is safely removed (versus surprise removed) + */ +static void remove_board(struct controller *ctrl, bool safe_removal) +{ + sunway_pciehp_unconfigure_device(ctrl, safe_removal); + + if (POWER_CTRL(ctrl)) { + sunway_pciehp_power_off_slot(ctrl); + + /* + * After turning power off, we must wait for at least 1 second + * before taking any action that relies on power having been + * removed from the slot/adapter. + */ + msleep(1000); + + /* Ignore link or presence changes caused by power off */ + atomic_and(~(PCI_EXP_SLTSTA_DLLSC | PCI_EXP_SLTSTA_PDC), + &ctrl->pending_events); + } + + /* turn off Green LED */ + sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF, + INDICATOR_NOOP); +} + +static int sunway_pciehp_enable_slot(struct controller *ctrl); +static int sunway_pciehp_disable_slot(struct controller *ctrl, bool safe_removal); + +void sunway_pciehp_request(struct controller *ctrl, int action) +{ + atomic_or(action, &ctrl->pending_events); + if (!sunway_pciehp_poll_mode) + irq_wake_thread(ctrl->pci_dev->irq, ctrl); +} + +void sunway_pciehp_queue_pushbutton_work(struct work_struct *work) +{ + struct controller *ctrl = container_of(work, struct controller, + button_work.work); + + mutex_lock(&ctrl->state_lock); + switch (ctrl->state) { + case BLINKINGOFF_STATE: + sunway_pciehp_request(ctrl, DISABLE_SLOT); + break; + case BLINKINGON_STATE: + sunway_pciehp_request(ctrl, PCI_EXP_SLTSTA_PDC); + break; + default: + break; + } + mutex_unlock(&ctrl->state_lock); +} + +void sunway_pciehp_handle_button_press(struct controller *ctrl) +{ + mutex_lock(&ctrl->state_lock); + switch (ctrl->state) { + case OFF_STATE: + case ON_STATE: + if (ctrl->state == ON_STATE) { + ctrl->state = BLINKINGOFF_STATE; + ctrl_info(ctrl, "Slot(%s): Powering off due to button press\n", + slot_name(ctrl)); + } else { + ctrl->state = BLINKINGON_STATE; + ctrl_info(ctrl, "Slot(%s) Powering on due to button press\n", + slot_name(ctrl)); + } + /* blink power indicator and turn off attention */ + sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_BLINK, + PCI_EXP_SLTCTL_ATTN_IND_OFF); + schedule_delayed_work(&ctrl->button_work, 5 * HZ); + break; + case BLINKINGOFF_STATE: + case BLINKINGON_STATE: + /* + * Cancel if we are still blinking; this means that we + * press the attention again before the 5 sec. limit + * expires to cancel hot-add or hot-remove + */ + ctrl_info(ctrl, "Slot(%s): Button cancel\n", slot_name(ctrl)); + cancel_delayed_work(&ctrl->button_work); + if (ctrl->state == BLINKINGOFF_STATE) { + ctrl->state = ON_STATE; + sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_ON, + PCI_EXP_SLTCTL_ATTN_IND_OFF); + } else { + ctrl->state = OFF_STATE; + sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF, + PCI_EXP_SLTCTL_ATTN_IND_OFF); + } + ctrl_info(ctrl, "Slot(%s): Action canceled due to button press\n", + slot_name(ctrl)); + break; + default: + ctrl_err(ctrl, "Slot(%s): Ignoring invalid state %#x\n", + slot_name(ctrl), ctrl->state); + break; + } + mutex_unlock(&ctrl->state_lock); +} + +void sunway_pciehp_handle_disable_request(struct controller *ctrl) +{ + mutex_lock(&ctrl->state_lock); + switch (ctrl->state) { + case BLINKINGON_STATE: + case BLINKINGOFF_STATE: + cancel_delayed_work(&ctrl->button_work); + break; + } + ctrl->state = POWEROFF_STATE; + mutex_unlock(&ctrl->state_lock); + + ctrl->request_result = sunway_pciehp_disable_slot(ctrl, SAFE_REMOVAL); +} + +void sunway_pciehp_save_config_space(struct controller *ctrl) +{ + struct pci_dev *pdev = ctrl->pci_dev; + int i; + + if (!pdev->state_saved) { + for (i = 0; i < 16; i++) + pci_read_config_dword(pdev, i * 4, &pdev->saved_config_space[i]); + pdev->state_saved = true; + } +} + +void sunway_pciehp_save_rc_piu(struct controller *ctrl) +{ + struct pci_bus *bus = ctrl->pci_dev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + unsigned long node, rc_index; + + node = hose->node; + rc_index = hose->index; + + sunway_pciehp_save_config_space(ctrl); + + if (!ctrl->saved_piu.state_saved) { + ctrl->saved_piu.epdmabar = read_piu_ior0(node, rc_index, EPDMABAR); + ctrl->saved_piu.msiaddr = read_piu_ior0(node, rc_index, MSIADDR); + ctrl->saved_piu.iommuexcpt_ctrl = read_piu_ior0(node, rc_index, IOMMUEXCPT_CTRL); + ctrl->saved_piu.dtbaseaddr = read_piu_ior0(node, rc_index, DTBASEADDR); + + ctrl->saved_piu.intaconfig = read_piu_ior0(node, rc_index, INTACONFIG); + ctrl->saved_piu.intbconfig = read_piu_ior0(node, rc_index, INTBCONFIG); + ctrl->saved_piu.intcconfig = read_piu_ior0(node, rc_index, INTCCONFIG); + ctrl->saved_piu.intdconfig = read_piu_ior0(node, rc_index, INTDCONFIG); + ctrl->saved_piu.pmeintconfig = read_piu_ior0(node, rc_index, PMEINTCONFIG); + ctrl->saved_piu.aererrintconfig = read_piu_ior0(node, rc_index, AERERRINTCONFIG); + ctrl->saved_piu.hpintconfig = read_piu_ior0(node, rc_index, HPINTCONFIG); + + ctrl->saved_piu.state_saved = true; + } +} + +void sunway_pciehp_start(struct hotplug_slot *hotplug_slot) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + struct pci_dev *pdev = ctrl->pci_dev; + struct pci_bus *bus = pdev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + unsigned long piu_value; + unsigned long node, rc_index; + bool hardware_auto = true; + u16 slot_ctrl; + + node = hose->node; + rc_index = hose->index; + + switch (ctrl->state) { + case OFF_STATE: + if (sunway_pciehp_poll_mode) { + ctrl_dbg(ctrl, "%s: poll mode\n", __func__); + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + /* poll mode */ + slot_ctrl &= ~PCI_EXP_SLTCTL_HPIE; + pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); + + sunway_pciehp_request(ctrl, SW64_POLL_ENABLE_SLOT); + } else { + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + /* interrupt mode */ + if (hardware_auto) { + ctrl_dbg(ctrl, "%s: hardware auto enable\n", __func__); + slot_ctrl &= ~(PCI_EXP_SLTCTL_ABPE | + PCI_EXP_SLTCTL_MRLSCE | + PCI_EXP_SLTCTL_PDCE | + PCI_EXP_SLTCTL_CCIE); + slot_ctrl |= (PCI_EXP_SLTCTL_PFDE | + PCI_EXP_SLTCTL_HPIE | + PCI_EXP_SLTCTL_DLLSCE); + } else { + ctrl_dbg(ctrl, "%s: hardware auto disable\n", __func__); + slot_ctrl &= ~(PCI_EXP_SLTCTL_ABPE | PCI_EXP_SLTCTL_MRLSCE); + slot_ctrl |= (PCI_EXP_SLTCTL_PFDE | + PCI_EXP_SLTCTL_PDCE | + PCI_EXP_SLTCTL_CCIE | + PCI_EXP_SLTCTL_HPIE | + PCI_EXP_SLTCTL_DLLSCE); + } + pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); + + sunway_pciehp_set_indicators(ctrl, INDICATOR_NOOP, + PCI_EXP_SLTCTL_ATTN_IND_BLINK); + + write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_INSERT); + } + break; + case ON_STATE: + sunway_pciehp_save_rc_piu(ctrl); + if (sunway_pciehp_poll_mode) { + ctrl_dbg(ctrl, "%s: poll mode\n", __func__); + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + /* poll mode */ + slot_ctrl &= ~PCI_EXP_SLTCTL_HPIE; + pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); + + sunway_pciehp_request(ctrl, SW64_POLL_DISABLE_SLOT); + } else { + ctrl_dbg(ctrl, "%s: int mode\n", __func__); + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + /* interrupt mode */ + slot_ctrl &= ~(PCI_EXP_SLTCTL_ABPE | PCI_EXP_SLTCTL_MRLSCE); + slot_ctrl |= (PCI_EXP_SLTCTL_PFDE | + PCI_EXP_SLTCTL_PDCE | + PCI_EXP_SLTCTL_CCIE | + PCI_EXP_SLTCTL_HPIE | + PCI_EXP_SLTCTL_DLLSCE); + pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); + + sunway_pciehp_set_indicators(ctrl, INDICATOR_NOOP, + PCI_EXP_SLTCTL_ATTN_IND_BLINK); + sunway_pciehp_link_disable(ctrl); + + while (1) { + piu_value = read_piu_ior1(node, rc_index, NEWLTSSMSTATE0); + piu_value &= 0xff; + + if (piu_value == 0x19) + break; + + udelay(10); + } + + write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_REMOVE); + + sunway_pciehp_request(ctrl, DISABLE_SLOT); + } + break; + default: + break; + } +} + +static void pciehp_restore_config_space_range(struct pci_dev *pdev, + int start, int end) +{ + int index; + + for (index = end; index >= start; index--) { + pci_write_config_dword(pdev, 4 * index, + pdev->saved_config_space[index]); + } +} + +void sunway_pciehp_restore_config_space(struct controller *ctrl) +{ + struct pci_dev *pdev = ctrl->pci_dev; + + if (pdev->state_saved) { + pciehp_restore_config_space_range(pdev, 12, 15); + pciehp_restore_config_space_range(pdev, 9, 11); + pciehp_restore_config_space_range(pdev, 0, 8); + pdev->state_saved = false; + } +} + +void sunway_pciehp_restore_rc_piu(struct controller *ctrl) +{ + struct pci_bus *bus = ctrl->pci_dev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + unsigned long node, rc_index; + + node = hose->node; + rc_index = hose->index; + + sunway_pciehp_restore_config_space(ctrl); + + if (ctrl->saved_piu.state_saved) { + write_piu_ior0(node, rc_index, EPDMABAR, ctrl->saved_piu.epdmabar); + write_piu_ior0(node, rc_index, MSIADDR, ctrl->saved_piu.msiaddr); + write_piu_ior0(node, rc_index, IOMMUEXCPT_CTRL, ctrl->saved_piu.iommuexcpt_ctrl); + write_piu_ior0(node, rc_index, DTBASEADDR, ctrl->saved_piu.dtbaseaddr); + + write_piu_ior0(node, rc_index, INTACONFIG, ctrl->saved_piu.intaconfig); + write_piu_ior0(node, rc_index, INTBCONFIG, ctrl->saved_piu.intbconfig); + write_piu_ior0(node, rc_index, INTCCONFIG, ctrl->saved_piu.intcconfig); + write_piu_ior0(node, rc_index, INTDCONFIG, ctrl->saved_piu.intdconfig); + write_piu_ior0(node, rc_index, PMEINTCONFIG, ctrl->saved_piu.pmeintconfig); + write_piu_ior0(node, rc_index, AERERRINTCONFIG, ctrl->saved_piu.aererrintconfig); + write_piu_ior0(node, rc_index, HPINTCONFIG, ctrl->saved_piu.hpintconfig); + + ctrl->saved_piu.state_saved = false; + } +} + +void sunway_pciehp_end(struct controller *ctrl, bool insert) +{ + struct pci_dev *pdev = ctrl->pci_dev; + struct pci_bus *bus = pdev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + unsigned long piu_value; + unsigned long node, rc_index; + u16 slot_ctrl; + + node = hose->node; + rc_index = hose->index; + + if (insert) { + write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_FINISH); + } else { + sunway_pciehp_set_indicators(ctrl, INDICATOR_NOOP, + PCI_EXP_SLTCTL_ATTN_IND_OFF); + sunway_pciehp_link_enable(ctrl); + + mdelay(100); + + while (1) { + piu_value = read_piu_ior1(node, rc_index, NEWLTSSMSTATE0); + piu_value &= 0xff; + + if (piu_value == 0x0) + break; + + udelay(10); + } + + write_piu_ior0(node, rc_index, HPINTCONFIG, HP_ENABLE_INTD_CORE0); + + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + slot_ctrl |= (PCI_EXP_SLTCTL_PFDE | + PCI_EXP_SLTCTL_PDCE | + PCI_EXP_SLTCTL_CCIE | + PCI_EXP_SLTCTL_HPIE | + PCI_EXP_SLTCTL_PCC | + PCI_EXP_SLTCTL_DLLSCE); + pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); + + write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_FINISH); + } +} + +void sunway_pciehp_handle_presence_or_link_change(struct controller *ctrl, u32 events) +{ + int present, link_active; + + /* + * If the slot is on and presence or link has changed, turn it off. + * Even if it's occupied again, we cannot assume the card is the same. + */ + mutex_lock(&ctrl->state_lock); + switch (ctrl->state) { + case BLINKINGOFF_STATE: + cancel_delayed_work(&ctrl->button_work); + fallthrough; + case ON_STATE: + ctrl->state = POWEROFF_STATE; + mutex_unlock(&ctrl->state_lock); + if (events & PCI_EXP_SLTSTA_DLLSC) + ctrl_info(ctrl, "Slot(%s): Link Down\n", + slot_name(ctrl)); + if (events & PCI_EXP_SLTSTA_PDC) + ctrl_info(ctrl, "Slot(%s): Card not present\n", + slot_name(ctrl)); + sunway_pciehp_disable_slot(ctrl, SURPRISE_REMOVAL); + break; + default: + mutex_unlock(&ctrl->state_lock); + break; + } + + /* Turn the slot on if it's occupied or link is up */ + mutex_lock(&ctrl->state_lock); + present = sunway_pciehp_card_present(ctrl); + link_active = sunway_pciehp_check_link_active(ctrl); + if (present <= 0 && link_active <= 0) { + sunway_pciehp_end(ctrl, false); + mutex_unlock(&ctrl->state_lock); + return; + } + + switch (ctrl->state) { + case BLINKINGON_STATE: + cancel_delayed_work(&ctrl->button_work); + fallthrough; + case OFF_STATE: + ctrl->state = POWERON_STATE; + mutex_unlock(&ctrl->state_lock); + if (present) + ctrl_info(ctrl, "Slot(%s): Card present\n", + slot_name(ctrl)); + if (link_active) + ctrl_info(ctrl, "Slot(%s): Link Up\n", + slot_name(ctrl)); + ctrl->request_result = sunway_pciehp_enable_slot(ctrl); + sunway_pciehp_end(ctrl, true); + break; + default: + mutex_unlock(&ctrl->state_lock); + break; + } +} + +static int __sunway_pciehp_enable_slot(struct controller *ctrl) +{ + u8 getstatus = 0; + + if (MRL_SENS(ctrl)) { + sunway_pciehp_get_latch_status(ctrl, &getstatus); + if (getstatus) { + ctrl_info(ctrl, "Slot(%s): Latch open\n", + slot_name(ctrl)); + return -ENODEV; + } + } + + if (POWER_CTRL(ctrl)) { + sunway_pciehp_get_power_status(ctrl, &getstatus); + if (getstatus) { + ctrl_info(ctrl, "Slot(%s): Already enabled\n", + slot_name(ctrl)); + return 0; + } + } + + return board_added(ctrl); +} + +static int sunway_pciehp_enable_slot(struct controller *ctrl) +{ + int ret; + + pm_runtime_get_sync(&ctrl->pci_dev->dev); + ret = __sunway_pciehp_enable_slot(ctrl); + if (ret && ATTN_BUTTN(ctrl)) + /* may be blinking */ + sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF, + INDICATOR_NOOP); + pm_runtime_put(&ctrl->pci_dev->dev); + + mutex_lock(&ctrl->state_lock); + ctrl->state = ret ? OFF_STATE : ON_STATE; + mutex_unlock(&ctrl->state_lock); + + return ret; +} + +static int __sunway_pciehp_disable_slot(struct controller *ctrl, bool safe_removal) +{ + u8 getstatus = 0; + + if (POWER_CTRL(ctrl)) { + sunway_pciehp_get_power_status(ctrl, &getstatus); + if (!getstatus) { + ctrl_info(ctrl, "Slot(%s): Already disabled\n", + slot_name(ctrl)); + return -EINVAL; + } + } + + remove_board(ctrl, safe_removal); + return 0; +} + +static int sunway_pciehp_disable_slot(struct controller *ctrl, bool safe_removal) +{ + int ret; + + pm_runtime_get_sync(&ctrl->pci_dev->dev); + ret = __sunway_pciehp_disable_slot(ctrl, safe_removal); + pm_runtime_put(&ctrl->pci_dev->dev); + + mutex_lock(&ctrl->state_lock); + ctrl->state = OFF_STATE; + mutex_unlock(&ctrl->state_lock); + + return ret; +} + +int sunway_pciehp_sysfs_enable_slot(struct hotplug_slot *hotplug_slot) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + + mutex_lock(&ctrl->state_lock); + switch (ctrl->state) { + case BLINKINGON_STATE: + case OFF_STATE: + mutex_unlock(&ctrl->state_lock); + /* + * The IRQ thread becomes a no-op if the user pulls out the + * card before the thread wakes up, so initialize to -ENODEV. + */ + ctrl->request_result = -ENODEV; + sunway_pciehp_request(ctrl, PCI_EXP_SLTSTA_PDC); + + wait_event(ctrl->requester, + !atomic_read(&ctrl->pending_events) && + !ctrl->ist_running); + + sunway_pciehp_start(hotplug_slot); + return ctrl->request_result; + case POWERON_STATE: + ctrl_info(ctrl, "Slot(%s): Already in powering on state\n", + slot_name(ctrl)); + break; + case BLINKINGOFF_STATE: + case ON_STATE: + case POWEROFF_STATE: + ctrl_info(ctrl, "Slot(%s): Already enabled\n", + slot_name(ctrl)); + break; + default: + ctrl_err(ctrl, "Slot(%s): Invalid state %#x\n", + slot_name(ctrl), ctrl->state); + break; + } + mutex_unlock(&ctrl->state_lock); + + return -ENODEV; +} + +int sunway_pciehp_sysfs_disable_slot(struct hotplug_slot *hotplug_slot) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + + mutex_lock(&ctrl->state_lock); + switch (ctrl->state) { + case BLINKINGOFF_STATE: + case ON_STATE: + sunway_pciehp_start(hotplug_slot); + + mutex_unlock(&ctrl->state_lock); + wait_event(ctrl->requester, + !atomic_read(&ctrl->pending_events) && + !ctrl->ist_running); + + return ctrl->request_result; + case POWEROFF_STATE: + ctrl_info(ctrl, "Slot(%s): Already in powering off state\n", + slot_name(ctrl)); + break; + case BLINKINGON_STATE: + case OFF_STATE: + case POWERON_STATE: + ctrl_info(ctrl, "Slot(%s): Already disabled\n", + slot_name(ctrl)); + break; + default: + ctrl_err(ctrl, "Slot(%s): Invalid state %#x\n", + slot_name(ctrl), ctrl->state); + break; + } + mutex_unlock(&ctrl->state_lock); + + return -ENODEV; +} diff --git a/drivers/pci/hotplug/sunway_pciehp_hpc.c b/drivers/pci/hotplug/sunway_pciehp_hpc.c new file mode 100644 index 000000000000..d30ee235b029 --- /dev/null +++ b/drivers/pci/hotplug/sunway_pciehp_hpc.c @@ -0,0 +1,1152 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Sunway PCI Express PCI Hot Plug Driver + */ + +#define dev_fmt(fmt) "sunway_pciehp: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../pci.h" +#include "sunway_pciehp.h" + +#include +#include + +static const struct dmi_system_id inband_presence_disabled_dmi_table[] = { + /* + * Match all Dell systems, as some Dell systems have inband + * presence disabled on NVMe slots (but don't support the bit to + * report it). Setting inband presence disabled should have no + * negative effect, except on broken hotplug slots that never + * assert presence detect--and those will still work, they will + * just have a bit of extra delay before being probed. + */ + { + .ident = "Dell System", + .matches = { + DMI_MATCH(DMI_OEM_STRING, "Dell System"), + }, + }, + {} +}; + +static inline struct pci_dev *ctrl_dev(struct controller *ctrl) +{ + return ctrl->pci_dev; +} + +static irqreturn_t sunway_pciehp_isr(int irq, void *dev_id); +static irqreturn_t sunway_pciehp_ist(int irq, void *dev_id); +static int sunway_pciehp_poll_start(void *data); + +static inline int sunway_pciehp_request_irq(struct controller *ctrl) +{ + int retval, irq = ctrl->pci_dev->irq; + + if (sunway_pciehp_poll_mode) { + ctrl->poll_thread = kthread_run(&sunway_pciehp_poll_start, ctrl, + "sunway_pciehp_poll_start-%s", + slot_name(ctrl)); + return PTR_ERR_OR_ZERO(ctrl->poll_thread); + } + + /* Installs the interrupt handler */ + retval = request_threaded_irq(irq, sunway_pciehp_isr, sunway_pciehp_ist, + IRQF_SHARED, MY_NAME, ctrl); + if (retval) + ctrl_err(ctrl, "Cannot get irq %d for the hotplug controller\n", + irq); + return retval; +} + +static inline void sunway_pciehp_free_irq(struct controller *ctrl) +{ + if (sunway_pciehp_poll_mode) + kthread_stop(ctrl->poll_thread); + else + free_irq(ctrl->pci_dev->irq, ctrl); +} + +static int pcie_poll_cmd(struct controller *ctrl, int timeout) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 slot_status; + + do { + pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status); + if (slot_status == (u16) ~0) { + ctrl_info(ctrl, "%s: no response from device\n", + __func__); + return 0; + } + + if (slot_status & PCI_EXP_SLTSTA_CC) { + pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, + PCI_EXP_SLTSTA_CC); + return 1; + } + msleep(10); + timeout -= 10; + } while (timeout >= 0); + return 0; /* timeout */ +} + +static void pcie_wait_cmd(struct controller *ctrl) +{ + unsigned int msecs = sunway_pciehp_poll_mode ? 2500 : 1000; + unsigned long duration = msecs_to_jiffies(msecs); + unsigned long cmd_timeout = ctrl->cmd_started + duration; + unsigned long now, timeout; + int rc; + + /* + * If the controller does not generate notifications for command + * completions, we never need to wait between writes. + */ + if (NO_CMD_CMPL(ctrl)) + return; + + if (!ctrl->cmd_busy) + return; + + /* + * Even if the command has already timed out, we want to call + * pcie_poll_cmd() so it can clear PCI_EXP_SLTSTA_CC. + */ + now = jiffies; + if (time_before_eq(cmd_timeout, now)) + timeout = 1; + else + timeout = cmd_timeout - now; + + if (ctrl->slot_ctrl & PCI_EXP_SLTCTL_HPIE && + ctrl->slot_ctrl & PCI_EXP_SLTCTL_CCIE) + rc = wait_event_timeout(ctrl->queue, !ctrl->cmd_busy, timeout); + else + rc = pcie_poll_cmd(ctrl, jiffies_to_msecs(timeout)); + + if (!rc) + ctrl_info(ctrl, "Timeout on hotplug command %#06x (issued %u msec ago)\n", + ctrl->slot_ctrl, + jiffies_to_msecs(jiffies - ctrl->cmd_started)); +} + +#define CC_ERRATUM_MASK (PCI_EXP_SLTCTL_PCC | \ + PCI_EXP_SLTCTL_PIC | \ + PCI_EXP_SLTCTL_AIC | \ + PCI_EXP_SLTCTL_EIC) + +static void pcie_do_write_cmd(struct controller *ctrl, u16 cmd, + u16 mask, bool wait) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 slot_ctrl_orig, slot_ctrl; + + mutex_lock(&ctrl->ctrl_lock); + + /* + * Always wait for any previous command that might still be in progress + */ + pcie_wait_cmd(ctrl); + + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + if (slot_ctrl == (u16) ~0) { + ctrl_info(ctrl, "%s: no response from device\n", __func__); + goto out; + } + + slot_ctrl_orig = slot_ctrl; + slot_ctrl &= ~mask; + slot_ctrl |= (cmd & mask); + ctrl->cmd_busy = 1; + smp_mb(); + ctrl->slot_ctrl = slot_ctrl; + pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); + ctrl->cmd_started = jiffies; + + /* + * Controllers with the Intel CF118 and similar errata advertise + * Command Completed support, but they only set Command Completed + * if we change the "Control" bits for power, power indicator, + * attention indicator, or interlock. If we only change the + * "Enable" bits, they never set the Command Completed bit. + */ + if ((slot_ctrl_orig & CC_ERRATUM_MASK) == (slot_ctrl & CC_ERRATUM_MASK)) + ctrl->cmd_busy = 0; + + /* + * Optionally wait for the hardware to be ready for a new command, + * indicating completion of the above issued command. + */ + if (wait) + pcie_wait_cmd(ctrl); + +out: + mutex_unlock(&ctrl->ctrl_lock); +} + +/** + * pcie_write_cmd - Issue controller command + * @ctrl: controller to which the command is issued + * @cmd: command value written to slot control register + * @mask: bitmask of slot control register to be modified + */ +static void pcie_write_cmd(struct controller *ctrl, u16 cmd, u16 mask) +{ + pcie_do_write_cmd(ctrl, cmd, mask, true); +} + +/* Same as above without waiting for the hardware to latch */ +static void pcie_write_cmd_nowait(struct controller *ctrl, u16 cmd, u16 mask) +{ + pcie_do_write_cmd(ctrl, cmd, mask, false); +} + +/** + * sunway_pciehp_check_link_active() - Is the link active + * @ctrl: PCIe hotplug controller + * + * Check whether the downstream link is currently active. Note it is + * possible that the card is removed immediately after this so the + * caller may need to take it into account. + * + * If the hotplug controller itself is not available anymore returns + * %-ENODEV. + */ +int sunway_pciehp_check_link_active(struct controller *ctrl) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 lnk_status; + int ret; + + ret = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status); + if (ret == PCIBIOS_DEVICE_NOT_FOUND || lnk_status == (u16)~0) + return -ENODEV; + + ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA); + ctrl_dbg(ctrl, "%s: lnk_status = %x\n", __func__, lnk_status); + + return ret; +} + +static bool pcie_wait_link_active(struct controller *ctrl) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + struct pci_bus *bus = pdev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + if (pcie_wait_for_link(pdev, true)) { + pci_mark_rc_linkup(hose->node, hose->index); + sunway_pciehp_restore_rc_piu(ctrl); + return true; + } + + return false; +} + +static bool pci_bus_check_dev(struct pci_bus *bus, int devfn) +{ + u32 l; + int count = 0; + int delay = 1000, step = 20; + bool found = false; + + do { + found = pci_bus_read_dev_vendor_id(bus, devfn, &l, 0); + count++; + + if (found) + break; + + msleep(step); + delay -= step; + } while (delay > 0); + + if (count > 1) + pr_debug("pci %04x:%02x:%02x.%d id reading try %d times with interval %d ms to get %08x\n", + pci_domain_nr(bus), bus->number, PCI_SLOT(devfn), + PCI_FUNC(devfn), count, step, l); + + return found; +} + +static void pcie_wait_for_presence(struct pci_dev *pdev) +{ + int timeout = 1250; + u16 slot_status; + + do { + pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status); + if (slot_status & PCI_EXP_SLTSTA_PDS) + return; + msleep(10); + timeout -= 10; + } while (timeout > 0); +} + +static bool pci_bus_check_linkup(struct pci_controller *hose) +{ + int count = 0; + int delay = 1000, step = 20; + bool linkup = false; + + do { + if (sw64_chip_init->pci_init.check_pci_linkup(hose->node, hose->index) == 0) { + udelay(10); + if (sw64_chip_init->pci_init.check_pci_linkup(hose->node, hose->index) == 0) + linkup = true; + } + count++; + + if (linkup) + break; + + msleep(step); + delay -= step; + } while (delay > 0); + + if (count > 1) + pr_debug("Node %ld RC %ld linkup reading try %d times with interval %d ms\n", + hose->node, hose->index, count, step); + + return linkup; +} + +int sunway_pciehp_check_link_status(struct controller *ctrl) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + struct pci_bus *bus = pdev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + bool found, linkup; + u16 lnk_status; + + if (!pcie_wait_link_active(ctrl)) { + ctrl_info(ctrl, "Slot(%s): No link\n", slot_name(ctrl)); + return -1; + } + + if (ctrl->inband_presence_disabled) + pcie_wait_for_presence(pdev); + + /* wait 1000ms and check RC linkup before read pci conf */ + msleep(1000); + linkup = pci_bus_check_linkup(hose); + + if (!linkup) + return -1; + + /* try read device id in 1s */ + found = pci_bus_check_dev(ctrl->pci_dev->subordinate, + PCI_DEVFN(0, 0)); + + /* ignore link or presence changes up to this point */ + if (found) + atomic_and(~(PCI_EXP_SLTSTA_DLLSC | PCI_EXP_SLTSTA_PDC), + &ctrl->pending_events); + + pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status); + ctrl_dbg(ctrl, "%s: lnk_status = %x\n", __func__, lnk_status); + if ((lnk_status & PCI_EXP_LNKSTA_LT) || + !(lnk_status & PCI_EXP_LNKSTA_NLW)) { + ctrl_info(ctrl, "Slot(%s): Cannot train link: status %#06x\n", + slot_name(ctrl), lnk_status); + return -1; + } + + pcie_update_link_speed(ctrl->pci_dev->subordinate, lnk_status); + + if (!found) { + ctrl_info(ctrl, "Slot(%s): No device found\n", + slot_name(ctrl)); + return -1; + } + + return 0; +} + +static int __sunway_pciehp_link_set(struct controller *ctrl, bool enable) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 lnk_ctrl; + + pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &lnk_ctrl); + + if (enable) + lnk_ctrl &= ~PCI_EXP_LNKCTL_LD; + else + lnk_ctrl |= PCI_EXP_LNKCTL_LD; + + pcie_capability_write_word(pdev, PCI_EXP_LNKCTL, lnk_ctrl); + ctrl_dbg(ctrl, "%s: lnk_ctrl = %x\n", __func__, lnk_ctrl); + + return 0; +} + +int sunway_pciehp_link_enable(struct controller *ctrl) +{ + return __sunway_pciehp_link_set(ctrl, true); +} + +int sunway_pciehp_link_disable(struct controller *ctrl) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + struct pci_bus *bus = pdev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + int ret; + + ret = __sunway_pciehp_link_set(ctrl, false); + pci_clear_rc_linkup(hose->node, hose->index); + + return ret; +} + +int sunway_pciehp_get_raw_indicator_status(struct hotplug_slot *hotplug_slot, + u8 *status) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 slot_ctrl; + + pci_config_pm_runtime_get(pdev); + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + pci_config_pm_runtime_put(pdev); + *status = (slot_ctrl & (PCI_EXP_SLTCTL_AIC | PCI_EXP_SLTCTL_PIC)) >> 6; + return 0; +} + +int sunway_pciehp_get_attention_status(struct hotplug_slot *hotplug_slot, u8 *status) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 slot_ctrl; + + pci_config_pm_runtime_get(pdev); + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + pci_config_pm_runtime_put(pdev); + ctrl_dbg(ctrl, "%s: SLOTCTRL %x, value read %x\n", __func__, + pci_pcie_cap(ctrl->pci_dev) + PCI_EXP_SLTCTL, slot_ctrl); + + switch (slot_ctrl & PCI_EXP_SLTCTL_AIC) { + case PCI_EXP_SLTCTL_ATTN_IND_ON: + *status = 1; /* On */ + break; + case PCI_EXP_SLTCTL_ATTN_IND_BLINK: + *status = 2; /* Blink */ + break; + case PCI_EXP_SLTCTL_ATTN_IND_OFF: + *status = 0; /* Off */ + break; + default: + *status = 0xFF; + break; + } + + return 0; +} + +void sunway_pciehp_get_power_status(struct controller *ctrl, u8 *status) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 slot_ctrl; + + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + + ctrl_dbg(ctrl, "%s: SLOTCTRL %x value read %x\n", __func__, + pci_pcie_cap(ctrl->pci_dev) + PCI_EXP_SLTCTL, slot_ctrl); + + switch (slot_ctrl & PCI_EXP_SLTCTL_PCC) { + case PCI_EXP_SLTCTL_PWR_ON: + *status = 1; /* On */ + break; + case PCI_EXP_SLTCTL_PWR_OFF: + *status = 0; /* Off */ + break; + default: + *status = 0xFF; + break; + } +} + +void sunway_pciehp_get_latch_status(struct controller *ctrl, u8 *status) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 slot_status; + + pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status); + *status = !!(slot_status & PCI_EXP_SLTSTA_MRLSS); +} + +/** + * sunway_pciehp_card_present() - Is the card present + * @ctrl: PCIe hotplug controller + * + * Function checks whether the card is currently present in the slot and + * in that case returns true. Note it is possible that the card is + * removed immediately after the check so the caller may need to take + * this into account. + * + * It the hotplug controller itself is not available anymore returns + * %-ENODEV. + */ +int sunway_pciehp_card_present(struct controller *ctrl) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 slot_status; + int ret; + + ret = pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status); + if (ret == PCIBIOS_DEVICE_NOT_FOUND || slot_status == (u16)~0) + return -ENODEV; + + return !!(slot_status & PCI_EXP_SLTSTA_PDS); +} + +/** + * sunway_pciehp_card_present_or_link_active() - whether given slot is occupied + * @ctrl: PCIe hotplug controller + * + * Unlike pciehp_card_present(), which determines presence solely from the + * Presence Detect State bit, this helper also returns true if the Link Active + * bit is set. This is a concession to broken hotplug ports which hardwire + * Presence Detect State to zero, such as Wilocity's [1ae9:0200]. + * + * Returns: %1 if the slot is occupied and %0 if it is not. If the hotplug + * port is not present anymore returns %-ENODEV. + */ +int sunway_pciehp_card_present_or_link_active(struct controller *ctrl) +{ + int ret; + + ret = sunway_pciehp_card_present(ctrl); + if (ret) + return ret; + + return sunway_pciehp_check_link_active(ctrl); +} + +int sunway_pciehp_query_power_fault(struct controller *ctrl) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 slot_status; + + pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status); + return !!(slot_status & PCI_EXP_SLTSTA_PFD); +} + +int sunway_pciehp_set_raw_indicator_status(struct hotplug_slot *hotplug_slot, + u8 status) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + struct pci_dev *pdev = ctrl_dev(ctrl); + + pci_config_pm_runtime_get(pdev); + pcie_write_cmd_nowait(ctrl, status << 6, + PCI_EXP_SLTCTL_AIC | PCI_EXP_SLTCTL_PIC); + pci_config_pm_runtime_put(pdev); + return 0; +} + +/** + * sunway_pciehp_set_indicators() - set attention indicator, power indicator, or both + * @ctrl: PCIe hotplug controller + * @pwr: one of: + * PCI_EXP_SLTCTL_PWR_IND_ON + * PCI_EXP_SLTCTL_PWR_IND_BLINK + * PCI_EXP_SLTCTL_PWR_IND_OFF + * @attn: one of: + * PCI_EXP_SLTCTL_ATTN_IND_ON + * PCI_EXP_SLTCTL_ATTN_IND_BLINK + * PCI_EXP_SLTCTL_ATTN_IND_OFF + * + * Either @pwr or @attn can also be INDICATOR_NOOP to leave that indicator + * unchanged. + */ +void sunway_pciehp_set_indicators(struct controller *ctrl, int pwr, int attn) +{ + u16 cmd = 0, mask = 0; + + if (PWR_LED(ctrl) && pwr != INDICATOR_NOOP) { + cmd |= (pwr & PCI_EXP_SLTCTL_PIC); + mask |= PCI_EXP_SLTCTL_PIC; + } + + if (ATTN_LED(ctrl) && attn != INDICATOR_NOOP) { + cmd |= (attn & PCI_EXP_SLTCTL_AIC); + mask |= PCI_EXP_SLTCTL_AIC; + } + + if (cmd) { + pcie_write_cmd_nowait(ctrl, cmd, mask); + ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__, + pci_pcie_cap(ctrl->pci_dev) + PCI_EXP_SLTCTL, cmd); + } +} + +int sunway_pciehp_power_on_slot(struct controller *ctrl) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 slot_status; + int retval; + + /* Clear power-fault bit from previous power failures */ + pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status); + if (slot_status & PCI_EXP_SLTSTA_PFD) + pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, + PCI_EXP_SLTSTA_PFD); + ctrl->power_fault_detected = 0; + + pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_ON, PCI_EXP_SLTCTL_PCC); + ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__, + pci_pcie_cap(ctrl->pci_dev) + PCI_EXP_SLTCTL, + PCI_EXP_SLTCTL_PWR_ON); + + retval = sunway_pciehp_link_enable(ctrl); + if (retval) + ctrl_err(ctrl, "%s: Can not enable the link!\n", __func__); + + return retval; +} + +void sunway_pciehp_power_off_slot(struct controller *ctrl) +{ + pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_OFF, PCI_EXP_SLTCTL_PCC); + ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__, + pci_pcie_cap(ctrl->pci_dev) + PCI_EXP_SLTCTL, + PCI_EXP_SLTCTL_PWR_OFF); +} + +void sunway_pciehp_poll(struct controller *ctrl, u32 events) +{ + struct pci_dev *pdev = ctrl_dev(ctrl); + struct pci_bus *bus = pdev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + unsigned long piu_value; + unsigned long node, rc_index; + u16 slot_ctrl; + int i; + + node = hose->node; + rc_index = hose->index; + + if (events & SW64_POLL_DISABLE_SLOT) { + while (1) { + piu_value = read_piu_ior1(node, rc_index, NEWLTSSMSTATE0); + piu_value &= 0xff; + + if (piu_value == 0x19) + break; + + udelay(10); + } + + write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_REMOVE); + + while (1) { + piu_value = read_piu_ior0(node, rc_index, HP_WATCHOUT); + piu_value >>= 24; + + if (piu_value == 0x19) + break; + + udelay(10); + } + + sunway_pciehp_link_enable(ctrl); + + mdelay(100); + while (1) { + piu_value = read_piu_ior1(node, rc_index, NEWLTSSMSTATE0); + piu_value &= 0xff; + + if (piu_value == 0x0) + break; + + udelay(10); + } + + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + slot_ctrl &= ~(PCI_EXP_SLTCTL_ABPE | + PCI_EXP_SLTCTL_PFDE | + PCI_EXP_SLTCTL_MRLSCE | + PCI_EXP_SLTCTL_PDCE | + PCI_EXP_SLTCTL_CCIE | + PCI_EXP_SLTCTL_HPIE); + slot_ctrl |= (PCI_EXP_SLTCTL_ATTN_IND_OFF | + PCI_EXP_SLTCTL_PWR_IND_OFF | + PCI_EXP_SLTCTL_DLLSCE); + pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); + + write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_FINISH); + + ctrl->state = OFF_STATE; + } + + if (events & SW64_POLL_ENABLE_SLOT) { + write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_INSERT); + + for (i = 0; i < 30; i++) { + if (pcie_wait_for_link(pdev, true)) { + pci_mark_rc_linkup(hose->node, hose->index); + sunway_pciehp_restore_rc_piu(ctrl); + + write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_FINISH); + + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); + slot_ctrl &= ~PCI_EXP_SLTCTL_PWR_OFF; + pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); + + ctrl->state = ON_STATE; + return; + } + + if (sunway_pciehp_query_power_fault(ctrl)) { + pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, + PCI_EXP_SLTSTA_PFD); + udelay(10); + if (sunway_pciehp_query_power_fault(ctrl)) { + ctrl_err(ctrl, "power fault\n"); + return; + } + } + } + + ctrl_err(ctrl, "insert failed\n"); + } +} + +static irqreturn_t sunway_pciehp_isr(int irq, void *dev_id) +{ + struct controller *ctrl = (struct controller *)dev_id; + struct pci_dev *pdev = ctrl_dev(ctrl); + struct device *parent = pdev->dev.parent; + u16 status, events = 0; + + /* + * Interrupts only occur in D3hot or shallower and only if enabled + * in the Slot Control register (PCIe r4.0, sec 6.7.3.4). + */ + if (pdev->current_state == PCI_D3cold || + (!(ctrl->slot_ctrl & PCI_EXP_SLTCTL_HPIE) && !sunway_pciehp_poll_mode)) + return IRQ_NONE; + + /* + * Keep the port accessible by holding a runtime PM ref on its parent. + * Defer resume of the parent to the IRQ thread if it's suspended. + * Mask the interrupt until then. + */ + if (parent) { + pm_runtime_get_noresume(parent); + if (!pm_runtime_active(parent)) { + pm_runtime_put(parent); + disable_irq_nosync(irq); + atomic_or(RERUN_ISR, &ctrl->pending_events); + return IRQ_WAKE_THREAD; + } + } + + if (sunway_pciehp_poll_mode) { + if (atomic_read(&ctrl->pending_events) & SW64_POLL_DISABLE_SLOT) { + sunway_pciehp_link_disable(ctrl); + goto out; + } else if (atomic_read(&ctrl->pending_events) & SW64_POLL_ENABLE_SLOT) + goto out; + + return IRQ_NONE; + } + +read_status: + pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &status); + if (status == (u16) ~0) { + ctrl_info(ctrl, "%s: no response from device\n", __func__); + if (parent) + pm_runtime_put(parent); + return IRQ_NONE; + } + + /* + * Slot Status contains plain status bits as well as event + * notification bits; right now we only want the event bits. + */ + status &= PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD | + PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_CC | + PCI_EXP_SLTSTA_DLLSC; + + /* + * If we've already reported a power fault, don't report it again + * until we've done something to handle it. + */ + if (ctrl->power_fault_detected) + status &= ~PCI_EXP_SLTSTA_PFD; + + events |= status; + if (!events) { + if (parent) + pm_runtime_put(parent); + return IRQ_NONE; + } + + if (status) { + pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, events); + + /* + * In MSI mode, all event bits must be zero before the port + * will send a new interrupt (PCIe Base Spec r5.0 sec 6.7.3.4). + * So re-read the Slot Status register in case a bit was set + * between read and write. + */ + if (pci_dev_msi_enabled(pdev) && !sunway_pciehp_poll_mode) + goto read_status; + } + + ctrl_dbg(ctrl, "pending interrupts %#06x from Slot Status\n", events); + if (parent) + pm_runtime_put(parent); + + /* + * Command Completed notifications are not deferred to the + * IRQ thread because it may be waiting for their arrival. + */ + if (events & PCI_EXP_SLTSTA_CC) { + ctrl->cmd_busy = 0; + smp_mb(); + wake_up(&ctrl->queue); + + if (events == PCI_EXP_SLTSTA_CC) + return IRQ_HANDLED; + + events &= ~PCI_EXP_SLTSTA_CC; + } + +out: + if (pdev->ignore_hotplug) { + ctrl_dbg(ctrl, "ignoring hotplug event %#06x\n", events); + return IRQ_HANDLED; + } + + /* Save pending events for consumption by IRQ thread. */ + atomic_or(events, &ctrl->pending_events); + return IRQ_WAKE_THREAD; +} + +static irqreturn_t sunway_pciehp_ist(int irq, void *dev_id) +{ + struct controller *ctrl = (struct controller *)dev_id; + struct pci_dev *pdev = ctrl_dev(ctrl); + irqreturn_t ret; + u32 events; + + ctrl->ist_running = true; + pci_config_pm_runtime_get(pdev); + + /* rerun sunway_pciehp_isr() if the port was inaccessible on interrupt */ + if (atomic_fetch_and(~RERUN_ISR, &ctrl->pending_events) & RERUN_ISR) { + ret = sunway_pciehp_isr(irq, dev_id); + enable_irq(irq); + if (ret != IRQ_WAKE_THREAD) + goto out; + } + + synchronize_hardirq(irq); + events = atomic_xchg(&ctrl->pending_events, 0); + if (!events) { + ret = IRQ_NONE; + goto out; + } + + if (sunway_pciehp_poll_mode) { + sunway_pciehp_poll(ctrl, events); + ret = IRQ_HANDLED; + goto out; + } + + /* Check Attention Button Pressed */ + if (events & PCI_EXP_SLTSTA_ABP) { + ctrl_info(ctrl, "Slot(%s): Attention button pressed\n", + slot_name(ctrl)); + sunway_pciehp_handle_button_press(ctrl); + } + + /* Check Power Fault Detected */ + if ((events & PCI_EXP_SLTSTA_PFD) && !ctrl->power_fault_detected) { + ctrl->power_fault_detected = 1; + ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(ctrl)); + sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF, + PCI_EXP_SLTCTL_ATTN_IND_ON); + } + + /* + * Disable requests have higher priority than Presence Detect Changed + * or Data Link Layer State Changed events. + */ + down_read(&ctrl->reset_lock); + + if (events & DISABLE_SLOT) + sunway_pciehp_handle_disable_request(ctrl); + else if (events & (PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC)) + sunway_pciehp_handle_presence_or_link_change(ctrl, events); + + up_read(&ctrl->reset_lock); + + ret = IRQ_HANDLED; +out: + pci_config_pm_runtime_put(pdev); + ctrl->ist_running = false; + wake_up(&ctrl->requester); + return ret; +} + +static int sunway_pciehp_poll_start(void *data) +{ + struct controller *ctrl = data; + + schedule_timeout_idle(10 * HZ); /* start with 10 sec delay */ + + while (!kthread_should_stop()) { + /* poll for interrupt events or user requests */ + while (sunway_pciehp_isr(IRQ_NOTCONNECTED, ctrl) == IRQ_WAKE_THREAD || + atomic_read(&ctrl->pending_events)) + sunway_pciehp_ist(IRQ_NOTCONNECTED, ctrl); + + if (sunway_pciehp_poll_time <= 0 || sunway_pciehp_poll_time > 60) + sunway_pciehp_poll_time = 2; /* clamp to sane value */ + + schedule_timeout_idle(sunway_pciehp_poll_time * HZ); + } + + return 0; +} + +static void pcie_enable_notification(struct controller *ctrl) +{ + u16 cmd, mask; + + /* + * TBD: Power fault detected software notification support. + * + * Power fault detected software notification is not enabled + * now, because it caused power fault detected interrupt storm + * on some machines. On those machines, power fault detected + * bit in the slot status register was set again immediately + * when it is cleared in the interrupt service routine, and + * next power fault detected interrupt was notified again. + */ + + /* + * Always enable link events: thus link-up and link-down shall + * always be treated as hotplug and unplug respectively. Enable + * presence detect only if Attention Button is not present. + */ + cmd = PCI_EXP_SLTCTL_DLLSCE; + if (ATTN_BUTTN(ctrl)) + cmd |= PCI_EXP_SLTCTL_ABPE; + else + cmd |= PCI_EXP_SLTCTL_PDCE; + if (!sunway_pciehp_poll_mode) + cmd |= PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_CCIE; + + mask = (PCI_EXP_SLTCTL_PDCE | PCI_EXP_SLTCTL_ABPE | + PCI_EXP_SLTCTL_PFDE | + PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_CCIE | + PCI_EXP_SLTCTL_DLLSCE); + + pcie_write_cmd_nowait(ctrl, cmd, mask); + ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__, + pci_pcie_cap(ctrl->pci_dev) + PCI_EXP_SLTCTL, cmd); +} + +static void pcie_disable_notification(struct controller *ctrl) +{ + u16 mask; + + mask = (PCI_EXP_SLTCTL_PDCE | PCI_EXP_SLTCTL_ABPE | + PCI_EXP_SLTCTL_MRLSCE | PCI_EXP_SLTCTL_PFDE | + PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_CCIE | + PCI_EXP_SLTCTL_DLLSCE); + pcie_write_cmd(ctrl, 0, mask); + ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__, + pci_pcie_cap(ctrl->pci_dev) + PCI_EXP_SLTCTL, 0); +} + +void sunway_pcie_clear_hotplug_events(struct controller *ctrl) +{ + pcie_capability_write_word(ctrl_dev(ctrl), PCI_EXP_SLTSTA, + PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC); +} + +/* + * sunway_pciehp has a 1:1 bus:slot relationship so we ultimately want a secondary + * bus reset of the bridge, but at the same time we want to ensure that it is + * not seen as a hot-unplug, followed by the hot-plug of the device. Thus, + * disable link state notification and presence detection change notification + * momentarily, if we see that they could interfere. Also, clear any spurious + * events after. + */ +int sunway_pciehp_reset_slot(struct hotplug_slot *hotplug_slot, int probe) +{ + struct controller *ctrl = to_ctrl(hotplug_slot); + struct pci_dev *pdev = ctrl_dev(ctrl); + u16 stat_mask = 0, ctrl_mask = 0; + int rc; + + if (probe) + return 0; + + down_write(&ctrl->reset_lock); + + if (!ATTN_BUTTN(ctrl)) { + ctrl_mask |= PCI_EXP_SLTCTL_PDCE; + stat_mask |= PCI_EXP_SLTSTA_PDC; + } + ctrl_mask |= PCI_EXP_SLTCTL_DLLSCE; + stat_mask |= PCI_EXP_SLTSTA_DLLSC; + + pcie_write_cmd(ctrl, 0, ctrl_mask); + ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__, + pci_pcie_cap(ctrl->pci_dev) + PCI_EXP_SLTCTL, 0); + + rc = pci_bridge_secondary_bus_reset(ctrl->pci_dev); + + pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, stat_mask); + pcie_write_cmd_nowait(ctrl, ctrl_mask, ctrl_mask); + ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__, + pci_pcie_cap(ctrl->pci_dev) + PCI_EXP_SLTCTL, ctrl_mask); + + up_write(&ctrl->reset_lock); + return rc; +} + +int sunway_pcie_init_notification(struct controller *ctrl) +{ + if (sunway_pciehp_request_irq(ctrl)) + return -1; + pcie_enable_notification(ctrl); + ctrl->notification_enabled = 1; + return 0; +} + +void sunway_pcie_shutdown_notification(struct controller *ctrl) +{ + if (ctrl->notification_enabled) { + pcie_disable_notification(ctrl); + sunway_pciehp_free_irq(ctrl); + ctrl->notification_enabled = 0; + } +} + +static inline void dbg_ctrl(struct controller *ctrl) +{ + struct pci_dev *pdev = ctrl->pci_dev; + u16 reg16; + + ctrl_dbg(ctrl, "Slot Capabilities : 0x%08x\n", ctrl->slot_cap); + pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, ®16); + ctrl_dbg(ctrl, "Slot Status : 0x%04x\n", reg16); + pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, ®16); + ctrl_dbg(ctrl, "Slot Control : 0x%04x\n", reg16); +} + +#define FLAG(x, y) (((x) & (y)) ? '+' : '-') + +struct controller *sunwayhpc_init(struct pci_dev *dev) +{ + struct controller *ctrl; + u32 slot_cap, slot_cap2, link_cap; + u8 poweron; + struct pci_bus *subordinate = dev->subordinate; + + ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); + if (!ctrl) + return NULL; + + ctrl->pci_dev = dev; + pcie_capability_read_dword(dev, PCI_EXP_SLTCAP, &slot_cap); + + if (dev->hotplug_user_indicators) + slot_cap &= ~(PCI_EXP_SLTCAP_AIP | PCI_EXP_SLTCAP_PIP); + + /* + * We assume no Thunderbolt controllers support Command Complete events, + * but some controllers falsely claim they do. + */ + if (dev->is_thunderbolt) + slot_cap |= PCI_EXP_SLTCAP_NCCS; + + ctrl->slot_cap = slot_cap; + mutex_init(&ctrl->ctrl_lock); + mutex_init(&ctrl->state_lock); + init_rwsem(&ctrl->reset_lock); + init_waitqueue_head(&ctrl->requester); + init_waitqueue_head(&ctrl->queue); + INIT_DELAYED_WORK(&ctrl->button_work, sunway_pciehp_queue_pushbutton_work); + dbg_ctrl(ctrl); + + down_read(&pci_bus_sem); + ctrl->state = list_empty(&subordinate->devices) ? OFF_STATE : ON_STATE; + up_read(&pci_bus_sem); + + pcie_capability_read_dword(dev, PCI_EXP_SLTCAP2, &slot_cap2); + if (slot_cap2 & PCI_EXP_SLTCAP2_IBPD) { + pcie_write_cmd_nowait(ctrl, PCI_EXP_SLTCTL_IBPD_DISABLE, + PCI_EXP_SLTCTL_IBPD_DISABLE); + ctrl->inband_presence_disabled = 1; + } + + if (dmi_first_match(inband_presence_disabled_dmi_table)) + ctrl->inband_presence_disabled = 1; + + /* Check if Data Link Layer Link Active Reporting is implemented */ + pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &link_cap); + + /* Clear all remaining event bits in Slot Status register. */ + pcie_capability_write_word(dev, PCI_EXP_SLTSTA, + PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD | + PCI_EXP_SLTSTA_MRLSC | PCI_EXP_SLTSTA_CC | + PCI_EXP_SLTSTA_DLLSC | PCI_EXP_SLTSTA_PDC); + + ctrl_info(ctrl, "Slot #%d AttnBtn%c PwrCtrl%c MRL%c AttnInd%c PwrInd%c HotPlug%c Surprise%c Interlock%c NoCompl%c IbPresDis%c LLActRep%c\n", + (slot_cap & PCI_EXP_SLTCAP_PSN) >> 19, + FLAG(slot_cap, PCI_EXP_SLTCAP_ABP), + FLAG(slot_cap, PCI_EXP_SLTCAP_PCP), + FLAG(slot_cap, PCI_EXP_SLTCAP_MRLSP), + FLAG(slot_cap, PCI_EXP_SLTCAP_AIP), + FLAG(slot_cap, PCI_EXP_SLTCAP_PIP), + FLAG(slot_cap, PCI_EXP_SLTCAP_HPC), + FLAG(slot_cap, PCI_EXP_SLTCAP_HPS), + FLAG(slot_cap, PCI_EXP_SLTCAP_EIP), + FLAG(slot_cap, PCI_EXP_SLTCAP_NCCS), + FLAG(slot_cap2, PCI_EXP_SLTCAP2_IBPD), + FLAG(link_cap, PCI_EXP_LNKCAP_DLLLARC)); + + /* + * If empty slot's power status is on, turn power off. The IRQ isn't + * requested yet, so avoid triggering a notification with this command. + */ + if (POWER_CTRL(ctrl)) { + sunway_pciehp_get_power_status(ctrl, &poweron); + if (!sunway_pciehp_card_present_or_link_active(ctrl) && poweron) { + pcie_disable_notification(ctrl); + sunway_pciehp_power_off_slot(ctrl); + } + } + + return ctrl; +} + +void sunway_pciehp_release_ctrl(struct controller *ctrl) +{ + cancel_delayed_work_sync(&ctrl->button_work); + kfree(ctrl); +} diff --git a/drivers/pci/hotplug/sunway_pciehp_pci.c b/drivers/pci/hotplug/sunway_pciehp_pci.c new file mode 100644 index 000000000000..f627ea054342 --- /dev/null +++ b/drivers/pci/hotplug/sunway_pciehp_pci.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Sunway PCI Express Hot Plug Controller Driver + */ + +#define dev_fmt(fmt) "sunway_pciehp: " fmt + +#include +#include +#include +#include +#include "../pci.h" +#include "sunway_pciehp.h" + +/** + * pciehp_configure_device() - enumerate PCI devices below a hotplug bridge + * @ctrl: PCIe hotplug controller + * + * Enumerate PCI devices below a hotplug bridge and add them to the system. + * Return 0 on success, %-EEXIST if the devices are already enumerated or + * %-ENODEV if enumeration failed. + */ +int sunway_pciehp_configure_device(struct controller *ctrl) +{ + struct pci_dev *dev; + struct pci_dev *bridge = ctrl->pci_dev; + struct pci_bus *parent = bridge->subordinate; + int num, ret = 0; + + pci_lock_rescan_remove(); + + dev = pci_get_slot(parent, PCI_DEVFN(0, 0)); + if (dev) { + /* + * The device is already there. Either configured by the + * boot firmware or a previous hotplug event. + */ + ctrl_dbg(ctrl, "Device %s already exists at %04x:%02x:00, skipping hot-add\n", + pci_name(dev), pci_domain_nr(parent), parent->number); + pci_dev_put(dev); + ret = -EEXIST; + goto out; + } + + num = pci_scan_slot(parent, PCI_DEVFN(0, 0)); + if (num == 0) { + ctrl_err(ctrl, "No new device found\n"); + ret = -ENODEV; + goto out; + } + + for_each_pci_bridge(dev, parent) + pci_hp_add_bridge(dev); + + pci_assign_unassigned_bridge_resources(bridge); + pcie_bus_configure_settings(parent); + pci_bus_add_devices(parent); + + out: + pci_unlock_rescan_remove(); + return ret; +} + +/** + * pciehp_unconfigure_device() - remove PCI devices below a hotplug bridge + * @ctrl: PCIe hotplug controller + * @presence: whether the card is still present in the slot; + * true for safe removal via sysfs or an Attention Button press, + * false for surprise removal + * + * Unbind PCI devices below a hotplug bridge from their drivers and remove + * them from the system. Safely removed devices are quiesced. Surprise + * removed devices are marked as such to prevent further accesses. + */ +void sunway_pciehp_unconfigure_device(struct controller *ctrl, bool presence) +{ + struct pci_dev *dev, *temp; + struct pci_bus *parent = ctrl->pci_dev->subordinate; + u16 command; + + ctrl_dbg(ctrl, "%s: domain:bus:dev = %04x:%02x:00\n", + __func__, pci_domain_nr(parent), parent->number); + + if (!presence) + pci_walk_bus(parent, pci_dev_set_disconnected, NULL); + + pci_lock_rescan_remove(); + + /* + * Stopping an SR-IOV PF device removes all the associated VFs, + * which will update the bus->devices list and confuse the + * iterator. Therefore, iterate in reverse so we remove the VFs + * first, then the PF. We do the same in pci_stop_bus_device(). + */ + list_for_each_entry_safe_reverse(dev, temp, &parent->devices, + bus_list) { + pci_dev_get(dev); + pci_stop_and_remove_bus_device(dev); + /* + * Ensure that no new Requests will be generated from + * the device. + */ + if (presence) { + pci_read_config_word(dev, PCI_COMMAND, &command); + command &= ~(PCI_COMMAND_MASTER | PCI_COMMAND_SERR); + command |= PCI_COMMAND_INTX_DISABLE; + pci_write_config_word(dev, PCI_COMMAND, command); + } + pci_dev_put(dev); + } + + pci_unlock_rescan_remove(); +} -- Gitee From ee8f133418e8d96a61250ac982de2b024ca0a059 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Tue, 14 May 2024 09:43:02 +0000 Subject: [PATCH 196/524] sw64: gpu: memset_io and memcpy_toio/fromio for iomem on AMD swsmu This commit complements commit cb7b5777c70b ("sw64: gpu: correct low-level mmio memset/memcpy direct calls"). Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 12618a583e97..5aa0bdbde7bf 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -876,7 +876,11 @@ int smu_cmn_update_table(struct smu_context *smu, table_size = smu_table->tables[table_index].size; if (drv2smu) { +#if IS_ENABLED(CONFIG_SW64) + memcpy_toio(table->cpu_addr, table_data, table_size); +#else memcpy(table->cpu_addr, table_data, table_size); +#endif /* * Flush hdp cache: to guard the content seen by * GPU is consitent with CPU. @@ -894,7 +898,11 @@ int smu_cmn_update_table(struct smu_context *smu, if (!drv2smu) { amdgpu_asic_invalidate_hdp(adev, NULL); +#if IS_ENABLED(CONFIG_SW64) + memcpy_fromio(table_data, table->cpu_addr, table_size); +#else memcpy(table_data, table->cpu_addr, table_size); +#endif } return 0; @@ -950,7 +958,11 @@ int smu_cmn_get_metrics_table(struct smu_context *smu, } if (metrics_table) +#if IS_ENABLED(CONFIG_SW64) + memcpy_toio(metrics_table, smu_table->metrics_table, table_size); +#else memcpy(metrics_table, smu_table->metrics_table, table_size); +#endif return 0; } -- Gitee From 6cfa9a9534a51d177f2334462bddebb49351a7cf Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Wed, 15 May 2024 16:07:57 +0000 Subject: [PATCH 197/524] sw64: pci: use readq/writeq to read/write RC and PIU IO registers Because kernel get all register base addresses of PCIe controller from firmware, use the base addresses to instead of macro. Remove functions read/write_rc_conf, read/write_piu_ior0 and read/write_piu_ior1, and use unified functions readq/writeq to access all PCI related IO registers. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/msi.h | 3 +- arch/sw_64/include/asm/sw64_init.h | 6 +- arch/sw_64/include/asm/sw64io.h | 60 -------------- arch/sw_64/kernel/chip_setup.c | 91 ++++++++++---------- arch/sw_64/kernel/reset.c | 10 +-- arch/sw_64/pci/pci-legacy.c | 10 +-- arch/sw_64/pci/pci.c | 15 ++-- drivers/irqchip/irq-sunway-cpu.c | 24 +++--- drivers/irqchip/irq-sunway-msi.c | 9 +- drivers/irqchip/irq-sunway-pci-intx.c | 71 ++++++++-------- drivers/pci/controller/pci-sunway.c | 101 ++++++++++++----------- drivers/pci/hotplug/sunway_pciehp_ctrl.c | 86 +++++++++---------- drivers/pci/hotplug/sunway_pciehp_hpc.c | 25 +++--- 13 files changed, 228 insertions(+), 283 deletions(-) diff --git a/arch/sw_64/include/asm/msi.h b/arch/sw_64/include/asm/msi.h index dbf6f81843be..e5d93040b2d3 100644 --- a/arch/sw_64/include/asm/msi.h +++ b/arch/sw_64/include/asm/msi.h @@ -48,8 +48,7 @@ struct sw64_msi_chip_data { unsigned long msi_config; unsigned long msiaddr; }; - unsigned long rc_node; - unsigned long rc_index; + struct pci_controller *hose; unsigned int msi_config_index; unsigned int dst_cpu; unsigned int vector; diff --git a/arch/sw_64/include/asm/sw64_init.h b/arch/sw_64/include/asm/sw64_init.h index 546be1a35250..2d553242487d 100644 --- a/arch/sw_64/include/asm/sw64_init.h +++ b/arch/sw_64/include/asm/sw64_init.h @@ -17,10 +17,8 @@ struct sw64_pci_init_ops { int (*map_irq)(const struct pci_dev *dev, u8 slot, u8 pin); unsigned long (*get_rc_enable)(unsigned long node); void (*hose_init)(struct pci_controller *hose); - void (*set_rc_piu)(unsigned long node, unsigned long index); - int (*check_pci_linkup)(unsigned long node, unsigned long index); - void (*set_intx)(unsigned long node, unsigned long index, - unsigned long int_conf); + void (*set_rc_piu)(struct pci_controller *hose); + int (*check_pci_linkup)(struct pci_controller *hose); }; diff --git a/arch/sw_64/include/asm/sw64io.h b/arch/sw_64/include/asm/sw64io.h index d52cd8cc86bf..e9c4a3be95ef 100644 --- a/arch/sw_64/include/asm/sw64io.h +++ b/arch/sw_64/include/asm/sw64io.h @@ -20,66 +20,6 @@ #define MK_PIU_IOR1(nid, idx) \ (SW64_PCI_IO_BASE((nid), (idx)) | PCI_IOR1_BASE) -static inline unsigned int -read_rc_conf(unsigned long node, unsigned long rc, - unsigned int offset) -{ - void __iomem *addr; - - addr = __va(MK_RC_CFG(node, rc) | offset); - return readl(addr); -} - -static inline void -write_rc_conf(unsigned long node, unsigned long rc, - unsigned int offset, unsigned int data) -{ - void __iomem *addr; - - addr = __va(MK_RC_CFG(node, rc) | offset); - writel(data, addr); -} - -static inline unsigned long -read_piu_ior0(unsigned long node, unsigned long rc, - unsigned int reg) -{ - void __iomem *addr; - - addr = __va(MK_PIU_IOR0(node, rc) + reg); - return readq(addr); -} - -static inline void -write_piu_ior0(unsigned long node, unsigned long rc, - unsigned int reg, unsigned long data) -{ - void __iomem *addr; - - addr = __va(MK_PIU_IOR0(node, rc) + reg); - writeq(data, addr); -} - -static inline unsigned long -read_piu_ior1(unsigned long node, unsigned long rc, - unsigned int reg) -{ - void __iomem *addr; - - addr = __va(MK_PIU_IOR1(node, rc) + reg); - return readq(addr); -} - -static inline void -write_piu_ior1(unsigned long node, unsigned long rc, - unsigned int reg, unsigned long data) -{ - void __iomem *addr; - - addr = __va(MK_PIU_IOR1(node, rc) + reg); - writeq(data, addr); -} - static inline unsigned long sw64_io_read(unsigned long node, unsigned long reg) { diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index 60373429a64e..c6374ae18138 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -78,38 +78,39 @@ static void pcie_save(void) { struct pci_controller *hose; struct piu_saved *piu_save; - unsigned long node, index; unsigned long i; + void __iomem *piu_ior0_base; + void __iomem *piu_ior1_base; for (hose = hose_head; hose; hose = hose->next) { - piu_save = kzalloc(sizeof(*piu_save), GFP_KERNEL); + piu_ior0_base = hose->piu_ior0_base; + piu_ior1_base = hose->piu_ior1_base; - node = hose->node; - index = hose->index; + piu_save = kzalloc(sizeof(*piu_save), GFP_KERNEL); hose->sysdata = piu_save; - piu_save->piuconfig0 = read_piu_ior0(node, index, PIUCONFIG0); - piu_save->piuconfig1 = read_piu_ior1(node, index, PIUCONFIG1); - piu_save->epdmabar = read_piu_ior0(node, index, EPDMABAR); - piu_save->msiaddr = read_piu_ior0(node, index, MSIADDR); + piu_save->piuconfig0 = readq(piu_ior0_base + PIUCONFIG0); + piu_save->piuconfig1 = readq(piu_ior1_base + PIUCONFIG1); + piu_save->epdmabar = readq(piu_ior0_base + EPDMABAR); + piu_save->msiaddr = readq(piu_ior1_base + MSIADDR); if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { for (i = 0; i < 256; i++) { - piu_save->msiconfig[i] = read_piu_ior0(node, index, - MSICONFIG0 + (i << 7)); + piu_save->msiconfig[i] = + readq(piu_ior0_base + MSICONFIG0 + (i << 7)); } } - piu_save->iommuexcpt_ctrl = read_piu_ior0(node, index, IOMMUEXCPT_CTRL); - piu_save->dtbaseaddr = read_piu_ior0(node, index, DTBASEADDR); + piu_save->iommuexcpt_ctrl = readq(piu_ior0_base + IOMMUEXCPT_CTRL); + piu_save->dtbaseaddr = readq(piu_ior0_base + DTBASEADDR); - piu_save->intaconfig = read_piu_ior0(node, index, INTACONFIG); - piu_save->intbconfig = read_piu_ior0(node, index, INTBCONFIG); - piu_save->intcconfig = read_piu_ior0(node, index, INTCCONFIG); - piu_save->intdconfig = read_piu_ior0(node, index, INTDCONFIG); - piu_save->pmeintconfig = read_piu_ior0(node, index, PMEINTCONFIG); - piu_save->aererrintconfig = read_piu_ior0(node, index, AERERRINTCONFIG); - piu_save->hpintconfig = read_piu_ior0(node, index, HPINTCONFIG); + piu_save->intaconfig = readq(piu_ior0_base + INTACONFIG); + piu_save->intbconfig = readq(piu_ior0_base + INTBCONFIG); + piu_save->intcconfig = readq(piu_ior0_base + INTCCONFIG); + piu_save->intdconfig = readq(piu_ior0_base + INTDCONFIG); + piu_save->pmeintconfig = readq(piu_ior0_base + PMEINTCONFIG); + piu_save->aererrintconfig = readq(piu_ior0_base + AERERRINTCONFIG); + piu_save->hpintconfig = readq(piu_ior0_base + HPINTCONFIG); } } @@ -118,53 +119,57 @@ static void pcie_restore(void) { struct pci_controller *hose; struct piu_saved *piu_save; - unsigned long node, index; u32 rc_misc_ctrl; unsigned int value; unsigned long i; + void __iomem *rc_config_space_base; + void __iomem *piu_ior0_base; + void __iomem *piu_ior1_base; for (hose = hose_head; hose; hose = hose->next) { - node = hose->node; - index = hose->index; + rc_config_space_base = hose->rc_config_space_base; + piu_ior0_base = hose->piu_ior0_base; + piu_ior1_base = hose->piu_ior1_base; piu_save = hose->sysdata; - write_piu_ior0(node, index, PIUCONFIG0, piu_save->piuconfig0); - write_piu_ior1(node, index, PIUCONFIG1, piu_save->piuconfig1); - write_piu_ior0(node, index, EPDMABAR, piu_save->epdmabar); - write_piu_ior0(node, index, MSIADDR, piu_save->msiaddr); + writeq(piu_save->piuconfig0, (piu_ior0_base + PIUCONFIG0)); + writeq(piu_save->piuconfig1, (piu_ior1_base + PIUCONFIG1)); + writeq(piu_save->epdmabar, (piu_ior0_base + EPDMABAR)); + writeq(piu_save->msiaddr, (piu_ior0_base + MSIADDR)); + if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { for (i = 0; i < 256; i++) { - write_piu_ior0(node, index, MSICONFIG0 + (i << 7), - piu_save->msiconfig[i]); + writeq(piu_save->msiconfig[i], + (piu_ior0_base + (MSICONFIG0 + (i << 7)))); } } - write_piu_ior0(node, index, IOMMUEXCPT_CTRL, piu_save->iommuexcpt_ctrl); - write_piu_ior0(node, index, DTBASEADDR, piu_save->dtbaseaddr); + writeq(piu_save->iommuexcpt_ctrl, (piu_ior0_base + IOMMUEXCPT_CTRL)); + writeq(piu_save->dtbaseaddr, (piu_ior0_base + DTBASEADDR)); - write_piu_ior0(node, index, INTACONFIG, piu_save->intaconfig); - write_piu_ior0(node, index, INTBCONFIG, piu_save->intbconfig); - write_piu_ior0(node, index, INTCCONFIG, piu_save->intcconfig); - write_piu_ior0(node, index, INTDCONFIG, piu_save->intdconfig); - write_piu_ior0(node, index, PMEINTCONFIG, piu_save->pmeintconfig); - write_piu_ior0(node, index, AERERRINTCONFIG, piu_save->aererrintconfig); - write_piu_ior0(node, index, HPINTCONFIG, piu_save->hpintconfig); + writeq(piu_save->intaconfig, (piu_ior0_base + INTACONFIG)); + writeq(piu_save->intbconfig, (piu_ior0_base + INTBCONFIG)); + writeq(piu_save->intcconfig, (piu_ior0_base + INTCCONFIG)); + writeq(piu_save->intdconfig, (piu_ior0_base + INTDCONFIG)); + writeq(piu_save->pmeintconfig, (piu_ior0_base + PMEINTCONFIG)); + writeq(piu_save->aererrintconfig, (piu_ior0_base + AERERRINTCONFIG)); + writeq(piu_save->hpintconfig, (piu_ior0_base + HPINTCONFIG)); /* Enable DBI_RO_WR_EN */ - rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); - write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); + rc_misc_ctrl = readl(rc_config_space_base + RC_MISC_CONTROL_1); + writel((rc_misc_ctrl | 0x1), (rc_config_space_base + RC_MISC_CONTROL_1)); /* Fix up DEVICE_ID_VENDOR_ID register */ value = (PCI_DEVICE_ID_SW64_ROOT_BRIDGE << 16) | PCI_VENDOR_ID_JN; - write_rc_conf(node, index, RC_VENDOR_ID, value); + writel(value, (rc_config_space_base + RC_VENDOR_ID)); /* Set PCI-E root class code */ - value = read_rc_conf(node, index, RC_REVISION_ID); - write_rc_conf(node, index, RC_REVISION_ID, (PCI_CLASS_BRIDGE_HOST << 16) | value); + value = readl(rc_config_space_base + RC_REVISION_ID); + writel((PCI_CLASS_BRIDGE_HOST << 16) | value, (rc_config_space_base + RC_REVISION_ID)); /* Disable DBI_RO_WR_EN */ - write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); + writel(rc_misc_ctrl, (rc_config_space_base + RC_MISC_CONTROL_1)); } } diff --git a/arch/sw_64/kernel/reset.c b/arch/sw_64/kernel/reset.c index 955339557a7a..3f1961ce85de 100644 --- a/arch/sw_64/kernel/reset.c +++ b/arch/sw_64/kernel/reset.c @@ -27,12 +27,10 @@ void fix_jm585_reset(void) 0x0585, NULL); if (pdev) { hose = pci_bus_to_pci_controller(pdev->bus); - val = read_rc_conf(hose->node, hose->index, - RC_PORT_LINK_CTL); - write_rc_conf(hose->node, hose->index, - RC_PORT_LINK_CTL, val | 0x8); - write_rc_conf(hose->node, hose->index, - RC_PORT_LINK_CTL, val); + val = readl(hose->rc_config_space_base + RC_PORT_LINK_CTL); + writel((val | 0x8), (hose->rc_config_space_base + RC_PORT_LINK_CTL)); + writel(val, (hose->rc_config_space_base + RC_PORT_LINK_CTL)); + } } diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index c968b3543dbe..52ba53a1fc0d 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -84,7 +84,7 @@ void __init common_init_pci(void) continue; hose->busn_space->start = last_bus; init_busnr = (0xff << 16) + ((last_bus + 1) << 8) + (last_bus); - write_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS, init_busnr); + writel(init_busnr, (hose->rc_config_space_base + RC_PRIMARY_BUS)); offset = hose->mem_space->start - PCI_32BIT_MEMIO; if (is_in_host()) hose->first_busno = last_bus + 1; @@ -117,10 +117,10 @@ void __init common_init_pci(void) last_bus++; hose->last_busno = hose->busn_space->end = last_bus; - init_busnr = read_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS); + init_busnr = readl(hose->rc_config_space_base + RC_PRIMARY_BUS); init_busnr &= ~(0xff << 16); init_busnr |= last_bus << 16; - write_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS, init_busnr); + writel(init_busnr, (hose->rc_config_space_base + RC_PRIMARY_BUS)); pci_bus_update_busn_res_end(bus, last_bus); last_bus++; } @@ -228,9 +228,9 @@ sw64_init_host(unsigned long node, unsigned long index) sw64_chip_init->pci_init.hose_init(hose); if (sw64_chip_init->pci_init.set_rc_piu) - sw64_chip_init->pci_init.set_rc_piu(node, index); + sw64_chip_init->pci_init.set_rc_piu(hose); - ret = sw64_chip_init->pci_init.check_pci_linkup(node, index); + ret = sw64_chip_init->pci_init.check_pci_linkup(hose); if (ret == 0) { /* Root Complex downstream port is link up */ pci_mark_rc_linkup(node, index); // 8-bit per node diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index 77dbd2f4bf37..073781645f32 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -249,23 +249,22 @@ static void enable_sw_dca(struct pci_dev *dev) rc_index = hose->index; for (i = 0; i < 256; i++) { - dca_conf = read_piu_ior1(node, rc_index, DEVICEID0 + (i << 7)); + dca_conf = readq(hose->piu_ior1_base + DEVICEID0 + (i << 7)); if (dca_conf >> 63) continue; else { dca_conf = (1UL << 63) | (dev->bus->number << 8) | dev->devfn; pr_info("dca device index %d, dca_conf = %#lx\n", i, dca_conf); - write_piu_ior1(node, rc_index, - DEVICEID0 + (i << 7), dca_conf); + writeq(dca_conf, (hose->piu_ior1_base + DEVICEID0 + (i << 7))); break; } } - dca_ctl = read_piu_ior1(node, rc_index, DCACONTROL); + dca_ctl = readq(hose->piu_ior1_base + DCACONTROL); if (dca_ctl & 0x1) { dca_ctl = 0x2; - write_piu_ior1(node, rc_index, DCACONTROL, dca_ctl); + writeq(dca_ctl, (hose->piu_ior1_base + DCACONTROL)); pr_info("Node %ld RC %ld enable DCA 1.0\n", node, rc_index); } } @@ -342,7 +341,7 @@ void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge) bridge->map_irq = sw64_pci_map_irq; init_busnr = (0xff << 16) + ((last_bus + 1) << 8) + (last_bus); - write_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS, init_busnr); + writel(init_busnr, (hose->rc_config_space_base + RC_PRIMARY_BUS)); hose->first_busno = last_bus + (is_in_host() ? 1 : 0); @@ -400,10 +399,10 @@ void sw64_pci_root_bridge_scan_finish_up(struct pci_host_bridge *bridge) hose->last_busno = last_bus; hose->busn_space->end = last_bus; - init_busnr = read_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS); + init_busnr = readl(hose->rc_config_space_base + RC_PRIMARY_BUS); init_busnr &= ~(0xff << 16); init_busnr |= last_bus << 16; - write_rc_conf(hose->node, hose->index, RC_PRIMARY_BUS, init_busnr); + writel(init_busnr, (hose->rc_config_space_base + RC_PRIMARY_BUS)); pci_bus_update_busn_res_end(bus, last_bus); last_bus++; diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 250ddbd61cf4..23ad3b16af08 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -43,45 +43,49 @@ static void handle_intx(unsigned int offset) { struct pci_controller *hose; unsigned long value; + void __iomem *piu_ior0_base; hose = hose_head; + offset <<= 7; for (hose = hose_head; hose; hose = hose->next) { - value = read_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7)); + piu_ior0_base = hose->piu_ior0_base; + + value = readq(piu_ior0_base + INTACONFIG + offset); if (value >> 63) { value = value & (~(1UL << 62)); - write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); + writeq(value, (piu_ior0_base + INTACONFIG + offset)); handle_irq(hose->int_irq); value = value | (1UL << 62); - write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); + writeq(value, (piu_ior0_base + INTACONFIG + offset)); } if (IS_ENABLED(CONFIG_PCIE_PME)) { - value = read_piu_ior0(hose->node, hose->index, PMEINTCONFIG); + value = readq(piu_ior0_base + PMEINTCONFIG); if (value >> 63) { handle_irq(hose->service_irq); - write_piu_ior0(hose->node, hose->index, PMEINTCONFIG, value); + writeq(value, (piu_ior0_base + PMEINTCONFIG)); } } if (IS_ENABLED(CONFIG_PCIEAER)) { - value = read_piu_ior0(hose->node, hose->index, AERERRINTCONFIG); + value = readq(piu_ior0_base + AERERRINTCONFIG); if (value >> 63) { handle_irq(hose->service_irq); - write_piu_ior0(hose->node, hose->index, AERERRINTCONFIG, value); + writeq(value, (piu_ior0_base + AERERRINTCONFIG)); } } if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE_SUNWAY)) { - value = read_piu_ior0(hose->node, hose->index, HPINTCONFIG); + value = readq(piu_ior0_base + HPINTCONFIG); if (value >> 63) { handle_irq(hose->service_irq); - write_piu_ior0(hose->node, hose->index, HPINTCONFIG, value); + writeq(value, (piu_ior0_base + HPINTCONFIG)); } } if (hose->iommu_enable) { - value = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); + value = readq(piu_ior0_base + IOMMUEXCPT_STATUS); if (value >> 63) handle_irq(hose->int_irq); } diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index c231e504464b..6de689e5106f 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -93,8 +93,8 @@ static unsigned long set_piu_msi_config(struct pci_controller *hose, int cpu, phy_cpu = cpu_to_rcid(cpu); msi_config |= ((phy_cpu >> 5) << 6) | (phy_cpu & 0x1f); reg = MSICONFIG0 + (unsigned long)(msiconf_index << 7); - write_piu_ior0(hose->node, hose->index, reg, msi_config); - msi_config = read_piu_ior0(hose->node, hose->index, reg); + writeq(msi_config, (hose->piu_ior0_base + reg)); + msi_config = readq(hose->piu_ior0_base + reg); set_bit(msiconf_index, hose->piu_msiconfig); return msi_config; @@ -236,8 +236,7 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, cdata->dst_cpu = cpu; cdata->vector = vector; - cdata->rc_index = hose->index; - cdata->rc_node = hose->node; + cdata->hose = hose; cdata->msi_config = msi_config; cdata->msi_config_index = msiconf_index; cdata->prev_cpu = cpu; @@ -463,7 +462,7 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned irq_move_complete(cdata, cpu, vector_index + msi_index); piu_index = cdata->msi_config_index; value = cdata->msi_config | (1UL << 63); - write_piu_ior0(cdata->rc_node, cdata->rc_index, MSICONFIG0 + (piu_index << 7), value); + writeq(value, (cdata->hose->piu_ior0_base + MSICONFIG0 + (piu_index << 7))); spin_unlock(&cdata->cdata_lock); handle_irq(irq); diff --git a/drivers/irqchip/irq-sunway-pci-intx.c b/drivers/irqchip/irq-sunway-pci-intx.c index 93ee2361e958..480335044377 100644 --- a/drivers/irqchip/irq-sunway-pci-intx.c +++ b/drivers/irqchip/irq-sunway-pci-intx.c @@ -19,33 +19,33 @@ static void unlock_legacy_lock(void) raw_spin_unlock(&legacy_lock); } -static void set_intx(unsigned long node, unsigned long index, unsigned long intx_conf) +static void set_intx(struct pci_controller *hose, unsigned long intx_conf) { + void __iomem *piu_ior0_base; + if (is_guest_or_emul()) return; - write_piu_ior0(node, index, INTACONFIG, intx_conf | (0x8UL << 10)); - write_piu_ior0(node, index, INTBCONFIG, intx_conf | (0x4UL << 10)); - write_piu_ior0(node, index, INTCCONFIG, intx_conf | (0x2UL << 10)); - write_piu_ior0(node, index, INTDCONFIG, intx_conf | (0x1UL << 10)); + piu_ior0_base = hose->piu_ior0_base; + + writeq(intx_conf | (0x8UL << 10), (piu_ior0_base + INTACONFIG)); + writeq(intx_conf | (0x4UL << 10), (piu_ior0_base + INTBCONFIG)); + writeq(intx_conf | (0x2UL << 10), (piu_ior0_base + INTCCONFIG)); + writeq(intx_conf | (0x1UL << 10), (piu_ior0_base + INTDCONFIG)); } static int __assign_piu_intx_config(struct pci_controller *hose, cpumask_t *targets) { unsigned long intx_conf; unsigned int cpu; - int node, index; int phy_cpu; - node = hose->node; - index = hose->index; - /* Use the last cpu in valid cpus to avoid core 0. */ cpu = cpumask_last(targets); phy_cpu = cpu_to_rcid(cpu); intx_conf = ((phy_cpu >> 5) << 6) | (phy_cpu & 0x1f); - set_intx(node, index, intx_conf); + set_intx(hose, intx_conf); return 0; } @@ -64,59 +64,59 @@ static int assign_piu_intx_config(struct pci_controller *hose, cpumask_t *target static void intx_irq_enable(struct irq_data *irq_data) { struct pci_controller *hose = irq_data->chip_data; - unsigned long intx_conf, node, index; + unsigned long intx_conf; + void __iomem *piu_ior0_base; if (is_guest_or_emul()) return; BUG_ON(!hose); - node = hose->node; - index = hose->index; + piu_ior0_base = hose->piu_ior0_base; - intx_conf = read_piu_ior0(node, index, INTACONFIG); + intx_conf = readq(piu_ior0_base + INTACONFIG); intx_conf |= PCI_INTX_ENABLE; - write_piu_ior0(node, index, INTACONFIG, intx_conf); + writeq(intx_conf, (piu_ior0_base + INTACONFIG)); - intx_conf = read_piu_ior0(node, index, INTBCONFIG); + intx_conf = readq(piu_ior0_base + INTBCONFIG); intx_conf |= PCI_INTX_ENABLE; - write_piu_ior0(node, index, INTBCONFIG, intx_conf); + writeq(intx_conf, (piu_ior0_base + INTBCONFIG)); - intx_conf = read_piu_ior0(node, index, INTCCONFIG); + intx_conf = readq(piu_ior0_base + INTCCONFIG); intx_conf |= PCI_INTX_ENABLE; - write_piu_ior0(node, index, INTCCONFIG, intx_conf); + writeq(intx_conf, (piu_ior0_base + INTCCONFIG)); - intx_conf = read_piu_ior0(node, index, INTDCONFIG); + intx_conf = readq(piu_ior0_base + INTDCONFIG); intx_conf |= PCI_INTX_ENABLE; - write_piu_ior0(node, index, INTDCONFIG, intx_conf); + writeq(intx_conf, (piu_ior0_base + INTDCONFIG)); } static void intx_irq_disable(struct irq_data *irq_data) { struct pci_controller *hose = irq_data->chip_data; - unsigned long intx_conf, node, index; + unsigned long intx_conf; + void __iomem *piu_ior0_base; if (is_guest_or_emul()) return; BUG_ON(!hose); - node = hose->node; - index = hose->index; + piu_ior0_base = hose->piu_ior0_base; - intx_conf = read_piu_ior0(node, index, INTACONFIG); + intx_conf = readq(piu_ior0_base + INTACONFIG); intx_conf &= PCI_INTX_DISABLE; - write_piu_ior0(node, index, INTACONFIG, intx_conf); + writeq(intx_conf, (piu_ior0_base + INTACONFIG)); - intx_conf = read_piu_ior0(node, index, INTBCONFIG); + intx_conf = readq(piu_ior0_base + INTBCONFIG); intx_conf &= PCI_INTX_DISABLE; - write_piu_ior0(node, index, INTBCONFIG, intx_conf); + writeq(intx_conf, (piu_ior0_base + INTBCONFIG)); - intx_conf = read_piu_ior0(node, index, INTCCONFIG); + intx_conf = readq(piu_ior0_base + INTCCONFIG); intx_conf &= PCI_INTX_DISABLE; - write_piu_ior0(node, index, INTCCONFIG, intx_conf); + writeq(intx_conf, (piu_ior0_base + INTCCONFIG)); - intx_conf = read_piu_ior0(node, index, INTDCONFIG); + intx_conf = readq(piu_ior0_base + INTDCONFIG); intx_conf &= PCI_INTX_DISABLE; - write_piu_ior0(node, index, INTDCONFIG, intx_conf); + writeq(intx_conf, (piu_ior0_base + INTDCONFIG)); } static int intx_set_affinity(struct irq_data *irq_data, @@ -149,14 +149,13 @@ static struct irq_chip sw64_intx_chip = { .flags = IRQCHIP_SKIP_SET_WAKE, }; -void __weak set_pcieport_service_irq(int node, int index) {} +void __weak set_pcieport_service_irq(struct pci_controller *hose) {} void setup_intx_irqs(struct pci_controller *hose) { - unsigned long irq, node, index, val_node; + unsigned long irq, node, val_node; node = hose->node; - index = hose->index; if (!node_online(node)) val_node = next_node_in(node, node_online_map); @@ -172,7 +171,7 @@ void setup_intx_irqs(struct pci_controller *hose) irq_set_chip_and_handler(irq + 1, &dummy_irq_chip, handle_level_irq); hose->service_irq = irq + 1; - set_pcieport_service_irq(node, index); + set_pcieport_service_irq(hose); } void __init sw64_init_irq(void) diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 688793d1b136..6af0020ce1ee 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -24,20 +24,17 @@ void set_adr_int(int node) } #endif -void set_pcieport_service_irq(int node, int index) +void set_pcieport_service_irq(struct pci_controller *hose) { if (IS_ENABLED(CONFIG_PCIE_PME)) - write_piu_ior0(node, index, - PMEINTCONFIG, PME_ENABLE_INTD_CORE0); + writeq(PME_ENABLE_INTD_CORE0, (hose->piu_ior0_base + PMEINTCONFIG)); if (IS_ENABLED(CONFIG_PCIEAER)) - write_piu_ior0(node, index, - AERERRINTCONFIG, AER_ENABLE_INTD_CORE0); + writeq(AER_ENABLE_INTD_CORE0, (hose->piu_ior0_base + AERERRINTCONFIG)); #ifdef CONFIG_UNCORE_JUNZHANG if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE_SUNWAY)) - write_piu_ior0(node, index, - HPINTCONFIG, HP_ENABLE_INTD_CORE0); + writeq(HP_ENABLE_INTD_CORE0, (hose->piu_ior0_base + HPINTCONFIG)); #endif } @@ -59,17 +56,16 @@ int chip_pcie_configure(struct pci_controller *hose) struct list_head *next; unsigned int max_read_size, smallest_max_payload; int max_payloadsize; - unsigned long rc_index, node; unsigned long piuconfig0, value; unsigned int pcie_caps_offset; unsigned int rc_conf_value; u16 devctl, new_values; bool rc_ari_disabled = false, found = false; unsigned char bus_max_num; + void __iomem *rc_config_space_base; - node = hose->node; - rc_index = hose->index; - smallest_max_payload = read_rc_conf(node, rc_index, RC_EXP_DEVCAP); + rc_config_space_base = hose->rc_config_space_base; + smallest_max_payload = readl(rc_config_space_base + RC_EXP_DEVCAP); smallest_max_payload &= PCI_EXP_DEVCAP_PAYLOAD; bus_max_num = hose->busn_space->start; @@ -120,40 +116,40 @@ int chip_pcie_configure(struct pci_controller *hose) } if (rc_ari_disabled) { - rc_conf_value = read_rc_conf(node, rc_index, RC_EXP_DEVCTL2); + rc_conf_value = readl(rc_config_space_base + RC_EXP_DEVCTL2); rc_conf_value &= ~PCI_EXP_DEVCTL2_ARI; - write_rc_conf(node, rc_index, RC_EXP_DEVCTL2, rc_conf_value); + writel(rc_conf_value, (rc_config_space_base + RC_EXP_DEVCTL2)); } else { - rc_conf_value = read_rc_conf(node, rc_index, RC_EXP_DEVCTL2); + rc_conf_value = readl(rc_config_space_base + RC_EXP_DEVCTL2); rc_conf_value |= PCI_EXP_DEVCTL2_ARI; - write_rc_conf(node, rc_index, RC_EXP_DEVCTL2, rc_conf_value); + writel(rc_conf_value, (rc_config_space_base + RC_EXP_DEVCTL2)); } - rc_conf_value = read_rc_conf(node, rc_index, RC_EXP_DEVCAP); + rc_conf_value = readl(rc_config_space_base + RC_EXP_DEVCAP); rc_conf_value &= PCI_EXP_DEVCAP_PAYLOAD; max_payloadsize = rc_conf_value; if (max_payloadsize < smallest_max_payload) smallest_max_payload = max_payloadsize; max_read_size = 0x2; /* Limit to 512B */ - value = read_rc_conf(node, rc_index, RC_EXP_DEVCTL); + value = readl(rc_config_space_base + RC_EXP_DEVCTL); value &= ~(PCI_EXP_DEVCTL_PAYLOAD | PCI_EXP_DEVCTL_READRQ); value |= (max_read_size << 12) | (smallest_max_payload << 5); - write_rc_conf(node, rc_index, RC_EXP_DEVCTL, value); + writel(value, (rc_config_space_base + RC_EXP_DEVCTL)); new_values = (max_read_size << 12) | (smallest_max_payload << 5); - piuconfig0 = read_piu_ior0(node, rc_index, PIUCONFIG0); + piuconfig0 = readq(hose->piu_ior0_base + PIUCONFIG0); piuconfig0 &= ~(0x7fUL << 9); if (smallest_max_payload == 0x2) { piuconfig0 |= (0x20UL << 9); - write_piu_ior0(node, rc_index, PIUCONFIG0, piuconfig0); + writeq(piuconfig0, (hose->piu_ior0_base + PIUCONFIG0)); } else { piuconfig0 |= (0x40UL << 9); - write_piu_ior0(node, rc_index, PIUCONFIG0, piuconfig0); + writeq(piuconfig0, (hose->piu_ior0_base + PIUCONFIG0)); } pr_info("Node%ld RC%ld MPSS %luB, MRRS %luB, Piuconfig0 %#lx, ARI %s\n", - node, rc_index, (1UL << smallest_max_payload) << 7, + hose->node, hose->index, (1UL << smallest_max_payload) << 7, (1UL << max_read_size) << 7, piuconfig0, rc_ari_disabled ? "disabled" : "enabled"); @@ -192,69 +188,74 @@ int chip_pcie_configure(struct pci_controller *hose) return bus_max_num; } -static int check_pci_linkup(unsigned long node, unsigned long index) +static int check_pci_linkup(struct pci_controller *hose) { unsigned long rc_debug; if (is_guest_or_emul()) { - if (node == 0 && index == 0) + if (hose->node == 0 && hose->index == 0) return 0; else return 1; - } else { - rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); } - return !(rc_debug == 0x111); + rc_debug = readq(hose->piu_ior1_base + RCDEBUGINF1); + + return !((rc_debug & 0x3fful) == 0x111); } -static void set_rc_piu(unsigned long node, unsigned long index) +static void set_rc_piu(struct pci_controller *hose) { unsigned int i __maybe_unused; unsigned int value; u32 rc_misc_ctrl; + void __iomem *rc_config_space_base; + void __iomem *piu_ior0_base; + void __iomem *piu_ior1_base; if (is_guest_or_emul()) return; + rc_config_space_base = hose->rc_config_space_base; + piu_ior0_base = hose->piu_ior0_base; + piu_ior1_base = hose->piu_ior1_base; + /* configure RC, set PCI-E root controller */ - write_rc_conf(node, index, RC_COMMAND, 0x00100007); - write_rc_conf(node, index, RC_PORT_LINK_CTL, 0x1f0020); - write_rc_conf(node, index, RC_EXP_DEVCTL, 0x2850); - write_rc_conf(node, index, RC_EXP_DEVCTL2, 0x6); + writel(0x00100007, (rc_config_space_base + RC_COMMAND)); + writel(0x1f0020, (rc_config_space_base + RC_PORT_LINK_CTL)); + writel(0x2850, (rc_config_space_base + RC_EXP_DEVCTL)); + writel(0x6, (rc_config_space_base + RC_EXP_DEVCTL2)); #ifdef CONFIG_UNCORE_XUELANG - write_rc_conf(node, index, RC_ORDER_RULE_CTL, 0x0100); + writel(0x0100, (rc_config_space_base + RC_ORDER_RULE_CTL)); #endif /* enable DBI_RO_WR_EN */ - rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); - write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); + rc_misc_ctrl = readl(rc_config_space_base + RC_MISC_CONTROL_1); + writel(rc_misc_ctrl | 0x1, (rc_config_space_base + RC_MISC_CONTROL_1)); /* fix up DEVICE_ID_VENDOR_ID register */ value = (PCI_DEVICE_ID_SW64_ROOT_BRIDGE << 16) | PCI_VENDOR_ID_JN; - write_rc_conf(node, index, RC_VENDOR_ID, value); + writel(value, (rc_config_space_base + RC_VENDOR_ID)); /* set PCI-E root class code */ - value = read_rc_conf(node, index, RC_REVISION_ID); - write_rc_conf(node, index, RC_REVISION_ID, - (PCI_CLASS_BRIDGE_HOST << 16) | value); + value = readl(rc_config_space_base + RC_REVISION_ID); + writel((PCI_CLASS_BRIDGE_HOST << 16) | value, (rc_config_space_base + RC_REVISION_ID)); /* disable DBI_RO_WR_EN */ - write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); + writel(rc_misc_ctrl, (rc_config_space_base + RC_MISC_CONTROL_1)); - write_rc_conf(node, index, RC_PRIMARY_BUS, 0xffffff); - write_piu_ior0(node, index, PIUCONFIG0, PIUCONFIG0_INIT_VAL); + writeq(PIUCONFIG0_INIT_VAL, (piu_ior0_base + PIUCONFIG0)); - write_piu_ior1(node, index, PIUCONFIG1, 0x2); - write_piu_ior1(node, index, ERRENABLE, -1); + writeq(0x2, (piu_ior1_base + PIUCONFIG1)); + writeq(-1, (piu_ior1_base + ERRENABLE)); /* set DMA offset value PCITODMA_OFFSET */ - write_piu_ior0(node, index, EPDMABAR, PCITODMA_OFFSET); + writeq(PCITODMA_OFFSET, (piu_ior0_base + EPDMABAR)); if (IS_ENABLED(CONFIG_PCI_MSI)) { - write_piu_ior0(node, index, MSIADDR, MSIX_MSG_ADDR); + writeq(MSIX_MSG_ADDR, (piu_ior0_base + MSIADDR)); #ifdef CONFIG_UNCORE_XUELANG for (i = 0; i < 256; i++) - write_piu_ior0(node, index, MSICONFIG0 + (i << 7), 0); + writeq(0, (piu_ior0_base + MSICONFIG0 + (i << 7))); #endif } } @@ -294,6 +295,8 @@ static void hose_init(struct pci_controller *hose) hose->dense_io_base = pci_io_base | PCI_LEGACY_IO; hose->ep_config_space_base = __va(pci_io_base | PCI_EP_CFG); hose->rc_config_space_base = __va(pci_io_base | PCI_RC_CFG); + hose->piu_ior0_base = __va(MK_PIU_IOR0(hose->node, hose->index)); + hose->piu_ior1_base = __va(MK_PIU_IOR1(hose->node, hose->index)); hose->mem_space->start = pci_io_base + PCI_32BIT_MEMIO; hose->mem_space->end = hose->mem_space->start + PCI_32BIT_MEMIO_SIZE - 1; @@ -706,8 +709,8 @@ static int sw64_pci_prepare_controller(struct pci_controller *hose, * 1. Root Complex enable * 2. Root Complex link up */ - set_rc_piu(hose->node, hose->index); - if (check_pci_linkup(hose->node, hose->index)) { + set_rc_piu(hose); + if (check_pci_linkup(hose)) { /** * Root Complex link up failed. * This usually means that no device on the slot. diff --git a/drivers/pci/hotplug/sunway_pciehp_ctrl.c b/drivers/pci/hotplug/sunway_pciehp_ctrl.c index c85ba8e684cc..52e79bfe7dd6 100644 --- a/drivers/pci/hotplug/sunway_pciehp_ctrl.c +++ b/drivers/pci/hotplug/sunway_pciehp_ctrl.c @@ -233,26 +233,25 @@ void sunway_pciehp_save_rc_piu(struct controller *ctrl) { struct pci_bus *bus = ctrl->pci_dev->bus; struct pci_controller *hose = pci_bus_to_pci_controller(bus); - unsigned long node, rc_index; + void __iomem *piu_ior0_base; - node = hose->node; - rc_index = hose->index; + piu_ior0_base = hose->piu_ior0_base; sunway_pciehp_save_config_space(ctrl); if (!ctrl->saved_piu.state_saved) { - ctrl->saved_piu.epdmabar = read_piu_ior0(node, rc_index, EPDMABAR); - ctrl->saved_piu.msiaddr = read_piu_ior0(node, rc_index, MSIADDR); - ctrl->saved_piu.iommuexcpt_ctrl = read_piu_ior0(node, rc_index, IOMMUEXCPT_CTRL); - ctrl->saved_piu.dtbaseaddr = read_piu_ior0(node, rc_index, DTBASEADDR); - - ctrl->saved_piu.intaconfig = read_piu_ior0(node, rc_index, INTACONFIG); - ctrl->saved_piu.intbconfig = read_piu_ior0(node, rc_index, INTBCONFIG); - ctrl->saved_piu.intcconfig = read_piu_ior0(node, rc_index, INTCCONFIG); - ctrl->saved_piu.intdconfig = read_piu_ior0(node, rc_index, INTDCONFIG); - ctrl->saved_piu.pmeintconfig = read_piu_ior0(node, rc_index, PMEINTCONFIG); - ctrl->saved_piu.aererrintconfig = read_piu_ior0(node, rc_index, AERERRINTCONFIG); - ctrl->saved_piu.hpintconfig = read_piu_ior0(node, rc_index, HPINTCONFIG); + ctrl->saved_piu.epdmabar = readq(piu_ior0_base + EPDMABAR); + ctrl->saved_piu.msiaddr = readq(piu_ior0_base + MSIADDR); + ctrl->saved_piu.iommuexcpt_ctrl = readq(piu_ior0_base + IOMMUEXCPT_CTRL); + ctrl->saved_piu.dtbaseaddr = readq(piu_ior0_base + DTBASEADDR); + + ctrl->saved_piu.intaconfig = readq(piu_ior0_base + INTACONFIG); + ctrl->saved_piu.intbconfig = readq(piu_ior0_base + INTBCONFIG); + ctrl->saved_piu.intcconfig = readq(piu_ior0_base + INTCCONFIG); + ctrl->saved_piu.intdconfig = readq(piu_ior0_base + INTDCONFIG); + ctrl->saved_piu.pmeintconfig = readq(piu_ior0_base + PMEINTCONFIG); + ctrl->saved_piu.aererrintconfig = readq(piu_ior0_base + AERERRINTCONFIG); + ctrl->saved_piu.hpintconfig = readq(piu_ior0_base + HPINTCONFIG); ctrl->saved_piu.state_saved = true; } @@ -265,12 +264,13 @@ void sunway_pciehp_start(struct hotplug_slot *hotplug_slot) struct pci_bus *bus = pdev->bus; struct pci_controller *hose = pci_bus_to_pci_controller(bus); unsigned long piu_value; - unsigned long node, rc_index; bool hardware_auto = true; u16 slot_ctrl; + void __iomem *piu_ior0_base; + void __iomem *piu_ior1_base; - node = hose->node; - rc_index = hose->index; + piu_ior0_base = hose->piu_ior0_base; + piu_ior1_base = hose->piu_ior1_base; switch (ctrl->state) { case OFF_STATE: @@ -308,7 +308,7 @@ void sunway_pciehp_start(struct hotplug_slot *hotplug_slot) sunway_pciehp_set_indicators(ctrl, INDICATOR_NOOP, PCI_EXP_SLTCTL_ATTN_IND_BLINK); - write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_INSERT); + writeq(HP_CTRL_INSERT, (piu_ior0_base + HP_CTRL)); } break; case ON_STATE: @@ -338,7 +338,7 @@ void sunway_pciehp_start(struct hotplug_slot *hotplug_slot) sunway_pciehp_link_disable(ctrl); while (1) { - piu_value = read_piu_ior1(node, rc_index, NEWLTSSMSTATE0); + piu_value = readq(piu_ior1_base + NEWLTSSMSTATE0); piu_value &= 0xff; if (piu_value == 0x19) @@ -347,7 +347,7 @@ void sunway_pciehp_start(struct hotplug_slot *hotplug_slot) udelay(10); } - write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_REMOVE); + writeq(HP_CTRL_REMOVE, (piu_ior0_base + HP_CTRL)); sunway_pciehp_request(ctrl, DISABLE_SLOT); } @@ -384,26 +384,25 @@ void sunway_pciehp_restore_rc_piu(struct controller *ctrl) { struct pci_bus *bus = ctrl->pci_dev->bus; struct pci_controller *hose = pci_bus_to_pci_controller(bus); - unsigned long node, rc_index; + void __iomem *piu_ior0_base; - node = hose->node; - rc_index = hose->index; + piu_ior0_base = hose->piu_ior0_base; sunway_pciehp_restore_config_space(ctrl); if (ctrl->saved_piu.state_saved) { - write_piu_ior0(node, rc_index, EPDMABAR, ctrl->saved_piu.epdmabar); - write_piu_ior0(node, rc_index, MSIADDR, ctrl->saved_piu.msiaddr); - write_piu_ior0(node, rc_index, IOMMUEXCPT_CTRL, ctrl->saved_piu.iommuexcpt_ctrl); - write_piu_ior0(node, rc_index, DTBASEADDR, ctrl->saved_piu.dtbaseaddr); - - write_piu_ior0(node, rc_index, INTACONFIG, ctrl->saved_piu.intaconfig); - write_piu_ior0(node, rc_index, INTBCONFIG, ctrl->saved_piu.intbconfig); - write_piu_ior0(node, rc_index, INTCCONFIG, ctrl->saved_piu.intcconfig); - write_piu_ior0(node, rc_index, INTDCONFIG, ctrl->saved_piu.intdconfig); - write_piu_ior0(node, rc_index, PMEINTCONFIG, ctrl->saved_piu.pmeintconfig); - write_piu_ior0(node, rc_index, AERERRINTCONFIG, ctrl->saved_piu.aererrintconfig); - write_piu_ior0(node, rc_index, HPINTCONFIG, ctrl->saved_piu.hpintconfig); + writeq(ctrl->saved_piu.epdmabar, (piu_ior0_base + EPDMABAR)); + writeq(ctrl->saved_piu.msiaddr, (piu_ior0_base + MSIADDR)); + writeq(ctrl->saved_piu.iommuexcpt_ctrl, (piu_ior0_base + IOMMUEXCPT_CTRL)); + writeq(ctrl->saved_piu.dtbaseaddr, (piu_ior0_base + DTBASEADDR)); + + writeq(ctrl->saved_piu.intaconfig, (piu_ior0_base + INTACONFIG)); + writeq(ctrl->saved_piu.intbconfig, (piu_ior0_base + INTBCONFIG)); + writeq(ctrl->saved_piu.intcconfig, (piu_ior0_base + INTCCONFIG)); + writeq(ctrl->saved_piu.intdconfig, (piu_ior0_base + INTDCONFIG)); + writeq(ctrl->saved_piu.pmeintconfig, (piu_ior0_base + PMEINTCONFIG)); + writeq(ctrl->saved_piu.aererrintconfig, (piu_ior0_base + AERERRINTCONFIG)); + writeq(ctrl->saved_piu.hpintconfig, (piu_ior0_base + HPINTCONFIG)); ctrl->saved_piu.state_saved = false; } @@ -415,14 +414,15 @@ void sunway_pciehp_end(struct controller *ctrl, bool insert) struct pci_bus *bus = pdev->bus; struct pci_controller *hose = pci_bus_to_pci_controller(bus); unsigned long piu_value; - unsigned long node, rc_index; u16 slot_ctrl; + void __iomem *piu_ior0_base; + void __iomem *piu_ior1_base; - node = hose->node; - rc_index = hose->index; + piu_ior0_base = hose->piu_ior0_base; + piu_ior1_base = hose->piu_ior1_base; if (insert) { - write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_FINISH); + writeq(HP_CTRL_FINISH, (piu_ior0_base + HP_CTRL)); } else { sunway_pciehp_set_indicators(ctrl, INDICATOR_NOOP, PCI_EXP_SLTCTL_ATTN_IND_OFF); @@ -431,7 +431,7 @@ void sunway_pciehp_end(struct controller *ctrl, bool insert) mdelay(100); while (1) { - piu_value = read_piu_ior1(node, rc_index, NEWLTSSMSTATE0); + piu_value = readq(piu_ior1_base + NEWLTSSMSTATE0); piu_value &= 0xff; if (piu_value == 0x0) @@ -440,7 +440,7 @@ void sunway_pciehp_end(struct controller *ctrl, bool insert) udelay(10); } - write_piu_ior0(node, rc_index, HPINTCONFIG, HP_ENABLE_INTD_CORE0); + writeq(HP_ENABLE_INTD_CORE0, (piu_ior0_base + HPINTCONFIG)); pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); slot_ctrl |= (PCI_EXP_SLTCTL_PFDE | @@ -451,7 +451,7 @@ void sunway_pciehp_end(struct controller *ctrl, bool insert) PCI_EXP_SLTCTL_DLLSCE); pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); - write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_FINISH); + writeq(HP_CTRL_FINISH, (piu_ior0_base + HP_CTRL)); } } diff --git a/drivers/pci/hotplug/sunway_pciehp_hpc.c b/drivers/pci/hotplug/sunway_pciehp_hpc.c index d30ee235b029..bd4556dbc3d4 100644 --- a/drivers/pci/hotplug/sunway_pciehp_hpc.c +++ b/drivers/pci/hotplug/sunway_pciehp_hpc.c @@ -304,9 +304,9 @@ static bool pci_bus_check_linkup(struct pci_controller *hose) bool linkup = false; do { - if (sw64_chip_init->pci_init.check_pci_linkup(hose->node, hose->index) == 0) { + if (sw64_chip_init->pci_init.check_pci_linkup(hose) == 0) { udelay(10); - if (sw64_chip_init->pci_init.check_pci_linkup(hose->node, hose->index) == 0) + if (sw64_chip_init->pci_init.check_pci_linkup(hose) == 0) linkup = true; } count++; @@ -634,16 +634,17 @@ void sunway_pciehp_poll(struct controller *ctrl, u32 events) struct pci_bus *bus = pdev->bus; struct pci_controller *hose = pci_bus_to_pci_controller(bus); unsigned long piu_value; - unsigned long node, rc_index; u16 slot_ctrl; int i; + void __iomem *piu_ior0_base; + void __iomem *piu_ior1_base; - node = hose->node; - rc_index = hose->index; + piu_ior0_base = hose->piu_ior0_base; + piu_ior1_base = hose->piu_ior1_base; if (events & SW64_POLL_DISABLE_SLOT) { while (1) { - piu_value = read_piu_ior1(node, rc_index, NEWLTSSMSTATE0); + piu_value = readq(piu_ior1_base + NEWLTSSMSTATE0); piu_value &= 0xff; if (piu_value == 0x19) @@ -652,10 +653,10 @@ void sunway_pciehp_poll(struct controller *ctrl, u32 events) udelay(10); } - write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_REMOVE); + writeq(HP_CTRL_REMOVE, (piu_ior0_base + HP_CTRL)); while (1) { - piu_value = read_piu_ior0(node, rc_index, HP_WATCHOUT); + piu_value = readq(piu_ior0_base + HP_WATCHOUT); piu_value >>= 24; if (piu_value == 0x19) @@ -668,7 +669,7 @@ void sunway_pciehp_poll(struct controller *ctrl, u32 events) mdelay(100); while (1) { - piu_value = read_piu_ior1(node, rc_index, NEWLTSSMSTATE0); + piu_value = readq(piu_ior1_base + NEWLTSSMSTATE0); piu_value &= 0xff; if (piu_value == 0x0) @@ -689,20 +690,20 @@ void sunway_pciehp_poll(struct controller *ctrl, u32 events) PCI_EXP_SLTCTL_DLLSCE); pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); - write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_FINISH); + writeq(HP_CTRL_FINISH, (piu_ior0_base + HP_CTRL)); ctrl->state = OFF_STATE; } if (events & SW64_POLL_ENABLE_SLOT) { - write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_INSERT); + writeq(HP_CTRL_INSERT, (piu_ior0_base + HP_CTRL)); for (i = 0; i < 30; i++) { if (pcie_wait_for_link(pdev, true)) { pci_mark_rc_linkup(hose->node, hose->index); sunway_pciehp_restore_rc_piu(ctrl); - write_piu_ior0(node, rc_index, HP_CTRL, HP_CTRL_FINISH); + writeq(HP_CTRL_FINISH, (piu_ior0_base + HP_CTRL)); pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); slot_ctrl &= ~PCI_EXP_SLTCTL_PWR_OFF; -- Gitee From 60f60b84cb9acf54c800d4b90d5465b66061aed0 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 31 May 2024 17:54:14 +0800 Subject: [PATCH 198/524] sw64: fix a bug in gpio-sunway.c Change spin_unlock_irqrestore() and spin_lock_irqsave() into raw_spin_unlock_irqrestore() and raw_spin_lock_irqsave(). Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/gpio/gpio-sunway.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/gpio/gpio-sunway.c b/drivers/gpio/gpio-sunway.c index 0fd113ff6b11..8532e31edbec 100644 --- a/drivers/gpio/gpio-sunway.c +++ b/drivers/gpio/gpio-sunway.c @@ -231,11 +231,11 @@ static void sunway_irq_enable(struct irq_data *d) unsigned long flags; u32 val; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); val = sunway_read(gpio, GPIO_INTEN); val |= BIT(d->hwirq); sunway_write(gpio, GPIO_INTEN, val); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void sunway_irq_disable(struct irq_data *d) @@ -246,11 +246,11 @@ static void sunway_irq_disable(struct irq_data *d) unsigned long flags; u32 val; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); val = sunway_read(gpio, GPIO_INTEN); val &= ~BIT(d->hwirq); sunway_write(gpio, GPIO_INTEN, val); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static int sunway_irq_reqres(struct irq_data *d) @@ -290,7 +290,7 @@ static int sunway_irq_set_type(struct irq_data *d, u32 type) IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW)) return -EINVAL; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); level = sunway_read(gpio, GPIO_INTTYPE_LEVEL); polarity = sunway_read(gpio, GPIO_INT_POLARITY); @@ -322,7 +322,7 @@ static int sunway_irq_set_type(struct irq_data *d, u32 type) sunway_write(gpio, GPIO_INTTYPE_LEVEL, level); if (type != IRQ_TYPE_EDGE_BOTH) sunway_write(gpio, GPIO_INT_POLARITY, polarity); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return 0; } @@ -351,7 +351,7 @@ static int sunway_gpio_set_debounce(struct gpio_chip *gc, unsigned long flags, val_deb; unsigned long mask = BIT(offset); - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); val_deb = sunway_read(gpio, GPIO_PORTA_DEBOUNCE); if (debounce) @@ -359,7 +359,7 @@ static int sunway_gpio_set_debounce(struct gpio_chip *gc, else sunway_write(gpio, GPIO_PORTA_DEBOUNCE, val_deb & ~mask); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return 0; } @@ -763,7 +763,7 @@ static int sunway_gpio_suspend(struct device *dev) unsigned long flags; int i; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); for (i = 0; i < gpio->nr_ports; i++) { unsigned int offset; unsigned int idx = gpio->ports[i].idx; @@ -793,7 +793,7 @@ static int sunway_gpio_suspend(struct device *dev) 0xffffffff & ~ctx->wake_en); } } - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); clk_disable_unprepare(gpio->clk); @@ -811,7 +811,7 @@ static int sunway_gpio_resume(struct device *dev) if (!IS_ERR(gpio->clk)) clk_prepare_enable(gpio->clk); - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); for (i = 0; i < gpio->nr_ports; i++) { unsigned int offset; unsigned int idx = gpio->ports[i].idx; @@ -840,7 +840,7 @@ static int sunway_gpio_resume(struct device *dev) sunway_write(gpio, GPIO_PORTA_EOI, 0xffffffff); } } - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return 0; } -- Gitee From 7b645fa8c3d0eb377f344aa3a8587398cb94b941 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 16 May 2024 08:51:42 +0800 Subject: [PATCH 199/524] =?UTF-8?q?sw64:=20kconfig:=20set=20default=20y=20?= =?UTF-8?q?for=20GPIO=20if=C2=A0ACPI=C2=A0enabled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set the driver of GPIO to default y, so that the GPIO is always available, which is used to signal ACPI events. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + drivers/gpio/Kconfig | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index a7c378d679cc..ccda16232cde 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -68,6 +68,7 @@ config SW64 select GENERIC_STRNCPY_FROM_USER select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL + select GPIOLIB if ACPI select HANDLE_DOMAIN_IRQ select HARDIRQS_SW_RESEND select HAVE_ARCH_AUDITSYSCALL diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 509f42e6ab6a..5ddd4a6badf8 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -251,6 +251,7 @@ config GPIO_SUNWAY depends on SW64 select GPIO_GENERIC select GENERIC_IRQ_CHIP + default y if ACPI help Say Y or M here to build support for the Sunway GPIO block. -- Gitee From fb7743d3879399b35246cebceb2b43c500deefe4 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 16 May 2024 09:00:55 +0800 Subject: [PATCH 200/524] sw64: kconfig: add I2C and SPI defconfig for junzhang The following configuration items are added and set to y: - CONFIG_I2C - CONFIG_I2C_DESIGNWARE_PLATFORM - CONFIG_SPI_CHIP - CONFIG_SPI_CHIP_MMIO Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/configs/junzhang_defconfig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/sw_64/configs/junzhang_defconfig b/arch/sw_64/configs/junzhang_defconfig index 62fccb2a8117..866d20c644d9 100644 --- a/arch/sw_64/configs/junzhang_defconfig +++ b/arch/sw_64/configs/junzhang_defconfig @@ -520,9 +520,13 @@ CONFIG_SERIAL_OF_PLATFORM=y CONFIG_VIRTIO_CONSOLE=y # CONFIG_HW_RANDOM is not set # CONFIG_I2C_COMPAT is not set +CONFIG_I2C=y CONFIG_I2C_CHARDEV=y +CONFIG_I2C_DESIGNWARE_PLATFORM=y CONFIG_I2C_MUX=y CONFIG_SPI=y +CONFIG_SPI_CHIP=y +CONFIG_SPI_CHIP_MMIO=y CONFIG_SPI_SPIDEV=y CONFIG_SENSORS_PVT=y CONFIG_SENSORS_LM75=y -- Gitee From 0a1133d7f10e0e484dd17bcdbf8fd704c17c79d0 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 21 May 2024 11:20:04 +0000 Subject: [PATCH 201/524] sw64: ata: add ata_hrst_delay cmdline option The delay time added by commit ec745d2cf875 ("sw64: ahci: fix port reset") is insufficient. With this patch, it can set delay time for port reset via cmdline 'ata_hrst_delay='. For example, 'ata_hrst_delay=1000' means wait for 1000 ms. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/ata/libata-sata.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/ata/libata-sata.c b/drivers/ata/libata-sata.c index be72030a500d..3fb6f09713e6 100644 --- a/drivers/ata/libata-sata.c +++ b/drivers/ata/libata-sata.c @@ -517,6 +517,20 @@ int sata_set_spd(struct ata_link *link) } EXPORT_SYMBOL_GPL(sata_set_spd); +#ifdef CONFIG_SW64 +unsigned int port_reset_delay __read_mostly = 1; +static int __init port_reset_setup(char *str) +{ + int delay; + + get_option(&str, &delay); + if (delay > 0) + port_reset_delay = delay; + + return 0; +} +__setup("ata_hrst_delay=", port_reset_setup); +#endif /** * sata_link_hardreset - reset link via SATA phy reset * @link: link to reset @@ -580,7 +594,11 @@ int sata_link_hardreset(struct ata_link *link, const unsigned int *timing, /* Couldn't find anything in SATA I/II specs, but AHCI-1.1 * 10.4.2 says at least 1 ms. */ +#ifdef CONFIG_SW64 + ata_msleep(link->ap, port_reset_delay); +#else ata_msleep(link->ap, 1); +#endif /* bring link back */ rc = sata_link_resume(link, timing, deadline); -- Gitee From 9404f550a51ef91b21767f3ebf3f1f01504bef71 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Sun, 28 Apr 2024 09:03:52 +0800 Subject: [PATCH 202/524] sw64: dtb: disable built-in DTB for junzhang Starting from junzhang, we use DTB provided by firmware to pass some boot parameters that are dynamically generated by firmware and cannot be added to built-in DTB. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index ccda16232cde..a165e5ed13a2 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -514,6 +514,7 @@ menu "Boot options" config BUILTIN_DTB bool "Embed DTB in kernel image" depends on OF + depends on SUBARCH_C3B default n help Embeds a device tree binary in the kernel image. -- Gitee From 02156422998c7f916ec814411e21b2cb82d4e96f Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 29 Apr 2024 14:04:53 +0800 Subject: [PATCH 203/524] sw64: smp: support SMP initialization based on device tree When ACPI is disabled, we prefer to take core related information from device tree rather than I/O operations in kernel. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/smp.h | 6 ++ arch/sw_64/kernel/acpi.c | 8 +-- arch/sw_64/kernel/setup.c | 14 ++-- arch/sw_64/kernel/smp.c | 120 ++++++++++++++++++++++++++++++++++- 4 files changed, 133 insertions(+), 15 deletions(-) diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h index 097f22b72c25..130c52c9e306 100644 --- a/arch/sw_64/include/asm/smp.h +++ b/arch/sw_64/include/asm/smp.h @@ -36,6 +36,8 @@ struct smp_rcb_struct { unsigned long init_done; }; +extern bool __init is_rcid_duplicate(int rcid); + #ifdef CONFIG_SMP /* SMP initialization hook for setup_arch */ void __init setup_smp(void); @@ -90,6 +92,9 @@ extern int get_thread_id_from_rcid(int rcid); extern int get_domain_id_from_rcid(int rcid); #else /* CONFIG_SMP */ + +static inline void __init setup_smp(void) { store_cpu_data(0); } + #define hard_smp_processor_id() 0 #define smp_call_function_on_cpu(func, info, wait, cpu) ({ 0; }) /* The map from sequential logical cpu number to hard cid. */ @@ -108,6 +113,7 @@ static inline void rcid_information_init(int core_version) { } static inline int get_core_id_from_rcid(int rcid) { return 0; } static inline int get_thread_id_from_rcid(int rcid) { return 0; } static inline int get_domain_id_from_rcid(int rcid) { return 0; } + #endif /* CONFIG_SMP */ #define NO_PROC_ID (-1) diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 5be5d93018cc..990db687657e 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -226,7 +226,7 @@ int acpi_unmap_cpu(int cpu) EXPORT_SYMBOL(acpi_unmap_cpu); #endif /* CONFIG_ACPI_HOTPLUG_CPU */ -static bool __init is_rcid_duplicate(int rcid) +bool __init is_rcid_duplicate(int rcid) { int i; @@ -249,14 +249,14 @@ setup_rcid_and_core_mask(struct acpi_madt_sw_cintc *sw_cintc) * represents the maximum number of cores in the system. */ if (possible_cores >= nr_cpu_ids) { - pr_err(PREFIX "Max core num [%u] reached, core [0x%x] ignored\n", - nr_cpu_ids, rcid); + pr_err(PREFIX "Core [0x%x] exceeds max core num [%u]\n", + rcid, nr_cpu_ids); return -ENODEV; } /* The rcid of each core is unique */ if (is_rcid_duplicate(rcid)) { - pr_err(PREFIX "Duplicate core [0x%x] in MADT\n", rcid); + pr_err(PREFIX "Duplicate core [0x%x]\n", rcid); return -EINVAL; } diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index aceaecb5bf8e..411810dbf10d 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -688,7 +688,6 @@ setup_arch(char **cmdline_p) setup_cpu_info(); setup_run_mode(); setup_chip_ops(); - sw64_chip_init->early_init.setup_core_map(&core_start); if (is_guest_or_emul()) get_vt_smp_info(); @@ -730,13 +729,10 @@ setup_arch(char **cmdline_p) /* Parse the ACPI tables for possible boot-time configuration */ acpi_boot_table_init(); - if (acpi_disabled) { -#ifdef CONFIG_SMP - setup_smp(); -#else - store_cpu_data(0); -#endif - } + if (acpi_disabled) + device_tree_init(); + + setup_smp(); sw64_numa_init(); @@ -779,11 +775,9 @@ setup_arch(char **cmdline_p) #ifdef CONFIG_NUMA cpu_set_node(); #endif - device_tree_init(); } } - static int show_cpuinfo(struct seq_file *f, void *slot) { diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 78d4082e1a58..03118ebc56ee 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include @@ -220,6 +222,103 @@ void __init smp_rcb_init(struct smp_rcb_struct *smp_rcb_base_addr) mb(); } +static int __init sw64_of_core_version(const struct device_node *dn, + int *version) +{ + if (!dn || !version) + return -EINVAL; + + if (of_device_is_compatible(dn, "sw64,xuelang")) { + *version = CORE_VERSION_C3B; + return 0; + } + + if (of_device_is_compatible(dn, "sw64,junzhang")) { + *version = CORE_VERSION_C4; + return 0; + } + + return -EINVAL; +} + +static int __init fdt_setup_smp(void) +{ + struct device_node *dn = NULL; + u64 boot_flag_address; + u32 rcid, logical_core_id = 0; + int ret, i, version; + + /* Clean the map from logical core ID to physical core ID */ + for (i = 0; i < ARRAY_SIZE(__cpu_to_rcid); ++i) + set_rcid_map(i, -1); + + /* Clean core mask */ + init_cpu_possible(cpu_none_mask); + init_cpu_present(cpu_none_mask); + + while ((dn = of_find_node_by_type(dn, "cpu"))) { + if (!of_device_is_available(dn)) { + pr_info("OF: Core is not available\n"); + continue; + } + + ret = of_property_read_u32(dn, "reg", &rcid); + if (ret) { + pr_err("OF: Found core without rcid\n"); + return -ENODEV; + } + + if (logical_core_id >= nr_cpu_ids) { + pr_err("OF: Core [0x%x] exceeds max core num [%u]\n", + rcid, nr_cpu_ids); + return -ENODEV; + } + + if (is_rcid_duplicate(rcid)) { + pr_err("OF: Duplicate core [0x%x]\n", rcid); + return -EINVAL; + } + + ret = sw64_of_core_version(dn, &version); + if (ret) { + pr_err("OF: No valid core version found\n"); + return ret; + } + + ret = of_property_read_u64(dn, "sw64,boot_flag_address", + &boot_flag_address); + if (ret) { + pr_err("OF: No boot_flag_address found\n"); + return ret; + } + + set_rcid_map(logical_core_id, rcid); + set_cpu_possible(logical_core_id, true); + store_cpu_data(logical_core_id); + + if (!cpumask_test_cpu(logical_core_id, &cpu_offline)) + set_cpu_present(logical_core_id, true); + + rcid_information_init(version); + + smp_rcb_init(__va(boot_flag_address)); + + logical_core_id++; + } + + /* No valid cpu node found */ + if (!num_possible_cpus()) + return -EINVAL; + + /* It's time to update nr_cpu_ids */ + nr_cpu_ids = num_possible_cpus(); + + pr_info("OF: Detected %u possible CPU(s), %u CPU(s) are present\n", + nr_cpu_ids, num_present_cpus()); + + return 0; +} + /* * Called from setup_arch. Detect an SMP system and which processors * are present. @@ -228,10 +327,29 @@ void __init setup_smp(void) { int i = 0, num = 0; + /* First try SMP initialization via ACPI */ + if (!acpi_disabled) + return; + + /* Next try SMP initialization via device tree */ + if (!fdt_setup_smp()) + return; + + /* Fallback to legacy SMP initialization */ + + /* Clean the map from logical core ID to physical core ID */ + for (i = 0; i < ARRAY_SIZE(__cpu_to_rcid); ++i) + set_rcid_map(i, -1); + + /* Clean core mask */ init_cpu_possible(cpu_none_mask); + init_cpu_present(cpu_none_mask); + + /* Legacy core detect */ + sw64_chip_init->early_init.setup_core_map(&core_start); /* For unified kernel, NR_CPUS is the maximum possible value */ - for (; i < NR_CPUS; i++) { + for (i = 0; i < NR_CPUS; i++) { if (cpu_to_rcid(i) != -1) { set_cpu_possible(num, true); store_cpu_data(num); -- Gitee From 83df31dc8ac1ca1c8b0610b20b317a3d113d0858 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 29 Apr 2024 14:33:39 +0800 Subject: [PATCH 204/524] sw64: numa: support NUMA initialization based on device tree Make the generic function of_numa_init() work on sunway platform. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index a165e5ed13a2..b685807536af 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -429,6 +429,7 @@ config NUMA bool "NUMA Support" depends on SMP && !FLATMEM select ACPI_NUMA if ACPI + select OF_NUMA help Say Y to compile the kernel to support NUMA (Non-Uniform Memory Access). This option is for configuring high-end multiprocessor -- Gitee From 3a4d4496f0997bbbb7e67a9705b4fcd218f9ea8f Mon Sep 17 00:00:00 2001 From: He Sheng Date: Wed, 17 Apr 2024 14:56:56 +0800 Subject: [PATCH 205/524] sw64: remove some unused module init/exit/license This patch cleans up some module related code, because these source will never be compiled into kernel modules. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/pci/msi.c | 1 - drivers/irqchip/irq-sunway-msi-vt.c | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/pci/msi.c b/arch/sw_64/pci/msi.c index fc2c122c37ef..98dfb1c3b3e7 100644 --- a/arch/sw_64/pci/msi.c +++ b/arch/sw_64/pci/msi.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -#include #include #include #include diff --git a/drivers/irqchip/irq-sunway-msi-vt.c b/drivers/irqchip/irq-sunway-msi-vt.c index df8c7d72671b..65d4ea559447 100644 --- a/drivers/irqchip/irq-sunway-msi-vt.c +++ b/drivers/irqchip/irq-sunway-msi-vt.c @@ -2,6 +2,7 @@ #include #include #include +#include static DEFINE_RAW_SPINLOCK(vector_lock); -- Gitee From 68074bf070d27b753f7e6500974493c64969e083 Mon Sep 17 00:00:00 2001 From: Hang Xiaoqian Date: Thu, 9 May 2024 10:27:54 +0800 Subject: [PATCH 206/524] sw64: fix sw64_is_fake_mcount() of recordmcount The recordmcount may misidentify reloc type ELF_LITERAL_GOT as _mcount, which causes the original instruction to be replaced with NOP. Signed-off-by: Hang Xiaoqian Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- scripts/recordmcount.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index 31a0243df35b..0a5bac4ac02b 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c @@ -499,6 +499,9 @@ static int sw64_is_fake_mcount(Elf64_Rel const *rp) Elf64_Addr current_r_offset = _w(rp->r_offset); int is_fake; + if (Elf_r_sym(rp) == 0) + return 1; + is_fake = (old_r_offset != ~(Elf64_Addr)0) && (current_r_offset - old_r_offset == SW64_FAKEMCOUNT_OFFSET); old_r_offset = current_r_offset; -- Gitee From 25113703bd0b29d88ff469311b5c856ec7564599 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 7 May 2024 15:24:51 +0000 Subject: [PATCH 207/524] sw64: modify some interrupt target core to logic 0 core Actually, the logic 0 core is the first online core, and its cid does not have to be zero. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uncore_io_junzhang.h | 9 ++++++--- arch/sw_64/include/asm/uncore_io_xuelang.h | 6 ++++-- drivers/pci/controller/pci-sunway.c | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index 8c14890e5c1d..770af3907b5c 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -62,9 +62,12 @@ #define LPC_FIRMWARE_IO (0x3UL << 28 | IO_BASE | LPC_BASE) #define PCI_VT_LEGACY_IO (IO_BASE | PCI_BASE | PCI_LEGACY_IO) -#define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) -#define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) -#define HP_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) +#define CORE0_CID (rcid_to_domain_id(cpu_to_rcid(0)) << 7 | \ + rcid_to_thread_id(cpu_to_rcid(0)) << 6 | \ + rcid_to_core_id(cpu_to_rcid(0))) +#define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10 | CORE0_CID) +#define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10 | CORE0_CID) +#define HP_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10 | CORE0_CID) #define PIUCONFIG0_INIT_VAL 0x38016 diff --git a/arch/sw_64/include/asm/uncore_io_xuelang.h b/arch/sw_64/include/asm/uncore_io_xuelang.h index 695ce68dded9..7a8c9edc8bfb 100644 --- a/arch/sw_64/include/asm/uncore_io_xuelang.h +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -66,8 +66,10 @@ #define DLI_PHY_CTL (0x10UL << 24) #define PCI_VT_LEGACY_IO (IO_BASE | PCI_BASE | PCI_LEGACY_IO) -#define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x1UL << 10) -#define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x1UL << 10) +#define CORE0_CID (rcid_to_domain_id(cpu_to_rcid(0)) << 6 | \ + rcid_to_core_id(cpu_to_rcid(0))) +#define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x1UL << 10 | CORE0_CID) +#define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x1UL << 10 | CORE0_CID) #define PIUCONFIG0_INIT_VAL 0x38056 diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 6af0020ce1ee..139c9515fe6f 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -19,7 +19,7 @@ void set_devint_wken(int node) #ifdef CONFIG_UNCORE_JUNZHANG void set_adr_int(int node) { - sw64_io_write(node, ADR_INT_CONFIG, (0x0 << 16 | 0x3f)); + sw64_io_write(node, ADR_INT_CONFIG, (CORE0_CID << 16 | 0x3f)); sw64_io_write(node, ADR_CTL, 0xc); } #endif -- Gitee From d1d450b483f95d9aa504d4d090bc36cabda95bfe Mon Sep 17 00:00:00 2001 From: Zhi Tongze Date: Wed, 8 May 2024 15:37:09 +0800 Subject: [PATCH 208/524] sw64: fix regs.pc on single-step with ptrace Previous patch had unexpectedly removed single step treatment from breakpoint handler, result in wrong pc after ptrace single step. Fixes: 1db0ae17cfe4 ("sw64: fix instruction fault handler") Signed-off-by: Zhi Tongze Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/traps.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 8dd522d09860..c9e956b01dc9 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -275,6 +275,8 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) switch (type) { case IF_BREAKPOINT: /* gdb do pc-4 for sigtrap */ + if (ptrace_cancel_bpt(current)) + regs->pc -= 4; force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc); return; -- Gitee From b0722bc24fae91d209e871bfb7c97ea1165034ed Mon Sep 17 00:00:00 2001 From: Deng Xiaoyun Date: Thu, 9 May 2024 15:41:41 +0800 Subject: [PATCH 209/524] sw64: fdt: map physical node ID to logical node ID On the Qemu command, NUMA node ID may be not in natural sequence as CPU ID, for example: -numa node,memdev=m0,cpus=4-7,nodeid=0 -numa node,memdev=m1,cpus=0-3,nodeid=1 But we want to create a NUMA topology as follow: Node 0: cpu 0-3 Node 1: cpu 4-7 Device tree has to make node ID convertion. This patch learns ACPI implementation to establishes mapping between physical node ID and logical node ID, and then set cpu_to_node_map with logical node ID. Signed-off-by: Deng Xiaoyun Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 03118ebc56ee..64571b45dd68 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -23,6 +23,12 @@ struct smp_rcb_struct *smp_rcb; extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; +static nodemask_t nodes_found_map = NODE_MASK_NONE; + /* maps to convert between physical node ID and logical node ID */ +static int pnode_to_lnode_map[MAX_NUMNODES] + = { [0 ... MAX_NUMNODES - 1] = NUMA_NO_NODE }; +static int lnode_to_pnode_map[MAX_NUMNODES] + = { [0 ... MAX_NUMNODES - 1] = NUMA_NO_NODE }; void *idle_task_pointer[NR_CPUS]; @@ -241,12 +247,39 @@ static int __init sw64_of_core_version(const struct device_node *dn, return -EINVAL; } +static void __fdt_map_pnode_to_lnode(int pnode, int lnode) +{ + if (pnode_to_lnode_map[pnode] == NUMA_NO_NODE || lnode < pnode_to_lnode_map[pnode]) + pnode_to_lnode_map[pnode] = lnode; + if (lnode_to_pnode_map[lnode] == NUMA_NO_NODE || pnode < lnode_to_pnode_map[lnode]) + lnode_to_pnode_map[lnode] = pnode; +} + +static int fdt_map_pnode_to_lnode(int pnode) +{ + int lnode; + + if (pnode < 0 || pnode >= MAX_NUMNODES || numa_off) + return NUMA_NO_NODE; + lnode = pnode_to_lnode_map[pnode]; + + if (lnode == NUMA_NO_NODE) { + if (nodes_weight(nodes_found_map) >= MAX_NUMNODES) + return NUMA_NO_NODE; + lnode = first_unset_node(nodes_found_map); + __fdt_map_pnode_to_lnode(pnode, lnode); + node_set(lnode, nodes_found_map); + } + + return lnode; +} + static int __init fdt_setup_smp(void) { struct device_node *dn = NULL; u64 boot_flag_address; u32 rcid, logical_core_id = 0; - int ret, i, version; + int ret, i, version, lnode, pnode; /* Clean the map from logical core ID to physical core ID */ for (i = 0; i < ARRAY_SIZE(__cpu_to_rcid); ++i) @@ -303,6 +336,12 @@ static int __init fdt_setup_smp(void) smp_rcb_init(__va(boot_flag_address)); + /* Set core affinity */ + pnode = of_node_to_nid(dn); + lnode = fdt_map_pnode_to_lnode(pnode); + + early_map_cpu_to_node(logical_core_id, lnode); + logical_core_id++; } -- Gitee From ee56da29e68aaab4603366ac5e1ba4f2b19e5bf6 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 6 Jun 2024 09:02:18 +0800 Subject: [PATCH 210/524] sw64: gpio: fix compile error of sunway_gpio_add_port The struct gpio_chip no longer has member 'of_node'. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/gpio/gpio-sunway.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/gpio/gpio-sunway.c b/drivers/gpio/gpio-sunway.c index 8532e31edbec..1b42ecbe25ee 100644 --- a/drivers/gpio/gpio-sunway.c +++ b/drivers/gpio/gpio-sunway.c @@ -525,9 +525,7 @@ static int sunway_gpio_add_port(struct sunway_gpio *gpio, return err; } -#ifdef CONFIG_OF_GPIO - port->gc.of_node = to_of_node(pp->fwnode); -#endif + port->gc.fwnode = pp->fwnode; port->gc.ngpio = pp->ngpio; port->gc.base = pp->gpio_base; -- Gitee From 1f6bc350893a2d8e4a0d7e1a770810bed00566c5 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 11 Jun 2024 14:52:33 +0000 Subject: [PATCH 211/524] sw64: adapt irq-sunway-msi.c and irq-sunway-msi-v2.c to upstream Due to the changes in the upstream, this patch adapts the above for irq-sunway-msi.c and irq-sunway-msi-v2.c. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-msi-v2.c | 77 ++++++++++++++--------------- drivers/irqchip/irq-sunway-msi.c | 55 +++++++++------------ 2 files changed, 61 insertions(+), 71 deletions(-) diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c index 2beacb1be39d..4a4f4f6ca040 100644 --- a/drivers/irqchip/irq-sunway-msi-v2.c +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -51,23 +51,19 @@ bool find_free_cpu_vector(const struct cpumask *search_mask, cpu = cpumask_first(search_mask); try_again: - if (is_guest_or_emul()) { - vector = IRQ_PENDING_MSI_VECTORS_SHIFT; - max_vector = SWVM_IRQS; - } else { - vector = 0; - max_vector = 256; - } + vector = 0; + max_vector = 256; for (; vector < max_vector; vector++) { while (per_cpu(vector_irq, cpu)[vector]) { cpu = cpumask_next(cpu, search_mask); if (cpu >= nr_cpu_ids) { if (vector == 255) { if (find_once_global) { - pr_warn("No global free vector\n"); + pr_info("No global free vector\n"); return false; } - pr_warn("No local free vector\n"); + pr_info("No local free vector, search_mask:%*pbl\n", + cpumask_pr_args(search_mask)); search_mask = cpu_online_mask; cpu = cpumask_first(search_mask); find_once_global = true; @@ -111,10 +107,11 @@ static bool find_free_cpu_vectors(const struct cpumask *search_mask, int *found_ goto try_again; else { if (find_once_global) { - pr_warn("No global free vectors\n"); + pr_info("No global free vectors\n"); return found; } - pr_warn("No local free vectors\n"); + pr_info("No local free vectors, search_mask:%*pbl\n", + cpumask_pr_args(search_mask)); search_mask = cpu_online_mask; cpu = cpumask_first(search_mask); find_once_global = true; @@ -146,6 +143,9 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, if (!cdata) return -ENOMEM; + if (cdata->move_in_progress) + return -EBUSY; + /* * If existing target cpu is already in the new mask and is online * then do nothing. @@ -172,15 +172,22 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, /* update new setting */ entry = irq_get_msi_desc(irqd->irq); spin_lock(&cdata->cdata_lock); + if (cpu_online(cdata->dst_cpu)) { + cdata->move_in_progress = true; + cdata->prev_vector = cdata->vector; + cdata->prev_cpu = cdata->dst_cpu; + } else { + for (i = 0; i < cdata->multi_msi; i++) + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; + } + + cdata->vector = vector; + cdata->dst_cpu = cpu; for (i = 0; i < cdata->multi_msi; i++) per_cpu(vector_irq, cpu)[vector + i] = entry->irq + i; - BUG_ON(irq_chip_compose_msi_msg(irqd, &msg)); + + irq_msi_compose_msg(d, &msg); __pci_write_msi_msg(entry, &msg); - cdata->prev_vector = cdata->vector; - cdata->prev_cpu = cdata->dst_cpu; - cdata->dst_cpu = cpu; - cdata->vector = vector; - cdata->move_in_progress = true; spin_unlock(&cdata->cdata_lock); cpumask_copy(irq_data_get_affinity_mask(irqd), &searchmask); @@ -238,7 +245,7 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, cdata = alloc_sw_msi_chip_data(irq_data); if (!cdata) { - pr_warn("error alloc irq chip data\n"); + pr_info("error alloc irq chip data\n"); return -ENOMEM; } @@ -274,7 +281,7 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, cdata = alloc_sw_msi_chip_data(irq_data); if (!cdata) { - pr_warn("error alloc irq chip data\n"); + pr_info("error alloc irq chip data\n"); return -ENOMEM; } @@ -334,11 +341,6 @@ static void sw64_vector_free_irqs(struct irq_domain *domain, static void sw64_irq_free_descs(unsigned int virq, unsigned int nr_irqs) { - if (is_guest_or_emul()) { - vt_sw64_vector_free_irqs(virq, nr_irqs); - return irq_free_descs(virq, nr_irqs); - } - return irq_domain_free_irqs(virq, nr_irqs); } @@ -347,7 +349,7 @@ void arch_teardown_msi_irqs(struct pci_dev *dev) struct msi_desc *desc; int i; - for_each_pci_msi_entry(desc, dev) { + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ASSOCIATED) { if (desc->irq) { for (i = 0; i < desc->nvec_used; i++) sw64_irq_free_descs(desc->irq + i, 1); @@ -379,11 +381,11 @@ static int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec, msi_alloc_info_t *arg) { struct pci_dev *pdev = to_pci_dev(dev); - struct msi_desc *desc = first_pci_msi_entry(pdev); + struct msi_desc *desc = msi_first_desc(dev, MSI_DESC_ALL); memset(arg, 0, sizeof(*arg)); arg->msi_dev = pdev; - if (desc->msi_attrib.is_msix) + if (desc->pci.msi_attrib.is_msix) arg->type = IRQ_ALLOC_TYPE_MSIX; else arg->type = IRQ_ALLOC_TYPE_MSI; @@ -425,7 +427,7 @@ int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) if (!msi_default_domain) return -EIO; - err = msi_domain_alloc_irqs(msi_default_domain, &pdev->dev, nvec); + err = msi_domain_alloc_irqs_all_locked(&pdev->dev, MSI_DEFAULT_DOMAIN, nvec); return err; } @@ -433,16 +435,20 @@ void arch_init_msi_domain(struct irq_domain *parent) { struct irq_domain *sw64_irq_domain; - if (is_guest_or_emul()) - return; - sw64_irq_domain = irq_domain_add_tree(NULL, &sw64_msi_domain_ops, NULL); BUG_ON(sw64_irq_domain == NULL); irq_set_default_host(sw64_irq_domain); msi_default_domain = pci_msi_create_irq_domain(NULL, &pci_msi_domain_info, sw64_irq_domain); if (!msi_default_domain) - pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n"); + pr_info("failed to initialize irqdomain for MSI/MSI-x.\n"); +} + +int pcibios_device_add(struct pci_dev *dev) +{ + if (msi_default_domain) + dev_set_msi_domain(&dev->dev, msi_default_domain); + return 0; } static void irq_move_complete(struct sw64_msi_chip_data *cdata, int cpu, int vector) @@ -472,13 +478,6 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned struct irq_data *irq_data; struct sw64_msi_chip_data *cdata; - if (is_guest_or_emul()) { - cpu = smp_processor_id(); - irq = per_cpu(vector_irq, cpu)[vector]; - handle_irq(irq); - return; - } - ptr = (unsigned long *)pci_msi1_addr; int_pci_msi[0] = *ptr; int_pci_msi[1] = *(ptr + 1); diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index 6de689e5106f..e8fdcc5d744b 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -47,23 +47,19 @@ bool find_free_cpu_vector(const struct cpumask *search_mask, cpu = cpumask_first(search_mask); try_again: - if (is_guest_or_emul()) { - vector = IRQ_PENDING_MSI_VECTORS_SHIFT; - max_vector = SWVM_IRQS; - } else { - vector = 0; - max_vector = 256; - } + vector = 0; + max_vector = 256; for (; vector < max_vector; vector++) { while (per_cpu(vector_irq, cpu)[vector]) { cpu = cpumask_next(cpu, search_mask); if (cpu >= nr_cpu_ids) { if (vector == 255) { if (find_once_global) { - pr_warn("No global free vector\n"); + pr_info("No global free vector\n"); return false; } - pr_warn("No local free vector\n"); + pr_info("No local free vector, search_mask:%*pbl\n", + cpumask_pr_args(search_mask)); search_mask = cpu_online_mask; cpu = cpumask_first(search_mask); find_once_global = true; @@ -124,6 +120,9 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, if (!cdata) return -ENOMEM; + if (cdata->move_in_progress) + return -EBUSY; + /* * If existing target cpu is already in the new mask and is online * then do nothing. @@ -144,14 +143,20 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, pdev = (struct pci_dev *)msi_desc_to_pci_dev(entry); hose = pci_bus_to_pci_controller(pdev->bus); spin_lock(&cdata->cdata_lock); - per_cpu(vector_irq, cpu)[vector] = irqd->irq; - msi_config = set_piu_msi_config(hose, cpu, cdata->msi_config_index, vector); - cdata->prev_vector = cdata->vector; - cdata->prev_cpu = cdata->dst_cpu; + if (cpu_online(cdata->dst_cpu)) { + cdata->move_in_progress = true; + cdata->prev_vector = cdata->vector; + cdata->prev_cpu = cdata->dst_cpu; + } else { + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; + } + cdata->dst_cpu = cpu; cdata->vector = vector; + per_cpu(vector_irq, cpu)[vector] = irqd->irq; + + msi_config = set_piu_msi_config(hose, cpu, cdata->msi_config_index, vector); cdata->msi_config = msi_config; - cdata->move_in_progress = true; spin_unlock(&cdata->cdata_lock); cpumask_copy((struct cpumask *)irq_data_get_affinity_mask(irqd), &searchmask); @@ -193,7 +198,8 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, nr_irqs, nr_irqs - 1); if (msiconf_index >= 256) { - pr_warn("No free msi on PIU!\n"); + pr_info("No free msi on PIU! node:%ld index:%ld\n", + hose->node, hose->index); return -ENOSPC; } @@ -226,7 +232,7 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, cdata = alloc_sw_msi_chip_data(irq_data); if (!cdata) { - pr_warn("error alloc irq chip data\n"); + pr_info("error alloc irq chip data\n"); return -ENOMEM; } @@ -291,11 +297,6 @@ static void sw64_vector_free_irqs(struct irq_domain *domain, static void sw64_irq_free_descs(unsigned int virq, unsigned int nr_irqs) { - if (is_guest_or_emul()) { - vt_sw64_vector_free_irqs(virq, nr_irqs); - return irq_free_descs(virq, nr_irqs); - } - return irq_domain_free_irqs(virq, nr_irqs); } @@ -391,16 +392,13 @@ void arch_init_msi_domain(struct irq_domain *parent) { struct irq_domain *sw64_irq_domain; - if (is_guest_or_emul()) - return; - sw64_irq_domain = irq_domain_add_tree(NULL, &sw64_msi_domain_ops, NULL); BUG_ON(sw64_irq_domain == NULL); irq_set_default_host(sw64_irq_domain); msi_default_domain = pci_msi_create_irq_domain(NULL, &pci_msi_domain_info, sw64_irq_domain); if (!msi_default_domain) - pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n"); + pr_info("failed to initialize irqdomain for MSI/MSI-x.\n"); } int pcibios_device_add(struct pci_dev *dev) @@ -432,13 +430,6 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned struct irq_data *irq_data; struct sw64_msi_chip_data *cdata; - if (is_guest_or_emul()) { - cpu = smp_processor_id(); - irq = per_cpu(vector_irq, cpu)[vector]; - handle_irq(irq); - return; - } - ptr = (unsigned long *)pci_msi1_addr; int_pci_msi[0] = *ptr; int_pci_msi[1] = *(ptr + 1); -- Gitee From 8e624ede891b1e45e1541ad3c5cf000dce7a532f Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 11 Jun 2024 16:37:52 +0800 Subject: [PATCH 212/524] sw64: fix msi interrupt for guest os This patch: - Add msi_domain for guset os, including vt_pci_msi_domain_ops and pci_vt_msi_domain_info which support msi interrupt initialization in a standard way. - Modify the method to find free cpu vector to ensure that msi irq number can be set on the proper cpu. When migrate an interrupt, it can find the vector which was used before but not cleaned up. - Add vt_irq_move_complete() for guest os. If the interrupt migration has completed, it is necessory to clear its information on previous cpu before irq is handled. After that, it can handle msi interrupt correctly during cpu hotplug. Besides, move guset related code into irq-sunway-msi-vt.c. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm.h | 2 +- arch/sw_64/include/asm/msi.h | 9 +- arch/sw_64/include/uapi/asm/kvm.h | 2 +- drivers/irqchip/irq-sunway-cpu.c | 7 +- drivers/irqchip/irq-sunway-msi-v2.c | 6 + drivers/irqchip/irq-sunway-msi-vt.c | 387 +++++++++++++++++----------- drivers/irqchip/irq-sunway-msi.c | 6 + 7 files changed, 263 insertions(+), 156 deletions(-) diff --git a/arch/sw_64/include/asm/kvm.h b/arch/sw_64/include/asm/kvm.h index f6e3022efcfe..41bb657bcacc 100644 --- a/arch/sw_64/include/asm/kvm.h +++ b/arch/sw_64/include/asm/kvm.h @@ -8,7 +8,7 @@ */ #define SWVM_IRQS 256 #define IRQ_PENDING_INTX_SHIFT 16 -#define IRQ_PENDING_MSI_VECTORS_SHIFT 17 +#define IRQ_PENDING_MSI_VECTORS_SHIFT 18 #define SWVM_NUM_NUMA_MEMBANKS 1 diff --git a/arch/sw_64/include/asm/msi.h b/arch/sw_64/include/asm/msi.h index e5d93040b2d3..4bec4b80fbe6 100644 --- a/arch/sw_64/include/asm/msi.h +++ b/arch/sw_64/include/asm/msi.h @@ -23,10 +23,15 @@ #define VT_MSIX_ADDR_DEST_ID(dest) \ (((dest) << VT_MSIX_ADDR_DEST_ID_SHIFT) & VT_MSIX_ADDR_DEST_ID_MASK) +enum irq_alloc_type; #ifdef CONFIG_PCI_MSI -extern void vt_sw64_vector_free_irqs(unsigned int virq, unsigned int nr_irqs); -extern int sw64_setup_vt_msi_irqs(struct pci_dev *dev, int nvec, int type); +extern void vt_handle_pci_msi_interrupt(unsigned long type, + unsigned long vector, + unsigned long pci_msi1_addr); +extern void sw64_init_vt_msi_domain(struct irq_domain *parent); +extern int sw64_setup_vt_msi_irqs(struct pci_dev *pdev, int nvec, int type); +extern int vt_pcibios_device_add(struct pci_dev *dev); extern bool find_free_cpu_vector(const struct cpumask *search_mask, int *found_cpu, int *found_vector); extern int msi_compose_msg(unsigned int irq, struct msi_msg *msg); diff --git a/arch/sw_64/include/uapi/asm/kvm.h b/arch/sw_64/include/uapi/asm/kvm.h index 2ae0cc74bb2b..3215aaa5f2eb 100644 --- a/arch/sw_64/include/uapi/asm/kvm.h +++ b/arch/sw_64/include/uapi/asm/kvm.h @@ -7,7 +7,7 @@ */ #define SWVM_IRQS 256 #define IRQ_PENDING_INTX_SHIFT 16 -#define IRQ_PENDING_MSI_VECTORS_SHIFT 17 +#define IRQ_PENDING_MSI_VECTORS_SHIFT 18 enum SW64_KVM_IRQ { SW64_KVM_IRQ_IPI = 27, diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 23ad3b16af08..109bf4acedde 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -198,7 +198,7 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, if (is_guest_or_emul()) { if ((type & 0xffff) > 15) { vector = type; - if (vector == 16) + if (vector == 16 || vector == 17) type = INT_INTx; else type = INT_MSI; @@ -212,7 +212,10 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, switch (type & 0xffff) { case INT_MSI: old_regs = set_irq_regs(regs); - handle_pci_msi_interrupt(type, vector, irq_arg); + if (is_guest_or_emul()) + vt_handle_pci_msi_interrupt(type, vector, irq_arg); + else + handle_pci_msi_interrupt(type, vector, irq_arg); set_irq_regs(old_regs); return; case INT_INTx: diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c index 4a4f4f6ca040..4e98a4c10f85 100644 --- a/drivers/irqchip/irq-sunway-msi-v2.c +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -435,6 +435,9 @@ void arch_init_msi_domain(struct irq_domain *parent) { struct irq_domain *sw64_irq_domain; + if (is_guest_or_emul()) + return sw64_init_vt_msi_domain(parent); + sw64_irq_domain = irq_domain_add_tree(NULL, &sw64_msi_domain_ops, NULL); BUG_ON(sw64_irq_domain == NULL); irq_set_default_host(sw64_irq_domain); @@ -446,6 +449,9 @@ void arch_init_msi_domain(struct irq_domain *parent) int pcibios_device_add(struct pci_dev *dev) { + if (is_guest_or_emul()) + return vt_pcibios_device_add(dev); + if (msi_default_domain) dev_set_msi_domain(&dev->dev, msi_default_domain); return 0; diff --git a/drivers/irqchip/irq-sunway-msi-vt.c b/drivers/irqchip/irq-sunway-msi-vt.c index 65d4ea559447..a28f23799117 100644 --- a/drivers/irqchip/irq-sunway-msi-vt.c +++ b/drivers/irqchip/irq-sunway-msi-vt.c @@ -1,11 +1,100 @@ // SPDX-License-Identifier: GPL-2.0 -#include #include +#include #include +#include #include +#include +#include + +static struct irq_domain *vt_msi_default_domain; static DEFINE_RAW_SPINLOCK(vector_lock); +static void vt_irq_move_complete(struct sw64_msi_chip_data *cdata, int cpu) +{ + if (likely(!cdata->move_in_progress)) + return; + if (cdata->dst_cpu == cpu) { + raw_spin_lock(&vector_lock); + cdata->move_in_progress = false; + per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector] = 0; + raw_spin_unlock(&vector_lock); + } +} + +void vt_handle_pci_msi_interrupt(unsigned long type, unsigned long vector, + unsigned long pci_msi1_addr) +{ + int irq, cpu; + struct irq_data *irq_data; + struct sw64_msi_chip_data *cdata; + + cpu = smp_processor_id(); + irq = per_cpu(vector_irq, cpu)[vector]; + irq_data = irq_domain_get_irq_data(vt_msi_default_domain->parent, irq); + cdata = irq_data_get_irq_chip_data(irq_data); + + spin_lock(&cdata->cdata_lock); + vt_irq_move_complete(cdata, cpu); + spin_unlock(&cdata->cdata_lock); + + handle_irq(irq); +} + +static bool vt_find_free_cpu_vector(const struct cpumask *search_mask, + int *found_cpu, int *found_vector, struct irq_data *d) +{ + int vector, max_vector, cpu; + bool find_once_global = false; + + cpu = cpumask_first(search_mask); +try_again: + vector = IRQ_PENDING_MSI_VECTORS_SHIFT; + max_vector = SWVM_IRQS; + + for (; vector < max_vector; vector++) { + while (per_cpu(vector_irq, cpu)[vector]) { + if (per_cpu(vector_irq, cpu)[vector] == d->irq) + break; + + if (!irqd_affinity_is_managed(d)) + cpu = cpumask_next(cpu, search_mask); + else + vector++; + + if (vector >= max_vector) { + cpu = cpumask_next(cpu, search_mask); + vector = IRQ_PENDING_MSI_VECTORS_SHIFT; + } + + if (cpu >= nr_cpu_ids) { + if (vector == max_vector-1) { + if (find_once_global) { + pr_err("No global free vector\n"); + return false; + } + pr_err("No local free vector\n"); + search_mask = cpu_online_mask; + cpu = cpumask_first(search_mask); + find_once_global = true; + goto try_again; + } + cpu = cpumask_first(search_mask); + break; + } + } + if (per_cpu(vector_irq, cpu)[vector] == d->irq) + break; + if (!per_cpu(vector_irq, cpu)[vector]) + break; + } + + *found_cpu = cpu; + *found_vector = vector; + return true; +} + static void __vt_irq_msi_compose_msg(struct sw64_msi_chip_data *cdata, struct msi_msg *msg) { @@ -18,8 +107,11 @@ static void __vt_irq_msi_compose_msg(struct sw64_msi_chip_data *cdata, static void vt_irq_msi_compose_msg(struct irq_data *irqd, struct msi_msg *msg) { struct sw64_msi_chip_data *cdata; + struct irq_data *d; + + d = irq_domain_get_irq_data(vt_msi_default_domain->parent, irqd->irq); + cdata = d->chip_data; - cdata = irqd->chip_data; __vt_irq_msi_compose_msg(cdata, msg); } @@ -33,10 +125,11 @@ static void vt_irq_msi_update_msg(struct irq_data *irqd, } static int -vt_set_affinity(struct irq_data *irqd, const struct cpumask *cpumask, +vt_set_affinity(struct irq_data *d, const struct cpumask *cpumask, bool force) { struct sw64_msi_chip_data *cdata; + struct irq_data *irqd; struct cpumask searchmask; int cpu, vector; @@ -44,6 +137,7 @@ vt_set_affinity(struct irq_data *irqd, const struct cpumask *cpumask, if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) return -EINVAL; + irqd = irq_domain_get_irq_data(vt_msi_default_domain->parent, d->irq); if (!irqd_is_started(irqd)) return IRQ_SET_MASK_OK; @@ -59,63 +153,53 @@ vt_set_affinity(struct irq_data *irqd, const struct cpumask *cpumask, return IRQ_SET_MASK_OK; cpumask_and(&searchmask, cpumask, cpu_online_mask); - if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + if (!vt_find_free_cpu_vector(&searchmask, &cpu, &vector, irqd)) return -ENOSPC; per_cpu(vector_irq, cpu)[vector] = irqd->irq; spin_lock(&cdata->cdata_lock); - cdata->dst_cpu = cpu; - cdata->vector = vector; cdata->prev_cpu = cdata->dst_cpu; cdata->prev_vector = cdata->vector; + cdata->dst_cpu = cpu; + cdata->vector = vector; cdata->move_in_progress = true; spin_unlock(&cdata->cdata_lock); - cpumask_copy((struct cpumask *)irq_data_get_affinity_mask(irqd), &searchmask); + irq_data_update_effective_affinity(irqd, &searchmask); vt_irq_msi_update_msg(irqd, irqd->chip_data); return 0; } static struct irq_chip vt_pci_msi_controller = { - .name = "PCI-MSI", - .irq_unmask = pci_msi_unmask_irq, - .irq_mask = pci_msi_mask_irq, - .irq_ack = sw64_irq_noop, - .irq_compose_msi_msg = vt_irq_msi_compose_msg, - .irq_set_affinity = vt_set_affinity, + .name = "PCI-MSI", + .irq_unmask = pci_msi_unmask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_ack = sw64_irq_noop, + .irq_compose_msi_msg = vt_irq_msi_compose_msg, + .flags = IRQCHIP_SKIP_SET_WAKE, + .irq_set_affinity = vt_set_affinity, }; -int chip_setup_vt_msix_irq(struct pci_dev *dev, struct msi_desc *desc) +int chip_setup_vt_msi_irqs(int virq, unsigned int nr_irqs, + struct irq_domain *domain, enum irq_alloc_type type) { - int virq, val_node = 0; struct irq_data *irq_data; struct sw64_msi_chip_data *cdata; - struct pci_controller *hose = pci_bus_to_pci_controller(dev->bus); - unsigned long flags, node, rc_index; - const struct cpumask *mask; + unsigned long node; + const struct cpumask *mask; struct cpumask searchmask; - int cpu, vector; - - node = hose->node; - rc_index = hose->index; - mask = cpumask_of_node(node); - - raw_spin_lock_irqsave(&vector_lock, flags); - /* Find unused msi config reg in PIU-IOR0 */ - if (!node_online(node)) - val_node = next_node_in(node, node_online_map); - else - val_node = node; + int i, vector, cpu; - virq = irq_alloc_descs_from(NR_IRQS_LEGACY, desc->nvec_used, val_node); - if (virq < 0) { - pr_err("Failed to allocate IRQ(base 16, count %d)\n", desc->nvec_used); - raw_spin_unlock_irqrestore(&vector_lock, flags); - return virq; + if (type != IRQ_ALLOC_TYPE_MSI && type != IRQ_ALLOC_TYPE_MSIX) { + pr_info("SW arch do not identify ID:%d\n", type); + return -ENOMEM; } - irq_data = irq_get_irq_data(virq); + irq_data = irq_domain_get_irq_data(domain, virq); + if (!irq_data) + return -EINVAL; + irq_data->chip = &vt_pci_msi_controller; if (irqd_affinity_is_managed(irq_data)) { mask = irq_data_get_affinity_mask(irq_data); @@ -127,155 +211,158 @@ int chip_setup_vt_msix_irq(struct pci_dev *dev, struct msi_desc *desc) if (cpumask_first(&searchmask) >= nr_cpu_ids) cpumask_copy(&searchmask, cpu_online_mask); - if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) - return -ENOSPC; - - cdata = kzalloc(sizeof(*cdata), GFP_KERNEL); - if (!cdata) - return -ENOMEM; - - per_cpu(vector_irq, cpu)[vector] = virq; - - irq_set_msi_desc(virq, desc); - irq_set_chip_and_handler_name(virq, &vt_pci_msi_controller, - handle_edge_irq, "edge"); - - cdata->dst_cpu = cpu; - cdata->vector = vector; - cdata->rc_index = hose->index; - cdata->rc_node = hose->node; - cdata->prev_cpu = cpu; - cdata->prev_vector = vector; - - irq_data->chip_data = cdata; - - vt_irq_msi_update_msg(irq_data, irq_data->chip_data); - raw_spin_unlock_irqrestore(&vector_lock, flags); - return 0; -} -EXPORT_SYMBOL(chip_setup_vt_msix_irq); - -int chip_setup_vt_msi_irqs(struct pci_dev *dev, int nvec, int type) -{ - struct msi_desc *desc; - struct pci_controller *hose = pci_bus_to_pci_controller(dev->bus); - struct irq_data *irq_data; - struct sw64_msi_chip_data *cdata; - unsigned long node, rc_index; - int virq = -1, val_node = 0; - unsigned long flags; - - const struct cpumask *mask; - struct cpumask searchmask; - int i, vector, cpu; - - if (type == PCI_CAP_ID_MSI && nvec > 32) - return 1; - - node = hose->node; - rc_index = hose->index; - raw_spin_lock_irqsave(&vector_lock, flags); - msi_for_each_desc(desc, &(dev->dev), MSI_DESC_ALL) { - /* Find unused msi config reg in PIU-IOR0 */ - if (!node_online(node)) - val_node = next_node_in(node, node_online_map); - else - val_node = node; - virq = irq_alloc_descs_from(NR_IRQS_LEGACY, desc->nvec_used, val_node); - if (virq < 0) { - pr_err("Failed to allocate IRQ(base 16, count %d)\n", desc->nvec_used); - raw_spin_unlock_irqrestore(&vector_lock, flags); - return virq; - } - - irq_data = irq_get_irq_data(virq); - if (irqd_affinity_is_managed(irq_data)) { - mask = irq_data_get_affinity_mask(irq_data); - cpumask_and(&searchmask, mask, cpu_online_mask); - } else { - node = irq_data_get_node(irq_data); - cpumask_copy(&searchmask, cpumask_of_node(node)); + for (i = 0; i < nr_irqs; i++) { + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &vt_pci_msi_controller; } - if (cpumask_first(&searchmask) >= nr_cpu_ids) - cpumask_copy(&searchmask, cpu_online_mask); - - for (i = 0; i < desc->nvec_used; i++) { - if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) - return -ENOSPC; - cdata = kzalloc(sizeof(*cdata), GFP_KERNEL); - if (!cdata) - return -ENOMEM; + if (!vt_find_free_cpu_vector(&searchmask, &cpu, &vector, irq_data)) + return -ENOSPC; - per_cpu(vector_irq, cpu)[vector] = virq + i; - irq_set_msi_desc_off(virq, i, desc); - irq_set_chip_and_handler_name(virq + i, &vt_pci_msi_controller, handle_edge_irq, "edge"); - irq_data = irq_get_irq_data(virq + i); + cdata = kzalloc(sizeof(*cdata), GFP_KERNEL); + if (!cdata) + return -ENOMEM; - cdata->dst_cpu = cpu; - cdata->vector = vector; - cdata->rc_index = hose->index; - cdata->rc_node = hose->node; - cdata->prev_cpu = cpu; - cdata->prev_vector = vector; + per_cpu(vector_irq, cpu)[vector] = virq + i; - irq_data->chip_data = cdata; + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + cdata->move_in_progress = false; - vt_irq_msi_update_msg(irq_data, irq_data->chip_data); - } + irq_data->chip_data = cdata; } - raw_spin_unlock_irqrestore(&vector_lock, flags); return 0; } EXPORT_SYMBOL(chip_setup_vt_msi_irqs); -void vt_sw64_vector_free_irqs(unsigned int virq, unsigned int nr_irqs) +static void sw64_vt_vector_free_irqs(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) { int i; - unsigned long flags; struct irq_data *irq_data; - struct sw64_msi_chip_data *cdata; + unsigned long flags; for (i = 0; i < nr_irqs; i++) { - irq_data = irq_get_irq_data(virq + i); + irq_data = irq_domain_get_irq_data(domain, virq + i); if (irq_data && irq_data->chip_data) { + struct sw64_msi_chip_data *cdata; + raw_spin_lock_irqsave(&vector_lock, flags); cdata = irq_data->chip_data; - irq_data->hwirq = 0; - irq_data->chip = &no_irq_chip; - irq_data->chip_data = NULL; + irq_domain_reset_irq_data(irq_data); per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; kfree(cdata); + raw_spin_unlock_irqrestore(&vector_lock, flags); } } } -int __arch_setup_vt_msix_irqs(struct pci_dev *dev, int nvec, int type) +static int assign_vt_irq_vector(int irq, unsigned int nr_irqs, + struct irq_domain *domain, enum irq_alloc_type type) { - struct msi_desc *entry; - int ret; + int err; + unsigned long flags; - msi_for_each_desc(entry, &dev->dev, MSI_DESC_ALL) { - ret = chip_setup_vt_msix_irq(dev, entry); - if (ret) - return ret; - } + raw_spin_lock_irqsave(&vector_lock, flags); + err = chip_setup_vt_msi_irqs(irq, nr_irqs, domain, type); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return err; +} +static int sw64_vt_vector_alloc_irqs(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, void *arg) +{ + int err; + struct irq_alloc_info *info = arg; + enum irq_alloc_type msi_type; + + if (arg == NULL) + return -ENODEV; + msi_type = info->type; + err = assign_vt_irq_vector(virq, nr_irqs, domain, msi_type); + if (err) + goto error; return 0; +error: + sw64_vt_vector_free_irqs(domain, virq, nr_irqs); + return err; } -int sw64_setup_vt_msi_irqs(struct pci_dev *dev, int nvec, int type) + +static int vt_pci_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *arg) { - int ret = 0; + struct pci_dev *pdev = to_pci_dev(dev); + struct msi_desc *desc = msi_first_desc(dev, MSI_DESC_ALL); - if (type == PCI_CAP_ID_MSI) - ret = chip_setup_vt_msi_irqs(dev, nvec, type); - else if (type == PCI_CAP_ID_MSIX) - ret = __arch_setup_vt_msix_irqs(dev, nvec, type); + memset(arg, 0, sizeof(*arg)); + arg->msi_dev = pdev; + if (desc->pci.msi_attrib.is_msix) + arg->type = IRQ_ALLOC_TYPE_MSIX; else - pr_info("SW arch do not identify ID:%d\n", type); + arg->type = IRQ_ALLOC_TYPE_MSI; + return 0; +} + +static struct msi_domain_ops vt_pci_msi_domain_ops = { + .msi_prepare = vt_pci_msi_prepare, +}; + +static struct msi_domain_info pci_vt_msi_domain_info = { + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, + .ops = &vt_pci_msi_domain_ops, + .chip = &vt_pci_msi_controller, + .handler = handle_edge_irq, + .handler_name = "edge", +}; + +static int sw64_vt_irq_map(struct irq_domain *d, unsigned int virq, irq_hw_number_t hw) +{ + irq_set_chip_and_handler(virq, &sw64_irq_chip, handle_level_irq); + irq_set_status_flags(virq, IRQ_LEVEL); + return 0; +} - return ret; +const struct irq_domain_ops sw64_vt_msi_domain_ops = { + .map = sw64_vt_irq_map, + .alloc = sw64_vt_vector_alloc_irqs, + .free = sw64_vt_vector_free_irqs, +}; + +int sw64_setup_vt_msi_irqs(struct pci_dev *pdev, int nvec, int type) +{ + struct irq_domain *domain; + int err; + + domain = vt_msi_default_domain; + if (domain == NULL) + return -EIO; + err = msi_domain_alloc_irqs_all_locked(&pdev->dev, MSI_DEFAULT_DOMAIN, nvec); + return err; +} + +void sw64_init_vt_msi_domain(struct irq_domain *parent) +{ + struct irq_domain *sw64_irq_domain; + + sw64_irq_domain = irq_domain_add_tree(NULL, &sw64_vt_msi_domain_ops, NULL); + BUG_ON(sw64_irq_domain == NULL); + irq_set_default_host(sw64_irq_domain); + vt_msi_default_domain = pci_msi_create_irq_domain(NULL, + &pci_vt_msi_domain_info, sw64_irq_domain); + if (!vt_msi_default_domain) + pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n"); +} + +int vt_pcibios_device_add(struct pci_dev *dev) +{ + if (vt_msi_default_domain) + dev_set_msi_domain(&dev->dev, vt_msi_default_domain); + return 0; } diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index e8fdcc5d744b..7f55f675caa1 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -392,6 +392,9 @@ void arch_init_msi_domain(struct irq_domain *parent) { struct irq_domain *sw64_irq_domain; + if (is_guest_or_emul()) + return sw64_init_vt_msi_domain(parent); + sw64_irq_domain = irq_domain_add_tree(NULL, &sw64_msi_domain_ops, NULL); BUG_ON(sw64_irq_domain == NULL); irq_set_default_host(sw64_irq_domain); @@ -403,6 +406,9 @@ void arch_init_msi_domain(struct irq_domain *parent) int pcibios_device_add(struct pci_dev *dev) { + if (is_guest_or_emul()) + return vt_pcibios_device_add(dev); + if (msi_default_domain) dev_set_msi_domain(&dev->dev, msi_default_domain); return 0; -- Gitee From 78d88f5cbbcbb7ba48b43dcaa15a592887b9a17f Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Wed, 12 Jun 2024 16:22:43 +0000 Subject: [PATCH 213/524] sw64: adapt iommu for SW64 from upstream The interfaces from the upstream have changed. This adapts the modifications from the upstream for sw64 iommu. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/Kconfig | 5 +- drivers/iommu/sw64/iommu.c | 410 +++--- .../iommu/sw64/{sunway_iommu.h => iommu.h} | 15 +- drivers/iommu/sw64/iommu_v2.c | 1128 ++++++++--------- include/acpi/actbl1.h | 28 + 5 files changed, 818 insertions(+), 768 deletions(-) rename drivers/iommu/sw64/{sunway_iommu.h => iommu.h} (88%) diff --git a/drivers/iommu/sw64/Kconfig b/drivers/iommu/sw64/Kconfig index 3a6a1e994f31..4cc96bdcd233 100644 --- a/drivers/iommu/sw64/Kconfig +++ b/drivers/iommu/sw64/Kconfig @@ -8,14 +8,15 @@ config SUNWAY_IOMMU depends on SW64 && PCI && SUBARCH_C3B help Support for IOMMU on SW64 platform. It can enable or bypass specific device by - adding boot param "iommu_enable" and "iommu.passthrough". + adding boot param "sunway_iommu" and "iommu.passthrough". # SW64 IOMMU V2 SUPPORT config SUNWAY_IOMMU_V2 bool "Sunway IOMMU V2 Support" select IOMMU_API select IOMMU_IOVA + select IOMMU_DMA depends on SW64 && PCI && SUBARCH_C4 help Support for IOMMU V2 on SW64 platform. It can enable or bypass specific device by - adding boot param "iommu_enable" and "iommu.passthrough". + adding boot param "sunway_iommu" and "iommu.passthrough". diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index 32b18f726fd9..94b01ca35ecd 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -6,6 +6,7 @@ * in later chips, then it should work just as well. * */ +#define pr_fmt(fmt) "IOMMU: " fmt #include #include @@ -31,22 +32,17 @@ #include #include -#include "sunway_iommu.h" +#include "iommu.h" -#define MAX_DOMAIN_NUM 65536 -#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) -#define SW64_DMA_LIMIT (0xe0000000 - 1) #define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) -#define SW64_IOMMU_LEVEL1_OFFSET 0x1ff -#define SW64_IOMMU_LEVEL2_OFFSET 0x3ff - #define SW64_IOMMU_GRN_8K ((0UL) << 4) /* page size as 8KB */ #define SW64_IOMMU_GRN_8M ((0x2UL) << 4) /* page size as 8MB */ #define SW64_IOMMU_PGSIZES (((1ULL) << PAGE_SHIFT) | ((1ULL) << PAGE_8M_SHIFT)) -#define IDENTMAP_ALL ((1U) << 0) -#define DMA_MASK64 ((1U) << 1) +#define IDENTMAP_ALL ((1U) << 0) + +#define MAX_NR_IOMMU_PER_NODE 8 /* IOMMU Exceptional Status */ enum exceptype { @@ -62,7 +58,7 @@ enum exceptype { PTE_LEVEL2_VAL, }; -u64 iommu_enable_cmd; /* default IOMMU boot param: 0 */ +DECLARE_BITMAP(iommu_bitmap, 32); unsigned long *sunway_iommu_domain_bitmap; @@ -81,58 +77,104 @@ const struct iommu_ops sunway_iommu_ops; static int iommu_identity_mapping; +static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) +{ + *(u16 *)data = alias; + return 0; +} + +static int get_alias(struct pci_dev *pdev) +{ + u16 pci_alias; + + /* As far as I know, few devices are using more than 2 aliases. */ + pci_for_each_dma_alias(pdev, __last_alias, &pci_alias); + + return pci_alias; +} + /* flush helpers */ -static void piu_flush_all(struct pci_controller *hose) +static void piu_flush_all(struct sunway_iommu *iommu) { - write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); - write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); - write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); + void __iomem *base; + + base = iommu->reg_base_addr; + if (!base) + return; + + writeq(0, base + DTLB_FLUSHALL); + writeq(0, base + PTLB_FLUSHALL); + writeq(0, base + PCACHE_FLUSHALL); } -void dev_flush_dtlb(struct sunway_iommu_domain *sdomain, - struct sunway_iommu_dev *sdev_data) +static void do_pcache_flush(struct sunway_iommu *iommu, + unsigned long flush_addr) { - struct pci_controller *hose; - int devid; + void __iomem *base; - list_for_each_entry(sdev_data, &sdomain->dev_list, list) { - hose = pci_bus_to_pci_controller(sdev_data->pdev->bus); - devid = sdev_data->devid; + base = iommu->reg_base_addr; + if (!base) + return; - write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, devid); - } + writeq(flush_addr, base + PCACHE_FLUSHPADDR); } void flush_pcache_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) { struct pci_controller *hose; + struct sunway_iommu *iommu; struct sunway_iommu_dev *sdev_data; list_for_each_entry(sdev_data, &sdomain->dev_list, list) { hose = pci_bus_to_pci_controller(sdev_data->pdev->bus); + iommu = hose->pci_iommu; flush_addr = __pa(flush_addr); - write_piu_ior0(hose->node, hose->index, - PCACHE_FLUSHPADDR, flush_addr); + do_pcache_flush(iommu, flush_addr); } } +static void do_ptlb_flush(struct sunway_iommu *iommu, + unsigned long flush_addr) +{ + void __iomem *base; + + base = iommu->reg_base_addr; + if (!base) + return; + + writeq(flush_addr, base + PTLB_FLUSHVADDR); +} + void flush_ptlb_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) { struct pci_controller *hose; struct pci_dev *pdev; struct sunway_iommu_dev *sdev_data; + struct sunway_iommu *iommu; + unsigned long address; + u16 alias, bus_number, devfn; list_for_each_entry(sdev_data, &sdomain->dev_list, list) { pdev = sdev_data->pdev; hose = pci_bus_to_pci_controller(pdev->bus); + iommu = hose->pci_iommu; - flush_addr = (pdev->bus->number << 8) + address = (pdev->bus->number << 8) | pdev->devfn | (flush_addr << 16); - write_piu_ior0(hose->node, hose->index, - PTLB_FLUSHVADDR, flush_addr); + do_ptlb_flush(iommu, address); + + if (sdev_data->alias != sdev_data->devid) { + alias = sdev_data->alias; + bus_number = PCI_BUS_NUM(alias); + devfn = alias & 0xff; + + address = (bus_number << 8) + | devfn | (flush_addr << 16); + do_ptlb_flush(iommu, address); + } } } @@ -279,18 +321,29 @@ static struct dma_domain *dma_domain_alloc(void) sunway_domain_init(&dma_dom->sdomain); dma_dom->sdomain.type = IOMMU_DOMAIN_DMA; - dma_dom->sdomain.pt_root = (unsigned long *)get_zeroed_page(GFP_KERNEL); - if (dma_dom->sdomain.pt_root == NULL) { - pr_err("Allocating a new sdomain pt_root failed!\n"); - dma_domain_free(dma_dom); - return NULL; - } - add_domain_to_list(&dma_dom->sdomain); return dma_dom; } +static void do_flush_dev(struct pci_controller *hose, u16 devid) +{ + struct sunway_iommu *iommu; + void __iomem *base; + + iommu = hose->pci_iommu; + if (!iommu) + return; + + base = iommu->reg_base_addr; + if (!base) + return; + + writeq(devid, base + DTLB_FLUSHDEV); + writeq(devid, base + PTLB_FLUSHDEV); + writeq(devid, base + PCACHE_FLUSHDEV); +} + static void device_flush_all(struct sunway_iommu_dev *sdata) { struct pci_controller *hose = pci_bus_to_pci_controller(sdata->pdev->bus); @@ -298,51 +351,80 @@ static void device_flush_all(struct sunway_iommu_dev *sdata) if (hose == NULL) return; - write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, sdata->devid); - write_piu_ior0(hose->node, hose->index, PTLB_FLUSHDEV, sdata->devid); - write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHDEV, sdata->devid); + do_flush_dev(hose, sdata->devid); + + if (sdata->devid != sdata->alias) + do_flush_dev(hose, sdata->alias); } /* iommu_ops device attach/unattach helpers */ -static void -set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain) +static int +set_entry_by_devid(u16 devid, + struct sunway_iommu_domain *sdomain, + struct sunway_iommu *iommu) { - struct sunway_iommu *iommu; - struct pci_dev *pdev; - struct page *page; + struct page *dt_page, *pt_page; unsigned long *dte_l1, *dte_l2; unsigned long dte_l1_val, dte_l2_base, dte_l2_val; + u16 bus_number, devfn; + int node; - pdev = sdev->pdev; - if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) - return; + bus_number = PCI_BUS_NUM(devid); + devfn = devid & 0xff; - sdev->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); - iommu = sdev->iommu; - dte_l1 = iommu->iommu_dtbr + (pdev->bus->number); + dte_l1 = iommu->iommu_dtbr + bus_number; dte_l1_val = *dte_l1; + node = node_online(iommu->node) ? iommu->node : NUMA_NO_NODE; if (!dte_l1_val) { /* Alloc a new level-2 device table page */ - page = alloc_pages_node(iommu->node, __GFP_ZERO, + dt_page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, get_order(PAGE_SIZE)); - if (!page) { - pr_err("Allocating a new level-2 device table page failed.\n"); - return; - } + if (!dt_page) + return -ENOMEM; - dte_l2_base = (unsigned long)page_address(page); + dte_l2_base = (unsigned long)page_address(dt_page); dte_l1_val = (__pa(dte_l2_base) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; *dte_l1 = dte_l1_val; } - dte_l2 = __va(dte_l1_val & ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + (pdev->devfn << 3); + if (!sdomain->pt_root) { + pt_page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0); + if (!pt_page) + return -ENOMEM; + + sdomain->pt_root = page_address(pt_page); + } + + dte_l2 = __va(dte_l1_val & ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + (devfn << 3); dte_l2_val = (__pa(sdomain->pt_root) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; - if (iommu_identity_mapping) { + + if (sdomain->type == IOMMU_DOMAIN_IDENTITY) dte_l2_val |= 0x1; - sdev->passthrough = IDENTMAP_ALL; - } + *dte_l2 = dte_l2_val; + pr_debug("iommu: device with id %d added to domain: %d\n", devid, sdomain->id); + + return 0; +} + +static void +set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu *iommu; + struct pci_dev *pdev; + + pdev = sdev->pdev; + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + return; + + iommu = sdev->iommu; + set_entry_by_devid(sdev->devid, sdomain, iommu); + if (sdev->devid != sdev->alias) + set_entry_by_devid(sdev->alias, sdomain, iommu); + + if (sdomain->type == IOMMU_DOMAIN_IDENTITY) + sdev->passthrough = IDENTMAP_ALL; device_flush_all(sdev); } @@ -356,7 +438,7 @@ do_attach(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomai sdomain->dev_cnt++; set_dte_entry(sdev_data, sdomain); - pr_debug("iommu: device %d add to domain: %d\n", + pr_debug("device %d add to domain: %d\n", sdev_data->devid, sdomain->id); } @@ -460,43 +542,6 @@ static struct sunway_iommu_dev *search_dev_data(u16 devid) * **********************************************************************/ -static struct sunway_iommu *sunway_iommu_early_init(struct pci_controller *hose) -{ - struct sunway_iommu *iommu; - struct page *page; - unsigned long base; - - hose->pci_iommu = kzalloc(sizeof(struct sunway_iommu), GFP_KERNEL); - if (!hose->pci_iommu) - return 0; - - iommu = hose->pci_iommu; - spin_lock_init(&iommu->dt_lock); - - iommu->node = hose->node; - if (!node_online(hose->node)) - iommu->node = -1; - - page = alloc_pages_node(iommu->node, __GFP_ZERO, get_order(PAGE_SIZE)); - if (!page) { - pr_err("Allocating a new iommu_dtbr page failed.\n"); - kfree(hose->pci_iommu); - return NULL; - } - - iommu->iommu_dtbr = page_address(page); - - iommu->hose_pt = hose; - iommu->index = hose->index; - - iommu->enabled = true; - - base = __pa(iommu->iommu_dtbr) & PAGE_MASK; - write_piu_ior0(hose->node, hose->index, DTBASEADDR, base); - - return iommu; -} - unsigned long fetch_dte(struct sunway_iommu *iommu, unsigned long devid, enum exceptype type) { @@ -571,11 +616,13 @@ irqreturn_t iommu_interrupt(int irq, void *dev) struct pci_controller *hose = (struct pci_controller *)dev; struct sunway_iommu_domain *sdomain; struct sunway_iommu_dev *sdev; + struct sunway_iommu *iommu; unsigned long iommu_status; unsigned long type; unsigned long devid, dva; - iommu_status = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); + iommu = hose->pci_iommu; + iommu_status = readq(iommu->reg_base_addr + IOMMUEXCPT_STATUS); if (!(iommu_status >> 63)) return IRQ_NONE; @@ -590,13 +637,13 @@ irqreturn_t iommu_interrupt(int irq, void *dev) pr_info("no such dev!!!\n"); iommu_status &= ~(1UL << 62); - write_piu_ior0(hose->node, hose->index, - IOMMUEXCPT_STATUS, iommu_status); + writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); return IRQ_HANDLED; } sdomain = sdev->domain; + switch (type) { case DTE_LEVEL1: pr_info("invalid level1 dte, addr:%#lx, val:%#lx\n", @@ -619,8 +666,7 @@ irqreturn_t iommu_interrupt(int irq, void *dev) fetch_pte(sdomain, dva, PTE_LEVEL2_VAL)); iommu_status &= ~(1UL << 62); - write_piu_ior0(hose->node, hose->index, - IOMMUEXCPT_STATUS, iommu_status); + writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); break; case UNAUTHORIZED_ACCESS: @@ -645,6 +691,7 @@ struct irqaction iommu_irqaction = { void sunway_enable_iommu_func(struct pci_controller *hose) { + struct sunway_iommu *iommu; unsigned int iommu_irq, err; unsigned long iommu_conf, iommu_ctrl; @@ -656,28 +703,18 @@ void sunway_enable_iommu_func(struct pci_controller *hose) if (err < 0) pr_info("sw iommu request irq failed!\n"); + iommu = hose->pci_iommu; iommu_ctrl = (1UL << 63) | (0x100UL << 10); - write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_CTRL, iommu_ctrl); - iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + writeq(iommu_ctrl, iommu->reg_base_addr + IOMMUEXCPT_CTRL); + iommu_conf = readq(iommu->reg_base_addr + PIUCONFIG0); iommu_conf = iommu_conf | (0x3 << 7); - write_piu_ior0(hose->node, hose->index, PIUCONFIG0, iommu_conf); - write_piu_ior0(hose->node, hose->index, TIMEOUT_CONFIG, 0xf); - iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + writeq(iommu_conf, iommu->reg_base_addr + PIUCONFIG0); + writeq(0xf, iommu->reg_base_addr + TIMEOUT_CONFIG); + iommu_conf = readq(iommu->reg_base_addr + PIUCONFIG0); pr_debug("SW arch configure node %ld hose-%ld iommu_conf = %#lx\n", hose->node, hose->index, iommu_conf); } -static bool is_iommu_enable(struct pci_controller *hose) -{ - u64 rc_mask = 0x1; - - rc_mask <<= (8 * hose->node + hose->index); - if (iommu_enable_cmd & rc_mask) - return true; - - return false; -} - /* iommu cpu syscore ops */ static int iommu_cpu_suspend(void) { @@ -696,12 +733,53 @@ struct syscore_ops iommu_cpu_syscore_ops = { static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type); +static struct sunway_iommu *sunway_iommu_early_init(struct pci_controller *hose) +{ + struct sunway_iommu *iommu; + struct page *page; + unsigned long base; + int ret = 0, node; + + iommu = kzalloc(sizeof(struct sunway_iommu), GFP_KERNEL); + if (!iommu) { + ret = -ENOMEM; + return 0; + } + + spin_lock_init(&iommu->dt_lock); + + iommu->node = hose->node; + iommu->index = hose->index; + + node = node_online(iommu->node) ? iommu->node : NUMA_NO_NODE; + page = alloc_pages_node(node, __GFP_ZERO, get_order(PAGE_SIZE)); + if (!page) { + ret = -ENOMEM; + goto free_iommu; + } + + iommu->iommu_dtbr = page_address(page); + base = __pa(iommu->iommu_dtbr) & PAGE_MASK; + iommu->reg_base_addr = __va(MK_PIU_IOR0(iommu->node, iommu->index)); + writeq(base, iommu->reg_base_addr + DTBASEADDR); + + hose->pci_iommu = iommu; + iommu->enabled = true; + return iommu; + +free_iommu: + kfree(iommu); + + return ERR_PTR(ret); +} + static int sunway_iommu_init(void) { struct pci_controller *hose; struct sunway_iommu *iommu; - int ret; + unsigned long rc_mask; int iommu_index = 0; + int ret; sunway_iommu_domain_bitmap = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, @@ -712,7 +790,8 @@ static int sunway_iommu_init(void) /* Do the loop */ for (hose = hose_head; hose; hose = hose->next) { - if (!is_iommu_enable(hose)) { + rc_mask = MAX_NR_IOMMU_PER_NODE * hose->node + hose->index; + if (!test_bit(rc_mask, iommu_bitmap)) { hose->iommu_enable = false; continue; } @@ -726,26 +805,24 @@ static int sunway_iommu_init(void) iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", iommu_index); - iommu_index++; + iommu_device_register(&iommu->iommu, &sunway_iommu_ops, NULL); + sunway_enable_iommu_func(hose); hose->iommu_enable = true; + piu_flush_all(iommu); - iommu_device_register(&iommu->iommu, &sunway_iommu_ops, NULL); + iommu_index++; } ret = iova_cache_get(); if (ret) return ret; - for (hose = hose_head; hose; hose = hose->next) - if (hose->iommu_enable) - piu_flush_all(hose); - register_syscore_ops(&iommu_cpu_syscore_ops); - return 1; + return 0; } -device_initcall(sunway_iommu_init); +subsys_initcall_sync(sunway_iommu_init); /******************************************************************************* * @@ -790,7 +867,7 @@ sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, unsigned long bus_addr, unsigned long paddr, - size_t page_size) + size_t page_size, int iommu_prot) { /* * pde: page table entry @@ -856,7 +933,12 @@ int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, } pte = (paddr & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID - | SW64_IOMMU_GRN_8K | SW64_IOMMU_ENABLE; + | SW64_IOMMU_GRN_8K; + + pte |= PTE_READE; + if (iommu_prot & IOMMU_WRITE) + pte |= PTE_WRITEE; + *(volatile u64 *)ptebaseaddr = pte; flush_pcache_by_addr(sunway_domain, ptebaseaddr); /* case 8M */ @@ -869,7 +951,11 @@ int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, ptes_one_cache = L1_CACHE_BYTES/sizeof(pte); pte = (paddr & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID - | SW64_IOMMU_GRN_8M | SW64_IOMMU_ENABLE; + | SW64_IOMMU_GRN_8M; + + pte |= PTE_READE; + if (iommu_prot & IOMMU_WRITE) + pte |= PTE_WRITEE; for (i = 0; i < ptes_one_page; i++) { if (*ptr) { @@ -1084,7 +1170,7 @@ sunway_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, mutex_lock(&sdomain->api_lock); while (pgcount--) { - ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size); + ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size, iommu_prot); if (ret) { pr_info("Failed to map page from IOVA %lx.\n", iova); return ret; @@ -1143,6 +1229,9 @@ static int iommu_init_device(struct device *dev) return -ENOMEM; pdev = to_pci_dev(dev); + sdev->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); + sdev->alias = get_alias(pdev); + hose = pci_bus_to_pci_controller(pdev->bus); iommu = hose->pci_iommu; llist_add(&sdev->dev_data_list, &dev_data_list); @@ -1219,10 +1308,9 @@ static struct iommu_device *sunway_iommu_probe_device(struct device *dev) static int sunway_iommu_def_domain_type(struct device *dev) { - if (dev_is_pci(dev)) { + if (dev_is_pci(dev)) if (iommu_identity_mapping) return IOMMU_DOMAIN_IDENTITY; - } return 0; } @@ -1234,8 +1322,13 @@ static bool sunway_iommu_capable(struct device *dev, enum iommu_cap cap) static void sunway_iommu_probe_finalize(struct device *dev) { - set_dma_ops(dev, NULL); - iommu_setup_dma_ops(dev, 0, SW64_DMA_LIMIT); + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (domain->type == IOMMU_DOMAIN_DMA) { + iommu_setup_dma_ops(dev, 0, SW64_32BIT_DMA_LIMIT); + } else + set_dma_ops(dev, get_arch_dma_ops()); } const struct iommu_ops sunway_iommu_ops = { @@ -1264,14 +1357,49 @@ const struct iommu_ops sunway_iommu_ops = { * rc0 for cpu node 1. * *****************************************************************************/ +static int __init sunway_iommu_setup(char *str) +{ + unsigned long rc_val; + int ret; + + if (!str) + return -EINVAL; + + bitmap_zero(iommu_bitmap, 64); + + if (!strncmp(str, "on", 2)) { + bitmap_fill(iommu_bitmap, 64); + } else if (!strncmp(str, "off", 3)) { + bitmap_zero(iommu_bitmap, 64); + } else { + ret = kstrtoul(str, 16, &rc_val); + if (!ret) + return -EINVAL; + + bitmap_from_u64(iommu_bitmap, rc_val); + } + + return 1; +} +__setup("sunway_iommu=", sunway_iommu_setup); + static int __init iommu_enable_setup(char *str) { + unsigned long rc_val; int ret; - unsigned long rc_bitmap = 0xffffffffUL; - ret = kstrtoul(str, 16, &rc_bitmap); - iommu_enable_cmd = rc_bitmap; + if (!str) + return -EINVAL; - return ret; + pr_info("iommu_enable= deprecated; use sunway_iommu=on/off instead.\n"); + bitmap_zero(iommu_bitmap, 64); + + ret = kstrtoul(str, 16, &rc_val); + if (!ret) + return -EINVAL; + + bitmap_from_u64(iommu_bitmap, rc_val); + + return 1; } -early_param("iommu_enable", iommu_enable_setup); +__setup("iommu_enable=", iommu_enable_setup); diff --git a/drivers/iommu/sw64/sunway_iommu.h b/drivers/iommu/sw64/iommu.h similarity index 88% rename from drivers/iommu/sw64/sunway_iommu.h rename to drivers/iommu/sw64/iommu.h index 94a155001d1b..f203d19234e7 100644 --- a/drivers/iommu/sw64/sunway_iommu.h +++ b/drivers/iommu/sw64/iommu.h @@ -14,14 +14,16 @@ struct sunway_iommu_bypass_id { }; struct sunway_iommu { - int index; bool enabled; + unsigned long index; + unsigned long node; + void __iomem *reg_base_addr; unsigned long *iommu_dtbr; spinlock_t dt_lock; /* Device Table Lock */ - int node; /* NUMA node */ struct pci_controller *hose_pt; struct iommu_device iommu; /* IOMMU core code handle */ + struct list_head list; }; struct sunway_iommu_dev { @@ -58,6 +60,11 @@ struct sunway_iommu_group { struct iommu_group *group; }; +#define MAX_DOMAIN_NUM 65536 +#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) + +#define SW64_32BIT_DMA_LIMIT (0xe0000000 - 1) + #define SW64_IOMMU_ENTRY_VALID ((1UL) << 63) #define SW64_PTE_LAST_MASK ((1UL) << 8) /*last stage valid*/ #define SW64_DMA_START 0x1000000 @@ -65,8 +72,8 @@ struct sunway_iommu_group { #define PAGE_8M_SHIFT 23 #define PAGE_512M_SHIFT 29 #define PAGE_8G_SHIFT 33 -#define SW64_IOMMU_ENABLE 3 -#define SW64_IOMMU_DISABLE 0 +#define PTE_WRITEE 0x2UL +#define PTE_READE 0x1UL #define SW64_IOMMU_LEVEL1_OFFSET 0x1ff #define SW64_IOMMU_LEVEL2_OFFSET 0x3ff #define SW64_IOMMU_LEVEL3_OFFSET 0x3ff diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index 4a8b576c2530..b2b68c9836a2 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -1,12 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 /* - * iommu.c: Generic sw64 IOMMU support - * - * This is designed and tested for 3231. If there are no changes in hardware - * in later chips, then it should work just as well. + * iommu.c: Generic sw64 IOMMU v2 support * */ +#define pr_fmt(fmt) "IOMMU: " fmt + #include #include #include @@ -17,48 +16,46 @@ #include #include #include -#include #include -#include #include #include #include #include #include #include -#include +#include #include +#include #include #include -#include "sunway_iommu.h" +#include "iommu.h" -#define MAX_DOMAIN_NUM 65536 -#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) -#define SW64_32BIT_DMA_LIMIT (0xe0000000 - 1) -#define SW64_64BIT_DMA_LIMIT ((1UL << 41) - 1) +#define SW64_64BIT_DMA_LIMIT ((1UL << 42) - 1) #define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) - -#define SW64_IOMMU_PGSIZES (((1ULL) << PAGE_SHIFT) \ - | ((1ULL) << PAGE_8M_SHIFT) \ - | ((1ULL) << PAGE_512M_SHIFT) \ - | ((1ULL) << PAGE_8G_SHIFT)) +#define IOVA_MAX_ADDRESS_WIDTH 42 #define IDENTMAP_ALL ((1U) << 0) -#define DMA_MASK64 ((1U) << 1) #define PTE_VALID 0x8000000000000000UL #define LAST_STAGE 0x100UL #define PTE_GRN_8M 0x10UL #define PTE_GRN_512M 0x20UL #define PTE_GRN_8G 0x30UL -#define PTE_WRITEE 0x2UL -#define PTE_READE 0x1UL -#define PTE_RWE 0x3UL #define PTE_FLAGS_MASK 0x8000000000000133UL #define PAGE_8G_OFFSET_MASK ((1UL << PAGE_8G_SHIFT) - 1) #define PAGE_512M_OFFSET_MASK ((1UL << PAGE_512M_SHIFT) - 1) #define PAGE_8M_OFFSET_MASK ((1UL << PAGE_8M_SHIFT) - 1) +#define MAX_NR_IOMMU_PER_NODE 16 + +DECLARE_BITMAP(iommu_bitmap, 64); +struct acpi_table_header *dmar_tbl; +#define for_each_iommu(iommu) \ + list_for_each_entry(iommu, &iommu_list, list) + +#define MAX_NR_IOMMU_PER_NODE 16 + +LIST_HEAD(iommu_list); /* IOMMU Exceptional Status */ enum exceptype { @@ -76,8 +73,6 @@ enum exceptype { PTE_LEVEL3_VAL, }; -u64 iommu_enable_cmd; /* default IOMMU boot param: 0 */ - unsigned long *sunway_iommu_domain_bitmap; static DEFINE_SPINLOCK(domain_bitmap_lock); @@ -93,46 +88,104 @@ struct dma_domain { }; const struct iommu_ops sunway_iommu_ops; static const struct dma_map_ops sunway_dma_ops; +static int iommu_identity_mapping; + +static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) +{ + *(u16 *)data = alias; + return 0; +} + +static int get_alias(struct pci_dev *pdev) +{ + u16 pci_alias; + + /* As far as I know, few devices are using more than 2 aliases. */ + pci_for_each_dma_alias(pdev, __last_alias, &pci_alias); + return pci_alias; +} /* flush helpers */ -static void piu_flush_all(struct pci_controller *hose) +static void piu_flush_all(struct sunway_iommu *iommu) +{ + void __iomem *base; + + base = iommu->reg_base_addr; + if (!base) + return; + + writeq(0, base + DTLB_FLUSHALL); + writeq(0, base + PTLB_FLUSHALL); + writeq(0, base + PCACHE_FLUSHALL); +} + +static void do_pcache_flush(struct sunway_iommu *iommu, + unsigned long flush_addr) { - write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); - write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); - write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); + void __iomem *base; + + base = iommu->reg_base_addr; + if (!base) + return; + + writeq(flush_addr, base + PCACHE_FLUSHPADDR); } void flush_pcache_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) { struct pci_controller *hose; struct sunway_iommu_dev *sdev; + struct sunway_iommu *iommu; list_for_each_entry(sdev, &sdomain->dev_list, list) { hose = pci_bus_to_pci_controller(sdev->pdev->bus); + iommu = hose->pci_iommu; flush_addr = __pa(flush_addr); - /* Set memory bar here */ - mb(); - write_piu_ior0(hose->node, hose->index, - PCACHE_FLUSHPADDR, flush_addr); + do_pcache_flush(iommu, flush_addr); } } +static void do_ptlb_flush(struct sunway_iommu *iommu, + unsigned long flush_addr) +{ + void __iomem *base; + + base = iommu->reg_base_addr; + if (!base) + return; + + writeq(flush_addr, base + PTLB_FLUSHVADDR); +} + void flush_ptlb_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) { struct pci_controller *hose; struct sunway_iommu_dev *sdev; + struct sunway_iommu *iommu; struct pci_dev *pdev; + unsigned long address; + u16 alias, bus_number, devfn; list_for_each_entry(sdev, &sdomain->dev_list, list) { pdev = sdev->pdev; hose = pci_bus_to_pci_controller(pdev->bus); + iommu = hose->pci_iommu; - flush_addr = (pdev->bus->number << 8) + address = (pdev->bus->number << 8) | pdev->devfn | (flush_addr << 16); - write_piu_ior0(hose->node, hose->index, - PTLB_FLUSHVADDR, flush_addr); + do_ptlb_flush(iommu, address); + + if (sdev->alias != sdev->devid) { + alias = sdev->alias; + bus_number = PCI_BUS_NUM(alias); + devfn = alias & 0xff; + + address = (bus_number << 8) + | devfn | (flush_addr << 16); + do_ptlb_flush(iommu, address); + } } } @@ -288,14 +341,30 @@ static struct dma_domain *dma_domain_alloc(void) sunway_domain_init(&dma_dom->sdomain); dma_dom->sdomain.type = IOMMU_DOMAIN_DMA; - init_iova_domain(&dma_dom->iovad, PAGE_SIZE, IOVA_PFN(SW64_DMA_START)); - reserve_iova(&dma_dom->iovad, (0xe0000000UL >> PAGE_SHIFT), (0x100000000UL >> PAGE_SHIFT)); add_domain_to_list(&dma_dom->sdomain); return dma_dom; } +static void do_flush_dev(struct pci_controller *hose, u16 devid) +{ + struct sunway_iommu *iommu; + void __iomem *base; + + iommu = hose->pci_iommu; + if (!iommu) + return; + + base = iommu->reg_base_addr; + if (!base) + return; + + writeq(devid, base + DTLB_FLUSHDEV); + writeq(devid, base + PTLB_FLUSHDEV); + writeq(devid, base + PCACHE_FLUSHDEV); +} + static void device_flush_all(struct sunway_iommu_dev *sdata) { struct pci_controller *hose = pci_bus_to_pci_controller(sdata->pdev->bus); @@ -303,38 +372,37 @@ static void device_flush_all(struct sunway_iommu_dev *sdata) if (hose == NULL) return; - write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, sdata->devid); - write_piu_ior0(hose->node, hose->index, PTLB_FLUSHDEV, sdata->devid); - write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHDEV, sdata->devid); + do_flush_dev(hose, sdata->devid); + + if (sdata->devid != sdata->alias) + do_flush_dev(hose, sdata->alias); } /* iommu_ops device attach/unattach helpers */ -static void -set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain) +static int +set_entry_by_devid(u16 devid, + struct sunway_iommu_domain *sdomain, + struct sunway_iommu *iommu) { - struct sunway_iommu *iommu; - struct pci_dev *pdev; struct page *dt_page, *pt_page; unsigned long *dte_l1, *dte_l2; unsigned long dte_l1_val, dte_l2_base, dte_l2_val; + u16 bus_number, devfn; + int node; - pdev = sdev->pdev; - if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) - return; + bus_number = PCI_BUS_NUM(devid); + devfn = devid & 0xff; - sdev->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); - iommu = sdev->iommu; - dte_l1 = iommu->iommu_dtbr + (pdev->bus->number); + dte_l1 = iommu->iommu_dtbr + bus_number; dte_l1_val = *dte_l1; if (!dte_l1_val) { + node = node_online(iommu->node) ? iommu->node : NUMA_NO_NODE; /* Alloc a new level-2 device table page */ dt_page = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, get_order(PAGE_SIZE)); - if (!dt_page) { - pr_err("Allocating a new level-2 device table page failed.\n"); - return; - } + if (!dt_page) + return -ENOMEM; dte_l2_base = (unsigned long)page_address(dt_page); dte_l1_val = (__pa(dte_l2_base) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; @@ -342,22 +410,42 @@ set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain } if (!sdomain->pt_root) { + node = node_online(iommu->node) ? iommu->node : NUMA_NO_NODE; pt_page = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, 0); - if (!pt_page) { - pr_err("Allocating pt_root failed!\n"); - return; - } + if (!pt_page) + return -ENOMEM; sdomain->pt_root = page_address(pt_page); } - dte_l2 = __va(dte_l1_val & ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + (pdev->devfn << 3); + dte_l2 = __va(dte_l1_val & ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + (devfn << 3); dte_l2_val = (__pa(sdomain->pt_root) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; - if (sdomain->type == IOMMU_DOMAIN_IDENTITY) { + if (sdomain->type == IOMMU_DOMAIN_IDENTITY) dte_l2_val |= 0x1; - sdev->passthrough = IDENTMAP_ALL; - } + *dte_l2 = dte_l2_val; + pr_debug("iommu: device with id %d added to domain: %d\n", devid, sdomain->id); + + return 0; +} + +static void set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu *iommu; + struct pci_dev *pdev; + + pdev = sdev->pdev; + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + return; + + iommu = sdev->iommu; + set_entry_by_devid(sdev->devid, sdomain, iommu); + if (sdev->devid != sdev->alias) + set_entry_by_devid(sdev->alias, sdomain, iommu); + + if (sdomain->type == IOMMU_DOMAIN_IDENTITY) + sdev->passthrough = IDENTMAP_ALL; + device_flush_all(sdev); } @@ -468,32 +556,6 @@ static struct sunway_iommu_dev *search_dev_data(u16 devid) return NULL; } -/* dma_ops helpers*/ -static struct sunway_iommu_domain *get_sunway_domain(struct device *dev) -{ - struct sunway_iommu_domain *sdomain; - struct iommu_domain *domain; - struct pci_dev *pdev; - struct sunway_iommu_dev *sdev; - - pdev = to_pci_dev(dev); - if (!pdev) - return ERR_PTR(-ENODEV); - - sdev = dev_iommu_priv_get(dev); - sdomain = sdev->domain; - if (sdomain == NULL) { - domain = iommu_get_domain_for_dev(dev); - sdomain = to_sunway_domain(domain); - attach_device(dev, sdomain); - } - - if (sdomain == NULL) - return ERR_PTR(-EBUSY); - - return sdomain; -} - /********************************************************************** * * Following functions describe IOMMU init ops @@ -505,35 +567,41 @@ static struct sunway_iommu *sunway_iommu_early_init(struct pci_controller *hose) struct sunway_iommu *iommu; struct page *page; unsigned long base; + int ret = 0; + int node; - hose->pci_iommu = kzalloc(sizeof(struct sunway_iommu), GFP_KERNEL); - if (!hose->pci_iommu) - return 0; + iommu = kzalloc(sizeof(struct sunway_iommu), GFP_KERNEL); + if (!iommu) { + ret = -ENOMEM; + goto out; + } - iommu = hose->pci_iommu; spin_lock_init(&iommu->dt_lock); iommu->node = hose->node; - if (!node_online(hose->node)) - iommu->node = -1; + iommu->index = hose->index; - page = alloc_pages_node(iommu->node, __GFP_ZERO, get_order(PAGE_SIZE)); + node = node_online(iommu->node) ? iommu->node : NUMA_NO_NODE; + page = alloc_pages_node(node, __GFP_ZERO, get_order(PAGE_SIZE)); if (!page) { - pr_err("Allocating a new iommu_dtbr page failed.\n"); - kfree(hose->pci_iommu); - return NULL; + ret = -ENOMEM; + goto free_iommu; } - iommu->iommu_dtbr = page_address(page); - iommu->hose_pt = hose; - iommu->index = hose->index; + iommu->iommu_dtbr = page_address(page); + base = __pa(iommu->iommu_dtbr) & PAGE_MASK; + iommu->reg_base_addr = __va(MK_PIU_IOR0(iommu->node, iommu->index)); + writeq(base, iommu->reg_base_addr + DTBASEADDR); + hose->pci_iommu = iommu; iommu->enabled = true; - base = __pa(iommu->iommu_dtbr) & PAGE_MASK; - write_piu_ior0(hose->node, hose->index, DTBASEADDR, base); - return iommu; + +free_iommu: + kfree(iommu); +out: + return ERR_PTR(ret); } unsigned long fetch_dte(struct sunway_iommu *iommu, unsigned long devid, @@ -621,12 +689,14 @@ irqreturn_t iommu_interrupt(int irq, void *dev) { struct pci_controller *hose = (struct pci_controller *)dev; struct sunway_iommu_domain *sdomain; + struct sunway_iommu *iommu; struct sunway_iommu_dev *sdev; unsigned long iommu_status; unsigned long type; unsigned long devid, dva; - iommu_status = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); + iommu = hose->pci_iommu; + iommu_status = readq(iommu->reg_base_addr + IOMMUEXCPT_STATUS); if (!(iommu_status >> 63)) return IRQ_NONE; @@ -641,23 +711,22 @@ irqreturn_t iommu_interrupt(int irq, void *dev) pr_info("no such dev!!!\n"); iommu_status &= ~(1UL << 62); - write_piu_ior0(hose->node, hose->index, - IOMMUEXCPT_STATUS, iommu_status); + writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); return IRQ_HANDLED; } - sdomain = sdev->domain; + sdomain = sdev->domain; switch (type) { case DTE_LEVEL1: pr_info("invalid level1 dte, addr:%#lx, val:%#lx\n", - fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1), - fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1_VAL)); + fetch_dte(iommu, devid, DTE_LEVEL1), + fetch_dte(iommu, devid, DTE_LEVEL1_VAL)); break; case DTE_LEVEL2: pr_info("invalid level2 dte, addr:%#lx, val:%#lx\n", - fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2), - fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2_VAL)); + fetch_dte(iommu, devid, DTE_LEVEL2), + fetch_dte(iommu, devid, DTE_LEVEL2_VAL)); break; case PTE_LEVEL1: pr_info("invalid level1 pte, addr: %#lx, val:%#lx\n", @@ -665,8 +734,7 @@ irqreturn_t iommu_interrupt(int irq, void *dev) fetch_pte(sdomain, dva, PTE_LEVEL1_VAL)); iommu_status &= ~(1UL << 62); - write_piu_ior0(hose->node, hose->index, - IOMMUEXCPT_STATUS, iommu_status); + writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); break; case PTE_LEVEL2: pr_info("invalid level2 pte, addr: %#lx, val: %#lx\n", @@ -674,8 +742,7 @@ irqreturn_t iommu_interrupt(int irq, void *dev) fetch_pte(sdomain, dva, PTE_LEVEL2_VAL)); iommu_status &= ~(1UL << 62); - write_piu_ior0(hose->node, hose->index, - IOMMUEXCPT_STATUS, iommu_status); + writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); break; case PTE_LEVEL3: @@ -684,8 +751,7 @@ irqreturn_t iommu_interrupt(int irq, void *dev) fetch_pte(sdomain, dva, PTE_LEVEL3_VAL)); iommu_status &= ~(1UL << 62); - write_piu_ior0(hose->node, hose->index, - IOMMUEXCPT_STATUS, iommu_status); + writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); break; default: pr_info("iommu exception type %ld\n", type); @@ -703,6 +769,7 @@ struct irqaction iommu_irqaction = { void sunway_enable_iommu_func(struct pci_controller *hose) { + struct sunway_iommu *iommu; unsigned int iommu_irq, err; unsigned long iommu_conf, iommu_ctrl; @@ -714,28 +781,18 @@ void sunway_enable_iommu_func(struct pci_controller *hose) if (err < 0) pr_info("sw iommu request irq failed!\n"); + iommu = hose->pci_iommu; iommu_ctrl = (1UL << 63) | (0x100UL << 10); - write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_CTRL, iommu_ctrl); - iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + writeq(iommu_ctrl, iommu->reg_base_addr + IOMMUEXCPT_CTRL); + iommu_conf = readq(iommu->reg_base_addr + PIUCONFIG0); iommu_conf = iommu_conf | (0x3 << 7); - write_piu_ior0(hose->node, hose->index, PIUCONFIG0, iommu_conf); - write_piu_ior0(hose->node, hose->index, TIMEOUT_CONFIG, 0xf); - iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + writeq(iommu_conf, iommu->reg_base_addr + PIUCONFIG0); + writeq(0xf, iommu->reg_base_addr + TIMEOUT_CONFIG); + iommu_conf = readq(iommu->reg_base_addr + PIUCONFIG0); pr_debug("SW arch configure node %ld hose-%ld iommu_conf = %#lx\n", hose->node, hose->index, iommu_conf); } -static bool is_iommu_enable(struct pci_controller *hose) -{ - u64 rc_mask = 0x1; - - rc_mask <<= (8 * hose->node + hose->index); - if (iommu_enable_cmd & rc_mask) - return true; - - return false; -} - /* iommu cpu syscore ops */ static int iommu_cpu_suspend(void) { @@ -754,58 +811,239 @@ struct syscore_ops iommu_cpu_syscore_ops = { static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type); -static int sunway_iommu_init(void) +/* Init functions */ +static int do_detect(void) +{ + acpi_status status = AE_OK; + + status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl); + + if (ACPI_SUCCESS(status) && !dmar_tbl) { + pr_warn("No DMAR found!\n"); + status = AE_NOT_FOUND; + } + + return ACPI_SUCCESS(status) ? 0 : -ENOENT; +} + +static struct pci_controller *find_hose_by_rcid(int node, int index) { struct pci_controller *hose; + + for (hose = hose_head; hose; hose = hose->next) + if (hose->node == node && hose->index == index) + return hose; + + return NULL; +} + +static int parse_one_drhd_unit(struct acpi_sw_dmar_header *header) +{ + struct acpi_dmar_sw_hardware_unit *drhd; struct sunway_iommu *iommu; + struct pci_controller *hose; + struct page *page; + unsigned long base; + int cmdline_enabled; + int rc_mask, ret, node; + int rc_node, rc_index; + + drhd = (struct acpi_dmar_sw_hardware_unit *)header; + if (!drhd->enable) + return 0; + + rc_node = (drhd->index >> 8) & 0xff; + rc_index = drhd->index & 0xff; + + hose = find_hose_by_rcid(rc_node, rc_index); + if (!hose) + return 0; + + iommu = kzalloc(sizeof(struct sunway_iommu), GFP_KERNEL); + if (!iommu) + return -ENOMEM; + + iommu->node = rc_node; + iommu->index = rc_index; + iommu->reg_base_addr = ioremap(drhd->address, drhd->size); + + rc_mask = MAX_NR_IOMMU_PER_NODE * iommu->node + iommu->index; + cmdline_enabled = test_bit(rc_mask, iommu_bitmap); + if (!cmdline_enabled) { + iommu->enabled = false; + ret = 0; + goto free_iommu; + } + + node = node_online(iommu->node) ? iommu->node : NUMA_NO_NODE; + page = alloc_pages_node(node, __GFP_ZERO, get_order(PAGE_SIZE)); + if (!page) { + ret = -ENOMEM; + goto free_iommu; + } + + iommu->iommu_dtbr = page_address(page); + base = __pa(iommu->iommu_dtbr) & PAGE_MASK; + writeq(base, iommu->reg_base_addr + DTBASEADDR); + + list_add(&iommu->list, &iommu_list); + iommu->enabled = true; + + hose->pci_iommu = iommu; + + pr_info("iommu: node: %ld index: %ld IOMMU enabled!\n", + iommu->node, iommu->index); + return 0; + +free_iommu: + kfree(iommu); + return ret; +} + +static int parse_drhd_units(struct acpi_table_sw_dmar *dmar) +{ + struct acpi_sw_dmar_header *iter, *start, *next, *end; + size_t len = dmar->header.length - sizeof(*dmar); + int ret, count = 0; + + /* Skip DMAR table, point to first DRHD table. */ + start = (struct acpi_sw_dmar_header *)(dmar + 1); + end = ((void *)start) + len; + + for (iter = start; iter < end; iter = next) { + next = (void *)iter + iter->length; + if (iter->length == 0) { + pr_warn(FW_BUG "Invalid 0-length structure\n"); + break; + } else if (next > end) { + pr_warn(FW_BUG "Record passes table end\n"); + return -EINVAL; + } + + if (iter->type >= ACPI_SW_DMAR_TYPE_RESERVED) { + pr_info("Unknown DMAR structure type %d\n", + iter->type); + } else if (iter->type == 0) { + ret = parse_one_drhd_unit(iter); + if (ret) + return ret; + } + count++; + } + + return 0; +} + +static int sunway_iommu_acpi_early_init(void) +{ int ret; + + struct acpi_table_sw_dmar *dmar; + + ret = do_detect(); + if (ret) + return ret; + + dmar = (struct acpi_table_sw_dmar *)dmar_tbl; + if (!dmar) + return -ENODEV; + + if (dmar->width < 42) { + pr_warn("Invalid DMAR haw\n"); + return -EINVAL; + } + pr_info("Host address width: %d\n", dmar->width); + + ret = parse_drhd_units(dmar); + + return ret; +} + +static int sunway_iommu_acpi_init(void) +{ + struct sunway_iommu *iommu; + struct pci_controller *hose; int iommu_index = 0; + int ret; - sunway_iommu_domain_bitmap = - (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(MAX_DOMAIN_NUM / 8)); - if (sunway_iommu_domain_bitmap == NULL) - return 0; - __set_bit(0, sunway_iommu_domain_bitmap); + ret = sunway_iommu_acpi_early_init(); + if (ret) + return ret; + + for_each_iommu(iommu) { + if (!iommu->enabled) + continue; + iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", + iommu_index); + iommu_device_register(&iommu->iommu, &sunway_iommu_ops, NULL); + iommu_index++; + hose = find_hose_by_rcid(iommu->node, iommu->index); + sunway_enable_iommu_func(hose); + hose->iommu_enable = true; + piu_flush_all(iommu); + } + + ret = iova_cache_get(); + if (ret) + return ret; + + register_syscore_ops(&iommu_cpu_syscore_ops); + + return 0; +} + +static int sunway_iommu_legacy_init(void) +{ + struct pci_controller *hose; + struct sunway_iommu *iommu; + unsigned long rc_mask; + int iommu_index = 0; + int ret; /* Do the loop */ for (hose = hose_head; hose; hose = hose->next) { - if (!is_iommu_enable(hose)) { + rc_mask = MAX_NR_IOMMU_PER_NODE * hose->node + hose->index; + if (!test_bit(rc_mask, iommu_bitmap)) { hose->iommu_enable = false; continue; } iommu = sunway_iommu_early_init(hose); - if (!iommu) { - pr_err("Allocating sunway_iommu failed\n"); - hose->iommu_enable = false; - continue; - } - iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", iommu_index); - iommu_device_set_ops(&iommu->iommu, &sunway_iommu_ops); - iommu_device_register(&iommu->iommu); + iommu_device_register(&iommu->iommu, &sunway_iommu_ops, NULL); iommu_index++; sunway_enable_iommu_func(hose); hose->iommu_enable = true; + piu_flush_all(iommu); } ret = iova_cache_get(); if (ret) return ret; - ret = bus_set_iommu(&pci_bus_type, &sunway_iommu_ops); - if (ret) - return ret; + register_syscore_ops(&iommu_cpu_syscore_ops); - for (hose = hose_head; hose; hose = hose->next) - if (hose->iommu_enable) - piu_flush_all(hose); + return 0; +} - register_syscore_ops(&iommu_cpu_syscore_ops); +static int sunway_iommu_init(void) +{ + int ret; - return 1; + sunway_iommu_domain_bitmap = + (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(MAX_DOMAIN_NUM / 8)); + if (!sunway_iommu_domain_bitmap) + return 0; + __set_bit(0, sunway_iommu_domain_bitmap); + + if (!acpi_disabled) + ret = sunway_iommu_acpi_init(); + else + ret = sunway_iommu_legacy_init(); + + return ret; } subsys_initcall_sync(sunway_iommu_init); @@ -890,7 +1128,7 @@ sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, unsigned long bus_addr, unsigned long paddr, - size_t page_size) + size_t page_size, int iommu_prot) { struct page *page; struct sunway_iommu *iommu; @@ -930,7 +1168,7 @@ int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, pte = &pte_base[offset]; if (!(*pte) || (current_level == level)) { - pte_val = PTE_VALID | PTE_RWE | grn; + pte_val = PTE_VALID | grn; if (current_level == level) { *(volatile u64 *)(pte) = 0; pte_val |= ((paddr & PAGE_MASK) | LAST_STAGE); @@ -944,6 +1182,10 @@ int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, pte_val |= (page_to_phys(page) & PAGE_MASK); } + pte_val |= PTE_READE; + if (iommu_prot & IOMMU_WRITE) + pte_val |= PTE_WRITEE; + if ((grn == PTE_GRN_512M) && (current_level == 2)) { int i; @@ -967,410 +1209,6 @@ int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, return 0; } -static unsigned long -sunway_alloc_iova(struct dma_domain *dma_dom, unsigned long pages, struct pci_dev *pdev) -{ - struct device *dev; - unsigned long pfn = 0; - - pages = __roundup_pow_of_two(pages); - dev = &(pdev->dev); - if (min(dev->coherent_dma_mask, *dev->dma_mask) == DMA_BIT_MASK(32)) { - pfn = alloc_iova_fast(&dma_dom->iovad, pages, - IOVA_PFN(SW64_32BIT_DMA_LIMIT), true); - } else { - /* IOVA boundary should be 16M ~ 3.5G */ - pfn = alloc_iova_fast(&dma_dom->iovad, pages, - IOVA_PFN(SW64_64BIT_DMA_LIMIT), true); - } - - return (pfn << PAGE_SHIFT); -} - -static void sunway_free_iova(struct dma_domain *dma_dom, - unsigned long address, unsigned long pages) -{ - pages = __roundup_pow_of_two(pages); - address >>= PAGE_SHIFT; - - free_iova_fast(&dma_dom->iovad, address, pages); -} - -static dma_addr_t -__sunway_map_single(struct dma_domain *dma_dom, - struct pci_dev *pdev, phys_addr_t paddr, size_t size) -{ - dma_addr_t ret, address, start; - unsigned long npages, i; - - npages = iommu_num_pages(paddr, size, PAGE_SIZE); - - address = sunway_alloc_iova(dma_dom, npages, pdev); - if (!address) - return 0; - - start = address; - for (i = 0; i < npages; ++i) { - ret = sunway_iommu_map_page(&dma_dom->sdomain, start, - paddr, PAGE_SIZE); - if (ret) { - pr_info("error when map page.\n"); - goto out_unmap; - } - - start += PAGE_SIZE; - paddr += PAGE_SIZE; - } - - address += paddr & ~PAGE_MASK; - return address; - -out_unmap: - for (--i; i >= 0; --i) { - start -= PAGE_SIZE; - sunway_iommu_unmap_page(&dma_dom->sdomain, start, PAGE_SIZE); - } - - sunway_free_iova(dma_dom, address, npages); - return 0; -} - -static dma_addr_t -pci_iommu_map_single(struct pci_dev *pdev, - struct dma_domain *dma_dom, void *cpu_addr, size_t size) -{ - struct pci_controller *hose = pci_bus_to_pci_controller(pdev->bus); - unsigned long paddr; - - if (hose == NULL) { - pr_err("%s: hose does not exist!\n", __func__); - return 0; - } - - paddr = __sunway_map_single(dma_dom, pdev, __pa(cpu_addr), size); - - pr_debug("pci_alloc_consistent: %zx -> [%px,%lx] from %ps\n", - size, cpu_addr, paddr, __builtin_return_address(0)); - - return paddr; -} - -static void *sunway_alloc_coherent(struct device *dev, - size_t size, - dma_addr_t *dma_addr, gfp_t gfp, - unsigned long attrs) -{ - struct pci_dev *pdev = to_pci_dev(dev); - struct pci_controller *hose; - struct sunway_iommu_domain *sdomain; - struct dma_domain *dma_dom; - struct sunway_iommu_dev *sdev; - struct page *page; - void *cpu_addr; - - if (!pdev) - return NULL; - - hose = pci_bus_to_pci_controller(pdev->bus); - if (!hose) - return NULL; - - gfp &= ~GFP_DMA; - -try_again: - page = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, get_order(size)); - if (!page) { - pr_err("Allocating pages failed.\n"); - return NULL; - } - - cpu_addr = page_address(page); - if (!cpu_addr) { - pr_info - ("pci_alloc_consistent: get_free_pages failed from %ps\n", - __builtin_return_address(0)); - - return NULL; - } - - *dma_addr = __pa(cpu_addr); - if (!(hose->iommu_enable)) - return cpu_addr; - - sdev = dev_iommu_priv_get(dev); - if (sdev->passthrough & DMA_MASK64) - return cpu_addr; - else if (sdev->passthrough) { - if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { - sdev->passthrough |= DMA_MASK64; - return cpu_addr; - } - - __free_pages(page, get_order(size)); - set_dma_ops(dev, get_arch_dma_ops(dev->bus)); - return dev->dma_ops->alloc(dev, size, dma_addr, gfp, attrs); - } - - sdomain = get_sunway_domain(dev); - dma_dom = to_dma_domain(sdomain); - - *dma_addr = pci_iommu_map_single(pdev, dma_dom, cpu_addr, size); - if (*dma_addr == 0) { - free_pages((unsigned long)cpu_addr, get_order(size)); - if (gfp & GFP_DMA) - return NULL; - - gfp |= GFP_DMA; - goto try_again; - } - - return cpu_addr; -} - -static void -__sunway_unmap_single(struct dma_domain *dma_dom, dma_addr_t dma_addr, size_t size) -{ - dma_addr_t start; - unsigned long npages; - int i; - - npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); - dma_addr &= PAGE_MASK; - start = dma_addr; - - for (i = 0; i < npages; i++) { - sunway_iommu_unmap_page(&dma_dom->sdomain, start, PAGE_SIZE); - start += PAGE_SIZE; - } - - sunway_free_iova(dma_dom, dma_addr, npages); - pr_debug("pci_free_consistent: %zx -> [%llx] from %ps\n", - size, dma_addr, __builtin_return_address(0)); - -} - -static void -sunway_free_coherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_addr, unsigned long attrs) -{ - struct sunway_iommu_domain *sdomain; - struct dma_domain *dma_dom; - struct pci_dev *pdev = to_pci_dev(dev); - struct pci_controller *hose; - struct sunway_iommu_dev *sdev; - - if (!pdev) - goto out_unmap; - - hose = pci_bus_to_pci_controller(pdev->bus); - if (!hose || !(hose->iommu_enable)) - goto out_unmap; - - sdev = dev_iommu_priv_get(dev); - if (sdev->passthrough) - goto out_unmap; - - sdomain = get_sunway_domain(dev); - dma_dom = to_dma_domain(sdomain); - __sunway_unmap_single(dma_dom, dma_addr, size); - goto out_free; - -out_unmap: - pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); - -out_free: - pr_debug("sunway_free_consistent: [%llx,%zx] from %ps\n", - dma_addr, size, __builtin_return_address(0)); - - free_pages((unsigned long)vaddr, get_order(size)); -} - -static dma_addr_t -sunway_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, unsigned long attrs) -{ - struct pci_dev *pdev = to_pci_dev(dev); - struct sunway_iommu_domain *sdomain; - struct dma_domain *dma_dom; - struct pci_controller *hose; - struct sunway_iommu_dev *sdev; - phys_addr_t paddr = page_to_phys(page) + offset; - - if (!pdev) - return 0; - - hose = pci_bus_to_pci_controller(pdev->bus); - if (!hose || !(hose->iommu_enable)) - return paddr; - - sdev = dev_iommu_priv_get(dev); - if (sdev->passthrough & DMA_MASK64) - return paddr; - else if (sdev->passthrough) { - if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { - sdev->passthrough |= DMA_MASK64; - return paddr; - } - - set_dma_ops(dev, get_arch_dma_ops(dev->bus)); - return dev->dma_ops->map_page(dev, page, offset, size, dir, attrs); - } - - sdomain = get_sunway_domain(dev); - dma_dom = to_dma_domain(sdomain); - - return pci_iommu_map_single(pdev, dma_dom, - (char *)page_address(page) + offset, size); -} - -static void -sunway_unmap_page(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction dir, unsigned long attrs) -{ - struct sunway_iommu_domain *sdomain; - struct dma_domain *dma_dom; - struct pci_dev *pdev; - struct pci_controller *hose; - struct sunway_iommu_dev *sdev; - - pdev = to_pci_dev(dev); - if (!pdev) - return; - - hose = pci_bus_to_pci_controller(pdev->bus); - if (hose == NULL) - return; - - if (!hose->iommu_enable) - return; - - sdev = dev_iommu_priv_get(dev); - if (sdev->passthrough) - return; - - sdomain = get_sunway_domain(dev); - dma_dom = to_dma_domain(sdomain); - __sunway_unmap_single(dma_dom, dma_addr, size); -} - -#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) -static int -sunway_map_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - struct sunway_iommu_domain *sdomain; - struct dma_domain *dma_dom = NULL; - struct scatterlist *sg; - struct pci_dev *pdev = to_pci_dev(dev); - struct pci_controller *hose; - struct sunway_iommu_dev *sdev; - int i, out_nents = 0; - - if (dir == PCI_DMA_NONE) - BUG(); - - if (!pdev) - return 0; - - hose = pci_bus_to_pci_controller(pdev->bus); - if (!hose) - return 0; - - sdomain = get_sunway_domain(dev); - dma_dom = to_dma_domain(sdomain); - - for_each_sg(sgl, sg, nents, i) { - BUG_ON(!sg_page(sg)); - - sg_dma_address(sg) = __pa(SG_ENT_VIRT_ADDRESS(sg)); - if (!(hose->iommu_enable)) - goto check; - - sdev = dev_iommu_priv_get(dev); - if (sdev->passthrough & DMA_MASK64) - goto check; - else if (sdev->passthrough) { - if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { - sdev->passthrough |= DMA_MASK64; - goto check; - } - - set_dma_ops(dev, get_arch_dma_ops(dev->bus)); - return dev->dma_ops->map_sg(dev, sgl, nents, dir, attrs); - } - - sg_dma_address(sg) = - pci_iommu_map_single(pdev, dma_dom, - SG_ENT_VIRT_ADDRESS(sg), sg->length); -check: - if (sg_dma_address(sg) == 0) - goto error; - - sg_dma_len(sg) = sg->length; - out_nents++; - } - - return nents; - -error: - pr_warn("pci_map_sg failed:"); - pr_warn("could not allocate dma page tables\n"); - - if (out_nents) - pci_unmap_sg(pdev, sgl, out_nents, dir); - return 0; -} - -static void -sunway_unmap_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - struct sunway_iommu_domain *sdomain; - struct dma_domain *dma_dom; - struct scatterlist *sg; - struct pci_dev *pdev; - struct pci_controller *hose; - struct sunway_iommu_dev *sdev; - dma_addr_t dma_addr; - long size; - int j; - - pdev = to_pci_dev(dev); - if (!pdev) - return; - - hose = pci_bus_to_pci_controller(pdev->bus); - if (!hose->iommu_enable) - return; - - sdev = dev_iommu_priv_get(dev); - if (sdev->passthrough) - return; - - sdomain = get_sunway_domain(dev); - dma_dom = to_dma_domain(sdomain); - - for_each_sg(sgl, sg, nents, j) { - dma_addr = sg->dma_address; - size = sg->dma_length; - if (!size) - break; - - __sunway_unmap_single(dma_dom, dma_addr, size); - } -} - -static const struct dma_map_ops sunway_dma_ops = { - .alloc = sunway_alloc_coherent, - .free = sunway_free_coherent, - .map_sg = sunway_map_sg, - .unmap_sg = sunway_unmap_sg, - .map_page = sunway_map_page, - .unmap_page = sunway_unmap_page, - .dma_supported = dma_direct_supported, -}; - /********************************************************************** * * IOMMU OPS Functions @@ -1499,19 +1337,6 @@ static int sunway_iommu_attach_device(struct iommu_domain *dom, struct device *d return ret; } -static void sunway_iommu_detach_device(struct iommu_domain *dom, struct device *dev) -{ - struct sunway_iommu_dev *sdev; - struct pci_dev *pdev = to_pci_dev(dev); - - if (!pdev) - return; - - sdev = dev_iommu_priv_get(dev); - if (sdev->domain != NULL) - detach_device(dev); -} - static phys_addr_t sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) { @@ -1519,7 +1344,7 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) unsigned long paddr, grn; unsigned long is_last; - if (iova > SW64_BAR_ADDRESS) + if (iova >= SW64_BAR_ADDRESS) return iova; paddr = fetch_pte(sdomain, iova, PTE_LEVEL1_VAL); @@ -1574,10 +1399,12 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) } static int -sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, - phys_addr_t paddr, size_t page_size, int iommu_prot, gfp_t gfp) +sunway_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, + phys_addr_t paddr, size_t page_size, size_t pgcount, + int iommu_prot, gfp_t gfp, size_t *mapped) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + size_t size = pgcount << PAGE_SHIFT; int ret; /* @@ -1585,37 +1412,59 @@ sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, * and pci device BAR, check should be introduced manually * to avoid VFIO trying to map pci config space. */ - if (iova > SW64_BAR_ADDRESS) + if (iova >= SW64_BAR_ADDRESS) return 0; - mutex_lock(&sdomain->api_lock); - ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size); - mutex_unlock(&sdomain->api_lock); + if (iova >= (1UL << IOVA_MAX_ADDRESS_WIDTH)) { + pr_err("IOMMU cannot map provided address: %lx\n", iova); + return -EFAULT; + } + + while (pgcount--) { + ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size, iommu_prot); + if (ret) { + pr_info("Failed to map page from IOVA %lx.\n", iova); + return ret; + } + iova += page_size; + paddr += page_size; + } + + if (!ret && mapped) + *mapped = size; return ret; } static size_t -sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, - size_t page_size, - struct iommu_iotlb_gather *gather) +sunway_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova, + size_t page_size, size_t pgcount, + struct iommu_iotlb_gather *gather) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); size_t unmap_size; + size_t total_unmap = 0; - if (iova > SW64_BAR_ADDRESS) + if (iova >= SW64_BAR_ADDRESS) return page_size; - mutex_lock(&sdomain->api_lock); - unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); - mutex_unlock(&sdomain->api_lock); + if (iova >= (1UL << IOVA_MAX_ADDRESS_WIDTH)) { + pr_err("IOMMU cannot map provided address: %lx\n", iova); + return -EFAULT; + } - return unmap_size; + while (pgcount--) { + unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); + iova += page_size; + total_unmap += page_size; + } + + return total_unmap; } static struct iommu_group *sunway_iommu_device_group(struct device *dev) { - return pci_device_group(dev); + return generic_device_group(dev); } static void iommu_uninit_device(struct device *dev) @@ -1663,6 +1512,9 @@ static int iommu_init_device(struct device *dev) return -ENOMEM; pdev = to_pci_dev(dev); + sdev->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); + sdev->alias = get_alias(pdev); + hose = pci_bus_to_pci_controller(pdev->bus); iommu = hose->pci_iommu; llist_add(&sdev->dev_data_list, &dev_data_list); @@ -1714,19 +1566,19 @@ static struct iommu_device *sunway_iommu_probe_device(struct device *dev) static int sunway_iommu_def_domain_type(struct device *dev) { - struct sunway_iommu_dev *sdev; - - sdev = dev_iommu_priv_get(dev); - if (sdev->domain) - return 0; + if (dev_is_pci(dev)) + if (iommu_identity_mapping) + return IOMMU_DOMAIN_IDENTITY; - return sdev->domain->type; + return 0; } -static bool sunway_iommu_capable(enum iommu_cap cap) +static bool sunway_iommu_capable(struct device *dev, enum iommu_cap cap) { switch (cap) { - case IOMMU_CAP_INTR_REMAP: + case IOMMU_CAP_CACHE_COHERENCY: + return true; + case IOMMU_CAP_NOEXEC: return true; default: return false; @@ -1738,43 +1590,77 @@ static void sunway_iommu_probe_finalize(struct device *dev) struct iommu_domain *domain; domain = iommu_get_domain_for_dev(dev); - if (domain) - set_dma_ops(dev, &sunway_dma_ops); + if (domain->type == IOMMU_DOMAIN_DMA) { + if (min(dev->coherent_dma_mask, *dev->dma_mask) == DMA_BIT_MASK(32)) + iommu_setup_dma_ops(dev, SW64_DMA_START, SW64_32BIT_DMA_LIMIT); + else + iommu_setup_dma_ops(dev, SW64_DMA_START, SW64_64BIT_DMA_LIMIT); + } else + set_dma_ops(dev, get_arch_dma_ops()); } const struct iommu_ops sunway_iommu_ops = { .capable = sunway_iommu_capable, .domain_alloc = sunway_iommu_domain_alloc, - .domain_free = sunway_iommu_domain_free, - .attach_dev = sunway_iommu_attach_device, - .detach_dev = sunway_iommu_detach_device, .probe_device = sunway_iommu_probe_device, .probe_finalize = sunway_iommu_probe_finalize, .release_device = sunway_iommu_release_device, - .map = sunway_iommu_map, - .unmap = sunway_iommu_unmap, - .iova_to_phys = sunway_iommu_iova_to_phys, .device_group = sunway_iommu_device_group, - .pgsize_bitmap = SW64_IOMMU_PGSIZES, + .pgsize_bitmap = SZ_8K | SZ_8M | SZ_512M | SZ_8G, .def_domain_type = sunway_iommu_def_domain_type, + .default_domain_ops = &(const struct iommu_domain_ops) { + .attach_dev = sunway_iommu_attach_device, + .map_pages = sunway_iommu_map_pages, + .unmap_pages = sunway_iommu_unmap_pages, + .iova_to_phys = sunway_iommu_iova_to_phys, + .free = sunway_iommu_domain_free, + } }; -/***************************************************************************** - * - * Boot param handle - * Each bit of iommu_enable bitmap represents an rc enable, and every 8 bits - * represents one cpu node. For example, iommu_enable=0x0100 means enabling - * rc0 for cpu node 1. - * - *****************************************************************************/ +/* Handle boot parameters */ +static int __init sunway_iommu_setup(char *str) +{ + unsigned long rc_val; + int ret; + + if (!str) + return -EINVAL; + + bitmap_zero(iommu_bitmap, 64); + + if (!strncmp(str, "on", 2)) { + bitmap_fill(iommu_bitmap, 64); + } else if (!strncmp(str, "off", 3)) { + bitmap_zero(iommu_bitmap, 64); + } else { + ret = kstrtoul(str, 16, &rc_val); + if (!ret) + return -EINVAL; + + bitmap_from_u64(iommu_bitmap, rc_val); + } + + return 1; +} +__setup("sunway_iommu=", sunway_iommu_setup); + static int __init iommu_enable_setup(char *str) { + unsigned long rc_val; int ret; - unsigned long rc_bitmap = 0xffffffffUL; - ret = kstrtoul(str, 16, &rc_bitmap); - iommu_enable_cmd = rc_bitmap; + if (!str) + return -EINVAL; - return ret; + pr_info("iommu_enable= deprecated; use sunway_iommu=on/off instead\n"); + bitmap_zero(iommu_bitmap, 64); + + ret = kstrtoul(str, 16, &rc_val); + if (!ret) + return -EINVAL; + + bitmap_from_u64(iommu_bitmap, rc_val); + + return 1; } __setup("iommu_enable=", iommu_enable_setup); diff --git a/include/acpi/actbl1.h b/include/acpi/actbl1.h index 8d5572ad48cb..e490b4c37a48 100644 --- a/include/acpi/actbl1.h +++ b/include/acpi/actbl1.h @@ -915,6 +915,34 @@ struct acpi_dmar_satc { u8 reserved; u16 segment; }; + +#ifdef CONFIG_SW64 +struct acpi_table_sw_dmar { + struct acpi_table_header header; /* Common ACPI table header*/ + u8 version; /* IOMMU version */ + u8 width; /* Host Address Width */ + u8 flags; +}; + +struct acpi_sw_dmar_header { + u8 type; + u8 length; +}; + +enum acpi_sw_dmar_type { + ACPI_SW_DMAR_TYPE_HARDWARE_UNIT = 0, + ACPI_SW_DMAR_TYPE_RESERVED = 1 +}; + +struct acpi_dmar_sw_hardware_unit { + struct acpi_sw_dmar_header header; + u16 index; + u32 size; + u8 enable; + u64 address; +}; +#endif + /******************************************************************************* * * DRTM - Dynamic Root of Trust for Measurement table -- Gitee From 485b9464c67622f9915dca8679ffcec21491d649 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 11 Jun 2024 15:32:52 +0800 Subject: [PATCH 214/524] sw64: adapt IO interfaces for SW64 from upstream The interfaces from the upstream have changed. This adapts the modifications from the upstream for SW64 IO interfaces. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/io.h | 219 +++++------------------------------- arch/sw_64/kernel/setup.c | 4 - arch/sw_64/lib/iomap.c | 207 ---------------------------------- 3 files changed, 28 insertions(+), 402 deletions(-) diff --git a/arch/sw_64/include/asm/io.h b/arch/sw_64/include/asm/io.h index 2b045be5257e..95ca8c88ea50 100644 --- a/arch/sw_64/include/asm/io.h +++ b/arch/sw_64/include/asm/io.h @@ -9,121 +9,11 @@ #include #include -/* The generic header contains only prototypes. Including it ensures that - * the implementation we have here matches that interface. - */ -#include - -/* We don't use IO slowdowns on the sw64, but.. */ -#define __SLOW_DOWN_IO do { } while (0) -#define SLOW_DOWN_IO do { } while (0) - #define page_to_phys(page) page_to_pa(page) /* Maximum PIO space address supported? */ #define IO_SPACE_LIMIT 0xffffffffffffffff -/* - * Generic IO read/write. These perform native-endian accesses. - */ - -#define __raw_writeb __raw_writeb -static inline void __raw_writeb(u8 val, volatile void __iomem *addr) -{ - asm volatile("stb %0, 0(%1)" : : "r" (val), "r" (addr)); -} - -#define __raw_writew __raw_writew -static inline void __raw_writew(u16 val, volatile void __iomem *addr) -{ - asm volatile("sth %0, 0(%1)" : : "r" (val), "r" (addr)); -} - -#define __raw_writel __raw_writel -static inline void __raw_writel(u32 val, volatile void __iomem *addr) -{ - asm volatile("stw %0, 0(%1)" : : "r" (val), "r" (addr)); -} - -#define __raw_writeq __raw_writeq -static inline void __raw_writeq(u64 val, volatile void __iomem *addr) -{ - asm volatile("stl %0, 0(%1)" : : "r" (val), "r" (addr)); -} - -#define __raw_readb __raw_readb -static inline u8 __raw_readb(const volatile void __iomem *addr) -{ - u8 val; - - asm volatile("ldbu %0, 0(%1)" : "=r" (val) : "r" (addr)); - return val; -} - -#define __raw_readw __raw_readw -static inline u16 __raw_readw(const volatile void __iomem *addr) -{ - u16 val; - - asm volatile("ldhu %0, 0(%1)" : "=r" (val) : "r" (addr)); - return val; -} - -#define __raw_readl __raw_readl -static inline u32 __raw_readl(const volatile void __iomem *addr) -{ - u32 val; - - asm volatile("ldw %0, 0(%1)\n" - "zapnot %0, 0xf, %0\n" - : "=r" (val) : "r" (addr)); - return val; -} - -#define __raw_readq __raw_readq -static inline u64 __raw_readq(const volatile void __iomem *addr) -{ - u64 val; - - asm volatile("ldl %0, 0(%1)" : "=r" (val) : "r" (addr)); - return val; -} - -/* IO barriers */ - -#define __iormb() rmb() -#define __iowmb() wmb() -#define mmiowb() do { } while (0) - -/* - * Relaxed I/O memory access primitives. These follow the Device memory - * ordering rules but do not guarantee any ordering relative to Normal memory - * accesses. - */ -#define readb_relaxed(c) __raw_readb(c) -#define readw_relaxed(c) __raw_readw(c) -#define readl_relaxed(c) __raw_readl(c) -#define readq_relaxed(c) __raw_readq(c) - -#define writeb_relaxed(v, c) __raw_writeb((v), (c)) -#define writew_relaxed(v, c) __raw_writew((v), (c)) -#define writel_relaxed(v, c) __raw_writel((v), (c)) -#define writeq_relaxed(v, c) __raw_writeq((v), (c)) - -/* - * I/O memory access primitives. Reads are ordered relative to any - * following Normal memory access. Writes are ordered relative to any prior - * Normal memory access. - */ -#define readb(c) ({ u8 __v = readb_relaxed(c); __iormb(); __v; }) -#define readw(c) ({ u16 __v = readw_relaxed(c); __iormb(); __v; }) -#define readl(c) ({ u32 __v = readl_relaxed(c); __iormb(); __v; }) -#define readq(c) ({ u64 __v = readq_relaxed(c); __iormb(); __v; }) - -#define writeb(v, c) ({ __iowmb(); writeb_relaxed((v), (c)); }) -#define writew(v, c) ({ __iowmb(); writew_relaxed((v), (c)); }) -#define writel(v, c) ({ __iowmb(); writel_relaxed((v), (c)); }) -#define writeq(v, c) ({ __iowmb(); writeq_relaxed((v), (c)); }) /* * We always have external versions of these routines. */ @@ -154,27 +44,12 @@ static inline void __iomem *__ioremap(phys_addr_t addr, size_t size, #define ioremap_uc ioremap_nocache #define ioport_map ioport_map -#define ioport_unmap ioport_unmap +extern void __iomem *ioport_map(unsigned long port, unsigned int nr); -static inline void __iounmap(volatile void __iomem *addr) +static inline void iounmap(volatile void __iomem *addr) { } -#define iounmap __iounmap - -#define ioread16be(p) be16_to_cpu(ioread16(p)) -#define ioread32be(p) be32_to_cpu(ioread32(p)) -#define iowrite16be(v, p) iowrite16(cpu_to_be16(v), (p)) -#define iowrite32be(v, p) iowrite32(cpu_to_be32(v), (p)) - -#define inb_p inb -#define inw_p inw -#define inl_p inl -#define outb_p outb -#define outw_p outw -#define outl_p outl - - /* * String version of IO memory access ops: */ @@ -214,74 +89,36 @@ extern void outsl(unsigned long port, const void *src, unsigned long count); #define outsw outsw #define outsl outsl -/* - * These defines will override the defaults when doing RTC queries - */ - -#define RTC_PORT(x) (0x70 + (x)) -#define RTC_ALWAYS_BCD 0 - -/* - * Convert a physical pointer to a virtual kernel pointer for /dev/mem - * access - */ -#define xlate_dev_mem_ptr(p) __va(p) - -/* - * Convert a virtual cached pointer to an uncached pointer - */ -#define xlate_dev_kmem_ptr(p) p - -/* - * These get provided from since sw64 does not - * select GENERIC_IOMAP. - */ -#define ioread8 ioread8 -#define ioread16 ioread16 -#define ioread32 ioread32 -#define ioread64 ioread64 -#define iowrite8 iowrite8 -#define iowrite16 iowrite16 -#define iowrite32 iowrite32 -#define iowrite64 iowrite64 -#define ioread64be ioread64be -#define iowrite64be iowrite64be -#define ioread8_rep ioread8_rep -#define ioread16_rep ioread16_rep -#define ioread32_rep ioread32_rep -#define iowrite8_rep iowrite8_rep -#define iowrite16_rep iowrite16_rep -#define iowrite32_rep iowrite32_rep #define pci_iounmap pci_iounmap #include -/* - * Change addresses as seen by the kernel (virtual) to addresses as - * seen by a device (bus), and vice versa. - * - * Note that this only works for a limited range of kernel addresses, - * and very well may not span all memory. Consider this interface - * deprecated in favour of the DMA-mapping API. - */ -static inline unsigned long __deprecated virt_to_bus(void *address) -{ - return virt_to_phys(address); -} -#define isa_virt_to_bus virt_to_bus - -static inline void * __deprecated bus_to_virt(unsigned long address) -{ - void *virt; - - /* This check is a sanity check but also ensures that bus address 0 - * maps to virtual address 0 which is useful to detect null pointers - * (the NCR driver is much simpler if NULL pointers are preserved). - */ - virt = phys_to_virt(address); - return (long)address <= 0 ? NULL : virt; -} -#define isa_bus_to_virt bus_to_virt +///* +// * Change addresses as seen by the kernel (virtual) to addresses as +// * seen by a device (bus), and vice versa. +// * +// * Note that this only works for a limited range of kernel addresses, +// * and very well may not span all memory. Consider this interface +// * deprecated in favour of the DMA-mapping API. +// */ +//static inline unsigned long __deprecated virt_to_bus(void *address) +//{ +// return virt_to_phys(address); +//} +//#define isa_virt_to_bus virt_to_bus +// +//static inline void * __deprecated bus_to_virt(unsigned long address) +//{ +// void *virt; +// +// /* This check is a sanity check but also ensures that bus address 0 +// * maps to virtual address 0 which is useful to detect null pointers +// * (the NCR driver is much simpler if NULL pointers are preserved). +// */ +// virt = phys_to_virt(address); +// return (long)address <= 0 ? NULL : virt; +//} +//#define isa_bus_to_virt bus_to_virt #endif /* __KERNEL__ */ diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 411810dbf10d..ee0b7850e820 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -180,10 +180,6 @@ reserve_std_resources(void) } } - /* Fix up for the Jensen's queer RTC placement. */ - standard_io_resources[0].start = RTC_PORT(0); - standard_io_resources[0].end = RTC_PORT(0) + 0x10; - for (i = 0; i < ARRAY_SIZE(standard_io_resources); ++i) request_resource(io, standard_io_resources+i); } diff --git a/arch/sw_64/lib/iomap.c b/arch/sw_64/lib/iomap.c index d9c66a89131e..f89cf16f683c 100644 --- a/arch/sw_64/lib/iomap.c +++ b/arch/sw_64/lib/iomap.c @@ -11,42 +11,6 @@ /* * Here comes the sw64 implementation of the IOMAP interfaces. */ -unsigned int ioread8(const void __iomem *addr) -{ - return readb(addr); -} -EXPORT_SYMBOL(ioread8); - -unsigned int ioread16(const void __iomem *addr) -{ - return readw(addr); -} -EXPORT_SYMBOL(ioread16); - -unsigned int ioread32(const void __iomem *addr) -{ - return readl(addr); -} -EXPORT_SYMBOL(ioread32); - -void iowrite8(u8 b, void __iomem *addr) -{ - writeb(b, addr); -} -EXPORT_SYMBOL(iowrite8); - -void iowrite16(u16 b, void __iomem *addr) -{ - writew(b, addr); -} -EXPORT_SYMBOL(iowrite16); - -void iowrite32(u32 b, void __iomem *addr) -{ - writel(b, addr); -} -EXPORT_SYMBOL(iowrite32); - u8 inb(unsigned long port) { return ioread8(ioport_map(port, 1)); @@ -83,202 +47,36 @@ void outl(u32 b, unsigned long port) } EXPORT_SYMBOL(outl); - -/* - * Read COUNT 8-bit bytes from port PORT into memory starting at SRC. - */ -void ioread8_rep(const void __iomem *port, void *dst, unsigned long count) -{ - while ((unsigned long)dst & 0x3) { - if (!count) - return; - count--; - *(unsigned char *)dst = ioread8(port); - dst += 1; - } - - while (count >= 4) { - unsigned int w; - - count -= 4; - w = ioread8(port); - w |= ioread8(port) << 8; - w |= ioread8(port) << 16; - w |= ioread8(port) << 24; - *(unsigned int *)dst = w; - dst += 4; - } - - while (count) { - --count; - *(unsigned char *)dst = ioread8(port); - dst += 1; - } -} -EXPORT_SYMBOL(ioread8_rep); - void insb(unsigned long port, void *dst, unsigned long count) { ioread8_rep(ioport_map(port, 1), dst, count); } EXPORT_SYMBOL(insb); -/* - * Read COUNT 16-bit words from port PORT into memory starting at - * SRC. SRC must be at least short aligned. This is used by the - * IDE driver to read disk sectors. Performance is important, but - * the interfaces seems to be slow: just using the inlined version - * of the inw() breaks things. - */ -void ioread16_rep(const void __iomem *port, void *dst, unsigned long count) -{ - if (unlikely((unsigned long)dst & 0x3)) { - if (!count) - return; - BUG_ON((unsigned long)dst & 0x1); - count--; - *(unsigned short *)dst = ioread16(port); - dst += 2; - } - - while (count >= 2) { - unsigned int w; - - count -= 2; - w = ioread16(port); - w |= ioread16(port) << 16; - *(unsigned int *)dst = w; - dst += 4; - } - - if (count) - *(unsigned short *)dst = ioread16(port); -} -EXPORT_SYMBOL(ioread16_rep); - void insw(unsigned long port, void *dst, unsigned long count) { ioread16_rep(ioport_map(port, 2), dst, count); } EXPORT_SYMBOL(insw); - -/* - * Read COUNT 32-bit words from port PORT into memory starting at - * SRC. Now works with any alignment in SRC. Performance is important, - * but the interfaces seems to be slow: just using the inlined version - * of the inl() breaks things. - */ -void ioread32_rep(const void __iomem *port, void *dst, unsigned long count) -{ - if (unlikely((unsigned long)dst & 0x3)) { - while (count--) { - struct S { int x __packed; }; - ((struct S *)dst)->x = ioread32(port); - dst += 4; - } - } else { - /* Buffer 32-bit aligned. */ - while (count--) { - *(unsigned int *)dst = ioread32(port); - dst += 4; - } - } -} -EXPORT_SYMBOL(ioread32_rep); - void insl(unsigned long port, void *dst, unsigned long count) { ioread32_rep(ioport_map(port, 4), dst, count); } EXPORT_SYMBOL(insl); - -/* - * Like insb but in the opposite direction. - * Don't worry as much about doing aligned memory transfers: - * doing byte reads the "slow" way isn't nearly as slow as - * doing byte writes the slow way (no r-m-w cycle). - */ -void iowrite8_rep(void __iomem *port, const void *xsrc, unsigned long count) -{ - const unsigned char *src = xsrc; - - while (count--) - iowrite8(*src++, port); -} -EXPORT_SYMBOL(iowrite8_rep); - void outsb(unsigned long port, const void *src, unsigned long count) { iowrite8_rep(ioport_map(port, 1), src, count); } EXPORT_SYMBOL(outsb); - -/* - * Like insw but in the opposite direction. This is used by the IDE - * driver to write disk sectors. Performance is important, but the - * interfaces seems to be slow: just using the inlined version of the - * outw() breaks things. - */ -void iowrite16_rep(void __iomem *port, const void *src, unsigned long count) -{ - if (unlikely((unsigned long)src & 0x3)) { - if (!count) - return; - BUG_ON((unsigned long)src & 0x1); - iowrite16(*(unsigned short *)src, port); - src += 2; - --count; - } - - while (count >= 2) { - unsigned int w; - - count -= 2; - w = *(unsigned int *)src; - src += 4; - iowrite16(w >> 0, port); - iowrite16(w >> 16, port); - } - - if (count) - iowrite16(*(unsigned short *)src, port); -} -EXPORT_SYMBOL(iowrite16_rep); - void outsw(unsigned long port, const void *src, unsigned long count) { iowrite16_rep(ioport_map(port, 2), src, count); } EXPORT_SYMBOL(outsw); - -/* - * Like insl but in the opposite direction. This is used by the IDE - * driver to write disk sectors. Works with any alignment in SRC. - * Performance is important, but the interfaces seems to be slow: - * just using the inlined version of the outl() breaks things. - */ -void iowrite32_rep(void __iomem *port, const void *src, unsigned long count) -{ - if (unlikely((unsigned long)src & 0x3)) { - while (count--) { - struct S { int x __packed; }; - iowrite32(((struct S *)src)->x, port); - src += 4; - } - } else { - /* Buffer 32-bit aligned. */ - while (count--) { - iowrite32(*(unsigned int *)src, port); - src += 4; - } - } -} -EXPORT_SYMBOL(iowrite32_rep); - void outsl(unsigned long port, const void *src, unsigned long count) { iowrite32_rep(ioport_map(port, 4), src, count); @@ -470,8 +268,3 @@ void __iomem *ioport_map(unsigned long port, unsigned int size) return __va(port); } EXPORT_SYMBOL(ioport_map); - -void ioport_unmap(void __iomem *addr) -{ -} -EXPORT_SYMBOL(ioport_unmap); -- Gitee From f318da23208f9c3475e0998744b3c64979f1235f Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Thu, 20 Jun 2024 16:38:08 +0000 Subject: [PATCH 215/524] sw64: fix MSI compilation warnings Use a encapsulated function in MSI to resolve compilation warnings. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-msi-v2.c | 2 +- drivers/irqchip/irq-sunway-msi.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c index 4e98a4c10f85..01ac9f468b49 100644 --- a/drivers/irqchip/irq-sunway-msi-v2.c +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -189,7 +189,7 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, irq_msi_compose_msg(d, &msg); __pci_write_msi_msg(entry, &msg); spin_unlock(&cdata->cdata_lock); - cpumask_copy(irq_data_get_affinity_mask(irqd), &searchmask); + irq_data_update_effective_affinity(irqd, &searchmask); raw_spin_unlock_irqrestore(&vector_lock, flags); diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index 7f55f675caa1..299d98f4128a 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -158,7 +158,7 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, msi_config = set_piu_msi_config(hose, cpu, cdata->msi_config_index, vector); cdata->msi_config = msi_config; spin_unlock(&cdata->cdata_lock); - cpumask_copy((struct cpumask *)irq_data_get_affinity_mask(irqd), &searchmask); + irq_data_update_effective_affinity(irqd, &searchmask); raw_spin_unlock_irqrestore(&vector_lock, flags); -- Gitee From ce7512da9aaefdd08b7ad51ce66b99bdcd4e4993 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Mon, 24 Jun 2024 10:23:32 +0000 Subject: [PATCH 216/524] sw64: fix pci resource assignment bug This commit fixes two errors in pci resource assignment: -It includes RC in the resource calculation -It allows RC to participate in the resource assignment process Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/pci/pci.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index 073781645f32..6eac8f5b9d8e 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -119,7 +119,7 @@ void pcibios_fixup_bus(struct pci_bus *bus) struct pci_controller *hose = pci_bus_to_pci_controller(bus); struct pci_dev *dev = bus->self; - if (!dev || bus->number == hose->first_busno) { + if (!dev) { bus->resource[0] = hose->io_space; bus->resource[1] = hose->mem_space; bus->resource[2] = hose->pre_mem_space; @@ -209,11 +209,6 @@ static void fixup_root_complex(struct pci_dev *dev) dev->class &= 0xff; dev->class |= PCI_CLASS_BRIDGE_PCI << 8; - for (i = 0; i < PCI_NUM_RESOURCES; i++) { - dev->resource[i].start = 0; - dev->resource[i].end = 0; - dev->resource[i].flags = IORESOURCE_PCI_FIXED; - } } atomic_inc(&dev->enable_cnt); -- Gitee From a6e66df600960aa2aa4dfe731cc4e863ca54fa40 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Wed, 26 Jun 2024 14:15:07 +0000 Subject: [PATCH 217/524] sw64: amdgpu/radeon: correct low-level mmio memset/memcpy direct calls Driver codes of the direct calls, via the SIMD-optimized memset and memcpy functions, may raise default when using RX580/R7 under sw64 architecture, so work around 'memset' references to 'memset_io' and 'memcpy' to 'memcpy_fromio/toio'. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 8 ++++++++ drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 29 +++++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 29 +++++++++++++++++++++++++++ drivers/gpu/drm/radeon/radeon_vce.c | 4 ++++ drivers/gpu/drm/radeon/vce_v1_0.c | 5 +++++ 5 files changed, 75 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index ab7a214431d3..7505516519f8 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -2752,7 +2752,11 @@ static int gfx_v7_0_mec_init(struct amdgpu_device *adev) } /* clear memory. Not sure if this is required or not */ +#if IS_ENABLED(CONFIG_SW64) + memset_io(hpd, 0, mec_hpd_size); +#else memset(hpd, 0, mec_hpd_size); +#endif amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); @@ -2861,7 +2865,11 @@ static void gfx_v7_0_mqd_init(struct amdgpu_device *adev, u64 wb_gpu_addr; /* init the mqd struct */ +#if IS_ENABLED(CONFIG_SW64) + memset_io(mqd, 0, sizeof(struct cik_mqd)); +#else memset(mqd, 0, sizeof(struct cik_mqd)); +#endif mqd->header = 0xC0310800; mqd->compute_static_thread_mgmt_se0 = 0xffffffff; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index d754c70d11ef..68cf37f45c45 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -1322,7 +1322,11 @@ static int gfx_v8_0_mec_init(struct amdgpu_device *adev) return r; } +#if IS_ENABLED(CONFIG_SW64) + memset_io(hpd, 0, mec_hpd_size); +#else memset(hpd, 0, mec_hpd_size); +#endif amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); @@ -4602,7 +4606,11 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring) if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ /* reset MQD to a clean status */ if (adev->gfx.kiq[0].mqd_backup) +#if IS_ENABLED(CONFIG_SW64) + memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct vi_mqd_allocation)); +#else memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct vi_mqd_allocation)); +#endif /* reset ring buffer */ ring->wptr = 0; @@ -4613,7 +4621,11 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring) vi_srbm_select(adev, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); } else { +#if IS_ENABLED(CONFIG_SW64) + memset_io((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); +#else memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); +#endif ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; if (amdgpu_sriov_vf(adev) && adev->in_suspend) @@ -4626,7 +4638,11 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring) mutex_unlock(&adev->srbm_mutex); if (adev->gfx.kiq[0].mqd_backup) +#if IS_ENABLED(CONFIG_SW64) + memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct vi_mqd_allocation)); +#else memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct vi_mqd_allocation)); +#endif } return 0; @@ -4639,7 +4655,11 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring) int mqd_idx = ring - &adev->gfx.compute_ring[0]; if (!amdgpu_in_reset(adev) && !adev->in_suspend) { +#if IS_ENABLED(CONFIG_SW64) + memset_io((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); +#else memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); +#endif ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; mutex_lock(&adev->srbm_mutex); @@ -4649,11 +4669,20 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring) mutex_unlock(&adev->srbm_mutex); if (adev->gfx.mec.mqd_backup[mqd_idx]) +#if IS_ENABLED(CONFIG_SW64) + memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation)); +#else memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation)); +#endif } else { /* restore MQD to a clean status */ if (adev->gfx.mec.mqd_backup[mqd_idx]) +#if IS_ENABLED(CONFIG_SW64) + memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); +#else memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); +#endif + /* reset ring buffer */ ring->wptr = 0; amdgpu_ring_clear_ring(ring); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 895060f6948f..676c576b72cb 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -1711,7 +1711,11 @@ static int gfx_v9_0_mec_init(struct amdgpu_device *adev) return r; } +#if IS_ENABLED(CONFIG_SW64) + memset_io(hpd, 0, mec_hpd_size); +#else memset(hpd, 0, mec_hpd_size); +#endif amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); @@ -3547,7 +3551,11 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){ /* for GPU_RESET case , reset MQD to a clean status */ if (adev->gfx.kiq[0].mqd_backup) +#if IS_ENABLED(CONFIG_SW64) + memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation)); +#else memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation)); +#endif /* reset ring buffer */ ring->wptr = 0; @@ -3559,7 +3567,11 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) soc15_grbm_select(adev, 0, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); } else { +#if IS_ENABLED(CONFIG_SW64) + memset_io((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); +#else memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); +#endif ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; if (amdgpu_sriov_vf(adev) && adev->in_suspend) @@ -3572,7 +3584,11 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) mutex_unlock(&adev->srbm_mutex); if (adev->gfx.kiq[0].mqd_backup) +#if IS_ENABLED(CONFIG_SW64) + memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation)); +#else memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation)); +#endif } return 0; @@ -3592,7 +3608,11 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) if (!tmp_mqd->cp_hqd_pq_control || (!amdgpu_in_reset(adev) && !adev->in_suspend)) { +#if IS_ENABLED(CONFIG_SW64) + memset_io((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); +#else memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); +#endif ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; mutex_lock(&adev->srbm_mutex); @@ -3602,11 +3622,20 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) mutex_unlock(&adev->srbm_mutex); if (adev->gfx.mec.mqd_backup[mqd_idx]) +#if IS_ENABLED(CONFIG_SW64) + memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); +#else memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); +#endif } else { /* restore MQD to a clean status */ if (adev->gfx.mec.mqd_backup[mqd_idx]) +#if IS_ENABLED(CONFIG_SW64) + memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); +#else memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); +#endif + /* reset ring buffer */ ring->wptr = 0; atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); diff --git a/drivers/gpu/drm/radeon/radeon_vce.c b/drivers/gpu/drm/radeon/radeon_vce.c index 2355a78e1b69..2eb1636c560e 100644 --- a/drivers/gpu/drm/radeon/radeon_vce.c +++ b/drivers/gpu/drm/radeon/radeon_vce.c @@ -238,7 +238,11 @@ int radeon_vce_resume(struct radeon_device *rdev) return r; } +#if IS_ENABLED(CONFIG_SW64) + memset_io(cpu_addr, 0, radeon_bo_size(rdev->vce.vcpu_bo)); +#else memset(cpu_addr, 0, radeon_bo_size(rdev->vce.vcpu_bo)); +#endif if (rdev->family < CHIP_BONAIRE) r = vce_v1_0_load_fw(rdev, cpu_addr); else diff --git a/drivers/gpu/drm/radeon/vce_v1_0.c b/drivers/gpu/drm/radeon/vce_v1_0.c index bdfbcf14b864..faab1addef26 100644 --- a/drivers/gpu/drm/radeon/vce_v1_0.c +++ b/drivers/gpu/drm/radeon/vce_v1_0.c @@ -193,8 +193,13 @@ int vce_v1_0_load_fw(struct radeon_device *rdev, uint32_t *data) data[3] = sign->val[i].nonce[3]; data[4] = cpu_to_le32(le32_to_cpu(sign->len) + 64); +#if IS_ENABLED(CONFIG_SW64) + memset_io(&data[5], 0, 44); + memcpy_toio(&data[16], &sign[1], rdev->vce_fw->size - sizeof(*sign)); +#else memset(&data[5], 0, 44); memcpy(&data[16], &sign[1], rdev->vce_fw->size - sizeof(*sign)); +#endif data += (le32_to_cpu(sign->len) + 64) / 4; data[0] = sign->val[i].sigval[0]; -- Gitee From 322dec4054a4fe2bbd6c0db152d011836f1b4d00 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Wed, 3 Jul 2024 14:28:36 +0000 Subject: [PATCH 218/524] sw64: fix io resource remapping During the pci scan, the assigned IORESOURCE_IO addresses are not the expected address range. This is because asm-generic/io.h defines PCI_IOBASE, which results in the function acpi_pci_root_remap_iospace() doing remapping. To resolve this issue, the definition of PCI_IOBASE is removed. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/io.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/include/asm/io.h b/arch/sw_64/include/asm/io.h index 95ca8c88ea50..4e0822f46518 100644 --- a/arch/sw_64/include/asm/io.h +++ b/arch/sw_64/include/asm/io.h @@ -92,6 +92,7 @@ extern void outsl(unsigned long port, const void *src, unsigned long count); #define pci_iounmap pci_iounmap #include +#undef PCI_IOBASE ///* // * Change addresses as seen by the kernel (virtual) to addresses as -- Gitee From 24d5a9473173463ad4d3e12d95ab9c21d8aa26d5 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 25 Jun 2024 09:42:03 +0000 Subject: [PATCH 219/524] sw64: fix some S3 CSRs problems for C4 This patch: - Restore CSR:KTP after wake-up. - Enable longtime to make CSR:SHTCLOCK continue to count after wake-up. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uncore_io_ops_junzhang.h | 7 ++++++- arch/sw_64/kernel/chip_setup.c | 3 +++ arch/sw_64/kernel/suspend_asm.S | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/uncore_io_ops_junzhang.h b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h index 95a3b5c80531..00f9898f8540 100644 --- a/arch/sw_64/include/asm/uncore_io_ops_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h @@ -28,7 +28,12 @@ static inline unsigned long __get_node_mem(int node) #define __io_read_longtime(node) (0UL) #define __io_write_longtime(node, data) do { } while (0) -#define __io_write_longtime_start_en(node, data) do { } while (0) + +static inline void +__io_write_longtime_start_en(int node, unsigned long data) +{ + sw64_io_write(node, GPIO_SWPORTA_DDR, data); +} static inline void __io_write_fault_int_en(int node, unsigned long data) diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index c6374ae18138..e6784968323d 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -193,6 +193,9 @@ static inline void intpu_restore(void) __io_write_longtime(0, saved_long_time); __io_write_longtime_start_en(0, 0x1); break; + case CPU_SW8A: + __io_write_longtime_start_en(0, 0x1); + break; default: pr_info("long time start is disable!"); break; diff --git a/arch/sw_64/kernel/suspend_asm.S b/arch/sw_64/kernel/suspend_asm.S index 34ee349515a7..c245e657503a 100644 --- a/arch/sw_64/kernel/suspend_asm.S +++ b/arch/sw_64/kernel/suspend_asm.S @@ -3,6 +3,7 @@ #include #include #include +#include .text .set noat @@ -56,6 +57,9 @@ $subloop: ldl $8, PSTATE_KTP($18) +#ifdef CONFIG_SUBARCH_C4 + csrw $8, CSR_KTP +#endif ldi $1, PSTATE_REGS($18) ldl $9, CALLEE_R9($1) ldl $10, CALLEE_R10($1) -- Gitee From 799c04e96677448e868d705d30bb9a6920727af3 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 25 Jun 2024 09:44:54 +0000 Subject: [PATCH 220/524] sw64: fix a base address of msiaddr register Fix base address of msiaddr register to make S3 work. Fixes: 43d035704fa6 ("sw64: pci: use readq/writeq to read/write RC and PIU IO registers") Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/chip_setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index e6784968323d..f6e92adab5e6 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -92,7 +92,7 @@ static void pcie_save(void) piu_save->piuconfig0 = readq(piu_ior0_base + PIUCONFIG0); piu_save->piuconfig1 = readq(piu_ior1_base + PIUCONFIG1); piu_save->epdmabar = readq(piu_ior0_base + EPDMABAR); - piu_save->msiaddr = readq(piu_ior1_base + MSIADDR); + piu_save->msiaddr = readq(piu_ior0_base + MSIADDR); if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { for (i = 0; i < 256; i++) { -- Gitee From db85e24f76d748f19ec5b3e1e2026940b06a3811 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Wed, 3 Jul 2024 15:31:50 +0000 Subject: [PATCH 221/524] sw64: fix a compilation warning Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/pci/pci.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index 6eac8f5b9d8e..fbbd54dd181c 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -194,7 +194,6 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82378, */ static void fixup_root_complex(struct pci_dev *dev) { - int i; struct pci_bus *bus = dev->bus; struct pci_controller *hose = pci_bus_to_pci_controller(bus); -- Gitee From 1f7cd5a9e6038568702b4b02ea080adbd0efe324 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Mon, 17 Jun 2024 17:13:09 +0000 Subject: [PATCH 222/524] sw64: handle null MSI irq Add a check after getting irq number from CPU vector_irq table. Otherwise, kernel will hang after getting a null irq. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-msi-v2.c | 5 +++++ drivers/irqchip/irq-sunway-msi-vt.c | 3 +++ drivers/irqchip/irq-sunway-msi.c | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c index 01ac9f468b49..553ccd0afa07 100644 --- a/drivers/irqchip/irq-sunway-msi-v2.c +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -501,6 +501,11 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned } irq = per_cpu(vector_irq, cpu)[vector_index + msi_index]; + if (unlikely(!irq)) { + vector = vector & (~(1UL << msi_index)); + continue; + } + irq_data = irq_domain_get_irq_data(msi_default_domain->parent, irq); cdata = irq_data_get_irq_chip_data(irq_data); spin_lock(&cdata->cdata_lock); diff --git a/drivers/irqchip/irq-sunway-msi-vt.c b/drivers/irqchip/irq-sunway-msi-vt.c index a28f23799117..ac2e78757b51 100644 --- a/drivers/irqchip/irq-sunway-msi-vt.c +++ b/drivers/irqchip/irq-sunway-msi-vt.c @@ -32,6 +32,9 @@ void vt_handle_pci_msi_interrupt(unsigned long type, unsigned long vector, cpu = smp_processor_id(); irq = per_cpu(vector_irq, cpu)[vector]; + if (unlikely(!irq)) + return; + irq_data = irq_domain_get_irq_data(vt_msi_default_domain->parent, irq); cdata = irq_data_get_irq_chip_data(irq_data); diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index 299d98f4128a..ba96eab6a3de 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -453,6 +453,11 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned } irq = per_cpu(vector_irq, cpu)[vector_index + msi_index]; + if (unlikely(!irq)) { + vector = vector & (~(1UL << msi_index)); + continue; + } + irq_data = irq_domain_get_irq_data(msi_default_domain->parent, irq); cdata = irq_data_get_irq_chip_data(irq_data); spin_lock(&cdata->cdata_lock); -- Gitee From f8a2d52148da76c4f2d357c3e2a272f3b12c9547 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Wed, 19 Jun 2024 19:25:12 +0000 Subject: [PATCH 223/524] sw64: seperate IRQ_MSI configs for sub-architectures Current kernel configuration has two different MSI handling codes specific to sub-architectures controlled by one single config, which is confusing and unclear. To better organize and clarify the MSI config logic, this patch introduces two IRQ_MSI configs, each dependent on the respective SUBARCH, to differentiate the two MSI handling process. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 3 +-- drivers/irqchip/Kconfig | 9 +++++++-- drivers/irqchip/Makefile | 7 +------ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index b685807536af..9890ec38f695 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -120,8 +120,7 @@ config SW64 select SET_FS select SPARSEMEM_EXTREME if SPARSEMEM select SW64_IRQ_CPU - select SW64_IRQ_MSI if PCI_MSI - select SW64_IRQ_MSI_VT if PCI_MSI + select SW64_PCI_INTX if PCI select SW64_TIMER select SWIOTLB select THREAD_INFO_IN_TASK diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 21aeae9af675..6be0af85a0b5 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -44,12 +44,17 @@ config SW64_PCI_INTX config SW64_IRQ_MSI bool - depends on SW64 && PCI_MSI + depends on SW64 && PCI_MSI && SUBARCH_C3B + default y + +config SW64_IRQ_MSI_V2 + bool + depends on SW64 && PCI_MSI && SUBARCH_C4 default y config SW64_IRQ_MSI_VT bool - depends on SW64_IRQ_MSI + depends on SW64_IRQ_MSI || SW64_IRQ_MSI_V2 default y config ARM_GIC_PM diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index cd085dde0d35..f4697c1a39c0 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -31,13 +31,8 @@ obj-$(CONFIG_SW64_PINTC) += irq-sunway-pintc.o obj-$(CONFIG_SW64_LPC_INTC) += irq-sunway-lpc-intc.o obj-$(CONFIG_SW64_PCI_INTX) += irq-sunway-pci-intx.o obj-$(CONFIG_SW64_IRQ_CPU) += irq-sunway-cpu.o - -ifeq ($(CONFIG_UNCORE_XUELANG),y) obj-$(CONFIG_SW64_IRQ_MSI) += irq-sunway-msi.o -else -obj-$(CONFIG_SW64_IRQ_MSI) += irq-sunway-msi-v2.o -endif - +obj-$(CONFIG_SW64_IRQ_MSI_V2) += irq-sunway-msi-v2.o obj-$(CONFIG_SW64_IRQ_MSI_VT) += irq-sunway-msi-vt.o obj-$(CONFIG_ARM_GIC_PM) += irq-gic-pm.o obj-$(CONFIG_ARCH_REALVIEW) += irq-gic-realview.o -- Gitee From 227dac388d518e1413b4ea4b7ee543bb3882c9a8 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 24 May 2024 17:06:14 +0800 Subject: [PATCH 224/524] sw64: optimize the code in do_entInt() Modify the location of the following function: - set_irq_regs() - irq_enter() - irq_exit() These modifications make the do_entInt() more readable. They do not affect the original program logic. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/irq.c | 2 - arch/sw_64/kernel/smp.c | 2 - drivers/clocksource/timer-sw64.c | 3 -- drivers/irqchip/irq-sunway-cpu.c | 63 +++++++++++++++----------------- 4 files changed, 30 insertions(+), 40 deletions(-) diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c index bdc92b40c4c0..6f367a2db319 100644 --- a/arch/sw_64/kernel/irq.c +++ b/arch/sw_64/kernel/irq.c @@ -98,9 +98,7 @@ handle_irq(int irq) return; } - irq_enter(); generic_handle_irq_desc(desc); - irq_exit(); } #ifdef CONFIG_HOTPLUG_CPU diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 64571b45dd68..aa110f555a29 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -594,9 +594,7 @@ void handle_ipi(struct pt_regs *regs) break; case IPI_CALL_FUNC: - irq_enter(); generic_smp_call_function_interrupt(); - irq_exit(); break; case IPI_CPU_STOP: diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index c70cf340a4b9..c7233617283f 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -418,7 +418,6 @@ void sw64_timer_interrupt(void) { struct clock_event_device *evt = this_cpu_ptr(&timer_events); - irq_enter(); if (!evt->event_handler) { pr_warn("Spurious local timer interrupt on cpu %d\n", smp_processor_id()); @@ -429,6 +428,4 @@ void sw64_timer_interrupt(void) inc_irq_stat(timer_irqs_event); evt->event_handler(evt); - - irq_exit(); } diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 109bf4acedde..31f170e0a5aa 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -181,16 +181,21 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, struct pt_regs *old_regs; extern char __idle_start[], __idle_end[]; + /* restart idle routine if it is interrupted */ + if (regs->pc > (u64)__idle_start && regs->pc < (u64)__idle_end) + regs->pc = (u64)__idle_start; + + irq_enter(); + old_regs = set_irq_regs(regs); + #ifdef CONFIG_SUBARCH_C4 if (pme_state == PME_WFW) { pme_state = PME_PENDING; - return; + goto out; } if (pme_state == PME_PENDING) { - old_regs = set_irq_regs(regs); handle_device_interrupt(vector); - set_irq_regs(old_regs); pme_state = PME_CLEAR; } #endif @@ -205,74 +210,66 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, } } - /* restart idle routine if it is interrupted */ - if (regs->pc > (u64)__idle_start && regs->pc < (u64)__idle_end) - regs->pc = (u64)__idle_start; - switch (type & 0xffff) { case INT_MSI: - old_regs = set_irq_regs(regs); if (is_guest_or_emul()) vt_handle_pci_msi_interrupt(type, vector, irq_arg); else handle_pci_msi_interrupt(type, vector, irq_arg); - set_irq_regs(old_regs); - return; + goto out; case INT_INTx: - old_regs = set_irq_regs(regs); handle_device_interrupt(vector); - set_irq_regs(old_regs); - return; + goto out; case INT_IPI: #ifdef CONFIG_SMP handle_ipi(regs); - return; + goto out; #else irq_err_count++; pr_crit("Interprocessor interrupt? You must be kidding!\n"); -#endif break; +#endif case INT_RTC: - old_regs = set_irq_regs(regs); sw64_timer_interrupt(); - set_irq_regs(old_regs); - return; + goto out; case INT_VT_SERIAL: case INT_VT_HOTPLUG: case INT_VT_GPIOA_PIN0: - old_regs = set_irq_regs(regs); handle_irq(type); - set_irq_regs(old_regs); - return; + goto out; +#if defined(CONFIG_SUBARCH_C3B) case INT_PC0: perf_irq(PMC_PC0, regs); - return; + goto out; case INT_PC1: perf_irq(PMC_PC1, regs); - return; + goto out; +#elif defined(CONFIG_SUBARCH_C4) + case INT_PC: + perf_irq(PMC_PC0, regs); + goto out; +#endif case INT_DEV: handle_dev_int(regs); - return; + goto out; case INT_FAULT: - old_regs = set_irq_regs(regs); handle_fault_int(); - set_irq_regs(old_regs); - return; + goto out; case INT_MT: - old_regs = set_irq_regs(regs); handle_mt_int(); - set_irq_regs(old_regs); - return; + goto out; case INT_NMI: - old_regs = set_irq_regs(regs); handle_nmi_int(); - set_irq_regs(old_regs); - return; + goto out; default: pr_crit("Hardware intr %ld %lx? uh?\n", type, vector); } pr_crit("PC = %016lx PS = %04lx\n", regs->pc, regs->ps); + +out: + set_irq_regs(old_regs); + irq_exit(); } EXPORT_SYMBOL(do_entInt); -- Gitee From 575727de1c29a3949cbb74785144d6f411f26a8d Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 24 May 2024 17:07:21 +0800 Subject: [PATCH 225/524] sw64: add CSR_CAUSE into pt_regs Add a soft_csr called CSR_CAUSE into pt_regs to record the cause of trap. It can be used to unify the trap entries and identify the nmi to be added in the future. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/csr.h | 1 + arch/sw_64/include/asm/ptrace.h | 1 + arch/sw_64/kernel/asm-offsets.c | 1 + arch/sw_64/kernel/entry_c4.S | 2 ++ 4 files changed, 5 insertions(+) diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h index cdb9249eedd2..917894152e36 100644 --- a/arch/sw_64/include/asm/csr.h +++ b/arch/sw_64/include/asm/csr.h @@ -59,6 +59,7 @@ #define CSR_SCRATCH 0xed #define CSR_SP 0xee #define CSR_KTP 0xef +#define CSR_CAUSE 0xf0 #define DA_MATCH_EN_S 4 #define DV_MATCH_EN_S 6 diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h index 44c9e2d1fe6a..0bbe85297f62 100644 --- a/arch/sw_64/include/asm/ptrace.h +++ b/arch/sw_64/include/asm/ptrace.h @@ -27,6 +27,7 @@ struct pt_regs { }; unsigned long orig_r0; unsigned long orig_r19; + unsigned long cause; /* These are saved by HMcode: */ unsigned long hm_ps; unsigned long hm_pc; diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c index 334f7f2e9550..337386c163ff 100644 --- a/arch/sw_64/kernel/asm-offsets.c +++ b/arch/sw_64/kernel/asm-offsets.c @@ -104,6 +104,7 @@ void foo(void) DEFINE(PT_REGS_PS, offsetof(struct pt_regs, ps)); DEFINE(PT_REGS_ORIG_R0, offsetof(struct pt_regs, orig_r0)); DEFINE(PT_REGS_ORIG_R19, offsetof(struct pt_regs, orig_r19)); + DEFINE(PT_REGS_CAUSE, offsetof(struct pt_regs, cause)); DEFINE(PT_REGS_HM_PS, offsetof(struct pt_regs, hm_ps)); DEFINE(PT_REGS_HM_PC, offsetof(struct pt_regs, hm_pc)); DEFINE(PT_REGS_HM_GP, offsetof(struct pt_regs, hm_gp)); diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S index c31143645b6f..08760f2f9f73 100644 --- a/arch/sw_64/kernel/entry_c4.S +++ b/arch/sw_64/kernel/entry_c4.S @@ -66,6 +66,7 @@ csrr $1, CSR_SP csrr $2, CSR_PS csrr $3, CSR_PC + csrr $4, CSR_CAUSE csrr $16, CSR_EARG0 csrr $17, CSR_EARG1 csrr $18, CSR_EARG2 @@ -76,6 +77,7 @@ stl $1, PT_REGS_SP($sp) stl $2, PT_REGS_PS($sp) stl $3, PT_REGS_PC($sp) + stl $4, PT_REGS_CAUSE($sp) ldi $1, NO_SYSCALL stl $1, PT_REGS_ORIG_R0($sp) csrr $8, CSR_KTP -- Gitee From 1560ec95358594e9a1b2ebf17fe30979df5f1ac2 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 24 May 2024 17:08:08 +0800 Subject: [PATCH 226/524] sw64: add NMI support Support the Non-Maskable Interrupt(NMI). Any type of interrupt can be configured to be non-maskable. NMI has its separate stack, also has a separate SAVE_ALL_NMI and RESTORE_ALL_NMI. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/core.h | 1 + arch/sw_64/include/asm/csr.h | 6 +- arch/sw_64/include/asm/hmcall.h | 4 + arch/sw_64/kernel/entry_c4.S | 139 +++++++++++++++++++++++++++++++ arch/sw_64/kernel/irq.c | 10 +++ arch/sw_64/kernel/smp.c | 14 ++++ arch/sw_64/kernel/traps.c | 19 +++++ arch/sw_64/kvm/entry_core4.S | 2 + drivers/irqchip/irq-sunway-cpu.c | 11 ++- 9 files changed, 202 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/include/asm/core.h b/arch/sw_64/include/asm/core.h index 2b6748cec93d..9ce1d6b7418e 100644 --- a/arch/sw_64/include/asm/core.h +++ b/arch/sw_64/include/asm/core.h @@ -78,6 +78,7 @@ static inline bool core_is_ht(void) extern void entArith(void); extern void entIF(void); extern void entInt(void); +extern void entNMI(void); extern void entMM(void); extern void entSys(void); extern void entUna(void); diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h index 917894152e36..9101e4c91fe1 100644 --- a/arch/sw_64/include/asm/csr.h +++ b/arch/sw_64/include/asm/csr.h @@ -51,6 +51,9 @@ #define CSR_IDA_MATCH 0xc5 #define CSR_IDA_MASK 0xc6 #define CSR_BASE_KREGS 0xe0 +#define CSR_NMI_STACK 0xe5 +#define CSR_NMI_SCRATCH 0xe6 +#define CSR_NMI_MASK 0xe7 #define CSR_PS 0xe8 #define CSR_PC 0xe9 #define CSR_EARG0 0xea @@ -79,7 +82,8 @@ static inline unsigned long sw64_read_csr(unsigned long x) { unsigned long __val; - __asm__ __volatile__("csrr %0,%1" : "=r"(__val) : "i"(x)); + + __asm__ __volatile__("csrr %0,%1; csrr %0,%1" : "=r"(__val) : "i"(x)); return __val; } diff --git a/arch/sw_64/include/asm/hmcall.h b/arch/sw_64/include/asm/hmcall.h index c9509327a229..fd873e825ae1 100644 --- a/arch/sw_64/include/asm/hmcall.h +++ b/arch/sw_64/include/asm/hmcall.h @@ -44,6 +44,9 @@ #define HMC_sendii 0x3E #define HMC_rti 0x3F +/* 0x40 - 0x7F : Hypervisor Level HMC routine */ +#define HMC_rti_nmi 0x40 +#define HMC_setup_nmi 0x41 /* 0x80 - 0xBF : User Level HMC routine */ #include @@ -185,6 +188,7 @@ __CALL_HMC_RW1(rdio64, unsigned long, unsigned long); __CALL_HMC_RW1(rdio32, unsigned int, unsigned long); __CALL_HMC_W2(wrent, void*, unsigned long); __CALL_HMC_W2(tbisasid, unsigned long, unsigned long); +__CALL_HMC_W2(setup_nmi, unsigned long, unsigned long); __CALL_HMC_W1(wrkgp, unsigned long); __CALL_HMC_RW2(wrperfmon, unsigned long, unsigned long, unsigned long); __CALL_HMC_RW3(sendii, unsigned long, unsigned long, unsigned long, unsigned long); diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S index 08760f2f9f73..219016ef05b2 100644 --- a/arch/sw_64/kernel/entry_c4.S +++ b/arch/sw_64/kernel/entry_c4.S @@ -14,6 +14,123 @@ .text .set noat + .macro SAVE_ALL_NMI + csrw $sp, CSR_NMI_SCRATCH + csrr $sp, CSR_NMI_STACK + + ldi $sp, -PT_REGS_SIZE($sp) + stl $0, PT_REGS_R0($sp) + stl $1, PT_REGS_R1($sp) + stl $2, PT_REGS_R2($sp) + stl $3, PT_REGS_R3($sp) + stl $4, PT_REGS_R4($sp) + stl $5, PT_REGS_R5($sp) + stl $6, PT_REGS_R6($sp) + stl $7, PT_REGS_R7($sp) + stl $8, PT_REGS_R8($sp) + stl $9, PT_REGS_R9($sp) + stl $10, PT_REGS_R10($sp) + stl $11, PT_REGS_R11($sp) + stl $12, PT_REGS_R12($sp) + stl $13, PT_REGS_R13($sp) + stl $14, PT_REGS_R14($sp) + stl $15, PT_REGS_R15($sp) + stl $16, PT_REGS_R16($sp) + stl $17, PT_REGS_R17($sp) + stl $18, PT_REGS_R18($sp) + stl $19, PT_REGS_R19($sp) + stl $20, PT_REGS_R20($sp) + stl $21, PT_REGS_R21($sp) + stl $22, PT_REGS_R22($sp) + stl $23, PT_REGS_R23($sp) + stl $24, PT_REGS_R24($sp) + stl $25, PT_REGS_R25($sp) + stl $26, PT_REGS_R26($sp) + stl $27, PT_REGS_R27($sp) + stl $28, PT_REGS_R28($sp) + stl $29, PT_REGS_GP($sp) + + /* Due to a flaw in the CSR design, in some cases the + * read CSR instruction is sent before the write instruction, + * so it needs to be read twice to ensure that the correct + * CSR value is read. Don`t delete it! + */ + csrr $1, CSR_NMI_SCRATCH + csrr $1, CSR_NMI_SCRATCH + csrr $2, CSR_PS + csrr $3, CSR_PC + csrr $4, CSR_CAUSE + csrr $16, CSR_EARG0 + csrr $17, CSR_EARG1 + csrr $18, CSR_EARG2 + stl $16, PT_REGS_EARG0($sp) + stl $17, PT_REGS_EARG1($sp) + stl $18, PT_REGS_EARG2($sp) + + stl $1, PT_REGS_SP($sp) + stl $2, PT_REGS_PS($sp) + stl $3, PT_REGS_PC($sp) + stl $4, PT_REGS_CAUSE($sp) + ldi $1, NO_SYSCALL + stl $1, PT_REGS_ORIG_R0($sp) + csrr $8, CSR_KTP +#ifdef CONFIG_FRAME_POINTER + ldi $sp, -STACKFRAME_SIZE($sp) + stl $3, STACKFRAME_PC($sp) + stl $15, STACKFRAME_FP($sp) + mov $sp, $15 +#endif + br $27, 1f +1: ldgp $29, 0($27) + call $26, save_nmi_ctx + ldl $16, (PT_REGS_EARG0 + STACKFRAME_SIZE)($sp) + ldl $17, (PT_REGS_EARG1 + STACKFRAME_SIZE)($sp) + ldl $18, (PT_REGS_EARG2 + STACKFRAME_SIZE)($sp) + .endm + + .macro RESTORE_ALL_NMI + call $26, restore_nmi_ctx + + ldi $sp, STACKFRAME_SIZE($sp) + + ldl $1, PT_REGS_PS($sp) + ldl $2, PT_REGS_PC($sp) + csrw $1, CSR_PS + csrw $2, CSR_PC + + ldl $0, PT_REGS_R0($sp) + ldl $1, PT_REGS_R1($sp) + ldl $2, PT_REGS_R2($sp) + ldl $3, PT_REGS_R3($sp) + ldl $4, PT_REGS_R4($sp) + ldl $5, PT_REGS_R5($sp) + ldl $6, PT_REGS_R6($sp) + ldl $7, PT_REGS_R7($sp) + ldl $8, PT_REGS_R8($sp) + ldl $9, PT_REGS_R9($sp) + ldl $10, PT_REGS_R10($sp) + ldl $11, PT_REGS_R11($sp) + ldl $12, PT_REGS_R12($sp) + ldl $13, PT_REGS_R13($sp) + ldl $14, PT_REGS_R14($sp) + ldl $15, PT_REGS_R15($sp) + ldl $16, PT_REGS_R16($sp) + ldl $17, PT_REGS_R17($sp) + ldl $18, PT_REGS_R18($sp) + ldl $19, PT_REGS_R19($sp) + ldl $20, PT_REGS_R20($sp) + ldl $21, PT_REGS_R21($sp) + ldl $22, PT_REGS_R22($sp) + ldl $23, PT_REGS_R23($sp) + ldl $24, PT_REGS_R24($sp) + ldl $25, PT_REGS_R25($sp) + ldl $26, PT_REGS_R26($sp) + ldl $27, PT_REGS_R27($sp) + ldl $28, PT_REGS_R28($sp) + ldl $29, PT_REGS_GP($sp) + ldl $sp, PT_REGS_SP($sp) + .endm + .macro SAVE_ALL csrw $sp, CSR_SP csrw $1, CSR_SCRATCH @@ -146,6 +263,18 @@ * Non-syscall kernel entry points. */ + .align 4 + .globl entNMI + .ent entNMI +entNMI: + SAVE_ALL_NMI + br $27, 1f +1: ldgp $29, 0($27) + ldi $19, STACKFRAME_SIZE($sp) + call $26, do_entInt + br ret_from_nmi + .end entNMI + .align 4 .globl entInt .ent entInt @@ -260,6 +389,16 @@ entSys: br ret_from_sys_call .end entSys + .align 4 + .globl ret_from_nmi + .ent ret_from_nmi +ret_from_nmi: + br $27, 1f +1: ldgp $29, 0($27) + RESTORE_ALL_NMI + sys_call HMC_rti_nmi + .end ret_from_nmi + .align 4 .globl ret_from_sys_call .ent ret_from_sys_call diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c index 6f367a2db319..8066dcf91424 100644 --- a/arch/sw_64/kernel/irq.c +++ b/arch/sw_64/kernel/irq.c @@ -114,7 +114,17 @@ void __init init_IRQ(void) * Just in case the platform init_irq() causes interrupts/mchecks * (as is the case with RAWHIDE, at least). */ + struct page __maybe_unused *nmi_stack_page = alloc_pages_node( + cpu_to_node(smp_processor_id()), + THREADINFO_GFP, THREAD_SIZE_ORDER); + unsigned long nmi_stack __maybe_unused = nmi_stack_page ? + (unsigned long)page_address(nmi_stack_page) : 0; + wrent(entInt, 0); + if (IS_ENABLED(CONFIG_SUBARCH_C4) && is_in_host()) { + sw64_write_csr_imb(nmi_stack + THREAD_SIZE, CSR_NMI_STACK); + wrent(entNMI, 6); + } sw64_init_irq(); irqchip_init(); diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index aa110f555a29..664da9f66cea 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -106,6 +106,9 @@ static void downshift_freq(void) { } void smp_callin(void) { int cpuid; + struct page __maybe_unused *nmi_stack_page; + unsigned long __maybe_unused nmi_stack; + save_ktp(); upshift_freq(); cpuid = smp_processor_id(); @@ -133,6 +136,17 @@ void smp_callin(void) /* update csr:ptbr */ update_ptbr_sys(virt_to_phys(init_mm.pgd)); + if (IS_ENABLED(CONFIG_SUBARCH_C4) && is_in_host()) { + nmi_stack_page = alloc_pages_node( + cpu_to_node(smp_processor_id()), + THREADINFO_GFP, + THREAD_SIZE_ORDER); + nmi_stack = nmi_stack_page ? + (unsigned long)page_address(nmi_stack_page) : 0; + sw64_write_csr_imb(nmi_stack + THREAD_SIZE, CSR_NMI_STACK); + wrent(entNMI, 6); + } + /* inform the notifiers about the new cpu */ notify_cpu_starting(cpuid); diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index c9e956b01dc9..e54db9a41be6 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -2006,6 +2006,25 @@ asmlinkage void do_entSys(struct pt_regs *regs) syscall_trace_leave(); } +struct nmi_ctx { + unsigned long csr_sp; + unsigned long csr_scratch; +}; + +DEFINE_PER_CPU(struct nmi_ctx, nmi_context); + +void save_nmi_ctx(void) +{ + this_cpu_write(nmi_context.csr_sp, sw64_read_csr(CSR_SP)); + this_cpu_write(nmi_context.csr_scratch, sw64_read_csr(CSR_SCRATCH)); +} + +void restore_nmi_ctx(void) +{ + sw64_write_csr_imb(this_cpu_read(nmi_context.csr_sp), CSR_SP); + sw64_write_csr_imb(this_cpu_read(nmi_context.csr_scratch), CSR_SCRATCH); +} + void trap_init(void) { diff --git a/arch/sw_64/kvm/entry_core4.S b/arch/sw_64/kvm/entry_core4.S index fb6757425f76..98993884fd08 100644 --- a/arch/sw_64/kvm/entry_core4.S +++ b/arch/sw_64/kvm/entry_core4.S @@ -266,6 +266,8 @@ $setfpec_over: csrr $18, CSR_EARG2 ldi $19, -PT_REGS_SIZE(sp) + csrr $1, CSR_CAUSE + stl $1, PT_REGS_CAUSE($19) call $26, do_entInt ldl $26, VCPU_RET_RA(sp) ldl $0, VCPU_RET_R0(sp) diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 31f170e0a5aa..f68bca18923f 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -184,8 +184,10 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, /* restart idle routine if it is interrupted */ if (regs->pc > (u64)__idle_start && regs->pc < (u64)__idle_end) regs->pc = (u64)__idle_start; - - irq_enter(); + if (regs->cause != -2) + irq_enter(); + else + nmi_enter(); old_regs = set_irq_regs(regs); #ifdef CONFIG_SUBARCH_C4 @@ -269,7 +271,10 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, out: set_irq_regs(old_regs); - irq_exit(); + if (regs->cause != -2) + irq_exit(); + else + nmi_exit(); } EXPORT_SYMBOL(do_entInt); -- Gitee From bb8e4da153edc4766330b89c502db8212542c082 Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Tue, 21 May 2024 19:53:19 +0800 Subject: [PATCH 227/524] sw64: fix bugs in decreasing frequency The issue arose because the downshift_freq() was attempting to scale down the frequency of a physicial core when only one of its logical core was offline. The correct behavior should be to scale down the frequency only when both logical cores of a pyhsicial core are offline. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 664da9f66cea..25a281c9394a 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -68,6 +68,7 @@ static void upshift_freq(void) if (is_guest_or_emul()) return; + cpu_num = sw64_chip->get_cpu_num(); for (i = 0; i < cpu_num; i++) { sw64_io_write(i, CLU_LV2_SELH, -1UL); @@ -79,11 +80,21 @@ static void upshift_freq(void) static void downshift_freq(void) { unsigned long value; - int cpuid, core_id, node_id; + int core_id, node_id, cpu; + int cpuid = smp_processor_id(); + struct cpu_topology *cpu_topo = &cpu_topology[cpuid]; if (is_guest_or_emul()) return; - cpuid = smp_processor_id(); + + for_each_online_cpu(cpu) { + struct cpu_topology *sib_topo = &cpu_topology[cpu]; + + if ((cpu_topo->package_id == sib_topo->package_id) && + (cpu_topo->core_id == sib_topo->core_id)) + return; + } + core_id = rcid_to_core_id(cpu_to_rcid(cpuid)); node_id = rcid_to_domain_id(cpu_to_rcid(cpuid)); -- Gitee From 4b427fa752dafc838353af2d01bc54e0d978fa96 Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Fri, 31 May 2024 13:55:09 +0800 Subject: [PATCH 228/524] sw64: add HARDLOCKUP_DETECTOR support Hardlockup depends on non-maskable perf interrupts, so set PMI as NMI and make hw_nmi_get_sample_period() based on current frequecy. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 2 ++ arch/sw_64/include/asm/hmcall.h | 3 +++ arch/sw_64/include/asm/irq_impl.h | 5 +++-- arch/sw_64/kernel/irq.c | 1 + arch/sw_64/kernel/setup.c | 7 +++++++ arch/sw_64/kernel/smp.c | 2 ++ 6 files changed, 18 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 9890ec38f695..9465fe569608 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -98,6 +98,8 @@ config SW64 select HAVE_PCI select HAVE_PCSPKR_PLATFORM select HAVE_PERF_EVENTS + select HAVE_PERF_EVENTS_NMI if SUBARCH_C4 + select HAVE_HARDLOCKUP_DETECTOR_PERF if PERF_EVENTS && HAVE_PERF_EVENTS_NMI select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_REGS_AND_STACK_ACCESS_API diff --git a/arch/sw_64/include/asm/hmcall.h b/arch/sw_64/include/asm/hmcall.h index fd873e825ae1..574144eab22b 100644 --- a/arch/sw_64/include/asm/hmcall.h +++ b/arch/sw_64/include/asm/hmcall.h @@ -248,6 +248,9 @@ static inline void wrap_asid(unsigned long asid, unsigned long ptbr) #define save_ktp() wrktp() #endif +#define set_nmi(irq) setup_nmi(1, (irq)) +#define clear_nmi(irq) setup_nmi(0, (irq)) + #endif /* !__ASSEMBLY__ */ #endif /* __KERNEL__ */ diff --git a/arch/sw_64/include/asm/irq_impl.h b/arch/sw_64/include/asm/irq_impl.h index 8040d22c53a5..c4f2690bca2e 100644 --- a/arch/sw_64/include/asm/irq_impl.h +++ b/arch/sw_64/include/asm/irq_impl.h @@ -24,8 +24,9 @@ enum sw64_irq_type { INT_IPI = 1, - INT_PC0 = 2, - INT_PC1 = 3, + INT_PC = 2, /* C4 PMI */ + INT_PC0 = 2, /* C3 PMI 0 */ + INT_PC1 = 3, /* C3 PMI 1 */ INT_INTx = 5, INT_MSI = 6, INT_MT = 7, diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c index 8066dcf91424..5706df471a8d 100644 --- a/arch/sw_64/kernel/irq.c +++ b/arch/sw_64/kernel/irq.c @@ -124,6 +124,7 @@ void __init init_IRQ(void) if (IS_ENABLED(CONFIG_SUBARCH_C4) && is_in_host()) { sw64_write_csr_imb(nmi_stack + THREAD_SIZE, CSR_NMI_STACK); wrent(entNMI, 6); + set_nmi(INT_PC); } sw64_init_irq(); diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index ee0b7850e820..839176939efb 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -145,6 +145,13 @@ void store_cpu_data(int cpu) cpu_data[cpu].last_asid = ASID_FIRST_VERSION; } +#ifdef CONFIG_HARDLOCKUP_DETECTOR_PERF +u64 hw_nmi_get_sample_period(int watchdog_thresh) +{ + return get_cpu_freq() * watchdog_thresh; +} +#endif + /* * I/O resources inherited from PeeCees. Except for perhaps the * turbochannel SWs, everyone has these on some sort of SuperIO chip. diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 25a281c9394a..3696d9f9f657 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -156,6 +157,7 @@ void smp_callin(void) (unsigned long)page_address(nmi_stack_page) : 0; sw64_write_csr_imb(nmi_stack + THREAD_SIZE, CSR_NMI_STACK); wrent(entNMI, 6); + set_nmi(INT_PC); } /* inform the notifiers about the new cpu */ -- Gitee From a36ad6cd19e292ae7398b4bfbad14244fe4e6abb Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 3 Jun 2024 14:46:41 +0800 Subject: [PATCH 229/524] sw64: defconfig: update defconfig for kata-containers For kata-containers in sw64, we rename kata_guest_defconfig to kata xuelang_defconfig for C3B, and add kata_junzhang_defconfig for C4. Signed-off-by: Chen Wang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- ...uest_defconfig => kata_junzhang_defconfig} | 129 +++++++++++++--- arch/sw_64/configs/kata_xuelang_defconfig | 145 +++++++++++++++--- 2 files changed, 226 insertions(+), 48 deletions(-) rename arch/sw_64/configs/{kata_guest_defconfig => kata_junzhang_defconfig} (85%) diff --git a/arch/sw_64/configs/kata_guest_defconfig b/arch/sw_64/configs/kata_junzhang_defconfig similarity index 85% rename from arch/sw_64/configs/kata_guest_defconfig rename to arch/sw_64/configs/kata_junzhang_defconfig index 6a7c442b94a7..160944e463dc 100644 --- a/arch/sw_64/configs/kata_guest_defconfig +++ b/arch/sw_64/configs/kata_junzhang_defconfig @@ -1,4 +1,4 @@ -CONFIG_LOCALVERSION="-xuelang" +CONFIG_LOCALVERSION="-junzhang" CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y # CONFIG_CROSS_MEMORY_ATTACH is not set @@ -6,6 +6,7 @@ CONFIG_USELIB=y CONFIG_HIGH_RES_TIMERS=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y +CONFIG_IKHEADERS=y CONFIG_LOG_BUF_SHIFT=18 CONFIG_MEMCG=y CONFIG_BLK_CGROUP=y @@ -17,6 +18,7 @@ CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y CONFIG_CGROUP_PERF=y CONFIG_NAMESPACES=y +CONFIG_USER_NS=y CONFIG_SCHED_AUTOGROUP=y CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y @@ -26,21 +28,17 @@ CONFIG_BPF_SYSCALL=y CONFIG_PERF_EVENTS=y CONFIG_DEBUG_PERF_USE_VMALLOC=y # CONFIG_COMPAT_BRK is not set -CONFIG_CPUFREQ_DEBUGFS=y -# CONFIG_LOCK_MEMB is not set +CONFIG_SUBARCH_C4=y CONFIG_SMP=y +CONFIG_SCHED_SMT=y CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_NUMA=y CONFIG_HZ=100 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=y -CONFIG_BUILTIN_DTB=y -CONFIG_BUILTIN_DTB_NAME="chip_vt" CONFIG_FIRMWARE_MEMMAP=y CONFIG_DMI_SYSFS=m -CONFIG_GOOGLE_FIRMWARE=y -CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE=y -CONFIG_SW64_SUSPEND_DEEPSLEEP_BOOTCORE=y +CONFIG_ACPI_TAD=y # CONFIG_CPU_IDLE is not set CONFIG_VIRTUALIZATION=y CONFIG_KVM=y @@ -55,6 +53,7 @@ CONFIG_MODULE_FORCE_LOAD=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_MODVERSIONS=y +CONFIG_BLK_DEV_THROTTLING=y CONFIG_PARTITION_ADVANCED=y CONFIG_OSF_PARTITION=y CONFIG_BSD_DISKLABEL=y @@ -67,6 +66,7 @@ CONFIG_ULTRIX_PARTITION=y CONFIG_MEMORY_HOTPLUG=y CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE=y CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_CMA_AREAS=7 CONFIG_NET=y CONFIG_PACKET=y CONFIG_PACKET_DIAG=y @@ -108,7 +108,7 @@ CONFIG_INET6_IPCOMP=m CONFIG_IPV6_MIP6=m CONFIG_IPV6_ILA=m CONFIG_IPV6_VTI=m -CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_SIT=m CONFIG_IPV6_GRE=m CONFIG_IPV6_SUBTREES=y CONFIG_IPV6_MROUTE=y @@ -117,7 +117,8 @@ CONFIG_IPV6_PIMSM_V2=y CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPV6_SEG6_HMAC=y CONFIG_NETFILTER=y -CONFIG_NF_CONNTRACK=m +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NF_CONNTRACK=y CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y CONFIG_NF_CONNTRACK_EVENTS=y @@ -137,7 +138,7 @@ CONFIG_NF_CT_NETLINK=m CONFIG_NF_CT_NETLINK_TIMEOUT=m CONFIG_NF_CT_NETLINK_HELPER=m CONFIG_NETFILTER_NETLINK_GLUE_CT=y -CONFIG_NF_TABLES=y +CONFIG_NF_TABLES=m CONFIG_NF_TABLES_NETDEV=y CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m @@ -209,6 +210,7 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -239,7 +241,11 @@ CONFIG_IP_SET_HASH_NETNET=m CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m -CONFIG_IP_VS=m +CONFIG_IP_VS=y +CONFIG_IP_VS_PROTO_TCP=y +CONFIG_IP_VS_PROTO_UDP=y +CONFIG_IP_VS_RR=y +CONFIG_IP_VS_NFCT=y CONFIG_NF_TABLES_IPV4=y CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -267,7 +273,7 @@ CONFIG_IP_NF_SECURITY=m CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_NF_TABLES_BRIDGE=y +CONFIG_NF_TABLES_BRIDGE=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -333,7 +339,7 @@ CONFIG_CLS_U32_MARK=y CONFIG_NET_CLS_RSVP=m CONFIG_NET_CLS_RSVP6=m CONFIG_NET_CLS_FLOW=m -CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_CGROUP=y CONFIG_NET_CLS_BPF=m CONFIG_NET_CLS_FLOWER=m CONFIG_NET_CLS_MATCHALL=m @@ -364,7 +370,6 @@ CONFIG_NET_IFE_SKBPRIO=m CONFIG_NET_IFE_SKBTCINDEX=m CONFIG_OPENVSWITCH=m CONFIG_VSOCKETS=y -CONFIG_VSOCKETS_DIAG=m CONFIG_VIRTIO_VSOCKETS=y CONFIG_NETLINK_DIAG=m CONFIG_CGROUP_NET_PRIO=y @@ -372,6 +377,15 @@ CONFIG_BPF_JIT=y # CONFIG_WIRELESS is not set CONFIG_NET_9P=y CONFIG_NET_9P_VIRTIO=y +CONFIG_PCI=y +CONFIG_PCIEPORTBUS=y +CONFIG_PCIEAER=y +# CONFIG_PCIEASPM is not set +CONFIG_PCI_MSI=y +CONFIG_PCI_IOV=y +CONFIG_UEVENT_HELPER=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y # CONFIG_STANDALONE is not set # CONFIG_PREVENT_FIRMWARE_BUILD is not set CONFIG_MTD=y @@ -390,12 +404,12 @@ CONFIG_MTD_PHYSMAP=y CONFIG_MTD_PHYSMAP_OF=y CONFIG_MTD_PLATRAM=y CONFIG_MTD_SPI_NOR=y -CONFIG_OF_OVERLAY=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=5000000 CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_NVME=y CONFIG_NVME_MULTIPATH=y CONFIG_NVME_RDMA=m CONFIG_NVME_FC=y @@ -420,12 +434,22 @@ CONFIG_SCSI_SAS_ATA=y CONFIG_SCSI_SRP_ATTRS=y CONFIG_ISCSI_TCP=m CONFIG_ISCSI_BOOT_SYSFS=y +CONFIG_SCSI_CXGB3_ISCSI=m +CONFIG_SCSI_CXGB4_ISCSI=m +CONFIG_SCSI_BNX2_ISCSI=m +CONFIG_MEGARAID_NEWGEN=y +CONFIG_MEGARAID_MM=m +CONFIG_MEGARAID_MAILBOX=m +CONFIG_MEGARAID_LEGACY=m +CONFIG_MEGARAID_SAS=m +CONFIG_SCSI_MPT3SAS=m CONFIG_SCSI_DH=y CONFIG_SCSI_DH_RDAC=y CONFIG_SCSI_DH_HP_SW=y CONFIG_SCSI_DH_EMC=y CONFIG_SCSI_DH_ALUA=y CONFIG_ATA=y +CONFIG_SATA_AHCI=y # CONFIG_ATA_SFF is not set CONFIG_MD=y CONFIG_MD_LINEAR=m @@ -467,6 +491,49 @@ CONFIG_TCM_PSCSI=m CONFIG_TCM_USER2=m CONFIG_LOOPBACK_TARGET=m CONFIG_ISCSI_TARGET=m +CONFIG_NET_FC=y +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_VIRTIO_NET=y +# CONFIG_NET_VENDOR_3COM is not set +# CONFIG_NET_VENDOR_AMD is not set +# CONFIG_NET_VENDOR_ARC is not set +# CONFIG_NET_VENDOR_AURORA is not set +CONFIG_CAVIUM_PTP=y +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_EZCHIP is not set +CONFIG_E100=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_IGB=y +CONFIG_IGBVF=m +CONFIG_IXGB=m +CONFIG_IXGBE=m +CONFIG_IXGBEVF=m +CONFIG_I40E=y +CONFIG_I40EVF=y +# CONFIG_NET_VENDOR_MARVELL is not set +CONFIG_MLX4_EN=y +CONFIG_MLX5_CORE=m +CONFIG_MLX5_FPGA=y +CONFIG_MLX5_CORE_EN=y +CONFIG_MLXSW_CORE=y +CONFIG_MLXSW_PCI=y +CONFIG_MLXSW_I2C=y +CONFIG_MLXSW_MINIMAL=y +# CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_QUALCOMM is not set +# CONFIG_NET_VENDOR_RENESAS is not set +# CONFIG_NET_VENDOR_ROCKER is not set +# CONFIG_NET_VENDOR_SAMSUNG is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SMSC is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set +# CONFIG_NET_VENDOR_VIA is not set +# CONFIG_NET_VENDOR_WIZNET is not set +# CONFIG_WLAN is not set CONFIG_INPUT_FF_MEMLESS=y CONFIG_INPUT_POLLDEV=y CONFIG_INPUT_MOUSEDEV=y @@ -477,22 +544,24 @@ CONFIG_INPUT_EVDEV=y CONFIG_SERIO_LIBPS2=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y +# CONFIG_SERIAL_8250_PCI is not set CONFIG_SERIAL_8250_SUNWAY=y CONFIG_SERIAL_OF_PLATFORM=y CONFIG_VIRTIO_CONSOLE=y -CONFIG_HW_RANDOM=y -CONFIG_HW_RANDOM_VIRTIO=y +# CONFIG_HW_RANDOM is not set # CONFIG_DEVPORT is not set # CONFIG_I2C_COMPAT is not set CONFIG_I2C_CHARDEV=y CONFIG_I2C_MUX=y +CONFIG_I2C_DESIGNWARE_PLATFORM=y CONFIG_SPI=y CONFIG_SPI_SPIDEV=y CONFIG_SENSORS_PVT=y CONFIG_SENSORS_LM75=y CONFIG_SSB=y -CONFIG_SUNWAY_SUPERIO_AST2400=y CONFIG_DRM=y +CONFIG_DRM_RADEON=y +CONFIG_DRM_AST=y CONFIG_DRM_VIRTIO_GPU=y CONFIG_FIRMWARE_EDID=y CONFIG_LCD_CLASS_DEVICE=y @@ -507,14 +576,26 @@ CONFIG_USB_STORAGE=y CONFIG_INFINIBAND=m CONFIG_INFINIBAND_USER_MAD=m CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_INFINIBAND_MTHCA=m +# CONFIG_INFINIBAND_MTHCA_DEBUG is not set +CONFIG_MLX4_INFINIBAND=m +CONFIG_MLX5_INFINIBAND=m +CONFIG_INFINIBAND_IPOIB=m +CONFIG_INFINIBAND_IPOIB_CM=y CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set # CONFIG_RTC_INTF_PROC is not set CONFIG_RTC_DRV_PCF8523=y CONFIG_UIO=y +CONFIG_UIO_PCI_GENERIC=m +CONFIG_VFIO=y +CONFIG_VIRTIO_PCI=y +# CONFIG_VIRTIO_PCI_LEGACY is not set CONFIG_VIRTIO_MMIO=y CONFIG_STAGING=y -CONFIG_SW64_LPC_INTC=y +CONFIG_FB_SM750=y +CONFIG_IOMMU_DEFAULT_PASSTHROUGH=y +CONFIG_SUNWAY_IOMMU_V2=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_EXT4_FS_SECURITY=y @@ -544,7 +625,6 @@ CONFIG_TMPFS=y CONFIG_TMPFS_POSIX_ACL=y CONFIG_HUGETLBFS=y CONFIG_CONFIGFS_FS=y -CONFIG_SQUASHFS=y CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_NFS_V4=y @@ -564,7 +644,7 @@ CONFIG_9P_FS=y CONFIG_9P_FSCACHE=y CONFIG_9P_FS_POSIX_ACL=y CONFIG_9P_FS_SECURITY=y -CONFIG_NLS_CODEPAGE_437=m +CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_737=m CONFIG_NLS_CODEPAGE_775=m CONFIG_NLS_CODEPAGE_850=m @@ -588,7 +668,7 @@ CONFIG_NLS_ISO8859_8=m CONFIG_NLS_CODEPAGE_1250=m CONFIG_NLS_CODEPAGE_1251=m CONFIG_NLS_ASCII=m -CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_1=y CONFIG_NLS_ISO8859_2=m CONFIG_NLS_ISO8859_3=m CONFIG_NLS_ISO8859_4=m @@ -619,14 +699,15 @@ CONFIG_SECURITY_INFINIBAND=y CONFIG_SECURITY_PATH=y CONFIG_CRYPTO_AUTHENC=y CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_SEQIV=y CONFIG_CRYPTO_ECHAINIV=y CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_AES=y CONFIG_CRYPTO_DES=y CONFIG_CRYPTO_DEFLATE=y CONFIG_CRYPTO_LZO=y # CONFIG_CRYPTO_HW is not set CONFIG_CONSOLE_LOGLEVEL_QUIET=7 # CONFIG_ENABLE_MUST_CHECK is not set -# CONFIG_FRAME_POINTER is not set CONFIG_SCHEDSTATS=y # CONFIG_RCU_TRACE is not set diff --git a/arch/sw_64/configs/kata_xuelang_defconfig b/arch/sw_64/configs/kata_xuelang_defconfig index c059bbb7a576..b6c4b4045b1f 100644 --- a/arch/sw_64/configs/kata_xuelang_defconfig +++ b/arch/sw_64/configs/kata_xuelang_defconfig @@ -6,14 +6,13 @@ CONFIG_USELIB=y CONFIG_HIGH_RES_TIMERS=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y +CONFIG_IKHEADERS=y CONFIG_LOG_BUF_SHIFT=18 CONFIG_MEMCG=y CONFIG_BLK_CGROUP=y CONFIG_CFS_BANDWIDTH=y -CONFIG_RT_GROUP_SCHED=y CONFIG_CGROUP_PIDS=y CONFIG_CGROUP_FREEZER=y -CONFIG_CGROUP_HUGETLB=y CONFIG_CPUSETS=y CONFIG_CGROUP_DEVICE=y CONFIG_CGROUP_CPUACCT=y @@ -25,25 +24,28 @@ CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y CONFIG_KALLSYMS_ALL=y +CONFIG_BPF_SYSCALL=y CONFIG_PERF_EVENTS=y CONFIG_DEBUG_PERF_USE_VMALLOC=y # CONFIG_COMPAT_BRK is not set -CONFIG_CPUFREQ_DEBUGFS=y -# CONFIG_LOCK_MEMB is not set CONFIG_SMP=y -CONFIG_HOTPLUG_CPU=y CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_NUMA=y CONFIG_HZ=100 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_BINFMT_MISC=y CONFIG_FIRMWARE_MEMMAP=y CONFIG_DMI_SYSFS=m -# CONFIG_SUSPEND is not set +CONFIG_ACPI_TAD=y # CONFIG_CPU_IDLE is not set CONFIG_VIRTUALIZATION=y CONFIG_KVM=y CONFIG_VHOST_NET=m -CONFIG_VHOST_VSOCK=y +CONFIG_VHOST_SCSI=m +CONFIG_VHOST_VSOCK=m +CONFIG_VHOST_CROSS_ENDIAN_LEGACY=y +CONFIG_KPROBES=y +CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_FORCE_LOAD=y CONFIG_MODULE_UNLOAD=y @@ -59,7 +61,10 @@ CONFIG_UNIXWARE_DISKLABEL=y CONFIG_LDM_PARTITION=y CONFIG_SGI_PARTITION=y CONFIG_ULTRIX_PARTITION=y +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE=y CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_CMA_AREAS=7 CONFIG_NET=y CONFIG_PACKET=y CONFIG_PACKET_DIAG=y @@ -67,7 +72,7 @@ CONFIG_UNIX=y CONFIG_UNIX_DIAG=y CONFIG_TLS=m CONFIG_TLS_DEVICE=y -CONFIG_XFRM_USER=m +CONFIG_XFRM_USER=y CONFIG_XFRM_INTERFACE=m CONFIG_XFRM_SUB_POLICY=y CONFIG_XFRM_STATISTICS=y @@ -91,7 +96,6 @@ CONFIG_INET_IPCOMP=m CONFIG_INET_UDP_DIAG=m CONFIG_TCP_CONG_ADVANCED=y CONFIG_TCP_MD5SIG=y -CONFIG_IPV6=m CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y CONFIG_IPV6_OPTIMISTIC_DAD=y @@ -111,7 +115,8 @@ CONFIG_IPV6_PIMSM_V2=y CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPV6_SEG6_HMAC=y CONFIG_NETFILTER=y -CONFIG_NF_CONNTRACK=m +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NF_CONNTRACK=y CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y CONFIG_NF_CONNTRACK_EVENTS=y @@ -203,6 +208,7 @@ CONFIG_NETFILTER_XT_MATCH_NFACCT=m CONFIG_NETFILTER_XT_MATCH_OSF=m CONFIG_NETFILTER_XT_MATCH_OWNER=m CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m CONFIG_NETFILTER_XT_MATCH_QUOTA=m CONFIG_NETFILTER_XT_MATCH_RATEEST=m @@ -233,10 +239,10 @@ CONFIG_IP_SET_HASH_NETNET=m CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m -CONFIG_IP_VS=m +CONFIG_IP_VS=y CONFIG_IP_VS_PROTO_TCP=y CONFIG_IP_VS_PROTO_UDP=y -CONFIG_IP_VS_RR=m +CONFIG_IP_VS_RR=y CONFIG_IP_VS_NFCT=y CONFIG_NF_TABLES_IPV4=y CONFIG_NFT_DUP_IPV4=m @@ -287,8 +293,7 @@ CONFIG_BRIDGE_EBT_REDIRECT=m CONFIG_BRIDGE_EBT_SNAT=m CONFIG_BRIDGE_EBT_LOG=m CONFIG_BRIDGE_EBT_NFLOG=m -CONFIG_BRIDGE=m -CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_BRIDGE=y CONFIG_VLAN_8021Q=m CONFIG_VLAN_8021Q_GVRP=y CONFIG_VLAN_8021Q_MVRP=y @@ -332,7 +337,7 @@ CONFIG_CLS_U32_MARK=y CONFIG_NET_CLS_RSVP=m CONFIG_NET_CLS_RSVP6=m CONFIG_NET_CLS_FLOW=m -CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_CGROUP=y CONFIG_NET_CLS_BPF=m CONFIG_NET_CLS_FLOWER=m CONFIG_NET_CLS_MATCHALL=m @@ -363,13 +368,20 @@ CONFIG_NET_IFE_SKBPRIO=m CONFIG_NET_IFE_SKBTCINDEX=m CONFIG_OPENVSWITCH=m CONFIG_VSOCKETS=y -CONFIG_VSOCKETS_DIAG=m +CONFIG_VIRTIO_VSOCKETS=y CONFIG_NETLINK_DIAG=m CONFIG_CGROUP_NET_PRIO=y CONFIG_BPF_JIT=y -CONFIG_NET_DROP_MONITOR=m # CONFIG_WIRELESS is not set -CONFIG_CAIF=m +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_PCI=y +CONFIG_PCIEPORTBUS=y +CONFIG_PCIEAER=y +# CONFIG_PCIEASPM is not set +CONFIG_PCI_MSI=y +CONFIG_PCI_IOV=y +CONFIG_UEVENT_HELPER=y CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y # CONFIG_STANDALONE is not set @@ -394,6 +406,8 @@ CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=5000000 +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_NVME=y CONFIG_NVME_MULTIPATH=y CONFIG_NVME_RDMA=m CONFIG_NVME_FC=y @@ -418,12 +432,22 @@ CONFIG_SCSI_SAS_ATA=y CONFIG_SCSI_SRP_ATTRS=y CONFIG_ISCSI_TCP=m CONFIG_ISCSI_BOOT_SYSFS=y +CONFIG_SCSI_CXGB3_ISCSI=m +CONFIG_SCSI_CXGB4_ISCSI=m +CONFIG_SCSI_BNX2_ISCSI=m +CONFIG_MEGARAID_NEWGEN=y +CONFIG_MEGARAID_MM=m +CONFIG_MEGARAID_MAILBOX=m +CONFIG_MEGARAID_LEGACY=m +CONFIG_MEGARAID_SAS=m +CONFIG_SCSI_MPT3SAS=m CONFIG_SCSI_DH=y CONFIG_SCSI_DH_RDAC=y CONFIG_SCSI_DH_HP_SW=y CONFIG_SCSI_DH_EMC=y CONFIG_SCSI_DH_ALUA=y CONFIG_ATA=y +CONFIG_SATA_AHCI=y # CONFIG_ATA_SFF is not set CONFIG_MD=y CONFIG_MD_LINEAR=m @@ -465,6 +489,49 @@ CONFIG_TCM_PSCSI=m CONFIG_TCM_USER2=m CONFIG_LOOPBACK_TARGET=m CONFIG_ISCSI_TARGET=m +CONFIG_NET_FC=y +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_VIRTIO_NET=y +# CONFIG_NET_VENDOR_3COM is not set +# CONFIG_NET_VENDOR_AMD is not set +# CONFIG_NET_VENDOR_ARC is not set +# CONFIG_NET_VENDOR_AURORA is not set +CONFIG_CAVIUM_PTP=y +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_EZCHIP is not set +CONFIG_E100=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_IGB=y +CONFIG_IGBVF=m +CONFIG_IXGB=m +CONFIG_IXGBE=m +CONFIG_IXGBEVF=m +CONFIG_I40E=y +CONFIG_I40EVF=y +# CONFIG_NET_VENDOR_MARVELL is not set +CONFIG_MLX4_EN=y +CONFIG_MLX5_CORE=m +CONFIG_MLX5_FPGA=y +CONFIG_MLX5_CORE_EN=y +CONFIG_MLXSW_CORE=y +CONFIG_MLXSW_PCI=y +CONFIG_MLXSW_I2C=y +CONFIG_MLXSW_MINIMAL=y +# CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_QUALCOMM is not set +# CONFIG_NET_VENDOR_RENESAS is not set +# CONFIG_NET_VENDOR_ROCKER is not set +# CONFIG_NET_VENDOR_SAMSUNG is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SMSC is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set +# CONFIG_NET_VENDOR_VIA is not set +# CONFIG_NET_VENDOR_WIZNET is not set +# CONFIG_WLAN is not set CONFIG_INPUT_FF_MEMLESS=y CONFIG_INPUT_POLLDEV=y CONFIG_INPUT_MOUSEDEV=y @@ -475,15 +542,24 @@ CONFIG_INPUT_EVDEV=y CONFIG_SERIO_LIBPS2=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y +# CONFIG_SERIAL_8250_PCI is not set CONFIG_SERIAL_8250_SUNWAY=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_VIRTIO_CONSOLE=y # CONFIG_HW_RANDOM is not set +# CONFIG_DEVPORT is not set # CONFIG_I2C_COMPAT is not set CONFIG_I2C_CHARDEV=y CONFIG_I2C_MUX=y CONFIG_SPI=y CONFIG_SPI_SPIDEV=y +CONFIG_SENSORS_PVT=y +CONFIG_SENSORS_LM75=y CONFIG_SSB=y CONFIG_DRM=y +CONFIG_DRM_RADEON=y +CONFIG_DRM_AST=y +CONFIG_DRM_VIRTIO_GPU=y CONFIG_FIRMWARE_EDID=y CONFIG_LCD_CLASS_DEVICE=y # CONFIG_VGA_CONSOLE is not set @@ -497,25 +573,40 @@ CONFIG_USB_STORAGE=y CONFIG_INFINIBAND=m CONFIG_INFINIBAND_USER_MAD=m CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_INFINIBAND_MTHCA=m +# CONFIG_INFINIBAND_MTHCA_DEBUG is not set +CONFIG_MLX4_INFINIBAND=m +CONFIG_MLX5_INFINIBAND=m +CONFIG_INFINIBAND_IPOIB=m +CONFIG_INFINIBAND_IPOIB_CM=y CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set # CONFIG_RTC_INTF_PROC is not set CONFIG_RTC_DRV_PCF8523=y CONFIG_UIO=y +CONFIG_UIO_PCI_GENERIC=m +CONFIG_VFIO=y +CONFIG_VIRTIO_PCI=y +# CONFIG_VIRTIO_PCI_LEGACY is not set +CONFIG_VIRTIO_MMIO=y CONFIG_STAGING=y +CONFIG_FB_SM750=y +CONFIG_IOMMU_DEFAULT_PASSTHROUGH=y +CONFIG_SUNWAY_IOMMU=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_EXT4_FS_SECURITY=y CONFIG_EXT4_DEBUG=y CONFIG_XFS_FS=y CONFIG_GFS2_FS=y -CONFIG_BTRFS_FS=m -CONFIG_BTRFS_FS_POSIX_ACL=y CONFIG_FANOTIFY=y CONFIG_QUOTA=y CONFIG_AUTOFS4_FS=y CONFIG_FUSE_FS=y -CONFIG_OVERLAY_FS=m +CONFIG_OVERLAY_FS=y +CONFIG_OVERLAY_FS_INDEX=y +CONFIG_OVERLAY_FS_XINO_AUTO=y +CONFIG_OVERLAY_FS_METACOPY=y CONFIG_FSCACHE=y CONFIG_ISO9660_FS=y CONFIG_JOLIET=y @@ -547,7 +638,11 @@ CONFIG_NFSD_V3_ACL=y CONFIG_NFSD_V4=y CONFIG_NFSD_SCSILAYOUT=y CONFIG_NFSD_V4_SECURITY_LABEL=y -CONFIG_NLS_CODEPAGE_437=m +CONFIG_9P_FS=y +CONFIG_9P_FSCACHE=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_9P_FS_SECURITY=y +CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_737=m CONFIG_NLS_CODEPAGE_775=m CONFIG_NLS_CODEPAGE_850=m @@ -571,7 +666,7 @@ CONFIG_NLS_ISO8859_8=m CONFIG_NLS_CODEPAGE_1250=m CONFIG_NLS_CODEPAGE_1251=m CONFIG_NLS_ASCII=m -CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_1=y CONFIG_NLS_ISO8859_2=m CONFIG_NLS_ISO8859_3=m CONFIG_NLS_ISO8859_4=m @@ -602,14 +697,16 @@ CONFIG_SECURITY_INFINIBAND=y CONFIG_SECURITY_PATH=y CONFIG_CRYPTO_AUTHENC=y CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_SEQIV=y CONFIG_CRYPTO_ECHAINIV=y CONFIG_CRYPTO_CBC=y -CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_AES=y CONFIG_CRYPTO_DES=y CONFIG_CRYPTO_DEFLATE=y CONFIG_CRYPTO_LZO=y # CONFIG_CRYPTO_HW is not set CONFIG_CONSOLE_LOGLEVEL_QUIET=7 # CONFIG_ENABLE_MUST_CHECK is not set +# CONFIG_FRAME_POINTER is not set CONFIG_SCHEDSTATS=y # CONFIG_RCU_TRACE is not set -- Gitee From a35512b9fc5aeed4525726cc6ab8205cb3caf05e Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 26 Jun 2024 14:15:01 +0800 Subject: [PATCH 230/524] sw64: bypass sunway pcie hotplug driver in non-physical scenario Sunway PCIe hotplug driver is specifically designed for physical hardware, and should not be activated in guest and emulator. Signed-off-by: Chen Wang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/hotplug/sunway_pciehp_core.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/pci/hotplug/sunway_pciehp_core.c b/drivers/pci/hotplug/sunway_pciehp_core.c index e79a074d2557..6d155d3a0595 100644 --- a/drivers/pci/hotplug/sunway_pciehp_core.c +++ b/drivers/pci/hotplug/sunway_pciehp_core.c @@ -248,6 +248,11 @@ static int __init sunway_pciehp_init(void) struct pci_dev *pdev = NULL; struct pci_controller *hose = NULL; + if (is_guest_or_emul()) { + pr_info(DRIVER_DESC " does not support for VM and emulator.\n"); + return -ENODEV; + } + pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); for (hose = hose_head; hose; hose = hose->next) { -- Gitee From 9ae1c4c32e0b6e422d0b08796a41e2d64fe572db Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Tue, 9 Jul 2024 09:11:45 +0800 Subject: [PATCH 231/524] sw64: adapt to the upstream changes in arch_remove_memory Due to the upstream changes, arch_remove_memory() no longer requires "int nid" as a paramenter. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/init.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index 05c0b8350480..ed4f213c7165 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -461,8 +461,7 @@ int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params) return ret; } -void arch_remove_memory(int nid, u64 start, u64 size, - struct vmem_altmap *altmap) +void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) { unsigned long start_pfn = start >> PAGE_SHIFT; unsigned long nr_pages = size >> PAGE_SHIFT; -- Gitee From 5556b9b618cf5192f5d1af5d308644ec6746f976 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Tue, 9 Jul 2024 09:33:19 +0800 Subject: [PATCH 232/524] sw64: adapt to the upstream changes in hotplug_slot_ops The upstream changes the parameter passing in reset_slot of struct hotplug_slot_ops from "int probe" to "bool probe". This patch adapts the above changes to sunway_pciehp_reset_slot(). Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/hotplug/sunway_pciehp.h | 2 +- drivers/pci/hotplug/sunway_pciehp_hpc.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/hotplug/sunway_pciehp.h b/drivers/pci/hotplug/sunway_pciehp.h index 5ef5e745543b..d1addc487d07 100644 --- a/drivers/pci/hotplug/sunway_pciehp.h +++ b/drivers/pci/hotplug/sunway_pciehp.h @@ -193,7 +193,7 @@ void sunway_pciehp_release_ctrl(struct controller *ctrl); int sunway_pciehp_sysfs_enable_slot(struct hotplug_slot *hotplug_slot); int sunway_pciehp_sysfs_disable_slot(struct hotplug_slot *hotplug_slot); -int sunway_pciehp_reset_slot(struct hotplug_slot *hotplug_slot, int probe); +int sunway_pciehp_reset_slot(struct hotplug_slot *hotplug_slot, bool probe); int sunway_pciehp_get_attention_status(struct hotplug_slot *hotplug_slot, u8 *status); int sunway_pciehp_set_raw_indicator_status(struct hotplug_slot *hotplug_slot, u8 status); int sunway_pciehp_get_raw_indicator_status(struct hotplug_slot *hotplug_slot, u8 *status); diff --git a/drivers/pci/hotplug/sunway_pciehp_hpc.c b/drivers/pci/hotplug/sunway_pciehp_hpc.c index bd4556dbc3d4..e6559261788d 100644 --- a/drivers/pci/hotplug/sunway_pciehp_hpc.c +++ b/drivers/pci/hotplug/sunway_pciehp_hpc.c @@ -995,7 +995,7 @@ void sunway_pcie_clear_hotplug_events(struct controller *ctrl) * momentarily, if we see that they could interfere. Also, clear any spurious * events after. */ -int sunway_pciehp_reset_slot(struct hotplug_slot *hotplug_slot, int probe) +int sunway_pciehp_reset_slot(struct hotplug_slot *hotplug_slot, bool probe) { struct controller *ctrl = to_ctrl(hotplug_slot); struct pci_dev *pdev = ctrl_dev(ctrl); -- Gitee From 411462899e6eac2e8637a2f960d1ac61fc7ed176 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Tue, 9 Jul 2024 09:54:32 +0800 Subject: [PATCH 233/524] sw64: add virt_to_bus() and bus_to_virt() This patch add virt_to_bus() and bus_to_virt() for sw64. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/io.h | 52 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/arch/sw_64/include/asm/io.h b/arch/sw_64/include/asm/io.h index 4e0822f46518..15abf97e9d57 100644 --- a/arch/sw_64/include/asm/io.h +++ b/arch/sw_64/include/asm/io.h @@ -94,32 +94,32 @@ extern void outsl(unsigned long port, const void *src, unsigned long count); #include #undef PCI_IOBASE -///* -// * Change addresses as seen by the kernel (virtual) to addresses as -// * seen by a device (bus), and vice versa. -// * -// * Note that this only works for a limited range of kernel addresses, -// * and very well may not span all memory. Consider this interface -// * deprecated in favour of the DMA-mapping API. -// */ -//static inline unsigned long __deprecated virt_to_bus(void *address) -//{ -// return virt_to_phys(address); -//} -//#define isa_virt_to_bus virt_to_bus -// -//static inline void * __deprecated bus_to_virt(unsigned long address) -//{ -// void *virt; -// -// /* This check is a sanity check but also ensures that bus address 0 -// * maps to virtual address 0 which is useful to detect null pointers -// * (the NCR driver is much simpler if NULL pointers are preserved). -// */ -// virt = phys_to_virt(address); -// return (long)address <= 0 ? NULL : virt; -//} -//#define isa_bus_to_virt bus_to_virt +/* + * Change addresses as seen by the kernel (virtual) to addresses as + * seen by a device (bus), and vice versa. + * + * Note that this only works for a limited range of kernel addresses, + * and very well may not span all memory. Consider this interface + * deprecated in favour of the DMA-mapping API. + */ +static inline unsigned long __deprecated virt_to_bus(void *address) +{ + return virt_to_phys(address); +} +#define isa_virt_to_bus virt_to_bus + +static inline void * __deprecated bus_to_virt(unsigned long address) +{ + void *virt; + + /* This check is a sanity check but also ensures that bus address 0 + * maps to virtual address 0 which is useful to detect null pointers + * (the NCR driver is much simpler if NULL pointers are preserved). + */ + virt = phys_to_virt(address); + return (long)address <= 0 ? NULL : virt; +} +#define isa_bus_to_virt bus_to_virt #endif /* __KERNEL__ */ -- Gitee From ccc6bcb384f4850c5e757b2267282195180e7e58 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 9 Jul 2024 17:18:45 +0000 Subject: [PATCH 234/524] sw64: fix rc resource setup The increment of 'enable_cnt' for rc is removed. This ensures that the pci correctly sets up the resources for rc. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/pci/pci.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index fbbd54dd181c..de56ccfda574 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -209,7 +209,6 @@ static void fixup_root_complex(struct pci_dev *dev) dev->class &= 0xff; dev->class |= PCI_CLASS_BRIDGE_PCI << 8; } - atomic_inc(&dev->enable_cnt); dev->no_msi = 1; } -- Gitee From 6beec2dfb0827fdf3f93328337db6998989fa5c6 Mon Sep 17 00:00:00 2001 From: Hang Yiyi Date: Wed, 20 Dec 2023 15:50:35 +0800 Subject: [PATCH 235/524] sw64: kvm: add trace kvm function Add trace_kvm related functions to sw64. These functions can facilitate performance analysis and information tracking of kvm. Signed-off-by: Hang Yiyi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmio.c | 13 ++- arch/sw_64/kvm/mmu.c | 10 +- arch/sw_64/kvm/sw64.c | 3 + arch/sw_64/kvm/trace.h | 205 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 226 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/kvm/mmio.c b/arch/sw_64/kvm/mmio.c index 0bde3406eb89..3214a4efb0b1 100644 --- a/arch/sw_64/kvm/mmio.c +++ b/arch/sw_64/kvm/mmio.c @@ -8,6 +8,9 @@ #include #include #include +#include + +#include "trace.h" static unsigned long mmio_read_buf(char *buf, unsigned int len) { @@ -50,6 +53,7 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run) return -EINVAL; data = mmio_read_buf(run->mmio.data, len); + trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr, &data); vcpu_set_reg(vcpu, vcpu->arch.mmio_decode.rt, data); } @@ -71,12 +75,17 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, run->mmio.phys_addr = sw64_read_csr(CSR_DVA) & 0xfffffffffffffUL; sw64_decode(vcpu, 0, run); #endif - if (run->mmio.is_write) + if (run->mmio.is_write) { + trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, run->mmio.len, + run->mmio.phys_addr, run->mmio.data); ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, run->mmio.len, run->mmio.data); - else + } else { + trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, run->mmio.len, + run->mmio.phys_addr, NULL); ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, run->mmio.len, run->mmio.data); + } if (!ret) { /* We handled the access successfully in the kernel. */ diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 054856c5a499..095bcff09fb6 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -27,7 +27,7 @@ #include #include #include - +#include #include #include @@ -36,6 +36,7 @@ #include #include +#include "trace.h" #define KVM_APT_FLAG_LOGGING_ACTIVE (1UL << 1) static bool memslot_is_logging(struct kvm_memory_slot *memslot) @@ -1367,6 +1368,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) write_fault = kvm_is_write_fault(access_type); + trace_kvm_guest_fault(vcpu->arch.regs.pc, as_info, fault_entry_addr, fault_gpa); /* The memory slot for IO doesn't register in memory region * with kvm, if hva == KVM_HVA_ERR_BAD, the gpa used for MMIO * needs emulation. @@ -1427,6 +1429,7 @@ int kvm_unmap_hva_range(struct kvm *kvm, if (!kvm->arch.pgd) return 0; + trace_kvm_unmap_hva_range(start, end); handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL); return 1; } @@ -1490,7 +1493,7 @@ int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) { if (!kvm->arch.pgd) return 0; - + trace_kvm_age_hva(start, end); return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL); } @@ -1498,6 +1501,7 @@ int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) { if (!kvm->arch.pgd) return 0; + trace_kvm_test_age_hva(hva); return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL); } @@ -1519,6 +1523,8 @@ int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) if (!kvm->arch.pgd) return 0; + trace_kvm_set_spte_hva(hva); + apt_pte = pte_wrprotect(pte); handle_hva_to_gpa(kvm, hva, end, &kvm_set_apte_handler, &apt_pte); return 0; diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index f6bfb2452938..518aa35363c7 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -318,6 +318,7 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) { + trace_kvm_set_guest_debug(vcpu, dbg->control); return 0; } @@ -549,6 +550,8 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, bool level = irq_level->level; irq_num = irq; + trace_kvm_irq_line(0, irq_num, irq_level->level); + /* target core for Intx is core0 */ vcpu = kvm_get_vcpu(kvm, 0); if (!vcpu) diff --git a/arch/sw_64/kvm/trace.h b/arch/sw_64/kvm/trace.h index 2611df3d3fa5..2b7b1de1e5d5 100644 --- a/arch/sw_64/kvm/trace.h +++ b/arch/sw_64/kvm/trace.h @@ -51,7 +51,210 @@ TRACE_EVENT(kvm_sw64_exit, __entry->vcpu_pc) ); -#endif /* _SW64_KVM_TRACE_H */ +TRACE_EVENT(kvm_guest_fault, + TP_PROTO(unsigned long vcpu_pc, unsigned long as_info, + unsigned long fault_entry_addr, + phys_addr_t fault_gpa), + TP_ARGS(vcpu_pc, as_info, fault_entry_addr, fault_gpa), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + __field(unsigned long, as_info) + __field(unsigned long, fault_entry_addr) + __field(unsigned long long, fault_gpa) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->as_info = as_info; + __entry->fault_entry_addr = fault_entry_addr; + __entry->fault_gpa = fault_gpa; + ), + + TP_printk("fault_gpa %#llx, as_info %#08lx, fault_entry_adr %#08lx, pc 0x%08lx", + __entry->fault_gpa, __entry->as_info, + __entry->fault_entry_addr, __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_access_fault, + TP_PROTO(unsigned long ipa), + TP_ARGS(ipa), + + TP_STRUCT__entry( + __field(unsigned long, ipa) + ), + + TP_fast_assign( + __entry->ipa = ipa; + ), + + TP_printk("IPA: %lx", __entry->ipa) +); + +TRACE_EVENT(kvm_irq_line, + TP_PROTO(int vcpu_idx, int irq_num, int level), + TP_ARGS(vcpu_idx, irq_num, level), + + TP_STRUCT__entry( + __field(int, vcpu_idx) + __field(int, irq_num) + __field(int, level) + ), + + TP_fast_assign( + __entry->vcpu_idx = vcpu_idx; + __entry->irq_num = irq_num; + __entry->level = level; + ), + + TP_printk("Inject interrupt, vcpu->idx: %d, num: %d, level: %d", + __entry->vcpu_idx, __entry->irq_num, __entry->level) +); + +TRACE_EVENT(kvm_mmio_emulate, + TP_PROTO(unsigned long vcpu_pc, unsigned long instr, + unsigned long cpsr), + TP_ARGS(vcpu_pc, instr, cpsr), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + __field(unsigned long, instr) + __field(unsigned long, cpsr) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->instr = instr; + __entry->cpsr = cpsr; + ), + + TP_printk("Emulate MMIO at: 0x%016lx (instr: %08lx, cpsr: %08lx)", + __entry->vcpu_pc, __entry->instr, __entry->cpsr) +); + +TRACE_EVENT(kvm_unmap_hva_range, + TP_PROTO(unsigned long start, unsigned long end), + TP_ARGS(start, end), + + TP_STRUCT__entry( + __field(unsigned long, start) + __field(unsigned long, end) + ), + + TP_fast_assign( + __entry->start = start; + __entry->end = end; + ), + + TP_printk("mmu notifier unmap range: %#016lx -- %#016lx", + __entry->start, __entry->end) +); + +TRACE_EVENT(kvm_set_spte_hva, + TP_PROTO(unsigned long hva), + TP_ARGS(hva), + + TP_STRUCT__entry( + __field(unsigned long, hva) + ), + + TP_fast_assign( + __entry->hva = hva; + ), + + TP_printk("mmu notifier set pte hva: %#016lx", __entry->hva) +); + +TRACE_EVENT(kvm_age_hva, + TP_PROTO(unsigned long start, unsigned long end), + TP_ARGS(start, end), + + TP_STRUCT__entry( + __field(unsigned long, start) + __field(unsigned long, end) + ), + + TP_fast_assign( + __entry->start = start; + __entry->end = end; + ), + + TP_printk("mmu notifier age hva: %#016lx -- %#016lx", + __entry->start, __entry->end) +); + +TRACE_EVENT(kvm_test_age_hva, + TP_PROTO(unsigned long hva), + TP_ARGS(hva), + + TP_STRUCT__entry( + __field(unsigned long, hva) + ), + + TP_fast_assign( + __entry->hva = hva; + ), + + TP_printk("mmu notifier test age hva: %#016lx", __entry->hva) +); + +TRACE_EVENT(kvm_set_way_flush, + TP_PROTO(unsigned long vcpu_pc, bool cache), + TP_ARGS(vcpu_pc, cache), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + __field(bool, cache) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->cache = cache; + ), + + TP_printk("S/W flush at 0x%016lx (cache %s)", + __entry->vcpu_pc, __entry->cache ? "on" : "off") +); + +TRACE_EVENT(kvm_toggle_cache, + TP_PROTO(unsigned long vcpu_pc, bool was, bool now), + TP_ARGS(vcpu_pc, was, now), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + __field(bool, was) + __field(bool, now) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->was = was; + __entry->now = now; + ), + + TP_printk("VM op at 0x%016lx (cache was %s, now %s)", + __entry->vcpu_pc, __entry->was ? "on" : "off", + __entry->now ? "on" : "off") +); + +TRACE_EVENT(kvm_set_guest_debug, + TP_PROTO(struct kvm_vcpu *vcpu, __u32 guest_debug), + TP_ARGS(vcpu, guest_debug), + + TP_STRUCT__entry( + __field(struct kvm_vcpu *, vcpu) + __field(__u32, guest_debug) + ), + + TP_fast_assign( + __entry->vcpu = vcpu; + __entry->guest_debug = guest_debug; + ), + + TP_printk("vcpu: %p, flags: 0x%08x", __entry->vcpu, __entry->guest_debug) +); + +#endif /* _TRACE_ARM_SW_64_KVM_H */ #undef TRACE_INCLUDE_PATH #define TRACE_INCLUDE_PATH . -- Gitee From 89fe409500ef3b8c88e0882e1631525d24b06b75 Mon Sep 17 00:00:00 2001 From: Du Yilong Date: Mon, 28 Aug 2023 16:30:47 +0800 Subject: [PATCH 236/524] sw64: kvm: fix return value of vmem_mmap The ret is uninitialized and may lead to unexpected result. Signed-off-by: Du Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/vmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kvm/vmem.c b/arch/sw_64/kvm/vmem.c index 688449b65fa5..945667f25955 100644 --- a/arch/sw_64/kvm/vmem.c +++ b/arch/sw_64/kvm/vmem.c @@ -144,7 +144,7 @@ static int vmem_mmap(struct file *flip, struct vm_area_struct *vma) /*to do if size bigger than vm_mem_size*/ pr_info("sw64_vmem: vm_start=%#lx, size= %#lx\n", vma->vm_start, size); - vmem_vm_insert_page(vma); + ret = vmem_vm_insert_page(vma); if (ret < 0) return ret; -- Gitee From e48d9b647dedad612a91d06cf350b25580a39c72 Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Sat, 7 Oct 2023 15:04:51 +0800 Subject: [PATCH 237/524] sw64: kvm: scale up the range of target vcpu parsed by hypervisor The range of target vcpu parsed by hypervisor was 0-31 when hypervisor injected msi into guest. It's not always, but it's very likely to cause guest runs slowly when the number of vcpus is more than 32. This patch scales up the range of target vcpu parsed by hypervisor to 0-255. Signed-off-by: Zhang Yi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/sw64.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index 518aa35363c7..eb6b891ab47f 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -52,19 +52,17 @@ int kvm_arch_check_processor_compat(void *opaque) int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, bool line_status) { - unsigned int vcid; - unsigned int vcpu_idx; + unsigned int dest_id; struct kvm_vcpu *vcpu = NULL; - int irq = e->msi.data & 0xff; + int vector = e->msi.data & 0xff; - vcid = (e->msi.address_lo & VT_MSIX_ADDR_DEST_ID_MASK) >> VT_MSIX_ADDR_DEST_ID_SHIFT; - vcpu_idx = vcid & 0x1f; - vcpu = kvm_get_vcpu(kvm, vcpu_idx); + dest_id = (e->msi.address_lo & VT_MSIX_ADDR_DEST_ID_MASK) >> VT_MSIX_ADDR_DEST_ID_SHIFT; + vcpu = kvm_get_vcpu(kvm, dest_id); if (!vcpu) return -EINVAL; - return vcpu_interrupt_line(vcpu, irq, true); + return vcpu_interrupt_line(vcpu, vector, true); } void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) -- Gitee From 72267658c15a284cfe637ca92fffc231a15ceb4d Mon Sep 17 00:00:00 2001 From: Deng Xiaoyun Date: Tue, 21 Feb 2023 11:12:46 +0800 Subject: [PATCH 238/524] sw64: add support for vmtop vmtop is a user-mode tool that runs on the host. It dynamically displays VM resource usage, including CPU and memory usage, and vCPU KVM exit events. To implement these functions, the kernel need to provide interfaces. Therefore, the kernel need to make the following changes. Add the contents of the kvm_vcpu_stat structure, which vmtop monitors. Add debugfs_entries structure and dfx_sw64_debugfs_entries structure, which generate corresponding files in the /sys/kernel/debug/kvm path for vmtop to read when VM is created. Add dfx_sw64_stat_fops, which is used for reading and writing vcpu_stat file in the /sys/kernel/debug/kvm path. Signed-off-by: Deng Xiaoyun Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/sw64.c | 76 +++++++++++++++++++++++++++++++++++++++- include/linux/kvm_host.h | 13 +++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index eb6b891ab47f..630ea907592c 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -22,7 +22,12 @@ bool set_msi_flag; - +#define VCPU_STAT(n, x, ...) \ + { n, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ } +#define VM_STAT(n, x, ...) \ + { n, offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ } +#define DFX_STAT(n, x, ...) \ + { n, offsetof(struct kvm_vcpu_stat, x), DFX_SW64_STAT_U64, ## __VA_ARGS__ } static unsigned long get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) { @@ -115,6 +120,52 @@ void check_vcpu_requests(struct kvm_vcpu *vcpu) } } +struct kvm_stats_debugfs_item debugfs_entries[] = { + VCPU_STAT("exits", exits), + VCPU_STAT("io_exits", io_exits), + VCPU_STAT("mmio_exits", mmio_exits), + VCPU_STAT("migration_set_dirty", migration_set_dirty), + VCPU_STAT("shutdown_exits", shutdown_exits), + VCPU_STAT("restart_exits", restart_exits), + VCPU_STAT("ipi_exits", ipi_exits), + VCPU_STAT("timer_exits", timer_exits), + VCPU_STAT("debug_exits", debug_exits), + VCPU_STAT("fatal_error_exits", fatal_error_exits), + VCPU_STAT("halt_exits", halt_exits), + VCPU_STAT("halt_successful_poll", halt_successful_poll), + VCPU_STAT("halt_attempted_poll", halt_attempted_poll), + VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("halt_poll_invalid", halt_poll_invalid), + VCPU_STAT("signal_exits", signal_exits), + { "vcpu_stat", 0, KVM_STAT_DFX_SW64 }, + { NULL } +}; + +struct dfx_sw64_kvm_stats_debugfs_item dfx_sw64_debugfs_entries[] = { + DFX_STAT("pid", pid), + DFX_STAT("exits", exits), + DFX_STAT("io_exits", io_exits), + DFX_STAT("mmio_exits", mmio_exits), + DFX_STAT("migration_set_dirty", migration_set_dirty), + DFX_STAT("shutdown_exits", shutdown_exits), + DFX_STAT("restart_exits", restart_exits), + DFX_STAT("ipi_exits", ipi_exits), + DFX_STAT("timer_exits", timer_exits), + DFX_STAT("debug_exits", debug_exits), + DFX_STAT("fatal_error_exits", fatal_error_exits), + DFX_STAT("halt_exits", halt_exits), + DFX_STAT("halt_successful_poll", halt_successful_poll), + DFX_STAT("halt_attempted_poll", halt_attempted_poll), + DFX_STAT("halt_wakeup", halt_wakeup), + DFX_STAT("halt_poll_invalid", halt_poll_invalid), + DFX_STAT("signal_exits", signal_exits), + DFX_STAT("steal", steal), + DFX_STAT("st_max", st_max), + DFX_STAT("utime", utime), + DFX_STAT("stime", stime), + DFX_STAT("gtime", gtime), + { NULL } +}; int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) { @@ -273,10 +324,24 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) return 0; } +void kvm_arch_vcpu_stat_reset(struct kvm_vcpu_stat *vcpu_stat) +{ + vcpu_stat->st_max = 0; +} + +static void update_steal_time(struct kvm_vcpu *vcpu) +{ + u64 delta; + + delta = current->sched_info.run_delay - vcpu->stat.steal; + vcpu->stat.steal = current->sched_info.run_delay; + vcpu->stat.st_max = max(vcpu->stat.st_max, delta); +} void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { vcpu->cpu = cpu; + update_steal_time(vcpu); } void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) @@ -320,6 +385,12 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, return 0; } +void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat) +{ + vcpu_stat->utime = current->utime; + vcpu_stat->stime = current->stime; + vcpu_stat->gtime = current->gtime; +} /* * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on @@ -358,6 +429,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) if (signal_pending(current)) { ret = -EINTR; run->exit_reason = KVM_EXIT_INTR; + vcpu->stat.signal_exits++; } if (ret <= 0) { @@ -398,6 +470,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) /* Back from guest */ vcpu->mode = OUTSIDE_GUEST_MODE; + vcpu->stat.exits++; local_irq_enable(); guest_exit_irqoff(); @@ -407,6 +480,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) /* ret = 0 indicate interrupt in guest mode, ret > 0 indicate hcall */ ret = handle_exit(vcpu, run, ret, &hargs); + update_vcpu_stat_time(&vcpu->stat); } if (vcpu->sigset_active) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 4388cfc5bb74..de2b643ac217 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1787,6 +1787,13 @@ struct kvm_stat_data { enum kvm_stat_kind kind; }; +struct kvm_stats_debugfs_item { + const char *name; + int offset; + enum kvm_stat_kind kind; + int mode; +}; + struct _kvm_stats_desc { struct kvm_stats_desc desc; char name[KVM_STATS_NAME_SIZE]; @@ -1908,6 +1915,11 @@ struct _kvm_stats_desc { HALT_POLL_HIST_COUNT), \ STATS_DESC_IBOOLEAN(VCPU_GENERIC, blocking) +#define VM_STAT(n, x, ...) \ + { n, offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ } +#define VCPU_STAT(n, x, ...) \ + { n, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ } + #ifdef CONFIG_SW64 enum dfx_sw64_stat_kind { DFX_SW64_STAT_U64, @@ -1923,6 +1935,7 @@ struct dfx_sw64_kvm_stats_debugfs_item { }; extern struct dfx_sw64_kvm_stats_debugfs_item dfx_sw64_debugfs_entries[]; #endif +extern struct kvm_stats_debugfs_item debugfs_entries[]; extern struct dentry *kvm_debugfs_dir; ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, -- Gitee From ae231ce59ef24eb334a3ad9a6033324bdfcf975a Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Thu, 4 Jan 2024 09:59:06 +0800 Subject: [PATCH 239/524] sw64: add missing c4 legacy power management code The c4 legacy power management is missing, which is causing errors when guest is excuting poweroff command. Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/platform/sw64/Makefile | 1 + drivers/platform/sw64/legacy_junzhang.c | 62 +++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 drivers/platform/sw64/legacy_junzhang.c diff --git a/drivers/platform/sw64/Makefile b/drivers/platform/sw64/Makefile index 28922224fb17..3efa835ade5d 100644 --- a/drivers/platform/sw64/Makefile +++ b/drivers/platform/sw64/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_PLATFORM_XUELANG) += legacy_xuelang.o +obj-$(CONFIG_PLATFORM_JUNZHANG) += legacy_junzhang.o diff --git a/drivers/platform/sw64/legacy_junzhang.c b/drivers/platform/sw64/legacy_junzhang.c new file mode 100644 index 000000000000..17d6d151d0b8 --- /dev/null +++ b/drivers/platform/sw64/legacy_junzhang.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +static void vt_mode_kill_arch(int mode) +{ + hcall(HCALL_SET_CLOCKEVENT, 0, 0, 0); + + switch (mode) { + case LINUX_REBOOT_CMD_RESTART: + hcall(HCALL_RESTART, 0, 0, 0); + mb(); + break; + case LINUX_REBOOT_CMD_HALT: + case LINUX_REBOOT_CMD_POWER_OFF: + hcall(HCALL_SHUTDOWN, 0, 0, 0); + mb(); + break; + default: + break; + } +} + +void sw64_halt(void) +{ + if (is_in_host()) + cpld_write(0x64, 0x00, 0xf0); + else + vt_mode_kill_arch(LINUX_REBOOT_CMD_HALT); +} + +void sw64_poweroff(void) +{ + if (is_in_host()) + cpld_write(0x64, 0x00, 0xf0); + else + vt_mode_kill_arch(LINUX_REBOOT_CMD_POWER_OFF); +} + +void sw64_restart(void) +{ + if (is_in_host()) { + fix_jm585_reset(); + cpld_write(0x64, 0x00, 0xc3); + } else + vt_mode_kill_arch(LINUX_REBOOT_CMD_RESTART); +} + +static int sw64_reset_init(void) +{ +#ifdef CONFIG_EFI + if (BIOS_SUPPORT_RESET_CLALLBACK((void *)bios_version)) + return 0; +#endif + pm_restart = sw64_restart; + pm_power_off = sw64_poweroff; + pm_halt = sw64_halt; + return 0; +} +subsys_initcall(sw64_reset_init); -- Gitee From 5e3b3336ba87741ea06e144d8e10a98280a78337 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 2 Jan 2024 14:26:36 +0800 Subject: [PATCH 240/524] sw64: kvm: delete memory hotplug codes for CORE3B Guest memory hotplug of CORE3B is not support, so delete it to make the code clearer. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hcall.h | 1 - arch/sw_64/include/asm/kvm_asm.h | 2 - arch/sw_64/include/asm/kvm_host.h | 11 ---- arch/sw_64/include/asm/memory.h | 1 - arch/sw_64/kvm/Kconfig | 7 --- arch/sw_64/kvm/handle_exit.c | 6 --- arch/sw_64/kvm/kvm_core3.c | 85 ------------------------------- arch/sw_64/mm/init.c | 5 +- drivers/misc/sunway-ged.c | 2 - 9 files changed, 1 insertion(+), 119 deletions(-) diff --git a/arch/sw_64/include/asm/hcall.h b/arch/sw_64/include/asm/hcall.h index bded05779db7..40eab985fcbd 100644 --- a/arch/sw_64/include/asm/hcall.h +++ b/arch/sw_64/include/asm/hcall.h @@ -18,7 +18,6 @@ enum HCALL_TYPE { HCALL_SWNET = 20, /* guest request swnet service */ HCALL_SWNET_IRQ = 21, /* guest request swnet intr */ HCALL_FATAL_ERROR = 22, /* guest fatal error, issued by hmcode */ - HCALL_MEMHOTPLUG = 23, /* guest memory hotplug event */ NR_HCALL }; diff --git a/arch/sw_64/include/asm/kvm_asm.h b/arch/sw_64/include/asm/kvm_asm.h index fd1b25018fc8..a8e8ef3d68a0 100644 --- a/arch/sw_64/include/asm/kvm_asm.h +++ b/arch/sw_64/include/asm/kvm_asm.h @@ -14,7 +14,6 @@ #define SW64_KVM_EXIT_RESTART 17 #define SW64_KVM_EXIT_APT_FAULT 18 #define SW64_KVM_EXIT_FATAL_ERROR 22 -#define SW64_KVM_EXIT_MEMHOTPLUG 23 #define SW64_KVM_EXIT_DEBUG 24 @@ -29,7 +28,6 @@ {17, "RESTART" }, \ {18, "APT_FAULT" }, \ {22, "FATAL_ERROR" }, \ - {23, "MEMHOTPLUG" }, \ {24, "DEBUG" } diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index 09a995218a2c..b398d72eb840 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -65,17 +65,12 @@ #define KVM_PHYS_MASK (KVM_PHYS_SIZE - _AC(1, ULL)) struct kvm_arch_memory_slot { - unsigned long host_phys_addr; - bool valid; }; struct kvm_arch { unsigned long host_phys_addr; unsigned long size; - /* segment table */ - unsigned long *seg_pgd; - struct swvm_mem mem; /* Addtional stage page table*/ pgd_t *pgd; @@ -157,9 +152,6 @@ struct kvm_vcpu_stat { u64 ipi_exits; u64 timer_exits; u64 debug_exits; -#ifdef CONFIG_KVM_MEMHOTPLUG - u64 memhotplug_exits; -#endif u64 fatal_error_exits; u64 halt_exits; u64 halt_successful_poll; @@ -176,9 +168,6 @@ struct kvm_vcpu_stat { u64 gtime; }; -#ifdef CONFIG_KVM_MEMHOTPLUG -void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr); -#endif #ifdef CONFIG_SUBARCH_C4 #define KVM_ARCH_WANT_MMU_NOTIFIER #endif diff --git a/arch/sw_64/include/asm/memory.h b/arch/sw_64/include/asm/memory.h index 65e5f8f79727..f0a20cbb096c 100644 --- a/arch/sw_64/include/asm/memory.h +++ b/arch/sw_64/include/asm/memory.h @@ -6,7 +6,6 @@ #include #endif -#define MIN_MEMORY_BLOCK_SIZE_VM_MEMHP (1UL << 30) #define NODE0_START (_TEXT_START - __START_KERNEL_map) #define MAX_PHYSMEM_BITS 48 diff --git a/arch/sw_64/kvm/Kconfig b/arch/sw_64/kvm/Kconfig index b7e43d0bae51..fdff308aed22 100644 --- a/arch/sw_64/kvm/Kconfig +++ b/arch/sw_64/kvm/Kconfig @@ -37,13 +37,6 @@ config KVM If unsure, say N. -config KVM_MEMHOTPLUG - bool "Memory hotplug support for guest" - depends on KVM && MEMORY_HOTPLUG && SUBARCH_C3B - help - Provides memory hotplug support for SW64 guest. - - source "drivers/vhost/Kconfig" endif # VIRTUALIZATION diff --git a/arch/sw_64/kvm/handle_exit.c b/arch/sw_64/kvm/handle_exit.c index 69b97860db88..e0c62dc1112e 100644 --- a/arch/sw_64/kvm/handle_exit.c +++ b/arch/sw_64/kvm/handle_exit.c @@ -63,12 +63,6 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, vcpu->run->exit_reason = KVM_EXIT_DEBUG; vcpu->run->debug.arch.epc = vcpu->arch.regs.pc; return 0; -#ifdef CONFIG_KVM_MEMHOTPLUG - case SW64_KVM_EXIT_MEMHOTPLUG: - vcpu->stat.memhotplug_exits++; - vcpu_mem_hotplug(vcpu, hargs->arg0); - return 1; -#endif #ifdef CONFIG_SUBARCH_C4 case SW64_KVM_EXIT_APT_FAULT: return kvm_handle_guest_abort(vcpu, run); diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c index f7e9150d40e0..b9a5019ee9c0 100644 --- a/arch/sw_64/kvm/kvm_core3.c +++ b/arch/sw_64/kvm/kvm_core3.c @@ -52,15 +52,6 @@ static void bind_vcpu_exit(void) { } static unsigned long longtime_offset; -#ifdef CONFIG_KVM_MEMHOTPLUG -static unsigned long get_vpcr(struct kvm_vcpu *vcpu, u64 vpn) -{ - unsigned long base; - - base = virt_to_phys(vcpu->kvm->arch.seg_pgd); - return base | ((vpn & VPN_MASK) << 44); -} -#else static unsigned long get_vpcr(struct kvm_vcpu *vcpu, u64 vpn) { unsigned long base, size; @@ -69,13 +60,11 @@ static unsigned long get_vpcr(struct kvm_vcpu *vcpu, u64 vpn) size = vcpu->kvm->arch.size; return (base >> 23) | ((size >> 23) << 16) | ((vpn & VPN_MASK) << 44); } -#endif void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu) { if (vcpu->arch.vcb.vpcr == 0) { vcpu->arch.vcb.vpcr = get_vpcr(vcpu, 0); -#ifndef CONFIG_KVM_MEMHOTPLUG if (unlikely(bind_vcpu_enabled)) { int nid; unsigned long end; @@ -85,7 +74,6 @@ void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu) if (pfn_to_nid(PHYS_PFN(end)) == nid) set_cpus_allowed_ptr(vcpu->arch.tsk, cpumask_of_node(nid)); } -#endif vcpu->arch.vcb.upcr = 0x7; } } @@ -103,57 +91,14 @@ void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) int kvm_sw64_init_vm(struct kvm *kvm) { -#ifdef CONFIG_KVM_MEMHOTPLUG - unsigned long *seg_pgd; - - if (kvm->arch.seg_pgd != NULL) { - kvm_err("kvm_arch already initialized?\n"); - return -EINVAL; - } - - seg_pgd = alloc_pages_exact(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO); - if (!seg_pgd) - return -ENOMEM; - - kvm->arch.seg_pgd = seg_pgd; - #endif return 0; } void kvm_sw64_destroy_vm(struct kvm *kvm) { - #ifdef CONFIG_KVM_MEMHOTPLUG - void *seg_pgd = NULL; - - if (kvm->arch.seg_pgd) { - seg_pgd = READ_ONCE(kvm->arch.seg_pgd); - kvm->arch.seg_pgd = NULL; - } - - if (seg_pgd) - free_pages_exact(seg_pgd, PAGE_SIZE); - #endif kvm_destroy_vcpus(kvm); } -#ifdef CONFIG_KVM_MEMHOTPLUG -static void setup_segment_table(struct kvm *kvm, - struct kvm_memory_slot *memslot, unsigned long addr, size_t size) -{ - unsigned long *seg_pgd = kvm->arch.seg_pgd; - unsigned long num_of_entry; - unsigned long base_hpa = addr; - unsigned long i; - - num_of_entry = round_up(size, 1 << 30) >> 30; - - for (i = 0; i < num_of_entry; i++) { - *seg_pgd = base_hpa + (i << 30); - seg_pgd++; - } -} -#endif - int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_memory_slot *old, struct kvm_memory_slot *new, @@ -182,12 +127,6 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, if (test_bit(IO_MARK_BIT + 1, (unsigned long *)(&(mem->guest_phys_addr)))) return 0; -#ifndef CONFIG_KVM_MEMHOTPLUG - if (mem->guest_phys_addr) { - pr_info("%s, No KVM MEMHOTPLUG support!\n", __func__); - return 0; - } -#endif if (!sw64_kvm_pool) return -ENOMEM; @@ -337,30 +276,6 @@ long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) return 0; } -#ifdef CONFIG_KVM_MEMHOTPLUG -void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr) -{ - struct kvm *kvm = vcpu->kvm; - struct kvm_memory_slot *slot; - unsigned long start_pfn = start_addr >> PAGE_SHIFT; - - kvm_for_each_memslot(slot, kvm_memslots(kvm)) { - if (start_pfn == slot->base_gfn) { - unsigned long *seg_pgd; - unsigned long num_of_entry = slot->npages >> 17; - unsigned long base_hpa = slot->arch.host_phys_addr; - unsigned long i; - - seg_pgd = kvm->arch.seg_pgd + (start_pfn >> 17); - for (i = 0; i < num_of_entry; i++) { - *seg_pgd = base_hpa + (i << 30); - seg_pgd++; - } - } - } -} -#endif - void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) { } diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index ed4f213c7165..0ac65a2fd61c 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -46,10 +46,7 @@ static phys_addr_t mem_size_limit; #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE unsigned long memory_block_size_bytes(void) { - if (is_in_guest()) - return MIN_MEMORY_BLOCK_SIZE_VM_MEMHP; - else - return MIN_MEMORY_BLOCK_SIZE; + return MIN_MEMORY_BLOCK_SIZE; } #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ diff --git a/drivers/misc/sunway-ged.c b/drivers/misc/sunway-ged.c index 5c397cb454c9..aa500421355d 100644 --- a/drivers/misc/sunway-ged.c +++ b/drivers/misc/sunway-ged.c @@ -161,8 +161,6 @@ static int sunway_memory_device_add(struct sunway_ged_device *device) list_add_tail(&mem_device->list, &device->dev_list); dev_dbg(device->dev, "Memory device configured\n"); - hcall(HCALL_MEMHOTPLUG, mem_device->start_addr, 0, 0); - return 1; } -- Gitee From c14aabb0965fce5c39f63cdfdec91484b96858e3 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 2 Jan 2024 10:51:13 +0800 Subject: [PATCH 241/524] sw64: kvm: fix clock sync of CORE4 live migration Move "migration_mark" and "shtclock" from struct kvm_vcpu_arch to struct vcpucb, so that they can be synced between the src and dst guest. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_host.h | 4 ---- arch/sw_64/kvm/kvm_core4.c | 13 +++++++------ arch/sw_64/kvm/mmu.c | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index b398d72eb840..b726b0f5d94e 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -123,10 +123,6 @@ struct kvm_vcpu_arch { /* Cache some mmu pages needed inside spinlock regions */ struct kvm_mmu_memory_cache mmu_page_cache; - - /* guest live migration */ - unsigned long migration_mark; - unsigned long shtclock; }; struct vmem_info { diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c index 3c52b0d7776c..81a999b3fd30 100644 --- a/arch/sw_64/kvm/kvm_core4.c +++ b/arch/sw_64/kvm/kvm_core4.c @@ -68,9 +68,9 @@ long kvm_sw64_get_vcb(struct file *filp, unsigned long arg) { struct kvm_vcpu *vcpu = filp->private_data; - if (vcpu->arch.migration_mark) - vcpu->arch.shtclock = sw64_read_csr(CSR_SHTCLOCK) - + vcpu->arch.vcb.shtclock_offset; + if (vcpu->arch.vcb.migration_mark) + vcpu->arch.vcb.shtclock = sw64_read_csr(CSR_SHTCLOCK) + + vcpu->arch.vcb.shtclock_offset; if (copy_to_user((void __user *)arg, &(vcpu->arch.vcb), sizeof(struct vcpucb))) return -EINVAL; @@ -85,13 +85,14 @@ long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) kvm_vcb = memdup_user((void __user *)arg, sizeof(*kvm_vcb)); memcpy(&(vcpu->arch.vcb), kvm_vcb, sizeof(struct vcpucb)); - if (vcpu->arch.migration_mark) { + if (vcpu->arch.vcb.migration_mark) { /* synchronize the longtime of source and destination */ if (vcpu->arch.vcb.soft_cid == 0) - shtclock_offset = vcpu->arch.shtclock - sw64_read_csr(CSR_SHTCLOCK); + shtclock_offset = vcpu->arch.vcb.shtclock - + sw64_read_csr(CSR_SHTCLOCK); vcpu->arch.vcb.shtclock_offset = shtclock_offset; set_timer(vcpu, 200000000); - vcpu->arch.migration_mark = 0; + vcpu->arch.vcb.migration_mark = 0; } return 0; } diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 095bcff09fb6..7523ac3a6c71 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -544,7 +544,7 @@ void kvm_mark_migration(struct kvm *kvm, int mark) unsigned long cpu; kvm_for_each_vcpu(cpu, vcpu, kvm) - vcpu->arch.migration_mark = mark; + vcpu->arch.vcb.migration_mark = mark; } void kvm_arch_commit_memory_region(struct kvm *kvm, -- Gitee From 31628256ee6fbf23103bab7e5760f49be854367c Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Mon, 11 Mar 2024 10:18:52 +0800 Subject: [PATCH 242/524] sw64: kvm: fix unmap_apt_{ptes, pmds} It has to flush remote tlbs after APT pte/pmd/pud is set. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 7523ac3a6c71..9c1d52883228 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -149,6 +149,7 @@ static void unmap_apt_ptes(struct kvm *kvm, pmd_t *pmd, if (!pte_none(*pte)) { /* Do we need WRITE_ONCE(pte, 0)? */ set_pte(pte, __pte(0)); + kvm_flush_remote_tlbs(kvm); put_page(virt_to_page(pte)); } } while (pte++, addr += PAGE_SIZE, addr != end); @@ -158,6 +159,7 @@ static void unmap_apt_ptes(struct kvm *kvm, pmd_t *pmd, pte_t *pte_table = pte_offset_kernel(pmd, 0); pmd_clear(pmd); + kvm_flush_remote_tlbs(kvm); free_page((unsigned long)pte_table); put_page(virt_to_page(pmd)); } @@ -195,6 +197,7 @@ static void unmap_apt_pmds(struct kvm *kvm, pud_t *pud, pmd_t *pmd_table __maybe_unused = pmd_offset(pud, 0UL); pud_clear(pud); + kvm_flush_remote_tlbs(kvm); free_page((unsigned long)pmd_table); put_page(virt_to_page(pud)); } -- Gitee From 429d409c552898bbe1bd46d10844b9996dc70214 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 26 Mar 2024 18:35:15 +0800 Subject: [PATCH 243/524] sw64: kvm: fix APT fault handler To prevent CSR:AS_INFO and CSR:EXC_GPA being corrupted by vCPU scheduling, save them in struct vcpucb, then hypervisor can get right values to handle APT_FAULT. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_mmu.h | 3 +- arch/sw_64/include/asm/vcpu.h | 4 +- arch/sw_64/kvm/emulate.c | 10 ----- arch/sw_64/kvm/handle_exit.c | 2 +- arch/sw_64/kvm/mmio.c | 6 +-- arch/sw_64/kvm/mmu.c | 66 ++++++++++++++++++-------------- 6 files changed, 43 insertions(+), 48 deletions(-) diff --git a/arch/sw_64/include/asm/kvm_mmu.h b/arch/sw_64/include/asm/kvm_mmu.h index f4493de934ba..f137e1c39e0c 100644 --- a/arch/sw_64/include/asm/kvm_mmu.h +++ b/arch/sw_64/include/asm/kvm_mmu.h @@ -126,6 +126,7 @@ void kvm_handle_apt_fault(struct kvm_vcpu *vcpu); int kvm_alloc_addtional_stage_pgd(struct kvm *kvm); void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot); -int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run); +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, + struct hcall_args *hargs); void apt_unmap_vm(struct kvm *kvm); #endif /* _ASM_SW64_KVM_MMU_H */ diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h index a68e8ff56af5..daa0c7e3913d 100644 --- a/arch/sw_64/include/asm/vcpu.h +++ b/arch/sw_64/include/asm/vcpu.h @@ -59,7 +59,7 @@ struct vcpucb { struct vcpucb { unsigned long ktp; - unsigned long pcbb; + unsigned long as_info; unsigned long ksp; unsigned long usp; unsigned long kgp; @@ -95,7 +95,7 @@ struct vcpucb { unsigned long reset_entry; unsigned long pvcpu; unsigned long exit_reason; - unsigned long ipaddr; + unsigned long fault_gpa; /* CSR:EXC_GPA */ unsigned long vcpu_pc_save; unsigned long shtclock_offset; unsigned long migration_mark; diff --git a/arch/sw_64/kvm/emulate.c b/arch/sw_64/kvm/emulate.c index 66e6335d9cce..9f2ededd5a1e 100644 --- a/arch/sw_64/kvm/emulate.c +++ b/arch/sw_64/kvm/emulate.c @@ -13,18 +13,8 @@ void sw64_decode(struct kvm_vcpu *vcpu, unsigned int insn, struct kvm_run *run) { int opc, ra; -#ifdef CONFIG_SUBARCH_C3B opc = (insn >> 26) & 0x3f; ra = (insn >> 21) & 0x1f; -#elif defined(CONFIG_SUBARCH_C4) - unsigned long ds_stat, exc_sum; - - ds_stat = sw64_read_csr(CSR_DS_STAT); - exc_sum = sw64_read_csr(CSR_EXC_SUM); - - opc = (ds_stat >> 4) & 0x3f; - ra = (exc_sum >> 8) & 0x1f; -#endif switch (opc) { case 0x20: /* LDBU */ diff --git a/arch/sw_64/kvm/handle_exit.c b/arch/sw_64/kvm/handle_exit.c index e0c62dc1112e..728d323a9f4d 100644 --- a/arch/sw_64/kvm/handle_exit.c +++ b/arch/sw_64/kvm/handle_exit.c @@ -65,7 +65,7 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, return 0; #ifdef CONFIG_SUBARCH_C4 case SW64_KVM_EXIT_APT_FAULT: - return kvm_handle_guest_abort(vcpu, run); + return kvm_handle_guest_abort(vcpu, run, hargs); #endif case SW64_KVM_EXIT_FATAL_ERROR: vcpu->stat.fatal_error_exits++; diff --git a/arch/sw_64/kvm/mmio.c b/arch/sw_64/kvm/mmio.c index 3214a4efb0b1..bc29dfbd2af9 100644 --- a/arch/sw_64/kvm/mmio.c +++ b/arch/sw_64/kvm/mmio.c @@ -68,13 +68,9 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, { int ret; -#ifdef CONFIG_SUBARCH_C3B run->mmio.phys_addr = hargs->arg1 & 0xfffffffffffffUL; sw64_decode(vcpu, hargs->arg2, run); -#elif defined(CONFIG_SUBARCH_C4) - run->mmio.phys_addr = sw64_read_csr(CSR_DVA) & 0xfffffffffffffUL; - sw64_decode(vcpu, 0, run); -#endif + if (run->mmio.is_write) { trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, run->mmio.len, run->mmio.phys_addr, run->mmio.data); diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 9c1d52883228..11866910f1a8 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -786,16 +786,22 @@ static bool apt_is_exec(struct kvm *kvm, phys_addr_t addr) return kvm_pte_exec(ptep); } -static int apt_set_pte_fast(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, - phys_addr_t addr, const pte_t *new_pte, - unsigned long flags) +static int apt_set_pte_fast(struct kvm_vcpu *vcpu, + struct kvm_mmu_memory_cache *cache, + const pte_t *new_pte, unsigned long flags) { pud_t *pud; pmd_t *pmd; pte_t *pte, old_pte; + unsigned long as_info, inv_hpa; + int inv_level; + struct kvm *kvm = vcpu->kvm; bool logging_active = flags & KVM_APT_FLAG_LOGGING_ACTIVE; - int inv_level = ((sw64_read_csr(CSR_AS_INFO)) >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; - unsigned long inv_hpa = sw64_read_csr(CSR_AS_INFO) & AF_ENTRY_ADDR_MASK; + phys_addr_t addr = vcpu->arch.vcb.fault_gpa; + + as_info = vcpu->arch.vcb.as_info; + inv_level = (as_info >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; + inv_hpa = as_info & AF_ENTRY_ADDR_MASK; VM_BUG_ON(logging_active && !cache); @@ -1134,25 +1140,28 @@ transparent_hugepage_adjust(struct kvm_memory_slot *memslot, return PAGE_SIZE; } -static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_gpa, - struct kvm_memory_slot *memslot, unsigned long hva, - unsigned long fault_status) +static int user_mem_abort(struct kvm_vcpu *vcpu, + struct kvm_memory_slot *memslot, + unsigned long hva, unsigned long fault_status) { int ret; - bool write_fault, exec_fault, writable, force_pte = false; - unsigned long mmu_seq; - gfn_t gfn = fault_gpa >> PAGE_SHIFT; + gfn_t gfn; + kvm_pfn_t pfn; + phys_addr_t fault_gpa; + unsigned int vma_shift; struct kvm *kvm = vcpu->kvm; - struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; struct vm_area_struct *vma; - kvm_pfn_t pfn; pgprot_t mem_type = PAGE_READONLY; - bool logging_active = memslot_is_logging(memslot); - unsigned long vma_pagesize, flags = 0; unsigned long as_info, access_type; - unsigned int vma_shift; + unsigned long mmu_seq, vma_pagesize, flags = 0; + struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; + bool logging_active, write_fault, exec_fault, writable, force_pte; - as_info = sw64_read_csr(CSR_AS_INFO); + force_pte = false; + logging_active = memslot_is_logging(memslot); + fault_gpa = vcpu->arch.vcb.fault_gpa; + gfn = fault_gpa >> PAGE_SHIFT; + as_info = vcpu->arch.vcb.as_info; access_type = (as_info >> AF_ACCESS_TYPE_SHIFT) & AF_ACCESS_TYPE_MASK; write_fault = kvm_is_write_fault(access_type); exec_fault = kvm_is_exec_fault(access_type); @@ -1317,7 +1326,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_gpa, new_pte = kvm_pte_mkexec(new_pte); } - ret = apt_set_pte_fast(kvm, memcache, fault_gpa, &new_pte, flags); + ret = apt_set_pte_fast(vcpu, memcache, &new_pte, flags); if (!ret) goto out_unlock; } @@ -1342,28 +1351,27 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_gpa, * memory region has been registered as standard RAM by user space. */ #ifdef CONFIG_SUBARCH_C4 -int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, + struct hcall_args *hargs) { - unsigned long as_info; /* the value of CSR: AS_INFO */ - unsigned int access_type, inv_level; - unsigned int fault_status; + unsigned long as_info; + unsigned int access_type, fault_status; unsigned long fault_entry_addr; phys_addr_t fault_gpa; struct kvm_memory_slot *memslot; unsigned long hva; bool write_fault, writable; gfn_t gfn; - int ret, idx; - as_info = sw64_read_csr(CSR_AS_INFO); + idx = srcu_read_lock(&vcpu->kvm->srcu); + + as_info = vcpu->arch.vcb.as_info; access_type = (as_info >> AF_ACCESS_TYPE_SHIFT) & AF_ACCESS_TYPE_MASK; - inv_level = (as_info >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; fault_status = (as_info >> AF_FAULT_STATUS_SHIFT) & AF_FAULT_STATUS_MASK; fault_entry_addr = (as_info & AF_ENTRY_ADDR_MASK) >> 3; - fault_gpa = sw64_read_csr(CSR_EXC_GPA); - idx = srcu_read_lock(&vcpu->kvm->srcu); + fault_gpa = vcpu->arch.vcb.fault_gpa; gfn = fault_gpa >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); @@ -1378,13 +1386,13 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) */ if (hva == KVM_HVA_ERR_BAD) { - ret = io_mem_abort(vcpu, run, NULL); + ret = io_mem_abort(vcpu, run, hargs); goto out_unlock; } /* Userspace should not be able to register out-of-bounds IPAs */ VM_BUG_ON(fault_gpa >= KVM_PHYS_SIZE); - ret = user_mem_abort(vcpu, fault_gpa, memslot, hva, fault_status); + ret = user_mem_abort(vcpu, memslot, hva, fault_status); if (ret == 0) ret = 1; out_unlock: -- Gitee From 0435d5cb78bc740836fdec70d82e10f32fc2cd92 Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Tue, 26 Mar 2024 09:32:03 +0800 Subject: [PATCH 244/524] sw64: kvm: fix live migration bug When it logs dirty pages it should dissolve hugepages, then reallocate pages. Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 11866910f1a8..52d627c5d1bc 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -805,7 +805,9 @@ static int apt_set_pte_fast(struct kvm_vcpu *vcpu, VM_BUG_ON(logging_active && !cache); - if (inv_level == 1) { + if (logging_active) { + goto dissolve; + } else if (inv_level == 1) { pud = (pud_t *)(inv_hpa | PAGE_OFFSET); goto find_pud; } else if (inv_level == 2) { @@ -816,6 +818,7 @@ static int apt_set_pte_fast(struct kvm_vcpu *vcpu, goto find_pte; } +dissolve: /* Create addtional page table mapping - Levels 0 and 1 */ pud = apt_get_pud(kvm->arch.pgd, cache, addr); if (!pud) { -- Gitee From 3ce3fbcf3f2024a8b67372f04479404922ecfd5a Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Wed, 27 Mar 2024 14:28:55 +0800 Subject: [PATCH 245/524] sw64: redesign struct kvm_regs for C4 Reuse struct user_pt_regs and struct user_fpsimd_state to redesign struct kvm_regs in a way that makes the code clearer. This requires corresponding modification to firmware. Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm.h | 52 ++++----------------- arch/sw_64/include/asm/kvm_emulate.h | 16 +++++++ arch/sw_64/kernel/asm-offsets.c | 69 +++++++++++++++++++++++++++- 3 files changed, 92 insertions(+), 45 deletions(-) diff --git a/arch/sw_64/include/asm/kvm.h b/arch/sw_64/include/asm/kvm.h index 41bb657bcacc..6794fe085e0d 100644 --- a/arch/sw_64/include/asm/kvm.h +++ b/arch/sw_64/include/asm/kvm.h @@ -64,49 +64,15 @@ struct kvm_regs { #elif CONFIG_SUBARCH_C4 struct kvm_regs { - unsigned long r0; - unsigned long r1; - unsigned long r2; - unsigned long r3; - - unsigned long r4; - unsigned long r5; - unsigned long r6; - unsigned long r7; - - unsigned long r8; - unsigned long r9; - unsigned long r10; - unsigned long r11; - - unsigned long r12; - unsigned long r13; - unsigned long r14; - unsigned long r15; - - unsigned long r16; - unsigned long r17; - unsigned long r18; - unsigned long r19; - - unsigned long r20; - unsigned long r21; - unsigned long r22; - unsigned long r23; - - unsigned long r24; - unsigned long r25; - unsigned long r26; - unsigned long r27; - - unsigned long r28; - unsigned long gp; - unsigned long sp; - unsigned long fpcr; - - unsigned long fp[124]; - unsigned long ps; - unsigned long pc; + union { + struct user_pt_regs regs; + struct { + unsigned long r[31]; + unsigned long pc; + unsigned long ps; + }; + }; + struct user_fpsimd_state fpstate; }; #endif diff --git a/arch/sw_64/include/asm/kvm_emulate.h b/arch/sw_64/include/asm/kvm_emulate.h index 915aa6c0bce2..d168f72a2deb 100644 --- a/arch/sw_64/include/asm/kvm_emulate.h +++ b/arch/sw_64/include/asm/kvm_emulate.h @@ -7,6 +7,7 @@ #define R(x) ((size_t) &((struct kvm_regs *)0)->x) +#ifdef CONFIG_SUBARCH_C3B static int reg_offsets[32] = { R(r0), R(r1), R(r2), R(r3), R(r4), R(r5), R(r6), R(r7), R(r8), R(r9), R(r10), R(r11), R(r12), R(r13), R(r14), R(r15), @@ -36,6 +37,21 @@ static inline unsigned long vcpu_get_reg(struct kvm_vcpu *vcpu, u8 reg_num) return *(unsigned long *)regs_ptr; } +#elif defined(CONFIG_SUBARCH_C4) +static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num, + unsigned long val) +{ + vcpu->arch.regs.r[reg_num] = val; +} + +static inline unsigned long vcpu_get_reg(struct kvm_vcpu *vcpu, u8 reg_num) +{ + if (reg_num == 31) + return 0; + return vcpu->arch.regs.r[reg_num]; +} +#endif + void sw64_decode(struct kvm_vcpu *vcpu, unsigned int insn, struct kvm_run *run); diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c index 337386c163ff..d865caffd014 100644 --- a/arch/sw_64/kernel/asm-offsets.c +++ b/arch/sw_64/kernel/asm-offsets.c @@ -114,6 +114,7 @@ void foo(void) BLANK(); DEFINE(KVM_REGS_SIZE, sizeof(struct kvm_regs)); +#ifdef CONFIG_SUBARCH_C3B DEFINE(KVM_REGS_R0, offsetof(struct kvm_regs, r0)); DEFINE(KVM_REGS_R1, offsetof(struct kvm_regs, r1)); DEFINE(KVM_REGS_R2, offsetof(struct kvm_regs, r2)); @@ -178,8 +179,72 @@ void foo(void) DEFINE(KVM_REGS_F30, offsetof(struct kvm_regs, fp[30 * 4])); DEFINE(KVM_REGS_PS, offsetof(struct kvm_regs, ps)); DEFINE(KVM_REGS_PC, offsetof(struct kvm_regs, pc)); -#ifdef CONFIG_SUBARCH_C4 - DEFINE(KVM_REGS_SP, offsetof(struct kvm_regs, sp)); +#elif CONFIG_SUBARCH_C4 + DEFINE(KVM_REGS_R0, offsetof(struct kvm_regs, regs.regs[0])); + DEFINE(KVM_REGS_R1, offsetof(struct kvm_regs, regs.regs[1])); + DEFINE(KVM_REGS_R2, offsetof(struct kvm_regs, regs.regs[2])); + DEFINE(KVM_REGS_R3, offsetof(struct kvm_regs, regs.regs[3])); + DEFINE(KVM_REGS_R4, offsetof(struct kvm_regs, regs.regs[4])); + DEFINE(KVM_REGS_R5, offsetof(struct kvm_regs, regs.regs[5])); + DEFINE(KVM_REGS_R6, offsetof(struct kvm_regs, regs.regs[6])); + DEFINE(KVM_REGS_R7, offsetof(struct kvm_regs, regs.regs[7])); + DEFINE(KVM_REGS_R8, offsetof(struct kvm_regs, regs.regs[8])); + DEFINE(KVM_REGS_R9, offsetof(struct kvm_regs, regs.regs[9])); + DEFINE(KVM_REGS_R10, offsetof(struct kvm_regs, regs.regs[10])); + DEFINE(KVM_REGS_R11, offsetof(struct kvm_regs, regs.regs[11])); + DEFINE(KVM_REGS_R12, offsetof(struct kvm_regs, regs.regs[12])); + DEFINE(KVM_REGS_R13, offsetof(struct kvm_regs, regs.regs[13])); + DEFINE(KVM_REGS_R14, offsetof(struct kvm_regs, regs.regs[14])); + DEFINE(KVM_REGS_R15, offsetof(struct kvm_regs, regs.regs[15])); + DEFINE(KVM_REGS_R16, offsetof(struct kvm_regs, regs.regs[16])); + DEFINE(KVM_REGS_R17, offsetof(struct kvm_regs, regs.regs[17])); + DEFINE(KVM_REGS_R18, offsetof(struct kvm_regs, regs.regs[18])); + DEFINE(KVM_REGS_R19, offsetof(struct kvm_regs, regs.regs[19])); + DEFINE(KVM_REGS_R20, offsetof(struct kvm_regs, regs.regs[20])); + DEFINE(KVM_REGS_R21, offsetof(struct kvm_regs, regs.regs[21])); + DEFINE(KVM_REGS_R22, offsetof(struct kvm_regs, regs.regs[22])); + DEFINE(KVM_REGS_R23, offsetof(struct kvm_regs, regs.regs[23])); + DEFINE(KVM_REGS_R24, offsetof(struct kvm_regs, regs.regs[24])); + DEFINE(KVM_REGS_R25, offsetof(struct kvm_regs, regs.regs[25])); + DEFINE(KVM_REGS_R26, offsetof(struct kvm_regs, regs.regs[26])); + DEFINE(KVM_REGS_R27, offsetof(struct kvm_regs, regs.regs[27])); + DEFINE(KVM_REGS_R28, offsetof(struct kvm_regs, regs.regs[28])); + DEFINE(KVM_REGS_GP, offsetof(struct kvm_regs, regs.regs[29])); + DEFINE(KVM_REGS_SP, offsetof(struct kvm_regs, regs.regs[30])); + DEFINE(KVM_REGS_PC, offsetof(struct kvm_regs, regs.pc)); + DEFINE(KVM_REGS_PS, offsetof(struct kvm_regs, regs.pstate)); + DEFINE(KVM_REGS_F0, offsetof(struct kvm_regs, fpstate.fp[0])); + DEFINE(KVM_REGS_F1, offsetof(struct kvm_regs, fpstate.fp[1])); + DEFINE(KVM_REGS_F2, offsetof(struct kvm_regs, fpstate.fp[2])); + DEFINE(KVM_REGS_F3, offsetof(struct kvm_regs, fpstate.fp[3])); + DEFINE(KVM_REGS_F4, offsetof(struct kvm_regs, fpstate.fp[4])); + DEFINE(KVM_REGS_F5, offsetof(struct kvm_regs, fpstate.fp[5])); + DEFINE(KVM_REGS_F6, offsetof(struct kvm_regs, fpstate.fp[6])); + DEFINE(KVM_REGS_F7, offsetof(struct kvm_regs, fpstate.fp[7])); + DEFINE(KVM_REGS_F8, offsetof(struct kvm_regs, fpstate.fp[8])); + DEFINE(KVM_REGS_F9, offsetof(struct kvm_regs, fpstate.fp[9])); + DEFINE(KVM_REGS_F10, offsetof(struct kvm_regs, fpstate.fp[10])); + DEFINE(KVM_REGS_F11, offsetof(struct kvm_regs, fpstate.fp[11])); + DEFINE(KVM_REGS_F12, offsetof(struct kvm_regs, fpstate.fp[12])); + DEFINE(KVM_REGS_F13, offsetof(struct kvm_regs, fpstate.fp[13])); + DEFINE(KVM_REGS_F14, offsetof(struct kvm_regs, fpstate.fp[14])); + DEFINE(KVM_REGS_F15, offsetof(struct kvm_regs, fpstate.fp[15])); + DEFINE(KVM_REGS_F16, offsetof(struct kvm_regs, fpstate.fp[16])); + DEFINE(KVM_REGS_F17, offsetof(struct kvm_regs, fpstate.fp[17])); + DEFINE(KVM_REGS_F18, offsetof(struct kvm_regs, fpstate.fp[18])); + DEFINE(KVM_REGS_F19, offsetof(struct kvm_regs, fpstate.fp[19])); + DEFINE(KVM_REGS_F20, offsetof(struct kvm_regs, fpstate.fp[20])); + DEFINE(KVM_REGS_F21, offsetof(struct kvm_regs, fpstate.fp[21])); + DEFINE(KVM_REGS_F22, offsetof(struct kvm_regs, fpstate.fp[22])); + DEFINE(KVM_REGS_F23, offsetof(struct kvm_regs, fpstate.fp[23])); + DEFINE(KVM_REGS_F24, offsetof(struct kvm_regs, fpstate.fp[24])); + DEFINE(KVM_REGS_F25, offsetof(struct kvm_regs, fpstate.fp[25])); + DEFINE(KVM_REGS_F26, offsetof(struct kvm_regs, fpstate.fp[26])); + DEFINE(KVM_REGS_F27, offsetof(struct kvm_regs, fpstate.fp[27])); + DEFINE(KVM_REGS_F28, offsetof(struct kvm_regs, fpstate.fp[28])); + DEFINE(KVM_REGS_F29, offsetof(struct kvm_regs, fpstate.fp[29])); + DEFINE(KVM_REGS_F30, offsetof(struct kvm_regs, fpstate.fp[30])); + DEFINE(KVM_REGS_FPCR, offsetof(struct kvm_regs, fpstate.fpcr)); #endif BLANK(); -- Gitee From 8164706ea09204d8e7847f25c593b785be100b28 Mon Sep 17 00:00:00 2001 From: Lu Feifei Date: Thu, 28 Mar 2024 13:49:59 +0800 Subject: [PATCH 246/524] sw64: kvm: fix page walk error of unmap_apt_range() There is an error in 5-level page walking, and fix it. Signed-off-by: Lu Feifei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 52d627c5d1bc..bcf7c28bdcd4 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -267,7 +267,7 @@ static void unmap_apt_range(struct kvm *kvm, phys_addr_t start, u64 size) */ if (!READ_ONCE(kvm->arch.pgd)) break; - next = p4d_addr_end(addr, end); + next = pgd_addr_end(addr, end); if (!p4d_none(*p4d)) unmap_apt_puds(kvm, p4d, addr, next); /* @@ -276,7 +276,7 @@ static void unmap_apt_range(struct kvm *kvm, phys_addr_t start, u64 size) */ if (next != end) cond_resched_lock(&kvm->mmu_lock); - } while (pgd++, addr = next, addr != end); + } while (p4d++, addr = next, addr != end); } static void apt_unmap_memslot(struct kvm *kvm, -- Gitee From d40f56a909020dd64d6890280056e93bf2ba206e Mon Sep 17 00:00:00 2001 From: Lu Feifei Date: Thu, 28 Mar 2024 14:00:56 +0800 Subject: [PATCH 247/524] sw64: kvm: fix the check for THP Use kvm_is_transparent_hugepage() to explicitly check whether a compound page is a THP when populating KVM's secondary MMU. Signed-off-by: Lu Feifei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index bcf7c28bdcd4..d53fafc79a65 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -1105,14 +1105,13 @@ transparent_hugepage_adjust(struct kvm_memory_slot *memslot, phys_addr_t *gpap) { kvm_pfn_t pfn = *pfnp; - struct page *page = pfn_to_page(pfn); /* * Make sure the adjustment is done only for THP pages. Also make * sure that the HVA and IPA are sufficiently aligned and that the * block map is contained within the memslot. */ - if (!PageHuge(page) && PageTransCompoundMap(page) && + if (kvm_is_transparent_hugepage(pfn) && fault_supports_apt_huge_mapping(memslot, hva, PMD_SIZE)) { /* * The address we faulted on is backed by a transparent huge -- Gitee From 65fc8e6fd2a46a9bd0832a7909c2334ad4998b26 Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Wed, 27 Mar 2024 17:20:19 +0800 Subject: [PATCH 248/524] sw64: kvm: add the member of the vcpucb structure Member atc is added for CSR_ATC to save and restore, and adjusted the size of the structure for future expansion. Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/vcpu.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h index daa0c7e3913d..8227166d6998 100644 --- a/arch/sw_64/include/asm/vcpu.h +++ b/arch/sw_64/include/asm/vcpu.h @@ -108,6 +108,8 @@ struct vcpucb { unsigned long csr_earg1; unsigned long csr_earg2; unsigned long csr_scratch; + unsigned long atc; + unsigned long reserved[45]; }; #endif -- Gitee From 3bf95452d6f0161e4efa85e4bd12b19dc7ccc68e Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Mon, 8 Apr 2024 18:36:14 +0800 Subject: [PATCH 249/524] sw64: kvm: fix size of struct kvm_regs for C3B The size of struct kvm_regs passed by Qemu is fixed and checked in kernel. Since Qemu has extended the struct kvm_regs, this patch adds some padding bytes to ensure that the size of struct kvm_regs is the same on both C3B and C4. Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/kvm.h b/arch/sw_64/include/asm/kvm.h index 6794fe085e0d..c728504d219e 100644 --- a/arch/sw_64/include/asm/kvm.h +++ b/arch/sw_64/include/asm/kvm.h @@ -49,7 +49,7 @@ struct kvm_regs { unsigned long r27; unsigned long r28; - unsigned long __padding0; + unsigned long reserved; unsigned long fpcr; unsigned long fp[124]; @@ -60,6 +60,7 @@ struct kvm_regs { unsigned long r16; unsigned long r17; unsigned long r18; + unsigned long __padding[6]; }; #elif CONFIG_SUBARCH_C4 -- Gitee From c61ba9461354df4a3b26fc79ebb2df12a3bffb7d Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Thu, 11 Apr 2024 14:39:59 +0800 Subject: [PATCH 250/524] sw64: kvm: fix pmd_trans_cont If CONFIG_TRANSPARENT_HUGEPAGE=n, it fails to complie because pmd_trans_cont() is undefined. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 10 +++++----- arch/sw_64/kvm/mmu.c | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index a7741a67e687..b9f98b9af33e 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -621,6 +621,11 @@ static inline int pte_devmap(pte_t a) } #endif +static inline int pmd_cont(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_CONT); +} + #ifdef CONFIG_TRANSPARENT_HUGEPAGE /* We don't have hardware dirty/accessed bits, generic_pmdp_establish is fine.*/ @@ -631,11 +636,6 @@ static inline int pmd_trans_splitting(pmd_t pmd) return !!(pmd_val(pmd) & _PAGE_SPLITTING); } -static inline int pmd_trans_cont(pmd_t pmd) -{ - return !!(pmd_val(pmd) & _PAGE_CONT); -} - static inline int pmd_trans_huge(pmd_t pmd) { return !!(pmd_val(pmd) & _PAGE_LEAF); diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index d53fafc79a65..4ec62bc4fc50 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -73,7 +73,7 @@ static void apt_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd) if (!pmd_trans_huge(*pmd)) return; - if (pmd_trans_cont(*pmd)) { + if (pmd_cont(*pmd)) { for (i = 0; i < CONT_PMDS; i++, pmd++) pmd_clear(pmd); } else @@ -178,7 +178,7 @@ static void unmap_apt_pmds(struct kvm *kvm, pud_t *pud, next = pmd_addr_end(addr, end); if (!pmd_none(*pmd)) { if (pmd_trans_huge(*pmd)) { - if (pmd_trans_cont(*pmd)) { + if (pmd_cont(*pmd)) { for (i = 0; i < CONT_PMDS; i++, pmd++) pmd_clear(pmd); } else -- Gitee From 64545190c8d80de2fabd06385a41225669828787 Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Fri, 12 Apr 2024 17:25:34 +0800 Subject: [PATCH 251/524] sw64: kvm: remove some unused codes Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/entry_core4.S | 2 -- arch/sw_64/kvm/kvm_core4.c | 2 -- arch/sw_64/kvm/sw64.c | 3 --- 3 files changed, 7 deletions(-) diff --git a/arch/sw_64/kvm/entry_core4.S b/arch/sw_64/kvm/entry_core4.S index 98993884fd08..48325cc7c2c5 100644 --- a/arch/sw_64/kvm/entry_core4.S +++ b/arch/sw_64/kvm/entry_core4.S @@ -219,8 +219,6 @@ $g_setfpec_over: ldl $15, PT_REGS_R15(sp) ldl $26, PT_REGS_R26(sp) ldl $29, PT_REGS_GP(sp) - csrr $2, CSR_PS - stl $2, PT_REGS_PS(sp) ldi sp, PT_REGS_SIZE(sp) /* restore host fpregs */ diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c index 81a999b3fd30..a9d9326072e4 100644 --- a/arch/sw_64/kvm/kvm_core4.c +++ b/arch/sw_64/kvm/kvm_core4.c @@ -55,8 +55,6 @@ int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) apt_unmap_vm(vcpu->kvm); hrtimer_cancel(&vcpu->arch.hrt); - vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; - vcpu->arch.vcb.vcpu_irq_disabled = 1; vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ vcpu->arch.power_off = 0; memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index 630ea907592c..86189243fc75 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -276,9 +276,6 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) hrtimer_init(&vcpu->arch.hrt, CLOCK_REALTIME, HRTIMER_MODE_ABS); vcpu->arch.hrt.function = clockdev_fn; vcpu->arch.tsk = current; - - vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; - vcpu->arch.vcb.vcpu_irq_disabled = 1; vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ return 0; -- Gitee From 02288b43ffb4a361cd73aa85a3cded5134db8d58 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Sun, 28 Apr 2024 10:36:54 +0800 Subject: [PATCH 252/524] sw64: kvm: use generic kvm mmu memory caches Switch to the generic MMU memory cache implementation. Besides, setting pud, pmd and pte page tables will call mmu_memory_cache_alloc() to request memory cache pages, so increase KVM_MMU_CACHE_MIN_PAGES to 3 to avoid kernel panic. Signed-off-by: Chen Wang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/Kbuild | 1 - arch/sw_64/include/asm/kvm_host.h | 11 ------- arch/sw_64/include/asm/kvm_mmu.h | 2 +- arch/sw_64/include/asm/kvm_types.h | 8 +++++ arch/sw_64/kvm/mmu.c | 53 +++++------------------------- arch/sw_64/kvm/sw64.c | 3 +- 6 files changed, 19 insertions(+), 59 deletions(-) create mode 100644 arch/sw_64/include/asm/kvm_types.h diff --git a/arch/sw_64/include/asm/Kbuild b/arch/sw_64/include/asm/Kbuild index 0dd0a704d8f1..e9cbabda1516 100644 --- a/arch/sw_64/include/asm/Kbuild +++ b/arch/sw_64/include/asm/Kbuild @@ -2,7 +2,6 @@ generic-y += clkdev.h generic-y += export.h -generic-y += kvm_types.h generic-y += mcs_spinlock.h generic-y += param.h generic-y += qrwlock.h diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index b726b0f5d94e..0ab02eb62e4e 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -76,17 +76,6 @@ struct kvm_arch { pgd_t *pgd; }; -#define KVM_NR_MEM_OBJS 40 - -/* - * We don't want allocation failures within the mmu code, so we preallocate - * enough memory for a single page fault in a cache. - */ -struct kvm_mmu_memory_cache { - int nobjs; - void *objects[KVM_NR_MEM_OBJS]; -}; - struct kvm_vcpu_arch { struct kvm_regs regs __aligned(32); struct vcpucb vcb; diff --git a/arch/sw_64/include/asm/kvm_mmu.h b/arch/sw_64/include/asm/kvm_mmu.h index f137e1c39e0c..94fe863ccafd 100644 --- a/arch/sw_64/include/asm/kvm_mmu.h +++ b/arch/sw_64/include/asm/kvm_mmu.h @@ -29,7 +29,7 @@ #define AF_STATUS_FOE 0x8 #define AF_STATUS_INV 0x10 -#define KVM_MMU_CACHE_MIN_PAGES 2 +#define KVM_MMU_CACHE_MIN_PAGES 3 static inline void kvm_set_aptpte_readonly(pte_t *pte) { diff --git a/arch/sw_64/include/asm/kvm_types.h b/arch/sw_64/include/asm/kvm_types.h new file mode 100644 index 000000000000..15b6c2845a92 --- /dev/null +++ b/arch/sw_64/include/asm/kvm_types.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_TYPES_H +#define _ASM_SW64_KVM_TYPES_H + +#define KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE 40 + +#endif /* _ASM_SW64_KVM_TYPES_H */ + diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 4ec62bc4fc50..9b4c770fa2d9 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -101,43 +101,6 @@ static void apt_dissolve_pud(struct kvm *kvm, phys_addr_t addr, pud_t *pudp) put_page(virt_to_page(pudp)); } -static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, - int min, int max) -{ - void *page; - - BUG_ON(max > KVM_NR_MEM_OBJS); - if (cache->nobjs >= min) - return 0; - while (cache->nobjs < max) { - page = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO); - if (!page) - return -ENOMEM; - cache->objects[cache->nobjs++] = page; - } - return 0; -} - -static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) -{ - while (mc->nobjs) - free_page((unsigned long)mc->objects[--mc->nobjs]); -} - -void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) -{ - mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); -} - -static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) -{ - void *p; - - BUG_ON(!mc || !mc->nobjs); - p = mc->objects[--mc->nobjs]; - return p; -} - static void unmap_apt_ptes(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr, phys_addr_t end) { @@ -362,7 +325,7 @@ static pud_t *apt_get_pud(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, if (p4d_none(*p4d)) { if (!cache) return NULL; - pud = mmu_memory_cache_alloc(cache); + pud = kvm_mmu_memory_cache_alloc(cache); p4d_populate(NULL, p4d, pud); get_page(virt_to_page(p4d)); } @@ -382,7 +345,7 @@ static pmd_t *apt_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, if (pud_none(*pud)) { if (!cache) return NULL; - pmd = mmu_memory_cache_alloc(cache); + pmd = kvm_mmu_memory_cache_alloc(cache); pud_populate(NULL, pud, pmd); get_page(virt_to_page(pud)); } @@ -840,7 +803,7 @@ static int apt_set_pte_fast(struct kvm_vcpu *vcpu, if (pud_none(*pud)) { if (!cache) return 0; /* ignore calls from kvm_set_spte_hva */ - pmd = mmu_memory_cache_alloc(cache); + pmd = kvm_mmu_memory_cache_alloc(cache); pud_populate(NULL, pud, pmd); get_page(virt_to_page(pud)); } @@ -866,7 +829,7 @@ static int apt_set_pte_fast(struct kvm_vcpu *vcpu, if (pmd_none(*pmd)) { if (!cache) return 0; /* ignore calls from kvm_set_spte_hva */ - pte = mmu_memory_cache_alloc(cache); + pte = kvm_mmu_memory_cache_alloc(cache); pmd_populate_kernel(NULL, pmd, pte); get_page(virt_to_page(pmd)); } @@ -928,7 +891,7 @@ static int apt_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, if (pud_none(*pud)) { if (!cache) return 0; /* ignore calls from kvm_set_spte_hva */ - pmd = mmu_memory_cache_alloc(cache); + pmd = kvm_mmu_memory_cache_alloc(cache); pud_populate(NULL, pud, pmd); get_page(virt_to_page(pud)); } @@ -953,7 +916,7 @@ static int apt_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, if (pmd_none(*pmd)) { if (!cache) return 0; /* ignore calls from kvm_set_spte_hva */ - pte = mmu_memory_cache_alloc(cache); + pte = kvm_mmu_memory_cache_alloc(cache); pmd_populate_kernel(NULL, pmd, pte); get_page(virt_to_page(pmd)); } @@ -1199,8 +1162,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, gfn = (fault_gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; up_read(¤t->mm->mmap_lock); /* We need minimum second+third level pages */ - ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES, - KVM_NR_MEM_OBJS); + ret = kvm_mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES); + if (ret) return ret; diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index 86189243fc75..4cf57af96eae 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -258,7 +258,7 @@ int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) { - kvm_mmu_free_memory_caches(vcpu); + kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); hrtimer_cancel(&vcpu->arch.hrt); } @@ -272,6 +272,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) /* Set up the timer for Guest */ pr_info("vcpu: [%d], regs addr = %#lx, vcpucb = %#lx\n", vcpu->vcpu_id, (unsigned long)&vcpu->arch.regs, (unsigned long)&vcpu->arch.vcb); + vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; vcpu->arch.vtimer_freq = cpuid(GET_CPU_FREQ, 0) * 1000UL * 1000UL; hrtimer_init(&vcpu->arch.hrt, CLOCK_REALTIME, HRTIMER_MODE_ABS); vcpu->arch.hrt.function = clockdev_fn; -- Gitee From 04f5ce19215ab42cc4a8922ae6c359442a664118 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 30 Apr 2024 14:42:20 +0800 Subject: [PATCH 253/524] sw64: kvm: acpi: match sunway_ged by acpi method Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/misc/sunway-ged.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/misc/sunway-ged.c b/drivers/misc/sunway-ged.c index aa500421355d..516770036962 100644 --- a/drivers/misc/sunway-ged.c +++ b/drivers/misc/sunway-ged.c @@ -2,6 +2,7 @@ /* Generic Event Device for ACPI. */ +#include #include #include #include @@ -236,10 +237,19 @@ static const struct of_device_id sunwayged_of_match[] = { }; MODULE_DEVICE_TABLE(of, sunwayged_of_match); +#ifdef CONFIG_ACPI +static const struct acpi_device_id sunwayged_acpi_match[] = { + { "SUNW1000", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, sunwayged_acpi_match); +#endif + static struct platform_driver sunwayged_platform_driver = { .driver = { .name = "sunway-ged", .of_match_table = sunwayged_of_match, + .acpi_match_table = ACPI_PTR(sunwayged_acpi_match), }, .probe = sunwayged_probe, .remove = sunwayged_remove, -- Gitee From 93074a7c93fafeb4591c008ff3e2c4aab266da85 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 14 May 2024 14:32:39 +0800 Subject: [PATCH 254/524] sw64: kvm: fix bug when vcpu disable irq Vcpu should not wake up when its irq disabled, unless a NMII interrupt is received, that is, the restart flag is set to ture. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/sw64.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index 4cf57af96eae..7305d060bb2e 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -169,6 +169,12 @@ struct dfx_sw64_kvm_stats_debugfs_item dfx_sw64_debugfs_entries[] = { int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) { + if (vcpu->arch.restart) + return 1; + + if (vcpu->arch.vcb.vcpu_irq_disabled) + return 0; + return ((!bitmap_empty(vcpu->arch.irqs_pending, SWVM_IRQS) || !vcpu->arch.halted) && !vcpu->arch.power_off); } -- Gitee From 48efabb0437d1f0965e331d8c608ab2264d156b9 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Wed, 22 May 2024 13:33:45 +0800 Subject: [PATCH 255/524] sw64: kvm: add numa support for memory hotplug Memory hotplug node id was always 0 in the past. Since vNUMA is supported, this patch adds 'node' into struct sunway_memory_device to hold numa node id which is prepared by Qemu, and then memory hotplug interfaces can handle argument nid correctly. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/misc/sunway-ged.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/misc/sunway-ged.c b/drivers/misc/sunway-ged.c index 516770036962..f2fda9f0dcb0 100644 --- a/drivers/misc/sunway-ged.c +++ b/drivers/misc/sunway-ged.c @@ -12,11 +12,14 @@ #include #include +/* offset should be the same as QEMU */ #define OFFSET_START_ADDR 0 #define OFFSET_LENGTH 8 #define OFFSET_STATUS 16 #define OFFSET_SLOT 24 +#define OFFSET_NODE 40 + /* Memory hotplug event */ #define SUNWAY_MEMHOTPLUG_ADD 0x1 #define SUNWAY_MEMHOTPLUG_REMOVE 0x2 @@ -29,6 +32,7 @@ struct sunway_memory_device { u64 start_addr; /* Memory Range start physical addr */ u64 length; /* Memory Range length */ u64 slot; /* Memory Range slot */ + u64 node; /* Memory Range node */ unsigned int enabled:1; }; @@ -59,7 +63,7 @@ static int sunway_memory_enable_device(struct sunway_memory_device *mem_device) lock_device_hotplug(); /* suppose node = 0, fix me! */ - result = __add_memory(0, mem_device->start_addr, mem_device->length, MHP_NONE); + result = __add_memory(mem_device->node, mem_device->start_addr, mem_device->length, MHP_NONE); unlock_device_hotplug(); /* * If the memory block has been used by the kernel, add_memory() @@ -106,7 +110,7 @@ static int sunway_memory_get_meminfo(struct sunway_memory_device *mem_device) static void sunway_memory_device_remove(struct sunway_ged_device *device) { struct sunway_memory_device *mem_dev, *n; - unsigned long start_addr, length, slot; + unsigned long start_addr, length, slot, node; if (!device) return; @@ -114,6 +118,7 @@ static void sunway_memory_device_remove(struct sunway_ged_device *device) start_addr = readq(device->membase + OFFSET_START_ADDR); length = readq(device->membase + OFFSET_LENGTH); slot = readq(device->membase + OFFSET_SLOT); + node = readq(device->membase + OFFSET_NODE); list_for_each_entry_safe(mem_dev, n, &device->dev_list, list) { if (!mem_dev->enabled) @@ -122,7 +127,7 @@ static void sunway_memory_device_remove(struct sunway_ged_device *device) if ((start_addr == mem_dev->start_addr) && (length == mem_dev->length)) { /* suppose node = 0, fix me! */ - remove_memory(0, start_addr, length); + remove_memory(node, start_addr, length); list_del(&mem_dev->list); kfree(mem_dev); } @@ -150,6 +155,7 @@ static int sunway_memory_device_add(struct sunway_ged_device *device) mem_device->start_addr = readq(device->membase + OFFSET_START_ADDR); mem_device->length = readq(device->membase + OFFSET_LENGTH); mem_device->slot = readq(device->membase + OFFSET_SLOT); + mem_device->node = readq(device->membase + OFFSET_NODE); result = sunway_memory_enable_device(mem_device); if (result) { -- Gitee From 5d08baa755e1295a88acb73389b031d3ab01fc99 Mon Sep 17 00:00:00 2001 From: Deng Xiaoyun Date: Thu, 9 May 2024 09:24:24 +0800 Subject: [PATCH 256/524] sw64: fix guest send ipi target For guest os, the target cpu id of IPI is vcpuid, and when vNUMA is supported, vcpuid is not equal to rcid. Signed-off-by: Deng Xiaoyun Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/smp.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h index 130c52c9e306..33a73ca25a6b 100644 --- a/arch/sw_64/include/asm/smp.h +++ b/arch/sw_64/include/asm/smp.h @@ -122,12 +122,12 @@ static inline void send_ipi(int cpu, unsigned long type) { int rcid; - rcid = cpu_to_rcid(cpu); - if (is_in_guest()) - hcall(HCALL_IVI, rcid, type, 0); - else + hcall(HCALL_IVI, cpu, type, 0); + else { + rcid = cpu_to_rcid(cpu); sendii(rcid, type, 0); + } } #define reset_cpu(cpu) send_ipi((cpu), II_RESET) -- Gitee From 7a69fce0a4310473d3490767dfee7d4e065cb8f4 Mon Sep 17 00:00:00 2001 From: Xu Linqin Date: Tue, 21 May 2024 18:11:13 +0800 Subject: [PATCH 257/524] sw64: kvm: implement kvm_arch_set_irq_inatomic() kvm_arch_set_irq_inatomic() is used in irqfd_wakeup() which is required by qemu to notify guest by irqfd. This patch improves IO performance of guest os because it injects irq directly instead of being scheduled. The virtualization rate has increased by 20% in guest randread/randwrite IOPS test when using SATA SSD. Signed-off-by: Xu Linqin Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/sw64.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index 7305d060bb2e..a22a7f78c2cb 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -42,6 +42,21 @@ static unsigned long get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) return next; } +int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, + int level, bool line_status) +{ + switch (e->type) { + case KVM_IRQ_ROUTING_MSI: + if (!kvm_set_msi(e, kvm, irq_source_id, level, line_status)) + return 0; + break; + default: + break; + } + return -EWOULDBLOCK; +} + int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) { set_bit(number, (vcpu->arch.irqs_pending)); -- Gitee From a41f7da734706bf45eb0c521eeb5d27a5be21a43 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Thu, 23 May 2024 15:00:25 +0800 Subject: [PATCH 258/524] sw64: kvm: fix inconsistent vcpucb of C4 The position of shtclock_offset in struct vcpucb is inconsistent between kvm and hmcode, leading to wrong SHTCLOCK after migration. Fix its position to solve the problem. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/vcpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h index 8227166d6998..b2ab8c7f8b86 100644 --- a/arch/sw_64/include/asm/vcpu.h +++ b/arch/sw_64/include/asm/vcpu.h @@ -97,9 +97,9 @@ struct vcpucb { unsigned long exit_reason; unsigned long fault_gpa; /* CSR:EXC_GPA */ unsigned long vcpu_pc_save; - unsigned long shtclock_offset; unsigned long migration_mark; unsigned long shtclock; + unsigned long shtclock_offset; unsigned long csr_pc; unsigned long csr_ps; unsigned long csr_sp; -- Gitee From d595d85d2a3e24f85841835ae8eda7ce0f609f52 Mon Sep 17 00:00:00 2001 From: Deng Xiaoyun Date: Mon, 13 May 2024 18:50:02 +0800 Subject: [PATCH 259/524] sw64: kvm: support up to 256 vCPUs for C4 Signed-off-by: Deng Xiaoyun Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_host.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index 0ab02eb62e4e..66434307523d 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -33,18 +33,19 @@ #ifdef CONFIG_SUBARCH_C3B #define VPN_BITS 8 #define GUEST_RESET_PC 0xffffffff80011100 +#define KVM_MAX_VCPUS 64 #endif #ifdef CONFIG_SUBARCH_C4 #define VPN_BITS 10 #define GUEST_RESET_PC 0xfff0000000011002 +#define KVM_MAX_VCPUS 256 #endif #define VPN_FIRST_VERSION (1UL << VPN_BITS) #define VPN_MASK ((1UL << VPN_BITS) - 1) #define VPN_SHIFT (64 - VPN_BITS) -#define KVM_MAX_VCPUS 64 #define KVM_INTERNAL_MEM_SLOTS (KVM_MEM_SLOTS_NUM - 512) #define KVM_HALT_POLL_NS_DEFAULT 0 -- Gitee From 7401f8eba153061434a4afa26b6844aaaeba934d Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Wed, 29 May 2024 10:56:03 +0800 Subject: [PATCH 260/524] sw64: add cpu hotplug support for C4 guest Qemu emulate IOR:VT_ONLINE_CPU to pass current online vCPUs to guest, which is used to initialize cpu_offline mask for C4 guest. Add sunway_cpu_device_add() and sunway_cpu_device_del() to sunway-ged driver to support cpu hotplug for C4 guest. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uncore_io_junzhang.h | 1 + arch/sw_64/kernel/acpi.c | 12 ++++++ arch/sw_64/kernel/smp.c | 11 +++++ drivers/misc/sunway-ged.c | 48 ++++++++++++++++++++- 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index 770af3907b5c..f553d2f61e64 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -100,6 +100,7 @@ enum { /* SPBU CSR */ enum { SMP_INFO = SPBU_BASE | 0x80UL, + VT_ONLINE_CPU = SPBU_BASE | 0x100UL, INIT_CTL = SPBU_BASE | 0x680UL, CORE_ONLINE = SPBU_BASE | 0x780UL, DLI_RLTD_FAULT = SPBU_BASE | 0x980UL, diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 990db687657e..23b5a90cc7b2 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -331,6 +332,17 @@ static int __init acpi_process_madt_sw_cintc(void) init_cpu_possible(cpu_none_mask); init_cpu_present(cpu_none_mask); +#ifdef CONFIG_SUBARCH_C4 + /* Set cpu_offline mask */ + if (is_guest_or_emul()) { + int vt_smp_cpu_num; + + vt_smp_cpu_num = sw64_io_read(0, VT_ONLINE_CPU); + for (i = vt_smp_cpu_num; i < KVM_MAX_VCPUS; i++) + cpumask_set_cpu(i, &cpu_offline); + } +#endif + /* Parse SW CINTC entries one by one */ ret = acpi_table_parse_madt(ACPI_MADT_TYPE_SW_CINTC, acpi_parse_sw_cintc, 0); diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 3696d9f9f657..ff92dc4cbd19 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -316,6 +317,15 @@ static int __init fdt_setup_smp(void) init_cpu_possible(cpu_none_mask); init_cpu_present(cpu_none_mask); +#ifdef CONFIG_SUBARCH_C4 + if (is_guest_or_emul()) { + int vt_smp_cpu_num; + + vt_smp_cpu_num = sw64_io_read(0, VT_ONLINE_CPU); + for (i = vt_smp_cpu_num; i < KVM_MAX_VCPUS; i++) + cpumask_set_cpu(i, &cpu_offline); + } +#endif while ((dn = of_find_node_by_type(dn, "cpu"))) { if (!of_device_is_available(dn)) { pr_info("OF: Core is not available\n"); @@ -833,6 +843,7 @@ void arch_cpu_idle_dead(void) if (is_in_guest()) { hcall(HCALL_SET_CLOCKEVENT, 0, 0, 0); hcall(HCALL_STOP, 0, 0, 0); + return; } else { wrtimer(0); } diff --git a/drivers/misc/sunway-ged.c b/drivers/misc/sunway-ged.c index f2fda9f0dcb0..c59d377ec8e5 100644 --- a/drivers/misc/sunway-ged.c +++ b/drivers/misc/sunway-ged.c @@ -18,11 +18,17 @@ #define OFFSET_STATUS 16 #define OFFSET_SLOT 24 +#define OFFSET_CPUID 32 + #define OFFSET_NODE 40 /* Memory hotplug event */ #define SUNWAY_MEMHOTPLUG_ADD 0x1 #define SUNWAY_MEMHOTPLUG_REMOVE 0x2 +/* Cpu hotplug event */ +#define SUNWAY_CPUHOTPLUG_ADD 0x4 +#define SUNWAY_CPUHOTPLUG_REMOVE 0x8 + struct sunway_memory_device { struct sunway_ged_device *device; @@ -171,12 +177,52 @@ static int sunway_memory_device_add(struct sunway_ged_device *device) return 1; } +static int sunway_cpu_device_add(struct sunway_ged_device *device) +{ + struct device *dev; + int cpuid; + + cpuid = readq(device->membase + OFFSET_CPUID); + + if (!device) + return -EINVAL; + set_cpu_present(cpuid, true); + dev = get_cpu_device(cpuid); + if (device_attach(dev) >= 0) + return 1; + dev_err(dev, "Processor driver could not be attached\n"); + set_cpu_present(cpuid, false); + return 0; +} + +static void sunway_cpu_device_del(struct sunway_ged_device *device) +{ + struct device *dev; + int cpuid; + + cpuid = readq(device->membase + OFFSET_CPUID); + + if (!device) + return; + set_cpu_present(cpuid, false); + dev = get_cpu_device(cpuid); + device_release_driver(dev); + + writeq(cpuid, device->membase + OFFSET_CPUID); +} + static irqreturn_t sunwayged_ist(int irq, void *data) { struct sunway_ged_device *sunwayged_dev = data; unsigned int status; - status = readl(sunwayged_dev->membase + OFFSET_STATUS); + status = readq(sunwayged_dev->membase + OFFSET_STATUS); + /* through IO status to add or remove cpu */ + if (status & SUNWAY_CPUHOTPLUG_ADD) + sunway_cpu_device_add(sunwayged_dev); + + if (status & SUNWAY_CPUHOTPLUG_REMOVE) + sunway_cpu_device_del(sunwayged_dev); /* through IO status to add or remove memory device */ if (status & SUNWAY_MEMHOTPLUG_ADD) -- Gitee From 296f3d02010b8a908b33060cc960a915bcb7f742 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 28 May 2024 13:59:12 +0800 Subject: [PATCH 261/524] sw64: Kconfig: activate ARCH_KEEP_MEMBLOCK This patch activates ARCH_KEEP_MEMBLOCK which add new memblock region within a NUMA node to 'memory' type. It fixes potential offnode page_structs warnings when new memory is added and used for page_structs. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 9465fe569608..d941bf5cc2e9 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -44,6 +44,7 @@ config SW64 select ARCH_INLINE_WRITE_UNLOCK_BH select ARCH_INLINE_WRITE_UNLOCK_IRQ select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE + select ARCH_KEEP_MEMBLOCK select ARCH_NO_PREEMPT select ARCH_SUPPORTS_ACPI select ARCH_SUPPORTS_ATOMIC_RMW -- Gitee From 6b963fffc1dc71993376a5219ebe41c08c800e0e Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 28 May 2024 15:13:10 +0800 Subject: [PATCH 262/524] sw64: remove unused sunway_memory_get_meminfo() This function is no longer used, so remove it. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/misc/sunway-ged.c | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/drivers/misc/sunway-ged.c b/drivers/misc/sunway-ged.c index c59d377ec8e5..7d0b6f7fc682 100644 --- a/drivers/misc/sunway-ged.c +++ b/drivers/misc/sunway-ged.c @@ -95,24 +95,6 @@ static int sunway_memory_enable_device(struct sunway_memory_device *mem_device) return 0; } -static int sunway_memory_get_meminfo(struct sunway_memory_device *mem_device) -{ - struct sunway_ged_device *geddev; - - if (!mem_device) - return -EINVAL; - - if (mem_device->enabled) - return 0; - - geddev = mem_device->device; - - mem_device->start_addr = readq(geddev->membase + OFFSET_START_ADDR); - mem_device->length = readq(geddev->membase + OFFSET_LENGTH); - - return 0; -} - static void sunway_memory_device_remove(struct sunway_ged_device *device) { struct sunway_memory_device *mem_dev, *n; -- Gitee From 2c2fa86359ee6e2ed9175ad1d2b9d872cc84143b Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 28 May 2024 15:32:34 +0800 Subject: [PATCH 263/524] sw64: fix compile error with CONFIG_SUNWAY_GED=m Fix the following errors: ERROR: modpost: "unlock_device_hotplug" [drivers/misc/sunway-ged.ko] undefined! ERROR: modpost: "__add_memory" [drivers/misc/sunway-ged.ko] undefined! ERROR: modpost: "lock_device_hotplug" [drivers/misc/sunway-ged.ko] undefined! Fixes: 51f31c3271e5 ("sw64: add memhotplug support for guest os") Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/misc/sunway-ged.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/misc/sunway-ged.c b/drivers/misc/sunway-ged.c index 7d0b6f7fc682..e2cd2218518b 100644 --- a/drivers/misc/sunway-ged.c +++ b/drivers/misc/sunway-ged.c @@ -67,10 +67,7 @@ static int sunway_memory_enable_device(struct sunway_memory_device *mem_device) if (!mem_device->length) goto out; - lock_device_hotplug(); - /* suppose node = 0, fix me! */ - result = __add_memory(mem_device->node, mem_device->start_addr, mem_device->length, MHP_NONE); - unlock_device_hotplug(); + result = add_memory(mem_device->node, mem_device->start_addr, mem_device->length, MHP_NONE); /* * If the memory block has been used by the kernel, add_memory() * returns -EEXIST. If add_memory() returns the other error, it -- Gitee From b3d9c3aa9e4936c52007ec9593677a010e6307be Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 28 May 2024 13:39:20 +0800 Subject: [PATCH 264/524] sw64: add dependency for sunway-ged driver Both memory hotplug and cpu hotplug are implemented through sunway-ged driver, so make SUNWAY_GED depend on MEMORY_HOTPLUG || HOTPLUG_CPU. On ther other hand, cpu hotplug and memory cpuplug are independt, so necessary #if guards have to be added to wrap corresponding codes. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/misc/Kconfig | 7 ++++--- drivers/misc/sunway-ged.c | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 1e9def44eb09..2916a2ac3804 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -377,12 +377,13 @@ config HMC6352 providing configuration and heading data via sysfs. config SUNWAY_GED - tristate "sunway generic device driver for memhotplug" + tristate "sunway generic device driver for memhotplug and cpuhotplug" depends on SW64 - depends on MEMORY_HOTPLUG + depends on MEMORY_HOTPLUG || HOTPLUG_CPU help This driver provides support for sunway generic device driver for - memhotplug, providing configuration and heading data via sysfs. + memhotplug and cpuhotplug, providing configuration and heading + data via sysfs. config DS1682 tristate "Dallas DS1682 Total Elapsed Time Recorder with Alarm" diff --git a/drivers/misc/sunway-ged.c b/drivers/misc/sunway-ged.c index e2cd2218518b..14d7ed3d3d92 100644 --- a/drivers/misc/sunway-ged.c +++ b/drivers/misc/sunway-ged.c @@ -50,6 +50,7 @@ struct sunway_ged_device { struct list_head dev_list; }; +#ifdef CONFIG_MEMORY_HOTPLUG static int sunway_memory_enable_device(struct sunway_memory_device *mem_device) { int num_enabled = 0; @@ -155,7 +156,9 @@ static int sunway_memory_device_add(struct sunway_ged_device *device) return 1; } +#endif +#ifdef CONFIG_HOTPLUG_CPU static int sunway_cpu_device_add(struct sunway_ged_device *device) { struct device *dev; @@ -189,6 +192,7 @@ static void sunway_cpu_device_del(struct sunway_ged_device *device) writeq(cpuid, device->membase + OFFSET_CPUID); } +#endif static irqreturn_t sunwayged_ist(int irq, void *data) { @@ -196,19 +200,23 @@ static irqreturn_t sunwayged_ist(int irq, void *data) unsigned int status; status = readq(sunwayged_dev->membase + OFFSET_STATUS); +#ifdef CONFIG_HOTPLUG_CPU /* through IO status to add or remove cpu */ if (status & SUNWAY_CPUHOTPLUG_ADD) sunway_cpu_device_add(sunwayged_dev); if (status & SUNWAY_CPUHOTPLUG_REMOVE) sunway_cpu_device_del(sunwayged_dev); +#endif +#ifdef CONFIG_MEMORY_HOTPLUG /* through IO status to add or remove memory device */ if (status & SUNWAY_MEMHOTPLUG_ADD) sunway_memory_device_add(sunwayged_dev); if (status & SUNWAY_MEMHOTPLUG_REMOVE) sunway_memory_device_remove(sunwayged_dev); +#endif return IRQ_HANDLED; } -- Gitee From 4c7231505d3a226225fd8c52efa95688c8f684c1 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Mon, 17 Jun 2024 14:37:46 +0800 Subject: [PATCH 265/524] sw64: kvm: fix build failure about KVM This patch: - built in kvm_cma.c to fix: arch/sw_64/mm/init.o: In function `sw64_kvm_reserve': (.init.text+0x424): undefined reference to `kvm_cma_declare_contiguous' - use Makefile.kvm for common files to avoid: virt/kvm/kvm_main.o: In function `kvm_vcpu_stats_read': (.text+0x3da4): undefined reference to `kvm_stats_read' Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kvm/Makefile b/arch/sw_64/kvm/Makefile index 8f594155f352..40f700f1a657 100644 --- a/arch/sw_64/kvm/Makefile +++ b/arch/sw_64/kvm/Makefile @@ -9,8 +9,7 @@ include $(srctree)/virt/kvm/Makefile.kvm obj-$(CONFIG_KVM) += kvm.o -kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o \ - sw64.o emulate.o mmio.o kvm_timer.o handle_exit.o perf.o +kvm-y += sw64.o emulate.o mmio.o kvm_timer.o handle_exit.o perf.o -kvm-$(CONFIG_SUBARCH_C3B) += kvm_core3.o entry_core3.o +kvm-$(CONFIG_SUBARCH_C3B) += kvm_core3.o entry_core3.o kvm_cma.o kvm-$(CONFIG_SUBARCH_C4) += kvm_core4.o mmu.o entry_core4.o -- Gitee From 46a7f0e035535bb04948d9ab135e99315eb6ee12 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Mon, 17 Jun 2024 15:02:19 +0800 Subject: [PATCH 266/524] sw64: kvm: fix link error with CONFIG_KVM=m Move kvm_cma.c to the arch/sw_64/kernel/ directory to fix: arch/sw_64/kvm/kvm_cma.o: In function `kvm_cma_init_reserved_areas': (.init.text+0x0): multiple definition of `init_module' arch/sw_64/kvm/kvm_core3.o:(.init.text+0x68): first defined here scripts/Makefile.build:438: recipe for target 'arch/sw_64/kvm/kvm.o' failed make[4]: *** [arch/sw_64/kvm/kvm.o] Error 1 Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/Makefile | 4 ++++ arch/sw_64/{kvm => kernel}/kvm_cma.c | 0 arch/sw_64/kvm/Makefile | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) rename arch/sw_64/{kvm => kernel}/kvm_cma.c (100%) diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 57124b8546ec..b5756003776f 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -40,6 +40,10 @@ ifndef CONFIG_PCI obj-y += pci-noop.o endif +ifdef CONFIG_KVM +obj-$(CONFIG_SUBARCH_C3B) += kvm_cma.o +endif + # Core logic support obj-$(CONFIG_SW64_CPUAUTOPLUG) += cpuautoplug.o diff --git a/arch/sw_64/kvm/kvm_cma.c b/arch/sw_64/kernel/kvm_cma.c similarity index 100% rename from arch/sw_64/kvm/kvm_cma.c rename to arch/sw_64/kernel/kvm_cma.c diff --git a/arch/sw_64/kvm/Makefile b/arch/sw_64/kvm/Makefile index 40f700f1a657..530170e9167d 100644 --- a/arch/sw_64/kvm/Makefile +++ b/arch/sw_64/kvm/Makefile @@ -11,5 +11,5 @@ obj-$(CONFIG_KVM) += kvm.o kvm-y += sw64.o emulate.o mmio.o kvm_timer.o handle_exit.o perf.o -kvm-$(CONFIG_SUBARCH_C3B) += kvm_core3.o entry_core3.o kvm_cma.o +kvm-$(CONFIG_SUBARCH_C3B) += kvm_core3.o entry_core3.o kvm-$(CONFIG_SUBARCH_C4) += kvm_core4.o mmu.o entry_core4.o -- Gitee From 958e02170f231f3a2460d032447c997c9cc76c6a Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 18 Jun 2024 14:10:02 +0800 Subject: [PATCH 267/524] sw64: kvm: fix NULL poniter access in kvm_arch_prepare_memory_region 'struct kvm_memory_slot *new' in kvm_arch_prepare_memory_region() should not be accessed if there is no memory slot creating, or it will result in kernel Oops due to the null *new. This patch also delete some unused codes. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/kvm_core3.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c index b9a5019ee9c0..364cabdd6407 100644 --- a/arch/sw_64/kvm/kvm_core3.c +++ b/arch/sw_64/kvm/kvm_core3.c @@ -113,14 +113,13 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, unsigned long ret; size_t size; - mem->flags = new->flags; + if (change == KVM_MR_FLAGS_ONLY || change == KVM_MR_DELETE) + return 0; + mem->guest_phys_addr = ((new->base_gfn) << PAGE_SHIFT); mem->memory_size = ((new->npages) << PAGE_SHIFT); mem->userspace_addr = new->userspace_addr; - if (change == KVM_MR_FLAGS_ONLY || change == KVM_MR_DELETE) - return 0; - if (test_bit(IO_MARK_BIT, (unsigned long *)(&(mem->guest_phys_addr)))) return 0; -- Gitee From d8d7f5ddc1b257fc13ef765850b760a16c7e1a6e Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 18 Jun 2024 15:18:56 +0800 Subject: [PATCH 268/524] sw64: kvm: fix the handling when the live migration fails To ensure that the source guest can still run properly when live migration fails, it is necessary to clear migration_mark and do tlb flush during kvm_arch_commit_memory_region(). Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/kvm_core3.c | 9 ++++++++- arch/sw_64/kvm/mmu.c | 5 +++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c index 364cabdd6407..355d7697c535 100644 --- a/arch/sw_64/kvm/kvm_core3.c +++ b/arch/sw_64/kvm/kvm_core3.c @@ -208,9 +208,16 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, * memory slot is write protected. */ + /* If dirty logging has been stopped, clear migration_mark for now. */ + if ((change == KVM_MR_FLAGS_ONLY) && (old->flags & KVM_MEM_LOG_DIRTY_PAGES) + && (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))) { + kvm_mark_migration(kvm, 0); + return; + } /* If it's the first time dirty logging, flush all vcpu tlbs. */ - if ((change == KVM_MR_FLAGS_ONLY) && (new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + if ((change == KVM_MR_FLAGS_ONLY) && (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES)) + && (new->flags & KVM_MEM_LOG_DIRTY_PAGES)) kvm_mark_migration(kvm, 1); } diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 9b4c770fa2d9..446ee845c6d3 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -528,8 +528,9 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, kvm_mark_migration(kvm, 1); kvm_mmu_wp_memory_region(kvm, new->id); } - /* If dirty logging has been stopped, do nothing for now. */ - if ((change != KVM_MR_DELETE) + + /* If dirty logging has been stopped, clear migration mark for now. */ + if ((change == KVM_MR_FLAGS_ONLY) && (old->flags & KVM_MEM_LOG_DIRTY_PAGES) && (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))) { kvm_mark_migration(kvm, 0); -- Gitee From eecb68df3f1d16178fe1c7b61ff725bc1dd99cd8 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 25 Jun 2024 11:04:57 +0800 Subject: [PATCH 269/524] sw64: kvm: use get_page() instead of kvm_get_pfn() When mapping a THP, it can be guaranteed that the page isn't reserved and safely avoid the kvm_is_reserved_pfn() call. And, kvm_get_pfn() has been removed in commit 36c3ce6c0d03 ("KVM: Get rid of kvm_get_pfn()"), so replace it with get_page(pfn_to_page()). Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 446ee845c6d3..24fb00282cb1 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -1098,7 +1098,7 @@ transparent_hugepage_adjust(struct kvm_memory_slot *memslot, *gpap &= PMD_MASK; kvm_release_pfn_clean(pfn); pfn &= ~(PTRS_PER_PMD - 1); - kvm_get_pfn(pfn); + get_page(pfn_to_page(pfn)); *pfnp = pfn; return PMD_SIZE; } -- Gitee From fb4d4b5e8a3bac40f48d93cc698033c6a583d56b Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 25 Jun 2024 13:37:20 +0800 Subject: [PATCH 270/524] sw64: kvm: fix compiler errors caused by mismatched parameters Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/kvm_core4.c | 6 +++--- arch/sw_64/kvm/mmu.c | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c index a9d9326072e4..6d518f28950c 100644 --- a/arch/sw_64/kvm/kvm_core4.c +++ b/arch/sw_64/kvm/kvm_core4.c @@ -96,9 +96,9 @@ long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) } int kvm_arch_prepare_memory_region(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, - enum kvm_mr_change change) + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) { return 0; } diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 24fb00282cb1..41706105468a 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -295,14 +295,14 @@ void apt_unmap_vm(struct kvm *kvm) { struct kvm_memslots *slots; struct kvm_memory_slot *memslot; - int idx; + int idx, bkt; idx = srcu_read_lock(&kvm->srcu); down_read(¤t->mm->mmap_lock); spin_lock(&kvm->mmu_lock); slots = kvm_memslots(kvm); - kvm_for_each_memslot(memslot, slots) + kvm_for_each_memslot(memslot, bkt, slots) apt_unmap_memslot(kvm, memslot); spin_unlock(&kvm->mmu_lock); up_read(¤t->mm->mmap_lock); @@ -1373,11 +1373,12 @@ static int handle_hva_to_gpa(struct kvm *kvm, unsigned long start, unsigned long struct kvm_memslots *slots; struct kvm_memory_slot *memslot; int ret = 0; + int bkt; slots = kvm_memslots(kvm); /* we only care about the pages that the guest sees */ - kvm_for_each_memslot(memslot, slots) { + kvm_for_each_memslot(memslot, bkt, slots) { unsigned long hva_start, hva_end; gfn_t gpa; -- Gitee From 09c12fd8e9890258dcd3d90bc7c991f847b95758 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 25 Jun 2024 13:53:06 +0800 Subject: [PATCH 271/524] sw64: kvm: rename mmu_notifier_* to mmu_invalidate_* Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 41706105468a..ff51acce5b4e 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -1085,7 +1085,7 @@ transparent_hugepage_adjust(struct kvm_memory_slot *memslot, * THP doesn't start to split while we are adjusting the * refcounts. * - * We are sure this doesn't happen, because mmu_notifier_retry + * We are sure this doesn't happen, because mmu_invalidate_retry * was successful and we are holding the mmu_lock, so if this * THP is trying to split, it will be blocked in the mmu * notifier before touching any of the pages, specifically @@ -1168,9 +1168,9 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, if (ret) return ret; - mmu_seq = vcpu->kvm->mmu_notifier_seq; + mmu_seq = vcpu->kvm->mmu_invalidate_seq; /* - * Ensure the read of mmu_notifier_seq happens before we call + * Ensure the read of mmu_invalidate_seq happens before we call * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk * the page we just got a reference to gets unmapped before we have a * chance to grab the mmu_lock, which ensure that if the page gets @@ -1205,7 +1205,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, } spin_lock(&kvm->mmu_lock); - if (mmu_notifier_retry(kvm, mmu_seq)) + if (mmu_invalidate_retry(kvm, mmu_seq)) goto out_unlock; /* -- Gitee From 89cb445c61c90f2d768015632f38ce3e83ed663c Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 25 Jun 2024 14:46:57 +0800 Subject: [PATCH 272/524] sw64: kvm: convert to the gfn-based MMU notifier callbacks Commit 3039bcc74498 ("KVM: Move x86's MMU notifier memslot walkers to generic code") has done the hva->gfn lookup in common code, so move sw64 to the gfn-base MMU notifier APIs too. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_host.h | 4 - arch/sw_64/kvm/mmu.c | 125 +++++++++--------------------- arch/sw_64/kvm/trace.h | 66 ---------------- 3 files changed, 35 insertions(+), 160 deletions(-) diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index 66434307523d..cce39b0289e5 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -157,10 +157,6 @@ struct kvm_vcpu_stat { #ifdef CONFIG_SUBARCH_C4 #define KVM_ARCH_WANT_MMU_NOTIFIER #endif -int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); -int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable); -int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); -int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat); void check_vcpu_requests(struct kvm_vcpu *vcpu); diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index ff51acce5b4e..5f5e7aca1622 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -1174,7 +1174,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk * the page we just got a reference to gets unmapped before we have a * chance to grab the mmu_lock, which ensure that if the page gets - * unmapped afterwards, the call to kvm_unmap_hva will take it away + * unmapped afterwards, the call to kvm_unmap_gfn will take it away * from us again properly. This smp_rmb() interacts with the smp_wmb() * in kvm_mmu_notifier_invalidate_. */ @@ -1366,72 +1366,38 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, return ret; } #endif -static int handle_hva_to_gpa(struct kvm *kvm, unsigned long start, unsigned long end, - int (*handler)(struct kvm *kvm, gpa_t gpa, u64 size, void *data), - void *data) -{ - struct kvm_memslots *slots; - struct kvm_memory_slot *memslot; - int ret = 0; - int bkt; - - slots = kvm_memslots(kvm); - - /* we only care about the pages that the guest sees */ - kvm_for_each_memslot(memslot, bkt, slots) { - unsigned long hva_start, hva_end; - gfn_t gpa; - hva_start = max(start, memslot->userspace_addr); - hva_end = min(end, memslot->userspace_addr + - (memslot->npages << PAGE_SHIFT)); - if (hva_start >= hva_end) - continue; - - gpa = hva_to_gfn_memslot(hva_start, memslot) << PAGE_SHIFT; - ret |= handler(kvm, gpa, (u64)(hva_end - hva_start), data); - } - - return ret; -} - -static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) -{ - unmap_apt_range(kvm, gpa, size); - return 0; -} - -int kvm_unmap_hva_range(struct kvm *kvm, - unsigned long start, unsigned long end, bool blockable) +bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) { if (!kvm->arch.pgd) - return 0; + return false; - trace_kvm_unmap_hva_range(start, end); - handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL); - return 1; + unmap_apt_range(kvm, range->start << PAGE_SHIFT, + (range->end - range->start) << PAGE_SHIFT); + + return false; } -static int apt_ptep_test_and_clear_young(pte_t *pte) +static bool apt_ptep_test_and_clear_young(pte_t *pte) { if (pte_young(*pte)) { *pte = pte_mkold(*pte); - return 1; + return true; } - return 0; + return false; } -static int apt_pmdp_test_and_clear_young(pmd_t *pmd) +static bool apt_pmdp_test_and_clear_young(pmd_t *pmd) { return apt_ptep_test_and_clear_young((pte_t *)pmd); } -static int apt_pudp_test_and_clear_young(pud_t *pud) +static bool apt_pudp_test_and_clear_young(pud_t *pud) { return apt_ptep_test_and_clear_young((pte_t *)pud); } -static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +static bool kvm_apt_test_clear_young(struct kvm *kvm, gpa_t gpa, u64 size, void *data) { pud_t *pud; pmd_t *pmd; @@ -1439,7 +1405,7 @@ static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); if (!apt_get_leaf_entry(kvm, gpa, &pud, &pmd, &pte)) - return 0; + return false; if (pud) return apt_pudp_test_and_clear_young(pud); @@ -1449,63 +1415,42 @@ static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) return apt_ptep_test_and_clear_young(pte); } -static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { - pud_t *pud; - pmd_t *pmd; - pte_t *pte; + gpa_t gpa = range->start << PAGE_SHIFT; + u64 size = (range->end - range->start) << PAGE_SHIFT; - WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); - if (!apt_get_leaf_entry(kvm, gpa, &pud, &pmd, &pte)) - return 0; - - if (pud) - return apt_pudp_test_and_clear_young(pud); - else if (pmd) - return apt_pmdp_test_and_clear_young(pmd); - else - return apt_ptep_test_and_clear_young(pte); -} - -int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) -{ if (!kvm->arch.pgd) - return 0; - trace_kvm_age_hva(start, end); - return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL); -} + return false; -int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) -{ - if (!kvm->arch.pgd) - return 0; - trace_kvm_test_age_hva(hva); - return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL); + return kvm_apt_test_clear_young(kvm, gpa, size, NULL); } -static int kvm_set_apte_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { - pte_t *pte = (pte_t *)data; + gpa_t gpa = range->start << PAGE_SHIFT; + u64 size = (range->end - range->start) << PAGE_SHIFT; - WARN_ON(size != PAGE_SIZE); + if (!kvm->arch.pgd) + return false; - apt_set_pte(kvm, NULL, gpa, pte, 0); - return 0; + return kvm_apt_test_clear_young(kvm, gpa, size, NULL); } -int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) +bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { - unsigned long end = hva + PAGE_SIZE; - pte_t apt_pte; - + gpa_t gpa = range->start << PAGE_SHIFT; + pte_t apt_pte = range->arg.pte; if (!kvm->arch.pgd) - return 0; + return false; - trace_kvm_set_spte_hva(hva); + WARN_ON(range->end - range->start != 1); - apt_pte = pte_wrprotect(pte); - handle_hva_to_gpa(kvm, hva, end, &kvm_set_apte_handler, &apt_pte); - return 0; + apt_pte = pte_wrprotect(apt_pte); + + apt_set_pte(kvm, NULL, gpa, &apt_pte, 0); + + return false; } /** diff --git a/arch/sw_64/kvm/trace.h b/arch/sw_64/kvm/trace.h index 2b7b1de1e5d5..0e6a75a57f11 100644 --- a/arch/sw_64/kvm/trace.h +++ b/arch/sw_64/kvm/trace.h @@ -132,72 +132,6 @@ TRACE_EVENT(kvm_mmio_emulate, __entry->vcpu_pc, __entry->instr, __entry->cpsr) ); -TRACE_EVENT(kvm_unmap_hva_range, - TP_PROTO(unsigned long start, unsigned long end), - TP_ARGS(start, end), - - TP_STRUCT__entry( - __field(unsigned long, start) - __field(unsigned long, end) - ), - - TP_fast_assign( - __entry->start = start; - __entry->end = end; - ), - - TP_printk("mmu notifier unmap range: %#016lx -- %#016lx", - __entry->start, __entry->end) -); - -TRACE_EVENT(kvm_set_spte_hva, - TP_PROTO(unsigned long hva), - TP_ARGS(hva), - - TP_STRUCT__entry( - __field(unsigned long, hva) - ), - - TP_fast_assign( - __entry->hva = hva; - ), - - TP_printk("mmu notifier set pte hva: %#016lx", __entry->hva) -); - -TRACE_EVENT(kvm_age_hva, - TP_PROTO(unsigned long start, unsigned long end), - TP_ARGS(start, end), - - TP_STRUCT__entry( - __field(unsigned long, start) - __field(unsigned long, end) - ), - - TP_fast_assign( - __entry->start = start; - __entry->end = end; - ), - - TP_printk("mmu notifier age hva: %#016lx -- %#016lx", - __entry->start, __entry->end) -); - -TRACE_EVENT(kvm_test_age_hva, - TP_PROTO(unsigned long hva), - TP_ARGS(hva), - - TP_STRUCT__entry( - __field(unsigned long, hva) - ), - - TP_fast_assign( - __entry->hva = hva; - ), - - TP_printk("mmu notifier test age hva: %#016lx", __entry->hva) -); - TRACE_EVENT(kvm_set_way_flush, TP_PROTO(unsigned long vcpu_pc, bool cache), TP_ARGS(vcpu_pc, cache), -- Gitee From 420af4aadb5320a940b2dcc4888b2fbde783cb87 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Fri, 28 Jun 2024 16:13:54 +0800 Subject: [PATCH 273/524] sw64: kvm: walk host page tables to find THP mapping size Since kvm_is_transparent_hugepage() has been removed in commit 205d76ff0684 ("KVM: Remove kvm_is_transparent_hugepage() and PageTransCompoundMap()"). This patch walks the host page tables to identify THP mappings instead of relying solely on the metadata in struct page to get the mapping size like other archs. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 56 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 5f5e7aca1622..bb4d35c5762a 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -1063,8 +1063,49 @@ static int apt_set_pud_huge(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, return 0; } +static int get_user_mapping_size(struct kvm *kvm, u64 hva) +{ + unsigned long flags; + int pgsize = PAGE_SIZE; + pgd_t pgd; + p4d_t p4d; + pud_t pud; + pmd_t pmd; + + /* + * Disable IRQs to prevent concurrent tear down of host page tables, + * e.g. if the primary MMU promotes a P*D to a huge page and then frees + * the original page table. + */ + local_irq_save(flags); + + pgd = *(pgd_offset(kvm->mm, hva)); + if (pgd_none(pgd)) + goto out; + + p4d = *(p4d_offset(&pgd, hva)); + if (p4d_none(p4d) || !p4d_present(p4d)) + goto out; + + pud = *(pud_offset(&p4d, hva)); + if (pud_none(pud) || !pud_present(pud)) + goto out; + + pmd = *(pmd_offset(&pud, hva)); + if (pmd_none(pmd) || !pmd_present(pmd)) + goto out; + + if (pmd_trans_huge(pmd)) + pgsize = PMD_SIZE; + +out: + local_irq_restore(flags); + return pgsize; + +} + static unsigned long -transparent_hugepage_adjust(struct kvm_memory_slot *memslot, +transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, unsigned long hva, kvm_pfn_t *pfnp, phys_addr_t *gpap) { @@ -1075,8 +1116,14 @@ transparent_hugepage_adjust(struct kvm_memory_slot *memslot, * sure that the HVA and IPA are sufficiently aligned and that the * block map is contained within the memslot. */ - if (kvm_is_transparent_hugepage(pfn) && - fault_supports_apt_huge_mapping(memslot, hva, PMD_SIZE)) { + if (fault_supports_apt_huge_mapping(memslot, hva, PMD_SIZE)) { + int pgsz = get_user_mapping_size(kvm, hva); + + if (pgsz < 0) + return pgsz; + + if (pgsz < PMD_SIZE) + return PAGE_SIZE; /* * The address we faulted on is backed by a transparent huge * page. However, because we map the compound huge page and @@ -1181,6 +1228,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, smp_rmb(); pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable); + if (pfn == KVM_PFN_ERR_HWPOISON) { kvm_send_hwpoison_signal(hva, vma); return 0; @@ -1213,7 +1261,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, * backed by a THP and thus use block mapping if possible. */ if (vma_pagesize == PAGE_SIZE && !force_pte) { - vma_pagesize = transparent_hugepage_adjust(memslot, hva, + vma_pagesize = transparent_hugepage_adjust(kvm, memslot, hva, &pfn, &fault_gpa); } -- Gitee From a0e901b2caf4f3869247c4cc6bbd2cd55c84eee8 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 9 Jul 2024 10:00:15 +0800 Subject: [PATCH 274/524] sw64: fix guest memory and cpu hotplug function This patch: - Remove node information in memory hot unplugging because nid parameter has been removed from remove_memory() and friends in commit e1c158e49566. - Add header file cpu.h to sunway-ged.c so that we can use get_cpu_device() function when we hotplug cpu. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/misc/sunway-ged.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/misc/sunway-ged.c b/drivers/misc/sunway-ged.c index 14d7ed3d3d92..4a0d6fc736ef 100644 --- a/drivers/misc/sunway-ged.c +++ b/drivers/misc/sunway-ged.c @@ -3,6 +3,7 @@ /* Generic Event Device for ACPI. */ #include +#include #include #include #include @@ -17,9 +18,7 @@ #define OFFSET_LENGTH 8 #define OFFSET_STATUS 16 #define OFFSET_SLOT 24 - #define OFFSET_CPUID 32 - #define OFFSET_NODE 40 /* Memory hotplug event */ @@ -96,7 +95,7 @@ static int sunway_memory_enable_device(struct sunway_memory_device *mem_device) static void sunway_memory_device_remove(struct sunway_ged_device *device) { struct sunway_memory_device *mem_dev, *n; - unsigned long start_addr, length, slot, node; + unsigned long start_addr, length, slot; if (!device) return; @@ -104,7 +103,6 @@ static void sunway_memory_device_remove(struct sunway_ged_device *device) start_addr = readq(device->membase + OFFSET_START_ADDR); length = readq(device->membase + OFFSET_LENGTH); slot = readq(device->membase + OFFSET_SLOT); - node = readq(device->membase + OFFSET_NODE); list_for_each_entry_safe(mem_dev, n, &device->dev_list, list) { if (!mem_dev->enabled) @@ -112,8 +110,7 @@ static void sunway_memory_device_remove(struct sunway_ged_device *device) if ((start_addr == mem_dev->start_addr) && (length == mem_dev->length)) { - /* suppose node = 0, fix me! */ - remove_memory(node, start_addr, length); + remove_memory(start_addr, length); list_del(&mem_dev->list); kfree(mem_dev); } -- Gitee From d86bcf928bb7061fd0847146064f7c563ca59c84 Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Mon, 17 Jun 2024 14:18:03 +0800 Subject: [PATCH 275/524] sw64: kvm: fix mmio GPA for C4 The original value of hargs->arg1 is from CSR:DVA, but io_mem_abort() will treat it as mmio GPA, so fix it with CSR:EXC_GPA. Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index bb4d35c5762a..a9bf514b81fd 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -1400,6 +1400,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, */ if (hva == KVM_HVA_ERR_BAD) { + hargs->arg1 = fault_gpa | (hargs->arg1 & 0x1fffUL); ret = io_mem_abort(vcpu, run, hargs); goto out_unlock; } -- Gitee From 2cbb2c2aa239deb1a5c6089c19a783abaf5b489d Mon Sep 17 00:00:00 2001 From: Xu Chenjiao Date: Wed, 10 Jul 2024 15:28:17 +0000 Subject: [PATCH 276/524] sw64: adapt the modifications from the upstream for pciehp The relevant interfaces from the upstream have changed. Therefore, this patch adapts the modifications from the upstream for the sunway pcie hotplug driver. Signed-off-by: Xu Chenjiao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 1 + drivers/pci/controller/pci-sunway.c | 7 +- drivers/pci/hotplug/sunway_pciehp.h | 10 +- drivers/pci/hotplug/sunway_pciehp_core.c | 56 ++++++++-- drivers/pci/hotplug/sunway_pciehp_ctrl.c | 25 +++-- drivers/pci/hotplug/sunway_pciehp_hpc.c | 131 +++++++++++++++++------ drivers/pci/hotplug/sunway_pciehp_pci.c | 15 +++ 7 files changed, 193 insertions(+), 52 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 1bd2123362f0..62fee7ba5dd6 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -65,6 +65,7 @@ struct pci_controller { unsigned int need_domain_info; bool iommu_enable; struct sunway_iommu *pci_iommu; + bool hotplug_enable; int first_busno; int last_busno; int self_busno; diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 139c9515fe6f..59962f9a6158 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -327,6 +327,8 @@ static void hose_init(struct pci_controller *hose) hose->first_busno = hose->self_busno = hose->busn_space->start; hose->last_busno = hose->busn_space->end; + hose->hotplug_enable = true; + if (is_in_host()) { if (IS_ENABLED(CONFIG_PCI_MSI)) memset(hose->piu_msiconfig, 0, 256/8); @@ -646,6 +648,7 @@ enum pci_props { PROP_PIU_IOR1_BASE, PROP_RC_INDEX, PROP_PCIE_IO_BASE, + PROP_HOTPLUG_ENABLE, PROP_NUM }; @@ -658,7 +661,8 @@ const char *prop_names[PROP_NUM] = { "sw64,piu_ior0_base", "sw64,piu_ior1_base", "sw64,rc_index", - "sw64,pcie_io_base" + "sw64,pcie_io_base", + "sw64,hot_plug_slot_enable" }; static int sw64_pci_prepare_controller(struct pci_controller *hose, @@ -681,6 +685,7 @@ static int sw64_pci_prepare_controller(struct pci_controller *hose, hose->iommu_enable = false; hose->index = props[PROP_RC_INDEX]; + hose->hotplug_enable = props[PROP_HOTPLUG_ENABLE]; hose->sparse_mem_base = 0; hose->sparse_io_base = 0; diff --git a/drivers/pci/hotplug/sunway_pciehp.h b/drivers/pci/hotplug/sunway_pciehp.h index d1addc487d07..2feecbcc5366 100644 --- a/drivers/pci/hotplug/sunway_pciehp.h +++ b/drivers/pci/hotplug/sunway_pciehp.h @@ -54,6 +54,9 @@ struct saved_piu_space { /** * struct controller - PCIe hotplug controller * @slot_cap: cached copy of the Slot Capabilities register + * @inband_presence_disabled: In-Band Presence Detect Disable supported by + * controller and disabled per spec recommendation (PCIe r5.0, appendix I + * implementation note) * @slot_ctrl: cached copy of the Slot Control register * @ctrl_lock: serializes writes to the Slot Control register * @cmd_started: jiffies when the Slot Control register was last written; @@ -79,6 +82,8 @@ struct saved_piu_space { * @reset_lock: prevents access to the Data Link Layer Link Active bit in the * Link Status register and to the Presence Detect State bit in the Slot * Status register during a slot reset which may cause them to flap + * @depth: Number of additional hotplug ports in the path to the root bus, + * used as lock subclass for @reset_lock * @ist_running: flag to keep user request waiting while IRQ thread is running * @request_result: result of last user request submitted to the IRQ thread * @requester: wait queue to wake up on completion of user request, @@ -109,6 +114,7 @@ struct controller { struct hotplug_slot hotplug_slot; /* hotplug core interface */ struct rw_semaphore reset_lock; + unsigned int depth; unsigned int ist_running; int request_result; wait_queue_head_t requester; @@ -202,6 +208,8 @@ int sunway_pciehp_link_enable(struct controller *ctrl); int sunway_pciehp_link_disable(struct controller *ctrl); void sunway_pciehp_restore_rc_piu(struct controller *ctrl); +int sunway_pciehp_slot_reset(struct pci_dev *dev); + static inline const char *slot_name(struct controller *ctrl) { return hotplug_slot_name(&ctrl->hotplug_slot); @@ -212,4 +220,4 @@ static inline struct controller *to_ctrl(struct hotplug_slot *hotplug_slot) return container_of(hotplug_slot, struct controller, hotplug_slot); } -#endif /* _PCIEHP_H */ +#endif /* _SUNWAYPCIEHP_H */ diff --git a/drivers/pci/hotplug/sunway_pciehp_core.c b/drivers/pci/hotplug/sunway_pciehp_core.c index 6d155d3a0595..acafc82b3224 100644 --- a/drivers/pci/hotplug/sunway_pciehp_core.c +++ b/drivers/pci/hotplug/sunway_pciehp_core.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include "../pci.h" #include "sunway_pciehp.h" @@ -161,7 +163,7 @@ static void sunway_pciehp_check_presence(struct controller *ctrl) { int occupied; - down_read(&ctrl->reset_lock); + down_read_nested(&ctrl->reset_lock, ctrl->depth); mutex_lock(&ctrl->state_lock); occupied = sunway_pciehp_card_present_or_link_active(ctrl); @@ -241,12 +243,42 @@ static void sunwayhp_remove(struct pci_dev *dev) sunway_pciehp_release_ctrl(ctrl); } -extern struct pci_controller *hose_head, **hose_tail; +static void sunway_hose_hotplug_init(void) +{ + int ret; + struct pci_dev *pdev = NULL; + struct pci_controller *hose; + struct pci_config_window *cfg; + struct acpi_device *adev; + u64 prop_hotplug_enable; + + while ((pdev = pci_get_device(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, pdev))) { + hose = pci_bus_to_pci_controller(pdev->bus); + hose->hotplug_enable = false; + + if (!acpi_disabled) { + cfg = (struct pci_config_window *)pdev->bus->sysdata; + adev = to_acpi_device(cfg->parent); + + ret = fwnode_property_read_u64_array(&adev->fwnode, + "sw64,hot_plug_slot_enable", &prop_hotplug_enable, 1); + + if (ret == 0) + hose->hotplug_enable = prop_hotplug_enable; + } + } +} + static int __init sunway_pciehp_init(void) { int retval; struct pci_dev *pdev = NULL; - struct pci_controller *hose = NULL; + struct pci_controller *hose; + + if (is_guest_or_emul()) { + pr_info(DRIVER_DESC " does not support for VM and emulator.\n"); + return -ENODEV; + } if (is_guest_or_emul()) { pr_info(DRIVER_DESC " does not support for VM and emulator.\n"); @@ -255,8 +287,13 @@ static int __init sunway_pciehp_init(void) pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); - for (hose = hose_head; hose; hose = hose->next) { - pdev = pci_get_device(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, pdev); + sunway_hose_hotplug_init(); + + while ((pdev = pci_get_device(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, pdev))) { + hose = pci_bus_to_pci_controller(pdev->bus); + + if (!hose->hotplug_enable) + continue; retval = sunwayhp_init(pdev); } @@ -267,12 +304,15 @@ static int __init sunway_pciehp_init(void) static void __exit sunway_pciehp_exit(void) { struct pci_dev *pdev = NULL; - struct pci_controller *hose = NULL; + struct pci_controller *hose; pr_info(DRIVER_DESC " version: " DRIVER_VERSION " unloaded\n"); - for (hose = hose_head; hose; hose = hose->next) { - pdev = pci_get_device(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, pdev); + while ((pdev = pci_get_device(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, pdev))) { + hose = pci_bus_to_pci_controller(pdev->bus); + + if (!hose->hotplug_enable) + continue; sunwayhp_remove(pdev); } diff --git a/drivers/pci/hotplug/sunway_pciehp_ctrl.c b/drivers/pci/hotplug/sunway_pciehp_ctrl.c index 52e79bfe7dd6..a1c4da32d0ee 100644 --- a/drivers/pci/hotplug/sunway_pciehp_ctrl.c +++ b/drivers/pci/hotplug/sunway_pciehp_ctrl.c @@ -161,11 +161,11 @@ void sunway_pciehp_handle_button_press(struct controller *ctrl) case ON_STATE: if (ctrl->state == ON_STATE) { ctrl->state = BLINKINGOFF_STATE; - ctrl_info(ctrl, "Slot(%s): Powering off due to button press\n", + ctrl_info(ctrl, "Slot(%s): Button press: will power off in 5 sec\n", slot_name(ctrl)); } else { ctrl->state = BLINKINGON_STATE; - ctrl_info(ctrl, "Slot(%s) Powering on due to button press\n", + ctrl_info(ctrl, "Slot(%s): Button press: will power on in 5 sec\n", slot_name(ctrl)); } /* blink power indicator and turn off attention */ @@ -180,22 +180,23 @@ void sunway_pciehp_handle_button_press(struct controller *ctrl) * press the attention again before the 5 sec. limit * expires to cancel hot-add or hot-remove */ - ctrl_info(ctrl, "Slot(%s): Button cancel\n", slot_name(ctrl)); cancel_delayed_work(&ctrl->button_work); if (ctrl->state == BLINKINGOFF_STATE) { ctrl->state = ON_STATE; sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_ON, PCI_EXP_SLTCTL_ATTN_IND_OFF); + ctrl_info(ctrl, "Slot(%s): Button press: canceling request to power off\n", + slot_name(ctrl)); } else { ctrl->state = OFF_STATE; sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF, PCI_EXP_SLTCTL_ATTN_IND_OFF); + ctrl_info(ctrl, "Slot(%s): Button press: canceling request to power on\n", + slot_name(ctrl)); } - ctrl_info(ctrl, "Slot(%s): Action canceled due to button press\n", - slot_name(ctrl)); break; default: - ctrl_err(ctrl, "Slot(%s): Ignoring invalid state %#x\n", + ctrl_err(ctrl, "Slot(%s): Button press: Ignoring invalid state %#x\n", slot_name(ctrl), ctrl->state); break; } @@ -489,6 +490,14 @@ void sunway_pciehp_handle_presence_or_link_change(struct controller *ctrl, u32 e present = sunway_pciehp_card_present(ctrl); link_active = sunway_pciehp_check_link_active(ctrl); if (present <= 0 && link_active <= 0) { + if (ctrl->state == BLINKINGON_STATE) { + ctrl->state = OFF_STATE; + cancel_delayed_work(&ctrl->button_work); + sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF, + INDICATOR_NOOP); + ctrl_info(ctrl, "Slot(%s): Card not present\n", + slot_name(ctrl)); + } sunway_pciehp_end(ctrl, false); mutex_unlock(&ctrl->state_lock); return; @@ -642,13 +651,11 @@ int sunway_pciehp_sysfs_disable_slot(struct hotplug_slot *hotplug_slot) switch (ctrl->state) { case BLINKINGOFF_STATE: case ON_STATE: - sunway_pciehp_start(hotplug_slot); - mutex_unlock(&ctrl->state_lock); + sunway_pciehp_start(hotplug_slot); wait_event(ctrl->requester, !atomic_read(&ctrl->pending_events) && !ctrl->ist_running); - return ctrl->request_result; case POWEROFF_STATE: ctrl_info(ctrl, "Slot(%s): Already in powering off state\n", diff --git a/drivers/pci/hotplug/sunway_pciehp_hpc.c b/drivers/pci/hotplug/sunway_pciehp_hpc.c index e6559261788d..2bb6d3ff3165 100644 --- a/drivers/pci/hotplug/sunway_pciehp_hpc.c +++ b/drivers/pci/hotplug/sunway_pciehp_hpc.c @@ -87,7 +87,7 @@ static int pcie_poll_cmd(struct controller *ctrl, int timeout) do { pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status); - if (slot_status == (u16) ~0) { + if (PCI_POSSIBLE_ERROR(slot_status)) { ctrl_info(ctrl, "%s: no response from device\n", __func__); return 0; @@ -96,6 +96,8 @@ static int pcie_poll_cmd(struct controller *ctrl, int timeout) if (slot_status & PCI_EXP_SLTSTA_CC) { pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, PCI_EXP_SLTSTA_CC); + ctrl->cmd_busy = 0; + smp_mb(); return 1; } msleep(10); @@ -163,7 +165,7 @@ static void pcie_do_write_cmd(struct controller *ctrl, u16 cmd, pcie_wait_cmd(ctrl); pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); - if (slot_ctrl == (u16) ~0) { + if (PCI_POSSIBLE_ERROR(slot_ctrl)) { ctrl_info(ctrl, "%s: no response from device\n", __func__); goto out; } @@ -233,7 +235,7 @@ int sunway_pciehp_check_link_active(struct controller *ctrl) int ret; ret = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status); - if (ret == PCIBIOS_DEVICE_NOT_FOUND || lnk_status == (u16)~0) + if (ret == PCIBIOS_DEVICE_NOT_FOUND || PCI_POSSIBLE_ERROR(lnk_status)) return -ENODEV; ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA); @@ -380,17 +382,10 @@ int sunway_pciehp_check_link_status(struct controller *ctrl) static int __sunway_pciehp_link_set(struct controller *ctrl, bool enable) { struct pci_dev *pdev = ctrl_dev(ctrl); - u16 lnk_ctrl; - pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &lnk_ctrl); - - if (enable) - lnk_ctrl &= ~PCI_EXP_LNKCTL_LD; - else - lnk_ctrl |= PCI_EXP_LNKCTL_LD; - - pcie_capability_write_word(pdev, PCI_EXP_LNKCTL, lnk_ctrl); - ctrl_dbg(ctrl, "%s: lnk_ctrl = %x\n", __func__, lnk_ctrl); + pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_LD, + enable ? 0 : PCI_EXP_LNKCTL_LD); return 0; } @@ -463,7 +458,6 @@ void sunway_pciehp_get_power_status(struct controller *ctrl, u8 *status) u16 slot_ctrl; pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); - ctrl_dbg(ctrl, "%s: SLOTCTRL %x value read %x\n", __func__, pci_pcie_cap(ctrl->pci_dev) + PCI_EXP_SLTCTL, slot_ctrl); @@ -508,7 +502,7 @@ int sunway_pciehp_card_present(struct controller *ctrl) int ret; ret = pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status); - if (ret == PCIBIOS_DEVICE_NOT_FOUND || slot_status == (u16)~0) + if (ret == PCIBIOS_DEVICE_NOT_FOUND || PCI_POSSIBLE_ERROR(slot_status)) return -ENODEV; return !!(slot_status & PCI_EXP_SLTSTA_PDS); @@ -728,6 +722,32 @@ void sunway_pciehp_poll(struct controller *ctrl, u32 events) } } +static void sunway_pciehp_ignore_dpc_link_change(struct controller *ctrl, + struct pci_dev *pdev, int irq) +{ + /* + * Ignore link changes which occurred while waiting for DPC recovery. + * Could be several if DPC triggered multiple times consecutively. + */ + synchronize_hardirq(irq); + atomic_and(~PCI_EXP_SLTSTA_DLLSC, &ctrl->pending_events); + if (sunway_pciehp_poll_mode) + pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, + PCI_EXP_SLTSTA_DLLSC); + ctrl_info(ctrl, "Slot(%s): Link Down/Up ignored (recovered by DPC)\n", + slot_name(ctrl)); + + /* + * If the link is unexpectedly down after successful recovery, + * the corresponding link change may have been ignored above. + * Synthesize it to ensure that it is acted on. + */ + down_read_nested(&ctrl->reset_lock, ctrl->depth); + if (!sunway_pciehp_check_link_active(ctrl)) + sunway_pciehp_request(ctrl, PCI_EXP_SLTSTA_DLLSC); + up_read(&ctrl->reset_lock); +} + static irqreturn_t sunway_pciehp_isr(int irq, void *dev_id) { struct controller *ctrl = (struct controller *)dev_id; @@ -770,7 +790,7 @@ static irqreturn_t sunway_pciehp_isr(int irq, void *dev_id) read_status: pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &status); - if (status == (u16) ~0) { + if (PCI_POSSIBLE_ERROR(status)) { ctrl_info(ctrl, "%s: no response from device\n", __func__); if (parent) pm_runtime_put(parent); @@ -791,6 +811,8 @@ static irqreturn_t sunway_pciehp_isr(int irq, void *dev_id) */ if (ctrl->power_fault_detected) status &= ~PCI_EXP_SLTSTA_PFD; + else if (status & PCI_EXP_SLTSTA_PFD) + ctrl->power_fault_detected = true; events |= status; if (!events) { @@ -800,7 +822,7 @@ static irqreturn_t sunway_pciehp_isr(int irq, void *dev_id) } if (status) { - pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, events); + pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, status); /* * In MSI mode, all event bits must be zero before the port @@ -874,31 +896,35 @@ static irqreturn_t sunway_pciehp_ist(int irq, void *dev_id) } /* Check Attention Button Pressed */ - if (events & PCI_EXP_SLTSTA_ABP) { - ctrl_info(ctrl, "Slot(%s): Attention button pressed\n", - slot_name(ctrl)); + if (events & PCI_EXP_SLTSTA_ABP) sunway_pciehp_handle_button_press(ctrl); - } /* Check Power Fault Detected */ - if ((events & PCI_EXP_SLTSTA_PFD) && !ctrl->power_fault_detected) { - ctrl->power_fault_detected = 1; + if (events & PCI_EXP_SLTSTA_PFD) { ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(ctrl)); sunway_pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF, PCI_EXP_SLTCTL_ATTN_IND_ON); } + /* + * Ignore Link Down/Up events caused by Downstream Port Containment + * if recovery from the error succeeded. + */ + if ((events & PCI_EXP_SLTSTA_DLLSC) && pci_dpc_recovered(pdev) && + ctrl->state == ON_STATE) { + events &= ~PCI_EXP_SLTSTA_DLLSC; + sunway_pciehp_ignore_dpc_link_change(ctrl, pdev, irq); + } + /* * Disable requests have higher priority than Presence Detect Changed * or Data Link Layer State Changed events. */ - down_read(&ctrl->reset_lock); - + down_read_nested(&ctrl->reset_lock, ctrl->depth); if (events & DISABLE_SLOT) sunway_pciehp_handle_disable_request(ctrl); else if (events & (PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC)) sunway_pciehp_handle_presence_or_link_change(ctrl, events); - up_read(&ctrl->reset_lock); ret = IRQ_HANDLED; @@ -956,7 +982,9 @@ static void pcie_enable_notification(struct controller *ctrl) else cmd |= PCI_EXP_SLTCTL_PDCE; if (!sunway_pciehp_poll_mode) - cmd |= PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_CCIE; + cmd |= PCI_EXP_SLTCTL_HPIE; + if (!sunway_pciehp_poll_mode && !NO_CMD_CMPL(ctrl)) + cmd |= PCI_EXP_SLTCTL_CCIE; mask = (PCI_EXP_SLTCTL_PDCE | PCI_EXP_SLTCTL_ABPE | PCI_EXP_SLTCTL_PFDE | @@ -987,6 +1015,31 @@ void sunway_pcie_clear_hotplug_events(struct controller *ctrl) PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC); } +/** + * sunway_pciehp_slot_reset() - ignore link event caused by error-induced hot reset + * + * Called from pcie_portdrv_slot_reset() after AER or DPC initiated a reset + * further up in the hierarchy to recover from an error. The reset was + * propagated down to this hotplug port. Ignore the resulting link flap. + * If the link failed to retrain successfully, synthesize the ignored event. + * Surprise removal during reset is detected through Presence Detect Changed. + */ +int sunway_pciehp_slot_reset(struct pci_dev *dev) +{ + struct controller *ctrl = pci_get_drvdata(dev); + + if (ctrl->state != ON_STATE) + return 0; + + pcie_capability_write_word(ctrl_dev(ctrl), PCI_EXP_SLTSTA, + PCI_EXP_SLTSTA_DLLSC); + + if (!sunway_pciehp_check_link_active(ctrl)) + sunway_pciehp_request(ctrl, PCI_EXP_SLTSTA_DLLSC); + + return 0; +} + /* * sunway_pciehp has a 1:1 bus:slot relationship so we ultimately want a secondary * bus reset of the bridge, but at the same time we want to ensure that it is @@ -1005,7 +1058,7 @@ int sunway_pciehp_reset_slot(struct hotplug_slot *hotplug_slot, bool probe) if (probe) return 0; - down_write(&ctrl->reset_lock); + down_write_nested(&ctrl->reset_lock, ctrl->depth); if (!ATTN_BUTTN(ctrl)) { ctrl_mask |= PCI_EXP_SLTCTL_PDCE; @@ -1061,10 +1114,24 @@ static inline void dbg_ctrl(struct controller *ctrl) #define FLAG(x, y) (((x) & (y)) ? '+' : '-') +static inline int sunway_pcie_hotplug_depth(struct pci_dev *dev) +{ + struct pci_bus *bus = dev->bus; + int depth = 0; + + while (bus->parent) { + bus = bus->parent; + if (bus->self && bus->self->is_hotplug_bridge) + depth++; + } + + return depth; +} + struct controller *sunwayhpc_init(struct pci_dev *dev) { struct controller *ctrl; - u32 slot_cap, slot_cap2, link_cap; + u32 slot_cap, slot_cap2; u8 poweron; struct pci_bus *subordinate = dev->subordinate; @@ -1073,6 +1140,7 @@ struct controller *sunwayhpc_init(struct pci_dev *dev) return NULL; ctrl->pci_dev = dev; + ctrl->depth = sunway_pcie_hotplug_depth(dev); pcie_capability_read_dword(dev, PCI_EXP_SLTCAP, &slot_cap); if (dev->hotplug_user_indicators) @@ -1108,9 +1176,6 @@ struct controller *sunwayhpc_init(struct pci_dev *dev) if (dmi_first_match(inband_presence_disabled_dmi_table)) ctrl->inband_presence_disabled = 1; - /* Check if Data Link Layer Link Active Reporting is implemented */ - pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &link_cap); - /* Clear all remaining event bits in Slot Status register. */ pcie_capability_write_word(dev, PCI_EXP_SLTSTA, PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD | @@ -1129,7 +1194,7 @@ struct controller *sunwayhpc_init(struct pci_dev *dev) FLAG(slot_cap, PCI_EXP_SLTCAP_EIP), FLAG(slot_cap, PCI_EXP_SLTCAP_NCCS), FLAG(slot_cap2, PCI_EXP_SLTCAP2_IBPD), - FLAG(link_cap, PCI_EXP_LNKCAP_DLLLARC)); + FLAG(dev->link_active_reporting, true)); /* * If empty slot's power status is on, turn power off. The IRQ isn't diff --git a/drivers/pci/hotplug/sunway_pciehp_pci.c b/drivers/pci/hotplug/sunway_pciehp_pci.c index f627ea054342..79e9646005ef 100644 --- a/drivers/pci/hotplug/sunway_pciehp_pci.c +++ b/drivers/pci/hotplug/sunway_pciehp_pci.c @@ -54,7 +54,14 @@ int sunway_pciehp_configure_device(struct controller *ctrl) pci_assign_unassigned_bridge_resources(bridge); pcie_bus_configure_settings(parent); + + /* + * Release reset_lock during driver binding + * to avoid AB-BA deadlock with device_lock. + */ + up_read(&ctrl->reset_lock); pci_bus_add_devices(parent); + down_read_nested(&ctrl->reset_lock, ctrl->depth); out: pci_unlock_rescan_remove(); @@ -95,7 +102,15 @@ void sunway_pciehp_unconfigure_device(struct controller *ctrl, bool presence) list_for_each_entry_safe_reverse(dev, temp, &parent->devices, bus_list) { pci_dev_get(dev); + + /* + * Release reset_lock during driver unbinding + * to avoid AB-BA deadlock with device_lock. + */ + up_read(&ctrl->reset_lock); pci_stop_and_remove_bus_device(dev); + down_read_nested(&ctrl->reset_lock, ctrl->depth); + /* * Ensure that no new Requests will be generated from * the device. -- Gitee From 6b26541eacb2d43f4cb3f81025a503181a9f3f28 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Wed, 10 Jul 2024 16:41:41 +0800 Subject: [PATCH 277/524] sw64: kvm: resolve a compile warning In upstream, the function arch_cpu_idle_dead() adds the "__noreturn" modifier. Threfore, the code "return;" in SW's arch_cpu_idle_dead() throws a warning. This patch deletes the meaningless "return;". Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index ff92dc4cbd19..68731747368a 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -843,7 +843,6 @@ void arch_cpu_idle_dead(void) if (is_in_guest()) { hcall(HCALL_SET_CLOCKEVENT, 0, 0, 0); hcall(HCALL_STOP, 0, 0, 0); - return; } else { wrtimer(0); } -- Gitee From 83a8107d9a4448dd717452878f7eb61dac17075e Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 4 Jul 2024 12:03:31 +0800 Subject: [PATCH 278/524] sw64: dts: support online-capable property for cpu device node On sunway platform, we add a property named online-capable to cpu device node if the related core is hot-pluggable. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uncore_io_junzhang.h | 1 - arch/sw_64/kernel/acpi.c | 12 ------------ arch/sw_64/kernel/smp.c | 21 +++++++++------------ 3 files changed, 9 insertions(+), 25 deletions(-) diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index f553d2f61e64..770af3907b5c 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -100,7 +100,6 @@ enum { /* SPBU CSR */ enum { SMP_INFO = SPBU_BASE | 0x80UL, - VT_ONLINE_CPU = SPBU_BASE | 0x100UL, INIT_CTL = SPBU_BASE | 0x680UL, CORE_ONLINE = SPBU_BASE | 0x780UL, DLI_RLTD_FAULT = SPBU_BASE | 0x980UL, diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 23b5a90cc7b2..990db687657e 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -332,17 +331,6 @@ static int __init acpi_process_madt_sw_cintc(void) init_cpu_possible(cpu_none_mask); init_cpu_present(cpu_none_mask); -#ifdef CONFIG_SUBARCH_C4 - /* Set cpu_offline mask */ - if (is_guest_or_emul()) { - int vt_smp_cpu_num; - - vt_smp_cpu_num = sw64_io_read(0, VT_ONLINE_CPU); - for (i = vt_smp_cpu_num; i < KVM_MAX_VCPUS; i++) - cpumask_set_cpu(i, &cpu_offline); - } -#endif - /* Parse SW CINTC entries one by one */ ret = acpi_table_parse_madt(ACPI_MADT_TYPE_SW_CINTC, acpi_parse_sw_cintc, 0); diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 68731747368a..34aca031dff9 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -307,6 +306,8 @@ static int __init fdt_setup_smp(void) struct device_node *dn = NULL; u64 boot_flag_address; u32 rcid, logical_core_id = 0; + u32 online_capable = 0; + bool available; int ret, i, version, lnode, pnode; /* Clean the map from logical core ID to physical core ID */ @@ -317,17 +318,12 @@ static int __init fdt_setup_smp(void) init_cpu_possible(cpu_none_mask); init_cpu_present(cpu_none_mask); -#ifdef CONFIG_SUBARCH_C4 - if (is_guest_or_emul()) { - int vt_smp_cpu_num; - - vt_smp_cpu_num = sw64_io_read(0, VT_ONLINE_CPU); - for (i = vt_smp_cpu_num; i < KVM_MAX_VCPUS; i++) - cpumask_set_cpu(i, &cpu_offline); - } -#endif while ((dn = of_find_node_by_type(dn, "cpu"))) { - if (!of_device_is_available(dn)) { + of_property_read_u32(dn, "online-capable", &online_capable); + + available = of_device_is_available(dn); + + if (!available && !online_capable) { pr_info("OF: Core is not available\n"); continue; } @@ -366,7 +362,8 @@ static int __init fdt_setup_smp(void) set_cpu_possible(logical_core_id, true); store_cpu_data(logical_core_id); - if (!cpumask_test_cpu(logical_core_id, &cpu_offline)) + if (!cpumask_test_cpu(logical_core_id, &cpu_offline) && + available) set_cpu_present(logical_core_id, true); rcid_information_init(version); -- Gitee From 166ba079acc3047cec7e5fd089ff6231b2c7ad6d Mon Sep 17 00:00:00 2001 From: Xu Chenjiao Date: Mon, 15 Jul 2024 09:08:21 +0000 Subject: [PATCH 279/524] sw64: pciehp: remove useless code Kernel get hotplug enabled/disabled status of PCIe controllers from ACPI firmware settings in sunway_pciehp driver. Signed-off-by: Xu Chenjiao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/controller/pci-sunway.c | 7 +------ drivers/pci/hotplug/sunway_pciehp_core.c | 5 ----- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 59962f9a6158..139c9515fe6f 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -327,8 +327,6 @@ static void hose_init(struct pci_controller *hose) hose->first_busno = hose->self_busno = hose->busn_space->start; hose->last_busno = hose->busn_space->end; - hose->hotplug_enable = true; - if (is_in_host()) { if (IS_ENABLED(CONFIG_PCI_MSI)) memset(hose->piu_msiconfig, 0, 256/8); @@ -648,7 +646,6 @@ enum pci_props { PROP_PIU_IOR1_BASE, PROP_RC_INDEX, PROP_PCIE_IO_BASE, - PROP_HOTPLUG_ENABLE, PROP_NUM }; @@ -661,8 +658,7 @@ const char *prop_names[PROP_NUM] = { "sw64,piu_ior0_base", "sw64,piu_ior1_base", "sw64,rc_index", - "sw64,pcie_io_base", - "sw64,hot_plug_slot_enable" + "sw64,pcie_io_base" }; static int sw64_pci_prepare_controller(struct pci_controller *hose, @@ -685,7 +681,6 @@ static int sw64_pci_prepare_controller(struct pci_controller *hose, hose->iommu_enable = false; hose->index = props[PROP_RC_INDEX]; - hose->hotplug_enable = props[PROP_HOTPLUG_ENABLE]; hose->sparse_mem_base = 0; hose->sparse_io_base = 0; diff --git a/drivers/pci/hotplug/sunway_pciehp_core.c b/drivers/pci/hotplug/sunway_pciehp_core.c index acafc82b3224..45972adcff36 100644 --- a/drivers/pci/hotplug/sunway_pciehp_core.c +++ b/drivers/pci/hotplug/sunway_pciehp_core.c @@ -280,11 +280,6 @@ static int __init sunway_pciehp_init(void) return -ENODEV; } - if (is_guest_or_emul()) { - pr_info(DRIVER_DESC " does not support for VM and emulator.\n"); - return -ENODEV; - } - pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); sunway_hose_hotplug_init(); -- Gitee From d5c32716737107d729fcd260de7e087c88194c19 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 15 Jul 2024 14:11:22 +0800 Subject: [PATCH 280/524] sw64: add uapi reg.h Add uapi reg.h which defines registers in Exception frame. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/reg.h | 53 +++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 arch/sw_64/include/uapi/asm/reg.h diff --git a/arch/sw_64/include/uapi/asm/reg.h b/arch/sw_64/include/uapi/asm/reg.h new file mode 100644 index 000000000000..e692e45a4936 --- /dev/null +++ b/arch/sw_64/include/uapi/asm/reg.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_REG_H +#define _UAPI_ASM_SW64_REG_H + +/* + * Exception frame offsets. + */ +#define EF_V0 0 +#define EF_T0 1 +#define EF_T1 2 +#define EF_T2 3 +#define EF_T3 4 +#define EF_T4 5 +#define EF_T5 6 +#define EF_T6 7 +#define EF_T7 8 +#define EF_S0 9 +#define EF_S1 10 +#define EF_S2 11 +#define EF_S3 12 +#define EF_S4 13 +#define EF_S5 14 +#define EF_S6 15 +#define EF_A3 16 +#define EF_A4 17 +#define EF_A5 18 +#define EF_T8 19 +#define EF_T9 20 +#define EF_T10 21 +#define EF_T11 22 +#define EF_RA 23 +#define EF_T12 24 +#define EF_AT 25 +#define EF_SP 26 +#define EF_PS 27 +#define EF_PC 28 +#define EF_GP 29 +#define EF_A0 30 +#define EF_A1 31 +#define EF_A2 32 + +#define EF_SIZE (33*8) +#define HWEF_SIZE (6*8) /* size of HMCODE frame (PS-A2) */ + +#define EF_SSIZE (EF_SIZE - HWEF_SIZE) + +/* + * Map register number into core file offset. + */ +#define CORE_REG(reg, ubase) \ + (((unsigned long *)((unsigned long)(ubase)))[reg]) + +#endif /* _UAPI_ASM_SW64_REG_H */ -- Gitee From 3dbfa16ae4b0c92b202821e6bba3485d454f7288 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 2 Jul 2024 15:01:52 +0800 Subject: [PATCH 281/524] sw64: pci: fix unable to get node ID when numa_off is set The function acpi_get_node() used to get node ID of PCIe controller cannot return the correct value when numa_off is set, further causing PCIe controller initialization failure. The reason why node ID is necessary is that it is used to mark whether PCIe controller is linked up. To solve this issue, add a linkup field in struct pci_controller to mark the link status. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 7 ++-- arch/sw_64/pci/pci-legacy.c | 9 ++++-- drivers/pci/controller/pci-sunway.c | 43 ++++++++++++------------- drivers/pci/hotplug/sunway_pciehp_hpc.c | 6 ++-- 4 files changed, 33 insertions(+), 32 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 62fee7ba5dd6..e43edf203e38 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -64,6 +64,7 @@ struct pci_controller { */ unsigned int need_domain_info; bool iommu_enable; + bool linkup; struct sunway_iommu *pci_iommu; bool hotplug_enable; int first_busno; @@ -110,9 +111,9 @@ extern int sw64_pcie_config_write(struct pci_bus *bus, unsigned int devfn, extern int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val); -extern void pci_mark_rc_linkup(unsigned long node, unsigned long index); -extern void pci_clear_rc_linkup(unsigned long node, unsigned long index); -extern int pci_get_rc_linkup(unsigned long node, unsigned long index); +extern void pci_mark_rc_linkup(struct pci_controller *hose); +extern void pci_clear_rc_linkup(struct pci_controller *hose); +extern int pci_get_rc_linkup(const struct pci_controller *hose); #ifdef CONFIG_PCI_DOMAINS static inline int pci_proc_domain(struct pci_bus *bus) diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index 52ba53a1fc0d..8eb5458dc2fc 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -206,6 +206,8 @@ int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) return sw64_chip_init->pci_init.map_irq(dev, slot, pin); } +static bool rc_linkup[MAX_NUMNODES][MAX_NR_RCS_PER_NODE]; + static void __init sw64_init_host(unsigned long node, unsigned long index) { @@ -233,7 +235,8 @@ sw64_init_host(unsigned long node, unsigned long index) ret = sw64_chip_init->pci_init.check_pci_linkup(hose); if (ret == 0) { /* Root Complex downstream port is link up */ - pci_mark_rc_linkup(node, index); // 8-bit per node + pci_mark_rc_linkup(hose); // 8-bit per node + rc_linkup[node][index] = true; } } @@ -245,7 +248,7 @@ static bool __init is_any_rc_linkup_one_node(unsigned long node) int i; for (i = 0; i < MAX_NR_RCS_PER_NODE; ++i) { - if (pci_get_rc_linkup(node, i)) + if (rc_linkup[node][i]) return true; } @@ -287,7 +290,7 @@ void __init sw64_init_arch(void) memset(msg, 0, 64); sprintf(msg, "Node %ld: RC [ ", node); for (i = 0; i < MAX_NR_RCS_PER_NODE; i++) { - if (pci_get_rc_linkup(node, i)) { + if (rc_linkup[node][i]) { memset(id, 0, 8); sprintf(id, "%d ", i); strcat(msg, id); diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 139c9515fe6f..db4ad9247887 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -348,25 +348,20 @@ void __init setup_chip_pci_ops(void) static struct pci_controller *head, **tail = &head; -static DECLARE_BITMAP(rc_linkup, (MAX_NUMNODES * MAX_NR_RCS_PER_NODE)); - -void pci_mark_rc_linkup(unsigned long node, unsigned long index) +void pci_mark_rc_linkup(struct pci_controller *hose) { - set_bit(node * MAX_NR_RCS_PER_NODE + index, rc_linkup); + hose->linkup = true; } -EXPORT_SYMBOL(pci_mark_rc_linkup); -void pci_clear_rc_linkup(unsigned long node, unsigned long index) +void pci_clear_rc_linkup(struct pci_controller *hose) { - clear_bit(node * MAX_NR_RCS_PER_NODE + index, rc_linkup); + hose->linkup = false; } -EXPORT_SYMBOL(pci_clear_rc_linkup); -int pci_get_rc_linkup(unsigned long node, unsigned long index) +int pci_get_rc_linkup(const struct pci_controller *hose) { - return test_bit(node * MAX_NR_RCS_PER_NODE + index, rc_linkup); + return hose->linkup; } -EXPORT_SYMBOL(pci_get_rc_linkup); /** * Link the specified pci controller to list @@ -551,7 +546,7 @@ int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, if (unlikely(bus->number == hose->self_busno)) { ret = sw64_pcie_read_rc_cfg(bus, devfn, where, size, val); } else { - if (pci_get_rc_linkup(hose->node, hose->index)) + if (pci_get_rc_linkup(hose)) ret = pci_generic_config_read(bus, devfn, where, size, val); else return ret; @@ -715,12 +710,10 @@ static int sw64_pci_prepare_controller(struct pci_controller *hose, * Root Complex link up failed. * This usually means that no device on the slot. */ - pr_info(": link down\n", - hose->node, hose->index); + pr_info("RC link down\n"); } else { - pci_mark_rc_linkup(hose->node, hose->index); - pr_info(": successfully link up\n", - hose->node, hose->index); + pci_mark_rc_linkup(hose); + pr_info("RC successfully link up\n"); } setup_intx_irqs(hose); @@ -765,13 +758,17 @@ static int sw64_pci_ecam_init(struct pci_config_window *cfg) if (!hose) return -ENOMEM; - /* Get node from ACPI namespace (_PXM) */ - hose->node = acpi_get_node(adev->handle); - if (hose->node == NUMA_NO_NODE) { - kfree(hose); - dev_err(&adev->dev, "unable to get node ID\n"); - return -EINVAL; +#ifdef CONFIG_ACPI_NUMA + if (!numa_off) { + /* Get node from ACPI namespace (_PXM) */ + hose->node = acpi_get_node(adev->handle); + if (hose->node == NUMA_NO_NODE) { + kfree(hose); + dev_err(&adev->dev, "unable to get node ID\n"); + return -EINVAL; + } } +#endif /* Init pci_controller */ ret = sw64_pci_prepare_controller(hose, &adev->fwnode); diff --git a/drivers/pci/hotplug/sunway_pciehp_hpc.c b/drivers/pci/hotplug/sunway_pciehp_hpc.c index 2bb6d3ff3165..5fd0167b55e6 100644 --- a/drivers/pci/hotplug/sunway_pciehp_hpc.c +++ b/drivers/pci/hotplug/sunway_pciehp_hpc.c @@ -251,7 +251,7 @@ static bool pcie_wait_link_active(struct controller *ctrl) struct pci_controller *hose = pci_bus_to_pci_controller(bus); if (pcie_wait_for_link(pdev, true)) { - pci_mark_rc_linkup(hose->node, hose->index); + pci_mark_rc_linkup(hose); sunway_pciehp_restore_rc_piu(ctrl); return true; } @@ -403,7 +403,7 @@ int sunway_pciehp_link_disable(struct controller *ctrl) int ret; ret = __sunway_pciehp_link_set(ctrl, false); - pci_clear_rc_linkup(hose->node, hose->index); + pci_clear_rc_linkup(hose); return ret; } @@ -694,7 +694,7 @@ void sunway_pciehp_poll(struct controller *ctrl, u32 events) for (i = 0; i < 30; i++) { if (pcie_wait_for_link(pdev, true)) { - pci_mark_rc_linkup(hose->node, hose->index); + pci_mark_rc_linkup(hose); sunway_pciehp_restore_rc_piu(ctrl); writeq(HP_CTRL_FINISH, (piu_ior0_base + HP_CTRL)); -- Gitee From e8868f2c78b31591c9fbc853d26cefeb6ea1862a Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 17 Jul 2024 08:41:50 +0800 Subject: [PATCH 282/524] sw64: pci: fix compile error when CONFIG_PCI_MSI=n Provide empty implementations of function vt_handle_pci_msi_interrupt() and msic_acpi_init() to fix compile error when CONFIG_PCI_MSI=n. This commit also removes some meaningless logs. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/irq.h | 3 --- arch/sw_64/include/asm/msi.h | 11 +++++++++++ arch/sw_64/pci/pci.c | 2 -- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/include/asm/irq.h b/arch/sw_64/include/asm/irq.h index 6a4abfc9687b..13f3dbac3491 100644 --- a/arch/sw_64/include/asm/irq.h +++ b/arch/sw_64/include/asm/irq.h @@ -38,9 +38,6 @@ struct acpi_madt_sw_lpc_intc; extern int __init sw64_add_gsi_domain_map(u32 gsi_base, u32 gsi_count, struct fwnode_handle *handle); -extern int __init msic_acpi_init(struct irq_domain *parent, - struct acpi_madt_sw_msic *msic); - #ifdef CONFIG_SW64_PINTC extern int __init pintc_acpi_init(struct irq_domain *parent, struct acpi_madt_sw_pintc *pintc); diff --git a/arch/sw_64/include/asm/msi.h b/arch/sw_64/include/asm/msi.h index 4bec4b80fbe6..5f2cfe7f4bd7 100644 --- a/arch/sw_64/include/asm/msi.h +++ b/arch/sw_64/include/asm/msi.h @@ -40,6 +40,8 @@ extern struct irq_chip sw64_irq_chip; extern void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned long pci_msi1_addr); +extern int __init msic_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_msic *msic); #define MSI_ADDR_BASE_HI 0 #define MSI_ADDR_BASE_LO 0x91abc0 @@ -93,5 +95,14 @@ static inline void handle_pci_msi_interrupt(unsigned long type, { pr_warn("SW arch disable CONFIG_PCI_MSI option.\n"); } + +static inline void vt_handle_pci_msi_interrupt(unsigned long type, + unsigned long vector, unsigned long pci_msi1_addr) +{ + pr_warn("SW arch disable CONFIG_PCI_MSI option.\n"); +} + +static inline int __init msic_acpi_init(struct irq_domain *parent, + struct acpi_madt_sw_msic *msic) { return 0; } #endif /* CONFIG_PCI_MSI */ #endif /* _ASM_SW64_MSI_H */ diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index de56ccfda574..de974eff019d 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -400,8 +400,6 @@ void sw64_pci_root_bridge_scan_finish_up(struct pci_host_bridge *bridge) pci_bus_update_busn_res_end(bus, last_bus); last_bus++; - pr_info("bus number update to %u\n", last_bus); - if (is_in_host()) sw64_pci_root_bridge_reserve_legacy_io(bridge); -- Gitee From 876a12dc2c2332f716637d630a81e17e6425547d Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Thu, 18 Jul 2024 11:34:43 +0000 Subject: [PATCH 283/524] sw64: fix C4 INTx configuration Correct the assembly logic for C4's INTxCONFIG to ensure that INTx interrupts delivered to the intended target. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-pci-intx.c | 30 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/drivers/irqchip/irq-sunway-pci-intx.c b/drivers/irqchip/irq-sunway-pci-intx.c index 480335044377..a60262fafeb3 100644 --- a/drivers/irqchip/irq-sunway-pci-intx.c +++ b/drivers/irqchip/irq-sunway-pci-intx.c @@ -28,23 +28,38 @@ static void set_intx(struct pci_controller *hose, unsigned long intx_conf) piu_ior0_base = hose->piu_ior0_base; - writeq(intx_conf | (0x8UL << 10), (piu_ior0_base + INTACONFIG)); - writeq(intx_conf | (0x4UL << 10), (piu_ior0_base + INTBCONFIG)); - writeq(intx_conf | (0x2UL << 10), (piu_ior0_base + INTCCONFIG)); - writeq(intx_conf | (0x1UL << 10), (piu_ior0_base + INTDCONFIG)); + if (IS_ENABLED(CONFIG_SUBARCH_C3B)) { + writeq(intx_conf | (0x8UL << 10), (piu_ior0_base + INTACONFIG)); + writeq(intx_conf | (0x4UL << 10), (piu_ior0_base + INTBCONFIG)); + writeq(intx_conf | (0x2UL << 10), (piu_ior0_base + INTCCONFIG)); + writeq(intx_conf | (0x1UL << 10), (piu_ior0_base + INTDCONFIG)); + } else { + writeq(intx_conf | (0x8UL << 10), (piu_ior0_base + INTDCONFIG)); + writeq(intx_conf | (0x4UL << 10), (piu_ior0_base + INTCCONFIG)); + writeq(intx_conf | (0x2UL << 10), (piu_ior0_base + INTBCONFIG)); + writeq(intx_conf | (0x1UL << 10), (piu_ior0_base + INTACONFIG)); + } } static int __assign_piu_intx_config(struct pci_controller *hose, cpumask_t *targets) { unsigned long intx_conf; unsigned int cpu; - int phy_cpu; + int thread, node, core, rcid; /* Use the last cpu in valid cpus to avoid core 0. */ cpu = cpumask_last(targets); - phy_cpu = cpu_to_rcid(cpu); + rcid = cpu_to_rcid(cpu); + + thread = rcid_to_thread_id(rcid); + node = rcid_to_domain_id(rcid); + core = rcid_to_core_id(rcid); + + if (IS_ENABLED(CONFIG_SUBARCH_C3B)) + intx_conf = core | (node << 6); + else + intx_conf = core | (thread << 6) | (node << 7); - intx_conf = ((phy_cpu >> 5) << 6) | (phy_cpu & 0x1f); set_intx(hose, intx_conf); return 0; @@ -181,4 +196,3 @@ void __init sw64_init_irq(void) for (hose = hose_head; hose; hose = hose->next) setup_intx_irqs(hose); } - -- Gitee From a39a52113eb13ab8fa3f47e0be5e9bf10c33f1dc Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 13 Jun 2024 09:37:53 +0800 Subject: [PATCH 284/524] sw64: smp: allow NR_CPUS less than the number detected by firmware When NR_CPUS is less than the number detected by firmware, smp initialization can still succeed, and the number of cores is limited to NR_CPUS. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/acpi.c | 4 ++-- arch/sw_64/kernel/smp.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 990db687657e..02a4e8fa9c95 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -249,9 +249,9 @@ setup_rcid_and_core_mask(struct acpi_madt_sw_cintc *sw_cintc) * represents the maximum number of cores in the system. */ if (possible_cores >= nr_cpu_ids) { - pr_err(PREFIX "Core [0x%x] exceeds max core num [%u]\n", + pr_warn_once(PREFIX "Core [0x%x] exceeds max core num [%u]\n", rcid, nr_cpu_ids); - return -ENODEV; + return 0; } /* The rcid of each core is unique */ diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 34aca031dff9..6bafc18f742c 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -335,9 +335,9 @@ static int __init fdt_setup_smp(void) } if (logical_core_id >= nr_cpu_ids) { - pr_err("OF: Core [0x%x] exceeds max core num [%u]\n", + pr_warn_once("OF: Core [0x%x] exceeds max core num [%u]\n", rcid, nr_cpu_ids); - return -ENODEV; + return 0; } if (is_rcid_duplicate(rcid)) { -- Gitee From a53d6b10a90cf61f547eb4d736e2fd955b286d8c Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 17 Jul 2024 20:01:16 +0800 Subject: [PATCH 285/524] sw64: kexec: remove code for compatibility with builtin DTB Starting from junzhang, builtin DTB is no longer supported, so we can remove related code. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/machine_kexec.c | 60 ------------------------------- 1 file changed, 60 deletions(-) diff --git a/arch/sw_64/kernel/machine_kexec.c b/arch/sw_64/kernel/machine_kexec.c index a85cca14444b..f0a2338ddcbd 100644 --- a/arch/sw_64/kernel/machine_kexec.c +++ b/arch/sw_64/kernel/machine_kexec.c @@ -285,53 +285,6 @@ static void *arch_kexec_alloc_and_setup_fdt(unsigned long initrd_start, return NULL; } -#ifdef CONFIG_EFI -static int update_efi_properties(const struct boot_params *params) -{ - int chosen_node, ret; - void *dtb_start = (void *)params->dtb_start; - - if (!dtb_start) - return -EINVAL; - - chosen_node = fdt_path_offset(dtb_start, "/chosen"); - if (chosen_node < 0) - return -EINVAL; - - ret = fdt_setprop_u64(dtb_start, chosen_node, - "linux,uefi-system-table", - params->efi_systab); - if (ret) - return ret; - - ret = fdt_setprop_u64(dtb_start, chosen_node, - "linux,uefi-mmap-start", - params->efi_memmap); - if (ret) - return ret; - - ret = fdt_setprop_u64(dtb_start, chosen_node, - "linux,uefi-mmap-size", - params->efi_memmap_size); - if (ret) - return ret; - - ret = fdt_setprop_u64(dtb_start, chosen_node, - "linux,uefi-mmap-desc-size", - params->efi_memdesc_size); - if (ret) - return ret; - - ret = fdt_setprop_u64(dtb_start, chosen_node, - "linux,uefi-mmap-desc-ver", - params->efi_memdesc_version); - if (ret) - return ret; - - return 0; -} -#endif - static void update_boot_params(void) { struct boot_params params = { 0 }; @@ -364,19 +317,6 @@ static void update_boot_params(void) params.efi_memmap_size = efi.memmap.map_end - efi.memmap.map; params.efi_memdesc_size = efi.memmap.desc_size; params.efi_memdesc_version = efi.memmap.desc_version; - - /** - * If current kernel take built-in DTB, it's possible that - * there are no efi related properties in "chosen" node. So, - * update these properties here. - * - * Harmless for the following cases: - * 1. Current kernel take DTB from firmware - * 2. New kernel with CONFIG_EFI=n - * 3. New kernel take built-in DTB - */ - if (update_efi_properties(¶ms)) - pr_err("Note: failed to update efi properties\n"); #endif /* update dtb base address */ sunway_dtb_address = params.dtb_start; -- Gitee From 5f49bc7408b348cb575b8d5a4ebe820bed19ad75 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 23 Jul 2024 15:32:16 +0800 Subject: [PATCH 286/524] sw64: kvm: don't retrieve memory slot again in page fault handler It is unnecessary to retrieve the memory slot again in user_mem_abort() because the corresponding memory slot has been passed from the caller. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index a9bf514b81fd..3c940544534e 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -1218,16 +1218,17 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, mmu_seq = vcpu->kvm->mmu_invalidate_seq; /* * Ensure the read of mmu_invalidate_seq happens before we call - * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk - * the page we just got a reference to gets unmapped before we have a - * chance to grab the mmu_lock, which ensure that if the page gets + * __gfn_to_pfn_memslot (which calls get_user_pages), so that we don't + * risk the page we just got a reference to gets unmapped before we have + * a chance to grab the mmu_lock, which ensure that if the page gets * unmapped afterwards, the call to kvm_unmap_gfn will take it away * from us again properly. This smp_rmb() interacts with the smp_wmb() * in kvm_mmu_notifier_invalidate_. */ smp_rmb(); - pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable); + pfn = __gfn_to_pfn_memslot(memslot, gfn, false, false, NULL, + write_fault, &writable, NULL); if (pfn == KVM_PFN_ERR_HWPOISON) { kvm_send_hwpoison_signal(hva, vma); @@ -1329,7 +1330,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, if (writable) { new_pte = kvm_pte_mkwrite(new_pte); kvm_set_pfn_dirty(pfn); - mark_page_dirty(kvm, gfn); + mark_page_dirty_in_slot(kvm, memslot, gfn); } if (exec_fault && fault_status == AF_STATUS_INV) { -- Gitee From 847afe3d8c09e76bd1c974ac935fa50d3ee6eef9 Mon Sep 17 00:00:00 2001 From: Xu Chenjiao Date: Mon, 22 Jul 2024 10:36:50 +0000 Subject: [PATCH 287/524] sw64: pciehp: fix compile error when CONFIG_PCIE_DPC=y The DPC function is not supported now. Signed-off-by: Xu Chenjiao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/hotplug/sunway_pciehp_hpc.c | 36 ------------------------- 1 file changed, 36 deletions(-) diff --git a/drivers/pci/hotplug/sunway_pciehp_hpc.c b/drivers/pci/hotplug/sunway_pciehp_hpc.c index 5fd0167b55e6..5e762cfd54f8 100644 --- a/drivers/pci/hotplug/sunway_pciehp_hpc.c +++ b/drivers/pci/hotplug/sunway_pciehp_hpc.c @@ -722,32 +722,6 @@ void sunway_pciehp_poll(struct controller *ctrl, u32 events) } } -static void sunway_pciehp_ignore_dpc_link_change(struct controller *ctrl, - struct pci_dev *pdev, int irq) -{ - /* - * Ignore link changes which occurred while waiting for DPC recovery. - * Could be several if DPC triggered multiple times consecutively. - */ - synchronize_hardirq(irq); - atomic_and(~PCI_EXP_SLTSTA_DLLSC, &ctrl->pending_events); - if (sunway_pciehp_poll_mode) - pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, - PCI_EXP_SLTSTA_DLLSC); - ctrl_info(ctrl, "Slot(%s): Link Down/Up ignored (recovered by DPC)\n", - slot_name(ctrl)); - - /* - * If the link is unexpectedly down after successful recovery, - * the corresponding link change may have been ignored above. - * Synthesize it to ensure that it is acted on. - */ - down_read_nested(&ctrl->reset_lock, ctrl->depth); - if (!sunway_pciehp_check_link_active(ctrl)) - sunway_pciehp_request(ctrl, PCI_EXP_SLTSTA_DLLSC); - up_read(&ctrl->reset_lock); -} - static irqreturn_t sunway_pciehp_isr(int irq, void *dev_id) { struct controller *ctrl = (struct controller *)dev_id; @@ -906,16 +880,6 @@ static irqreturn_t sunway_pciehp_ist(int irq, void *dev_id) PCI_EXP_SLTCTL_ATTN_IND_ON); } - /* - * Ignore Link Down/Up events caused by Downstream Port Containment - * if recovery from the error succeeded. - */ - if ((events & PCI_EXP_SLTSTA_DLLSC) && pci_dpc_recovered(pdev) && - ctrl->state == ON_STATE) { - events &= ~PCI_EXP_SLTSTA_DLLSC; - sunway_pciehp_ignore_dpc_link_change(ctrl, pdev, irq); - } - /* * Disable requests have higher priority than Presence Detect Changed * or Data Link Layer State Changed events. -- Gitee From fbc1e1e1643e266ea213661abe77a7e0d918bc7d Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 23 Jul 2024 15:06:21 +0800 Subject: [PATCH 288/524] sw64: kvm: fix invalid memslot flags checking during dirty logging We do not define __KVM_HAVE_READONLY_MEM now, so it is impossible to have KVM_MEM_READONLY flag set. And it is more reasonable to confirm the dirty logging status by KVM_MEM_LOG_DIRTY_PAGES. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 3c940544534e..a8506a8b7716 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -41,7 +41,7 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot) { - return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); + return memslot->dirty_bitmap && (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES); } /* -- Gitee From 6c4093a9ab90d52c14985673cbd4adeddb4893db Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 23 Jul 2024 15:31:10 +0800 Subject: [PATCH 289/524] sw64: kvm: enable ring-based dirty page tracking This patch introduces dirty ring feature. Different from dirty bitmap which needs to pull the whole bitmap to userspace and go through every bit even it is sparse, dirty ring records the dense dirty list of gfn which is mapped into userspace to allow fast dirty sync. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_host.h | 1 - arch/sw_64/include/uapi/asm/kvm.h | 2 ++ arch/sw_64/kvm/Kconfig | 1 + arch/sw_64/kvm/mmu.c | 10 +++++++--- arch/sw_64/kvm/sw64.c | 20 +++++++++++++++----- 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index cce39b0289e5..e2f367003509 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -159,7 +159,6 @@ struct kvm_vcpu_stat { #endif void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat); -void check_vcpu_requests(struct kvm_vcpu *vcpu); void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu); int vmem_init(void); void vmem_exit(void); diff --git a/arch/sw_64/include/uapi/asm/kvm.h b/arch/sw_64/include/uapi/asm/kvm.h index 3215aaa5f2eb..430e82acee3e 100644 --- a/arch/sw_64/include/uapi/asm/kvm.h +++ b/arch/sw_64/include/uapi/asm/kvm.h @@ -9,6 +9,8 @@ #define IRQ_PENDING_INTX_SHIFT 16 #define IRQ_PENDING_MSI_VECTORS_SHIFT 18 +#define KVM_DIRTY_LOG_PAGE_OFFSET 64 + enum SW64_KVM_IRQ { SW64_KVM_IRQ_IPI = 27, SW64_KVM_IRQ_TIMER = 9, diff --git a/arch/sw_64/kvm/Kconfig b/arch/sw_64/kvm/Kconfig index fdff308aed22..5fac5b1d9fb4 100644 --- a/arch/sw_64/kvm/Kconfig +++ b/arch/sw_64/kvm/Kconfig @@ -31,6 +31,7 @@ config KVM select TUN select GENERIC_ALLOCATOR select KVM_GENERIC_DIRTYLOG_READ_PROTECT + select HAVE_KVM_DIRTY_RING_ACQ_REL help Support for hosting Guest kernels. We don't support KVM with 3-level page tables yet. diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index a8506a8b7716..17383d721015 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -39,9 +39,13 @@ #include "trace.h" #define KVM_APT_FLAG_LOGGING_ACTIVE (1UL << 1) -static bool memslot_is_logging(struct kvm_memory_slot *memslot) +static bool memslot_is_logging(struct kvm *kvm, struct kvm_memory_slot *memslot) { - return memslot->dirty_bitmap && (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES); + if (!kvm->dirty_ring_size) + return memslot->dirty_bitmap + && (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES); + else + return memslot->flags & KVM_MEM_LOG_DIRTY_PAGES; } /* @@ -1171,7 +1175,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, bool logging_active, write_fault, exec_fault, writable, force_pte; force_pte = false; - logging_active = memslot_is_logging(memslot); + logging_active = memslot_is_logging(kvm, memslot); fault_gpa = vcpu->arch.vcb.fault_gpa; gfn = fault_gpa >> PAGE_SHIFT; as_info = vcpu->arch.vcb.as_info; diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index a22a7f78c2cb..d0510e567c32 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -122,7 +122,7 @@ void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) } } -void check_vcpu_requests(struct kvm_vcpu *vcpu) +static int check_vcpu_requests(struct kvm_vcpu *vcpu) { unsigned long vpn; long cpu = smp_processor_id(); @@ -132,7 +132,12 @@ void check_vcpu_requests(struct kvm_vcpu *vcpu) vpn = vcpu->arch.vpnc[cpu] & VPN_MASK; tbivpn(0, 0, vpn); } + + if (kvm_dirty_ring_check_request(vcpu)) + return 0; } + + return 1; } struct kvm_stats_debugfs_item debugfs_entries[] = { @@ -452,9 +457,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) } if (ret <= 0) { - local_irq_enable(); - preempt_enable(); - continue; + goto exit; } memset(&hargs, 0, sizeof(hargs)); @@ -474,7 +477,14 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) vcpu->arch.halted = 0; sw64_kvm_switch_vpn(vcpu); - check_vcpu_requests(vcpu); + ret = check_vcpu_requests(vcpu); + if (ret <= 0) { +exit: + local_irq_enable(); + preempt_enable(); + continue; + } + guest_enter_irqoff(); /* update aptp before the guest runs */ -- Gitee From c08ca548cced3872915b4f3f7a6c3f685a889dbc Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Thu, 8 Aug 2024 11:13:17 +0800 Subject: [PATCH 290/524] sw64: remove redundant link options Due to an issue with the binutils-dev package, extra link options were added to support libbfd. But a better approach is to fix the binutils-dev and remove the extra link options from kernel source. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/build/feature/Makefile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index cb57e46cec4b..2753d8f028c4 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 include ../../scripts/Makefile.include -ARCH ?= $(shell uname -m) FILES= \ test-all.bin \ test-backtrace.bin \ @@ -88,11 +87,7 @@ all: $(FILES) __BUILD = $(CC) $(CFLAGS) -MD -Wall -Werror -o $@ $(patsubst %.bin,%.c,$(@F)) $(LDFLAGS) BUILD = $(__BUILD) > $(@:.bin=.make.output) 2>&1 -ifeq ($(ARCH),sw_64) - BUILD_BFD = $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl -liberty -lz -else BUILD_BFD = $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl -endif BUILD_ALL = $(BUILD) -fstack-protector-all -O2 -D_FORTIFY_SOURCE=2 -ldw -lelf -lnuma -lelf -lslang $(FLAGS_PERL_EMBED) $(FLAGS_PYTHON_EMBED) -DPACKAGE='"perf"' -lbfd -ldl -lz -llzma -lzstd -lcap __BUILDXX = $(CXX) $(CXXFLAGS) -MD -Wall -Werror -o $@ $(patsubst %.bin,%.cpp,$(@F)) $(LDFLAGS) -- Gitee From 6506ec363faecfe856add52d696aa01c98490484 Mon Sep 17 00:00:00 2001 From: Zhi Tongze Date: Wed, 5 Jun 2024 09:26:51 +0800 Subject: [PATCH 291/524] sw64: add basic frequency scaling support for JunZhang This patch adds basic frequency scaling support for JunZhang SoC, and fixes mismatch of mhz and khz in sw64_cpufreq_debugfs. Signed-off-by: Zhi Tongze Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpufreq.h | 41 +++- arch/sw_64/include/asm/uncore_io_junzhang.h | 1 + arch/sw_64/platform/Makefile | 2 +- .../platform/{cpufreq_xuelang.c => cpufreq.c} | 192 +++++++++++------- drivers/cpufreq/Kconfig | 1 - drivers/cpufreq/sw64_cpufreq_debugfs.c | 28 +-- 6 files changed, 159 insertions(+), 106 deletions(-) rename arch/sw_64/platform/{cpufreq_xuelang.c => cpufreq.c} (31%) diff --git a/arch/sw_64/include/asm/cpufreq.h b/arch/sw_64/include/asm/cpufreq.h index e6f1d16e5620..70dfc67cf4e8 100644 --- a/arch/sw_64/include/asm/cpufreq.h +++ b/arch/sw_64/include/asm/cpufreq.h @@ -38,23 +38,42 @@ struct clk { #define CLK_ALWAYS_ENABLED (1 << 0) #define CLK_RATE_PROPAGATES (1 << 1) -#define CLK_PRT 0x1UL -#define CORE_CLK0_V (0x1UL << 1) -#define CORE_CLK0_R (0x1UL << 2) -#define CORE_CLK2_V (0x1UL << 15) -#define CORE_CLK2_R (0x1UL << 16) - -#define CLK_LV1_SEL_PRT 0x1UL -#define CLK_LV1_SEL_MUXA (0x1UL << 2) -#define CLK_LV1_SEL_MUXB (0x1UL << 3) - +#define CORE_CLK0_VALID (0x1UL << 1) +#define CORE_CLK0_RESET (0x1UL << 2) + +#define CLK_LV1_SEL_PROTECT (0x1UL << 0) +#define CLK_LV1_SEL_MUXA (0x1UL << 2) +#define CLK_LV1_SEL_MUXB (0x1UL << 3) + +#ifdef CONFIG_UNCORE_JUNZHANG +#define CLK0_PROTECT (0x1UL << 0) +#define CLK2_PROTECT (0x1UL << 32) +#define CORE_CLK2_VALID (0x1UL << 33) +#define CORE_CLK2_RESET (0x1UL << 34) +#define CORE_CLK2_LOCK (0x1UL << 35) +#define CORE_PLL0_CFG_SHIFT 20 +#define CORE_PLL2_CFG_SHIFT 36 +#define CORE_PLL2_CFG_MASK 0x1f +#define STARTUP_RATE (2000UL * 1000 * 1000) +#endif + +#ifdef CONFIG_UNCORE_XUELANG +#define CLK_PROTECT (0x1UL << 0) +#define CLK0_PROTECT CLK_PROTECT +#define CLK2_PROTECT CLK_PROTECT +#define CORE_CLK2_VALID (0x1UL << 15) +#define CORE_CLK2_RESET (0x1UL << 16) +#define CORE_CLK2_LOCK (0x1UL << 17) #define CORE_PLL0_CFG_SHIFT 4 #define CORE_PLL2_CFG_SHIFT 18 +#define CORE_PLL2_CFG_MASK 0xf +#define STARTUP_RATE (2400UL * 1000 * 1000) +#endif extern struct cpufreq_frequency_table freq_table[]; int clk_init(void); -void sw64_set_rate(unsigned int index); +int sw64_set_rate(unsigned int index); struct clk *sw64_clk_get(struct device *dev, const char *id); diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index 770af3907b5c..e21876f70d1e 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -120,6 +120,7 @@ enum { MC_ONLINE = SPBU_BASE | 0x3780UL, CLK_CTL = SPBU_BASE | 0x3b80UL, CLU_LV2_SELH = SPBU_BASE | 0x3a00UL, + CLU_LV1_SEL = SPBU_BASE | 0x3a80UL, CLU_LV2_SELL = SPBU_BASE | 0x3b00UL, PIU_TOP0_CONFIG = SPBU_BASE | 0x4c80UL, PIU_TOP1_CONFIG = SPBU_BASE | 0x4d00UL, diff --git a/arch/sw_64/platform/Makefile b/arch/sw_64/platform/Makefile index 4c0edceb4a2c..09ed3eb0f743 100644 --- a/arch/sw_64/platform/Makefile +++ b/arch/sw_64/platform/Makefile @@ -1,2 +1,2 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_PLATFORM_XUELANG) += cpufreq_xuelang.o +obj-y += cpufreq.o diff --git a/arch/sw_64/platform/cpufreq_xuelang.c b/arch/sw_64/platform/cpufreq.c similarity index 31% rename from arch/sw_64/platform/cpufreq_xuelang.c rename to arch/sw_64/platform/cpufreq.c index e639fdc34fb9..87f66b2313b5 100644 --- a/arch/sw_64/platform/cpufreq_xuelang.c +++ b/arch/sw_64/platform/cpufreq.c @@ -1,51 +1,101 @@ // SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include #include -#include -#include #include +#include +#include +#include +#include +#include -/* Minimum CLK support */ -enum { - DC_0, DC_1, DC_2, DC_3, DC_4, DC_5, DC_6, DC_7, DC_8, - DC_9, DC_10, DC_11, DC_12, DC_13, DC_14, DC_15, DC_RESV -}; - -struct cpufreq_frequency_table freq_table[] = { - {0, 200, CPUFREQ_ENTRY_INVALID}, - {0, DC_1, CPUFREQ_ENTRY_INVALID}, - {0, DC_2, 0}, - {0, DC_3, 0}, - {0, DC_4, 0}, - {0, DC_5, 0}, - {0, DC_6, 0}, - {0, DC_7, 0}, - {0, DC_8, 0}, - {0, DC_9, 0}, - {0, DC_10, 0}, - {0, DC_11, 0}, - {0, DC_12, 0}, - {0, DC_13, 0}, - {0, DC_14, 0}, - {0, DC_15, 0}, - {-1, DC_RESV, CPUFREQ_TABLE_END}, -}; - +#define MAX_RETRY 10 static struct platform_device sw64_cpufreq_device = { .name = "sw64_cpufreq", .id = -1, }; -static int __init sw64_cpufreq_init(void) +/* + * frequency in MHz, volts in mV and stored as "driver_data" in the structure. + * volts 0 means to be determined + */ +#define FV(mhz, mv) \ + { \ + .frequency = (mhz) * 1000, \ + .driver_data = (mv) \ + } + +#ifdef CONFIG_PLATFORM_JUNZHANG +struct cpufreq_frequency_table freq_table[] = { + {0, 0, CPUFREQ_ENTRY_INVALID}, /* 200Mhz is ignored */ + FV(1200, 850), + FV(1300, 850), + FV(1400, 850), + FV(1450, 850), + FV(1500, 850), + FV(1550, 850), + FV(1600, 850), + FV(1650, 900), + FV(1700, 900), + FV(1750, 900), + FV(1800, 900), + FV(1850, 900), + FV(1900, 900), + FV(1950, 900), + FV(2000, 900), + FV(2050, 950), + FV(2100, 950), + FV(2150, 950), + FV(2200, 950), + FV(2250, 0), + FV(2300, 0), + FV(2350, 0), + FV(2400, 0), + FV(2450, 0), + FV(2500, 0), + FV(2550, 0), + FV(2600, 0), + FV(2650, 0), + FV(2700, 0), + FV(2800, 0), + FV(2850, 0), + {0, 0, CPUFREQ_TABLE_END}, +}; +static void __init fill_freq_table(struct cpufreq_frequency_table *ft) +{ +} +#elif CONFIG_PLATFORM_XUELANG +struct cpufreq_frequency_table freq_table[] = { + {0, 0, CPUFREQ_ENTRY_INVALID}, /* 200Mhz is ignored */ + {0, 0, CPUFREQ_ENTRY_INVALID}, /* 1200Mhz is ignored */ + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_TABLE_END}, +}; + +static void __init fill_freq_table(struct cpufreq_frequency_table *ft) { int i; + unsigned long freq_off; unsigned char external_clk; - unsigned long max_rate, freq_off; - - max_rate = get_cpu_freq() / 1000; - external_clk = *((unsigned char *)__va(MB_EXTCLK)); if (external_clk == 240) @@ -53,19 +103,22 @@ static int __init sw64_cpufreq_init(void) else freq_off = 50000; - /* clock table init */ + freq_table[2].frequency = freq_off * 36; + for (i = 3; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) + freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); +} +#endif + +static int __init sw64_cpufreq_init(void) +{ + int i; + unsigned long max_rate = get_cpu_freq() / 1000; + + fill_freq_table(freq_table); for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { - if (i == 1) - freq_table[i].driver_data = freq_off * 24; - if (i == 2) - freq_table[i].frequency = freq_off * 36; - if (i > 2) - freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); - - if (freq_table[i].frequency == max_rate) - freq_table[i + 1].frequency = CPUFREQ_TABLE_END; + if (max_rate == freq_table[i].frequency) + freq_table[i+1].frequency = CPUFREQ_TABLE_END; } - return platform_device_register(&sw64_cpufreq_device); } arch_initcall(sw64_cpufreq_init); @@ -73,7 +126,7 @@ arch_initcall(sw64_cpufreq_init); static struct clk cpu_clk = { .name = "cpu_clk", .flags = CLK_ALWAYS_ENABLED | CLK_RATE_PROPAGATES, - .rate = 2400000000, + .rate = STARTUP_RATE, }; struct clk *sw64_clk_get(struct device *dev, const char *id) @@ -89,6 +142,7 @@ unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy) struct cpufreq_frequency_table *ft = policy->freq_table; val = sw64_io_read(0, CLK_CTL) >> CORE_PLL2_CFG_SHIFT; + val &= CORE_PLL2_CFG_MASK; for (i = 0; ft[i].frequency != CPUFREQ_TABLE_END; i++) { if (val == i) @@ -98,41 +152,29 @@ unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy) } EXPORT_SYMBOL(__sw64_cpufreq_get); -void sw64_set_rate(unsigned int index) +int sw64_set_rate(unsigned int index) { - unsigned int i, val; - int cpu_num; + int i, retry, cpu_num; cpu_num = sw64_chip->get_cpu_num(); - for (i = 0; i < cpu_num; i++) { - sw64_io_write(i, CLK_CTL, CORE_CLK2_R | CORE_CLK2_V | CLK_PRT); - val = sw64_io_read(i, CLK_CTL); - - sw64_io_write(i, CLK_CTL, val | index << CORE_PLL2_CFG_SHIFT); - + sw64_io_write(i, CLU_LV1_SEL, CLK_LV1_SEL_PROTECT); + sw64_io_write(i, CLK_CTL, CLK2_PROTECT | CORE_CLK2_RESET | CORE_CLK2_VALID | CLK0_PROTECT); + sw64_io_write(i, CLK_CTL, CLK2_PROTECT | CORE_CLK2_VALID | (unsigned long)index << CORE_PLL2_CFG_SHIFT); udelay(1); - - sw64_io_write(i, CLK_CTL, CORE_CLK2_V | CLK_PRT - | index << CORE_PLL2_CFG_SHIFT); - val = sw64_io_read(i, CLK_CTL); - - /* LV1 select PLL1/PLL2 */ - sw64_io_write(i, CLU_LV1_SEL, CLK_LV1_SEL_MUXA | CLK_LV1_SEL_PRT); - - /* Set CLK_CTL PLL0 */ - sw64_io_write(i, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V); - - sw64_io_write(i, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V - | index << CORE_PLL0_CFG_SHIFT); - - udelay(1); - - sw64_io_write(i, CLK_CTL, val | CORE_CLK0_V - | index << CORE_PLL0_CFG_SHIFT); - - /* LV1 select PLL0/PLL1 */ - sw64_io_write(i, CLU_LV1_SEL, CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PRT); + sw64_io_write(i, CLK_CTL, CORE_CLK2_VALID); + retry = 0; + while (retry < MAX_RETRY) { + if (sw64_io_read(i, CLK_CTL) & CORE_CLK2_LOCK) + break; + retry++; + udelay(100); + } + if (retry == MAX_RETRY) + return -ETIME; + sw64_io_write(i, CLK_CTL, 0); + sw64_io_write(i, CLU_LV1_SEL, CLK_LV1_SEL_MUXA | CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PROTECT); } + return 0; } EXPORT_SYMBOL_GPL(sw64_set_rate); diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 23b2e3b2d8c6..80d3dc2505bc 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -317,7 +317,6 @@ endif if SW64 config SW64_CPUFREQ bool "SW64 CPU Frequency interface" - depends on UNCORE_XUELANG default y help This adds the CPUFreq driver for SW64 processor which supports diff --git a/drivers/cpufreq/sw64_cpufreq_debugfs.c b/drivers/cpufreq/sw64_cpufreq_debugfs.c index bb4ae26bc22b..f72dbb9d5e16 100644 --- a/drivers/cpufreq/sw64_cpufreq_debugfs.c +++ b/drivers/cpufreq/sw64_cpufreq_debugfs.c @@ -8,25 +8,22 @@ #include #include +/* Show cpufreq in Mhz */ static int cpufreq_show(struct seq_file *m, void *v) { int i; u64 val; - int freq; - - val = sw64_io_read(0, CLK_CTL); - val = val >> CORE_PLL2_CFG_SHIFT; + val = sw64_io_read(0, CLK_CTL) >> CORE_PLL2_CFG_SHIFT; + val &= CORE_PLL2_CFG_MASK; + seq_puts(m, "CPU frequency in Mhz:\n"); for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { - if (freq_table[i].frequency != CPUFREQ_ENTRY_INVALID) - freq = freq_table[i].frequency; - else - freq = freq_table[i].driver_data; - + if (freq_table[i].frequency == CPUFREQ_ENTRY_INVALID) + continue; if (val == i) - seq_printf(m, "[%d] ", freq); + seq_printf(m, "[%d] ", freq_table[i].frequency / 1000); else - seq_printf(m, "%d ", freq); + seq_printf(m, "%d ", freq_table[i].frequency / 1000); } seq_puts(m, "\n"); @@ -53,15 +50,10 @@ static ssize_t cpufreq_set(struct file *file, const char __user *user_buf, err = kstrtoint(buf, 10, &cf); if (err) return err; - + cf *= 1000; /* convert Mhz to khz */ index = -1; for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { - if (freq_table[i].frequency != CPUFREQ_ENTRY_INVALID) - freq = freq_table[i].frequency; - else - freq = freq_table[i].driver_data; - - if (cf == freq) { + if (cf == freq_table[i].frequency) { index = i; break; } -- Gitee From 6867ce434d79b0aab2ee57753476c95ea735882c Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 23 May 2024 09:38:34 +0800 Subject: [PATCH 292/524] sw64: platform: add misc platform driver Add a platform driver to sunway platform. The driver is bound to platform devices that hold base addresses for some sunway platform specific peripherals. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/platform.h | 8 ++ drivers/platform/sw64/Makefile | 2 + drivers/platform/sw64/misc-platform.c | 169 ++++++++++++++++++++++++++ 3 files changed, 179 insertions(+) create mode 100644 drivers/platform/sw64/misc-platform.c diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h index 4f93d9532f9a..df5e570ccc5f 100644 --- a/arch/sw_64/include/asm/platform.h +++ b/arch/sw_64/include/asm/platform.h @@ -33,4 +33,12 @@ extern void fix_jm585_reset(void); extern void early_parse_fdt_property(const void *fdt, const char *path, const char *prop_name, u64 *property, int size); +extern void __iomem *misc_platform_get_spbu_base(unsigned long node); +extern void __iomem *misc_platform_get_intpu_base(unsigned long node); +extern void __iomem *misc_platform_get_gpio_base(unsigned long node); + +#ifdef CONFIG_SUBARCH_C3B +extern void __iomem *misc_platform_get_cab0_base(unsigned long node); +#endif + #endif /* _ASM_SW64_PLATFORM_H */ diff --git a/drivers/platform/sw64/Makefile b/drivers/platform/sw64/Makefile index 3efa835ade5d..7a7f7ac1d0da 100644 --- a/drivers/platform/sw64/Makefile +++ b/drivers/platform/sw64/Makefile @@ -1,3 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_PLATFORM_XUELANG) += legacy_xuelang.o obj-$(CONFIG_PLATFORM_JUNZHANG) += legacy_junzhang.o +obj-y += misc-platform.o + diff --git a/drivers/platform/sw64/misc-platform.c b/drivers/platform/sw64/misc-platform.c new file mode 100644 index 000000000000..8a5fe92b1d81 --- /dev/null +++ b/drivers/platform/sw64/misc-platform.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define pr_fmt(fmt) "sunway-misc-platform: " fmt + +#include +#include +#include + +#include + +struct misc_platform { + void __iomem *spbu_base; + void __iomem *intpu_base; + void __iomem *gpio_base; +}; + +static struct misc_platform misc_platform_devices[MAX_NUMNODES]; + +#ifdef CONFIG_OF +static const struct of_device_id misc_platform_of_match[] = { + { .compatible = "sunway,misc-platform" }, + {}, +}; +#endif + +#ifdef CONFIG_ACPI +static const struct acpi_device_id misc_platform_acpi_match[] = { + { "SUNW0200", 0 }, + {}, +}; +#endif + +static int misc_platform_get_node(struct device *dev) +{ + int nid = NUMA_NO_NODE; + unsigned long long pxm; + struct device_node *np; + acpi_status status; + + /* Try to directly get the physical node ID */ + if (acpi_disabled) { + /** + * We don't use the function of_node_to_nid() + * in case CONFIG_NUMA=n. + */ + np = of_node_get(dev->of_node); + of_property_read_u32(np, "numa-node-id", &nid); + of_node_put(np); + } else { + /** + * We don't use the function acpi_get_node() here + * beacuse we want physical node ID instead of the + * logical one. + */ + status = acpi_evaluate_integer(ACPI_HANDLE(dev), + "_PXM", NULL, &pxm); + if (ACPI_SUCCESS(status)) + nid = (int)pxm; + } + + return nid; +} + +static int misc_platform_probe(struct platform_device *pdev) +{ + int ret, node; + u64 base_address; + void __iomem *spbu_base = NULL; + void __iomem *intpu_base = NULL; + void __iomem *gpio_base = NULL; + struct device *dev = &pdev->dev; + + node = misc_platform_get_node(dev); + if (node == NUMA_NO_NODE) { + pr_err("unable to get node ID\n"); + return ret; + } + + if (!device_property_read_u64(dev, "sunway,spbu_base", + &base_address)) + spbu_base = __va(base_address); + + if (!device_property_read_u64(dev, "sunway,intpu_base", + &base_address)) + intpu_base = __va(base_address); + + if (!device_property_read_u64(dev, "sunway,gpio_base", + &base_address)) + gpio_base = __va(base_address); + + misc_platform_devices[node].spbu_base = spbu_base; + misc_platform_devices[node].intpu_base = intpu_base; + misc_platform_devices[node].gpio_base = gpio_base; + + pr_info("misc-platform on node %d found\n", node); + + return 0; +} + +void __iomem *misc_platform_get_spbu_base(unsigned long node) +{ + void __iomem *spbu_base; + + if (node >= MAX_NUMNODES) + return NULL; + + spbu_base = misc_platform_devices[node].spbu_base; + + /* Fallback to legacy address */ + if (!spbu_base) + return __va(SW64_IO_BASE(node) | SPBU_BASE); + + return spbu_base; +} + +void __iomem *misc_platform_get_intpu_base(unsigned long node) +{ + void __iomem *intpu_base; + + if (node >= MAX_NUMNODES) + return NULL; + + intpu_base = misc_platform_devices[node].intpu_base; + + /* Fallback to legacy address */ + if (!intpu_base) + return __va(SW64_IO_BASE(node) | INTPU_BASE); + + return intpu_base; +} + +void __iomem *misc_platform_get_gpio_base(unsigned long node) +{ + void __iomem *gpio_base; + + if (node >= MAX_NUMNODES) + return NULL; + + gpio_base = misc_platform_devices[node].gpio_base; + + /* Fallback to legacy address */ + if (!gpio_base) + return __va(SW64_IO_BASE(node) | GPIO_BASE); + + return gpio_base; +} + +#ifdef CONFIG_SUBARCH_C3B +void __iomem *misc_platform_get_cab0_base(unsigned long node) +{ + return __va(SW64_IO_BASE(node) | CAB0_BASE); +} +#endif + +static struct platform_driver misc_platform_driver = { + .probe = misc_platform_probe, + .driver = { + .name = "sunway-misc-platform", + .of_match_table = of_match_ptr(misc_platform_of_match), + .acpi_match_table = ACPI_PTR(misc_platform_acpi_match), + }, +}; + +static int __init misc_platform_driver_init(void) +{ + return platform_driver_register(&misc_platform_driver); +} +arch_initcall(misc_platform_driver_init); + -- Gitee From 5700375ea656e43f8fdede8d05ec92e860f82984 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 9 Jul 2024 09:35:10 +0800 Subject: [PATCH 293/524] sw64: remove sw64_io related functions Remove sw64_io related functions to decouple architecture related kernel code and hardware. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpufreq.h | 3 + arch/sw_64/include/asm/irq.h | 12 + arch/sw_64/include/asm/pci.h | 74 +++++- arch/sw_64/include/asm/platform.h | 2 + arch/sw_64/include/asm/sw64io.h | 18 -- arch/sw_64/include/asm/topology.h | 2 - arch/sw_64/include/asm/uncore_io_junzhang.h | 138 ---------- .../include/asm/uncore_io_ops_junzhang.h | 27 +- .../sw_64/include/asm/uncore_io_ops_xuelang.h | 46 +--- arch/sw_64/include/asm/uncore_io_xuelang.h | 247 +----------------- arch/sw_64/kernel/chip_setup.c | 52 +++- arch/sw_64/kernel/setup.c | 14 +- arch/sw_64/kernel/smp.c | 22 +- arch/sw_64/kernel/topology.c | 8 +- arch/sw_64/kvm/kvm_core3.c | 8 +- arch/sw_64/pci/pci-legacy.c | 28 +- arch/sw_64/platform/cpufreq.c | 31 ++- drivers/clocksource/timer-sw64.c | 23 +- drivers/cpufreq/sw64_cpufreq_debugfs.c | 3 +- drivers/irqchip/Kconfig | 1 + drivers/irqchip/irq-sunway-cpu.c | 52 ---- drivers/irqchip/irq-sunway-pintc.c | 207 +++++++++++++-- drivers/pci/controller/pci-sunway.c | 22 +- 23 files changed, 448 insertions(+), 592 deletions(-) diff --git a/arch/sw_64/include/asm/cpufreq.h b/arch/sw_64/include/asm/cpufreq.h index 70dfc67cf4e8..93244b3b18c6 100644 --- a/arch/sw_64/include/asm/cpufreq.h +++ b/arch/sw_64/include/asm/cpufreq.h @@ -70,6 +70,9 @@ struct clk { #define STARTUP_RATE (2400UL * 1000 * 1000) #endif +#define OFFSET_CLU_LV1_SEL 0x3a80UL +#define OFFSET_CLK_CTL 0x3b80UL + extern struct cpufreq_frequency_table freq_table[]; int clk_init(void); diff --git a/arch/sw_64/include/asm/irq.h b/arch/sw_64/include/asm/irq.h index 13f3dbac3491..de3530b687e0 100644 --- a/arch/sw_64/include/asm/irq.h +++ b/arch/sw_64/include/asm/irq.h @@ -41,12 +41,24 @@ extern int __init sw64_add_gsi_domain_map(u32 gsi_base, u32 gsi_count, #ifdef CONFIG_SW64_PINTC extern int __init pintc_acpi_init(struct irq_domain *parent, struct acpi_madt_sw_pintc *pintc); +extern void handle_dev_int(struct pt_regs *regs); +extern void handle_fault_int(void); #else static inline int __init pintc_acpi_init(struct irq_domain *parent, struct acpi_madt_sw_pintc *pintc) { return 0; } + +static inline void handle_dev_int(struct pt_regs *regs) +{ + pr_crit("Enter MCU int, but the driver is not configured!\n"); +} + +static inline void handle_fault_int(void) +{ + pr_crit("Enter fault int, but the driver is not configured!\n"); +} #endif #ifdef CONFIG_SW64_LPC_INTC diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index e43edf203e38..877aa0107539 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -8,9 +8,77 @@ #include #include -/* - * The following structure is used to manage multiple PCI busses. - */ +/* Offset for PCIe controller registers */ +enum SUNWAY_PIU_IOR0 { + PIUCONFIG0 = 0x0UL, + EPDMABAR = 0x80UL, + IOMMUSEGITEM0 = 0x100UL, + IOMMUEXCPT_CTRL = 0x2100UL, + MSIADDR = 0x2180UL, + MSICONFIG0 = 0x2200UL, + INTACONFIG = 0xa200UL, + INTBCONFIG = 0xa280UL, + INTCCONFIG = 0xa300UL, + INTDCONFIG = 0xa380UL, + AERERRINTCONFIG = 0xa400UL, + AERERRMSICONFIG = 0xa480UL, + PMEINTCONFIG = 0xa500UL, + PMEMSICONFIG = 0xa580UL, + HPINTCONFIG = 0xa600UL, + HPMSICONFIG = 0xa680UL, + HP_CTRL = 0xac80UL, /* verison >= junzhang */ + HP_WATCHOUT = 0xae00UL, /* version >= junzhang */ + DTBASEADDR = 0xb000UL, + DTLB_FLUSHALL = 0xb080UL, + DTLB_FLUSHDEV = 0xb100UL, + PTLB_FLUSHALL = 0xb180UL, + PTLB_FLUSHDEV = 0xb200UL, + PTLB_FLUSHVADDR = 0xb280UL, + PCACHE_FLUSHALL = 0xb300UL, + PCACHE_FLUSHDEV = 0xb380UL, + PCACHE_FLUSHPADDR = 0xb400UL, + TIMEOUT_CONFIG = 0xb480UL, + IOMMUEXCPT_STATUS = 0xb500UL, + IOMMUPAGE_PADDR1 = 0xb580UL, + IOMMUPAGE_PADDR2 = 0xb600UL, + IOMMUPAGE_PADDR3 = 0xb680UL, + PTLB_ACCESS = 0xb700UL, + PTLB_ITEM_TAG = 0xb780UL, + PTLB_ITEM_DATA = 0xb800UL, + PCACHE_ACCESS = 0xb880UL, + PCACHE_ITEM_TAG = 0xb900UL, + PCACHE_ITEM_DATA0 = 0xb980UL, +}; + +enum SUNWAY_PIU_IOR1 { + PIUCONFIG1 = 0x0UL, + NEWLTSSMSTATE0 = 0x300UL, /* verison >= junzhang */ + ERRENABLE = 0x880UL, + RCDEBUGINF1 = 0xc80UL, + DCACONTROL = 0x1a00UL, + DEVICEID0 = 0x1a80UL, +}; + +enum SUNWAY_RC { + RC_VENDOR_ID = 0x0UL, + RC_COMMAND = 0x80UL, + RC_REVISION_ID = 0x100UL, + RC_PRIMARY_BUS = 0x300UL, + RC_MSI_CONTROL = 0xa00UL, + RC_EXP_DEVCAP = 0xe80UL, + RC_EXP_DEVCTL = 0xf00UL, + RC_SLOT_CTRL = 0x1100UL, + RC_LINK_STAT = 0x1000UL, + RC_CONTROL = 0X1180UL, + RC_STATUS = 0X1200UL, + RC_EXP_DEVCTL2 = 0x1300UL, + RC_PORT_LINK_CTL = 0xe200UL, + RC_ORDER_RULE_CTL = 0x11680UL, + RC_MISC_CONTROL_1 = 0x11780UL, + RC_PHY_INT_REG = 0x80000UL, + RC_PHY_EXT_GEN1 = 0x82400UL, + RC_PHY_EXT_GEN2 = 0x82480UL, +}; struct pci_dev; struct pci_bus; diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h index df5e570ccc5f..c44116df8e16 100644 --- a/arch/sw_64/include/asm/platform.h +++ b/arch/sw_64/include/asm/platform.h @@ -41,4 +41,6 @@ extern void __iomem *misc_platform_get_gpio_base(unsigned long node); extern void __iomem *misc_platform_get_cab0_base(unsigned long node); #endif +extern bool sunway_machine_is_compatible(const char *compat); + #endif /* _ASM_SW64_PLATFORM_H */ diff --git a/arch/sw_64/include/asm/sw64io.h b/arch/sw_64/include/asm/sw64io.h index e9c4a3be95ef..925206ba7d6e 100644 --- a/arch/sw_64/include/asm/sw64io.h +++ b/arch/sw_64/include/asm/sw64io.h @@ -20,24 +20,6 @@ #define MK_PIU_IOR1(nid, idx) \ (SW64_PCI_IO_BASE((nid), (idx)) | PCI_IOR1_BASE) -static inline unsigned long -sw64_io_read(unsigned long node, unsigned long reg) -{ - void __iomem *addr; - - addr = __va(SW64_IO_BASE(node) | reg); - return readq(addr); -} - -static inline void -sw64_io_write(unsigned long node, unsigned long reg, unsigned long data) -{ - void __iomem *addr; - - addr = __va(SW64_IO_BASE(node) | reg); - writeq(data, addr); -} - #if defined(CONFIG_UNCORE_XUELANG) #include #endif diff --git a/arch/sw_64/include/asm/topology.h b/arch/sw_64/include/asm/topology.h index 25ec7b9e9431..c4da0cfcceff 100644 --- a/arch/sw_64/include/asm/topology.h +++ b/arch/sw_64/include/asm/topology.h @@ -62,8 +62,6 @@ static inline void numa_remove_cpu(unsigned int cpu) { } static inline void numa_store_cpu_info(unsigned int cpu) { } #endif /* CONFIG_NUMA */ -extern void get_vt_smp_info(void); - #include static inline void arch_fix_phys_package_id(int num, u32 slot) { } diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index e21876f70d1e..24c1e3be32f3 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -71,142 +71,4 @@ #define PIUCONFIG0_INIT_VAL 0x38016 -/*-----------------------addr-----------------------*/ -/* INTPU REG */ -enum { - DEVINT_MISS = INTPU_BASE | 0x100UL, - MT_INT_CONFIG = INTPU_BASE | 0x300UL, - DEV_INT_CONFIG = INTPU_BASE | 0x480UL, - FMT_ERR = INTPU_BASE | 0x700UL, - FAULT_INT_CONFIG = INTPU_BASE | 0x780UL, - SERR_CNTTH = INTPU_BASE | 0x880UL, - SPBUSERR_CNT = INTPU_BASE | 0x900UL, - IRUSERR_CNT = INTPU_BASE | 0xa80UL, - ERRRPT_EN = INTPU_BASE | 0xb00UL, - IINT_MISS_VECTOR0 = INTPU_BASE | 0x1080UL, - IINT_MISS_VECTOR1 = INTPU_BASE | 0x1100UL, - IINT_MISS = INTPU_BASE | 0x1180UL, - IINT_MISS_RPTEN = INTPU_BASE | 0x1200UL, - DEVINT_MISS_RPTEN = INTPU_BASE | 0x1280UL, - ECCSERR = INTPU_BASE | 0x1300UL, - ECCSERR_RPTEN = INTPU_BASE | 0x1380UL, - ECCMERR = INTPU_BASE | 0x1400UL, - ECCMERR_RPTEN = INTPU_BASE | 0x1480UL, - DEVINT_WKEN = INTPU_BASE | 0x1500UL, - ADR_INT_CONFIG = INTPU_BASE | 0x1580UL, - DEVINTWK_INTEN = INTPU_BASE | 0x1600UL, -}; - -/* SPBU CSR */ -enum { - SMP_INFO = SPBU_BASE | 0x80UL, - INIT_CTL = SPBU_BASE | 0x680UL, - CORE_ONLINE = SPBU_BASE | 0x780UL, - DLI_RLTD_FAULT = SPBU_BASE | 0x980UL, - DLI_RLTD_FAULT_EN = SPBU_BASE | 0xa00UL, - DLI_RLTD_FAULT_INTEN = SPBU_BASE | 0xa80UL, - CFG_INFO = SPBU_BASE | 0x1100UL, - IO_START = SPBU_BASE | 0x1300UL, - I2C0_SRST_L = SPBU_BASE | 0x1900UL, - I2C1_SRST_L = SPBU_BASE | 0x1980UL, - I2C2_SRST_L = SPBU_BASE | 0x1a00UL, - MCU_DVC_INT = SPBU_BASE | 0x3000UL, - MCU_DVC_INT_EN = SPBU_BASE | 0x3080UL, - SI_FAULT_STAT = SPBU_BASE | 0x3100UL, - SI_FAULT_STAT_EN = SPBU_BASE | 0x3180UL, - SI_FAULT_INT_EN = SPBU_BASE | 0x3200UL, - ADR_CTL = SPBU_BASE | 0x3600UL, - PIUH_CTRL = SPBU_BASE | 0x3680UL, - MC_ONLINE = SPBU_BASE | 0x3780UL, - CLK_CTL = SPBU_BASE | 0x3b80UL, - CLU_LV2_SELH = SPBU_BASE | 0x3a00UL, - CLU_LV1_SEL = SPBU_BASE | 0x3a80UL, - CLU_LV2_SELL = SPBU_BASE | 0x3b00UL, - PIU_TOP0_CONFIG = SPBU_BASE | 0x4c80UL, - PIU_TOP1_CONFIG = SPBU_BASE | 0x4d00UL, - PIU_PHY_SRST_H = SPBU_BASE | 0x6280UL, - BUTTON_RST_N_PCIE0 = SPBU_BASE | 0x6a80UL, - SOFT_INFO0 = SPBU_BASE | 0xa000UL, -}; - -/*--------------------------offset-----------------------------------*/ -/* PIU IOR0 */ -enum { - PIUCONFIG0 = 0x0UL, - EPDMABAR = 0x80UL, - IOMMUSEGITEM0 = 0x100UL, - IOMMUEXCPT_CTRL = 0x2100UL, - MSIADDR = 0x2180UL, - MSICONFIG0 = 0x2200UL, - INTACONFIG = 0xa200UL, - INTBCONFIG = 0xa280UL, - INTCCONFIG = 0xa300UL, - INTDCONFIG = 0xa380UL, - AERERRINTCONFIG = 0xa400UL, - AERERRMSICONFIG = 0xa480UL, - PMEINTCONFIG = 0xa500UL, - PMEMSICONFIG = 0xa580UL, - HPINTCONFIG = 0xa600UL, - HPMSICONFIG = 0xa680UL, - HP_CTRL = 0xac80UL, - HP_WATCHOUT = 0xae00UL, - DTBASEADDR = 0xb000UL, - DTLB_FLUSHALL = 0xb080UL, - DTLB_FLUSHDEV = 0xb100UL, - PTLB_FLUSHALL = 0xb180UL, - PTLB_FLUSHDEV = 0xb200UL, - PTLB_FLUSHVADDR = 0xb280UL, - PCACHE_FLUSHALL = 0xb300UL, - PCACHE_FLUSHDEV = 0xb380UL, - PCACHE_FLUSHPADDR = 0xb400UL, - TIMEOUT_CONFIG = 0xb480UL, - IOMMUEXCPT_STATUS = 0xb500UL, - IOMMUPAGE_PADDR1 = 0xb580UL, - IOMMUPAGE_PADDR2 = 0xb600UL, - IOMMUPAGE_PADDR3 = 0xb680UL, - PTLB_ACCESS = 0xb700UL, - PTLB_ITEM_TAG = 0xb780UL, - PTLB_ITEM_DATA = 0xb800UL, - PCACHE_ACCESS = 0xb880UL, - PCACHE_ITEM_TAG = 0xb900UL, - PCACHE_ITEM_DATA0 = 0xb980UL, -}; - -/* PIU IOR1 */ -enum { - PIUCONFIG1 = 0x0UL, - NEWLTSSMSTATE0 = 0x300UL, - ERRENABLE = 0x880UL, - RCDEBUGINF1 = 0xc80UL, - DCACONTROL = 0x1a00UL, - DEVICEID0 = 0x1a80UL, -}; - -/* RC */ -enum { - RC_VENDOR_ID = 0x0UL, - RC_COMMAND = 0x80UL, - RC_REVISION_ID = 0x100UL, - RC_PRIMARY_BUS = 0x300UL, - RC_MSI_CONTROL = 0xa00UL, - RC_EXP_DEVCAP = 0xe80UL, - RC_EXP_DEVCTL = 0xf00UL, - RC_SLOT_CTRL = 0x1100UL, - RC_LINK_STAT = 0x1000UL, - RC_CONTROL = 0X1180UL, - RC_STATUS = 0X1200UL, - RC_EXP_DEVCTL2 = 0x1300UL, - RC_PORT_LINK_CTL = 0xe200UL, - RC_ORDER_RULE_CTL = 0x11680UL, - RC_MISC_CONTROL_1 = 0x11780UL, - RC_PHY_INT_REG = 0x80000UL, - RC_PHY_EXT_GEN1 = 0x82400UL, - RC_PHY_EXT_GEN2 = 0x82480UL, -}; -/* GPIO */ -enum { - GPIO_SWPORTA_DR = GPIO_BASE | 0x0UL, - GPIO_SWPORTA_DDR = GPIO_BASE | 0x200UL, -}; -/*--------------------------------------------------------------------------*/ #endif /* _ASM_SW64_UNCORE_IO_JUNZHANG_H */ diff --git a/arch/sw_64/include/asm/uncore_io_ops_junzhang.h b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h index 00f9898f8540..eff4f40886d8 100644 --- a/arch/sw_64/include/asm/uncore_io_ops_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h @@ -2,12 +2,17 @@ #ifndef _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H #define _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H +#define OFFSET_CFG_INFO 0x1100UL + static inline int __get_cpu_nums(void) { int cpus; unsigned long cfg_info; + void __iomem *spbu_base; + + spbu_base = misc_platform_get_spbu_base(0); - cfg_info = sw64_io_read(0, CFG_INFO); + cfg_info = readq(spbu_base + OFFSET_CFG_INFO); cfg_info = (cfg_info >> 33) & 0x3; cpus = 1 << cfg_info; @@ -18,27 +23,15 @@ static inline unsigned long __get_node_mem(int node) { unsigned long node_mem; unsigned long total_mem; + void __iomem *spbu_base; + + spbu_base = misc_platform_get_spbu_base(node); - total_mem = sw64_io_read(node, CFG_INFO) >> 3; + total_mem = readq(spbu_base + OFFSET_CFG_INFO) >> 3; total_mem = (total_mem & 0xffff) << 28; node_mem = total_mem / __get_cpu_nums(); return node_mem; } -#define __io_read_longtime(node) (0UL) -#define __io_write_longtime(node, data) do { } while (0) - -static inline void -__io_write_longtime_start_en(int node, unsigned long data) -{ - sw64_io_write(node, GPIO_SWPORTA_DDR, data); -} - -static inline void -__io_write_fault_int_en(int node, unsigned long data) -{ - sw64_io_write(node, FAULT_INT_CONFIG, data); -} - #endif /* _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H */ diff --git a/arch/sw_64/include/asm/uncore_io_ops_xuelang.h b/arch/sw_64/include/asm/uncore_io_ops_xuelang.h index 9336e473211d..e96c3669b577 100644 --- a/arch/sw_64/include/asm/uncore_io_ops_xuelang.h +++ b/arch/sw_64/include/asm/uncore_io_ops_xuelang.h @@ -2,12 +2,20 @@ #ifndef _ASM_SW64_UNCORE_IO_OPS_XUELANG_H #define _ASM_SW64_UNCORE_IO_OPS_XUELANG_H +#define OFFSET_TRKMODE 0x80UL + +#define OFFSET_MC_CAP_CFG 0x1180UL +#define OFFSET_MC_ONLINE 0x3780UL + static inline int __get_cpu_nums(void) { int cpus; unsigned long trkmode; + void __iomem *cab0_base; - trkmode = sw64_io_read(0, TRKMODE); + cab0_base = misc_platform_get_cab0_base(0); + + trkmode = readq(cab0_base + OFFSET_TRKMODE); trkmode = (trkmode >> 6) & 0x3; cpus = 1 << trkmode; @@ -21,45 +29,15 @@ static inline unsigned long __get_node_mem(int node) unsigned long mc_online; unsigned long mc_cap; unsigned long mc_num; + void __iomem *mcu_base = misc_platform_get_spbu_base(node); - mc_config = sw64_io_read(node, MC_CAP_CFG) & 0xf; + mc_config = readq(mcu_base + OFFSET_MC_CAP_CFG) & 0xf; mc_cap = (1UL << mc_config) << 28; - mc_online = sw64_io_read(node, MC_ONLINE) & 0xff; + mc_online = readq(mcu_base + OFFSET_MC_ONLINE) & 0xff; mc_num = __kernel_ctpop(mc_online); node_mem = mc_cap * mc_num; return node_mem; } -static inline unsigned long -__io_read_longtime(int node) -{ - return sw64_io_read(node, LONG_TIME); -} - -static inline void -__io_write_longtime(int node, unsigned long data) -{ - sw64_io_write(node, LONG_TIME, data); -} - -static inline void -__io_write_longtime_start_en(int node, unsigned long data) -{ - sw64_io_write(node, LONG_TIME_START_EN, data); -} - -static inline void -__io_write_fault_int_en(int node, unsigned long data) -{ - sw64_io_write(node, DUAL_CG0_FAULT_INTEN, data); - sw64_io_write(node, DUAL_CG1_FAULT_INTEN, data); - sw64_io_write(node, DUAL_CG2_FAULT_INTEN, data); - sw64_io_write(node, DUAL_CG3_FAULT_INTEN, data); - sw64_io_write(node, DUAL_CG4_FAULT_INTEN, data); - sw64_io_write(node, DUAL_CG5_FAULT_INTEN, data); - sw64_io_write(node, DUAL_CG6_FAULT_INTEN, data); - sw64_io_write(node, DUAL_CG7_FAULT_INTEN, data); -} - #endif /* _ASM_SW64_UNCORE_IO_OPS_XUELANG_H */ diff --git a/arch/sw_64/include/asm/uncore_io_xuelang.h b/arch/sw_64/include/asm/uncore_io_xuelang.h index 7a8c9edc8bfb..c8f01cb01e52 100644 --- a/arch/sw_64/include/asm/uncore_io_xuelang.h +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -48,6 +48,7 @@ #define PCI0_BUS SW64_PCI0_BUS #define MCU_BASE (0x3UL << 36) +#define SPBU_BASE MCU_BASE #define CAB0_BASE (0x10UL << 32) #define INTPU_BASE (0x2aUL << 32) #define IIC0_BASE (0x31UL << 32) @@ -73,250 +74,4 @@ #define PIUCONFIG0_INIT_VAL 0x38056 -/*-----------------------addr-----------------------*/ -/* CAB0 REG */ -enum { - TRKMODE = CAB0_BASE | 0x80UL, -}; - -/* DLIA IO REG */ -enum { - DLIA_BWTEST_PAT = DLIA_BASE | 0x100980UL, - DLIA_PHY_VLDLANE = DLIA_BASE | DLI_PHY_CTL | 0x300UL, -}; - -/* DLIB IO REG */ -enum { - DLIB_BWTEST_PAT = DLIB_BASE | 0x100980UL, - DLIB_PHY_VLDLANE = DLIB_BASE | DLI_PHY_CTL | 0x300UL, -}; - -/* DLIC IO REG */ -enum { - DLIC_BWTEST_PAT = DLIC_BASE | 0x100980UL, - DLIC_PHY_VLDLANE = DLIC_BASE | DLI_PHY_CTL | 0x300UL, -}; -/* INTPU REG */ -enum { - LCORE_SLEEPY = INTPU_BASE | 0x0UL, - LCORE_SLEEP = INTPU_BASE | 0x80UL, - DEVICE_MISS = INTPU_BASE | 0x100UL, - LONG_TIME = INTPU_BASE | 0x180UL, - LCORE_IDLE = INTPU_BASE | 0x280UL, - MT_INT_CONFIG = INTPU_BASE | 0x300UL, - DEV_INT_CONFIG = INTPU_BASE | 0x480UL, - FMT_ERR = INTPU_BASE | 0x700UL, - FAULT_INT_CONFIG = INTPU_BASE | 0x780UL, - SERR_CNTTH = INTPU_BASE | 0x880UL, - MCUSERR_CNT = INTPU_BASE | 0x900UL, - IRUSERR_CNT = INTPU_BASE | 0xa80UL, - ERRRPT_EN = INTPU_BASE | 0xb00UL, - IINT_MISS_VECTOR = INTPU_BASE | 0x1100UL, - IINT_MIS = INTPU_BASE | 0x1180UL, - IINT_MISS_RPTEN = INTPU_BASE | 0x1200UL, - DEVINT_MISS_RPTEN = INTPU_BASE | 0x1280UL, - ECCSERR = INTPU_BASE | 0x1300UL, - ECCSERR_RPTEN = INTPU_BASE | 0x1380UL, - ECCMERR = INTPU_BASE | 0x1400UL, - ECCMERR_RPTEN = INTPU_BASE | 0x1480UL, - DEVINT_WKEN = INTPU_BASE | 0x1500UL, - NMI_INT_CONFIG = INTPU_BASE | 0x1580UL, - DEVINTWK_INTEN = INTPU_BASE | 0x1600UL, -}; - -/* MC IO REG */ -enum { - CFGDEC = 0x400UL, - CFGCR = 0x480UL, - INIT_CTRL = 0x580UL, - CFGERR = 0xd00UL, - FSMSTAT = 0xe00UL, - PUB_INTERFACE = 0x1000UL, - POWERCTRL = 0x1080UL, - CFGMR0 = 0x1280UL, - CFGMR1 = 0x1300UL, - CFGMR2 = 0x1380UL, - CFGMR3 = 0x1400UL, - PERF_CTRL = 0x1480UL, - MC_PERF0 = 0x1500UL, - CFGMR4 = 0x1800UL, - CFGMR5 = 0x1880UL, - CFGMR6 = 0x1900UL, - MC_CTRL = 0x1c00UL, - MEMSERR_P = 0x1c80UL, - MEMSERR = 0x1d00UL, -}; - -/* MCU CSR */ -enum { - SMP_INFO = MCU_BASE | 0x80UL, - INIT_CTL = MCU_BASE | 0x680UL, - MT_STATE = MCU_BASE | 0x700UL, - CORE_ONLINE = MCU_BASE | 0x780UL, - MT_INT = MCU_BASE | 0x800UL, - MT_INT_END = MCU_BASE | 0x880UL, - CPU_ID = MCU_BASE | 0x900UL, - DLI_RLTD_FAULT = MCU_BASE | 0x980UL, - DLI_RLTD_FAULT_EN = MCU_BASE | 0xa00UL, - DLI_RLTD_FAULT_INTEN = MCU_BASE | 0xa80UL, - FAULT_SOURCE = MCU_BASE | 0xb00UL, - INT_SOURCE = MCU_BASE | 0xb80UL, - CORE_STATE0 = MCU_BASE | 0xc00UL, - CORE_STATE1 = MCU_BASE | 0xc80UL, - CFG_INFO = MCU_BASE | 0x1100UL, - MC_CAP_CFG = MCU_BASE | 0x1180UL, - IO_START = MCU_BASE | 0x1300UL, - UART_ONLINE = MCU_BASE | 0x1780UL, - I2C0_SRST_L = MCU_BASE | 0x1900UL, - I2C1_SRST_L = MCU_BASE | 0x1980UL, - I2C2_SRST_L = MCU_BASE | 0x1a00UL, - MCU_DVC_INT = MCU_BASE | 0x3000UL, - MCU_DVC_INT_EN = MCU_BASE | 0x3080UL, - SI_FAULT_STAT = MCU_BASE | 0x3100UL, - SI_FAULT_EN = MCU_BASE | 0x3180UL, - SI_FAULT_INT_EN = MCU_BASE | 0x3200UL, - FIFO_SYNSEL = MCU_BASE | 0x3400UL, - CPU_INFO = MCU_BASE | 0x3480UL, - WAKEUP_CTL = MCU_BASE | 0x3500UL, - FLAGREG = MCU_BASE | 0x3580UL, - NMI_CTL = MCU_BASE | 0x3600UL, - PIUPLL_CNT = MCU_BASE | 0x3680UL, - MC_ONLINE = MCU_BASE | 0x3780UL, - FLASH_INFO = MCU_BASE | 0x3800UL, - RTPUSROMCNT = MCU_BASE | 0x3880UL, - CLU_LV1_SEL = MCU_BASE | 0x3a80UL, - CLU_LV2_SEL = MCU_BASE | 0x3b00UL, - CLK_CTL = MCU_BASE | 0x3b80UL, - SLEEP_WAIT_CNT = MCU_BASE | 0x4980UL, - CHIP_ID = MCU_BASE | 0x4b00UL, - PIU_TOP0_CONFIG = MCU_BASE | 0x4c80UL, - PIU_TOP1_CONFIG = MCU_BASE | 0x4d00UL, - LVDS_CTL = MCU_BASE | 0x4d80UL, - LPC_DMAREQ_TOTH = MCU_BASE | 0x5100UL, - DLI_ONLINE = MCU_BASE | 0x6180UL, - LPC_DMAREQ_HADR = MCU_BASE | 0x6200UL, - PIU_PHY_SRST_H = MCU_BASE | 0x6280UL, - CLK_SEL_PCIE0 = MCU_BASE | 0x6280UL, - CLK_SEL_PCIE1 = MCU_BASE | 0x6300UL, - CLK_SEL_PCIE2 = MCU_BASE | 0x6380UL, - CLK_SEL_PCIE3 = MCU_BASE | 0x6400UL, - CLK_SEL_PCIE4 = MCU_BASE | 0x6480UL, - CLK_SEL_PCIE5 = MCU_BASE | 0x6500UL, - PERST_N_PCIE0 = MCU_BASE | 0x6680UL, - PERST_N_PCIE1 = MCU_BASE | 0x6700UL, - PERST_N_PCIE2 = MCU_BASE | 0x6780UL, - PERST_N_PCIE3 = MCU_BASE | 0x6800UL, - PERST_N_PCIE4 = MCU_BASE | 0x6880UL, - PERST_N_PCIE5 = MCU_BASE | 0x6900UL, - BUTTON_RST_N_PCIE0 = MCU_BASE | 0x6a80UL, - BUTTON_RST_N_PCIE1 = MCU_BASE | 0x6b00UL, - BUTTON_RST_N_PCIE2 = MCU_BASE | 0x6b80UL, - BUTTON_RST_N_PCIE3 = MCU_BASE | 0x6c00UL, - BUTTON_RST_N_PCIE4 = MCU_BASE | 0x6c80UL, - BUTTON_RST_N_PCIE5 = MCU_BASE | 0x6d00UL, - DUAL_CG0_FAULT = MCU_BASE | 0x6d80UL, - DUAL_CG1_FAULT = MCU_BASE | 0x6e00UL, - DUAL_CG2_FAULT = MCU_BASE | 0x6e80UL, - DUAL_CG3_FAULT = MCU_BASE | 0x6f00UL, - DUAL_CG4_FAULT = MCU_BASE | 0x6f80UL, - DUAL_CG5_FAULT = MCU_BASE | 0x7000UL, - DUAL_CG6_FAULT = MCU_BASE | 0x7080UL, - DUAL_CG7_FAULT = MCU_BASE | 0x7100UL, - DUAL_CG0_FAULT_EN = MCU_BASE | 0x7180UL, - DUAL_CG1_FAULT_EN = MCU_BASE | 0x7200UL, - DUAL_CG2_FAULT_EN = MCU_BASE | 0x7280UL, - DUAL_CG3_FAULT_EN = MCU_BASE | 0x7300UL, - DUAL_CG4_FAULT_EN = MCU_BASE | 0x7380UL, - DUAL_CG5_FAULT_EN = MCU_BASE | 0x7400UL, - DUAL_CG6_FAULT_EN = MCU_BASE | 0x7480UL, - DUAL_CG7_FAULT_EN = MCU_BASE | 0x7500UL, - DUAL_CG0_FAULT_INTEN = MCU_BASE | 0x7580UL, - DUAL_CG1_FAULT_INTEN = MCU_BASE | 0x7600UL, - DUAL_CG2_FAULT_INTEN = MCU_BASE | 0x7680UL, - DUAL_CG3_FAULT_INTEN = MCU_BASE | 0x7700UL, - DUAL_CG4_FAULT_INTEN = MCU_BASE | 0x7780UL, - DUAL_CG5_FAULT_INTEN = MCU_BASE | 0x7800UL, - DUAL_CG6_FAULT_INTEN = MCU_BASE | 0x7880UL, - DUAL_CG7_FAULT_INTEN = MCU_BASE | 0x7900UL, - SOFT_INFO0 = MCU_BASE | 0x7f00UL, - LONG_TIME_START_EN = MCU_BASE | 0x9000UL, -}; - -/*--------------------------offset-----------------------------------*/ -/* PIU IOR0 */ -enum { - PIUCONFIG0 = 0x0UL, - EPDMABAR = 0x80UL, - IOMMUSEGITEM0 = 0x100UL, - IOMMUEXCPT_CTRL = 0x2100UL, - MSIADDR = 0x2180UL, - MSICONFIG0 = 0x2200UL, - INTACONFIG = 0xa200UL, - INTBCONFIG = 0xa280UL, - INTCCONFIG = 0xa300UL, - INTDCONFIG = 0xa380UL, - AERERRINTCONFIG = 0xa400UL, - AERERRMSICONFIG = 0xa480UL, - PMEINTCONFIG = 0xa500UL, - PMEMSICONFIG = 0xa580UL, - HPINTCONFIG = 0xa600UL, - HPMSICONFIG = 0xa680UL, - DTBASEADDR = 0xb000UL, - DTLB_FLUSHALL = 0xb080UL, - DTLB_FLUSHDEV = 0xb100UL, - PTLB_FLUSHALL = 0xb180UL, - PTLB_FLUSHDEV = 0xb200UL, - PTLB_FLUSHVADDR = 0xb280UL, - PCACHE_FLUSHALL = 0xb300UL, - PCACHE_FLUSHDEV = 0xb380UL, - PCACHE_FLUSHPADDR = 0xb400UL, - TIMEOUT_CONFIG = 0xb480UL, - IOMMUEXCPT_STATUS = 0xb500UL, - IOMMUPAGE_PADDR1 = 0xb580UL, - IOMMUPAGE_PADDR2 = 0xb600UL, - IOMMUPAGE_PADDR3 = 0xb680UL, - PTLB_ACCESS = 0xb700UL, - PTLB_ITEM_TAG = 0xb780UL, - PTLB_ITEM_DATA = 0xb800UL, - PCACHE_ACCESS = 0xb880UL, - PCACHE_ITEM_TAG = 0xb900UL, - PCACHE_ITEM_DATA0 = 0xb980UL, -}; - -/* PIU IOR1 */ -enum { - PIUCONFIG1 = 0x0UL, - ERRENABLE = 0x880UL, - RCDEBUGINF1 = 0xc80UL, - DCACONTROL = 0x1a00UL, - DEVICEID0 = 0x1a80UL, -}; - -/* RC */ -enum { - RC_VENDOR_ID = 0x0UL, - RC_COMMAND = 0x80UL, - RC_REVISION_ID = 0x100UL, - RC_PRIMARY_BUS = 0x300UL, - RC_MSI_CONTROL = 0xa00UL, - RC_EXP_DEVCAP = 0xe80UL, - RC_EXP_DEVCTL = 0xf00UL, - RC_SLOT_CTRL = 0x1100UL, - RC_LINK_STAT = 0x1000UL, - RC_CONTROL = 0X1180UL, - RC_STATUS = 0X1200UL, - RC_EXP_DEVCTL2 = 0x1300UL, - RC_PORT_LINK_CTL = 0xe200UL, - RC_ORDER_RULE_CTL = 0x11680UL, - RC_MISC_CONTROL_1 = 0x11780UL, - RC_PHY_INT_REG = 0x80000UL, - RC_PHY_EXT_GEN1 = 0x82400UL, - RC_PHY_EXT_GEN2 = 0x82480UL, -}; -/* GPIO */ -enum { - GPIO_SWPORTA_DR = GPIO_BASE | 0x0UL, - GPIO_SWPORTA_DDR = GPIO_BASE | 0x200UL, -}; -/*--------------------------------------------------------------------------*/ #endif /* _ASM_SW64_UNCORE_IO_XUELANG_H */ diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index f6e92adab5e6..211d673250c7 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -6,6 +6,18 @@ #include #include +#define OFFSET_CORE_ONLINE 0x780UL +#define OFFSET_MC_ONLINE 0x3780UL +#define OFFSET_I2C0_SRST_L 0x1900UL +#define OFFSET_I2C1_SRST_L 0x1980UL +#define OFFSET_I2C2_SRST_L 0x1a00UL +#define OFFSET_MCU_DVC_INT_EN 0x3080UL +#define OFFSET_LONG_TIME_START_EN 0x9000UL + +#define OFFSET_LONG_TIME 0x180UL + +#define OFFSET_GPIO_SWPORTA_DDR 0x200UL + struct sw64_chip_ops *sw64_chip; struct sw64_chip_init_ops *sw64_chip_init; @@ -30,11 +42,13 @@ static void __init setup_core_map(struct cpumask *cpumask) { int i, j, cpu_num, cpuid, max_cores_per_cpu; unsigned long coreonline; + void __iomem *spbu_base; cpu_num = get_cpu_nums(); cpuid = 0; for (i = 0; i < cpu_num; i++) { - coreonline = sw64_io_read(i, CORE_ONLINE); + spbu_base = misc_platform_get_spbu_base(i); + coreonline = readq(spbu_base + OFFSET_CORE_ONLINE); max_cores_per_cpu = MAX_CORES_PER_CPU; if (is_guest_or_emul()) @@ -64,14 +78,16 @@ static void __init setup_core_map(struct cpumask *cpumask) #ifdef CONFIG_PM static void i2c_srst(void) { - sw64_io_write(0, I2C0_SRST_L, 0x0); - sw64_io_write(0, I2C0_SRST_L, 0x1); + void __iomem *spbu_base = misc_platform_get_spbu_base(0); + + writeq(0x0, spbu_base + OFFSET_I2C0_SRST_L); + writeq(0x1, spbu_base + OFFSET_I2C0_SRST_L); - sw64_io_write(0, I2C1_SRST_L, 0x0); - sw64_io_write(0, I2C1_SRST_L, 0x1); + writeq(0x0, spbu_base + OFFSET_I2C1_SRST_L); + writeq(0x1, spbu_base + OFFSET_I2C1_SRST_L); - sw64_io_write(0, I2C2_SRST_L, 0x0); - sw64_io_write(0, I2C2_SRST_L, 0x1); + writeq(0x0, spbu_base + OFFSET_I2C2_SRST_L); + writeq(0x1, spbu_base + OFFSET_I2C2_SRST_L); } static void pcie_save(void) @@ -178,9 +194,11 @@ static unsigned long saved_dvc_int, saved_long_time; static inline void intpu_save(void) { + void __iomem *intpu_base = misc_platform_get_intpu_base(0); + switch (cpu_desc.model) { case CPU_SW831: - saved_long_time = __io_read_longtime(0); + saved_long_time = readq(intpu_base + OFFSET_LONG_TIME); default: break; } @@ -188,13 +206,17 @@ static inline void intpu_save(void) static inline void intpu_restore(void) { + void __iomem *intpu_base = misc_platform_get_intpu_base(0); + void __iomem *spbu_base = misc_platform_get_spbu_base(0); + void __iomem *gpio_base = misc_platform_get_gpio_base(0); + switch (cpu_desc.model) { case CPU_SW831: - __io_write_longtime(0, saved_long_time); - __io_write_longtime_start_en(0, 0x1); + writeq(saved_long_time, intpu_base + OFFSET_LONG_TIME); + writeq(0x1, spbu_base + OFFSET_LONG_TIME_START_EN); break; case CPU_SW8A: - __io_write_longtime_start_en(0, 0x1); + writeq(0x1, gpio_base + OFFSET_GPIO_SWPORTA_DDR); break; default: pr_info("long time start is disable!"); @@ -204,13 +226,17 @@ static inline void intpu_restore(void) static inline void spbu_save(void) { - saved_dvc_int = sw64_io_read(0, MCU_DVC_INT_EN); + void __iomem *spbu_base = misc_platform_get_spbu_base(0); + + saved_dvc_int = readq(spbu_base + OFFSET_MCU_DVC_INT_EN); } static inline void spbu_restore(void) { + void __iomem *spbu_base = misc_platform_get_spbu_base(0); + i2c_srst(); - sw64_io_write(0, MCU_DVC_INT_EN, saved_dvc_int); + writeq(saved_dvc_int, spbu_base + OFFSET_MCU_DVC_INT_EN); } static int io_suspend(void) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 839176939efb..f55ddc856df9 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -470,6 +470,18 @@ void early_parse_fdt_property(const void *fdt, const char *path, *property = of_read_number(prop, size / 4); } +bool sunway_machine_is_compatible(const char *compat) +{ + const void *fdt = initial_boot_params; + int offset; + + offset = fdt_path_offset(fdt, "/"); + if (offset < 0) + return false; + + return !fdt_node_check_compatible(fdt, offset, compat); +} + static void __init setup_firmware_fdt(void) { void *dt_virt; @@ -691,8 +703,6 @@ setup_arch(char **cmdline_p) setup_cpu_info(); setup_run_mode(); setup_chip_ops(); - if (is_guest_or_emul()) - get_vt_smp_info(); setup_sched_clock(); diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 6bafc18f742c..63a3139bf9ba 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -63,17 +63,25 @@ enum core_version { }; #ifdef CONFIG_SUBARCH_C4 +#define OFFSET_CLU_LV2_SELH 0x3a00UL +#define OFFSET_CLU_LV2_SELL 0x3b00UL + static void upshift_freq(void) { int i, cpu_num; + void __iomem *spbu_base; if (is_guest_or_emul()) return; + if (!sunway_machine_is_compatible("sunway,junzhang")) + return; + cpu_num = sw64_chip->get_cpu_num(); for (i = 0; i < cpu_num; i++) { - sw64_io_write(i, CLU_LV2_SELH, -1UL); - sw64_io_write(i, CLU_LV2_SELL, -1UL); + spbu_base = misc_platform_get_spbu_base(i); + writeq(-1UL, spbu_base + OFFSET_CLU_LV2_SELH); + writeq(-1UL, spbu_base + OFFSET_CLU_LV2_SELL); udelay(1000); } } @@ -84,10 +92,14 @@ static void downshift_freq(void) int core_id, node_id, cpu; int cpuid = smp_processor_id(); struct cpu_topology *cpu_topo = &cpu_topology[cpuid]; + void __iomem *spbu_base; if (is_guest_or_emul()) return; + if (!sunway_machine_is_compatible("sunway,junzhang")) + return; + for_each_online_cpu(cpu) { struct cpu_topology *sib_topo = &cpu_topology[cpu]; @@ -99,12 +111,14 @@ static void downshift_freq(void) core_id = rcid_to_core_id(cpu_to_rcid(cpuid)); node_id = rcid_to_domain_id(cpu_to_rcid(cpuid)); + spbu_base = misc_platform_get_spbu_base(node_id); + if (core_id > 31) { value = 1UL << (2 * (core_id - 32)); - sw64_io_write(node_id, CLU_LV2_SELH, value); + writeq(value, spbu_base + OFFSET_CLU_LV2_SELH); } else { value = 1UL << (2 * core_id); - sw64_io_write(node_id, CLU_LV2_SELL, value); + writeq(value, spbu_base + OFFSET_CLU_LV2_SELL); } } #else diff --git a/arch/sw_64/kernel/topology.c b/arch/sw_64/kernel/topology.c index 8371c013446f..b00f083517c3 100644 --- a/arch/sw_64/kernel/topology.c +++ b/arch/sw_64/kernel/topology.c @@ -6,6 +6,8 @@ #include #include +#define OFFSET_SMP_INFO 0x80UL + static int __init parse_dt_topology(void) { return 0; @@ -24,11 +26,12 @@ static int topo_threads[NR_CPUS]; static int topo_cores[NR_CPUS]; static int topo_packages[NR_CPUS]; -void __init get_vt_smp_info(void) +static void __init get_vt_smp_info(void) { unsigned long smp_info; + void __iomem *spbu_base = misc_platform_get_spbu_base(0); - smp_info = sw64_io_read(0, SMP_INFO); + smp_info = readq(spbu_base + OFFSET_SMP_INFO); if (smp_info == -1UL) smp_info = 0; topo_nr_threads = (smp_info >> VT_THREADS_SHIFT) & VT_THREADS_MASK; @@ -79,6 +82,7 @@ static void __init init_topo_packages(void) static void __init init_topology_array(void) { + get_vt_smp_info(); topo_nr_cpus = num_present_cpus(); if (topo_nr_maxcpus > topo_nr_cpus) topo_nr_cpus = topo_nr_maxcpus; diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c index 355d7697c535..8b751c76f121 100644 --- a/arch/sw_64/kvm/kvm_core3.c +++ b/arch/sw_64/kvm/kvm_core3.c @@ -24,6 +24,8 @@ #include "trace.h" #include "vmem.c" +#define OFFSET_LONG_TIME 0x180UL + __read_mostly bool bind_vcpu_enabled; #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) @@ -240,10 +242,11 @@ int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) long kvm_sw64_get_vcb(struct file *filp, unsigned long arg) { + void __iomem *intpu_base = misc_platform_get_intpu_base(0); struct kvm_vcpu *vcpu = filp->private_data; if (vcpu->arch.vcb.migration_mark) { - unsigned long result = sw64_io_read(0, LONG_TIME) + unsigned long result = readq(intpu_base + OFFSET_LONG_TIME) + vcpu->arch.vcb.guest_longtime_offset; vcpu->arch.vcb.guest_longtime = result; vcpu->arch.vcb.guest_irqs_pending = vcpu->arch.irqs_pending[0]; @@ -260,6 +263,7 @@ long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) unsigned long result; struct kvm_vcpu *vcpu = filp->private_data; struct vcpucb *kvm_vcb; + void __iomem *intpu_base = misc_platform_get_intpu_base(0); kvm_vcb = memdup_user((void __user *)arg, sizeof(*kvm_vcb)); memcpy(&(vcpu->arch.vcb), kvm_vcb, sizeof(struct vcpucb)); @@ -269,7 +273,7 @@ long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) vcpu->arch.vcb.vpcr = get_vpcr(vcpu, 0); /* synchronize the longtime of source and destination */ if (vcpu->arch.vcb.soft_cid == 0) { - result = sw64_io_read(0, LONG_TIME); + result = readq(intpu_base + OFFSET_LONG_TIME); vcpu->arch.vcb.guest_longtime_offset = vcpu->arch.vcb.guest_longtime - result; longtime_offset = vcpu->arch.vcb.guest_longtime_offset; } else diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index 8eb5458dc2fc..379a6566f0da 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -9,6 +9,9 @@ #include #include +#define OFFSET_DEVINT_WKEN 0x1500UL +#define OFFSET_DEVINTWK_INTEN 0x1600UL + /* * The PCI controller list. */ @@ -240,8 +243,16 @@ sw64_init_host(unsigned long node, unsigned long index) } } -void __weak set_devint_wken(int node) {} -void __weak set_adr_int(int node) {} +static void set_devint_wken(int node) +{ + unsigned long val; + void __iomem *intpu_base = misc_platform_get_intpu_base(node); + + /* enable INTD wakeup */ + val = 0x80; + writeq(val, intpu_base + OFFSET_DEVINT_WKEN); + writeq(val, intpu_base + OFFSET_DEVINTWK_INTEN); +} static bool __init is_any_rc_linkup_one_node(unsigned long node) { @@ -263,18 +274,19 @@ void __init sw64_init_arch(void) char id[8], msg[64]; int i; + if (!acpi_disabled) + return; + + if (sunway_machine_is_compatible("sunway,junzhang_v3")) + return; + cpu_num = sw64_chip->get_cpu_num(); for (node = 0; node < cpu_num; node++) { - if (is_in_host()) { + if (is_in_host()) set_devint_wken(node); - set_adr_int(node); - } } - if (!acpi_disabled) - return; - pr_info("SW arch PCI initialize!\n"); for (node = 0; node < cpu_num; node++) { rc_enable = sw64_chip_init->pci_init.get_rc_enable(node); diff --git a/arch/sw_64/platform/cpufreq.c b/arch/sw_64/platform/cpufreq.c index 87f66b2313b5..69d7611a2fb0 100644 --- a/arch/sw_64/platform/cpufreq.c +++ b/arch/sw_64/platform/cpufreq.c @@ -8,11 +8,10 @@ #include #include -#include #include #include #include -#include +#include #define MAX_RETRY 10 @@ -139,9 +138,10 @@ unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy) { int i; u64 val; + void __iomem *spbu_base = misc_platform_get_spbu_base(0); struct cpufreq_frequency_table *ft = policy->freq_table; - val = sw64_io_read(0, CLK_CTL) >> CORE_PLL2_CFG_SHIFT; + val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL2_CFG_SHIFT; val &= CORE_PLL2_CFG_MASK; for (i = 0; ft[i].frequency != CPUFREQ_TABLE_END; i++) { @@ -155,25 +155,36 @@ EXPORT_SYMBOL(__sw64_cpufreq_get); int sw64_set_rate(unsigned int index) { int i, retry, cpu_num; + void __iomem *spbu_base; cpu_num = sw64_chip->get_cpu_num(); for (i = 0; i < cpu_num; i++) { - sw64_io_write(i, CLU_LV1_SEL, CLK_LV1_SEL_PROTECT); - sw64_io_write(i, CLK_CTL, CLK2_PROTECT | CORE_CLK2_RESET | CORE_CLK2_VALID | CLK0_PROTECT); - sw64_io_write(i, CLK_CTL, CLK2_PROTECT | CORE_CLK2_VALID | (unsigned long)index << CORE_PLL2_CFG_SHIFT); + spbu_base = misc_platform_get_spbu_base(i); + + /* select PLL0/PLL1 */ + writeq(CLK_LV1_SEL_PROTECT, spbu_base + OFFSET_CLU_LV1_SEL); + /* reset PLL2 */ + writeq(CLK2_PROTECT | CORE_CLK2_RESET | CORE_CLK2_VALID, spbu_base + OFFSET_CLK_CTL); + /* configure PLL2_CFG */ + writeq(CLK2_PROTECT | CORE_CLK2_VALID | (unsigned long)index << CORE_PLL2_CFG_SHIFT, + spbu_base + OFFSET_CLK_CTL); udelay(1); - sw64_io_write(i, CLK_CTL, CORE_CLK2_VALID); + /* reset over */ + writeq(CORE_CLK2_VALID, spbu_base + OFFSET_CLK_CTL); retry = 0; while (retry < MAX_RETRY) { - if (sw64_io_read(i, CLK_CTL) & CORE_CLK2_LOCK) + if (readq(spbu_base + OFFSET_CLK_CTL) & CORE_CLK2_LOCK) break; retry++; udelay(100); } if (retry == MAX_RETRY) return -ETIME; - sw64_io_write(i, CLK_CTL, 0); - sw64_io_write(i, CLU_LV1_SEL, CLK_LV1_SEL_MUXA | CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PROTECT); + /* configure over */ + writeq(0, spbu_base + OFFSET_CLK_CTL); + /* select PLL2/PLL2 */ + writeq(CLK_LV1_SEL_MUXA | CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PROTECT, + spbu_base + OFFSET_CLU_LV1_SEL); } return 0; } diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index c7233617283f..5dad3496c0a0 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -71,23 +71,36 @@ void __init sw64_setup_clocksource(void) void __init setup_sched_clock(void) { } #elif defined(CONFIG_SUBARCH_C3B) #ifdef CONFIG_SMP +#define OFFSET_LONG_TIME_START_EN 0x9000UL + +#define OFFSET_LONG_TIME 0x180UL + +#define OFFSET_GPIO_SWPORTA_DR 0x0UL +#define OFFSET_GPIO_SWPORTA_DDR 0x200UL + static u64 read_longtime(struct clocksource *cs) { unsigned long node; + void __iomem *intpu_base; node = __this_cpu_read(hard_node_id); - return __io_read_longtime(node); + intpu_base = misc_platform_get_intpu_base(node); + + return readq(intpu_base + OFFSET_LONG_TIME); } static int longtime_enable(struct clocksource *cs) { + void __iomem *spbu_base = misc_platform_get_spbu_base(0); + void __iomem *gpio_base = misc_platform_get_gpio_base(0); + switch (cpu_desc.model) { case CPU_SW3231: - sw64_io_write(0, GPIO_SWPORTA_DR, 0); - sw64_io_write(0, GPIO_SWPORTA_DDR, 0xff); + writeq(0, gpio_base + OFFSET_GPIO_SWPORTA_DR); + writeq(0xff, gpio_base + OFFSET_GPIO_SWPORTA_DDR); break; case CPU_SW831: - __io_write_longtime_start_en(0, 0x1); + writeq(0x1, spbu_base + OFFSET_LONG_TIME_START_EN); break; default: break; @@ -111,7 +124,7 @@ static u64 read_vtime(struct clocksource *cs) { unsigned long vtime_addr; - vtime_addr = IO_BASE | LONG_TIME; + vtime_addr = IO_BASE | INTPU_BASE | OFFSET_LONG_TIME; return rdio64(vtime_addr); } diff --git a/drivers/cpufreq/sw64_cpufreq_debugfs.c b/drivers/cpufreq/sw64_cpufreq_debugfs.c index f72dbb9d5e16..e09c63495a0e 100644 --- a/drivers/cpufreq/sw64_cpufreq_debugfs.c +++ b/drivers/cpufreq/sw64_cpufreq_debugfs.c @@ -13,8 +13,9 @@ static int cpufreq_show(struct seq_file *m, void *v) { int i; u64 val; + void __iomem *spbu_base = misc_platform_get_spbu_base(0); - val = sw64_io_read(0, CLK_CTL) >> CORE_PLL2_CFG_SHIFT; + val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL2_CFG_SHIFT; val &= CORE_PLL2_CFG_MASK; seq_puts(m, "CPU frequency in Mhz:\n"); for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 6be0af85a0b5..44cb6b258a38 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -36,6 +36,7 @@ config SW64_IRQ_CPU bool depends on SW64 default y + select SW64_PINTC config SW64_PCI_INTX bool diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index f68bca18923f..4947805c33ab 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -117,25 +117,6 @@ static void dummy_perf(unsigned long vector, struct pt_regs *regs) void (*perf_irq)(unsigned long vector, struct pt_regs *regs) = dummy_perf; EXPORT_SYMBOL(perf_irq); -static void handle_fault_int(void) -{ - int node; - unsigned long value; - - node = __this_cpu_read(hard_node_id); - pr_info("enter fault int, si_fault_stat = %#lx\n", - sw64_io_read(node, SI_FAULT_STAT)); - sw64_io_write(node, SI_FAULT_INT_EN, 0); - sw64_io_write(node, DLI_RLTD_FAULT_INTEN, 0); -#if defined(CONFIG_UNCORE_XUELANG) - value = 0; -#elif defined(CONFIG_UNCORE_JUNZHANG) - value = sw64_io_read(node, FAULT_INT_CONFIG); - value |= (1 << 8); -#endif - __io_write_fault_int_en(node, value); -} - static void handle_mt_int(void) { pr_info("enter mt int\n"); @@ -146,33 +127,6 @@ static void handle_nmi_int(void) pr_info("enter nmi int\n"); } -#ifdef CONFIG_SW64_PINTC -static void handle_dev_int(struct pt_regs *regs) -{ - unsigned long config_val, val, stat; - int node = 0; - unsigned int hwirq; - - config_val = sw64_io_read(node, DEV_INT_CONFIG); - val = config_val & (~(1UL << 8)); - sw64_io_write(node, DEV_INT_CONFIG, val); - stat = sw64_io_read(node, MCU_DVC_INT); - - while (stat) { - hwirq = ffs(stat) - 1; - generic_handle_domain_irq(mcu_irq_domain, hwirq); - stat &= ~(1UL << hwirq); - } - - sw64_io_write(node, DEV_INT_CONFIG, config_val); -} -#else -static void handle_dev_int(struct pt_regs *regs) -{ - pr_crit(PREFIX "the child controller PINTC is not configured!\n"); -} -#endif - int pme_state; asmlinkage void do_entInt(unsigned long type, unsigned long vector, @@ -344,12 +298,6 @@ static int __init pintc_parse_madt(union acpi_subtable_headers *header, pintc = (struct acpi_madt_sw_pintc *)header; - /* Not yet supported */ - if (pintc->node > 0) { - pr_warn(PREFIX "PINTC and LPC-INTC on node x(x > 0) are not supported\n"); - return 0; - } - if ((pintc->version == ACPI_MADT_SW_PINTC_VERSION_NONE) || (pintc->version >= ACPI_MADT_SW_PINTC_VERSION_RESERVED)) { pr_err(PREFIX "invalid PINTC version\n"); diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index fa3af8c1c585..e2f1c39538d6 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -7,6 +7,8 @@ #include #include +#include + /** * Currently, Peripheral interrupt control logic of Sunway is mainly * distributed on the device side, which are hardware entities @@ -43,9 +45,26 @@ #define PREFIX "PINTC: " -#define OFFSET_MCU_DVC_INT_EN 0x3080UL - -#define OFFSET_DEV_INT_CONFIG 0x480UL +#define OFFSET_DLI_RLTD_FAULT_INTEN 0xa80UL +#define OFFSET_MCU_DVC_INT 0x3000UL +#define OFFSET_MCU_DVC_INT_EN 0x3080UL +#define OFFSET_SI_FAULT_STAT 0x3100UL +#define OFFSET_SI_FAULT_INT_EN 0x3200UL +#define OFFSET_ADR_CTL 0x3600UL /* PINTC version >= 2 */ +#define OFFSET_DUAL_CG0_FAULT_INTEN 0x7580UL /* PINTC version 1 only */ +#define OFFSET_DUAL_CG1_FAULT_INTEN 0x7600UL /* PINTC version 1 only */ +#define OFFSET_DUAL_CG2_FAULT_INTEN 0x7680UL /* PINTC version 1 only */ +#define OFFSET_DUAL_CG3_FAULT_INTEN 0x7700UL /* PINTC version 1 only */ +#define OFFSET_DUAL_CG4_FAULT_INTEN 0x7780UL /* PINTC version 1 only */ +#define OFFSET_DUAL_CG5_FAULT_INTEN 0x7800UL /* PINTC version 1 only */ +#define OFFSET_DUAL_CG6_FAULT_INTEN 0x7880UL /* PINTC version 1 only */ +#define OFFSET_DUAL_CG7_FAULT_INTEN 0x7900UL /* PINTC version 1 only */ + +#define OFFSET_DEV_INT_CONFIG 0x480UL +#define OFFSET_FAULT_INT_CONFIG 0x780UL +#define OFFSET_DEVINT_WKEN 0x1500UL +#define OFFSET_ADR_INT_CONFIG 0x1580UL /* PINTC version >= 2 */ +#define OFFSET_DEVINTWK_INTEN 0x1600UL #define SW_PINTC_MCU_GSI_BASE 64 @@ -55,6 +74,8 @@ #define MCU_BASE_V1 0x803000000000 #define MCU_SIZE_V1 0x8f00 +DECLARE_PER_CPU(unsigned long, hard_node_id); + struct pintc_chip_data { bool vt; /* virtual pintc */ u32 node; /* node ID */ @@ -65,6 +86,39 @@ struct pintc_chip_data { u32 mcu_irq_num; }; +static struct pintc_chip_data *chip_datas[MAX_NUMNODES]; + +static struct pintc_chip_data *pintc_alloc_chip_data(u32 node) +{ + struct pintc_chip_data *chip_data; + + if (WARN_ON(node >= MAX_NUMNODES)) + return NULL; + + chip_data = kzalloc_node(sizeof(struct pintc_chip_data), + GFP_KERNEL, node); + if (!chip_data) + chip_data = kzalloc(sizeof(struct pintc_chip_data), + GFP_KERNEL); + + chip_datas[node] = chip_data; + + return chip_data; +} + +static void pintc_free_chip_data(struct pintc_chip_data *chip_data) +{ + if (!chip_data) + return; + + if (WARN_ON((chip_data->node >= MAX_NUMNODES) || + (chip_datas[chip_data->node] != chip_data))) + return; + + chip_datas[chip_data->node] = NULL; + kfree(chip_data); +} + static DEFINE_RAW_SPINLOCK(pintc_lock); static void lock_dev_lock(void) { @@ -120,7 +174,7 @@ static void mcu_irq_disable(struct irq_data *irq_data) mcu_irq_mask(irq_data); } -static unsigned long make_mcu_int_target(u32 version, int rcid) +static unsigned long make_pintc_int_target(u32 version, int rcid) { int node, core, thread; unsigned long target = 0; @@ -168,7 +222,7 @@ static int __assign_mcu_irq_config(const struct pintc_chip_data *chip_data, rcid = cpu_to_rcid(cpu); val = readq(chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); - dev_int_tar = make_mcu_int_target(chip_data->version, rcid); + dev_int_tar = make_pintc_int_target(chip_data->version, rcid); val &= 0xffff; val |= dev_int_tar << 16; writeq(val, chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); @@ -333,13 +387,89 @@ static int __init pintc_init_mcu(struct pintc_chip_data *chip_data, return -ENOMEM; } - pr_info(PREFIX "MCU sub controller [%u:%u] initialized\n", + pr_info(PREFIX "MCU version [%u] on node [%u] initialized\n", chip_data->version, chip_data->node); return 0; } +void handle_dev_int(struct pt_regs *regs) +{ + void __iomem *mcu_base, *intpu_base; + unsigned long config_val, val, stat; + unsigned int hwirq; + + /* Currently, only MCU controller on node 0 is supported */ + mcu_base = chip_datas[0]->mcu_base; + intpu_base = chip_datas[0]->pintc_base; + + config_val = readq(intpu_base + OFFSET_DEV_INT_CONFIG); + val = config_val & (~(1UL << 8)); + writeq(val, intpu_base + OFFSET_DEV_INT_CONFIG); + stat = readq(mcu_base + OFFSET_MCU_DVC_INT); + + while (stat) { + hwirq = ffs(stat) - 1; + generic_handle_domain_irq(mcu_irq_domain, hwirq); + stat &= ~(1UL << hwirq); + } + + writeq(config_val, intpu_base + OFFSET_DEV_INT_CONFIG); +} + +void handle_fault_int(void) +{ + int node; + unsigned long value; + void __iomem *mcu_base, *intpu_base; + + node = __this_cpu_read(hard_node_id); + +#if defined(CONFIG_UNCORE_XUELANG) + mcu_base = misc_platform_get_spbu_base(node); + intpu_base = misc_platform_get_intpu_base(node); +#elif defined(CONFIG_UNCORE_JUNZHANG) + mcu_base = chip_datas[node]->mcu_base; + intpu_base = chip_datas[node]->pintc_base; +#endif + + pr_info("Enter fault int, si_fault_stat = %#llx\n", + readq(mcu_base + OFFSET_SI_FAULT_STAT)); + + writeq(0, mcu_base + OFFSET_SI_FAULT_INT_EN); + writeq(0, mcu_base + OFFSET_DLI_RLTD_FAULT_INTEN); + +#if defined(CONFIG_UNCORE_XUELANG) + value = 0; + writeq(value, mcu_base + OFFSET_DUAL_CG0_FAULT_INTEN); + writeq(value, mcu_base + OFFSET_DUAL_CG1_FAULT_INTEN); + writeq(value, mcu_base + OFFSET_DUAL_CG2_FAULT_INTEN); + writeq(value, mcu_base + OFFSET_DUAL_CG3_FAULT_INTEN); + writeq(value, mcu_base + OFFSET_DUAL_CG4_FAULT_INTEN); + writeq(value, mcu_base + OFFSET_DUAL_CG5_FAULT_INTEN); + writeq(value, mcu_base + OFFSET_DUAL_CG6_FAULT_INTEN); + writeq(value, mcu_base + OFFSET_DUAL_CG7_FAULT_INTEN); +#elif defined(CONFIG_UNCORE_JUNZHANG) + value = readq(intpu_base + OFFSET_FAULT_INT_CONFIG); + value |= (1 << 8); + writeq(value, intpu_base + OFFSET_FAULT_INT_CONFIG); +#endif +} + #ifdef CONFIG_OF +static int __init pintc_of_init_mcu(struct pintc_chip_data *chip_data, + struct device_node *pintc) +{ + /* Not yet supported */ + if (chip_data->node > 0) { + pr_info(PREFIX "MCU version [%u] on node [%u] skipped\n", + chip_data->version, chip_data->node); + return 0; + } + + return pintc_init_mcu(chip_data, of_node_to_fwnode(pintc)); +} + static int __init pintc_of_init_common(struct device_node *pintc, struct device_node *parent, bool vt) @@ -393,7 +523,7 @@ pintc_of_init_common(struct device_node *pintc, MCU_BASE_V1); } - chip_data = kzalloc_node(sizeof(*chip_data), GFP_KERNEL, node); + chip_data = pintc_alloc_chip_data(node); if (!chip_data) { ret = -ENOMEM; goto out_unmap; @@ -406,14 +536,20 @@ pintc_of_init_common(struct device_node *pintc, chip_data->mcu_base = mcu_base; chip_data->mcu_irq_num = nr_irqs; - ret = pintc_init_mcu(chip_data, of_node_to_fwnode(pintc)); + /* Enable S3 wakeup interrupt for physical environment */ + if (!vt && IS_ENABLED(CONFIG_SUSPEND)) { + writeq(0x80, chip_data->pintc_base + OFFSET_DEVINT_WKEN); + writeq(0x80, chip_data->pintc_base + OFFSET_DEVINTWK_INTEN); + } + + ret = pintc_of_init_mcu(chip_data, pintc); if (ret) goto out_free_mem; return 0; out_free_mem: - kfree(chip_data); + pintc_free_chip_data(chip_data); out_unmap: iounmap(mcu_base); iounmap(pintc_base); @@ -496,8 +632,15 @@ static int __init pintc_acpi_init_mcu(struct pintc_chip_data *chip_data, struct fwnode_handle *handle; int ret; + /* Not yet supported */ + if (chip_data->node > 0) { + pr_info(PREFIX "MCU version [%u] on node [%u] skipped\n", + chip_data->version, chip_data->node); + return 0; + } + if (!mcu->status) { - pr_info(PREFIX "MCU sub controller [%u:%u] disabled\n", + pr_info(PREFIX "MCU version [%u] on node [%u] disabled\n", chip_data->version, chip_data->node); return 0; } @@ -532,6 +675,10 @@ static int __init pintc_acpi_init_mcu(struct pintc_chip_data *chip_data, goto out_acpi_free_mcu_domain; } + /* Init SW LPC INTC */ + acpi_table_parse_madt(ACPI_MADT_TYPE_SW_LPC_INTC, + lpc_intc_parse_madt, 0); + return 0; out_acpi_free_mcu_domain: @@ -543,6 +690,28 @@ static int __init pintc_acpi_init_mcu(struct pintc_chip_data *chip_data, return ret; } +static int __init pintc_acpi_init_fault(struct pintc_chip_data *chip_data, + struct acpi_madt_sw_sub_pintc *fault) +{ + if (!fault->status) { + pr_info(PREFIX "Fault version [%u] on node [%u] disabled\n", + chip_data->version, chip_data->node); + return 0; + } + + /* Fault share the same base address with MCU currently */ + chip_data->mcu_base = ioremap(fault->address, fault->size); + if (!chip_data->mcu_base) { + pr_err(PREFIX "failed to map fault base address\n"); + return -ENXIO; + } + + pr_info(PREFIX "Fault version [%u] on node [%u] initialized\n", + chip_data->version, chip_data->node); + + return 0; +} + int __init pintc_acpi_init(struct irq_domain *parent, struct acpi_madt_sw_pintc *pintc) { @@ -566,8 +735,7 @@ int __init pintc_acpi_init(struct irq_domain *parent, return -EINVAL; } - chip_data = kzalloc_node(sizeof(*chip_data), GFP_KERNEL, - pintc->node); + chip_data = pintc_alloc_chip_data(pintc->node); if (!chip_data) return -ENOMEM; @@ -588,24 +756,29 @@ int __init pintc_acpi_init(struct irq_domain *parent, goto out_acpi_free_chip_data; } + /* Enable S3 wakeup interrupt for physical environment */ + if (!virtual && IS_ENABLED(CONFIG_SUSPEND)) { + writeq(0x80, chip_data->pintc_base + OFFSET_DEVINT_WKEN); + writeq(0x80, chip_data->pintc_base + OFFSET_DEVINTWK_INTEN); + } + for (i = 0; i < pintc->sub_num; ++i) { switch (pintc->sub[i].type) { case SW_PINTC_SUB_TYPE_MCU: pintc_acpi_init_mcu(chip_data, &pintc->sub[i]); break; + case SW_PINTC_SUB_TYPE_FAULT: + pintc_acpi_init_fault(chip_data, &pintc->sub[i]); + break; default: break; } } - /* Init SW LPC INTC */ - acpi_table_parse_madt(ACPI_MADT_TYPE_SW_LPC_INTC, - lpc_intc_parse_madt, 0); - return 0; out_acpi_free_chip_data: - kfree(chip_data); + pintc_free_chip_data(chip_data); return ret; } #endif diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index db4ad9247887..e6e178923fd1 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -6,23 +6,7 @@ #include -void set_devint_wken(int node) -{ - unsigned long val; - - /* enable INTD wakeup */ - val = 0x80; - sw64_io_write(node, DEVINT_WKEN, val); - sw64_io_write(node, DEVINTWK_INTEN, val); -} - -#ifdef CONFIG_UNCORE_JUNZHANG -void set_adr_int(int node) -{ - sw64_io_write(node, ADR_INT_CONFIG, (CORE0_CID << 16 | 0x3f)); - sw64_io_write(node, ADR_CTL, 0xc); -} -#endif +#define OFFSET_IO_START 0x1300UL void set_pcieport_service_irq(struct pci_controller *hose) { @@ -263,11 +247,13 @@ static void set_rc_piu(struct pci_controller *hose) static unsigned long get_rc_enable(unsigned long node) { unsigned long rc_enable; + void __iomem *spbu_base; if (is_guest_or_emul()) return 1; - rc_enable = sw64_io_read(node, IO_START); + spbu_base = misc_platform_get_spbu_base(node); + rc_enable = readq(spbu_base + OFFSET_IO_START); return rc_enable; } -- Gitee From ea12a51e5dea671ec6ae4ea89c39be646d3469d9 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 14 Aug 2024 10:12:35 +0800 Subject: [PATCH 294/524] sw64: pci: support PCIe controller driver based on device tree Now we can describe Sunway PCIe controller via device tree. Here is an example of a device tree description: pcie0: pcie@8800 { compatible = "sunway,pcie"; device_type = "pci"; numa-node-id = <0>; reg = <0x8806 0x0 0x1 0x0>; reg-names = "ecam"; sunway,rc-config-base = <0x8805 0x0>; sunway,ep-config-base = <0x8806 0x0>; sunway,ep-mem-32-base = <0x8800 0x0>; sunway,ep-mem-64-base = <0x8880 0x0>; sunway,ep-io-base = <0x8801 0x0>; sunway,piu-ior0-base = <0x8802 0x0>; sunway,piu-ior1-base = <0x8803 0x0>; sunway,rc-index = <0x0 0x0>; sunway,pcie-io-base = <0x8800 0x0>; #address-cells = <3>; #size-cells = <2>; bus-range = <0x0 0xff>; /* I/O, 32 bits mem and prefetchable 64 bits mem */ ranges = <0x01000000 0x0 0x0 0x8801 0x0 0x1 0x0>, <0x02000000 0x0 0xe0000000 0x8800 0xe0000000 0x0 0x20000000>, <0x43000000 0x8880 0x0 0x8880 0x0 0x80 0x0>; status = "okay"; }; Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/io.h | 14 ++ arch/sw_64/include/asm/pci.h | 3 +- arch/sw_64/pci/acpi.c | 61 +++--- arch/sw_64/pci/pci-legacy.c | 9 +- arch/sw_64/pci/pci.c | 26 ++- drivers/pci/controller/pci-sunway.c | 235 ++++++++++++++++++++--- drivers/pci/hotplug/sunway_pciehp_core.c | 37 ++-- 7 files changed, 309 insertions(+), 76 deletions(-) diff --git a/arch/sw_64/include/asm/io.h b/arch/sw_64/include/asm/io.h index 15abf97e9d57..1695801e6c4b 100644 --- a/arch/sw_64/include/asm/io.h +++ b/arch/sw_64/include/asm/io.h @@ -121,6 +121,20 @@ static inline void * __deprecated bus_to_virt(unsigned long address) } #define isa_bus_to_virt bus_to_virt +static inline int pci_remap_iospace(const struct resource *res, + phys_addr_t phys_addr) +{ + if (!(res->flags & IORESOURCE_IO)) + return -EINVAL; + + if (res->end > IO_SPACE_LIMIT) + return -EINVAL; + + return 0; +} + +#define pci_remap_iospace pci_remap_iospace + #endif /* __KERNEL__ */ #endif /* _ASM_SW64_IO_H */ diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 877aa0107539..7e9bf4fcb25d 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -158,6 +158,8 @@ extern void __init sw64_init_arch(void); extern struct pci_ops sw64_pci_ops; extern int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); extern struct pci_controller *hose_head; +extern bool sunway_legacy_pci; + #ifdef CONFIG_PCI_SW64 extern void __init setup_chip_pci_ops(void); #else @@ -168,7 +170,6 @@ extern struct pci_controller * pci_bus_to_pci_controller(const struct pci_bus *bus); extern struct pci_controller *bus_num_to_pci_controller(unsigned long bus_num); -extern void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge); extern void sw64_pci_root_bridge_scan_finish_up(struct pci_host_bridge *bridge); extern int sw64_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); diff --git a/arch/sw_64/pci/acpi.c b/arch/sw_64/pci/acpi.c index 6f03ea79131c..514e7838f3c8 100644 --- a/arch/sw_64/pci/acpi.c +++ b/arch/sw_64/pci/acpi.c @@ -5,8 +5,6 @@ #include #include -#define UPPER_32_BITS_OF_U64(u64_val) ((u64_val) & 0xFFFFFFFF00000000ULL) - struct pci_root_info { struct acpi_pci_root_info info; struct pci_config_window *cfg; @@ -87,6 +85,33 @@ pci_acpi_setup_ecam_mapping(struct acpi_pci_root *root) return cfg; } +static int upper_32_bits_of_ep_mem_32_base(struct acpi_device *adev, u64 *memh) +{ + int status = 0; + u64 val; + const char *prop = "sunway,ep-mem-32-base"; + const char *legacy_prop = "sw64,ep_mem_32_base"; + + status = fwnode_property_read_u64(&adev->fwnode, prop, &val); + + /* Fallback to legacy property name and try again */ + if (status) + status = fwnode_property_read_u64(&adev->fwnode, + legacy_prop, &val); + + /* This property is necessary */ + if (status) { + dev_err(&adev->dev, "failed to retrieve %s or %s\n", + prop, legacy_prop); + return status; + } + + *memh = upper_32_bits(val); + *memh <<= 32; + + return 0; +} + static int pci_acpi_prepare_root_resources(struct acpi_pci_root_info *ci) { int status = 0; @@ -100,14 +125,9 @@ static int pci_acpi_prepare_root_resources(struct acpi_pci_root_info *ci) * * Get the upper 32 bits here. */ - status = fwnode_property_read_u64_array(&device->fwnode, - "sw64,ep_mem_32_base", &memh, 1); - if (status) { - dev_err(&device->dev, "unable to retrieve \"sw64,ep_mem_32_base\"\n"); + status = upper_32_bits_of_ep_mem_32_base(device, &memh); + if (status) return status; - } - - memh = UPPER_32_BITS_OF_U64(memh); /** * Get host bridge resources via _CRS method, the return value @@ -131,7 +151,7 @@ static int pci_acpi_prepare_root_resources(struct acpi_pci_root_info *ci) resource_list_for_each_entry_safe(entry, tmp, &ci->resources) { if (entry->res->flags & IORESOURCE_MEM) { - if (!UPPER_32_BITS_OF_U64(entry->res->end)) { + if (!upper_32_bits(entry->res->end)) { /* Patch mem res with upper 32 bits */ entry->res->start |= memh; entry->res->end |= memh; @@ -225,27 +245,6 @@ struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root) return NULL; } -int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge) -{ - if (!acpi_disabled) { - struct pci_config_window *cfg = bridge->sysdata; - struct acpi_device *adev = to_acpi_device(cfg->parent); - struct pci_controller *hose = cfg->priv; - struct device *bus_dev = &bridge->bus->dev; - - ACPI_COMPANION_SET(&bridge->dev, adev); - set_dev_node(bus_dev, hose->node); - - /** - * Some quirks for pci controller of Sunway - * before scanning Root Complex - */ - sw64_pci_root_bridge_prepare(bridge); - } - - return 0; -} - void pcibios_add_bus(struct pci_bus *bus) { acpi_pci_add_bus(bus); diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index 379a6566f0da..caa64977fa74 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -12,6 +12,8 @@ #define OFFSET_DEVINT_WKEN 0x1500UL #define OFFSET_DEVINTWK_INTEN 0x1600UL +bool sunway_legacy_pci; + /* * The PCI controller list. */ @@ -22,7 +24,7 @@ static void __init pcibios_reserve_legacy_regions(struct pci_bus *bus); static int __init pcibios_init(void) { - if (acpi_disabled) + if (sunway_legacy_pci) sw64_init_pci(); return 0; } @@ -277,9 +279,12 @@ void __init sw64_init_arch(void) if (!acpi_disabled) return; - if (sunway_machine_is_compatible("sunway,junzhang_v3")) + if (!sunway_machine_is_compatible("sunway,chip3") && + !sunway_machine_is_compatible("sunway,junzhang")) return; + sunway_legacy_pci = true; + cpu_num = sw64_chip->get_cpu_num(); for (node = 0; node < cpu_num; node++) { diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index de974eff019d..23a3f8a1e2f8 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -287,7 +288,7 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, enable_sw_dca); */ static unsigned char last_bus; -void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge) +static void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge) { struct pci_controller *hose = NULL; struct resource_entry *entry = NULL; @@ -429,3 +430,26 @@ void sw64_pci_root_bridge_scan_finish_up(struct pci_host_bridge *bridge) */ pci_clear_flags(PCI_REASSIGN_ALL_BUS); } + +int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge) +{ + struct pci_config_window *cfg = bridge->sysdata; + struct acpi_device *adev = NULL; + struct pci_controller *hose = cfg->priv; + struct device *bus_dev = &bridge->bus->dev; + + if (sunway_legacy_pci) + return 0; + + if (!acpi_disabled) + adev = to_acpi_device(cfg->parent); + + ACPI_COMPANION_SET(&bridge->dev, adev); + set_dev_node(bus_dev, hose->node); + + /* Some quirks for Sunway PCIe controller before scanning */ + sw64_pci_root_bridge_prepare(bridge); + + return 0; +} + diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index e6e178923fd1..f8c24860edfd 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include #include @@ -385,7 +388,7 @@ struct pci_controller *pci_bus_to_pci_controller(const struct pci_bus *bus) if (unlikely(!bus)) return NULL; - if (acpi_disabled) + if (sunway_legacy_pci) return (struct pci_controller *)(bus->sysdata); cfg = (struct pci_config_window *)bus->sysdata; @@ -615,8 +618,6 @@ int sw64_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) return map_irq(dev, slot, pin); } -#ifdef CONFIG_ACPI - enum pci_props { PROP_RC_CONFIG_BASE = 0, PROP_EP_CONFIG_BASE, @@ -630,7 +631,7 @@ enum pci_props { PROP_NUM }; -const char *prop_names[PROP_NUM] = { +static const char *prop_names_legacy[PROP_NUM] = { "sw64,rc_config_base", "sw64,ep_config_base", "sw64,ep_mem_32_base", @@ -642,23 +643,68 @@ const char *prop_names[PROP_NUM] = { "sw64,pcie_io_base" }; +static const char *prop_names[PROP_NUM] = { + "sunway,rc-config-base", + "sunway,ep-config-base", + "sunway,ep-mem-32-base", + "sunway,ep-mem-64-base", + "sunway,ep-io-base", + "sunway,piu-ior0-base", + "sunway,piu-ior1-base", + "sunway,rc-index", + "sunway,pcie-io-base" +}; + +#ifdef CONFIG_NUMA +static void pci_controller_set_node(struct pci_controller *hose, + struct fwnode_handle *fwnode) +{ + if (numa_off) + return; + + /* Get node from ACPI namespace (_PXM) or DTB */ + if (acpi_disabled) + hose->node = of_node_to_nid(to_of_node(fwnode)); + else + hose->node = acpi_get_node(ACPI_HANDLE_FWNODE(fwnode)); + + /** + * If numa_off is not set, we expect a valid node ID. + * If not, fallback to node 0. + */ + if (hose->node == NUMA_NO_NODE) { + pr_warn("Invalid node ID\n"); + hose->node = 0; + } +} +#endif + static int sw64_pci_prepare_controller(struct pci_controller *hose, struct fwnode_handle *fwnode) { u64 props[PROP_NUM]; int i, ret; - /* Get properties of Root Complex */ + /* Get necessary properties of Root Complex */ for (i = 0; i < PROP_NUM; ++i) { - ret = fwnode_property_read_u64_array(fwnode, prop_names[i], - &props[i], 1); + ret = fwnode_property_read_u64(fwnode, prop_names[i], + &props[i]); + + /* Fallback to legacy names and try again */ + if (ret) + ret = fwnode_property_read_u64(fwnode, + prop_names_legacy[i], &props[i]); + if (ret) { - pr_err("unable to retrieve \"%s\"\n", - prop_names[i]); + pr_err("unable to retrieve \"%s\"\n", prop_names[i]); return ret; } } +#ifdef CONFIG_NUMA + pci_controller_set_node(hose, fwnode); +#endif + hose->iommu_enable = false; hose->index = props[PROP_RC_INDEX]; @@ -709,13 +755,9 @@ static int sw64_pci_prepare_controller(struct pci_controller *hose, return 0; } -/** - * Use the info from ACPI to init pci_controller - */ -static int sw64_pci_ecam_init(struct pci_config_window *cfg) +#ifdef CONFIG_ACPI +static int sw64_pci_acpi_present(struct device *dev) { - struct pci_controller *hose = NULL; - struct device *dev = cfg->parent; struct acpi_device *adev = to_acpi_device(dev); int ret; @@ -740,27 +782,39 @@ static int sw64_pci_ecam_init(struct pci_config_window *cfg) return -ENODEV; } - hose = kzalloc(sizeof(*hose), GFP_KERNEL); - if (!hose) - return -ENOMEM; + return 0; +} +#endif -#ifdef CONFIG_ACPI_NUMA - if (!numa_off) { - /* Get node from ACPI namespace (_PXM) */ - hose->node = acpi_get_node(adev->handle); - if (hose->node == NUMA_NO_NODE) { - kfree(hose); - dev_err(&adev->dev, "unable to get node ID\n"); - return -EINVAL; - } +/** + * Use the information from ACPI or DTB to init pci_controller + */ +static int sw64_pci_ecam_init(struct pci_config_window *cfg) +{ + struct pci_controller *hose = NULL; + struct device *dev = cfg->parent; + struct fwnode_handle *fwnode = dev->fwnode; + int ret; + +#ifdef CONFIG_ACPI + if (!acpi_disabled) { + ret = sw64_pci_acpi_present(dev); + if (ret) + return ret; + + fwnode = acpi_fwnode_handle(to_acpi_device(dev)); } #endif + hose = kzalloc(sizeof(*hose), GFP_KERNEL); + if (!hose) + return -ENOMEM; + /* Init pci_controller */ - ret = sw64_pci_prepare_controller(hose, &adev->fwnode); + ret = sw64_pci_prepare_controller(hose, fwnode); if (ret) { kfree(hose); - dev_err(&adev->dev, "failed to init pci controller\n"); + dev_err(dev, "failed to init pci controller\n"); return ret; } @@ -778,4 +832,125 @@ const struct pci_ecam_ops sw64_pci_ecam_ops = { .write = sw64_pcie_config_write, } }; -#endif + +#ifdef CONFIG_OF + +static const struct of_device_id sunway_pcie_of_match[] = { + { + .compatible = "sunway,pcie", + .data = &sw64_pci_ecam_ops, + }, + {}, +}; + +static void sunway_pcie_ecam_free(void *p) +{ + pci_ecam_free((struct pci_config_window *)p); +} + +static struct pci_host_bridge *sunway_pcie_of_init(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct pci_config_window *cfg; + struct pci_ecam_ops *ops; + struct resource *ecam_res = NULL; + struct resource_entry *bus_range = NULL; + int ret; + + ops = (struct pci_ecam_ops *)of_device_get_match_data(dev); + if (!ops) { + dev_err(dev, "failed to get ecam ops\n"); + return NULL; + } + + ecam_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ecam"); + if (!ecam_res) { + dev_err(dev, "failed to get ecam\n"); + return NULL; + } + + bridge = devm_pci_alloc_host_bridge(dev, 0); + if (!bridge) { + dev_err(dev, "failed to create host bridge\n"); + return NULL; + } + + bus_range = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); + if (!bus_range) { + dev_err(dev, "failed to get bus resource\n"); + return NULL; + } + + cfg = pci_ecam_create(dev, ecam_res, bus_range->res, ops); + if (IS_ERR(cfg)) { + dev_err(dev, "failed to create ecam\n"); + return NULL; + } + + ret = devm_add_action_or_reset(dev, sunway_pcie_ecam_free, cfg); + if (ret) + return NULL; + + bridge->sysdata = cfg; + bridge->ops = (struct pci_ops *)&ops->pci_ops; + + return bridge; +} + +static int sunway_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct pci_bus *bus, *child; + int ret; + + /* Prevent conflicts between legacy PCI and DT-based PCI */ + if (sunway_legacy_pci) + return 0; + + bridge = sunway_pcie_of_init(pdev); + if (!bridge) + return -ENODEV; + + ret = pci_scan_root_bus_bridge(bridge); + if (ret < 0) { + dev_err(dev, "failed to scan root bridge\n"); + return ret; + } + + bus = bridge->bus; + + /** + * Some quirks for Sunway PCIe controller after scanning, + * that's why we don't directly call function pci_host_probe(). + */ + sw64_pci_root_bridge_scan_finish_up(bridge); + + pci_bus_size_bridges(bus); + pci_bus_assign_resources(bus); + + list_for_each_entry(child, &bus->children, node) + pcie_bus_configure_settings(child); + + pci_bus_add_devices(bus); + + return 0; +} + +static struct platform_driver sunway_pcie_driver = { + .probe = sunway_pcie_probe, + .driver = { + .name = "sunway-pcie", + .of_match_table = sunway_pcie_of_match, + }, +}; + +static int __init sunway_pcie_driver_init(void) +{ + return platform_driver_register(&sunway_pcie_driver); +} +subsys_initcall(sunway_pcie_driver_init); + +#endif /* CONFIG_OF */ + diff --git a/drivers/pci/hotplug/sunway_pciehp_core.c b/drivers/pci/hotplug/sunway_pciehp_core.c index 45972adcff36..43c1a4facfae 100644 --- a/drivers/pci/hotplug/sunway_pciehp_core.c +++ b/drivers/pci/hotplug/sunway_pciehp_core.c @@ -249,23 +249,38 @@ static void sunway_hose_hotplug_init(void) struct pci_dev *pdev = NULL; struct pci_controller *hose; struct pci_config_window *cfg; - struct acpi_device *adev; - u64 prop_hotplug_enable; + struct device *dev; + struct fwnode_handle *fwnode; + u64 hotplug_enable; - while ((pdev = pci_get_device(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, pdev))) { + while ((pdev = pci_get_device(PCI_VENDOR_ID_JN, + PCI_DEVICE_ID_SW64_ROOT_BRIDGE, pdev))) { hose = pci_bus_to_pci_controller(pdev->bus); + + /* disable by default */ hose->hotplug_enable = false; - if (!acpi_disabled) { - cfg = (struct pci_config_window *)pdev->bus->sysdata; - adev = to_acpi_device(cfg->parent); + if (sunway_legacy_pci) + continue; + + cfg = (struct pci_config_window *)pdev->bus->sysdata; + dev = cfg->parent; + + if (acpi_disabled) + fwnode = dev->fwnode; + else + fwnode = acpi_fwnode_handle(to_acpi_device(dev)); + + ret = fwnode_property_read_u64(fwnode, + "sunway,hotplug-enable", &hotplug_enable); - ret = fwnode_property_read_u64_array(&adev->fwnode, - "sw64,hot_plug_slot_enable", &prop_hotplug_enable, 1); + /* Fallback to legacy prop name */ + if (ret) + ret = fwnode_property_read_u64(fwnode, + "sw64,hot_plug_slot_enable", &hotplug_enable); - if (ret == 0) - hose->hotplug_enable = prop_hotplug_enable; - } + if (!ret) + hose->hotplug_enable = hotplug_enable; } } -- Gitee From e07d910c2ba791052d788c8b02b7dcb287754cad Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 16 Aug 2024 14:45:58 +0800 Subject: [PATCH 295/524] sw64: pci: rename some functions Rename some PCI related functions to improve code readability. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 11 +++-- arch/sw_64/include/asm/sw64_init.h | 1 - arch/sw_64/pci/acpi.c | 11 ++--- arch/sw_64/pci/pci-legacy.c | 21 ++++------ arch/sw_64/pci/pci.c | 8 ++-- drivers/acpi/pci_mcfg.c | 18 ++++---- drivers/pci/controller/pci-sunway.c | 64 +++++++++++++---------------- include/linux/pci-ecam.h | 2 +- 8 files changed, 60 insertions(+), 76 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 7e9bf4fcb25d..f28a3acb015c 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -155,7 +155,6 @@ extern void __init sw64_device_interrupt(unsigned long vector); extern void setup_intx_irqs(struct pci_controller *hose); extern void __init sw64_init_irq(void); extern void __init sw64_init_arch(void); -extern struct pci_ops sw64_pci_ops; extern int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); extern struct pci_controller *hose_head; extern bool sunway_legacy_pci; @@ -170,14 +169,14 @@ extern struct pci_controller * pci_bus_to_pci_controller(const struct pci_bus *bus); extern struct pci_controller *bus_num_to_pci_controller(unsigned long bus_num); -extern void sw64_pci_root_bridge_scan_finish_up(struct pci_host_bridge *bridge); -extern int sw64_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); +extern int sunway_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); +extern void sunway_pci_root_bridge_scan_finish(struct pci_host_bridge *bridge); -extern void __iomem *sw64_pcie_map_bus(struct pci_bus *bus, +extern void __iomem *sunway_pci_map_bus(struct pci_bus *bus, unsigned int devfn, int where); -extern int sw64_pcie_config_write(struct pci_bus *bus, unsigned int devfn, +extern int sunway_pci_config_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val); -extern int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, +extern int sunway_pci_config_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val); extern void pci_mark_rc_linkup(struct pci_controller *hose); diff --git a/arch/sw_64/include/asm/sw64_init.h b/arch/sw_64/include/asm/sw64_init.h index 2d553242487d..34a28e365ec6 100644 --- a/arch/sw_64/include/asm/sw64_init.h +++ b/arch/sw_64/include/asm/sw64_init.h @@ -14,7 +14,6 @@ struct sw64_early_init_ops { }; struct sw64_pci_init_ops { - int (*map_irq)(const struct pci_dev *dev, u8 slot, u8 pin); unsigned long (*get_rc_enable)(unsigned long node); void (*hose_init)(struct pci_controller *hose); void (*set_rc_piu)(struct pci_controller *hose); diff --git a/arch/sw_64/pci/acpi.c b/arch/sw_64/pci/acpi.c index 514e7838f3c8..1b748074fdf8 100644 --- a/arch/sw_64/pci/acpi.c +++ b/arch/sw_64/pci/acpi.c @@ -85,7 +85,7 @@ pci_acpi_setup_ecam_mapping(struct acpi_pci_root *root) return cfg; } -static int upper_32_bits_of_ep_mem_32_base(struct acpi_device *adev, u64 *memh) +static int ep_32bits_memio_base(struct acpi_device *adev, u64 *memh) { int status = 0; u64 val; @@ -125,7 +125,7 @@ static int pci_acpi_prepare_root_resources(struct acpi_pci_root_info *ci) * * Get the upper 32 bits here. */ - status = upper_32_bits_of_ep_mem_32_base(device, &memh); + status = ep_32bits_memio_base(device, &memh); if (status) return status; @@ -219,11 +219,8 @@ struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root) if (!bus) return NULL; - /** - * Some quirks for pci controller of Sunway - * after scanning Root Complex - */ - sw64_pci_root_bridge_scan_finish_up(pci_find_host_bridge(bus)); + /* Some quirks for Sunway PCIe controller after scanning */ + sunway_pci_root_bridge_scan_finish(pci_find_host_bridge(bus)); pci_bus_size_bridges(bus); pci_bus_assign_resources(bus); diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index caa64977fa74..ccf87520c72d 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -70,6 +70,12 @@ int __weak chip_pcie_configure(struct pci_controller *hose) return 0; } +static struct pci_ops sunway_pci_ops = { + .map_bus = sunway_pci_map_bus, + .read = sunway_pci_config_read, + .write = sunway_pci_config_write, +}; + unsigned char last_bus = PCI0_BUS; void __init common_init_pci(void) { @@ -102,9 +108,9 @@ void __init common_init_pci(void) bridge->dev.parent = NULL; bridge->sysdata = hose; bridge->busnr = hose->busn_space->start; - bridge->ops = &sw64_pci_ops; + bridge->ops = &sunway_pci_ops; bridge->swizzle_irq = pci_common_swizzle; - bridge->map_irq = sw64_map_irq; + bridge->map_irq = sunway_pci_map_irq; ret = pci_scan_root_bus_bridge(bridge); if (ret) { @@ -200,17 +206,6 @@ static void __init pcibios_reserve_legacy_regions(struct pci_bus *bus) return; } -struct pci_ops sw64_pci_ops = { - .map_bus = sw64_pcie_map_bus, - .read = sw64_pcie_config_read, - .write = sw64_pcie_config_write, -}; - -int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) -{ - return sw64_chip_init->pci_init.map_irq(dev, slot, pin); -} - static bool rc_linkup[MAX_NUMNODES][MAX_NR_RCS_PER_NODE]; static void __init diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index 23a3f8a1e2f8..d697cc0f246f 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -288,7 +288,7 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, enable_sw_dca); */ static unsigned char last_bus; -static void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge) +static void sunway_pci_root_bridge_prepare(struct pci_host_bridge *bridge) { struct pci_controller *hose = NULL; struct resource_entry *entry = NULL; @@ -332,7 +332,7 @@ static void sw64_pci_root_bridge_prepare(struct pci_host_bridge *bridge) bus->number = last_bus; bridge->swizzle_irq = pci_common_swizzle; - bridge->map_irq = sw64_pci_map_irq; + bridge->map_irq = sunway_pci_map_irq; init_busnr = (0xff << 16) + ((last_bus + 1) << 8) + (last_bus); writel(init_busnr, (hose->rc_config_space_base + RC_PRIMARY_BUS)); @@ -372,7 +372,7 @@ sw64_pci_root_bridge_reserve_legacy_io(struct pci_host_bridge *bridge) } } -void sw64_pci_root_bridge_scan_finish_up(struct pci_host_bridge *bridge) +void sunway_pci_root_bridge_scan_finish(struct pci_host_bridge *bridge) { struct pci_controller *hose = NULL; struct pci_bus *bus = NULL; @@ -448,7 +448,7 @@ int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge) set_dev_node(bus_dev, hose->node); /* Some quirks for Sunway PCIe controller before scanning */ - sw64_pci_root_bridge_prepare(bridge); + sunway_pci_root_bridge_prepare(bridge); return 0; } diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c index 58d19e109f61..0b2e7bb60707 100644 --- a/drivers/acpi/pci_mcfg.c +++ b/drivers/acpi/pci_mcfg.c @@ -211,15 +211,15 @@ static struct mcfg_fixup mcfg_quirks[] = { { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 10), MCFG_BUS_ANY, ops }, \ { "SUNWAY", table_id, rev, ((node) * MAX_NR_RCS_PER_NODE + 11), MCFG_BUS_ANY, ops } \ - /* up to 8 nodes for SW64 series */ - SW64_ECAM_QUIRK("SUNWAY ", 1, 0x00, &sw64_pci_ecam_ops), - SW64_ECAM_QUIRK("SUNWAY ", 1, 0x01, &sw64_pci_ecam_ops), - SW64_ECAM_QUIRK("SUNWAY ", 1, 0x02, &sw64_pci_ecam_ops), - SW64_ECAM_QUIRK("SUNWAY ", 1, 0x03, &sw64_pci_ecam_ops), - SW64_ECAM_QUIRK("SUNWAY ", 1, 0x04, &sw64_pci_ecam_ops), - SW64_ECAM_QUIRK("SUNWAY ", 1, 0x05, &sw64_pci_ecam_ops), - SW64_ECAM_QUIRK("SUNWAY ", 1, 0x06, &sw64_pci_ecam_ops), - SW64_ECAM_QUIRK("SUNWAY ", 1, 0x07, &sw64_pci_ecam_ops), + /* Up to 8 nodes for Sunway PCIe controller */ + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x00, &sunway_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x01, &sunway_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x02, &sunway_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x03, &sunway_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x04, &sunway_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x05, &sunway_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x06, &sunway_pci_ecam_ops), + SW64_ECAM_QUIRK("SUNWAY ", 1, 0x07, &sunway_pci_ecam_ops), #endif /* SW64 */ }; diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index f8c24860edfd..bce0659e6bf0 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -261,7 +261,7 @@ static unsigned long get_rc_enable(unsigned long node) return rc_enable; } -static int map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +int sunway_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) { struct pci_controller *hose = pci_bus_to_pci_controller(dev->bus); @@ -323,7 +323,6 @@ static void hose_init(struct pci_controller *hose) }; static struct sw64_pci_init_ops chip_pci_init_ops = { - .map_irq = map_irq, .get_rc_enable = get_rc_enable, .hose_init = hose_init, .set_rc_piu = set_rc_piu, @@ -399,7 +398,7 @@ EXPORT_SYMBOL(pci_bus_to_pci_controller); /** * PCIe Root Complex read config space operations */ -static int sw64_pcie_read_rc_cfg(struct pci_bus *bus, unsigned int devfn, +static int pci_read_rc_cfg(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { u32 data; @@ -449,7 +448,7 @@ static int sw64_pcie_read_rc_cfg(struct pci_bus *bus, unsigned int devfn, /** * PCIe Root Complex write config space operations */ -static int sw64_pcie_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, +static int pci_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { u32 data; @@ -488,7 +487,7 @@ static int sw64_pcie_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, } /** - * sw64_pcie_valid_device - check if a valid device is present + * pci_valid_device - check if a valid device is present * on bus * * @bus : PCI bus structure @@ -496,7 +495,7 @@ static int sw64_pcie_write_rc_cfg(struct pci_bus *bus, unsigned int devfn, * * @return: 'true' on success and 'false' if invalid device is found */ -static bool sw64_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) +static bool pci_valid_device(struct pci_bus *bus, unsigned int devfn) { struct pci_controller *hose = pci_bus_to_pci_controller(bus); @@ -510,7 +509,7 @@ static bool sw64_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) } /** - * sw64_pcie_config_read - read val from config space of + * sunway_pci_config_read - read val from config space of * PCI host controller or device * * @bus : PCI bus structure @@ -521,7 +520,7 @@ static bool sw64_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) * * @return: Whether read operation success */ -int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, +int sunway_pci_config_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { struct pci_controller *hose = pci_bus_to_pci_controller(bus); @@ -533,7 +532,7 @@ int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, hose->self_busno = hose->busn_space->start; if (unlikely(bus->number == hose->self_busno)) { - ret = sw64_pcie_read_rc_cfg(bus, devfn, where, size, val); + ret = pci_read_rc_cfg(bus, devfn, where, size, val); } else { if (pci_get_rc_linkup(hose)) ret = pci_generic_config_read(bus, devfn, where, size, val); @@ -542,10 +541,10 @@ int sw64_pcie_config_read(struct pci_bus *bus, unsigned int devfn, } return ret; } -EXPORT_SYMBOL(sw64_pcie_config_read); +EXPORT_SYMBOL(sunway_pci_config_read); /** - * sw64_pcie_config_write - write val to config space of PCI + * sunway_pci_config_write - write val to config space of PCI * host controller or device * * @bus : PCI bus structure @@ -556,7 +555,7 @@ EXPORT_SYMBOL(sw64_pcie_config_read); * * @return: Whether write operation success */ -int sw64_pcie_config_write(struct pci_bus *bus, unsigned int devfn, +int sunway_pci_config_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { struct pci_controller *hose = pci_bus_to_pci_controller(bus); @@ -567,14 +566,14 @@ int sw64_pcie_config_write(struct pci_bus *bus, unsigned int devfn, hose->self_busno = hose->busn_space->start; if (unlikely(bus->number == hose->self_busno)) - return sw64_pcie_write_rc_cfg(bus, devfn, where, size, val); + return pci_write_rc_cfg(bus, devfn, where, size, val); else return pci_generic_config_write(bus, devfn, where, size, val); } -EXPORT_SYMBOL(sw64_pcie_config_write); +EXPORT_SYMBOL(sunway_pci_config_write); /** - * sw64_pcie_map_bus - get configuration base address + * sunway_pci_map_bus - get configuration base address * @bus : PCI bus structure * @devfn: device/function * @where: offset from base @@ -582,14 +581,14 @@ EXPORT_SYMBOL(sw64_pcie_config_write); * @return: base address of the configuration space needed to be * accessed. */ -void __iomem *sw64_pcie_map_bus(struct pci_bus *bus, +void __iomem *sunway_pci_map_bus(struct pci_bus *bus, unsigned int devfn, int where) { struct pci_controller *hose = pci_bus_to_pci_controller(bus); void __iomem *cfg_iobase; unsigned long relbus; - if (!sw64_pcie_valid_device(bus, devfn)) + if (!pci_valid_device(bus, devfn)) return NULL; /** @@ -611,12 +610,7 @@ void __iomem *sw64_pcie_map_bus(struct pci_bus *bus, cfg_iobase, bus->number, devfn, where); return cfg_iobase; } -EXPORT_SYMBOL(sw64_pcie_map_bus); - -int sw64_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) -{ - return map_irq(dev, slot, pin); -} +EXPORT_SYMBOL(sunway_pci_map_bus); enum pci_props { PROP_RC_CONFIG_BASE = 0, @@ -679,7 +673,7 @@ static void pci_controller_set_node(struct pci_controller *hose, } #endif -static int sw64_pci_prepare_controller(struct pci_controller *hose, +static int pci_prepare_controller(struct pci_controller *hose, struct fwnode_handle *fwnode) { u64 props[PROP_NUM]; @@ -756,7 +750,7 @@ static int sw64_pci_prepare_controller(struct pci_controller *hose, } #ifdef CONFIG_ACPI -static int sw64_pci_acpi_present(struct device *dev) +static int pci_acpi_present(struct device *dev) { struct acpi_device *adev = to_acpi_device(dev); int ret; @@ -789,7 +783,7 @@ static int sw64_pci_acpi_present(struct device *dev) /** * Use the information from ACPI or DTB to init pci_controller */ -static int sw64_pci_ecam_init(struct pci_config_window *cfg) +static int sunway_pci_ecam_init(struct pci_config_window *cfg) { struct pci_controller *hose = NULL; struct device *dev = cfg->parent; @@ -798,7 +792,7 @@ static int sw64_pci_ecam_init(struct pci_config_window *cfg) #ifdef CONFIG_ACPI if (!acpi_disabled) { - ret = sw64_pci_acpi_present(dev); + ret = pci_acpi_present(dev); if (ret) return ret; @@ -811,7 +805,7 @@ static int sw64_pci_ecam_init(struct pci_config_window *cfg) return -ENOMEM; /* Init pci_controller */ - ret = sw64_pci_prepare_controller(hose, fwnode); + ret = pci_prepare_controller(hose, fwnode); if (ret) { kfree(hose); dev_err(dev, "failed to init pci controller\n"); @@ -823,13 +817,13 @@ static int sw64_pci_ecam_init(struct pci_config_window *cfg) return 0; } -const struct pci_ecam_ops sw64_pci_ecam_ops = { +const struct pci_ecam_ops sunway_pci_ecam_ops = { .bus_shift = 24, - .init = sw64_pci_ecam_init, + .init = sunway_pci_ecam_init, .pci_ops = { - .map_bus = sw64_pcie_map_bus, - .read = sw64_pcie_config_read, - .write = sw64_pcie_config_write, + .map_bus = sunway_pci_map_bus, + .read = sunway_pci_config_read, + .write = sunway_pci_config_write, } }; @@ -838,7 +832,7 @@ const struct pci_ecam_ops sw64_pci_ecam_ops = { static const struct of_device_id sunway_pcie_of_match[] = { { .compatible = "sunway,pcie", - .data = &sw64_pci_ecam_ops, + .data = &sunway_pci_ecam_ops, }, {}, }; @@ -925,7 +919,7 @@ static int sunway_pcie_probe(struct platform_device *pdev) * Some quirks for Sunway PCIe controller after scanning, * that's why we don't directly call function pci_host_probe(). */ - sw64_pci_root_bridge_scan_finish_up(bridge); + sunway_pci_root_bridge_scan_finish(bridge); pci_bus_size_bridges(bus); pci_bus_assign_resources(bus); diff --git a/include/linux/pci-ecam.h b/include/linux/pci-ecam.h index 863e572202e2..ac473d00e9a1 100644 --- a/include/linux/pci-ecam.h +++ b/include/linux/pci-ecam.h @@ -88,7 +88,7 @@ extern const struct pci_ecam_ops xgene_v2_pcie_ecam_ops; /* APM X-Gene PCIe v2.x extern const struct pci_ecam_ops al_pcie_ops; /* Amazon Annapurna Labs PCIe */ extern const struct pci_ecam_ops tegra194_pcie_ops; /* Tegra194 PCIe */ extern const struct pci_ecam_ops loongson_pci_ecam_ops; /* Loongson PCIe */ -extern const struct pci_ecam_ops sw64_pci_ecam_ops; /* SW64 PCIe */ +extern const struct pci_ecam_ops sunway_pci_ecam_ops; /* Sunway PCIe */ #endif #if IS_ENABLED(CONFIG_PCI_HOST_COMMON) -- Gitee From 4deb6ff7dd3a276dd7395db99f379ce18846ec06 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Wed, 14 Aug 2024 09:55:00 +0800 Subject: [PATCH 296/524] sw64: fix a bug in huge_pte_offset() In the function huge_pte_offset(), when the value of pmd is detected as "unpresent", it returns an unexpected "NULL" instead of the address of the pmd. This patch ensures that huge_pte_offset() returns the expected value. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/hugetlbpage.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/mm/hugetlbpage.c b/arch/sw_64/mm/hugetlbpage.c index fae1fa8bf7df..f986422398ef 100644 --- a/arch/sw_64/mm/hugetlbpage.c +++ b/arch/sw_64/mm/hugetlbpage.c @@ -94,11 +94,12 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, pud = pud_offset(p4d, addr); if (pud_present(*pud)) { pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) - return NULL; - if (pmd_val(*pmd) & _PAGE_CONT) - pte = pte_offset_map(pmd, addr); - else + if (sz == CONT_PMD_SIZE) { + if (!pmd_none(*pmd)) + pte = pte_offset_map(pmd, addr); + else + return NULL; + } else if (sz == PMD_SIZE) pte = (pte_t *) pmd; } } -- Gitee From e61903926057bffe0bf3fb7f9dea33de91fab9eb Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 2 Aug 2024 10:29:43 +0800 Subject: [PATCH 297/524] sw64: clk: fix the clock frequency of SPI and I2C Fix the clock frequency of SPI and I2C to make these peripherals work properly. This commit also fixes some format issues and typos for junzhang's default dts file. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/junzhang.dts | 25 ++++++++++++------------- drivers/acpi/acpi_apd.c | 31 ++++++++++++++++++++++++------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/arch/sw_64/boot/dts/junzhang.dts b/arch/sw_64/boot/dts/junzhang.dts index 8712678a66e8..7ec626f00faa 100644 --- a/arch/sw_64/boot/dts/junzhang.dts +++ b/arch/sw_64/boot/dts/junzhang.dts @@ -21,18 +21,17 @@ soc { clocks { i2cclk: i2cclk { - compatible = "fixed-clock"; - clock-frequency = <12500000>; - #clock-cells = <0>; - clock-output-names = "i2cclk_12.5mhz"; + compatible = "fixed-clock"; + clock-frequency = <12000000>; + #clock-cells = <0>; + clock-output-names = "i2cclk_12mhz"; }; spiclk: spiclk { - compatible = "fixed-clock"; - clock-frequency = <12500000>; - #clock-cells = <0>; - clock-output-names = "spiclk_12.5mhz"; + compatible = "fixed-clock"; + clock-frequency = <12000000>; + #clock-cells = <0>; + clock-output-names = "spiclk_12mhz"; }; - }; pintc: interrupt-controller { @@ -135,7 +134,7 @@ pvt: pvt@0x8030 { compatible = "sw64,pvt-vol"; reg = <0x8030 0x0 0x0 0x7c00>; status = "okay"; - }; + }; spi: spi@0x8032 { #address-cells = <1>; @@ -235,7 +234,7 @@ gpio: gpio@8036 { reg = <0x8036 0x0 0x0 0x8000>; status = "okay"; - porta: gpio-contraller@0 { + porta: gpio-controller@0 { compatible = "snps,dw-apb-gpio-port"; gpio-controller; #gpio-cells = <2>; @@ -244,7 +243,7 @@ porta: gpio-contraller@0 { interrupt-controller; #interrupt-cells = <2>; interrupts-extended = <&pintc 0>, <&pintc 1>; - }; - }; + }; + }; }; }; diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index c5afed1eeb9c..e62f024f8d39 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -180,16 +180,33 @@ static const struct apd_device_desc hip08_spi_desc = { #endif /* CONFIG_ARM64 */ #ifdef CONFIG_SW64 -static const struct apd_device_desc sunway_i2c_desc = { - .setup = acpi_apd_setup, - .fixed_clk_rate = 25000000, +#include + +extern u64 sunway_mclk_hz; +static int sw64_acpi_apd_setup(struct apd_private_data *pdata); + +static struct apd_device_desc sunway_i2c_desc = { + .setup = sw64_acpi_apd_setup, + .fixed_clk_rate = 50000000, }; -static const struct apd_device_desc sunway_spi_desc = { - .setup = acpi_apd_setup, - .fixed_clk_rate = 25000000, +static struct apd_device_desc sunway_spi_desc = { + .setup = sw64_acpi_apd_setup, + .fixed_clk_rate = 50000000, }; -#endif + +static int sw64_acpi_apd_setup(struct apd_private_data *pdata) +{ + struct apd_device_desc *dev_desc = (struct apd_device_desc *)pdata->dev_desc; + + if (sunway_machine_is_compatible("sunway,junzhang")) + dev_desc->fixed_clk_rate = 12000000; + else + dev_desc->fixed_clk_rate = sunway_mclk_hz; + + return acpi_apd_setup(pdata); +} +#endif /* CONFIG_SW64 */ #endif -- Gitee From 9ad297d2a9f5184cd31fe4c4af7d59ab408ea741 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 23 Aug 2024 07:52:27 +0000 Subject: [PATCH 298/524] sw64: iommu: improve iommu initialization After restructuring of pci controller initialization, IOMMU should get its reg_base_addr from pci_controller instead of making it up again to match the semantic. Add the following two methods to help improve these conditions: - Setup iommu->reg_base by fetching pci_controller->piu_ior0_base. - Check whether the IOMMU node has been initialized to make sure there are no unexpected behaviours happening if PCI accidentally init the same IOMMU twice. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 5 ++++- drivers/iommu/sw64/iommu_v2.c | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index 94b01ca35ecd..f705d10830ff 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -760,7 +760,7 @@ static struct sunway_iommu *sunway_iommu_early_init(struct pci_controller *hose) iommu->iommu_dtbr = page_address(page); base = __pa(iommu->iommu_dtbr) & PAGE_MASK; - iommu->reg_base_addr = __va(MK_PIU_IOR0(iommu->node, iommu->index)); + iommu->reg_base_addr = hose->piu_ior0_base; writeq(base, iommu->reg_base_addr + DTBASEADDR); hose->pci_iommu = iommu; @@ -796,6 +796,9 @@ static int sunway_iommu_init(void) continue; } + if (hose->iommu_enable) + continue; + iommu = sunway_iommu_early_init(hose); if (!iommu) { pr_err("Allocating sunway_iommu failed\n"); diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index b2b68c9836a2..d9e3b75e6c9e 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -590,7 +590,7 @@ static struct sunway_iommu *sunway_iommu_early_init(struct pci_controller *hose) iommu->iommu_dtbr = page_address(page); base = __pa(iommu->iommu_dtbr) & PAGE_MASK; - iommu->reg_base_addr = __va(MK_PIU_IOR0(iommu->node, iommu->index)); + iommu->reg_base_addr = hose->piu_ior0_base; writeq(base, iommu->reg_base_addr + DTBASEADDR); hose->pci_iommu = iommu; @@ -971,8 +971,9 @@ static int sunway_iommu_acpi_init(void) return ret; for_each_iommu(iommu) { - if (!iommu->enabled) + if (!iommu->enabled || hose->iommu_enable) continue; + iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", iommu_index); iommu_device_register(&iommu->iommu, &sunway_iommu_ops, NULL); @@ -1008,6 +1009,9 @@ static int sunway_iommu_legacy_init(void) continue; } + if (hose->iommu_enable) + continue; + iommu = sunway_iommu_early_init(hose); iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", iommu_index); -- Gitee From f6b408df30ee487214f11db6464514643cf91eb5 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 20 Aug 2024 12:13:43 +0800 Subject: [PATCH 299/524] sw64: mm: remove legacy memory detection for JunZhang platform We prefer to get memory information from firmware instead of detecting memory in kernel to improve kernel compatibility. Due to the latest firmware being able to provide comprehensive memory information, it's time to remove legacy memory detection for JunZhang platform. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/init.c | 18 +++++++++--------- arch/sw_64/mm/numa.c | 9 +++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index 0ac65a2fd61c..6e976d7b20e4 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -328,16 +328,16 @@ core_initcall_sync(sw64_kvm_pool_init); void __init sw64_memblock_init(void) { - /** - * Detect all memory on all nodes, used in the following - * cases: - * 1. Legacy memory detect - * 2. Legacy NUMA initialization - */ - setup_socket_info(); - show_socket_mem_layout(); - if (sunway_boot_magic != 0xDEED2024UL) { + /** + * Detect all memory on all nodes, used in the following + * cases: + * 1. Legacy memory detect + * 2. Legacy NUMA initialization + */ + setup_socket_info(); + show_socket_mem_layout(); + /* Find our usable memory */ mem_detect(); diff --git a/arch/sw_64/mm/numa.c b/arch/sw_64/mm/numa.c index e46372f3ac77..2793718c66f6 100644 --- a/arch/sw_64/mm/numa.c +++ b/arch/sw_64/mm/numa.c @@ -9,6 +9,7 @@ #include #include +#include int cpu_to_node_map[NR_CPUS]; cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; @@ -294,6 +295,14 @@ static int __init manual_numa_init(void) struct memblock_region *mblk; phys_addr_t node_base, node_size, node_end; + /** + * When boot magic is 0xDEED2024UL, legacy memory detection is + * completely bypassed, causing manual_numa_init failure. So we + * disable numa here. + */ + if (sunway_boot_magic == 0xDEED2024UL) + numa_off = 1; + if (numa_off) { pr_info("NUMA disabled\n"); /* Forced off on command line. */ pr_info("Faking one node at [mem %#018llx-%#018llx]\n", -- Gitee From c6b7705089b1c0b587109bd34ddd9b3e2d881a36 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 25 Apr 2025 19:06:46 +0800 Subject: [PATCH 300/524] sw64: fix compiler error when enabling CONFIG_SW64_CPUAUTOPLUG This patch adapts get_idle_time() and get_idle_time_jiffy() to sw64_get_idle_time() and sw64_get_idle_time_jiffy(), fixing the complier error that occurs when CONFIG_SW64_CPUAUTOPLUG is enabled. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/cpuautoplug.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/arch/sw_64/kernel/cpuautoplug.c b/arch/sw_64/kernel/cpuautoplug.c index a7571a77a72c..4fdac8a1f726 100644 --- a/arch/sw_64/kernel/cpuautoplug.c +++ b/arch/sw_64/kernel/cpuautoplug.c @@ -213,7 +213,7 @@ static cputime64_t calc_busy_time(unsigned int cpu) return busy_time; } -static inline cputime64_t get_idle_time_jiffy(cputime64_t *wall) +static inline cputime64_t sw64_get_idle_time_jiffy(cputime64_t *wall) { unsigned int cpu; cputime64_t idle_time = 0; @@ -234,7 +234,7 @@ static inline cputime64_t get_idle_time_jiffy(cputime64_t *wall) return (cputime64_t)jiffies_to_usecs(idle_time); } -static inline cputime64_t get_idle_time(cputime64_t *wall) +static inline cputime64_t sw64_get_idle_time(cputime64_t *wall) { unsigned int cpu; u64 idle_time = 0; @@ -242,7 +242,7 @@ static inline cputime64_t get_idle_time(cputime64_t *wall) for_each_online_cpu(cpu) { idle_time += get_cpu_idle_time_us(cpu, wall); if (idle_time == -1ULL) - return get_idle_time_jiffy(wall); + return sw64_get_idle_time_jiffy(wall); } return idle_time; @@ -378,7 +378,7 @@ static void do_autoplug_timer(struct work_struct *work) goto out; } - cur_idle_time = get_idle_time(&cur_wall_time); + cur_idle_time = sw64_get_idle_time(&cur_wall_time); if (cur_wall_time == 0) cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); @@ -441,11 +441,16 @@ static struct platform_driver platform_driver = { static int __init cpuautoplug_init(void) { int i, ret, delay; + struct device *dev_root; - ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, + dev_root = bus_get_dev_root(&cpu_subsys); + if (dev_root) { + ret = sysfs_create_group(&dev_root->kobj, &cpuclass_attr_group); - if (ret) - return ret; + put_device(dev_root); + if (ret) + return ret; + } ret = platform_driver_register(&platform_driver); if (ret) -- Gitee From 70a6e3e6c99b0095828a1663ca5f56cfef477d28 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 25 Apr 2025 19:17:10 +0800 Subject: [PATCH 301/524] sw64: fix compiler error when enabling CONFIG_DEBUG_MATCH This patch fixes the compilation error that occurred when selecting CONFIG_DEBUG_MATCH due to changes in the upstream. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/match.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/arch/sw_64/kernel/match.c b/arch/sw_64/kernel/match.c index 9ecd5d838846..2d7eec649e31 100644 --- a/arch/sw_64/kernel/match.c +++ b/arch/sw_64/kernel/match.c @@ -211,8 +211,7 @@ static ssize_t da_match_set(struct file *file, const char __user *user_buf, if (err) return err; - if (on_each_cpu(write_da_match, NULL, 1)) - pr_crit("%s: timed out\n", __func__); + on_each_cpu(write_da_match, NULL, 1); return len; } @@ -258,8 +257,7 @@ static ssize_t dv_match_set(struct file *file, const char __user *user_buf, if (err) return err; - if (on_each_cpu(write_dv_match, NULL, 1)) - pr_crit("%s: timed out\n", __func__); + on_each_cpu(write_dv_match, NULL, 1); return len; } @@ -313,9 +311,7 @@ static ssize_t dav_match_set(struct file *file, const char __user *user_buf, if (err) return err; - - if (on_each_cpu(write_dav_match, NULL, 1)) - pr_crit("%s: timed out\n", __func__); + on_each_cpu(write_dav_match, NULL, 1); return len; } @@ -364,8 +360,7 @@ static ssize_t ia_match_set(struct file *file, const char __user *user_buf, if (err) return err; - if (on_each_cpu(write_ia_match, NULL, 1)) - pr_crit("%s: timed out\n", __func__); + on_each_cpu(write_ia_match, NULL, 1); return len; } @@ -406,8 +401,7 @@ static ssize_t iv_match_set(struct file *file, const char __user *user_buf, if (err) return err; - if (on_each_cpu(write_iv_match, NULL, 1)) - pr_crit("%s: timed out\n", __func__); + on_each_cpu(write_iv_match, NULL, 1); return len; } @@ -448,8 +442,7 @@ static ssize_t ida_match_set(struct file *file, const char __user *user_buf, if (err) return err; - if (on_each_cpu(write_ida_match, NULL, 1)) - pr_crit("%s: timed out\n", __func__); + on_each_cpu(write_ida_match, NULL, 1); return len; } -- Gitee From 654456903523bc5ea0d2efc4b06bbb389ec58c65 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 25 Apr 2025 21:40:14 +0800 Subject: [PATCH 302/524] sw64: fix complier error when enabling CONFIG_DYNAMIC_FTARCE Due to the change of the upstream, kprobe_ftrace_handler() no longer accepts the paramenter pt_regs but ftrace_regs. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/kprobes/kprobes-ftrace.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c index 89d7dba9dc25..ca3cee3a0316 100644 --- a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c +++ b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c @@ -11,8 +11,9 @@ /* Ftrace callback handler for kprobes */ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, - struct ftrace_ops *ops, struct pt_regs *regs) + struct ftrace_ops *ops, struct ftrace_regs *fregs) { + struct pt_regs *regs = ftrace_get_regs(fregs); struct kprobe *p; struct kprobe_ctlblk *kcb; -- Gitee From 11eb51202dd7305bd2867efff031eb290cc0134a Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 25 Apr 2025 21:45:58 +0800 Subject: [PATCH 303/524] sw64: fix compiler error when enabling CONFIG_RELOCATABLE Due to the changes of upstream, we need to reference the header file linux/panic_notifier.h to fix the compiler error. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 2 +- arch/sw_64/kernel/relocate.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index d941bf5cc2e9..6a1ba7b3799b 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -458,7 +458,7 @@ config RELOCATION_TABLE_SIZE hex "Relocation table size" depends on RELOCATABLE range 0x0 0x01000000 - default "0x80000" + default "0x100000" help A table of relocation data will be appended to the kernel binary and parsed at boot to fix up the relocated kernel. diff --git a/arch/sw_64/kernel/relocate.c b/arch/sw_64/kernel/relocate.c index ebdf7d894805..dfd323d76597 100644 --- a/arch/sw_64/kernel/relocate.c +++ b/arch/sw_64/kernel/relocate.c @@ -9,6 +9,7 @@ */ #include #include +#include #include #include -- Gitee From 5776a8423f219e53d3b99e6f36ac5002a19e3fe8 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 25 Apr 2025 21:52:27 +0800 Subject: [PATCH 304/524] sw64: fix compiler error of perf tests Solve the problem of compiler errors in perf tests caused by changes in the upstream. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/perf/arch/sw_64/include/arch-tests.h | 2 +- tools/perf/arch/sw_64/tests/arch-tests.c | 11 +++-------- tools/perf/arch/sw_64/tests/regs_load.S | 6 +++--- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/tools/perf/arch/sw_64/include/arch-tests.h b/tools/perf/arch/sw_64/include/arch-tests.h index 90ec4c8cb880..35bb987d797b 100644 --- a/tools/perf/arch/sw_64/include/arch-tests.h +++ b/tools/perf/arch/sw_64/include/arch-tests.h @@ -7,6 +7,6 @@ struct thread; struct perf_sample; #endif -extern struct test arch_tests[]; +extern struct test_suite *arch_tests[]; #endif diff --git a/tools/perf/arch/sw_64/tests/arch-tests.c b/tools/perf/arch/sw_64/tests/arch-tests.c index 5b1543c98022..ad16b4f8f63e 100644 --- a/tools/perf/arch/sw_64/tests/arch-tests.c +++ b/tools/perf/arch/sw_64/tests/arch-tests.c @@ -3,14 +3,9 @@ #include "tests/tests.h" #include "arch-tests.h" -struct test arch_tests[] = { +struct test_suite *arch_tests[] = { #ifdef HAVE_DWARF_UNWIND_SUPPORT - { - .desc = "DWARF unwind", - .func = test__dwarf_unwind, - }, + &suite__dwarf_unwind, #endif - { - .func = NULL, - }, + NULL, }; diff --git a/tools/perf/arch/sw_64/tests/regs_load.S b/tools/perf/arch/sw_64/tests/regs_load.S index 8c5aabc2c6fb..35f7fb65012e 100644 --- a/tools/perf/arch/sw_64/tests/regs_load.S +++ b/tools/perf/arch/sw_64/tests/regs_load.S @@ -3,12 +3,12 @@ .text .set noat -.type perf_regs_load,%function +.globl perf_regs_load #define STL_REG(r) stl $r, (8 * r)($16) #define LDL_REG(r) ldl $r, (8 * r)($16) #define SP (8 * 30) #define PC (8 * 31) -SYM_FUNC_START(perf_regs_load) +perf_regs_load: STL_REG(0) STL_REG(1) STL_REG(2) @@ -44,4 +44,4 @@ SYM_FUNC_START(perf_regs_load) stl $26, (PC)($16) LDL_REG(17) ret -SYM_FUNC_END(perf_regs_load) +.end perf_regs_load -- Gitee From c2064ff9b998be883624cee3f772cb4d66dee622 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 23 Aug 2024 03:24:42 +0000 Subject: [PATCH 305/524] sw64: fix LPC legacy IO space on chip junzhang and junzhang_v2 According to commit a15ee190f55d ("sw64: lpc: work around hardware flaws"), PCI Legacy IO address can not be accessed normally, so we borrow memory IO address to access legacy IO on chip junchang/junzhang_v2. The above commit can handle most access cases, as LPC driver can be successfully directed to the new address space by retrieving lpc->hst_regs from ACPI or DTS. However, if an uninformed user use inb/outb directly from userspace to initiate an access, it will be transformed into accessing an illegal address space as current inb/outb still use the original legacy IO space address. To optimize the iomap performance, initiate the corresponding enhancements globally during arch setup by adding a check on chip types to differentiate LPC legacy IO space. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 30 ++++++++++++++++++++++++++++++ arch/sw_64/lib/iomap.c | 13 ++++++------- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index f55ddc856df9..8e255480470a 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -118,6 +118,9 @@ EXPORT_SYMBOL(sunway_boot_magic); unsigned long sunway_dtb_address; EXPORT_SYMBOL(sunway_dtb_address); +unsigned long legacy_io_base; +unsigned long legacy_io_shift; + u64 sunway_mclk_hz; u64 sunway_extclk_hz; @@ -552,6 +555,30 @@ static void __init setup_firmware_fdt(void) } } +static void __init setup_legacy_io(void) +{ + if (is_guest_or_emul()) { + legacy_io_base = PCI_VT_LEGACY_IO; + legacy_io_shift = 0; + return; + } + + if (sunway_machine_is_compatible("sunway,junzhang") || + sunway_machine_is_compatible("sunway,junzhang_v2")) { + /* + * Due to a hardware defect, chip junzhang and junzhang_v2 cannot + * recognize accesses to LPC legacy IO. The workaround is using some + * of the LPC MEMIO space to access Legacy IO space. Thus, + * legacy_io_base should be LPC_MEM_IO instead on these chips. + */ + legacy_io_base = LPC_MEM_IO; + legacy_io_shift = 12; + } else { + legacy_io_base = LPC_LEGACY_IO; + legacy_io_shift = 0; + } +} + static void __init setup_builtin_fdt(void) { void *dt_virt; @@ -712,6 +739,9 @@ setup_arch(char **cmdline_p) /* Now we get the final boot_command_line */ *cmdline_p = boot_command_line; + /* Decide legacy IO base addr based on chips */ + setup_legacy_io(); + /* Register a call for panic conditions. */ atomic_notifier_chain_register(&panic_notifier_list, &sw64_panic_block); diff --git a/arch/sw_64/lib/iomap.c b/arch/sw_64/lib/iomap.c index f89cf16f683c..c73004a87dcd 100644 --- a/arch/sw_64/lib/iomap.c +++ b/arch/sw_64/lib/iomap.c @@ -8,6 +8,9 @@ #include #include +extern unsigned long legacy_io_base; +extern unsigned long legacy_io_shift; + /* * Here comes the sw64 implementation of the IOMAP interfaces. */ @@ -258,13 +261,9 @@ EXPORT_SYMBOL(_memset_c_io); void __iomem *ioport_map(unsigned long port, unsigned int size) { - unsigned long io_offset; - - if (port < 0x100000) { - io_offset = is_in_host() ? LPC_LEGACY_IO : PCI_VT_LEGACY_IO; - port = port | io_offset; - } + if (port >= 0x100000) + return __va(port); - return __va(port); + return __va((port << legacy_io_shift) | legacy_io_base); } EXPORT_SYMBOL(ioport_map); -- Gitee From 4bb8f58df9a99c3d540d64ed55a68074fdd0bf60 Mon Sep 17 00:00:00 2001 From: Zhou Xuemei Date: Tue, 3 Sep 2024 06:13:20 +0000 Subject: [PATCH 306/524] sw64: pci: remove legacy io reservation This patch removes the legacy io reservation code, which is no longer necessary in sw64 architecture. Additionally, the 'PCIBIOS_MIN_IO' value is adjusted to raise the base address for legacy io resource assignment. Signed-off-by: Zhou Xuemei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 2 +- arch/sw_64/pci/pci-legacy.c | 36 ------------------------------------ arch/sw_64/pci/pci.c | 33 --------------------------------- 3 files changed, 1 insertion(+), 70 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index f28a3acb015c..9662fdd87cb1 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -147,7 +147,7 @@ struct pci_controller { #define pcibios_assign_all_busses() (pci_has_flag(PCI_REASSIGN_ALL_BUS)) -#define PCIBIOS_MIN_IO 0 +#define PCIBIOS_MIN_IO 0x1000 #define PCIBIOS_MIN_MEM 0 extern void __init sw64_init_pci(void); diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index ccf87520c72d..8a9ac6f1146b 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -19,7 +19,6 @@ bool sunway_legacy_pci; */ struct pci_controller *hose_head, **hose_tail = &hose_head; -static void __init pcibios_reserve_legacy_regions(struct pci_bus *bus); static int __init pcibios_init(void) @@ -138,13 +137,7 @@ void __init common_init_pci(void) pcibios_claim_console_setup(); - if (is_in_host()) { - list_for_each_entry(bus, &pci_root_buses, node) - pcibios_reserve_legacy_regions(bus); - } - pr_info("SW arch assign unassigned resources.\n"); - pci_assign_unassigned_resources(); for (hose = hose_head; hose; hose = hose->next) { @@ -177,35 +170,6 @@ alloc_resource(void) return res; } -static void __init pcibios_reserve_legacy_regions(struct pci_bus *bus) -{ - struct pci_controller *hose = pci_bus_to_pci_controller(bus); - resource_size_t offset; - struct resource *res; - - pr_debug("Reserving legacy ranges for domain %04x\n", pci_domain_nr(bus)); - - /* Check for IO */ - if (!(hose->io_space->flags & IORESOURCE_IO)) - goto no_io; - offset = (unsigned long)hose->io_space->start; - res = kzalloc(sizeof(struct resource), GFP_KERNEL); - BUG_ON(res == NULL); - res->name = "Legacy IO"; - res->flags = IORESOURCE_IO; - res->start = offset; - res->end = (offset + 0xfff) & 0xfffffffffffffffful; - pr_debug("Candidate legacy IO: %pR\n", res); - if (request_resource(hose->io_space, res)) { - pr_debug("PCI %04x:%02x Cannot reserve Legacy IO %pR\n", - pci_domain_nr(bus), bus->number, res); - kfree(res); - } - -no_io: - return; -} - static bool rc_linkup[MAX_NUMNODES][MAX_NR_RCS_PER_NODE]; static void __init diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index d697cc0f246f..47b8067fed03 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -342,36 +342,6 @@ static void sunway_pci_root_bridge_prepare(struct pci_host_bridge *bridge) pci_add_flags(PCI_REASSIGN_ALL_BUS); } -static void -sw64_pci_root_bridge_reserve_legacy_io(struct pci_host_bridge *bridge) -{ - struct pci_bus *bus = bridge->bus; - struct resource_entry *entry = NULL; - struct resource *res = NULL; - - resource_list_for_each_entry(entry, &bridge->windows) { - if (!(entry->res->flags & IORESOURCE_IO)) - continue; - - res = kzalloc(sizeof(struct resource), GFP_KERNEL); - if (WARN_ON(!res)) - return; - - res->name = "legacy io"; - res->flags = IORESOURCE_IO; - res->start = entry->res->start; - res->end = (res->start + 0xFFF) & 0xFFFFFFFFFFFFFFFFUL; - - pr_info("reserving legacy io %pR for domain %04x\n", - res, pci_domain_nr(bus)); - if (request_resource(entry->res, res)) { - pr_err("pci %04x:%02x reserve legacy io %pR failed\n", - pci_domain_nr(bus), bus->number, res); - kfree(res); - } - } -} - void sunway_pci_root_bridge_scan_finish(struct pci_host_bridge *bridge) { struct pci_controller *hose = NULL; @@ -401,9 +371,6 @@ void sunway_pci_root_bridge_scan_finish(struct pci_host_bridge *bridge) pci_bus_update_busn_res_end(bus, last_bus); last_bus++; - if (is_in_host()) - sw64_pci_root_bridge_reserve_legacy_io(bridge); - /** * Root Complex of SW64 does not support ASPM, causing * control field(_OSC) unable to be updated. -- Gitee From 2af581b661bef401003b4cb5b0c750e537c1e686 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 2 Sep 2024 15:59:19 +0800 Subject: [PATCH 307/524] Revert "sw64: fdt: map physical node ID to logical node ID" This reverts commit 412cac38c1e83432b586064b6fa6be29fa46c015. We can not map physical node ID to logical node ID for CPU affinity without considering memory affinity. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 41 ++--------------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 63a3139bf9ba..a12fae659cdf 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -24,13 +24,6 @@ struct smp_rcb_struct *smp_rcb; extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; -static nodemask_t nodes_found_map = NODE_MASK_NONE; - /* maps to convert between physical node ID and logical node ID */ -static int pnode_to_lnode_map[MAX_NUMNODES] - = { [0 ... MAX_NUMNODES - 1] = NUMA_NO_NODE }; -static int lnode_to_pnode_map[MAX_NUMNODES] - = { [0 ... MAX_NUMNODES - 1] = NUMA_NO_NODE }; - void *idle_task_pointer[NR_CPUS]; /* State of each CPU */ @@ -288,33 +281,6 @@ static int __init sw64_of_core_version(const struct device_node *dn, return -EINVAL; } -static void __fdt_map_pnode_to_lnode(int pnode, int lnode) -{ - if (pnode_to_lnode_map[pnode] == NUMA_NO_NODE || lnode < pnode_to_lnode_map[pnode]) - pnode_to_lnode_map[pnode] = lnode; - if (lnode_to_pnode_map[lnode] == NUMA_NO_NODE || pnode < lnode_to_pnode_map[lnode]) - lnode_to_pnode_map[lnode] = pnode; -} - -static int fdt_map_pnode_to_lnode(int pnode) -{ - int lnode; - - if (pnode < 0 || pnode >= MAX_NUMNODES || numa_off) - return NUMA_NO_NODE; - lnode = pnode_to_lnode_map[pnode]; - - if (lnode == NUMA_NO_NODE) { - if (nodes_weight(nodes_found_map) >= MAX_NUMNODES) - return NUMA_NO_NODE; - lnode = first_unset_node(nodes_found_map); - __fdt_map_pnode_to_lnode(pnode, lnode); - node_set(lnode, nodes_found_map); - } - - return lnode; -} - static int __init fdt_setup_smp(void) { struct device_node *dn = NULL; @@ -322,7 +288,7 @@ static int __init fdt_setup_smp(void) u32 rcid, logical_core_id = 0; u32 online_capable = 0; bool available; - int ret, i, version, lnode, pnode; + int ret, i, version; /* Clean the map from logical core ID to physical core ID */ for (i = 0; i < ARRAY_SIZE(__cpu_to_rcid); ++i) @@ -385,10 +351,7 @@ static int __init fdt_setup_smp(void) smp_rcb_init(__va(boot_flag_address)); /* Set core affinity */ - pnode = of_node_to_nid(dn); - lnode = fdt_map_pnode_to_lnode(pnode); - - early_map_cpu_to_node(logical_core_id, lnode); + early_map_cpu_to_node(logical_core_id, of_node_to_nid(dn)); logical_core_id++; } -- Gitee From b8d2ee4a366bf88407df40052f14fdf8e2b64dff Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 29 Aug 2024 13:47:58 +0800 Subject: [PATCH 308/524] sw64: smp: fix function fdt_setup_smp() This commit includes some small fixes: - Provide compatibility for CPU device node with vendor prefix "sunway". - Remove unnecessary logs. - When NR_CPUS less than the number detected by firmware, break the loop instead of the function return. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index a12fae659cdf..dfeedbc3e349 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -268,12 +268,14 @@ static int __init sw64_of_core_version(const struct device_node *dn, if (!dn || !version) return -EINVAL; - if (of_device_is_compatible(dn, "sw64,xuelang")) { + if (of_device_is_compatible(dn, "sw64,xuelang") || + of_device_is_compatible(dn, "sunway,xuelang")) { *version = CORE_VERSION_C3B; return 0; } - if (of_device_is_compatible(dn, "sw64,junzhang")) { + if (of_device_is_compatible(dn, "sw64,junzhang") || + of_device_is_compatible(dn, "sunway,junzhang")) { *version = CORE_VERSION_C4; return 0; } @@ -303,10 +305,8 @@ static int __init fdt_setup_smp(void) available = of_device_is_available(dn); - if (!available && !online_capable) { - pr_info("OF: Core is not available\n"); + if (!available && !online_capable) continue; - } ret = of_property_read_u32(dn, "reg", &rcid); if (ret) { @@ -317,7 +317,7 @@ static int __init fdt_setup_smp(void) if (logical_core_id >= nr_cpu_ids) { pr_warn_once("OF: Core [0x%x] exceeds max core num [%u]\n", rcid, nr_cpu_ids); - return 0; + break; } if (is_rcid_duplicate(rcid)) { @@ -333,6 +333,9 @@ static int __init fdt_setup_smp(void) ret = of_property_read_u64(dn, "sw64,boot_flag_address", &boot_flag_address); + if (ret) + ret = of_property_read_u64(dn, "sunway,boot_flag_address", + &boot_flag_address); if (ret) { pr_err("OF: No boot_flag_address found\n"); return ret; -- Gitee From 27abd662c9142d2cd8ebfbe0856c02fbe215bae3 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 29 Aug 2024 14:37:45 +0800 Subject: [PATCH 309/524] sw64: clk: further fix the clock frequency of SPI and I2C The clock frequency of SPI and I2C is equal to Sunway MCLK. Fixes: 889b6b1dba80 ("sw64: clk: fix the clock frequency of SPI and I2C") Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/acpi/acpi_apd.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index e62f024f8d39..6b1b31eabfdd 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -199,10 +199,7 @@ static int sw64_acpi_apd_setup(struct apd_private_data *pdata) { struct apd_device_desc *dev_desc = (struct apd_device_desc *)pdata->dev_desc; - if (sunway_machine_is_compatible("sunway,junzhang")) - dev_desc->fixed_clk_rate = 12000000; - else - dev_desc->fixed_clk_rate = sunway_mclk_hz; + dev_desc->fixed_clk_rate = sunway_mclk_hz; return acpi_apd_setup(pdata); } -- Gitee From 6f4bd7da3ab4e00937cb95abaad6e790f8879172 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 29 Aug 2024 14:53:13 +0800 Subject: [PATCH 310/524] sw64: acpi: suppress log of function acpi_numa_x2apic_affinity_init() Suppress unnecessary logs to make the kernel startup log clean. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/acpi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 02a4e8fa9c95..515728920cfd 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -148,7 +148,7 @@ acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) /* Record the mapping from logical core id to node id */ cpu = rcid_to_cpu(rcid); if (cpu < 0) { - pr_err("SRAT: Can not find the logical id for physical Core 0x%04x\n", + pr_warn_once("SRAT: Can not find the logical id for physical Core 0x%04x\n", rcid); return; } -- Gitee From 585bc7d40a7de9d3eca7a1bde2a0ccd037bf2b62 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 29 Aug 2024 15:01:11 +0800 Subject: [PATCH 311/524] sw64: fix null pointer issue when CONFIG_BUILTIN_DTB=y If CONFIG_BUILTIN_DTB=y, then initial_boot_params is initialized after function setup_builtin_fdt(), causing null pointer issue in function setup_legacy_io(). So we need to move setup_legacy_io() after setup_builtin_fdt(). Fixes: 158c423d4329 ("sw64: fix LPC legacy IO space on chip junzhang and junzhang_v2") Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 8e255480470a..b735fe0e3e59 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -739,9 +739,6 @@ setup_arch(char **cmdline_p) /* Now we get the final boot_command_line */ *cmdline_p = boot_command_line; - /* Decide legacy IO base addr based on chips */ - setup_legacy_io(); - /* Register a call for panic conditions. */ atomic_notifier_chain_register(&panic_notifier_list, &sw64_panic_block); @@ -764,6 +761,9 @@ setup_arch(char **cmdline_p) if (IS_ENABLED(CONFIG_BUILTIN_DTB)) setup_builtin_fdt(); + /* Decide legacy IO base addr based on chips */ + setup_legacy_io(); + sw64_memblock_init(); /* Try to upgrade ACPI tables via initrd */ -- Gitee From ed946d25b7fac5a84ed7e6ce8a37cd1cb6a13283 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 29 Aug 2024 15:27:28 +0800 Subject: [PATCH 312/524] sw64: irqchip: add version 3 for PINTC For forward compatibility. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-pintc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index e2f1c39538d6..eb7128b79c6f 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -188,6 +188,7 @@ static unsigned long make_pintc_int_target(u32 version, int rcid) target = core | (thread << 5) | (node << 6); break; case 0x2: /* PINTC v2 */ + case 0x3: /* PINTC v3 */ target = core | (thread << 6) | (node << 7); break; default: -- Gitee From cd9074bad1176eb04649c95ce4995170c2d37887 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 2 Sep 2024 15:40:44 +0800 Subject: [PATCH 313/524] sw64: defconfig: fix SPI related configuration items SPI_CHIP is renamed to SPI_SUNWAY, and SPI_CHIP_MMIO is renamed to SPI_SUNWAY_MMIO. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/configs/junzhang_defconfig | 4 ++-- arch/sw_64/configs/xuelang_defconfig | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/configs/junzhang_defconfig b/arch/sw_64/configs/junzhang_defconfig index 866d20c644d9..ba2eaafde797 100644 --- a/arch/sw_64/configs/junzhang_defconfig +++ b/arch/sw_64/configs/junzhang_defconfig @@ -525,8 +525,8 @@ CONFIG_I2C_CHARDEV=y CONFIG_I2C_DESIGNWARE_PLATFORM=y CONFIG_I2C_MUX=y CONFIG_SPI=y -CONFIG_SPI_CHIP=y -CONFIG_SPI_CHIP_MMIO=y +CONFIG_SPI_SUNWAY=y +CONFIG_SPI_SUNWAY_MMIO=y CONFIG_SPI_SPIDEV=y CONFIG_SENSORS_PVT=y CONFIG_SENSORS_LM75=y diff --git a/arch/sw_64/configs/xuelang_defconfig b/arch/sw_64/configs/xuelang_defconfig index 3da2cffa2085..a255565c411b 100644 --- a/arch/sw_64/configs/xuelang_defconfig +++ b/arch/sw_64/configs/xuelang_defconfig @@ -520,7 +520,7 @@ CONFIG_VIRTIO_CONSOLE=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_MUX=y CONFIG_SPI=y -CONFIG_SPI_CHIP3=y +CONFIG_SPI_SUNWAY=y CONFIG_SPI_SPIDEV=y CONFIG_SENSORS_PVT=y CONFIG_SENSORS_LM75=y -- Gitee From 3ec8903fb9e16acdcfbe093773592a5fad8d96cc Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 2 Sep 2024 18:37:19 +0800 Subject: [PATCH 314/524] sw64: spi: convert SPI_MASTER_GPIO_SS to SPI_CONTROLLER_GPIO_SS SPI_MASTER_GPIO_SS is renamed to SPI_CONTROLLER_GPIO_SS. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/spi/spi-sunway.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-sunway.c b/drivers/spi/spi-sunway.c index 6133173ea09a..37de04209c3b 100644 --- a/drivers/spi/spi-sunway.c +++ b/drivers/spi/spi-sunway.c @@ -350,7 +350,7 @@ int spi_chip_add_host(struct device *dev, struct spi_chip *spi_chip) master->handle_err = spi_chip_handle_err; master->max_speed_hz = spi_chip->max_freq; master->dev.of_node = dev->of_node; - master->flags = SPI_MASTER_GPIO_SS; + master->flags = SPI_CONTROLLER_GPIO_SS; master->max_transfer_size = spi_chip_max_length; master->max_message_size = spi_chip_max_length; -- Gitee From a8cd6df4c82c59e7451e4ff454c7be072a374b23 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Sat, 20 Jul 2024 09:52:34 +0800 Subject: [PATCH 315/524] sw64: topology: support initializing topology via DT or ACPI Implement functions parse_acpi_topology() and parse_dt_topology() to support getting topology information from device tree or ACPI PPTT table. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 2 + arch/sw_64/include/asm/acpi.h | 7 + arch/sw_64/include/asm/topology.h | 13 -- arch/sw_64/kernel/topology.c | 356 ++++++++++++++++++++++-------- 4 files changed, 276 insertions(+), 102 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 6a1ba7b3799b..7d64c5188688 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -5,6 +5,7 @@ config SW64 select ACPI select ACPI_GENERIC_GSI if ACPI select ACPI_MCFG if (ACPI && PCI) + select ACPI_PPTT if ACPI select ACPI_REDUCED_HARDWARE_ONLY select ARCH_ATOMIC select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI @@ -59,6 +60,7 @@ config SW64 select AUDIT_ARCH select COMMON_CLK select DMA_OPS if PCI + select GENERIC_ARCH_TOPOLOGY select GENERIC_CLOCKEVENTS select GENERIC_IRQ_LEGACY select GENERIC_IRQ_MIGRATION if SMP diff --git a/arch/sw_64/include/asm/acpi.h b/arch/sw_64/include/asm/acpi.h index ef46f481e1fd..7c3310168798 100644 --- a/arch/sw_64/include/asm/acpi.h +++ b/arch/sw_64/include/asm/acpi.h @@ -7,6 +7,7 @@ #include #include #include +#include #ifdef CONFIG_ACPI extern int acpi_noirq; @@ -105,6 +106,12 @@ static inline bool arch_has_acpi_pdc(void) static inline void arch_acpi_set_pdc_bits(u32 *buf) { } + +static inline u32 get_acpi_id_for_cpu(unsigned int cpu) +{ + /* We take rcid as processor _UID */ + return cpu_physical_id(cpu); +} #else /* !CONFIG_ACPI */ static inline void acpi_noirq_set(void) { } diff --git a/arch/sw_64/include/asm/topology.h b/arch/sw_64/include/asm/topology.h index c4da0cfcceff..48e66c51cfc9 100644 --- a/arch/sw_64/include/asm/topology.h +++ b/arch/sw_64/include/asm/topology.h @@ -9,19 +9,6 @@ #include #include -extern struct cpu_topology cpu_topology[NR_CPUS]; - -#define topology_physical_package_id(cpu) (cpu_topology[cpu].package_id) -#define topology_core_id(cpu) (cpu_topology[cpu].core_id) -#define topology_core_cpumask(cpu) (&cpu_topology[cpu].core_sibling) -#define topology_sibling_cpumask(cpu) (&cpu_topology[cpu].thread_sibling) -#define topology_llc_cpumask(cpu) (&cpu_topology[cpu].llc_sibling) - -void init_cpu_topology(void); -void store_cpu_topology(int cpuid); -void remove_cpu_topology(int cpuid); -const struct cpumask *cpu_coregroup_mask(int cpu); - static inline int rcid_to_thread_id(int rcid) { return (rcid & THREAD_ID_MASK) >> THREAD_ID_SHIFT; diff --git a/arch/sw_64/kernel/topology.c b/arch/sw_64/kernel/topology.c index b00f083517c3..2758b563b119 100644 --- a/arch/sw_64/kernel/topology.c +++ b/arch/sw_64/kernel/topology.c @@ -3,22 +3,14 @@ #include #include #include +#include +#include + #include #include #define OFFSET_SMP_INFO 0x80UL -static int __init parse_dt_topology(void) -{ - return 0; -} - -/* - * cpu topology table - */ -struct cpu_topology cpu_topology[NR_CPUS]; -EXPORT_SYMBOL_GPL(cpu_topology); - int topo_nr_threads, topo_nr_cores, topo_nr_maxcpus; static int topo_nr_cpus; @@ -91,126 +83,312 @@ static void __init init_topology_array(void) init_topo_packages(); } -const struct cpumask *cpu_coregroup_mask(int cpu) +/* + * This function returns the logic cpu number of the node. + * There are basically three kinds of return values: + * (1) logic cpu number which is > 0. + * (2) -ENODEV when the device tree(DT) node is valid and found in the DT but + * there is no possible logical CPU in the kernel to match. This happens + * when CONFIG_NR_CPUS is configure to be smaller than the number of + * CPU nodes in DT. We need to just ignore this case. + * (3) -1 if the node does not exist in the device tree + */ +static int __init get_cpu_for_node(struct device_node *node) { - return topology_llc_cpumask(cpu); -} + struct device_node *cpu_node; + int cpu; -static void update_siblings_masks(int cpu) -{ - struct cpu_topology *cpu_topo = &cpu_topology[cpu]; - int sib; + cpu_node = of_parse_phandle(node, "cpu", 0); + if (!cpu_node) + return -1; - /* update core and thread sibling masks */ - for_each_online_cpu(sib) { - struct cpu_topology *sib_topo = &cpu_topology[sib]; + cpu = of_cpu_node_to_id(cpu_node); + if (cpu >= 0) + topology_parse_cpu_capacity(cpu_node, cpu); + else + pr_info("CPU node for %pOF exist but the possible cpu range is :%*pbl\n", + cpu_node, cpumask_pr_args(cpu_possible_mask)); - if (cpu_topo->package_id == sib_topo->package_id) { - cpumask_set_cpu(cpu, &sib_topo->core_sibling); - cpumask_set_cpu(sib, &cpu_topo->core_sibling); - cpumask_set_cpu(cpu, &sib_topo->llc_sibling); - cpumask_set_cpu(sib, &cpu_topo->llc_sibling); + of_node_put(cpu_node); + return cpu; +} - if (cpu_topo->core_id == sib_topo->core_id) { - cpumask_set_cpu(cpu, &sib_topo->thread_sibling); - cpumask_set_cpu(sib, &cpu_topo->thread_sibling); +static int __init parse_core(struct device_node *core, int package_id, + int cluster_id, int core_id) +{ + char name[20]; + bool leaf = true; + int i = 0; + int cpu; + struct device_node *t; + + do { + snprintf(name, sizeof(name), "thread%d", i); + t = of_get_child_by_name(core, name); + if (t) { + leaf = false; + cpu = get_cpu_for_node(t); + if (cpu >= 0) { + cpu_topology[cpu].package_id = package_id; + cpu_topology[cpu].cluster_id = cluster_id; + cpu_topology[cpu].core_id = core_id; + cpu_topology[cpu].thread_id = i; + } else if (cpu != -ENODEV) { + pr_err("%pOF: Can't get CPU for thread\n", t); + of_node_put(t); + return -EINVAL; } + of_node_put(t); + } + i++; + } while (t); + + cpu = get_cpu_for_node(core); + if (cpu >= 0) { + if (!leaf) { + pr_err("%pOF: Core has both threads and CPU\n", + core); + return -EINVAL; } + + cpu_topology[cpu].package_id = package_id; + cpu_topology[cpu].cluster_id = cluster_id; + cpu_topology[cpu].core_id = core_id; + } else if (leaf && cpu != -ENODEV) { + pr_err("%pOF: Can't get CPU for leaf core\n", core); + return -EINVAL; } + + return 0; } -void store_cpu_topology(int cpu) +static int __init parse_cluster(struct device_node *cluster, int package_id, + int cluster_id, int depth) { - struct cpu_topology *cpu_topo = &cpu_topology[cpu]; + char name[20]; + bool leaf = true; + bool has_cores = false; + struct device_node *c; + int core_id = 0; + int i, ret; - if (cpu_topo->package_id != -1) - goto topology_populated; + /* + * First check for child clusters; we currently ignore any + * information about the nesting of clusters and present the + * scheduler with a flat list of them. + */ + i = 0; + do { + snprintf(name, sizeof(name), "cluster%d", i); + c = of_get_child_by_name(cluster, name); + if (c) { + leaf = false; + ret = parse_cluster(c, package_id, i, depth + 1); + if (depth > 0) + pr_warn("Topology for clusters of clusters not yet supported\n"); + of_node_put(c); + if (ret != 0) + return ret; + } + i++; + } while (c); + + /* Now check for cores */ + i = 0; + do { + snprintf(name, sizeof(name), "core%d", i); + c = of_get_child_by_name(cluster, name); + if (c) { + has_cores = true; + + if (depth == 0) { + pr_err("%pOF: cpu-map children should be clusters\n", + c); + of_node_put(c); + return -EINVAL; + } - if (is_guest_or_emul()) { - cpu_topo->package_id = topo_packages[cpu]; - cpu_topo->core_id = topo_cores[cpu]; - cpu_topo->thread_id = topo_threads[cpu]; - goto topology_populated; - } + if (leaf) { + ret = parse_core(c, package_id, cluster_id, + core_id++); + } else { + pr_err("%pOF: Non-leaf cluster with core %s\n", + cluster, name); + ret = -EINVAL; + } - cpu_topo->package_id = rcid_to_domain_id(cpu_to_rcid(cpu)); - cpu_topo->core_id = rcid_to_core_id(cpu_to_rcid(cpu)); - cpu_topo->thread_id = rcid_to_thread_id(cpu_to_rcid(cpu)); + of_node_put(c); + if (ret != 0) + return ret; + } + i++; + } while (c); - pr_debug("CPU%u: socket %d core %d thread %d\n", - cpu, cpu_topo->package_id, cpu_topo->core_id, - cpu_topo->thread_id); + if (leaf && !has_cores) + pr_warn("%pOF: empty cluster\n", cluster); -topology_populated: - update_siblings_masks(cpu); + return 0; } -static void clear_cpu_topology(int cpu) +static int __init parse_socket(struct device_node *socket) { - struct cpu_topology *cpu_topo = &cpu_topology[cpu]; + char name[20]; + struct device_node *c; + bool has_socket = false; + int package_id = 0, ret; + + do { + snprintf(name, sizeof(name), "socket%d", package_id); + c = of_get_child_by_name(socket, name); + if (c) { + has_socket = true; + ret = parse_cluster(c, package_id, -1, 0); + of_node_put(c); + if (ret != 0) + return ret; + } + package_id++; + } while (c); - cpumask_clear(&cpu_topo->llc_sibling); - cpumask_set_cpu(cpu, &cpu_topo->llc_sibling); + if (!has_socket) + ret = parse_cluster(socket, 0, -1, 0); - cpumask_clear(&cpu_topo->core_sibling); - cpumask_set_cpu(cpu, &cpu_topo->core_sibling); - cpumask_clear(&cpu_topo->thread_sibling); - cpumask_set_cpu(cpu, &cpu_topo->thread_sibling); + return ret; } -static void __init reset_cpu_topology(void) +static int __init parse_dt_topology(void) { + struct device_node *cn, *map; + int ret = 0; int cpu; - for_each_possible_cpu(cpu) { - struct cpu_topology *cpu_topo = &cpu_topology[cpu]; + cn = of_find_node_by_path("/cpus"); + if (!cn) { + pr_err("No CPU information found in DT\n"); + return 0; + } - cpu_topo->thread_id = -1; - cpu_topo->core_id = 0; - cpu_topo->package_id = -1; + /* + * When topology is provided cpu-map is essentially a root + * cluster with restricted subnodes. + */ + map = of_get_child_by_name(cn, "cpu-map"); + if (!map) + goto out; - clear_cpu_topology(cpu); - } -} + ret = parse_socket(map); + if (ret != 0) + goto out_map; -void remove_cpu_topology(int cpu) -{ - int sibling; + topology_normalize_cpu_scale(); - for_each_cpu(sibling, topology_core_cpumask(cpu)) - cpumask_clear_cpu(cpu, topology_core_cpumask(sibling)); - for_each_cpu(sibling, topology_sibling_cpumask(cpu)) - cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling)); - for_each_cpu(sibling, topology_llc_cpumask(cpu)) - cpumask_clear_cpu(cpu, topology_llc_cpumask(sibling)); + /* + * Check that all cores are in the topology; the SMP code will + * only mark cores described in the DT as possible. + */ + for_each_possible_cpu(cpu) + if (cpu_topology[cpu].package_id < 0) { + ret = -EINVAL; + break; + } - clear_cpu_topology(cpu); +out_map: + of_node_put(map); +out: + of_node_put(cn); + return ret; } #ifdef CONFIG_ACPI -static int __init parse_acpi_topology(void) +/* + * Propagate the topology information of the processor_topology_node tree to the + * cpu_topology array. + */ +int __init parse_acpi_topology(void) { + int cpu, topology_id; + + if (acpi_disabled) + return 0; + + for_each_possible_cpu(cpu) { + topology_id = find_acpi_cpu_topology(cpu, 0); + if (topology_id < 0) + return topology_id; + + if (acpi_pptt_cpu_is_thread(cpu) == 1) { + cpu_topology[cpu].thread_id = topology_id; + topology_id = find_acpi_cpu_topology(cpu, 1); + cpu_topology[cpu].core_id = topology_id; + } else { + cpu_topology[cpu].thread_id = -1; + cpu_topology[cpu].core_id = topology_id; + } + topology_id = find_acpi_cpu_topology_cluster(cpu); + cpu_topology[cpu].cluster_id = topology_id; + topology_id = find_acpi_cpu_topology_package(cpu); + cpu_topology[cpu].package_id = topology_id; + } + return 0; } -#else -static inline int __init parse_acpi_topology(void) -{ - return -EINVAL; -} #endif void __init init_cpu_topology(void) { + struct cpu_topology *boot_cpu_topo = &cpu_topology[0]; + int cpu, ret; + reset_cpu_topology(); + ret = parse_acpi_topology(); + if (!ret) + ret = of_have_populated_dt() && parse_dt_topology(); + + if (ret) { + /* + * Discard anything that was parsed if we hit an error so we + * don't use partial information. But do not return yet to give + * arch-specific early cache level detection a chance to run. + */ + reset_cpu_topology(); + } - if (is_guest_or_emul()) + /* Backward compatibility */ + if (is_guest_or_emul() && (boot_cpu_topo->package_id == -1)) init_topology_array(); - /* - * Discard anything that was parsed if we hit an error so we - * don't use partial information. - */ - if (!acpi_disabled && parse_acpi_topology()) - reset_cpu_topology(); - else if (of_have_populated_dt() && parse_dt_topology()) - reset_cpu_topology(); + + for_each_possible_cpu(cpu) { + ret = fetch_cache_info(cpu); + if (!ret) + continue; + else if (ret != -ENOENT) + pr_err("Early cacheinfo failed, ret = %d\n", ret); + return; + } +} + +void store_cpu_topology(unsigned int cpu) +{ + struct cpu_topology *cpu_topo = &cpu_topology[cpu]; + + if (cpu_topo->package_id != -1) + goto topology_populated; + + if (is_guest_or_emul()) { + cpu_topo->package_id = topo_packages[cpu]; + cpu_topo->core_id = topo_cores[cpu]; + cpu_topo->thread_id = topo_threads[cpu]; + goto topology_populated; + } + + cpu_topo->package_id = rcid_to_domain_id(cpu_to_rcid(cpu)); + cpu_topo->core_id = rcid_to_core_id(cpu_to_rcid(cpu)); + cpu_topo->thread_id = rcid_to_thread_id(cpu_to_rcid(cpu)); + + pr_debug("CPU%u: socket %d core %d thread %d\n", + cpu, cpu_topo->package_id, cpu_topo->core_id, + cpu_topo->thread_id); + +topology_populated: + update_siblings_masks(cpu); } -- Gitee From 9f34ab5738585bef6cd8c964bcd200e19f53ea7b Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 22 Jul 2024 19:20:23 +0800 Subject: [PATCH 316/524] sw64: cpu: move cpu related code to the newly created cpu.c To improve the code structure, create the source file cpu.c to store cpu related code. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cache.h | 28 ++++- arch/sw_64/include/asm/cpu.h | 99 +++++++++++++++++ arch/sw_64/include/asm/hw_init.h | 111 ------------------- arch/sw_64/include/asm/kvm_host.h | 1 + arch/sw_64/include/asm/mmu_context.h | 2 +- arch/sw_64/include/asm/sw64_init.h | 2 +- arch/sw_64/kernel/Makefile | 2 +- arch/sw_64/kernel/acpi.c | 1 + arch/sw_64/kernel/cacheinfo.c | 1 + arch/sw_64/kernel/chip_setup.c | 4 +- arch/sw_64/kernel/cpu.c | 155 +++++++++++++++++++++++++++ arch/sw_64/kernel/irq.c | 1 + arch/sw_64/kernel/setup.c | 152 -------------------------- arch/sw_64/kernel/smp.c | 2 +- arch/sw_64/kernel/time.c | 1 + arch/sw_64/kvm/sw64.c | 1 + arch/sw_64/lib/udelay.c | 2 + arch/sw_64/platform/cpufreq.c | 2 +- drivers/clocksource/timer-sw64.c | 2 +- 19 files changed, 294 insertions(+), 275 deletions(-) create mode 100644 arch/sw_64/kernel/cpu.c diff --git a/arch/sw_64/include/asm/cache.h b/arch/sw_64/include/asm/cache.h index 6a6ce4e99265..5848ebc23ffc 100644 --- a/arch/sw_64/include/asm/cache.h +++ b/arch/sw_64/include/asm/cache.h @@ -1,13 +1,33 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * include/asm/cache.h - */ + #ifndef _ASM_SW64_CACHE_H #define _ASM_SW64_CACHE_H #define L1_CACHE_SHIFT 7 #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -#define SMP_CACHE_BYTES L1_CACHE_BYTES +#define CACHE_SIZE_SHIFT 0 +#define CACHE_SIZE_MASK (0xffffffffUL << CACHE_SIZE_SHIFT) +#define CACHE_SIZE(val) \ + (((val) & CACHE_SIZE_MASK) >> CACHE_SIZE_SHIFT) +#define CACHE_LINE_BITS_SHIFT 32 +#define CACHE_LINE_BITS_MASK (0xfUL << CACHE_LINE_BITS_SHIFT) +#define CACHE_LINE_BITS(val) \ + (((val) & CACHE_LINE_BITS_MASK) >> CACHE_LINE_BITS_SHIFT) +#define CACHE_INDEX_BITS_SHIFT 36 +#define CACHE_INDEX_BITS_MASK (0x3fUL << CACHE_INDEX_BITS_SHIFT) +#define CACHE_INDEX_BITS(val) \ + (((val) & CACHE_INDEX_BITS_MASK) >> CACHE_INDEX_BITS_SHIFT) + +#ifndef __ASSEMBLY__ + +enum sunway_cache_type { + L1_ICACHE = 0, + L1_DCACHE = 1, + L2_CACHE = 2, + L3_CACHE = 3 +}; + +#endif #endif /* _ASM_SW64_CACHE_H */ diff --git a/arch/sw_64/include/asm/cpu.h b/arch/sw_64/include/asm/cpu.h index 4da30bb91d89..962b2957bc03 100644 --- a/arch/sw_64/include/asm/cpu.h +++ b/arch/sw_64/include/asm/cpu.h @@ -1,5 +1,104 @@ /* SPDX-License-Identifier: GPL-2.0 */ + #ifndef _ASM_SW64_CPU_H #define _ASM_SW64_CPU_H +#include +#include + +#define TABLE_ENTRY_MAX 32 +#define VENDOR_ID_MAX 2 +#define MODEL_MAX 8 + +#define CPUID_ARCH_REV_MASK 0xf +#define CPUID_ARCH_REV(val) ((val) & CPUID_ARCH_REV_MASK) +#define CPUID_ARCH_VAR_SHIFT 4 +#define CPUID_ARCH_VAR_MASK (0xf << CPUID_ARCH_VAR_SHIFT) +#define CPUID_ARCH_VAR(val) \ + (((val) & CPUID_ARCH_VAR_MASK) >> CPUID_ARCH_VAR_SHIFT) +#define CPUID_CHIP_VAR_SHIFT 8 +#define CPUID_CHIP_VAR_MASK (0xf << CPUID_CHIP_VAR_SHIFT) +#define CPUID_CHIP_VAR(val) \ + (((val) & CPUID_CHIP_VAR_MASK) >> CPUID_CHIP_VAR_SHIFT) +#define CPUID_FAMILY_SHIFT 12 +#define CPUID_FAMILY_MASK (0xf << CPUID_FAMILY_SHIFT) +#define CPUID_FAMILY(val) \ + (((val) & CPUID_FAMILY_MASK) >> CPUID_FAMILY_SHIFT) +#define CPUID_MODEL_SHIFT 24 +#define CPUID_MODEL_MASK (0xff << CPUID_MODEL_SHIFT) +#define CPUID_MODEL(val) \ + (((val) & CPUID_MODEL_MASK) >> CPUID_MODEL_SHIFT) +#define CPUID_PA_BITS_SHIFT 32 +#define CPUID_PA_BITS_MASK (0x7fUL << CPUID_PA_BITS_SHIFT) +#define CPUID_PA_BITS(val) \ + (((val) & CPUID_PA_BITS_MASK) >> CPUID_PA_BITS_SHIFT) +#define CPUID_VA_BITS_SHIFT 39 +#define CPUID_VA_BITS_MASK (0x7fUL << CPUID_VA_BITS_SHIFT) +#define CPUID_VA_BITS(val) \ + (((val) & CPUID_VA_BITS_MASK) >> CPUID_VA_BITS_SHIFT) + +#define current_cpu_data cpu_data[smp_processor_id()] + +enum hmcall_cpuid_cmd { + GET_TABLE_ENTRY = 1, + GET_VENDOR_ID = 2, + GET_MODEL = 3, + GET_CPU_FREQ = 4, + GET_CACHE_INFO = 5 +}; + +enum sunway_cpu_model { + CPU_SW3231 = 0x31, + CPU_SW831 = 0x32, + CPU_SW8A = 0x41 +}; + +struct cache_desc { + unsigned int size; /* Bytes per way */ + unsigned int sets; /* Number of lines per set */ + unsigned char ways; /* Number of ways */ + unsigned char linesz; /* Size of line in bytes */ + unsigned char flags; /* Flags describing cache properties */ +}; + +struct cpuinfo_sw64 { + unsigned long last_asid; + unsigned long last_vpn; + unsigned long ipi_count; + struct cache_desc icache; /* Primary I-cache */ + struct cache_desc dcache; /* Primary D or combined I/D cache */ + struct cache_desc scache; /* Secondary cache */ + struct cache_desc tcache; /* Tertiary/split secondary cache */ +} __aligned(SMP_CACHE_BYTES); + +struct cpu_desc_t { + __u8 model; + __u8 family; + __u8 chip_var; + __u8 arch_var; + __u8 arch_rev; + __u8 pa_bits; + __u8 va_bits; + char vendor_id[16]; + char model_id[64]; + unsigned long frequency; +} __randomize_layout; + +extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; +extern struct cpu_desc_t cpu_desc; +extern cpumask_t cpu_offline; + +extern void store_cpu_data(int cpu); +extern void __init setup_cpu_info(void); + +static inline unsigned long get_cpu_freq(void) +{ + return cpu_desc.frequency; +} + +static inline void update_cpu_freq(unsigned long khz) +{ + cpu_desc.frequency = khz * 1000; +} + #endif /* _ASM_SW64_CPU_H */ diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h index 5af3336d5fd6..b2d36df86bf2 100644 --- a/arch/sw_64/include/asm/hw_init.h +++ b/arch/sw_64/include/asm/hw_init.h @@ -10,40 +10,6 @@ #define MM_SIZE __va(0x2040) #define VPCR_SHIFT 44 -/* - * Descriptor for a cache - */ -struct cache_desc { - unsigned int size; /* Bytes per way */ - unsigned int sets; /* Number of lines per set */ - unsigned char ways; /* Number of ways */ - unsigned char linesz; /* Size of line in bytes */ - unsigned char flags; /* Flags describing cache properties */ -}; - -struct cpuinfo_sw64 { - unsigned long last_asid; - unsigned long last_vpn; - unsigned long ipi_count; - struct cache_desc icache; /* Primary I-cache */ - struct cache_desc dcache; /* Primary D or combined I/D cache */ - struct cache_desc scache; /* Secondary cache */ - struct cache_desc tcache; /* Tertiary/split secondary cache */ -} __aligned(SMP_CACHE_BYTES); - -struct cpu_desc_t { - __u8 model; - __u8 family; - __u8 chip_var; - __u8 arch_var; - __u8 arch_rev; - __u8 pa_bits; - __u8 va_bits; - char vendor_id[16]; - char model_id[64]; - unsigned long frequency; -} __randomize_layout; - #define MAX_NUMSOCKETS 8 struct socket_desc_t { bool is_online; /* 1 for online, 0 for offline */ @@ -69,29 +35,14 @@ struct memmap_entry { enum memmap_types type; }; -extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; -extern void store_cpu_data(int cpu); - -extern struct cpu_desc_t cpu_desc; extern struct socket_desc_t socket_desc[MAX_NUMSOCKETS]; extern int memmap_nr; extern struct memmap_entry memmap_map[MAX_NUMMEMMAPS]; -extern cpumask_t cpu_offline; extern bool memblock_initialized; int __init add_memmap_region(u64 addr, u64 size, enum memmap_types type); void __init process_memmap(void); -static inline unsigned long get_cpu_freq(void) -{ - return cpu_desc.frequency; -} - -static inline void update_cpu_freq(unsigned long khz) -{ - cpu_desc.frequency = khz * 1000; -} - #define EMUL_FLAG (0x1UL << 63) #define MM_SIZE_MASK (EMUL_FLAG - 1) @@ -106,66 +57,4 @@ DECLARE_STATIC_KEY_FALSE(hw_una_enabled); #define is_in_emul() static_branch_unlikely(&run_mode_emul_key) #define is_guest_or_emul() !static_branch_likely(&run_mode_host_key) -#define CPU_SW3231 0x31 -#define CPU_SW831 0x32 -#define CPU_SW8A 0x41 - -#define GET_TABLE_ENTRY 1 -#define GET_VENDOR_ID 2 -#define GET_MODEL 3 -#define GET_CPU_FREQ 4 -#define GET_CACHE_INFO 5 - -#define TABLE_ENTRY_MAX 32 -#define VENDOR_ID_MAX 2 -#define MODEL_MAX 8 -#define CACHE_INFO_MAX 4 - -#define L1_ICACHE 0 -#define L1_DCACHE 1 -#define L2_CACHE 2 -#define L3_CACHE 3 - -#define CPUID_ARCH_REV_MASK 0xf -#define CPUID_ARCH_REV(val) ((val) & CPUID_ARCH_REV_MASK) -#define CPUID_ARCH_VAR_SHIFT 4 -#define CPUID_ARCH_VAR_MASK (0xf << CPUID_ARCH_VAR_SHIFT) -#define CPUID_ARCH_VAR(val) \ - (((val) & CPUID_ARCH_VAR_MASK) >> CPUID_ARCH_VAR_SHIFT) -#define CPUID_CHIP_VAR_SHIFT 8 -#define CPUID_CHIP_VAR_MASK (0xf << CPUID_CHIP_VAR_SHIFT) -#define CPUID_CHIP_VAR(val) \ - (((val) & CPUID_CHIP_VAR_MASK) >> CPUID_CHIP_VAR_SHIFT) -#define CPUID_FAMILY_SHIFT 12 -#define CPUID_FAMILY_MASK (0xf << CPUID_FAMILY_SHIFT) -#define CPUID_FAMILY(val) \ - (((val) & CPUID_FAMILY_MASK) >> CPUID_FAMILY_SHIFT) -#define CPUID_MODEL_SHIFT 24 -#define CPUID_MODEL_MASK (0xff << CPUID_MODEL_SHIFT) -#define CPUID_MODEL(val) \ - (((val) & CPUID_MODEL_MASK) >> CPUID_MODEL_SHIFT) -#define CPUID_PA_BITS_SHIFT 32 -#define CPUID_PA_BITS_MASK (0x7fUL << CPUID_PA_BITS_SHIFT) -#define CPUID_PA_BITS(val) \ - (((val) & CPUID_PA_BITS_MASK) >> CPUID_PA_BITS_SHIFT) -#define CPUID_VA_BITS_SHIFT 39 -#define CPUID_VA_BITS_MASK (0x7fUL << CPUID_VA_BITS_SHIFT) -#define CPUID_VA_BITS(val) \ - (((val) & CPUID_VA_BITS_MASK) >> CPUID_VA_BITS_SHIFT) - - -#define CACHE_SIZE_SHIFT 0 -#define CACHE_SIZE_MASK (0xffffffffUL << CACHE_SIZE_SHIFT) -#define CACHE_SIZE(val) \ - (((val) & CACHE_SIZE_MASK) >> CACHE_SIZE_SHIFT) -#define CACHE_LINE_BITS_SHIFT 32 -#define CACHE_LINE_BITS_MASK (0xfUL << CACHE_LINE_BITS_SHIFT) -#define CACHE_LINE_BITS(val) \ - (((val) & CACHE_LINE_BITS_MASK) >> CACHE_LINE_BITS_SHIFT) -#define CACHE_INDEX_BITS_SHIFT 36 -#define CACHE_INDEX_BITS_MASK (0x3fUL << CACHE_INDEX_BITS_SHIFT) -#define CACHE_INDEX_BITS(val) \ - (((val) & CACHE_INDEX_BITS_MASK) >> CACHE_INDEX_BITS_SHIFT) -#define current_cpu_data cpu_data[smp_processor_id()] - #endif /* _ASM_SW64_HW_INIT_H */ diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index e2f367003509..221a7db26f0f 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -27,6 +27,7 @@ #include #include +#include #define last_vpn(cpu) (cpu_data[cpu].last_vpn) diff --git a/arch/sw_64/include/asm/mmu_context.h b/arch/sw_64/include/asm/mmu_context.h index 1ff7df9539c5..84dfb15b43bf 100644 --- a/arch/sw_64/include/asm/mmu_context.h +++ b/arch/sw_64/include/asm/mmu_context.h @@ -15,7 +15,7 @@ #define ASID_BITS 10 #endif -#include +#include #define last_asid(cpu) (cpu_data[cpu].last_asid) #define ASID_FIRST_VERSION (1UL << ASID_BITS) diff --git a/arch/sw_64/include/asm/sw64_init.h b/arch/sw_64/include/asm/sw64_init.h index 34a28e365ec6..20901ede574c 100644 --- a/arch/sw_64/include/asm/sw64_init.h +++ b/arch/sw_64/include/asm/sw64_init.h @@ -8,7 +8,7 @@ #include struct sw64_early_init_ops { - void (*setup_core_map)(struct cpumask *cpumask); + void (*setup_core_map)(void); unsigned long (*get_node_mem)(int nodeid); void (*get_smp_info)(void); }; diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index b5756003776f..18a3cab6a9b6 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -13,7 +13,7 @@ CFLAGS_REMOVE_insn.o = -pg CFLAGS_REMOVE_printk.o = -pg endif -obj-y := fpu.o traps.o process.o sys_sw64.o irq.o \ +obj-y := fpu.o traps.o process.o sys_sw64.o irq.o cpu.o \ signal.o setup.o ptrace.o time.o \ systbls.o dup_print.o chip_setup.o \ insn.o early_init.o topology.o cacheinfo.o \ diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 515728920cfd..ebe344752035 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -6,6 +6,7 @@ #include #include +#include #include #ifdef CONFIG_ACPI_HOTPLUG_CPU diff --git a/arch/sw_64/kernel/cacheinfo.c b/arch/sw_64/kernel/cacheinfo.c index e340c53690a9..d56ba4d6b909 100644 --- a/arch/sw_64/kernel/cacheinfo.c +++ b/arch/sw_64/kernel/cacheinfo.c @@ -16,6 +16,7 @@ */ #include +#include #include /* Populates leaf and increments to next leaf */ diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index 211d673250c7..efc07f0d0cab 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -3,7 +3,7 @@ #include #include -#include +#include #include #define OFFSET_CORE_ONLINE 0x780UL @@ -38,7 +38,7 @@ static unsigned long __init get_node_mem(int nodeid) return __get_node_mem(nodeid); } -static void __init setup_core_map(struct cpumask *cpumask) +static void __init setup_core_map(void) { int i, j, cpu_num, cpuid, max_cores_per_cpu; unsigned long coreonline; diff --git a/arch/sw_64/kernel/cpu.c b/arch/sw_64/kernel/cpu.c new file mode 100644 index 000000000000..073f359380ca --- /dev/null +++ b/arch/sw_64/kernel/cpu.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#include +#include +#include + +/* Map logical to physical */ +int __cpu_to_rcid[NR_CPUS]; +EXPORT_SYMBOL(__cpu_to_rcid); + +/* A collection of per-processor data. */ +struct cpuinfo_sw64 cpu_data[NR_CPUS]; +EXPORT_SYMBOL(cpu_data); + +struct cpu_desc_t cpu_desc; + +cpumask_t cpu_offline = CPU_MASK_NONE; + +/* Move global data into per-processor storage */ +void store_cpu_data(int cpu) +{ + cpu_data[cpu].last_asid = ASID_FIRST_VERSION; +} + +void __init setup_cpu_info(void) +{ + int i; + struct cache_desc *c; + unsigned long val; + + val = cpuid(GET_TABLE_ENTRY, 0); + cpu_desc.model = CPUID_MODEL(val); + cpu_desc.family = CPUID_FAMILY(val); + cpu_desc.chip_var = CPUID_CHIP_VAR(val); + cpu_desc.arch_var = CPUID_ARCH_VAR(val); + cpu_desc.arch_rev = CPUID_ARCH_REV(val); + cpu_desc.pa_bits = CPUID_PA_BITS(val); + cpu_desc.va_bits = CPUID_VA_BITS(val); + + for (i = 0; i < VENDOR_ID_MAX; i++) { + val = cpuid(GET_VENDOR_ID, i); + memcpy(cpu_desc.vendor_id + (i * 8), &val, 8); + } + + for (i = 0; i < MODEL_MAX; i++) { + val = cpuid(GET_MODEL, i); + memcpy(cpu_desc.model_id + (i * 8), &val, 8); + } + + cpu_desc.frequency = cpuid(GET_CPU_FREQ, 0) * 1000UL * 1000UL; + + for (i = 0; i < NR_CPUS; i++) { + c = &(cpu_data[i].icache); + val = cpuid(GET_CACHE_INFO, L1_ICACHE); + c->size = CACHE_SIZE(val); + c->linesz = 1 << (CACHE_LINE_BITS(val)); + c->sets = 1 << (CACHE_INDEX_BITS(val)); + c->ways = c->size / c->sets / c->linesz; + + c = &(cpu_data[i].dcache); + val = cpuid(GET_CACHE_INFO, L1_DCACHE); + c->size = CACHE_SIZE(val); + c->linesz = 1 << (CACHE_LINE_BITS(val)); + c->sets = 1 << (CACHE_INDEX_BITS(val)); + c->ways = c->size / c->sets / c->linesz; + + c = &(cpu_data[i].scache); + val = cpuid(GET_CACHE_INFO, L2_CACHE); + c->size = CACHE_SIZE(val); + c->linesz = 1 << (CACHE_LINE_BITS(val)); + c->sets = 1 << (CACHE_INDEX_BITS(val)); + c->ways = c->size / c->sets / c->linesz; + + c = &(cpu_data[i].tcache); + val = cpuid(GET_CACHE_INFO, L3_CACHE); + c->size = CACHE_SIZE(val); + c->linesz = 1 << (CACHE_LINE_BITS(val)); + c->sets = 1 << (CACHE_INDEX_BITS(val)); + c->ways = c->size / c->sets / c->linesz; + } +} + +static int show_cpuinfo(struct seq_file *f, void *slot) +{ + int i; + unsigned long freq; + + freq = cpuid(GET_CPU_FREQ, 0); + + for_each_online_cpu(i) { + /* + * glibc reads /proc/cpuinfo to determine the number of + * online processors, looking for lines beginning with + * "processor". Give glibc what it expects. + */ + seq_printf(f, "processor\t: %u\n" + "vendor_id\t: %s\n" + "cpu family\t: %d\n" + "model\t\t: %u\n" + "model name\t: %s CPU @ %lu.%lu%luGHz\n" + "cpu variation\t: %u\n" + "cpu revision\t: %u\n", + i, cpu_desc.vendor_id, cpu_desc.family, + cpu_desc.model, cpu_desc.model_id, + freq / 1000, (freq % 1000) / 100, + (freq % 100) / 10, + cpu_desc.arch_var, cpu_desc.arch_rev); + seq_printf(f, "cpu MHz\t\t: %lu.00\n" + "cache size\t: %u KB\n" + "physical id\t: %d\n" + "bogomips\t: %lu.%02lu\n", + get_cpu_freq() / 1000 / 1000, cpu_data[i].tcache.size >> 10, + cpu_topology[i].package_id, + loops_per_jiffy / (500000/HZ), + (loops_per_jiffy / (5000/HZ)) % 100); + + seq_printf(f, "flags\t\t: fpu simd vpn upn cpuid\n"); + seq_printf(f, "page size\t: %d\n", 8192); + seq_printf(f, "cache_alignment\t: %d\n", cpu_data[i].tcache.linesz); + seq_printf(f, "address sizes\t: %u bits physical, %u bits virtual\n\n", + cpu_desc.pa_bits, cpu_desc.va_bits); + } + + return 0; +} + +/* + * We show only CPU #0 info. + */ +static void *c_start(struct seq_file *f, loff_t *pos) +{ + return *pos < 1 ? (void *)1 : NULL; +} + +static void *c_next(struct seq_file *f, void *v, loff_t *pos) +{ + (*pos)++; + return NULL; +} + +static void c_stop(struct seq_file *f, void *v) +{ +} + +const struct seq_operations cpuinfo_op = { + .start = c_start, + .next = c_next, + .stop = c_stop, + .show = show_cpuinfo, +}; + diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c index 5706df471a8d..d916a484b2fb 100644 --- a/arch/sw_64/kernel/irq.c +++ b/arch/sw_64/kernel/irq.c @@ -17,6 +17,7 @@ #include #include +#include #include volatile unsigned long irq_err_count; diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index b735fe0e3e59..c7712c9eca06 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -39,9 +39,6 @@ #define DBGDCONT(args...) #endif -int __cpu_to_rcid[NR_CPUS]; /* Map logical to physical */ -EXPORT_SYMBOL(__cpu_to_rcid); - DEFINE_PER_CPU(unsigned long, hard_node_id) = { 0 }; static DEFINE_PER_CPU(struct cpu, cpu_devices); @@ -63,9 +60,6 @@ static struct notifier_block sw64_panic_block = { INT_MAX /* try to do it first */ }; -/* the value is IOR: CORE_ONLIE*/ -cpumask_t core_start = CPU_MASK_NONE; - static struct resource data_resource = { .name = "Kernel data", .start = 0, @@ -87,24 +81,17 @@ static struct resource bss_resource = { .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM }; -/* A collection of per-processor data. */ -struct cpuinfo_sw64 cpu_data[NR_CPUS]; -EXPORT_SYMBOL(cpu_data); - DEFINE_STATIC_KEY_TRUE(run_mode_host_key); DEFINE_STATIC_KEY_FALSE(run_mode_guest_key); DEFINE_STATIC_KEY_FALSE(run_mode_emul_key); DEFINE_STATIC_KEY_FALSE(hw_una_enabled); -struct cpu_desc_t cpu_desc; struct socket_desc_t socket_desc[MAX_NUMSOCKETS]; int memmap_nr; struct memmap_entry memmap_map[MAX_NUMMEMMAPS]; bool memblock_initialized; -cpumask_t cpu_offline = CPU_MASK_NONE; - /* boot_params */ /** * Keep sunway_boot_params for backward compatibility. All related code @@ -140,14 +127,6 @@ struct screen_info screen_info = { }; EXPORT_SYMBOL(screen_info); -/* - * Move global data into per-processor storage. - */ -void store_cpu_data(int cpu) -{ - cpu_data[cpu].last_asid = ASID_FIRST_VERSION; -} - #ifdef CONFIG_HARDLOCKUP_DETECTOR_PERF u64 hw_nmi_get_sample_period(int watchdog_thresh) { @@ -613,64 +592,6 @@ static void __init device_tree_init(void) unflatten_device_tree(); } -static void __init setup_cpu_info(void) -{ - int i; - struct cache_desc *c; - unsigned long val; - - val = cpuid(GET_TABLE_ENTRY, 0); - cpu_desc.model = CPUID_MODEL(val); - cpu_desc.family = CPUID_FAMILY(val); - cpu_desc.chip_var = CPUID_CHIP_VAR(val); - cpu_desc.arch_var = CPUID_ARCH_VAR(val); - cpu_desc.arch_rev = CPUID_ARCH_REV(val); - cpu_desc.pa_bits = CPUID_PA_BITS(val); - cpu_desc.va_bits = CPUID_VA_BITS(val); - - for (i = 0; i < VENDOR_ID_MAX; i++) { - val = cpuid(GET_VENDOR_ID, i); - memcpy(cpu_desc.vendor_id + (i * 8), &val, 8); - } - - for (i = 0; i < MODEL_MAX; i++) { - val = cpuid(GET_MODEL, i); - memcpy(cpu_desc.model_id + (i * 8), &val, 8); - } - - cpu_desc.frequency = cpuid(GET_CPU_FREQ, 0) * 1000UL * 1000UL; - - for (i = 0; i < NR_CPUS; i++) { - c = &(cpu_data[i].icache); - val = cpuid(GET_CACHE_INFO, L1_ICACHE); - c->size = CACHE_SIZE(val); - c->linesz = 1 << (CACHE_LINE_BITS(val)); - c->sets = 1 << (CACHE_INDEX_BITS(val)); - c->ways = c->size / c->sets / c->linesz; - - c = &(cpu_data[i].dcache); - val = cpuid(GET_CACHE_INFO, L1_DCACHE); - c->size = CACHE_SIZE(val); - c->linesz = 1 << (CACHE_LINE_BITS(val)); - c->sets = 1 << (CACHE_INDEX_BITS(val)); - c->ways = c->size / c->sets / c->linesz; - - c = &(cpu_data[i].scache); - val = cpuid(GET_CACHE_INFO, L2_CACHE); - c->size = CACHE_SIZE(val); - c->linesz = 1 << (CACHE_LINE_BITS(val)); - c->sets = 1 << (CACHE_INDEX_BITS(val)); - c->ways = c->size / c->sets / c->linesz; - - c = &(cpu_data[i].tcache); - val = cpuid(GET_CACHE_INFO, L3_CACHE); - c->size = CACHE_SIZE(val); - c->linesz = 1 << (CACHE_LINE_BITS(val)); - c->sets = 1 << (CACHE_INDEX_BITS(val)); - c->ways = c->size / c->sets / c->linesz; - } -} - #ifdef CONFIG_SUBARCH_C3B static void __init setup_run_mode(void) { @@ -821,79 +742,6 @@ setup_arch(char **cmdline_p) } } -static int -show_cpuinfo(struct seq_file *f, void *slot) -{ - int i; - unsigned long cpu_freq; - - cpu_freq = cpuid(GET_CPU_FREQ, 0); - - for_each_online_cpu(i) { - /* - * glibc reads /proc/cpuinfo to determine the number of - * online processors, looking for lines beginning with - * "processor". Give glibc what it expects. - */ - seq_printf(f, "processor\t: %u\n" - "vendor_id\t: %s\n" - "cpu family\t: %d\n" - "model\t\t: %u\n" - "model name\t: %s CPU @ %lu.%lu%luGHz\n" - "cpu variation\t: %u\n" - "cpu revision\t: %u\n", - i, cpu_desc.vendor_id, cpu_desc.family, - cpu_desc.model, cpu_desc.model_id, - cpu_freq / 1000, (cpu_freq % 1000) / 100, - (cpu_freq % 100) / 10, - cpu_desc.arch_var, cpu_desc.arch_rev); - seq_printf(f, "cpu MHz\t\t: %lu.00\n" - "cache size\t: %u KB\n" - "physical id\t: %d\n" - "bogomips\t: %lu.%02lu\n", - get_cpu_freq() / 1000 / 1000, cpu_data[i].tcache.size >> 10, - cpu_topology[i].package_id, - loops_per_jiffy / (500000/HZ), - (loops_per_jiffy / (5000/HZ)) % 100); - - seq_printf(f, "flags\t\t: fpu simd vpn upn cpuid\n"); - seq_printf(f, "page size\t: %d\n", 8192); - seq_printf(f, "cache_alignment\t: %d\n", cpu_data[i].tcache.linesz); - seq_printf(f, "address sizes\t: %u bits physical, %u bits virtual\n\n", - cpu_desc.pa_bits, cpu_desc.va_bits); - } - return 0; -} - -/* - * We show only CPU #0 info. - */ -static void * -c_start(struct seq_file *f, loff_t *pos) -{ - return *pos < 1 ? (void *)1 : NULL; -} - -static void * -c_next(struct seq_file *f, void *v, loff_t *pos) -{ - (*pos)++; - return NULL; -} - -static void -c_stop(struct seq_file *f, void *v) -{ -} - -const struct seq_operations cpuinfo_op = { - .start = c_start, - .next = c_next, - .stop = c_stop, - .show = show_cpuinfo, -}; - - static int sw64_panic_event(struct notifier_block *this, unsigned long event, void *ptr) { diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index dfeedbc3e349..a48b1cb999c6 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -399,7 +399,7 @@ void __init setup_smp(void) init_cpu_present(cpu_none_mask); /* Legacy core detect */ - sw64_chip_init->early_init.setup_core_map(&core_start); + sw64_chip_init->early_init.setup_core_map(); /* For unified kernel, NR_CPUS is the maximum possible value */ for (i = 0; i < NR_CPUS; i++) { diff --git a/arch/sw_64/kernel/time.c b/arch/sw_64/kernel/time.c index 533a6a14c200..6eaf8fab4f28 100644 --- a/arch/sw_64/kernel/time.c +++ b/arch/sw_64/kernel/time.c @@ -5,6 +5,7 @@ #include #include +#include #include #include diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index d0510e567c32..99f7df5468fe 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -15,6 +15,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include "trace.h" diff --git a/arch/sw_64/lib/udelay.c b/arch/sw_64/lib/udelay.c index 59ca8a97d748..c60601d56b0e 100644 --- a/arch/sw_64/lib/udelay.c +++ b/arch/sw_64/lib/udelay.c @@ -7,6 +7,8 @@ #include +#include + /* * Use only for very small delays (< 1 msec). * diff --git a/arch/sw_64/platform/cpufreq.c b/arch/sw_64/platform/cpufreq.c index 69d7611a2fb0..0d552d9a753d 100644 --- a/arch/sw_64/platform/cpufreq.c +++ b/arch/sw_64/platform/cpufreq.c @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index 5dad3496c0a0..e3720a3a1df4 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -8,9 +8,9 @@ #include #include +#include #include #include -#include #include #define DEFAULT_MCLK 25000 /* Khz */ -- Gitee From 267d4155bd2ae8cc525e739fb348afaa15d60f92 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 23 Jul 2024 16:50:04 +0800 Subject: [PATCH 317/524] sw64: cache: refactor cacheinfo related code Remove struct cache_desc and separate the initialization of cache information from the initialization of cpu information to improve code readability. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cache.h | 13 ---- arch/sw_64/include/asm/cpu.h | 16 ++--- arch/sw_64/kernel/cacheinfo.c | 117 ++++++++++++++++++++++----------- arch/sw_64/kernel/cpu.c | 40 ++--------- 4 files changed, 91 insertions(+), 95 deletions(-) diff --git a/arch/sw_64/include/asm/cache.h b/arch/sw_64/include/asm/cache.h index 5848ebc23ffc..411218f5eb0e 100644 --- a/arch/sw_64/include/asm/cache.h +++ b/arch/sw_64/include/asm/cache.h @@ -6,19 +6,6 @@ #define L1_CACHE_SHIFT 7 #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) -#define CACHE_SIZE_SHIFT 0 -#define CACHE_SIZE_MASK (0xffffffffUL << CACHE_SIZE_SHIFT) -#define CACHE_SIZE(val) \ - (((val) & CACHE_SIZE_MASK) >> CACHE_SIZE_SHIFT) -#define CACHE_LINE_BITS_SHIFT 32 -#define CACHE_LINE_BITS_MASK (0xfUL << CACHE_LINE_BITS_SHIFT) -#define CACHE_LINE_BITS(val) \ - (((val) & CACHE_LINE_BITS_MASK) >> CACHE_LINE_BITS_SHIFT) -#define CACHE_INDEX_BITS_SHIFT 36 -#define CACHE_INDEX_BITS_MASK (0x3fUL << CACHE_INDEX_BITS_SHIFT) -#define CACHE_INDEX_BITS(val) \ - (((val) & CACHE_INDEX_BITS_MASK) >> CACHE_INDEX_BITS_SHIFT) - #ifndef __ASSEMBLY__ enum sunway_cache_type { diff --git a/arch/sw_64/include/asm/cpu.h b/arch/sw_64/include/asm/cpu.h index 962b2957bc03..77341687f99e 100644 --- a/arch/sw_64/include/asm/cpu.h +++ b/arch/sw_64/include/asm/cpu.h @@ -5,6 +5,7 @@ #include #include +#include #define TABLE_ENTRY_MAX 32 #define VENDOR_ID_MAX 2 @@ -53,22 +54,10 @@ enum sunway_cpu_model { CPU_SW8A = 0x41 }; -struct cache_desc { - unsigned int size; /* Bytes per way */ - unsigned int sets; /* Number of lines per set */ - unsigned char ways; /* Number of ways */ - unsigned char linesz; /* Size of line in bytes */ - unsigned char flags; /* Flags describing cache properties */ -}; - struct cpuinfo_sw64 { unsigned long last_asid; unsigned long last_vpn; unsigned long ipi_count; - struct cache_desc icache; /* Primary I-cache */ - struct cache_desc dcache; /* Primary D or combined I/D cache */ - struct cache_desc scache; /* Secondary cache */ - struct cache_desc tcache; /* Tertiary/split secondary cache */ } __aligned(SMP_CACHE_BYTES); struct cpu_desc_t { @@ -101,4 +90,7 @@ static inline void update_cpu_freq(unsigned long khz) cpu_desc.frequency = khz * 1000; } +extern unsigned int get_cpu_cache_size(int cpu, int level, enum cache_type type); +extern unsigned int get_cpu_cacheline_size(int cpu, int level, enum cache_type type); + #endif /* _ASM_SW64_CPU_H */ diff --git a/arch/sw_64/kernel/cacheinfo.c b/arch/sw_64/kernel/cacheinfo.c index d56ba4d6b909..69a32554e255 100644 --- a/arch/sw_64/kernel/cacheinfo.c +++ b/arch/sw_64/kernel/cacheinfo.c @@ -14,28 +14,75 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ + #include #include #include +#include + +#define get_cache_info(type) cpuid(GET_CACHE_INFO, (type)) +#define get_cache_size(info) ((info) & 0xffffffffUL) +#define get_cacheline_size(info) (1 << (((info) >> 32) & 0xfUL)) +#define get_cache_sets(info) (1 << (((info) >> 36) & 0x3fUL)) +#define get_cache_ways(info) (get_cache_size(info) / get_cache_sets(info) / get_cacheline_size(info)) +#define cache_size(type) get_cache_size(get_cache_info((type))) +#define cache_level(type) ((type) < L2_CACHE ? 1 : (type)) /* Populates leaf and increments to next leaf */ -#define populate_cache(cache, leaf, c_level, c_type, c_id) \ -do { \ - leaf->id = c_id; \ - leaf->attributes = CACHE_ID; \ - leaf->type = c_type; \ - leaf->level = c_level; \ - leaf->coherency_line_size = c->cache.linesz; \ - leaf->number_of_sets = c->cache.sets; \ - leaf->ways_of_associativity = c->cache.ways; \ - leaf->size = c->cache.size; \ - leaf++; \ +#define populate_cache(cache_info, leaf, c_level, c_type, c_id) \ +do { \ + leaf->id = c_id; \ + leaf->attributes = CACHE_ID; \ + leaf->type = c_type; \ + leaf->level = c_level; \ + leaf->coherency_line_size = get_cacheline_size(cache_info); \ + leaf->number_of_sets = get_cache_sets(cache_info); \ + leaf->ways_of_associativity = get_cache_ways(cache_info); \ + leaf->size = get_cache_size(cache_info); \ + leaf++; \ } while (0) +static struct cacheinfo *get_cacheinfo(int cpu, int level, enum cache_type type) +{ + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *leaf; + int index; + + for (index = 0; index < this_cpu_ci->num_leaves; index++) { + leaf = this_cpu_ci->info_list + index; + if ((leaf->level == level) && (leaf->type == type)) + return leaf; + } + + return NULL; +} + +unsigned int get_cpu_cache_size(int cpu, int level, enum cache_type type) +{ + struct cacheinfo *leaf = get_cacheinfo(cpu, level, type); + + return leaf ? leaf->size : 0; +} + +unsigned int get_cpu_cacheline_size(int cpu, int level, enum cache_type type) +{ + struct cacheinfo *leaf = get_cacheinfo(cpu, level, type); + + return leaf ? leaf->coherency_line_size : 0; +} + + +static inline enum cache_type kernel_cache_type(enum sunway_cache_type type) +{ + if ((type > L1_DCACHE) || !cache_size(L1_ICACHE)) + return CACHE_TYPE_UNIFIED; + + return (type == L1_DCACHE) ? CACHE_TYPE_DATA : CACHE_TYPE_INST; +} + int init_cache_level(unsigned int cpu) { - struct cpuinfo_sw64 *c = &cpu_data[cpu]; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); int levels = 0, leaves = 0; @@ -43,55 +90,51 @@ int init_cache_level(unsigned int cpu) * If Dcache is not set, we assume the cache structures * are not properly initialized. */ - if (c->dcache.size) + if (cache_size(L1_DCACHE)) levels += 1; else return -ENOENT; + leaves += cache_size(L1_ICACHE) ? 2 : 1; - leaves += (c->icache.size) ? 2 : 1; - - if (c->scache.size) { + if (cache_size(L2_CACHE)) { levels++; leaves++; } - if (c->tcache.size) { + if (cache_size(L3_CACHE)) { levels++; leaves++; } this_cpu_ci->num_levels = levels; this_cpu_ci->num_leaves = leaves; + return 0; } int populate_cache_leaves(unsigned int cpu) { - struct cpuinfo_sw64 *c = &cpu_data[cpu]; + enum sunway_cache_type type; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *this_leaf = this_cpu_ci->info_list; struct cpu_topology *topo = &cpu_topology[cpu]; - if (c->icache.size) { - cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); - populate_cache(dcache, this_leaf, 1, CACHE_TYPE_DATA, cpu); - cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); - populate_cache(icache, this_leaf, 1, CACHE_TYPE_INST, cpu); - - } else { - cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); - populate_cache(dcache, this_leaf, 1, CACHE_TYPE_UNIFIED, cpu); - } - - if (c->scache.size) { - cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); - populate_cache(scache, this_leaf, 2, CACHE_TYPE_UNIFIED, cpu); - } - - if (c->tcache.size) { - cpumask_copy(&this_leaf->shared_cpu_map, topology_llc_cpumask(cpu)); - populate_cache(tcache, this_leaf, 3, CACHE_TYPE_UNIFIED, topo->package_id); + for (type = L1_ICACHE; type <= L3_CACHE; type++) { + if (!cache_size(type)) + continue; + + /* L3 Cache is shared */ + if (type == L3_CACHE) { + cpumask_copy(&this_leaf->shared_cpu_map, + topology_llc_cpumask(cpu)); + populate_cache(get_cache_info(type), this_leaf, cache_level(type), + kernel_cache_type(type), topo->package_id); + } else { + cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); + populate_cache(get_cache_info(type), this_leaf, cache_level(type), + kernel_cache_type(type), cpu); + } } this_cpu_ci->cpu_map_populated = true; diff --git a/arch/sw_64/kernel/cpu.c b/arch/sw_64/kernel/cpu.c index 073f359380ca..fafbb23433b8 100644 --- a/arch/sw_64/kernel/cpu.c +++ b/arch/sw_64/kernel/cpu.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -29,7 +30,6 @@ void store_cpu_data(int cpu) void __init setup_cpu_info(void) { int i; - struct cache_desc *c; unsigned long val; val = cpuid(GET_TABLE_ENTRY, 0); @@ -52,46 +52,20 @@ void __init setup_cpu_info(void) } cpu_desc.frequency = cpuid(GET_CPU_FREQ, 0) * 1000UL * 1000UL; - - for (i = 0; i < NR_CPUS; i++) { - c = &(cpu_data[i].icache); - val = cpuid(GET_CACHE_INFO, L1_ICACHE); - c->size = CACHE_SIZE(val); - c->linesz = 1 << (CACHE_LINE_BITS(val)); - c->sets = 1 << (CACHE_INDEX_BITS(val)); - c->ways = c->size / c->sets / c->linesz; - - c = &(cpu_data[i].dcache); - val = cpuid(GET_CACHE_INFO, L1_DCACHE); - c->size = CACHE_SIZE(val); - c->linesz = 1 << (CACHE_LINE_BITS(val)); - c->sets = 1 << (CACHE_INDEX_BITS(val)); - c->ways = c->size / c->sets / c->linesz; - - c = &(cpu_data[i].scache); - val = cpuid(GET_CACHE_INFO, L2_CACHE); - c->size = CACHE_SIZE(val); - c->linesz = 1 << (CACHE_LINE_BITS(val)); - c->sets = 1 << (CACHE_INDEX_BITS(val)); - c->ways = c->size / c->sets / c->linesz; - - c = &(cpu_data[i].tcache); - val = cpuid(GET_CACHE_INFO, L3_CACHE); - c->size = CACHE_SIZE(val); - c->linesz = 1 << (CACHE_LINE_BITS(val)); - c->sets = 1 << (CACHE_INDEX_BITS(val)); - c->ways = c->size / c->sets / c->linesz; - } } static int show_cpuinfo(struct seq_file *f, void *slot) { int i; + unsigned int l3_cache_size, l3_cachline_size; unsigned long freq; freq = cpuid(GET_CPU_FREQ, 0); for_each_online_cpu(i) { + l3_cache_size = get_cpu_cache_size(i, 3, CACHE_TYPE_UNIFIED); + l3_cachline_size = get_cpu_cacheline_size(i, 3, CACHE_TYPE_UNIFIED); + /* * glibc reads /proc/cpuinfo to determine the number of * online processors, looking for lines beginning with @@ -113,14 +87,14 @@ static int show_cpuinfo(struct seq_file *f, void *slot) "cache size\t: %u KB\n" "physical id\t: %d\n" "bogomips\t: %lu.%02lu\n", - get_cpu_freq() / 1000 / 1000, cpu_data[i].tcache.size >> 10, + get_cpu_freq() / 1000 / 1000, l3_cache_size >> 10, cpu_topology[i].package_id, loops_per_jiffy / (500000/HZ), (loops_per_jiffy / (5000/HZ)) % 100); seq_printf(f, "flags\t\t: fpu simd vpn upn cpuid\n"); seq_printf(f, "page size\t: %d\n", 8192); - seq_printf(f, "cache_alignment\t: %d\n", cpu_data[i].tcache.linesz); + seq_printf(f, "cache_alignment\t: %d\n", l3_cachline_size); seq_printf(f, "address sizes\t: %u bits physical, %u bits virtual\n\n", cpu_desc.pa_bits, cpu_desc.va_bits); } -- Gitee From 15e190df6364735217a20eed50d5c798eea0f388 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 24 Jul 2024 09:07:35 +0800 Subject: [PATCH 318/524] sw64: cpu: refactor cpuinfo related code Remove struct cpu_desc_t to improve code readability. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpu.h | 61 +++------------------ arch/sw_64/kernel/chip_setup.c | 4 +- arch/sw_64/kernel/cpu.c | 93 +++++++++++++++++++++++++------- arch/sw_64/kernel/setup.c | 3 +- drivers/clocksource/timer-sw64.c | 2 +- 5 files changed, 86 insertions(+), 77 deletions(-) diff --git a/arch/sw_64/include/asm/cpu.h b/arch/sw_64/include/asm/cpu.h index 77341687f99e..e6f5cfdbaa3a 100644 --- a/arch/sw_64/include/asm/cpu.h +++ b/arch/sw_64/include/asm/cpu.h @@ -7,37 +7,6 @@ #include #include -#define TABLE_ENTRY_MAX 32 -#define VENDOR_ID_MAX 2 -#define MODEL_MAX 8 - -#define CPUID_ARCH_REV_MASK 0xf -#define CPUID_ARCH_REV(val) ((val) & CPUID_ARCH_REV_MASK) -#define CPUID_ARCH_VAR_SHIFT 4 -#define CPUID_ARCH_VAR_MASK (0xf << CPUID_ARCH_VAR_SHIFT) -#define CPUID_ARCH_VAR(val) \ - (((val) & CPUID_ARCH_VAR_MASK) >> CPUID_ARCH_VAR_SHIFT) -#define CPUID_CHIP_VAR_SHIFT 8 -#define CPUID_CHIP_VAR_MASK (0xf << CPUID_CHIP_VAR_SHIFT) -#define CPUID_CHIP_VAR(val) \ - (((val) & CPUID_CHIP_VAR_MASK) >> CPUID_CHIP_VAR_SHIFT) -#define CPUID_FAMILY_SHIFT 12 -#define CPUID_FAMILY_MASK (0xf << CPUID_FAMILY_SHIFT) -#define CPUID_FAMILY(val) \ - (((val) & CPUID_FAMILY_MASK) >> CPUID_FAMILY_SHIFT) -#define CPUID_MODEL_SHIFT 24 -#define CPUID_MODEL_MASK (0xff << CPUID_MODEL_SHIFT) -#define CPUID_MODEL(val) \ - (((val) & CPUID_MODEL_MASK) >> CPUID_MODEL_SHIFT) -#define CPUID_PA_BITS_SHIFT 32 -#define CPUID_PA_BITS_MASK (0x7fUL << CPUID_PA_BITS_SHIFT) -#define CPUID_PA_BITS(val) \ - (((val) & CPUID_PA_BITS_MASK) >> CPUID_PA_BITS_SHIFT) -#define CPUID_VA_BITS_SHIFT 39 -#define CPUID_VA_BITS_MASK (0x7fUL << CPUID_VA_BITS_SHIFT) -#define CPUID_VA_BITS(val) \ - (((val) & CPUID_VA_BITS_MASK) >> CPUID_VA_BITS_SHIFT) - #define current_cpu_data cpu_data[smp_processor_id()] enum hmcall_cpuid_cmd { @@ -55,12 +24,6 @@ enum sunway_cpu_model { }; struct cpuinfo_sw64 { - unsigned long last_asid; - unsigned long last_vpn; - unsigned long ipi_count; -} __aligned(SMP_CACHE_BYTES); - -struct cpu_desc_t { __u8 model; __u8 family; __u8 chip_var; @@ -68,27 +31,19 @@ struct cpu_desc_t { __u8 arch_rev; __u8 pa_bits; __u8 va_bits; - char vendor_id[16]; - char model_id[64]; - unsigned long frequency; -} __randomize_layout; + const char *vendor_id; + const char *model_id; + unsigned long last_asid; + unsigned long last_vpn; + unsigned long ipi_count; +} __aligned(SMP_CACHE_BYTES); extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; -extern struct cpu_desc_t cpu_desc; extern cpumask_t cpu_offline; extern void store_cpu_data(int cpu); -extern void __init setup_cpu_info(void); - -static inline unsigned long get_cpu_freq(void) -{ - return cpu_desc.frequency; -} - -static inline void update_cpu_freq(unsigned long khz) -{ - cpu_desc.frequency = khz * 1000; -} +extern unsigned long get_cpu_freq(void); +extern void update_cpu_freq(unsigned long khz); extern unsigned int get_cpu_cache_size(int cpu, int level, enum cache_type type); extern unsigned int get_cpu_cacheline_size(int cpu, int level, enum cache_type type); diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index efc07f0d0cab..732943cb1859 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -196,7 +196,7 @@ static inline void intpu_save(void) { void __iomem *intpu_base = misc_platform_get_intpu_base(0); - switch (cpu_desc.model) { + switch (current_cpu_data.model) { case CPU_SW831: saved_long_time = readq(intpu_base + OFFSET_LONG_TIME); default: @@ -210,7 +210,7 @@ static inline void intpu_restore(void) void __iomem *spbu_base = misc_platform_get_spbu_base(0); void __iomem *gpio_base = misc_platform_get_gpio_base(0); - switch (cpu_desc.model) { + switch (current_cpu_data.model) { case CPU_SW831: writeq(saved_long_time, intpu_base + OFFSET_LONG_TIME); writeq(0x1, spbu_base + OFFSET_LONG_TIME_START_EN); diff --git a/arch/sw_64/kernel/cpu.c b/arch/sw_64/kernel/cpu.c index fafbb23433b8..43ac70253adf 100644 --- a/arch/sw_64/kernel/cpu.c +++ b/arch/sw_64/kernel/cpu.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -9,6 +10,18 @@ #include #include +#define TABLE_ENTRY_MAX 32 +#define VENDOR_ID_MAX 2 +#define MODEL_MAX 8 + +#define cpuinfo_arch_rev(cpu_info) ((cpu_info) & 0xf) +#define cpuinfo_arch_var(cpu_info) (((cpu_info) >> 4) & 0xf) +#define cpuinfo_chip_var(cpu_info) (((cpu_info) >> 8) & 0xf) +#define cpuinfo_family(cpu_info) (((cpu_info) >> 12) & 0xf) +#define cpuinfo_model(cpu_info) (((cpu_info) >> 24) & 0xff) +#define cpuinfo_pa_bits(cpu_info) (((cpu_info) >> 32) & 0x7fUL) +#define cpuinfo_va_bits(cpu_info) (((cpu_info) >> 39) & 0x7fUL) + /* Map logical to physical */ int __cpu_to_rcid[NR_CPUS]; EXPORT_SYMBOL(__cpu_to_rcid); @@ -17,42 +30,79 @@ EXPORT_SYMBOL(__cpu_to_rcid); struct cpuinfo_sw64 cpu_data[NR_CPUS]; EXPORT_SYMBOL(cpu_data); -struct cpu_desc_t cpu_desc; - cpumask_t cpu_offline = CPU_MASK_NONE; +static unsigned long cpu_freq; +static unsigned long cpu_info; +static char vendor_id[16]; +static char model_id[64]; + +unsigned long get_cpu_freq(void) +{ + if (likely(cpu_freq)) + return cpu_freq; + + return cpuid(GET_CPU_FREQ, 0) * 1000UL * 1000UL; +} + +void update_cpu_freq(unsigned long khz) +{ + cpu_freq = khz * 1000; +} + /* Move global data into per-processor storage */ void store_cpu_data(int cpu) { cpu_data[cpu].last_asid = ASID_FIRST_VERSION; } -void __init setup_cpu_info(void) +static int cpuinfo_cpu_online(unsigned int cpu) { - int i; + /* Currently, cpu info is shared by all cores */ + cpu_data[cpu].model = cpuinfo_model(cpu_info); + cpu_data[cpu].family = cpuinfo_family(cpu_info); + cpu_data[cpu].chip_var = cpuinfo_chip_var(cpu_info); + cpu_data[cpu].arch_var = cpuinfo_arch_var(cpu_info); + cpu_data[cpu].arch_rev = cpuinfo_arch_rev(cpu_info); + cpu_data[cpu].pa_bits = cpuinfo_pa_bits(cpu_info); + cpu_data[cpu].va_bits = cpuinfo_va_bits(cpu_info); + + cpu_data[cpu].vendor_id = vendor_id; + cpu_data[cpu].model_id = model_id; + + return 0; +} + +static int __init sw64_cpuinfo_init(void) +{ + int i, ret; unsigned long val; - val = cpuid(GET_TABLE_ENTRY, 0); - cpu_desc.model = CPUID_MODEL(val); - cpu_desc.family = CPUID_FAMILY(val); - cpu_desc.chip_var = CPUID_CHIP_VAR(val); - cpu_desc.arch_var = CPUID_ARCH_VAR(val); - cpu_desc.arch_rev = CPUID_ARCH_REV(val); - cpu_desc.pa_bits = CPUID_PA_BITS(val); - cpu_desc.va_bits = CPUID_VA_BITS(val); + /* Get cpu info */ + cpu_info = cpuid(GET_TABLE_ENTRY, 0); + /* Get vendor name in string format */ for (i = 0; i < VENDOR_ID_MAX; i++) { val = cpuid(GET_VENDOR_ID, i); - memcpy(cpu_desc.vendor_id + (i * 8), &val, 8); + memcpy(vendor_id + (i * 8), &val, 8); } + /* Get model name in string format */ for (i = 0; i < MODEL_MAX; i++) { val = cpuid(GET_MODEL, i); - memcpy(cpu_desc.model_id + (i * 8), &val, 8); + memcpy(model_id + (i * 8), &val, 8); } - cpu_desc.frequency = cpuid(GET_CPU_FREQ, 0) * 1000UL * 1000UL; + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "sw64/cpuinfo:online", + cpuinfo_cpu_online, NULL); + if (ret < 0) { + pr_err("cpuinfo: failed to register cpuinfo_cpu_online\n"); + return ret; + } + + return 0; } +arch_initcall(sw64_cpuinfo_init); static int show_cpuinfo(struct seq_file *f, void *slot) { @@ -78,11 +128,11 @@ static int show_cpuinfo(struct seq_file *f, void *slot) "model name\t: %s CPU @ %lu.%lu%luGHz\n" "cpu variation\t: %u\n" "cpu revision\t: %u\n", - i, cpu_desc.vendor_id, cpu_desc.family, - cpu_desc.model, cpu_desc.model_id, + i, vendor_id, cpu_data[i].family, + cpu_data[i].model, model_id, freq / 1000, (freq % 1000) / 100, (freq % 100) / 10, - cpu_desc.arch_var, cpu_desc.arch_rev); + cpu_data[i].arch_var, cpu_data[i].arch_rev); seq_printf(f, "cpu MHz\t\t: %lu.00\n" "cache size\t: %u KB\n" "physical id\t: %d\n" @@ -96,7 +146,7 @@ static int show_cpuinfo(struct seq_file *f, void *slot) seq_printf(f, "page size\t: %d\n", 8192); seq_printf(f, "cache_alignment\t: %d\n", l3_cachline_size); seq_printf(f, "address sizes\t: %u bits physical, %u bits virtual\n\n", - cpu_desc.pa_bits, cpu_desc.va_bits); + cpu_data[i].pa_bits, cpu_data[i].va_bits); } return 0; @@ -127,3 +177,8 @@ const struct seq_operations cpuinfo_op = { .show = show_cpuinfo, }; +bool arch_match_cpu_phys_id(int cpu, u64 phys_id) +{ + return phys_id == cpu_physical_id(cpu); +} + diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index c7712c9eca06..6c36fedfa6e3 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -49,7 +49,7 @@ static inline int phys_addr_valid(unsigned long addr) * and other physical address variables cannot be used, so let's * roughly judge physical address based on arch specific bit. */ - return !(addr >> (cpu_desc.pa_bits - 1)); + return !(addr >> (current_cpu_data.pa_bits - 1)); } extern struct atomic_notifier_head panic_notifier_list; @@ -648,7 +648,6 @@ setup_arch(char **cmdline_p) trap_init(); jump_label_init(); - setup_cpu_info(); setup_run_mode(); setup_chip_ops(); diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index e3720a3a1df4..8b022f455c78 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -94,7 +94,7 @@ static int longtime_enable(struct clocksource *cs) void __iomem *spbu_base = misc_platform_get_spbu_base(0); void __iomem *gpio_base = misc_platform_get_gpio_base(0); - switch (cpu_desc.model) { + switch (current_cpu_data.model) { case CPU_SW3231: writeq(0, gpio_base + OFFSET_GPIO_SWPORTA_DR); writeq(0xff, gpio_base + OFFSET_GPIO_SWPORTA_DDR); -- Gitee From 70cee51922dc437f6816a0926e2a690cc6cfddef Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 4 Sep 2024 18:26:51 +0800 Subject: [PATCH 319/524] sw64: cache: fix shared_cpu_map of cacheinfo Currently, L1 and L2 caches are privately owned by the physical core, which has two hardware threads in the JunZhang series. This means that two logical cores belonging to the same physical core share L1 and L2 caches. Besides, in the generic update_siblings_masks(), cache information is initialized before cpu siblings. Therefore, we cannot initialize shared_cpu_map with topology_llc_cpumask. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/cacheinfo.c | 49 ++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/arch/sw_64/kernel/cacheinfo.c b/arch/sw_64/kernel/cacheinfo.c index 69a32554e255..da24b0e5b989 100644 --- a/arch/sw_64/kernel/cacheinfo.c +++ b/arch/sw_64/kernel/cacheinfo.c @@ -18,7 +18,6 @@ #include #include -#include #include #define get_cache_info(type) cpuid(GET_CACHE_INFO, (type)) @@ -113,30 +112,56 @@ int init_cache_level(unsigned int cpu) return 0; } +static void setup_shared_cpu_map(unsigned int cpu) +{ + unsigned int index; + unsigned int rcid = cpu_to_rcid(cpu); + struct cacheinfo *this_leaf; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + + for (index = 0; index < this_cpu_ci->num_leaves; index++) { + unsigned int i; + + this_leaf = this_cpu_ci->info_list + index; + + cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); + + for_each_possible_cpu(i) { + unsigned int sib_rcid = cpu_to_rcid(i); + + if ((rcid_to_domain_id(sib_rcid) != rcid_to_domain_id(rcid)) || + (i == cpu)) + continue; + + if ((rcid_to_core_id(rcid) == rcid_to_core_id(sib_rcid)) || + (this_leaf->level == 3)) + cpumask_set_cpu(i, &this_leaf->shared_cpu_map); + } + } +} + int populate_cache_leaves(unsigned int cpu) { enum sunway_cache_type type; + unsigned int cache_id; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *this_leaf = this_cpu_ci->info_list; - struct cpu_topology *topo = &cpu_topology[cpu]; for (type = L1_ICACHE; type <= L3_CACHE; type++) { if (!cache_size(type)) continue; /* L3 Cache is shared */ - if (type == L3_CACHE) { - cpumask_copy(&this_leaf->shared_cpu_map, - topology_llc_cpumask(cpu)); - populate_cache(get_cache_info(type), this_leaf, cache_level(type), - kernel_cache_type(type), topo->package_id); - } else { - cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); - populate_cache(get_cache_info(type), this_leaf, cache_level(type), - kernel_cache_type(type), cpu); - } + cache_id = (type == L3_CACHE) ? rcid_to_domain_id(cpu_to_rcid(cpu)) : + rcid_to_core_id(cpu_to_rcid(cpu)); + + populate_cache(get_cache_info(type), this_leaf, cache_level(type), + kernel_cache_type(type), cache_id); + } + setup_shared_cpu_map(cpu); + this_cpu_ci->cpu_map_populated = true; return 0; -- Gitee From 600cea8679d17def21ee7c456668347c0b9efa8e Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 4 Sep 2024 19:50:08 +0800 Subject: [PATCH 320/524] sw64: cacheinfo: give preference to cache information from PPTT table We prefer the cache information provided by the firmware in ACPI PPTT table. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/cacheinfo.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/kernel/cacheinfo.c b/arch/sw_64/kernel/cacheinfo.c index da24b0e5b989..c8528cf9f3ba 100644 --- a/arch/sw_64/kernel/cacheinfo.c +++ b/arch/sw_64/kernel/cacheinfo.c @@ -16,6 +16,7 @@ */ #include +#include #include #include @@ -39,7 +40,6 @@ do { \ leaf->number_of_sets = get_cache_sets(cache_info); \ leaf->ways_of_associativity = get_cache_ways(cache_info); \ leaf->size = get_cache_size(cache_info); \ - leaf++; \ } while (0) static struct cacheinfo *get_cacheinfo(int cpu, int level, enum cache_type type) @@ -140,14 +140,32 @@ static void setup_shared_cpu_map(unsigned int cpu) } } +static bool is_pptt_cache_info_valid(void) +{ + struct acpi_table_header *table; + acpi_status status; + + if (is_guest_or_emul() || acpi_disabled) + return false; + + status = acpi_get_table(ACPI_SIG_PPTT, 0, &table); + if (ACPI_FAILURE(status)) + return false; + + acpi_put_table(table); + + return true; +} + int populate_cache_leaves(unsigned int cpu) { enum sunway_cache_type type; unsigned int cache_id; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *this_leaf = this_cpu_ci->info_list; + bool pptt_valid = is_pptt_cache_info_valid(); - for (type = L1_ICACHE; type <= L3_CACHE; type++) { + for (type = L1_ICACHE; type <= L3_CACHE; type++, this_leaf++) { if (!cache_size(type)) continue; @@ -158,11 +176,15 @@ int populate_cache_leaves(unsigned int cpu) populate_cache(get_cache_info(type), this_leaf, cache_level(type), kernel_cache_type(type), cache_id); - } + if (pptt_valid) + this_leaf->attributes &= ~CACHE_ID; - setup_shared_cpu_map(cpu); + } - this_cpu_ci->cpu_map_populated = true; + if (!pptt_valid) { + setup_shared_cpu_map(cpu); + this_cpu_ci->cpu_map_populated = true; + } return 0; } -- Gitee From e9c735e46aa4e0ef297fab80f68296c406dc3bf4 Mon Sep 17 00:00:00 2001 From: Deng Xiaoyun Date: Thu, 29 Aug 2024 15:43:54 +0800 Subject: [PATCH 321/524] sw64: fix judgements about legacy pci support Starting with junzhang-v2, the host kernel gets pci controller information from the device tree. Starting with junzhang, the guest kernel gets pci controller information from the device tree. In order to be compatible with traditional pci information acquisition method, we add relevant judgments. Signed-off-by: Deng Xiaoyun Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/pci/pci-legacy.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index 8a9ac6f1146b..417852cbfe95 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -227,6 +227,20 @@ static bool __init is_any_rc_linkup_one_node(unsigned long node) return false; } +static bool __init is_sunway_legacy_pci(void) +{ + if (IS_ENABLED(CONFIG_SUBARCH_C3B)) + return true; + + if (sunway_machine_is_compatible("sunway,chip4")) + return true; + + if (is_in_host() && sunway_machine_is_compatible("sunway,junzhang")) + return true; + + return false; +} + void __init sw64_init_arch(void) { if (IS_ENABLED(CONFIG_PCI)) { @@ -238,8 +252,7 @@ void __init sw64_init_arch(void) if (!acpi_disabled) return; - if (!sunway_machine_is_compatible("sunway,chip3") && - !sunway_machine_is_compatible("sunway,junzhang")) + if (!is_sunway_legacy_pci()) return; sunway_legacy_pci = true; -- Gitee From c4832f1e429be3e788e239d7c9849632e6c5cfab Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 13 Sep 2024 07:20:49 +0000 Subject: [PATCH 322/524] sw64: iommu: fix NULL hose check Fix an oops issue of encountering NULL hose when using IOMMU ACPI initialization method. Fixes: 40c93f0b54f9 ("sw64: iommu: improve iommu initialization"). Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu_v2.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index d9e3b75e6c9e..bf4ff7d16d2a 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -971,6 +971,10 @@ static int sunway_iommu_acpi_init(void) return ret; for_each_iommu(iommu) { + hose = find_hose_by_rcid(iommu->node, iommu->index); + if (!hose) + continue; + if (!iommu->enabled || hose->iommu_enable) continue; @@ -978,7 +982,6 @@ static int sunway_iommu_acpi_init(void) iommu_index); iommu_device_register(&iommu->iommu, &sunway_iommu_ops, NULL); iommu_index++; - hose = find_hose_by_rcid(iommu->node, iommu->index); sunway_enable_iommu_func(hose); hose->iommu_enable = true; piu_flush_all(iommu); -- Gitee From 30b4552bdd7edb625f276d27f90566d99e631699 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Sat, 14 Sep 2024 06:19:47 +0000 Subject: [PATCH 323/524] sw64: move handle_intx() into pci-intx handle Move handle_intx() function into pci_intx handle code. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/irq.h | 9 +++++ drivers/irqchip/irq-sunway-cpu.c | 53 --------------------------- drivers/irqchip/irq-sunway-pci-intx.c | 53 +++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 53 deletions(-) diff --git a/arch/sw_64/include/asm/irq.h b/arch/sw_64/include/asm/irq.h index de3530b687e0..6a2bc75ffe3f 100644 --- a/arch/sw_64/include/asm/irq.h +++ b/arch/sw_64/include/asm/irq.h @@ -38,6 +38,15 @@ struct acpi_madt_sw_lpc_intc; extern int __init sw64_add_gsi_domain_map(u32 gsi_base, u32 gsi_count, struct fwnode_handle *handle); +#ifdef CONFIG_SW64_PCI_INTX +extern void handle_intx(unsigned int offset); +#else +static inline void handle_intx(unsigned int offset) +{ + pr_crit("Enter PCI INTx, but no handle configured!\n"); +} +#endif + #ifdef CONFIG_SW64_PINTC extern int __init pintc_acpi_init(struct irq_domain *parent, struct acpi_madt_sw_pintc *pintc); diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 4947805c33ab..9dddf6786eab 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -39,59 +39,6 @@ struct fwnode_handle *cintc_handle; -static void handle_intx(unsigned int offset) -{ - struct pci_controller *hose; - unsigned long value; - void __iomem *piu_ior0_base; - - hose = hose_head; - offset <<= 7; - for (hose = hose_head; hose; hose = hose->next) { - piu_ior0_base = hose->piu_ior0_base; - - value = readq(piu_ior0_base + INTACONFIG + offset); - if (value >> 63) { - value = value & (~(1UL << 62)); - writeq(value, (piu_ior0_base + INTACONFIG + offset)); - handle_irq(hose->int_irq); - value = value | (1UL << 62); - writeq(value, (piu_ior0_base + INTACONFIG + offset)); - } - - if (IS_ENABLED(CONFIG_PCIE_PME)) { - value = readq(piu_ior0_base + PMEINTCONFIG); - if (value >> 63) { - handle_irq(hose->service_irq); - writeq(value, (piu_ior0_base + PMEINTCONFIG)); - } - } - - if (IS_ENABLED(CONFIG_PCIEAER)) { - value = readq(piu_ior0_base + AERERRINTCONFIG); - if (value >> 63) { - handle_irq(hose->service_irq); - writeq(value, (piu_ior0_base + AERERRINTCONFIG)); - } - } - - if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE_SUNWAY)) { - value = readq(piu_ior0_base + HPINTCONFIG); - if (value >> 63) { - handle_irq(hose->service_irq); - writeq(value, (piu_ior0_base + HPINTCONFIG)); - } - - } - - if (hose->iommu_enable) { - value = readq(piu_ior0_base + IOMMUEXCPT_STATUS); - if (value >> 63) - handle_irq(hose->int_irq); - } - } -} - static void handle_device_interrupt(unsigned long irq_info) { unsigned int i; diff --git a/drivers/irqchip/irq-sunway-pci-intx.c b/drivers/irqchip/irq-sunway-pci-intx.c index a60262fafeb3..2d7df695bf49 100644 --- a/drivers/irqchip/irq-sunway-pci-intx.c +++ b/drivers/irqchip/irq-sunway-pci-intx.c @@ -7,6 +7,7 @@ #include #include #include +#include static DEFINE_RAW_SPINLOCK(legacy_lock); static void lock_legacy_lock(void) @@ -196,3 +197,55 @@ void __init sw64_init_irq(void) for (hose = hose_head; hose; hose = hose->next) setup_intx_irqs(hose); } + +void handle_intx(unsigned int offset) +{ + struct pci_controller *hose; + unsigned long value; + void __iomem *piu_ior0_base; + + hose = hose_head; + offset <<= 7; + for (hose = hose_head; hose; hose = hose->next) { + piu_ior0_base = hose->piu_ior0_base; + + value = readq(piu_ior0_base + INTACONFIG + offset); + if (value & PCI_INTX_VALID) { + value &= PCI_INTX_DISABLE; + writeq(value, (piu_ior0_base + INTACONFIG + offset)); + handle_irq(hose->int_irq); + value |= PCI_INTX_ENABLE; + writeq(value, (piu_ior0_base + INTACONFIG + offset)); + } + + if (IS_ENABLED(CONFIG_PCIE_PME)) { + value = readq(piu_ior0_base + PMEINTCONFIG); + if (value & PCI_INTX_VALID) { + handle_irq(hose->service_irq); + writeq(value, (piu_ior0_base + PMEINTCONFIG)); + } + } + + if (IS_ENABLED(CONFIG_PCIEAER)) { + value = readq(piu_ior0_base + AERERRINTCONFIG); + if (value & PCI_INTX_VALID) { + handle_irq(hose->service_irq); + writeq(value, (piu_ior0_base + AERERRINTCONFIG)); + } + } + + if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE_SUNWAY)) { + value = readq(piu_ior0_base + HPINTCONFIG); + if (value & PCI_INTX_VALID) { + handle_irq(hose->service_irq); + writeq(value, (piu_ior0_base + HPINTCONFIG)); + } + } + + if (hose->iommu_enable) { + value = readq(piu_ior0_base + IOMMUEXCPT_STATUS); + if (value & PCI_INTX_VALID) + handle_irq(hose->int_irq); + } + } +} -- Gitee From 0d8324fe412dde87549c4c02370355e6bfda087c Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Wed, 25 Sep 2024 08:44:38 +0000 Subject: [PATCH 324/524] sw64: introduce a fine-grained intx mask/unmask system We have recently discovered an unexpected shutdown failure that if an intx irq arrived RC after removing its registered action, but before disabling the RC's INTxCONFIG, handle_intx() may accidentally re-enable it and keep sending it to CPU, causing CPU hang up and shutdown failure. This is because we are following a non-standard handle_irq procedure of not registering proper irq_mask/unmask callbacks. The kernel's handle_irq procedure provides an irq state check to return directly if an irq is relieved from its actions or is marked as disabled before actually handling the irq. This can avoid the re-enabling problem we've encountered easily and thoroughly. This patch introduces a more fine-grained irq_mask/unmask, which is capable of disabling a designated INTxCONFIG instead of enabling/disabling all 4 INTxCONFIGs every time to improve the efficiency. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 1 + drivers/irqchip/irq-sunway-pci-intx.c | 229 ++++++++++++++++---------- 2 files changed, 147 insertions(+), 83 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 9662fdd87cb1..3527b4a92413 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -236,6 +236,7 @@ extern int chip_pcie_configure(struct pci_controller *hose); #define PCI_INTX_ENABLE ((1UL) << 62) #define PCI_INTX_DISABLE ~((1UL) << 62) #define PCI_INTX_VALID (1UL << 63) +#define PCI_INTX_INTDST_MASK 0x3ffUL #define PCI_VENDOR_ID_JN 0x5656 #define PCI_DEVICE_ID_SW64_ROOT_BRIDGE 0x3231 diff --git a/drivers/irqchip/irq-sunway-pci-intx.c b/drivers/irqchip/irq-sunway-pci-intx.c index 2d7df695bf49..d6f23c666237 100644 --- a/drivers/irqchip/irq-sunway-pci-intx.c +++ b/drivers/irqchip/irq-sunway-pci-intx.c @@ -10,6 +10,13 @@ #include static DEFINE_RAW_SPINLOCK(legacy_lock); + +struct intx_chip_data { + struct pci_controller *hose; + unsigned long intxconfig[4]; + unsigned int offset; +}; + static void lock_legacy_lock(void) { raw_spin_lock(&legacy_lock); @@ -20,33 +27,34 @@ static void unlock_legacy_lock(void) raw_spin_unlock(&legacy_lock); } -static void set_intx(struct pci_controller *hose, unsigned long intx_conf) +struct intx_chip_data *alloc_intx_chip_data(u32 node) { - void __iomem *piu_ior0_base; + struct intx_chip_data *chip_data; - if (is_guest_or_emul()) - return; + if (WARN_ON(node >= MAX_NUMNODES)) + return NULL; - piu_ior0_base = hose->piu_ior0_base; + chip_data = kzalloc_node(sizeof(struct intx_chip_data), + GFP_KERNEL, node); + if (!chip_data) + chip_data = kzalloc(sizeof(struct intx_chip_data), + GFP_KERNEL); - if (IS_ENABLED(CONFIG_SUBARCH_C3B)) { - writeq(intx_conf | (0x8UL << 10), (piu_ior0_base + INTACONFIG)); - writeq(intx_conf | (0x4UL << 10), (piu_ior0_base + INTBCONFIG)); - writeq(intx_conf | (0x2UL << 10), (piu_ior0_base + INTCCONFIG)); - writeq(intx_conf | (0x1UL << 10), (piu_ior0_base + INTDCONFIG)); - } else { - writeq(intx_conf | (0x8UL << 10), (piu_ior0_base + INTDCONFIG)); - writeq(intx_conf | (0x4UL << 10), (piu_ior0_base + INTCCONFIG)); - writeq(intx_conf | (0x2UL << 10), (piu_ior0_base + INTBCONFIG)); - writeq(intx_conf | (0x1UL << 10), (piu_ior0_base + INTACONFIG)); - } + return chip_data; } -static int __assign_piu_intx_config(struct pci_controller *hose, cpumask_t *targets) +static int __assign_piu_intx_config(struct intx_chip_data *chip_data, + cpumask_t *targets) { - unsigned long intx_conf; + struct pci_controller *hose; + unsigned long intxconfig; + void __iomem *piu_ior0_base; unsigned int cpu; int thread, node, core, rcid; + unsigned int i; + + if (is_guest_or_emul()) + return 0; /* Use the last cpu in valid cpus to avoid core 0. */ cpu = cpumask_last(targets); @@ -56,89 +64,80 @@ static int __assign_piu_intx_config(struct pci_controller *hose, cpumask_t *targ node = rcid_to_domain_id(rcid); core = rcid_to_core_id(rcid); - if (IS_ENABLED(CONFIG_SUBARCH_C3B)) - intx_conf = core | (node << 6); - else - intx_conf = core | (thread << 6) | (node << 7); + hose = chip_data->hose; + piu_ior0_base = hose->piu_ior0_base; - set_intx(hose, intx_conf); + for (i = 0; i < 4; i++) { + intxconfig = chip_data->intxconfig[i]; + intxconfig &= ~PCI_INTX_INTDST_MASK; + if (IS_ENABLED(CONFIG_SUBARCH_C3B)) + intxconfig |= core | (node << 6); + else + intxconfig |= core | (thread << 6) | (node << 7); + + writeq(intxconfig, piu_ior0_base + INTACONFIG + (i << 7)); + chip_data->intxconfig[i] = intxconfig; + } return 0; } -static int assign_piu_intx_config(struct pci_controller *hose, cpumask_t *targets) +static int assign_piu_intx_config(struct intx_chip_data *chip_data, + cpumask_t *targets) { int ret; lock_legacy_lock(); - ret = __assign_piu_intx_config(hose, targets); + ret = __assign_piu_intx_config(chip_data, targets); unlock_legacy_lock(); return ret; } -static void intx_irq_enable(struct irq_data *irq_data) +static void set_intx_enable(struct irq_data *irq_data, u32 flag) { - struct pci_controller *hose = irq_data->chip_data; - unsigned long intx_conf; + struct intx_chip_data *chip_data = irq_data->chip_data; + struct pci_controller *hose; void __iomem *piu_ior0_base; + unsigned long intxconfig; + unsigned int i; - if (is_guest_or_emul()) + if (!chip_data) return; - BUG_ON(!hose); + hose = chip_data->hose; piu_ior0_base = hose->piu_ior0_base; - intx_conf = readq(piu_ior0_base + INTACONFIG); - intx_conf |= PCI_INTX_ENABLE; - writeq(intx_conf, (piu_ior0_base + INTACONFIG)); - - intx_conf = readq(piu_ior0_base + INTBCONFIG); - intx_conf |= PCI_INTX_ENABLE; - writeq(intx_conf, (piu_ior0_base + INTBCONFIG)); + for (i = 0; i < 4; i++) { + intxconfig = chip_data->intxconfig[i]; + if (flag) + intxconfig |= PCI_INTX_ENABLE; + else + intxconfig &= PCI_INTX_DISABLE; + writeq(intxconfig, piu_ior0_base + INTACONFIG + (i << 7)); + } +} - intx_conf = readq(piu_ior0_base + INTCCONFIG); - intx_conf |= PCI_INTX_ENABLE; - writeq(intx_conf, (piu_ior0_base + INTCCONFIG)); +static void intx_irq_enable(struct irq_data *irq_data) +{ + if (is_guest_or_emul()) + return; - intx_conf = readq(piu_ior0_base + INTDCONFIG); - intx_conf |= PCI_INTX_ENABLE; - writeq(intx_conf, (piu_ior0_base + INTDCONFIG)); + set_intx_enable(irq_data, 1); } static void intx_irq_disable(struct irq_data *irq_data) { - struct pci_controller *hose = irq_data->chip_data; - unsigned long intx_conf; - void __iomem *piu_ior0_base; - if (is_guest_or_emul()) return; - BUG_ON(!hose); - piu_ior0_base = hose->piu_ior0_base; - - intx_conf = readq(piu_ior0_base + INTACONFIG); - intx_conf &= PCI_INTX_DISABLE; - writeq(intx_conf, (piu_ior0_base + INTACONFIG)); - - intx_conf = readq(piu_ior0_base + INTBCONFIG); - intx_conf &= PCI_INTX_DISABLE; - writeq(intx_conf, (piu_ior0_base + INTBCONFIG)); - - intx_conf = readq(piu_ior0_base + INTCCONFIG); - intx_conf &= PCI_INTX_DISABLE; - writeq(intx_conf, (piu_ior0_base + INTCCONFIG)); - - intx_conf = readq(piu_ior0_base + INTDCONFIG); - intx_conf &= PCI_INTX_DISABLE; - writeq(intx_conf, (piu_ior0_base + INTDCONFIG)); + set_intx_enable(irq_data, 0); } static int intx_set_affinity(struct irq_data *irq_data, const struct cpumask *dest, bool force) { - struct pci_controller *hose = irq_data->chip_data; + struct intx_chip_data *chip_data = irq_data->chip_data; cpumask_t targets; int ret = 0; @@ -148,20 +147,62 @@ static int intx_set_affinity(struct irq_data *irq_data, cpumask_copy(&targets, dest); intx_irq_disable(irq_data); - ret = assign_piu_intx_config(hose, &targets); + ret = assign_piu_intx_config(chip_data, &targets); intx_irq_enable(irq_data); return ret; } +static void intx_mask_irq(struct irq_data *irq_data, u32 flag) +{ + struct intx_chip_data *chip_data = irq_data->chip_data; + struct pci_controller *hose; + void __iomem *piu_ior0_base; + unsigned long intxconfig; + unsigned int offset; + + if (!chip_data) + return; + + hose = chip_data->hose; + piu_ior0_base = hose->piu_ior0_base; + offset = chip_data->offset; + intxconfig = chip_data->intxconfig[offset]; + + if (flag) + intxconfig &= PCI_INTX_DISABLE; + else + intxconfig |= PCI_INTX_ENABLE; + + writeq(intxconfig, piu_ior0_base + INTACONFIG + (offset << 7)); +} + +static void intx_irq_mask(struct irq_data *irq_data) +{ + if (is_guest_or_emul()) + return; + + intx_mask_irq(irq_data, 1); +} + +static void intx_irq_unmask(struct irq_data *irq_data) +{ + if (is_guest_or_emul()) + return; + + intx_mask_irq(irq_data, 0); +} + static void noop(struct irq_data *d) {} static struct irq_chip sw64_intx_chip = { .name = "PCI_INTX", .irq_enable = intx_irq_enable, .irq_disable = intx_irq_disable, - .irq_set_affinity = intx_set_affinity, - .irq_ack = noop, + .irq_mask = intx_irq_mask, + .irq_unmask = intx_irq_unmask, + .irq_set_affinity = intx_set_affinity, + .irq_ack = noop, .flags = IRQCHIP_SKIP_SET_WAKE, }; @@ -170,8 +211,12 @@ void __weak set_pcieport_service_irq(struct pci_controller *hose) {} void setup_intx_irqs(struct pci_controller *hose) { unsigned long irq, node, val_node; + struct intx_chip_data *chip_data; + void __iomem *piu_ior0_base; + int i = 0; node = hose->node; + piu_ior0_base = hose->piu_ior0_base; if (!node_online(node)) val_node = next_node_in(node, node_online_map); @@ -182,8 +227,23 @@ void setup_intx_irqs(struct pci_controller *hose) WARN_ON(irq < 0); irq_set_chip_and_handler(irq, &sw64_intx_chip, handle_level_irq); irq_set_status_flags(irq, IRQ_LEVEL); - irq_set_chip_data(irq, hose); + + chip_data = alloc_intx_chip_data(val_node); + if (!chip_data) + return; + + chip_data->hose = hose; + + for (i = 0; i < 4; i++) { + if (IS_ENABLED(CONFIG_SUBARCH_C3B)) + chip_data->intxconfig[i] = (0x1UL << (3 - i)) << 10; + else + chip_data->intxconfig[i] = (0x1UL << i) << 10; + } + + irq_set_chip_data(irq, chip_data); hose->int_irq = irq; + irq_set_chip_and_handler(irq + 1, &dummy_irq_chip, handle_level_irq); hose->service_irq = irq + 1; @@ -200,22 +260,31 @@ void __init sw64_init_irq(void) void handle_intx(unsigned int offset) { + struct irq_data *irq_data; + struct intx_chip_data *chip_data; struct pci_controller *hose; unsigned long value; void __iomem *piu_ior0_base; hose = hose_head; - offset <<= 7; for (hose = hose_head; hose; hose = hose->next) { piu_ior0_base = hose->piu_ior0_base; - value = readq(piu_ior0_base + INTACONFIG + offset); - if (value & PCI_INTX_VALID) { - value &= PCI_INTX_DISABLE; - writeq(value, (piu_ior0_base + INTACONFIG + offset)); + value = readq(piu_ior0_base + INTACONFIG + (offset << 7)); + if ((value & (PCI_INTX_VALID)) && (value & PCI_INTX_ENABLE)) { + irq_data = irq_get_irq_data(hose->int_irq); + if (irq_data) { + chip_data = irq_data->chip_data; + if (chip_data) + chip_data->offset = offset; + } handle_irq(hose->int_irq); - value |= PCI_INTX_ENABLE; - writeq(value, (piu_ior0_base + INTACONFIG + offset)); + } + + if (hose->iommu_enable) { + value = readq(piu_ior0_base + IOMMUEXCPT_STATUS); + if (value & PCI_INTX_VALID) + handle_irq(hose->int_irq); } if (IS_ENABLED(CONFIG_PCIE_PME)) { @@ -241,11 +310,5 @@ void handle_intx(unsigned int offset) writeq(value, (piu_ior0_base + HPINTCONFIG)); } } - - if (hose->iommu_enable) { - value = readq(piu_ior0_base + IOMMUEXCPT_STATUS); - if (value & PCI_INTX_VALID) - handle_irq(hose->int_irq); - } } } -- Gitee From 25746dbe51f0980cd1130838ab831dd4f229bd18 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 27 Sep 2024 09:08:59 +0000 Subject: [PATCH 325/524] sw64: remove deprecated annotation in irq.h Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/irq.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arch/sw_64/include/asm/irq.h b/arch/sw_64/include/asm/irq.h index 6a2bc75ffe3f..ae837f19b2d8 100644 --- a/arch/sw_64/include/asm/irq.h +++ b/arch/sw_64/include/asm/irq.h @@ -2,12 +2,6 @@ #ifndef _ASM_SW64_IRQ_H #define _ASM_SW64_IRQ_H -/* - * arch/sw/include/asm/irq.h - * - * (C) 2012 OSKernel JN - */ - #include #define NR_VECTORS_PERCPU 256 -- Gitee From adff9083f3f8bf495bec4f3f1b809953a7de56e3 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 13 Sep 2024 09:04:12 +0800 Subject: [PATCH 326/524] sw64: provide a cleaner raw_smp_processor_id() Use cpu in thread_info directly. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/smp.h | 3 +-- arch/sw_64/kernel/asm-offsets.c | 3 --- arch/sw_64/kernel/process.c | 1 + 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h index 33a73ca25a6b..4249b10dc550 100644 --- a/arch/sw_64/include/asm/smp.h +++ b/arch/sw_64/include/asm/smp.h @@ -50,8 +50,7 @@ extern void __init smp_rcb_init(struct smp_rcb_struct *smp_rcb_base_addr); #ifdef GENERATING_ASM_OFFSETS #define raw_smp_processor_id() (0) #else -#include -#define raw_smp_processor_id() (*((unsigned int *)((void *)current + TASK_CPU))) +#define raw_smp_processor_id() (current_thread_info()->cpu) #endif #define hard_smp_processor_id() cpu_to_rcid(raw_smp_processor_id()) diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c index d865caffd014..c4e4e40a03d5 100644 --- a/arch/sw_64/kernel/asm-offsets.c +++ b/arch/sw_64/kernel/asm-offsets.c @@ -30,9 +30,6 @@ void foo(void) DEFINE(TASK_GROUP_LEADER, offsetof(struct task_struct, group_leader)); DEFINE(TASK_TGID, offsetof(struct task_struct, tgid)); DEFINE(TASK_STACK, offsetof(struct task_struct, stack)); -#ifdef CONFIG_SMP - DEFINE(TASK_CPU, offsetof(struct task_struct, thread_info.cpu)); -#endif BLANK(); OFFSET(PSTATE_REGS, processor_state, regs); diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c index ded9f780497a..1447ea34276c 100644 --- a/arch/sw_64/kernel/process.c +++ b/arch/sw_64/kernel/process.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include -- Gitee From 18666b448089905211b90ddf49a998e3ab74282b Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 13 Sep 2024 17:21:17 +0800 Subject: [PATCH 327/524] sw64: mm: support fdt memory reservation Now we can reserve memory via the device tree. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/init.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index 6e976d7b20e4..29ee809c3ac5 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -385,6 +385,8 @@ void __init sw64_memblock_init(void) } } + early_init_fdt_scan_reserved_mem(); + /* end of DRAM range may have been changed */ max_pfn = max_low_pfn = PFN_DOWN(memblock_end_of_DRAM()); } -- Gitee From 4ae7e3b8aa68aa527d7f6a7bfecc170c6ebacb6a Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Tue, 15 Oct 2024 14:16:40 +0800 Subject: [PATCH 328/524] sw64: perf: add core4 pmu support This patch introduces core4 pmu support to make perf work, and moves common callchain code into a standalone file. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/csr.h | 4 + arch/sw_64/include/asm/perf_event.h | 17 ++++ arch/sw_64/include/asm/pmc_c4.h | 52 ++++++++++ arch/sw_64/kernel/Makefile | 7 +- arch/sw_64/kernel/perf_callchain.c | 142 ++++++++++++++++++++++++++++ arch/sw_64/kernel/perf_event.c | 138 --------------------------- arch/sw_64/kernel/perf_event_c4.c | 15 +-- 7 files changed, 230 insertions(+), 145 deletions(-) create mode 100644 arch/sw_64/include/asm/pmc_c4.h create mode 100644 arch/sw_64/kernel/perf_callchain.c diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h index 9101e4c91fe1..dac5b6a29b47 100644 --- a/arch/sw_64/include/asm/csr.h +++ b/arch/sw_64/include/asm/csr.h @@ -29,6 +29,10 @@ #define CSR_PTBR_SYS 0x68 #define CSR_PTBR_USR 0x69 #define CSR_APTP 0x6a +#define CSR_IDR_PCCTL 0x7a +#define CSR_IACC 0x7b +#define CSR_IMISC 0x7c +#define CSR_RETIC 0x7f #define CSR_CID 0xc4 #define CSR_WR_FREGS 0xc8 #define CSR_SHTCLOCK 0xca diff --git a/arch/sw_64/include/asm/perf_event.h b/arch/sw_64/include/asm/perf_event.h index dc55a361babd..a92545694c65 100644 --- a/arch/sw_64/include/asm/perf_event.h +++ b/arch/sw_64/include/asm/perf_event.h @@ -2,8 +2,14 @@ #ifndef _ASM_SW64_PERF_EVENT_H #define _ASM_SW64_PERF_EVENT_H +#if defined(CONFIG_SUBARCH_C3B) #include +#elif defined(CONFIG_SUBARCH_C4) +#include +#endif #include +#include +#include #ifdef CONFIG_PERF_EVENTS struct pt_regs; @@ -13,4 +19,15 @@ extern unsigned long perf_misc_flags(struct pt_regs *regs); #define perf_arch_bpf_user_pt_regs(regs) ®s->user_regs #endif +/* For tracking PMCs and the hw events they monitor on each CPU. */ +struct cpu_hw_events { + /* + * Set the bit (indexed by the counter number) when the counter + * is used for an event. + */ + unsigned long used_mask[BITS_TO_LONGS(MAX_HWEVENTS)]; + /* Array of events current scheduled on this cpu. */ + struct perf_event *event[MAX_HWEVENTS]; +}; + #endif /* _ASM_SW64_PERF_EVENT_H */ diff --git a/arch/sw_64/include/asm/pmc_c4.h b/arch/sw_64/include/asm/pmc_c4.h new file mode 100644 index 000000000000..cdee1041ce3b --- /dev/null +++ b/arch/sw_64/include/asm/pmc_c4.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Definitions for use with the sw64 wrperfmon HMCODE call. + */ + +#ifndef _ASM_SW64_WRPMC_H +#define _ASM_SW64_WRPMC_H + +/* Following commands are implemented on all CPUs */ +/* core4 */ +#define PMC_CMD_READ_PC0 5 +#define PMC_CMD_READ_PC1 6 +#define PMC_CMD_READ_PC2 7 +#define PMC_CMD_READ_PC3 8 +#define PMC_CMD_READ_PC4 9 +#define PMC_CMD_ENABLE 11 +#define PMC_CMD_DISABLE 12 +#define PMC_CMD_WRITE_BASE 16 + +#define PC_RAW_BASE 0x0 +#define PC_MAX 0x8D + +#define SW64_PERFCTRL_AM 0x0 +#define SW64_PERFCTRL_VM 0x3 +#define SW64_PERFCTRL_KM 0x5 +#define SW64_PERFCTRL_UM 0x7 + +/* pc0-4 events */ +#define SW64_PMU_INSTRUCTIONS 0x3 +#define SW64_PMU_BRANCH 0x4 +#define SW64_PMU_BRANCH_MISSES 0x5 +#define SW64_L1I_CACHE 0x6 +#define SW64_L1I_CACHE_MISSES 0x7 +#define SW64_PMU_CYCLE 0x30 +#define SW64_DTB 0x31 +#define SW64_DTB_MISSES 0x32 +#define SW64_L1D_CACHE 0x3D +#define SW64_L1D_CACHE_MISSES 0x3E +#define SW64_PMU_L2_REFERENCES 0x50 +#define SW64_PMU_L2_MISSES 0x53 + +#define PC_ALL_PM_SET 3 +#define MAX_HWEVENTS 5 +#define PMC_COUNT_MASK (-1UL) + +#define IACC_EN 0x4 +#define IMISC_EN 0x8 +#define RETIC_EN 0x10 +#define BRRETC_EN 0x20 +#define BRFAILC_EN 0x40 + +#endif /* _ASM_SW64_WRPMC_H */ diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 18a3cab6a9b6..6b5172f50056 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -28,7 +28,7 @@ obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_PM) += pm.o obj-$(CONFIG_SUSPEND) += suspend_asm.o suspend.o -obj-$(CONFIG_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o obj-$(CONFIG_HIBERNATION) += hibernate_asm.o hibernate.o obj-$(CONFIG_AUDIT) += audit.o obj-$(CONFIG_RELOCATABLE) += relocate.o @@ -36,6 +36,11 @@ obj-$(CONFIG_DEBUG_FS) += segvdbg.o unaligned.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_DEBUG_MATCH) += match.o +ifdef CONFIG_PERF_EVENTS +obj-$(CONFIG_SUBARCH_C3B) += perf_event.o +obj-$(CONFIG_SUBARCH_C4) += perf_event_c4.o +endif + ifndef CONFIG_PCI obj-y += pci-noop.o endif diff --git a/arch/sw_64/kernel/perf_callchain.c b/arch/sw_64/kernel/perf_callchain.c new file mode 100644 index 000000000000..0c0a4c180919 --- /dev/null +++ b/arch/sw_64/kernel/perf_callchain.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sw64 callchain support + * + * Copyright (C) 2023 SW64 Limited + */ +#include +#include + +#include + +bool valid_utext_addr(unsigned long addr) +{ + return addr >= current->mm->start_code && addr <= current->mm->end_code; +} + +bool valid_dy_addr(unsigned long addr) +{ + bool ret = false; + struct vm_area_struct *vma; + struct mm_struct *mm = current->mm; + + if (addr > TASK_SIZE || addr < TASK_UNMAPPED_BASE) + return ret; + vma = find_vma(mm, addr); + if (vma && vma->vm_start <= addr && (vma->vm_flags & VM_EXEC)) + ret = true; + return ret; +} + +#ifdef CONFIG_FRAME_POINTER +void perf_callchain_user(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + + struct stack_frame frame; + unsigned long __user *fp; + int err; + + perf_callchain_store(entry, regs->pc); + + fp = (unsigned long __user *)regs->regs[15]; + + while (entry->nr < entry->max_stack && + (unsigned long)fp < current->mm->start_stack) { + if (!access_ok(fp, sizeof(frame))) + break; + + pagefault_disable(); + err = __copy_from_user_inatomic(&frame, fp, sizeof(frame)); + pagefault_enable(); + + if (err) + break; + + if (valid_utext_addr(frame.return_address) || + valid_dy_addr(frame.return_address)) + perf_callchain_store(entry, frame.return_address); + else + break; + fp = (void __user *)frame.next_frame; + } +} +#else /* !CONFIG_FRAME_POINTER */ +void perf_callchain_user(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + unsigned long usp = current_user_stack_pointer(); + unsigned long user_addr; + int err; + + perf_callchain_store(entry, regs->pc); + + while (entry->nr < entry->max_stack && usp < current->mm->start_stack) { + if (!access_ok((const void __user *)usp, 8)) + break; + + pagefault_disable(); + err = __get_user(user_addr, (unsigned long *)usp); + pagefault_enable(); + + if (err) + break; + + if (valid_utext_addr(user_addr) || valid_dy_addr(user_addr)) + perf_callchain_store(entry, user_addr); + usp = usp + 8; + } +} +#endif/* CONFIG_FRAME_POINTER */ + +/* + * Gets called by walk_stackframe() for every stackframe. This will be called + * whist unwinding the stackframe and is like a subroutine return so we use + * the PC. + */ +static int callchain_trace(unsigned long pc, void *data) +{ + struct perf_callchain_entry_ctx *entry = data; + + perf_callchain_store(entry, pc); + return 0; +} + +void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + walk_stackframe(NULL, regs, callchain_trace, entry); +} + +/* + * Gets the perf_instruction_pointer and perf_misc_flags for guest os. + */ +#undef is_in_guest + +unsigned long perf_instruction_pointer(struct pt_regs *regs) +{ + if (perf_guest_state()) + return perf_guest_get_ip(); + + return instruction_pointer(regs); +} + +unsigned long perf_misc_flags(struct pt_regs *regs) +{ + unsigned int guest_state = perf_guest_state(); + int misc = 0; + + if (guest_state) { + if (guest_state & PERF_GUEST_USER) + misc |= PERF_RECORD_MISC_GUEST_USER; + else + misc |= PERF_RECORD_MISC_GUEST_KERNEL; + } else { + if (user_mode(regs)) + misc |= PERF_RECORD_MISC_USER; + else + misc |= PERF_RECORD_MISC_KERNEL; + } + + return misc; +} diff --git a/arch/sw_64/kernel/perf_event.c b/arch/sw_64/kernel/perf_event.c index 7544814615a8..c0aede9321b4 100644 --- a/arch/sw_64/kernel/perf_event.c +++ b/arch/sw_64/kernel/perf_event.c @@ -8,17 +8,6 @@ #include #include -/* For tracking PMCs and the hw events they monitor on each CPU. */ -struct cpu_hw_events { - /* - * Set the bit (indexed by the counter number) when the counter - * is used for an event. - */ - unsigned long used_mask[BITS_TO_LONGS(MAX_HWEVENTS)]; - /* Array of events current scheduled on this cpu. */ - struct perf_event *event[MAX_HWEVENTS]; -}; - DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); struct sw64_perf_event { @@ -629,101 +618,6 @@ static void sw64_perf_event_irq_handler(unsigned long idx, } } -bool valid_utext_addr(unsigned long addr) -{ - return addr >= current->mm->start_code && addr <= current->mm->end_code; -} - -bool valid_dy_addr(unsigned long addr) -{ - bool ret = false; - struct vm_area_struct *vma; - struct mm_struct *mm = current->mm; - - if (addr > TASK_SIZE || addr < TASK_UNMAPPED_BASE) - return ret; - vma = find_vma(mm, addr); - if (vma && vma->vm_start <= addr && (vma->vm_flags & VM_EXEC)) - ret = true; - return ret; -} - -#ifdef CONFIG_FRAME_POINTER -void perf_callchain_user(struct perf_callchain_entry_ctx *entry, - struct pt_regs *regs) -{ - - struct stack_frame frame; - unsigned long __user *fp; - int err; - - perf_callchain_store(entry, regs->pc); - - fp = (unsigned long __user *)regs->regs[15]; - - while (entry->nr < entry->max_stack && (unsigned long)fp < current->mm->start_stack) { - if (!access_ok(fp, sizeof(frame))) - break; - - pagefault_disable(); - err = __copy_from_user_inatomic(&frame, fp, sizeof(frame)); - pagefault_enable(); - - if (err) - break; - - if (valid_utext_addr(frame.return_address) || valid_dy_addr(frame.return_address)) - perf_callchain_store(entry, frame.return_address); - fp = (void __user *)frame.next_frame; - } -} -#else /* !CONFIG_FRAME_POINTER */ -void perf_callchain_user(struct perf_callchain_entry_ctx *entry, - struct pt_regs *regs) -{ - unsigned long usp = rdusp(); - unsigned long user_addr; - int err; - - perf_callchain_store(entry, regs->pc); - - while (entry->nr < entry->max_stack && usp < current->mm->start_stack) { - if (!access_ok((const void __user *)usp, 8)) - break; - - pagefault_disable(); - err = __get_user(user_addr, (unsigned long *)usp); - pagefault_enable(); - - if (err) - break; - - if (valid_utext_addr(user_addr) || valid_dy_addr(user_addr)) - perf_callchain_store(entry, user_addr); - usp = usp + 8; - } -} -#endif/* CONFIG_FRAME_POINTER */ - -/* - * Gets called by walk_stackframe() for every stackframe. This will be called - * whist unwinding the stackframe and is like a subroutine return so we use - * the PC. - */ -static int callchain_trace(unsigned long pc, void *data) -{ - struct perf_callchain_entry_ctx *entry = data; - - perf_callchain_store(entry, pc); - return 0; -} - -void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, - struct pt_regs *regs) -{ - walk_stackframe(NULL, regs, callchain_trace, entry); -} - /* * Init call to initialise performance events at kernel startup. */ @@ -755,35 +649,3 @@ int __init init_hw_perf_events(void) return 0; } early_initcall(init_hw_perf_events); - -/* - * Gets the perf_instruction_pointer and perf_misc_flags for guest os. - */ - -unsigned long perf_instruction_pointer(struct pt_regs *regs) -{ - if (perf_guest_state()) - return perf_guest_get_ip(); - - return instruction_pointer(regs); -} - -unsigned long perf_misc_flags(struct pt_regs *regs) -{ - unsigned int guest_state = perf_guest_state(); - int misc = 0; - - if (guest_state) { - if (guest_state & PERF_GUEST_USER) - misc |= PERF_RECORD_MISC_GUEST_USER; - else - misc |= PERF_RECORD_MISC_GUEST_KERNEL; - } else { - if (user_mode(regs)) - misc |= PERF_RECORD_MISC_USER; - else - misc |= PERF_RECORD_MISC_KERNEL; - } - - return misc; -} diff --git a/arch/sw_64/kernel/perf_event_c4.c b/arch/sw_64/kernel/perf_event_c4.c index 7a5e8deba0ee..302d3c65b61f 100644 --- a/arch/sw_64/kernel/perf_event_c4.c +++ b/arch/sw_64/kernel/perf_event_c4.c @@ -507,7 +507,7 @@ static void hw_perf_event_destroy(struct perf_event *event) /* Nothing to be done! */ } -static int __hw_perf_event_init(struct perf_event *event) +static void __hw_perf_event_init(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -523,8 +523,6 @@ static int __hw_perf_event_init(struct perf_event *event) hwc->last_period = hwc->sample_period; local64_set(&hwc->period_left, hwc->sample_period); } - - return 0; } /* @@ -632,6 +630,7 @@ static void sw64_perf_event_irq_handler(unsigned long perfmon_num, int idx; u64 val; + __this_cpu_inc(irq_pmi_count); cpuc = this_cpu_ptr(&cpu_hw_events); for (idx = 0; idx < sw64_pmu->num_pmcs; ++idx) { @@ -660,8 +659,6 @@ static void sw64_perf_event_irq_handler(unsigned long perfmon_num, if (perf_event_overflow(event, &data, regs)) sw64_pmu_stop(event, 0); } - - } /* @@ -669,12 +666,18 @@ static void sw64_perf_event_irq_handler(unsigned long perfmon_num, */ int __init init_hw_perf_events(void) { + pr_info("Performance Events: "); if (!supported_cpu()) { pr_info("Performance events: Unsupported CPU type!\n"); return 0; } - pr_info("Performance events: Supported CPU type!\n"); + if (is_in_guest()) { + pr_cont("No PMU driver, software events only.\n"); + return 0; + } + + pr_cont("Supported CPU type!\n"); /* Override performance counter IRQ vector */ -- Gitee From 79e5334a8e4e1542544aa557b1c7f4d90a074b83 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 18 Oct 2024 16:59:02 +0800 Subject: [PATCH 329/524] sw64: pci: fix incorrect pointer of struct pci_controller In any case, we should get pointer of struct pci_controller via function pci_bus_to_pci_controller(). Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/pci/pci-sysfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/pci/pci-sysfs.c b/arch/sw_64/pci/pci-sysfs.c index fd097d52b16a..4b9a29f2f348 100644 --- a/arch/sw_64/pci/pci-sysfs.c +++ b/arch/sw_64/pci/pci-sysfs.c @@ -58,6 +58,7 @@ static int pci_mmap_resource(struct kobject *kobj, struct vm_area_struct *vma, int sparse) { struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + struct pci_controller *hose = pci_bus_to_pci_controller(pdev->bus); struct resource *res = attr->private; enum pci_mmap_state mmap_type; struct pci_bus_region bar; @@ -79,7 +80,7 @@ static int pci_mmap_resource(struct kobject *kobj, vma->vm_pgoff += bar.start >> (PAGE_SHIFT - (sparse ? 5 : 0)); mmap_type = res->flags & IORESOURCE_MEM ? pci_mmap_mem : pci_mmap_io; - return hose_mmap_page_range(pdev->sysdata, vma, mmap_type, sparse); + return hose_mmap_page_range(hose, vma, mmap_type, sparse); } static int pci_mmap_resource_sparse(struct file *filp, struct kobject *kobj, -- Gitee From 7667324f15098b4e87c6b836d0603829f5c328e1 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Tue, 24 Sep 2024 10:54:54 +0800 Subject: [PATCH 330/524] sw64: kvm: add KVM_CAP_READONLY_MEM support When KVM_CAP_READONLY_MEM is enabled, we can access ROM device such as pflash device. But Writing pflash device needs to be treated as mmio write, so we adds this write request handling in kvm. Signed-off-by: Chen Wang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/kvm.h | 1 + arch/sw_64/kvm/mmu.c | 2 +- arch/sw_64/kvm/sw64.c | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/uapi/asm/kvm.h b/arch/sw_64/include/uapi/asm/kvm.h index 430e82acee3e..1949e106563d 100644 --- a/arch/sw_64/include/uapi/asm/kvm.h +++ b/arch/sw_64/include/uapi/asm/kvm.h @@ -19,6 +19,7 @@ enum SW64_KVM_IRQ { }; #define __KVM_HAVE_IRQ_LINE +#define __KVM_HAVE_READONLY_MEM #define KVM_NR_IRQCHIPS 1 diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 17383d721015..12e9dd43228f 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -1404,7 +1404,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, * needs emulation. */ - if (hva == KVM_HVA_ERR_BAD) { + if (hva == KVM_HVA_ERR_BAD || (write_fault && !writable)) { hargs->arg1 = fault_gpa | (hargs->arg1 & 0x1fffUL); ret = io_mem_abort(vcpu, run, hargs); goto out_unlock; diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index 99f7df5468fe..a2d2b1739dca 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -232,6 +232,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_IRQCHIP: case KVM_CAP_IOEVENTFD: case KVM_CAP_SYNC_MMU: + case KVM_CAP_READONLY_MEM: r = 1; break; case KVM_CAP_NR_VCPUS: -- Gitee From 47aec00f124f212dc446702586c304d4c80c01e0 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Tue, 25 Jun 2024 16:44:44 +0800 Subject: [PATCH 331/524] sw64: reset the CSR:PTBR_USR In the initalization phase, the value of CSR:PTBR_USR is pointed to ZERO_PAGE to avoid address substitutions that cause errors. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/csr.h | 6 ++++++ arch/sw_64/kernel/smp.c | 3 +++ arch/sw_64/mm/init.c | 3 +++ 3 files changed, 12 insertions(+) diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h index dac5b6a29b47..e205c3f1427e 100644 --- a/arch/sw_64/include/asm/csr.h +++ b/arch/sw_64/include/asm/csr.h @@ -108,6 +108,12 @@ static inline void update_ptbr_sys(unsigned long ptbr) sw64_write_csr_imb(ptbr, CSR_PTBR_SYS); } +static inline void update_ptbr_usr(unsigned long ptbr) +{ + imemb(); + sw64_write_csr_imb(ptbr, CSR_PTBR_USR); +} + #endif #else #define sw64_read_csr(x) (0) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index a48b1cb999c6..2c7cd4110ebb 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -154,6 +154,9 @@ void smp_callin(void) current->active_mm = &init_mm; /* update csr:ptbr */ update_ptbr_sys(virt_to_phys(init_mm.pgd)); +#ifdef CONFIG_SUBARCH_C4 + update_ptbr_usr(__pa_symbol(empty_zero_page)); +#endif if (IS_ENABLED(CONFIG_SUBARCH_C4) && is_in_host()) { nmi_stack_page = alloc_pages_node( diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index 29ee809c3ac5..c13abbbe26f3 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -109,6 +109,9 @@ switch_to_system_map(void) { memset(swapper_pg_dir, 0, PAGE_SIZE); update_ptbr_sys(virt_to_phys(swapper_pg_dir)); +#ifdef CONFIG_SUBARCH_C4 + update_ptbr_usr(__pa_symbol(empty_zero_page)); +#endif tbiv(); } -- Gitee From 178682d10b22cb15971ff4613fc846b00cd52e03 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 25 Jun 2024 09:19:54 +0800 Subject: [PATCH 332/524] sw64: fix irq work Send an IPI to the cpu itself when trying to raise a irq_work and call irq_work_run() when handling this IPI. Clear the abandoned code. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/irq_work.h | 12 ++++++++++++ arch/sw_64/kernel/smp.c | 13 +++++++++++++ arch/sw_64/kernel/time.c | 20 -------------------- 3 files changed, 25 insertions(+), 20 deletions(-) create mode 100644 arch/sw_64/include/asm/irq_work.h diff --git a/arch/sw_64/include/asm/irq_work.h b/arch/sw_64/include/asm/irq_work.h new file mode 100644 index 000000000000..f4d0cd857be7 --- /dev/null +++ b/arch/sw_64/include/asm/irq_work.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SW64_IRQ_WORK_H +#define __ASM_SW64_IRQ_WORK_H + +extern void arch_irq_work_raise(void); + +static inline bool arch_irq_work_has_interrupt(void) +{ + return IS_ENABLED(CONFIG_SMP); +} + +#endif /* __ASM_SW64_IRQ_WORK_H */ diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 2c7cd4110ebb..833b99ef66e5 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ enum ipi_message_type { IPI_RESCHEDULE, IPI_CALL_FUNC, IPI_CPU_STOP, + IPI_IRQ_WORK, }; int smp_num_cpus = 1; /* Number that came online. */ @@ -589,6 +591,13 @@ static void ipi_cpu_stop(int cpu) wait_for_interrupt(); } +#ifdef CONFIG_IRQ_WORK +void arch_irq_work_raise(void) +{ + send_ipi_message(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); +} +#endif + void handle_ipi(struct pt_regs *regs) { int cpu = smp_processor_id(); @@ -618,6 +627,10 @@ void handle_ipi(struct pt_regs *regs) ipi_cpu_stop(cpu); break; + case IPI_IRQ_WORK: + irq_work_run(); + break; + default: pr_crit("Unknown IPI on CPU %d: %lu\n", cpu, which); break; diff --git a/arch/sw_64/kernel/time.c b/arch/sw_64/kernel/time.c index 6eaf8fab4f28..c6957d70a551 100644 --- a/arch/sw_64/kernel/time.c +++ b/arch/sw_64/kernel/time.c @@ -25,26 +25,6 @@ EXPORT_SYMBOL(rtc_lock); unsigned long est_cycle_freq; -#ifdef CONFIG_IRQ_WORK - -DEFINE_PER_CPU(u8, irq_work_pending); - -#define set_irq_work_pending_flag() __this_cpu_write(irq_work_pending, 1) -#define test_irq_work_pending() __this_cpu_read(irq_work_pending) -#define clear_irq_work_pending() __this_cpu_write(irq_work_pending, 0) - -void arch_irq_work_raise(void) -{ - set_irq_work_pending_flag(); -} - -#else /* CONFIG_IRQ_WORK */ - -#define test_irq_work_pending() 0 -#define clear_irq_work_pending() - -#endif /* CONFIG_IRQ_WORK */ - void __init time_init(void) { -- Gitee From f1cea82d8aa3d8bb81e0c40c65c06ab2775882ec Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 3 Jul 2024 16:48:09 +0800 Subject: [PATCH 333/524] sw64: bpf: fix BPF_CALL address Use the new bpf_jit_get_func_addr() helper for properly fetching the address through prog->aux->func[off]->bpf_func. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/net/bpf_jit_comp.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index 31202dd0f9cf..64ed06693924 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -661,7 +661,8 @@ static int add_exception_handler(const struct bpf_insn *insn, * >0 - successfully JITed a 16-byte eBPF instruction. * <0 - failed to JIT. */ -static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) +static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, + bool extra_pass) { const u8 code = insn->code; u8 dst = bpf2sw64[insn->dst_reg]; @@ -672,7 +673,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) const s32 imm = insn->imm; const int bpf_idx = insn - ctx->prog->insnsi; s32 jmp_offset; - u64 func; struct bpf_insn insn1; u64 imm64; int ret; @@ -1142,13 +1142,18 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) break; case BPF_JMP | BPF_CALL: - func = (u64)__bpf_call_base + imm; - if ((func & ~(KERNEL_IMAGE_SIZE - 1)) != __START_KERNEL_map) - /* calling bpf program, switch to vmalloc addr */ - func = (func & U32_MAX) | VMALLOC_START; + { + bool fixed; + u64 func; + + ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &func, &fixed); + if (ret < 0) + return ret; + emit_sw64_ldu64(SW64_BPF_REG_PV, func, ctx); emit(SW64_BPF_CALL(SW64_BPF_REG_RA, SW64_BPF_REG_PV), ctx); break; + } case BPF_JMP | BPF_TAIL_CALL: if (emit_bpf_tail_call(ctx)) @@ -1269,7 +1274,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) return 0; } -static int build_body(struct jit_ctx *ctx) +static int build_body(struct jit_ctx *ctx, bool extra_pass) { const struct bpf_prog *prog = ctx->prog; int i; @@ -1280,7 +1285,7 @@ static int build_body(struct jit_ctx *ctx) if (ctx->image == NULL) ctx->insn_offset[i] = ctx->idx; - ret = build_insn(insn, ctx); + ret = build_insn(insn, ctx, extra_pass); if (ret < 0) return ret; while (ret > 0) { @@ -1371,7 +1376,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) /* Fake pass to fill in ctx->offset. */ build_prologue(&ctx, was_classic); - if (build_body(&ctx)) { + if (build_body(&ctx, extra_pass)) { prog = orig_prog; goto out_off; } @@ -1405,7 +1410,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) build_prologue(&ctx, was_classic); - if (build_body(&ctx)) { + if (build_body(&ctx, extra_pass)) { bpf_jit_binary_free(header); prog = orig_prog; goto out_off; -- Gitee From 56c6354be339f1f61c3c04ba4f8bc67d602ba466 Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Mon, 8 Jul 2024 17:19:42 +0800 Subject: [PATCH 334/524] sw64: kexec: fix kernel crashdump bugs This patch: - Remove the max_addr limit for memblock_alloc_base(), because alloc can now be performed in any memory space. - Delete kernel_restart_prepare(NULL) in order to prevent both 0 and panic cores from entering machine_kexec(). - fix the check for crash_base. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/machine_kexec.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/kernel/machine_kexec.c b/arch/sw_64/kernel/machine_kexec.c index f0a2338ddcbd..4ef59d6022fc 100644 --- a/arch/sw_64/kernel/machine_kexec.c +++ b/arch/sw_64/kernel/machine_kexec.c @@ -54,7 +54,7 @@ void __init kexec_control_page_init(void) phys_addr_t addr; addr = memblock_phys_alloc_range(KEXEC_CONTROL_PAGE_SIZE, PAGE_SIZE, - 0, KTEXT_MAX); + 0, 0); kexec_control_page = (void *)(__START_KERNEL_map + addr); } @@ -105,8 +105,8 @@ void __init reserve_crashkernel(void) pr_warn("Add crash kernel area [mem %#018llx-%#018llx] to memmap region failed.\n", crash_base, crash_base + crash_size - 1); - if (crash_base >= KERNEL_IMAGE_SIZE) - pr_warn("Crash base should be less than %#x\n", KERNEL_IMAGE_SIZE); + if (crash_base < PCI_LEGACY_IO_SIZE) + pr_warn("Crash base should be greater than or equal to %#lx\n", PCI_LEGACY_IO_SIZE); crashk_res.start = crash_base; crashk_res.end = crash_base + crash_size - 1; @@ -192,7 +192,6 @@ void machine_crash_shutdown(struct pt_regs *regs) cpu = smp_processor_id(); local_irq_disable(); - kernel_restart_prepare(NULL); atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1); smp_call_function(machine_crash_nonpanic_core, NULL, false); msecs = 1000; /* Wait at most a second for the other cpus to stop */ -- Gitee From 5adb6315b2c1f1a7855edfcdad2fc7a1969fca73 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Tue, 25 Jun 2024 15:23:18 +0800 Subject: [PATCH 335/524] sw64: add __vdso_getcpu support This patch implement getcpu() through vDSO. For C3, there is no way to get logical cpu id from userspace, so we have to maintain mappings from whami to cpu id and node id. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hmcall.h | 2 ++ arch/sw_64/include/asm/vdso.h | 7 +++++ arch/sw_64/kernel/hmcall.c | 9 ++++++ arch/sw_64/kernel/vdso.c | 37 ++++++++++++++++++++++++ arch/sw_64/kernel/vdso/Makefile | 4 ++- arch/sw_64/kernel/vdso/vdso.lds.S | 1 + arch/sw_64/kernel/vdso/vgetcpu.c | 48 +++++++++++++++++++++++++++++++ 7 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 arch/sw_64/kernel/vdso/vgetcpu.c diff --git a/arch/sw_64/include/asm/hmcall.h b/arch/sw_64/include/asm/hmcall.h index 574144eab22b..60f68e31c568 100644 --- a/arch/sw_64/include/asm/hmcall.h +++ b/arch/sw_64/include/asm/hmcall.h @@ -54,6 +54,7 @@ /* Following will be deprecated from user level invocation */ #define HMC_rwreg 0x87 #define HMC_sz_uflush 0xA8 +#define HMC_uwhami 0xA0 #define HMC_longtime 0xB1 #ifdef __KERNEL__ @@ -162,6 +163,7 @@ __CALL_HMC_VOID(wrktp); __CALL_HMC_R0(rdps, unsigned long); __CALL_HMC_R0(rvpcr, unsigned long); +__CALL_HMC_R0(uwhami, unsigned long); __CALL_HMC_R0(rdusp, unsigned long); __CALL_HMC_W1(wrusp, unsigned long); diff --git a/arch/sw_64/include/asm/vdso.h b/arch/sw_64/include/asm/vdso.h index 7a2e23c648f3..3f6ba2d72b62 100644 --- a/arch/sw_64/include/asm/vdso.h +++ b/arch/sw_64/include/asm/vdso.h @@ -50,6 +50,13 @@ struct vdso_data { s32 tz_minuteswest; s32 tz_dsttime; u32 seq_count; +#ifdef CONFIG_SUBARCH_C3B + u8 vdso_whami_to_cpu[NR_CPUS]; + u8 vdso_whami_to_node[NR_CPUS]; +#endif +#ifdef CONFIG_SUBARCH_C4 + u8 vdso_cpu_to_node[NR_CPUS]; +#endif }; static inline unsigned long get_vdso_base(void) diff --git a/arch/sw_64/kernel/hmcall.c b/arch/sw_64/kernel/hmcall.c index d2054a930bd7..50aaa2b9b5b4 100644 --- a/arch/sw_64/kernel/hmcall.c +++ b/arch/sw_64/kernel/hmcall.c @@ -108,6 +108,14 @@ static inline void fixup_wrusp(void) entry[1] = 0x1ee00000; /* pri_ret $23 */ } +static inline void fixup_uwhami(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(uwhami)); + + entry[0] = 0x94161078; /* pri_ldl/p r0, VC__WHAMI(vcpucb) */ + entry[1] = 0x1ef00000; /* pri_ret/b $23 */ +} + void __init fixup_hmcall(void) { #if defined(CONFIG_SUBARCH_C3B) @@ -119,6 +127,7 @@ void __init fixup_hmcall(void) fixup_wrktp(); fixup_rdusp(); fixup_wrusp(); + fixup_uwhami(); imemb(); #endif } diff --git a/arch/sw_64/kernel/vdso.c b/arch/sw_64/kernel/vdso.c index 15f1f244a82f..0d0ca5d0a38b 100644 --- a/arch/sw_64/kernel/vdso.c +++ b/arch/sw_64/kernel/vdso.c @@ -36,6 +36,41 @@ struct vdso_data *vdso_data = &vdso_data_store.data; static struct vm_special_mapping vdso_spec[2]; +#ifdef CONFIG_SUBARCH_C3B +#define V_NODE_SHIFT 6 +static void init_cpu_map(void) +{ + + int i, whami, domain; + unsigned int shift, mask; + + if (is_in_host()) + shift = DOMAIN_ID_SHIFT; + else + shift = V_NODE_SHIFT; + mask = (1 << shift) - 1; + + vdso_data_write_begin(vdso_data); + for (i = 0; i < num_possible_cpus(); i++) { + domain = cpu_to_rcid(i) >> shift; + whami = (domain << DOMAIN_ID_SHIFT) | (cpu_to_rcid(i) & mask); + vdso_data->vdso_whami_to_cpu[whami] = i; + vdso_data->vdso_whami_to_node[whami] = domain; + } + vdso_data_write_end(vdso_data); +} +#else +static void init_cpu_map(void) +{ + int i; + + vdso_data_write_begin(vdso_data); + for (i = 0; i < num_possible_cpus(); i++) + vdso_data->vdso_cpu_to_node[i] = (cpu_to_rcid(i) & DOMAIN_ID_MASK) >> DOMAIN_ID_SHIFT; + vdso_data_write_end(vdso_data); +} +#endif + static int __init vdso_init(void) { int i; @@ -73,6 +108,8 @@ static int __init vdso_init(void) .pages = &vdso_pagelist[1], }; + init_cpu_map(); + return 0; } arch_initcall(vdso_init); diff --git a/arch/sw_64/kernel/vdso/Makefile b/arch/sw_64/kernel/vdso/Makefile index 190cc345dbb9..e5640abbd727 100644 --- a/arch/sw_64/kernel/vdso/Makefile +++ b/arch/sw_64/kernel/vdso/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 # Symbols present in the vdso -vdso-syms = rt_sigreturn gettimeofday +vdso-syms = rt_sigreturn gettimeofday getcpu # Files to link into the vdso obj-vdso = $(patsubst %, v%.o, $(vdso-syms)) @@ -17,12 +17,14 @@ CPPFLAGS_vdso.lds += -P -C -U$(ARCH) CFLAGS_REMOVE_vdso.o = -pg CFLAGS_REMOVE_vrt_sigreturn.o = -pg CFLAGS_REMOVE_vgettimeofday.o = -pg +CFLAGS_REMOVE_vgetcpu = -pg ifdef CONFIG_FEEDBACK_COLLECT # vDSO code runs in userspace, not collecting feedback data. CFLAGS_REMOVE_vdso.o = -ffeedback-generate CFLAGS_REMOVE_vrt_sigreturn.o = -ffeedback-generate CFLAGS_REMOVE_vgettimeofday.o = -ffeedback-generate +CFLAGS_REMOVE_vgetcpu.o = -ffeedback-generate endif # Disable gcov profiling for VDSO code diff --git a/arch/sw_64/kernel/vdso/vdso.lds.S b/arch/sw_64/kernel/vdso/vdso.lds.S index de1782ccb7b6..08134a86f66c 100644 --- a/arch/sw_64/kernel/vdso/vdso.lds.S +++ b/arch/sw_64/kernel/vdso/vdso.lds.S @@ -84,6 +84,7 @@ VERSION __vdso_rt_sigreturn; __vdso_gettimeofday; __vdso_clock_gettime; + __vdso_getcpu; local: *; }; } diff --git a/arch/sw_64/kernel/vdso/vgetcpu.c b/arch/sw_64/kernel/vdso/vgetcpu.c new file mode 100644 index 000000000000..d17f1b16ccb8 --- /dev/null +++ b/arch/sw_64/kernel/vdso/vgetcpu.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or + * NON INFRINGEMENT. See the GNU General Public License for + * more details. + */ + +#include +#include +#include +#include +#include + +static void __getcpu(unsigned int *cpu, unsigned int *node, + const struct vdso_data *data) +{ + unsigned int cpuid; +#ifdef CONFIG_SUBARCH_C3B + asm volatile ("sys_call %1\n" + "mov $0, %0\n" + : "=&r"(cpuid) + : "i"(HMC_uwhami)); + *cpu = data->vdso_whami_to_cpu[cpuid]; + *node = data->vdso_whami_to_node[cpuid]; +#else + asm volatile ("csrr %0, %1" : "=&r"(cpuid) : "i"(CSR_SOFTCID)); + *cpu = cpuid; + *node = data->vdso_cpu_to_node[*cpu]; +#endif +} + + +long __vdso_getcpu(unsigned int *cpu, unsigned int *node, + struct getcpu_cache *unused) +{ + const struct vdso_data *data = get_vdso_data(); + + __getcpu(cpu, node, data); + return 0; +} + -- Gitee From 0fc0c4a8fd9c5b514e4cdd76fa611d3570bc31da Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Tue, 23 Jul 2024 14:45:05 +0800 Subject: [PATCH 336/524] sw64: remove unnecessary parameter passing Compiler has already prepared parameters for reboot_code_buffer(), so there is no need to get parameters through these four kexec_args. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/relocate_kernel.S | 38 ----------------------------- 1 file changed, 38 deletions(-) diff --git a/arch/sw_64/kernel/relocate_kernel.S b/arch/sw_64/kernel/relocate_kernel.S index f1a160636212..a4b0d27778b9 100644 --- a/arch/sw_64/kernel/relocate_kernel.S +++ b/arch/sw_64/kernel/relocate_kernel.S @@ -16,11 +16,6 @@ relocate_new_kernel: .prologue 0 - ldl a0, arg0 - ldl a1, arg1 - ldl a2, arg2 - ldl a3, arg3 - ldl s0, kexec_indirection_page ldl s1, kexec_start_address @@ -100,10 +95,6 @@ done: .globl kexec_smp_wait .ent kexec_smp_wait kexec_smp_wait: - ldl a0, s_arg0 - ldl a1, s_arg1 - ldl a2, s_arg2 - ldl a3, s_arg3 ldl s1, kexec_start_address /* Non-relocated address works for args and kexec_start_address (old @@ -124,35 +115,6 @@ kexec_smp_wait: jmp ra, (s1) .end kexec_smp_wait .size kexec_smp_wait, .-kexec_smp_wait -#endif - - .align 3 - - /* All parameters to new kernel are passed in registers a0-a3. - * kexec_args[0..3] are uses to prepare register values. - */ - -kexec_args: - .globl kexec_args -arg0: .quad 0x0 -arg1: .quad 0x0 -arg2: .quad 0x0 -arg3: .quad 0x0 - .size kexec_args, 8*4 - -#ifdef CONFIG_CRASH_SMP - /* - * Secondary CPUs may have different kernel parameters in - * their registers a0-a3. secondary_kexec_args[0..3] are used - * to prepare register values. - */ -secondary_kexec_args: - .globl secondary_kexec_args -s_arg0: .quad 0x0 -s_arg1: .quad 0x0 -s_arg2: .quad 0x0 -s_arg3: .quad 0x0 - .size secondary_kexec_args, 8*4 kexec_flag: .quad 0x1 -- Gitee From ee32d67ed8d13c8dcdcdc2972039833a48fcfee7 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 4 Jul 2024 17:29:43 +0800 Subject: [PATCH 337/524] sw64: improve lib performance for new archs Always use cached memory write instead of non-cache memory write for architectures other than the old C3B. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/lib/copy_simd_align_template.S | 6 ++++++ arch/sw_64/lib/set_simd_align_template.S | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/arch/sw_64/lib/copy_simd_align_template.S b/arch/sw_64/lib/copy_simd_align_template.S index fbb5912e1213..aa648b7c822a 100644 --- a/arch/sw_64/lib/copy_simd_align_template.S +++ b/arch/sw_64/lib/copy_simd_align_template.S @@ -15,7 +15,9 @@ * */ +#ifdef CONFIG_SUBARCH_C3B #define NC_STORE_THRESHOLD 2048 +#endif #define SAVE_SIMD_REGS \ ldi $sp, -0x60($sp); \ @@ -97,6 +99,7 @@ $dest_aligned_32: $prep_simd_loop: SAVE_SIMD_REGS +#ifdef CONFIG_SUBARCH_C3B ldi $1, NC_STORE_THRESHOLD($31) cmple $18, $1, $1 bne $1, $simd_loop @@ -114,6 +117,7 @@ $simd_loop_nc: beq $1, $simd_loop_nc memb # required for _nc store instructions br $31, $simd_loop_end +#endif .align 4 $simd_loop: @@ -207,6 +211,7 @@ $prep_simd_u_loop: ifmovd $1, $f1 ifmovd $2, $f2 FIXUP_LDST( vldd $f4, 0($3) ) +#ifdef CONFIG_SUBARCH_C3B ldi $1, NC_STORE_THRESHOLD($31) cmple $18, $1, $1 bne $1, $simd_u_loop @@ -230,6 +235,7 @@ $simd_u_loop_nc: beq $1, $simd_u_loop_nc memb # required for _nc store instructions br $31, $simd_u_loop_end +#endif .align 4 $simd_u_loop: diff --git a/arch/sw_64/lib/set_simd_align_template.S b/arch/sw_64/lib/set_simd_align_template.S index fa77cacf684f..6a8b4118f2a0 100644 --- a/arch/sw_64/lib/set_simd_align_template.S +++ b/arch/sw_64/lib/set_simd_align_template.S @@ -13,7 +13,9 @@ * */ +#ifdef CONFIG_SUBARCH_C3B #define NC_STORE_THRESHOLD 2048 +#endif #define SAVE_SIMD_REGS \ ldi $sp, -0x40($sp); \ @@ -65,6 +67,7 @@ $prep_simd_loop: SAVE_SIMD_REGS ifmovd $17, $f1 vcpyf $f1, $f1 +#ifdef CONFIG_SUBARCH_C3B ldi $1, NC_STORE_THRESHOLD($31) cmple $18, $1, $1 bne $1, $simd_loop @@ -79,6 +82,7 @@ $simd_loop_nc: beq $1, $simd_loop_nc memb # required for _nc store instructions br $31, $simd_loop_end +#endif .align 3 $simd_loop: -- Gitee From 494164a25d54c526bebd072b1078aa25937e8743 Mon Sep 17 00:00:00 2001 From: Xu Chenjiao Date: Fri, 26 Jul 2024 11:22:20 +0000 Subject: [PATCH 338/524] sw64: lpc: fix ast2400 driver error for C4 Port ast2400 driver to work around the hardware flaws of Low Pin Count controller of C4, since the address passed in is 48 bit length, so we change address type from unsigned int to unsigned long. Besides the resource index should be 0 to get the valid resource of super I/O. Signed-off-by: Xu Chenjiao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/ast2400.h | 19 +++++++++++++------ drivers/mfd/lpc_sunway.c | 4 ++++ drivers/mfd/sunway_ast2400.c | 6 +++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/arch/sw_64/include/asm/ast2400.h b/arch/sw_64/include/asm/ast2400.h index 5f4cc84ff3a8..30d45eff62a2 100644 --- a/arch/sw_64/include/asm/ast2400.h +++ b/arch/sw_64/include/asm/ast2400.h @@ -49,9 +49,17 @@ #define SUPERIO_PNP_PORT 0x2E #define SUPERIO_CHIPID 0xC333 +#ifdef CONFIG_SUBARCH_C3B +#define PORT_OFFSET 0 +#endif + +#ifdef CONFIG_SUBARCH_C4 +#define PORT_OFFSET 12 +#endif + struct device_operations; typedef struct pnp_device { - unsigned int port; + unsigned long port; unsigned int device; struct device_operations *ops; @@ -100,8 +108,8 @@ typedef struct superio_ast2400_device { struct device *dev; const char *name; unsigned int enabled : 1; /* set if we should enable the device */ - unsigned int superio_ast2400_efir; /* extended function index register */ - unsigned int superio_ast2400_efdr; /* extended function data register */ + unsigned long superio_ast2400_efir; /* extended function index register */ + unsigned long superio_ast2400_efdr; /* extended function data register */ struct chip_operations *chip_ops; const void *chip_info; } *superio_device_t; @@ -136,19 +144,18 @@ static inline void pnp_exit_conf_mode(device_t dev) static inline u8 pnp_read_config(device_t dev, u8 reg) { outb(reg, dev->port); - return inb(dev->port + 1); + return inb(dev->port + (1 << PORT_OFFSET)); } static inline void pnp_write_config(device_t dev, u8 reg, u8 value) { outb(reg, dev->port); - outb(value, dev->port + 1); + outb(value, dev->port + (1 << PORT_OFFSET)); } static inline void pnp_set_logical_device(device_t dev) { pnp_write_config(dev, 0x07, dev->device & 0xff); -// pnp_write_config(dev, 0x07, 0x3); } static inline void pnp_set_enable(device_t dev, int enable) diff --git a/drivers/mfd/lpc_sunway.c b/drivers/mfd/lpc_sunway.c index a70971e7912d..ebad6f202145 100644 --- a/drivers/mfd/lpc_sunway.c +++ b/drivers/mfd/lpc_sunway.c @@ -78,6 +78,10 @@ struct lpc_chip3_adapter { static struct resource superio_chip3_resources[] = { { .flags = IORESOURCE_IO, +#ifdef CONFIG_SUBARCH_C4 + .start = LPC_MEM_IO, + .end = LPC_MEM_IO + 0x10, +#endif } }; diff --git a/drivers/mfd/sunway_ast2400.c b/drivers/mfd/sunway_ast2400.c index 4e1e3e3d21e2..5fd7fc951c96 100644 --- a/drivers/mfd/sunway_ast2400.c +++ b/drivers/mfd/sunway_ast2400.c @@ -163,7 +163,7 @@ static int superio_ast2400_probe(struct platform_device *pdev) return err; } - res = platform_get_resource(pdev, IORESOURCE_IO, 1); + res = platform_get_resource(pdev, IORESOURCE_IO, 0); if (res) { physaddr = res->start; dev_info(&pdev->dev, "request memory region %pR\n", res); @@ -171,8 +171,8 @@ static int superio_ast2400_probe(struct platform_device *pdev) superio_device->dev = &pdev->dev; superio_device->enabled = 1; - superio_device->superio_ast2400_efir = physaddr + SUPERIO_PNP_PORT; - superio_device->superio_ast2400_efdr = physaddr + SUPERIO_PNP_PORT + 1; + superio_device->superio_ast2400_efir = physaddr + (SUPERIO_PNP_PORT << PORT_OFFSET); + superio_device->superio_ast2400_efdr = physaddr + ((SUPERIO_PNP_PORT + 1) << PORT_OFFSET); superio_uart0_irq = platform_get_irq_byname(pdev, "uart0_irq"); superio_uart1_irq = platform_get_irq_byname(pdev, "uart1_irq"); -- Gitee From 5ace5d8ecf3804afe70c827833b4cd2ebbc8845a Mon Sep 17 00:00:00 2001 From: Zhi Tongze Date: Thu, 22 Aug 2024 09:22:01 +0800 Subject: [PATCH 339/524] sw64: select different MUX to read cpu frequency MUX will not be fixed to use pll2, so we need to check current MUX before reading CPU frequency. Signed-off-by: Zhi Tongze Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpufreq.h | 4 +++- arch/sw_64/platform/cpufreq.c | 12 ++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/include/asm/cpufreq.h b/arch/sw_64/include/asm/cpufreq.h index 93244b3b18c6..0d3e794b16ea 100644 --- a/arch/sw_64/include/asm/cpufreq.h +++ b/arch/sw_64/include/asm/cpufreq.h @@ -51,7 +51,8 @@ struct clk { #define CORE_CLK2_VALID (0x1UL << 33) #define CORE_CLK2_RESET (0x1UL << 34) #define CORE_CLK2_LOCK (0x1UL << 35) -#define CORE_PLL0_CFG_SHIFT 20 +#define CORE_PLL0_CFG_SHIFT 4 +#define CORE_PLL1_CFG_SHIFT 20 #define CORE_PLL2_CFG_SHIFT 36 #define CORE_PLL2_CFG_MASK 0x1f #define STARTUP_RATE (2000UL * 1000 * 1000) @@ -65,6 +66,7 @@ struct clk { #define CORE_CLK2_RESET (0x1UL << 16) #define CORE_CLK2_LOCK (0x1UL << 17) #define CORE_PLL0_CFG_SHIFT 4 +#define CORE_PLL1_CFG_SHIFT 11 #define CORE_PLL2_CFG_SHIFT 18 #define CORE_PLL2_CFG_MASK 0xf #define STARTUP_RATE (2400UL * 1000 * 1000) diff --git a/arch/sw_64/platform/cpufreq.c b/arch/sw_64/platform/cpufreq.c index 0d552d9a753d..e18201069f67 100644 --- a/arch/sw_64/platform/cpufreq.c +++ b/arch/sw_64/platform/cpufreq.c @@ -136,12 +136,20 @@ EXPORT_SYMBOL(sw64_clk_get); unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy) { - int i; + int i, clu_lv1_sel; u64 val; void __iomem *spbu_base = misc_platform_get_spbu_base(0); struct cpufreq_frequency_table *ft = policy->freq_table; - val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL2_CFG_SHIFT; + clu_lv1_sel = (readq(spbu_base + OFFSET_CLU_LV1_SEL) >> 2) & 0x3; + + if (clu_lv1_sel == 0) + val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL0_CFG_SHIFT; + else if (clu_lv1_sel == 2) + val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL1_CFG_SHIFT; + else + val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL2_CFG_SHIFT; + val &= CORE_PLL2_CFG_MASK; for (i = 0; ft[i].frequency != CPUFREQ_TABLE_END; i++) { -- Gitee From 90f86d07f08de4c43fa351a672cf2a200fb7b34e Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Tue, 27 Aug 2024 08:54:28 +0800 Subject: [PATCH 340/524] sw64: add unaligned access handling code for SIMD in kernel mode (5.10) The current kernel does not support unaligned access handling for SIMD. Unaligned access handling code for SIMD in kernel mode is added. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/traps.c | 528 +++++++++++++++++++++++++++++++++++++- 1 file changed, 527 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index e54db9a41be6..bc7bb4c04456 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -392,7 +392,8 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, { long error, disp; unsigned int insn, fncode, rb; - unsigned long tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + unsigned long tmp, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, vb; + unsigned long fp[4]; unsigned long pc = regs->pc - 4; /* @@ -402,6 +403,531 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, */ switch (opcode) { + + case 0x0c: /* vlds */ + if ((unsigned long)va<<61 == 0) { + __asm__ __volatile__( + "1: ldl %1, 0(%5)\n" + "2: ldl %2, 8(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + + sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); + + return; + } else { + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + + tmp1 = tmp1 | tmp4; + tmp2 = tmp5 | tmp3; + + sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); + + return; + } + + case 0x0d: /* vldd */ + if ((unsigned long)va<<61 == 0) { + __asm__ __volatile__( + "1: ldl %1, 0(%5)\n" + "2: ldl %2, 8(%5)\n" + "3: ldl %3, 16(%5)\n" + "4: ldl %4, 24(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi %4, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + + sw64_write_simd_fp_reg_d(reg, tmp1, tmp2, tmp3, tmp4); + + return; + } else { + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + + tmp7 = tmp1 | tmp4; //f0 + tmp8 = tmp5 | tmp3; //f1 + + vb = ((unsigned long)(va))+16; + + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(vb), "0"(0)); + + if (error) + goto got_exception; + + tmp = tmp1 | tmp4; // f2 + tmp2 = tmp5 | tmp3; // f3 + + sw64_write_simd_fp_reg_d(reg, tmp7, tmp8, tmp, tmp2); + return; + } + + case 0x0e: /* vsts */ + sw64_read_simd_fp_m_s(reg, fp); + if ((unsigned long)va<<61 == 0) { + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); + + if (error) + goto got_exception; + + return; + } else { + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(fp[0]), "0"(0)); + + if (error) + goto got_exception; + + + vb = ((unsigned long)va) + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[1]), "0"(0)); + + if (error) + goto got_exception; + + return; + } + + case 0x0f: /* vstd */ + sw64_read_simd_fp_m_d(reg, fp); + if ((unsigned long)va<<61 == 0) { + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); + + if (error) + goto got_exception; + + vb = ((unsigned long)va)+16; + + + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(vb), "r"(fp[2]), "r"(fp[3]), "0"(0)); + + if (error) + goto got_exception; + + return; + } else { + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(fp[0]), "0"(0)); + + if (error) + goto got_exception; + + vb = ((unsigned long)va) + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[1]), "0"(0)); + + if (error) + goto got_exception; + + vb = vb + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[2]), "0"(0)); + + if (error) + goto got_exception; + + vb = vb + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[3]), "0"(0)); + + if (error) + goto got_exception; + + return; + } + case 0x1e: insn = *(unsigned int *)pc; fncode = (insn >> 12) & 0xf; -- Gitee From df5b7a18ec269ac3c17360b8b9cb736552df6938 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Tue, 27 Aug 2024 17:19:55 +0800 Subject: [PATCH 341/524] sw64: fix the register bug in do_entUna The position is not reserved for $31 in pt_regs. An overflow will occur during storing in do_entUna, which could corrupt PC. A judgement statement is used to set reg_addr, and (reg->regs[reg]) has been changed to *reg_addr. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/traps.c | 47 ++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index bc7bb4c04456..9a9b43c13286 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -386,6 +386,16 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->pc); } +#define OP_INT_MASK (1L << 0x22 | 1L << 0x2a | /* ldw stw */ \ + 1L << 0x23 | 1L << 0x2b | /* ldl stl */ \ + 1L << 0x21 | 1L << 0x29 | /* ldhu sth */ \ + 1L << 0x20 | 1L << 0x28) /* ldbu stb */ + +#define FN_INT_MASK (1L << 0x0 | 1L << 0x6 | /* ldbu_a stb_a */ \ + 1L << 0x1 | 1L << 0x7 | /* ldhu_a sth_a */ \ + 1L << 0x2 | 1L << 0x8 | /* ldw_a stw_a */ \ + 1L << 0x3 | 1L << 0x9) /* ldl_a stl_a */ + asmlinkage void do_entUna(void *va, unsigned long opcode, unsigned long reg, struct pt_regs *regs) @@ -394,8 +404,23 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, unsigned int insn, fncode, rb; unsigned long tmp, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, vb; unsigned long fp[4]; + unsigned long fake_reg, *reg_addr = &fake_reg; unsigned long pc = regs->pc - 4; + insn = *(unsigned int *)pc; + fncode = (insn >> 12) & 0xf; + + if (((1L << opcode) & OP_INT_MASK) || + ((opcode == 0x1e) && ((1L << fncode) & FN_INT_MASK))) { + /* it's an integer load/store */ + if (reg < 31) { + reg_addr = ®s->regs[reg]; + } else { + /* zero "register" */ + fake_reg = 0; + } + } + /* * We don't want to use the generic get/put unaligned macros as * we want to trap exceptions. Only if we actually get an @@ -1017,7 +1042,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(regs->regs[reg]), "0"(0)); + : "r"(va), "r"(*reg_addr), "0"(0)); if (error) goto got_exception; @@ -1050,7 +1075,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(regs->regs[reg]), "0"(0)); + : "r"(va), "r"(*reg_addr), "0"(0)); if (error) goto got_exception; @@ -1103,7 +1128,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(regs->regs[reg]), "0"(0)); + : "r"(va), "r"(*reg_addr), "0"(0)); if (error) goto got_exception; @@ -1190,7 +1215,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(regs->regs[reg]), "0"(0)); + : "r"(va), "r"(*reg_addr), "0"(0)); if (error) goto got_exception; @@ -1222,7 +1247,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(regs->regs[reg]), "0"(0)); + : "r"(va), "r"(*reg_addr), "0"(0)); if (error) goto got_exception; @@ -1274,7 +1299,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(regs->regs[reg]), "0"(0)); + : "r"(va), "r"(*reg_addr), "0"(0)); if (error) goto got_exception; @@ -1322,16 +1347,6 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, * uses them as temporary storage for integer memory to memory copies. * However, we need to deal with stt/ldt and sts/lds only. */ -#define OP_INT_MASK (1L << 0x22 | 1L << 0x2a | /* ldw stw */ \ - 1L << 0x23 | 1L << 0x2b | /* ldl stl */ \ - 1L << 0x21 | 1L << 0x29 | /* ldhu sth */ \ - 1L << 0x20 | 1L << 0x28) /* ldbu stb */ - -#define FN_INT_MASK (1L << 0x0 | 1L << 0x6 | /* ldbu_a stb_a */ \ - 1L << 0x1 | 1L << 0x7 | /* ldhu_a sth_a */ \ - 1L << 0x2 | 1L << 0x8 | /* ldw_a stw_a */ \ - 1L << 0x3 | 1L << 0x9) /* ldl_a stl_a */ - asmlinkage void do_entUnaUser(void __user *va, unsigned long opcode, unsigned long reg, struct pt_regs *regs) -- Gitee From bbf0247c08831f7f9ba6c0901393afd86fe748bb Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 26 Aug 2024 10:03:37 +0800 Subject: [PATCH 342/524] sw64: fix setup_mem_size() Recalculate mem_size_limit before mem_start is updated, otherwise the min() will always return 0; Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index c13abbbe26f3..f01d2ebef70f 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -68,9 +68,9 @@ static int __init setup_mem_size(char *p) mem_size_limit = size; if (mem_start < NODE0_START) { - mem_start = NODE0_START; mem_size_limit -= min(mem_size_limit, NODE0_START - mem_start); + mem_start = NODE0_START; } return 0; -- Gitee From 5995f962c50ad6026ce666036b72678fc7a73d90 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 12 Aug 2024 17:15:36 +0800 Subject: [PATCH 343/524] sw64: fix numa setup when acpi is disabled If acpi is disabled and there's no memory on 2nd node, the 2nd node would not be registered and all cpus would be registered to the 1st node. Fix this issue by moving cpu_set_node() inside numa_init() and make it set numa_nodes_parsed. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 10 ------- arch/sw_64/mm/numa.c | 55 +++++++++++++++++---------------------- 2 files changed, 24 insertions(+), 41 deletions(-) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 6c36fedfa6e3..1cb42570c0d1 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -382,10 +382,6 @@ static int __init request_standard_resources(void) } subsys_initcall(request_standard_resources); -#ifdef CONFIG_NUMA -extern void cpu_set_node(void); -#endif - static int __init topology_init(void) { int i, ret; @@ -733,12 +729,6 @@ setup_arch(char **cmdline_p) /* Default root filesystem to sda2. */ ROOT_DEV = MKDEV(SCSI_DISK0_MAJOR, 2); - - if (acpi_disabled) { -#ifdef CONFIG_NUMA - cpu_set_node(); -#endif - } } static int diff --git a/arch/sw_64/mm/numa.c b/arch/sw_64/mm/numa.c index 2793718c66f6..8736a49e54ee 100644 --- a/arch/sw_64/mm/numa.c +++ b/arch/sw_64/mm/numa.c @@ -289,6 +289,28 @@ static void __init get_numa_info_socket(void) } } +static void cpu_set_node(void) +{ + int i; + + if (numa_off) { + for (i = 0; i < nr_cpu_ids; i++) + cpu_to_node_map[i] = 0; + } else { + for (i = 0; i < nr_cpu_ids; i++) { + int nid = rcid_to_domain_id(cpu_to_rcid(i)); + + cpu_to_node_map[i] = nid; + node_set(nid, numa_nodes_parsed); + } + } + /* + * Setup numa_node for cpu 0 before per_cpu area for booting. + * Actual setup of numa_node will be done in native_smp_prepare_cpus(). + */ + set_cpu_numa_node(0, cpu_to_node_map[0]); +} + static int __init manual_numa_init(void) { int ret, nid; @@ -350,6 +372,8 @@ static int __init manual_numa_init(void) } } + cpu_set_node(); + return 0; } @@ -385,37 +409,6 @@ void __init sw64_numa_init(void) numa_init(manual_numa_init); } -void cpu_set_node(void) -{ - int i; - - if (numa_off) { - for (i = 0; i < nr_cpu_ids; i++) - cpu_to_node_map[i] = 0; - } else { - int rr, default_node, cid; - - rr = first_node(node_online_map); - for (i = 0; i < nr_cpu_ids; i++) { - cid = cpu_to_rcid(i); - default_node = rcid_to_domain_id(cid); - if (node_online(default_node)) { - cpu_to_node_map[i] = default_node; - } else { - cpu_to_node_map[i] = rr; - rr = next_node(rr, node_online_map); - if (rr == MAX_NUMNODES) - rr = first_node(node_online_map); - } - } - } - /* - * Setup numa_node for cpu 0 before per_cpu area for booting. - * Actual setup of numa_node will be done in native_smp_prepare_cpus(). - */ - set_cpu_numa_node(0, cpu_to_node_map[0]); -} - void numa_store_cpu_info(unsigned int cpu) { set_cpu_numa_node(cpu, cpu_to_node_map[cpu]); -- Gitee From 8ae96663f8e1ff0ef4c41ff82d3e28a1d66d1a5e Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Wed, 14 Aug 2024 18:50:13 +0800 Subject: [PATCH 344/524] sw64: add junzhang_v1/2/3_key Add junzhang_v1_key/junzhang_v2_key/junzhang_v3_key to identify firmware junzhang/junzhang_v2/junzhang_v3. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hw_init.h | 6 ++++++ arch/sw_64/kernel/setup.c | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h index b2d36df86bf2..c13fbb4d620e 100644 --- a/arch/sw_64/include/asm/hw_init.h +++ b/arch/sw_64/include/asm/hw_init.h @@ -51,10 +51,16 @@ DECLARE_STATIC_KEY_FALSE(run_mode_guest_key); DECLARE_STATIC_KEY_FALSE(run_mode_emul_key); DECLARE_STATIC_KEY_FALSE(hw_una_enabled); +DECLARE_STATIC_KEY_FALSE(junzhang_v1_key); +DECLARE_STATIC_KEY_FALSE(junzhang_v2_key); +DECLARE_STATIC_KEY_FALSE(junzhang_v3_key); #define is_in_host() static_branch_likely(&run_mode_host_key) #define is_in_guest() static_branch_unlikely(&run_mode_guest_key) #define is_in_emul() static_branch_unlikely(&run_mode_emul_key) #define is_guest_or_emul() !static_branch_likely(&run_mode_host_key) +#define is_junzhang_v1() static_branch_unlikely(&junzhang_v1_key) +#define is_junzhang_v2() static_branch_likely(&junzhang_v2_key) +#define is_junzhang_v3() static_branch_unlikely(&junzhang_v3_key) #endif /* _ASM_SW64_HW_INIT_H */ diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 1cb42570c0d1..e3608c601cba 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -86,6 +86,9 @@ DEFINE_STATIC_KEY_FALSE(run_mode_guest_key); DEFINE_STATIC_KEY_FALSE(run_mode_emul_key); DEFINE_STATIC_KEY_FALSE(hw_una_enabled); +DEFINE_STATIC_KEY_FALSE(junzhang_v1_key); +DEFINE_STATIC_KEY_FALSE(junzhang_v2_key); +DEFINE_STATIC_KEY_FALSE(junzhang_v3_key); struct socket_desc_t socket_desc[MAX_NUMSOCKETS]; int memmap_nr; @@ -500,6 +503,20 @@ static void __init setup_firmware_fdt(void) pr_info("EXTCLK: %llu Hz\n", sunway_extclk_hz); } + if (sunway_machine_is_compatible("sunway,junzhang")) { + static_branch_enable(&junzhang_v1_key); + static_branch_disable(&junzhang_v2_key); + static_branch_disable(&junzhang_v3_key); + } else if (sunway_machine_is_compatible("sunway,junzhang_v2")) { + static_branch_enable(&junzhang_v2_key); + static_branch_disable(&junzhang_v1_key); + static_branch_disable(&junzhang_v3_key); + } else if (sunway_machine_is_compatible("sunway,junzhang_v3")) { + static_branch_enable(&junzhang_v3_key); + static_branch_disable(&junzhang_v1_key); + static_branch_disable(&junzhang_v2_key); + } + name = of_flat_dt_get_machine_name(); if (name) pr_info("DTB(from firmware): Machine model: %s\n", name); -- Gitee From 1b8ab20feb49bbe7522f80c35c43fc9c69347f53 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Wed, 14 Aug 2024 19:08:29 +0800 Subject: [PATCH 345/524] sw64: compatible with suspend implementation Modify the suspend function to adapt to different firmware. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 33 ++++++++++++++++---------------- arch/sw_64/kernel/suspend.c | 16 ++++++++-------- drivers/irqchip/irq-sunway-cpu.c | 18 ++++++++--------- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 833b99ef66e5..aa66fa87c728 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -99,10 +99,11 @@ static void downshift_freq(void) struct cpu_topology *sib_topo = &cpu_topology[cpu]; if ((cpu_topo->package_id == sib_topo->package_id) && - (cpu_topo->core_id == sib_topo->core_id)) + (cpu_topo->core_id == sib_topo->core_id)) return; } + core_id = rcid_to_core_id(cpu_to_rcid(cpuid)); node_id = rcid_to_domain_id(cpu_to_rcid(cpuid)); @@ -542,10 +543,11 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) wmb(); smp_rcb->ready = 0; -#ifdef CONFIG_SUBARCH_C3B - /* send wake up signal */ - send_wakeup_interrupt(cpu); -#endif + if (!is_junzhang_v1()) { + /* send wake up signal */ + send_wakeup_interrupt(cpu); + } + smp_boot_one_cpu(cpu, tidle); #ifdef CONFIG_SUBARCH_C3B @@ -841,17 +843,16 @@ void arch_cpu_idle_dead(void) } #ifdef CONFIG_SUSPEND - -#ifdef CONFIG_SUBARCH_C3B - sleepen(); - send_sleep_interrupt(smp_processor_id()); - while (1) - asm("nop"); -#else - asm volatile("halt"); - while (1) - asm("nop"); -#endif + if (!is_junzhang_v1()) { + sleepen(); + send_sleep_interrupt(smp_processor_id()); + while (1) + asm("nop"); + } else { + asm volatile("halt"); + while (1) + asm("nop"); + } #else asm volatile("memb"); diff --git a/arch/sw_64/kernel/suspend.c b/arch/sw_64/kernel/suspend.c index 97e5478a678f..955cf41fd8c9 100644 --- a/arch/sw_64/kernel/suspend.c +++ b/arch/sw_64/kernel/suspend.c @@ -38,14 +38,14 @@ void sw64_suspend_enter(void) disable_local_timer(); current_thread_info()->pcb.tp = rtid(); -#ifdef CONFIG_SUBARCH_C3B - sw64_suspend_deep_sleep(&suspend_state); -#else - pme_state = PME_WFW; - sw64_write_csr_imb(PME_EN, CSR_INT_EN); - asm("halt"); - local_irq_disable(); -#endif + if (!is_junzhang_v1()) + sw64_suspend_deep_sleep(&suspend_state); + else { + pme_state = PME_WFW; + sw64_write_csr_imb(PME_EN, CSR_INT_EN); + asm("halt"); + local_irq_disable(); + } wrtp(current_thread_info()->pcb.tp); diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 9dddf6786eab..6a3db0dc183d 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -91,17 +91,17 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, nmi_enter(); old_regs = set_irq_regs(regs); -#ifdef CONFIG_SUBARCH_C4 - if (pme_state == PME_WFW) { - pme_state = PME_PENDING; - goto out; - } + if (is_junzhang_v1()) { + if (pme_state == PME_WFW) { + pme_state = PME_PENDING; + goto out; + } - if (pme_state == PME_PENDING) { - handle_device_interrupt(vector); - pme_state = PME_CLEAR; + if (pme_state == PME_PENDING) { + handle_device_interrupt(vector); + pme_state = PME_CLEAR; + } } -#endif if (is_guest_or_emul()) { if ((type & 0xffff) > 15) { -- Gitee From 1b1adb7af4338d683c4aafc310e2dc146c061610 Mon Sep 17 00:00:00 2001 From: Xu Linqin Date: Fri, 16 Aug 2024 10:14:17 +0800 Subject: [PATCH 346/524] sw64: kvm: adjust interrupt priority for guest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit According to architecture manual, adjust interrupt priority of guest in descending order as follows: RTC, IPI, MSI, INTx and VT_SERIAL. It does not follow manual completely due to flat interrupt management in sw64 kvm. The previous priorities were not appropriate, which may cause system lag when numerous IPIs are generated by applications like mariadb within a short period of time. This patch has no negative effect on the performance of Unixbench and fio. Signed-off-by: Xu Linqin Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/emulate.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/sw_64/kvm/emulate.c b/arch/sw_64/kvm/emulate.c index 9f2ededd5a1e..f103a5f3b9d4 100644 --- a/arch/sw_64/kvm/emulate.c +++ b/arch/sw_64/kvm/emulate.c @@ -4,6 +4,7 @@ * Author: fire3 yangzh * linhn */ +#include #include #include #include @@ -75,6 +76,12 @@ unsigned int interrupt_pending(struct kvm_vcpu *vcpu, bool *more) bitmap_copy(blk, vcpu->arch.irqs_pending, SWVM_IRQS); + if (test_bit(INT_RTC, blk)) + return INT_RTC; + + if (test_bit(INT_IPI, blk)) + return INT_IPI; + irq = find_last_bit(blk, SWVM_IRQS); return irq; -- Gitee From 20283429c81efca91443aa85b9cdd2e08c3fc20e Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 3 Sep 2024 16:36:27 +0800 Subject: [PATCH 347/524] sw64: add arch_hugetlb_valid_size for C4 Add arch_hugetlb_valid_size routine used to validate passed huge page size. Remove hugepagesz= command line parsing which has been moved into an arch independent routine. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/hugetlbpage_c4.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/arch/sw_64/mm/hugetlbpage_c4.c b/arch/sw_64/mm/hugetlbpage_c4.c index 913389cd2577..b3649e19bff1 100644 --- a/arch/sw_64/mm/hugetlbpage_c4.c +++ b/arch/sw_64/mm/hugetlbpage_c4.c @@ -433,20 +433,14 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, } #endif /* CONFIG_HUGETLB_PAGE */ -static __init int setup_hugepagesz(char *opt) +bool __init arch_hugetlb_valid_size(unsigned long size) { - unsigned long ps = memparse(opt, &opt); - - switch (ps) { + switch (size) { case PUD_SIZE: case PMD_SIZE * CONT_PMDS: case PMD_SIZE: - hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT); - return 1; + return true; } - pr_err("hugepagesz: Unsupported page size %lu M\n", - ps >> 20); - return 0; + return false; } -__setup("hugepagesz=", setup_hugepagesz); -- Gitee From 6c3ab6dedb2e734bf0c3aced7206655aae0f842e Mon Sep 17 00:00:00 2001 From: Wang Yicheng Date: Wed, 4 Sep 2024 09:34:23 +0800 Subject: [PATCH 348/524] sw64: fix compilation issues Current code causes a compilation error when the CONFIG_PM is not selected. This commit modifies the relevant code logic to fix this issue. Signed-off-by: Wang Yicheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpufreq.h | 1 + drivers/irqchip/irq-sunway-cpu.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/arch/sw_64/include/asm/cpufreq.h b/arch/sw_64/include/asm/cpufreq.h index 0d3e794b16ea..d89db83b42a6 100644 --- a/arch/sw_64/include/asm/cpufreq.h +++ b/arch/sw_64/include/asm/cpufreq.h @@ -8,6 +8,7 @@ #include #include #include +#include struct clk; diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 6a3db0dc183d..406db782e763 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -91,6 +91,7 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, nmi_enter(); old_regs = set_irq_regs(regs); +#ifdef CONFIG_PM if (is_junzhang_v1()) { if (pme_state == PME_WFW) { pme_state = PME_PENDING; @@ -102,6 +103,7 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, pme_state = PME_CLEAR; } } +#endif if (is_guest_or_emul()) { if ((type & 0xffff) > 15) { -- Gitee From ea6c6f21be244ff82f7238a9a3907a12ce30d83a Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Tue, 22 Oct 2024 09:37:01 +0800 Subject: [PATCH 349/524] sw64: add the missed clear_flush() This patch adds clear_flush() back, which was deleted by mistake. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/hugetlbpage_c4.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/sw_64/mm/hugetlbpage_c4.c b/arch/sw_64/mm/hugetlbpage_c4.c index b3649e19bff1..07ec95b734d4 100644 --- a/arch/sw_64/mm/hugetlbpage_c4.c +++ b/arch/sw_64/mm/hugetlbpage_c4.c @@ -88,6 +88,19 @@ static pte_t get_and_clear(struct mm_struct *mm, return orig_pte; } +static void clear_flush(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long pgsize, + unsigned long ncontig) +{ + struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0); + unsigned long i, saddr = addr; + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) + pte_clear(mm, addr, ptep); + + flush_tlb_range(&vma, saddr, addr); +} + static pte_t get_clear_contig_flush(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long pgsize, unsigned long ncontig) -- Gitee From 98cd9b001c10a56a7d0454bc735dabf641861eb2 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 3 Sep 2024 16:49:04 +0800 Subject: [PATCH 350/524] sw64: fix set_huge_pte_at for C4 Use clear_flush instead of get_and_clear to make sure tlb would be flushd. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/hugetlbpage_c4.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/mm/hugetlbpage_c4.c b/arch/sw_64/mm/hugetlbpage_c4.c index 07ec95b734d4..ce3f5168f561 100644 --- a/arch/sw_64/mm/hugetlbpage_c4.c +++ b/arch/sw_64/mm/hugetlbpage_c4.c @@ -233,7 +233,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pfn = pte_pfn(pte); hugeprot = pte_pgprot(pte); - get_and_clear(mm, addr, ptep, pgsize, ncontig); + clear_flush(mm, addr, ptep, pgsize, ncontig); for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); -- Gitee From cab2923850755eb20d33cffe632641d5a81bd554 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 10 Sep 2024 08:49:08 +0800 Subject: [PATCH 351/524] sw64: modify sys_pfh_ops In order to be compatible with new chip which extends each field in CSR:PFH_CNT, assign a unique id for each field the user may want to modify, and let the user modify each field individually. Since this syscall has not been widely used, we will abandon the old one directly, and assign a new number for this new version. Only root on host machine is allowed to use this syscall. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/sys_sw64.c | 187 ++++++++++++++++++++----- arch/sw_64/kernel/syscalls/syscall.tbl | 4 +- 2 files changed, 157 insertions(+), 34 deletions(-) diff --git a/arch/sw_64/kernel/sys_sw64.c b/arch/sw_64/kernel/sys_sw64.c index 31e6a8d65cf9..7efebce4af11 100644 --- a/arch/sw_64/kernel/sys_sw64.c +++ b/arch/sw_64/kernel/sys_sw64.c @@ -154,53 +154,176 @@ SYSCALL_DEFINE0(sw64_pipe) #ifdef CONFIG_SUBARCH_C4 -struct pfh_val { - unsigned long pfh_ctl; - unsigned long pfh_cnt; -}; +static void local_set_pfh_ctl(void *info) +{ + unsigned long *kcsr = info; -static void local_set_pfh(void *info) + sw64_write_csr(*kcsr, CSR_PFH_CTL); +} + +static void local_set_pfh_cnt(void *info) { - struct pfh_val *kbuf = info; + unsigned long *kcsr = info; - if (kbuf->pfh_ctl) - sw64_write_csr(kbuf->pfh_ctl, CSR_PFH_CTL); - if (kbuf->pfh_cnt) - sw64_write_csr(kbuf->pfh_cnt, CSR_PFH_CNT); + sw64_write_csr(*kcsr, CSR_PFH_CNT); } -SYSCALL_DEFINE3(pfh_ops, unsigned long, op, - unsigned long __user *, pfh_ctl_p, - unsigned long __user *, pfh_cnt_p) +enum pfh_field_id { + L1_CCNT, + L1_RCNT, + L1_MCNT, + L2_CCNT, + L2_RCNT, + L2_MCNT, + L2_RAMP, + L3_CCNT, + L3_RCNT, + L3_MCNT, + L3_RAMP, + L1PFH_EN = 0x10, + L2PFH_EN, + L3PFH_EN, + PFH_FIELD_MAX +}; + +struct pfh_field { + unsigned long shift; + unsigned long mask; +}; + +struct pfh_field pfh_fields_c4[PFH_FIELD_MAX] = { + [L1_CCNT] = {0, 0x0f}, + [L1_RCNT] = {4, 0x0f}, + [L1_MCNT] = {8, 0x0f}, + [L2_CCNT] = {12, 0x0f}, + [L2_RCNT] = {16, 0x0f}, + [L2_MCNT] = {20, 0x0f}, + [L2_RAMP] = {24, 0x03}, + [L3_CCNT] = {26, 0x1f}, + [L3_RCNT] = {31, 0x1f}, + [L3_MCNT] = {36, 0x1f}, + [L3_RAMP] = {41, 0x03}, + [L1PFH_EN] = {0, 0x01}, + [L2PFH_EN] = {1, 0x01}, + [L3PFH_EN] = {2, 0x01} +}; + +struct pfh_field pfh_fields_c4b[PFH_FIELD_MAX] = { + [L1_CCNT] = {0, 0x3f}, + [L1_RCNT] = {6, 0x3f}, + [L1_MCNT] = {12, 0x3f}, + [L2_CCNT] = {18, 0x3f}, + [L2_RCNT] = {24, 0x3f}, + [L2_MCNT] = {30, 0x3f}, + [L2_RAMP] = {36, 0x03}, + [L3_CCNT] = {38, 0x7f}, + [L3_RCNT] = {45, 0x7f}, + [L3_MCNT] = {52, 0x7f}, + [L3_RAMP] = {59, 0x03}, + [L1PFH_EN] = {0, 0x01}, + [L2PFH_EN] = {1, 0x01}, + [L3PFH_EN] = {2, 0x01} +}; + +/* + * id: + * 0x00: PFH_CNT: L1_CCNT + * 0x01: PFH_CNT: L1_RCNT + * 0x02: PFH_CNT: L1_MCNT + * 0x03: PFH_CNT: L2_CCNT + * 0x04: PFH_CNT: L2_RCNT + * 0x05: PFH_CNT: L2_MCNT + * 0x06: PFH_CNT: L2_RAMP + * 0x07: PFH_CNT: L3_CCNT + * 0x08: PFH_CNT: L3_RCNT + * 0x09: PFH_CNT: L3_MCNT + * 0x0a: PFH_CNT: L3_RAMP + * 0x10: PFH_CTL: L1PFH_EN + * 0x11: PFH_CTL: L2PFH_EN + * 0x12: PFH_CTL: L3PFH_EN + * op: 0 for get, 1 for set + */ +SYSCALL_DEFINE3(pfh_ops, unsigned long, id, unsigned long, op, + unsigned long __user *, buf) { - struct pfh_val kbuf = {0, 0}; + unsigned long kcsr = 0; + unsigned long kbuf = 0; + unsigned long field_shift; + unsigned long field_mask; + unsigned long csr_idx; long error = 0; + struct pfh_field *pfh_fields_arr; + + if (!is_in_host()) + return -EPERM; if (!capable(CAP_SYS_ADMIN)) return -EPERM; - if (op) { // op != 0, set - if (pfh_ctl_p) - error |= get_user(kbuf.pfh_ctl, pfh_ctl_p); - if (pfh_cnt_p) - error |= get_user(kbuf.pfh_cnt, pfh_cnt_p); + switch (id & 0xf0) { + case 0x00: + csr_idx = CSR_PFH_CNT; + break; + case 0x10: + csr_idx = CSR_PFH_CTL; + break; + default: + error = -EINVAL; + goto out; + } - if (!error && (kbuf.pfh_ctl || kbuf.pfh_cnt)) { - smp_call_function(local_set_pfh, &kbuf, 1); - local_set_pfh(&kbuf); - } - } else { // op == 0, get - if (pfh_ctl_p) { - kbuf.pfh_ctl = sw64_read_csr(CSR_PFH_CTL); - error |= put_user(kbuf.pfh_ctl, pfh_ctl_p); - } + if (!is_junzhang_v3()) + pfh_fields_arr = pfh_fields_c4; + else + pfh_fields_arr = pfh_fields_c4b; - if (pfh_cnt_p) { - kbuf.pfh_cnt = sw64_read_csr(CSR_PFH_CNT); - error |= put_user(kbuf.pfh_cnt, pfh_cnt_p); - } + field_shift = pfh_fields_arr[id].shift; + field_mask = pfh_fields_arr[id].mask << field_shift; + + switch (csr_idx) { + case CSR_PFH_CTL: + kcsr = sw64_read_csr(CSR_PFH_CTL); + break; + case CSR_PFH_CNT: + kcsr = sw64_read_csr(CSR_PFH_CNT); + break; + default: + /* should never reach here */ + BUG(); + } + + switch (op) { + case 0: // get + kbuf = (kcsr & field_mask) >> field_shift; + error = put_user(kbuf, buf); + goto out; + case 1: // set + error = get_user(kbuf, buf); + if (error) + goto out; + kcsr = (kcsr & (~field_mask)) | + ((kbuf << field_shift) & field_mask); + break; + default: + error = -EINVAL; + goto out; + } + + switch (csr_idx) { + case CSR_PFH_CTL: + smp_call_function(local_set_pfh_ctl, &kcsr, 1); + local_set_pfh_ctl(&kcsr); + break; + case CSR_PFH_CNT: + smp_call_function(local_set_pfh_cnt, &kcsr, 1); + local_set_pfh_cnt(&kcsr); + break; + default: + /* should never reach here */ + BUG(); } +out: return error; } diff --git a/arch/sw_64/kernel/syscalls/syscall.tbl b/arch/sw_64/kernel/syscalls/syscall.tbl index 3acd8b5ea7db..fe75af3652e3 100644 --- a/arch/sw_64/kernel/syscalls/syscall.tbl +++ b/arch/sw_64/kernel/syscalls/syscall.tbl @@ -116,8 +116,8 @@ 106 common listen sys_listen #107 is unused #108 is unused -#109 is unused -110 common pfh_ops sys_pfh_ops +109 common pfh_ops sys_pfh_ops +110 common old_pfh_ops sys_ni_syscall 111 common sigsuspend sys_sigsuspend #112 is unused 113 common recvmsg sys_recvmsg -- Gitee From 234e877cecc671298b64589c72317b4f984df9c7 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 10 Sep 2024 09:21:17 +0800 Subject: [PATCH 352/524] sw64: kvm: fix an error when unmapping 512M hugepages In sw64 architecture, 512M hugepage consist of 64 consecutive 8M hugepages. Therefore, at the time of unmapping, we should get the address of the next 512M hugepage. Current code get the address of the next 8M hugepage, it will be out of range when we unmap the page, so we fix it. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 12e9dd43228f..209a216f6c7d 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -146,8 +146,10 @@ static void unmap_apt_pmds(struct kvm *kvm, pud_t *pud, if (!pmd_none(*pmd)) { if (pmd_trans_huge(*pmd)) { if (pmd_cont(*pmd)) { - for (i = 0; i < CONT_PMDS; i++, pmd++) - pmd_clear(pmd); + for (i = 0; i < CONT_PMDS; i++) + pmd_clear(pmd + i); + pmd += CONT_PMDS - 1; + next += CONT_PMD_SIZE - PMD_SIZE; } else pmd_clear(pmd); /* Do we need flush tlb???? edited by lff */ -- Gitee From a30cfd7f9172fe439792e13a1324bb3559c400f7 Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Thu, 19 Sep 2024 17:05:52 +0800 Subject: [PATCH 353/524] sw64: mm: fix PFN of PMDs for 512M hugepage A 512M hugepage is composed of 64 contiguous 8M hugepages. When setting up the page table entries, it assigned the PFN of first 8M hugepage to all the 64 contiguous PMDs and set the HPFN bit. This caused incorrect page in gup_huge_pmd() because of PMD_MASK, and resulted in incorrect address access in some scenarios when 512M hugepages were used. This patch fixes the page table setup to ensure that each PMD's PFN corresponds to an individual 8M hugepage. For kvm, although the original code works fine, this patch makes similar modifications to Additional Page Table(APT) to maintain consistency in the 512M hugepage setup method. Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 8 ++++++-- arch/sw_64/mm/hugetlbpage_c4.c | 15 +++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 209a216f6c7d..b019a294ba1c 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -956,10 +956,11 @@ static int apt_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, static int apt_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache - *cache, phys_addr_t addr, const pmd_t *new_pmd, unsigned long sz) + *cache, phys_addr_t addr, pmd_t *new_pmd, unsigned long sz) { pmd_t *pmd, old_pmd, *ori_pmd; int i; + unsigned long dpfn; retry: pmd = apt_get_pmd(kvm, cache, addr, sz); VM_BUG_ON(!pmd); @@ -1022,8 +1023,11 @@ static int apt_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache /* Do we need WRITE_ONCE(pmd, new_pmd)? */ if (sz == CONT_PMD_SIZE) { - for (i = 0; i < CONT_PMDS; i++, ori_pmd++) + dpfn = 1UL << (_PFN_SHIFT + PMD_SHIFT - PAGE_SHIFT); + for (i = 0; i < CONT_PMDS; i++, ori_pmd++) { set_pmd(ori_pmd, *new_pmd); + new_pmd->pmd += dpfn; + } } else set_pmd(pmd, *new_pmd); return 0; diff --git a/arch/sw_64/mm/hugetlbpage_c4.c b/arch/sw_64/mm/hugetlbpage_c4.c index ce3f5168f561..2611af9cf780 100644 --- a/arch/sw_64/mm/hugetlbpage_c4.c +++ b/arch/sw_64/mm/hugetlbpage_c4.c @@ -215,7 +215,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, size_t pgsize; int i; int ncontig; - unsigned long pfn; + unsigned long pfn, dpfn; pgprot_t hugeprot; /* @@ -231,11 +231,12 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, ncontig = num_contig_ptes(sz, &pgsize); pfn = pte_pfn(pte); + dpfn = PMD_SIZE >> PAGE_SHIFT; hugeprot = pte_pgprot(pte); clear_flush(mm, addr, ptep, pgsize, ncontig); - for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); } @@ -254,7 +255,7 @@ void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, void huge_ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - unsigned long pfn; + unsigned long pfn, dpfn; pgprot_t hugeprot; int ncontig, i; size_t pgsize; @@ -266,6 +267,7 @@ void huge_ptep_set_wrprotect(struct mm_struct *mm, } ncontig = CONT_PMDS; + dpfn = PMD_SIZE >> PAGE_SHIFT; pte = get_and_clear(mm, addr, ptep, pgsize, ncontig); pte = pte_wrprotect(pte); @@ -273,7 +275,7 @@ void huge_ptep_set_wrprotect(struct mm_struct *mm, hugeprot = pte_pgprot(pte); pfn = pte_pfn(pte); - for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); } @@ -332,7 +334,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma, { int ncontig, i; size_t pgsize = 0; - unsigned long pfn = pte_pfn(pte); + unsigned long pfn = pte_pfn(pte), dpfn; pgprot_t hugeprot; pte_t orig_pte; @@ -340,6 +342,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma, return ptep_set_access_flags(vma, addr, ptep, pte, dirty); ncontig = CONT_PMDS; + dpfn = PMD_SIZE >> PAGE_SHIFT; if (!__cont_access_flags_changed(ptep, pte, ncontig)) return 0; @@ -355,7 +358,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma, pte = pte_mkyoung(pte); hugeprot = pte_pgprot(pte); - for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot)); return 1; -- Gitee From bdab732f2d7541bf3584e776c4c4638a3944f805 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Wed, 25 Sep 2024 11:36:49 +0800 Subject: [PATCH 354/524] sw64: fix compilation issues on match.c The current code has a compilation error in the match.c file on Core3 environment. The DEBUG_MATCH option relies on Core4. This commit modifies the relevant code to fix this issue. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig.debug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/Kconfig.debug b/arch/sw_64/Kconfig.debug index 6cb3c2488b36..985fe68c29b9 100644 --- a/arch/sw_64/Kconfig.debug +++ b/arch/sw_64/Kconfig.debug @@ -47,7 +47,7 @@ config SW64_RRK config DEBUG_MATCH bool "instruction-flow and data-flow match debugfs interface" - depends on DEBUG_FS + depends on DEBUG_FS && SUBARCH_C4 default n help Turns on the DebugFS interface for instruction-flow and data-flow match. -- Gitee From 26f526a6880db88d8e8d2dbddb617df64456ce6a Mon Sep 17 00:00:00 2001 From: Zhi Tongze Date: Sat, 12 Oct 2024 09:18:08 +0800 Subject: [PATCH 355/524] sw64: fix the error of cpufreq update When adjusting the cpu frequency, if 0 is returned, it is a correct execution return. if a non-zero value is returned, it is an incorrect execution return. the current logic is wrong and has been fixed. Signed-off-by: Zhi Tongze Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/cpufreq/sw64_cpufreq.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/sw64_cpufreq.c b/drivers/cpufreq/sw64_cpufreq.c index f4bf5f3cc550..93163da904a2 100644 --- a/drivers/cpufreq/sw64_cpufreq.c +++ b/drivers/cpufreq/sw64_cpufreq.c @@ -66,13 +66,16 @@ static unsigned int sw64_cpufreq_get(unsigned int cpu) static int sw64_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) { + int ret; unsigned int cpu = policy->cpu; if (!cpu_online(cpu)) return -ENODEV; /* setting the cpu frequency */ - sw64_set_rate(index); + ret = sw64_set_rate(index); + if (ret) + return ret; update_cpu_freq(freq_table[index].frequency); return 0; -- Gitee From 3bf3f66f1bc867f5d9b209e70df813d997780f87 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Oct 2024 08:59:15 +0800 Subject: [PATCH 356/524] sw64: show CPU feature UNA in /proc/cpuinfo Show whether the cpu hardware support unaligned memory access in cpuinfo. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpu.h | 7 ++++++- arch/sw_64/kernel/cpu.c | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/include/asm/cpu.h b/arch/sw_64/include/asm/cpu.h index e6f5cfdbaa3a..37a588fe12d9 100644 --- a/arch/sw_64/include/asm/cpu.h +++ b/arch/sw_64/include/asm/cpu.h @@ -14,9 +14,14 @@ enum hmcall_cpuid_cmd { GET_VENDOR_ID = 2, GET_MODEL = 3, GET_CPU_FREQ = 4, - GET_CACHE_INFO = 5 + GET_CACHE_INFO = 5, + GET_FEATURES = 6 }; +#define CPU_FEAT_FPU 0x1 +#define CPU_FEAT_SIMD 0x2 +#define CPU_FEAT_UNA 0x4 + enum sunway_cpu_model { CPU_SW3231 = 0x31, CPU_SW831 = 0x32, diff --git a/arch/sw_64/kernel/cpu.c b/arch/sw_64/kernel/cpu.c index 43ac70253adf..0f44e58fbe52 100644 --- a/arch/sw_64/kernel/cpu.c +++ b/arch/sw_64/kernel/cpu.c @@ -142,7 +142,8 @@ static int show_cpuinfo(struct seq_file *f, void *slot) loops_per_jiffy / (500000/HZ), (loops_per_jiffy / (5000/HZ)) % 100); - seq_printf(f, "flags\t\t: fpu simd vpn upn cpuid\n"); + seq_printf(f, "flags\t\t: fpu simd vpn upn cpuid%s\n", + (cpuid(GET_FEATURES, 0) & CPU_FEAT_UNA) ? " una" : ""); seq_printf(f, "page size\t: %d\n", 8192); seq_printf(f, "cache_alignment\t: %d\n", l3_cachline_size); seq_printf(f, "address sizes\t: %u bits physical, %u bits virtual\n\n", -- Gitee From 72da8d00fa785c898c4f1c33f8ccb1b4867a8f53 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 14 Oct 2024 14:42:03 +0800 Subject: [PATCH 357/524] sw64: fix mmap protection_map Set FOW bit for shared writable page so the software could set dirty-bit in exception handling. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index b9f98b9af33e..e9843b19916e 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -207,7 +207,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL | _PAGE_LEAF | _PAGE_CONT) #define _PAGE_P(x) _PAGE_NORMAL((x) | _PAGE_FOW) -#define _PAGE_S(x) _PAGE_NORMAL(x) +#define _PAGE_S(x) _PAGE_NORMAL((x) | _PAGE_FOW) /* * pgprot_noncached() is only for infiniband pci support, and a real -- Gitee From 028b37bfe74d0eb8d3177ee736c7a8e11e5bef4d Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 12 Sep 2024 14:55:40 +0800 Subject: [PATCH 358/524] sw64: pci: remove function fix_jm585_reset() Remove function fix_jm585_reset() and add a PCI quirk to fix the issue of JMicron 585 SATA card when reboot. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/platform.h | 1 - arch/sw_64/kernel/reset.c | 26 +++------------------ arch/sw_64/pci/pci.c | 30 +++++++++++++++++++++++++ drivers/platform/sw64/legacy_junzhang.c | 5 ++--- drivers/platform/sw64/legacy_xuelang.c | 5 ++--- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h index c44116df8e16..3fbe9362c568 100644 --- a/arch/sw_64/include/asm/platform.h +++ b/arch/sw_64/include/asm/platform.h @@ -29,7 +29,6 @@ extern void (*pm_restart)(void); extern void (*pm_halt)(void); extern int i2c_set_adapter(void); extern void cpld_write(uint8_t slave_addr, uint8_t reg, uint8_t data); -extern void fix_jm585_reset(void); extern void early_parse_fdt_property(const void *fdt, const char *path, const char *prop_name, u64 *property, int size); diff --git a/arch/sw_64/kernel/reset.c b/arch/sw_64/kernel/reset.c index 3f1961ce85de..7bbd6ce8e07a 100644 --- a/arch/sw_64/kernel/reset.c +++ b/arch/sw_64/kernel/reset.c @@ -8,32 +8,13 @@ #include #include #include -#include #include #include #include #include #include -#include -void fix_jm585_reset(void) -{ - struct pci_dev *pdev; - struct pci_controller *hose; - int val; - - pdev = pci_get_device(PCI_VENDOR_ID_JMICRON, - 0x0585, NULL); - if (pdev) { - hose = pci_bus_to_pci_controller(pdev->bus); - val = readl(hose->rc_config_space_base + RC_PORT_LINK_CTL); - writel((val | 0x8), (hose->rc_config_space_base + RC_PORT_LINK_CTL)); - writel(val, (hose->rc_config_space_base + RC_PORT_LINK_CTL)); - - } - -} static void default_halt(void) { local_irq_disable(); @@ -57,10 +38,6 @@ static void default_poweroff(void) static void default_restart(void) { - /* No point in taking interrupts anymore. */ - local_irq_disable(); - - fix_jm585_reset(); #ifdef CONFIG_EFI if (efi_capsule_pending(NULL)) efi_reboot(REBOOT_WARM, NULL); @@ -103,6 +80,9 @@ void machine_restart(char *command) preempt_disable(); smp_send_stop(); #endif + /* No point in taking interrupts anymore. */ + local_irq_disable(); + do_kernel_restart(command); pm_restart(); } diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index 47b8067fed03..ea68291a0b37 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -265,6 +266,35 @@ static void enable_sw_dca(struct pci_dev *dev) DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, enable_sw_dca); #endif +static int jm585_restart_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct pci_dev *pdev; + struct pci_controller *hose; + int val; + + pdev = pci_get_device(PCI_VENDOR_ID_JMICRON, 0x0585, NULL); + if (pdev) { + hose = pci_bus_to_pci_controller(pdev->bus); + val = readl(hose->rc_config_space_base + RC_PORT_LINK_CTL); + writel((val | 0x8), (hose->rc_config_space_base + RC_PORT_LINK_CTL)); + writel(val, (hose->rc_config_space_base + RC_PORT_LINK_CTL)); + } + + return NOTIFY_DONE; +} + +static void quirk_jm585_restart(struct pci_dev *dev) +{ + static struct notifier_block jm585_restart_nb = { + .notifier_call = jm585_restart_notify, + .priority = 128, + }; + + register_restart_handler(&jm585_restart_nb); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_JMICRON, 0x0585, quirk_jm585_restart); + /** * There are some special aspects to the Root Complex of Sunway: * 1. Root Complex config space base addr is different diff --git a/drivers/platform/sw64/legacy_junzhang.c b/drivers/platform/sw64/legacy_junzhang.c index 17d6d151d0b8..56fc57d46923 100644 --- a/drivers/platform/sw64/legacy_junzhang.c +++ b/drivers/platform/sw64/legacy_junzhang.c @@ -41,10 +41,9 @@ void sw64_poweroff(void) void sw64_restart(void) { - if (is_in_host()) { - fix_jm585_reset(); + if (is_in_host()) cpld_write(0x64, 0x00, 0xc3); - } else + else vt_mode_kill_arch(LINUX_REBOOT_CMD_RESTART); } diff --git a/drivers/platform/sw64/legacy_xuelang.c b/drivers/platform/sw64/legacy_xuelang.c index 8a63d9edf9f2..67b4436b5a55 100644 --- a/drivers/platform/sw64/legacy_xuelang.c +++ b/drivers/platform/sw64/legacy_xuelang.c @@ -42,10 +42,9 @@ void sw64_poweroff(void) void sw64_restart(void) { - if (is_in_host()) { - fix_jm585_reset(); + if (is_in_host()) cpld_write(0x64, 0x00, 0xc3); - } else + else vt_mode_kill_arch(LINUX_REBOOT_CMD_RESTART); } -- Gitee From 10d3296d3ebea2514476b78375ab08eed99cfa05 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 11 Oct 2024 08:51:39 +0000 Subject: [PATCH 359/524] sw64: irqchip: improve intx implementation Improve INTx implementation by applying the following modifications: - Replace number with macros in irq-sunway-pci-intx.c. - Rename some ambiguous functions. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/irq_impl.h | 2 +- arch/sw_64/include/asm/pci.h | 2 +- arch/sw_64/kernel/irq.c | 2 +- arch/sw_64/kernel/pci-noop.c | 2 +- drivers/irqchip/irq-sunway-cpu.c | 8 ++--- drivers/irqchip/irq-sunway-pci-intx.c | 45 ++++++++++++++++++--------- 6 files changed, 39 insertions(+), 22 deletions(-) diff --git a/arch/sw_64/include/asm/irq_impl.h b/arch/sw_64/include/asm/irq_impl.h index c4f2690bca2e..0e31aaf37ac6 100644 --- a/arch/sw_64/include/asm/irq_impl.h +++ b/arch/sw_64/include/asm/irq_impl.h @@ -44,7 +44,7 @@ extern struct irqaction timer_irqaction; extern void init_rtc_irq(irq_handler_t handler); extern void handle_irq(int irq); extern void handle_ipi(struct pt_regs *regs); -extern void __init sw64_init_irq(void); +extern void __init sunway_init_pci_intx(void); extern irqreturn_t timer_interrupt(int irq, void *dev); #endif /* _ASM_SW64_IRQ_IMPL_H */ diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 3527b4a92413..a1acda1c166c 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -153,7 +153,7 @@ struct pci_controller { extern void __init sw64_init_pci(void); extern void __init sw64_device_interrupt(unsigned long vector); extern void setup_intx_irqs(struct pci_controller *hose); -extern void __init sw64_init_irq(void); +extern void __init sunway_init_pci_intx(void); extern void __init sw64_init_arch(void); extern int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); extern struct pci_controller *hose_head; diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c index d916a484b2fb..82854ba75a94 100644 --- a/arch/sw_64/kernel/irq.c +++ b/arch/sw_64/kernel/irq.c @@ -128,7 +128,7 @@ void __init init_IRQ(void) set_nmi(INT_PC); } - sw64_init_irq(); + sunway_init_pci_intx(); irqchip_init(); } diff --git a/arch/sw_64/kernel/pci-noop.c b/arch/sw_64/kernel/pci-noop.c index abfba92fa6a9..9fc6bb373605 100644 --- a/arch/sw_64/kernel/pci-noop.c +++ b/arch/sw_64/kernel/pci-noop.c @@ -135,4 +135,4 @@ void __init common_init_pci(void) } void __init sw64_init_arch(void) { } -void __init sw64_init_irq(void) { } +void __init sunway_init_pci_intx(void) { } diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 406db782e763..5aeda6a673aa 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -39,7 +39,7 @@ struct fwnode_handle *cintc_handle; -static void handle_device_interrupt(unsigned long irq_info) +static void handle_pci_intx_interrupt(unsigned long irq_info) { unsigned int i; @@ -48,7 +48,7 @@ static void handle_device_interrupt(unsigned long irq_info) return; } - for (i = 0; i < 4; i++) { + for (i = 0; i < PCI_NUM_INTX; i++) { if ((irq_info >> i) & 0x1) handle_intx(i); } @@ -99,7 +99,7 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, } if (pme_state == PME_PENDING) { - handle_device_interrupt(vector); + handle_pci_intx_interrupt(vector); pme_state = PME_CLEAR; } } @@ -123,7 +123,7 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, handle_pci_msi_interrupt(type, vector, irq_arg); goto out; case INT_INTx: - handle_device_interrupt(vector); + handle_pci_intx_interrupt(vector); goto out; case INT_IPI: diff --git a/drivers/irqchip/irq-sunway-pci-intx.c b/drivers/irqchip/irq-sunway-pci-intx.c index d6f23c666237..89eb6ca86d26 100644 --- a/drivers/irqchip/irq-sunway-pci-intx.c +++ b/drivers/irqchip/irq-sunway-pci-intx.c @@ -9,11 +9,22 @@ #include #include +#define PCI_INTXCONFIG_OFFSET 7 +#define PCI_INTTYPE_OFFSET 10 + +#if defined(CONFIG_SUBARCH_C3B) + #define PCI_INTDST_DOMAIN_ID_SHIFT 6 +#elif defined(CONFIG_SUBARCH_C4) + #define PCI_INTDST_DOMAIN_ID_SHIFT 7 +#endif + +#define PCI_INTDST_THREAD_ID_SHIFT 6 + static DEFINE_RAW_SPINLOCK(legacy_lock); struct intx_chip_data { struct pci_controller *hose; - unsigned long intxconfig[4]; + unsigned long intxconfig[PCI_NUM_INTX]; unsigned int offset; }; @@ -67,16 +78,18 @@ static int __assign_piu_intx_config(struct intx_chip_data *chip_data, hose = chip_data->hose; piu_ior0_base = hose->piu_ior0_base; - for (i = 0; i < 4; i++) { + for (i = 0; i < PCI_NUM_INTX; i++) { intxconfig = chip_data->intxconfig[i]; intxconfig &= ~PCI_INTX_INTDST_MASK; if (IS_ENABLED(CONFIG_SUBARCH_C3B)) - intxconfig |= core | (node << 6); + intxconfig |= core | (node << PCI_INTDST_DOMAIN_ID_SHIFT); else - intxconfig |= core | (thread << 6) | (node << 7); + intxconfig |= core | (thread << PCI_INTDST_THREAD_ID_SHIFT) + | (node << PCI_INTDST_DOMAIN_ID_SHIFT); - writeq(intxconfig, piu_ior0_base + INTACONFIG + (i << 7)); + writeq(intxconfig, piu_ior0_base + INTACONFIG + + (i << PCI_INTXCONFIG_OFFSET)); chip_data->intxconfig[i] = intxconfig; } return 0; @@ -108,13 +121,14 @@ static void set_intx_enable(struct irq_data *irq_data, u32 flag) hose = chip_data->hose; piu_ior0_base = hose->piu_ior0_base; - for (i = 0; i < 4; i++) { + for (i = 0; i < PCI_NUM_INTX; i++) { intxconfig = chip_data->intxconfig[i]; if (flag) intxconfig |= PCI_INTX_ENABLE; else intxconfig &= PCI_INTX_DISABLE; - writeq(intxconfig, piu_ior0_base + INTACONFIG + (i << 7)); + writeq(intxconfig, piu_ior0_base + INTACONFIG + + (i << PCI_INTXCONFIG_OFFSET)); } } @@ -174,7 +188,8 @@ static void intx_mask_irq(struct irq_data *irq_data, u32 flag) else intxconfig |= PCI_INTX_ENABLE; - writeq(intxconfig, piu_ior0_base + INTACONFIG + (offset << 7)); + writeq(intxconfig, piu_ior0_base + INTACONFIG + + (offset << PCI_INTXCONFIG_OFFSET)); } static void intx_irq_mask(struct irq_data *irq_data) @@ -196,7 +211,7 @@ static void intx_irq_unmask(struct irq_data *irq_data) static void noop(struct irq_data *d) {} static struct irq_chip sw64_intx_chip = { - .name = "PCI_INTX", + .name = "PCI-INTX", .irq_enable = intx_irq_enable, .irq_disable = intx_irq_disable, .irq_mask = intx_irq_mask, @@ -234,11 +249,11 @@ void setup_intx_irqs(struct pci_controller *hose) chip_data->hose = hose; - for (i = 0; i < 4; i++) { + for (i = 0; i < PCI_NUM_INTX; i++) { if (IS_ENABLED(CONFIG_SUBARCH_C3B)) - chip_data->intxconfig[i] = (0x1UL << (3 - i)) << 10; + chip_data->intxconfig[i] = (0x1UL << (3 - i)) << PCI_INTTYPE_OFFSET; else - chip_data->intxconfig[i] = (0x1UL << i) << 10; + chip_data->intxconfig[i] = (0x1UL << i) << PCI_INTTYPE_OFFSET; } irq_set_chip_data(irq, chip_data); @@ -250,7 +265,7 @@ void setup_intx_irqs(struct pci_controller *hose) set_pcieport_service_irq(hose); } -void __init sw64_init_irq(void) +void __init sunway_init_pci_intx(void) { struct pci_controller *hose = hose_head; @@ -270,7 +285,9 @@ void handle_intx(unsigned int offset) for (hose = hose_head; hose; hose = hose->next) { piu_ior0_base = hose->piu_ior0_base; - value = readq(piu_ior0_base + INTACONFIG + (offset << 7)); + value = readq(piu_ior0_base + INTACONFIG + + (offset << PCI_INTXCONFIG_OFFSET)); + if ((value & (PCI_INTX_VALID)) && (value & PCI_INTX_ENABLE)) { irq_data = irq_get_irq_data(hose->int_irq); if (irq_data) { -- Gitee From f3196896a915d388e4d36eef0cc3f4ce262d4249 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 11 Oct 2024 08:56:41 +0000 Subject: [PATCH 360/524] sw64: irqchip: remove fallback after kzalloc_node() Following similar changes of commit 4beaacc6fea5 ("net/mlx4_en: remove fallback after kzalloc_node()"). Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-pci-intx.c | 3 --- drivers/irqchip/irq-sunway-pintc.c | 3 --- 2 files changed, 6 deletions(-) diff --git a/drivers/irqchip/irq-sunway-pci-intx.c b/drivers/irqchip/irq-sunway-pci-intx.c index 89eb6ca86d26..bd108d58ff53 100644 --- a/drivers/irqchip/irq-sunway-pci-intx.c +++ b/drivers/irqchip/irq-sunway-pci-intx.c @@ -47,9 +47,6 @@ struct intx_chip_data *alloc_intx_chip_data(u32 node) chip_data = kzalloc_node(sizeof(struct intx_chip_data), GFP_KERNEL, node); - if (!chip_data) - chip_data = kzalloc(sizeof(struct intx_chip_data), - GFP_KERNEL); return chip_data; } diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index eb7128b79c6f..6e883a060d85 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -97,9 +97,6 @@ static struct pintc_chip_data *pintc_alloc_chip_data(u32 node) chip_data = kzalloc_node(sizeof(struct pintc_chip_data), GFP_KERNEL, node); - if (!chip_data) - chip_data = kzalloc(sizeof(struct pintc_chip_data), - GFP_KERNEL); chip_datas[node] = chip_data; -- Gitee From b3dbe47c9023be4f3a08a8f77dea8e2d0e34c587 Mon Sep 17 00:00:00 2001 From: Yu Jiayi Date: Wed, 16 Oct 2024 17:18:54 +0800 Subject: [PATCH 361/524] sw64: kvm: optimize the calling and implementation of functions in mmu.c This patch: - The kvm_set_pfn_accessed() calls kvm_set_page_accessed(), and the kvm_release_pfn_clean() eventually calls kvm_set_page_accessed() too. So, remove the redundant call to kvm_set_pfn_accessed(). - Move the functionality of kvm_mmu_write_protect_pt_masked() into its caller, kvm_arch_mmu_enable_log_dirty_pt_masked(). This will be used to share some of the code. Signed-off-by: Yu Jiayi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index b019a294ba1c..784c956ac61d 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -1358,7 +1358,6 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, out_unlock: spin_unlock(&kvm->mmu_lock); - kvm_set_pfn_accessed(pfn); kvm_release_pfn_clean(pfn); return ret; } @@ -1513,18 +1512,19 @@ bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) return false; } -/** - * kvm_mmu_write_protect_pt_masked() - write protect dirty pages + +/* + * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected pages. * @kvm: The KVM pointer * @slot: The memory slot associated with mask * @gfn_offset: The gfn offset in memory slot - * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory - * slot to be write protected + * @mask: The mask of pages at offset 'gfn_offset' in this memory + * slot to enable dirty logging on * - * Walks bits set in mask write protects the associated pte's. Caller must + * Write protect selected pages to enable dirty logging for them. Caller must * acquire kvm_mmu_lock. */ -static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, +void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask) { @@ -1534,17 +1534,3 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, apt_wp_range(kvm, start, end); } - -/* - * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected - * dirty pages. - * - * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to - * enable dirty logging for them. - */ -void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, - struct kvm_memory_slot *slot, - gfn_t gfn_offset, unsigned long mask) -{ - kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); -} -- Gitee From 53e4e543fdc2f76e91ac7593e94c2235061bee14 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Wed, 30 Oct 2024 16:11:28 +0800 Subject: [PATCH 362/524] sw64: kvm: remove unused kvm_arch_flush_remote_tlbs_memslot() The kvm_arch_flush_remote_tlbs_memslot() has been moved to common code, so delete it. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/sw64.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index a2d2b1739dca..1213121728c4 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -607,13 +607,6 @@ vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) return VM_FAULT_SIGBUS; } -void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, - struct kvm_memory_slot *memslot) -{ - /* Let implementation handle TLB/GVA invalidation */ - kvm_arch_flush_shadow_memslot(kvm, memslot); -} - int kvm_dev_ioctl_check_extension(long ext) { int r; -- Gitee From 39917af6489dff05b244645aec34e371ebe72156 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 19 Sep 2024 10:23:13 +0800 Subject: [PATCH 363/524] sw64: refactor reset routines with new API Refactor reset routines by using new do_kernel_power_off() instead of old pm_power_off(), simplifying the source file (reset.c). In addition, the routines of virtual and physical machine are separated, and the legacy routines (legacy_xuelang.c and legacy_junzhang.c) are removed. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/efi.h | 10 +- arch/sw_64/include/asm/platform.h | 16 +--- arch/sw_64/kernel/Makefile | 1 + arch/sw_64/kernel/efi.c | 19 ++++ arch/sw_64/kernel/reset.c | 118 ++++++++++++++---------- drivers/firmware/efi/sunway-init.c | 6 +- drivers/platform/sw64/Makefile | 2 - drivers/platform/sw64/legacy_junzhang.c | 61 ------------ drivers/platform/sw64/legacy_xuelang.c | 62 ------------- 9 files changed, 101 insertions(+), 194 deletions(-) create mode 100644 arch/sw_64/kernel/efi.c delete mode 100644 drivers/platform/sw64/legacy_junzhang.c delete mode 100644 drivers/platform/sw64/legacy_xuelang.c diff --git a/arch/sw_64/include/asm/efi.h b/arch/sw_64/include/asm/efi.h index 34d5637e23c2..ef2eda461f94 100644 --- a/arch/sw_64/include/asm/efi.h +++ b/arch/sw_64/include/asm/efi.h @@ -4,16 +4,22 @@ #include #include + #ifdef CONFIG_EFI + extern void efi_init(void); extern unsigned long entSuspend; -#define SLEEP_ENTRY_GUID EFI_GUID(0x59cb76bb, 0x9c3a, 0x4c8f, 0xbd, 0x5c, 0xc0, 0x0f, 0x20, 0x61, 0x18, 0x4b) +#define SLEEP_ENTRY_GUID EFI_GUID(0x59cb76bb, 0x9c3a, 0x4c8f, 0xbd, 0x5c, 0xc0, 0x0f, 0x20, 0x61, 0x18, 0x4b) +#define BIOS_VERSION_GUID EFI_GUID(0xc47a23c3, 0xcebb, 0x4cc9, 0xa5, 0xe2, 0xde, 0xd0, 0x8f, 0xe4, 0x20, 0xb5) + +extern unsigned long sunway_bios_version; #else #define efi_init() #define efi_idmap_init() -#endif +#define sunway_bios_version (0) +#endif /* CONFIG_EFI */ #define arch_efi_call_virt_setup() #define arch_efi_call_virt_teardown() diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h index 3fbe9362c568..96656f112c40 100644 --- a/arch/sw_64/include/asm/platform.h +++ b/arch/sw_64/include/asm/platform.h @@ -9,26 +9,12 @@ #include #endif -#ifdef CONFIG_EFI -#define BIOS_VERSION_GUID EFI_GUID(0xc47a23c3, 0xcebb, 0x4cc9, 0xa5, 0xe2, 0xde, 0xd0, 0x8f, 0xe4, 0x20, 0xb5) - -#define BIOS_SUPPORT_RESET_CLALLBACK(bios_version) ((bios_version) != NULL) - -extern unsigned long bios_version; - -#endif - extern struct boot_params *sunway_boot_params; extern unsigned long sunway_boot_magic; extern unsigned long sunway_dtb_address; -extern void sw64_halt(void); -extern void sw64_poweroff(void); -extern void sw64_restart(void); -extern void (*pm_restart)(void); -extern void (*pm_halt)(void); -extern int i2c_set_adapter(void); extern void cpld_write(uint8_t slave_addr, uint8_t reg, uint8_t data); + extern void early_parse_fdt_property(const void *fdt, const char *path, const char *prop_name, u64 *property, int size); diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 6b5172f50056..0b0012462517 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -35,6 +35,7 @@ obj-$(CONFIG_RELOCATABLE) += relocate.o obj-$(CONFIG_DEBUG_FS) += segvdbg.o unaligned.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_DEBUG_MATCH) += match.o +obj-$(CONFIG_EFI) += efi.o ifdef CONFIG_PERF_EVENTS obj-$(CONFIG_SUBARCH_C3B) += perf_event.o diff --git a/arch/sw_64/kernel/efi.c b/arch/sw_64/kernel/efi.c new file mode 100644 index 000000000000..8bf80b271581 --- /dev/null +++ b/arch/sw_64/kernel/efi.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include + +bool efi_poweroff_required(void) +{ + /* VM has its own poweroff interface */ + if (!is_in_host()) + return false; + + /* Prefer ACPI S5 */ + if (acpi_sleep_state_supported(ACPI_STATE_S5)) + return false; + + return efi_enabled(EFI_RUNTIME_SERVICES); +} diff --git a/arch/sw_64/kernel/reset.c b/arch/sw_64/kernel/reset.c index 7bbd6ce8e07a..a7f332fa31ee 100644 --- a/arch/sw_64/kernel/reset.c +++ b/arch/sw_64/kernel/reset.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (C) 2020-2022 Sunway Technology Corporation Limited + * Copyright (C) 2020-2024 Sunway Technology Corporation Limited */ + #include #include #include @@ -14,10 +15,16 @@ #include #include +#include + +void (*pm_power_off)(void); +EXPORT_SYMBOL(pm_power_off); -static void default_halt(void) +void machine_halt(void) { + preempt_disable(); local_irq_disable(); + smp_send_stop(); pr_notice("\n\n** You can safely turn off the power now **\n\n"); @@ -25,74 +32,87 @@ static void default_halt(void) arch_cpu_idle(); } -static void default_poweroff(void) +void machine_power_off(void) { - /* No point in taking interrupts anymore. */ + preempt_disable(); local_irq_disable(); -#ifdef CONFIG_EFI - efi.reset_system(EFI_RESET_SHUTDOWN, EFI_SUCCESS, 0, NULL); -#endif - while (true) - arch_cpu_idle(); -} + smp_send_stop(); -static void default_restart(void) -{ -#ifdef CONFIG_EFI - if (efi_capsule_pending(NULL)) - efi_reboot(REBOOT_WARM, NULL); - else - efi_reboot(REBOOT_COLD, NULL); -#endif + do_kernel_power_off(); + + /* VM cannot reach here */ + WARN_ON(!is_in_host()); + + /** + * Compatibility with old firmware, can be removed + * when no longer support SW3231. + */ + if (!sunway_bios_version) + cpld_write(0x64, 0x00, 0xf0); while (true) arch_cpu_idle(); } -void (*pm_restart)(void); - -void (*pm_power_off)(void); -EXPORT_SYMBOL(pm_power_off); - -void (*pm_halt)(void); - -void machine_halt(void) +void machine_restart(char *command) { -#ifdef CONFIG_SMP preempt_disable(); + local_irq_disable(); smp_send_stop(); -#endif - pm_halt(); + + do_kernel_restart(command); + + /* VM cannot reach here */ + WARN_ON(!is_in_host()); + + /** + * Compatibility with old firmware, can be removed + * when no longer support SW3231. + */ + if (!sunway_bios_version) + cpld_write(0x64, 0x00, 0xc3); + else if (efi_enabled(EFI_RUNTIME_SERVICES)) + efi_reboot(reboot_mode, NULL); + + while (true) + arch_cpu_idle(); } -void machine_power_off(void) +static int vm_restart(struct sys_off_data *data) { -#ifdef CONFIG_SMP - preempt_disable(); - smp_send_stop(); -#endif - pm_power_off(); + hcall(HCALL_SET_CLOCKEVENT, 0, 0, 0); + hcall(HCALL_RESTART, 0, 0, 0); + mb(); + + return NOTIFY_DONE; } -void machine_restart(char *command) +static int vm_power_off(struct sys_off_data *data) { -#ifdef CONFIG_SMP - preempt_disable(); - smp_send_stop(); -#endif - /* No point in taking interrupts anymore. */ - local_irq_disable(); + hcall(HCALL_SET_CLOCKEVENT, 0, 0, 0); + hcall(HCALL_SHUTDOWN, 0, 0, 0); + mb(); - do_kernel_restart(command); - pm_restart(); + return NOTIFY_DONE; } -static int __init sw64_reboot_setup(void) +static int __init vm_power_init(void) { - pm_restart = default_restart; - pm_power_off = default_poweroff; - pm_halt = default_halt; + struct sys_off_handler *handler; + + if (is_in_host()) + return 0; + + handler = register_sys_off_handler(SYS_OFF_MODE_RESTART, + SYS_OFF_PRIO_DEFAULT, vm_restart, NULL); + if (WARN_ON(IS_ERR(handler))) + return PTR_ERR(handler); + + handler = register_sys_off_handler(SYS_OFF_MODE_POWER_OFF, + SYS_OFF_PRIO_DEFAULT, vm_power_off, NULL); + if (WARN_ON(IS_ERR(handler))) + return PTR_ERR(handler); return 0; } -arch_initcall(sw64_reboot_setup); +arch_initcall(vm_power_init); diff --git a/drivers/firmware/efi/sunway-init.c b/drivers/firmware/efi/sunway-init.c index 8557c181c28c..ec678b403270 100644 --- a/drivers/firmware/efi/sunway-init.c +++ b/drivers/firmware/efi/sunway-init.c @@ -27,7 +27,7 @@ #include unsigned long entSuspend; -unsigned long bios_version; +unsigned long sunway_bios_version; static int __init is_memory(efi_memory_desc_t *md) { @@ -38,7 +38,7 @@ static int __init is_memory(efi_memory_desc_t *md) static efi_config_table_type_t arch_tables[] __initdata = { {SMBIOS3_TABLE_GUID, NULL, ""}, {SLEEP_ENTRY_GUID, &entSuspend, "SLEEP ENTRY"}, - {BIOS_VERSION_GUID, &bios_version, "BIOS VERSION"}, + {BIOS_VERSION_GUID, &sunway_bios_version, "BIOS VERSION"}, {}, }; @@ -107,7 +107,7 @@ static int __init uefi_init(u64 efi_system_table) out: early_memunmap(systab, sizeof(efi_system_table_t)); - if (!bios_version) + if (!sunway_bios_version) retval = -EINVAL; return retval; diff --git a/drivers/platform/sw64/Makefile b/drivers/platform/sw64/Makefile index 7a7f7ac1d0da..4c8f5e44def6 100644 --- a/drivers/platform/sw64/Makefile +++ b/drivers/platform/sw64/Makefile @@ -1,5 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_PLATFORM_XUELANG) += legacy_xuelang.o -obj-$(CONFIG_PLATFORM_JUNZHANG) += legacy_junzhang.o obj-y += misc-platform.o diff --git a/drivers/platform/sw64/legacy_junzhang.c b/drivers/platform/sw64/legacy_junzhang.c deleted file mode 100644 index 56fc57d46923..000000000000 --- a/drivers/platform/sw64/legacy_junzhang.c +++ /dev/null @@ -1,61 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include - -static void vt_mode_kill_arch(int mode) -{ - hcall(HCALL_SET_CLOCKEVENT, 0, 0, 0); - - switch (mode) { - case LINUX_REBOOT_CMD_RESTART: - hcall(HCALL_RESTART, 0, 0, 0); - mb(); - break; - case LINUX_REBOOT_CMD_HALT: - case LINUX_REBOOT_CMD_POWER_OFF: - hcall(HCALL_SHUTDOWN, 0, 0, 0); - mb(); - break; - default: - break; - } -} - -void sw64_halt(void) -{ - if (is_in_host()) - cpld_write(0x64, 0x00, 0xf0); - else - vt_mode_kill_arch(LINUX_REBOOT_CMD_HALT); -} - -void sw64_poweroff(void) -{ - if (is_in_host()) - cpld_write(0x64, 0x00, 0xf0); - else - vt_mode_kill_arch(LINUX_REBOOT_CMD_POWER_OFF); -} - -void sw64_restart(void) -{ - if (is_in_host()) - cpld_write(0x64, 0x00, 0xc3); - else - vt_mode_kill_arch(LINUX_REBOOT_CMD_RESTART); -} - -static int sw64_reset_init(void) -{ -#ifdef CONFIG_EFI - if (BIOS_SUPPORT_RESET_CLALLBACK((void *)bios_version)) - return 0; -#endif - pm_restart = sw64_restart; - pm_power_off = sw64_poweroff; - pm_halt = sw64_halt; - return 0; -} -subsys_initcall(sw64_reset_init); diff --git a/drivers/platform/sw64/legacy_xuelang.c b/drivers/platform/sw64/legacy_xuelang.c deleted file mode 100644 index 67b4436b5a55..000000000000 --- a/drivers/platform/sw64/legacy_xuelang.c +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -#include -#include - -static void vt_mode_kill_arch(int mode) -{ - hcall(HCALL_SET_CLOCKEVENT, 0, 0, 0); - - switch (mode) { - case LINUX_REBOOT_CMD_RESTART: - hcall(HCALL_RESTART, 0, 0, 0); - mb(); - break; - case LINUX_REBOOT_CMD_HALT: - case LINUX_REBOOT_CMD_POWER_OFF: - hcall(HCALL_SHUTDOWN, 0, 0, 0); - mb(); - break; - default: - break; - } -} - -void sw64_halt(void) -{ - if (is_in_host()) - cpld_write(0x64, 0x00, 0xf0); - else - vt_mode_kill_arch(LINUX_REBOOT_CMD_HALT); -} - -void sw64_poweroff(void) -{ - if (is_in_host()) - cpld_write(0x64, 0x00, 0xf0); - else - vt_mode_kill_arch(LINUX_REBOOT_CMD_POWER_OFF); -} - -void sw64_restart(void) -{ - if (is_in_host()) - cpld_write(0x64, 0x00, 0xc3); - else - vt_mode_kill_arch(LINUX_REBOOT_CMD_RESTART); -} - -static int sw64_reset_init(void) -{ -#ifdef CONFIG_EFI - if (BIOS_SUPPORT_RESET_CLALLBACK((void *)bios_version)) - return 0; -#endif - pm_restart = sw64_restart; - pm_power_off = sw64_poweroff; - pm_halt = sw64_halt; - return 0; -} -subsys_initcall(sw64_reset_init); -- Gitee From 47f54bd017a8dc03c9c97f103a0cbf6a61644c22 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Mon, 5 Aug 2024 10:53:19 +0800 Subject: [PATCH 364/524] sw64: kvm: use vma_lookup() instead of find_vma_intersection() vma_lookup() finds the vma of a specific address with a cleaner interface and is more readable. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 784c956ac61d..4786e4a4b4b9 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -1197,7 +1197,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, /* Let's check if we will get back a huge page backed by hugetlbfs */ down_read(¤t->mm->mmap_lock); - vma = find_vma_intersection(current->mm, hva, hva + 1); + vma = vma_lookup(current->mm, hva); if (unlikely(!vma)) { kvm_err("Failed to find VMA for hva 0x%lx\n", hva); up_read(¤t->mm->mmap_lock); -- Gitee From 516e591a6917105d0f124684f2476673d441b266 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 26 Sep 2024 13:44:34 +0800 Subject: [PATCH 365/524] sw64: acpi: support reset and S5 Now acpi_power_off() and acpi_reboot() can work if the firmware provides necessary information. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/include/asm/acpi.h | 12 ++++++++++++ arch/sw_64/kernel/reset.c | 2 ++ 3 files changed, 15 insertions(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 7d64c5188688..8aa38e26d9b2 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -7,6 +7,7 @@ config SW64 select ACPI_MCFG if (ACPI && PCI) select ACPI_PPTT if ACPI select ACPI_REDUCED_HARDWARE_ONLY + select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI select ARCH_ATOMIC select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_ELF_RANDOMIZE diff --git a/arch/sw_64/include/asm/acpi.h b/arch/sw_64/include/asm/acpi.h index 7c3310168798..e386d07d1e73 100644 --- a/arch/sw_64/include/asm/acpi.h +++ b/arch/sw_64/include/asm/acpi.h @@ -112,6 +112,18 @@ static inline u32 get_acpi_id_for_cpu(unsigned int cpu) /* We take rcid as processor _UID */ return cpu_physical_id(cpu); } + +static inline unsigned long acpi_get_wakeup_address(void) +{ + return 0; +} + +static inline bool acpi_skip_set_wakeup_address(void) +{ + return true; +} + +#define acpi_skip_set_wakeup_address acpi_skip_set_wakeup_address #else /* !CONFIG_ACPI */ static inline void acpi_noirq_set(void) { } diff --git a/arch/sw_64/kernel/reset.c b/arch/sw_64/kernel/reset.c index a7f332fa31ee..8181923e676e 100644 --- a/arch/sw_64/kernel/reset.c +++ b/arch/sw_64/kernel/reset.c @@ -65,6 +65,8 @@ void machine_restart(char *command) /* VM cannot reach here */ WARN_ON(!is_in_host()); + acpi_reboot(); + /** * Compatibility with old firmware, can be removed * when no longer support SW3231. -- Gitee From 87c3c3d492b3441d0186424f2dc5675efcf9be8c Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Mon, 11 Nov 2024 02:57:49 -0500 Subject: [PATCH 366/524] sw64: kvm: remove unused argument 'addr' of apt_dissolve_pud/pmd() Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/mmu.c | 54 ++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 4786e4a4b4b9..92d10fdbd97c 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -70,21 +70,35 @@ enum { * * Function clears a PMD entry, flushes TLBs. */ -static void apt_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd) +static void apt_dissolve_pmd(struct kvm *kvm, pmd_t *pmd) { - int i; - if (!pmd_trans_huge(*pmd)) return; - if (pmd_cont(*pmd)) { - for (i = 0; i < CONT_PMDS; i++, pmd++) - pmd_clear(pmd); - } else + pmd_clear(pmd); + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pmd)); +} + +/** + * apt_dissolve_cont_pmd() - clear and flush huge cont PMD entry + * @kvm: pointer to kvm structure. + * @addr: IPA + * @pmd: pmd pointer for IPA + * + * Function clears a cont PMD entry, flushes TLBs. + */ +static void apt_dissolve_cont_pmd(struct kvm *kvm, pmd_t *pmd) +{ + int i; + pmd_t *start_pmd; + + start_pmd = pmd; + for (i = 0; i < CONT_PMDS; i++, pmd++) pmd_clear(pmd); kvm_flush_remote_tlbs(kvm); - put_page(virt_to_page(pmd)); + put_page(virt_to_page(start_pmd)); } /** @@ -95,7 +109,7 @@ static void apt_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd) * * Function clears a PUD entry, flushes TLBs. */ -static void apt_dissolve_pud(struct kvm *kvm, phys_addr_t addr, pud_t *pudp) +static void apt_dissolve_pud(struct kvm *kvm, pud_t *pudp) { if (!pud_huge(*pudp)) return; @@ -804,7 +818,7 @@ static int apt_set_pte_fast(struct kvm_vcpu *vcpu, * on to allocate page. */ if (logging_active) - apt_dissolve_pud(kvm, addr, pud); + apt_dissolve_pud(kvm, pud); find_pud: if (pud_none(*pud)) { @@ -828,8 +842,13 @@ static int apt_set_pte_fast(struct kvm_vcpu *vcpu, * While dirty page logging - dissolve huge PMD, then continue on to * allocate page. */ - if (logging_active) - apt_dissolve_pmd(kvm, addr, pmd); + if (logging_active) { + if (pmd_cont(*pmd)) + apt_dissolve_cont_pmd(kvm, + pmd_offset(pud, addr & CONT_PMD_MASK)); + else + apt_dissolve_pmd(kvm, pmd); + } find_pmd: /* Create stage-2 page mappings - Level 2 */ @@ -893,7 +912,7 @@ static int apt_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, * on to allocate page. */ if (logging_active) - apt_dissolve_pud(kvm, addr, pud); + apt_dissolve_pud(kvm, pud); if (pud_none(*pud)) { if (!cache) @@ -916,8 +935,13 @@ static int apt_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, * While dirty page logging - dissolve huge PMD, then continue on to * allocate page. */ - if (logging_active) - apt_dissolve_pmd(kvm, addr, pmd); + if (logging_active) { + if (pmd_cont(*pmd)) + apt_dissolve_cont_pmd(kvm, + pmd_offset(pud, addr & CONT_PMD_MASK)); + else + apt_dissolve_pmd(kvm, pmd); + } /* Create stage-2 page mappings - Level 2 */ if (pmd_none(*pmd)) { -- Gitee From 055f96ec3ff30479a46d44251e8132849b269574 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 30 Oct 2024 19:41:46 +0800 Subject: [PATCH 367/524] sw64: irqchip: fix irq_enable/disable callback for MCU controller Enable or disable the corresponding IRQ instead of all IRQs in irq_enable/disable callback of MCU controller. In addition, spin locks are added to protect access to registers. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-pintc.c | 150 +++++++++++++++++------------ 1 file changed, 89 insertions(+), 61 deletions(-) diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index 6e883a060d85..934ad3d60607 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -84,6 +84,8 @@ struct pintc_chip_data { void __iomem *mcu_base; /* MCU/SPBU base address */ struct irq_chip *mcu_chip; u32 mcu_irq_num; + raw_spinlock_t pintc_lock; + raw_spinlock_t mcu_lock; }; static struct pintc_chip_data *chip_datas[MAX_NUMNODES]; @@ -116,59 +118,82 @@ static void pintc_free_chip_data(struct pintc_chip_data *chip_data) kfree(chip_data); } -static DEFINE_RAW_SPINLOCK(pintc_lock); -static void lock_dev_lock(void) -{ - raw_spin_lock(&pintc_lock); -} - -static void unlock_dev_lock(void) -{ - raw_spin_unlock(&pintc_lock); -} - -static void mcu_irq_mask(struct irq_data *data) +static void mcu_irq_disable(struct irq_data *data) { struct pintc_chip_data *chip_data = data->chip_data; - unsigned long mask; + unsigned long mask, flags; int hwirq = data->hwirq; + raw_spin_lock_irqsave(&chip_data->mcu_lock, flags); + mask = readq(chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); mask &= ~(0x1UL << hwirq); writeq(mask, chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); + + raw_spin_unlock_irqrestore(&chip_data->mcu_lock, flags); } -static void mcu_irq_unmask(struct irq_data *data) +static void mcu_irq_enable(struct irq_data *data) { struct pintc_chip_data *chip_data = data->chip_data; - unsigned long mask; + unsigned long mask, flags; int hwirq = data->hwirq; + raw_spin_lock_irqsave(&chip_data->mcu_lock, flags); + mask = readq(chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); mask |= (0x1UL << hwirq); writeq(mask, chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); + + raw_spin_unlock_irqrestore(&chip_data->mcu_lock, flags); } -static void mcu_irq_enable(struct irq_data *irq_data) +static void pintc_mcu_enable(void __iomem *pintc_base) { - struct pintc_chip_data *chip_data = irq_data->chip_data; unsigned long devint_conf; - devint_conf = readq(chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); + devint_conf = readq(pintc_base + OFFSET_DEV_INT_CONFIG); devint_conf |= (1UL << 8); - writeq(devint_conf, chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); - mcu_irq_unmask(irq_data); + writeq(devint_conf, pintc_base + OFFSET_DEV_INT_CONFIG); } -static void mcu_irq_disable(struct irq_data *irq_data) +static void pintc_mcu_disable(void __iomem *pintc_base) { - struct pintc_chip_data *chip_data = irq_data->chip_data; unsigned long devint_conf; - devint_conf = readq(chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); + devint_conf = readq(pintc_base + OFFSET_DEV_INT_CONFIG); devint_conf &= ~(1UL << 8); - writeq(devint_conf, chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); - mcu_irq_mask(irq_data); + writeq(devint_conf, pintc_base + OFFSET_DEV_INT_CONFIG); +} + +static unsigned long +pintc_mcu_disable_and_save(struct pintc_chip_data *chip_data) +{ + unsigned long val; + + raw_spin_lock(&chip_data->pintc_lock); + + val = readq(chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); + pintc_mcu_disable(chip_data->pintc_base); + + raw_spin_unlock(&chip_data->pintc_lock); + + return val & (1UL << 8); +} + +static void +pintc_mcu_restore(struct pintc_chip_data *chip_data, unsigned long val) +{ + unsigned long current_val; + + raw_spin_lock(&chip_data->pintc_lock); + + current_val = readq(chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); + current_val &= ~(1UL << 8); + current_val |= val; + writeq(current_val, chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); + + raw_spin_unlock(&chip_data->pintc_lock); } static unsigned long make_pintc_int_target(u32 version, int rcid) @@ -195,10 +220,30 @@ static unsigned long make_pintc_int_target(u32 version, int rcid) return target; } -static int __assign_mcu_irq_config(const struct pintc_chip_data *chip_data, +static void update_pintc_mcu_target(struct pintc_chip_data *chip_data, + unsigned long target) +{ + unsigned long val, flags; + + raw_spin_lock_irqsave(&chip_data->pintc_lock, flags); + + val = readq(chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); + + /* Disable MCU irqs until affinity setting is completed */ + pintc_mcu_disable(chip_data->pintc_base); + + val &= 0xffff; + val |= (target << 16); + + writeq(val, chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); + + raw_spin_unlock_irqrestore(&chip_data->pintc_lock, flags); +} + +static int assign_mcu_irq_config(struct pintc_chip_data *chip_data, cpumask_t *targets) { - unsigned long dev_int_tar, val; + unsigned long dev_int_tar; unsigned int cpu; int rcid; @@ -219,52 +264,32 @@ static int __assign_mcu_irq_config(const struct pintc_chip_data *chip_data, rcid = cpu_to_rcid(cpu); - val = readq(chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); dev_int_tar = make_pintc_int_target(chip_data->version, rcid); - val &= 0xffff; - val |= dev_int_tar << 16; - writeq(val, chip_data->pintc_base + OFFSET_DEV_INT_CONFIG); + update_pintc_mcu_target(chip_data, dev_int_tar); return 0; } -static int assign_mcu_irq_config(const struct pintc_chip_data *chip_data, - cpumask_t *targets) -{ - int ret; - - lock_dev_lock(); - ret = __assign_mcu_irq_config(chip_data, targets); - unlock_dev_lock(); - - return ret; -} - static int mcu_irq_set_affinity(struct irq_data *irq_data, const struct cpumask *dest, bool force) { struct pintc_chip_data *chip_data = irq_data->chip_data; cpumask_t targets; - int ret = 0; if (cpumask_any_and(dest, cpu_online_mask) >= nr_cpu_ids) return -EINVAL; cpumask_and(&targets, dest, cpu_online_mask); - mcu_irq_disable(irq_data); - ret = assign_mcu_irq_config(chip_data, &targets); - mcu_irq_enable(irq_data); - - return ret; + return assign_mcu_irq_config(chip_data, &targets); } static struct irq_chip pintc_mcu_chip = { .name = "MCU-INT", .irq_enable = mcu_irq_enable, .irq_disable = mcu_irq_disable, - .irq_mask = mcu_irq_mask, - .irq_unmask = mcu_irq_unmask, + .irq_mask = mcu_irq_disable, + .irq_unmask = mcu_irq_enable, .irq_set_affinity = mcu_irq_set_affinity, }; @@ -378,6 +403,10 @@ static int __init pintc_init_mcu(struct pintc_chip_data *chip_data, &pintc_mcu_domain_ops, chip_data); /* Mask all interrupts for now */ writeq(0x0, chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); + + /* When building the root domain, move it to a better location */ + if (mcu_irq_domain) + pintc_mcu_enable(chip_data->pintc_base); } if (!mcu_irq_domain) { @@ -385,26 +414,25 @@ static int __init pintc_init_mcu(struct pintc_chip_data *chip_data, return -ENOMEM; } + raw_spin_lock_init(&chip_data->pintc_lock); + raw_spin_lock_init(&chip_data->mcu_lock); + pr_info(PREFIX "MCU version [%u] on node [%u] initialized\n", chip_data->version, chip_data->node); return 0; } +/* Currently, only MCU controller on node 0 is supported */ void handle_dev_int(struct pt_regs *regs) { - void __iomem *mcu_base, *intpu_base; - unsigned long config_val, val, stat; + unsigned long stat, val; unsigned int hwirq; - /* Currently, only MCU controller on node 0 is supported */ - mcu_base = chip_datas[0]->mcu_base; - intpu_base = chip_datas[0]->pintc_base; + /* Disable global irq of MCU due to some hardware reasons */ + val = pintc_mcu_disable_and_save(chip_datas[0]); - config_val = readq(intpu_base + OFFSET_DEV_INT_CONFIG); - val = config_val & (~(1UL << 8)); - writeq(val, intpu_base + OFFSET_DEV_INT_CONFIG); - stat = readq(mcu_base + OFFSET_MCU_DVC_INT); + stat = readq(chip_datas[0]->mcu_base + OFFSET_MCU_DVC_INT); while (stat) { hwirq = ffs(stat) - 1; @@ -412,7 +440,7 @@ void handle_dev_int(struct pt_regs *regs) stat &= ~(1UL << hwirq); } - writeq(config_val, intpu_base + OFFSET_DEV_INT_CONFIG); + pintc_mcu_restore(chip_datas[0], val); } void handle_fault_int(void) -- Gitee From e65999d8739ec3a2ce99c889b8bbb7cb6bca4865 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Mon, 18 Nov 2024 14:06:03 +0800 Subject: [PATCH 368/524] sw64: implementing VDSO with generic code The vdso has been changed to a generic code implementation, adapting to the GENERIC_GETTIMEOFDAY, HAVE_GENERIC_VDSO, and ARCH_CLOCKSOURCE_INIT options. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 3 + arch/sw_64/Makefile | 4 + arch/sw_64/include/asm/clocksource.h | 7 + arch/sw_64/include/asm/vdso/clocksource.h | 8 + arch/sw_64/include/asm/vdso/gettimeofday.h | 105 +++++++++++ arch/sw_64/include/asm/vdso/processor.h | 7 + arch/sw_64/include/asm/vdso/vsyscall.h | 27 +++ arch/sw_64/kernel/time.c | 6 + arch/sw_64/kernel/vdso.c | 22 --- arch/sw_64/kernel/vdso/Makefile | 8 +- arch/sw_64/kernel/vdso/vgettimeofday.c | 192 ++------------------- 11 files changed, 187 insertions(+), 202 deletions(-) create mode 100644 arch/sw_64/include/asm/clocksource.h create mode 100644 arch/sw_64/include/asm/vdso/clocksource.h create mode 100644 arch/sw_64/include/asm/vdso/gettimeofday.h create mode 100644 arch/sw_64/include/asm/vdso/processor.h create mode 100644 arch/sw_64/include/asm/vdso/vsyscall.h diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 8aa38e26d9b2..d538d3ce5ed9 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -9,6 +9,7 @@ config SW64 select ACPI_REDUCED_HARDWARE_ONLY select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI select ARCH_ATOMIC + select ARCH_CLOCKSOURCE_INIT select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_PHYS_TO_DMA @@ -63,6 +64,7 @@ config SW64 select DMA_OPS if PCI select GENERIC_ARCH_TOPOLOGY select GENERIC_CLOCKEVENTS + select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO select GENERIC_IRQ_LEGACY select GENERIC_IRQ_MIGRATION if SMP select GENERIC_IRQ_PROBE @@ -91,6 +93,7 @@ config SW64 select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER + select HAVE_GENERIC_VDSO if MMU && 64BIT select HAVE_IDE select HAVE_KPROBES select HAVE_KPROBES_ON_FTRACE diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index cfea96316c94..ccf2f5d6e378 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -49,6 +49,10 @@ export LIBS_Y boot := arch/sw_64/boot +PHONY += vdso_install +vdso_install: + $(Q)$(MAKE) $(build)=arch/sw64/kernel/vdso $@ + #Default target when executing make with no arguments all: $(boot)/vmlinux.bin.gz diff --git a/arch/sw_64/include/asm/clocksource.h b/arch/sw_64/include/asm/clocksource.h new file mode 100644 index 000000000000..482185566b0c --- /dev/null +++ b/arch/sw_64/include/asm/clocksource.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_CLOCKSOURCE_H +#define _ASM_CLOCKSOURCE_H + +#include + +#endif diff --git a/arch/sw_64/include/asm/vdso/clocksource.h b/arch/sw_64/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..df6ea65c1dec --- /dev/null +++ b/arch/sw_64/include/asm/vdso/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif diff --git a/arch/sw_64/include/asm/vdso/gettimeofday.h b/arch/sw_64/include/asm/vdso/gettimeofday.h new file mode 100644 index 000000000000..57eb602550c7 --- /dev/null +++ b/arch/sw_64/include/asm/vdso/gettimeofday.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_GETTIMEOFDAY_H +#define __ASM_VDSO_GETTIMEOFDAY_H + +#ifndef __ASSEMBLY__ + +#include +#include +#include +#include +#include +#include +#define VDSO_HAS_CLOCK_GETRES 1 + +static __always_inline +int gettimeofday_fallback(struct __kernel_old_timeval *_tv, + struct timezone *_tz) +{ + long retval; + long error; + asm volatile( + " mov %2, $16\n" + " mov %3, $17\n" + " ldi $0, %4\n" + " sys_call %5\n" + " mov $0, %0\n" + " mov $19, %1" + : "=r"(retval), "=r"(error) + : "r"(_tv), "r"(_tz), "i"(__NR_gettimeofday), "i"(HMC_callsys) + : "$0", "$16", "$17", "$19"); + if (unlikely(error)) + return -retval; + else + return retval; +} + +static __always_inline +long clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + long retval; + long error; + asm volatile( + " mov %2, $16\n" + " mov %3, $17\n" + " ldi $0, %4\n" + " sys_call %5\n" + " mov $0, %0\n" + " mov $19, %1" + : "=r"(retval), "=r"(error) + : "r"(_clkid), "r"(_ts), "i"(__NR_clock_gettime), "i"(HMC_callsys) + : "$0", "$16", "$17", "$19"); + if (unlikely(error)) + return -retval; + else + return retval; +} + +static __always_inline +int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + long retval; + long error; + asm volatile( + " mov %2, $16\n" + " mov %3, $17\n" + " ldi $0, %4\n" + " sys_call %5\n" + " mov $0, %0\n" + " mov $19, %1" + : "=r"(retval), "=r"(error) + : "r"(_clkid), "r"(_ts), "i"(__NR_clock_getres), "i"(HMC_callsys) + : "$0", "$16", "$17", "$19"); + if (unlikely(error)) + return -retval; + else + return retval; +} + +#if defined(CONFIG_SUBARCH_C3B) +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, + const struct vdso_data *vd) +{ + register unsigned long __r0 __asm__("$0"); + + __asm__ __volatile__( + "sys_call %1" : "=r"(__r0) : "i" (HMC_longtime)); + + return __r0; +} +#elif defined(CONFIG_SUBARCH_C4) +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, + const struct vdso_data *vd) +{ + return sw64_read_csr(CSR_SHTCLOCK); +} +#endif + +static __always_inline const struct vdso_data *__arch_get_vdso_data(void) +{ + return _vdso_data; +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/sw_64/include/asm/vdso/processor.h b/arch/sw_64/include/asm/vdso/processor.h new file mode 100644 index 000000000000..8bd332e1db87 --- /dev/null +++ b/arch/sw_64/include/asm/vdso/processor.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#include + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/sw_64/include/asm/vdso/vsyscall.h b/arch/sw_64/include/asm/vdso/vsyscall.h new file mode 100644 index 000000000000..edd710ccd851 --- /dev/null +++ b/arch/sw_64/include/asm/vdso/vsyscall.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_VSYSCALL_H +#define __ASM_VDSO_VSYSCALL_H + +#ifndef __ASSEMBLY__ + +#include +#include + +extern struct vdso_data *vdso_data; + +/* + * Update the vDSO data page to keep in sync with kernel timekeeping. + */ +static __always_inline struct vdso_data *__sw64_get_k_vdso_data(void) +{ + return vdso_data; +} + +#define __arch_get_k_vdso_data __sw64_get_k_vdso_data + +/* The asm-generic header needs to be included after the definitions above */ +#include + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_VSYSCALL_H */ diff --git a/arch/sw_64/kernel/time.c b/arch/sw_64/kernel/time.c index c6957d70a551..503f9e1beb43 100644 --- a/arch/sw_64/kernel/time.c +++ b/arch/sw_64/kernel/time.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "proto.h" @@ -42,3 +43,8 @@ time_init(void) /* Calibrate the delay loop directly */ lpj_fine = cycle_freq / HZ; } + +void clocksource_arch_init(struct clocksource *cs) +{ + cs->vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER; +} diff --git a/arch/sw_64/kernel/vdso.c b/arch/sw_64/kernel/vdso.c index 0d0ca5d0a38b..375f9c60a49c 100644 --- a/arch/sw_64/kernel/vdso.c +++ b/arch/sw_64/kernel/vdso.c @@ -156,25 +156,3 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, return PTR_ERR(ret); } -void update_vsyscall(struct timekeeper *tk) -{ - vdso_data_write_begin(vdso_data); - - vdso_data->xtime_sec = tk->xtime_sec; - vdso_data->xtime_nsec = tk->tkr_mono.xtime_nsec; - vdso_data->wall_to_mono_sec = tk->wall_to_monotonic.tv_sec; - vdso_data->wall_to_mono_nsec = tk->wall_to_monotonic.tv_nsec; - vdso_data->cs_shift = tk->tkr_mono.shift; - - vdso_data->cs_mult = tk->tkr_mono.mult; - vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last; - vdso_data->cs_mask = tk->tkr_mono.mask; - - vdso_data_write_end(vdso_data); -} - -void update_vsyscall_tz(void) -{ - vdso_data->tz_minuteswest = sys_tz.tz_minuteswest; - vdso_data->tz_dsttime = sys_tz.tz_dsttime; -} diff --git a/arch/sw_64/kernel/vdso/Makefile b/arch/sw_64/kernel/vdso/Makefile index e5640abbd727..8ae5bf3d1770 100644 --- a/arch/sw_64/kernel/vdso/Makefile +++ b/arch/sw_64/kernel/vdso/Makefile @@ -1,10 +1,16 @@ # SPDX-License-Identifier: GPL-2.0 # Symbols present in the vdso +ARCH_REL_TYPE_ABS := R_SW64_REFLONG|R_SW64_REFQUAD|R_SW64_JMP_SLOT +include $(srctree)/lib/vdso/Makefile vdso-syms = rt_sigreturn gettimeofday getcpu # Files to link into the vdso obj-vdso = $(patsubst %, v%.o, $(vdso-syms)) +ifneq ($(c-gettimeofday-y),) + CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y) +endif + # Build rules targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds vdso-syms.S obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) @@ -35,7 +41,7 @@ $(obj)/vdso.o: $(obj)/vdso.so # link rule for the .so file, .lds has to be first SYSCFLAGS_vdso.so.dbg = $(c_flags) -$(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) FORCE +$(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) $(call if_changed,vdsold) SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ $(call cc-ldoption, -Wl$(comma)--hash-style=both) diff --git a/arch/sw_64/kernel/vdso/vgettimeofday.c b/arch/sw_64/kernel/vdso/vgettimeofday.c index 69dcab66cb10..aecd96bea710 100644 --- a/arch/sw_64/kernel/vdso/vgettimeofday.c +++ b/arch/sw_64/kernel/vdso/vgettimeofday.c @@ -13,192 +13,26 @@ */ #include +#include -#include -#include -#include -#include - -static __always_inline int syscall_fallback(clockid_t clkid, struct timespec64 *ts) -{ - long retval; - long error; - asm volatile( - " mov %2, $16\n" - " mov %3, $17\n" - " ldi $0, %4\n" - " sys_call %5\n" - " mov $0, %0\n" - " mov $19, %1" - : "=r"(retval), "=r"(error) - : "r"(clkid), "r"(ts), "i"(__NR_clock_gettime), "i"(HMC_callsys) - : "$0", "$16", "$17", "$19"); - if (unlikely(error)) - return -retval; - else - return retval; -} - -static __always_inline int do_realtime_coarse(struct timespec64 *ts, - const struct vdso_data *data) +extern +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts); +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) { - u32 start_seq; - - do { - start_seq = vdso_data_read_begin(data); - - ts->tv_sec = data->xtime_sec; - ts->tv_nsec = data->xtime_nsec >> data->cs_shift; - } while (vdso_data_read_retry(data, start_seq)); - - return 0; + return __cvdso_clock_gettime(clock, ts); } - -static __always_inline int do_monotonic_coarse(struct timespec64 *ts, - const struct vdso_data *data) -{ - u32 start_seq; - u64 to_mono_sec; - u64 to_mono_nsec; - - do { - start_seq = vdso_data_read_begin(data); - - ts->tv_sec = data->xtime_sec; - ts->tv_nsec = data->xtime_nsec >> data->cs_shift; - - to_mono_sec = data->wall_to_mono_sec; - to_mono_nsec = data->wall_to_mono_nsec; - } while (vdso_data_read_retry(data, start_seq)); - - ts->tv_sec += to_mono_sec; - timespec64_add_ns(ts, to_mono_nsec); - - return 0; -} - -#if defined(CONFIG_SUBARCH_C3B) -static __always_inline u64 read_longtime(void) -{ - register unsigned long __r0 __asm__("$0"); - - __asm__ __volatile__( - "sys_call %1" : "=r"(__r0) : "i" (HMC_longtime)); - - return __r0; -} -#elif defined(CONFIG_SUBARCH_C4) -static __always_inline u64 read_longtime(void) -{ - return sw64_read_csr(CSR_SHTCLOCK); -} -#endif - -static __always_inline u64 get_ns(const struct vdso_data *data) -{ - u64 cycle_now, delta, nsec; - - cycle_now = read_longtime(); - delta = (cycle_now - data->cs_cycle_last) & data->cs_mask; - - nsec = (delta * data->cs_mult) + data->xtime_nsec; - nsec >>= data->cs_shift; - - return nsec; -} - - -static __always_inline int do_realtime(struct timespec64 *ts, - const struct vdso_data *data) -{ - u32 start_seq; - u64 ns; - - do { - start_seq = vdso_data_read_begin(data); - - ts->tv_sec = data->xtime_sec; - ns = get_ns(data); - } while (vdso_data_read_retry(data, start_seq)); - - ts->tv_nsec = 0; - timespec64_add_ns(ts, ns); - - return 0; -} - -static __always_inline int do_monotonic(struct timespec64 *ts, - const struct vdso_data *data) -{ - u32 start_seq; - u64 ns; - u64 to_mono_sec; - u64 to_mono_nsec; - - do { - start_seq = vdso_data_read_begin(data); - - ts->tv_sec = data->xtime_sec; - ns = get_ns(data); - - to_mono_sec = data->wall_to_mono_sec; - to_mono_nsec = data->wall_to_mono_nsec; - } while (vdso_data_read_retry(data, start_seq)); - - ts->tv_sec += to_mono_sec; - ts->tv_nsec = 0; - timespec64_add_ns(ts, ns + to_mono_nsec); - - return 0; -} - - +extern +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz); int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) { - const struct vdso_data *data = get_vdso_data(); - struct timespec64 ts; - int ret; - - ret = do_realtime(&ts, data); - if (ret) - return ret; - - if (tv) { - tv->tv_sec = ts.tv_sec; - tv->tv_usec = ts.tv_nsec / 1000; - } - - if (tz) { - tz->tz_minuteswest = data->tz_minuteswest; - tz->tz_dsttime = data->tz_dsttime; - } - - return 0; + return __cvdso_gettimeofday(tv, tz); } -int __vdso_clock_gettime(clockid_t clkid, struct timespec64 *ts) +extern +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res); +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res) { - const struct vdso_data *data = get_vdso_data(); - int ret; - - switch (clkid) { - case CLOCK_REALTIME_COARSE: - ret = do_realtime_coarse(ts, data); - break; - case CLOCK_MONOTONIC_COARSE: - ret = do_monotonic_coarse(ts, data); - break; - case CLOCK_REALTIME: - ret = do_realtime(ts, data); - break; - case CLOCK_MONOTONIC: - ret = do_monotonic(ts, data); - break; - default: - /* fall back to a syscall */ - ret = syscall_fallback(clkid, ts); - } - - return ret; + return __cvdso_clock_getres(clock_id, res); } + -- Gitee From d9e32ccdd8f833a1a77f7d86da0d2bf39c03a51a Mon Sep 17 00:00:00 2001 From: Deng Xiaoyun Date: Wed, 16 Oct 2024 17:06:10 +0800 Subject: [PATCH 369/524] sw64: kvm: declare KVM_CAP_SET_GUEST_DEBUG Because QEMU 8.2.0 needs this feature support, so we fix it according to the upstream community. Signed-off-by: Deng Xiaoyun Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/kvm.h | 1 + arch/sw_64/kvm/sw64.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/sw_64/include/uapi/asm/kvm.h b/arch/sw_64/include/uapi/asm/kvm.h index 1949e106563d..8cc5f752bc6b 100644 --- a/arch/sw_64/include/uapi/asm/kvm.h +++ b/arch/sw_64/include/uapi/asm/kvm.h @@ -20,6 +20,7 @@ enum SW64_KVM_IRQ { #define __KVM_HAVE_IRQ_LINE #define __KVM_HAVE_READONLY_MEM +#define __KVM_HAVE_GUEST_DEBUG #define KVM_NR_IRQCHIPS 1 diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index 1213121728c4..b70f50e5610f 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -233,6 +233,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_IOEVENTFD: case KVM_CAP_SYNC_MMU: case KVM_CAP_READONLY_MEM: + case KVM_CAP_SET_GUEST_DEBUG: r = 1; break; case KVM_CAP_NR_VCPUS: -- Gitee From 5c798bce2d2ffc05734f525e59976f2a978604c0 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 12 Nov 2024 14:43:39 +0800 Subject: [PATCH 370/524] sw64: update defconfig Enable NO_HZ_IDLE and disable UDF_FS by default. Recreate defconfig by `make savedefconfig`. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/configs/junzhang_defconfig | 7 +++---- arch/sw_64/configs/xuelang_defconfig | 5 +++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/configs/junzhang_defconfig b/arch/sw_64/configs/junzhang_defconfig index ba2eaafde797..566d0e1a5075 100644 --- a/arch/sw_64/configs/junzhang_defconfig +++ b/arch/sw_64/configs/junzhang_defconfig @@ -3,6 +3,7 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y # CONFIG_CROSS_MEMORY_ATTACH is not set CONFIG_USELIB=y +CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y @@ -28,7 +29,6 @@ CONFIG_DEBUG_PERF_USE_VMALLOC=y CONFIG_SUBARCH_C4=y CONFIG_SMP=y CONFIG_SCHED_SMT=y -CONFIG_NR_CPUS=512 CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_NUMA=y CONFIG_HZ=100 @@ -520,10 +520,9 @@ CONFIG_SERIAL_OF_PLATFORM=y CONFIG_VIRTIO_CONSOLE=y # CONFIG_HW_RANDOM is not set # CONFIG_I2C_COMPAT is not set -CONFIG_I2C=y CONFIG_I2C_CHARDEV=y -CONFIG_I2C_DESIGNWARE_PLATFORM=y CONFIG_I2C_MUX=y +CONFIG_I2C_DESIGNWARE_PLATFORM=y CONFIG_SPI=y CONFIG_SPI_SUNWAY=y CONFIG_SPI_SUNWAY_MMIO=y @@ -573,7 +572,6 @@ CONFIG_FSCACHE=y CONFIG_ISO9660_FS=y CONFIG_JOLIET=y CONFIG_ZISOFS=y -CONFIG_UDF_FS=y CONFIG_MSDOS_FS=y CONFIG_VFAT_FS=y CONFIG_FAT_DEFAULT_UTF8=y @@ -664,6 +662,7 @@ CONFIG_CRYPTO_SHA1=y CONFIG_CRYPTO_DEFLATE=y CONFIG_CRYPTO_LZO=y # CONFIG_CRYPTO_HW is not set +CONFIG_CRC_ITU_T=y CONFIG_CONSOLE_LOGLEVEL_QUIET=7 # CONFIG_ENABLE_MUST_CHECK is not set CONFIG_SCHEDSTATS=y diff --git a/arch/sw_64/configs/xuelang_defconfig b/arch/sw_64/configs/xuelang_defconfig index a255565c411b..64ab45a5095d 100644 --- a/arch/sw_64/configs/xuelang_defconfig +++ b/arch/sw_64/configs/xuelang_defconfig @@ -3,6 +3,7 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y # CONFIG_CROSS_MEMORY_ATTACH is not set CONFIG_USELIB=y +CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y @@ -25,6 +26,7 @@ CONFIG_EXPERT=y CONFIG_KALLSYMS_ALL=y CONFIG_PERF_EVENTS=y CONFIG_DEBUG_PERF_USE_VMALLOC=y +# CONFIG_COMPAT_BRK is not set CONFIG_SMP=y CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_NUMA=y @@ -556,7 +558,6 @@ CONFIG_VIRTIO_MMIO=y CONFIG_STAGING=y CONFIG_IOMMU_DEFAULT_PASSTHROUGH=y CONFIG_SUNWAY_IOMMU=y -CONFIG_SW64_LPC_INTC=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_EXT4_FS_SECURITY=y @@ -570,7 +571,6 @@ CONFIG_FSCACHE=y CONFIG_ISO9660_FS=y CONFIG_JOLIET=y CONFIG_ZISOFS=y -CONFIG_UDF_FS=y CONFIG_MSDOS_FS=y CONFIG_VFAT_FS=y CONFIG_FAT_DEFAULT_UTF8=y @@ -661,6 +661,7 @@ CONFIG_CRYPTO_SHA1=y CONFIG_CRYPTO_DEFLATE=y CONFIG_CRYPTO_LZO=y # CONFIG_CRYPTO_HW is not set +CONFIG_CRC_ITU_T=y CONFIG_CONSOLE_LOGLEVEL_QUIET=7 # CONFIG_FRAME_POINTER is not set CONFIG_SCHEDSTATS=y -- Gitee From 14d7da6173735850d3d899968c27d6f5d5b0cb28 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 3 Jul 2024 16:31:28 +0800 Subject: [PATCH 371/524] sw64: rename _PFN_BITS to __PFN_BITS Rename _PFN_BITS to __PFN_BITS to avoid "_PFN_BITS" redefined when CONFIG_ZSMALLOC is enabled. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index e9843b19916e..ddabb9e1774f 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -200,8 +200,8 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define PFN_PTE_SHIFT _PFN_SHIFT -#define _PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) -#define _PFN_MASK (GENMASK(_PFN_BITS - 1, 0) << _PFN_SHIFT) +#define __PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) +#define _PFN_MASK (GENMASK(__PFN_BITS - 1, 0) << _PFN_SHIFT) #define _PAGE_TABLE (_PAGE_VALID | __DIRTY_BITS | __ACCESS_BITS) #define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL | _PAGE_LEAF | _PAGE_CONT) -- Gitee From c2237aa8861e7a861464abfb89207eec4aa2a7b8 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 14 Nov 2024 14:36:50 +0800 Subject: [PATCH 372/524] sw64 bpf: add BPF_NOSPEC jit Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/net/bpf_jit_comp.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index 64ed06693924..477ec889d217 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -1217,6 +1217,13 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, return ret; break; + /* speculation barrier */ + case BPF_ST | BPF_NOSPEC: + /* + * Nothing required here. + */ + break; + /* ST: *(size *)(dst + off) = imm */ case BPF_ST | BPF_MEM | BPF_W: case BPF_ST | BPF_MEM | BPF_H: -- Gitee From 2eed70dc5a7ad885ae1e7e589e07183273dd89b7 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 3 Jul 2024 17:01:32 +0800 Subject: [PATCH 373/524] sw64 bpf: Add missing uapi header for BPF_PROG_TYPE_PERF_EVENT programs Add missing uapi header for BPF_PROG_TYPE_PERF_EVENT programs by exporting struct user_regs_struct instead of struct pt_regs which is in-kernel only. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/arch/sw_64/include/uapi/asm/bpf_perf_event.h | 9 +++++++++ tools/include/uapi/asm/bpf_perf_event.h | 2 ++ 2 files changed, 11 insertions(+) create mode 100644 tools/arch/sw_64/include/uapi/asm/bpf_perf_event.h diff --git a/tools/arch/sw_64/include/uapi/asm/bpf_perf_event.h b/tools/arch/sw_64/include/uapi/asm/bpf_perf_event.h new file mode 100644 index 000000000000..52f6f1e555f1 --- /dev/null +++ b/tools/arch/sw_64/include/uapi/asm/bpf_perf_event.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SW64_BPF_PERF_EVENT_H +#define _UAPI_ASM_SW64_BPF_PERF_EVENT_H + +#include + +typedef struct user_pt_regs bpf_user_pt_regs_t; + +#endif /* _UAPI_ASM_SW64_BPF_PERF_EVENT_H */ diff --git a/tools/include/uapi/asm/bpf_perf_event.h b/tools/include/uapi/asm/bpf_perf_event.h index ff52668abf8c..8f95170bf4d5 100644 --- a/tools/include/uapi/asm/bpf_perf_event.h +++ b/tools/include/uapi/asm/bpf_perf_event.h @@ -8,6 +8,8 @@ #include "../../arch/riscv/include/uapi/asm/bpf_perf_event.h" #elif defined(__loongarch__) #include "../../arch/loongarch/include/uapi/asm/bpf_perf_event.h" +#elif defined(__sw_64__) +#include "../../arch/sw_64/include/uapi/asm/bpf_perf_event.h" #else #include #endif -- Gitee From cb650cae97be0f9b90cb25dd15e821cef733af1e Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 10 Oct 2023 10:59:33 +0800 Subject: [PATCH 374/524] sw64: ptrace: add NT_SW64_SYSTEM_CALL regset This regeset is intended to be used to get and set a system call number while tracing. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/ptrace.c | 32 ++++++++++++++++++++++++++++++++ include/uapi/linux/elf.h | 1 + 2 files changed, 33 insertions(+) diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index 0af3d35163f9..9c5539f58ded 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -290,9 +290,33 @@ static int fpr_set(struct task_struct *target, sizeof(struct user_fpsimd_state)); } +static int syscall_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + return membuf_store(&to, task_pt_regs(target)->orig_r0); +} + +static int syscall_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned long nr = task_pt_regs(target)->orig_r0; + int ret; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &nr, 0, -1); + if (ret) + return ret; + + task_pt_regs(target)->orig_r0 = nr; + return ret; +} + enum sw64_regset { REGSET_GPR, REGSET_FPR, + REGSET_SYSCALL, }; static const struct user_regset sw64_regsets[] = { @@ -312,6 +336,14 @@ static const struct user_regset sw64_regsets[] = { .regset_get = fpr_get, .set = fpr_set }, + [REGSET_SYSCALL] = { + .core_note_type = NT_SW64_SYSTEM_CALL, + .n = 1, + .size = sizeof(u64), + .align = sizeof(u64), + .regset_get = syscall_get, + .set = syscall_set + } }; static const struct user_regset_view user_sw64_view = { diff --git a/include/uapi/linux/elf.h b/include/uapi/linux/elf.h index 9b731976ce2f..52a2ea81256e 100644 --- a/include/uapi/linux/elf.h +++ b/include/uapi/linux/elf.h @@ -454,6 +454,7 @@ typedef struct elf64_shdr { #define NT_LOONGARCH_LBT 0xa04 /* LoongArch Loongson Binary Translation registers */ #define NT_LOONGARCH_HW_BREAK 0xa05 /* LoongArch hardware breakpoint registers */ #define NT_LOONGARCH_HW_WATCH 0xa06 /* LoongArch hardware watchpoint registers */ +#define NT_SW64_SYSTEM_CALL 0x7f00 /* SW64 system call number */ /* Note types with note name "GNU" */ #define NT_GNU_PROPERTY_TYPE_0 5 -- Gitee From 7cf8dd67c27103447ec9b691bb0d892d28570230 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 14 Oct 2024 14:42:03 +0800 Subject: [PATCH 375/524] sw64: rename mmap protection_map Rename mmap protection_map based on their functions. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 50 +++++++++++++++++++++++++++----- arch/sw_64/mm/init.c | 34 ++++++++++------------ 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index ddabb9e1774f..0b9b65685e74 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -142,9 +142,6 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) * the page is accessed. They are cleared only by the page-out routines */ #define PAGE_NONE __pgprot(__ACCESS_BITS | _PAGE_FOR | _PAGE_FOW | _PAGE_FOE | _PAGE_PROTNONE) -#define PAGE_SHARED __pgprot(_PAGE_VALID | __ACCESS_BITS) -#define PAGE_COPY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW) -#define PAGE_READONLY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW) #define PAGE_KERNEL __pgprot(_PAGE_VALID | _PAGE_ASM | _PAGE_KRE | _PAGE_KWE) #define _PAGE_NORMAL(x) __pgprot(_PAGE_VALID | __ACCESS_BITS | (x)) @@ -189,9 +186,6 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) * the page is accessed. They are cleared only by the page-out routines */ #define PAGE_NONE __pgprot(__ACCESS_BITS | _PAGE_FOR | _PAGE_FOW | _PAGE_FOE | _PAGE_LEAF | _PAGE_PROTNONE) -#define PAGE_SHARED __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_LEAF) -#define PAGE_COPY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW | _PAGE_LEAF) -#define PAGE_READONLY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW | _PAGE_LEAF) #define PAGE_KERNEL __pgprot(_PAGE_VALID | _PAGE_KERN | _PAGE_LEAF) #define _PAGE_NORMAL(x) __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_LEAF | (x)) @@ -206,8 +200,48 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define _PAGE_TABLE (_PAGE_VALID | __DIRTY_BITS | __ACCESS_BITS) #define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL | _PAGE_LEAF | _PAGE_CONT) -#define _PAGE_P(x) _PAGE_NORMAL((x) | _PAGE_FOW) -#define _PAGE_S(x) _PAGE_NORMAL((x) | _PAGE_FOW) +#define PAGE_READONLY_NOEXEC _PAGE_NORMAL(_PAGE_FOE | _PAGE_FOW) +#define PAGE_EXEC _PAGE_NORMAL(_PAGE_FOW | _PAGE_FOR) +#define PAGE_READONLY_EXEC _PAGE_NORMAL(_PAGE_FOW) +#define PAGE_COPY_NOEXEC PAGE_READONLY_NOEXEC +#define PAGE_COPY_EXEC PAGE_READONLY_EXEC +/* + * Since we don't have hardware dirty-bit management yet, shared + * writable page has FOW bit set to make sure dirty-bit could be + * set properly. + */ +#define PAGE_SHARED_NOEXEC PAGE_READONLY_NOEXEC +#define PAGE_SHARED_EXEC PAGE_READONLY_EXEC + +/* For backward compatibility */ +#define PAGE_READONLY PAGE_READONLY_EXEC +#define PAGE_COPY PAGE_COPY_EXEC +#define PAGE_SHARED PAGE_SHARED_EXEC + +/* + * The hardware can handle write-only mappings, but as the sw64 + * architecture does byte-wide writes with a read-modify-write + * sequence, it's not practical to have write-without-read privs. + * Thus the "-w- -> rw-" and "-wx -> rwx" mapping here (and in + * arch/sw_64/mm/fault.c) + */ +#define __P000 PAGE_NONE +#define __P001 PAGE_READONLY_NOEXEC +#define __P010 PAGE_COPY_NOEXEC +#define __P011 PAGE_COPY_NOEXEC +#define __P100 PAGE_EXEC +#define __P101 PAGE_READONLY_EXEC +#define __P110 PAGE_COPY_EXEC +#define __P111 PAGE_COPY_EXEC + +#define __S000 PAGE_NONE +#define __S001 PAGE_READONLY_NOEXEC +#define __S010 PAGE_SHARED_NOEXEC +#define __S011 PAGE_SHARED_NOEXEC +#define __S100 PAGE_EXEC +#define __S101 PAGE_READONLY_EXEC +#define __S110 PAGE_SHARED_EXEC +#define __S111 PAGE_SHARED_EXEC /* * pgprot_noncached() is only for infiniband pci support, and a real diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index f01d2ebef70f..d5e1baf23f72 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -473,23 +473,21 @@ void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) #endif static const pgprot_t protection_map[16] = { - [VM_NONE] = _PAGE_P(_PAGE_FOE | _PAGE_FOW | - _PAGE_FOR), - [VM_READ] = _PAGE_P(_PAGE_FOE | _PAGE_FOW), - [VM_WRITE] = _PAGE_P(_PAGE_FOE), - [VM_WRITE | VM_READ] = _PAGE_P(_PAGE_FOE), - [VM_EXEC] = _PAGE_P(_PAGE_FOW | _PAGE_FOR), - [VM_EXEC | VM_READ] = _PAGE_P(_PAGE_FOW), - [VM_EXEC | VM_WRITE] = _PAGE_P(0), - [VM_EXEC | VM_WRITE | VM_READ] = _PAGE_P(0), - [VM_SHARED] = _PAGE_S(_PAGE_FOE | _PAGE_FOW | - _PAGE_FOR), - [VM_SHARED | VM_READ] = _PAGE_S(_PAGE_FOE | _PAGE_FOW), - [VM_SHARED | VM_WRITE] = _PAGE_S(_PAGE_FOE), - [VM_SHARED | VM_WRITE | VM_READ] = _PAGE_S(_PAGE_FOE), - [VM_SHARED | VM_EXEC] = _PAGE_S(_PAGE_FOW | _PAGE_FOR), - [VM_SHARED | VM_EXEC | VM_READ] = _PAGE_S(_PAGE_FOW), - [VM_SHARED | VM_EXEC | VM_WRITE] = _PAGE_S(0), - [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = _PAGE_S(0) + [VM_NONE] = PAGE_NONE, + [VM_READ] = PAGE_READONLY_NOEXEC, + [VM_WRITE] = PAGE_COPY_NOEXEC, + [VM_WRITE | VM_READ] = PAGE_COPY_NOEXEC, + [VM_EXEC] = PAGE_EXEC, + [VM_EXEC | VM_READ] = PAGE_READONLY_EXEC, + [VM_EXEC | VM_WRITE] = PAGE_COPY_EXEC, + [VM_EXEC | VM_WRITE | VM_READ] = PAGE_COPY_EXEC, + [VM_SHARED] = PAGE_NONE, + [VM_SHARED | VM_READ] = PAGE_READONLY_NOEXEC, + [VM_SHARED | VM_WRITE] = PAGE_SHARED_NOEXEC, + [VM_SHARED | VM_WRITE | VM_READ] = PAGE_SHARED_NOEXEC, + [VM_SHARED | VM_EXEC] = PAGE_EXEC, + [VM_SHARED | VM_EXEC | VM_READ] = PAGE_READONLY_EXEC, + [VM_SHARED | VM_EXEC | VM_WRITE] = PAGE_SHARED_EXEC, + [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_EXEC }; DECLARE_VM_GET_PAGE_PROT -- Gitee From 135ebfa412764562e4afe17f951a0090ab87c8b8 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 3 Jul 2024 17:03:05 +0800 Subject: [PATCH 376/524] libbpf: add sw64 support Add __NR_bpf, bpf_target_sw64, PT_REGS_xxx and libbpf_smp barriers for sw64. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/lib/bpf/bpf.c | 2 ++ tools/lib/bpf/bpf_tracing.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index b0f1913763a3..a5e21d9baf61 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -59,6 +59,8 @@ # define __NR_bpf 6319 # elif defined(__mips__) && defined(_ABI64) # define __NR_bpf 5315 +# elif defined(__sw_64__) +# define __NR_bpf 170 # else # error __NR_bpf not defined. libbpf does not support your arch. # endif diff --git a/tools/lib/bpf/bpf_tracing.h b/tools/lib/bpf/bpf_tracing.h index 1c13f8e88833..8d34cb534ca6 100644 --- a/tools/lib/bpf/bpf_tracing.h +++ b/tools/lib/bpf/bpf_tracing.h @@ -35,6 +35,9 @@ #elif defined(__TARGET_ARCH_loongarch) #define bpf_target_loongarch #define bpf_target_defined +#elif defined(__TARGET_ARCH_sw_64) + #define bpf_target_sw64 + #define bpf_target_defined #else /* Fall back to what the compiler says */ @@ -68,6 +71,9 @@ #elif defined(__loongarch__) #define bpf_target_loongarch #define bpf_target_defined +#elif defined(__sw_64__) + #define bpf_target_sw64 + #define bpf_target_defined #endif /* no compiler target */ #endif @@ -441,6 +447,34 @@ struct pt_regs___arm64 { #define __PT_SP_REG regs[3] #define __PT_IP_REG csr_era +#elif defined(bpf_target_sw64) + +/* sw64 provides struct user_pt_regs instead of struct pt_regs to userspace */ +struct pt_regs; +#define PT_REGS_SW64 const volatile struct user_pt_regs +#define PT_REGS_PARM1(x) (((PT_REGS_SW64 *)(x))->regs[16]) +#define PT_REGS_PARM2(x) (((PT_REGS_SW64 *)(x))->regs[17]) +#define PT_REGS_PARM3(x) (((PT_REGS_SW64 *)(x))->regs[18]) +#define PT_REGS_PARM4(x) (((PT_REGS_SW64 *)(x))->regs[19]) +#define PT_REGS_PARM5(x) (((PT_REGS_SW64 *)(x))->regs[20]) +#define PT_REGS_RET(x) (((PT_REGS_SW64 *)(x))->regs[26]) +/* Works only with CONFIG_FRAME_POINTER */ +#define PT_REGS_FP(x) (((PT_REGS_SW64 *)(x))->regs[15]) +#define PT_REGS_RC(x) (((PT_REGS_SW64 *)(x))->regs[0]) +#define PT_REGS_SP(x) (((PT_REGS_SW64 *)(x))->regs[30]) +#define PT_REGS_IP(x) (((PT_REGS_SW64 *)(x))->pc) + +#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_SW64 *)(x), regs[16]) +#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_SW64 *)(x), regs[17]) +#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_SW64 *)(x), regs[18]) +#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_SW64 *)(x), regs[19]) +#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_SW64 *)(x), regs[20]) +#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_SW64 *)(x), regs[26]) +#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_SW64 *)(x), regs[15]) +#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_SW64 *)(x), regs[0]) +#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_SW64 *)(x), regs[30]) +#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_SW64 *)(x), pc) + #endif #if defined(bpf_target_defined) -- Gitee From ec8e6872fed1758eb6aba37ab027734ebb8e9c22 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Mon, 18 Nov 2024 09:27:21 +0000 Subject: [PATCH 377/524] sw64: iommu: introduce PCI device check before iommu probing Introduce a device type check before iommu probing, as sw64 IOMMUs currently support only PCI devices. Otherwise, IOMMU may try to probe platform devices and cause undefined behaviour. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu_v2.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index bf4ff7d16d2a..a22f9f5e734c 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -1540,14 +1540,11 @@ static struct iommu_device *sunway_iommu_probe_device(struct device *dev) struct sunway_iommu *iommu; int ret; - pdev = to_pci_dev(dev); - if (!pdev) - return ERR_PTR(-ENODEV); - - if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + if (!dev_is_pci(dev)) return ERR_PTR(-ENODEV); - if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) + pdev = to_pci_dev(dev); + if (!pdev) return ERR_PTR(-ENODEV); hose = pci_bus_to_pci_controller(pdev->bus); -- Gitee From f10c7edb72e894c0ad248c9a283f2f2154f69f2a Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Wed, 20 Nov 2024 06:43:35 +0000 Subject: [PATCH 378/524] sw64: iommu: fix def_domain_type callback The current def_domain_type implementation only works if domains have been allocated as IOMMU_DOMAIN_IDENTITY, while default domain allocation requires def_domain_type returns the desired identity type. As a result, domain initialization fails to get the default type and ends unexpectedly. The def_domain_type callback should designate default domains as passthrough in the first place, especially before their allocation. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 9 +-------- drivers/iommu/sw64/iommu_v2.c | 7 +------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index f705d10830ff..75a3f34236d8 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -75,8 +75,6 @@ struct dma_domain { }; const struct iommu_ops sunway_iommu_ops; -static int iommu_identity_mapping; - static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) { *(u16 *)data = alias; @@ -1035,7 +1033,6 @@ static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type) } sdomain->type = IOMMU_DOMAIN_IDENTITY; - iommu_identity_mapping = 1; break; default: @@ -1311,11 +1308,7 @@ static struct iommu_device *sunway_iommu_probe_device(struct device *dev) static int sunway_iommu_def_domain_type(struct device *dev) { - if (dev_is_pci(dev)) - if (iommu_identity_mapping) - return IOMMU_DOMAIN_IDENTITY; - - return 0; + return IOMMU_DOMAIN_IDENTITY; } static bool sunway_iommu_capable(struct device *dev, enum iommu_cap cap) diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index a22f9f5e734c..4ac51591661b 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -88,7 +88,6 @@ struct dma_domain { }; const struct iommu_ops sunway_iommu_ops; static const struct dma_map_ops sunway_dma_ops; -static int iommu_identity_mapping; static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) { @@ -1570,11 +1569,7 @@ static struct iommu_device *sunway_iommu_probe_device(struct device *dev) static int sunway_iommu_def_domain_type(struct device *dev) { - if (dev_is_pci(dev)) - if (iommu_identity_mapping) - return IOMMU_DOMAIN_IDENTITY; - - return 0; + return IOMMU_DOMAIN_IDENTITY; } static bool sunway_iommu_capable(struct device *dev, enum iommu_cap cap) -- Gitee From f29641fd09a6cff6a2d114909a9c5b98f093cbc3 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Wed, 20 Nov 2024 09:32:53 +0000 Subject: [PATCH 379/524] sw64: iommu: fix work around of iommu interrupt remapping capability According to commit b062007c63eb ("iommu: Remove IOMMU_CAP_INTR_REMAP"), the original interrupt remapping capability work around is now ineffective. Introduce a new flag to work around the interrupt remapping check. The discussion upon this can be seen in commit 9a662748100e ("sw64: iommu: work around interrupt remapping capability"). Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/msi.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/include/asm/msi.h b/arch/sw_64/include/asm/msi.h index 5f2cfe7f4bd7..92e0109eec31 100644 --- a/arch/sw_64/include/asm/msi.h +++ b/arch/sw_64/include/asm/msi.h @@ -49,6 +49,8 @@ extern int __init msic_acpi_init(struct irq_domain *parent, #define MSI_ADDR_SHIFT 20 #define MSI_ADDR_DEST_ID_SHIFT 10 +#define arch_is_isolated_msi() true + struct sw64_msi_chip_data { spinlock_t cdata_lock; union { -- Gitee From 1a8d9c42fe36f105209f01ba3e7a20caa2e207fc Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Mon, 25 Nov 2024 03:23:21 +0000 Subject: [PATCH 380/524] sw64: iommu: mark hose->iommu_enable as true before probing The iommu_device_register callback requires hose->iommu_enable become true before probing devices. Adjust the initialization process to fix this problem. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu_v2.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index 4ac51591661b..de8c57e333ca 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -977,12 +977,12 @@ static int sunway_iommu_acpi_init(void) if (!iommu->enabled || hose->iommu_enable) continue; + sunway_enable_iommu_func(hose); + hose->iommu_enable = true; iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", iommu_index); iommu_device_register(&iommu->iommu, &sunway_iommu_ops, NULL); iommu_index++; - sunway_enable_iommu_func(hose); - hose->iommu_enable = true; piu_flush_all(iommu); } @@ -1015,12 +1015,12 @@ static int sunway_iommu_legacy_init(void) continue; iommu = sunway_iommu_early_init(hose); + sunway_enable_iommu_func(hose); + hose->iommu_enable = true; iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", iommu_index); iommu_device_register(&iommu->iommu, &sunway_iommu_ops, NULL); iommu_index++; - sunway_enable_iommu_func(hose); - hose->iommu_enable = true; piu_flush_all(iommu); } -- Gitee From 7a3fa0a64fbe7b35da96111c145aebd15d89c5f6 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Wed, 13 Nov 2024 11:17:05 +0800 Subject: [PATCH 381/524] sw64: kexec: fix a compilation error This patch fixes a compilation error that resulted when KEXEC and CRASH_DUMP are selected. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/elf.h | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/include/asm/elf.h b/arch/sw_64/include/asm/elf.h index 95ba89a1aa9d..f94d69a5b110 100644 --- a/arch/sw_64/include/asm/elf.h +++ b/arch/sw_64/include/asm/elf.h @@ -4,6 +4,7 @@ #ifdef __KERNEL__ #include #endif +#include /* Special values for the st_other field in the symbol table. */ @@ -59,7 +60,7 @@ * * For now, we just leave it at 33 (32 general regs + processor status word). */ -#define ELF_NGREG 33 +#define ELF_NGREG (sizeof(struct user_pt_regs) / sizeof(elf_greg_t)) typedef unsigned long elf_greg_t; typedef elf_greg_t elf_gregset_t[ELF_NGREG]; @@ -118,8 +119,14 @@ extern int arch_setup_additional_pages(struct linux_binprm *bprm, #ifdef __KERNEL__ struct pt_regs; struct task_struct; -extern void sw64_elf_core_copy_regs(elf_greg_t *dest, struct pt_regs *pt); -#define ELF_CORE_COPY_REGS(DEST, REGS) sw64_elf_core_copy_regs(DEST, REGS); +#define ELF_CORE_COPY_REGS(dest, regs) \ +do { \ + struct thread_info *ti; \ + \ + *(struct user_pt_regs *)&(dest) = (regs)->user_regs; \ + ti = (void *)((__u64)regs & ~(THREAD_SIZE - 1)); \ + dest[ELF_NGREG] = ti->pcb.tp; \ +} while (0); /* * This yields a mask that user programs can use to figure out what -- Gitee From d1d7488c235ff30af12aa60f278a92d45ca60ca3 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Wed, 13 Nov 2024 13:59:13 +0800 Subject: [PATCH 382/524] sw64: kexec: fix a bug in menuconfig This patch fixes a bug where menuconfig could not select KEXEC and CRASH_DUMP due to upstream changes in kernel/Kconfig.kexec. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index d538d3ce5ed9..eda9e2a74b4a 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -317,34 +317,6 @@ config PHYSICAL_START (the "X" value as specified in the "crashkernel=YM@XM" command line boot parameter passed to the panic-ed kernel). -config KEXEC - bool "Kexec system call (EXPERIMENTAL)" - select KEXEC_CORE - help - kexec is a system call that implements the ability to shutdown your - current kernel, and to start another kernel. It is like a reboot - but it is independent of the system firmware. And like a reboot - you can start any kernel with it, not just Linux. - - The name comes from the similarity to the exec system call. - - It is an ongoing process to be certain the hardware in a machine - is properly shutdown, so do not be surprised if this code does not - initially work for you. As of this writing the exact hardware - interface is strongly in flux, so no good recommendation can be - made. - -config CRASH_DUMP - bool "Kernel crash dumps (EXPERIMENTAL)" - help - Generate crash dump after being started by kexec. - This should be normally only set in special crash dump kernels - which are loaded in the main kernel with kexec-tools into - a specially reserved region and then later executed after - a crash by kdump/kexec. The crash dump kernel must be compiled - to a memory address not used by the main kernel or firmware using - PHYSICAL_START. - config SECCOMP def_bool y prompt "Enable seccomp to safely compute untrusted bytecode" @@ -399,6 +371,12 @@ config TRACE_IRQFLAGS_SUPPORT config ARCH_SUPPORTS_UPROBES def_bool y +config ARCH_SUPPORTS_KEXEC + def_bool y + +config ARCH_SUPPORTS_CRASH_DUMP + def_bool y + config SCHED_SMT bool "SMT scheduler support" depends on SMP && SUBARCH_C4 -- Gitee From 27c50607a3edfaf5eadbf946eff716d0899ee1d3 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 29 Nov 2024 01:21:59 +0000 Subject: [PATCH 383/524] sw64: iommu: mark hose->iommu_enable as true before probing on iommu_v1 Following the same behaviour of commit e36b50274e3d ("sw64: iommu: mark hose->iommu_enable as true before probing"), but on iommu_v1. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index 75a3f34236d8..2336e2704b01 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -804,12 +804,12 @@ static int sunway_iommu_init(void) continue; } + sunway_enable_iommu_func(hose); + hose->iommu_enable = true; iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", iommu_index); iommu_device_register(&iommu->iommu, &sunway_iommu_ops, NULL); - sunway_enable_iommu_func(hose); - hose->iommu_enable = true; piu_flush_all(iommu); iommu_index++; @@ -1294,8 +1294,10 @@ static struct iommu_device *sunway_iommu_probe_device(struct device *dev) if (!hose->iommu_enable) return ERR_PTR(-ENODEV); - if (dev_iommu_priv_get(dev)) + if (dev_iommu_priv_get(dev)) { + iommu = hose->pci_iommu; return &iommu->iommu; + } ret = iommu_init_device(dev); if (ret) -- Gitee From fadd222960e3d9b78695ace148899aefccab707d Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Wed, 6 Nov 2024 14:59:14 +0800 Subject: [PATCH 384/524] sw64: bpf: fix XADD32 and XADD64 This patch fixes a bug in emit_sw64_xadd32 and emit_sw64_xadd64 that reaults in instruction faults. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/net/bpf_jit_comp.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index 477ec889d217..4229904d9006 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -331,7 +331,7 @@ static void emit_sw64_xadd32(const int src, int dst, s16 off, struct jit_ctx *ct int atomic_end; u8 tmp1 = get_tmp_reg(ctx); u8 tmp2 = get_tmp_reg(ctx); - u8 tmp3 = get_tmp_reg(ctx); + u8 __maybe_unused tmp3 = get_tmp_reg(ctx); if (off < -0x800 || off > 0x7ff) { emit(SW64_BPF_LDI(tmp1, dst, off), ctx); @@ -341,15 +341,19 @@ static void emit_sw64_xadd32(const int src, int dst, s16 off, struct jit_ctx *ct atomic_start = ctx->idx; emit(SW64_BPF_LLDW(tmp2, dst, off), ctx); +#if defined(CONFIG_SUBARCH_C3B) emit(SW64_BPF_LDI(tmp3, SW64_BPF_REG_ZR, 1), ctx); emit(SW64_BPF_WR_F(tmp3), ctx); +#endif emit(SW64_BPF_ADDW_REG(tmp2, src, tmp2), ctx); if (ctx->idx & 1) emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, SW64_BPF_REG_ZR), ctx); emit(SW64_BPF_LSTW(tmp2, dst, off), ctx); - emit(SW64_BPF_RD_F(tmp3), ctx); +#if defined(CONFIG_SUBARCH_C3B) + emit(SW64_BPF_RD_F(tmp2), ctx); +#endif atomic_end = ctx->idx; - emit(SW64_BPF_BEQ(tmp3, atomic_start - atomic_end - 1), ctx); + emit(SW64_BPF_BEQ(tmp2, atomic_start - atomic_end - 1), ctx); put_tmp_reg(ctx); put_tmp_reg(ctx); @@ -363,7 +367,7 @@ static void emit_sw64_xadd64(const int src, int dst, s16 off, struct jit_ctx *ct int atomic_end; u8 tmp1 = get_tmp_reg(ctx); u8 tmp2 = get_tmp_reg(ctx); - u8 tmp3 = get_tmp_reg(ctx); + u8 __maybe_unused tmp3 = get_tmp_reg(ctx); if (off < -0x800 || off > 0x7ff) { emit(SW64_BPF_LDI(tmp1, dst, off), ctx); @@ -373,15 +377,19 @@ static void emit_sw64_xadd64(const int src, int dst, s16 off, struct jit_ctx *ct atomic_start = ctx->idx; emit(SW64_BPF_LLDL(tmp2, dst, off), ctx); +#if defined(CONFIG_SUBARCH_C3B) emit(SW64_BPF_LDI(tmp3, SW64_BPF_REG_ZR, 1), ctx); emit(SW64_BPF_WR_F(tmp3), ctx); +#endif emit(SW64_BPF_ADDL_REG(tmp2, src, tmp2), ctx); if (ctx->idx & 1) emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, SW64_BPF_REG_ZR), ctx); emit(SW64_BPF_LSTL(tmp2, dst, off), ctx); - emit(SW64_BPF_RD_F(tmp3), ctx); +#if defined(CONFIG_SUBARCH_C3B) + emit(SW64_BPF_RD_F(tmp2), ctx); +#endif atomic_end = ctx->idx; - emit(SW64_BPF_BEQ(tmp3, atomic_start - atomic_end - 1), ctx); + emit(SW64_BPF_BEQ(tmp2, atomic_start - atomic_end - 1), ctx); put_tmp_reg(ctx); put_tmp_reg(ctx); -- Gitee From 21dea9d13194b11ccf4cabd4e78ffe87c7b78318 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Wed, 6 Nov 2024 15:09:59 +0800 Subject: [PATCH 385/524] sw64: bpf: optimize some instructions This patch adds BPF_DIV, BPF_MOD and BPF_TO_BE instructions to core4. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/net/bpf_jit.h | 28 ++++++++++++++++++ arch/sw_64/net/bpf_jit_comp.c | 53 +++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/arch/sw_64/net/bpf_jit.h b/arch/sw_64/net/bpf_jit.h index 929036d8ea6b..bb6399d61e38 100644 --- a/arch/sw_64/net/bpf_jit.h +++ b/arch/sw_64/net/bpf_jit.h @@ -75,7 +75,11 @@ #define SW64_BPF_FUNC_ALU_ADDL 0x08 #define SW64_BPF_FUNC_ALU_SUBL 0x09 #define SW64_BPF_FUNC_ALU_MULW 0x10 +#define SW64_BPF_FUNC_ALU_UDIVW 0x12 +#define SW64_BPF_FUNC_ALU_UREMW 0x14 #define SW64_BPF_FUNC_ALU_MULL 0x18 +#define SW64_BPF_FUNC_ALU_UDIVL 0x1b +#define SW64_BPF_FUNC_ALU_UREML 0x1d #define SW64_BPF_FUNC_ALU_CMPEQ 0x28 #define SW64_BPF_FUNC_ALU_CMPLT 0x29 #define SW64_BPF_FUNC_ALU_CMPLE 0x2A @@ -90,6 +94,9 @@ #define SW64_BPF_FUNC_ALU_SLL 0x48 #define SW64_BPF_FUNC_ALU_SRL 0x49 #define SW64_BPF_FUNC_ALU_SRA 0x4A +#define SW64_BPF_FUNC_ALU_REVBH 0x5b +#define SW64_BPF_FUNC_ALU_REVBW 0x5c +#define SW64_BPF_FUNC_ALU_REVBL 0x5d #define SW64_BPF_FUNC_ALU_ZAP 0x68 #define SW64_BPF_FUNC_ALU_ZAPNOT 0x69 #define SW64_BPF_FUNC_ALU_SEXTB 0x6A @@ -193,9 +200,30 @@ enum sw64_bpf_registers { #define SW64_BPF_MULW_REG(ra, rb, dst) \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ ra, rb, dst, SW64_BPF_FUNC_ALU_MULW) +#define SW64_BPF_UDIVW_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_UDIVW) +#define SW64_BPF_UREMW_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_UREMW) #define SW64_BPF_MULL_REG(ra, rb, dst) \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ ra, rb, dst, SW64_BPF_FUNC_ALU_MULL) +#define SW64_BPF_UDIVL_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_UDIVL) +#define SW64_BPF_UREML_REG(ra, rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + ra, rb, dst, SW64_BPF_FUNC_ALU_UREML) +#define SW64_BPF_REVBH_REG(rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + SW64_BPF_REG_ZR, rb, dst, SW64_BPF_FUNC_ALU_REVBH) +#define SW64_BPF_REVBW_REG(rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + SW64_BPF_REG_ZR, rb, dst, SW64_BPF_FUNC_ALU_REVBW) +#define SW64_BPF_REVBL_REG(rb, dst) \ + sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ + SW64_BPF_REG_ZR, rb, dst, SW64_BPF_FUNC_ALU_REVBL) #define SW64_BPF_ZAP_REG(ra, rb, dst) \ sw64_bpf_gen_format_simple_alu_reg(SW64_BPF_OPCODE_ALU_REG, \ ra, rb, dst, SW64_BPF_FUNC_ALU_ZAP) diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index 4229904d9006..352769e42ce4 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -277,6 +277,7 @@ static void emit_sw64_ldu64(const int dst, const u64 imm, struct jit_ctx *ctx) put_tmp_reg(ctx); } +#if defined(CONFIG_SUBARCH_C3B) /* Do not change!!! See arch/sw_64/lib/divide.S for more detail */ #define REG(x) "$"str(x) #define str(x) #x @@ -323,6 +324,7 @@ static void emit_sw64_divmod(const int dst, const int src, struct jit_ctx *ctx, #undef DIVIDEND #undef DIVISOR #undef RESULT +#endif /* STX XADD: lock *(u32 *)(dst + off) += src */ static void emit_sw64_xadd32(const int src, int dst, s16 off, struct jit_ctx *ctx) @@ -396,6 +398,7 @@ static void emit_sw64_xadd64(const int src, int dst, s16 off, struct jit_ctx *ct put_tmp_reg(ctx); } +#if defined(CONFIG_SUBARCH_C3B) static void emit_sw64_htobe16(const int dst, struct jit_ctx *ctx) { u8 tmp = get_tmp_reg(ctx); @@ -472,6 +475,7 @@ static void emit_sw64_htobe64(const int dst, struct jit_ctx *ctx) put_tmp_reg(ctx); put_tmp_reg(ctx); } +#endif static void jit_fill_hole(void *area, unsigned int size) { @@ -715,16 +719,34 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, emit(SW64_BPF_MULL_REG(dst, src, dst), ctx); break; case BPF_ALU | BPF_DIV | BPF_X: +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_divmod(dst, src, ctx, code); +#else + emit(SW64_BPF_UDIVW_REG(dst, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); +#endif break; case BPF_ALU64 | BPF_DIV | BPF_X: +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_divmod(dst, src, ctx, code); +#else + emit(SW64_BPF_UDIVL_REG(dst, src, dst), ctx); +#endif break; case BPF_ALU | BPF_MOD | BPF_X: +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_divmod(dst, src, ctx, code); +#else + emit(SW64_BPF_UREMW_REG(dst, src, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); +#endif break; case BPF_ALU64 | BPF_MOD | BPF_X: +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_divmod(dst, src, ctx, code); +#else + emit(SW64_BPF_UREML_REG(dst, src, dst), ctx); +#endif break; case BPF_ALU | BPF_LSH | BPF_X: emit(SW64_BPF_SLL_REG(dst, src, dst), ctx); @@ -794,13 +816,26 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, case BPF_ALU | BPF_END | BPF_TO_BE: switch (imm) { case 16: +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_htobe16(dst, ctx); +#else + emit(SW64_BPF_REVBH_REG(dst, dst), ctx); +#endif break; case 32: +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_htobe32(dst, ctx); +#else + emit(SW64_BPF_REVBW_REG(dst, dst), ctx); + emit(SW64_BPF_ZAPNOT_IMM(dst, 0xf, dst), ctx); +#endif break; case 64: +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_htobe64(dst, ctx); +#else + emit(SW64_BPF_REVBL_REG(dst, dst), ctx); +#endif break; default: pr_err("eBPF JIT %s[%d]: BPF_TO_BE unknown size\n", @@ -875,19 +910,37 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, break; case BPF_ALU | BPF_DIV | BPF_K: emit_sw64_ldu32(tmp1, imm, ctx); +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_divmod(dst, tmp1, ctx, code); +#else + emit(SW64_BPF_UDIVW_REG(dst, tmp1, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); +#endif break; case BPF_ALU64 | BPF_DIV | BPF_K: emit_sw64_lds32(tmp1, imm, ctx); +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_divmod(dst, tmp1, ctx, code); +#else + emit(SW64_BPF_UDIVL_REG(dst, tmp1, dst), ctx); +#endif break; case BPF_ALU | BPF_MOD | BPF_K: emit_sw64_ldu32(tmp1, imm, ctx); +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_divmod(dst, tmp1, ctx, code); +#else + emit(SW64_BPF_UREMW_REG(dst, tmp1, dst), ctx); + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); +#endif break; case BPF_ALU64 | BPF_MOD | BPF_K: emit_sw64_lds32(tmp1, imm, ctx); +#if defined(CONFIG_SUBARCH_C3B) emit_sw64_divmod(dst, tmp1, ctx, code); +#else + emit(SW64_BPF_UREML_REG(dst, tmp1, dst), ctx); +#endif break; case BPF_ALU | BPF_LSH | BPF_K: if (imm >= 0 && imm <= U8_MAX) { -- Gitee From 9126c3011c70ce07b433132ebc7647e987a9406c Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 7 Nov 2024 17:26:24 +0800 Subject: [PATCH 386/524] sw64: bpf: fix BPF_CALL JIT for multi-function programs With bpf-to-bpf calls the dynamically allocated function start is unknown until all functions of the program are JITed, and we get 0 as BPF_CALL target addr until the final pass. This causes incorrect ctx->epilogue_offset and BPF_EXIT jump offset in the final image. To fix it, add a new function emit_sw64_load_call_addr() to emit BPF_CALL target address. It's similar to emit_sw64_ldu64, except it always emit 12 instructions. This will make sure ctx->idx does not change in multiple jit passes. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/net/bpf_jit_comp.c | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index 352769e42ce4..0e846b00e1fc 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -277,6 +277,38 @@ static void emit_sw64_ldu64(const int dst, const u64 imm, struct jit_ctx *ctx) put_tmp_reg(ctx); } +/* constant insn count */ +static void emit_sw64_load_call_addr(u64 addr, struct jit_ctx *ctx) +{ + u16 imm_tmp; + u8 dst = SW64_BPF_REG_PV; + u8 reg_tmp = get_tmp_reg(ctx); + + imm_tmp = (addr >> 60) & 0xf; + emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm_tmp), ctx); + emit(SW64_BPF_SLL_IMM(dst, 60, dst), ctx); + + imm_tmp = (addr >> 45) & 0x7fff; + emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); + emit(SW64_BPF_SLL_IMM(reg_tmp, 45, reg_tmp), ctx); + emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); + + imm_tmp = (addr >> 30) & 0x7fff; + emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); + emit(SW64_BPF_SLL_IMM(reg_tmp, 30, reg_tmp), ctx); + emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); + + imm_tmp = (addr >> 15) & 0x7fff; + emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); + emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx); + emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); + + imm_tmp = addr & 0x7fff; + emit(SW64_BPF_LDI(dst, dst, imm_tmp), ctx); + + put_tmp_reg(ctx); +} + #if defined(CONFIG_SUBARCH_C3B) /* Do not change!!! See arch/sw_64/lib/divide.S for more detail */ #define REG(x) "$"str(x) @@ -1211,7 +1243,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, if (ret < 0) return ret; - emit_sw64_ldu64(SW64_BPF_REG_PV, func, ctx); + emit_sw64_load_call_addr(func, ctx); emit(SW64_BPF_CALL(SW64_BPF_REG_RA, SW64_BPF_REG_PV), ctx); break; } -- Gitee From 1ba68cd385119f9e531ab7e72c21ab041e1fc57a Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 11 Nov 2024 14:08:01 +0800 Subject: [PATCH 387/524] sw64: bpf: optimize JIT for load imm Optimize 32-bit and 64-bit load immediate. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/net/bpf_jit_comp.c | 153 ++++++++++++---------------------- 1 file changed, 51 insertions(+), 102 deletions(-) diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index 0e846b00e1fc..5c06c224cfee 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -145,134 +145,83 @@ static inline void put_tmp_reg(struct jit_ctx *ctx) ctx->current_tmp_reg = 7; } -static void emit_sw64_ldu32(const int dst, const u32 imm, struct jit_ctx *ctx) +static void emit_sw64_lds32(const int dst, const s32 imm, struct jit_ctx *ctx) { - u16 imm_tmp; - u8 reg_tmp = get_tmp_reg(ctx); + s16 hi = imm >> 16; + s16 lo = imm & 0xffff; - if (!imm) { - emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx); - put_tmp_reg(ctx); - return; - } + if (!imm) + return emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx); - if (imm <= S16_MAX) { - emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); - put_tmp_reg(ctx); - return; - } + if (imm >= S16_MIN && imm <= S16_MAX) + return emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); - if (imm >= U32_MAX - S16_MAX) { - emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); - emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); - put_tmp_reg(ctx); - return; - } - - imm_tmp = (imm >> 30) & 3; - emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm_tmp), ctx); - if (imm_tmp) - emit(SW64_BPF_SLL_IMM(dst, 30, dst), ctx); - - imm_tmp = (imm >> 15) & 0x7fff; - if (imm_tmp) { - emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); - emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx); - emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); + if (lo < 0) { + if (hi == S16_MAX) { + emit(SW64_BPF_LDIH(dst, SW64_BPF_REG_ZR, (hi >> 1) + 1), ctx); + emit(SW64_BPF_ADDL_REG(dst, dst, dst), ctx); + } else { + emit(SW64_BPF_LDIH(dst, SW64_BPF_REG_ZR, hi + 1), ctx); + } + } else { + emit(SW64_BPF_LDIH(dst, SW64_BPF_REG_ZR, hi), ctx); } + if (lo) + emit(SW64_BPF_LDI(dst, dst, lo), ctx); +} - imm_tmp = imm & 0x7fff; - if (imm_tmp) - emit(SW64_BPF_LDI(dst, dst, imm_tmp), ctx); - - put_tmp_reg(ctx); +static void emit_sw64_ldu32(const int dst, const u32 imm, struct jit_ctx *ctx) +{ + emit_sw64_lds32(dst, imm, ctx); + if ((s32)imm < 0) + emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); } -static void emit_sw64_lds32(const int dst, const s32 imm, struct jit_ctx *ctx) +/* + * Load high 32 bits for emit_sw64_ldu64(). + * + * Same as emit_sw64_lds32() except this one does not worry about sign extension + * and generates a 32 bits left shift at the end. + */ +static void emit_sw64_ld32_hi(const int dst, const s32 imm, struct jit_ctx *ctx) { s16 hi = imm >> 16; s16 lo = imm & 0xffff; - u8 reg_tmp = get_tmp_reg(ctx); - if (!imm) { - emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx); - put_tmp_reg(ctx); - return; - } + if (!imm) + return emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx); if (imm >= S16_MIN && imm <= S16_MAX) { emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); - put_tmp_reg(ctx); - return; + goto out; } + if (lo < 0) + hi++; emit(SW64_BPF_LDIH(dst, SW64_BPF_REG_ZR, hi), ctx); - if (lo & 0x8000) { // sign bit is 1 - lo = lo & 0x7fff; - emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, 1), ctx); - emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx); - emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); - if (lo) - emit(SW64_BPF_LDI(dst, dst, lo), ctx); - } else { // sign bit is 0 - if (lo) - emit(SW64_BPF_LDI(dst, dst, lo), ctx); - } - - put_tmp_reg(ctx); + if (lo) + emit(SW64_BPF_LDI(dst, dst, lo), ctx); +out: + emit(SW64_BPF_SLL_IMM(dst, 32, dst), ctx); } static void emit_sw64_ldu64(const int dst, const u64 imm, struct jit_ctx *ctx) { - u16 imm_tmp; - u8 reg_tmp = get_tmp_reg(ctx); + u32 hi = imm >> 32; + u32 lo = imm & 0xffffffff; + u8 reg_tmp; - if (!imm) { - emit(SW64_BPF_BIS_REG(SW64_BPF_REG_ZR, SW64_BPF_REG_ZR, dst), ctx); - put_tmp_reg(ctx); - return; - } - - if (imm <= U32_MAX) { - put_tmp_reg(ctx); - return emit_sw64_ldu32(dst, (u32)imm, ctx); - } + if (!hi) + return emit_sw64_ldu32(dst, lo, ctx); - if (imm >= (U64_MAX - S16_MAX) || imm <= S16_MAX) { - emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm), ctx); - put_tmp_reg(ctx); - return; - } + if (!lo) + return emit_sw64_ld32_hi(dst, hi, ctx); - imm_tmp = (imm >> 60) & 0xf; - emit(SW64_BPF_LDI(dst, SW64_BPF_REG_ZR, imm_tmp), ctx); - if (imm_tmp) - emit(SW64_BPF_SLL_IMM(dst, 60, dst), ctx); - - imm_tmp = (imm >> 45) & 0x7fff; - if (imm_tmp) { - emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); - emit(SW64_BPF_SLL_IMM(reg_tmp, 45, reg_tmp), ctx); - emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); - } - - imm_tmp = (imm >> 30) & 0x7fff; - if (imm_tmp) { - emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); - emit(SW64_BPF_SLL_IMM(reg_tmp, 30, reg_tmp), ctx); - emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); - } - - imm_tmp = (imm >> 15) & 0x7fff; - if (imm_tmp) { - emit(SW64_BPF_LDI(reg_tmp, SW64_BPF_REG_ZR, imm_tmp), ctx); - emit(SW64_BPF_SLL_IMM(reg_tmp, 15, reg_tmp), ctx); - emit(SW64_BPF_ADDL_REG(dst, reg_tmp, dst), ctx); - } + reg_tmp = get_tmp_reg(ctx); - imm_tmp = imm & 0x7fff; - if (imm_tmp) - emit(SW64_BPF_LDI(dst, dst, imm_tmp), ctx); + emit_sw64_ld32_hi(dst, hi, ctx); + emit_sw64_ldu32(reg_tmp, lo, ctx); + emit(SW64_BPF_BIS_REG(dst, reg_tmp, dst), ctx); put_tmp_reg(ctx); } -- Gitee From f396173a56839b733f1bb11050395e86fd5de831 Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Wed, 13 Nov 2024 17:05:51 +0800 Subject: [PATCH 388/524] sw64: perf: fix specific performance events can be sampled Some of the general-purpose performance counting events are fetched by reading the specific performance counters, but unfortunately the specific performance counters are unable to generate performance counting interrupts. Now, we mask the corresponding events in sampling. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/perf_event_c4.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/kernel/perf_event_c4.c b/arch/sw_64/kernel/perf_event_c4.c index 302d3c65b61f..b5a9c8363800 100644 --- a/arch/sw_64/kernel/perf_event_c4.c +++ b/arch/sw_64/kernel/perf_event_c4.c @@ -507,7 +507,7 @@ static void hw_perf_event_destroy(struct perf_event *event) /* Nothing to be done! */ } -static void __hw_perf_event_init(struct perf_event *event) +static int __hw_perf_event_init(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -515,6 +515,18 @@ static void __hw_perf_event_init(struct perf_event *event) if (!is_sampling_event(event)) pr_debug("not sampling event\n"); + else { + switch (hwc->config) { + case SW64_L1I_CACHE: + case SW64_L1I_CACHE_MISSES: + case SW64_PMU_INSTRUCTIONS: + case SW64_PMU_BRANCH: + case SW64_PMU_BRANCH_MISSES: + return -EOPNOTSUPP; + default: + break; + } + } event->destroy = hw_perf_event_destroy; @@ -523,6 +535,8 @@ static void __hw_perf_event_init(struct perf_event *event) hwc->last_period = hwc->sample_period; local64_set(&hwc->period_left, hwc->sample_period); } + + return 0; } /* @@ -581,9 +595,7 @@ static int sw64_pmu_event_init(struct perf_event *event) hwc->config = config; /* Do the real initialisation work. */ - __hw_perf_event_init(event); - - return 0; + return __hw_perf_event_init(event); } static struct pmu pmu = { -- Gitee From 269066a965898340217ef457840a23bbe43702a0 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Mon, 2 Dec 2024 10:41:16 +0800 Subject: [PATCH 389/524] sw64: fix kprobe function Some function symbols should not be detected by kprobe, such as those in entry.S and debugging-related functions, and should be added to the kprobebacklist. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry-ftrace.S | 3 ++- arch/sw_64/kernel/entry_c3.S | 2 ++ arch/sw_64/kernel/entry_c4.S | 2 ++ arch/sw_64/kernel/idle.c | 2 +- arch/sw_64/kernel/kprobes/kprobes.c | 13 +++++++++++++ arch/sw_64/kernel/stacktrace.c | 4 ++-- arch/sw_64/kernel/traps.c | 10 +++++----- arch/sw_64/kernel/vmlinux.lds.S | 1 + arch/sw_64/mm/fault.c | 2 +- drivers/irqchip/irq-sunway-cpu.c | 2 +- 10 files changed, 30 insertions(+), 11 deletions(-) diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S index 73e8e043fc9d..6878b773968d 100644 --- a/arch/sw_64/kernel/entry-ftrace.S +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -206,6 +206,7 @@ END(return_to_handler) #endif + .pushsection ".entry.text", "ax" #ifdef CONFIG_DYNAMIC_FTRACE .global _mcount .ent _mcount @@ -283,8 +284,8 @@ skip_ftrace: 3: mcount_end ret $31, ($28), 1 .end _mcount - #endif /* CONFIG_DYNAMIC_FTRACE */ + .popsection #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS .global ftrace_regs_caller diff --git a/arch/sw_64/kernel/entry_c3.S b/arch/sw_64/kernel/entry_c3.S index df8bff5d09b3..fe1e9a8635a2 100644 --- a/arch/sw_64/kernel/entry_c3.S +++ b/arch/sw_64/kernel/entry_c3.S @@ -130,6 +130,7 @@ * Non-syscall kernel entry points. */ + .pushsection ".entry.text", "ax" .align 4 .globl entInt .ent entInt @@ -311,3 +312,4 @@ ret_from_kernel_thread: call $26, ($9) br ret_to_user .end ret_from_kernel_thread + .popsection diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S index 219016ef05b2..3a8912fb6287 100644 --- a/arch/sw_64/kernel/entry_c4.S +++ b/arch/sw_64/kernel/entry_c4.S @@ -263,6 +263,7 @@ * Non-syscall kernel entry points. */ + .pushsection ".entry.text", "ax" .align 4 .globl entNMI .ent entNMI @@ -489,3 +490,4 @@ ret_from_kernel_thread: call $26, ($9) br ret_to_user .end ret_from_kernel_thread + .popsection diff --git a/arch/sw_64/kernel/idle.c b/arch/sw_64/kernel/idle.c index d26bdc405b53..9ee6338bfef6 100644 --- a/arch/sw_64/kernel/idle.c +++ b/arch/sw_64/kernel/idle.c @@ -9,7 +9,7 @@ #include #include -void arch_cpu_idle(void) +void noinstr arch_cpu_idle(void) { local_irq_enable(); cpu_relax(); diff --git a/arch/sw_64/kernel/kprobes/kprobes.c b/arch/sw_64/kernel/kprobes/kprobes.c index 024ce7d99e61..2ee18260f2a3 100644 --- a/arch/sw_64/kernel/kprobes/kprobes.c +++ b/arch/sw_64/kernel/kprobes/kprobes.c @@ -270,6 +270,19 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, regs->regs[26] = (unsigned long)__kretprobe_trampoline; } +/* + * Provide a blacklist of symbols identifying ranges which cannot be kprobed. + * This blacklist is exposed to userspace via debugfs (kprobes/blacklist). + */ +int __init arch_populate_kprobe_blacklist(void) +{ + int ret; + + ret = kprobe_add_area_blacklist((unsigned long)__entry_text_start, + (unsigned long)__entry_text_end); + return ret; +} + /* * Called when the probe at kretprobe trampoline is hit */ diff --git a/arch/sw_64/kernel/stacktrace.c b/arch/sw_64/kernel/stacktrace.c index ff00506d5b82..ed8a9941612a 100644 --- a/arch/sw_64/kernel/stacktrace.c +++ b/arch/sw_64/kernel/stacktrace.c @@ -61,7 +61,7 @@ int unwind_frame(struct task_struct *tsk, struct stackframe *frame) } EXPORT_SYMBOL_GPL(unwind_frame); -void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, +void noinstr walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, int (*fn)(unsigned long, void *), void *data) { unsigned long pc, fp; @@ -109,7 +109,7 @@ void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, EXPORT_SYMBOL_GPL(walk_stackframe); #else /* !CONFIG_FRAME_POINTER */ -void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, +void noinstr walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, int (*fn)(unsigned long, void *), void *data) { unsigned long *ksp; diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 9a9b43c13286..cb7b3141b73a 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -148,7 +148,7 @@ extern long sw64_fp_emul(unsigned long pc); #endif asmlinkage void -do_entArith(unsigned long exc_sum, unsigned long write_mask, +noinstr do_entArith(unsigned long exc_sum, unsigned long write_mask, struct pt_regs *regs) { long si_code = FPE_FLTINV; @@ -250,7 +250,7 @@ static int try_fix_rd_f(unsigned int inst, struct pt_regs *regs) * do something necessary to handle it correctly. */ asmlinkage void -do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) +noinstr do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) { int signo, code; unsigned int inst, type; @@ -397,7 +397,7 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) 1L << 0x3 | 1L << 0x9) /* ldl_a stl_a */ asmlinkage void -do_entUna(void *va, unsigned long opcode, unsigned long reg, +noinstr do_entUna(void *va, unsigned long opcode, unsigned long reg, struct pt_regs *regs) { long error, disp; @@ -1348,7 +1348,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, * However, we need to deal with stt/ldt and sts/lds only. */ asmlinkage void -do_entUnaUser(void __user *va, unsigned long opcode, +noinstr do_entUnaUser(void __user *va, unsigned long opcode, unsigned long reg, struct pt_regs *regs) { #ifdef CONFIG_UNA_PRINT @@ -2508,7 +2508,7 @@ do_entUnaUser(void __user *va, unsigned long opcode, force_sig_fault(SIGBUS, BUS_ADRALN, va); } -asmlinkage void do_entSys(struct pt_regs *regs) +asmlinkage void noinstr do_entSys(struct pt_regs *regs) { long ret = -ENOSYS; unsigned long nr; diff --git a/arch/sw_64/kernel/vmlinux.lds.S b/arch/sw_64/kernel/vmlinux.lds.S index 9b81b2c7afb8..8bbce3e743fe 100644 --- a/arch/sw_64/kernel/vmlinux.lds.S +++ b/arch/sw_64/kernel/vmlinux.lds.S @@ -30,6 +30,7 @@ SECTIONS IRQENTRY_TEXT SOFTIRQENTRY_TEXT KPROBES_TEXT + ENTRY_TEXT *(.fixup) *(.gnu.warning) } :text diff --git a/arch/sw_64/mm/fault.c b/arch/sw_64/mm/fault.c index e76560a7edca..a43e38cc21b8 100644 --- a/arch/sw_64/mm/fault.c +++ b/arch/sw_64/mm/fault.c @@ -133,7 +133,7 @@ unsigned long show_va_to_pa(struct mm_struct *mm, unsigned long addr) extern int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_regs *regs); asmlinkage void notrace -do_page_fault(unsigned long address, unsigned long mmcsr, +noinstr do_page_fault(unsigned long address, unsigned long mmcsr, long cause, struct pt_regs *regs) { struct vm_area_struct *vma; diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 5aeda6a673aa..1030bfa51cca 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -76,7 +76,7 @@ static void handle_nmi_int(void) int pme_state; -asmlinkage void do_entInt(unsigned long type, unsigned long vector, +asmlinkage void noinstr do_entInt(unsigned long type, unsigned long vector, unsigned long irq_arg, struct pt_regs *regs) { struct pt_regs *old_regs; -- Gitee From c9ae707d6a20fbcf597ffe60fc23adb504ccd4e7 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Mon, 2 Dec 2024 11:09:10 +0800 Subject: [PATCH 390/524] sw64: fix no CONFIG_DYNAMIC_FTRACE_WITH_REGS error When no CONFIG_DYNAMIC_FTRACE_WITH_REGS, sw_64 don`t supports passing the contents of function_trace_op as the third parameter back from the _mcount call. fix it! Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/ftrace.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/include/asm/ftrace.h b/arch/sw_64/include/asm/ftrace.h index 7ed6e3c06a33..3478cd4ad850 100644 --- a/arch/sw_64/include/asm/ftrace.h +++ b/arch/sw_64/include/asm/ftrace.h @@ -15,7 +15,9 @@ #define MCOUNT_INSN_SIZE 20 /* 5 * SW64_INSN_SIZE */ #define MCOUNT_LDGP_SIZE 8 /* 2 * SW64_INSN_SIZE */ +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS #define ARCH_SUPPORTS_FTRACE_OPS 1 +#endif #ifndef __ASSEMBLY__ #include -- Gitee From a906f5c9355a07a8933aae723a37783b6909a3f4 Mon Sep 17 00:00:00 2001 From: Yan Bo Date: Wed, 21 Feb 2024 16:35:15 +0000 Subject: [PATCH 391/524] sw64: iommu: remove unnecessary locking in map/unmap_pages We encounter a `scheduling while atomic` bug when using an nvme ssd or mellanox card. This bug is caused by locks in sunway_iommu_map and sunway_iommu_unmap. Remove these locks to fix it. Detailed in commit 37ec8eb851c1 ("iommu/amd: Remove unnecessary locking from AMD iommu driver"). Signed-off-by: Yan Bo Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index 2336e2704b01..d7ae34970994 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -1168,7 +1168,6 @@ sunway_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, if (iova >= SW64_BAR_ADDRESS) return 0; - mutex_lock(&sdomain->api_lock); while (pgcount--) { ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size, iommu_prot); if (ret) { @@ -1178,7 +1177,6 @@ sunway_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, iova += page_size; paddr += page_size; } - mutex_unlock(&sdomain->api_lock); if (!ret && mapped) *mapped = size; @@ -1198,13 +1196,11 @@ sunway_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova, if (iova >= SW64_BAR_ADDRESS) return page_size; - mutex_lock(&sdomain->api_lock); while (pgcount--) { unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); iova += page_size; total_unmap += page_size; } - mutex_unlock(&sdomain->api_lock); return total_unmap; } -- Gitee From 350cbeaaff3a6865a3228bc71de85f364a15dd78 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Tue, 3 Dec 2024 06:07:05 +0000 Subject: [PATCH 392/524] sw64: fix iommu commandline parameter Fix iommu commandline parameters. Make the following changes: - Check the return value of kstrtoul correctly. - Adjust iommu_bitmap size on iommu_v1. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 16 ++++++++-------- drivers/iommu/sw64/iommu_v2.c | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index d7ae34970994..9c78526e3dd2 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -1359,16 +1359,16 @@ static int __init sunway_iommu_setup(char *str) if (!str) return -EINVAL; - bitmap_zero(iommu_bitmap, 64); + bitmap_zero(iommu_bitmap, 32); if (!strncmp(str, "on", 2)) { - bitmap_fill(iommu_bitmap, 64); + bitmap_fill(iommu_bitmap, 32); } else if (!strncmp(str, "off", 3)) { - bitmap_zero(iommu_bitmap, 64); + bitmap_zero(iommu_bitmap, 32); } else { ret = kstrtoul(str, 16, &rc_val); - if (!ret) - return -EINVAL; + if (ret) + return ret; bitmap_from_u64(iommu_bitmap, rc_val); } @@ -1386,11 +1386,11 @@ static int __init iommu_enable_setup(char *str) return -EINVAL; pr_info("iommu_enable= deprecated; use sunway_iommu=on/off instead.\n"); - bitmap_zero(iommu_bitmap, 64); + bitmap_zero(iommu_bitmap, 32); ret = kstrtoul(str, 16, &rc_val); - if (!ret) - return -EINVAL; + if (ret) + return ret; bitmap_from_u64(iommu_bitmap, rc_val); diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index de8c57e333ca..dccb2b744238 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -1633,8 +1633,8 @@ static int __init sunway_iommu_setup(char *str) bitmap_zero(iommu_bitmap, 64); } else { ret = kstrtoul(str, 16, &rc_val); - if (!ret) - return -EINVAL; + if (ret) + return ret; bitmap_from_u64(iommu_bitmap, rc_val); } @@ -1655,8 +1655,8 @@ static int __init iommu_enable_setup(char *str) bitmap_zero(iommu_bitmap, 64); ret = kstrtoul(str, 16, &rc_val); - if (!ret) - return -EINVAL; + if (ret) + return ret; bitmap_from_u64(iommu_bitmap, rc_val); -- Gitee From 71f8c3c53b3f4af424c5e6788d64a2b622d19e78 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Tue, 3 Dec 2024 06:18:11 +0000 Subject: [PATCH 393/524] sw64: iommu: fix iommu_v1 capability Provide iommu_v1 with certain capability flags to pass vfio initialization. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index 9c78526e3dd2..b813491f2eed 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -1311,7 +1311,14 @@ static int sunway_iommu_def_domain_type(struct device *dev) static bool sunway_iommu_capable(struct device *dev, enum iommu_cap cap) { - return false; + switch (cap) { + case IOMMU_CAP_CACHE_COHERENCY: + return true; + case IOMMU_CAP_NOEXEC: + return true; + default: + return false; + } } static void sunway_iommu_probe_finalize(struct device *dev) -- Gitee From 9626d415b764ce2108e3cc6a8198b6340ae98d7e Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Tue, 3 Dec 2024 06:36:27 +0000 Subject: [PATCH 394/524] sw64: iommu: adjust def_domain_type The current def_domain_type is too aggressive as it sets all initialized domain to IDENTITY. This is causing failures when users intentionally set initialized domain to DOMAIN_DMA. More research indicate the def_domain_type is used to initialize domains for devices that can only be directly mapped. Thus, we adjust current def_domain_type to 0, indicating the common code is responsible for determining domain types of these devices. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 2 +- drivers/iommu/sw64/iommu_v2.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index b813491f2eed..b71865d47e82 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -1306,7 +1306,7 @@ static struct iommu_device *sunway_iommu_probe_device(struct device *dev) static int sunway_iommu_def_domain_type(struct device *dev) { - return IOMMU_DOMAIN_IDENTITY; + return 0; } static bool sunway_iommu_capable(struct device *dev, enum iommu_cap cap) diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index dccb2b744238..d31c64e11c46 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -1569,7 +1569,7 @@ static struct iommu_device *sunway_iommu_probe_device(struct device *dev) static int sunway_iommu_def_domain_type(struct device *dev) { - return IOMMU_DOMAIN_IDENTITY; + return 0; } static bool sunway_iommu_capable(struct device *dev, enum iommu_cap cap) -- Gitee From 3010bbc5e8b584c8dc99e66bd7de1a5427e40123 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Wed, 4 Dec 2024 02:44:16 +0000 Subject: [PATCH 395/524] sw64: iommu: work around lazy translation initialization Users can still somehow manage to get into lazy translation domain and find devices fail to register any DMA ops. This is because we do not distinguish between lazy and strict translation domains right now. Currently, we have to register lazy translation domains with iommu_dma_ops as well to work around this issue. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 4 ++-- drivers/iommu/sw64/iommu_v2.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index b71865d47e82..ae8e76a0c6ca 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -1326,8 +1326,8 @@ static void sunway_iommu_probe_finalize(struct device *dev) struct iommu_domain *domain; domain = iommu_get_domain_for_dev(dev); - if (domain->type == IOMMU_DOMAIN_DMA) { - iommu_setup_dma_ops(dev, 0, SW64_32BIT_DMA_LIMIT); + if (domain->type & __IOMMU_DOMAIN_DMA_API) { + iommu_setup_dma_ops(dev, SW64_DMA_START, SW64_32BIT_DMA_LIMIT); } else set_dma_ops(dev, get_arch_dma_ops()); } diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index d31c64e11c46..fe007a7ed878 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -1589,7 +1589,7 @@ static void sunway_iommu_probe_finalize(struct device *dev) struct iommu_domain *domain; domain = iommu_get_domain_for_dev(dev); - if (domain->type == IOMMU_DOMAIN_DMA) { + if (domain->type & __IOMMU_DOMAIN_DMA_API) { if (min(dev->coherent_dma_mask, *dev->dma_mask) == DMA_BIT_MASK(32)) iommu_setup_dma_ops(dev, SW64_DMA_START, SW64_32BIT_DMA_LIMIT); else -- Gitee From 666affc3eb3973990d90c1a8139e669863047627 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Wed, 4 Dec 2024 03:40:03 +0000 Subject: [PATCH 396/524] sw64: iommu: restore iommu_map/unmap callbacks As we do not support lazy translation domains now, mapping/unmapping pages together will not do any good to reduce iotlb flush times. Restore iommu_map/unmap callbacks to match the semantic. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 37 ++++++------------------ drivers/iommu/sw64/iommu.h | 1 + drivers/iommu/sw64/iommu_v2.c | 54 +++++++++++------------------------ 3 files changed, 26 insertions(+), 66 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index ae8e76a0c6ca..e4fbc0b9fa11 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -1152,12 +1152,10 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) } static int -sunway_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, - phys_addr_t paddr, size_t page_size, size_t pgcount, - int iommu_prot, gfp_t gfp, size_t *mapped) +sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, + phys_addr_t paddr, size_t page_size, int iommu_prot, gfp_t gfp) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); - size_t size = pgcount << PAGE_SHIFT; int ret; /* @@ -1168,41 +1166,24 @@ sunway_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, if (iova >= SW64_BAR_ADDRESS) return 0; - while (pgcount--) { - ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size, iommu_prot); - if (ret) { - pr_info("Failed to map page from IOVA %lx.\n", iova); - return ret; - } - iova += page_size; - paddr += page_size; - } - - if (!ret && mapped) - *mapped = size; + ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size, iommu_prot); return ret; } static size_t -sunway_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova, - size_t page_size, size_t pgcount, - struct iommu_iotlb_gather *gather) +sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, + size_t page_size, struct iommu_iotlb_gather *gather) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); size_t unmap_size; - size_t total_unmap = 0; if (iova >= SW64_BAR_ADDRESS) return page_size; - while (pgcount--) { - unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); - iova += page_size; - total_unmap += page_size; - } + unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); - return total_unmap; + return unmap_size; } static struct iommu_group *sunway_iommu_device_group(struct device *dev) @@ -1343,8 +1324,8 @@ const struct iommu_ops sunway_iommu_ops = { .def_domain_type = sunway_iommu_def_domain_type, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = sunway_iommu_attach_device, - .map_pages = sunway_iommu_map_pages, - .unmap_pages = sunway_iommu_unmap_pages, + .map = sunway_iommu_map, + .unmap = sunway_iommu_unmap, .iova_to_phys = sunway_iommu_iova_to_phys, .free = sunway_iommu_domain_free, } diff --git a/drivers/iommu/sw64/iommu.h b/drivers/iommu/sw64/iommu.h index f203d19234e7..a8b49139fade 100644 --- a/drivers/iommu/sw64/iommu.h +++ b/drivers/iommu/sw64/iommu.h @@ -79,6 +79,7 @@ struct sunway_iommu_group { #define SW64_IOMMU_LEVEL3_OFFSET 0x3ff #define SW64_IOMMU_BYPASS 0x1 #define SW64_IOMMU_MAP_FLAG ((0x1UL) << 20) +#define MAX_IOVA_WIDTH (1UL << 42) #define PAGE_SHIFT_IOMMU 18 #define PAGE_SIZE_IOMMU (_AC(1, UL) << PAGE_SHIFT_IOMMU) diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index fe007a7ed878..33f25fa5b2c5 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -423,7 +423,8 @@ set_entry_by_devid(u16 devid, dte_l2_val |= 0x1; *dte_l2 = dte_l2_val; - pr_debug("iommu: device with id %d added to domain: %d\n", devid, sdomain->id); + printk("device with id %d added to domain: %d with pte_root: %lx\n", + devid, sdomain->id, dte_l2_val); return 0; } @@ -1403,69 +1404,46 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) paddr += iova & ~PAGE_MASK; return paddr; } - static int -sunway_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, - phys_addr_t paddr, size_t page_size, size_t pgcount, - int iommu_prot, gfp_t gfp, size_t *mapped) +sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, + phys_addr_t paddr, size_t page_size, int iommu_prot, gfp_t gfp) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); - size_t size = pgcount << PAGE_SHIFT; int ret; - /* - * As VFIO cannot distinguish between normal DMA request - * and pci device BAR, check should be introduced manually - * to avoid VFIO trying to map pci config space. - */ if (iova >= SW64_BAR_ADDRESS) return 0; - if (iova >= (1UL << IOVA_MAX_ADDRESS_WIDTH)) { + /* IOMMU v2 supports 42 bit mapped address width*/ + if (iova >= MAX_IOVA_WIDTH) { pr_err("IOMMU cannot map provided address: %lx\n", iova); return -EFAULT; } - while (pgcount--) { - ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size, iommu_prot); - if (ret) { - pr_info("Failed to map page from IOVA %lx.\n", iova); - return ret; - } - iova += page_size; - paddr += page_size; - } - - if (!ret && mapped) - *mapped = size; + ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size, iommu_prot); return ret; } static size_t -sunway_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova, - size_t page_size, size_t pgcount, - struct iommu_iotlb_gather *gather) +sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, + size_t page_size, struct iommu_iotlb_gather *gather) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); size_t unmap_size; - size_t total_unmap = 0; if (iova >= SW64_BAR_ADDRESS) return page_size; - if (iova >= (1UL << IOVA_MAX_ADDRESS_WIDTH)) { - pr_err("IOMMU cannot map provided address: %lx\n", iova); + /* IOMMU v2 supports 42 bit mapped address width*/ + if (iova >= MAX_IOVA_WIDTH) { + pr_err("Trying to unmap illegal IOVA : %lx\n", iova); return -EFAULT; } - while (pgcount--) { - unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); - iova += page_size; - total_unmap += page_size; - } + unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); - return total_unmap; + return unmap_size; } static struct iommu_group *sunway_iommu_device_group(struct device *dev) @@ -1609,8 +1587,8 @@ const struct iommu_ops sunway_iommu_ops = { .def_domain_type = sunway_iommu_def_domain_type, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = sunway_iommu_attach_device, - .map_pages = sunway_iommu_map_pages, - .unmap_pages = sunway_iommu_unmap_pages, + .map = sunway_iommu_map, + .unmap = sunway_iommu_unmap, .iova_to_phys = sunway_iommu_iova_to_phys, .free = sunway_iommu_domain_free, } -- Gitee From dccb50ef1888942f5d3f69d88a7b6665e84a3245 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 12 Dec 2024 18:09:48 +0800 Subject: [PATCH 397/524] sw64: acpi: add support for ACPI S3 Now the ACPI S3 flow can work if the firmware provides the _S3_ Method. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/suspend.h | 2 ++ arch/sw_64/kernel/pm.c | 6 +++++- arch/sw_64/kernel/suspend.c | 8 +++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/include/asm/suspend.h b/arch/sw_64/include/asm/suspend.h index 833e27f9d5e1..b76db1580543 100644 --- a/arch/sw_64/include/asm/suspend.h +++ b/arch/sw_64/include/asm/suspend.h @@ -46,5 +46,7 @@ struct processor_state { }; extern void sw64_suspend_deep_sleep(struct processor_state *state); +extern int sw64_suspend_enter(void); + extern const struct platform_suspend_ops native_suspend_ops; #endif /* _ASM_SW64_SUSPEND_H */ diff --git a/arch/sw_64/kernel/pm.c b/arch/sw_64/kernel/pm.c index f0a35e5d0486..15867bdb14f7 100644 --- a/arch/sw_64/kernel/pm.c +++ b/arch/sw_64/kernel/pm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include @@ -9,7 +10,10 @@ struct syscore_ops io_syscore_ops; static int __init sw64_pm_init(void) { #ifdef CONFIG_SUSPEND - suspend_set_ops(&native_suspend_ops); + if (acpi_sleep_state_supported(ACPI_STATE_S3)) + acpi_suspend_lowlevel = sw64_suspend_enter; + else + suspend_set_ops(&native_suspend_ops); #endif register_syscore_ops(&io_syscore_ops); diff --git a/arch/sw_64/kernel/suspend.c b/arch/sw_64/kernel/suspend.c index 955cf41fd8c9..9947937c71b5 100644 --- a/arch/sw_64/kernel/suspend.c +++ b/arch/sw_64/kernel/suspend.c @@ -30,7 +30,7 @@ extern struct pci_controller *hose_head; /* * Boot Core will enter suspend stat here. */ -void sw64_suspend_enter(void) +int sw64_suspend_enter(void) { /* boot processor will go to deep sleep mode from here * After wake up boot processor, pc will go here @@ -50,15 +50,17 @@ void sw64_suspend_enter(void) wrtp(current_thread_info()->pcb.tp); disable_local_timer(); + + return 0; } static int native_suspend_enter(suspend_state_t state) { if (is_in_guest()) return 0; + /* processor specific suspend */ - sw64_suspend_enter(); - return 0; + return sw64_suspend_enter(); } const struct platform_suspend_ops native_suspend_ops = { -- Gitee From ca1b8b83fae68aa97a070d71f856bb2175db7d96 Mon Sep 17 00:00:00 2001 From: Yu Jiayi Date: Fri, 13 Dec 2024 12:22:45 +0800 Subject: [PATCH 398/524] sw64: fix bad address error for vfio Due to version upgrade introducing a check for pte page flag in follow_fault_pfn(). In our framework, the FOW bit is set for shared writable page. So the added check return EFAULT error. The same problem in function of hva_to_pfn_remapped(). Therefore, adding CONFIG_SW64 judgement to skip this check. Signed-off-by: Yu Jiayi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/vfio/vfio_iommu_type1.c | 4 ++++ virt/kvm/kvm_main.c | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 0a6868716ccc..a222d1c248f0 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -539,11 +539,15 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm, pte = ptep_get(ptep); +#ifdef CONFIG_SW64 + *pfn = pte_pfn(pte); +#else if (write_fault && !pte_write(pte)) ret = -EFAULT; else *pfn = pte_pfn(pte); +#endif pte_unmap_unlock(ptep, ptl); return ret; } diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 3f13223bcfc3..e4077b389c80 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2661,6 +2661,10 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, pte = ptep_get(ptep); +#ifdef CONFIG_SW64 + if (writable) + *writable = true; +#else if (write_fault && !pte_write(pte)) { pfn = KVM_PFN_ERR_RO_FAULT; goto out; @@ -2668,6 +2672,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, if (writable) *writable = pte_write(pte); +#endif pfn = pte_pfn(pte); /* @@ -2690,7 +2695,9 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, if (!kvm_try_get_pfn(pfn)) r = -EFAULT; +#ifndef CONFIG_SW64 out: +#endif pte_unmap_unlock(ptep, ptl); *p_pfn = pfn; -- Gitee From 96f8b88f1ec08a4fd2c499e9fca02986204c7a16 Mon Sep 17 00:00:00 2001 From: Deng Xiaoyun Date: Fri, 6 Dec 2024 14:11:34 +0800 Subject: [PATCH 399/524] sw64: emulator: fix the multi-core boot issue When the slave core of the emulator is in the halt state during startup and waiting for the interruption of the II_RESET, the halt state will be interrupted by the II_WAKE interruption, resulting in the salve core falling to start. Therefore, the II_WAKE interrupt is required to mask the emulator. Signed-off-by: Deng Xiaoyun Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index aa66fa87c728..24dc63317c56 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -543,7 +543,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) wmb(); smp_rcb->ready = 0; - if (!is_junzhang_v1()) { + if (!is_junzhang_v1() && is_in_host()) { /* send wake up signal */ send_wakeup_interrupt(cpu); } -- Gitee From 2eabf1ccb498b7bedfb040998f5f5ed3db3b500e Mon Sep 17 00:00:00 2001 From: Deng Xiaoyun Date: Fri, 6 Dec 2024 14:11:34 +0800 Subject: [PATCH 400/524] sw64: emulator: add shutdown and restart functions Triggers the shutdown or restart process by writing emulated IOR. Signed-off-by: Deng Xiaoyun Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uncore_io_junzhang.h | 1 + arch/sw_64/include/asm/uncore_io_xuelang.h | 1 + arch/sw_64/kernel/reset.c | 15 +++++++++++++++ 3 files changed, 17 insertions(+) diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index 24c1e3be32f3..1e54339da52e 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -33,6 +33,7 @@ #define VT_THREADS_MASK 0xfff #define QEMU_PRINTF_BUFF_BASE (IO_BASE | SPBU_BASE | 0x40000UL) +#define QEMU_RESTART_SHUTDOWN_BASE (IO_BASE | SPBU_BASE | 0x50000UL) /* MSIConfig */ #define MSICONFIG_VALID (0x1UL << 63) diff --git a/arch/sw_64/include/asm/uncore_io_xuelang.h b/arch/sw_64/include/asm/uncore_io_xuelang.h index c8f01cb01e52..3b72561b94ae 100644 --- a/arch/sw_64/include/asm/uncore_io_xuelang.h +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -32,6 +32,7 @@ #define VT_THREADS_MASK 0xfff #define QEMU_PRINTF_BUFF_BASE (IO_BASE | MCU_BASE | 0x40000UL) +#define QEMU_RESTART_SHUTDOWN_BASE (IO_BASE | SPBU_BASE | 0x50000UL) /* MSIConfig */ #define MSICONFIG_VALID (0x1UL << 63) diff --git a/arch/sw_64/kernel/reset.c b/arch/sw_64/kernel/reset.c index 8181923e676e..b6eede36b72c 100644 --- a/arch/sw_64/kernel/reset.c +++ b/arch/sw_64/kernel/reset.c @@ -16,6 +16,7 @@ #include #include #include +#include void (*pm_power_off)(void); EXPORT_SYMBOL(pm_power_off); @@ -38,6 +39,13 @@ void machine_power_off(void) local_irq_disable(); smp_send_stop(); + if (is_in_emul()) { + void __iomem *addr = __va(QEMU_RESTART_SHUTDOWN_BASE); + u64 data = 1; + + *(u64 *)addr = data; + } + do_kernel_power_off(); /* VM cannot reach here */ @@ -60,6 +68,13 @@ void machine_restart(char *command) local_irq_disable(); smp_send_stop(); + if (is_in_emul()) { + void __iomem *addr = __va(QEMU_RESTART_SHUTDOWN_BASE); + u64 data = 2; + + *(u64 *)addr = data; + } + do_kernel_restart(command); /* VM cannot reach here */ -- Gitee From e8060a4ccc04bf7986e0271476785fb000b861c1 Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Thu, 21 Nov 2024 11:23:18 +0800 Subject: [PATCH 401/524] perf jitdump: Add SW64 support Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/perf/arch/sw_64/Makefile | 1 + .../arch/sw_64/include/dwarf-regs-table.h | 14 +++ tools/perf/arch/sw_64/include/perf_regs.h | 83 +---------------- tools/perf/util/dwarf-regs.c | 7 ++ tools/perf/util/genelf.h | 4 + tools/perf/util/perf-regs-arch/Build | 1 + .../perf/util/perf-regs-arch/perf_regs_sw64.c | 92 +++++++++++++++++++ tools/perf/util/perf_regs.c | 6 ++ tools/perf/util/perf_regs.h | 3 + tools/perf/util/python-ext-sources | 1 + 10 files changed, 132 insertions(+), 80 deletions(-) create mode 100644 tools/perf/arch/sw_64/include/dwarf-regs-table.h create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_sw64.c diff --git a/tools/perf/arch/sw_64/Makefile b/tools/perf/arch/sw_64/Makefile index 1aa9dd772489..eebe1ec9d2ee 100644 --- a/tools/perf/arch/sw_64/Makefile +++ b/tools/perf/arch/sw_64/Makefile @@ -1,4 +1,5 @@ ifndef NO_DWARF PERF_HAVE_DWARF_REGS := 1 endif +PERF_HAVE_JITDUMP := 1 PERF_HAVE_ARCH_REGS_QUERY_REGISTER_OFFSET := 1 diff --git a/tools/perf/arch/sw_64/include/dwarf-regs-table.h b/tools/perf/arch/sw_64/include/dwarf-regs-table.h new file mode 100644 index 000000000000..20386ee78b01 --- /dev/null +++ b/tools/perf/arch/sw_64/include/dwarf-regs-table.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef DEFINE_DWARF_REGSTR_TABLE +/* This is included in perf/util/dwarf-regs.c */ + +static const char * const sw_64_regstr_tbl[] = { + "%r0", "%r1", "%r2", "%r3", "%r4", + "%r5", "%r6", "%r7", "%r8", "%r9", + "%r10", "%r11", "%r12", "%r13", "%r14", + "%r15", "%r16", "%r17", "%r18", "%r19", + "%r20", "%r21", "%r22", "%r23", "%r24", + "%r25", "%r26", "%r27", "%r28", "%r29", + "%30" +}; +#endif diff --git a/tools/perf/arch/sw_64/include/perf_regs.h b/tools/perf/arch/sw_64/include/perf_regs.h index e0c1b15375b5..89a223a357a1 100644 --- a/tools/perf/arch/sw_64/include/perf_regs.h +++ b/tools/perf/arch/sw_64/include/perf_regs.h @@ -8,85 +8,8 @@ void perf_regs_load(u64 *regs); -#define PERF_REGS_MASK ((1ULL << PERF_REG_SW64_MAX) - 1) -#define PERF_REGS_MAX PERF_REG_SW64_MAX -#define PERF_SAMPLE_REGS_ABI PERF_SAMPLE_REGS_ABI_64 - -#define PERF_REG_IP PERF_REG_SW64_PC -#define PERF_REG_SP PERF_REG_SW64_SP - -static inline const char *perf_reg_name(int id) -{ - switch (id) { - case PERF_REG_SW64_R0: - return "r0"; - case PERF_REG_SW64_R1: - return "r1"; - case PERF_REG_SW64_R2: - return "r2"; - case PERF_REG_SW64_R3: - return "r3"; - case PERF_REG_SW64_R4: - return "r4"; - case PERF_REG_SW64_R5: - return "r5"; - case PERF_REG_SW64_R6: - return "r6"; - case PERF_REG_SW64_R7: - return "r7"; - case PERF_REG_SW64_R8: - return "r8"; - case PERF_REG_SW64_R9: - return "r9"; - case PERF_REG_SW64_R10: - return "r10"; - case PERF_REG_SW64_R11: - return "r11"; - case PERF_REG_SW64_R12: - return "r12"; - case PERF_REG_SW64_R13: - return "r13"; - case PERF_REG_SW64_R14: - return "r14"; - case PERF_REG_SW64_R15: - return "r15"; - case PERF_REG_SW64_R16: - return "r16"; - case PERF_REG_SW64_R17: - return "r17"; - case PERF_REG_SW64_R18: - return "r18"; - case PERF_REG_SW64_R19: - return "r19"; - case PERF_REG_SW64_R20: - return "r20"; - case PERF_REG_SW64_R21: - return "r21"; - case PERF_REG_SW64_R22: - return "r22"; - case PERF_REG_SW64_R23: - return "r23"; - case PERF_REG_SW64_R24: - return "r24"; - case PERF_REG_SW64_R25: - return "r25"; - case PERF_REG_SW64_R26: - return "r26"; - case PERF_REG_SW64_R27: - return "r27"; - case PERF_REG_SW64_R28: - return "r28"; - case PERF_REG_SW64_GP: - return "gp"; - case PERF_REG_SW64_SP: - return "sp"; - case PERF_REG_SW64_PC: - return "pc"; - default: - return NULL; - } - - return NULL; -} +#define PERF_REGS_MASK ((1ULL << PERF_REG_SW64_MAX) - 1) +#define PERF_REGS_MAX PERF_REG_SW64_MAX +#define PERF_SAMPLE_REGS_ABI PERF_SAMPLE_REGS_ABI_64 #endif /* ARCH_PERF_REGS_H */ diff --git a/tools/perf/util/dwarf-regs.c b/tools/perf/util/dwarf-regs.c index 69cfaa5953bf..5650fb9e6ba9 100644 --- a/tools/perf/util/dwarf-regs.c +++ b/tools/perf/util/dwarf-regs.c @@ -18,6 +18,10 @@ #define EM_LOONGARCH 258 /* LoongArch */ #endif +#ifndef EM_SW64 +#define EM_SW64 0x9916 /* SW64 */ +#endif + /* Define const char * {arch}_register_tbl[] */ #define DEFINE_DWARF_REGSTR_TABLE #include "../arch/x86/include/dwarf-regs-table.h" @@ -27,6 +31,7 @@ #include "../arch/powerpc/include/dwarf-regs-table.h" #include "../arch/s390/include/dwarf-regs-table.h" #include "../arch/sparc/include/dwarf-regs-table.h" +#include "../arch/sw_64/include/dwarf-regs-table.h" #include "../arch/xtensa/include/dwarf-regs-table.h" #include "../arch/mips/include/dwarf-regs-table.h" #include "../arch/loongarch/include/dwarf-regs-table.h" @@ -57,6 +62,8 @@ const char *get_dwarf_regstr(unsigned int n, unsigned int machine) case EM_SPARC: case EM_SPARCV9: return __get_dwarf_regstr(sparc_regstr_tbl, n); + case EM_SW64: + return __get_dwarf_regstr(sw_64_regstr_tbl, n); case EM_XTENSA: return __get_dwarf_regstr(xtensa_regstr_tbl, n); case EM_MIPS: diff --git a/tools/perf/util/genelf.h b/tools/perf/util/genelf.h index 5f18d20ea903..1c780df8997a 100644 --- a/tools/perf/util/genelf.h +++ b/tools/perf/util/genelf.h @@ -37,6 +37,10 @@ int jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_ent #elif defined(__sparc__) #define GEN_ELF_ARCH EM_SPARC #define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__sw_64__) +#define EM_SW64 0x9916 +#define GEN_ELF_ARCH EM_SW64 +#define GEN_ELF_CLASS ELFCLASS64 #elif defined(__s390x__) #define GEN_ELF_ARCH EM_S390 #define GEN_ELF_CLASS ELFCLASS64 diff --git a/tools/perf/util/perf-regs-arch/Build b/tools/perf/util/perf-regs-arch/Build index d9d596d330a7..d47893db4268 100644 --- a/tools/perf/util/perf-regs-arch/Build +++ b/tools/perf/util/perf-regs-arch/Build @@ -6,4 +6,5 @@ perf-y += perf_regs_mips.o perf-y += perf_regs_powerpc.o perf-y += perf_regs_riscv.o perf-y += perf_regs_s390.o +perf-y += perf_regs_sw64.o perf-y += perf_regs_x86.o diff --git a/tools/perf/util/perf-regs-arch/perf_regs_sw64.c b/tools/perf/util/perf-regs-arch/perf_regs_sw64.c new file mode 100644 index 000000000000..1f81a4e67d8d --- /dev/null +++ b/tools/perf/util/perf-regs-arch/perf_regs_sw64.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 + +#ifdef HAVE_PERF_REGS_SUPPORT + +#include "../perf_regs.h" +#include "../../../arch/sw_64/include/uapi/asm/perf_regs.h" + +const char *__perf_reg_name_sw64(int id) +{ + switch (id) { + case PERF_REG_SW64_R0: + return "r0"; + case PERF_REG_SW64_R1: + return "r1"; + case PERF_REG_SW64_R2: + return "r2"; + case PERF_REG_SW64_R3: + return "r3"; + case PERF_REG_SW64_R4: + return "r4"; + case PERF_REG_SW64_R5: + return "r5"; + case PERF_REG_SW64_R6: + return "r6"; + case PERF_REG_SW64_R7: + return "r7"; + case PERF_REG_SW64_R8: + return "r8"; + case PERF_REG_SW64_R9: + return "r9"; + case PERF_REG_SW64_R10: + return "r10"; + case PERF_REG_SW64_R11: + return "r11"; + case PERF_REG_SW64_R12: + return "r12"; + case PERF_REG_SW64_R13: + return "r13"; + case PERF_REG_SW64_R14: + return "r14"; + case PERF_REG_SW64_R15: + return "r15"; + case PERF_REG_SW64_R16: + return "r16"; + case PERF_REG_SW64_R17: + return "r17"; + case PERF_REG_SW64_R18: + return "r18"; + case PERF_REG_SW64_R19: + return "r19"; + case PERF_REG_SW64_R20: + return "r20"; + case PERF_REG_SW64_R21: + return "r21"; + case PERF_REG_SW64_R22: + return "r22"; + case PERF_REG_SW64_R23: + return "r23"; + case PERF_REG_SW64_R24: + return "r24"; + case PERF_REG_SW64_R25: + return "r25"; + case PERF_REG_SW64_R26: + return "r26"; + case PERF_REG_SW64_R27: + return "r27"; + case PERF_REG_SW64_R28: + return "r28"; + case PERF_REG_SW64_GP: + return "gp"; + case PERF_REG_SW64_SP: + return "sp"; + case PERF_REG_SW64_PC: + return "pc"; + default: + return NULL; + } + + return NULL; +} + +uint64_t __perf_reg_ip_sw64(void) +{ + return PERF_REG_SW64_PC; +} + +uint64_t __perf_reg_sp_sw64(void) +{ + return PERF_REG_SW64_SP; +} + +#endif diff --git a/tools/perf/util/perf_regs.c b/tools/perf/util/perf_regs.c index e2275856b570..90dd226928ad 100644 --- a/tools/perf/util/perf_regs.c +++ b/tools/perf/util/perf_regs.c @@ -39,6 +39,8 @@ const char *perf_reg_name(int id, const char *arch) reg_name = __perf_reg_name_riscv(id); else if (!strcmp(arch, "s390")) reg_name = __perf_reg_name_s390(id); + else if (!strcmp(arch, "sw_64")) + reg_name = __perf_reg_name_sw64(id); else if (!strcmp(arch, "x86")) reg_name = __perf_reg_name_x86(id); else if (!strcmp(arch, "arm")) @@ -94,6 +96,8 @@ uint64_t perf_arch_reg_ip(const char *arch) return __perf_reg_ip_riscv(); else if (!strcmp(arch, "s390")) return __perf_reg_ip_s390(); + else if (!strcmp(arch, "sw_64")) + return __perf_reg_ip_sw64(); else if (!strcmp(arch, "x86")) return __perf_reg_ip_x86(); @@ -119,6 +123,8 @@ uint64_t perf_arch_reg_sp(const char *arch) return __perf_reg_sp_riscv(); else if (!strcmp(arch, "s390")) return __perf_reg_sp_s390(); + else if (!strcmp(arch, "sw_64")) + return __perf_reg_sp_sw64(); else if (!strcmp(arch, "x86")) return __perf_reg_sp_x86(); diff --git a/tools/perf/util/perf_regs.h b/tools/perf/util/perf_regs.h index ecd2a5362042..ea68adaa19a0 100644 --- a/tools/perf/util/perf_regs.h +++ b/tools/perf/util/perf_regs.h @@ -58,6 +58,9 @@ uint64_t __perf_reg_sp_riscv(void); const char *__perf_reg_name_s390(int id); uint64_t __perf_reg_ip_s390(void); uint64_t __perf_reg_sp_s390(void); +const char *__perf_reg_name_sw64(int id); +uint64_t __perf_reg_ip_sw64(void); +uint64_t __perf_reg_sp_sw64(void); const char *__perf_reg_name_x86(int id); uint64_t __perf_reg_ip_x86(void); uint64_t __perf_reg_sp_x86(void); diff --git a/tools/perf/util/python-ext-sources b/tools/perf/util/python-ext-sources index 26e1c8d973ea..080a2091e53f 100644 --- a/tools/perf/util/python-ext-sources +++ b/tools/perf/util/python-ext-sources @@ -48,4 +48,5 @@ util/perf-regs-arch/perf_regs_mips.c util/perf-regs-arch/perf_regs_powerpc.c util/perf-regs-arch/perf_regs_riscv.c util/perf-regs-arch/perf_regs_s390.c +util/perf-regs-arch/perf_regs_sw64.c util/perf-regs-arch/perf_regs_x86.c -- Gitee From 783ee58b274275018598a10b4591ab495afac8d7 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 31 Dec 2024 14:15:52 +0800 Subject: [PATCH 402/524] sw64: emulator: refactor shutdown and restart functions Refactor shutdown and restart functions for emulator to avoid hard coding of I/O address. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uncore_io_junzhang.h | 1 - arch/sw_64/include/asm/uncore_io_xuelang.h | 1 - arch/sw_64/kernel/reset.c | 17 +------- drivers/platform/sw64/misc-platform.c | 43 +++++++++++++++++++-- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index 1e54339da52e..24c1e3be32f3 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -33,7 +33,6 @@ #define VT_THREADS_MASK 0xfff #define QEMU_PRINTF_BUFF_BASE (IO_BASE | SPBU_BASE | 0x40000UL) -#define QEMU_RESTART_SHUTDOWN_BASE (IO_BASE | SPBU_BASE | 0x50000UL) /* MSIConfig */ #define MSICONFIG_VALID (0x1UL << 63) diff --git a/arch/sw_64/include/asm/uncore_io_xuelang.h b/arch/sw_64/include/asm/uncore_io_xuelang.h index 3b72561b94ae..c8f01cb01e52 100644 --- a/arch/sw_64/include/asm/uncore_io_xuelang.h +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -32,7 +32,6 @@ #define VT_THREADS_MASK 0xfff #define QEMU_PRINTF_BUFF_BASE (IO_BASE | MCU_BASE | 0x40000UL) -#define QEMU_RESTART_SHUTDOWN_BASE (IO_BASE | SPBU_BASE | 0x50000UL) /* MSIConfig */ #define MSICONFIG_VALID (0x1UL << 63) diff --git a/arch/sw_64/kernel/reset.c b/arch/sw_64/kernel/reset.c index b6eede36b72c..e7852211fc2a 100644 --- a/arch/sw_64/kernel/reset.c +++ b/arch/sw_64/kernel/reset.c @@ -16,7 +16,6 @@ #include #include #include -#include void (*pm_power_off)(void); EXPORT_SYMBOL(pm_power_off); @@ -39,13 +38,6 @@ void machine_power_off(void) local_irq_disable(); smp_send_stop(); - if (is_in_emul()) { - void __iomem *addr = __va(QEMU_RESTART_SHUTDOWN_BASE); - u64 data = 1; - - *(u64 *)addr = data; - } - do_kernel_power_off(); /* VM cannot reach here */ @@ -68,13 +60,6 @@ void machine_restart(char *command) local_irq_disable(); smp_send_stop(); - if (is_in_emul()) { - void __iomem *addr = __va(QEMU_RESTART_SHUTDOWN_BASE); - u64 data = 2; - - *(u64 *)addr = data; - } - do_kernel_restart(command); /* VM cannot reach here */ @@ -117,7 +102,7 @@ static int __init vm_power_init(void) { struct sys_off_handler *handler; - if (is_in_host()) + if (is_in_host() || is_in_emul()) return 0; handler = register_sys_off_handler(SYS_OFF_MODE_RESTART, diff --git a/drivers/platform/sw64/misc-platform.c b/drivers/platform/sw64/misc-platform.c index 8a5fe92b1d81..4d47ebb90546 100644 --- a/drivers/platform/sw64/misc-platform.c +++ b/drivers/platform/sw64/misc-platform.c @@ -5,9 +5,12 @@ #include #include #include +#include #include +#define OFFSET_EMUL_POWER_CONTROL 0x50000UL + struct misc_platform { void __iomem *spbu_base; void __iomem *intpu_base; @@ -61,10 +64,27 @@ static int misc_platform_get_node(struct device *dev) return nid; } +static int emul_restart(struct sys_off_data *data) +{ + void __iomem *spbu_base = misc_platform_devices[0].spbu_base; + + writeq(2, spbu_base + OFFSET_EMUL_POWER_CONTROL); + + return NOTIFY_DONE; +} + +static int emul_power_off(struct sys_off_data *data) +{ + void __iomem *spbu_base = misc_platform_devices[0].spbu_base; + + writeq(1, spbu_base + OFFSET_EMUL_POWER_CONTROL); + + return NOTIFY_DONE; +} + static int misc_platform_probe(struct platform_device *pdev) { - int ret, node; - u64 base_address; + u64 node, base_address; void __iomem *spbu_base = NULL; void __iomem *intpu_base = NULL; void __iomem *gpio_base = NULL; @@ -73,9 +93,14 @@ static int misc_platform_probe(struct platform_device *pdev) node = misc_platform_get_node(dev); if (node == NUMA_NO_NODE) { pr_err("unable to get node ID\n"); - return ret; + return -ENODATA; } + /* Set default value */ + spbu_base = __va(SW64_IO_BASE(node) | SPBU_BASE); + intpu_base = __va(SW64_IO_BASE(node) | INTPU_BASE); + gpio_base = __va(SW64_IO_BASE(node) | GPIO_BASE); + if (!device_property_read_u64(dev, "sunway,spbu_base", &base_address)) spbu_base = __va(base_address); @@ -92,7 +117,17 @@ static int misc_platform_probe(struct platform_device *pdev) misc_platform_devices[node].intpu_base = intpu_base; misc_platform_devices[node].gpio_base = gpio_base; - pr_info("misc-platform on node %d found\n", node); + if (is_in_emul() && (node == 0)) { + if (IS_ERR(register_sys_off_handler(SYS_OFF_MODE_RESTART, + SYS_OFF_PRIO_DEFAULT, emul_restart, NULL))) + pr_info("unable to register emul_restart\n"); + + if (IS_ERR(register_sys_off_handler(SYS_OFF_MODE_POWER_OFF, + SYS_OFF_PRIO_DEFAULT, emul_power_off, NULL))) + pr_info("unable to register emul_power_off\n"); + } + + pr_info("misc-platform on node %llu found\n", node); return 0; } -- Gitee From 0524b1dde040c62c19b54c1421b3672602273470 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 26 Dec 2024 19:12:35 +0800 Subject: [PATCH 403/524] sw64: acpi: suppress meaningless error log when acpi=off Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/efi.c | 2 +- arch/sw_64/kernel/pm.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/kernel/efi.c b/arch/sw_64/kernel/efi.c index 8bf80b271581..7955fa840a43 100644 --- a/arch/sw_64/kernel/efi.c +++ b/arch/sw_64/kernel/efi.c @@ -12,7 +12,7 @@ bool efi_poweroff_required(void) return false; /* Prefer ACPI S5 */ - if (acpi_sleep_state_supported(ACPI_STATE_S5)) + if (!acpi_disabled && acpi_sleep_state_supported(ACPI_STATE_S5)) return false; return efi_enabled(EFI_RUNTIME_SERVICES); diff --git a/arch/sw_64/kernel/pm.c b/arch/sw_64/kernel/pm.c index 15867bdb14f7..d723d4307811 100644 --- a/arch/sw_64/kernel/pm.c +++ b/arch/sw_64/kernel/pm.c @@ -10,7 +10,7 @@ struct syscore_ops io_syscore_ops; static int __init sw64_pm_init(void) { #ifdef CONFIG_SUSPEND - if (acpi_sleep_state_supported(ACPI_STATE_S3)) + if (!acpi_disabled && acpi_sleep_state_supported(ACPI_STATE_S3)) acpi_suspend_lowlevel = sw64_suspend_enter; else suspend_set_ops(&native_suspend_ops); -- Gitee From da71f8b53d23e3d1e2e7399071b27d57b7303b7c Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 19 Dec 2024 10:38:49 +0800 Subject: [PATCH 404/524] sw64: numa: decrease CONFIG_NODES_SHIFT to 3 According to the hardware, adjust CONFIG_NODES_SHIFT to 3. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index eda9e2a74b4a..9ce83cb0e9f8 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -427,7 +427,7 @@ config USE_PERCPU_NUMA_NODE_ID config NODES_SHIFT int - default "7" + default "3" depends on NUMA config RELOCATABLE -- Gitee From 9ec1efe98c049e01baf7b76766005f0a08dd7c30 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 23 Dec 2024 15:06:35 +0800 Subject: [PATCH 405/524] sw64: irqchip: use pr_fmt instead of prefix string macro Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-cpu.c | 12 ++++---- drivers/irqchip/irq-sunway-lpc-intc.c | 27 +++++++++-------- drivers/irqchip/irq-sunway-msi-v2.c | 7 +++-- drivers/irqchip/irq-sunway-msi.c | 7 +++-- drivers/irqchip/irq-sunway-pintc.c | 43 ++++++++++++++------------- 5 files changed, 50 insertions(+), 46 deletions(-) diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 1030bfa51cca..e381c19b9cae 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "CINTC: " fmt + #include #include #include @@ -35,8 +37,6 @@ * +-----------------------------------------------------------+ */ -#define PREFIX "CINTC: " - struct fwnode_handle *cintc_handle; static void handle_pci_intx_interrupt(unsigned long irq_info) @@ -249,7 +249,7 @@ static int __init pintc_parse_madt(union acpi_subtable_headers *header, if ((pintc->version == ACPI_MADT_SW_PINTC_VERSION_NONE) || (pintc->version >= ACPI_MADT_SW_PINTC_VERSION_RESERVED)) { - pr_err(PREFIX "invalid PINTC version\n"); + pr_err("invalid PINTC version\n"); return -EINVAL; } @@ -265,7 +265,7 @@ static int __init msic_parse_madt(union acpi_subtable_headers *header, msic = (struct acpi_madt_sw_msic *)header; if ((msic->version == ACPI_MADT_SW_MSIC_VERSION_NONE) || (msic->version >= ACPI_MADT_SW_MSIC_VERSION_RESERVED)) { - pr_err(PREFIX "invalid MSIC version\n"); + pr_err("invalid MSIC version\n"); return -EINVAL; } @@ -302,7 +302,7 @@ static __init int cintc_acpi_init(union acpi_subtable_headers *header, cintc = (struct acpi_madt_sw_cintc *)header; virtual = is_core_virtual(cintc->flags); - pr_info(PREFIX "version [%u] (%s) found\n", cintc->version, + pr_info("version [%u] (%s) found\n", cintc->version, virtual ? "virtual" : "physical"); /** @@ -315,7 +315,7 @@ static __init int cintc_acpi_init(union acpi_subtable_headers *header, */ cintc_handle = irq_domain_alloc_named_fwnode("CINTC"); if (!cintc_handle) { - pr_err(PREFIX "failed to alloc fwnode\n"); + pr_err("failed to alloc fwnode\n"); return -ENOMEM; } diff --git a/drivers/irqchip/irq-sunway-lpc-intc.c b/drivers/irqchip/irq-sunway-lpc-intc.c index 80442dae812f..7786488f86a6 100644 --- a/drivers/irqchip/irq-sunway-lpc-intc.c +++ b/drivers/irqchip/irq-sunway-lpc-intc.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 + +#define pr_fmt(fmt) "LPC-INTC: " fmt + #include #include #include @@ -7,8 +10,6 @@ #include #include -#define PREFIX "LPC-INTC: " - #define LPC_IRQ 0x4 #define LPC_IRQ_MASK 0x8 @@ -156,7 +157,7 @@ static int __init lpc_intc_init(struct fwnode_handle *handle, lpc_irq_domain = __irq_domain_add(handle, irqnr, irqnr, 0, &sw64_lpc_domain_ops, base_addr); if (!lpc_irq_domain) { - pr_info(PREFIX "failed to create irq domain\n"); + pr_info("failed to create irq domain\n"); return -ENOMEM; } @@ -208,13 +209,13 @@ static int __init lpc_intc_of_init(struct device_node *np, base = of_iomap(np, 0); if (!base) { - pr_err(PREFIX "failed to remap lpc intc registers\n"); + pr_err("failed to remap lpc intc registers\n"); return -ENXIO; } parent_irq = irq_of_parse_and_map(np, 0); if (!parent_irq) { - pr_err(PREFIX "failed to find parent interrupt\n"); + pr_err("failed to find parent interrupt\n"); ret = -EINVAL; goto out_unmap; } @@ -223,7 +224,7 @@ static int __init lpc_intc_of_init(struct device_node *np, if (ret) goto out_unmap; - pr_info(PREFIX "version [%u] on node [%u] initialized\n", + pr_info("version [%u] on node [%u] initialized\n", version, node); return 0; @@ -250,20 +251,20 @@ int __init lpc_intc_acpi_init(struct irq_domain *parent, bool enabled; enabled = is_lpc_intc_enabled(lpc_intc->flags); - pr_info(PREFIX "version [%u] on node [%u] %s\n", + pr_info("version [%u] on node [%u] %s\n", lpc_intc->version, lpc_intc->node, enabled ? "found" : "disabled"); if (!enabled) return 0; if (lpc_intc->gsi_base != SW_LPC_INTC_GSI_BASE) { - pr_err(PREFIX "invalid GSI\n"); + pr_err("invalid GSI\n"); return -EINVAL; } handle = irq_domain_alloc_named_id_fwnode("LPC-INTC", lpc_intc->node); if (!handle) { - pr_err(PREFIX "failed to alloc fwnode\n"); + pr_err("failed to alloc fwnode\n"); return -ENOMEM; } @@ -273,14 +274,14 @@ int __init lpc_intc_acpi_init(struct irq_domain *parent, parent_irq = irq_create_fwspec_mapping(&fwspec); if (parent_irq <= 0) { - pr_err(PREFIX "failed to map parent irq\n"); + pr_err("failed to map parent irq\n"); ret = -EINVAL; goto out_acpi_free_fwnode; } base_addr = ioremap(lpc_intc->address, lpc_intc->size); if (!base_addr) { - pr_err(PREFIX "failed to map base address\n"); + pr_err("failed to map base address\n"); ret = -ENXIO; goto out_acpi_free_fwnode; } @@ -293,11 +294,11 @@ int __init lpc_intc_acpi_init(struct irq_domain *parent, ret = sw64_add_gsi_domain_map(lpc_intc->gsi_base, lpc_intc->gsi_count, handle); if (ret) { - pr_info(PREFIX "failed to add GSI map\n"); + pr_info("failed to add GSI map\n"); goto out_acpi_free_lpc_domain; } - pr_info(PREFIX "version [%u] on node [%u] initialized\n", + pr_info("version [%u] on node [%u] initialized\n", lpc_intc->version, lpc_intc->node); return 0; diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c index 553ccd0afa07..41df065ad045 100644 --- a/drivers/irqchip/irq-sunway-msi-v2.c +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 + +#define pr_fmt(fmt) "MSIC: " fmt + #include #include #include @@ -8,8 +11,6 @@ #include #include -#define PREFIX "MSIC: " - static struct irq_domain *msi_default_domain; static DEFINE_RAW_SPINLOCK(vector_lock); DEFINE_PER_CPU(vector_irq_t, vector_irq) = { @@ -537,7 +538,7 @@ int __init msic_acpi_init(struct irq_domain *parent, enabled = is_msic_enabled(msic->flags); virtual = is_msic_virtual(msic->flags); - pr_info(PREFIX "version [%u] on node [%u] Root Complex [%u] (%s) %s\n", + pr_info("version [%u] on node [%u] Root Complex [%u] (%s) %s\n", msic->version, msic->node, msic->rc, virtual ? "virtual" : "physical", enabled ? "found" : "disabled"); diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index ba96eab6a3de..f6127eb33884 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 + +#define pr_fmt(fmt) "MSIC: " fmt + #include #include #include @@ -7,8 +10,6 @@ #include #include -#define PREFIX "MSIC: " - static struct irq_domain *msi_default_domain; static DEFINE_RAW_SPINLOCK(vector_lock); DEFINE_PER_CPU(vector_irq_t, vector_irq) = { @@ -492,7 +493,7 @@ int __init msic_acpi_init(struct irq_domain *parent, enabled = is_msic_enabled(msic->flags); virtual = is_msic_virtual(msic->flags); - pr_info(PREFIX "version [%u] on node [%u] Root Complex [%u] (%s) %s\n", + pr_info("version [%u] on node [%u] Root Complex [%u] (%s) %s\n", msic->version, msic->node, msic->rc, virtual ? "virtual" : "physical", enabled ? "found" : "disabled"); diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index 934ad3d60607..185f2a5b99da 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 + +#define pr_fmt(fmt) "PINTC: " fmt + #include #include #include @@ -43,8 +46,6 @@ * +----------------------------------------------------------------+ */ -#define PREFIX "PINTC: " - #define OFFSET_DLI_RLTD_FAULT_INTEN 0xa80UL #define OFFSET_MCU_DVC_INT 0x3000UL #define OFFSET_MCU_DVC_INT_EN 0x3080UL @@ -410,14 +411,14 @@ static int __init pintc_init_mcu(struct pintc_chip_data *chip_data, } if (!mcu_irq_domain) { - pr_err(PREFIX "failed to create MCU irq domain\n"); + pr_err("failed to create MCU irq domain\n"); return -ENOMEM; } raw_spin_lock_init(&chip_data->pintc_lock); raw_spin_lock_init(&chip_data->mcu_lock); - pr_info(PREFIX "MCU version [%u] on node [%u] initialized\n", + pr_info("MCU version [%u] on node [%u] initialized\n", chip_data->version, chip_data->node); return 0; @@ -488,7 +489,7 @@ static int __init pintc_of_init_mcu(struct pintc_chip_data *chip_data, { /* Not yet supported */ if (chip_data->node > 0) { - pr_info(PREFIX "MCU version [%u] on node [%u] skipped\n", + pr_info("MCU version [%u] on node [%u] skipped\n", chip_data->version, chip_data->node); return 0; } @@ -510,7 +511,7 @@ pintc_of_init_common(struct device_node *pintc, return -ENODEV; if (vt && parent) { - pr_err(PREFIX "virtual pintc has no parent controller\n"); + pr_err("virtual pintc has no parent controller\n"); return -EINVAL; } @@ -538,14 +539,14 @@ pintc_of_init_common(struct device_node *pintc, pintc_base = of_iomap(pintc, 0); if (!vt && !pintc_base) { pintc_base = ioremap(INTPU_BASE_V1, INTPU_SIZE_V1); - pr_warn(PREFIX "pintc base address fallback to 0x%lx\n", + pr_warn("pintc base address fallback to 0x%lx\n", INTPU_BASE_V1); } mcu_base = of_iomap(pintc, 1); if (!vt && !mcu_base) { mcu_base = ioremap(MCU_BASE_V1, MCU_SIZE_V1); - pr_warn(PREFIX "mcu base address fallback to 0x%lx\n", + pr_warn("mcu base address fallback to 0x%lx\n", MCU_BASE_V1); } @@ -632,7 +633,7 @@ static int __init lpc_intc_parse_madt(union acpi_subtable_headers *header, if ((lpc_intc->version == ACPI_MADT_SW_LPC_INTC_VERSION_NONE) || (lpc_intc->version >= ACPI_MADT_SW_LPC_INTC_VERSION_RESERVED)) { - pr_err(PREFIX "invalid LPC-INTC version\n"); + pr_err("invalid LPC-INTC version\n"); return -EINVAL; } @@ -660,25 +661,25 @@ static int __init pintc_acpi_init_mcu(struct pintc_chip_data *chip_data, /* Not yet supported */ if (chip_data->node > 0) { - pr_info(PREFIX "MCU version [%u] on node [%u] skipped\n", + pr_info("MCU version [%u] on node [%u] skipped\n", chip_data->version, chip_data->node); return 0; } if (!mcu->status) { - pr_info(PREFIX "MCU version [%u] on node [%u] disabled\n", + pr_info("MCU version [%u] on node [%u] disabled\n", chip_data->version, chip_data->node); return 0; } if (mcu->gsi_base != SW_PINTC_MCU_GSI_BASE) { - pr_err(PREFIX "invalid MCU GSI\n"); + pr_err("invalid MCU GSI\n"); return -EINVAL; } handle = irq_domain_alloc_named_id_fwnode("PINTC-MCU", chip_data->node); if (!handle) { - pr_err(PREFIX "failed to alloc fwnode\n"); + pr_err("failed to alloc fwnode\n"); return -ENOMEM; } @@ -686,7 +687,7 @@ static int __init pintc_acpi_init_mcu(struct pintc_chip_data *chip_data, chip_data->mcu_base = ioremap(mcu->address, mcu->size); if (!chip_data->mcu_base) { - pr_err(PREFIX "failed to map mcu base address\n"); + pr_err("failed to map mcu base address\n"); ret = -ENXIO; goto out_acpi_free_fwnode; } @@ -697,7 +698,7 @@ static int __init pintc_acpi_init_mcu(struct pintc_chip_data *chip_data, ret = sw64_add_gsi_domain_map(mcu->gsi_base, mcu->gsi_count, handle); if (ret) { - pr_info(PREFIX "failed to add GSI map\n"); + pr_info("failed to add GSI map\n"); goto out_acpi_free_mcu_domain; } @@ -720,7 +721,7 @@ static int __init pintc_acpi_init_fault(struct pintc_chip_data *chip_data, struct acpi_madt_sw_sub_pintc *fault) { if (!fault->status) { - pr_info(PREFIX "Fault version [%u] on node [%u] disabled\n", + pr_info("Fault version [%u] on node [%u] disabled\n", chip_data->version, chip_data->node); return 0; } @@ -728,11 +729,11 @@ static int __init pintc_acpi_init_fault(struct pintc_chip_data *chip_data, /* Fault share the same base address with MCU currently */ chip_data->mcu_base = ioremap(fault->address, fault->size); if (!chip_data->mcu_base) { - pr_err(PREFIX "failed to map fault base address\n"); + pr_err("failed to map fault base address\n"); return -ENXIO; } - pr_info(PREFIX "Fault version [%u] on node [%u] initialized\n", + pr_info("Fault version [%u] on node [%u] initialized\n", chip_data->version, chip_data->node); return 0; @@ -748,7 +749,7 @@ int __init pintc_acpi_init(struct irq_domain *parent, enabled = is_pintc_enabled(pintc->flags); virtual = is_pintc_virtual(pintc->flags); - pr_info(PREFIX "version [%u] on node [%u] (%s) %s\n", + pr_info("version [%u] on node [%u] (%s) %s\n", pintc->version, pintc->node, virtual ? "virtual" : "physical", enabled ? "found" : "disabled"); @@ -757,7 +758,7 @@ int __init pintc_acpi_init(struct irq_domain *parent, return 0; if (pintc_sub_type_check(pintc)) { - pr_err(PREFIX "invalid sub type\n"); + pr_err("invalid sub type\n"); return -EINVAL; } @@ -777,7 +778,7 @@ int __init pintc_acpi_init(struct irq_domain *parent, chip_data->pintc_base = ioremap(pintc->address, pintc->size); if (!chip_data->pintc_base) { - pr_err(PREFIX "failed to map pintc base address\n"); + pr_err("failed to map pintc base address\n"); ret = -ENXIO; goto out_acpi_free_chip_data; } -- Gitee From d24e3f685612feeb033c8b8e5348970f23308cca Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 23 Dec 2024 15:08:22 +0800 Subject: [PATCH 406/524] sw64: irqchip: use sunway as vendor prefix in device tree Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/platform.h | 49 +++++++++++++++++++++++++++ drivers/irqchip/irq-sunway-lpc-intc.c | 29 +++++----------- drivers/irqchip/irq-sunway-pintc.c | 33 +++++------------- 3 files changed, 66 insertions(+), 45 deletions(-) diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h index 96656f112c40..227f8eeb7f95 100644 --- a/arch/sw_64/include/asm/platform.h +++ b/arch/sw_64/include/asm/platform.h @@ -28,4 +28,53 @@ extern void __iomem *misc_platform_get_cab0_base(unsigned long node); extern bool sunway_machine_is_compatible(const char *compat); +#ifdef CONFIG_OF +#include +#include + +static inline void __init +sunway_of_get_numa_node(struct device_node *np, u32 *node, u32 fallback) +{ + int nid; + + nid = of_node_to_nid(np); + if (nid != NUMA_NO_NODE) { + *node = nid; + return; + } + + /* Fallback to legacy property */ + if (of_property_read_u32(np, "sw64,node", node)) { + *node = fallback; + pr_warn("node fallback to %u\n", fallback); + } +} + +static inline void __init +sunway_of_get_irq_num(struct device_node *np, u32 *irq_num, u32 fallback) +{ + if (!of_property_read_u32(np, "sunway,irq-num", irq_num)) + return; + + /* Fallback to legacy property */ + if (of_property_read_u32(np, "sw64,irq-num", irq_num)) { + *irq_num = fallback; + pr_warn("irq-num fallback to %u\n", fallback); + } +} + +static inline void __init +sunway_of_get_version(struct device_node *np, u32 *version, u32 fallback) +{ + if (!of_property_read_u32(np, "sunway,version", version)) + return; + + /* Fallback to legacy property */ + if (of_property_read_u32(np, "sw64,ver", version)) { + *version = fallback; + pr_warn("version fallback to %u\n", fallback); + } +} +#endif + #endif /* _ASM_SW64_PLATFORM_H */ diff --git a/drivers/irqchip/irq-sunway-lpc-intc.c b/drivers/irqchip/irq-sunway-lpc-intc.c index 7786488f86a6..8ad6109cdcf3 100644 --- a/drivers/irqchip/irq-sunway-lpc-intc.c +++ b/drivers/irqchip/irq-sunway-lpc-intc.c @@ -10,6 +10,8 @@ #include #include +#include + #define LPC_IRQ 0x4 #define LPC_IRQ_MASK 0x8 @@ -186,26 +188,9 @@ static int __init lpc_intc_of_init(struct device_node *np, sw_lpc_intc_node = np; - ret = of_property_read_u32(np, "sw64,node", &node); - if (ret) { - node = 0; - pr_warn(PREFIX "\"sw64,node\" fallback to %u\n", - node); - } - - ret = of_property_read_u32(np, "sw64,irq-num", &nr_irqs); - if (ret) { - nr_irqs = 16; - pr_warn(PREFIX "\"sw64,irq-num\" fallback to %u\n", - nr_irqs); - } - - ret = of_property_read_u32(np, "sw64,ver", &version); - if (ret) { - version = 1; - pr_warn(PREFIX "\"sw64,ver\" fallback to %u\n", - version); - } + sunway_of_get_numa_node(np, &node, 0); + sunway_of_get_irq_num(np, &nr_irqs, 16); + sunway_of_get_version(np, &version, 1); base = of_iomap(np, 0); if (!base) { @@ -233,7 +218,9 @@ static int __init lpc_intc_of_init(struct device_node *np, iounmap(base); return ret; } -IRQCHIP_DECLARE(sw_lpc_intc, "sw64,lpc_intc", lpc_intc_of_init); + +IRQCHIP_DECLARE(sunway_lpc_intc, "sunway,lpc-intc", lpc_intc_of_init); +IRQCHIP_DECLARE(sunway_lpc_intc_legacy, "sw64,lpc_intc", lpc_intc_of_init); #endif #ifdef CONFIG_ACPI diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index 185f2a5b99da..6aaebea7c74b 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -515,26 +515,9 @@ pintc_of_init_common(struct device_node *pintc, return -EINVAL; } - ret = of_property_read_u32(pintc, "sw64,node", &node); - if (ret) { - node = 0; - pr_warn(PREFIX "\"sw64,node\" fallback to %u\n", - node); - } - - ret = of_property_read_u32(pintc, "sw64,irq-num", &nr_irqs); - if (ret) { - nr_irqs = vt ? 16 : 8; - pr_warn(PREFIX "\"sw64,irq-num\" fallback to %u\n", - nr_irqs); - } - - ret = of_property_read_u32(pintc, "sw64,ver", &version); - if (ret) { - version = 1; - pr_warn(PREFIX "\"sw64,ver\" fallback to %u\n", - version); - } + sunway_of_get_numa_node(pintc, &node, 0); + sunway_of_get_irq_num(pintc, &nr_irqs, vt ? 16 : 8); + sunway_of_get_version(pintc, &version, 1); pintc_base = of_iomap(pintc, 0); if (!vt && !pintc_base) { @@ -589,8 +572,9 @@ pintc_of_init(struct device_node *pintc, struct device_node *parent) return pintc_of_init_common(pintc, parent, false); } -IRQCHIP_DECLARE(sw64_pintc, "sw64,pintc", pintc_of_init); -IRQCHIP_DECLARE(sw64_pintc_legacy, "sw64,sw6_irq_controller", pintc_of_init); +IRQCHIP_DECLARE(sunway_pintc, "sunway,pintc", pintc_of_init); +IRQCHIP_DECLARE(sunway_pintc_legacy1, "sw64,pintc", pintc_of_init); +IRQCHIP_DECLARE(sunway_pintc_legacy0, "sw64,sw6_irq_controller", pintc_of_init); static int __init pintc_vt_of_init(struct device_node *pintc, struct device_node *parent) @@ -598,8 +582,9 @@ pintc_vt_of_init(struct device_node *pintc, struct device_node *parent) return pintc_of_init_common(pintc, parent, true); } -IRQCHIP_DECLARE(sw64_pintc_vt, "sw64,pintc_vt", pintc_vt_of_init); -IRQCHIP_DECLARE(sw64_pintc_vt_legacy, "sw64,sw6_irq_vt_controller", pintc_vt_of_init); +IRQCHIP_DECLARE(sunway_pintc_vt, "sunway,pintc-vt", pintc_vt_of_init); +IRQCHIP_DECLARE(sunway_pintc_vt_legacy1, "sw64,pintc_vt", pintc_vt_of_init); +IRQCHIP_DECLARE(sunway_pintc_vt_legacy0, "sw64,sw6_irq_vt_controller", pintc_vt_of_init); #endif #ifdef CONFIG_ACPI -- Gitee From 4554ad099e4d794b23fb2f6872368d62ea1b7882 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 17 Oct 2024 08:26:42 +0800 Subject: [PATCH 407/524] sw64: cpu: give preference to CPU information from SMBIOS Prioritize obtaining CPU information from SMBIOS, and if the information from SMBIOS is invalid, fallback to obtaining CPU information from hmcode. Please note that this commit will not break the backward compatibility of C3B. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpu.h | 2 +- arch/sw_64/kernel/cpu.c | 154 ++++++++++++++++++++++++++++++++--- 2 files changed, 143 insertions(+), 13 deletions(-) diff --git a/arch/sw_64/include/asm/cpu.h b/arch/sw_64/include/asm/cpu.h index 37a588fe12d9..ab67d723c069 100644 --- a/arch/sw_64/include/asm/cpu.h +++ b/arch/sw_64/include/asm/cpu.h @@ -30,7 +30,7 @@ enum sunway_cpu_model { struct cpuinfo_sw64 { __u8 model; - __u8 family; + __u16 family; __u8 chip_var; __u8 arch_var; __u8 arch_rev; diff --git a/arch/sw_64/kernel/cpu.c b/arch/sw_64/kernel/cpu.c index 0f44e58fbe52..538f1fcfb686 100644 --- a/arch/sw_64/kernel/cpu.c +++ b/arch/sw_64/kernel/cpu.c @@ -5,15 +5,23 @@ #include #include #include +#include #include #include #include +#include #define TABLE_ENTRY_MAX 32 #define VENDOR_ID_MAX 2 #define MODEL_MAX 8 +/* Offset in the DMI processor structure (Type 4) */ +#define DMI_PROCESSOR_FAMILY 0x06 +#define DMI_PROCESSOR_MANUFACTURER 0x07 +#define DMI_PROCESSOR_VERSION 0x10 +#define DMI_PROCESSOR_FAMILY2 0x28 + #define cpuinfo_arch_rev(cpu_info) ((cpu_info) & 0xf) #define cpuinfo_arch_var(cpu_info) (((cpu_info) >> 4) & 0xf) #define cpuinfo_chip_var(cpu_info) (((cpu_info) >> 8) & 0xf) @@ -34,7 +42,8 @@ cpumask_t cpu_offline = CPU_MASK_NONE; static unsigned long cpu_freq; static unsigned long cpu_info; -static char vendor_id[16]; +static __u16 family; +static char vendor_id[64]; static char model_id[64]; unsigned long get_cpu_freq(void) @@ -60,38 +69,159 @@ static int cpuinfo_cpu_online(unsigned int cpu) { /* Currently, cpu info is shared by all cores */ cpu_data[cpu].model = cpuinfo_model(cpu_info); - cpu_data[cpu].family = cpuinfo_family(cpu_info); cpu_data[cpu].chip_var = cpuinfo_chip_var(cpu_info); cpu_data[cpu].arch_var = cpuinfo_arch_var(cpu_info); cpu_data[cpu].arch_rev = cpuinfo_arch_rev(cpu_info); cpu_data[cpu].pa_bits = cpuinfo_pa_bits(cpu_info); cpu_data[cpu].va_bits = cpuinfo_va_bits(cpu_info); + cpu_data[cpu].family = family; + cpu_data[cpu].vendor_id = vendor_id; cpu_data[cpu].model_id = model_id; return 0; } -static int __init sw64_cpuinfo_init(void) +static const char * __init dmi_get_string(const struct dmi_header *dm, u8 s) +{ + const u8 *bp = ((u8 *) dm) + dm->length; + const u8 *nsp; + + if (s) { + while (--s > 0 && *bp) + bp += strlen(bp) + 1; + + /* Strings containing only spaces are considered empty */ + nsp = bp; + while (*nsp == ' ') + nsp++; + if (*nsp != '\0') + return bp; + } + + return ""; +} + +static void __init find_dmi_processor_version(const struct dmi_header *dm, + void *private) { - int i, ret; + char *dmi_data = (char *)dm; + const char *p; + size_t len; + + if (dm->type != DMI_ENTRY_PROCESSOR) + return; + + p = dmi_get_string(dm, dmi_data[DMI_PROCESSOR_VERSION]); + + len = strlen(p); + + if ((len > 0) && (len < ARRAY_SIZE(model_id))) + strcpy(model_id, p); +} + +static void __init get_model_id(void) +{ + int i; unsigned long val; - /* Get cpu info */ - cpu_info = cpuid(GET_TABLE_ENTRY, 0); + /* Prefer model id from SMBIOS */ + if (!IS_ENABLED(CONFIG_SUBARCH_C3B) && + sunway_bios_version) + dmi_walk(find_dmi_processor_version, NULL); - /* Get vendor name in string format */ - for (i = 0; i < VENDOR_ID_MAX; i++) { - val = cpuid(GET_VENDOR_ID, i); - memcpy(vendor_id + (i * 8), &val, 8); - } + if (strlen(model_id) > 0) + return; - /* Get model name in string format */ + /* Fallback to HMCode */ for (i = 0; i < MODEL_MAX; i++) { val = cpuid(GET_MODEL, i); memcpy(model_id + (i * 8), &val, 8); } +} + +static void __init find_dmi_processor_manufacturer(const struct dmi_header *dm, + void *private) +{ + char *dmi_data = (char *)dm; + const char *p; + size_t len; + + if (dm->type != DMI_ENTRY_PROCESSOR) + return; + + p = dmi_get_string(dm, dmi_data[DMI_PROCESSOR_MANUFACTURER]); + + len = strlen(p); + + if ((len > 0) && (len < ARRAY_SIZE(vendor_id))) + strcpy(vendor_id, p); +} + +static void __init get_vendor_id(void) +{ + int i; + unsigned long val; + + /* Prefer vendor id from SMBIOS */ + if (!IS_ENABLED(CONFIG_SUBARCH_C3B) && + sunway_bios_version) + dmi_walk(find_dmi_processor_manufacturer, NULL); + + if (strlen(vendor_id) > 0) + return; + + /* Fallback to HMCode */ + for (i = 0; i < VENDOR_ID_MAX; i++) { + val = cpuid(GET_VENDOR_ID, i); + memcpy(vendor_id + (i * 8), &val, 8); + } +} + +static void __init find_dmi_processor_family(const struct dmi_header *dm, + void *private) +{ + char *dmi_data = (char *)dm; + + if (dm->type != DMI_ENTRY_PROCESSOR) + return; + + family = *(__u8 *)(dmi_data + DMI_PROCESSOR_FAMILY); + + if (family == 0xfe) + family = *(__u16 *)(dmi_data + DMI_PROCESSOR_FAMILY2); +} + +static void __init get_family(void) +{ + /* Prefer processor family from SMBIOS */ + if (!IS_ENABLED(CONFIG_SUBARCH_C3B) && + sunway_bios_version) + dmi_walk(find_dmi_processor_family, NULL); + + if (family) + return; + + /* Fallback to HMCode */ + family = cpuinfo_family(cpu_info); +} + +static int __init sw64_cpuinfo_init(void) +{ + int ret; + + /* Get CPU information from HMCode */ + cpu_info = cpuid(GET_TABLE_ENTRY, 0); + + /* Get processor family */ + get_family(); + + /* Get vendor name in string format */ + get_vendor_id(); + + /* Get processor name in string format */ + get_model_id(); ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "sw64/cpuinfo:online", cpuinfo_cpu_online, NULL); -- Gitee From 0460ddea6b65686b2c70a9e89db0d0b4537befba Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 10 Feb 2025 14:20:49 +0800 Subject: [PATCH 408/524] sw64: kvm: fix incorrect vm_flags in vmem_mmap() When adding KVM support for CORE3B, vm_flags_init() was mistakenly called, causing the vm_flags of related shared memory mapping to be corrupted, and the write permission for this mapping is cleared. In commit f2d23021ce74 ("sw64: fix mmap protection_map"), FOW bit is set for shared writable page to timely mark dirty pages. Subsequently, a write fault is triggered while writing to the abovementioned shared writable memory. Due to the incorrect vm_flags, it can't be handled properly, and segment fault is thrown eventually. This patch fixes vm_flags with vm_flags_set() which performs an OR operation. Fixes: 46814e6da2f8 ("sw64: add KVM support") Signed-off-by: Chen Wang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/vmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kvm/vmem.c b/arch/sw_64/kvm/vmem.c index 945667f25955..597251364695 100644 --- a/arch/sw_64/kvm/vmem.c +++ b/arch/sw_64/kvm/vmem.c @@ -41,7 +41,7 @@ static int vmem_vm_insert_page(struct vm_area_struct *vma) size = info->size; uaddr = vma->vm_start; - vm_flags_init(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP); + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP); vmem_page = pfn_to_page(addr >> PAGE_SHIFT); do { ret = vm_insert_page(vma, uaddr, vmem_page); -- Gitee From 24f141d02bb28530ed96b27f91a222705961a841 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 14 Feb 2025 15:53:01 +0800 Subject: [PATCH 409/524] sw64: ftrace: implement ftrace_modify_call In some cases, ftrace needs to call ftrace_modify_call() to change the target address of "call r28" from ftrace_caller to ftrace_regs_caller, the latter of which prepares an additional $19 to pass pt_regs. However, since we implememted ftrace_modify_call() as empty, a subsequent function might use a random value of $19, which could cause panic. This patch implements the ftrace_modify_call(). Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/ftrace.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/sw_64/kernel/ftrace.c b/arch/sw_64/kernel/ftrace.c index fb25ffe3dbda..6a8446e59961 100644 --- a/arch/sw_64/kernel/ftrace.c +++ b/arch/sw_64/kernel/ftrace.c @@ -115,6 +115,19 @@ int __init ftrace_dyn_arch_init(void) int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { + unsigned int insn[1]; + unsigned long pc = rec->ip + MCOUNT_LDGP_SIZE + 4; + unsigned long offset; + + if (addr == FTRACE_ADDR) + offset = TI_FTRACE_ADDR; + else + offset = TI_FTRACE_REGS_ADDR; + + /* ldl r28,(ftrace_addr_offset)(r8) */ + insn[0] = (0x23U << 26) | (28U << 21) | (8U << 16) | offset; + copy_to_kernel_nofault((void *)pc, insn, SW64_INSN_SIZE); + return 0; } #endif -- Gitee From 9caf76556396cda22a5bc465d9b7bbe351e1d96e Mon Sep 17 00:00:00 2001 From: Jing Li Date: Sat, 8 Feb 2025 14:24:04 +0800 Subject: [PATCH 410/524] sw64: cache: fix cache ID is not unique when ACPI disabled When ACPI disabled, Cache information is populated by the arch-specific implementation that does not provide a unique cache ID for L1/L2 cache located on different NUMA nodes. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/cacheinfo.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kernel/cacheinfo.c b/arch/sw_64/kernel/cacheinfo.c index c8528cf9f3ba..fb4c9ca7650b 100644 --- a/arch/sw_64/kernel/cacheinfo.c +++ b/arch/sw_64/kernel/cacheinfo.c @@ -166,19 +166,20 @@ int populate_cache_leaves(unsigned int cpu) bool pptt_valid = is_pptt_cache_info_valid(); for (type = L1_ICACHE; type <= L3_CACHE; type++, this_leaf++) { + unsigned int node = rcid_to_domain_id(cpu_to_rcid(cpu)); + unsigned int core = rcid_to_core_id(cpu_to_rcid(cpu)); + if (!cache_size(type)) continue; /* L3 Cache is shared */ - cache_id = (type == L3_CACHE) ? rcid_to_domain_id(cpu_to_rcid(cpu)) : - rcid_to_core_id(cpu_to_rcid(cpu)); + cache_id = (type == L3_CACHE) ? node : ((node << 16) | core); populate_cache(get_cache_info(type), this_leaf, cache_level(type), kernel_cache_type(type), cache_id); if (pptt_valid) this_leaf->attributes &= ~CACHE_ID; - } if (!pptt_valid) { -- Gitee From d143a7953b0e34028718eaf2fcf754c298acbc2a Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Tue, 11 Feb 2025 10:32:10 +0800 Subject: [PATCH 411/524] sw64: fix an error when running no main() program While running no main() program, it can not update gp through r27. So, make regs->regs[27] = pc in start_thread() to fix it. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/process.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c index 1447ea34276c..429941af6cc6 100644 --- a/arch/sw_64/kernel/process.c +++ b/arch/sw_64/kernel/process.c @@ -23,6 +23,7 @@ void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) { regs->pc = pc; + regs->regs[27] = pc; regs->ps = 8; regs->regs[30] = sp; } -- Gitee From 1141d0e15b8347153ed268fb4ca2c2a20705a8f7 Mon Sep 17 00:00:00 2001 From: Zhi Tongze Date: Mon, 10 Feb 2025 15:22:33 +0800 Subject: [PATCH 412/524] sw64: flush TLB after modifying ptbr_sys We need to flush TLB after modifying ptbr_sys to clear old TLB entries. Signed-off-by: Zhi Tongze Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/csr.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h index e205c3f1427e..cc36137d6dd6 100644 --- a/arch/sw_64/include/asm/csr.h +++ b/arch/sw_64/include/asm/csr.h @@ -104,8 +104,10 @@ static inline void sw64_write_csr_imb(unsigned long x, unsigned long y) #include static inline void update_ptbr_sys(unsigned long ptbr) { + mb(); imemb(); sw64_write_csr_imb(ptbr, CSR_PTBR_SYS); + tbiv(); } static inline void update_ptbr_usr(unsigned long ptbr) -- Gitee From 303df98ee8c498d8678ad5b29ef3545092c9ee63 Mon Sep 17 00:00:00 2001 From: Ye Si Date: Tue, 18 Feb 2025 09:45:27 +0000 Subject: [PATCH 413/524] sw64: sound: avoid pagefault in pcm driver's memset sw64 does not support SIMD instuction to access iomem region. memset by default is optimized by SIMD instructions, memset_io is not. Signed-off-by: wenlunpeng Signed-off-by: Ye Si Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- sound/core/pcm_misc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sound/core/pcm_misc.c b/sound/core/pcm_misc.c index 5588b6a1ee8b..a8061ce9479d 100644 --- a/sound/core/pcm_misc.c +++ b/sound/core/pcm_misc.c @@ -438,7 +438,12 @@ int snd_pcm_format_set_silence(snd_pcm_format_t format, void *data, unsigned int /* signed or 1 byte data */ if (pcm_formats[(INT)format].signd == 1 || width <= 8) { unsigned int bytes = samples * width / 8; +#ifdef CONFIG_SW64 +#include + memset_io(data, *pat, bytes); +#else memset(data, *pat, bytes); +#endif return 0; } /* non-zero samples, fill using a loop */ -- Gitee From e8b3707a2fe3ccd3733a74c1bf11334ebbe3486a Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 18 Feb 2025 09:08:35 +0000 Subject: [PATCH 414/524] sw64: pci: fix secondary bus reset issue This patch makes two modifications: - Secondary bus reset may reset PIU and RC, so we need to save and restore PIU and RC. - Unify the save and restore interfaces for PIU and RC. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 28 ++-- arch/sw_64/kernel/chip_setup.c | 69 +-------- arch/sw_64/pci/pci.c | 19 +++ drivers/pci/controller/pci-sunway.c | 171 +++++++++++++++++++++++ drivers/pci/hotplug/sunway_pciehp_ctrl.c | 85 +---------- 5 files changed, 212 insertions(+), 160 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index a1acda1c166c..c0981bc22682 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -86,21 +86,10 @@ struct resource; struct sunway_iommu; struct page; -struct piu_saved { - unsigned long piuconfig0; - unsigned long piuconfig1; - unsigned long epdmabar; - unsigned long msiaddr; - unsigned long msiconfig[256]; - unsigned long iommuexcpt_ctrl; - unsigned long dtbaseaddr; - unsigned long hpintconfig; - unsigned long pmeintconfig; - unsigned long aererrintconfig; - unsigned long intaconfig; - unsigned long intbconfig; - unsigned long intcconfig; - unsigned long intdconfig; +struct piu_saved_data { + unsigned int size; + bool saved_state; + u64 data[0]; }; /* A controller. Used to manage multiple PCI busses. */ @@ -139,6 +128,8 @@ struct pci_controller { int last_busno; int self_busno; void *sysdata; + struct piu_saved_data *piu_ior0; + struct piu_saved_data *piu_ior1; }; /* Override the logic in pci_scan_bus for skipping already-configured @@ -165,6 +156,13 @@ extern void __init setup_chip_pci_ops(void); #define setup_chip_pci_ops() do { } while (0) #endif +extern void save_piu_ior0(struct pci_controller *hose); +extern void restore_piu_ior0(struct pci_controller *hose); +extern void save_piu_ior1(struct pci_controller *hose); +extern void restore_piu_ior1(struct pci_controller *hose); +extern void save_rc_piu(struct pci_dev *dev); +extern void restore_rc_piu(struct pci_dev *dev); + extern struct pci_controller * pci_bus_to_pci_controller(const struct pci_bus *bus); extern struct pci_controller *bus_num_to_pci_controller(unsigned long bus_num); diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index 732943cb1859..df98e141b14f 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -93,84 +93,22 @@ static void i2c_srst(void) static void pcie_save(void) { struct pci_controller *hose; - struct piu_saved *piu_save; - unsigned long i; - void __iomem *piu_ior0_base; - void __iomem *piu_ior1_base; for (hose = hose_head; hose; hose = hose->next) { - piu_ior0_base = hose->piu_ior0_base; - piu_ior1_base = hose->piu_ior1_base; - - piu_save = kzalloc(sizeof(*piu_save), GFP_KERNEL); - hose->sysdata = piu_save; - - piu_save->piuconfig0 = readq(piu_ior0_base + PIUCONFIG0); - piu_save->piuconfig1 = readq(piu_ior1_base + PIUCONFIG1); - piu_save->epdmabar = readq(piu_ior0_base + EPDMABAR); - piu_save->msiaddr = readq(piu_ior0_base + MSIADDR); - - if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { - for (i = 0; i < 256; i++) { - piu_save->msiconfig[i] = - readq(piu_ior0_base + MSICONFIG0 + (i << 7)); - } - } - - piu_save->iommuexcpt_ctrl = readq(piu_ior0_base + IOMMUEXCPT_CTRL); - piu_save->dtbaseaddr = readq(piu_ior0_base + DTBASEADDR); - - piu_save->intaconfig = readq(piu_ior0_base + INTACONFIG); - piu_save->intbconfig = readq(piu_ior0_base + INTBCONFIG); - piu_save->intcconfig = readq(piu_ior0_base + INTCCONFIG); - piu_save->intdconfig = readq(piu_ior0_base + INTDCONFIG); - piu_save->pmeintconfig = readq(piu_ior0_base + PMEINTCONFIG); - piu_save->aererrintconfig = readq(piu_ior0_base + AERERRINTCONFIG); - piu_save->hpintconfig = readq(piu_ior0_base + HPINTCONFIG); - + save_piu_ior0(hose); + save_piu_ior1(hose); } } static void pcie_restore(void) { struct pci_controller *hose; - struct piu_saved *piu_save; u32 rc_misc_ctrl; unsigned int value; - unsigned long i; void __iomem *rc_config_space_base; - void __iomem *piu_ior0_base; - void __iomem *piu_ior1_base; for (hose = hose_head; hose; hose = hose->next) { rc_config_space_base = hose->rc_config_space_base; - piu_ior0_base = hose->piu_ior0_base; - piu_ior1_base = hose->piu_ior1_base; - piu_save = hose->sysdata; - - writeq(piu_save->piuconfig0, (piu_ior0_base + PIUCONFIG0)); - writeq(piu_save->piuconfig1, (piu_ior1_base + PIUCONFIG1)); - writeq(piu_save->epdmabar, (piu_ior0_base + EPDMABAR)); - writeq(piu_save->msiaddr, (piu_ior0_base + MSIADDR)); - - - if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { - for (i = 0; i < 256; i++) { - writeq(piu_save->msiconfig[i], - (piu_ior0_base + (MSICONFIG0 + (i << 7)))); - } - } - - writeq(piu_save->iommuexcpt_ctrl, (piu_ior0_base + IOMMUEXCPT_CTRL)); - writeq(piu_save->dtbaseaddr, (piu_ior0_base + DTBASEADDR)); - - writeq(piu_save->intaconfig, (piu_ior0_base + INTACONFIG)); - writeq(piu_save->intbconfig, (piu_ior0_base + INTBCONFIG)); - writeq(piu_save->intcconfig, (piu_ior0_base + INTCCONFIG)); - writeq(piu_save->intdconfig, (piu_ior0_base + INTDCONFIG)); - writeq(piu_save->pmeintconfig, (piu_ior0_base + PMEINTCONFIG)); - writeq(piu_save->aererrintconfig, (piu_ior0_base + AERERRINTCONFIG)); - writeq(piu_save->hpintconfig, (piu_ior0_base + HPINTCONFIG)); /* Enable DBI_RO_WR_EN */ rc_misc_ctrl = readl(rc_config_space_base + RC_MISC_CONTROL_1); @@ -186,6 +124,9 @@ static void pcie_restore(void) /* Disable DBI_RO_WR_EN */ writel(rc_misc_ctrl, (rc_config_space_base + RC_MISC_CONTROL_1)); + + restore_piu_ior0(hose); + restore_piu_ior1(hose); } } diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index ea68291a0b37..deb5d8d052d4 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,24 @@ int raw_pci_write(unsigned int domain, unsigned int bus, unsigned int devfn, return -EINVAL; } +void pcibios_reset_secondary_bus(struct pci_dev *dev) +{ + if (dev->bus->self) { + pci_reset_secondary_bus(dev); + } else { + if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { + pr_warn("The chip does not support secondary bus reset! \ + No further action is required.\n"); + return; + } + + save_rc_piu(dev); + pci_reset_secondary_bus(dev); + ssleep(1); + restore_rc_piu(dev); + } +} + resource_size_t pcibios_default_alignment(void) { if (is_in_guest()) diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index bce0659e6bf0..6a63adab8b04 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -36,6 +36,171 @@ int pcibios_enable_device(struct pci_dev *dev, int bars) return pci_enable_resources(dev, bars); } +#ifdef CONFIG_UNCORE_XUELANG +#define PIU_IOR0_SAVE_REGS (12 + 256) +#define PIU_IOR1_SAVE_REGS 1 +#else +#define PIU_IOR0_SAVE_REGS 12 +#define PIU_IOR1_SAVE_REGS 1 +#endif + +static struct piu_saved_data *alloc_piu_saved_data(unsigned int size) +{ + struct piu_saved_data *piu_saved_data; + + piu_saved_data = kzalloc(sizeof(*piu_saved_data) + size, GFP_KERNEL); + if (!piu_saved_data) { + pr_err("alloc piu saved data failed!\n"); + return NULL; + } + + piu_saved_data->size = size; + + return piu_saved_data; +} + +void save_piu_ior0(struct pci_controller *hose) +{ + int i = 0, j; + void __iomem *piu_ior0_base; + struct piu_saved_data *piu_ior0; + u64 *ior; + + piu_ior0 = hose->piu_ior0; + if (!piu_ior0) + return; + + piu_ior0_base = hose->piu_ior0_base; + + ior = piu_ior0->data; + ior[i++] = readq(piu_ior0_base + PIUCONFIG0); + ior[i++] = readq(piu_ior0_base + EPDMABAR); + ior[i++] = readq(piu_ior0_base + IOMMUEXCPT_CTRL); + ior[i++] = readq(piu_ior0_base + MSIADDR); + + if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { + for (j = 0; j < 256; j++) + ior[i++] = readq(piu_ior0_base + MSICONFIG0 + (j << 7)); + } + + ior[i++] = readq(piu_ior0_base + INTACONFIG); + ior[i++] = readq(piu_ior0_base + INTBCONFIG); + ior[i++] = readq(piu_ior0_base + INTCCONFIG); + ior[i++] = readq(piu_ior0_base + INTDCONFIG); + ior[i++] = readq(piu_ior0_base + PMEINTCONFIG); + ior[i++] = readq(piu_ior0_base + AERERRINTCONFIG); + ior[i++] = readq(piu_ior0_base + HPINTCONFIG); + ior[i++] = readq(piu_ior0_base + DTBASEADDR); + + piu_ior0->saved_state = true; + +} + +void restore_piu_ior0(struct pci_controller *hose) +{ + int i = 0, j; + void __iomem *piu_ior0_base; + struct piu_saved_data *piu_ior0; + u64 *ior; + + piu_ior0 = hose->piu_ior0; + if (!piu_ior0) + return; + + if (piu_ior0->saved_state) { + piu_ior0_base = hose->piu_ior0_base; + + ior = piu_ior0->data; + writeq(ior[i++], piu_ior0_base + PIUCONFIG0); + writeq(ior[i++], piu_ior0_base + EPDMABAR); + writeq(ior[i++], piu_ior0_base + IOMMUEXCPT_CTRL); + writeq(ior[i++], piu_ior0_base + MSIADDR); + + if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { + for (j = 0; j < 256; j++) + writeq(ior[i++], piu_ior0_base + MSICONFIG0 + (j << 7)); + } + writeq(ior[i++], piu_ior0_base + INTACONFIG); + writeq(ior[i++], piu_ior0_base + INTBCONFIG); + writeq(ior[i++], piu_ior0_base + INTCCONFIG); + writeq(ior[i++], piu_ior0_base + INTDCONFIG); + writeq(ior[i++], piu_ior0_base + PMEINTCONFIG); + writeq(ior[i++], piu_ior0_base + AERERRINTCONFIG); + writeq(ior[i++], piu_ior0_base + HPINTCONFIG); + writeq(ior[i++], piu_ior0_base + DTBASEADDR); + + piu_ior0->saved_state = false; + } +} + +void save_piu_ior1(struct pci_controller *hose) +{ + int i = 0; + void __iomem *piu_ior1_base; + struct piu_saved_data *piu_ior1; + u64 *ior; + + piu_ior1 = hose->piu_ior1; + if (!piu_ior1) + return; + + piu_ior1_base = hose->piu_ior1_base; + + ior = piu_ior1->data; + ior[i++] = readq(piu_ior1_base + PIUCONFIG1); + + piu_ior1->saved_state = true; +} + +void restore_piu_ior1(struct pci_controller *hose) +{ + int i = 0; + void __iomem *piu_ior1_base; + struct piu_saved_data *piu_ior1; + u64 *ior; + + piu_ior1 = hose->piu_ior1; + if (!piu_ior1) + return; + + if (piu_ior1->saved_state) { + piu_ior1_base = hose->piu_ior1_base; + + ior = piu_ior1->data; + writeq(ior[i++], piu_ior1_base + PIUCONFIG1); + + piu_ior1->saved_state = false; + } +} + +void save_rc_piu(struct pci_dev *dev) +{ + struct pci_bus *bus = dev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + if (bus->self) + return; + + pci_save_state(dev); + + save_piu_ior0(hose); + save_piu_ior1(hose); +} + +void restore_rc_piu(struct pci_dev *dev) +{ + struct pci_bus *bus = dev->bus; + struct pci_controller *hose = pci_bus_to_pci_controller(bus); + + if (bus->self) + return; + + pci_restore_state(dev); + + restore_piu_ior0(hose); + restore_piu_ior1(hose); +} + int chip_pcie_configure(struct pci_controller *hose) { struct pci_dev *dev; @@ -316,6 +481,9 @@ static void hose_init(struct pci_controller *hose) hose->first_busno = hose->self_busno = hose->busn_space->start; hose->last_busno = hose->busn_space->end; + hose->piu_ior0 = alloc_piu_saved_data(PIU_IOR0_SAVE_REGS * sizeof(u64)); + hose->piu_ior1 = alloc_piu_saved_data(PIU_IOR1_SAVE_REGS * sizeof(u64)); + if (is_in_host()) { if (IS_ENABLED(CONFIG_PCI_MSI)) memset(hose->piu_msiconfig, 0, 256/8); @@ -719,6 +887,9 @@ static int pci_prepare_controller(struct pci_controller *hose, hose->need_domain_info = 0; + hose->piu_ior0 = alloc_piu_saved_data(PIU_IOR0_SAVE_REGS * sizeof(u64)); + hose->piu_ior1 = alloc_piu_saved_data(PIU_IOR1_SAVE_REGS * sizeof(u64)); + #if IS_ENABLED(CONFIG_PCI_MSI) if (is_in_host()) memset(hose->piu_msiconfig, 0, 256 / 8); /* 256 bits bitmap */ diff --git a/drivers/pci/hotplug/sunway_pciehp_ctrl.c b/drivers/pci/hotplug/sunway_pciehp_ctrl.c index a1c4da32d0ee..b2a0a437445c 100644 --- a/drivers/pci/hotplug/sunway_pciehp_ctrl.c +++ b/drivers/pci/hotplug/sunway_pciehp_ctrl.c @@ -218,44 +218,11 @@ void sunway_pciehp_handle_disable_request(struct controller *ctrl) ctrl->request_result = sunway_pciehp_disable_slot(ctrl, SAFE_REMOVAL); } -void sunway_pciehp_save_config_space(struct controller *ctrl) -{ - struct pci_dev *pdev = ctrl->pci_dev; - int i; - - if (!pdev->state_saved) { - for (i = 0; i < 16; i++) - pci_read_config_dword(pdev, i * 4, &pdev->saved_config_space[i]); - pdev->state_saved = true; - } -} - void sunway_pciehp_save_rc_piu(struct controller *ctrl) { - struct pci_bus *bus = ctrl->pci_dev->bus; - struct pci_controller *hose = pci_bus_to_pci_controller(bus); - void __iomem *piu_ior0_base; - - piu_ior0_base = hose->piu_ior0_base; - - sunway_pciehp_save_config_space(ctrl); - - if (!ctrl->saved_piu.state_saved) { - ctrl->saved_piu.epdmabar = readq(piu_ior0_base + EPDMABAR); - ctrl->saved_piu.msiaddr = readq(piu_ior0_base + MSIADDR); - ctrl->saved_piu.iommuexcpt_ctrl = readq(piu_ior0_base + IOMMUEXCPT_CTRL); - ctrl->saved_piu.dtbaseaddr = readq(piu_ior0_base + DTBASEADDR); - - ctrl->saved_piu.intaconfig = readq(piu_ior0_base + INTACONFIG); - ctrl->saved_piu.intbconfig = readq(piu_ior0_base + INTBCONFIG); - ctrl->saved_piu.intcconfig = readq(piu_ior0_base + INTCCONFIG); - ctrl->saved_piu.intdconfig = readq(piu_ior0_base + INTDCONFIG); - ctrl->saved_piu.pmeintconfig = readq(piu_ior0_base + PMEINTCONFIG); - ctrl->saved_piu.aererrintconfig = readq(piu_ior0_base + AERERRINTCONFIG); - ctrl->saved_piu.hpintconfig = readq(piu_ior0_base + HPINTCONFIG); + struct pci_dev *pdev = ctrl->pci_dev; - ctrl->saved_piu.state_saved = true; - } + save_rc_piu(pdev); } void sunway_pciehp_start(struct hotplug_slot *hotplug_slot) @@ -358,55 +325,11 @@ void sunway_pciehp_start(struct hotplug_slot *hotplug_slot) } } -static void pciehp_restore_config_space_range(struct pci_dev *pdev, - int start, int end) -{ - int index; - - for (index = end; index >= start; index--) { - pci_write_config_dword(pdev, 4 * index, - pdev->saved_config_space[index]); - } -} - -void sunway_pciehp_restore_config_space(struct controller *ctrl) -{ - struct pci_dev *pdev = ctrl->pci_dev; - - if (pdev->state_saved) { - pciehp_restore_config_space_range(pdev, 12, 15); - pciehp_restore_config_space_range(pdev, 9, 11); - pciehp_restore_config_space_range(pdev, 0, 8); - pdev->state_saved = false; - } -} - void sunway_pciehp_restore_rc_piu(struct controller *ctrl) { - struct pci_bus *bus = ctrl->pci_dev->bus; - struct pci_controller *hose = pci_bus_to_pci_controller(bus); - void __iomem *piu_ior0_base; - - piu_ior0_base = hose->piu_ior0_base; - - sunway_pciehp_restore_config_space(ctrl); - - if (ctrl->saved_piu.state_saved) { - writeq(ctrl->saved_piu.epdmabar, (piu_ior0_base + EPDMABAR)); - writeq(ctrl->saved_piu.msiaddr, (piu_ior0_base + MSIADDR)); - writeq(ctrl->saved_piu.iommuexcpt_ctrl, (piu_ior0_base + IOMMUEXCPT_CTRL)); - writeq(ctrl->saved_piu.dtbaseaddr, (piu_ior0_base + DTBASEADDR)); - - writeq(ctrl->saved_piu.intaconfig, (piu_ior0_base + INTACONFIG)); - writeq(ctrl->saved_piu.intbconfig, (piu_ior0_base + INTBCONFIG)); - writeq(ctrl->saved_piu.intcconfig, (piu_ior0_base + INTCCONFIG)); - writeq(ctrl->saved_piu.intdconfig, (piu_ior0_base + INTDCONFIG)); - writeq(ctrl->saved_piu.pmeintconfig, (piu_ior0_base + PMEINTCONFIG)); - writeq(ctrl->saved_piu.aererrintconfig, (piu_ior0_base + AERERRINTCONFIG)); - writeq(ctrl->saved_piu.hpintconfig, (piu_ior0_base + HPINTCONFIG)); + struct pci_dev *pdev = ctrl->pci_dev; - ctrl->saved_piu.state_saved = false; - } + restore_rc_piu(pdev); } void sunway_pciehp_end(struct controller *ctrl, bool insert) -- Gitee From 52def5df4e75843c9f4b75328c8120f311c0d91b Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 7 Feb 2025 09:46:15 +0000 Subject: [PATCH 415/524] sw64: iommu: implement resv_region APIs Implement sw64 resv_region API to avoid DMA access on unexpected memory regions. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 35 +++++++++++++++++++-- drivers/iommu/sw64/iommu_v2.c | 59 +++++++++++++++++++++++++++++------ 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index e4fbc0b9fa11..10b83f07f96f 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -33,6 +33,7 @@ #include #include "iommu.h" +#include "../dma-iommu.h" #define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) @@ -71,7 +72,6 @@ LIST_HEAD(sunway_domain_list); struct dma_domain { struct sunway_iommu_domain sdomain; - struct iova_domain iovad; }; const struct iommu_ops sunway_iommu_ops; @@ -243,7 +243,6 @@ static void dma_domain_free(struct dma_domain *dma_dom) return; del_domain_from_list(&dma_dom->sdomain); - put_iova_domain(&dma_dom->iovad); free_pagetable(&dma_dom->sdomain); if (dma_dom->sdomain.id) domain_id_free(dma_dom->sdomain.id); @@ -1018,6 +1017,9 @@ static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type) } sdomain = &dma_dom->sdomain; + sdomain->domain.geometry.aperture_start = 0ULL; + sdomain->domain.geometry.aperture_end = DMA_BIT_MASK(32); + sdomain->domain.geometry.force_aperture = true; break; case IOMMU_DOMAIN_IDENTITY: @@ -1313,6 +1315,34 @@ static void sunway_iommu_probe_finalize(struct device *dev) set_dma_ops(dev, get_arch_dma_ops()); } +static void sunway_iommu_get_resv_regions(struct device *dev, + struct list_head *head) +{ + struct iommu_resv_region *region; + + region = iommu_alloc_resv_region(SW64_32BIT_DMA_LIMIT, + (DMA_BIT_MASK(32) - SW64_32BIT_DMA_LIMIT), + IOMMU_NOEXEC | IOMMU_MMIO, + IOMMU_RESV_RESERVED, GFP_KERNEL); + if (!region) + return; + + list_add_tail(®ion->list, head); + + if (dev_is_pci(dev)) { + struct pci_dev *pdev = to_pci_dev(dev); + + if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) { + region = iommu_alloc_resv_region(0, 1UL << 24, + IOMMU_READ | IOMMU_WRITE, + IOMMU_RESV_DIRECT_RELAXABLE, + GFP_KERNEL); + if (region) + list_add_tail(®ion->list, head); + } + } +} + const struct iommu_ops sunway_iommu_ops = { .capable = sunway_iommu_capable, .domain_alloc = sunway_iommu_domain_alloc, @@ -1320,6 +1350,7 @@ const struct iommu_ops sunway_iommu_ops = { .probe_finalize = sunway_iommu_probe_finalize, .release_device = sunway_iommu_release_device, .device_group = sunway_iommu_device_group, + .get_resv_regions = sunway_iommu_get_resv_regions, .pgsize_bitmap = SW64_IOMMU_PGSIZES, .def_domain_type = sunway_iommu_def_domain_type, .default_domain_ops = &(const struct iommu_domain_ops) { diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index 33f25fa5b2c5..3e51963714bc 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -30,6 +30,7 @@ #include #include "iommu.h" +#include "../dma-iommu.h" #define SW64_64BIT_DMA_LIMIT ((1UL << 42) - 1) #define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) @@ -84,8 +85,8 @@ LIST_HEAD(sunway_domain_list); struct dma_domain { struct sunway_iommu_domain sdomain; - struct iova_domain iovad; }; + const struct iommu_ops sunway_iommu_ops; static const struct dma_map_ops sunway_dma_ops; @@ -265,7 +266,6 @@ static void dma_domain_free(struct dma_domain *dma_dom) return; del_domain_from_list(&dma_dom->sdomain); - put_iova_domain(&dma_dom->iovad); free_pagetable(&dma_dom->sdomain); if (dma_dom->sdomain.id) domain_id_free(dma_dom->sdomain.id); @@ -1236,7 +1236,7 @@ static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type) } sdomain->domain.geometry.aperture_start = 0UL; - sdomain->domain.geometry.aperture_end = ~0ULL; + sdomain->domain.geometry.aperture_end = SW64_64BIT_DMA_LIMIT; sdomain->domain.geometry.force_aperture = true; sdomain->type = IOMMU_DOMAIN_UNMANAGED; break; @@ -1249,6 +1249,9 @@ static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type) } sdomain = &dma_dom->sdomain; + sdomain->domain.geometry.aperture_start = 0UL; + sdomain->domain.geometry.aperture_end = SW64_64BIT_DMA_LIMIT; + sdomain->domain.geometry.force_aperture = true; break; case IOMMU_DOMAIN_IDENTITY: @@ -1411,6 +1414,16 @@ sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); int ret; + /* + * As VFIO cannot distinguish between normal DMA request + * and pci device BAR, check should be introduced manually + * to avoid VFIO trying to map pci config space. + */ + if (iova > IO_BASE) { + pr_err("iova %#lx is out of memory!\n", iova); + return -ENOMEM; + } + if (iova >= SW64_BAR_ADDRESS) return 0; @@ -1567,15 +1580,42 @@ static void sunway_iommu_probe_finalize(struct device *dev) struct iommu_domain *domain; domain = iommu_get_domain_for_dev(dev); - if (domain->type & __IOMMU_DOMAIN_DMA_API) { - if (min(dev->coherent_dma_mask, *dev->dma_mask) == DMA_BIT_MASK(32)) - iommu_setup_dma_ops(dev, SW64_DMA_START, SW64_32BIT_DMA_LIMIT); - else - iommu_setup_dma_ops(dev, SW64_DMA_START, SW64_64BIT_DMA_LIMIT); - } else + if (domain->type & __IOMMU_DOMAIN_DMA_API) + iommu_setup_dma_ops(dev, SW64_DMA_START, + SW64_64BIT_DMA_LIMIT - SW64_DMA_START); + else set_dma_ops(dev, get_arch_dma_ops()); } +static void sunway_iommu_get_resv_regions(struct device *dev, + struct list_head *head) +{ + struct iommu_resv_region *region; + + /* Reserve 3.5~4G for device */ + region = iommu_alloc_resv_region(SW64_32BIT_DMA_LIMIT, + (DMA_BIT_MASK(32) - SW64_32BIT_DMA_LIMIT), + IOMMU_NOEXEC | IOMMU_MMIO, + IOMMU_RESV_RESERVED, GFP_KERNEL); + if (!region) + return; + + list_add_tail(®ion->list, head); + + if (dev_is_pci(dev)) { + struct pci_dev *pdev = to_pci_dev(dev); + + if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) { + region = iommu_alloc_resv_region(0, 1UL << 24, + IOMMU_READ | IOMMU_WRITE, + IOMMU_RESV_DIRECT_RELAXABLE, + GFP_KERNEL); + if (region) + list_add_tail(®ion->list, head); + } + } +} + const struct iommu_ops sunway_iommu_ops = { .capable = sunway_iommu_capable, .domain_alloc = sunway_iommu_domain_alloc, @@ -1584,6 +1624,7 @@ const struct iommu_ops sunway_iommu_ops = { .release_device = sunway_iommu_release_device, .device_group = sunway_iommu_device_group, .pgsize_bitmap = SZ_8K | SZ_8M | SZ_512M | SZ_8G, + .get_resv_regions = sunway_iommu_get_resv_regions, .def_domain_type = sunway_iommu_def_domain_type, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = sunway_iommu_attach_device, -- Gitee From 1ba00e1d00eba4760c71ce74a1ce63e0c5fb6c7c Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 25 Feb 2025 18:04:59 +0800 Subject: [PATCH 416/524] sw64: topology: discard topology information from PPTT Using the architecture implemented store_cpu_topology() to initialize topology information, which provides more readable topology IDs. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/topology.c | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/arch/sw_64/kernel/topology.c b/arch/sw_64/kernel/topology.c index 2758b563b119..76b80593a9d1 100644 --- a/arch/sw_64/kernel/topology.c +++ b/arch/sw_64/kernel/topology.c @@ -299,41 +299,6 @@ static int __init parse_dt_topology(void) return ret; } -#ifdef CONFIG_ACPI -/* - * Propagate the topology information of the processor_topology_node tree to the - * cpu_topology array. - */ -int __init parse_acpi_topology(void) -{ - int cpu, topology_id; - - if (acpi_disabled) - return 0; - - for_each_possible_cpu(cpu) { - topology_id = find_acpi_cpu_topology(cpu, 0); - if (topology_id < 0) - return topology_id; - - if (acpi_pptt_cpu_is_thread(cpu) == 1) { - cpu_topology[cpu].thread_id = topology_id; - topology_id = find_acpi_cpu_topology(cpu, 1); - cpu_topology[cpu].core_id = topology_id; - } else { - cpu_topology[cpu].thread_id = -1; - cpu_topology[cpu].core_id = topology_id; - } - topology_id = find_acpi_cpu_topology_cluster(cpu); - cpu_topology[cpu].cluster_id = topology_id; - topology_id = find_acpi_cpu_topology_package(cpu); - cpu_topology[cpu].package_id = topology_id; - } - - return 0; -} -#endif - void __init init_cpu_topology(void) { struct cpu_topology *boot_cpu_topo = &cpu_topology[0]; -- Gitee From 1e8f662a2e4cd9d78647f1010d932d8a2feff58a Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 14 Feb 2025 08:36:39 +0000 Subject: [PATCH 417/524] sw64: add a 1ms delay before CPU offline In-flight irqs will cause CPU hang up if CPU fails to handle them before its actual shutdown. Add a 1ms delay after irq migration to wait for these irqs get handled. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/irq.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c index 82854ba75a94..c650e3a4faf0 100644 --- a/arch/sw_64/kernel/irq.c +++ b/arch/sw_64/kernel/irq.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -106,6 +107,8 @@ handle_irq(int irq) void fixup_irqs(void) { irq_migrate_all_off_this_cpu(); + + mdelay(1); } #endif -- Gitee From b4fc3221321886c6c34123f57a6680603f2a2781 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Wed, 5 Mar 2025 10:08:07 +0800 Subject: [PATCH 418/524] sw64: fix nmi_enter error in CORE3B In commit fbdc856644d6 ("sw64: add NMI support"), since CORE3B does not have regs->cause, if some random case happens to take the value of -2, then it enters the wrong judgment. Fixes: fbdc856644d6 ("sw64: add NMI support") Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-cpu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index e381c19b9cae..edab9ddd9982 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -85,10 +85,10 @@ asmlinkage void noinstr do_entInt(unsigned long type, unsigned long vector, /* restart idle routine if it is interrupted */ if (regs->pc > (u64)__idle_start && regs->pc < (u64)__idle_end) regs->pc = (u64)__idle_start; - if (regs->cause != -2) - irq_enter(); - else + if (unlikely(regs->cause == -2 && IS_ENABLED(CONFIG_SUBARCH_C4))) nmi_enter(); + else + irq_enter(); old_regs = set_irq_regs(regs); #ifdef CONFIG_PM -- Gitee From 9874e19a95e06db85411500d9732f2d22f180a9c Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Wed, 5 Mar 2025 14:45:52 +0800 Subject: [PATCH 419/524] sw64: fix copy_oldmem_page according to upstream The upstream has modified the implementation of copy_oldmem_page, which has led to the failure of generating /proc/vmcore in the second kernel of kdump. So, fix it. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/crash_dump.c | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/arch/sw_64/kernel/crash_dump.c b/arch/sw_64/kernel/crash_dump.c index 4484673823b8..7335abdf3bb5 100644 --- a/arch/sw_64/kernel/crash_dump.c +++ b/arch/sw_64/kernel/crash_dump.c @@ -14,24 +14,11 @@ * published by the Free Software Foundation. */ -#include #include +#include -/** - * copy_oldmem_page() - copy one page from old kernel memory - * @pfn: page frame number to be copied - * @buf: buffer where the copied page is placed - * @csize: number of bytes to copy - * @offset: offset in bytes into the page - * @userbuf: if set, @buf is int he user address space - * - * This function copies one page from old kernel memory into buffer pointed by - * @buf. If @buf is in userspace, set @userbuf to %1. Returns number of bytes - * copied or negative error in case of failure. - */ -ssize_t copy_oldmem_page(unsigned long pfn, char *buf, - size_t csize, unsigned long offset, - int userbuf) +ssize_t copy_oldmem_page(struct iov_iter *iter, unsigned long pfn, + size_t csize, unsigned long offset) { void *vaddr; @@ -42,14 +29,7 @@ ssize_t copy_oldmem_page(unsigned long pfn, char *buf, if (!vaddr) return -ENOMEM; - if (userbuf) { - if (copy_to_user(buf, vaddr + offset, csize)) { - iounmap(vaddr); - return -EFAULT; - } - } else { - memcpy(buf, vaddr + offset, csize); - } + csize = copy_to_iter(vaddr + offset, csize, iter); iounmap(vaddr); return csize; -- Gitee From 28491c668d474e794be8bf1ff7161966d4ab6f65 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 28 Feb 2025 11:00:47 +0800 Subject: [PATCH 420/524] sw64: ftrace: fix atomicity bugs in ftrace When running a stress test such as LTP:ftrace-stress-test, the following atomicity bugs may occur: 1. When running the ftrace instruction sequence, ftrace_make_nop() has just modified the LDL but not yet modified the CALL, causing the executor to use the an incorrect $28 for the CALL. * This bug is fixed by reversing the nop modification order. 2. When an interrupt occurs while running the nop sequence, ftrace_make_call() modifies the last instruction to CALL, causing the executor to CALL with an unrelated $28 after returning from the interrupt. * This bug is fixed by moving the ftrace instruction sequence forward to form an instruction set that cannot be interrupted. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/ftrace.h | 6 +++++- arch/sw_64/kernel/ftrace.c | 23 ++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/arch/sw_64/include/asm/ftrace.h b/arch/sw_64/include/asm/ftrace.h index 3478cd4ad850..871c0b7b0aef 100644 --- a/arch/sw_64/include/asm/ftrace.h +++ b/arch/sw_64/include/asm/ftrace.h @@ -12,7 +12,11 @@ #define _ASM_SW64_FTRACE_H #define MCOUNT_ADDR ((unsigned long)_mcount) -#define MCOUNT_INSN_SIZE 20 /* 5 * SW64_INSN_SIZE */ +#ifdef CONFIG_DYNAMIC_FTRACE +#define MCOUNT_INSN_SIZE 16 /* 4 * SW64_INSN_SIZE */ +#else +#define MCOUNT_INSN_SIZE 20 /* 5 * SW64_INSN_SIZE */ +#endif #define MCOUNT_LDGP_SIZE 8 /* 2 * SW64_INSN_SIZE */ #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS diff --git a/arch/sw_64/kernel/ftrace.c b/arch/sw_64/kernel/ftrace.c index 6a8446e59961..511760ac18c2 100644 --- a/arch/sw_64/kernel/ftrace.c +++ b/arch/sw_64/kernel/ftrace.c @@ -72,13 +72,17 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) else offset = TI_FTRACE_REGS_ADDR; - insn[0] = SW64_NOP; /* ldl r28,(ftrace_addr_offset)(r8) */ - insn[1] = (0x23U << 26) | (28U << 21) | (8U << 16) | offset; - insn[2] = SW64_CALL(R28, R28, 0); + insn[0] = (0x23U << 26) | (28U << 21) | (8U << 16) | offset; + insn[1] = SW64_CALL(R28, R28, 0); + insn[2] = SW64_NOP; + + *((u32 *)pc) = insn[0]; + mb(); + *((u32 *)(pc + 4)) = insn[1]; + *((u32 *)(pc + 8)) = insn[2]; - /* replace the 3 mcount instructions at once */ - return copy_to_kernel_nofault((void *)pc, insn, 3 * SW64_INSN_SIZE); + return 0; } /* @@ -90,7 +94,12 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long pc = rec->ip + MCOUNT_LDGP_SIZE; unsigned int insn[3] = {SW64_NOP, SW64_NOP, SW64_NOP}; - return copy_to_kernel_nofault((void *)pc, insn, 3 * SW64_INSN_SIZE); + *((u32 *)(pc + 8)) = insn[2]; + *((u32 *)(pc + 4)) = insn[1]; + mb(); + *((u32 *)pc) = insn[0]; + + return 0; } void arch_ftrace_update_code(int command) @@ -116,7 +125,7 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { unsigned int insn[1]; - unsigned long pc = rec->ip + MCOUNT_LDGP_SIZE + 4; + unsigned long pc = rec->ip + MCOUNT_LDGP_SIZE; unsigned long offset; if (addr == FTRACE_ADDR) -- Gitee From 9b616b69a4f9efc88c505a32d8a4c7b8dceeca0f Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Thu, 6 Mar 2025 17:41:45 +0800 Subject: [PATCH 421/524] sw64: fix nmi_exit error in CORE3B The cause of the error is the same as commit f4325aa64b69 ("sw64: fix nmi_enter error in CORE3B"). Fixes: f4325aa64b69 ("sw64: fix nmi_enter error in CORE3B") Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-cpu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index edab9ddd9982..3e8c3f54712a 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -174,10 +174,10 @@ asmlinkage void noinstr do_entInt(unsigned long type, unsigned long vector, out: set_irq_regs(old_regs); - if (regs->cause != -2) - irq_exit(); - else + if (unlikely(regs->cause == -2 && IS_ENABLED(CONFIG_SUBARCH_C4))) nmi_exit(); + else + irq_exit(); } EXPORT_SYMBOL(do_entInt); -- Gitee From 923b7d399710ed39bef627bbe0cc401600338d65 Mon Sep 17 00:00:00 2001 From: Xu Linqin Date: Wed, 26 Feb 2025 09:20:40 +0800 Subject: [PATCH 422/524] =?UTF-8?q?sw64/kvm:=20fix=C2=A0a=C2=A0bug=C2=A0fo?= =?UTF-8?q?r=C2=A0starting=C2=A0CORE3=C2=A0VM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In QEMU, cpu reset occurs before memory initialized.After adding 'cpu_reset' action in funciton 'sw64_cpu_realizefn' in QEMU,which make code more standardized,this will cause the value of 'addr' is 0 in 'kvm_sw64_vcpu_reset',leading to the host's kernel being overwritten after memset action. Considering the factors of product version maintance, the bug is fixed simply by checking 'addr' not equal 0.In the future, the 'memset' operation can be removed from 'kvm_sw64_vcpu_reset', and the relevant memory can be initialized by QEMU. Signed-off-by: Xu Linqin Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/kvm_core3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c index 8b751c76f121..142602c67831 100644 --- a/arch/sw_64/kvm/kvm_core3.c +++ b/arch/sw_64/kvm/kvm_core3.c @@ -234,7 +234,7 @@ int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) vcpu->arch.power_off = 0; memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); - if (vcpu->vcpu_id == 0) + if (vcpu->vcpu_id == 0 && addr) memset(__va(addr), 0, 0x2000000); return 0; -- Gitee From 7ab50a717c366085dc30ebac3765ebb599fb02a0 Mon Sep 17 00:00:00 2001 From: Wang Yuanheng Date: Mon, 10 Mar 2025 10:27:15 +0800 Subject: [PATCH 423/524] sw64/vdso: Support mremap() for vDSO vDSO VMA address is saved in mm_context for the purpose of using restorer from vDSO page to return to userspace after signal handling. In Checkpoint Restore in Userspace (CRIU) project we place vDSO VMA on restore back to the place where it was on the dump. With the exception for x86 (where there is API to map vDSO with arch_prctl()), we move vDSO inherited from CRIU task to restoree position by mremap(). Make vDSO code track the VMA address by supplying .mremap() fops the same way it's done for arm64 by: commit 739586951b8a ("arm64/vdso: Support mremap() for vDSO"). Signed-off-by: Wang Yuanheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/vdso.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/sw_64/kernel/vdso.c b/arch/sw_64/kernel/vdso.c index 375f9c60a49c..42af417dd70e 100644 --- a/arch/sw_64/kernel/vdso.c +++ b/arch/sw_64/kernel/vdso.c @@ -71,6 +71,19 @@ static void init_cpu_map(void) } #endif +static int vdso_mremap(const struct vm_special_mapping *sm, + struct vm_area_struct *new_vma) +{ + unsigned long new_size = new_vma->vm_end - new_vma->vm_start; + unsigned long vdso_size = vdso_end - vdso_start; + + if (vdso_size != new_size) + return -EINVAL; + + current->mm->context.vdso = (void *)new_vma->vm_start; + return 0; +} + static int __init vdso_init(void) { int i; @@ -106,6 +119,7 @@ static int __init vdso_init(void) vdso_spec[1] = (struct vm_special_mapping) { .name = "[vdso]", .pages = &vdso_pagelist[1], + .mremap = vdso_mremap, }; init_cpu_map(); -- Gitee From 7e323dc4e589d90cb29aaf58e24d689d95845c8b Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 4 Mar 2025 16:14:35 +0800 Subject: [PATCH 424/524] sw64: kvm: modify parameters of the exception functions These parameters for handling exception functions already stored in earg0, earg1, and earg2 of struct pt_regs, so KVM needs an adaptive modification of it. And we can see that in __sw64_vcpu_run, after we call do_entInt, the function push stack will cause pt_regs to be modified, so we should pop stack after we ret from do_entInt. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/entry_core3.S | 8 +++++++- arch/sw_64/kvm/entry_core4.S | 10 ++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kvm/entry_core3.S b/arch/sw_64/kvm/entry_core3.S index a61ecc387d26..d2e5373ee632 100644 --- a/arch/sw_64/kvm/entry_core3.S +++ b/arch/sw_64/kvm/entry_core3.S @@ -253,8 +253,14 @@ $setfpec_over: ldl $18, HOST_INT_R18(sp) ldi sp, (HOST_INT_SIZE + PT_REGS_SIZE)(sp) - ldi $19, -PT_REGS_SIZE(sp) + ldi sp, -PT_REGS_SIZE(sp) + stl $16, PT_REGS_EARG0(sp) + stl $17, PT_REGS_EARG1(sp) + stl $18, PT_REGS_EARG2(sp) + ldi $16, 0(sp) + call $26, do_entInt + ldi sp, PT_REGS_SIZE(sp) ldl $26, VCPU_RET_RA(sp) ldl $0, VCPU_RET_R0(sp) $ret_to: diff --git a/arch/sw_64/kvm/entry_core4.S b/arch/sw_64/kvm/entry_core4.S index 48325cc7c2c5..1556d637ff8b 100644 --- a/arch/sw_64/kvm/entry_core4.S +++ b/arch/sw_64/kvm/entry_core4.S @@ -263,10 +263,16 @@ $setfpec_over: csrr $17, CSR_EARG1 csrr $18, CSR_EARG2 - ldi $19, -PT_REGS_SIZE(sp) + ldi sp, -PT_REGS_SIZE(sp) + stl $16, PT_REGS_EARG0(sp) + stl $17, PT_REGS_EARG1(sp) + stl $18, PT_REGS_EARG2(sp) + ldi $16, 0(sp) + csrr $1, CSR_CAUSE - stl $1, PT_REGS_CAUSE($19) + stl $1, PT_REGS_CAUSE($16) call $26, do_entInt + ldi sp, PT_REGS_SIZE(sp) ldl $26, VCPU_RET_RA(sp) ldl $0, VCPU_RET_R0(sp) $ret_to: -- Gitee From 728b2fe9bbe3e65688f604e2f61af45b3a5212dd Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Thu, 2 Jan 2025 10:11:26 +0800 Subject: [PATCH 425/524] sw64: modify parameters of the exception functions These parameters for handing exception functions already stored in earg0, earg1, and earg2 of struct pt_regs and don't need to be arranged in $16, $17, $18 in entry_c3/4.S. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry_c3.S | 13 ++++++++----- arch/sw_64/kernel/entry_c4.S | 12 ++++++------ arch/sw_64/kernel/traps.c | 24 +++++++++++++----------- arch/sw_64/mm/fault.c | 7 ++++--- drivers/irqchip/irq-sunway-cpu.c | 6 ++++-- 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/arch/sw_64/kernel/entry_c3.S b/arch/sw_64/kernel/entry_c3.S index fe1e9a8635a2..c8823bfd664e 100644 --- a/arch/sw_64/kernel/entry_c3.S +++ b/arch/sw_64/kernel/entry_c3.S @@ -61,6 +61,9 @@ stl $4, PT_REGS_GP($sp) stl $5, PT_REGS_PC($sp) stl $6, PT_REGS_PS($sp) + stl $16, PT_REGS_EARG0($sp) + stl $17, PT_REGS_EARG1($sp) + stl $18, PT_REGS_EARG2($sp) and $6, 0x8, $7 beq $7, 1f sys_call HMC_rdusp @@ -136,7 +139,7 @@ .ent entInt entInt: SAVE_ALL - ldi $19, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) call $26, do_entInt br ret_from_sys_call .end entInt @@ -146,7 +149,7 @@ entInt: .ent entArith entArith: SAVE_ALL - ldi $18, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) call $26, do_entArith br ret_from_sys_call .end entArith @@ -156,7 +159,7 @@ entArith: .ent entMM entMM: SAVE_ALL - ldi $19, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) call $26, do_page_fault br ret_from_sys_call .end entMM @@ -166,7 +169,7 @@ entMM: .ent entIF entIF: SAVE_ALL - ldi $18, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) call $26, do_entIF br ret_from_sys_call .end entIF @@ -182,7 +185,7 @@ entIF: .ent entUna entUna: SAVE_ALL - ldi $19, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) ldl $0, (PT_REGS_PS + STACKFRAME_SIZE)($sp) and $0, 8, $0 /* user mode ? */ beq $0, 1f diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S index 3a8912fb6287..5473b988e2e2 100644 --- a/arch/sw_64/kernel/entry_c4.S +++ b/arch/sw_64/kernel/entry_c4.S @@ -271,7 +271,7 @@ entNMI: SAVE_ALL_NMI br $27, 1f 1: ldgp $29, 0($27) - ldi $19, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) call $26, do_entInt br ret_from_nmi .end entNMI @@ -283,7 +283,7 @@ entInt: SAVE_ALL br $27, 1f 1: ldgp $29, 0($27) - ldi $19, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) call $26, do_entInt br ret_from_sys_call .end entInt @@ -299,7 +299,7 @@ entArith: br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ - ldi $18, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) call $26, do_entArith br ret_from_sys_call .end entArith @@ -312,7 +312,7 @@ entMM: br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ - ldi $19, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) call $26, do_page_fault br ret_from_sys_call .end entMM @@ -328,7 +328,7 @@ entIF: br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ - ldi $18, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) call $26, do_entIF br ret_from_sys_call .end entIF @@ -350,7 +350,7 @@ entUna: br $27, 1f 1: ldgp $29, 0($27) RESTORE_IRQ - ldi $19, STACKFRAME_SIZE($sp) + ldi $16, STACKFRAME_SIZE($sp) ldl $0, (PT_REGS_PS + STACKFRAME_SIZE)($sp) and $0, 8, $0 /* user mode ? */ beq $0, 1f diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index cb7b3141b73a..1bce82e95f5f 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -147,11 +147,10 @@ extern long sw64_fp_emul_imprecise(struct pt_regs *regs, unsigned long writemask extern long sw64_fp_emul(unsigned long pc); #endif -asmlinkage void -noinstr do_entArith(unsigned long exc_sum, unsigned long write_mask, - struct pt_regs *regs) +asmlinkage void noinstr do_entArith(struct pt_regs *regs) { long si_code = FPE_FLTINV; + unsigned long exc_sum = regs->earg0; #ifndef CONFIG_SUBARCH_C3B /* integer divide by zero */ @@ -249,11 +248,12 @@ static int try_fix_rd_f(unsigned int inst, struct pt_regs *regs) * BPT/GENTRAP/OPDEC make regs->pc = exc_pc + 4. debugger should * do something necessary to handle it correctly. */ -asmlinkage void -noinstr do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) +asmlinkage void noinstr do_entIF(struct pt_regs *regs) { int signo, code; unsigned int inst, type; + unsigned long inst_type = regs->earg0; + unsigned long va = regs->earg1; type = inst_type & 0xffffffff; inst = inst_type >> 32; @@ -396,9 +396,7 @@ noinstr do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs 1L << 0x2 | 1L << 0x8 | /* ldw_a stw_a */ \ 1L << 0x3 | 1L << 0x9) /* ldl_a stl_a */ -asmlinkage void -noinstr do_entUna(void *va, unsigned long opcode, unsigned long reg, - struct pt_regs *regs) +asmlinkage void noinstr do_entUna(struct pt_regs *regs) { long error, disp; unsigned int insn, fncode, rb; @@ -406,6 +404,9 @@ noinstr do_entUna(void *va, unsigned long opcode, unsigned long reg, unsigned long fp[4]; unsigned long fake_reg, *reg_addr = &fake_reg; unsigned long pc = regs->pc - 4; + void *va = (void *)regs->earg0; + unsigned long opcode = regs->earg1; + unsigned long reg = regs->earg2; insn = *(unsigned int *)pc; fncode = (insn >> 12) & 0xf; @@ -1347,9 +1348,7 @@ noinstr do_entUna(void *va, unsigned long opcode, unsigned long reg, * uses them as temporary storage for integer memory to memory copies. * However, we need to deal with stt/ldt and sts/lds only. */ -asmlinkage void -noinstr do_entUnaUser(void __user *va, unsigned long opcode, - unsigned long reg, struct pt_regs *regs) +asmlinkage void noinstr do_entUnaUser(struct pt_regs *regs) { #ifdef CONFIG_UNA_PRINT static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5); @@ -1364,6 +1363,9 @@ noinstr do_entUnaUser(void __user *va, unsigned long opcode, unsigned long instr, instr_op, value, fncode; unsigned int rb = -1U; long disp; + void __user *va = (void *)regs->earg0; + unsigned long opcode = regs->earg1; + unsigned long reg = regs->earg2; #ifdef CONFIG_DEBUG_FS /* diff --git a/arch/sw_64/mm/fault.c b/arch/sw_64/mm/fault.c index a43e38cc21b8..7b58aafece1f 100644 --- a/arch/sw_64/mm/fault.c +++ b/arch/sw_64/mm/fault.c @@ -132,15 +132,16 @@ unsigned long show_va_to_pa(struct mm_struct *mm, unsigned long addr) extern int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_regs *regs); -asmlinkage void notrace -noinstr do_page_fault(unsigned long address, unsigned long mmcsr, - long cause, struct pt_regs *regs) +asmlinkage void notrace noinstr do_page_fault(struct pt_regs *regs) { struct vm_area_struct *vma; struct mm_struct *mm = current->mm; int si_code = SEGV_MAPERR; vm_fault_t fault; unsigned int flags = FAULT_FLAG_DEFAULT; + unsigned long address = regs->earg0; + unsigned long mmcsr = regs->earg1; + long cause = regs->earg2; if (notify_page_fault(regs, mmcsr)) return; diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 3e8c3f54712a..58579c1044e0 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -76,11 +76,13 @@ static void handle_nmi_int(void) int pme_state; -asmlinkage void noinstr do_entInt(unsigned long type, unsigned long vector, - unsigned long irq_arg, struct pt_regs *regs) +asmlinkage void noinstr do_entInt(struct pt_regs *regs) { struct pt_regs *old_regs; extern char __idle_start[], __idle_end[]; + unsigned long type = regs->earg0; + unsigned long vector = regs->earg1; + unsigned long irq_arg = regs->earg2; /* restart idle routine if it is interrupted */ if (regs->pc > (u64)__idle_start && regs->pc < (u64)__idle_end) -- Gitee From 51142a78e52267c29e509bb6ff063b0caf906c20 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 26 Nov 2024 11:06:55 +0800 Subject: [PATCH 426/524] sw64: rename PT_REGS_SP to PT_REGS_R30 Avoid redefining "PT_REGS_SP" which is already defined in tools/lib/bpf/bpf_tracing.h. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/asm-offsets.c | 2 +- arch/sw_64/kernel/entry-ftrace.S | 2 +- arch/sw_64/kernel/entry_c3.S | 4 ++-- arch/sw_64/kernel/entry_c4.S | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c index c4e4e40a03d5..1a68486da270 100644 --- a/arch/sw_64/kernel/asm-offsets.c +++ b/arch/sw_64/kernel/asm-offsets.c @@ -96,7 +96,7 @@ void foo(void) DEFINE(PT_REGS_R27, offsetof(struct pt_regs, regs[27])); DEFINE(PT_REGS_R28, offsetof(struct pt_regs, regs[28])); DEFINE(PT_REGS_GP, offsetof(struct pt_regs, regs[29])); - DEFINE(PT_REGS_SP, offsetof(struct pt_regs, regs[30])); + DEFINE(PT_REGS_R30, offsetof(struct pt_regs, regs[30])); DEFINE(PT_REGS_PC, offsetof(struct pt_regs, pc)); DEFINE(PT_REGS_PS, offsetof(struct pt_regs, ps)); DEFINE(PT_REGS_ORIG_R0, offsetof(struct pt_regs, orig_r0)); diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S index 6878b773968d..01e7cb4f9d22 100644 --- a/arch/sw_64/kernel/entry-ftrace.S +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -91,7 +91,7 @@ stl $28, PT_REGS_R28($sp) stl $29, PT_REGS_GP($sp) ldi $0, PT_REGS_SIZE($sp) - stl $0, PT_REGS_SP($sp) + stl $0, PT_REGS_R30($sp) .endm .macro RESTORE_PT_REGS diff --git a/arch/sw_64/kernel/entry_c3.S b/arch/sw_64/kernel/entry_c3.S index c8823bfd664e..dd5603469634 100644 --- a/arch/sw_64/kernel/entry_c3.S +++ b/arch/sw_64/kernel/entry_c3.S @@ -69,7 +69,7 @@ sys_call HMC_rdusp br 2f 1: ldi $0, PT_REGS_SIZE($sp) -2: stl $0, PT_REGS_SP($sp) +2: stl $0, PT_REGS_R30($sp) ldi $1, NO_SYSCALL stl $1, PT_REGS_ORIG_R0($sp) sys_call HMC_rdktp @@ -84,7 +84,7 @@ .macro RESTORE_ALL ldi $sp, STACKFRAME_SIZE($sp) - ldl $16, PT_REGS_SP($sp) + ldl $16, PT_REGS_R30($sp) /* skip wrusp if returning to kernel */ blt $16, 1f sys_call HMC_wrusp diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S index 5473b988e2e2..f6b620c36756 100644 --- a/arch/sw_64/kernel/entry_c4.S +++ b/arch/sw_64/kernel/entry_c4.S @@ -67,7 +67,7 @@ stl $17, PT_REGS_EARG1($sp) stl $18, PT_REGS_EARG2($sp) - stl $1, PT_REGS_SP($sp) + stl $1, PT_REGS_R30($sp) stl $2, PT_REGS_PS($sp) stl $3, PT_REGS_PC($sp) stl $4, PT_REGS_CAUSE($sp) @@ -128,7 +128,7 @@ ldl $27, PT_REGS_R27($sp) ldl $28, PT_REGS_R28($sp) ldl $29, PT_REGS_GP($sp) - ldl $sp, PT_REGS_SP($sp) + ldl $sp, PT_REGS_R30($sp) .endm .macro SAVE_ALL @@ -191,7 +191,7 @@ stl $17, PT_REGS_EARG1($sp) stl $18, PT_REGS_EARG2($sp) - stl $1, PT_REGS_SP($sp) + stl $1, PT_REGS_R30($sp) stl $2, PT_REGS_PS($sp) stl $3, PT_REGS_PC($sp) stl $4, PT_REGS_CAUSE($sp) @@ -251,7 +251,7 @@ ldl $27, PT_REGS_R27($sp) ldl $28, PT_REGS_R28($sp) ldl $29, PT_REGS_GP($sp) - ldl $sp, PT_REGS_SP($sp) + ldl $sp, PT_REGS_R30($sp) .endm .macro RESTORE_IRQ -- Gitee From 105d8d4ff76bed4ae9a57e7c8f2b3be04ef90a57 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Mon, 10 Mar 2025 09:52:12 +0800 Subject: [PATCH 427/524] sw64: vdso: add support for time namespaces Implement generic vdso time namespace support which also enables time namespaces for sw64. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/Makefile | 2 +- arch/sw_64/include/asm/page.h | 1 + arch/sw_64/include/asm/vdso.h | 4 +- arch/sw_64/include/asm/vdso/gettimeofday.h | 8 + arch/sw_64/kernel/vdso.c | 219 ++++++++++++++++----- arch/sw_64/kernel/vdso/Makefile | 4 +- arch/sw_64/kernel/vdso/note.S | 12 ++ arch/sw_64/kernel/vdso/vdso.lds.S | 5 +- 9 files changed, 199 insertions(+), 57 deletions(-) create mode 100644 arch/sw_64/kernel/vdso/note.S diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 9ce83cb0e9f8..3681e9550e60 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -74,6 +74,7 @@ config SW64 select GENERIC_STRNCPY_FROM_USER select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL + select GENERIC_VDSO_TIME_NS if HAVE_GENERIC_VDSO select GPIOLIB if ACPI select HANDLE_DOMAIN_IRQ select HARDIRQS_SW_RESEND diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index ccf2f5d6e378..9b25e3797f52 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -51,7 +51,7 @@ boot := arch/sw_64/boot PHONY += vdso_install vdso_install: - $(Q)$(MAKE) $(build)=arch/sw64/kernel/vdso $@ + $(Q)$(MAKE) $(build)=arch/sw_64/kernel/vdso $@ #Default target when executing make with no arguments all: $(boot)/vmlinux.bin.gz diff --git a/arch/sw_64/include/asm/page.h b/arch/sw_64/include/asm/page.h index 3a95151b67f8..1ba314b342a0 100644 --- a/arch/sw_64/include/asm/page.h +++ b/arch/sw_64/include/asm/page.h @@ -58,6 +58,7 @@ extern unsigned long __boot_phys_addr(unsigned long addr); #define virt_to_pfn(vaddr) (PHYS_PFN(__pa(vaddr))) #define pfn_to_virt(pfn) (__va(PFN_PHYS(pfn))) +#define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) #ifdef CONFIG_FLATMEM #define pfn_valid(pfn) ((pfn) < max_mapnr) diff --git a/arch/sw_64/include/asm/vdso.h b/arch/sw_64/include/asm/vdso.h index 3f6ba2d72b62..8e2dba1b6c27 100644 --- a/arch/sw_64/include/asm/vdso.h +++ b/arch/sw_64/include/asm/vdso.h @@ -25,6 +25,7 @@ * to prelink this. */ #define VDSO_LBASE 0x0 +#define __VVAR_PAGES 2 #ifndef __ASSEMBLY__ @@ -37,6 +38,7 @@ ((unsigned long)(base) + __vdso_##name); \ }) +extern char vdso_start[], vdso_end[]; struct vdso_data { u64 xtime_sec; @@ -74,7 +76,7 @@ static inline unsigned long get_vdso_base(void) static inline const struct vdso_data *get_vdso_data(void) { - return (const struct vdso_data *)(get_vdso_base() - PAGE_SIZE); + return (const struct vdso_data *)(get_vdso_base() - __VVAR_PAGES * PAGE_SIZE); } static inline u32 vdso_data_read_begin(const struct vdso_data *data) diff --git a/arch/sw_64/include/asm/vdso/gettimeofday.h b/arch/sw_64/include/asm/vdso/gettimeofday.h index 57eb602550c7..318b095ec54d 100644 --- a/arch/sw_64/include/asm/vdso/gettimeofday.h +++ b/arch/sw_64/include/asm/vdso/gettimeofday.h @@ -100,6 +100,14 @@ static __always_inline const struct vdso_data *__arch_get_vdso_data(void) return _vdso_data; } +#ifdef CONFIG_TIME_NS +static __always_inline +const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) +{ + return _timens_data; +} +#endif + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/sw_64/kernel/vdso.c b/arch/sw_64/kernel/vdso.c index 42af417dd70e..d40e1ea700f4 100644 --- a/arch/sw_64/kernel/vdso.c +++ b/arch/sw_64/kernel/vdso.c @@ -17,13 +17,24 @@ #include #include #include -#include - +#include +#include +#include +#include #include -extern char vdso_start[], vdso_end[]; -static unsigned long vdso_pages; -static struct page **vdso_pagelist; +enum vvar_pages { + VVAR_DATA_PAGE_OFFSET, + VVAR_TIMENS_PAGE_OFFSET, + VVAR_NR_PAGES, +}; + +enum sw64_vdso_map { + SW64_VDSO_MAP_VVAR, + SW64_VDSO_MAP_VDSO, +}; + +#define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT) /* * The vDSO data page. @@ -34,7 +45,18 @@ static union { } vdso_data_store __page_aligned_data; struct vdso_data *vdso_data = &vdso_data_store.data; -static struct vm_special_mapping vdso_spec[2]; +struct __vdso_info { + const char *name; + const char *vdso_code_start; + const char *vdso_code_end; + unsigned long vdso_pages; + /* Data Mapping */ + struct vm_special_mapping *dm; + /* Code Mapping */ + struct vm_special_mapping *cm; +}; + +static struct __vdso_info vdso_info; #ifdef CONFIG_SUBARCH_C3B #define V_NODE_SHIFT 6 @@ -84,89 +106,182 @@ static int vdso_mremap(const struct vm_special_mapping *sm, return 0; } -static int __init vdso_init(void) +static void __init __vdso_init(struct __vdso_info *vdso_info) { - int i; + unsigned int i; + struct page **vdso_pagelist; + unsigned long pfn; - if (memcmp(vdso_start, "\177ELF", 4)) { - pr_err("vDSO is not a valid ELF object!\n"); - return -EINVAL; - } + if (memcmp(vdso_info->vdso_code_start, "\177ELF", 4)) + panic("vDSO is not a valid ELF object!\n"); - vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; - pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n", - vdso_pages + 1, vdso_pages, vdso_start, 1L, vdso_data); + vdso_info->vdso_pages = ( + vdso_info->vdso_code_end - + vdso_info->vdso_code_start) >> + PAGE_SHIFT; - /* Allocate the vDSO pagelist, plus a page for the data. */ - vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *), + vdso_pagelist = kcalloc(vdso_info->vdso_pages, + sizeof(struct page *), GFP_KERNEL); if (vdso_pagelist == NULL) - return -ENOMEM; - - /* Grab the vDSO data page. */ - vdso_pagelist[0] = virt_to_page(vdso_data); + panic("vDSO kcalloc failed!\n"); /* Grab the vDSO code pages. */ - for (i = 0; i < vdso_pages; i++) - vdso_pagelist[i + 1] = virt_to_page(vdso_start + i * PAGE_SIZE); - - /* Populate the special mapping structures */ - vdso_spec[0] = (struct vm_special_mapping) { - .name = "[vvar]", - .pages = vdso_pagelist, - }; - - vdso_spec[1] = (struct vm_special_mapping) { - .name = "[vdso]", - .pages = &vdso_pagelist[1], - .mremap = vdso_mremap, - }; + pfn = sym_to_pfn(vdso_info->vdso_code_start); + + for (i = 0; i < vdso_info->vdso_pages; i++) + vdso_pagelist[i] = pfn_to_page(pfn + i); + + vdso_info->cm->pages = vdso_pagelist; init_cpu_map(); +} + +#ifdef CONFIG_TIME_NS +struct vdso_data *arch_get_vdso_data(void *vvar_page) +{ + return (struct vdso_data *)(vvar_page); +} + +/* + * The vvar mapping contains data for a specific time namespace, so when a task + * changes namespace we must unmap its vvar data for the old namespace. + * Subsequent faults will map in data for the new namespace. + * + * For more details see timens_setup_vdso_data(). + */ +int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) +{ + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); + + mmap_read_lock(mm); + + for_each_vma(vmi, vma) { + if (vma_is_special_mapping(vma, vdso_info.dm)) + zap_vma_pages(vma); + } + + mmap_read_unlock(mm); + return 0; +} +#endif + +static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, + struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct page *timens_page = find_timens_vvar_page(vma); + unsigned long pfn; + + switch (vmf->pgoff) { + case VVAR_DATA_PAGE_OFFSET: + if (timens_page) + pfn = page_to_pfn(timens_page); + else + pfn = sym_to_pfn(vdso_data); + break; +#ifdef CONFIG_TIME_NS + case VVAR_TIMENS_PAGE_OFFSET: + /* + * If a task belongs to a time namespace then a namespace + * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and + * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET + * offset. + * See also the comment near timens_setup_vdso_data(). + */ + if (!timens_page) + return VM_FAULT_SIGBUS; + pfn = sym_to_pfn(vdso_data); + break; +#endif /* CONFIG_TIME_NS */ + default: + return VM_FAULT_SIGBUS; + } + + return vmf_insert_pfn(vma, vmf->address, pfn); +} + +static struct vm_special_mapping sw64_vdso_maps[] __ro_after_init = { + [SW64_VDSO_MAP_VVAR] = { + .name = "[vvar]", + .fault = vvar_fault, + }, + [SW64_VDSO_MAP_VDSO] = { + .name = "[vdso]", + .mremap = vdso_mremap, + }, +}; + +static struct __vdso_info vdso_info __ro_after_init = { + .name = "vdso", + .vdso_code_start = vdso_start, + .vdso_code_end = vdso_end, + .dm = &sw64_vdso_maps[SW64_VDSO_MAP_VVAR], + .cm = &sw64_vdso_maps[SW64_VDSO_MAP_VDSO], +}; + +static int __init vdso_init(void) +{ + __vdso_init(&vdso_info); return 0; } arch_initcall(vdso_init); -int arch_setup_additional_pages(struct linux_binprm *bprm, - int uses_interp) +static int __setup_additional_pages(struct mm_struct *mm, + struct linux_binprm *bprm, + int uses_interp, + struct __vdso_info *vdso_info) { - struct mm_struct *mm = current->mm; unsigned long vdso_base, vdso_text_len, vdso_mapping_len; void *ret; - vdso_text_len = vdso_pages << PAGE_SHIFT; + BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES); + + vdso_text_len = vdso_info->vdso_pages << PAGE_SHIFT; /* Be sure to map the data page */ - vdso_mapping_len = vdso_text_len + PAGE_SIZE; + vdso_mapping_len = vdso_text_len + VVAR_SIZE; - if (down_write_killable(&mm->mmap_lock)) - return -EINTR; vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); if (IS_ERR_VALUE(vdso_base)) { ret = ERR_PTR(vdso_base); goto up_fail; } - ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE, - VM_READ|VM_MAYREAD, - &vdso_spec[0]); + + ret = _install_special_mapping(mm, vdso_base, VVAR_SIZE, + (VM_READ | VM_MAYREAD | VM_PFNMAP), vdso_info->dm); if (IS_ERR(ret)) goto up_fail; - vdso_base += PAGE_SIZE; + vdso_base += VVAR_SIZE; mm->context.vdso = (void *)vdso_base; - ret = _install_special_mapping(mm, vdso_base, vdso_text_len, - VM_READ|VM_EXEC| - VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, - &vdso_spec[1]); + + ret = + _install_special_mapping(mm, vdso_base, vdso_text_len, + (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), + vdso_info->cm); + if (IS_ERR(ret)) goto up_fail; - up_write(&mm->mmap_lock); return 0; up_fail: mm->context.vdso = NULL; - up_write(&mm->mmap_lock); return PTR_ERR(ret); } +int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) +{ + struct mm_struct *mm = current->mm; + int ret; + + if (mmap_write_lock_killable(mm)) + return -EINTR; + + ret = __setup_additional_pages(mm, bprm, uses_interp, &vdso_info); + mmap_write_unlock(mm); + + return ret; +} diff --git a/arch/sw_64/kernel/vdso/Makefile b/arch/sw_64/kernel/vdso/Makefile index 8ae5bf3d1770..51db491cb418 100644 --- a/arch/sw_64/kernel/vdso/Makefile +++ b/arch/sw_64/kernel/vdso/Makefile @@ -5,7 +5,7 @@ include $(srctree)/lib/vdso/Makefile vdso-syms = rt_sigreturn gettimeofday getcpu # Files to link into the vdso -obj-vdso = $(patsubst %, v%.o, $(vdso-syms)) +obj-vdso = $(patsubst %, v%.o, $(vdso-syms)) note.o ifneq ($(c-gettimeofday-y),) CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y) @@ -41,7 +41,7 @@ $(obj)/vdso.o: $(obj)/vdso.so # link rule for the .so file, .lds has to be first SYSCFLAGS_vdso.so.dbg = $(c_flags) -$(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) +$(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) FORCE $(call if_changed,vdsold) SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ $(call cc-ldoption, -Wl$(comma)--hash-style=both) diff --git a/arch/sw_64/kernel/vdso/note.S b/arch/sw_64/kernel/vdso/note.S new file mode 100644 index 000000000000..2a956c942211 --- /dev/null +++ b/arch/sw_64/kernel/vdso/note.S @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. + * Here we can supply some information useful to userland. + */ + +#include +#include + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/sw_64/kernel/vdso/vdso.lds.S b/arch/sw_64/kernel/vdso/vdso.lds.S index 08134a86f66c..0dde95a9e912 100644 --- a/arch/sw_64/kernel/vdso/vdso.lds.S +++ b/arch/sw_64/kernel/vdso/vdso.lds.S @@ -26,7 +26,10 @@ OUTPUT_ARCH(sw_64) SECTIONS { - PROVIDE(_vdso_data = . - PAGE_SIZE); + PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE); +#ifdef CONFIG_TIME_NS + PROVIDE(_timens_data = _vdso_data + PAGE_SIZE); +#endif . = VDSO_LBASE + SIZEOF_HEADERS; .hash : { *(.hash) } :text -- Gitee From ef7be5db5bfe5b98a159ed286ac5906c29d8ca9d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 25 Nov 2024 15:48:18 +0800 Subject: [PATCH 428/524] sw64: compile divide.S for c3b only New subarchs support hardware DIV and MOD instructions, no need to include software implementation in kernel. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/lib/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/lib/Makefile b/arch/sw_64/lib/Makefile index a37d1ac0ca07..aa6d8eee29c2 100644 --- a/arch/sw_64/lib/Makefile +++ b/arch/sw_64/lib/Makefile @@ -6,8 +6,7 @@ asflags-y := $(KBUILD_CFLAGS) ccflags-y := -Werror -lib-y = __divlu.o __remlu.o __divwu.o __remwu.o \ - udelay.o \ +lib-y = udelay.o \ memmove.o \ checksum.o \ csum_partial_copy.o \ @@ -18,6 +17,7 @@ lib-y = __divlu.o __remlu.o __divwu.o __remwu.o \ csum_ipv6_magic.o \ string.o \ uaccess.o +lib-$(CONFIG_SUBARCH_C3B) += __divlu.o __remlu.o __divwu.o __remwu.o lib-clear_page-y := clear_page.o lib-clear_page-$(CONFIG_DEEP_CLEAR_PAGE) := clear_page_simd.o @@ -40,6 +40,7 @@ lib-y += $(lib-clear_page-y) $(lib-clear_user-y) $(lib-copy_page-y) $(lib-copy_u obj-y = iomap.o obj-y += iomap_copy.o +ifeq ($(CONFIG_SUBARCH_C3B),y) # The division routines are built from single source, with different defines. AFLAGS___divlu.o = -DDIV AFLAGS___remlu.o = -DREM @@ -49,3 +50,4 @@ AFLAGS___remwu.o = -DREM -DINTSIZE $(addprefix $(obj)/,__divlu.o __remlu.o __divwu.o __remwu.o): \ $(src)/divide.S FORCE $(call if_changed_rule,as_o_S) +endif -- Gitee From 824714df04b86c4cae84e2bb0332c1a84e012ae8 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 6 Dec 2024 13:01:31 +0800 Subject: [PATCH 429/524] sw64: make sw64_(read/write)_csr always inline The csr index is hard coded in csrr/csrw instruction, so these functions has to be inlined. Otherwise the csr index will be stored in a register and compilation will fail. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/csr.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h index cc36137d6dd6..f88e2bd36d9f 100644 --- a/arch/sw_64/include/asm/csr.h +++ b/arch/sw_64/include/asm/csr.h @@ -83,7 +83,7 @@ #ifdef CONFIG_HAVE_CSRRW #ifndef __ASSEMBLY__ -static inline unsigned long sw64_read_csr(unsigned long x) +static __always_inline unsigned long sw64_read_csr(unsigned long x) { unsigned long __val; @@ -91,12 +91,12 @@ static inline unsigned long sw64_read_csr(unsigned long x) return __val; } -static inline void sw64_write_csr(unsigned long x, unsigned long y) +static __always_inline void sw64_write_csr(unsigned long x, unsigned long y) { __asm__ __volatile__("csrw %0,%1" ::"r"(x), "i"(y)); } -static inline void sw64_write_csr_imb(unsigned long x, unsigned long y) +static __always_inline void sw64_write_csr_imb(unsigned long x, unsigned long y) { __asm__ __volatile__("csrw %0,%1; imemb" ::"r"(x), "i"(y)); } -- Gitee From b864a58ea48d9999b44bc9fe693932bfc3c2a5a9 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 6 Dec 2024 13:20:41 +0800 Subject: [PATCH 430/524] sw64: put cpu_set_node() in init text This functions is used in initialization only. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/numa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/mm/numa.c b/arch/sw_64/mm/numa.c index 8736a49e54ee..e6416726c64a 100644 --- a/arch/sw_64/mm/numa.c +++ b/arch/sw_64/mm/numa.c @@ -289,7 +289,7 @@ static void __init get_numa_info_socket(void) } } -static void cpu_set_node(void) +static void __init cpu_set_node(void) { int i; -- Gitee From 09360c2863b27eb934670ccf73a70fe510512ca9 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 2 Dec 2024 10:15:20 +0800 Subject: [PATCH 431/524] sw64: change condition of some cflags Change -fsw-rev and -fno-sw-unalign-bytefrom CONFIG_SUBARCH_C4 to !CONFIG_SUBARCH_C3B. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index 9b25e3797f52..b7267453a9f1 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -27,7 +27,7 @@ endif CHECKFLAGS += -D__sw__ cflags-y := -pipe -ffixed-8 -mno-fp-regs #-msmall-data -ifeq ($(CONFIG_SUBARCH_C4),y) +ifneq ($(CONFIG_SUBARCH_C3B),y) cflags-y += -fsw-rev -fno-sw-unalign-byte endif cflags-y += $(call cc-option, -fno-jump-tables) -- Gitee From 5fcd20c46cc6b5e7f67cd5be04c445617f4054f9 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Tue, 29 Oct 2024 17:25:12 +0800 Subject: [PATCH 432/524] sw64: reimplement cmpxchg and xchg C4 has casl instruction to implement cmpxchg, reimplement it for better understanding. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/xchg.h | 187 +++++++++++++++------------------- 1 file changed, 84 insertions(+), 103 deletions(-) diff --git a/arch/sw_64/include/asm/xchg.h b/arch/sw_64/include/asm/xchg.h index 38f067d5ed04..00f2af06df26 100644 --- a/arch/sw_64/include/asm/xchg.h +++ b/arch/sw_64/include/asm/xchg.h @@ -244,22 +244,25 @@ ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) static inline unsigned long ____xchg(_u8, volatile char *m, unsigned long val) { - unsigned long ret, tmp, addr64; + unsigned long ret, tmp, addr64, tmp2; __asm__ __volatile__( - " andnot %4, 7, %3\n" - " inslb %1, %4, %1\n" - "1: lldl %2, 0(%3)\n" - " extlb %2, %4, %0\n" - " masklb %2, %4, %2\n" - " or %1, %2, %2\n" - " lstl %2, 0(%3)\n" - " beq %2, 2f\n" + " andnot %[m], 7, %[addr64]\n" + " inslb %[new_v], %[m], %[new_v]\n" + " ldl %[ret], 0(%[addr64])\n" + "1: masklb %[ret], %[m], %[tmp]\n" + " or %[new_v], %[tmp], %[tmp]\n" + " casl %[ret], %[addr64], %[tmp]\n" + " cmpeq %[ret], %[tmp], %[tmp2]\n" + " beq %[tmp2], 2f\n" + " extlb %[ret], %[m], %[ret]\n" ".subsection 2\n" - "2: lbr 1b\n" + "2: mov %[tmp], %[ret]\n" + " br 1b\n" ".previous" - : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) - : "r" ((long)m), "1" (val) : "memory"); + : [ret]"=&r"(ret), [new_v]"=&r"(val), [tmp]"=&r"(tmp), + [addr64]"=&r"(addr64), [tmp2]"=&r"(tmp2) + : [m]"r" ((long)m), "1" (val) : "memory"); return ret; } @@ -267,22 +270,25 @@ ____xchg(_u8, volatile char *m, unsigned long val) static inline unsigned long ____xchg(_u16, volatile short *m, unsigned long val) { - unsigned long ret, tmp, addr64; + unsigned long ret, tmp, addr64, tmp2; __asm__ __volatile__( - " andnot %4, 7, %3\n" - " inslh %1, %4, %1\n" - "1: lldl %2, 0(%3)\n" - " extlh %2, %4, %0\n" - " masklh %2, %4, %2\n" - " or %1, %2, %2\n" - " lstl %2, 0(%3)\n" - " beq %2, 2f\n" + " andnot %[m], 7, %[addr64]\n" + " inslh %[new_v], %[m], %[new_v]\n" + " ldl %[ret], 0(%[addr64])\n" + "1: masklh %[ret], %[m], %[tmp]\n" + " or %[new_v], %[tmp], %[tmp]\n" + " casl %[ret], %[addr64], %[tmp]\n" + " cmpeq %[ret], %[tmp], %[tmp2]\n" + " beq %[tmp2], 2f\n" + " extlh %[ret], %[m], %[ret]\n" ".subsection 2\n" - "2: lbr 1b\n" + "2: mov %[tmp], %[ret]\n" + " br 1b\n" ".previous" - : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) - : "r" ((long)m), "1" (val) : "memory"); + : [ret]"=&r"(ret), [new_v]"=&r"(val), [tmp]"=&r"(tmp), + [addr64]"=&r"(addr64), [tmp2]"=&r"(tmp2) + : [m]"r" ((long)m), "1" (val) : "memory"); return ret; } @@ -290,19 +296,21 @@ ____xchg(_u16, volatile short *m, unsigned long val) static inline unsigned long ____xchg(_u32, volatile int *m, unsigned long val) { - unsigned long dummy, addr; + unsigned long tmp1, tmp2; __asm__ __volatile__( - " ldi %3, %5\n" - "1: lldw %0, 0(%3)\n" - " bis $31, %4, %1\n" - " lstw %1, 0(%3)\n" - " beq %1, 2f\n" + " ldw %[old_v], 0(%[m])\n" + "1: mov %[new_v], %[tmp1]\n" + " casw %[old_v], %[m], %[tmp1]\n" + " cmpeq %[old_v], %[tmp1], %[tmp2]\n" + " beq %[tmp2], 2f\n" ".subsection 2\n" - "2: lbr 1b\n" + "2: mov %[tmp1], %[old_v]\n" + " br 1b\n" ".previous" - : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) - : "rI" (val), "m" (*m) : "memory"); + : [old_v]"=&r"(val), [tmp1]"=&r"(tmp1), [tmp2]"=&r"(tmp2) + : [new_v]"r"(val), [m]"r"(m) + ); return val; } @@ -310,19 +318,21 @@ ____xchg(_u32, volatile int *m, unsigned long val) static inline unsigned long ____xchg(_u64, volatile long *m, unsigned long val) { - unsigned long dummy, addr; + unsigned long tmp1, tmp2; __asm__ __volatile__( - " ldi %3, %5\n" - "1: lldl %0, 0(%3)\n" - " bis $31, %4, %1\n" - " lstl %1, 0(%3)\n" - " beq %1, 2f\n" + " ldl %[old_v], 0(%[m])\n" + "1: mov %[new_v], %[tmp1]\n" + " casl %[old_v], %[m], %[tmp1]\n" + " cmpeq %[old_v], %[tmp1], %[tmp2]\n" + " beq %[tmp2], 2f\n" ".subsection 2\n" - "2: lbr 1b\n" + "2: mov %[tmp1], %[old_v]\n" + " br 1b\n" ".previous" - : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) - : "rI" (val), "m" (*m) : "memory"); + : [old_v]"=&r"(val), [tmp1]"=&r"(tmp1), [tmp2]"=&r"(tmp2) + : [new_v]"r"(val), [m]"r"(m) + ); return val; } @@ -343,22 +353,18 @@ ____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new) unsigned long prev, tmp, cmp, addr64; __asm__ __volatile__( - " andnot %5, 7, %4\n" - " inslb %1, %5, %1\n" - "1: lldl %2, 0(%4)\n" - " extlb %2, %5, %0\n" - " cmpeq %0, %6, %3\n" - " beq %3, 2f\n" - " masklb %2, %5, %2\n" - " or %1, %2, %2\n" - " lstl %2, 0(%4)\n" - " beq %2, 3f\n" - "2:\n" - ".subsection 2\n" - "3: lbr 1b\n" - ".previous" - : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) - : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); + " andnot %[m], 7, %[addr64]\n" + " inslb %[new_v], %[m], %[new_v]\n" + " inslb %[old], %[m], %[cmp]\n" + " ldl %[prev], 0(%[addr64])\n" + " masklb %[prev], %[m], %[tmp]\n" + " or %[new_v], %[tmp], %[new_v]\n" + " or %[cmp], %[tmp], %[cmp]\n" + " casl %[cmp], %[addr64], %[new_v]\n" + " extlb %[new_v], %[m], %[prev]\n" + : [prev]"=&r" (prev), [new_v]"=&r" (new), [tmp]"=&r" (tmp), + [cmp]"=&r" (cmp), [addr64]"=&r" (addr64) + : [m]"r" ((long)m), [old]"r" (old), "1" (new) : "memory"); return prev; } @@ -369,45 +375,30 @@ ____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new) unsigned long prev, tmp, cmp, addr64; __asm__ __volatile__( - " andnot %5, 7, %4\n" - " inslh %1, %5, %1\n" - "1: lldl %2, 0(%4)\n" - " extlh %2, %5, %0\n" - " cmpeq %0, %6, %3\n" - " beq %3, 2f\n" - " masklh %2, %5, %2\n" - " or %1, %2, %2\n" - " lstl %2, 0(%4)\n" - " beq %2, 3f\n" - "2:\n" - ".subsection 2\n" - "3: lbr 1b\n" - ".previous" - : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) - : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); - + " andnot %[m], 7, %[addr64]\n" + " inslh %[new_v], %[m], %[new_v]\n" + " inslh %[old], %[m], %[cmp]\n" + " ldl %[prev], 0(%[addr64])\n" + " masklh %[prev], %[m], %[tmp]\n" + " or %[new_v], %[tmp], %[new_v]\n" + " or %[cmp], %[tmp], %[cmp]\n" + " casl %[cmp], %[addr64], %[new_v]\n" + " extlh %[new_v], %[m], %[prev]\n" + : [prev]"=&r" (prev), [new_v]"=&r" (new), [tmp]"=&r" (tmp), + [cmp]"=&r" (cmp), [addr64]"=&r" (addr64) + : [m]"r" ((long)m), [old]"r" (old), "1" (new) : "memory"); return prev; } static inline unsigned long ____cmpxchg(_u32, volatile int *m, int old, int new) { - unsigned long prev, cmp, addr, tmp; + unsigned long prev; __asm__ __volatile__( - " ldi %3, %7\n" - "1: lldw %0, 0(%3)\n" - " cmpeq %0, %5, %1\n" - " beq %1, 2f\n" - " bis $31, %6, %4\n" - " lstw %4, 0(%3)\n" - " beq %4, 3f\n" - "2:\n" - ".subsection 2\n" - "3: lbr 1b\n" - ".previous" - : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) - : "r"((long) old), "r"(new), "m"(*m) : "memory"); + " casw %[old], %[m], %[prev]\n" + : [prev]"=r"(prev) + : [old]"r"((long) old), "0"(new), [m]"r"(m) : "memory"); return prev; } @@ -415,22 +406,12 @@ ____cmpxchg(_u32, volatile int *m, int old, int new) static inline unsigned long ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) { - unsigned long prev, cmp, addr, tmp; + unsigned long prev; __asm__ __volatile__( - " ldi %3, %7\n" - "1: lldl %0, 0(%3)\n" - " cmpeq %0, %5, %1\n" - " beq %1, 2f\n" - " bis $31, %6, %4\n" - " lstl %4, 0(%3)\n" - " beq %4, 3f\n" - "2:\n" - ".subsection 2\n" - "3: lbr 1b\n" - ".previous" - : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) - : "r"((long) old), "r"(new), "m"(*m) : "memory"); + " casl %[old], %[m], %[prev]\n" + : [prev]"=r"(prev) + : [old]"r"((long) old), "0"(new), [m]"r"(m) : "memory"); return prev; } -- Gitee From f6104ad0eeab7b78f00f1ae04d460877e3ca9bd2 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 21 Mar 2025 11:43:30 +0800 Subject: [PATCH 433/524] sw64: fix secondary_cpu_start for CORE3B For the startup process of secondary cpus, in hmcode, C3B uses cyclic busy read to wait for flags, and C4 uses HALT to optimize the process. However, in secondary_cpu_start(), the interval between set_secondary_ready() and reset_cpu() may be too long, so that the secondary cpu has entered the kernel during this time, resulting in a boot error of C3B. This patch resolves this issue by removing the reset_cpu() in C3B. Additionally, to ensure correctness during cpu transition from offline to online, this patch adds reset_cpu() for C3B in __cpu_up(). Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 24dc63317c56..484605689a84 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -215,8 +215,10 @@ static int secondary_cpu_start(int cpuid, struct task_struct *idle) set_secondary_ready(cpuid); +#ifdef CONFIG_SUBARCH_C4 /* send reset signal */ reset_cpu(cpuid); +#endif /* Wait 10 seconds for secondary cpu. */ timeout = jiffies + 10*HZ; @@ -546,6 +548,9 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) if (!is_junzhang_v1() && is_in_host()) { /* send wake up signal */ send_wakeup_interrupt(cpu); +#ifdef CONFIG_SUBARCH_C3B + reset_cpu(cpu); +#endif } smp_boot_one_cpu(cpu, tidle); -- Gitee From 71da6c8f67141dd8b25adb2fccddaec7b9e5fb3f Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 27 Feb 2025 08:58:43 +0000 Subject: [PATCH 434/524] sw64: simplify GP handling for entUna If the target register is GP in do_entUna, return directly. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry_c4.S | 4 +--- arch/sw_64/kernel/traps.c | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S index f6b620c36756..8aa87acd4f25 100644 --- a/arch/sw_64/kernel/entry_c4.S +++ b/arch/sw_64/kernel/entry_c4.S @@ -356,9 +356,7 @@ entUna: beq $0, 1f call $26, do_entUnaUser /* return to ret_from_syscall */ br ret_from_sys_call -1: ldl $9, (PT_REGS_GP + STACKFRAME_SIZE)($sp) - call $26, do_entUna - stl $9, (PT_REGS_GP + STACKFRAME_SIZE)($sp) +1: call $26, do_entUna RESTORE_ALL sys_call HMC_rti .end entUna diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 1bce82e95f5f..b87363f35801 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -408,6 +408,9 @@ asmlinkage void noinstr do_entUna(struct pt_regs *regs) unsigned long opcode = regs->earg1; unsigned long reg = regs->earg2; + if (reg == 29) + return; + insn = *(unsigned int *)pc; fncode = (insn >> 12) & 0xf; -- Gitee From 5ef3bac5b08144a9c40d3a4468b609936b1dfa8b Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 2 Dec 2024 10:17:20 +0800 Subject: [PATCH 435/524] sw64: reduce jump instructions in entUna Use seleq to select the target for call, instead of writing two calls. Make sure call and bne instructions are not next to each other to let L2BTAC work correctly. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry_c4.S | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S index 8aa87acd4f25..4e64ae6da08b 100644 --- a/arch/sw_64/kernel/entry_c4.S +++ b/arch/sw_64/kernel/entry_c4.S @@ -351,12 +351,14 @@ entUna: 1: ldgp $29, 0($27) RESTORE_IRQ ldi $16, STACKFRAME_SIZE($sp) - ldl $0, (PT_REGS_PS + STACKFRAME_SIZE)($sp) - and $0, 8, $0 /* user mode ? */ - beq $0, 1f - call $26, do_entUnaUser /* return to ret_from_syscall */ - br ret_from_sys_call -1: call $26, do_entUna + ldl $9, (PT_REGS_PS + STACKFRAME_SIZE)($sp) + and $9, 8, $9 /* user mode ? */ + ldi $1, do_entUnaUser + ldi $2, do_entUna + seleq $9, $2, $1, $27 + call $26, ($27), 0 + .align 2 + bne $9, ret_from_sys_call RESTORE_ALL sys_call HMC_rti .end entUna -- Gitee From a041610af0ab82b79b75a4e2982f00e539561247 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 2 Dec 2024 10:23:49 +0800 Subject: [PATCH 436/524] sw64: enable -fsw-unalign-byte for new subarchs New subarchs support hardware unaligned memory access in most cases, no need to force compiler to avoid unaligned memory access for most part of the kernel. However, we do need to make sure there is no unaligned memory access in software handling process, so compile trap_unalign.c with -fno-sw-unalign-byte and let entUna leave kernel directly without doing anything else. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Makefile | 2 +- arch/sw_64/kernel/Makefile | 7 +- arch/sw_64/kernel/entry_c4.S | 2 - arch/sw_64/kernel/trap_unalign.c | 2141 ++++++++++++++++++++++++++++++ arch/sw_64/kernel/traps.c | 2127 ----------------------------- 5 files changed, 2148 insertions(+), 2131 deletions(-) create mode 100644 arch/sw_64/kernel/trap_unalign.c diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index b7267453a9f1..41b90403ee3c 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -28,7 +28,7 @@ endif CHECKFLAGS += -D__sw__ cflags-y := -pipe -ffixed-8 -mno-fp-regs #-msmall-data ifneq ($(CONFIG_SUBARCH_C3B),y) - cflags-y += -fsw-rev -fno-sw-unalign-byte + cflags-y += -fsw-rev -fsw-unalign-byte endif cflags-y += $(call cc-option, -fno-jump-tables) diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 0b0012462517..539b3011c854 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -13,12 +13,17 @@ CFLAGS_REMOVE_insn.o = -pg CFLAGS_REMOVE_printk.o = -pg endif +ifneq ($(CONFIG_SUBARCH_C3B),y) + CFLAGS_REMOVE_trap_unalign.o = -fsw-unalign-byte + CFLAGS_trap_unalign.o += -fno-sw-unalign-byte +endif + obj-y := fpu.o traps.o process.o sys_sw64.o irq.o cpu.o \ signal.o setup.o ptrace.o time.o \ systbls.o dup_print.o chip_setup.o \ insn.o early_init.o topology.o cacheinfo.o \ vdso.o vdso/ hmcall.o stacktrace.o idle.o reset.o \ - head.o termios.o + head.o termios.o trap_unalign.o obj-$(CONFIG_SUBARCH_C3B) += entry_c3.o tc.o obj-$(CONFIG_SUBARCH_C4) += entry_c4.o diff --git a/arch/sw_64/kernel/entry_c4.S b/arch/sw_64/kernel/entry_c4.S index 4e64ae6da08b..2734a1d874f3 100644 --- a/arch/sw_64/kernel/entry_c4.S +++ b/arch/sw_64/kernel/entry_c4.S @@ -357,8 +357,6 @@ entUna: ldi $2, do_entUna seleq $9, $2, $1, $27 call $26, ($27), 0 - .align 2 - bne $9, ret_from_sys_call RESTORE_ALL sys_call HMC_rti .end entUna diff --git a/arch/sw_64/kernel/trap_unalign.c b/arch/sw_64/kernel/trap_unalign.c new file mode 100644 index 000000000000..a07f6fe8f9e8 --- /dev/null +++ b/arch/sw_64/kernel/trap_unalign.c @@ -0,0 +1,2141 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "proto.h" + +#define OP_INT_MASK (1L << 0x22 | 1L << 0x2a | /* ldw stw */ \ + 1L << 0x23 | 1L << 0x2b | /* ldl stl */ \ + 1L << 0x21 | 1L << 0x29 | /* ldhu sth */ \ + 1L << 0x20 | 1L << 0x28) /* ldbu stb */ + +#define FN_INT_MASK (1L << 0x0 | 1L << 0x6 | /* ldbu_a stb_a */ \ + 1L << 0x1 | 1L << 0x7 | /* ldhu_a sth_a */ \ + 1L << 0x2 | 1L << 0x8 | /* ldw_a stw_a */ \ + 1L << 0x3 | 1L << 0x9) /* ldl_a stl_a */ + +asmlinkage void noinstr do_entUna(struct pt_regs *regs) +{ + long error, disp; + unsigned int insn, fncode, rb; + unsigned long tmp, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, vb; + unsigned long fp[4]; + unsigned long fake_reg, *reg_addr = &fake_reg; + unsigned long pc = regs->pc - 4; + void *va = (void *)regs->earg0; + unsigned long opcode = regs->earg1; + unsigned long reg = regs->earg2; + + if (reg == 29) + return; + + insn = *(unsigned int *)pc; + fncode = (insn >> 12) & 0xf; + + if (((1L << opcode) & OP_INT_MASK) || + ((opcode == 0x1e) && ((1L << fncode) & FN_INT_MASK))) { + /* it's an integer load/store */ + if (reg < 31) { + reg_addr = ®s->regs[reg]; + } else { + /* zero "register" */ + fake_reg = 0; + } + } + + /* + * We don't want to use the generic get/put unaligned macros as + * we want to trap exceptions. Only if we actually get an + * exception will we decide whether we should have caught it. + */ + + switch (opcode) { + + case 0x0c: /* vlds */ + if ((unsigned long)va<<61 == 0) { + __asm__ __volatile__( + "1: ldl %1, 0(%5)\n" + "2: ldl %2, 8(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + + sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); + + return; + } else { + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + + tmp1 = tmp1 | tmp4; + tmp2 = tmp5 | tmp3; + + sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); + + return; + } + + case 0x0d: /* vldd */ + if ((unsigned long)va<<61 == 0) { + __asm__ __volatile__( + "1: ldl %1, 0(%5)\n" + "2: ldl %2, 8(%5)\n" + "3: ldl %3, 16(%5)\n" + "4: ldl %4, 24(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi %4, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + + sw64_write_simd_fp_reg_d(reg, tmp1, tmp2, tmp3, tmp4); + + return; + } else { + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + + tmp7 = tmp1 | tmp4; //f0 + tmp8 = tmp5 | tmp3; //f1 + + vb = ((unsigned long)(va))+16; + + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(vb), "0"(0)); + + if (error) + goto got_exception; + + tmp = tmp1 | tmp4; // f2 + tmp2 = tmp5 | tmp3; // f3 + + sw64_write_simd_fp_reg_d(reg, tmp7, tmp8, tmp, tmp2); + return; + } + + case 0x0e: /* vsts */ + sw64_read_simd_fp_m_s(reg, fp); + if ((unsigned long)va<<61 == 0) { + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); + + if (error) + goto got_exception; + + return; + } else { + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(fp[0]), "0"(0)); + + if (error) + goto got_exception; + + + vb = ((unsigned long)va) + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[1]), "0"(0)); + + if (error) + goto got_exception; + + return; + } + + case 0x0f: /* vstd */ + sw64_read_simd_fp_m_d(reg, fp); + if ((unsigned long)va<<61 == 0) { + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); + + if (error) + goto got_exception; + + vb = ((unsigned long)va)+16; + + + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(vb), "r"(fp[2]), "r"(fp[3]), "0"(0)); + + if (error) + goto got_exception; + + return; + } else { + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(fp[0]), "0"(0)); + + if (error) + goto got_exception; + + vb = ((unsigned long)va) + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[1]), "0"(0)); + + if (error) + goto got_exception; + + vb = vb + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[2]), "0"(0)); + + if (error) + goto got_exception; + + vb = vb + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[3]), "0"(0)); + + if (error) + goto got_exception; + + return; + } + + case 0x1e: + insn = *(unsigned int *)pc; + fncode = (insn >> 12) & 0xf; + rb = (insn >> 16) & 0x1f; + disp = insn & 0xfff; + + disp = (disp << 52) >> 52; /* sext */ + + switch (fncode) { + case 0x1: /* ldhu_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 1(%3)\n" + " extlh %1, %3, %1\n" + " exthh %2, %3, %2\n" + "3:\n" + ".section __ex_table,\"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto got_exception; + regs->regs[reg] = tmp1 | tmp2; + regs->regs[rb] = regs->regs[rb] + disp; + return; + + case 0x2: /* ldw_a */ + __asm__ __volatile__( + "1: ldl_u %1,0(%3)\n" + "2: ldl_u %2,3(%3)\n" + " extlw %1,%3,%1\n" + " exthw %2,%3,%2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + regs->regs[reg] = (int)(tmp1 | tmp2); + regs->regs[rb] = regs->regs[rb] + disp; + return; + + case 0x3: /* ldl_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + regs->regs[reg] = tmp1 | tmp2; + regs->regs[rb] = regs->regs[rb] + disp; + return; + + case 0x7: /* sth_a */ + __asm__ __volatile__( + " zap %6, 2, %1\n" + " srl %6, 8, %2\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %2, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %1, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto got_exception; + regs->regs[rb] = regs->regs[rb] + disp; + return; + + case 0x8: /* stw_a */ + __asm__ __volatile__( + " zapnot %6, 0x1, %1\n" + " srl %6, 8, %2\n" + " zapnot %2, 0x1,%2\n" + " srl %6, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %6, 24, %4\n" + " zapnot %4, 0x1, %4\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3: stb %3, 0x2(%5)\n" + "4: stb %4, 0x3(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi $31, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto got_exception; + regs->regs[rb] = regs->regs[rb] + disp; + return; + + case 0x9: /* stl_a */ + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto got_exception; + regs->regs[rb] = regs->regs[rb] + disp; + return; + } + return; + + case 0x21: + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 1(%3)\n" + " extlh %1, %3, %1\n" + " exthh %2, %3, %2\n" + "3:\n" + ".section __ex_table,\"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + regs->regs[reg] = tmp1 | tmp2; + return; + + case 0x22: + __asm__ __volatile__( + "1: ldl_u %1,0(%3)\n" + "2: ldl_u %2,3(%3)\n" + " extlw %1,%3,%1\n" + " exthw %2,%3,%2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + regs->regs[reg] = (int)(tmp1 | tmp2); + return; + + case 0x23: /* ldl */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto got_exception; + regs->regs[reg] = tmp1 | tmp2; + return; + + case 0x29: /* sth */ + __asm__ __volatile__( + " zap %6, 2, %1\n" + " srl %6, 8, %2\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %2, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %1, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto got_exception; + return; + + case 0x2a: /* stw */ + __asm__ __volatile__( + " zapnot %6, 0x1, %1\n" + " srl %6, 8, %2\n" + " zapnot %2, 0x1,%2\n" + " srl %6, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %6, 24, %4\n" + " zapnot %4, 0x1, %4\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3: stb %3, 0x2(%5)\n" + "4: stb %4, 0x3(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi $31, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto got_exception; + return; + + case 0x2b: /* stl */ + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto got_exception; + return; + } + + pr_warn("Bad unaligned kernel access at %016lx: %p %lx %lu\n", + pc, va, opcode, reg); + make_task_dead(SIGSEGV); + +got_exception: + /* Ok, we caught the exception, but we don't want it. Is there + * someone to pass it along to? + */ + if (fixup_exception(regs, pc)) { + pr_info("Forwarding unaligned exception at %lx (%lx)\n", + pc, regs->pc); + return; + } + + /* + * Yikes! No one to forward the exception to. + * Since the registers are in a weird format, dump them ourselves. + */ + + die("Unhandled unaligned exception", regs, error); +} + +/* + * Handle user-level unaligned fault. Handling user-level unaligned + * faults is *extremely* slow and produces nasty messages. A user + * program *should* fix unaligned faults ASAP. + * + * Notice that we have (almost) the regular kernel stack layout here, + * so finding the appropriate registers is a little more difficult + * than in the kernel case. + * + * Finally, we handle regular integer load/stores only. In + * particular, load-linked/store-conditionally and floating point + * load/stores are not supported. The former make no sense with + * unaligned faults (they are guaranteed to fail) and I don't think + * the latter will occur in any decent program. + * + * Sigh. We *do* have to handle some FP operations, because GCC will + * uses them as temporary storage for integer memory to memory copies. + * However, we need to deal with stt/ldt and sts/lds only. + */ +asmlinkage void noinstr do_entUnaUser(struct pt_regs *regs) +{ +#ifdef CONFIG_UNA_PRINT + static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5); +#endif + + unsigned long tmp1, tmp2, tmp3, tmp4; + unsigned long fake_reg, *reg_addr = &fake_reg; + int si_code; + long error; + unsigned long tmp, tmp5, tmp6, tmp7, tmp8, vb; + unsigned long fp[4]; + unsigned long instr, instr_op, value, fncode; + unsigned int rb = -1U; + long disp; + void __user *va = (void *)regs->earg0; + unsigned long opcode = regs->earg1; + unsigned long reg = regs->earg2; + +#ifdef CONFIG_DEBUG_FS + /* + * If command name is specified, record some information + * to debugfs. + */ + if (unaligned_task[0] && !strcmp(unaligned_task, current->comm)) { + int idx; + + idx = unaligned_count % UNA_MAX_ENTRIES; + unaligned[idx].va = (unsigned long)va; + unaligned[idx].pc = regs->pc; + unaligned_count++; + } +#endif + + /* Check the UAC bits to decide what the user wants us to do + * with the unaliged access. + */ + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, + 1, regs, regs->pc - 4); + +#ifdef CONFIG_UNA_PRINT + if (!(current_thread_info()->status & TS_UAC_NOPRINT)) { + if (__ratelimit(&ratelimit)) { + pr_info("%s(%d): unaligned trap at %016lx: %p %lx %ld\n", + current->comm, task_pid_nr(current), + regs->pc - 4, va, opcode, reg); + } + } +#endif + if ((current_thread_info()->status & TS_UAC_SIGBUS)) + goto give_sigbus; + /* Not sure why you'd want to use this, but... */ + if ((current_thread_info()->status & TS_UAC_NOFIX)) + return; + + /* Don't bother reading ds in the access check since we already + * know that this came from the user. Also rely on the fact that + * the page at TASK_SIZE is unmapped and so can't be touched anyway. + */ + if ((unsigned long)va >= TASK_SIZE) + goto give_sigsegv; + + get_user(instr, (__u32 *)(regs->pc - 4)); + fncode = (instr >> 12) & 0xf; + + if (((1L << opcode) & OP_INT_MASK) || + ((opcode == 0x1e) && ((1L << fncode) & FN_INT_MASK))) { + /* it's an integer load/store */ + if (reg < 31) { + reg_addr = ®s->regs[reg]; + } else { + /* zero "register" */ + fake_reg = 0; + } + } + + get_user(instr, (__u32 *)(regs->pc - 4)); + instr_op = (instr >> 26) & 0x3f; + + get_user(value, (__u64 *)va); + + switch (instr_op) { + + case 0x0c: /* vlds */ + if ((unsigned long)va << 61 == 0) { + __asm__ __volatile__( + "1: ldl %1, 0(%5)\n" + "2: ldl %2, 8(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); + + return; + } else { + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + tmp1 = tmp1 | tmp4; + tmp2 = tmp5 | tmp3; + + sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); + + return; + } + case 0x0a: /* ldse */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + tmp = tmp1 | tmp2; + tmp = tmp | (tmp << 32); + + sw64_write_simd_fp_reg_s(reg, tmp, tmp); + + return; + + case 0x0d: /* vldd */ + if ((unsigned long)va << 61 == 0) { + __asm__ __volatile__( + "1: ldl %1, 0(%5)\n" + "2: ldl %2, 8(%5)\n" + "3: ldl %3, 16(%5)\n" + "4: ldl %4, 24(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi %4, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + sw64_write_simd_fp_reg_d(reg, tmp1, tmp2, tmp3, tmp4); + + return; + } else { + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + tmp7 = tmp1 | tmp4; //f0 + tmp8 = tmp5 | tmp3; //f1 + + vb = ((unsigned long)(va))+16; + + __asm__ __volatile__( + "1: ldl_u %1, 0(%6)\n" + "2: ldl_u %2, 7(%6)\n" + "3: ldl_u %3, 15(%6)\n" + " extll %1, %6, %1\n" + " extll %2, %6, %5\n" + " exthl %2, %6, %4\n" + " exthl %3, %6, %3\n" + "4:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 4b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 4b-2b(%0)\n" + " .long 3b - .\n" + " ldi %3, 4b-3b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5) + : "r"(vb), "0"(0)); + + if (error) + goto give_sigsegv; + + tmp = tmp1 | tmp4; // f2 + tmp2 = tmp5 | tmp3; // f3 + + sw64_write_simd_fp_reg_d(reg, tmp7, tmp8, tmp, tmp2); + return; + } + + case 0x0b: /* ldde */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + tmp = tmp1 | tmp2; + + sw64_write_simd_fp_reg_d(reg, tmp, tmp, tmp, tmp); + return; + + case 0x09: /* ldwe */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + + if (error) + goto give_sigsegv; + + sw64_write_simd_fp_reg_ldwe(reg, (int)(tmp1 | tmp2)); + + return; + + case 0x0e: /* vsts */ + sw64_read_simd_fp_m_s(reg, fp); + if ((unsigned long)va << 61 == 0) { + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); + + if (error) + goto give_sigsegv; + + return; + } else { + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(fp[0]), "0"(0)); + + if (error) + goto give_sigsegv; + + + vb = ((unsigned long)va) + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[1]), "0"(0)); + + if (error) + goto give_sigsegv; + + return; + } + + case 0x0f: /* vstd */ + sw64_read_simd_fp_m_d(reg, fp); + if ((unsigned long)va << 61 == 0) { + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); + + if (error) + goto give_sigsegv; + + vb = ((unsigned long)va)+16; + + + __asm__ __volatile__( + " bis %4, %4, %1\n" + " bis %5, %5, %2\n" + "1: stl %1, 0(%3)\n" + "2: stl %2, 8(%3)\n" + "3:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(vb), "r"(fp[2]), "r"(fp[3]), "0"(0)); + + if (error) + goto give_sigsegv; + + return; + } else { + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(fp[0]), "0"(0)); + + if (error) + goto give_sigsegv; + + vb = ((unsigned long)va) + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[1]), "0"(0)); + + if (error) + goto give_sigsegv; + + vb = vb + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[2]), "0"(0)); + + if (error) + goto give_sigsegv; + + vb = vb + 8; + + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(vb), "r"(fp[3]), "0"(0)); + + if (error) + goto give_sigsegv; + + return; + } + } + switch (opcode) { + case 0x1e: + rb = (instr >> 16) & 0x1f; + disp = instr & 0xfff; + disp = (disp << 52) >> 52; + + switch (fncode) { + case 0x1: /* ldhu_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 1(%3)\n" + " extlh %1, %3, %1\n" + " exthh %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = tmp1 | tmp2; + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0x2: /* ldw_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = (int)(tmp1 | tmp2); + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0x3: /* ldl_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = tmp1 | tmp2; + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0x4: /* flds_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + sw64_write_fp_reg_s(reg, tmp1 | tmp2); + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0x5: /* fldd_a */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + sw64_write_fp_reg(reg, tmp1 | tmp2); + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0x7: /* sth_a */ + __asm__ __volatile__( + " zap %6, 2, %1\n" + " srl %6, 8, %2\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %2, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %1, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0xa: /* fsts_a */ + fake_reg = sw64_read_fp_reg_s(reg); + regs->regs[rb] = regs->regs[rb] + disp; + break; + /* fallthrough; */ + case 0x8: /* stw_a */ + __asm__ __volatile__( + " zapnot %6, 0x1, %1\n" + " srl %6, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %6, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %6, 24, %4\n" + " zapnot %4, 0x1, %4\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3: stb %3, 0x2(%5)\n" + "4: stb %4, 0x3(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi $31, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + regs->regs[rb] = regs->regs[rb] + disp; + break; + + case 0xb: /* fstd_a */ + fake_reg = sw64_read_fp_reg(reg); + regs->regs[rb] = regs->regs[rb] + disp; + break; + /* fallthrough; */ + case 0x9: /* stl_a */ + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + regs->regs[rb] = regs->regs[rb] + disp; + break; + } + break; + + case 0x21: /* ldhu */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 1(%3)\n" + " extlh %1, %3, %1\n" + " exthh %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = tmp1 | tmp2; + break; + + case 0x26: /* flds */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + sw64_write_fp_reg_s(reg, tmp1 | tmp2); + return; + + case 0x27: /* fldd */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + sw64_write_fp_reg(reg, tmp1 | tmp2); + return; + + case 0x22: /* ldw */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 3(%3)\n" + " extlw %1, %3, %1\n" + " exthw %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = (int)(tmp1 | tmp2); + break; + + case 0x23: /* ldl */ + __asm__ __volatile__( + "1: ldl_u %1, 0(%3)\n" + "2: ldl_u %2, 7(%3)\n" + " extll %1, %3, %1\n" + " exthl %2, %3, %2\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %1, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %2, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) + : "r"(va), "0"(0)); + if (error) + goto give_sigsegv; + *reg_addr = tmp1 | tmp2; + break; + + /* Note that the store sequences do not indicate that they change + * memory because it _should_ be affecting nothing in this context. + * (Otherwise we have other, much larger, problems.) + */ + case 0x29: /* sth with stb */ + __asm__ __volatile__( + " zap %6, 2, %1\n" + " srl %6, 8, %2\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi %2, 3b-1b(%0)\n" + " .long 2b - .\n" + " ldi %1, 3b-2b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + return; + + case 0x2e: /* fsts*/ + fake_reg = sw64_read_fp_reg_s(reg); + fallthrough; + + case 0x2a: /* stw with stb*/ + __asm__ __volatile__( + " zapnot %6, 0x1, %1\n" + " srl %6, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %6, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %6, 24, %4\n" + " zapnot %4, 0x1, %4\n" + "1: stb %1, 0x0(%5)\n" + "2: stb %2, 0x1(%5)\n" + "3: stb %3, 0x2(%5)\n" + "4: stb %4, 0x3(%5)\n" + "5:\n" + ".section __ex_table, \"a\"\n" + " .long 1b - .\n" + " ldi $31, 5b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 5b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 5b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 5b-4b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), + "=&r"(tmp3), "=&r"(tmp4) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + return; + + case 0x2f: /* fstd */ + fake_reg = sw64_read_fp_reg(reg); + fallthrough; + + case 0x2b: /* stl */ + __asm__ __volatile__( + " zapnot %10, 0x1, %1\n" + " srl %10, 8, %2\n" + " zapnot %2, 0x1, %2\n" + " srl %10, 16, %3\n" + " zapnot %3, 0x1, %3\n" + " srl %10, 24, %4\n" + " zapnot %4, 0x1, %4\n" + " srl %10, 32, %5\n" + " zapnot %5, 0x1, %5\n" + " srl %10, 40, %6\n" + " zapnot %6, 0x1, %6\n" + " srl %10, 48, %7\n" + " zapnot %7, 0x1, %7\n" + " srl %10, 56, %8\n" + " zapnot %8, 0x1, %8\n" + "1: stb %1, 0(%9)\n" + "2: stb %2, 1(%9)\n" + "3: stb %3, 2(%9)\n" + "4: stb %4, 3(%9)\n" + "5: stb %5, 4(%9)\n" + "6: stb %6, 5(%9)\n" + "7: stb %7, 6(%9)\n" + "8: stb %8, 7(%9)\n" + "9:\n" + ".section __ex_table, \"a\"\n\t" + " .long 1b - .\n" + " ldi $31, 9b-1b(%0)\n" + " .long 2b - .\n" + " ldi $31, 9b-2b(%0)\n" + " .long 3b - .\n" + " ldi $31, 9b-3b(%0)\n" + " .long 4b - .\n" + " ldi $31, 9b-4b(%0)\n" + " .long 5b - .\n" + " ldi $31, 9b-5b(%0)\n" + " .long 6b - .\n" + " ldi $31, 9b-6b(%0)\n" + " .long 7b - .\n" + " ldi $31, 9b-7b(%0)\n" + " .long 8b - .\n" + " ldi $31, 9b-8b(%0)\n" + ".previous" + : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), + "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) + : "r"(va), "r"(*reg_addr), "0"(0)); + + if (error) + goto give_sigsegv; + return; + + default: + /* What instruction were you trying to use, exactly? */ + goto give_sigbus; + } + + return; + +give_sigsegv: + regs->pc -= 4; /* make pc point to faulting insn */ + + /* We need to replicate some of the logic in mm/fault.c, + * since we don't have access to the fault code in the + * exception handling return path. + */ + if ((unsigned long)va >= TASK_SIZE) + si_code = SEGV_ACCERR; + else { + struct mm_struct *mm = current->mm; + + down_read(&mm->mmap_lock); + if (find_vma(mm, (unsigned long)va)) + si_code = SEGV_ACCERR; + else + si_code = SEGV_MAPERR; + up_read(&mm->mmap_lock); + } + force_sig_fault(SIGSEGV, si_code, va); + return; + +give_sigbus: + regs->pc -= 4; + force_sig_fault(SIGBUS, BUS_ADRALN, va); +} diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index b87363f35801..a84441c79b61 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -386,2133 +386,6 @@ asmlinkage void noinstr do_entIF(struct pt_regs *regs) force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->pc); } -#define OP_INT_MASK (1L << 0x22 | 1L << 0x2a | /* ldw stw */ \ - 1L << 0x23 | 1L << 0x2b | /* ldl stl */ \ - 1L << 0x21 | 1L << 0x29 | /* ldhu sth */ \ - 1L << 0x20 | 1L << 0x28) /* ldbu stb */ - -#define FN_INT_MASK (1L << 0x0 | 1L << 0x6 | /* ldbu_a stb_a */ \ - 1L << 0x1 | 1L << 0x7 | /* ldhu_a sth_a */ \ - 1L << 0x2 | 1L << 0x8 | /* ldw_a stw_a */ \ - 1L << 0x3 | 1L << 0x9) /* ldl_a stl_a */ - -asmlinkage void noinstr do_entUna(struct pt_regs *regs) -{ - long error, disp; - unsigned int insn, fncode, rb; - unsigned long tmp, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, vb; - unsigned long fp[4]; - unsigned long fake_reg, *reg_addr = &fake_reg; - unsigned long pc = regs->pc - 4; - void *va = (void *)regs->earg0; - unsigned long opcode = regs->earg1; - unsigned long reg = regs->earg2; - - if (reg == 29) - return; - - insn = *(unsigned int *)pc; - fncode = (insn >> 12) & 0xf; - - if (((1L << opcode) & OP_INT_MASK) || - ((opcode == 0x1e) && ((1L << fncode) & FN_INT_MASK))) { - /* it's an integer load/store */ - if (reg < 31) { - reg_addr = ®s->regs[reg]; - } else { - /* zero "register" */ - fake_reg = 0; - } - } - - /* - * We don't want to use the generic get/put unaligned macros as - * we want to trap exceptions. Only if we actually get an - * exception will we decide whether we should have caught it. - */ - - switch (opcode) { - - case 0x0c: /* vlds */ - if ((unsigned long)va<<61 == 0) { - __asm__ __volatile__( - "1: ldl %1, 0(%5)\n" - "2: ldl %2, 8(%5)\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "0"(0)); - - if (error) - goto got_exception; - - sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); - - return; - } else { - __asm__ __volatile__( - "1: ldl_u %1, 0(%6)\n" - "2: ldl_u %2, 7(%6)\n" - "3: ldl_u %3, 15(%6)\n" - " extll %1, %6, %1\n" - " extll %2, %6, %5\n" - " exthl %2, %6, %4\n" - " exthl %3, %6, %3\n" - "4:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 4b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 4b-2b(%0)\n" - " .long 3b - .\n" - " ldi %3, 4b-3b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5) - : "r"(va), "0"(0)); - - if (error) - goto got_exception; - - tmp1 = tmp1 | tmp4; - tmp2 = tmp5 | tmp3; - - sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); - - return; - } - - case 0x0d: /* vldd */ - if ((unsigned long)va<<61 == 0) { - __asm__ __volatile__( - "1: ldl %1, 0(%5)\n" - "2: ldl %2, 8(%5)\n" - "3: ldl %3, 16(%5)\n" - "4: ldl %4, 24(%5)\n" - "5:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 5b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 5b-2b(%0)\n" - " .long 3b - .\n" - " ldi %3, 5b-3b(%0)\n" - " .long 4b - .\n" - " ldi %4, 5b-4b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "0"(0)); - - if (error) - goto got_exception; - - sw64_write_simd_fp_reg_d(reg, tmp1, tmp2, tmp3, tmp4); - - return; - } else { - __asm__ __volatile__( - "1: ldl_u %1, 0(%6)\n" - "2: ldl_u %2, 7(%6)\n" - "3: ldl_u %3, 15(%6)\n" - " extll %1, %6, %1\n" - " extll %2, %6, %5\n" - " exthl %2, %6, %4\n" - " exthl %3, %6, %3\n" - "4:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 4b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 4b-2b(%0)\n" - " .long 3b - .\n" - " ldi %3, 4b-3b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5) - : "r"(va), "0"(0)); - - if (error) - goto got_exception; - - tmp7 = tmp1 | tmp4; //f0 - tmp8 = tmp5 | tmp3; //f1 - - vb = ((unsigned long)(va))+16; - - __asm__ __volatile__( - "1: ldl_u %1, 0(%6)\n" - "2: ldl_u %2, 7(%6)\n" - "3: ldl_u %3, 15(%6)\n" - " extll %1, %6, %1\n" - " extll %2, %6, %5\n" - " exthl %2, %6, %4\n" - " exthl %3, %6, %3\n" - "4:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 4b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 4b-2b(%0)\n" - " .long 3b - .\n" - " ldi %3, 4b-3b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5) - : "r"(vb), "0"(0)); - - if (error) - goto got_exception; - - tmp = tmp1 | tmp4; // f2 - tmp2 = tmp5 | tmp3; // f3 - - sw64_write_simd_fp_reg_d(reg, tmp7, tmp8, tmp, tmp2); - return; - } - - case 0x0e: /* vsts */ - sw64_read_simd_fp_m_s(reg, fp); - if ((unsigned long)va<<61 == 0) { - __asm__ __volatile__( - " bis %4, %4, %1\n" - " bis %5, %5, %2\n" - "1: stl %1, 0(%3)\n" - "2: stl %2, 8(%3)\n" - "3:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); - - if (error) - goto got_exception; - - return; - } else { - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(fp[0]), "0"(0)); - - if (error) - goto got_exception; - - - vb = ((unsigned long)va) + 8; - - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(vb), "r"(fp[1]), "0"(0)); - - if (error) - goto got_exception; - - return; - } - - case 0x0f: /* vstd */ - sw64_read_simd_fp_m_d(reg, fp); - if ((unsigned long)va<<61 == 0) { - __asm__ __volatile__( - " bis %4, %4, %1\n" - " bis %5, %5, %2\n" - "1: stl %1, 0(%3)\n" - "2: stl %2, 8(%3)\n" - "3:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); - - if (error) - goto got_exception; - - vb = ((unsigned long)va)+16; - - - __asm__ __volatile__( - " bis %4, %4, %1\n" - " bis %5, %5, %2\n" - "1: stl %1, 0(%3)\n" - "2: stl %2, 8(%3)\n" - "3:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(vb), "r"(fp[2]), "r"(fp[3]), "0"(0)); - - if (error) - goto got_exception; - - return; - } else { - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(fp[0]), "0"(0)); - - if (error) - goto got_exception; - - vb = ((unsigned long)va) + 8; - - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(vb), "r"(fp[1]), "0"(0)); - - if (error) - goto got_exception; - - vb = vb + 8; - - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(vb), "r"(fp[2]), "0"(0)); - - if (error) - goto got_exception; - - vb = vb + 8; - - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(vb), "r"(fp[3]), "0"(0)); - - if (error) - goto got_exception; - - return; - } - - case 0x1e: - insn = *(unsigned int *)pc; - fncode = (insn >> 12) & 0xf; - rb = (insn >> 16) & 0x1f; - disp = insn & 0xfff; - - disp = (disp << 52) >> 52; /* sext */ - - switch (fncode) { - case 0x1: /* ldhu_a */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 1(%3)\n" - " extlh %1, %3, %1\n" - " exthh %2, %3, %2\n" - "3:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto got_exception; - regs->regs[reg] = tmp1 | tmp2; - regs->regs[rb] = regs->regs[rb] + disp; - return; - - case 0x2: /* ldw_a */ - __asm__ __volatile__( - "1: ldl_u %1,0(%3)\n" - "2: ldl_u %2,3(%3)\n" - " extlw %1,%3,%1\n" - " exthw %2,%3,%2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - - if (error) - goto got_exception; - regs->regs[reg] = (int)(tmp1 | tmp2); - regs->regs[rb] = regs->regs[rb] + disp; - return; - - case 0x3: /* ldl_a */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 7(%3)\n" - " extll %1, %3, %1\n" - " exthl %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - - if (error) - goto got_exception; - regs->regs[reg] = tmp1 | tmp2; - regs->regs[rb] = regs->regs[rb] + disp; - return; - - case 0x7: /* sth_a */ - __asm__ __volatile__( - " zap %6, 2, %1\n" - " srl %6, 8, %2\n" - "1: stb %1, 0x0(%5)\n" - "2: stb %2, 0x1(%5)\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %2, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %1, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), - "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto got_exception; - regs->regs[rb] = regs->regs[rb] + disp; - return; - - case 0x8: /* stw_a */ - __asm__ __volatile__( - " zapnot %6, 0x1, %1\n" - " srl %6, 8, %2\n" - " zapnot %2, 0x1,%2\n" - " srl %6, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %6, 24, %4\n" - " zapnot %4, 0x1, %4\n" - "1: stb %1, 0x0(%5)\n" - "2: stb %2, 0x1(%5)\n" - "3: stb %3, 0x2(%5)\n" - "4: stb %4, 0x3(%5)\n" - "5:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi $31, 5b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 5b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 5b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 5b-4b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), - "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto got_exception; - regs->regs[rb] = regs->regs[rb] + disp; - return; - - case 0x9: /* stl_a */ - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto got_exception; - regs->regs[rb] = regs->regs[rb] + disp; - return; - } - return; - - case 0x21: - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 1(%3)\n" - " extlh %1, %3, %1\n" - " exthh %2, %3, %2\n" - "3:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - - if (error) - goto got_exception; - regs->regs[reg] = tmp1 | tmp2; - return; - - case 0x22: - __asm__ __volatile__( - "1: ldl_u %1,0(%3)\n" - "2: ldl_u %2,3(%3)\n" - " extlw %1,%3,%1\n" - " exthw %2,%3,%2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - - if (error) - goto got_exception; - regs->regs[reg] = (int)(tmp1 | tmp2); - return; - - case 0x23: /* ldl */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 7(%3)\n" - " extll %1, %3, %1\n" - " exthl %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - - if (error) - goto got_exception; - regs->regs[reg] = tmp1 | tmp2; - return; - - case 0x29: /* sth */ - __asm__ __volatile__( - " zap %6, 2, %1\n" - " srl %6, 8, %2\n" - "1: stb %1, 0x0(%5)\n" - "2: stb %2, 0x1(%5)\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %2, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %1, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), - "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto got_exception; - return; - - case 0x2a: /* stw */ - __asm__ __volatile__( - " zapnot %6, 0x1, %1\n" - " srl %6, 8, %2\n" - " zapnot %2, 0x1,%2\n" - " srl %6, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %6, 24, %4\n" - " zapnot %4, 0x1, %4\n" - "1: stb %1, 0x0(%5)\n" - "2: stb %2, 0x1(%5)\n" - "3: stb %3, 0x2(%5)\n" - "4: stb %4, 0x3(%5)\n" - "5:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi $31, 5b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 5b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 5b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 5b-4b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), - "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto got_exception; - return; - - case 0x2b: /* stl */ - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto got_exception; - return; - } - - pr_warn("Bad unaligned kernel access at %016lx: %p %lx %lu\n", - pc, va, opcode, reg); - make_task_dead(SIGSEGV); - -got_exception: - /* Ok, we caught the exception, but we don't want it. Is there - * someone to pass it along to? - */ - if (fixup_exception(regs, pc)) { - pr_info("Forwarding unaligned exception at %lx (%lx)\n", - pc, regs->pc); - return; - } - - /* - * Yikes! No one to forward the exception to. - * Since the registers are in a weird format, dump them ourselves. - */ - - die("Unhandled unaligned exception", regs, error); -} - -/* - * Handle user-level unaligned fault. Handling user-level unaligned - * faults is *extremely* slow and produces nasty messages. A user - * program *should* fix unaligned faults ASAP. - * - * Notice that we have (almost) the regular kernel stack layout here, - * so finding the appropriate registers is a little more difficult - * than in the kernel case. - * - * Finally, we handle regular integer load/stores only. In - * particular, load-linked/store-conditionally and floating point - * load/stores are not supported. The former make no sense with - * unaligned faults (they are guaranteed to fail) and I don't think - * the latter will occur in any decent program. - * - * Sigh. We *do* have to handle some FP operations, because GCC will - * uses them as temporary storage for integer memory to memory copies. - * However, we need to deal with stt/ldt and sts/lds only. - */ -asmlinkage void noinstr do_entUnaUser(struct pt_regs *regs) -{ -#ifdef CONFIG_UNA_PRINT - static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5); -#endif - - unsigned long tmp1, tmp2, tmp3, tmp4; - unsigned long fake_reg, *reg_addr = &fake_reg; - int si_code; - long error; - unsigned long tmp, tmp5, tmp6, tmp7, tmp8, vb; - unsigned long fp[4]; - unsigned long instr, instr_op, value, fncode; - unsigned int rb = -1U; - long disp; - void __user *va = (void *)regs->earg0; - unsigned long opcode = regs->earg1; - unsigned long reg = regs->earg2; - -#ifdef CONFIG_DEBUG_FS - /* - * If command name is specified, record some information - * to debugfs. - */ - if (unaligned_task[0] && !strcmp(unaligned_task, current->comm)) { - int idx; - - idx = unaligned_count % UNA_MAX_ENTRIES; - unaligned[idx].va = (unsigned long)va; - unaligned[idx].pc = regs->pc; - unaligned_count++; - } -#endif - - /* Check the UAC bits to decide what the user wants us to do - * with the unaliged access. - */ - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, - 1, regs, regs->pc - 4); - -#ifdef CONFIG_UNA_PRINT - if (!(current_thread_info()->status & TS_UAC_NOPRINT)) { - if (__ratelimit(&ratelimit)) { - pr_info("%s(%d): unaligned trap at %016lx: %p %lx %ld\n", - current->comm, task_pid_nr(current), - regs->pc - 4, va, opcode, reg); - } - } -#endif - if ((current_thread_info()->status & TS_UAC_SIGBUS)) - goto give_sigbus; - /* Not sure why you'd want to use this, but... */ - if ((current_thread_info()->status & TS_UAC_NOFIX)) - return; - - /* Don't bother reading ds in the access check since we already - * know that this came from the user. Also rely on the fact that - * the page at TASK_SIZE is unmapped and so can't be touched anyway. - */ - if ((unsigned long)va >= TASK_SIZE) - goto give_sigsegv; - - get_user(instr, (__u32 *)(regs->pc - 4)); - fncode = (instr >> 12) & 0xf; - - if (((1L << opcode) & OP_INT_MASK) || - ((opcode == 0x1e) && ((1L << fncode) & FN_INT_MASK))) { - /* it's an integer load/store */ - if (reg < 31) { - reg_addr = ®s->regs[reg]; - } else { - /* zero "register" */ - fake_reg = 0; - } - } - - get_user(instr, (__u32 *)(regs->pc - 4)); - instr_op = (instr >> 26) & 0x3f; - - get_user(value, (__u64 *)va); - - switch (instr_op) { - - case 0x0c: /* vlds */ - if ((unsigned long)va << 61 == 0) { - __asm__ __volatile__( - "1: ldl %1, 0(%5)\n" - "2: ldl %2, 8(%5)\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "0"(0)); - - if (error) - goto give_sigsegv; - - sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); - - return; - } else { - __asm__ __volatile__( - "1: ldl_u %1, 0(%6)\n" - "2: ldl_u %2, 7(%6)\n" - "3: ldl_u %3, 15(%6)\n" - " extll %1, %6, %1\n" - " extll %2, %6, %5\n" - " exthl %2, %6, %4\n" - " exthl %3, %6, %3\n" - "4:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 4b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 4b-2b(%0)\n" - " .long 3b - .\n" - " ldi %3, 4b-3b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5) - : "r"(va), "0"(0)); - - if (error) - goto give_sigsegv; - - tmp1 = tmp1 | tmp4; - tmp2 = tmp5 | tmp3; - - sw64_write_simd_fp_reg_s(reg, tmp1, tmp2); - - return; - } - case 0x0a: /* ldse */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 3(%3)\n" - " extlw %1, %3, %1\n" - " exthw %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - - if (error) - goto give_sigsegv; - - tmp = tmp1 | tmp2; - tmp = tmp | (tmp << 32); - - sw64_write_simd_fp_reg_s(reg, tmp, tmp); - - return; - - case 0x0d: /* vldd */ - if ((unsigned long)va << 61 == 0) { - __asm__ __volatile__( - "1: ldl %1, 0(%5)\n" - "2: ldl %2, 8(%5)\n" - "3: ldl %3, 16(%5)\n" - "4: ldl %4, 24(%5)\n" - "5:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 5b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 5b-2b(%0)\n" - " .long 3b - .\n" - " ldi %3, 5b-3b(%0)\n" - " .long 4b - .\n" - " ldi %4, 5b-4b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "0"(0)); - - if (error) - goto give_sigsegv; - - sw64_write_simd_fp_reg_d(reg, tmp1, tmp2, tmp3, tmp4); - - return; - } else { - __asm__ __volatile__( - "1: ldl_u %1, 0(%6)\n" - "2: ldl_u %2, 7(%6)\n" - "3: ldl_u %3, 15(%6)\n" - " extll %1, %6, %1\n" - " extll %2, %6, %5\n" - " exthl %2, %6, %4\n" - " exthl %3, %6, %3\n" - "4:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 4b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 4b-2b(%0)\n" - " .long 3b - .\n" - " ldi %3, 4b-3b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5) - : "r"(va), "0"(0)); - - if (error) - goto give_sigsegv; - - tmp7 = tmp1 | tmp4; //f0 - tmp8 = tmp5 | tmp3; //f1 - - vb = ((unsigned long)(va))+16; - - __asm__ __volatile__( - "1: ldl_u %1, 0(%6)\n" - "2: ldl_u %2, 7(%6)\n" - "3: ldl_u %3, 15(%6)\n" - " extll %1, %6, %1\n" - " extll %2, %6, %5\n" - " exthl %2, %6, %4\n" - " exthl %3, %6, %3\n" - "4:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 4b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 4b-2b(%0)\n" - " .long 3b - .\n" - " ldi %3, 4b-3b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5) - : "r"(vb), "0"(0)); - - if (error) - goto give_sigsegv; - - tmp = tmp1 | tmp4; // f2 - tmp2 = tmp5 | tmp3; // f3 - - sw64_write_simd_fp_reg_d(reg, tmp7, tmp8, tmp, tmp2); - return; - } - - case 0x0b: /* ldde */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 7(%3)\n" - " extll %1, %3, %1\n" - " exthl %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - - if (error) - goto give_sigsegv; - - tmp = tmp1 | tmp2; - - sw64_write_simd_fp_reg_d(reg, tmp, tmp, tmp, tmp); - return; - - case 0x09: /* ldwe */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 3(%3)\n" - " extlw %1, %3, %1\n" - " exthw %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - - if (error) - goto give_sigsegv; - - sw64_write_simd_fp_reg_ldwe(reg, (int)(tmp1 | tmp2)); - - return; - - case 0x0e: /* vsts */ - sw64_read_simd_fp_m_s(reg, fp); - if ((unsigned long)va << 61 == 0) { - __asm__ __volatile__( - " bis %4, %4, %1\n" - " bis %5, %5, %2\n" - "1: stl %1, 0(%3)\n" - "2: stl %2, 8(%3)\n" - "3:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); - - if (error) - goto give_sigsegv; - - return; - } else { - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(fp[0]), "0"(0)); - - if (error) - goto give_sigsegv; - - - vb = ((unsigned long)va) + 8; - - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(vb), "r"(fp[1]), "0"(0)); - - if (error) - goto give_sigsegv; - - return; - } - - case 0x0f: /* vstd */ - sw64_read_simd_fp_m_d(reg, fp); - if ((unsigned long)va << 61 == 0) { - __asm__ __volatile__( - " bis %4, %4, %1\n" - " bis %5, %5, %2\n" - "1: stl %1, 0(%3)\n" - "2: stl %2, 8(%3)\n" - "3:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "r"(fp[0]), "r"(fp[1]), "0"(0)); - - if (error) - goto give_sigsegv; - - vb = ((unsigned long)va)+16; - - - __asm__ __volatile__( - " bis %4, %4, %1\n" - " bis %5, %5, %2\n" - "1: stl %1, 0(%3)\n" - "2: stl %2, 8(%3)\n" - "3:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(vb), "r"(fp[2]), "r"(fp[3]), "0"(0)); - - if (error) - goto give_sigsegv; - - return; - } else { - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(fp[0]), "0"(0)); - - if (error) - goto give_sigsegv; - - vb = ((unsigned long)va) + 8; - - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(vb), "r"(fp[1]), "0"(0)); - - if (error) - goto give_sigsegv; - - vb = vb + 8; - - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(vb), "r"(fp[2]), "0"(0)); - - if (error) - goto give_sigsegv; - - vb = vb + 8; - - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(vb), "r"(fp[3]), "0"(0)); - - if (error) - goto give_sigsegv; - - return; - } - } - switch (opcode) { - case 0x1e: - rb = (instr >> 16) & 0x1f; - disp = instr & 0xfff; - disp = (disp << 52) >> 52; - - switch (fncode) { - case 0x1: /* ldhu_a */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 1(%3)\n" - " extlh %1, %3, %1\n" - " exthh %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto give_sigsegv; - *reg_addr = tmp1 | tmp2; - regs->regs[rb] = regs->regs[rb] + disp; - break; - - case 0x2: /* ldw_a */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 3(%3)\n" - " extlw %1, %3, %1\n" - " exthw %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto give_sigsegv; - *reg_addr = (int)(tmp1 | tmp2); - regs->regs[rb] = regs->regs[rb] + disp; - break; - - case 0x3: /* ldl_a */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 7(%3)\n" - " extll %1, %3, %1\n" - " exthl %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto give_sigsegv; - *reg_addr = tmp1 | tmp2; - regs->regs[rb] = regs->regs[rb] + disp; - break; - - case 0x4: /* flds_a */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 3(%3)\n" - " extlw %1, %3, %1\n" - " exthw %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto give_sigsegv; - sw64_write_fp_reg_s(reg, tmp1 | tmp2); - regs->regs[rb] = regs->regs[rb] + disp; - break; - - case 0x5: /* fldd_a */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 7(%3)\n" - " extll %1, %3, %1\n" - " exthl %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto give_sigsegv; - sw64_write_fp_reg(reg, tmp1 | tmp2); - regs->regs[rb] = regs->regs[rb] + disp; - break; - - case 0x7: /* sth_a */ - __asm__ __volatile__( - " zap %6, 2, %1\n" - " srl %6, 8, %2\n" - "1: stb %1, 0x0(%5)\n" - "2: stb %2, 0x1(%5)\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %2, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %1, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), - "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto give_sigsegv; - regs->regs[rb] = regs->regs[rb] + disp; - break; - - case 0xa: /* fsts_a */ - fake_reg = sw64_read_fp_reg_s(reg); - regs->regs[rb] = regs->regs[rb] + disp; - break; - /* fallthrough; */ - case 0x8: /* stw_a */ - __asm__ __volatile__( - " zapnot %6, 0x1, %1\n" - " srl %6, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %6, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %6, 24, %4\n" - " zapnot %4, 0x1, %4\n" - "1: stb %1, 0x0(%5)\n" - "2: stb %2, 0x1(%5)\n" - "3: stb %3, 0x2(%5)\n" - "4: stb %4, 0x3(%5)\n" - "5:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi $31, 5b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 5b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 5b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 5b-4b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), - "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto give_sigsegv; - regs->regs[rb] = regs->regs[rb] + disp; - break; - - case 0xb: /* fstd_a */ - fake_reg = sw64_read_fp_reg(reg); - regs->regs[rb] = regs->regs[rb] + disp; - break; - /* fallthrough; */ - case 0x9: /* stl_a */ - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto give_sigsegv; - regs->regs[rb] = regs->regs[rb] + disp; - break; - } - break; - - case 0x21: /* ldhu */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 1(%3)\n" - " extlh %1, %3, %1\n" - " exthh %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto give_sigsegv; - *reg_addr = tmp1 | tmp2; - break; - - case 0x26: /* flds */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 3(%3)\n" - " extlw %1, %3, %1\n" - " exthw %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto give_sigsegv; - sw64_write_fp_reg_s(reg, tmp1 | tmp2); - return; - - case 0x27: /* fldd */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 7(%3)\n" - " extll %1, %3, %1\n" - " exthl %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto give_sigsegv; - sw64_write_fp_reg(reg, tmp1 | tmp2); - return; - - case 0x22: /* ldw */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 3(%3)\n" - " extlw %1, %3, %1\n" - " exthw %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto give_sigsegv; - *reg_addr = (int)(tmp1 | tmp2); - break; - - case 0x23: /* ldl */ - __asm__ __volatile__( - "1: ldl_u %1, 0(%3)\n" - "2: ldl_u %2, 7(%3)\n" - " extll %1, %3, %1\n" - " exthl %2, %3, %2\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %1, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %2, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) - : "r"(va), "0"(0)); - if (error) - goto give_sigsegv; - *reg_addr = tmp1 | tmp2; - break; - - /* Note that the store sequences do not indicate that they change - * memory because it _should_ be affecting nothing in this context. - * (Otherwise we have other, much larger, problems.) - */ - case 0x29: /* sth with stb */ - __asm__ __volatile__( - " zap %6, 2, %1\n" - " srl %6, 8, %2\n" - "1: stb %1, 0x0(%5)\n" - "2: stb %2, 0x1(%5)\n" - "3:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi %2, 3b-1b(%0)\n" - " .long 2b - .\n" - " ldi %1, 3b-2b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), - "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto give_sigsegv; - return; - - case 0x2e: /* fsts*/ - fake_reg = sw64_read_fp_reg_s(reg); - fallthrough; - - case 0x2a: /* stw with stb*/ - __asm__ __volatile__( - " zapnot %6, 0x1, %1\n" - " srl %6, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %6, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %6, 24, %4\n" - " zapnot %4, 0x1, %4\n" - "1: stb %1, 0x0(%5)\n" - "2: stb %2, 0x1(%5)\n" - "3: stb %3, 0x2(%5)\n" - "4: stb %4, 0x3(%5)\n" - "5:\n" - ".section __ex_table, \"a\"\n" - " .long 1b - .\n" - " ldi $31, 5b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 5b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 5b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 5b-4b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), - "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto give_sigsegv; - return; - - case 0x2f: /* fstd */ - fake_reg = sw64_read_fp_reg(reg); - fallthrough; - - case 0x2b: /* stl */ - __asm__ __volatile__( - " zapnot %10, 0x1, %1\n" - " srl %10, 8, %2\n" - " zapnot %2, 0x1, %2\n" - " srl %10, 16, %3\n" - " zapnot %3, 0x1, %3\n" - " srl %10, 24, %4\n" - " zapnot %4, 0x1, %4\n" - " srl %10, 32, %5\n" - " zapnot %5, 0x1, %5\n" - " srl %10, 40, %6\n" - " zapnot %6, 0x1, %6\n" - " srl %10, 48, %7\n" - " zapnot %7, 0x1, %7\n" - " srl %10, 56, %8\n" - " zapnot %8, 0x1, %8\n" - "1: stb %1, 0(%9)\n" - "2: stb %2, 1(%9)\n" - "3: stb %3, 2(%9)\n" - "4: stb %4, 3(%9)\n" - "5: stb %5, 4(%9)\n" - "6: stb %6, 5(%9)\n" - "7: stb %7, 6(%9)\n" - "8: stb %8, 7(%9)\n" - "9:\n" - ".section __ex_table, \"a\"\n\t" - " .long 1b - .\n" - " ldi $31, 9b-1b(%0)\n" - " .long 2b - .\n" - " ldi $31, 9b-2b(%0)\n" - " .long 3b - .\n" - " ldi $31, 9b-3b(%0)\n" - " .long 4b - .\n" - " ldi $31, 9b-4b(%0)\n" - " .long 5b - .\n" - " ldi $31, 9b-5b(%0)\n" - " .long 6b - .\n" - " ldi $31, 9b-6b(%0)\n" - " .long 7b - .\n" - " ldi $31, 9b-7b(%0)\n" - " .long 8b - .\n" - " ldi $31, 9b-8b(%0)\n" - ".previous" - : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), - "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(*reg_addr), "0"(0)); - - if (error) - goto give_sigsegv; - return; - - default: - /* What instruction were you trying to use, exactly? */ - goto give_sigbus; - } - - return; - -give_sigsegv: - regs->pc -= 4; /* make pc point to faulting insn */ - - /* We need to replicate some of the logic in mm/fault.c, - * since we don't have access to the fault code in the - * exception handling return path. - */ - if ((unsigned long)va >= TASK_SIZE) - si_code = SEGV_ACCERR; - else { - struct mm_struct *mm = current->mm; - - down_read(&mm->mmap_lock); - if (find_vma(mm, (unsigned long)va)) - si_code = SEGV_ACCERR; - else - si_code = SEGV_MAPERR; - up_read(&mm->mmap_lock); - } - force_sig_fault(SIGSEGV, si_code, va); - return; - -give_sigbus: - regs->pc -= 4; - force_sig_fault(SIGBUS, BUS_ADRALN, va); -} - asmlinkage void noinstr do_entSys(struct pt_regs *regs) { long ret = -ENOSYS; -- Gitee From e71fb9fef7540d710ef6085f1e25366543cb20f4 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Thu, 20 Mar 2025 15:18:01 +0800 Subject: [PATCH 437/524] sw64: ftrace: save&restore more stack frame in ftrace_caller When stack trace is performed on the target function "A", it calls to ftrace_caller at the beginning of the function and then to stace_trace_call(). However, in this process we didn't save the stack ftame of "A", nor did we save the stack frame of the function "B" that calls "A". This will cause the result of stack_trace to be incorrect. This patch fixes above by pushing and popping stack frames of "A" & "B" at the beginning of mcount_enter and the end of mcount_exit. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry-ftrace.S | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S index 01e7cb4f9d22..75d604c14db4 100644 --- a/arch/sw_64/kernel/entry-ftrace.S +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -19,6 +19,19 @@ #define FTRACE_SP_OFF 0x50 .macro mcount_enter + /* save $26 & fp of the function before caller */ + subl $sp, 0x10, $sp + stl $26, 0($sp) + stl $15, 0x8($sp) + ldi $15, 0($sp) + + /* save $28 & fp of caller */ + subl $sp, 0x10, $sp + stl $28, 0($sp) + stl $15, 0x8($sp) + ldi $15, 0($sp) + + /* save other regs */ subl $sp, FTRACE_SP_OFF, $sp stl $16, 0($sp) stl $17, 0x8($sp) @@ -35,6 +48,7 @@ .endm .macro mcount_end + /* restore all regs */ ldl $16, 0($sp) ldl $17, 0x8($sp) ldl $18, 0x10($sp) @@ -48,6 +62,11 @@ ldl $20, 0x40($sp) ldl $21, 0x48($sp) addl $sp, FTRACE_SP_OFF, $sp + + ldl $28, 0($sp) + ldl $26, 0x10($sp) + ldl $15, 0x18($sp) + addl $sp, 0x20, $sp .endm .macro RESTORE_GRAPH_ARGS @@ -59,6 +78,19 @@ .endm .macro SAVE_PT_REGS + /* save $26 & fp of the function before caller */ + subl $sp, 0x10, $sp + stl $26, 0($sp) + stl $15, 0x8($sp) + ldi $15, 0($sp) + + /* save $28 & fp of caller */ + subl $sp, 0x10, $sp + stl $28, 0($sp) + stl $15, 0x8($sp) + ldi $15, 0($sp) + + /* save pt_regs */ ldi $sp, -PT_REGS_SIZE($sp) stl $0, PT_REGS_R0($sp) stl $1, PT_REGS_R1($sp) @@ -95,6 +127,7 @@ .endm .macro RESTORE_PT_REGS + /* restore pt_regs */ ldl $0, PT_REGS_R0($sp) ldl $1, PT_REGS_R1($sp) ldl $2, PT_REGS_R2($sp) @@ -126,6 +159,12 @@ ldl $28, PT_REGS_R28($sp) ldl $29, PT_REGS_GP($sp) ldi $sp, PT_REGS_SIZE($sp) + + /* restore $28 & $26 & fp */ + ldl $28, 0($sp) + ldl $26, 0x10($sp) + ldl $15, 0x18($sp) + addl $sp, 0x20, $sp .endm .macro RESTORE_GRAPH_REG_ARGS -- Gitee From 1e86da90bc7918e813be92b609298b07e4a26967 Mon Sep 17 00:00:00 2001 From: Deng Xiaoyun Date: Fri, 28 Mar 2025 11:44:26 +0800 Subject: [PATCH 438/524] sw64: bpf: add ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE to Kconfig According to the upstream community, adding this configuration enables the kernel to support the probe_read and probe_read_str helper functions. Signed-off-by: Deng Xiaoyun Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 3681e9550e60..bcb57e532105 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -12,6 +12,7 @@ config SW64 select ARCH_CLOCKSOURCE_INIT select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_ELF_RANDOMIZE + select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE select ARCH_HAS_PHYS_TO_DMA select ARCH_HAS_PMEM_API select ARCH_HAS_PTE_DEVMAP -- Gitee From 22aed5422693611e8244e31e3a28df3143ef7950 Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Fri, 21 Mar 2025 08:44:24 +0800 Subject: [PATCH 439/524] sw64: kexec: add support for crashkernel=size[KMG] Originally, the crash_base was restricted to above 4G. However, this would cause some devices using dma32 to fail to reset properly when the second kernle starts. To solve this problem, memblock_find_in_range() is added to automatically search for space below 4G memory. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/machine_kexec.c | 32 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/arch/sw_64/kernel/machine_kexec.c b/arch/sw_64/kernel/machine_kexec.c index 4ef59d6022fc..5d54e7cf6823 100644 --- a/arch/sw_64/kernel/machine_kexec.c +++ b/arch/sw_64/kernel/machine_kexec.c @@ -80,19 +80,24 @@ void __init reserve_crashkernel(void) pr_warn("size of crash kernel memory unspecified, no memory reserved for crash kernel\n"); return; } - if (!crash_base) { - pr_warn("base of crash kernel memory unspecified, no memory reserved for crash kernel\n"); - return; - } - if (!memblock_is_region_memory(crash_base, crash_size)) - memblock_add(crash_base, crash_size); - - ret = memblock_reserve(crash_base, crash_size); - if (ret < 0) { - pr_warn("crashkernel reservation failed - memory is in use [mem %#018llx-%#018llx]\n", - crash_base, crash_base + crash_size - 1); - return; + if (crash_base == 0) { + crash_base = memblock_phys_alloc_range(crash_size, SZ_8M, 0, SZ_4G); + if (crash_base == 0) { + pr_warn("cannot allocate crashkernel (size:0x%llx)\n", + crash_size); + return; + } + } else { + if (!memblock_is_region_memory(crash_base, crash_size)) + memblock_add(crash_base, crash_size); + + ret = memblock_reserve(crash_base, crash_size); + if (ret < 0) { + pr_warn("crashkernel reservation failed - memory is in use [mem %#018llx-%#018llx]\n", + crash_base, crash_base + crash_size - 1); + return; + } } pr_info("Reserving %ldMB of memory at %ldMB for crashkernel (System RAM: %ldMB)\n", @@ -105,9 +110,6 @@ void __init reserve_crashkernel(void) pr_warn("Add crash kernel area [mem %#018llx-%#018llx] to memmap region failed.\n", crash_base, crash_base + crash_size - 1); - if (crash_base < PCI_LEGACY_IO_SIZE) - pr_warn("Crash base should be greater than or equal to %#lx\n", PCI_LEGACY_IO_SIZE); - crashk_res.start = crash_base; crashk_res.end = crash_base + crash_size - 1; insert_resource(&iomem_resource, &crashk_res); -- Gitee From bd7e3fda45f070f080360c16f4219cfcc8184823 Mon Sep 17 00:00:00 2001 From: Si Ye Date: Wed, 12 Mar 2025 06:24:14 +0000 Subject: [PATCH 440/524] sw64: sound: fix compile error when CONFIG_SND_DUMMY=y In a previous commit, #include was added inside a function, which caused compilation errors. To resolve this issue, the header file has been moved to the beginning of the file to ensure successful compilation. This change does not affect functionality and is purely a fix for the compilation error. Fixes: ce16ff1a7368 ("sw64: sound: avoid pagefault in pcm driver's memset") Signed-off-by: Si Ye Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- sound/core/pcm_misc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sound/core/pcm_misc.c b/sound/core/pcm_misc.c index a8061ce9479d..b4f59cc2900a 100644 --- a/sound/core/pcm_misc.c +++ b/sound/core/pcm_misc.c @@ -23,6 +23,9 @@ #include #include #include +#ifdef CONFIG_SW64 +#include +#endif #include "pcm_local.h" @@ -439,7 +442,6 @@ int snd_pcm_format_set_silence(snd_pcm_format_t format, void *data, unsigned int if (pcm_formats[(INT)format].signd == 1 || width <= 8) { unsigned int bytes = samples * width / 8; #ifdef CONFIG_SW64 -#include memset_io(data, *pat, bytes); #else memset(data, *pat, bytes); -- Gitee From 6ab3880c343045ed1eba7f761869a751976055bc Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Mon, 24 Mar 2025 17:29:51 +0800 Subject: [PATCH 441/524] sw64: locking/atomic: make atomic*_{cmp,}xchg optional The fallback of atomic*_{cmp,}xchg has been added to the generic code in include/linux/atomic/atomic-arch-fallback.h. Remove macro definitions of atomic*_{cmp,}xchg in the architecture code. More details can be seen in commit id = d12157efc8e0. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/atomic.h | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/arch/sw_64/include/asm/atomic.h b/arch/sw_64/include/asm/atomic.h index 4a68da09722c..73a6b7c9b76e 100644 --- a/arch/sw_64/include/asm/atomic.h +++ b/arch/sw_64/include/asm/atomic.h @@ -23,18 +23,6 @@ #define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i)) #define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i)) -/* - * To get proper branch prediction for the main line, we must branch - * forward to code at the end of this object's .text section, then - * branch back to restart the operation. - */ -#define arch_atomic64_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), old, new)) -#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new)) - -#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), old, new)) -#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) - - #ifdef CONFIG_SUBARCH_C3B /** * arch_atomic_fetch_add_unless - add unless the number is a given value -- Gitee From 9ff031a20872abf9b0b9ae7b770560b0e2b1bec6 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 4 Mar 2025 09:34:21 +0800 Subject: [PATCH 442/524] sw64: fix perf event L1I cache write support L1I cache is not writable on SW64. Set L1I cache write event to unsupported. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/perf_event_c4.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/kernel/perf_event_c4.c b/arch/sw_64/kernel/perf_event_c4.c index b5a9c8363800..462394e78887 100644 --- a/arch/sw_64/kernel/perf_event_c4.c +++ b/arch/sw_64/kernel/perf_event_c4.c @@ -101,8 +101,8 @@ static const int core4_cache_event_map [C(RESULT_MISS)] = SW64_L1I_CACHE_MISSES, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = SW64_L1I_CACHE, - [C(RESULT_MISS)] = SW64_L1I_CACHE_MISSES, + [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_OP_UNSUPP, }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, -- Gitee From 3fcde1af16dc70f63339ec19c0520e90fc91d7f3 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 10 Dec 2024 17:32:46 +0800 Subject: [PATCH 443/524] sw64: update exclusive counter handling Modify arch perf event interface so that hardware PMUs that have exclusive counters does not occupy the generic counter. All macros for hardware PMUs now use SW64_PMU_ as prefix to avoid ambiguity. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/csr.h | 2 + arch/sw_64/include/asm/perf_event.h | 4 +- arch/sw_64/include/asm/pmc.h | 3 +- arch/sw_64/include/asm/pmc_c4.h | 27 ++-- arch/sw_64/kernel/perf_event.c | 4 +- arch/sw_64/kernel/perf_event_c4.c | 236 +++++++++++++++++----------- 6 files changed, 165 insertions(+), 111 deletions(-) diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h index f88e2bd36d9f..39688acbda36 100644 --- a/arch/sw_64/include/asm/csr.h +++ b/arch/sw_64/include/asm/csr.h @@ -32,6 +32,8 @@ #define CSR_IDR_PCCTL 0x7a #define CSR_IACC 0x7b #define CSR_IMISC 0x7c +#define CSR_DACC 0x7d +#define CSR_DMISC 0x7e #define CSR_RETIC 0x7f #define CSR_CID 0xc4 #define CSR_WR_FREGS 0xc8 diff --git a/arch/sw_64/include/asm/perf_event.h b/arch/sw_64/include/asm/perf_event.h index a92545694c65..6d8a8cf2f9c8 100644 --- a/arch/sw_64/include/asm/perf_event.h +++ b/arch/sw_64/include/asm/perf_event.h @@ -25,9 +25,9 @@ struct cpu_hw_events { * Set the bit (indexed by the counter number) when the counter * is used for an event. */ - unsigned long used_mask[BITS_TO_LONGS(MAX_HWEVENTS)]; + unsigned long used_mask[BITS_TO_LONGS(PMU_NUM_GENERIC_COUNTERS)]; /* Array of events current scheduled on this cpu. */ - struct perf_event *event[MAX_HWEVENTS]; + struct perf_event *event[PMU_NUM_GENERIC_COUNTERS + PMU_NUM_EXCLUSIVE_COUNTERS]; }; #endif /* _ASM_SW64_PERF_EVENT_H */ diff --git a/arch/sw_64/include/asm/pmc.h b/arch/sw_64/include/asm/pmc.h index d5672dd940a7..a3a22945e095 100644 --- a/arch/sw_64/include/asm/pmc.h +++ b/arch/sw_64/include/asm/pmc.h @@ -49,7 +49,8 @@ #define PC1_DTB_SINGLE_MISSES 0x30 #define PC1_DCACHE_MISSES 0x32 -#define MAX_HWEVENTS 2 +#define PMU_NUM_GENERIC_COUNTERS 2 #define PMC_COUNT_MASK ((1UL << 58) - 1) +#define PMU_NUM_EXCLUSIVE_COUNTERS 5 #endif /* _ASM_SW64_PMC_H */ diff --git a/arch/sw_64/include/asm/pmc_c4.h b/arch/sw_64/include/asm/pmc_c4.h index cdee1041ce3b..de813dcacc55 100644 --- a/arch/sw_64/include/asm/pmc_c4.h +++ b/arch/sw_64/include/asm/pmc_c4.h @@ -26,23 +26,30 @@ #define SW64_PERFCTRL_UM 0x7 /* pc0-4 events */ -#define SW64_PMU_INSTRUCTIONS 0x3 -#define SW64_PMU_BRANCH 0x4 -#define SW64_PMU_BRANCH_MISSES 0x5 -#define SW64_L1I_CACHE 0x6 -#define SW64_L1I_CACHE_MISSES 0x7 #define SW64_PMU_CYCLE 0x30 -#define SW64_DTB 0x31 -#define SW64_DTB_MISSES 0x32 -#define SW64_L1D_CACHE 0x3D -#define SW64_L1D_CACHE_MISSES 0x3E +#define SW64_PMU_DTB 0x31 +#define SW64_PMU_DTB_MISSES 0x32 +#define SW64_PMU_L1D_CACHE 0x3D +#define SW64_PMU_L1D_CACHE_MISSES 0x3E #define SW64_PMU_L2_REFERENCES 0x50 #define SW64_PMU_L2_MISSES 0x53 +/* these events has exclusive counters */ +#define SW64_PMU_EXCLUSIVE_INSTRUCTIONS 0x400 +#define SW64_PMU_EXCLUSIVE_L1I_CACHE 0x401 +#define SW64_PMU_EXCLUSIVE_L1I_CACHE_MISSES 0x402 +#define SW64_PMU_EXCLUSIVE_L1D_CACHE 0x403 +#define SW64_PMU_EXCLUSIVE_L1D_CACHE_MISSES 0x404 +#define SW64_PMU_EXCLUSIVE_BRANCH 0x405 +#define SW64_PMU_EXCLUSIVE_BRANCH_MISSES 0x406 + #define PC_ALL_PM_SET 3 -#define MAX_HWEVENTS 5 +#define PMU_NUM_GENERIC_COUNTERS 5 +#define PMU_NUM_EXCLUSIVE_COUNTERS 7 #define PMC_COUNT_MASK (-1UL) +#define DACC_EN 0x1 +#define DMISC_EN 0x2 #define IACC_EN 0x4 #define IMISC_EN 0x8 #define RETIC_EN 0x10 diff --git a/arch/sw_64/kernel/perf_event.c b/arch/sw_64/kernel/perf_event.c index c0aede9321b4..ae02878a0414 100644 --- a/arch/sw_64/kernel/perf_event.c +++ b/arch/sw_64/kernel/perf_event.c @@ -42,7 +42,7 @@ struct sw64_pmu_t { * All PMC counters reside in the IBOX register PCTR. This is the * LSB of the counter. */ - int pmc_count_shift[MAX_HWEVENTS]; + int pmc_count_shift[PMU_NUM_GENERIC_COUNTERS]; /* * The mask that isolates the PMC bits when the LSB of the counter @@ -244,7 +244,7 @@ static const struct sw64_pmu_t core3_pmu = { .map_hw_event = core3_map_hw_event, .cache_events = &core3_cache_event_map, .map_cache_event = core3_map_cache_event, - .num_pmcs = MAX_HWEVENTS, + .num_pmcs = PMU_NUM_GENERIC_COUNTERS, .pmc_count_mask = PMC_COUNT_MASK, .pmc_max_period = PMC_COUNT_MASK, .pmc_left = 4, diff --git a/arch/sw_64/kernel/perf_event_c4.c b/arch/sw_64/kernel/perf_event_c4.c index 462394e78887..c1d2c7872428 100644 --- a/arch/sw_64/kernel/perf_event_c4.c +++ b/arch/sw_64/kernel/perf_event_c4.c @@ -63,16 +63,16 @@ static const struct sw64_pmu_t *sw64_pmu; * actual codes that are used to program the PMCs hence we introduce our * own hw event type identifiers. */ -#define SW64_OP_UNSUPP (-EOPNOTSUPP) +#define SW64_PMU_OP_UNSUPP (-EOPNOTSUPP) /* Mapping of the hw event types to the perf tool interface */ static const int core4_hw_event_map[] = { [PERF_COUNT_HW_CPU_CYCLES] = SW64_PMU_CYCLE, - [PERF_COUNT_HW_INSTRUCTIONS] = SW64_PMU_INSTRUCTIONS, - [PERF_COUNT_HW_CACHE_REFERENCES] = SW64_PMU_L2_REFERENCES, - [PERF_COUNT_HW_CACHE_MISSES] = SW64_PMU_L2_MISSES, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = SW64_PMU_BRANCH, - [PERF_COUNT_HW_BRANCH_MISSES] = SW64_PMU_BRANCH_MISSES, + [PERF_COUNT_HW_INSTRUCTIONS] = SW64_PMU_EXCLUSIVE_INSTRUCTIONS, + [PERF_COUNT_HW_CACHE_REFERENCES] = SW64_PMU_L1D_CACHE, + [PERF_COUNT_HW_CACHE_MISSES] = SW64_PMU_L1D_CACHE_MISSES, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = SW64_PMU_EXCLUSIVE_BRANCH, + [PERF_COUNT_HW_BRANCH_MISSES] = SW64_PMU_EXCLUSIVE_BRANCH_MISSES, }; /* Mapping of the hw cache event types to the perf tool interface */ @@ -83,105 +83,121 @@ static const int core4_cache_event_map [PERF_COUNT_HW_CACHE_RESULT_MAX] = { [C(L1D)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = SW64_L1D_CACHE, - [C(RESULT_MISS)] = SW64_L1D_CACHE_MISSES, + [C(RESULT_ACCESS)] = SW64_PMU_L1D_CACHE, + [C(RESULT_MISS)] = SW64_PMU_L1D_CACHE_MISSES, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = SW64_L1D_CACHE, - [C(RESULT_MISS)] = SW64_L1D_CACHE_MISSES, + [C(RESULT_ACCESS)] = SW64_PMU_L1D_CACHE, + [C(RESULT_MISS)] = SW64_PMU_L1D_CACHE_MISSES, }, [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, }, [C(L1I)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = SW64_L1I_CACHE, - [C(RESULT_MISS)] = SW64_L1I_CACHE_MISSES, + [C(RESULT_ACCESS)] = SW64_PMU_EXCLUSIVE_L1I_CACHE, + [C(RESULT_MISS)] = SW64_PMU_EXCLUSIVE_L1I_CACHE_MISSES, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, }, [C(LL)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, }, [C(DTLB)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = SW64_DTB, - [C(RESULT_MISS)] = SW64_DTB_MISSES, + [C(RESULT_ACCESS)] = SW64_PMU_DTB, + [C(RESULT_MISS)] = SW64_PMU_DTB_MISSES, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = SW64_DTB, - [C(RESULT_MISS)] = SW64_DTB_MISSES, + [C(RESULT_ACCESS)] = SW64_PMU_DTB, + [C(RESULT_MISS)] = SW64_PMU_DTB_MISSES, }, [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, }, [C(ITLB)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, }, [C(BPU)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, }, [C(NODE)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = SW64_OP_UNSUPP, - [C(RESULT_MISS)] = SW64_OP_UNSUPP, + [C(RESULT_ACCESS)] = SW64_PMU_OP_UNSUPP, + [C(RESULT_MISS)] = SW64_PMU_OP_UNSUPP, }, }, }; +static inline int event_has_exclusive_counter(int event) +{ + return event == SW64_PMU_EXCLUSIVE_INSTRUCTIONS || + event == SW64_PMU_EXCLUSIVE_L1I_CACHE || + event == SW64_PMU_EXCLUSIVE_L1I_CACHE_MISSES || + event == SW64_PMU_EXCLUSIVE_L1D_CACHE || + event == SW64_PMU_EXCLUSIVE_L1D_CACHE_MISSES || + event == SW64_PMU_EXCLUSIVE_BRANCH || + event == SW64_PMU_EXCLUSIVE_BRANCH_MISSES; +} + +static inline int event_support_sampling(int event) +{ + return !event_has_exclusive_counter(event); +} + static const int core4_map_hw_event(u64 config) { return core4_hw_event_map[config]; @@ -219,7 +235,10 @@ static bool core4_raw_event_valid(u64 config) int idx = config >> 8; int event = config & 0xff; - if (idx >= 0 && idx < MAX_HWEVENTS && + if (event_has_exclusive_counter(event)) + return true; + + if (idx >= 0 && idx < PMU_NUM_GENERIC_COUNTERS && event >= PC_RAW_BASE && event <= (PC_RAW_BASE + PC_MAX)) return true; @@ -232,7 +251,7 @@ static const struct sw64_pmu_t core4_pmu = { .map_hw_event = core4_map_hw_event, .cache_events = &core4_cache_event_map, .map_cache_event = core4_map_cache_event, - .num_pmcs = MAX_HWEVENTS, + .num_pmcs = PMU_NUM_GENERIC_COUNTERS, .pmc_count_mask = PMC_COUNT_MASK, .pmc_max_period = PMC_COUNT_MASK, .raw_event_valid = core4_raw_event_valid, @@ -245,7 +264,10 @@ static int sw64_perf_event_set_period(struct perf_event *event) long left = local64_read(&hwc->period_left); long period = hwc->sample_period; int overflow = 0, idx = hwc->idx; - long value; + long value = 0; + + if (!event_support_sampling(hwc->config)) + goto set_counter; if (unlikely(left <= -period)) { left = period; @@ -265,23 +287,31 @@ static int sw64_perf_event_set_period(struct perf_event *event) left = sw64_pmu->pmc_max_period; value = sw64_pmu->pmc_max_period - left; + +set_counter: local64_set(&hwc->prev_count, value); switch (hwc->config) { - case SW64_PMU_INSTRUCTIONS: + case SW64_PMU_EXCLUSIVE_INSTRUCTIONS: sw64_write_csr(value, CSR_RETIC); break; - case SW64_PMU_BRANCH: - sw64_write_csr(value, CSR_BRRETC); - break; - case SW64_PMU_BRANCH_MISSES: - sw64_write_csr(value, CSR_BRFAILC); - break; - case SW64_L1I_CACHE: + case SW64_PMU_EXCLUSIVE_L1I_CACHE: sw64_write_csr(value, CSR_IACC); break; - case SW64_L1I_CACHE_MISSES: + case SW64_PMU_EXCLUSIVE_L1I_CACHE_MISSES: sw64_write_csr(value, CSR_IMISC); break; + case SW64_PMU_EXCLUSIVE_L1D_CACHE: + sw64_write_csr(value, CSR_DACC); + break; + case SW64_PMU_EXCLUSIVE_L1D_CACHE_MISSES: + sw64_write_csr(value, CSR_DMISC); + break; + case SW64_PMU_EXCLUSIVE_BRANCH: + sw64_write_csr(value, CSR_BRRETC); + break; + case SW64_PMU_EXCLUSIVE_BRANCH_MISSES: + sw64_write_csr(value, CSR_BRFAILC); + break; default: wrperfmon(idx + PMC_CMD_WRITE_BASE, value); } @@ -315,23 +345,29 @@ static unsigned long sw64_perf_event_update(struct perf_event *event) again: prev_raw_count = local64_read(&hwc->prev_count); switch (hwc->config) { - case SW64_PMU_INSTRUCTIONS: + case SW64_PMU_EXCLUSIVE_INSTRUCTIONS: new_raw_count = sw64_read_csr(CSR_RETIC); break; - case SW64_PMU_BRANCH: - new_raw_count = sw64_read_csr(CSR_BRRETC); - break; - case SW64_PMU_BRANCH_MISSES: - new_raw_count = sw64_read_csr(CSR_BRFAILC); - break; - case SW64_L1I_CACHE: + case SW64_PMU_EXCLUSIVE_L1I_CACHE: new_raw_count = sw64_read_csr(CSR_IACC); break; - case SW64_L1I_CACHE_MISSES: + case SW64_PMU_EXCLUSIVE_L1I_CACHE_MISSES: new_raw_count = sw64_read_csr(CSR_IMISC); break; + case SW64_PMU_EXCLUSIVE_L1D_CACHE: + new_raw_count = sw64_read_csr(CSR_DACC); + break; + case SW64_PMU_EXCLUSIVE_L1D_CACHE_MISSES: + new_raw_count = sw64_read_csr(CSR_DMISC); + break; + case SW64_PMU_EXCLUSIVE_BRANCH: + new_raw_count = sw64_read_csr(CSR_BRRETC); + break; + case SW64_PMU_EXCLUSIVE_BRANCH_MISSES: + new_raw_count = sw64_read_csr(CSR_BRFAILC); + break; default: - new_raw_count = wrperfmon(idx + MAX_HWEVENTS, 0); + new_raw_count = wrperfmon(idx + PMU_NUM_GENERIC_COUNTERS, 0); } if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, @@ -341,7 +377,8 @@ static unsigned long sw64_perf_event_update(struct perf_event *event) delta = new_raw_count - prev_raw_count; local64_add(delta, &event->count); - local64_sub(delta, &hwc->period_left); + if (event_support_sampling(hwc->config)) + local64_sub(delta, &hwc->period_left); return new_raw_count; } @@ -366,19 +403,25 @@ static void sw64_pmu_stop(struct perf_event *event, int flags) if (!(hwc->state & PERF_HES_STOPPED)) { value = sw64_read_csr(CSR_IDR_PCCTL); switch (hwc->config) { - case SW64_L1I_CACHE: + case SW64_PMU_EXCLUSIVE_INSTRUCTIONS: + sw64_write_csr(value & ~RETIC_EN, CSR_IDR_PCCTL); + break; + case SW64_PMU_EXCLUSIVE_L1I_CACHE: sw64_write_csr(value & ~IACC_EN, CSR_IDR_PCCTL); break; - case SW64_L1I_CACHE_MISSES: + case SW64_PMU_EXCLUSIVE_L1I_CACHE_MISSES: sw64_write_csr(value & ~IMISC_EN, CSR_IDR_PCCTL); break; - case SW64_PMU_INSTRUCTIONS: - sw64_write_csr(value & ~RETIC_EN, CSR_IDR_PCCTL); + case SW64_PMU_EXCLUSIVE_L1D_CACHE: + sw64_write_csr(value & ~DACC_EN, CSR_IDR_PCCTL); break; - case SW64_PMU_BRANCH: + case SW64_PMU_EXCLUSIVE_L1D_CACHE_MISSES: + sw64_write_csr(value & ~DMISC_EN, CSR_IDR_PCCTL); + break; + case SW64_PMU_EXCLUSIVE_BRANCH: sw64_write_csr(value & ~BRRETC_EN, CSR_IDR_PCCTL); break; - case SW64_PMU_BRANCH_MISSES: + case SW64_PMU_EXCLUSIVE_BRANCH_MISSES: sw64_write_csr(value & ~BRFAILC_EN, CSR_IDR_PCCTL); break; default: @@ -419,19 +462,25 @@ static void sw64_pmu_start(struct perf_event *event, int flags) /* counting in selected modes, for both counters */ switch (hwc->config) { - case SW64_L1I_CACHE: + case SW64_PMU_EXCLUSIVE_INSTRUCTIONS: + sw64_write_csr(value | RETIC_EN, CSR_IDR_PCCTL); + break; + case SW64_PMU_EXCLUSIVE_L1I_CACHE: sw64_write_csr(value | IACC_EN, CSR_IDR_PCCTL); break; - case SW64_L1I_CACHE_MISSES: + case SW64_PMU_EXCLUSIVE_L1I_CACHE_MISSES: sw64_write_csr(value | IMISC_EN, CSR_IDR_PCCTL); break; - case SW64_PMU_INSTRUCTIONS: - sw64_write_csr(value | RETIC_EN, CSR_IDR_PCCTL); + case SW64_PMU_EXCLUSIVE_L1D_CACHE: + sw64_write_csr(value | DACC_EN, CSR_IDR_PCCTL); + break; + case SW64_PMU_EXCLUSIVE_L1D_CACHE_MISSES: + sw64_write_csr(value | DMISC_EN, CSR_IDR_PCCTL); break; - case SW64_PMU_BRANCH: + case SW64_PMU_EXCLUSIVE_BRANCH: sw64_write_csr(value | BRRETC_EN, CSR_IDR_PCCTL); break; - case SW64_PMU_BRANCH_MISSES: + case SW64_PMU_EXCLUSIVE_BRANCH_MISSES: sw64_write_csr(value | BRFAILC_EN, CSR_IDR_PCCTL); break; default: @@ -451,7 +500,9 @@ static int sw64_pmu_add(struct perf_event *event, int flags) int idx = hwc->idx; int err = -EAGAIN; - if (__test_and_set_bit(idx, cpuc->used_mask)) { + if (event_has_exclusive_counter(hwc->config)) { + hwc->idx = (hwc->config & 0xff) + PMU_NUM_GENERIC_COUNTERS; + } else if (__test_and_set_bit(idx, cpuc->used_mask)) { idx = find_first_zero_bit(cpuc->used_mask, sw64_pmu->num_pmcs); if (idx == sw64_pmu->num_pmcs) goto out; @@ -513,19 +564,12 @@ static int __hw_perf_event_init(struct perf_event *event) hwc->config_base = SW64_PERFCTRL_AM; - if (!is_sampling_event(event)) + if (!is_sampling_event(event)) { pr_debug("not sampling event\n"); - else { - switch (hwc->config) { - case SW64_L1I_CACHE: - case SW64_L1I_CACHE_MISSES: - case SW64_PMU_INSTRUCTIONS: - case SW64_PMU_BRANCH: - case SW64_PMU_BRANCH_MISSES: - return -EOPNOTSUPP; - default: - break; - } + } else if (!event_support_sampling(hwc->config)) { + pr_info("event %#llx does not support sampling(period=%#llx)", + hwc->config, event->attr.sample_period); + return -EOPNOTSUPP; } event->destroy = hw_perf_event_destroy; -- Gitee From d69f96a1d4897a2f9bc98c1da519a32dadf21b97 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 4 Mar 2025 09:56:47 +0800 Subject: [PATCH 444/524] sw64: enable all HW PMUs for max sampling period events Treat sampling events with maximum sampling period as non-sampling events and enable all hardware PMUs for them. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/perf_event_c4.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/sw_64/kernel/perf_event_c4.c b/arch/sw_64/kernel/perf_event_c4.c index c1d2c7872428..5977c12b8b98 100644 --- a/arch/sw_64/kernel/perf_event_c4.c +++ b/arch/sw_64/kernel/perf_event_c4.c @@ -558,6 +558,12 @@ static void hw_perf_event_destroy(struct perf_event *event) /* Nothing to be done! */ } +static inline bool max_sampling_period(struct perf_event *event) +{ + return ((u64)event->attr.sample_period == U64_MAX) || + ((s64)event->attr.sample_period == S64_MAX); +} + static int __hw_perf_event_init(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -566,6 +572,8 @@ static int __hw_perf_event_init(struct perf_event *event) if (!is_sampling_event(event)) { pr_debug("not sampling event\n"); + } else if (max_sampling_period(event)) { + pr_debug("max sampling period, allowing all events\n"); } else if (!event_support_sampling(hwc->config)) { pr_info("event %#llx does not support sampling(period=%#llx)", hwc->config, event->attr.sample_period); -- Gitee From d0f6f60ddbf8ef4c9bd86083fc37d609b9da575f Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Thu, 20 Mar 2025 09:37:17 +0800 Subject: [PATCH 445/524] sw64: fix specific performance counter error When splitting the specific performance counters from the general performance counters, we were not adding a corresponding bitmap for the specific performance counter. And the labeling used for the specific performance counter event would conflict with the raw event. Now, we fixed it. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/perf_event.h | 1 + arch/sw_64/include/asm/pmc_c4.h | 14 +++++++------- arch/sw_64/kernel/perf_event_c4.c | 16 +++++++++++++--- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/arch/sw_64/include/asm/perf_event.h b/arch/sw_64/include/asm/perf_event.h index 6d8a8cf2f9c8..391ab1c6d621 100644 --- a/arch/sw_64/include/asm/perf_event.h +++ b/arch/sw_64/include/asm/perf_event.h @@ -26,6 +26,7 @@ struct cpu_hw_events { * is used for an event. */ unsigned long used_mask[BITS_TO_LONGS(PMU_NUM_GENERIC_COUNTERS)]; + unsigned long exclusive_used_mask[BITS_TO_LONGS(PMU_NUM_EXCLUSIVE_COUNTERS)]; /* Array of events current scheduled on this cpu. */ struct perf_event *event[PMU_NUM_GENERIC_COUNTERS + PMU_NUM_EXCLUSIVE_COUNTERS]; }; diff --git a/arch/sw_64/include/asm/pmc_c4.h b/arch/sw_64/include/asm/pmc_c4.h index de813dcacc55..b4d560283aa5 100644 --- a/arch/sw_64/include/asm/pmc_c4.h +++ b/arch/sw_64/include/asm/pmc_c4.h @@ -35,13 +35,13 @@ #define SW64_PMU_L2_MISSES 0x53 /* these events has exclusive counters */ -#define SW64_PMU_EXCLUSIVE_INSTRUCTIONS 0x400 -#define SW64_PMU_EXCLUSIVE_L1I_CACHE 0x401 -#define SW64_PMU_EXCLUSIVE_L1I_CACHE_MISSES 0x402 -#define SW64_PMU_EXCLUSIVE_L1D_CACHE 0x403 -#define SW64_PMU_EXCLUSIVE_L1D_CACHE_MISSES 0x404 -#define SW64_PMU_EXCLUSIVE_BRANCH 0x405 -#define SW64_PMU_EXCLUSIVE_BRANCH_MISSES 0x406 +#define SW64_PMU_EXCLUSIVE_INSTRUCTIONS 0x1000 +#define SW64_PMU_EXCLUSIVE_L1I_CACHE 0x1001 +#define SW64_PMU_EXCLUSIVE_L1I_CACHE_MISSES 0x1002 +#define SW64_PMU_EXCLUSIVE_L1D_CACHE 0x1003 +#define SW64_PMU_EXCLUSIVE_L1D_CACHE_MISSES 0x1004 +#define SW64_PMU_EXCLUSIVE_BRANCH 0x1005 +#define SW64_PMU_EXCLUSIVE_BRANCH_MISSES 0x1006 #define PC_ALL_PM_SET 3 #define PMU_NUM_GENERIC_COUNTERS 5 diff --git a/arch/sw_64/kernel/perf_event_c4.c b/arch/sw_64/kernel/perf_event_c4.c index 5977c12b8b98..7edd75f09ce2 100644 --- a/arch/sw_64/kernel/perf_event_c4.c +++ b/arch/sw_64/kernel/perf_event_c4.c @@ -235,7 +235,7 @@ static bool core4_raw_event_valid(u64 config) int idx = config >> 8; int event = config & 0xff; - if (event_has_exclusive_counter(event)) + if (event_has_exclusive_counter(config)) return true; if (idx >= 0 && idx < PMU_NUM_GENERIC_COUNTERS && @@ -501,7 +501,11 @@ static int sw64_pmu_add(struct perf_event *event, int flags) int err = -EAGAIN; if (event_has_exclusive_counter(hwc->config)) { - hwc->idx = (hwc->config & 0xff) + PMU_NUM_GENERIC_COUNTERS; + idx = hwc->config & 0xff; + if (__test_and_set_bit(idx, cpuc->exclusive_used_mask)) + goto out; + __set_bit(idx, cpuc->exclusive_used_mask); + hwc->idx = idx + PMU_NUM_GENERIC_COUNTERS; } else if (__test_and_set_bit(idx, cpuc->used_mask)) { idx = find_first_zero_bit(cpuc->used_mask, sw64_pmu->num_pmcs); if (idx == sw64_pmu->num_pmcs) @@ -534,6 +538,8 @@ static void sw64_pmu_del(struct perf_event *event, int flags) return; sw64_pmu_stop(event, PERF_EF_UPDATE); + if (event_has_exclusive_counter(hwc->config)) + __clear_bit(event->hw.idx - PMU_NUM_GENERIC_COUNTERS, cpuc->exclusive_used_mask); __clear_bit(event->hw.idx, cpuc->used_mask); /* Absorb the final count and turn off the event. */ @@ -645,7 +651,11 @@ static int sw64_pmu_event_init(struct perf_event *event) attr->exclude_host || attr->exclude_guest) return -EINVAL; - hwc->config = config; + if ((hwc->idx & 0xf0) == 0) + hwc->config = config; + else + hwc->config = attr->config; + /* Do the real initialisation work. */ return __hw_perf_event_init(event); } -- Gitee From e93b7219487cac047944f0fd9725bf0ee75ed400 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 1 Apr 2025 17:42:04 +0800 Subject: [PATCH 446/524] sw64: use CONFIG_HZ_PERIODIC by default We have identified multiple issues confirmed to be related to CONFIG_NO_HZ, so we are temporarily reverting to the default CONFIG_HZ_PERIODIC. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/configs/junzhang_defconfig | 2 -- arch/sw_64/configs/xuelang_defconfig | 2 -- 2 files changed, 4 deletions(-) diff --git a/arch/sw_64/configs/junzhang_defconfig b/arch/sw_64/configs/junzhang_defconfig index 566d0e1a5075..c7e50cf9d6f2 100644 --- a/arch/sw_64/configs/junzhang_defconfig +++ b/arch/sw_64/configs/junzhang_defconfig @@ -3,7 +3,6 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y # CONFIG_CROSS_MEMORY_ATTACH is not set CONFIG_USELIB=y -CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y @@ -664,6 +663,5 @@ CONFIG_CRYPTO_LZO=y # CONFIG_CRYPTO_HW is not set CONFIG_CRC_ITU_T=y CONFIG_CONSOLE_LOGLEVEL_QUIET=7 -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_SCHEDSTATS=y # CONFIG_RCU_TRACE is not set diff --git a/arch/sw_64/configs/xuelang_defconfig b/arch/sw_64/configs/xuelang_defconfig index 64ab45a5095d..b04f6ba61d27 100644 --- a/arch/sw_64/configs/xuelang_defconfig +++ b/arch/sw_64/configs/xuelang_defconfig @@ -3,7 +3,6 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y # CONFIG_CROSS_MEMORY_ATTACH is not set CONFIG_USELIB=y -CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y @@ -26,7 +25,6 @@ CONFIG_EXPERT=y CONFIG_KALLSYMS_ALL=y CONFIG_PERF_EVENTS=y CONFIG_DEBUG_PERF_USE_VMALLOC=y -# CONFIG_COMPAT_BRK is not set CONFIG_SMP=y CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_NUMA=y -- Gitee From d200cf47d04b43a392a9f2abe8235ee8fc4dbf1e Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Wed, 19 Mar 2025 03:19:57 +0000 Subject: [PATCH 447/524] sw64: msi: fix irq affinity setting during migration When a managed irq's last active cpu in its affinity mask is offline, interrupts remain disabled until another cpu in the affinity comes online. Frequent cpu hotplug operations exacerbate this issue by progressively narrowing the affinity mask, reducing the available cpu pool for recovery. To address this, we retain the original affinity during migration and introduce effective affinity through the irq_data_update_effective_affinity. This approach achieves the following benefits: 1. Maintains a larger candidate cpu pool based on the original affinity. 2. Allows interrupts to resume as soon as any cpu in the original affinity set comes online. 3. Mitigates dependency on specific cpus coming online quickly. This solution ensures more robust interrupt handing under dynamic cpu configurations while preserving system responsiveness. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + drivers/irqchip/irq-sunway-msi-v2.c | 18 +++++++++++------- drivers/irqchip/irq-sunway-msi.c | 17 ++++++++++------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index bcb57e532105..be0c18a9a04d 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -67,6 +67,7 @@ config SW64 select GENERIC_CLOCKEVENTS select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO select GENERIC_IRQ_LEGACY + select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select GENERIC_IRQ_MIGRATION if SMP select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c index 41df065ad045..283e190cd72c 100644 --- a/drivers/irqchip/irq-sunway-msi-v2.c +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -137,22 +137,24 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, irqd = irq_domain_get_irq_data(msi_default_domain->parent, d->irq); /* Don't do anything if the interrupt isn't started */ - if (!irqd_is_started(irqd)) + if (!irqd_is_started(irqd) && !irqd_affinity_is_managed(irqd)) return IRQ_SET_MASK_OK; cdata = irqd->chip_data; if (!cdata) return -ENOMEM; - if (cdata->move_in_progress) - return -EBUSY; - /* * If existing target cpu is already in the new mask and is online * then do nothing. */ - if (cpu_online(cdata->dst_cpu) && cpumask_test_cpu(cdata->dst_cpu, cpumask)) - return IRQ_SET_MASK_OK; + if (cpu_online(cdata->dst_cpu)) { + if (cpumask_test_cpu(cdata->dst_cpu, cpumask)) + return IRQ_SET_MASK_OK; + + if (cdata->move_in_progress) + return -EBUSY; + } raw_spin_lock_irqsave(&vector_lock, flags); @@ -190,7 +192,7 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, irq_msi_compose_msg(d, &msg); __pci_write_msi_msg(entry, &msg); spin_unlock(&cdata->cdata_lock); - irq_data_update_effective_affinity(irqd, &searchmask); + irq_data_update_effective_affinity(d, cpumask_of(cpu)); raw_spin_unlock_irqrestore(&vector_lock, flags); @@ -259,6 +261,7 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, } irq_data->chip_data = cdata; + irq_data_update_effective_affinity(irq_data, cpumask_of(cpu)); } cdata->dst_cpu = cpu; @@ -295,6 +298,7 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, cdata->prev_vector = vector; cdata->multi_msi = 1; cdata->move_in_progress = false; + irq_data_update_effective_affinity(irq_data, cpumask_of(cpu)); } } return 0; diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index f6127eb33884..fb1dd4c60bb1 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -114,22 +114,24 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, irqd = irq_domain_get_irq_data(msi_default_domain->parent, d->irq); /* Don't do anything if the interrupt isn't started */ - if (!irqd_is_started(irqd)) + if (!irqd_is_started(irqd) && !irqd_affinity_is_managed(irqd)) return IRQ_SET_MASK_OK; cdata = irqd->chip_data; if (!cdata) return -ENOMEM; - if (cdata->move_in_progress) - return -EBUSY; - /* * If existing target cpu is already in the new mask and is online * then do nothing. */ - if (cpu_online(cdata->dst_cpu) && cpumask_test_cpu(cdata->dst_cpu, cpumask)) - return IRQ_SET_MASK_OK; + if (cpu_online(cdata->dst_cpu)) { + if (cpumask_test_cpu(cdata->dst_cpu, cpumask)) + return IRQ_SET_MASK_OK; + + if (cdata->move_in_progress) + return -EBUSY; + } raw_spin_lock_irqsave(&vector_lock, flags); @@ -159,7 +161,7 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, msi_config = set_piu_msi_config(hose, cpu, cdata->msi_config_index, vector); cdata->msi_config = msi_config; spin_unlock(&cdata->cdata_lock); - irq_data_update_effective_affinity(irqd, &searchmask); + irq_data_update_effective_affinity(d, cpumask_of(cpu)); raw_spin_unlock_irqrestore(&vector_lock, flags); @@ -249,6 +251,7 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, cdata->prev_cpu = cpu; cdata->prev_vector = vector; cdata->move_in_progress = false; + irq_data_update_effective_affinity(irq_data, cpumask_of(cpu)); } return 0; } -- Gitee From 3bc128bd554b678d8d428d5e728d2fecb45191d7 Mon Sep 17 00:00:00 2001 From: Yan Bo Date: Tue, 4 Mar 2025 02:51:00 +0000 Subject: [PATCH 448/524] sw64: iommu: print more info for interrupts Output more information for iommu interrupts. Signed-off-by: Yan Bo Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 21 ++++++++---- drivers/iommu/sw64/iommu_v2.c | 60 +++++++++++++++++++++++++++-------- 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index 10b83f07f96f..dfb0c79fc0ad 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -53,6 +53,8 @@ enum exceptype { PTE_LEVEL2, UNAUTHORIZED_ACCESS, ILLEGAL_RESPONSE, + SEGMENT_TRANSLATION_MISS, + SEGMENT_TRANSLATION_UNAUTHORIZED_ACCESS, DTE_LEVEL1_VAL, DTE_LEVEL2_VAL, PTE_LEVEL1_VAL, @@ -640,6 +642,7 @@ irqreturn_t iommu_interrupt(int irq, void *dev) } sdomain = sdev->domain; + pr_info("iommu exception type:%#lx\n", type); switch (type) { case DTE_LEVEL1: @@ -661,22 +664,28 @@ irqreturn_t iommu_interrupt(int irq, void *dev) pr_info("invalid level2 pte, addr: %#lx, val: %#lx\n", fetch_pte(sdomain, dva, PTE_LEVEL2), fetch_pte(sdomain, dva, PTE_LEVEL2_VAL)); - - iommu_status &= ~(1UL << 62); - writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); break; - case UNAUTHORIZED_ACCESS: - pr_info("unauthorized access\n"); + pr_info("page translation unauthorized access\n"); break; case ILLEGAL_RESPONSE: - pr_info("illegal response\n"); + pr_info("accessing the device table or page table \ + return an illegal response\n"); + break; + case SEGMENT_TRANSLATION_MISS: + pr_info("segment translation miss\n"); + break; + case SEGMENT_TRANSLATION_UNAUTHORIZED_ACCESS: + pr_info("segment translation unauthorized access\n"); break; default: pr_info("unknown error\n"); break; } + iommu_status &= ~(1UL << 62); + writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); + return IRQ_HANDLED; } diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index 3e51963714bc..65e8afb2099f 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -65,8 +65,16 @@ enum exceptype { PTE_LEVEL1, PTE_LEVEL2, PTE_LEVEL3, - UNAUTHORIZED_ACCESS, + LEVEL1_PTE_UNAUTHORIZED_ACCESS, + LEVEL2_PTE_UNAUTHORIZED_ACCESS, + LEVEL3_PTE_UNAUTHORIZED_ACCESS, + LEVEL1_PTE_GRANULARITY_ERROR, + LEVEL2_PTE_GRANULARITY_ERROR, + LEVEL3_PTE_GRANULARITY_ERROR, ILLEGAL_RESPONSE, + INVALID_HIGH_ADDRESS, + SEGMENT_TRANSLATION_MISS, + SEGMENT_TRANSLATION_UNAUTHORIZED_ACCESS, DTE_LEVEL1_VAL, DTE_LEVEL2_VAL, PTE_LEVEL1_VAL, @@ -716,7 +724,9 @@ irqreturn_t iommu_interrupt(int irq, void *dev) return IRQ_HANDLED; } - sdomain = sdev->domain; + sdomain = sdev->domain; + pr_info("iommu exception type:%#lx\n", type); + switch (type) { case DTE_LEVEL1: pr_info("invalid level1 dte, addr:%#lx, val:%#lx\n", @@ -732,32 +742,56 @@ irqreturn_t iommu_interrupt(int irq, void *dev) pr_info("invalid level1 pte, addr: %#lx, val:%#lx\n", fetch_pte(sdomain, dva, PTE_LEVEL1), fetch_pte(sdomain, dva, PTE_LEVEL1_VAL)); - - iommu_status &= ~(1UL << 62); - writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); break; case PTE_LEVEL2: pr_info("invalid level2 pte, addr: %#lx, val: %#lx\n", fetch_pte(sdomain, dva, PTE_LEVEL2), fetch_pte(sdomain, dva, PTE_LEVEL2_VAL)); - - iommu_status &= ~(1UL << 62); - writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); break; - case PTE_LEVEL3: pr_info("invalid level3 pte, addr: %#lx, val: %#lx\n", fetch_pte(sdomain, dva, PTE_LEVEL3), fetch_pte(sdomain, dva, PTE_LEVEL3_VAL)); - - iommu_status &= ~(1UL << 62); - writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); + break; + case LEVEL1_PTE_UNAUTHORIZED_ACCESS: + pr_info("level1 pte unauthorized access\n"); + break; + case LEVEL2_PTE_UNAUTHORIZED_ACCESS: + pr_info("level2 pte unauthorized access\n"); + break; + case LEVEL3_PTE_UNAUTHORIZED_ACCESS: + pr_info("level3 pte unauthorized access\n"); + break; + case LEVEL1_PTE_GRANULARITY_ERROR: + pr_info("level1 pte granularity error\n"); + break; + case LEVEL2_PTE_GRANULARITY_ERROR: + pr_info("level2 pte granularity error\n"); + break; + case LEVEL3_PTE_GRANULARITY_ERROR: + pr_info("level3 pte granularity error\n"); + break; + case ILLEGAL_RESPONSE: + pr_info("accessing the device table or page table \ + return an illegal response\n"); + break; + case INVALID_HIGH_ADDRESS: + pr_info("the address is longer than the allowed length\n"); + break; + case SEGMENT_TRANSLATION_MISS: + pr_info("segment translation miss\n"); + break; + case SEGMENT_TRANSLATION_UNAUTHORIZED_ACCESS: + pr_info("segment translation unauthorized access\n"); break; default: - pr_info("iommu exception type %ld\n", type); + pr_info("unknown error\n"); break; } + iommu_status &= ~(1UL << 62); + writeq(iommu_status, iommu->reg_base_addr + IOMMUEXCPT_STATUS); + return IRQ_HANDLED; } -- Gitee From fda39de2a930579460129bf6ff16a7a2616daa0c Mon Sep 17 00:00:00 2001 From: Zhi Tongze Date: Wed, 9 Oct 2024 17:16:36 +0800 Subject: [PATCH 449/524] sw64: modify CPU autoplug initialization process Allow the interface of CPU autoplug to be accessible after initialization. Optimize the CPU autoplug strategy. Currently, we can bring multiple cores online simultaneously to meet the system performance requirements during instantaneous high loads. Signed-off-by: Zhi Tongze Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/cpuautoplug.c | 191 +++++++++++++++++++++----------- 1 file changed, 127 insertions(+), 64 deletions(-) diff --git a/arch/sw_64/kernel/cpuautoplug.c b/arch/sw_64/kernel/cpuautoplug.c index 4fdac8a1f726..94a388b1116e 100644 --- a/arch/sw_64/kernel/cpuautoplug.c +++ b/arch/sw_64/kernel/cpuautoplug.c @@ -6,15 +6,23 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include #include +#include "../../../kernel/sched/sched.h" int autoplug_enabled; int autoplug_verbose; int autoplug_adjusting; + DEFINE_PER_CPU(int, cpu_adjusting); struct cpu_autoplug_info { @@ -41,11 +49,12 @@ static ssize_t enabled_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - char val[5]; - int n; + int err, n; + + err = kstrtoint(buf, 10, &n); - memcpy(val, buf, count); - n = kstrtol(val, 0, 0); + if (err) + return err; if (n > 1 || n < 0) return -EINVAL; @@ -65,11 +74,12 @@ static ssize_t verbose_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - char val[5]; - int n; + int err, n; + + err = kstrtoint(buf, 10, &n); - memcpy(val, buf, count); - n = kstrtol(val, 0, 0); + if (err) + return err; if (n > 1 || n < 0) return -EINVAL; @@ -89,11 +99,12 @@ static ssize_t maxcpus_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - char val[5]; - int n; + int err, n; - memcpy(val, buf, count); - n = kstrtol(val, 0, 0); + err = kstrtoint(buf, 10, &n); + + if (err) + return err; if (n > num_possible_cpus() || n < ap_info.mincpus) return -EINVAL; @@ -113,11 +124,12 @@ static ssize_t mincpus_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - char val[5]; - int n; + int err, n; + + err = kstrtoint(buf, 10, &n); - memcpy(val, buf, count); - n = kstrtol(val, 0, 0); + if (err) + return err; if (n > ap_info.maxcpus || n < 1) return -EINVAL; @@ -140,11 +152,12 @@ static ssize_t sampling_rate_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - char val[6]; - int n; + int err, n; - memcpy(val, buf, count); - n = kstrtol(val, 0, 0); + err = kstrtoint(buf, 10, &n); + + if (err) + return err; if (n > SAMPLING_RATE_MAX || n < SAMPLING_RATE_MIN) return -EINVAL; @@ -253,7 +266,7 @@ static cputime64_t get_min_busy_time(cputime64_t arr[], int size) int i, min_cpu_idx; cputime64_t min_time = arr[0]; - for (i = 0; i < size; i++) { + for (i = 1; i < size; i++) { if (arr[i] > 0 && arr[i] < min_time) { min_time = arr[i]; min_cpu_idx = i; @@ -268,7 +281,7 @@ static int find_min_busy_cpu(void) int nr_all_cpus = num_possible_cpus(); unsigned int cpus, target_cpu; cputime64_t busy_time; - cputime64_t b_time[NR_CPUS]; + static cputime64_t b_time[CONFIG_NR_CPUS]; memset(b_time, 0, sizeof(b_time)); for_each_online_cpu(cpus) { @@ -279,63 +292,102 @@ static int find_min_busy_cpu(void) return target_cpu; } +static void up_core(int target_cpu) +{ + if (target_cpu > 0 && target_cpu < CONFIG_NR_CPUS) { + per_cpu(cpu_adjusting, target_cpu) = 1; + lock_device_hotplug(); + + add_cpu(target_cpu); + pr_debug("The target_cpu is %d, After cpu_up, the cpu_num is %d\n", + target_cpu, num_online_cpus()); + get_cpu_device(target_cpu)->offline = false; + unlock_device_hotplug(); + per_cpu(cpu_adjusting, target_cpu) = 0; + } +} + +static void down_core(int target_cpu) +{ + if (target_cpu > 0 && target_cpu < CONFIG_NR_CPUS) { + per_cpu(cpu_adjusting, target_cpu) = -1; + lock_device_hotplug(); + + remove_cpu(target_cpu); + pr_debug("The target_cpu is %d. After cpu_down, the cpu_num is %d\n", + target_cpu, num_online_cpus()); + get_cpu_device(target_cpu)->offline = true; + unlock_device_hotplug(); + per_cpu(cpu_adjusting, target_cpu) = 0; + } +} + static void increase_cores(int cur_cpus) { - struct device *dev; + int cr; if (cur_cpus == ap_info.maxcpus) return; - cur_cpus = cpumask_next_zero(0, cpu_online_mask); + for (cr = 0; cr < ((num_possible_cpus() + 7) / 8); cr++) { + int target_cpu; + int target_rcid; + int next_cpu; + int next_rcid; + + target_cpu = cpumask_next_zero(0, cpu_online_mask); + target_rcid = cpu_to_rcid(target_cpu); + next_rcid = target_rcid ^ (1ULL << 8); - dev = get_cpu_device(cur_cpus); + for_each_possible_cpu(cr) { + if ((cpu_to_rcid(cr)) == next_rcid) + next_cpu = cr; + } - per_cpu(cpu_adjusting, dev->id) = 1; - lock_device_hotplug(); - cpu_device_up(dev); - pr_info("The target_cpu is %d, After cpu_up, the cpu_num is %d\n", - dev->id, num_online_cpus()); - get_cpu_device(dev->id)->offline = false; - unlock_device_hotplug(); - per_cpu(cpu_adjusting, dev->id) = 0; + pr_debug("increase_cores target_cpu = %d, next_cpu = %d\n", + target_cpu, next_cpu); + up_core(target_cpu); + up_core(next_cpu); + } } static void decrease_cores(int cur_cpus) { - struct device *dev; + int target_cpu; + int next_cpu; if (cur_cpus == ap_info.mincpus) return; - cur_cpus = find_min_busy_cpu(); - - dev = get_cpu_device(cur_cpus); - - if (dev->id > 0) { - per_cpu(cpu_adjusting, dev->id) = -1; - lock_device_hotplug(); - cpu_device_down(dev); - pr_info("The target_cpu is %d. After cpu_down, the cpu_num is %d\n", - cur_cpus, num_online_cpus()); - get_cpu_device(dev->id)->offline = true; - unlock_device_hotplug(); - per_cpu(cpu_adjusting, dev->id) = 0; + target_cpu = find_min_busy_cpu(); + for_each_cpu(next_cpu, &(cpu_topology[target_cpu].thread_sibling)) { + if (target_cpu != next_cpu) + break; } + pr_debug("decrease_cores target_cpu = %d, next_cpu = %d\n", + target_cpu, next_cpu); + down_core(target_cpu); + mdelay(500); + down_core(next_cpu); } -#define INC_THRESHOLD 80 -#define DEC_THRESHOLD 40 +#define INC_THRESHOLD 90 +#define DEC_THRESHOLD 80 static void do_autoplug_timer(struct work_struct *work) { - cputime64_t cur_wall_time = 0, cur_idle_time; - unsigned long idle_time, wall_time; int delay, load; int nr_cur_cpus = num_online_cpus(); - int nr_all_cpus = num_possible_cpus(); int inc_req = 1, dec_req = 2; struct cpufreq_policy *policy = cpufreq_cpu_get_raw(smp_processor_id()); - +#ifdef CONFIG_NO_HZ_COMMON + int nr_all_cpus = num_possible_cpus(); + cputime64_t cur_wall_time = 0, cur_idle_time; + unsigned long idle_time, wall_time; +#else + long active; + atomic_long_t calc_load_tasks; +#endif if (!policy || IS_ERR(policy->clk)) { pr_err("%s: No %s associated to cpu: %d\n", __func__, policy ? "clk" : "policy", 0); @@ -344,20 +396,20 @@ static void do_autoplug_timer(struct work_struct *work) ap_info.maxcpus = setup_max_cpus > nr_cpu_ids ? nr_cpu_ids : setup_max_cpus; - ap_info.mincpus = ap_info.maxcpus / 4; + ap_info.mincpus = ap_info.maxcpus / 16; if (strcmp(policy->governor->name, "performance") == 0) { ap_info.mincpus = ap_info.maxcpus; } else if (strcmp(policy->governor->name, "powersave") == 0) { ap_info.maxcpus = ap_info.mincpus; } else if (strcmp(policy->governor->name, "ondemand") == 0) { - ap_info.sampling_rate = 500; + ap_info.sampling_rate = 1000; inc_req = 0; dec_req = 2; } else if (strcmp(policy->governor->name, "conservative") == 0) { inc_req = 1; dec_req = 3; - ap_info.sampling_rate = 1000; /* 1s */ + ap_info.sampling_rate = 500; /* 1s */ } BUG_ON(smp_processor_id() != 0); @@ -378,7 +430,8 @@ static void do_autoplug_timer(struct work_struct *work) goto out; } - cur_idle_time = sw64_get_idle_time(&cur_wall_time); +#ifdef CONFIG_NO_HZ_COMMON + cur_idle_time = get_idle_time(&cur_wall_time); if (cur_wall_time == 0) cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); @@ -387,16 +440,22 @@ static void do_autoplug_timer(struct work_struct *work) idle_time = (unsigned int)(cur_idle_time - ap_info.prev_idle); idle_time += wall_time * (nr_all_cpus - nr_cur_cpus); - ap_info.prev_wall = cur_idle_time; + ap_info.prev_idle = cur_idle_time; if (unlikely(!wall_time || wall_time * nr_all_cpus < idle_time)) { autoplug_adjusting = 0; goto out; } - load = 100 * (wall_time * nr_all_cpus - idle_time) / wall_time; + load = (100 * (wall_time * nr_all_cpus - idle_time)) / wall_time; +#else + active = atomic_long_read(&calc_load_tasks); + active = active > 0 ? active * FIXED_1 : 0; + CALC_LOAD(avenrun[0], EXP_1, active); + load = avenrun[0] / 2; +#endif - if (load < (nr_cur_cpus - 1) * 100 - DEC_THRESHOLD) { + if (load < (nr_cur_cpus - 1) * (100 - DEC_THRESHOLD)) { ap_info.inc_reqs = 0; if (ap_info.dec_reqs < dec_req) ap_info.dec_reqs++; @@ -458,6 +517,9 @@ static int __init cpuautoplug_init(void) pr_info("cpuautoplug: SW64 CPU autoplug driver.\n"); + ap_info.prev_wall = 0; + ap_info.prev_idle = 0; + ap_info.maxcpus = setup_max_cpus > nr_cpu_ids ? nr_cpu_ids : setup_max_cpus; ap_info.mincpus = ap_info.maxcpus / 4; @@ -477,13 +539,14 @@ static int __init cpuautoplug_init(void) for_each_possible_cpu(i) per_cpu(cpu_adjusting, i) = 0; +#ifndef MODULE delay = msecs_to_jiffies(ap_info.sampling_rate * 24); - INIT_DEFERRABLE_WORK(&ap_info.work, do_autoplug_timer); +#else + delay = msecs_to_jiffies(ap_info.sampling_rate * 8); +#endif + INIT_DELAYED_WORK(&ap_info.work, do_autoplug_timer); schedule_delayed_work_on(0, &ap_info.work, delay); - if (!autoplug_enabled) - cancel_delayed_work_sync(&ap_info.work); - return ret; } -- Gitee From 4ecdf74d5a7d14968f156caf42398b36dde76ff8 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Thu, 3 Apr 2025 10:04:09 +0800 Subject: [PATCH 450/524] sw64: fix an error in BPF exception handling Fix the calculation method of the offset between the PC and the exception table entry in BPF. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/net/bpf_jit_comp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index 5c06c224cfee..ed78ae0a23c9 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -637,7 +637,7 @@ static int add_exception_handler(const struct bpf_insn *insn, ex = &ctx->prog->aux->extable[ctx->exentry_idx]; pc = (unsigned long)&ctx->image[ctx->idx - 1]; - offset = (long)&ex->insn - pc; + offset = pc - (long)&ex->insn; ex->insn = offset; ex->fixup.bits.nextinsn = sizeof(u32); -- Gitee From 30e8aae8321e18107af255bc3fff83805a65b52c Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 8 Apr 2025 07:21:23 +0000 Subject: [PATCH 451/524] sw64: pci: fix piu data allocation failure issue The issue is introduced by commit aff939d68d0a ("sw64: pci: fix secondary bus reset issue"), which triggered an early kzalloc call during initialization in legacy boot mode. At this stage, the slab allocator is not yet operational, leading to allocation failures for piu data. To address this, the memory allocation for piu data has been postponed to a later stage in the initialization process, ensuring that the slab allocator is fully initialized before any memory allocation attempts are made. Fixes: aff939d68d0a ("sw64: pci: fix secondary bus reset issue") Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 9 +++++++++ arch/sw_64/pci/pci-legacy.c | 7 +++++++ drivers/pci/controller/pci-sunway.c | 13 +------------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index c0981bc22682..74d33777f05d 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -156,6 +156,15 @@ extern void __init setup_chip_pci_ops(void); #define setup_chip_pci_ops() do { } while (0) #endif +#ifdef CONFIG_UNCORE_XUELANG +#define PIU_IOR0_SAVE_REGS (12 + 256) +#define PIU_IOR1_SAVE_REGS 1 +#else +#define PIU_IOR0_SAVE_REGS 12 +#define PIU_IOR1_SAVE_REGS 1 +#endif + +extern struct piu_saved_data *alloc_piu_saved_data(unsigned int size); extern void save_piu_ior0(struct pci_controller *hose); extern void restore_piu_ior0(struct pci_controller *hose); extern void save_piu_ior1(struct pci_controller *hose); diff --git a/arch/sw_64/pci/pci-legacy.c b/arch/sw_64/pci/pci-legacy.c index 417852cbfe95..4ee58b43f407 100644 --- a/arch/sw_64/pci/pci-legacy.c +++ b/arch/sw_64/pci/pci-legacy.c @@ -92,6 +92,13 @@ void __init common_init_pci(void) bridge = pci_alloc_host_bridge(0); if (!bridge) continue; + + hose->piu_ior0 = + alloc_piu_saved_data(PIU_IOR0_SAVE_REGS * sizeof(u64)); + + hose->piu_ior1 = + alloc_piu_saved_data(PIU_IOR1_SAVE_REGS * sizeof(u64)); + hose->busn_space->start = last_bus; init_busnr = (0xff << 16) + ((last_bus + 1) << 8) + (last_bus); writel(init_busnr, (hose->rc_config_space_base + RC_PRIMARY_BUS)); diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 6a63adab8b04..a4535c61c04e 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -36,15 +36,7 @@ int pcibios_enable_device(struct pci_dev *dev, int bars) return pci_enable_resources(dev, bars); } -#ifdef CONFIG_UNCORE_XUELANG -#define PIU_IOR0_SAVE_REGS (12 + 256) -#define PIU_IOR1_SAVE_REGS 1 -#else -#define PIU_IOR0_SAVE_REGS 12 -#define PIU_IOR1_SAVE_REGS 1 -#endif - -static struct piu_saved_data *alloc_piu_saved_data(unsigned int size) +struct piu_saved_data *alloc_piu_saved_data(unsigned int size) { struct piu_saved_data *piu_saved_data; @@ -481,9 +473,6 @@ static void hose_init(struct pci_controller *hose) hose->first_busno = hose->self_busno = hose->busn_space->start; hose->last_busno = hose->busn_space->end; - hose->piu_ior0 = alloc_piu_saved_data(PIU_IOR0_SAVE_REGS * sizeof(u64)); - hose->piu_ior1 = alloc_piu_saved_data(PIU_IOR1_SAVE_REGS * sizeof(u64)); - if (is_in_host()) { if (IS_ENABLED(CONFIG_PCI_MSI)) memset(hose->piu_msiconfig, 0, 256/8); -- Gitee From eda3a1d7ca40af46933a673357ed056b2b6522f8 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Sat, 12 Apr 2025 00:13:55 +0800 Subject: [PATCH 452/524] sw64: ftrace: fix bug in restoring regs The ftrace_call&ftrace_regs_call may modify the value of $26 saved in memory. So in entry-ftrace.S, when restoring registers, the $26 should not be restored to its original value. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry-ftrace.S | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S index 75d604c14db4..00e9d25931ec 100644 --- a/arch/sw_64/kernel/entry-ftrace.S +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -63,8 +63,6 @@ ldl $21, 0x48($sp) addl $sp, FTRACE_SP_OFF, $sp - ldl $28, 0($sp) - ldl $26, 0x10($sp) ldl $15, 0x18($sp) addl $sp, 0x20, $sp .endm @@ -160,9 +158,7 @@ ldl $29, PT_REGS_GP($sp) ldi $sp, PT_REGS_SIZE($sp) - /* restore $28 & $26 & fp */ - ldl $28, 0($sp) - ldl $26, 0x10($sp) + /* only restore $fp */ ldl $15, 0x18($sp) addl $sp, 0x20, $sp .endm -- Gitee From ab4f2e98ea610825643786a4c0bf42709116866f Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Wed, 16 Apr 2025 16:43:37 +0800 Subject: [PATCH 453/524] sw64: ftrace: add notrace attribute to memset&memcpy In ftrace_call(), memset&memcpy may be used. However, when memset&memcpy are traced by ftrace, the memset&memcpy in ftrace_call() will trigger ftrace_call() again, causing an infinite loop and eventually leading to a stack overflow. This patch fixes this by adding the notrace attribute to functions in arch/sw64/lib/string.c. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/lib/string.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/sw_64/lib/string.c b/arch/sw_64/lib/string.c index e1ff596d8667..5eed2016dc61 100644 --- a/arch/sw_64/lib/string.c +++ b/arch/sw_64/lib/string.c @@ -19,14 +19,14 @@ static inline void *____memcpy(void *dest, const void *src, size_t n) return ____memcpy_simd_align(dest, src, n); } -void *memcpy(void *dest, const void *src, size_t n) +void notrace *memcpy(void *dest, const void *src, size_t n) { return ____memcpy(dest, src, n); } EXPORT_SYMBOL(memcpy); /* For backward compatibility with modules. Unused otherwise. */ -void *__memcpy(void *dest, const void *src, size_t n) +void notrace *__memcpy(void *dest, const void *src, size_t n) { return ____memcpy(dest, src, n); } @@ -47,12 +47,12 @@ static inline void *____constant_c_memset(void *s, unsigned long c, size_t n) return ____constant_c_memset_simd_align(s, c, n); } -void *__constant_c_memset(void *s, unsigned long c, size_t n) +void notrace *__constant_c_memset(void *s, unsigned long c, size_t n) { return ____constant_c_memset(s, c, n); } -void *___memset(void *s, int c, size_t n) +void notrace *___memset(void *s, int c, size_t n) { unsigned long c_ul = (c & 0xff) * 0x0101010101010101UL; @@ -60,7 +60,7 @@ void *___memset(void *s, int c, size_t n) } EXPORT_SYMBOL(___memset); -void *__memset(void *s, int c, size_t n) +void notrace *__memset(void *s, int c, size_t n) { unsigned long c_ul = (c & 0xff) * 0x0101010101010101UL; @@ -68,7 +68,7 @@ void *__memset(void *s, int c, size_t n) } EXPORT_SYMBOL(__memset); -void *memset(void *s, int c, size_t n) +void notrace *memset(void *s, int c, size_t n) { unsigned long c_ul = (c & 0xff) * 0x0101010101010101UL; @@ -76,7 +76,7 @@ void *memset(void *s, int c, size_t n) } EXPORT_SYMBOL(memset); -void *__memsetw(void *dest, unsigned short c, size_t count) +void notrace *__memsetw(void *dest, unsigned short c, size_t count) { unsigned long c_ul = (c & 0xffff) * 0x0001000100010001UL; -- Gitee From c5e6e86929abdb72084c9c3f784d858857bab284 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Fri, 18 Apr 2025 10:39:05 +0800 Subject: [PATCH 454/524] sw64: msi: clear the residual vector_irq information of cpu When an irq is migrated, the irq may on the way to prev_cpu. If we unbind the driver immediately, we have to clear the residual vector_irq information of prev_cpu when msi_domain free irqs. So irq on the way can't get the vector_irq information that irq has been released. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-msi-vt.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/irqchip/irq-sunway-msi-vt.c b/drivers/irqchip/irq-sunway-msi-vt.c index ac2e78757b51..8855634465f7 100644 --- a/drivers/irqchip/irq-sunway-msi-vt.c +++ b/drivers/irqchip/irq-sunway-msi-vt.c @@ -258,6 +258,8 @@ static void sw64_vt_vector_free_irqs(struct irq_domain *domain, cdata = irq_data->chip_data; irq_domain_reset_irq_data(irq_data); per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; + if (cdata->move_in_progress) + per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector] = 0; kfree(cdata); raw_spin_unlock_irqrestore(&vector_lock, flags); -- Gitee From 087282b1c4eeb2132cc483570224090843640210 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Tue, 14 May 2024 17:43:02 +0000 Subject: [PATCH 455/524] sw64: msi: modify msi migration strategy Based on the issues that occur with testing cpuhotplug, modify msi interrupt migration strategy as follows: - If cdata have move_in_progress set, msi interrupt cannot be migrated. - If old cpu is offline, the migrating irq can then be immediately removed from the old cpu's vector_irq table. - If the remaining vector resource cannot hold all migrating irqs, then cpu is not allowed to be disabled. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/irq.c | 49 +++++++++++++++++++++++++++++++++++++++++ arch/sw_64/kernel/smp.c | 8 +++++++ 2 files changed, 57 insertions(+) diff --git a/arch/sw_64/kernel/irq.c b/arch/sw_64/kernel/irq.c index c650e3a4faf0..78cac3ce2056 100644 --- a/arch/sw_64/kernel/irq.c +++ b/arch/sw_64/kernel/irq.c @@ -110,6 +110,55 @@ void fixup_irqs(void) mdelay(1); } + +#ifdef CONFIG_SW64_IRQ_MSI +static int cpu_vector_available(int cpu) +{ + int vector, max_vector = 256; + int avl_vector = 0; + + for (vector = 0; vector < max_vector; vector++) + if (per_cpu(vector_irq, cpu)[vector] == 0) + avl_vector++; + + return avl_vector; +} + +static int cpu_vector_tomove(int cpu) +{ + int max_vector = 256; + + return max_vector - cpu_vector_available(cpu); +} + +static int vector_available(void) +{ + int cpu, avl_vector = 0; + + for_each_online_cpu(cpu) + avl_vector += cpu_vector_available(cpu); + + return avl_vector; +} + +int can_unplug_cpu(void) +{ + unsigned int free, tomove; + unsigned int cpu = smp_processor_id(); + + tomove = cpu_vector_tomove(cpu); + free = vector_available(); + if (free < tomove) { + pr_info("CPU %u has %u vectors, %u available, Cannot disable CPU\n", + cpu, tomove, free); + return -ENOSPC; + } + + return 0; +} +#else +int can_unplug_cpu(void) { return 0; } +#endif #endif void __init init_IRQ(void) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 484605689a84..3502decc5f3e 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -802,9 +802,17 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) EXPORT_SYMBOL(flush_tlb_kernel_range); #ifdef CONFIG_HOTPLUG_CPU +extern int can_unplug_cpu(void); int __cpu_disable(void) { int cpu = smp_processor_id(); + int ret; + + if (is_in_host()) { + ret = can_unplug_cpu(); + if (ret) + return ret; + } set_cpu_online(cpu, false); remove_cpu_topology(cpu); -- Gitee From 3290037a125340793c2608f7e201cf432d448cce Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Fri, 11 Apr 2025 09:42:30 +0800 Subject: [PATCH 456/524] sw64: msi: fix guest msi interrupt handling This patch: - If cdata->move_in_progress is set and dst_cpu is online, msi interrupt cannot be migrated. - If old cpu is offline, the migrating irq can then be immediately removed from the old cpu's vector_irq table. - If the remaining vector resource cannot hold all migrating irqs, then cpu is not allowed to be disabled. - Retain the original affinity during migration and introduce effective affinity (irq_data_update_effective_affinity). - Add vector_lock before modify vector_irq. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 8 +++----- drivers/irqchip/irq-sunway-msi-vt.c | 28 ++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 3502decc5f3e..3327f59f43d8 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -808,11 +808,9 @@ int __cpu_disable(void) int cpu = smp_processor_id(); int ret; - if (is_in_host()) { - ret = can_unplug_cpu(); - if (ret) - return ret; - } + ret = can_unplug_cpu(); + if (ret) + return ret; set_cpu_online(cpu, false); remove_cpu_topology(cpu); diff --git a/drivers/irqchip/irq-sunway-msi-vt.c b/drivers/irqchip/irq-sunway-msi-vt.c index 8855634465f7..98b1ecbf75f1 100644 --- a/drivers/irqchip/irq-sunway-msi-vt.c +++ b/drivers/irqchip/irq-sunway-msi-vt.c @@ -135,13 +135,14 @@ vt_set_affinity(struct irq_data *d, const struct cpumask *cpumask, struct irq_data *irqd; struct cpumask searchmask; int cpu, vector; + unsigned long flags; /* Is this valid ? */ if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) return -EINVAL; irqd = irq_domain_get_irq_data(vt_msi_default_domain->parent, d->irq); - if (!irqd_is_started(irqd)) + if (!irqd_is_started(irqd) && !irqd_affinity_is_managed(irqd)) return IRQ_SET_MASK_OK; cdata = irqd->chip_data; @@ -155,21 +156,35 @@ vt_set_affinity(struct irq_data *d, const struct cpumask *cpumask, if (cpu_online(cdata->dst_cpu) && cpumask_test_cpu(cdata->dst_cpu, cpumask)) return IRQ_SET_MASK_OK; + if (cdata->move_in_progress && cpu_online(cdata->dst_cpu)) + return -EBUSY; + + raw_spin_lock_irqsave(&vector_lock, flags); + cpumask_and(&searchmask, cpumask, cpu_online_mask); - if (!vt_find_free_cpu_vector(&searchmask, &cpu, &vector, irqd)) + if (!vt_find_free_cpu_vector(&searchmask, &cpu, &vector, irqd)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); return -ENOSPC; + } per_cpu(vector_irq, cpu)[vector] = irqd->irq; spin_lock(&cdata->cdata_lock); - cdata->prev_cpu = cdata->dst_cpu; - cdata->prev_vector = cdata->vector; + + if (cpu_online(cdata->dst_cpu)) { + cdata->move_in_progress = true; + cdata->prev_vector = cdata->vector; + cdata->prev_cpu = cdata->dst_cpu; + } else { + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; + } cdata->dst_cpu = cpu; cdata->vector = vector; - cdata->move_in_progress = true; spin_unlock(&cdata->cdata_lock); - irq_data_update_effective_affinity(irqd, &searchmask); + irq_data_update_effective_affinity(d, cpumask_of(cpu)); vt_irq_msi_update_msg(irqd, irqd->chip_data); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return 0; } @@ -234,6 +249,7 @@ int chip_setup_vt_msi_irqs(int virq, unsigned int nr_irqs, cdata->prev_cpu = cpu; cdata->prev_vector = vector; cdata->move_in_progress = false; + irq_data_update_effective_affinity(irq_data, cpumask_of(cpu)); irq_data->chip_data = cdata; } -- Gitee From f593002df3b8a9095d2f927f67e834b1844a15a9 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Wed, 16 Apr 2025 09:06:00 +0000 Subject: [PATCH 457/524] sw64: gpu: fix SIMD IO access issues This patch resolves issues related to SIMD IO access on GPU: - Set contiguous VRAM addresses to iomem type to avoid memcpy. - Prevent optimization of a zeroing loop. There is a loop that assigns zero to IO space, and Gcc 12.3 will optimize this loop into a memset call. So mark the loop variable as volatile to avoid incorrect optimization. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c | 4 ++++ drivers/gpu/drm/ttm/ttm_resource.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index b7441654e6fa..0c24d26110c8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -1196,7 +1196,11 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, { struct amdgpu_device *adev = ring->adev; struct amdgpu_bo *bo = adev->uvd.ib_bo; +#if IS_ENABLED(CONFIG_SW64) + volatile uint32_t *msg; +#else uint32_t *msg; +#endif int i; msg = amdgpu_bo_kptr(bo); diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 46ff9c75bb12..68b02e46c061 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -665,7 +665,11 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, } if (mem->bus.addr) { +#if IS_ENABLED(CONFIG_SW64) + iosys_map_set_vaddr_iomem(&iter_io->dmap, mem->bus.addr); +#else iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr); +#endif iter_io->needs_unmap = false; } else { iter_io->needs_unmap = true; -- Gitee From 70da9cd16da9386754b96a055ff45bae9d1409ff Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Mon, 21 Apr 2025 07:31:53 +0000 Subject: [PATCH 458/524] sw64: msi: clear residual vector irq mapping when free irqs This path complements commit 7b9e38345558 ("sw64: msi: clear the residual vector_irq information of cpu") by addressing the same problem on host OS. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-msi-v2.c | 6 +++++- drivers/irqchip/irq-sunway-msi.c | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c index 283e190cd72c..8ac98399b5d9 100644 --- a/drivers/irqchip/irq-sunway-msi-v2.c +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -333,9 +333,13 @@ static void sw64_vector_free_irqs(struct irq_domain *domain, irq_domain_reset_irq_data(irq_data); cdata->multi_msi--; per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; + if (cdata->move_in_progress) + per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector] = 0; - if (cdata->multi_msi) + if (cdata->multi_msi) { cdata->vector++; + cdata->prev_vector++; + } if (cdata->multi_msi == 0) kfree(cdata); diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index fb1dd4c60bb1..024d20c25dd7 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -293,6 +293,8 @@ static void sw64_vector_free_irqs(struct irq_domain *domain, } irq_domain_reset_irq_data(irq_data); per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; + if (cdata->move_in_progress) + per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector] = 0; kfree(cdata); raw_spin_unlock_irqrestore(&vector_lock, flags); } -- Gitee From 6da257f12b456c621f5327a53c059ec05ac6d4ae Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Thu, 17 Apr 2025 04:13:50 +0000 Subject: [PATCH 459/524] sw64: iommu: work around iova range check after resv_region 3.5G ~ 4G currently is seen as PCI 32-bit MEMIO space. In theory, this space should be excluded from memory space addressing (using resv_region APIs), which will leave a memory hole on the entire memory space naturally. However, some applications(especially qemu) under sunway do not support incontiguous memory allocation right now. This memory hole has to be seen as one of the valid IOVA ranges to pass VFIO validness check for qemu. In this case, CPU is still capable of allocating IOVA in this space, which is, frankly speaking, dangerous and buggy. We manage to find a compromise solution, which is allow these IOVA being allocated and "mapped" as usual, but with a warning issued to users at the same time. So users can quickly learn if they are using these "illegal" IOVA and thus change their strategies accordingly. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu_v2.c | 72 ++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index 65e8afb2099f..7ac13d6732f0 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -431,7 +432,7 @@ set_entry_by_devid(u16 devid, dte_l2_val |= 0x1; *dte_l2 = dte_l2_val; - printk("device with id %d added to domain: %d with pte_root: %lx\n", + pr_debug("device with id %d added to domain: %d with pte_root: %lx\n", devid, sdomain->id, dte_l2_val); return 0; @@ -1270,7 +1271,7 @@ static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type) } sdomain->domain.geometry.aperture_start = 0UL; - sdomain->domain.geometry.aperture_end = SW64_64BIT_DMA_LIMIT; + sdomain->domain.geometry.aperture_end = ~0UL; sdomain->domain.geometry.force_aperture = true; sdomain->type = IOMMU_DOMAIN_UNMANAGED; break; @@ -1388,9 +1389,16 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) unsigned long paddr, grn; unsigned long is_last; + if ((iova > SW64_32BIT_DMA_LIMIT) + && (iova <= DMA_BIT_MASK(32))) + return iova; + if (iova >= SW64_BAR_ADDRESS) return iova; + if (iova >= MAX_IOVA_WIDTH) + return 0; + paddr = fetch_pte(sdomain, iova, PTE_LEVEL1_VAL); if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) return 0; @@ -1449,21 +1457,41 @@ sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, int ret; /* - * As VFIO cannot distinguish between normal DMA request - * and pci device BAR, check should be introduced manually - * to avoid VFIO trying to map pci config space. + * 3.5G ~ 4G currently is seen as PCI 32-bit MEMIO space. In theory, + * this space should be excluded from memory space addressing (using + * resv_region APIs), which will leave a memory hole on the entire memory + * space naturally. + * + * However, some applications(especially qemu) under sunway do not + * support incontiguous memory allocation right now. This memory + * hole has to be seen as one of the valid IOVA ranges to pass VFIO + * validness check for qemu. In this case, CPU is still capable of + * allocating IOVA in this space, which is, frankly speaking, dangerous + * and buggy. + * + * We manage to find a compromise solution, which is allow these IOVA + * being allocated and "mapped" as usual, but with a warning issued to + * users at the same time. So users can quickly learn if they are using + * these "illegal" IOVA and thus change their strategies accordingly. */ - if (iova > IO_BASE) { - pr_err("iova %#lx is out of memory!\n", iova); - return -ENOMEM; + if ((iova > SW64_32BIT_DMA_LIMIT) + && (iova <= DMA_BIT_MASK(32))) { + pr_warn_once("Domain %d are using IOVA: %lx\n", sdomain->id, iova); + return 0; } - if (iova >= SW64_BAR_ADDRESS) + /* + * For the same reason, IOVA allocated from PCI dev BAR address should + * be warned as well. + */ + if (iova >= SW64_BAR_ADDRESS) { + pr_warn_once("Domain %d are using IOVA: %lx\n", sdomain->id, iova); return 0; + } /* IOMMU v2 supports 42 bit mapped address width*/ if (iova >= MAX_IOVA_WIDTH) { - pr_err("IOMMU cannot map provided address: %lx\n", iova); + pr_err("The IOMMU hardware cannot map provided address: %lx\n", iova); return -EFAULT; } @@ -1479,13 +1507,17 @@ sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); size_t unmap_size; + if ((iova > SW64_32BIT_DMA_LIMIT) + && (iova <= DMA_BIT_MASK(32))) + return page_size; + if (iova >= SW64_BAR_ADDRESS) return page_size; - /* IOMMU v2 supports 42 bit mapped address width*/ + /* IOMMU v2 supports 42 bit mapped address width */ if (iova >= MAX_IOVA_WIDTH) { pr_err("Trying to unmap illegal IOVA : %lx\n", iova); - return -EFAULT; + return 0; } unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); @@ -1625,8 +1657,22 @@ static void sunway_iommu_get_resv_regions(struct device *dev, struct list_head *head) { struct iommu_resv_region *region; + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + goto do_resv; + /* + * Allow user applications have access to a contiguous memory space, + * so no reserves for unmanaged domains. + * + * See comments in map API for more detail. + */ + if (domain->type == IOMMU_DOMAIN_UNMANAGED) + return; - /* Reserve 3.5~4G for device */ +do_resv: + /* Reserve 3.5~4G for MEMIO */ region = iommu_alloc_resv_region(SW64_32BIT_DMA_LIMIT, (DMA_BIT_MASK(32) - SW64_32BIT_DMA_LIMIT), IOMMU_NOEXEC | IOMMU_MMIO, -- Gitee From 4d614738d28e96d02c8dc00d263d1e0f5509eb4a Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Tue, 13 May 2025 14:36:44 +0800 Subject: [PATCH 460/524] sw64: vdso: adapt to the upstream changes in vdso_install Fix it according to upstream commit 56769ba4b297 ("kbuild: unify vdso_install rules"). Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Makefile | 4 +--- arch/sw_64/kernel/vdso/Makefile | 11 ----------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index 41b90403ee3c..b78a2f4d915c 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -49,9 +49,7 @@ export LIBS_Y boot := arch/sw_64/boot -PHONY += vdso_install -vdso_install: - $(Q)$(MAKE) $(build)=arch/sw_64/kernel/vdso $@ +vdso-install-y += arch/sw_64/vdso/vdso.so.dbg #Default target when executing make with no arguments all: $(boot)/vmlinux.bin.gz diff --git a/arch/sw_64/kernel/vdso/Makefile b/arch/sw_64/kernel/vdso/Makefile index 51db491cb418..6987903e60e4 100644 --- a/arch/sw_64/kernel/vdso/Makefile +++ b/arch/sw_64/kernel/vdso/Makefile @@ -69,14 +69,3 @@ quiet_cmd_vdsold = VDSOLD $@ # that contains the same symbols at the same offsets. quiet_cmd_so2s = SO2S $@ cmd_so2s = $(NM) -D $< | $(srctree)/$(src)/so2s.sh > $@ - -# install commands for the unstripped file -quiet_cmd_vdso_install = INSTALL $@ - cmd_vdso_install = cp $(obj)/$@.dbg $(MODLIB)/vdso/$@ - -vdso.so: $(obj)/vdso.so.dbg - @mkdir -p $(MODLIB)/vdso - $(call cmd,vdso_install) - - -vdso_install: vdso.so -- Gitee From 4bf42a177b015952a92152e45df501b433457ff2 Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Tue, 13 May 2025 14:46:29 +0800 Subject: [PATCH 461/524] sw64: adapt to the upstream changes in arch_static_branch* Fix it according to upstream commit 4356e9f841f7 ("work around gcc bugs with 'asm goto' with outputs"). Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/jump_label.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/arch/sw_64/include/asm/jump_label.h b/arch/sw_64/include/asm/jump_label.h index 32fbf7573b20..27898e8e2c75 100644 --- a/arch/sw_64/include/asm/jump_label.h +++ b/arch/sw_64/include/asm/jump_label.h @@ -12,12 +12,12 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm_volatile_goto("1: nop\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - ".align 3\n\t" - ".quad 1b, %l[l_yes], %0\n\t" - ".popsection\n\t" - : : "i"(&((char *)key)[branch]) : : l_yes); + asm goto("1: nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 3\n\t" + ".quad 1b, %l[l_yes], %0\n\t" + ".popsection\n\t" + : : "i"(&((char *)key)[branch]) : : l_yes); return false; l_yes: @@ -26,12 +26,12 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) { - asm_volatile_goto("1: br %l[l_yes]\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - ".align 3\n\t" - ".quad 1b, %l[l_yes], %0\n\t" - ".popsection\n\t" - : : "i"(&((char *)key)[branch]) : : l_yes); + asm goto("1: br %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 3\n\t" + ".quad 1b, %l[l_yes], %0\n\t" + ".popsection\n\t" + : : "i"(&((char *)key)[branch]) : : l_yes); return false; l_yes: -- Gitee From 79544d91a06e86d0bd1c367f67034b7c7e79b7a6 Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Tue, 13 May 2025 14:54:38 +0800 Subject: [PATCH 462/524] sw64: adapt to the upstream changes in early_init_dt_(scan/verify) Fix it according to upstream commit b2473a359763 ("of/fdt: add dt_phys arg to early_init_dt_scan and early_init_dt_verify"). Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index e3608c601cba..f8ca94ce24c6 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -483,7 +483,7 @@ static void __init setup_firmware_fdt(void) memblock_reserve(__boot_pa(dt_virt), fdt_totalsize(dt_virt)); if (!arch_dtb_verify(dt_virt, true) || - !early_init_dt_scan(dt_virt)) { + !early_init_dt_scan(dt_virt, __boot_pa(dt_virt))) { pr_crit("Invalid DTB(from firmware) at virtual address 0x%lx\n", (unsigned long)dt_virt); @@ -578,7 +578,7 @@ static void __init setup_builtin_fdt(void) dt_virt = (void *)__dtb_start; if (!arch_dtb_verify(dt_virt, false) || - !early_init_dt_verify(dt_virt)) { + !early_init_dt_verify(dt_virt, __boot_pa(dt_virt))) { pr_crit("Invalid DTB(built-in) at virtual address 0x%lx\n", (unsigned long)dt_virt); while (true) -- Gitee From 7990f400e56061a5563ed14e1a48aa291782511e Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Tue, 13 May 2025 15:18:59 +0800 Subject: [PATCH 463/524] sw64: adapt to the upstream changes in huge_ptep_get_and_clear Fix it according to upstream commit 02410ac72ac3 ("mm: hugetlb: Add huge page size param to huge_ptep_get_and_clear()"). Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hugetlb.h | 2 +- arch/sw_64/mm/hugetlbpage_c4.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/include/asm/hugetlb.h b/arch/sw_64/include/asm/hugetlb.h index f4c8cbe0891a..eeeef6530a57 100644 --- a/arch/sw_64/include/asm/hugetlb.h +++ b/arch/sw_64/include/asm/hugetlb.h @@ -15,7 +15,7 @@ extern void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR extern pte_t huge_ptep_get_and_clear(struct mm_struct *mm, - unsigned long addr, pte_t *ptep); + unsigned long addr, pte_t *ptep, unsigned long sz); #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH extern pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, diff --git a/arch/sw_64/mm/hugetlbpage_c4.c b/arch/sw_64/mm/hugetlbpage_c4.c index 2611af9cf780..eaeeaadb6546 100644 --- a/arch/sw_64/mm/hugetlbpage_c4.c +++ b/arch/sw_64/mm/hugetlbpage_c4.c @@ -280,7 +280,8 @@ void huge_ptep_set_wrprotect(struct mm_struct *mm, } pte_t huge_ptep_get_and_clear(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) + unsigned long addr, pte_t *ptep, + unsigned long sz) { int ncontig; size_t pgsize; -- Gitee From 10d816fafab6c38ccae43891201649883dea930e Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 11 Mar 2025 15:32:25 +0800 Subject: [PATCH 464/524] sw64: cpufreq: remove debugfs code Since cpufreq is already supported, the legacy debugfs interface is no longer needed. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/cpufreq/Kconfig | 9 --- drivers/cpufreq/Makefile | 1 - drivers/cpufreq/sw64_cpufreq_debugfs.c | 94 -------------------------- 3 files changed, 104 deletions(-) delete mode 100644 drivers/cpufreq/sw64_cpufreq_debugfs.c diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 80d3dc2505bc..4c13741a744f 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -325,15 +325,6 @@ config SW64_CPUFREQ For details, take a look at . If unsure, say N. - -config SW64_CPUFREQ_DEBUGFS - bool "SW64 CPU Frequency debugfs interface" - depends on SW64_CPUFREQ && DEBUG_FS - default y - help - Turns on the DebugFS interface for CPU Frequency. - - If you don't know what to do here, say N. endif config QORIQ_CPUFREQ diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index f9c1c9012ce7..844b67530491 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -109,4 +109,3 @@ obj-$(CONFIG_SH_CPU_FREQ) += sh-cpufreq.o obj-$(CONFIG_SPARC_US2E_CPUFREQ) += sparc-us2e-cpufreq.o obj-$(CONFIG_SPARC_US3_CPUFREQ) += sparc-us3-cpufreq.o obj-$(CONFIG_SW64_CPUFREQ) += sw64_cpufreq.o -obj-$(CONFIG_SW64_CPUFREQ_DEBUGFS) += sw64_cpufreq_debugfs.o diff --git a/drivers/cpufreq/sw64_cpufreq_debugfs.c b/drivers/cpufreq/sw64_cpufreq_debugfs.c deleted file mode 100644 index e09c63495a0e..000000000000 --- a/drivers/cpufreq/sw64_cpufreq_debugfs.c +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include - -#include -#include -#include - -/* Show cpufreq in Mhz */ -static int cpufreq_show(struct seq_file *m, void *v) -{ - int i; - u64 val; - void __iomem *spbu_base = misc_platform_get_spbu_base(0); - - val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL2_CFG_SHIFT; - val &= CORE_PLL2_CFG_MASK; - seq_puts(m, "CPU frequency in Mhz:\n"); - for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { - if (freq_table[i].frequency == CPUFREQ_ENTRY_INVALID) - continue; - if (val == i) - seq_printf(m, "[%d] ", freq_table[i].frequency / 1000); - else - seq_printf(m, "%d ", freq_table[i].frequency / 1000); - } - seq_puts(m, "\n"); - - return 0; -} - -static int cpufreq_open(struct inode *inode, struct file *file) -{ - return single_open(file, cpufreq_show, NULL); -} - -static ssize_t cpufreq_set(struct file *file, const char __user *user_buf, - size_t len, loff_t *ppos) -{ - char buf[5]; - size_t size; - int cf, i, err, index, freq; - - size = min(sizeof(buf) - 1, len); - if (copy_from_user(buf, user_buf, size)) - return -EFAULT; - buf[size] = '\0'; - - err = kstrtoint(buf, 10, &cf); - if (err) - return err; - cf *= 1000; /* convert Mhz to khz */ - index = -1; - for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { - if (cf == freq_table[i].frequency) { - index = i; - break; - } - } - - if (index < 0) - return -EINVAL; - - sw64_set_rate(index); - update_cpu_freq(freq); - return len; -} - -static const struct file_operations set_cpufreq_fops = { - .open = cpufreq_open, - .read = seq_read, - .write = cpufreq_set, - .llseek = seq_lseek, - .release = single_release, -}; - -static int __init cpufreq_debugfs_init(void) -{ - struct dentry *cpufreq_entry; - - if (!sw64_debugfs_dir) - return -ENODEV; - - cpufreq_entry = debugfs_create_file("cpufreq", 0600, - sw64_debugfs_dir, NULL, - &set_cpufreq_fops); - if (!cpufreq_entry) - return -ENOMEM; - - return 0; -} -late_initcall(cpufreq_debugfs_init); -- Gitee From 0b87f6ba1ddf75780f78cb3c589f90c3db0a684b Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 11 Mar 2025 16:34:57 +0800 Subject: [PATCH 465/524] sw64: cpufreq: rename sw64_cpufreq.c to sunway-cpufreq.c Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/cpufreq/Makefile | 2 +- drivers/cpufreq/{sw64_cpufreq.c => sunway-cpufreq.c} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename drivers/cpufreq/{sw64_cpufreq.c => sunway-cpufreq.c} (100%) diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 844b67530491..7deb3b194e5c 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -108,4 +108,4 @@ obj-$(CONFIG_LOONGSON3_ACPI_CPUFREQ) += loongson3-acpi-cpufreq.o obj-$(CONFIG_SH_CPU_FREQ) += sh-cpufreq.o obj-$(CONFIG_SPARC_US2E_CPUFREQ) += sparc-us2e-cpufreq.o obj-$(CONFIG_SPARC_US3_CPUFREQ) += sparc-us3-cpufreq.o -obj-$(CONFIG_SW64_CPUFREQ) += sw64_cpufreq.o +obj-$(CONFIG_SW64_CPUFREQ) += sunway-cpufreq.o diff --git a/drivers/cpufreq/sw64_cpufreq.c b/drivers/cpufreq/sunway-cpufreq.c similarity index 100% rename from drivers/cpufreq/sw64_cpufreq.c rename to drivers/cpufreq/sunway-cpufreq.c -- Gitee From f61fbb87880c61427256a01803b0274ad210778e Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 11 Mar 2025 18:52:45 +0800 Subject: [PATCH 466/524] sw64: cpufreq: move cpufreq notifier to its user Move cpufreq notifier to its user to make the code clean without any functional changes. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpufreq.h | 2 -- drivers/clocksource/timer-sw64.c | 26 +++++++++++++++++++++++++- drivers/cpufreq/sunway-cpufreq.c | 26 -------------------------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/arch/sw_64/include/asm/cpufreq.h b/arch/sw_64/include/asm/cpufreq.h index d89db83b42a6..b2b39e2f7476 100644 --- a/arch/sw_64/include/asm/cpufreq.h +++ b/arch/sw_64/include/asm/cpufreq.h @@ -83,7 +83,5 @@ int sw64_set_rate(unsigned int index); struct clk *sw64_clk_get(struct device *dev, const char *id); -void sw64_update_clockevents(unsigned long cpu, u32 freq); - unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy); #endif /* _ASM_SW64_CPUFREQ_H */ diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index 8b022f455c78..eed2540f7a14 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -385,7 +386,7 @@ static int timer_set_oneshot(struct clock_event_device *evt) return 0; } -void sw64_update_clockevents(unsigned long cpu, u32 freq) +static void sw64_update_clockevents(unsigned long cpu, u32 freq) { struct clock_event_device *swevt = &per_cpu(timer_events, cpu); @@ -398,6 +399,29 @@ void sw64_update_clockevents(unsigned long cpu, u32 freq) } } +static int sw64_cpufreq_notifier(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freqs = (struct cpufreq_freqs *)data; + unsigned long cpu = freqs->policy->cpu; + + if (val == CPUFREQ_POSTCHANGE) + sw64_update_clockevents(cpu, freqs->new * 1000); + + return 0; +} + +static struct notifier_block sw64_cpufreq_notifier_block = { + .notifier_call = sw64_cpufreq_notifier +}; + +static int __init register_cpufreq_notifier(void) +{ + return cpufreq_register_notifier(&sw64_cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); +} +arch_initcall(register_cpufreq_notifier); + /* * Setup the local timer for this CPU. Copy the initialized values * of the boot CPU and register the clock event in the framework. diff --git a/drivers/cpufreq/sunway-cpufreq.c b/drivers/cpufreq/sunway-cpufreq.c index 93163da904a2..294b0c606875 100644 --- a/drivers/cpufreq/sunway-cpufreq.c +++ b/drivers/cpufreq/sunway-cpufreq.c @@ -27,26 +27,6 @@ static uint nowait; static struct clk *cpuclk; - -static int sw64_cpu_freq_notifier(struct notifier_block *nb, - unsigned long val, void *data); - -static struct notifier_block sw64_cpufreq_notifier_block = { - .notifier_call = sw64_cpu_freq_notifier -}; - -static int sw64_cpu_freq_notifier(struct notifier_block *nb, - unsigned long val, void *data) -{ - struct cpufreq_freqs *freqs = (struct cpufreq_freqs *)data; - unsigned long cpu = freqs->policy->cpu; - - if (val == CPUFREQ_POSTCHANGE) - sw64_update_clockevents(cpu, freqs->new * 1000); - - return 0; -} - static unsigned int sw64_cpufreq_get(unsigned int cpu) { struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); @@ -153,18 +133,12 @@ static int __init cpufreq_init(void) pr_info("SW-64 CPU frequency driver\n"); - cpufreq_register_notifier(&sw64_cpufreq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - return cpufreq_register_driver(&sw64_cpufreq_driver); } static void __exit cpufreq_exit(void) { cpufreq_unregister_driver(&sw64_cpufreq_driver); - cpufreq_unregister_notifier(&sw64_cpufreq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - platform_driver_unregister(&platform_driver); } -- Gitee From 9a5285380cca94c10330a59ca71a8876e6662501 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 11 Mar 2025 20:00:55 +0800 Subject: [PATCH 467/524] sw64: cpufreq: remove some meaningless code The removed code makes no sense. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpufreq.h | 37 ----------------- arch/sw_64/kernel/cpuautoplug.c | 9 +++-- arch/sw_64/platform/cpufreq.c | 20 +-------- drivers/cpufreq/sunway-cpufreq.c | 69 ++++---------------------------- 4 files changed, 14 insertions(+), 121 deletions(-) diff --git a/arch/sw_64/include/asm/cpufreq.h b/arch/sw_64/include/asm/cpufreq.h index b2b39e2f7476..4438b143ab7b 100644 --- a/arch/sw_64/include/asm/cpufreq.h +++ b/arch/sw_64/include/asm/cpufreq.h @@ -10,38 +10,6 @@ #include #include -struct clk; - -struct clk_ops { - void (*init)(struct clk *clk); - void (*enable)(struct clk *clk); - void (*disable)(struct clk *clk); - void (*recalc)(struct clk *clk); - int (*set_rate)(struct clk *clk, unsigned long rate, int algo_id); - long (*round_rate)(struct clk *clk, unsigned long rate); -}; - -struct clk { - struct list_head node; - const char *name; - int id; - struct module *owner; - - struct clk *parent; - const struct clk_ops *ops; - - struct kref kref; - - unsigned long rate; - unsigned long flags; -}; - -#define CLK_ALWAYS_ENABLED (1 << 0) -#define CLK_RATE_PROPAGATES (1 << 1) - -#define CORE_CLK0_VALID (0x1UL << 1) -#define CORE_CLK0_RESET (0x1UL << 2) - #define CLK_LV1_SEL_PROTECT (0x1UL << 0) #define CLK_LV1_SEL_MUXA (0x1UL << 2) #define CLK_LV1_SEL_MUXB (0x1UL << 3) @@ -56,7 +24,6 @@ struct clk { #define CORE_PLL1_CFG_SHIFT 20 #define CORE_PLL2_CFG_SHIFT 36 #define CORE_PLL2_CFG_MASK 0x1f -#define STARTUP_RATE (2000UL * 1000 * 1000) #endif #ifdef CONFIG_UNCORE_XUELANG @@ -70,7 +37,6 @@ struct clk { #define CORE_PLL1_CFG_SHIFT 11 #define CORE_PLL2_CFG_SHIFT 18 #define CORE_PLL2_CFG_MASK 0xf -#define STARTUP_RATE (2400UL * 1000 * 1000) #endif #define OFFSET_CLU_LV1_SEL 0x3a80UL @@ -78,10 +44,7 @@ struct clk { extern struct cpufreq_frequency_table freq_table[]; -int clk_init(void); int sw64_set_rate(unsigned int index); -struct clk *sw64_clk_get(struct device *dev, const char *id); - unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy); #endif /* _ASM_SW64_CPUFREQ_H */ diff --git a/arch/sw_64/kernel/cpuautoplug.c b/arch/sw_64/kernel/cpuautoplug.c index 94a388b1116e..dadf95cf0aea 100644 --- a/arch/sw_64/kernel/cpuautoplug.c +++ b/arch/sw_64/kernel/cpuautoplug.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -13,7 +14,6 @@ #include #include -#include #include #include #include "../../../kernel/sched/sched.h" @@ -388,9 +388,10 @@ static void do_autoplug_timer(struct work_struct *work) long active; atomic_long_t calc_load_tasks; #endif - if (!policy || IS_ERR(policy->clk)) { - pr_err("%s: No %s associated to cpu: %d\n", - __func__, policy ? "clk" : "policy", 0); + + if (!policy) { + pr_err("%s: no policy associated to cpu: %d\n", + __func__, smp_processor_id()); return; } diff --git a/arch/sw_64/platform/cpufreq.c b/arch/sw_64/platform/cpufreq.c index e18201069f67..930ed4d1af08 100644 --- a/arch/sw_64/platform/cpufreq.c +++ b/arch/sw_64/platform/cpufreq.c @@ -15,11 +15,6 @@ #define MAX_RETRY 10 -static struct platform_device sw64_cpufreq_device = { - .name = "sw64_cpufreq", - .id = -1, -}; - /* * frequency in MHz, volts in mV and stored as "driver_data" in the structure. * volts 0 means to be determined @@ -118,21 +113,10 @@ static int __init sw64_cpufreq_init(void) if (max_rate == freq_table[i].frequency) freq_table[i+1].frequency = CPUFREQ_TABLE_END; } - return platform_device_register(&sw64_cpufreq_device); -} -arch_initcall(sw64_cpufreq_init); -static struct clk cpu_clk = { - .name = "cpu_clk", - .flags = CLK_ALWAYS_ENABLED | CLK_RATE_PROPAGATES, - .rate = STARTUP_RATE, -}; - -struct clk *sw64_clk_get(struct device *dev, const char *id) -{ - return &cpu_clk; + return 0; } -EXPORT_SYMBOL(sw64_clk_get); +arch_initcall(sw64_cpufreq_init); unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy) { diff --git a/drivers/cpufreq/sunway-cpufreq.c b/drivers/cpufreq/sunway-cpufreq.c index 294b0c606875..8fe7e929b3b8 100644 --- a/drivers/cpufreq/sunway-cpufreq.c +++ b/drivers/cpufreq/sunway-cpufreq.c @@ -1,20 +1,14 @@ // SPDX-License-Identifier: GPL-2.0 /* - * linux/arch/sw/kernel/setup.c - * - * Copyright (C) 1995 Linus Torvalds + * Copyright (C) 2025 WXIAT */ -/* - * Cpufreq driver for the sw64 processors - * - */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define pr_fmt(fmt) "cpufreq: " fmt #include #include #include -#include /* set_cpus_allowed() */ +#include #include #include #include @@ -23,17 +17,13 @@ #include #include -static uint nowait; - -static struct clk *cpuclk; - static unsigned int sw64_cpufreq_get(unsigned int cpu) { struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); - if (!policy || IS_ERR(policy->clk)) { - pr_err("%s: No %s associated to cpu: %d\n", - __func__, policy ? "clk" : "policy", cpu); + if (!policy) { + pr_err("%s: no policy associated to cpu: %d\n", + __func__, cpu); return 0; } @@ -63,14 +53,6 @@ static int sw64_cpufreq_target(struct cpufreq_policy *policy, static int sw64_cpufreq_cpu_init(struct cpufreq_policy *policy) { - cpuclk = sw64_clk_get(NULL, "cpu_clk"); - if (IS_ERR(cpuclk)) { - pr_err("couldn't get CPU clk\n"); - return PTR_ERR(cpuclk); - } - - policy->clk = cpuclk; - cpufreq_generic_init(policy, freq_table, 0); return 0; @@ -100,53 +82,16 @@ static struct cpufreq_driver sw64_cpufreq_driver = { .attr = sw64_table_attr, }; -static const struct platform_device_id platform_device_ids[] = { - { - .name = "sw64_cpufreq", - }, - {} -}; - -MODULE_DEVICE_TABLE(platform, platform_device_ids); - -static struct platform_driver platform_driver = { - .driver = { - .name = "sw64_cpufreq", - }, - .id_table = platform_device_ids, -}; - - static int __init cpufreq_init(void) { - int ret; - if (is_in_guest()) { pr_warn("Now sw_64 CPUFreq does not support virtual machines\n"); return -ENODEV; } - /* Register platform stuff */ - ret = platform_driver_register(&platform_driver); - if (ret) - return ret; - pr_info("SW-64 CPU frequency driver\n"); return cpufreq_register_driver(&sw64_cpufreq_driver); } +device_initcall(cpufreq_init); -static void __exit cpufreq_exit(void) -{ - cpufreq_unregister_driver(&sw64_cpufreq_driver); - platform_driver_unregister(&platform_driver); -} - -module_init(cpufreq_init); -module_exit(cpufreq_exit); - -module_param(nowait, uint, 0644); -MODULE_PARM_DESC(nowait, "Disable SW-64 specific wait"); - -MODULE_DESCRIPTION("cpufreq driver for sw64"); -MODULE_LICENSE("GPL"); -- Gitee From e313a7e8767a93f74ff86cc4929ceff93012ccb4 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 12 Mar 2025 09:05:38 +0800 Subject: [PATCH 468/524] sw64: cpufreq: refactor cpufreq related code This refactoring centralizes the cpufreq drivers of Sunway platforms in sunway-cpufreq.c, improving the maintainability of the code. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kbuild | 2 +- arch/sw_64/include/asm/cpufreq.h | 50 ------- arch/sw_64/platform/Makefile | 2 - arch/sw_64/platform/cpufreq.c | 183 ----------------------- drivers/cpufreq/sunway-cpufreq.c | 239 +++++++++++++++++++++++++++---- 5 files changed, 215 insertions(+), 261 deletions(-) delete mode 100644 arch/sw_64/include/asm/cpufreq.h delete mode 100644 arch/sw_64/platform/Makefile delete mode 100644 arch/sw_64/platform/cpufreq.c diff --git a/arch/sw_64/Kbuild b/arch/sw_64/Kbuild index aa0bf0507406..474e3479b5cb 100644 --- a/arch/sw_64/Kbuild +++ b/arch/sw_64/Kbuild @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-y += kernel/ mm/ platform/ +obj-y += kernel/ mm/ obj-$(CONFIG_NET) += net/ obj-$(CONFIG_KVM) += kvm/ obj-$(CONFIG_MATHEMU) += math-emu/ diff --git a/arch/sw_64/include/asm/cpufreq.h b/arch/sw_64/include/asm/cpufreq.h deleted file mode 100644 index 4438b143ab7b..000000000000 --- a/arch/sw_64/include/asm/cpufreq.h +++ /dev/null @@ -1,50 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef _ASM_SW64_CPUFREQ_H -#define _ASM_SW64_CPUFREQ_H - -#include -#include -#include -#include -#include -#include - -#define CLK_LV1_SEL_PROTECT (0x1UL << 0) -#define CLK_LV1_SEL_MUXA (0x1UL << 2) -#define CLK_LV1_SEL_MUXB (0x1UL << 3) - -#ifdef CONFIG_UNCORE_JUNZHANG -#define CLK0_PROTECT (0x1UL << 0) -#define CLK2_PROTECT (0x1UL << 32) -#define CORE_CLK2_VALID (0x1UL << 33) -#define CORE_CLK2_RESET (0x1UL << 34) -#define CORE_CLK2_LOCK (0x1UL << 35) -#define CORE_PLL0_CFG_SHIFT 4 -#define CORE_PLL1_CFG_SHIFT 20 -#define CORE_PLL2_CFG_SHIFT 36 -#define CORE_PLL2_CFG_MASK 0x1f -#endif - -#ifdef CONFIG_UNCORE_XUELANG -#define CLK_PROTECT (0x1UL << 0) -#define CLK0_PROTECT CLK_PROTECT -#define CLK2_PROTECT CLK_PROTECT -#define CORE_CLK2_VALID (0x1UL << 15) -#define CORE_CLK2_RESET (0x1UL << 16) -#define CORE_CLK2_LOCK (0x1UL << 17) -#define CORE_PLL0_CFG_SHIFT 4 -#define CORE_PLL1_CFG_SHIFT 11 -#define CORE_PLL2_CFG_SHIFT 18 -#define CORE_PLL2_CFG_MASK 0xf -#endif - -#define OFFSET_CLU_LV1_SEL 0x3a80UL -#define OFFSET_CLK_CTL 0x3b80UL - -extern struct cpufreq_frequency_table freq_table[]; - -int sw64_set_rate(unsigned int index); - -unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy); -#endif /* _ASM_SW64_CPUFREQ_H */ diff --git a/arch/sw_64/platform/Makefile b/arch/sw_64/platform/Makefile deleted file mode 100644 index 09ed3eb0f743..000000000000 --- a/arch/sw_64/platform/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-y += cpufreq.o diff --git a/arch/sw_64/platform/cpufreq.c b/arch/sw_64/platform/cpufreq.c deleted file mode 100644 index 930ed4d1af08..000000000000 --- a/arch/sw_64/platform/cpufreq.c +++ /dev/null @@ -1,183 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#define MAX_RETRY 10 - -/* - * frequency in MHz, volts in mV and stored as "driver_data" in the structure. - * volts 0 means to be determined - */ -#define FV(mhz, mv) \ - { \ - .frequency = (mhz) * 1000, \ - .driver_data = (mv) \ - } - -#ifdef CONFIG_PLATFORM_JUNZHANG -struct cpufreq_frequency_table freq_table[] = { - {0, 0, CPUFREQ_ENTRY_INVALID}, /* 200Mhz is ignored */ - FV(1200, 850), - FV(1300, 850), - FV(1400, 850), - FV(1450, 850), - FV(1500, 850), - FV(1550, 850), - FV(1600, 850), - FV(1650, 900), - FV(1700, 900), - FV(1750, 900), - FV(1800, 900), - FV(1850, 900), - FV(1900, 900), - FV(1950, 900), - FV(2000, 900), - FV(2050, 950), - FV(2100, 950), - FV(2150, 950), - FV(2200, 950), - FV(2250, 0), - FV(2300, 0), - FV(2350, 0), - FV(2400, 0), - FV(2450, 0), - FV(2500, 0), - FV(2550, 0), - FV(2600, 0), - FV(2650, 0), - FV(2700, 0), - FV(2800, 0), - FV(2850, 0), - {0, 0, CPUFREQ_TABLE_END}, -}; -static void __init fill_freq_table(struct cpufreq_frequency_table *ft) -{ -} -#elif CONFIG_PLATFORM_XUELANG -struct cpufreq_frequency_table freq_table[] = { - {0, 0, CPUFREQ_ENTRY_INVALID}, /* 200Mhz is ignored */ - {0, 0, CPUFREQ_ENTRY_INVALID}, /* 1200Mhz is ignored */ - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_ENTRY_INVALID}, - {0, 0, CPUFREQ_TABLE_END}, -}; - -static void __init fill_freq_table(struct cpufreq_frequency_table *ft) -{ - int i; - unsigned long freq_off; - unsigned char external_clk; - external_clk = *((unsigned char *)__va(MB_EXTCLK)); - - if (external_clk == 240) - freq_off = 60000; - else - freq_off = 50000; - - freq_table[2].frequency = freq_off * 36; - for (i = 3; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) - freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); -} -#endif - -static int __init sw64_cpufreq_init(void) -{ - int i; - unsigned long max_rate = get_cpu_freq() / 1000; - - fill_freq_table(freq_table); - for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { - if (max_rate == freq_table[i].frequency) - freq_table[i+1].frequency = CPUFREQ_TABLE_END; - } - - return 0; -} -arch_initcall(sw64_cpufreq_init); - -unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy) -{ - int i, clu_lv1_sel; - u64 val; - void __iomem *spbu_base = misc_platform_get_spbu_base(0); - struct cpufreq_frequency_table *ft = policy->freq_table; - - clu_lv1_sel = (readq(spbu_base + OFFSET_CLU_LV1_SEL) >> 2) & 0x3; - - if (clu_lv1_sel == 0) - val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL0_CFG_SHIFT; - else if (clu_lv1_sel == 2) - val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL1_CFG_SHIFT; - else - val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL2_CFG_SHIFT; - - val &= CORE_PLL2_CFG_MASK; - - for (i = 0; ft[i].frequency != CPUFREQ_TABLE_END; i++) { - if (val == i) - return ft[i].frequency; - } - return 0; -} -EXPORT_SYMBOL(__sw64_cpufreq_get); - -int sw64_set_rate(unsigned int index) -{ - int i, retry, cpu_num; - void __iomem *spbu_base; - - cpu_num = sw64_chip->get_cpu_num(); - for (i = 0; i < cpu_num; i++) { - spbu_base = misc_platform_get_spbu_base(i); - - /* select PLL0/PLL1 */ - writeq(CLK_LV1_SEL_PROTECT, spbu_base + OFFSET_CLU_LV1_SEL); - /* reset PLL2 */ - writeq(CLK2_PROTECT | CORE_CLK2_RESET | CORE_CLK2_VALID, spbu_base + OFFSET_CLK_CTL); - /* configure PLL2_CFG */ - writeq(CLK2_PROTECT | CORE_CLK2_VALID | (unsigned long)index << CORE_PLL2_CFG_SHIFT, - spbu_base + OFFSET_CLK_CTL); - udelay(1); - /* reset over */ - writeq(CORE_CLK2_VALID, spbu_base + OFFSET_CLK_CTL); - retry = 0; - while (retry < MAX_RETRY) { - if (readq(spbu_base + OFFSET_CLK_CTL) & CORE_CLK2_LOCK) - break; - retry++; - udelay(100); - } - if (retry == MAX_RETRY) - return -ETIME; - /* configure over */ - writeq(0, spbu_base + OFFSET_CLK_CTL); - /* select PLL2/PLL2 */ - writeq(CLK_LV1_SEL_MUXA | CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PROTECT, - spbu_base + OFFSET_CLU_LV1_SEL); - } - return 0; -} -EXPORT_SYMBOL_GPL(sw64_set_rate); diff --git a/drivers/cpufreq/sunway-cpufreq.c b/drivers/cpufreq/sunway-cpufreq.c index 8fe7e929b3b8..73dbb889063b 100644 --- a/drivers/cpufreq/sunway-cpufreq.c +++ b/drivers/cpufreq/sunway-cpufreq.c @@ -6,18 +6,196 @@ #define pr_fmt(fmt) "cpufreq: " fmt #include -#include #include -#include #include -#include -#include -#include -#include -#include +#include +#include -static unsigned int sw64_cpufreq_get(unsigned int cpu) +#define MAX_RETRY 10 + +#define CLK_LV1_SEL_PROTECT (0x1UL << 0) +#define CLK_LV1_SEL_MUXA (0x1UL << 2) +#define CLK_LV1_SEL_MUXB (0x1UL << 3) + +#define OFFSET_CLU_LV1_SEL 0x3a80UL +#define OFFSET_CLK_CTL 0x3b80UL + +/* + * frequency in MHz, volts in mV and stored as "driver_data" in the structure. + * volts 0 means to be determined + */ +#define FV(mhz, mv) \ + { \ + .frequency = (mhz) * 1000, \ + .driver_data = (mv) \ + } + +#ifdef CONFIG_PLATFORM_JUNZHANG +#define CLK0_PROTECT (0x1UL << 0) +#define CLK2_PROTECT (0x1UL << 32) +#define CORE_CLK2_VALID (0x1UL << 33) +#define CORE_CLK2_RESET (0x1UL << 34) +#define CORE_CLK2_LOCK (0x1UL << 35) +#define CORE_PLL0_CFG_SHIFT 4 +#define CORE_PLL1_CFG_SHIFT 20 +#define CORE_PLL2_CFG_SHIFT 36 +#define CORE_PLL2_CFG_MASK 0x1f + +struct cpufreq_frequency_table freq_table[] = { + {0, 0, CPUFREQ_ENTRY_INVALID}, /* 200Mhz is ignored */ + FV(1200, 850), + FV(1300, 850), + FV(1400, 850), + FV(1450, 850), + FV(1500, 850), + FV(1550, 850), + FV(1600, 850), + FV(1650, 900), + FV(1700, 900), + FV(1750, 900), + FV(1800, 900), + FV(1850, 900), + FV(1900, 900), + FV(1950, 900), + FV(2000, 900), + FV(2050, 950), + FV(2100, 950), + FV(2150, 950), + FV(2200, 950), + FV(2250, 0), + FV(2300, 0), + FV(2350, 0), + FV(2400, 0), + FV(2450, 0), + FV(2500, 0), + FV(2550, 0), + FV(2600, 0), + FV(2650, 0), + FV(2700, 0), + FV(2800, 0), + FV(2850, 0), + {0, 0, CPUFREQ_TABLE_END}, +}; +static void __init fill_freq_table(struct cpufreq_frequency_table *ft) +{ +} +#endif + +#ifdef CONFIG_PLATFORM_XUELANG +#define CLK_PROTECT (0x1UL << 0) +#define CLK0_PROTECT CLK_PROTECT +#define CLK2_PROTECT CLK_PROTECT +#define CORE_CLK2_VALID (0x1UL << 15) +#define CORE_CLK2_RESET (0x1UL << 16) +#define CORE_CLK2_LOCK (0x1UL << 17) +#define CORE_PLL0_CFG_SHIFT 4 +#define CORE_PLL1_CFG_SHIFT 11 +#define CORE_PLL2_CFG_SHIFT 18 +#define CORE_PLL2_CFG_MASK 0xf + +struct cpufreq_frequency_table freq_table[] = { + {0, 0, CPUFREQ_ENTRY_INVALID}, /* 200Mhz is ignored */ + {0, 0, CPUFREQ_ENTRY_INVALID}, /* 1200Mhz is ignored */ + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_ENTRY_INVALID}, + {0, 0, CPUFREQ_TABLE_END}, +}; + +static void __init fill_freq_table(struct cpufreq_frequency_table *ft) +{ + int i; + unsigned long freq_off; + unsigned char external_clk; + + external_clk = *((unsigned char *)__va(MB_EXTCLK)); + + if (external_clk == 240) + freq_off = 60000; + else + freq_off = 50000; + + freq_table[2].frequency = freq_off * 36; + for (i = 3; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) + freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); +} +#endif + +static unsigned int sunway_get_rate(struct cpufreq_policy *policy) +{ + int i, clu_lv1_sel; + u64 val; + void __iomem *spbu_base = misc_platform_get_spbu_base(0); + struct cpufreq_frequency_table *ft = policy->freq_table; + + clu_lv1_sel = (readq(spbu_base + OFFSET_CLU_LV1_SEL) >> 2) & 0x3; + + if (clu_lv1_sel == 0) + val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL0_CFG_SHIFT; + else if (clu_lv1_sel == 2) + val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL1_CFG_SHIFT; + else + val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL2_CFG_SHIFT; + + val &= CORE_PLL2_CFG_MASK; + + for (i = 0; ft[i].frequency != CPUFREQ_TABLE_END; i++) { + if (val == i) + return ft[i].frequency; + } + return 0; +} + +static int sunway_set_rate(unsigned int index) +{ + int i, retry, cpu_num; + void __iomem *spbu_base; + + cpu_num = sw64_chip->get_cpu_num(); + for (i = 0; i < cpu_num; i++) { + spbu_base = misc_platform_get_spbu_base(i); + + /* select PLL0/PLL1 */ + writeq(CLK_LV1_SEL_PROTECT, spbu_base + OFFSET_CLU_LV1_SEL); + /* reset PLL2 */ + writeq(CLK2_PROTECT | CORE_CLK2_RESET | CORE_CLK2_VALID, spbu_base + OFFSET_CLK_CTL); + /* configure PLL2_CFG */ + writeq(CLK2_PROTECT | CORE_CLK2_VALID | (unsigned long)index << CORE_PLL2_CFG_SHIFT, + spbu_base + OFFSET_CLK_CTL); + udelay(1); + /* reset over */ + writeq(CORE_CLK2_VALID, spbu_base + OFFSET_CLK_CTL); + retry = 0; + while (retry < MAX_RETRY) { + if (readq(spbu_base + OFFSET_CLK_CTL) & CORE_CLK2_LOCK) + break; + retry++; + udelay(100); + } + if (retry == MAX_RETRY) + return -ETIME; + /* configure over */ + writeq(0, spbu_base + OFFSET_CLK_CTL); + /* select PLL2/PLL2 */ + writeq(CLK_LV1_SEL_MUXA | CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PROTECT, + spbu_base + OFFSET_CLU_LV1_SEL); + } + return 0; +} + +static unsigned int sunway_cpufreq_get(unsigned int cpu) { struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); @@ -27,13 +205,13 @@ static unsigned int sw64_cpufreq_get(unsigned int cpu) return 0; } - return __sw64_cpufreq_get(policy); + return sunway_get_rate(policy); } /* * Here we notify other drivers of the proposed change and the final change. */ -static int sw64_cpufreq_target(struct cpufreq_policy *policy, +static int sunway_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) { int ret; @@ -43,7 +221,7 @@ static int sw64_cpufreq_target(struct cpufreq_policy *policy, return -ENODEV; /* setting the cpu frequency */ - ret = sw64_set_rate(index); + ret = sunway_set_rate(index); if (ret) return ret; update_cpu_freq(freq_table[index].frequency); @@ -51,47 +229,58 @@ static int sw64_cpufreq_target(struct cpufreq_policy *policy, return 0; } -static int sw64_cpufreq_cpu_init(struct cpufreq_policy *policy) +static int sunway_cpufreq_init(struct cpufreq_policy *policy) { cpufreq_generic_init(policy, freq_table, 0); return 0; } -static int sw64_cpufreq_verify(struct cpufreq_policy_data *policy) +static int sunway_cpufreq_verify(struct cpufreq_policy_data *policy) { return cpufreq_frequency_table_verify(policy, freq_table); } -static int sw64_cpufreq_exit(struct cpufreq_policy *policy) +static int sunway_cpufreq_exit(struct cpufreq_policy *policy) { return 0; } -static struct freq_attr *sw64_table_attr[] = { +static struct freq_attr *sunway_table_attr[] = { &cpufreq_freq_attr_scaling_available_freqs, NULL, }; -static struct cpufreq_driver sw64_cpufreq_driver = { - .name = "sw64", - .init = sw64_cpufreq_cpu_init, - .verify = sw64_cpufreq_verify, - .target_index = sw64_cpufreq_target, - .get = sw64_cpufreq_get, - .exit = sw64_cpufreq_exit, - .attr = sw64_table_attr, +static struct cpufreq_driver sunway_cpufreq_driver = { + .name = "sunway-cpufreq", + .init = sunway_cpufreq_init, + .verify = sunway_cpufreq_verify, + .target_index = sunway_cpufreq_target, + .get = sunway_cpufreq_get, + .exit = sunway_cpufreq_exit, + .attr = sunway_table_attr, }; static int __init cpufreq_init(void) { + int i, ret; + unsigned long max_rate = get_cpu_freq() / 1000; + if (is_in_guest()) { pr_warn("Now sw_64 CPUFreq does not support virtual machines\n"); return -ENODEV; } - pr_info("SW-64 CPU frequency driver\n"); + fill_freq_table(freq_table); + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (max_rate == freq_table[i].frequency) + freq_table[i+1].frequency = CPUFREQ_TABLE_END; + } - return cpufreq_register_driver(&sw64_cpufreq_driver); + ret = cpufreq_register_driver(&sunway_cpufreq_driver); + if (ret) + return ret; + + return 0; } device_initcall(cpufreq_init); -- Gitee From 6032e53b650b9969a9b4652f25029a822b96246d Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 12 Mar 2025 09:50:38 +0800 Subject: [PATCH 469/524] sw64: cpufreq: fix for emulator that do not support cpufreq Currently on Sunway platforms, cpufreq is only supported in host mode. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/cpufreq/sunway-cpufreq.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/sunway-cpufreq.c b/drivers/cpufreq/sunway-cpufreq.c index 73dbb889063b..ba22843d8eed 100644 --- a/drivers/cpufreq/sunway-cpufreq.c +++ b/drivers/cpufreq/sunway-cpufreq.c @@ -265,8 +265,8 @@ static int __init cpufreq_init(void) int i, ret; unsigned long max_rate = get_cpu_freq() / 1000; - if (is_in_guest()) { - pr_warn("Now sw_64 CPUFreq does not support virtual machines\n"); + if (!is_in_host()) { + pr_warn("cpufreq driver of Sunway platforms is only supported in host mode\n"); return -ENODEV; } -- Gitee From ac9fc25d43ace960551c933b2fa9728ff355239c Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 14 Apr 2025 11:26:00 +0800 Subject: [PATCH 470/524] sw64: fix some compile issues for cpuautoplug.c Fix compiler errors: In file included from arch/sw_64/kernel/cpuautoplug.c:19: arch/sw_64/kernel/../../../kernel/sched/sched.h: In function 'update_current_exec_runtime': arch/sw_64/kernel/../../../kernel/sched/sched.h:3287:9: error: implicit declaration of function 'account_group_exec_runtime' [-Werror=implicit-function-declaration] 3287 | account_group_exec_runtime(curr, delta_exec); | ^~~~~~~~~~~~~~~~~~~~~~~~~~ arch/sw_64/kernel/cpuautoplug.c: In function 'do_autoplug_timer': arch/sw_64/kernel/cpuautoplug.c:455:9: error: implicit declaration of function 'CALC_LOAD' [-Werror=implicit-function-declaration] 455 | CALC_LOAD(avenrun[0], EXP_1, active); | ^~~~~~~~~ Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/cpuautoplug.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/sw_64/kernel/cpuautoplug.c b/arch/sw_64/kernel/cpuautoplug.c index dadf95cf0aea..f5ea76fffa57 100644 --- a/arch/sw_64/kernel/cpuautoplug.c +++ b/arch/sw_64/kernel/cpuautoplug.c @@ -16,7 +16,6 @@ #include #include -#include "../../../kernel/sched/sched.h" int autoplug_enabled; int autoplug_verbose; @@ -452,7 +451,7 @@ static void do_autoplug_timer(struct work_struct *work) #else active = atomic_long_read(&calc_load_tasks); active = active > 0 ? active * FIXED_1 : 0; - CALC_LOAD(avenrun[0], EXP_1, active); + calc_load(avenrun[0], EXP_1, active); load = avenrun[0] / 2; #endif -- Gitee From 830f67ef57e319089e256cb13862a94c1652f8f8 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Wed, 26 Feb 2025 15:38:27 +0800 Subject: [PATCH 471/524] sw64: kvm: optimize the implementation of guest interrupt Add irqs_pending in vcpucb for interrupt deliver. The current guest interrupt can only be injected when entering the vm and vcpu is in interrupt enable state. So, we optimized the guest interrupt handling process. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_host.h | 6 ++++ arch/sw_64/include/asm/vcpu.h | 5 ++- arch/sw_64/kvm/emulate.c | 2 ++ arch/sw_64/kvm/handle_exit.c | 2 +- arch/sw_64/kvm/kvm_core3.c | 54 ++++++++++++++++++++++++++++ arch/sw_64/kvm/kvm_core4.c | 60 ++++++++++++++++++++++++++++++- arch/sw_64/kvm/kvm_timer.c | 12 +------ arch/sw_64/kvm/sw64.c | 49 +++---------------------- 8 files changed, 131 insertions(+), 59 deletions(-) diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index 221a7db26f0f..63b344ed0359 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -184,6 +184,12 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); int kvm_sw64_perf_init(void); int kvm_sw64_perf_teardown(void); void kvm_flush_tlb_all(void); +int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu); +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); +int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number); +void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type); +void sw64_kvm_clear_irq(struct kvm_vcpu *vcpu); +void sw64_kvm_try_deliver_interrupt(struct kvm_vcpu *vcpu); void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn); int kvm_sw64_init_vm(struct kvm *kvm); void kvm_sw64_destroy_vm(struct kvm *kvm); diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h index b2ab8c7f8b86..e96707253224 100644 --- a/arch/sw_64/include/asm/vcpu.h +++ b/arch/sw_64/include/asm/vcpu.h @@ -109,7 +109,10 @@ struct vcpucb { unsigned long csr_earg2; unsigned long csr_scratch; unsigned long atc; - unsigned long reserved[45]; + unsigned long reserved1[10]; + /* Pending virtual interrupts */ + DECLARE_BITMAP(irqs_pending, SWVM_IRQS); + unsigned long reserved2[31]; }; #endif diff --git a/arch/sw_64/kvm/emulate.c b/arch/sw_64/kvm/emulate.c index f103a5f3b9d4..b60bebd1fa22 100644 --- a/arch/sw_64/kvm/emulate.c +++ b/arch/sw_64/kvm/emulate.c @@ -66,6 +66,7 @@ void sw64_decode(struct kvm_vcpu *vcpu, unsigned int insn, struct kvm_run *run) } } +#ifdef CONFIG_SUBARCH_C3B /* * Virtual Interrupts. */ @@ -123,3 +124,4 @@ void try_deliver_interrupt(struct kvm_vcpu *vcpu, unsigned int irq, bool more) inject_vcpu_irq(vcpu, irq); } } +#endif diff --git a/arch/sw_64/kvm/handle_exit.c b/arch/sw_64/kvm/handle_exit.c index 728d323a9f4d..8c6c7325ccfc 100644 --- a/arch/sw_64/kvm/handle_exit.c +++ b/arch/sw_64/kvm/handle_exit.c @@ -47,7 +47,7 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, case SW64_KVM_EXIT_STOP: vcpu->stat.stop_exits++; vcpu->arch.halted = 1; - memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); + sw64_kvm_clear_irq(vcpu); kvm_vcpu_block(vcpu); return 1; case SW64_KVM_EXIT_TIMER: diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c index 142602c67831..26c27c69ba7b 100644 --- a/arch/sw_64/kvm/kvm_core3.c +++ b/arch/sw_64/kvm/kvm_core3.c @@ -286,6 +286,60 @@ long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) return 0; } +int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) +{ + return test_bit(SW64_KVM_IRQ_TIMER, vcpu->arch.irqs_pending); +} + +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.restart) + return 1; + + if (vcpu->arch.vcb.vcpu_irq_disabled) + return 0; + + return ((!bitmap_empty(vcpu->arch.irqs_pending, SWVM_IRQS) || !vcpu->arch.halted) + && !vcpu->arch.power_off); +} + +int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number) +{ + set_bit(number, (vcpu->arch.irqs_pending)); + kvm_vcpu_kick(vcpu); + return 0; +} + +void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type) +{ + struct kvm_vcpu *target_vcpu = kvm_get_vcpu(vcpu->kvm, target_vcpuid); + + if (target_vcpu == NULL) + return; + if (type == II_RESET) { + target_vcpu->arch.restart = 1; + kvm_vcpu_kick(target_vcpu); + } else + vcpu_interrupt_line(target_vcpu, 1); +} + +void sw64_kvm_clear_irq(struct kvm_vcpu *vcpu) +{ + memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); +} + +void sw64_kvm_try_deliver_interrupt(struct kvm_vcpu *vcpu) +{ + int irq; + bool more; + + clear_vcpu_irq(vcpu); + irq = interrupt_pending(vcpu, &more); + + if (irq < SWVM_IRQS) + try_deliver_interrupt(vcpu, irq, more); +} + void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) { } diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c index 6d518f28950c..4918ee335daf 100644 --- a/arch/sw_64/kvm/kvm_core4.c +++ b/arch/sw_64/kvm/kvm_core4.c @@ -57,7 +57,7 @@ int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) hrtimer_cancel(&vcpu->arch.hrt); vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ vcpu->arch.power_off = 0; - memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); + memset(&vcpu->arch.vcb.irqs_pending, 0, sizeof(vcpu->arch.vcb.irqs_pending)); return 0; } @@ -95,6 +95,64 @@ long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) return 0; } +int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) +{ + return test_bit(SW64_KVM_IRQ_TIMER, vcpu->arch.vcb.irqs_pending); +} + +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.restart) + return 1; + + if (vcpu->arch.vcb.vcpu_irq_disabled) + return 0; + + return ((!bitmap_empty(vcpu->arch.vcb.irqs_pending, SWVM_IRQS) || + !vcpu->arch.halted) && !vcpu->arch.power_off); +} + +int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number) +{ + int cpu = vcpu->cpu; + int me = smp_processor_id(); + + set_bit(number, (vcpu->arch.vcb.irqs_pending)); + + if (vcpu->mode == IN_GUEST_MODE) { + if (cpu != me && (unsigned int)cpu < nr_cpu_ids + && cpu_online(cpu)) { + if (vcpu->arch.vcb.vcpu_irq_disabled) + return 0; + send_ipi(cpu, II_II1); + } + } else + kvm_vcpu_kick(vcpu); + return 0; +} + +void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type) +{ + struct kvm_vcpu *target_vcpu = kvm_get_vcpu(vcpu->kvm, target_vcpuid); + + if (target_vcpu == NULL) + return; + if (type == II_RESET) { + target_vcpu->arch.restart = 1; + kvm_vcpu_kick(target_vcpu); + } else + vcpu_interrupt_line(target_vcpu, 1); +} + +void sw64_kvm_clear_irq(struct kvm_vcpu *vcpu) +{ + memset(&vcpu->arch.vcb.irqs_pending, 0, sizeof(vcpu->arch.vcb.irqs_pending)); +} + +void sw64_kvm_try_deliver_interrupt(struct kvm_vcpu *vcpu) +{ +} + int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_memory_slot *old, struct kvm_memory_slot *new, diff --git a/arch/sw_64/kvm/kvm_timer.c b/arch/sw_64/kvm/kvm_timer.c index 895be63cd8d1..04a4709c996d 100644 --- a/arch/sw_64/kvm/kvm_timer.c +++ b/arch/sw_64/kvm/kvm_timer.c @@ -50,17 +50,7 @@ void set_timer(struct kvm_vcpu *vcpu, unsigned long delta) /* And this is the routine when we want to set an interrupt for the Guest. */ void set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { - /* - * Next time the Guest runs, the core code will see if it can deliver - * this interrupt. - */ - set_bit(irq, (vcpu->arch.irqs_pending)); - - /* - * Make sure it sees it; it might be asleep (eg. halted), or running - * the Guest right now, in which case kick_process() will knock it out. - */ - kvm_vcpu_kick(vcpu); + vcpu_interrupt_line(vcpu, irq); } enum hrtimer_restart clockdev_fn(struct hrtimer *timer) diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index b70f50e5610f..7065c984d2b1 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -58,13 +58,6 @@ int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, return -EWOULDBLOCK; } -int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) -{ - set_bit(number, (vcpu->arch.irqs_pending)); - kvm_vcpu_kick(vcpu); - return 0; -} - int kvm_arch_check_processor_compat(void *opaque) { return 0; @@ -83,7 +76,7 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq if (!vcpu) return -EINVAL; - return vcpu_interrupt_line(vcpu, vector, true); + return vcpu_interrupt_line(vcpu, vector); } void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) @@ -188,18 +181,6 @@ struct dfx_sw64_kvm_stats_debugfs_item dfx_sw64_debugfs_entries[] = { { NULL } }; -int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) -{ - if (vcpu->arch.restart) - return 1; - - if (vcpu->arch.vcb.vcpu_irq_disabled) - return 0; - - return ((!bitmap_empty(vcpu->arch.irqs_pending, SWVM_IRQS) || !vcpu->arch.halted) - && !vcpu->arch.power_off); -} - int kvm_arch_hardware_enable(void) { return 0; @@ -251,11 +232,6 @@ void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { } -int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) -{ - return test_bit(SW64_KVM_IRQ_TIMER, vcpu->arch.irqs_pending); -} - int kvm_arch_hardware_setup(void *opaque) { return 0; @@ -428,8 +404,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) struct kvm_run *run = vcpu->run; struct vcpucb *vcb = &(vcpu->arch.vcb); struct hcall_args hargs; - int irq, ret; - bool more; + int ret; sigset_t sigsaved; /* Set guest vcb */ @@ -465,17 +440,13 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) memset(&hargs, 0, sizeof(hargs)); - clear_vcpu_irq(vcpu); - if (vcpu->arch.restart == 1) { /* handle reset vCPU */ vcpu->arch.regs.pc = GUEST_RESET_PC; vcpu->arch.restart = 0; } - irq = interrupt_pending(vcpu, &more); - if (irq < SWVM_IRQS) - try_deliver_interrupt(vcpu, irq, more); + sw64_kvm_try_deliver_interrupt(vcpu); vcpu->arch.halted = 0; @@ -627,24 +598,12 @@ int kvm_dev_ioctl_check_extension(long ext) return r; } -void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type) -{ - struct kvm_vcpu *target_vcpu = kvm_get_vcpu(vcpu->kvm, target_vcpuid); - - if (type == II_RESET) - target_vcpu->arch.restart = 1; - - if (target_vcpu != NULL) - vcpu_interrupt_line(target_vcpu, 1, 1); -} - int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, bool line_status) { u32 irq = irq_level->irq; unsigned int irq_num; struct kvm_vcpu *vcpu = NULL; - bool level = irq_level->level; irq_num = irq; trace_kvm_irq_line(0, irq_num, irq_level->level); @@ -654,7 +613,7 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, if (!vcpu) return -EINVAL; - return vcpu_interrupt_line(vcpu, irq_num, level); + return vcpu_interrupt_line(vcpu, irq_num); } -- Gitee From 4a4022d4449921719182c4eb2ac9bca846a9e08d Mon Sep 17 00:00:00 2001 From: Yu Jiayi Date: Thu, 27 Feb 2025 14:52:28 +0800 Subject: [PATCH 472/524] sw64: kvm: optimize the compatibility of guest interrupt The guest interrupt optimization enables the kernel and hmcode to handle the interrupt injection in a new way. In order to be compatible with old versions of kernel and hmcode, we set a global variable to get the version of hmcode and write a value to smp_rcb block to mark the version of kernel. Signed-off-by: Yu Jiayi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpu.h | 1 + arch/sw_64/include/asm/smp.h | 1 + arch/sw_64/kvm/emulate.c | 2 -- arch/sw_64/kvm/kvm_core4.c | 55 ++++++++++++++++++++++++++++++++---- arch/sw_64/kvm/sw64.c | 4 +++ 5 files changed, 56 insertions(+), 7 deletions(-) diff --git a/arch/sw_64/include/asm/cpu.h b/arch/sw_64/include/asm/cpu.h index ab67d723c069..37368a9d4d52 100644 --- a/arch/sw_64/include/asm/cpu.h +++ b/arch/sw_64/include/asm/cpu.h @@ -21,6 +21,7 @@ enum hmcall_cpuid_cmd { #define CPU_FEAT_FPU 0x1 #define CPU_FEAT_SIMD 0x2 #define CPU_FEAT_UNA 0x4 +#define CPU_FEAT_VINT 0x8 enum sunway_cpu_model { CPU_SW3231 = 0x31, diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h index 4249b10dc550..0b1ebf2143f2 100644 --- a/arch/sw_64/include/asm/smp.h +++ b/arch/sw_64/include/asm/smp.h @@ -34,6 +34,7 @@ struct smp_rcb_struct { unsigned long restart_args; unsigned long ready; unsigned long init_done; + unsigned long feat_vint; }; extern bool __init is_rcid_duplicate(int rcid); diff --git a/arch/sw_64/kvm/emulate.c b/arch/sw_64/kvm/emulate.c index b60bebd1fa22..f103a5f3b9d4 100644 --- a/arch/sw_64/kvm/emulate.c +++ b/arch/sw_64/kvm/emulate.c @@ -66,7 +66,6 @@ void sw64_decode(struct kvm_vcpu *vcpu, unsigned int insn, struct kvm_run *run) } } -#ifdef CONFIG_SUBARCH_C3B /* * Virtual Interrupts. */ @@ -124,4 +123,3 @@ void try_deliver_interrupt(struct kvm_vcpu *vcpu, unsigned int irq, bool more) inject_vcpu_irq(vcpu, irq); } } -#endif diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c index 4918ee335daf..57f1176472f3 100644 --- a/arch/sw_64/kvm/kvm_core4.c +++ b/arch/sw_64/kvm/kvm_core4.c @@ -20,6 +20,8 @@ #include #include "trace.h" +extern bool feature_vint; + static unsigned long shtclock_offset; void update_aptp(unsigned long pgd) @@ -57,7 +59,10 @@ int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) hrtimer_cancel(&vcpu->arch.hrt); vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ vcpu->arch.power_off = 0; - memset(&vcpu->arch.vcb.irqs_pending, 0, sizeof(vcpu->arch.vcb.irqs_pending)); + if (feature_vint) + memset(&vcpu->arch.vcb.irqs_pending, 0, sizeof(vcpu->arch.vcb.irqs_pending)); + else + memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); return 0; } @@ -97,7 +102,10 @@ long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) { - return test_bit(SW64_KVM_IRQ_TIMER, vcpu->arch.vcb.irqs_pending); + if (feature_vint) + return test_bit(SW64_KVM_IRQ_TIMER, vcpu->arch.vcb.irqs_pending); + else + return test_bit(SW64_KVM_IRQ_TIMER, vcpu->arch.irqs_pending); } int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) @@ -108,11 +116,15 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) if (vcpu->arch.vcb.vcpu_irq_disabled) return 0; - return ((!bitmap_empty(vcpu->arch.vcb.irqs_pending, SWVM_IRQS) || + if (feature_vint) + return ((!bitmap_empty(vcpu->arch.vcb.irqs_pending, SWVM_IRQS) || + !vcpu->arch.halted) && !vcpu->arch.power_off); + else + return ((!bitmap_empty(vcpu->arch.irqs_pending, SWVM_IRQS) || !vcpu->arch.halted) && !vcpu->arch.power_off); } -int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number) +static int feat_vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number) { int cpu = vcpu->cpu; int me = smp_processor_id(); @@ -131,6 +143,23 @@ int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number) return 0; } +int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number) +{ + if (feature_vint) + return feat_vcpu_interrupt_line(vcpu, number); + /* + * Next time the Guest runs, the core code will see if it can deliver + * this interrupt. + */ + set_bit(number, (vcpu->arch.irqs_pending)); + /* + * Make sure it sees it; it might be asleep (eg. halted), or running + * the Guest right now, in which case kick_process() will knock it out. + */ + kvm_vcpu_kick(vcpu); + return 0; +} + void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type) { struct kvm_vcpu *target_vcpu = kvm_get_vcpu(vcpu->kvm, target_vcpuid); @@ -146,11 +175,27 @@ void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type) void sw64_kvm_clear_irq(struct kvm_vcpu *vcpu) { - memset(&vcpu->arch.vcb.irqs_pending, 0, sizeof(vcpu->arch.vcb.irqs_pending)); + if (feature_vint) + memset(&vcpu->arch.vcb.irqs_pending, 0, sizeof(vcpu->arch.vcb.irqs_pending)); + else + memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); +} + +void sw64_kvm_try_deliver_interrupt_orig(struct kvm_vcpu *vcpu) +{ + int irq; + bool more; + + clear_vcpu_irq(vcpu); + irq = interrupt_pending(vcpu, &more); + if (irq < SWVM_IRQS) + try_deliver_interrupt(vcpu, irq, more); } void sw64_kvm_try_deliver_interrupt(struct kvm_vcpu *vcpu) { + if (!feature_vint) + sw64_kvm_try_deliver_interrupt_orig(vcpu); } int kvm_arch_prepare_memory_region(struct kvm *kvm, diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index 7065c984d2b1..053115c8284c 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -22,6 +22,8 @@ #include "irq.h" bool set_msi_flag; +bool feature_vint; +extern struct smp_rcb_struct *smp_rcb; #define VCPU_STAT(n, x, ...) \ { n, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ } @@ -242,6 +244,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (type) return -EINVAL; + feature_vint = (cpuid(GET_FEATURES, 0) & CPU_FEAT_VINT); + smp_rcb->feat_vint = 1; return kvm_sw64_init_vm(kvm); } -- Gitee From f864604d217ea8bddb3f9efdd224d65cf13cb7f3 Mon Sep 17 00:00:00 2001 From: Yu Jiayi Date: Mon, 10 Mar 2025 16:31:25 +0800 Subject: [PATCH 473/524] sw64: fix SWVM_IRQS undeclared when enabling HIBERNATION The SWVM_IRQS is defined in kvm.h. When enabling CONFIG_HIBERNATION, it involves the use of vcpucb structure. But the kvm has not been compiled. So, we define CORE4VM_IRQS to replace SWVM_IRQS when defining vcpucb structure. Signed-off-by: Yu Jiayi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/vcpu.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h index e96707253224..b069031add39 100644 --- a/arch/sw_64/include/asm/vcpu.h +++ b/arch/sw_64/include/asm/vcpu.h @@ -56,6 +56,7 @@ struct vcpucb { }; #elif CONFIG_SUBARCH_C4 +#define CORE4VM_IRQS 256 struct vcpucb { unsigned long ktp; @@ -111,7 +112,7 @@ struct vcpucb { unsigned long atc; unsigned long reserved1[10]; /* Pending virtual interrupts */ - DECLARE_BITMAP(irqs_pending, SWVM_IRQS); + DECLARE_BITMAP(irqs_pending, CORE4VM_IRQS); unsigned long reserved2[31]; }; #endif -- Gitee From 7a2626dff91fd25fdbc44b75caf326a422aa6af4 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Thu, 17 Apr 2025 19:16:23 +0800 Subject: [PATCH 474/524] sw64: fix shared_cpu_map of L3 Cache When a cpu comes online from offline, the shared_cpu_map that the other cpus get from /sys/devices/system/cpu/cpuX/cache/index3/shared_cpu_map are not updated. This is beacuse we only updated shared_cpu_map of this cpu but did not update the share_cpu_map of other cpus. This patch fixes the problem by traversing and updating the shared_cpu_map of the other cpus. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/cacheinfo.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kernel/cacheinfo.c b/arch/sw_64/kernel/cacheinfo.c index fb4c9ca7650b..a6e8298cff50 100644 --- a/arch/sw_64/kernel/cacheinfo.c +++ b/arch/sw_64/kernel/cacheinfo.c @@ -116,7 +116,7 @@ static void setup_shared_cpu_map(unsigned int cpu) { unsigned int index; unsigned int rcid = cpu_to_rcid(cpu); - struct cacheinfo *this_leaf; + struct cacheinfo *this_leaf, *sib_leaf; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); for (index = 0; index < this_cpu_ci->num_leaves; index++) { @@ -126,16 +126,20 @@ static void setup_shared_cpu_map(unsigned int cpu) cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); - for_each_possible_cpu(i) { + for_each_online_cpu(i) { unsigned int sib_rcid = cpu_to_rcid(i); + struct cpu_cacheinfo *sib_cpu_ci = get_cpu_cacheinfo(i); if ((rcid_to_domain_id(sib_rcid) != rcid_to_domain_id(rcid)) || (i == cpu)) continue; + sib_leaf = sib_cpu_ci->info_list + index; if ((rcid_to_core_id(rcid) == rcid_to_core_id(sib_rcid)) || - (this_leaf->level == 3)) + (this_leaf->level == 3)) { + cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map); cpumask_set_cpu(i, &this_leaf->shared_cpu_map); + } } } } -- Gitee From 8a2ffa0bd22d1fa0a406eabef1c89df802eaa8fe Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 21 Feb 2025 11:20:27 +0800 Subject: [PATCH 475/524] sw64: modify the interface for firmware to pass the running mode Starting from C4, the compatible string of the root node of device tree is used to pass the running mode: - sunway,emulator : emulator - sunway,virtual-machine : guest - sunway,xxx (e.g. sunway,junzhang) : host Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 58 +++++++++++++++++++------------- arch/sw_64/kernel/smp.c | 3 +- arch/sw_64/kernel/suspend.c | 2 +- drivers/irqchip/irq-sunway-cpu.c | 2 +- 4 files changed, 37 insertions(+), 28 deletions(-) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index f8ca94ce24c6..02c3df0f8265 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -463,6 +463,31 @@ bool sunway_machine_is_compatible(const char *compat) return !fdt_node_check_compatible(fdt, offset, compat); } +#ifndef CONFIG_SUBARCH_C3B +static void __init setup_run_mode(void) +{ + static_branch_disable(&run_mode_host_key); + static_branch_disable(&run_mode_guest_key); + static_branch_disable(&run_mode_emul_key); + + if (sunway_machine_is_compatible("sunway,emulator")) { + static_branch_enable(&run_mode_emul_key); + pr_info("Mode: Emul\n"); + return; + } + + if (sunway_machine_is_compatible("sunway,virtual-machine") || + (rvpcr() >> VPCR_SHIFT)) { + static_branch_enable(&run_mode_guest_key); + pr_info("Mode: Guest\n"); + return; + } + + static_branch_enable(&run_mode_host_key); + pr_info("Mode: Host\n"); +} +#endif + static void __init setup_firmware_fdt(void) { void *dt_virt; @@ -491,6 +516,10 @@ static void __init setup_firmware_fdt(void) cpu_relax(); } +#ifndef CONFIG_SUBARCH_C3B + setup_run_mode(); +#endif + if (sunway_boot_magic == 0xDEED2024UL) { /* Parse MCLK(Hz) from firmware DTB */ early_parse_fdt_property(dt_virt, "/soc/clocks/mclk", @@ -606,7 +635,7 @@ static void __init device_tree_init(void) } #ifdef CONFIG_SUBARCH_C3B -static void __init setup_run_mode(void) +static void __init setup_run_mode_legacy(void) { if (*(unsigned long *)MM_SIZE) { static_branch_disable(&run_mode_host_key); @@ -627,28 +656,6 @@ static void __init setup_run_mode(void) static_branch_disable(&run_mode_emul_key); } } -#elif CONFIG_SUBARCH_C4 -static void __init setup_run_mode(void) -{ - if (rvpcr() >> VPCR_SHIFT) { - pr_info("run mode: guest\n"); - static_branch_disable(&run_mode_host_key); - static_branch_disable(&run_mode_emul_key); - static_branch_enable(&run_mode_guest_key); - } else if (sunway_boot_magic == 0xA2024) { - pr_info("run mode: emul\n"); - static_branch_disable(&run_mode_host_key); - static_branch_disable(&run_mode_guest_key); - static_branch_enable(&run_mode_emul_key); - sunway_boot_magic = 0xDEED2024; - } else { - pr_info("run mode: host\n"); - static_branch_disable(&run_mode_guest_key); - static_branch_disable(&run_mode_emul_key); - static_branch_enable(&run_mode_host_key); - } - -} #endif void __init @@ -661,7 +668,10 @@ setup_arch(char **cmdline_p) trap_init(); jump_label_init(); - setup_run_mode(); + +#ifdef CONFIG_SUBARCH_C3B + setup_run_mode_legacy(); +#endif setup_chip_ops(); setup_sched_clock(); diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 3327f59f43d8..ef88fe2344e9 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -854,7 +854,7 @@ void arch_cpu_idle_dead(void) } #ifdef CONFIG_SUSPEND - if (!is_junzhang_v1()) { + if (is_in_host() && !is_junzhang_v1()) { sleepen(); send_sleep_interrupt(smp_processor_id()); while (1) @@ -864,7 +864,6 @@ void arch_cpu_idle_dead(void) while (1) asm("nop"); } - #else asm volatile("memb"); asm volatile("halt"); diff --git a/arch/sw_64/kernel/suspend.c b/arch/sw_64/kernel/suspend.c index 9947937c71b5..c647f9d3d8a7 100644 --- a/arch/sw_64/kernel/suspend.c +++ b/arch/sw_64/kernel/suspend.c @@ -56,7 +56,7 @@ int sw64_suspend_enter(void) static int native_suspend_enter(suspend_state_t state) { - if (is_in_guest()) + if (!is_in_host()) return 0; /* processor specific suspend */ diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 58579c1044e0..28b346f87bf3 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -94,7 +94,7 @@ asmlinkage void noinstr do_entInt(struct pt_regs *regs) old_regs = set_irq_regs(regs); #ifdef CONFIG_PM - if (is_junzhang_v1()) { + if (is_in_host() && is_junzhang_v1()) { if (pme_state == PME_WFW) { pme_state = PME_PENDING; goto out; -- Gitee From cbe0cc246ccfc2f08a46ecb3f2ca060b70dfe9f4 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 21 Apr 2025 15:18:36 +0800 Subject: [PATCH 476/524] sw64: clocksource: fix the issue of clockevents not being notified This patch ensures that all clockevents in the shared clock domain are notified when the cpu frequency changes. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/clocksource/timer-sw64.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index eed2540f7a14..9af19cb362fc 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -386,29 +386,24 @@ static int timer_set_oneshot(struct clock_event_device *evt) return 0; } -static void sw64_update_clockevents(unsigned long cpu, u32 freq) +static void sw64_update_clockevents(void *data) { - struct clock_event_device *swevt = &per_cpu(timer_events, cpu); + struct cpufreq_freqs *freqs = (struct cpufreq_freqs *)data; + struct clock_event_device *swevt = this_cpu_ptr(&timer_events); - if (cpu == smp_processor_id()) - clockevents_update_freq(swevt, freq); - else { - clockevents_calc_mult_shift(swevt, freq, 4); - swevt->min_delta_ns = clockevent_delta2ns(swevt->min_delta_ticks, swevt); - swevt->max_delta_ns = clockevent_delta2ns(swevt->max_delta_ticks, swevt); - } + clockevents_update_freq(swevt, freqs->new * 1000); } static int sw64_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data) { struct cpufreq_freqs *freqs = (struct cpufreq_freqs *)data; - unsigned long cpu = freqs->policy->cpu; if (val == CPUFREQ_POSTCHANGE) - sw64_update_clockevents(cpu, freqs->new * 1000); + on_each_cpu_mask(freqs->policy->cpus, + sw64_update_clockevents, data, 1); - return 0; + return NOTIFY_OK; } static struct notifier_block sw64_cpufreq_notifier_block = { -- Gitee From 4fd631b7d838da22f79f89c68f9401c564c10111 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 19 Mar 2025 20:10:56 +0800 Subject: [PATCH 477/524] sw64: add powercap driver This commit adds support for powercap driver on Sunway platform. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 8 + drivers/platform/sw64/Makefile | 1 + drivers/platform/sw64/powercap.c | 745 +++++++++++++++++++++++++++++++ 3 files changed, 754 insertions(+) create mode 100644 drivers/platform/sw64/powercap.c diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index be0c18a9a04d..d8c82f89d616 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -631,6 +631,14 @@ config ARCH_HIBERNATION_POSSIBLE depends on SW64 def_bool y +config SW64_POWERCAP + bool "Sunway powercap driver" + select IPMI_SI + depends on SW64 && CPU_FREQ && ACPI && IPMI_HANDLER + help + This enables support for the sunway powercap driver + based on BMC and IPMI system interface. + source "drivers/cpuidle/Kconfig" source "drivers/idle/Kconfig" diff --git a/drivers/platform/sw64/Makefile b/drivers/platform/sw64/Makefile index 4c8f5e44def6..3bde94020c42 100644 --- a/drivers/platform/sw64/Makefile +++ b/drivers/platform/sw64/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_SW64_POWERCAP) += powercap.o obj-y += misc-platform.o diff --git a/drivers/platform/sw64/powercap.c b/drivers/platform/sw64/powercap.c new file mode 100644 index 000000000000..3f380ace5601 --- /dev/null +++ b/drivers/platform/sw64/powercap.c @@ -0,0 +1,745 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025 WXIAT + */ + +#define pr_fmt(fmt) "sunway-powercap: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#define SUNWAY_POWERCAP_NETFN 0x3A + +#define SUNWAY_POWERCAP_ACPI_NOTIFY_VALUE 0x84 + +enum sunway_powercap_version { + SUNWAY_POWERCAP_V1 = 1, + SUNWAY_POWERCAP_VERSION_MAX, +}; + +enum sunway_powercap_mode { + SUNWAY_POWERCAP_MODE_POLL = 0, + SUNWAY_POWERCAP_MODE_INTERRUPT, +}; + +enum sunway_powercap_poll_interval { + SUNWAY_POWERCAP_POLL_INTERVAL0 = 50, + SUNWAY_POWERCAP_POLL_INTERVAL1 = 100, + SUNWAY_POWERCAP_POLL_INTERVAL2 = 200, + SUNWAY_POWERCAP_POLL_INTERVAL3 = 250, +}; + +enum sunway_powercap_cmd { + SUNWAY_POWERCAP_CMD_GET_CFG = 0x30, + SUNWAY_POWERCAP_CMD_GET_FREQ = 0x31, + SUNWAY_POWERCAP_CMD_ACK = 0x32, +}; + +enum sunway_powercap_state { + SUNWAY_POWERCAP_STATE_FREE = 0x0F, + SUNWAY_POWERCAP_STATE_LIMIT = 0xF0, +}; + +#pragma pack(1) + +struct sunway_powercap_cfg { + u8 version; + u8 mode; + u8 poll_interval; + u8 reserved; +}; + +#define FREQ_FLAG_ENABLE (1 << 0) +#define FREQ_FLAG_FREE (1 << 1) +#define FREQ_FLAG_TERMINATE (1 << 2) + +struct sunway_powercap_freq { + u32 target_freq; + u16 target_core; + u16 flags; +}; + +#define ACK_FLAG_VALID_SIZE (1 << 0) +#define ACK_FLAG_VALID_VERSION (1 << 1) +#define ACK_FLAG_VALID_MODE (1 << 2) +#define ACK_FLAG_VALID_INTERVAL (1 << 3) +#define ACK_FLAG_VALID_FREQ (1 << 4) +#define ACK_FLAG_VALID_NODE (1 << 5) +#define ACK_FLAG_VALID_CORE (1 << 6) + +struct sunway_powercap_ack { + u8 cmd; + u8 flags; + u16 reserved; +}; + +/* Reset to default packing */ +#pragma pack() + +struct sunway_powercap_bmc_data { + struct device *bmc_device; + struct ipmi_addr address; + struct ipmi_user *user; + struct completion complete; + int interface; + + struct kernel_ipmi_msg tx_message; + unsigned char tx_msg_data[IPMI_MAX_MSG_LENGTH]; + long tx_msgid; + + unsigned char rx_msg_data[IPMI_MAX_MSG_LENGTH]; + unsigned short rx_msg_len; + unsigned char rx_result; + int rx_recv_type; + + bool initialized; +}; + +struct sunway_powercap_driver_data { + struct device *dev; + + unsigned char version; + unsigned char mode; + unsigned char poll_interval; + + struct timer_list timer; + struct work_struct work; + + struct ipmi_smi_watcher bmc_events; + struct ipmi_user_hndl ipmi_hndlrs; + struct sunway_powercap_bmc_data bmc_data; +}; + +struct sunway_powercap_cpu { + unsigned int state; + unsigned int node; + unsigned int core; + struct cpufreq_policy *policy; + struct freq_qos_request *qos_req; +}; + +static void sunway_powercap_register_bmc(int iface, struct device *dev); +static void sunway_powercap_bmc_gone(int iface); +static void sunway_powercap_msg_handler(struct ipmi_recv_msg *msg, + void *user_msg_data); + +static struct sunway_powercap_driver_data driver_data = { + .bmc_events = { + .new_smi = sunway_powercap_register_bmc, + .smi_gone = sunway_powercap_bmc_gone, + }, + + .ipmi_hndlrs = { + .ipmi_recv_hndl = sunway_powercap_msg_handler, + }, +}; + +static struct sunway_powercap_cpu powercap_cpu_data[NR_CPUS]; + +static unsigned char powercap_freq_ack[IPMI_MAX_MSG_LENGTH]; + +static int sunway_powercap_send_message(struct sunway_powercap_bmc_data *bmc_data) +{ + int ret; + + ret = ipmi_validate_addr(&bmc_data->address, sizeof(bmc_data->address)); + if (ret) { + dev_err(bmc_data->bmc_device, "invalid ipmi addr (%d)\n", ret); + return ret; + } + + bmc_data->tx_msgid++; + ret = ipmi_request_settime(bmc_data->user, &bmc_data->address, + bmc_data->tx_msgid, &bmc_data->tx_message, + bmc_data, 0, 0, 0); + if (ret) { + dev_err(bmc_data->bmc_device, + "unable to send message (%d)\n", ret); + return ret; + } + + return 0; +} + +static int sunway_powercap_send_cmd(struct sunway_powercap_bmc_data *bmc_data, + unsigned char cmd, const unsigned char *data, unsigned short data_len) +{ + bmc_data->tx_message.cmd = cmd; + bmc_data->tx_message.data_len = data_len; + + if (data_len) + memcpy(bmc_data->tx_msg_data, data, data_len); + + return sunway_powercap_send_message(bmc_data); +} + +static int sunway_powercap_query(struct sunway_powercap_bmc_data *bmc_data, + unsigned char cmd, const char *info) +{ + int ret; + + ret = sunway_powercap_send_cmd(bmc_data, cmd, NULL, 0); + if (ret) { + dev_err(bmc_data->bmc_device, "unable to query %s\n", info); + return ret; + } + + wait_for_completion(&bmc_data->complete); + + if (bmc_data->rx_result) { + dev_err(bmc_data->bmc_device, "rx error 0x%x when query %s\n", + bmc_data->rx_result, info); + return -EINVAL; + } + + return 0; +} + +static int sunway_powercap_ack_bmc(struct sunway_powercap_bmc_data *bmc_data, + const struct sunway_powercap_ack *ack, int num) +{ + unsigned char cmd = SUNWAY_POWERCAP_CMD_ACK; + int ret; + + ret = sunway_powercap_send_cmd(bmc_data, cmd, + (const char *)ack, sizeof(*ack) * num); + if (ret) { + dev_err(bmc_data->bmc_device, "unable to send ack\n"); + return ret; + } + + wait_for_completion(&bmc_data->complete); + + return 0; +} + +static inline unsigned int +powercap_target_node(const struct sunway_powercap_freq *freq) +{ + return freq->target_core & 0x3F; +} + +static inline unsigned int +powercap_target_core(const struct sunway_powercap_freq *freq) +{ + return (freq->target_core >> 6) & 0x3FF; +} + +static inline bool is_powercap_cpu_match(const struct sunway_powercap_cpu *data, + unsigned int node, unsigned int core) +{ + if ((node != 0x3F) && (data->node != node)) + return false; + + if ((core != 0x3FF) && (data->core != core)) + return false; + + return true; +} + +static int sunway_powercap_validate_freq(const struct sunway_powercap_freq *freq, + struct sunway_powercap_ack *ack) +{ + unsigned int node = powercap_target_node(freq); + unsigned int core = powercap_target_core(freq); + int i; + + /* Currently, core must be 0x3FF(all bits are 1) */ + if (core != 0x3FF) + goto out_validate_freq; + + for (i = 0; i < ARRAY_SIZE(powercap_cpu_data); i++) { + struct cpufreq_policy *policy = powercap_cpu_data[i].policy; + unsigned int target_freq = freq->target_freq; + + if (!policy) + continue; + + if (!is_powercap_cpu_match(&powercap_cpu_data[i], node, core)) + continue; + + /* Now we confirm that core and node are valid */ + ack->flags |= ACK_FLAG_VALID_NODE; + ack->flags |= ACK_FLAG_VALID_CORE; + + if (cpufreq_frequency_table_get_index(policy, target_freq) < 0) { + pr_err("invalid target freq %u\n", target_freq); + return -EINVAL; + } + + ack->flags |= ACK_FLAG_VALID_FREQ; + + return 0; + } + +out_validate_freq: + pr_err("invalid core %u on node %u\n", core, node); + + return -EINVAL; +} + +static inline bool is_powercap_enabled(const struct sunway_powercap_freq *freq) +{ + return !!(freq->flags & FREQ_FLAG_ENABLE); +} + +static inline bool is_powercap_no_limit(const struct sunway_powercap_freq *freq) +{ + return !!(freq->flags & FREQ_FLAG_FREE); +} + +static inline bool +is_powercap_freq_data_terminate(const struct sunway_powercap_freq *freq) +{ + return !!(freq->flags & FREQ_FLAG_TERMINATE); +} + +static int sunway_powercap_handle_free_cpus(struct cpufreq_policy *policy, + struct freq_qos_request *req, const struct sunway_powercap_freq *freq) +{ + int ret, related_cpu; + + if (!is_powercap_enabled(freq) || is_powercap_no_limit(freq)) + return 0; + + ret = freq_qos_add_request(&policy->constraints, + req, FREQ_QOS_MAX, freq->target_freq); + if (ret < 0) { + pr_err("unable to add qos request on cpus %*pbl\n", + cpumask_pr_args(policy->related_cpus)); + return ret; + } + + for_each_cpu(related_cpu, policy->related_cpus) + powercap_cpu_data[related_cpu].state = SUNWAY_POWERCAP_STATE_LIMIT; + + return 0; +} + +static int sunway_powercap_handle_limit_cpus(struct cpufreq_policy *policy, + struct freq_qos_request *req, const struct sunway_powercap_freq *freq) +{ + int ret, related_cpu; + + if (is_powercap_enabled(freq) && !is_powercap_no_limit(freq)) { + ret = freq_qos_update_request(req, freq->target_freq); + if (ret < 0) + pr_err("unable to update qos request on cpus %*pbl\n", + cpumask_pr_args(policy->related_cpus)); + return ret; + } + + ret = freq_qos_remove_request(req); + if (ret < 0) { + pr_err("unable to remove qos request on cpus %*pbl\n", + cpumask_pr_args(policy->related_cpus)); + return ret; + } + + for_each_cpu(related_cpu, policy->related_cpus) + powercap_cpu_data[related_cpu].state = SUNWAY_POWERCAP_STATE_FREE; + + return 0; +} + +static int sunway_powercap_handle_one_freq(const struct sunway_powercap_freq *freq, + struct sunway_powercap_ack *ack) +{ + int i; + unsigned int node = powercap_target_node(freq); + unsigned int core = powercap_target_core(freq); + unsigned int state; + struct freq_qos_request *req; + struct cpufreq_policy *policy; + cpumask_var_t done; + + /* Ack freq */ + ack->cmd = SUNWAY_POWERCAP_CMD_GET_FREQ; + + /* Size must be valid here */ + ack->flags |= ACK_FLAG_VALID_SIZE; + + if (sunway_powercap_validate_freq(freq, ack)) + return -EINVAL; + + if (!alloc_cpumask_var(&done, GFP_KERNEL)) + return -ENOMEM; + + cpumask_clear(done); + + for (i = 0; i < ARRAY_SIZE(powercap_cpu_data); i++) { + policy = powercap_cpu_data[i].policy; + + if (!policy || policy_is_inactive(policy)) + continue; + + if (cpumask_test_cpu(i, done)) + continue; + + if (!is_powercap_cpu_match(&powercap_cpu_data[i], node, core)) + continue; + + state = powercap_cpu_data[i].state; + req = powercap_cpu_data[i].qos_req; + + if (state == SUNWAY_POWERCAP_STATE_FREE) + sunway_powercap_handle_free_cpus(policy, req, freq); + else if (state == SUNWAY_POWERCAP_STATE_LIMIT) + sunway_powercap_handle_limit_cpus(policy, req, freq); + else + pr_err("cpu %d with invalid state 0x%x\n", i, state); + + cpumask_or(done, done, policy->related_cpus); + } + + free_cpumask_var(done); + + return 0; +} + +static int sunway_powercap_poll_once(struct sunway_powercap_bmc_data *bmc_data) +{ + struct sunway_powercap_ack *ack; + struct sunway_powercap_freq *freq; + unsigned char cmd = SUNWAY_POWERCAP_CMD_GET_FREQ; + int ret, num, i; + +query_freq: + /* Clean ACK data */ + memset(powercap_freq_ack, 0, sizeof(powercap_freq_ack)); + + ret = sunway_powercap_query(bmc_data, cmd, "freq"); + if (ret) + return ret; + + ack = (struct sunway_powercap_ack *)&powercap_freq_ack[0]; + freq = (struct sunway_powercap_freq *)&bmc_data->rx_msg_data[0]; + + /* Number of freq data */ + num = bmc_data->rx_msg_len >> 3; + + if (!num || (bmc_data->rx_msg_len & 0x7)) { + dev_err(bmc_data->bmc_device, "invalid freq size %d\n", + bmc_data->rx_msg_len); + + /** + * The size must be multiple of 8 bytes, otherwise + * send only one ack with invalid size. + */ + ack->cmd = cmd; + ack->flags &= ~ACK_FLAG_VALID_SIZE; + sunway_powercap_ack_bmc(bmc_data, ack, 1); + + return -EINVAL; + } + + /* Handle freq data one by one */ + for (i = 0; i < num; i++) + sunway_powercap_handle_one_freq(freq + i, ack + i); + + sunway_powercap_ack_bmc(bmc_data, ack, num); + + /* More freq Data needs to be queried */ + if (!is_powercap_freq_data_terminate(&freq[num - 1])) + goto query_freq; + + return 0; +} + +static inline bool is_legal_poll_interval(u8 interval) +{ + return (interval == SUNWAY_POWERCAP_POLL_INTERVAL0) || + (interval == SUNWAY_POWERCAP_POLL_INTERVAL1) || + (interval == SUNWAY_POWERCAP_POLL_INTERVAL2) || + (interval == SUNWAY_POWERCAP_POLL_INTERVAL3); +} + +static int sunway_powercap_validate_cfg(const struct sunway_powercap_cfg *cfg, + struct sunway_powercap_ack *ack) +{ + bool valid = true; + + if (!cfg->version || (cfg->version >= SUNWAY_POWERCAP_VERSION_MAX)) { + pr_err("invalid version %d\n", cfg->version); + valid = false; + } else + ack->flags |= ACK_FLAG_VALID_VERSION; + + if (cfg->mode > SUNWAY_POWERCAP_MODE_INTERRUPT) { + pr_err("invalid mode %d\n", cfg->mode); + valid = false; + } else + ack->flags |= ACK_FLAG_VALID_MODE; + + if ((cfg->mode == SUNWAY_POWERCAP_MODE_POLL) && + !is_legal_poll_interval(cfg->poll_interval)) { + pr_err("invalid poll interval %dms\n", cfg->poll_interval); + valid = false; + } else + ack->flags |= ACK_FLAG_VALID_INTERVAL; + + return valid ? 0 : -EINVAL; +} + +static void sunway_powercap_add_timer(void) +{ + unsigned long expire; + struct timer_list *timer = &driver_data.timer; + + expire = jiffies + msecs_to_jiffies(driver_data.poll_interval); + timer->expires = round_jiffies_relative(expire); + add_timer(timer); +} + +static void sunway_powercap_poll_func(struct timer_list *t) +{ + struct work_struct *work = &driver_data.work; + + schedule_work(work); + sunway_powercap_add_timer(); +} + +static void sunway_powercap_acpi_notify(acpi_handle device, u32 value, void *data) +{ + struct device *dev = driver_data.dev; + struct work_struct *work = &driver_data.work; + + if (value != SUNWAY_POWERCAP_ACPI_NOTIFY_VALUE) { + dev_err(dev, "unknown acpi notify value\n"); + return; + } + + schedule_work(work); +} + +static int sunway_powercap_setup_cfg(const struct sunway_powercap_cfg *cfg) +{ + bool is_poll_mode = (cfg->mode == SUNWAY_POWERCAP_MODE_POLL); + struct device *dev = driver_data.dev; + struct acpi_device *adev; + acpi_status status; + + driver_data.version = cfg->version; + driver_data.mode = cfg->mode; + driver_data.poll_interval = cfg->poll_interval; + + if (is_poll_mode) { + timer_setup(&driver_data.timer, sunway_powercap_poll_func, 0); + sunway_powercap_add_timer(); + } else { + /* Must be interrupt mode */ + + adev = ACPI_COMPANION(dev); + if (WARN_ON(!adev)) + return -EINVAL; + + status = acpi_install_notify_handler(adev->handle, + ACPI_DEVICE_NOTIFY, + sunway_powercap_acpi_notify, + NULL); + if (ACPI_FAILURE(status)) { + dev_err(dev, "unable to register notifier %08x\n", + status); + return -EINVAL; + } + } + + dev_info(dev, "found with version %d and %s mode\n", + driver_data.version, + is_poll_mode ? "polling" : "interrupt"); + + return 0; +} + +static int sunway_powercap_init_cfg(struct sunway_powercap_bmc_data *bmc_data) +{ + struct sunway_powercap_cfg cfg = { 0 }; + struct sunway_powercap_ack ack = { 0 }; + unsigned char cmd = SUNWAY_POWERCAP_CMD_GET_CFG; + int ret; + + ret = sunway_powercap_query(bmc_data, cmd, "cfg"); + if (ret) + return ret; + + ack.cmd = cmd; + + if (bmc_data->rx_msg_len != sizeof(cfg)) { + dev_err(bmc_data->bmc_device, "invalid cfg size %d\n", + bmc_data->rx_msg_len); + ret = -EINVAL; + } + + if (!ret) { + ack.flags |= ACK_FLAG_VALID_SIZE; + memcpy(&cfg, bmc_data->rx_msg_data, sizeof(cfg)); + + ret = sunway_powercap_validate_cfg(&cfg, &ack); + if (!ret) + ret = sunway_powercap_setup_cfg(&cfg); + } + + sunway_powercap_ack_bmc(bmc_data, &ack, 1); + + return ret; +} + +static void sunway_powercap_register_bmc(int iface, struct device *dev) +{ + struct sunway_powercap_bmc_data *bmc_data = &driver_data.bmc_data; + int ret; + + /* Multiple BMC for suwnay powercap are not supported */ + if (bmc_data->initialized) { + dev_err(dev, "unable to register sunway-powercap repeatedly\n"); + return; + } + + bmc_data->address.addr_type = IPMI_SYSTEM_INTERFACE_ADDR_TYPE; + bmc_data->address.channel = IPMI_BMC_CHANNEL; + bmc_data->address.data[0] = 0; + bmc_data->interface = iface; + bmc_data->bmc_device = dev; + + /* Create IPMI user */ + ret = ipmi_create_user(bmc_data->interface, &driver_data.ipmi_hndlrs, + bmc_data, &bmc_data->user); + if (ret) { + dev_err(dev, "unable to register user with IPMI interface %d", + bmc_data->interface); + return; + } + + /* Initialize message */ + bmc_data->tx_msgid = 0; + bmc_data->tx_message.netfn = SUNWAY_POWERCAP_NETFN; + bmc_data->tx_message.data = bmc_data->tx_msg_data; + + init_completion(&bmc_data->complete); + + ret = sunway_powercap_init_cfg(bmc_data); + if (ret) { + dev_err(dev, "unable to initialize powercap configuration\n"); + goto out_destroy_user; + } + + bmc_data->initialized = true; + + return; + +out_destroy_user: + ipmi_destroy_user(bmc_data->user); +} + +static void sunway_powercap_bmc_gone(int iface) +{ + struct sunway_powercap_bmc_data *bmc_data = &driver_data.bmc_data; + + if (WARN_ON(bmc_data->interface != iface)) + return; + + ipmi_destroy_user(bmc_data->user); +} + +static void sunway_powercap_msg_handler(struct ipmi_recv_msg *msg, + void *user_msg_data) +{ + struct sunway_powercap_bmc_data *bmc_data = user_msg_data; + + if (msg->msgid != bmc_data->tx_msgid) { + dev_err(bmc_data->bmc_device, + "mismatch between rx msgid (0x%lx) and tx msgid (0x%lx)!\n", + msg->msgid, + bmc_data->tx_msgid); + ipmi_free_recv_msg(msg); + return; + } + + bmc_data->rx_recv_type = msg->recv_type; + if (msg->msg.data_len > 0) + bmc_data->rx_result = msg->msg.data[0]; + else + bmc_data->rx_result = IPMI_UNKNOWN_ERR_COMPLETION_CODE; + + if (msg->msg.data_len > 1) { + bmc_data->rx_msg_len = msg->msg.data_len - 1; + memcpy(bmc_data->rx_msg_data, msg->msg.data + 1, + bmc_data->rx_msg_len); + } else + bmc_data->rx_msg_len = 0; + + ipmi_free_recv_msg(msg); + complete(&bmc_data->complete); +} + +static void do_powercap(struct work_struct *work) +{ + sunway_powercap_poll_once(&driver_data.bmc_data); +} + +static int sunway_powercap_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct cpufreq_policy *policy; + int cpu; + + driver_data.dev = dev; + + INIT_WORK(&driver_data.work, do_powercap); + + for_each_possible_cpu(cpu) { + int related_cpu, rcid = cpu_physical_id(cpu); + struct freq_qos_request *req; + + /* Initial state */ + powercap_cpu_data[related_cpu].state = SUNWAY_POWERCAP_STATE_FREE; + + powercap_cpu_data[related_cpu].core = rcid_to_core_id(rcid); + powercap_cpu_data[related_cpu].node = rcid_to_domain_id(rcid); + + if (powercap_cpu_data[cpu].policy) + continue; + + policy = cpufreq_cpu_get(cpu); + if (!policy) + continue; + + req = devm_kzalloc(dev, sizeof(*req), GFP_KERNEL); + if (!req) + return -ENOMEM; + + for_each_cpu(related_cpu, policy->related_cpus) { + powercap_cpu_data[related_cpu].policy = policy; + powercap_cpu_data[related_cpu].qos_req = req; + } + } + + return ipmi_smi_watcher_register(&driver_data.bmc_events); +} + +#ifdef CONFIG_ACPI +static const struct acpi_device_id sunway_powercap_acpi_match[] = { + { "SUNW0203", 0 }, + {}, +}; +#endif + +static struct platform_driver sunway_powercap_driver = { + .probe = sunway_powercap_probe, + .driver = { + .name = "sunway-powercap", + .acpi_match_table = ACPI_PTR(sunway_powercap_acpi_match), + }, +}; + +static int __init sunway_powercap_driver_init(void) +{ + return platform_driver_register(&sunway_powercap_driver); +} +late_initcall(sunway_powercap_driver_init); -- Gitee From c8c0a80448668de5e0b9f905137e8870475e91f8 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Sun, 27 Apr 2025 23:44:13 +0800 Subject: [PATCH 478/524] sw64: fix printing issue when using kprobe on ftrace Currently, the SAVE_PT_REGS in entry-ftrace.S does not save pc, and the kprobe_ftrace_handler does not save cause, orig_r0, orig_r19. As a result, when FUNCTION_TRACER is enabled, using kprobe to print this information at the function entry does not meet expectations. This commit fixed the issue. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry-ftrace.S | 1 + arch/sw_64/kernel/kprobes/kprobes-ftrace.c | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S index 00e9d25931ec..426d7561f097 100644 --- a/arch/sw_64/kernel/entry-ftrace.S +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -331,6 +331,7 @@ ftrace_regs_caller: 1: ldgp $29, 0($27) subl $28, MCOUNT_INSN_SIZE, $16 + stl $16, PT_REGS_PC($sp) bis $26, $31, $17 ldi $4, function_trace_op ldl $18, 0($4) diff --git a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c index ca3cee3a0316..12ca8d122348 100644 --- a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c +++ b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c @@ -26,6 +26,16 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, kprobes_inc_nmissed_count(p); } else { regs->regs[28] -= MCOUNT_INSN_SIZE; + if (in_task()) { + regs->orig_r0 = current_pt_regs()->orig_r0; + regs->orig_r19 = current_pt_regs()->orig_r19; + regs->cause = current_pt_regs()->cause; + } + if (in_interrupt()) { + regs->orig_r0 = get_irq_regs()->orig_r0; + regs->orig_r19 = get_irq_regs()->orig_r19; + regs->cause = get_irq_regs()->cause; + } __this_cpu_write(current_kprobe, p); kcb->kprobe_status = KPROBE_HIT_ACTIVE; -- Gitee From 86dc9fbaff2657100a81fa31b0fc246838b2b755 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Apr 2025 16:46:36 +0800 Subject: [PATCH 479/524] sw64: improve sw64_rrk Implement an improved sw64_rrk which stores the timestamp and message level. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/dup_print.c | 86 +++++++++++++++++++++++++++++------ kernel/printk/printk.c | 22 ++++++--- 2 files changed, 87 insertions(+), 21 deletions(-) diff --git a/arch/sw_64/kernel/dup_print.c b/arch/sw_64/kernel/dup_print.c index eef31527dc13..fa604abef38c 100644 --- a/arch/sw_64/kernel/dup_print.c +++ b/arch/sw_64/kernel/dup_print.c @@ -12,33 +12,88 @@ static DEFINE_SPINLOCK(printk_lock); -unsigned long sw64_printk_offset; +static unsigned long sw64_printk_offset; #define PRINTK_SIZE 0x100000UL -void sw64_printk(const char *fmt, va_list args) +static bool rrk_last_newline_end; +static unsigned long rrk_last_id; +static const char * const level_str[] = { + "(0)EMERG", + "(1)ALERT", + "(2)CRIT", + "(3)ERR", + "(4)WARNING", + "(5)NOTICE", + "(6)INFO", + "(7)DEBUG" +}; +#define LEVEL_STR_MAX_LEN 10 // length of "(4)WARNING" + +void sw64_rrk_store(const char *text, u16 text_len, u64 ts_nsec, int level, + unsigned long id, bool newline_end) { char *sw64_printk_buf; unsigned long flags; - char textbuf[1024]; - const char *text; - size_t text_len; + size_t __maybe_unused rrk_len; + char header_buf[128]; + /* same time fmt as print_time() in printk.c */ + char header_fmt[] = "[%5llu.%06llu %-"__stringify(LEVEL_STR_MAX_LEN)"s] "; + size_t header_len; + char *newline; + /* if writing a new entry while the last one did not end with '\n', print '\n' first */ + bool newline_first = rrk_last_id && (rrk_last_id != id) && (!rrk_last_newline_end); + bool wrap = false; + unsigned long max_offset_allowed; spin_lock_irqsave(&printk_lock, flags); - sw64_printk_buf = (char *)(KERNEL_PRINTK_BUFF_BASE + sw64_printk_offset); + header_len = scnprintf(header_buf, sizeof(header_buf), header_fmt, + ts_nsec / NSEC_PER_SEC, (ts_nsec % NSEC_PER_SEC) / NSEC_PER_USEC, + level >= 0 ? level_str[level] : "CONT"); - text_len = vscnprintf(textbuf, sizeof(textbuf), fmt, args); - text = printk_skip_headers(textbuf); - text_len -= text - textbuf; - - if (sw64_printk_offset >= (PRINTK_SIZE - 1024)) { + max_offset_allowed = PRINTK_SIZE - text_len - header_len - (newline_first ? 1 : 0); + if (unlikely(sw64_printk_offset >= max_offset_allowed)) { sw64_printk_offset = 0; - sw64_printk_buf = (char *)(KERNEL_PRINTK_BUFF_BASE + sw64_printk_offset); memset(sw64_printk_buf, 0, PRINTK_SIZE); + wrap = true; + } + sw64_printk_buf = (char *)(KERNEL_PRINTK_BUFF_BASE + sw64_printk_offset); + + if (unlikely(newline_first)) { + sw64_printk_buf[0] = '\n'; + sw64_printk_buf++; + sw64_printk_offset++; + } + + if (likely(level != -1) || unlikely(wrap)) { + memcpy(sw64_printk_buf, header_buf, header_len); + sw64_printk_offset += header_len; + sw64_printk_buf += header_len; + } + + while (unlikely((newline = strnchr(text, text_len, '\n')))) { + size_t len; + + /* copy the first line */ + newline++; + len = newline - text; + memcpy(sw64_printk_buf, text, len); + + /* add padding for next line */ + memset(&sw64_printk_buf[len], ' ', header_len); + + text += len; + text_len -= len; + sw64_printk_buf += len + header_len; + sw64_printk_offset += len + header_len; } memcpy(sw64_printk_buf, text, text_len); sw64_printk_offset += text_len; + if (likely(sw64_printk_buf[text_len - 1] != '\n' && newline_end)) { + sw64_printk_buf[text_len] = '\n'; + sw64_printk_offset++; + } if (is_in_emul()) { void __iomem *addr = __va(QEMU_PRINTF_BUFF_BASE); @@ -47,6 +102,9 @@ void sw64_printk(const char *fmt, va_list args) *(u64 *)addr = data; } + rrk_last_id = id; + rrk_last_newline_end = newline_end; + spin_unlock_irqrestore(&printk_lock, flags); } #endif @@ -55,8 +113,8 @@ void sw64_printk(const char *fmt, va_list args) #include static DEFINE_SPINLOCK(printf_lock); -#define USER_PRINT_BUFF_BASE (0x600000UL + __START_KERNEL_map) -#define USER_PRINT_BUFF_LEN 0x100000UL +#define USER_PRINT_BUFF_BASE (0x600000UL + __START_KERNEL_map) +#define USER_PRINT_BUFF_LEN 0x100000UL #define USER_MESSAGE_MAX_LEN 0x100000UL unsigned long sw64_printf_offset; int sw64_user_printf(const char __user *buf, int len) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index a63883b046ef..b3c0e1267468 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2184,6 +2184,10 @@ static u16 printk_sprint(char *text, u16 size, int facility, return text_len; } +#ifdef CONFIG_SW64_RRK +extern void sw64_rrk_store(const char *text, u16 text_len, u64 ts_nsec, int level, + unsigned long id, bool final); +#endif __printf(4, 0) int vprintk_store(int facility, int level, const struct dev_printk_info *dev_info, @@ -2202,17 +2206,10 @@ int vprintk_store(int facility, int level, u16 text_len; int ret = 0; u64 ts_nsec; -#ifdef CONFIG_SW64_RRK - extern void sw64_printk(const char *fmt, va_list args); -#endif if (!printk_enter_irqsave(recursion_ptr, irqflags)) return 0; -#ifdef CONFIG_SW64_RRK - sw64_printk(fmt, args); -#endif - /* * Since the duration of printk() can vary depending on the message * and state of the ringbuffer, grab the timestamp now so that it is @@ -2260,6 +2257,12 @@ int vprintk_store(int facility, int level, prb_commit(&e); } +#ifdef CONFIG_SW64_RRK + sw64_rrk_store(&r.text_buf[r.info->text_len], text_len, + r.info->ts_nsec, -1, e.id, + !!(flags & LOG_NEWLINE)); +#endif + ret = text_len; goto out; } @@ -2299,6 +2302,11 @@ int vprintk_store(int facility, int level, else prb_final_commit(&e); +#ifdef CONFIG_SW64_RRK + sw64_rrk_store(&r.text_buf[0], r.info->text_len, r.info->ts_nsec, r.info->level, + e.id, !!(flags & LOG_NEWLINE)); +#endif + ret = text_len + trunc_msg_len; out: printk_exit_irqrestore(recursion_ptr, irqflags); -- Gitee From 20beff18c3e27a3bcf3ffa3ea3e24e3066ed7daa Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 2 Apr 2025 10:04:10 +0800 Subject: [PATCH 480/524] sw64: move syscall handling to sys_sw64.c Move do_entSys() to sys_sw64.c, where most arch syscalls are defined. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/sys_sw64.c | 41 ++++++++++++++++++++++++++++++++++++ arch/sw_64/kernel/traps.c | 41 ------------------------------------ 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/arch/sw_64/kernel/sys_sw64.c b/arch/sw_64/kernel/sys_sw64.c index 7efebce4af11..c11c02285bf3 100644 --- a/arch/sw_64/kernel/sys_sw64.c +++ b/arch/sw_64/kernel/sys_sw64.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 #include + #include +#include SYSCALL_DEFINE5(getsysinfo, unsigned long, op, void __user *, buffer, unsigned long, nbytes, int __user *, start, void __user *, arg) @@ -335,3 +337,42 @@ SYSCALL_DEFINE0(pfh_ops) } #endif /* CONFIG_SUBARCH_C4 */ + +asmlinkage void noinstr do_entSys(struct pt_regs *regs) +{ + long ret = -ENOSYS; + unsigned long nr; + unsigned long ti_flags = current_thread_info()->flags; + + regs->orig_r0 = regs->regs[0]; + regs->orig_r19 = regs->regs[19]; + nr = regs->regs[0]; + + if (ti_flags & _TIF_SYSCALL_WORK) { + nr = syscall_trace_enter(); + if (nr == NO_SYSCALL) + goto syscall_out; + regs->orig_r0 = regs->regs[0]; + regs->orig_r19 = regs->regs[19]; + } + + if (nr < __NR_syscalls) { + syscall_fn_t syscall_fn = sys_call_table[nr]; + + ret = syscall_fn(regs->regs[16], regs->regs[17], regs->regs[18], + regs->regs[19], regs->regs[20], regs->regs[21]); + } + + if ((nr != __NR_sigreturn) && (nr != __NR_rt_sigreturn)) { + if (likely((ret >= 0) || regs->orig_r0 == NO_SYSCALL)) + syscall_set_return_value(current, regs, 0, ret); + else + syscall_set_return_value(current, regs, ret, 0); + } + +syscall_out: + rseq_syscall(regs); + + if (ti_flags & _TIF_SYSCALL_WORK) + syscall_trace_leave(); +} diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index a84441c79b61..c2433236fb2d 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -31,7 +30,6 @@ #include #include #include -#include #include #include "proto.h" @@ -386,45 +384,6 @@ asmlinkage void noinstr do_entIF(struct pt_regs *regs) force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->pc); } -asmlinkage void noinstr do_entSys(struct pt_regs *regs) -{ - long ret = -ENOSYS; - unsigned long nr; - unsigned long ti_flags = current_thread_info()->flags; - - regs->orig_r0 = regs->regs[0]; - regs->orig_r19 = regs->regs[19]; - nr = regs->regs[0]; - - if (ti_flags & _TIF_SYSCALL_WORK) { - nr = syscall_trace_enter(); - if (nr == NO_SYSCALL) - goto syscall_out; - regs->orig_r0 = regs->regs[0]; - regs->orig_r19 = regs->regs[19]; - } - - if (nr < __NR_syscalls) { - syscall_fn_t syscall_fn = sys_call_table[nr]; - - ret = syscall_fn(regs->regs[16], regs->regs[17], regs->regs[18], - regs->regs[19], regs->regs[20], regs->regs[21]); - } - - if ((nr != __NR_sigreturn) && (nr != __NR_rt_sigreturn)) { - if (likely((ret >= 0) || regs->orig_r0 == NO_SYSCALL)) - syscall_set_return_value(current, regs, 0, ret); - else - syscall_set_return_value(current, regs, ret, 0); - } - -syscall_out: - rseq_syscall(regs); - - if (ti_flags & _TIF_SYSCALL_WORK) - syscall_trace_leave(); -} - struct nmi_ctx { unsigned long csr_sp; unsigned long csr_scratch; -- Gitee From 003c99b1cc35e88acbc5b0843da929abd7b6f070 Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Mon, 22 Jan 2024 09:20:32 +0800 Subject: [PATCH 481/524] sw64: qspinlock: add PARAVIRT SPINLOCK support for sw64 As kernel has used this interface, and Compact NUMA-aware lock depends on it, so introduce basic support into sw64. Note that pv_wait() and pv_kick() are left unimplemented. Signed-off-by: Gu Zitao Reviewed-by: He Sheng --- arch/sw_64/Kconfig | 21 ++++++ arch/sw_64/include/asm/paravirt.h | 72 +++++++++++++++++++++ arch/sw_64/include/asm/paravirt_api_clock.h | 1 + arch/sw_64/include/asm/qspinlock.h | 45 +++++++++++++ arch/sw_64/include/asm/qspinlock_paravirt.h | 7 ++ arch/sw_64/include/asm/spinlock.h | 4 ++ arch/sw_64/kernel/Makefile | 1 + arch/sw_64/kernel/paravirt-spinlocks.c | 14 ++++ arch/sw_64/kernel/paravirt.c | 58 +++++++++++++++++ arch/sw_64/kernel/setup.c | 4 ++ 10 files changed, 227 insertions(+) create mode 100644 arch/sw_64/include/asm/paravirt.h create mode 100644 arch/sw_64/include/asm/paravirt_api_clock.h create mode 100644 arch/sw_64/include/asm/qspinlock.h create mode 100644 arch/sw_64/include/asm/qspinlock_paravirt.h create mode 100644 arch/sw_64/kernel/paravirt-spinlocks.c create mode 100644 arch/sw_64/kernel/paravirt.c diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index d8c82f89d616..d936f96f5b8b 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -424,6 +424,27 @@ config NUMA Access). This option is for configuring high-end multiprocessor server machines. If in doubt, say N. +config PARAVIRT_SPINLOCKS + bool "Paravirtualization layer for spinlocks" + depends on PARAVIRT && SMP + help + Paravirtualized spinlocks allow a pvops backend to replace the + spinlock implementation with something virtualization-friendly + (for example, block the virtual CPU rather than spinning). + + It has a minimal impact on native kernels and gives a nice performance + benefit on paravirtualized KVM kernels. + + If you are unsure how to answer this question, answer Y. + +config PARAVIRT + bool "Enable paravirtualization code" + select PARAVIRT_SPINLOCKS + help + This changes the kernel so it can modify itself when it is run + under a hypervisor, potentially improving performance significantly + over full virtualization. + config USE_PERCPU_NUMA_NODE_ID def_bool y depends on NUMA diff --git a/arch/sw_64/include/asm/paravirt.h b/arch/sw_64/include/asm/paravirt.h new file mode 100644 index 000000000000..0ca6befc53ee --- /dev/null +++ b/arch/sw_64/include/asm/paravirt.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_PARAVIRT_H +#define _ASM_SW64_PARAVIRT_H + +#ifdef CONFIG_PARAVIRT +struct static_key; +extern struct static_key paravirt_steal_enabled; +extern struct static_key paravirt_steal_rq_enabled; + +struct pv_time_ops { + unsigned long long (*steal_clock)(int cpu); +}; + +struct pv_lock_ops { + void (*wait)(u8 *ptr, u8 val); + void (*kick)(int cpu); + void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); + void (*queued_spin_unlock)(struct qspinlock *lock); + bool (*vcpu_is_preempted)(int cpu); +}; + +struct paravirt_patch_template { + struct pv_time_ops time; + struct pv_lock_ops lock; +}; + +extern struct paravirt_patch_template pv_ops; + +static inline u64 paravirt_steal_clock(int cpu) +{ + return pv_ops.time.steal_clock(cpu); +} + +__visible bool __native_vcpu_is_preempted(int cpu); + +static inline bool pv_vcpu_is_preempted(int cpu) +{ + return pv_ops.lock.vcpu_is_preempted(cpu); +} + +#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) +bool pv_is_native_spin_unlock(void); +void __init pv_qspinlock_init(void); + +static inline void pv_wait(u8 *ptr, u8 val) +{ + return pv_ops.lock.wait(ptr, val); +} + +static inline void pv_kick(int cpu) +{ + return pv_ops.lock.kick(cpu); +} + +static inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) +{ + return pv_ops.lock.queued_spin_lock_slowpath(lock, val); +} + +static inline void pv_queued_spin_unlock(struct qspinlock *lock) +{ + return pv_ops.lock.queued_spin_unlock(lock); +} +#endif /* CONFIG_PARAVIRT_SPINLOCKS */ + +#else + +#define pv_qspinlock_init() do {} while (0) + +#endif /* CONFIG_PARAVIRT */ + +#endif /* _ASM_SW64_PARAVIRT_H */ diff --git a/arch/sw_64/include/asm/paravirt_api_clock.h b/arch/sw_64/include/asm/paravirt_api_clock.h new file mode 100644 index 000000000000..65ac7cee0dad --- /dev/null +++ b/arch/sw_64/include/asm/paravirt_api_clock.h @@ -0,0 +1 @@ +#include diff --git a/arch/sw_64/include/asm/qspinlock.h b/arch/sw_64/include/asm/qspinlock.h new file mode 100644 index 000000000000..11e5169a9717 --- /dev/null +++ b/arch/sw_64/include/asm/qspinlock.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_QSPINLOCK_H +#define _ASM_SW64_QSPINLOCK_H + +#include +#include + +#ifdef CONFIG_PARAVIRT_SPINLOCKS +/* keep the same as x86 */ +#define _Q_PENDING_LOOPS (1 << 9) + +extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +extern void __pv_init_lock_hash(void); +extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); + +#define queued_spin_unlock queued_spin_unlock +/* + * queued_spin_unlock - release a queued spinlock + * @lock : Pointer to queued spinlock structure + * + * A smp_store_release() on the least-significant byte. + */ +static inline void native_queued_spin_unlock(struct qspinlock *lock) +{ + /* + * Now that we have a reference to the (likely) + * blocked pv_node, release the lock. + */ + smp_store_release(&lock->locked, 0); +} + +static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) +{ + pv_queued_spin_lock_slowpath(lock, val); +} + +static inline void queued_spin_unlock(struct qspinlock *lock) +{ + pv_queued_spin_unlock(lock); +} +#endif + +#include + +#endif /* _ASM_SW64_QSPINLOCK_H */ diff --git a/arch/sw_64/include/asm/qspinlock_paravirt.h b/arch/sw_64/include/asm/qspinlock_paravirt.h new file mode 100644 index 000000000000..e504b135e2b5 --- /dev/null +++ b/arch/sw_64/include/asm/qspinlock_paravirt.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_QSPINLOCK_PARAVIRT_H +#define _ASM_SW64_QSPINLOCK_PARAVIRT_H + +extern void __pv_queued_spin_unlock(struct qspinlock *lock); + +#endif /* _ASM_SW64_QSPINLOCK_PARAVIRT_H */ diff --git a/arch/sw_64/include/asm/spinlock.h b/arch/sw_64/include/asm/spinlock.h index 64358f32cd9a..1770cd9a3376 100644 --- a/arch/sw_64/include/asm/spinlock.h +++ b/arch/sw_64/include/asm/spinlock.h @@ -17,6 +17,10 @@ #include #include +#include + +/* How long a lock should spin before we consider blocking */ +#define SPIN_THRESHOLD (1 << 15) /* See include/linux/spinlock.h */ #define smp_mb__after_spinlock() smp_mb() diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 539b3011c854..caf6de81dbde 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -66,3 +66,4 @@ obj-$(CONFIG_UPROBES) += uprobes.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o +obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt-spinlocks.o diff --git a/arch/sw_64/kernel/paravirt-spinlocks.c b/arch/sw_64/kernel/paravirt-spinlocks.c new file mode 100644 index 000000000000..cd4bd3e68721 --- /dev/null +++ b/arch/sw_64/kernel/paravirt-spinlocks.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +__visible bool __native_vcpu_is_preempted(int cpu) +{ + return false; +} + +bool pv_is_native_spin_unlock(void) +{ + return false; +} diff --git a/arch/sw_64/kernel/paravirt.c b/arch/sw_64/kernel/paravirt.c new file mode 100644 index 000000000000..e22a718fc525 --- /dev/null +++ b/arch/sw_64/kernel/paravirt.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct static_key paravirt_steal_enabled; +struct static_key paravirt_steal_rq_enabled; + +struct paravirt_patch_template pv_ops = { +#ifdef CONFIG_PARAVIRT_SPINLOCKS + .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, + .lock.queued_spin_unlock = native_queued_spin_unlock, +#endif + .lock.vcpu_is_preempted = __native_vcpu_is_preempted, +}; +EXPORT_SYMBOL_GPL(pv_ops); + +#ifdef CONFIG_PARAVIRT_SPINLOCKS +static bool pvqspinlock; + +static __init int parse_pvqspinlock(char *arg) +{ + pvqspinlock = true; + return 0; +} +early_param("pvqspinlock", parse_pvqspinlock); + +void __init pv_qspinlock_init(void) +{ + /* Don't use the PV qspinlock code if there is only 1 vCPU. */ + if (num_possible_cpus() == 1) + return; + + if (!pvqspinlock) { + pr_info("PV qspinlocks disabled\n"); + return; + } + + pr_info("PV qspinlocks enabled\n"); + + __pv_init_lock_hash(); + pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; + pv_ops.lock.queued_spin_unlock = __pv_queued_spin_unlock; + /* TODO: wait and kick */ + pv_ops.lock.wait = NULL; + pv_ops.lock.kick = NULL; +} +#endif diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 02c3df0f8265..27f3d3b25bbe 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -756,6 +756,10 @@ setup_arch(char **cmdline_p) /* Default root filesystem to sda2. */ ROOT_DEV = MKDEV(SCSI_DISK0_MAJOR, 2); + +#ifdef CONFIG_PARAVIRT_SPINLOCKS + pv_qspinlock_init(); +#endif } static int -- Gitee From e1002151fc1b26a044d6a39b653e56bcde4b711b Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Mon, 22 Jan 2024 09:29:18 +0800 Subject: [PATCH 482/524] sw64: qspinlock: add CNA support for sw64 Compact NUMA-aware lock(CNA) is a NUMA-aware version of the MCS lock. Enabled sw64 to support CNA by adding a new configuration option (NUMA_AWARE_SPINLOCKS) and delaying the initialization function of CNA until after the initialization of the cpu_to_node function. Signed-off-by: Gu Zitao Reviewed-by: He Sheng --- arch/sw_64/Kconfig | 17 +++++++++++++++++ arch/sw_64/include/asm/qspinlock.h | 4 ++++ arch/sw_64/kernel/smp.c | 3 +++ 3 files changed, 24 insertions(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index d936f96f5b8b..c1b74369e030 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -424,6 +424,23 @@ config NUMA Access). This option is for configuring high-end multiprocessor server machines. If in doubt, say N. +config NUMA_AWARE_SPINLOCKS + bool "Numa-aware spinlocks" + depends on NUMA + depends on QUEUED_SPINLOCKS + depends on 64BIT + depends on PARAVIRT_SPINLOCKS + default y + help + Introduce NUMA (Non Uniform Memory Access) awareness into + the slow path of spinlocks. + + In this variant of qspinlock, the kernel will try to keep the lock + on the same node, thus reducing the number of remote cache misses, + while trading some of the short term fairness for better performance. + + Say N if you want absolute first come first serve fairness. + config PARAVIRT_SPINLOCKS bool "Paravirtualization layer for spinlocks" depends on PARAVIRT && SMP diff --git a/arch/sw_64/include/asm/qspinlock.h b/arch/sw_64/include/asm/qspinlock.h index 11e5169a9717..15ae23e22cc6 100644 --- a/arch/sw_64/include/asm/qspinlock.h +++ b/arch/sw_64/include/asm/qspinlock.h @@ -5,6 +5,10 @@ #include #include +#ifdef CONFIG_NUMA_AWARE_SPINLOCKS +extern void cna_configure_spin_lock_slowpath(void); +#endif + #ifdef CONFIG_PARAVIRT_SPINLOCKS /* keep the same as x86 */ #define _Q_PENDING_LOOPS (1 << 9) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index ef88fe2344e9..11c442ed0f14 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -502,6 +502,9 @@ void __init smp_prepare_cpus(unsigned int max_cpus) for_each_possible_cpu(cpu) { numa_store_cpu_info(cpu); } +#ifdef CONFIG_NUMA_AWARE_SPINLOCKS + cna_configure_spin_lock_slowpath(); +#endif /* Nothing to do on a UP box, or when told not to. */ if (nr_cpu_ids == 1 || max_cpus == 0) { -- Gitee From 06235ef5bf1d3bb2cb40796d2d4e35573be8dda2 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 9 May 2025 01:06:47 +0000 Subject: [PATCH 483/524] sw64: iommu: do real map on iova in 32bit mmio space According to commit e50319254576 ("sw64: iommu: work around iova range check after resv_region"), the current map/unmap strategy will directly return success, without doing real map when application trying to use IOVA in 3.5G~4G. This solved the problem of passing IOVA check on initiating certain application, but what we do not expect is that VFIO will still pin the physical memory pages for these IOVA (in 3.5~4G). It seems to work fine during normal using. But when it comes to unmap/unpin, as VFIO cannot find the related physical pfns through mapping tables, (because we did not actually map them!), it will cause a 512M physical memory leak after application exits. The current strategy is to do real mappings for these IOVA. As for possible collisions, we rely on OS to not initiate access requests in this range. Fixes: e50319254576 ("sw64: iommu: work around iova range check after resv_region") Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu_v2.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index 7ac13d6732f0..b73439dd13e5 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -1389,10 +1389,6 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) unsigned long paddr, grn; unsigned long is_last; - if ((iova > SW64_32BIT_DMA_LIMIT) - && (iova <= DMA_BIT_MASK(32))) - return iova; - if (iova >= SW64_BAR_ADDRESS) return iova; @@ -1470,14 +1466,14 @@ sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, * and buggy. * * We manage to find a compromise solution, which is allow these IOVA - * being allocated and "mapped" as usual, but with a warning issued to + * being allocated and mapped as usual, and with a warning issued to * users at the same time. So users can quickly learn if they are using * these "illegal" IOVA and thus change their strategies accordingly. */ - if ((iova > SW64_32BIT_DMA_LIMIT) + if ((SW64_32BIT_DMA_LIMIT < iova + page_size) && (iova <= DMA_BIT_MASK(32))) { - pr_warn_once("Domain %d are using IOVA: %lx\n", sdomain->id, iova); - return 0; + pr_warn_once("process %s (pid:%d) is using domain %d with IOVA: %lx\n", + current->comm, current->pid, sdomain->id, iova); } /* @@ -1507,10 +1503,6 @@ sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); size_t unmap_size; - if ((iova > SW64_32BIT_DMA_LIMIT) - && (iova <= DMA_BIT_MASK(32))) - return page_size; - if (iova >= SW64_BAR_ADDRESS) return page_size; -- Gitee From 9b4b7728d2c290c3f78ef0522468732f3a3f9c55 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Thu, 15 May 2025 14:37:10 +0800 Subject: [PATCH 484/524] sw64: kprobe on ftrace: fix bug in restoring regs Currently, in kprobe_ftrace_handler, if it is in_interrupt, orig_r0, orig_r19, and cause are saved through get_irq_regs. However, this ignores the case of soft interrupts, where get_irq_regs will return 0. Therefore it is adjusted to save through get_irq_regs when it is in_irq. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/kprobes/kprobes-ftrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c index 12ca8d122348..9a6b6cf7c750 100644 --- a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c +++ b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c @@ -31,7 +31,7 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, regs->orig_r19 = current_pt_regs()->orig_r19; regs->cause = current_pt_regs()->cause; } - if (in_interrupt()) { + if (in_irq()) { regs->orig_r0 = get_irq_regs()->orig_r0; regs->orig_r19 = get_irq_regs()->orig_r19; regs->cause = get_irq_regs()->cause; -- Gitee From 90a2c91c395815f70aba8dcf700393dc730afe69 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Tue, 20 May 2025 22:29:09 +0800 Subject: [PATCH 485/524] sw64: ioremap: support generic ioremap for C4 Use generic ioremap instead of the architecture-specific ioremap. Note: When accessing I/O physical addresses: 1. Do not use __va(). 2. Always use ioremap() to map them to the vmalloc segment. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/include/asm/io.h | 17 ++++++++++------- arch/sw_64/include/asm/pgtable.h | 1 + arch/sw_64/lib/iomap.c | 4 ++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index c1b74369e030..e61c5245dc90 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -66,6 +66,7 @@ config SW64 select GENERIC_ARCH_TOPOLOGY select GENERIC_CLOCKEVENTS select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO + select GENERIC_IOREMAP if SUBARCH_C4 select GENERIC_IRQ_LEGACY select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select GENERIC_IRQ_MIGRATION if SMP diff --git a/arch/sw_64/include/asm/io.h b/arch/sw_64/include/asm/io.h index 1695801e6c4b..121e909a2cf9 100644 --- a/arch/sw_64/include/asm/io.h +++ b/arch/sw_64/include/asm/io.h @@ -30,6 +30,7 @@ extern void outl(u32 b, unsigned long port); #define outw outw #define outl outl +#ifndef CONFIG_GENERIC_IOREMAP static inline void __iomem *__ioremap(phys_addr_t addr, size_t size, pgprot_t prot) { @@ -37,19 +38,21 @@ static inline void __iomem *__ioremap(phys_addr_t addr, size_t size, return (void __iomem *)(tmp); } - #define ioremap(addr, size) __ioremap((addr), (size), PAGE_KERNEL) -#define ioremap_nocache(addr, size) __ioremap((addr), (size), PAGE_KERNEL) -#define ioremap_cache(addr, size) __ioremap((addr), (size), PAGE_KERNEL) + +static inline void __iounmap(volatile void __iomem *addr) +{ +} +#define iounmap __iounmap +#endif + +#define ioremap_nocache(addr, size) ioremap((addr), (size)) +#define ioremap_cache(addr, size) ioremap((addr), (size)) #define ioremap_uc ioremap_nocache #define ioport_map ioport_map extern void __iomem *ioport_map(unsigned long port, unsigned int nr); -static inline void iounmap(volatile void __iomem *addr) -{ -} - /* * String version of IO memory access ops: */ diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 0b9b65685e74..57c2639aeaf8 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -188,6 +188,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define PAGE_NONE __pgprot(__ACCESS_BITS | _PAGE_FOR | _PAGE_FOW | _PAGE_FOE | _PAGE_LEAF | _PAGE_PROTNONE) #define PAGE_KERNEL __pgprot(_PAGE_VALID | _PAGE_KERN | _PAGE_LEAF) #define _PAGE_NORMAL(x) __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_LEAF | (x)) +#define _PAGE_IOREMAP pgprot_val(PAGE_KERNEL) #define page_valid_kern(x) ((x & (_PAGE_VALID | _PAGE_KERN)) == (_PAGE_VALID | _PAGE_KERN)) #endif diff --git a/arch/sw_64/lib/iomap.c b/arch/sw_64/lib/iomap.c index c73004a87dcd..51e7600b0fec 100644 --- a/arch/sw_64/lib/iomap.c +++ b/arch/sw_64/lib/iomap.c @@ -262,8 +262,8 @@ EXPORT_SYMBOL(_memset_c_io); void __iomem *ioport_map(unsigned long port, unsigned int size) { if (port >= 0x100000) - return __va(port); + return ioremap(port, size); - return __va((port << legacy_io_shift) | legacy_io_base); + return ioremap((port << legacy_io_shift) | legacy_io_base, size); } EXPORT_SYMBOL(ioport_map); -- Gitee From 68326f34707948ccbde67011429b741d4e3f2f93 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 9 May 2025 19:16:30 +0800 Subject: [PATCH 486/524] sw64: ioremap: fix an overflow in pci_remap_cfgspace In pei_ecam_create(), the SW64 defines "bsz" as 1<<24, which is larger than other architectures. When calculating "bus_range * bsz", the result exceeds 32 bits, causing an unsigned int overflow. This overflow leads to pci_remap_cfgspace() receiving an incorrect size. To fix this, change the type of "bus_range" and "bsz" from unsigned int to unsigned long for SW64. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/ecam.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/pci/ecam.c b/drivers/pci/ecam.c index 1c40d2506aef..e568ca750522 100644 --- a/drivers/pci/ecam.c +++ b/drivers/pci/ecam.c @@ -79,7 +79,12 @@ struct pci_config_window *pci_ecam_create(struct device *dev, if (!cfg->winp) goto err_exit_malloc; } else { +#if IS_ENABLED(CONFIG_SW64) + cfg->win = pci_remap_cfgspace(cfgres->start, + (unsigned long)bus_range * (unsigned long)bsz); +#else cfg->win = pci_remap_cfgspace(cfgres->start, bus_range * bsz); +#endif if (!cfg->win) goto err_exit_iomap; } -- Gitee From 6e187bb5b6859ddbd2df2c021afb21497841c61e Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 9 May 2025 23:29:14 +0800 Subject: [PATCH 487/524] sw64: ioremap: map some IO space with ioremap Since we have generic ioremap, this patch replaces __va() with ioremap() for IO mapping of RC, EP, PIU and usb xhci. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 3 +++ drivers/pci/controller/pci-sunway.c | 9 +++++---- drivers/usb/host/pci-quirks.c | 4 +++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 74d33777f05d..6e8d0ed37567 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -48,6 +48,7 @@ enum SUNWAY_PIU_IOR0 { PCACHE_ACCESS = 0xb880UL, PCACHE_ITEM_TAG = 0xb900UL, PCACHE_ITEM_DATA0 = 0xb980UL, + SUNWAY_PIU_IOR0_SIZE = 0xba00UL, }; enum SUNWAY_PIU_IOR1 { @@ -57,6 +58,7 @@ enum SUNWAY_PIU_IOR1 { RCDEBUGINF1 = 0xc80UL, DCACONTROL = 0x1a00UL, DEVICEID0 = 0x1a80UL, + SUNWAY_PIU_IOR1_SIZE = 0x1b00UL, }; enum SUNWAY_RC { @@ -78,6 +80,7 @@ enum SUNWAY_RC { RC_PHY_INT_REG = 0x80000UL, RC_PHY_EXT_GEN1 = 0x82400UL, RC_PHY_EXT_GEN2 = 0x82480UL, + SUNWAY_RC_SIZE = 0x82500UL, }; struct pci_dev; diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index a4535c61c04e..76514c5695cb 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -865,10 +865,9 @@ static int pci_prepare_controller(struct pci_controller *hose, hose->dense_mem_base = props[PROP_PCIE_IO_BASE]; hose->dense_io_base = props[PROP_EP_IO_BASE]; - hose->rc_config_space_base = __va(props[PROP_RC_CONFIG_BASE]); - hose->ep_config_space_base = __va(props[PROP_EP_CONFIG_BASE]); - hose->piu_ior0_base = __va(props[PROP_PIU_IOR0_BASE]); - hose->piu_ior1_base = __va(props[PROP_PIU_IOR1_BASE]); + hose->rc_config_space_base = ioremap(props[PROP_RC_CONFIG_BASE], SUNWAY_RC_SIZE); + hose->piu_ior0_base = ioremap(props[PROP_PIU_IOR0_BASE], SUNWAY_PIU_IOR0_SIZE); + hose->piu_ior1_base = ioremap(props[PROP_PIU_IOR1_BASE], SUNWAY_PIU_IOR1_SIZE); hose->first_busno = 0xff; hose->last_busno = 0xff; @@ -964,6 +963,8 @@ static int sunway_pci_ecam_init(struct pci_config_window *cfg) if (!hose) return -ENOMEM; + hose->ep_config_space_base = cfg->win; + /* Init pci_controller */ ret = pci_prepare_controller(hose, fwnode); if (ret) { diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c index 80e5ca7b940f..88c493b60d9a 100644 --- a/drivers/usb/host/pci-quirks.c +++ b/drivers/usb/host/pci-quirks.c @@ -1328,7 +1328,7 @@ fixup_usb_xhci_reset(struct pci_dev *dev) if (offset == 0) return; - base = (void *)__va(SW64_PCI_IO_BASE(hose->node, hose->index) | offset); + base = ioremap(SW64_PCI_IO_BASE(hose->node, hose->index) | offset, SZ_8K); ext_cap_offset = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_LEGACY); if (!ext_cap_offset) @@ -1415,6 +1415,8 @@ fixup_usb_xhci_reset(struct pci_dev *dev) pci_read_config_dword(dev, PCI_COMMAND, &tmp); tmp &= ~(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); pci_write_config_dword(dev, PCI_COMMAND, tmp); + + iounmap(base); } DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_SERIAL_USB_XHCI, 0, fixup_usb_xhci_reset); -- Gitee From cfe7a52357366c1b502f8d8f4dd2cd4a6ef9322c Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Wed, 21 May 2025 13:51:39 +0800 Subject: [PATCH 488/524] sw64: remove redundant qspinlock.h from Kbuild Fix compiler warning: scripts/Makefile.asm-generic:25: redundant generic-y found in arch/sw_64/include/asm/Kbuild: qspinlock.h Signed-off-by: Gu Zitao Reviewed-by: He Sheng --- arch/sw_64/include/asm/Kbuild | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/sw_64/include/asm/Kbuild b/arch/sw_64/include/asm/Kbuild index e9cbabda1516..bbd410bda0a4 100644 --- a/arch/sw_64/include/asm/Kbuild +++ b/arch/sw_64/include/asm/Kbuild @@ -5,7 +5,6 @@ generic-y += export.h generic-y += mcs_spinlock.h generic-y += param.h generic-y += qrwlock.h -generic-y += qspinlock.h generic-y += rwsem.h generic-y += seccomp.h generic-y += segment.h -- Gitee From d87b3dc49ff295ac9c35875fa2a4b095ea3dde45 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Thu, 22 May 2025 15:13:57 +0800 Subject: [PATCH 489/524] sw64: cpufreq: fix the frequency returned by cpufreq_driver->get() Before setting the CPU frequency using the CCLK_CTL register, it is impossible to get the CPU frequency via PLL configuration. In this case, the hmcode should be called instead. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/cpufreq/sunway-cpufreq.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/drivers/cpufreq/sunway-cpufreq.c b/drivers/cpufreq/sunway-cpufreq.c index ba22843d8eed..41a6b21ff827 100644 --- a/drivers/cpufreq/sunway-cpufreq.c +++ b/drivers/cpufreq/sunway-cpufreq.c @@ -135,26 +135,23 @@ static void __init fill_freq_table(struct cpufreq_frequency_table *ft) static unsigned int sunway_get_rate(struct cpufreq_policy *policy) { - int i, clu_lv1_sel; + int i; u64 val; void __iomem *spbu_base = misc_platform_get_spbu_base(0); struct cpufreq_frequency_table *ft = policy->freq_table; - clu_lv1_sel = (readq(spbu_base + OFFSET_CLU_LV1_SEL) >> 2) & 0x3; - - if (clu_lv1_sel == 0) - val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL0_CFG_SHIFT; - else if (clu_lv1_sel == 2) - val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL1_CFG_SHIFT; - else - val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL2_CFG_SHIFT; - + /* PLL2 provides working frequency for core */ + val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL2_CFG_SHIFT; val &= CORE_PLL2_CFG_MASK; for (i = 0; ft[i].frequency != CPUFREQ_TABLE_END; i++) { - if (val == i) + if (val == i) { + if (ft[i].frequency == CPUFREQ_ENTRY_INVALID) + return cpuid(GET_CPU_FREQ, 0) * 1000UL; return ft[i].frequency; + } } + return 0; } @@ -252,6 +249,7 @@ static struct freq_attr *sunway_table_attr[] = { static struct cpufreq_driver sunway_cpufreq_driver = { .name = "sunway-cpufreq", + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, .init = sunway_cpufreq_init, .verify = sunway_cpufreq_verify, .target_index = sunway_cpufreq_target, -- Gitee From 6e59233f6f12ff5872957d81cead74c68ea59c03 Mon Sep 17 00:00:00 2001 From: Changwei Miao Date: Sat, 9 Sep 2023 08:50:30 +0800 Subject: [PATCH 490/524] sw64: mm: adjust userspace memory layout Historically, sw64 always put the stack below 0x120000000 which confilicts with arch_get_unmapped_area_topdown, and arch_get_unmapped_area usually allocate VA in the range of 47 bits. This patch adjusts the stack layout to grow downward from the high-end of user space, and makes some related changes as follow: * Enable CONFIG_SW64_VA_BITS_47=y as default, which keeps VAs size compatibility with old software. See arch/sw_64/Kconfig for details. * Ignore ADDR_LIMIT_32BIT. Unlike OSF/Alpha, there is no eflag the same as EF_ALPHA_32BIT. We may implement 32bit addr limit with TIF_32BIT like others in the future. Signed-off-by: Changwei Miao Signed-off-by: Wang Yicheng Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 2 + arch/sw_64/include/asm/elf.h | 6 +- arch/sw_64/include/asm/page.h | 3 + arch/sw_64/include/asm/pgtable.h | 2 +- arch/sw_64/include/asm/processor.h | 49 +++++++++--- arch/sw_64/mm/Kconfig | 41 ++++++++++ arch/sw_64/mm/hugetlbpage.c | 12 +-- arch/sw_64/mm/mmap.c | 119 ++++++++++++++++++++++++----- 8 files changed, 197 insertions(+), 37 deletions(-) create mode 100644 arch/sw_64/mm/Kconfig diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index e61c5245dc90..a45ee01644f8 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -685,3 +685,5 @@ source "drivers/idle/Kconfig" endmenu source "arch/sw_64/kvm/Kconfig" + +source "arch/sw_64/mm/Kconfig" diff --git a/arch/sw_64/include/asm/elf.h b/arch/sw_64/include/asm/elf.h index f94d69a5b110..7aec6327901f 100644 --- a/arch/sw_64/include/asm/elf.h +++ b/arch/sw_64/include/asm/elf.h @@ -91,7 +91,11 @@ typedef struct user_fpsimd_state elf_fpregset_t; * that it will "exec", and that there is sufficient room for the brk. */ -#define ELF_ET_DYN_BASE (TASK_UNMAPPED_BASE + 0x1000000) +#ifdef CONFIG_SW64_FORCE_52BIT +#define ELF_ET_DYN_BASE (2 * TASK_SIZE / 3) +#else +#define ELF_ET_DYN_BASE (2 * DEFAULT_MAP_WINDOW / 3) +#endif /* * $0 is set by ld.so to a pointer to a function which might be diff --git a/arch/sw_64/include/asm/page.h b/arch/sw_64/include/asm/page.h index 1ba314b342a0..89bf4ba56f22 100644 --- a/arch/sw_64/include/asm/page.h +++ b/arch/sw_64/include/asm/page.h @@ -67,6 +67,9 @@ extern unsigned long __boot_phys_addr(unsigned long addr); #define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_NON_EXEC #include #include + +#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA + #endif #endif /* _ASM_SW64_PAGE_H */ diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 57c2639aeaf8..0ad6f12c3275 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -830,7 +830,7 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) pr_err("%s: %d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) extern void paging_init(void); -/* We have our own get_unmapped_area to cope with ADDR_LIMIT_32BIT. */ #define HAVE_ARCH_UNMAPPED_AREA +#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN #endif /* _ASM_SW64_PGTABLE_H */ diff --git a/arch/sw_64/include/asm/processor.h b/arch/sw_64/include/asm/processor.h index f66a77f233c9..4360140e9535 100644 --- a/arch/sw_64/include/asm/processor.h +++ b/arch/sw_64/include/asm/processor.h @@ -8,7 +8,6 @@ #ifndef _ASM_SW64_PROCESSOR_H #define _ASM_SW64_PROCESSOR_H -#include /* for ADDR_LIMIT_32BIT */ #include #define task_pt_regs(task) \ @@ -27,19 +26,47 @@ /* * We have a 52-bit user address space: 4PB user VM... + * 20230728(mcw): + * To make sure that arch_get_unmapped_area_topdown and old + * software, e.g. golang runtime and v8 jit, works well at + * the same time, just providing 47-bit VAs unless a hint is + * supplied to mmap. */ -#define TASK_SIZE (0x10000000000000UL) -#define UNMAPPED_BASE (TASK_SIZE >> 6) -#define STACK_TOP \ - (current->personality & ADDR_LIMIT_32BIT ? 0x80000000 : 0x00120000000UL) -#define STACK_TOP_MAX 0x00120000000UL +#define VA_BITS (CONFIG_SW64_VA_BITS) +#if VA_BITS > 47 +#define VA_BITS_MIN (47) +#else +#define VA_BITS_MIN (VA_BITS) +#endif -/* This decides where the kernel will search for a free chunk of vm - * space during mmap's. - */ -#define TASK_UNMAPPED_BASE \ - ((current->personality & ADDR_LIMIT_32BIT) ? 0x40000000 : UNMAPPED_BASE) +#define DEFAULT_MAP_WINDOW_64 (1UL << VA_BITS_MIN) +#define TASK_SIZE_64 (1UL << VA_BITS) + +#define TASK_SIZE_MAX TASK_SIZE_64 +#define TASK_SIZE TASK_SIZE_64 +#define DEFAULT_MAP_WINDOW DEFAULT_MAP_WINDOW_64 + +#ifdef CONFIG_SW64_FORCE_52BIT +#define STACK_TOP_MAX TASK_SIZE +#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4)) +#else +#define STACK_TOP_MAX DEFAULT_MAP_WINDOW +#define TASK_UNMAPPED_BASE (PAGE_ALIGN(DEFAULT_MAP_WINDOW / 4)) +#endif + +#define STACK_TOP STACK_TOP_MAX + +#ifndef CONFIG_SW64_FORCE_52BIT +#define arch_get_mmap_end(addr, len, flags) \ + (((addr) > DEFAULT_MAP_WINDOW) ? TASK_SIZE : DEFAULT_MAP_WINDOW) +#define arch_get_mmap_base(addr, base) ((addr > DEFAULT_MAP_WINDOW) ? \ + base + TASK_SIZE - DEFAULT_MAP_WINDOW : \ + base) +#else +#define arch_get_mmap_end(addr, len, flags) (TASK_SIZE) +#define arch_get_mmap_base(addr, base) (base) +#endif struct thread_struct { struct user_fpsimd_state fpstate; diff --git a/arch/sw_64/mm/Kconfig b/arch/sw_64/mm/Kconfig new file mode 100644 index 000000000000..4f5f7f4599d7 --- /dev/null +++ b/arch/sw_64/mm/Kconfig @@ -0,0 +1,41 @@ +choice + prompt "Virtual address space size" + default SW64_VA_BITS_47 + help + Allows choosing one of multiple possible userspace virtual address space size. + +config SW64_VA_BITS_47 + bool "47-bit" + help + Limit the userspace VAs to 47-bit size. + + It may waste some userspace VA range, but is safe and enough for current + userland to works wells. + +config SW64_VA_BITS_52 + bool "52-bit" + help + Use the full 52-bit size userspace VAs. + + The kernel will attempt to maintain compatibility with older software by + providing 47-bit VAs unless a hint is supplied to mmap. + +endchoice + +config SW64_VA_BITS + int + default 47 if SW64_VA_BITS_47 + default 52 if SW64_VA_BITS_52 + default 47 + help + The size of userspace VA. + +config SW64_FORCE_52BIT + bool "Force 52-bit virtual address for userspace" + depends on SW64_VA_BITS_52 && EXPERT + help + This configuration option disable the 47-bit compatibility logic, and forces + all userspace address to be 52-bit if possible. + + Note, it is not 100% safe for software that unable to handle VA bigger than + 47-bit to works well with this option enabled. Do not enable it for now. diff --git a/arch/sw_64/mm/hugetlbpage.c b/arch/sw_64/mm/hugetlbpage.c index f986422398ef..d723a506e9b3 100644 --- a/arch/sw_64/mm/hugetlbpage.c +++ b/arch/sw_64/mm/hugetlbpage.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -217,25 +218,24 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, info.flags = 0; info.length = len; - info.low_limit = current->mm->mmap_legacy_base; - info.high_limit = TASK_SIZE; + info.low_limit = current->mm->mmap_base; + info.high_limit = arch_get_mmap_end(addr, len, flags); info.align_mask = PAGE_MASK & ~huge_page_mask(h); info.align_offset = 0; return vm_unmapped_area(&info); } static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, - unsigned long addr0, unsigned long len, + unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct hstate *h = hstate_file(file); struct vm_unmapped_area_info info; - unsigned long addr; info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; info.low_limit = PAGE_SIZE; - info.high_limit = current->mm->mmap_base; + info.high_limit = arch_get_mmap_base(addr, current->mm->mmap_base); info.align_mask = PAGE_MASK & ~huge_page_mask(h); info.align_offset = 0; addr = vm_unmapped_area(&info); @@ -250,7 +250,7 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, VM_BUG_ON(addr != -ENOMEM); info.flags = 0; info.low_limit = TASK_UNMAPPED_BASE; - info.high_limit = TASK_SIZE; + info.high_limit = arch_get_mmap_end(addr, len, flags); addr = vm_unmapped_area(&info); } diff --git a/arch/sw_64/mm/mmap.c b/arch/sw_64/mm/mmap.c index a7a189fc36d6..d4bf9a7d2627 100644 --- a/arch/sw_64/mm/mmap.c +++ b/arch/sw_64/mm/mmap.c @@ -5,10 +5,20 @@ #include #include #include +#include #include +#include #include +/* + * Top of mmap area (just below the process stack). + * Leave at least a ~128 MB hole. + */ + +#define MIN_GAP (SZ_128M) +#define MAX_GAP (STACK_TOP / 6 * 5) + unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, @@ -17,15 +27,9 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, struct mm_struct *mm = current->mm; struct vm_area_struct *vma; struct vm_unmapped_area_info info; - unsigned long limit; + const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); - /* Support 32 bit heap. */ - if (current->personality & ADDR_LIMIT_32BIT) - limit = 0x80000000; - else - limit = TASK_SIZE; - - if (len > limit) + if (unlikely(len > mmap_end - mmap_min_addr)) return -ENOMEM; if (flags & MAP_FIXED) { @@ -47,26 +51,100 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, info.flags = 0; info.length = len; info.low_limit = mm->mmap_base; - info.high_limit = limit; + info.high_limit = mmap_end; info.align_mask = 0; info.align_offset = pgoff << PAGE_SHIFT; return vm_unmapped_area(&info); } -unsigned long arch_mmap_rnd(void) +unsigned long +arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) { - unsigned long rnd; + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + struct vm_unmapped_area_info info; + const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); + + if (unlikely(len > mmap_end - mmap_min_addr)) + return -ENOMEM; - /* 8MB for 32bit, 256MB for 64bit */ - if (current->personality & ADDR_LIMIT_32BIT) - rnd = get_random_long() & 0x7ffffful; - else - rnd = get_random_long() & 0xffffffful; + if (flags & MAP_FIXED) { + if (addr + len > TASK_SIZE) + return -EINVAL; + + return addr; + } + + if (addr) { + addr = PAGE_ALIGN(addr); + + vma = find_vma(mm, addr); + if (TASK_SIZE - len >= addr && + (!vma || addr + len <= vm_start_gap(vma))) + return addr; + } + + info.flags = VM_UNMAPPED_AREA_TOPDOWN; + info.length = len; + info.low_limit = FIRST_USER_ADDRESS; + info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); + info.align_mask = 0; + info.align_offset = pgoff << PAGE_SHIFT; + addr = vm_unmapped_area(&info); + + /* + * A failed mmap() very likely causes application failure, + * so fall back to the bottom-up function here. This scenario + * can happen with large stack limits and large mmap() + * allocations. + */ + if (addr & ~PAGE_MASK) { + VM_BUG_ON(addr != -ENOMEM); + info.flags = 0; + info.low_limit = mm->mmap_base; + info.high_limit = mmap_end; + addr = vm_unmapped_area(&info); + } + return addr; +} + +unsigned long arch_mmap_rnd(void) +{ + unsigned long rnd = get_random_long() & 0x7fffffful; return rnd << PAGE_SHIFT; } +unsigned long mmap_is_legacy(struct rlimit *rlim_stack) +{ + if (current->personality & ADDR_COMPAT_LAYOUT) + return 1; + if (rlim_stack->rlim_cur == RLIM_INFINITY) + return 1; + + return sysctl_legacy_va_layout; +} + +static unsigned long mmap_base(unsigned long rnd, struct rlimit *rlim_stack) +{ + unsigned long gap = rlim_stack->rlim_cur; + unsigned long pad = stack_guard_gap; + + /* Account for stack randomization if necessary. 8M of VA. */ + if (current->flags & PF_RANDOMIZE) + pad += 0x7ff00; + /* Values close to RLIM_INFINITY can overflow. */ + if (gap + pad > gap) + gap = MIN_GAP; + else if (gap > MAX_GAP) + gap = MAX_GAP; + + return PAGE_ALIGN(STACK_TOP - gap - rnd); +} + /* * This function, called very early during the creation of a new process VM * image, sets up which VM layout function to use: @@ -82,8 +160,13 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) * Fall back to the standard layout if the personality bit is set, or * if the expected stack growth is unlimited: */ - mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; - mm->get_unmapped_area = arch_get_unmapped_area; + if (mmap_is_legacy(rlim_stack)) { + mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; + mm->get_unmapped_area = arch_get_unmapped_area; + } else { + mm->mmap_base = mmap_base(random_factor, rlim_stack); + mm->get_unmapped_area = arch_get_unmapped_area_topdown; + } } SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, -- Gitee From cf11ca5450eacaad138f99df70e14dd0f2f62fac Mon Sep 17 00:00:00 2001 From: xiedongliang Date: Wed, 21 May 2025 16:14:04 +0800 Subject: [PATCH 491/524] sw64: ftrace: Enable HAVE_FUNCTION_GRAPH_RETVAL The previous patch ("function_graph: Support recording and printing the return value of function") has laid the groundwork for the for the funcgraph-retval, and this modification makes it available on the sw64 platform. We introduce a new structure called fgraph_ret_regs for the sw64 platform to hold return registers and the frame pointer. We then fill its content in the return_to_handler and pass its address to the function ftrace_return_to_handler to record the return value. Signed-off-by: xiedongliang Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/include/asm/ftrace.h | 20 ++++++++++++++++++++ arch/sw_64/kernel/entry-ftrace.S | 12 +++++++----- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index a45ee01644f8..89973bbf8b71 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -95,6 +95,7 @@ config SW64 select HAVE_EBPF_JIT select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER select HAVE_GENERIC_VDSO if MMU && 64BIT diff --git a/arch/sw_64/include/asm/ftrace.h b/arch/sw_64/include/asm/ftrace.h index 871c0b7b0aef..c5ebb6e336b8 100644 --- a/arch/sw_64/include/asm/ftrace.h +++ b/arch/sw_64/include/asm/ftrace.h @@ -47,4 +47,24 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) } #endif /* ifndef __ASSEMBLY__ */ + +#ifndef __ASSEMBLY__ +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +struct fgraph_ret_regs { + unsigned long ax; + unsigned long bp; +}; + +static inline unsigned long fgraph_ret_regs_return_value(struct fgraph_ret_regs *ret_regs) +{ + return ret_regs->ax; +} + +static inline unsigned long fgraph_ret_regs_frame_pointer(struct fgraph_ret_regs *ret_regs) +{ + return ret_regs->bp; +} +#endif /* ifdef CONFIG_FUNCTION_GRAPH_TRACER */ +#endif + #endif /* _ASM_SW64_FTRACE_H */ diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S index 426d7561f097..2631610daee1 100644 --- a/arch/sw_64/kernel/entry-ftrace.S +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -173,14 +173,16 @@ /* save return value regs*/ .macro save_return_regs - subl $sp, 0x8, $sp - stl $0, 0x0($sp) + subl $sp, 16, $sp + stl $0, 0($sp) + stl $15, 8($sp) .endm /* restore return value regs*/ .macro restore_return_regs - ldl $0, 0x0($sp) - addl $sp, 0x8, $sp + ldl $15, 8($sp) + ldl $0, 0($sp) + addl $sp, 16, $sp .endm @@ -231,7 +233,7 @@ ENTRY(return_to_handler) br $27, 1f 1: ldgp $29, 0($27) save_return_regs - bis $31, $15, $16 /* parent's fp */ + bis $31, $sp, $16 /* ret_regs */ ldi $27, ftrace_return_to_handler call $26, ($27) bis $31, $0, $26 -- Gitee From 0688da1ddaac8c55734161818074eede2e782c34 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Thu, 8 May 2025 16:46:14 +0800 Subject: [PATCH 492/524] sw64: emit fixed-length instructions for BPF_PSEUDO_FUNC For BPF_PSEUDO_FUNC instruction, verifier will refill imm with correct addresses of bpf_calls and then run last pass of JIT. Since the emit_sw64_ldu64 is variable-length, which will emit appropriate length instructions accorroding to the imm, it may broke ctx->offset, and lead to unpredictable problem, such as inaccurate jump. So it is fixed with fixed-length instructions. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/net/bpf_jit_comp.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index ed78ae0a23c9..5a6b461c3832 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -227,10 +227,9 @@ static void emit_sw64_ldu64(const int dst, const u64 imm, struct jit_ctx *ctx) } /* constant insn count */ -static void emit_sw64_load_call_addr(u64 addr, struct jit_ctx *ctx) +static void emit_sw64_load_call_addr(u8 dst, u64 addr, struct jit_ctx *ctx) { u16 imm_tmp; - u8 dst = SW64_BPF_REG_PV; u8 reg_tmp = get_tmp_reg(ctx); imm_tmp = (addr >> 60) & 0xf; @@ -1192,7 +1191,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, if (ret < 0) return ret; - emit_sw64_load_call_addr(func, ctx); + emit_sw64_load_call_addr(SW64_BPF_REG_PV, func, ctx); emit(SW64_BPF_CALL(SW64_BPF_REG_RA, SW64_BPF_REG_PV), ctx); break; } @@ -1224,7 +1223,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, case BPF_LD | BPF_IMM | BPF_DW: insn1 = insn[1]; imm64 = ((u64)insn1.imm << 32) | (u32)imm; - emit_sw64_ldu64(dst, imm64, ctx); + if (bpf_pseudo_func(insn)) + emit_sw64_load_call_addr(SW64_BPF_REG_A1, imm64, ctx); + else + emit_sw64_ldu64(dst, imm64, ctx); put_tmp_reg(ctx); put_tmp_reg(ctx); return 1; -- Gitee From 3fb62d6287a39da5c69e9f0101e459945dca6583 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Mon, 28 Apr 2025 17:17:39 +0800 Subject: [PATCH 493/524] sw64: add sw64 rethook implementation Implement the kretprobes on sw64 arch by using rethook machenism which abstracts general kretprobe info into a struct rethook_node to be embedded in the struct kretprobe_instance. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/kernel/kprobes/Makefile | 3 + arch/sw_64/kernel/kprobes/kprobes.c | 59 +--------- arch/sw_64/kernel/kprobes/rethook.c | 22 ++++ arch/sw_64/kernel/kprobes/rethook.h | 8 ++ .../sw_64/kernel/kprobes/rethook_trampoline.S | 106 ++++++++++++++++++ 6 files changed, 141 insertions(+), 58 deletions(-) create mode 100644 arch/sw_64/kernel/kprobes/rethook.c create mode 100644 arch/sw_64/kernel/kprobes/rethook.h create mode 100644 arch/sw_64/kernel/kprobes/rethook_trampoline.S diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 89973bbf8b71..4fddce1e2936 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -116,6 +116,7 @@ config SW64 select HAVE_PERF_USER_STACK_DUMP select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if STACKTRACE + select HAVE_RETHOOK select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN diff --git a/arch/sw_64/kernel/kprobes/Makefile b/arch/sw_64/kernel/kprobes/Makefile index 110ba2bf7752..f07b3513c3a6 100644 --- a/arch/sw_64/kernel/kprobes/Makefile +++ b/arch/sw_64/kernel/kprobes/Makefile @@ -1,3 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_KPROBES) += kprobes.o decode-insn.o obj-$(CONFIG_KPROBES_ON_FTRACE) += kprobes-ftrace.o +obj-$(CONFIG_RETHOOK) += rethook.o rethook_trampoline.o +CFLAGS_REMOVE_rethook.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_rethook_trampoline.o = $(CC_FLAGS_FTRACE) diff --git a/arch/sw_64/kernel/kprobes/kprobes.c b/arch/sw_64/kernel/kprobes/kprobes.c index 2ee18260f2a3..662f65fb3381 100644 --- a/arch/sw_64/kernel/kprobes/kprobes.c +++ b/arch/sw_64/kernel/kprobes/kprobes.c @@ -240,35 +240,6 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, } return ret; } -/* - * Function return probe trampoline: - * - init_kprobes() establishes a probepoint here - * - When the probed function returns, this probe causes the - * handlers to fire - */ -static void __used kretprobe_trampoline_holder(void) -{ - asm volatile( - /* Keep the assembler from reordering and placing JR here. */ - ".set noreorder\n\t" - "nop\n\t" - ".global __kretprobe_trampoline\n" - "__kretprobe_trampoline:\n\t" - "nop\n\t" - : : : "memory"); -} - -void __kretprobe_trampoline(void); - -void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, - struct pt_regs *regs) -{ - ri->ret_addr = (kprobe_opcode_t *) regs->regs[26]; - ri->fp = NULL; - - /* Replace the return addr with trampoline addr */ - regs->regs[26] = (unsigned long)__kretprobe_trampoline; -} /* * Provide a blacklist of symbols identifying ranges which cannot be kprobed. @@ -283,40 +254,12 @@ int __init arch_populate_kprobe_blacklist(void) return ret; } -/* - * Called when the probe at kretprobe trampoline is hit - */ -static int __kprobes trampoline_probe_handler(struct kprobe *p, - struct pt_regs *regs) -{ - unsigned long orig_ret_address; - - orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); - instruction_pointer(regs) = orig_ret_address; - regs->regs[26] = orig_ret_address; - - /* - * By returning a non-zero value, we are telling - * kprobe_handler() that we don't want the post_handler - * to run (and have re-enabled preemption) - */ - return 1; -} - int __kprobes arch_trampoline_kprobe(struct kprobe *p) { - if (p->addr == (kprobe_opcode_t *)__kretprobe_trampoline) - return 1; - return 0; } -static struct kprobe trampoline_p = { - .addr = (kprobe_opcode_t *)__kretprobe_trampoline, - .pre_handler = trampoline_probe_handler -}; - int __init arch_init_kprobes(void) { - return register_kprobe(&trampoline_p); + return 0; } diff --git a/arch/sw_64/kernel/kprobes/rethook.c b/arch/sw_64/kernel/kprobes/rethook.c new file mode 100644 index 000000000000..e8895fac9403 --- /dev/null +++ b/arch/sw_64/kernel/kprobes/rethook.c @@ -0,0 +1,22 @@ +#include +#include +#include "rethook.h" + +/* This is called from arch_rethook_trampoline() */ +unsigned long __used arch_rethook_trampoline_callback(struct pt_regs *regs) +{ + return rethook_trampoline_handler(regs, regs->regs[15]); +} + +NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); + +void arch_rethook_prepare(struct rethook_node *rhn, struct pt_regs *regs, bool mcount) +{ + rhn->ret_addr = regs->regs[26]; + rhn->frame = regs->regs[15]; + + /* replace return addr with trampoline */ + regs->regs[26] = (unsigned long)arch_rethook_trampoline; +} + +NOKPROBE_SYMBOL(arch_rethook_prepare); diff --git a/arch/sw_64/kernel/kprobes/rethook.h b/arch/sw_64/kernel/kprobes/rethook.h new file mode 100644 index 000000000000..b4e2e5fe81c2 --- /dev/null +++ b/arch/sw_64/kernel/kprobes/rethook.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __SW64_RETHOOK_H +#define __SW64_RETHOOK_H + +unsigned long arch_rethook_trampoline_callback(struct pt_regs *regs); +void arch_rethook_prepare(struct rethook_node *rhn, struct pt_regs *regs, bool mcount); + +#endif diff --git a/arch/sw_64/kernel/kprobes/rethook_trampoline.S b/arch/sw_64/kernel/kprobes/rethook_trampoline.S new file mode 100644 index 000000000000..9b5ddf8e2ba2 --- /dev/null +++ b/arch/sw_64/kernel/kprobes/rethook_trampoline.S @@ -0,0 +1,106 @@ +#include +#include +#include + + .text + .set noat + .align 4 + + .macro save_all_base_regs + /* save $26 & fp of the function before caller */ + subl $sp, 0x10, $sp + stl $26, 0($sp) + stl $15, 0x8($sp) + ldi $15, 0($sp) + + /* save $28 & fp of caller */ + subl $sp, 0x10, $sp + stl $28, 0($sp) + stl $15, 0x8($sp) + ldi $15, 0($sp) + + /* save pt_regs */ + ldi $sp, -PT_REGS_SIZE($sp) + stl $0, PT_REGS_R0($sp) + stl $1, PT_REGS_R1($sp) + stl $2, PT_REGS_R2($sp) + stl $3, PT_REGS_R3($sp) + stl $4, PT_REGS_R4($sp) + stl $5, PT_REGS_R5($sp) + stl $6, PT_REGS_R6($sp) + stl $7, PT_REGS_R7($sp) + stl $8, PT_REGS_R8($sp) + stl $9, PT_REGS_R9($sp) + stl $10, PT_REGS_R10($sp) + stl $11, PT_REGS_R11($sp) + stl $12, PT_REGS_R12($sp) + stl $13, PT_REGS_R13($sp) + stl $14, PT_REGS_R14($sp) + stl $15, PT_REGS_R15($sp) + stl $16, PT_REGS_R16($sp) + stl $17, PT_REGS_R17($sp) + stl $18, PT_REGS_R18($sp) + stl $19, PT_REGS_R19($sp) + stl $20, PT_REGS_R20($sp) + stl $21, PT_REGS_R21($sp) + stl $22, PT_REGS_R22($sp) + stl $23, PT_REGS_R23($sp) + stl $24, PT_REGS_R24($sp) + stl $25, PT_REGS_R25($sp) + stl $26, PT_REGS_R26($sp) + stl $27, PT_REGS_R27($sp) + stl $28, PT_REGS_R28($sp) + stl $29, PT_REGS_GP($sp) + ldi $0, PT_REGS_SIZE($sp) + stl $0, PT_REGS_R30($sp) + .endm + + .macro restore_all_base_regs + /* restore pt_regs */ + ldl $0, PT_REGS_R0($sp) + ldl $1, PT_REGS_R1($sp) + ldl $2, PT_REGS_R2($sp) + ldl $3, PT_REGS_R3($sp) + ldl $4, PT_REGS_R4($sp) + ldl $5, PT_REGS_R5($sp) + ldl $6, PT_REGS_R6($sp) + ldl $7, PT_REGS_R7($sp) + ldl $8, PT_REGS_R8($sp) + ldl $9, PT_REGS_R9($sp) + ldl $10, PT_REGS_R10($sp) + ldl $11, PT_REGS_R11($sp) + ldl $12, PT_REGS_R12($sp) + ldl $13, PT_REGS_R13($sp) + ldl $14, PT_REGS_R14($sp) + ldl $15, PT_REGS_R15($sp) + ldl $16, PT_REGS_R16($sp) + ldl $17, PT_REGS_R17($sp) + ldl $18, PT_REGS_R18($sp) + ldl $19, PT_REGS_R19($sp) + ldl $20, PT_REGS_R20($sp) + ldl $21, PT_REGS_R21($sp) + ldl $22, PT_REGS_R22($sp) + ldl $23, PT_REGS_R23($sp) + ldl $24, PT_REGS_R24($sp) + ldl $25, PT_REGS_R25($sp) + ldl $27, PT_REGS_R27($sp) + ldl $28, PT_REGS_R28($sp) + ldl $29, PT_REGS_GP($sp) + ldi $sp, PT_REGS_SIZE($sp) + + /* only restore $fp */ + ldl $15, 0x18($sp) + addl $sp, 0x20, $sp + .endm + +ENTRY(arch_rethook_trampoline) + save_all_base_regs + + mov $sp, $16 + call arch_rethook_trampoline_callback + mov $0, $26 + + restore_all_base_regs + + ret +ENDPROC(arch_rethook_trampoline) -- Gitee From e5b0d776cc43a10a0fdbc521ba374303763f3f34 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Fri, 23 May 2025 19:33:05 +0800 Subject: [PATCH 494/524] sw64: add support for function error injection Inspired by the commit 42d038c4fb00 ("arm64: Add support for function error injection"), this patch supports function error injection for sw64. This patch mainly support two functions: one is regs_set_return_value() which is used to overwrite the return value; the another function is override_function_with_return() which is to override the probed function returning and jump to its caller. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/include/asm/ptrace.h | 5 +++++ arch/sw_64/lib/Makefile | 2 ++ arch/sw_64/lib/error-inject.c | 10 ++++++++++ 4 files changed, 18 insertions(+) create mode 100644 arch/sw_64/lib/error-inject.c diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 4fddce1e2936..950ea7826950 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -95,6 +95,7 @@ config SW64 select HAVE_EBPF_JIT select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_ERROR_INJECTION select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h index 0bbe85297f62..3e5730c8b6a2 100644 --- a/arch/sw_64/include/asm/ptrace.h +++ b/arch/sw_64/include/asm/ptrace.h @@ -87,6 +87,11 @@ static inline long regs_return_value(struct pt_regs *regs) return -regs->regs[0]; } +static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc) +{ + regs->regs[0] = rc; +} + #endif /* !__ASSEMBLY__ */ #endif /* __KERNEL__ */ diff --git a/arch/sw_64/lib/Makefile b/arch/sw_64/lib/Makefile index aa6d8eee29c2..c2927c1e98dd 100644 --- a/arch/sw_64/lib/Makefile +++ b/arch/sw_64/lib/Makefile @@ -40,6 +40,8 @@ lib-y += $(lib-clear_page-y) $(lib-clear_user-y) $(lib-copy_page-y) $(lib-copy_u obj-y = iomap.o obj-y += iomap_copy.o +obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o + ifeq ($(CONFIG_SUBARCH_C3B),y) # The division routines are built from single source, with different defines. AFLAGS___divlu.o = -DDIV diff --git a/arch/sw_64/lib/error-inject.c b/arch/sw_64/lib/error-inject.c new file mode 100644 index 000000000000..b1549c4529e5 --- /dev/null +++ b/arch/sw_64/lib/error-inject.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +void override_function_with_return(struct pt_regs *regs) +{ + instruction_pointer_set(regs, regs->regs[26]); +} +NOKPROBE_SYMBOL(override_function_with_return); -- Gitee From 2aa9b19ca597e5a6c948dce6b8aa0fa7b3a6cbd4 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Mon, 26 May 2025 21:35:15 +0800 Subject: [PATCH 495/524] sw64: ftrace: No need to save original function's temporary registers During context saving in entry-ftrace.S, there is no need to save original function's temporary registers. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry-ftrace.S | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S index 2631610daee1..5edf09209561 100644 --- a/arch/sw_64/kernel/entry-ftrace.S +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -91,14 +91,6 @@ /* save pt_regs */ ldi $sp, -PT_REGS_SIZE($sp) stl $0, PT_REGS_R0($sp) - stl $1, PT_REGS_R1($sp) - stl $2, PT_REGS_R2($sp) - stl $3, PT_REGS_R3($sp) - stl $4, PT_REGS_R4($sp) - stl $5, PT_REGS_R5($sp) - stl $6, PT_REGS_R6($sp) - stl $7, PT_REGS_R7($sp) - stl $8, PT_REGS_R8($sp) stl $9, PT_REGS_R9($sp) stl $10, PT_REGS_R10($sp) stl $11, PT_REGS_R11($sp) @@ -127,14 +119,6 @@ .macro RESTORE_PT_REGS /* restore pt_regs */ ldl $0, PT_REGS_R0($sp) - ldl $1, PT_REGS_R1($sp) - ldl $2, PT_REGS_R2($sp) - ldl $3, PT_REGS_R3($sp) - ldl $4, PT_REGS_R4($sp) - ldl $5, PT_REGS_R5($sp) - ldl $6, PT_REGS_R6($sp) - ldl $7, PT_REGS_R7($sp) - ldl $8, PT_REGS_R8($sp) ldl $9, PT_REGS_R9($sp) ldl $10, PT_REGS_R10($sp) ldl $11, PT_REGS_R11($sp) -- Gitee From 8bc40b1b4d9aec8a967f594f22564fa35a425025 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Wed, 28 May 2025 14:59:49 +0800 Subject: [PATCH 496/524] sw64: ftrace: Add direct call support Select the HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS to provide the register_ftrace_direct[_multi] interfaces allowing users to register the customed trampoline (direct_caller) as the mcount for one or more target functions. And modify_ftrace_direct[_multi] are also provided for modifying direct_caller. There are a few cases to distinguish: - If a direct call ops is the only one tracing a function -> the ftrace patchsite jumps to the trampoline - Else -> the ftrace patchsite jumps to the ftrace_regs_caller trampoline points to ftrace_list_ops so it iterates over all registered ftrace ops, including the direct call ops and calls its call_direct_funcs handler which stores the direct called trampoline's address in the ftrace_regs and the ftrace_regs_caller trampoline will return to that address instead of returning to the traced function Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/include/asm/ftrace.h | 11 +++++++++++ arch/sw_64/kernel/entry-ftrace.S | 9 ++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 950ea7826950..55fa28b564cd 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -91,6 +91,7 @@ config SW64 select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_BUGVERBOSE select HAVE_DYNAMIC_FTRACE + select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS select HAVE_DYNAMIC_FTRACE_WITH_REGS select HAVE_EBPF_JIT select HAVE_FAST_GUP diff --git a/arch/sw_64/include/asm/ftrace.h b/arch/sw_64/include/asm/ftrace.h index c5ebb6e336b8..33122cd7f91b 100644 --- a/arch/sw_64/include/asm/ftrace.h +++ b/arch/sw_64/include/asm/ftrace.h @@ -46,6 +46,17 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) return addr; } +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +static inline void +__arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr) +{ + regs->regs[2] = addr; +} + +#define arch_ftrace_set_direct_caller(fregs, addr) \ + __arch_ftrace_set_direct_caller(&(fregs)->regs, addr) +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ + #endif /* ifndef __ASSEMBLY__ */ #ifndef __ASSEMBLY__ diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S index 5edf09209561..5dbae4991875 100644 --- a/arch/sw_64/kernel/entry-ftrace.S +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -114,6 +114,10 @@ stl $29, PT_REGS_GP($sp) ldi $0, PT_REGS_SIZE($sp) stl $0, PT_REGS_R30($sp) + + /* save direct caller reg */ + ldi $2, 0($31) + stl $2, PT_REGS_R2($sp) .endm .macro RESTORE_PT_REGS @@ -338,7 +342,10 @@ ftrace_regs_call: call ftrace_graph_caller #endif RESTORE_PT_REGS - ret $31, ($28), 1 + bne $2, .Ldirect + ret $31, ($28), 1 +.Ldirect: + ret $31, ($2), 1 .end ftrace_regs_caller #endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ -- Gitee From e863ec8b3695074e468ae4ae7c521433c6cfbb4c Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Fri, 23 May 2025 18:24:49 +0800 Subject: [PATCH 497/524] sw64: ftrace: Add direct call trampoline samples support The ftrace samples need per-architecture trampoline implementations to save and restore argument registers around the calls to my_direct_func* and to restore polluted registers (eg: $26). Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 2 + samples/ftrace/ftrace-direct-modify.c | 68 +++++++++++++++++++ samples/ftrace/ftrace-direct-multi-modify.c | 74 +++++++++++++++++++++ samples/ftrace/ftrace-direct-multi.c | 41 ++++++++++++ samples/ftrace/ftrace-direct-too.c | 46 +++++++++++++ samples/ftrace/ftrace-direct.c | 40 +++++++++++ 6 files changed, 271 insertions(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 55fa28b564cd..071c1d2ae37e 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -120,6 +120,8 @@ config SW64 select HAVE_RELIABLE_STACKTRACE if STACKTRACE select HAVE_RETHOOK select HAVE_RSEQ + select HAVE_SAMPLE_FTRACE_DIRECT + select HAVE_SAMPLE_FTRACE_DIRECT_MULTI select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select IRQ_FORCED_THREADING diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c index e2a6a69352df..23ea4ac06582 100644 --- a/samples/ftrace/ftrace-direct-modify.c +++ b/samples/ftrace/ftrace-direct-modify.c @@ -164,6 +164,74 @@ asm ( #endif /* CONFIG_LOONGARCH */ +#ifdef CONFIG_SW64 + +asm ( +" .pushsection .text, \"ax\", @progbits\n" +" .type my_tramp1, @function\n" +" .globl my_tramp1\n" +" my_tramp1:\n" +" subl $sp, 0x10, $sp\n" +" stl $26, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save $28 & fp of caller */ +" subl $sp, 0x10, $sp\n" +" stl $28, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save other regs */ +" subl $sp, 0x18, $sp\n" +" stl $26, 0($sp)\n" +" stl $28, 0x8($sp)\n" +" stl $29, 0x10($sp)\n" +" br $27, 1f\n" +"1: ldgp $29, 0($27)\n" +" call my_direct_func1\n" + /* restore all regs */ +" ldl $26, 0x0($sp)\n" +" ldl $28, 0x8($sp)\n" +" ldl $29, 0x10($sp)\n" +" addl $sp, 0x18, $sp\n" +" ldl $15, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ret $31, ($28), 1\n" +" .size my_tramp1, .-my_tramp1\n" + +" .type my_tramp2, @function\n" +" .globl my_tramp2\n" +" my_tramp2:\n" +" subl $sp, 0x10, $sp\n" +" stl $26, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save $28 & fp of caller */ +" subl $sp, 0x10, $sp\n" +" stl $28, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save other regs */ +" subl $sp, 0x18, $sp\n" +" stl $26, 0($sp)\n" +" stl $28, 0x8($sp)\n" +" stl $29, 0x10($sp)\n" +" br $27, 1f\n" +"1: ldgp $29, 0($27)\n" +" call my_direct_func2\n" + /* restore all regs */ +" ldl $26, 0x0($sp)\n" +" ldl $28, 0x8($sp)\n" +" ldl $29, 0x10($sp)\n" +" addl $sp, 0x18, $sp\n" +" ldl $15, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ret $31, ($28), 1\n" +" .size my_tramp2, .-my_tramp2\n" +" .popsection\n" +); + +#endif /* CONFIG_SW64 */ + static struct ftrace_ops direct; static unsigned long my_tramp = (unsigned long)my_tramp1; diff --git a/samples/ftrace/ftrace-direct-multi-modify.c b/samples/ftrace/ftrace-direct-multi-modify.c index 2e349834d63c..455da065d89f 100644 --- a/samples/ftrace/ftrace-direct-multi-modify.c +++ b/samples/ftrace/ftrace-direct-multi-modify.c @@ -184,6 +184,80 @@ asm ( #endif /* CONFIG_LOONGARCH */ +#ifdef CONFIG_SW64 + +asm ( +" .pushsection .text, \"ax\", @progbits\n" +" .type my_tramp1, @function\n" +" .globl my_tramp1\n" +" my_tramp1:\n" +" subl $sp, 0x10, $sp\n" +" stl $26, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save $28 & fp of caller */ +" subl $sp, 0x10, $sp\n" +" stl $28, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save other regs */ +" subl $sp, 0x20, $sp\n" +" stl $16, 0($sp)\n" +" stl $26, 0x8($sp)\n" +" stl $28, 0x10($sp)\n" +" stl $29, 0x18($sp)\n" +" mov $28, $16\n" +" br $27, 1f\n" +"1: ldgp $29, 0($27)\n" +" call my_direct_func1\n" + /* restore all regs */ +" ldl $16, 0($sp)\n" +" ldl $26, 0x8($sp)\n" +" ldl $28, 0x10($sp)\n" +" ldl $29, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ldl $15, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ret $31, ($28), 1\n" +" .size my_tramp1, .-my_tramp1\n" + +" .type my_tramp2, @function\n" +" .globl my_tramp2\n" +" my_tramp2:\n" +" subl $sp, 0x10, $sp\n" +" stl $26, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save $28 & fp of caller */ +" subl $sp, 0x10, $sp\n" +" stl $28, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save other regs */ +" subl $sp, 0x20, $sp\n" +" stl $16, 0($sp)\n" +" stl $26, 0x8($sp)\n" +" stl $28, 0x10($sp)\n" +" stl $29, 0x18($sp)\n" +" mov $28, $16\n" +" br $27, 1f\n" +"1: ldgp $29, 0($2)\n" +" call my_direct_func2\n" + /* restore all regs */ +" ldl $16, 0($sp)\n" +" ldl $26, 0x8($sp)\n" +" ldl $28, 0x10($sp)\n" +" ldl $29, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ldl $15, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ret $31, ($28), 1\n" +" .size my_tramp2, .-my_tramp2\n" +" .popsection\n" +); + +#endif /* CONFIG_SW64 */ + static unsigned long my_tramp = (unsigned long)my_tramp1; static unsigned long tramps[2] = { (unsigned long)my_tramp1, diff --git a/samples/ftrace/ftrace-direct-multi.c b/samples/ftrace/ftrace-direct-multi.c index 9243dbfe4d0c..91b03bcf2f9c 100644 --- a/samples/ftrace/ftrace-direct-multi.c +++ b/samples/ftrace/ftrace-direct-multi.c @@ -116,6 +116,47 @@ asm ( #endif /* CONFIG_LOONGARCH */ +#ifdef CONFIG_SW64 + +asm ( +" .pushsection .text, \"ax\", @progbits\n" +" .type my_tramp, @function\n" +" .globl my_tramp\n" +" my_tramp:\n" +" subl $sp, 0x10, $sp\n" +" stl $26, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save $28 & fp of caller */ +" subl $sp, 0x10, $sp\n" +" stl $28, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save other regs */ +" subl $sp, 0x20, $sp\n" +" stl $16, 0($sp)\n" +" stl $26, 0x8($sp)\n" +" stl $28, 0x10($sp)\n" +" stl $29, 0x18($sp)\n" +" mov $28, $16\n" +" br $27, 1f\n" +"1: ldgp $29, 0($27)\n" +" call my_direct_func\n" + /* restore all regs */ +" ldl $16, 0($sp)\n" +" ldl $26, 0x8($sp)\n" +" ldl $28, 0x10($sp)\n" +" ldl $29, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ldl $15, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ret $31, ($28), 1\n" +" .size my_tramp, .-my_tramp\n" +" .popsection\n" +); + +#endif /* CONFIG_SW64 */ + static struct ftrace_ops direct; static int __init ftrace_direct_multi_init(void) diff --git a/samples/ftrace/ftrace-direct-too.c b/samples/ftrace/ftrace-direct-too.c index e39c3563ae4e..8d4bf161bda8 100644 --- a/samples/ftrace/ftrace-direct-too.c +++ b/samples/ftrace/ftrace-direct-too.c @@ -125,6 +125,52 @@ asm ( #endif /* CONFIG_LOONGARCH */ +#ifdef CONFIG_SW64 + +asm ( +" .pushsection .text, \"ax\", @progbits\n" +" .type my_tramp, @function\n" +" .globl my_tramp\n" +" my_tramp:\n" +" subl $sp, 0x10, $sp\n" +" stl $26, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save $28 & fp of caller */ +" subl $sp, 0x10, $sp\n" +" stl $28, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save other regs */ +" subl $sp, 0x38, $sp\n" +" stl $16, 0($sp)\n" +" stl $17, 0x8($sp)\n" +" stl $18, 0x10($sp)\n" +" stl $19, 0x18($sp)\n" +" stl $26, 0x20($sp)\n" +" stl $28, 0x28($sp)\n" +" stl $29, 0x30($sp)\n" +" br $27, 1f\n" +"1: ldgp $29, 0($27)\n" +" call my_direct_func\n" + /* restore all regs */ +" ldl $16, 0($sp)\n" +" ldl $17, 0x8($sp)\n" +" ldl $18, 0x10($sp)\n" +" stl $19, 0x18($sp)\n" +" ldl $26, 0x20($sp)\n" +" ldl $28, 0x28($sp)\n" +" ldl $29, 0x30($sp)\n" +" addl $sp, 0x38, $sp\n" +" ldl $15, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ret $31, ($28), 1\n" +" .size my_tramp, .-my_tramp\n" +" .popsection\n" +); + +#endif /* CONFIG_SW64 */ + static struct ftrace_ops direct; static int __init ftrace_direct_init(void) diff --git a/samples/ftrace/ftrace-direct.c b/samples/ftrace/ftrace-direct.c index 32c477da1e9a..4e062e97fe01 100644 --- a/samples/ftrace/ftrace-direct.c +++ b/samples/ftrace/ftrace-direct.c @@ -110,6 +110,46 @@ asm ( #endif /* CONFIG_LOONGARCH */ +#ifdef CONFIG_SW64 + +asm ( +" .pushsection .text, \"ax\", @progbits\n" +" .type my_tramp, @function\n" +" .globl my_tramp\n" +" my_tramp:\n" +" subl $sp, 0x10, $sp\n" +" stl $26, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save $28 & fp of caller */ +" subl $sp, 0x10, $sp\n" +" stl $28, 0($sp)\n" +" stl $15, 0x8($sp)\n" +" ldi $15, 0($sp)\n" + /* save other regs */ +" subl $sp, 0x20, $sp\n" +" stl $16, 0($sp)\n" +" stl $26, 0x8($sp)\n" +" stl $28, 0x10($sp)\n" +" stl $29, 0x18($sp)\n" +" br $27, 1f\n" +"1: ldgp $29, 0($27)\n" +" call my_direct_func\n" + /* restore all regs */ +" ldl $16, 0($sp)\n" +" ldl $26, 0x8($sp)\n" +" ldl $28, 0x10($sp)\n" +" ldl $29, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ldl $15, 0x18($sp)\n" +" addl $sp, 0x20, $sp\n" +" ret $31, ($28), 1\n" +" .size my_tramp, .-my_tramp\n" +" .popsection\n" +); + +#endif /* CONFIG_SW64 */ + static struct ftrace_ops direct; static int __init ftrace_direct_init(void) -- Gitee From 2603442c00cf319aebdc3ca640d689b169249f0b Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Wed, 28 May 2025 22:30:40 +0800 Subject: [PATCH 498/524] sw64: optimize unaligned access Since junzhang_v2 supports hardware unalinged access, this patch selects HAVE_EFFICIENT_UNALIGNED_ACCESS and enables hw_una_enabled, which can improve the efficiency of unaligned access. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/kernel/setup.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 071c1d2ae37e..25f77fc27603 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -93,6 +93,7 @@ config SW64 select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS select HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_EBPF_JIT select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 27f3d3b25bbe..f83fd7d7c42c 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -576,6 +576,12 @@ static void __init setup_firmware_fdt(void) } } +static void __init setup_cpu_caps(void) +{ + if (!IS_ENABLED(CONFIG_SUBARCH_C3B) && !is_junzhang_v1()) + static_branch_enable(&hw_una_enabled); +} + static void __init setup_legacy_io(void) { if (is_guest_or_emul()) { @@ -679,6 +685,9 @@ setup_arch(char **cmdline_p) /* Early initialization for device tree */ setup_firmware_fdt(); + /* Now we know who we are, setup caps */ + setup_cpu_caps(); + /* Now we get the final boot_command_line */ *cmdline_p = boot_command_line; -- Gitee From 9f1db3fbe139924b91b220fbaf28271e7165c971 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Thu, 29 May 2025 16:56:08 +0800 Subject: [PATCH 499/524] sw64: disable EFFICIENT_UNALIGNED_ACCESS for C3B Since C3B does not support hardware unaligned access, disable the EFFICIENT_UNALIGNED_ACCESS for C3B. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 25f77fc27603..c0bd7ace57c1 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -93,7 +93,7 @@ config SW64 select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS select HAVE_DYNAMIC_FTRACE_WITH_REGS - select HAVE_EFFICIENT_UNALIGNED_ACCESS + select HAVE_EFFICIENT_UNALIGNED_ACCESS if !SUBARCH_C3B select HAVE_EBPF_JIT select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD -- Gitee From 851931dc58f4bc9cd1bf4360a912b65d0d0a5191 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Tue, 8 Apr 2025 08:40:38 +0000 Subject: [PATCH 500/524] sw64: iommu: implement map_pages/unmap_pages API Implement sunway_iommu map_pages/unmap_pages API, as commit f9de146eb206 ("iommu: Retire map/unmap ops") removes original map/unmap API. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/iommu.c | 69 ++++++++++++++++++++++++++--------- drivers/iommu/sw64/iommu_v2.c | 69 ++++++++++++++++++++++++++--------- 2 files changed, 102 insertions(+), 36 deletions(-) diff --git a/drivers/iommu/sw64/iommu.c b/drivers/iommu/sw64/iommu.c index dfb0c79fc0ad..de135a7e648b 100644 --- a/drivers/iommu/sw64/iommu.c +++ b/drivers/iommu/sw64/iommu.c @@ -857,7 +857,6 @@ sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, unsigned long *pte_l2, unmapped; pr_debug("%s iova %#lx, page_size %#lx\n", __func__, iova, page_size); - BUG_ON(!is_power_of_2(page_size)); unmapped = 0; while (unmapped < page_size) { @@ -1162,39 +1161,73 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) return paddr; } -static int -sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, - phys_addr_t paddr, size_t page_size, int iommu_prot, gfp_t gfp) +static int sunway_iommu_map_pages(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int prot, gfp_t gfp, size_t *mapped) { - struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); - int ret; + struct sunway_iommu_domain *sdomain = to_sunway_domain(domain); + size_t size = pgcount << __ffs(pgsize); + unsigned long mapped_size = 0; + int ret = 0; /* * As VFIO cannot distinguish between normal DMA request * and pci device BAR, check should be introduced manually * to avoid VFIO trying to map pci config space. */ - if (iova >= SW64_BAR_ADDRESS) + if (iova >= SW64_BAR_ADDRESS) { + if (mapped) + *mapped = size; return 0; + } + + if (!(pgsize & domain->pgsize_bitmap)) { + pr_err("pgsize: %lx not supported.\n", pgsize); + return -EINVAL; + } + + while (mapped_size < size) { + ret = sunway_iommu_map_page(sdomain, iova, paddr, pgsize, prot); + if (ret) + goto out; + + iova += pgsize; + paddr += pgsize; + mapped_size += pgsize; + } - ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size, iommu_prot); + if (mapped) + *mapped = size; +out: return ret; } -static size_t -sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, - size_t page_size, struct iommu_iotlb_gather *gather) +static size_t sunway_iommu_unmap_pages(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, + struct iommu_iotlb_gather *iotlb_gather) { - struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); - size_t unmap_size; + struct sunway_iommu_domain *sdomain = to_sunway_domain(domain); + size_t size = pgcount << __ffs(pgsize); + unsigned long unmapped_size = 0; + unsigned long unmap_size; if (iova >= SW64_BAR_ADDRESS) - return page_size; + return size; - unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); + if (!(pgsize & domain->pgsize_bitmap)) { + pr_err("pgsize: %lx not supported.\n", pgsize); + return -EINVAL; + } + + while (unmapped_size < size) { + unmap_size = sunway_iommu_unmap_page(sdomain, iova, pgsize); + + iova += unmap_size; + unmapped_size += unmap_size; + } - return unmap_size; + return size; } static struct iommu_group *sunway_iommu_device_group(struct device *dev) @@ -1364,8 +1397,8 @@ const struct iommu_ops sunway_iommu_ops = { .def_domain_type = sunway_iommu_def_domain_type, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = sunway_iommu_attach_device, - .map = sunway_iommu_map, - .unmap = sunway_iommu_unmap, + .map_pages = sunway_iommu_map_pages, + .unmap_pages = sunway_iommu_unmap_pages, .iova_to_phys = sunway_iommu_iova_to_phys, .free = sunway_iommu_domain_free, } diff --git a/drivers/iommu/sw64/iommu_v2.c b/drivers/iommu/sw64/iommu_v2.c index b73439dd13e5..7e363bfbdaaf 100644 --- a/drivers/iommu/sw64/iommu_v2.c +++ b/drivers/iommu/sw64/iommu_v2.c @@ -1117,7 +1117,6 @@ sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, int tmp = 1; pr_debug("%s iova %#lx, page_size %#lx\n", __func__, iova, page_size); - BUG_ON(!is_power_of_2(page_size)); switch (page_size) { case (1UL << 33): @@ -1445,12 +1444,15 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) paddr += iova & ~PAGE_MASK; return paddr; } -static int -sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, - phys_addr_t paddr, size_t page_size, int iommu_prot, gfp_t gfp) + +static int sunway_iommu_map_pages(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int prot, gfp_t gfp, size_t *mapped) { - struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); - int ret; + struct sunway_iommu_domain *sdomain = to_sunway_domain(domain); + size_t size = pgcount << __ffs(pgsize); + unsigned long mapped_size = 0; + int ret = 0; /* * 3.5G ~ 4G currently is seen as PCI 32-bit MEMIO space. In theory, @@ -1470,7 +1472,7 @@ sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, * users at the same time. So users can quickly learn if they are using * these "illegal" IOVA and thus change their strategies accordingly. */ - if ((SW64_32BIT_DMA_LIMIT < iova + page_size) + if ((SW64_32BIT_DMA_LIMIT < iova + size) && (iova <= DMA_BIT_MASK(32))) { pr_warn_once("process %s (pid:%d) is using domain %d with IOVA: %lx\n", current->comm, current->pid, sdomain->id, iova); @@ -1482,6 +1484,8 @@ sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, */ if (iova >= SW64_BAR_ADDRESS) { pr_warn_once("Domain %d are using IOVA: %lx\n", sdomain->id, iova); + if (mapped) + *mapped = size; return 0; } @@ -1491,20 +1495,39 @@ sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, return -EFAULT; } - ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size, iommu_prot); + if (!(pgsize & domain->pgsize_bitmap)) { + pr_err("pgsize: %lx not supported\n", pgsize); + return -EINVAL; + } + while (mapped_size < size) { + ret = sunway_iommu_map_page(sdomain, iova, paddr, pgsize, prot); + if (ret) + goto out; + + iova += pgsize; + paddr += pgsize; + mapped_size += pgsize; + } + + if (mapped) + *mapped = size; + +out: return ret; } -static size_t -sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, - size_t page_size, struct iommu_iotlb_gather *gather) +static size_t sunway_iommu_unmap_pages(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, + struct iommu_iotlb_gather *iotlb_gather) { - struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); - size_t unmap_size; + struct sunway_iommu_domain *sdomain = to_sunway_domain(domain); + size_t size = pgcount << __ffs(pgsize); + unsigned long unmapped_size = 0; + unsigned long unmap_size; if (iova >= SW64_BAR_ADDRESS) - return page_size; + return size; /* IOMMU v2 supports 42 bit mapped address width */ if (iova >= MAX_IOVA_WIDTH) { @@ -1512,9 +1535,19 @@ sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, return 0; } - unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); + if (!(pgsize & domain->pgsize_bitmap)) { + pr_err("pgsize: %lx not supported\n", pgsize); + return -EINVAL; + } + + while (unmapped_size < size) { + unmap_size = sunway_iommu_unmap_page(sdomain, iova, pgsize); + + iova += unmap_size; + unmapped_size += unmap_size; + } - return unmap_size; + return size; } static struct iommu_group *sunway_iommu_device_group(struct device *dev) @@ -1700,8 +1733,8 @@ const struct iommu_ops sunway_iommu_ops = { .def_domain_type = sunway_iommu_def_domain_type, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = sunway_iommu_attach_device, - .map = sunway_iommu_map, - .unmap = sunway_iommu_unmap, + .map_pages = sunway_iommu_map_pages, + .unmap_pages = sunway_iommu_unmap_pages, .iova_to_phys = sunway_iommu_iova_to_phys, .free = sunway_iommu_domain_free, } -- Gitee From a07eaaafd36774537956f4096c409ff7e07f2c62 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Thu, 29 May 2025 17:37:48 +0800 Subject: [PATCH 501/524] sw64: sort arch/sw64/Kconfig Sort the configs in arch/sw64/Kconfig alphabetically. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index c0bd7ace57c1..11271d570693 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -67,8 +67,8 @@ config SW64 select GENERIC_CLOCKEVENTS select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO select GENERIC_IOREMAP if SUBARCH_C4 + select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select GENERIC_IRQ_LEGACY - select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select GENERIC_IRQ_MIGRATION if SMP select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW @@ -93,8 +93,8 @@ config SW64 select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS select HAVE_DYNAMIC_FTRACE_WITH_REGS - select HAVE_EFFICIENT_UNALIGNED_ACCESS if !SUBARCH_C3B select HAVE_EBPF_JIT + select HAVE_EFFICIENT_UNALIGNED_ACCESS if !SUBARCH_C3B select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_ERROR_INJECTION @@ -102,6 +102,7 @@ config SW64 select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER select HAVE_GENERIC_VDSO if MMU && 64BIT + select HAVE_HARDLOCKUP_DETECTOR_PERF if PERF_EVENTS && HAVE_PERF_EVENTS_NMI select HAVE_IDE select HAVE_KPROBES select HAVE_KPROBES_ON_FTRACE @@ -114,7 +115,6 @@ config SW64 select HAVE_PCSPKR_PLATFORM select HAVE_PERF_EVENTS select HAVE_PERF_EVENTS_NMI if SUBARCH_C4 - select HAVE_HARDLOCKUP_DETECTOR_PERF if PERF_EVENTS && HAVE_PERF_EVENTS_NMI select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_REGS_AND_STACK_ACCESS_API -- Gitee From ec48c19a95adeb5e2df9972bb0835b6540a7c3fe Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Tue, 3 Jun 2025 10:07:55 +0800 Subject: [PATCH 502/524] sw64: set module region to the 2GB space below vmalloc area The module region used to be allocated across the entire vmalloc area using generic module_alloc function. If a considerable portion of the vmalloc area is occupied by a large application, the sections of the same module are scattered beyond the relocation limit of the module. Architecture-specific module_alloc is implemented, moving modules to the 2GB region before vmalloc_start. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 4 +++- arch/sw_64/kernel/module.c | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 0ad6f12c3275..0092d88c199f 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -78,7 +78,9 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) /* Number of pointers that fit on a page: this will go away. */ #define PTRS_PER_PAGE (1UL << (PAGE_SHIFT - 3)) -#define VMALLOC_START (-2 * PGDIR_SIZE) +#define MODULES_VADDR 0xfffff00000000000 +#define MODULES_END 0xfffff0007fffffff +#define VMALLOC_START 0xfffff00080000000 #ifndef CONFIG_SPARSEMEM_VMEMMAP #define VMALLOC_END (-PGDIR_SIZE) #else diff --git a/arch/sw_64/kernel/module.c b/arch/sw_64/kernel/module.c index 67264e3644a7..fb7c61c1b481 100644 --- a/arch/sw_64/kernel/module.c +++ b/arch/sw_64/kernel/module.c @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include +#include +#include #define DEBUGP(fmt...) @@ -277,3 +280,10 @@ apply_relocate_add(Elf64_Shdr *sechdrs, const char *strtab, return 0; } + +void *module_alloc(unsigned long size) +{ + return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, + GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, + NUMA_NO_NODE, __builtin_return_address(0)); +} -- Gitee From 00f00180b53c5d836afd654a644347b83645897e Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 10 Jun 2025 09:31:34 +0800 Subject: [PATCH 503/524] sw64: pintc: fix guest pintc init Ioremap now map I/O physical addresses to the vmalloc segment instead of use __va() . Guest passes size 0 to ioremap, it will return NULL. Actually, the guest will not use this io address, so we skip this address check directly. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-pintc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index 6aaebea7c74b..9d8af2f042ad 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -671,7 +671,7 @@ static int __init pintc_acpi_init_mcu(struct pintc_chip_data *chip_data, chip_data->mcu_irq_num = mcu->gsi_count; chip_data->mcu_base = ioremap(mcu->address, mcu->size); - if (!chip_data->mcu_base) { + if (!chip_data->vt && !chip_data->mcu_base) { pr_err("failed to map mcu base address\n"); ret = -ENXIO; goto out_acpi_free_fwnode; @@ -762,7 +762,7 @@ int __init pintc_acpi_init(struct irq_domain *parent, chip_data->version = pintc->version; chip_data->pintc_base = ioremap(pintc->address, pintc->size); - if (!chip_data->pintc_base) { + if (!virtual && !chip_data->pintc_base) { pr_err("failed to map pintc base address\n"); ret = -ENXIO; goto out_acpi_free_chip_data; -- Gitee From 3a335e68adc24fb9d110c3d96863410bc01c48bb Mon Sep 17 00:00:00 2001 From: Yu Jiayi Date: Fri, 6 Jun 2025 10:50:19 +0800 Subject: [PATCH 504/524] sw64: kvm: fix undefined compile issues for kvm module If use 'CONFIG_KVM=m' to compile, smp_rcb and run_mode_guest_key are undefined. Their use is introduced by optimizing the guest interrupt. This patch we export smp_rcb definition to symbol table and avoid run_mode_guest_key call in guest interrupt notification. Signed-off-by: Yu Jiayi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 1 + arch/sw_64/kvm/kvm_core4.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 11c442ed0f14..93122a011f66 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -22,6 +22,7 @@ #include "proto.h" struct smp_rcb_struct *smp_rcb; +EXPORT_SYMBOL(smp_rcb); extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c index 57f1176472f3..4b6376f3c039 100644 --- a/arch/sw_64/kvm/kvm_core4.c +++ b/arch/sw_64/kvm/kvm_core4.c @@ -126,6 +126,7 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) static int feat_vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number) { + int rcid; int cpu = vcpu->cpu; int me = smp_processor_id(); @@ -136,7 +137,9 @@ static int feat_vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number) && cpu_online(cpu)) { if (vcpu->arch.vcb.vcpu_irq_disabled) return 0; - send_ipi(cpu, II_II1); + /* send_ipi */ + rcid = cpu_to_rcid(cpu); + sendii(rcid, II_II1, 0); } } else kvm_vcpu_kick(vcpu); -- Gitee From 638e5d6933486cbea4c18b86e64601ed150e3051 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Fri, 6 Jun 2025 23:44:59 +0800 Subject: [PATCH 505/524] sw64: ftrace: restore the register storing direct call address The register storing direct called trampoline's address is only saved, but not restored. This commit fixes this issue. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry-ftrace.S | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S index 5dbae4991875..9274ca8d9769 100644 --- a/arch/sw_64/kernel/entry-ftrace.S +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -341,6 +341,7 @@ ftrace_regs_call: RESTORE_GRAPH_REG_ARGS call ftrace_graph_caller #endif + ldl $2, PT_REGS_R2($sp) // load direct call addr RESTORE_PT_REGS bne $2, .Ldirect ret $31, ($28), 1 -- Gitee From 69513184b2d14c2e944bcb96724651427520a34e Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 28 Apr 2025 10:59:33 +0800 Subject: [PATCH 506/524] sw64: cpufreq: optimize the granularity of cpufreq Optimize the granularity of cpufreq according to the actual clock domain to improve system energy efficiency. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/cpu.h | 20 +++++- arch/sw_64/kernel/cpu.c | 32 +++++----- arch/sw_64/kernel/setup.c | 2 +- arch/sw_64/kernel/time.c | 2 +- arch/sw_64/lib/udelay.c | 4 +- drivers/clocksource/timer-sw64.c | 20 ++---- drivers/cpufreq/sunway-cpufreq.c | 103 ++++++++++++++++--------------- 7 files changed, 97 insertions(+), 86 deletions(-) diff --git a/arch/sw_64/include/asm/cpu.h b/arch/sw_64/include/asm/cpu.h index 37368a9d4d52..9fc739dacfab 100644 --- a/arch/sw_64/include/asm/cpu.h +++ b/arch/sw_64/include/asm/cpu.h @@ -7,6 +7,9 @@ #include #include +#define KHZ (1000UL) +#define MHZ (1000UL * 1000UL) + #define current_cpu_data cpu_data[smp_processor_id()] enum hmcall_cpuid_cmd { @@ -48,10 +51,23 @@ extern struct cpuinfo_sw64 cpu_data[NR_CPUS]; extern cpumask_t cpu_offline; extern void store_cpu_data(int cpu); -extern unsigned long get_cpu_freq(void); -extern void update_cpu_freq(unsigned long khz); extern unsigned int get_cpu_cache_size(int cpu, int level, enum cache_type type); extern unsigned int get_cpu_cacheline_size(int cpu, int level, enum cache_type type); +/* Hz */ +static inline unsigned long sunway_max_cpu_freq(void) +{ + return cpuid(GET_CPU_FREQ, 0) * MHZ; +} + +#ifdef CONFIG_SW64_CPUFREQ +extern unsigned long get_cpu_freq(unsigned int cpu); +#else +static inline unsigned long get_cpu_freq(unsigned int cpu) +{ + return sunway_max_cpu_freq(); +} +#endif + #endif /* _ASM_SW64_CPU_H */ diff --git a/arch/sw_64/kernel/cpu.c b/arch/sw_64/kernel/cpu.c index 538f1fcfb686..172cca36e42d 100644 --- a/arch/sw_64/kernel/cpu.c +++ b/arch/sw_64/kernel/cpu.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -40,25 +41,11 @@ EXPORT_SYMBOL(cpu_data); cpumask_t cpu_offline = CPU_MASK_NONE; -static unsigned long cpu_freq; static unsigned long cpu_info; static __u16 family; static char vendor_id[64]; static char model_id[64]; -unsigned long get_cpu_freq(void) -{ - if (likely(cpu_freq)) - return cpu_freq; - - return cpuid(GET_CPU_FREQ, 0) * 1000UL * 1000UL; -} - -void update_cpu_freq(unsigned long khz) -{ - cpu_freq = khz * 1000; -} - /* Move global data into per-processor storage */ void store_cpu_data(int cpu) { @@ -240,7 +227,7 @@ static int show_cpuinfo(struct seq_file *f, void *slot) unsigned int l3_cache_size, l3_cachline_size; unsigned long freq; - freq = cpuid(GET_CPU_FREQ, 0); + freq = sunway_max_cpu_freq() / MHZ; for_each_online_cpu(i) { l3_cache_size = get_cpu_cache_size(i, 3, CACHE_TYPE_UNIFIED); @@ -267,7 +254,7 @@ static int show_cpuinfo(struct seq_file *f, void *slot) "cache size\t: %u KB\n" "physical id\t: %d\n" "bogomips\t: %lu.%02lu\n", - get_cpu_freq() / 1000 / 1000, l3_cache_size >> 10, + get_cpu_freq(i) / 1000 / 1000, l3_cache_size >> 10, cpu_topology[i].package_id, loops_per_jiffy / (500000/HZ), (loops_per_jiffy / (5000/HZ)) % 100); @@ -313,3 +300,16 @@ bool arch_match_cpu_phys_id(int cpu, u64 phys_id) return phys_id == cpu_physical_id(cpu); } +#ifdef CONFIG_SW64_CPUFREQ + +unsigned long get_cpu_freq(unsigned int cpu) +{ + unsigned long freq = cpufreq_quick_get(cpu); + + if (likely(freq)) + return freq * KHZ; + + return sunway_max_cpu_freq(); +} + +#endif diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index f83fd7d7c42c..03974c001bf9 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -133,7 +133,7 @@ EXPORT_SYMBOL(screen_info); #ifdef CONFIG_HARDLOCKUP_DETECTOR_PERF u64 hw_nmi_get_sample_period(int watchdog_thresh) { - return get_cpu_freq() * watchdog_thresh; + return sunway_max_cpu_freq() * watchdog_thresh; } #endif diff --git a/arch/sw_64/kernel/time.c b/arch/sw_64/kernel/time.c index 503f9e1beb43..c6cefd4383b5 100644 --- a/arch/sw_64/kernel/time.c +++ b/arch/sw_64/kernel/time.c @@ -31,7 +31,7 @@ time_init(void) { unsigned long cycle_freq; - cycle_freq = get_cpu_freq(); + cycle_freq = sunway_max_cpu_freq(); pr_info("CPU Cycle frequency = %ld Hz\n", cycle_freq); diff --git a/arch/sw_64/lib/udelay.c b/arch/sw_64/lib/udelay.c index c60601d56b0e..ba5efe4b4549 100644 --- a/arch/sw_64/lib/udelay.c +++ b/arch/sw_64/lib/udelay.c @@ -32,7 +32,7 @@ EXPORT_SYMBOL(__delay); void udelay(unsigned long usecs) { - unsigned long loops = usecs * get_cpu_freq() / 1000000; + unsigned long loops = usecs * get_cpu_freq(smp_processor_id()) / 1000000; unsigned long tmp; __asm__ __volatile__( @@ -47,7 +47,7 @@ EXPORT_SYMBOL(udelay); void ndelay(unsigned long nsecs) { - unsigned long loops = nsecs * get_cpu_freq() / 1000000000; + unsigned long loops = nsecs * get_cpu_freq(smp_processor_id()) / 1000000000; unsigned long tmp; __asm__ __volatile__( diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index 9af19cb362fc..d925d2650bde 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -174,7 +174,7 @@ void __init sw64_setup_clocksource(void) else clocksource_register_khz(&clocksource_vtime, mclk_khz); #else - clocksource_register_hz(&clocksource_tc, get_cpu_freq()); + clocksource_register_hz(&clocksource_tc, sunway_max_cpu_freq()); pr_info("Setup clocksource TC, mult = %d\n", clocksource_tc.mult); #endif } @@ -208,23 +208,13 @@ void __init setup_sched_clock(void) sc_shift = 7; step = 1UL << sc_shift; - sc_multi = step * NSEC_PER_SEC / get_cpu_freq(); + sc_multi = step * NSEC_PER_SEC / sunway_max_cpu_freq(); calibrate_sched_clock(); pr_info("sched_clock: sc_multi=%llu, sc_shift=%llu\n", sc_multi, sc_shift); } -#ifdef CONFIG_GENERIC_SCHED_CLOCK -static u64 notrace read_sched_clock(void) -{ - return (rdtc() - sc_start) >> sc_shift; -} - -void __init sw64_sched_clock_init(void) -{ - sched_clock_register(sched_clock_read, BITS_PER_LONG, get_cpu_freq() >> sc_shift); -} -#else /* !CONFIG_GENERIC_SCHED_CLOCK */ +#ifndef CONFIG_GENERIC_SCHED_CLOCK /* * scheduler clock - returns current time in nanoseconds. */ @@ -428,7 +418,7 @@ void sw64_setup_timer(void) struct clock_event_device *swevt = &per_cpu(timer_events, cpu); /* min_delta ticks => 100ns */ - min_delta = get_cpu_freq()/1000/1000/10; + min_delta = sunway_max_cpu_freq()/1000/1000/10; if (is_in_guest()) { memcpy(swevt, &vtimer_clockevent, sizeof(*swevt)); @@ -443,7 +433,7 @@ void sw64_setup_timer(void) } swevt->cpumask = cpumask_of(cpu); swevt->set_state_shutdown(swevt); - clockevents_config_and_register(swevt, get_cpu_freq(), min_delta, ULONG_MAX); + clockevents_config_and_register(swevt, sunway_max_cpu_freq(), min_delta, ULONG_MAX); } void sw64_timer_interrupt(void) diff --git a/drivers/cpufreq/sunway-cpufreq.c b/drivers/cpufreq/sunway-cpufreq.c index 41a6b21ff827..ba12dad3b28b 100644 --- a/drivers/cpufreq/sunway-cpufreq.c +++ b/drivers/cpufreq/sunway-cpufreq.c @@ -77,9 +77,6 @@ struct cpufreq_frequency_table freq_table[] = { FV(2850, 0), {0, 0, CPUFREQ_TABLE_END}, }; -static void __init fill_freq_table(struct cpufreq_frequency_table *ft) -{ -} #endif #ifdef CONFIG_PLATFORM_XUELANG @@ -135,11 +132,14 @@ static void __init fill_freq_table(struct cpufreq_frequency_table *ft) static unsigned int sunway_get_rate(struct cpufreq_policy *policy) { - int i; + int i, node; u64 val; - void __iomem *spbu_base = misc_platform_get_spbu_base(0); + void __iomem *spbu_base; struct cpufreq_frequency_table *ft = policy->freq_table; + node = per_cpu(hard_node_id, policy->cpu); + spbu_base = misc_platform_get_spbu_base(node); + /* PLL2 provides working frequency for core */ val = readq(spbu_base + OFFSET_CLK_CTL) >> CORE_PLL2_CFG_SHIFT; val &= CORE_PLL2_CFG_MASK; @@ -147,7 +147,7 @@ static unsigned int sunway_get_rate(struct cpufreq_policy *policy) for (i = 0; ft[i].frequency != CPUFREQ_TABLE_END; i++) { if (val == i) { if (ft[i].frequency == CPUFREQ_ENTRY_INVALID) - return cpuid(GET_CPU_FREQ, 0) * 1000UL; + return sunway_max_cpu_freq() / KHZ; return ft[i].frequency; } } @@ -155,40 +155,42 @@ static unsigned int sunway_get_rate(struct cpufreq_policy *policy) return 0; } -static int sunway_set_rate(unsigned int index) +static int sunway_set_rate(struct cpufreq_policy *policy, unsigned int index) { - int i, retry, cpu_num; + int retry, node; void __iomem *spbu_base; - cpu_num = sw64_chip->get_cpu_num(); - for (i = 0; i < cpu_num; i++) { - spbu_base = misc_platform_get_spbu_base(i); - - /* select PLL0/PLL1 */ - writeq(CLK_LV1_SEL_PROTECT, spbu_base + OFFSET_CLU_LV1_SEL); - /* reset PLL2 */ - writeq(CLK2_PROTECT | CORE_CLK2_RESET | CORE_CLK2_VALID, spbu_base + OFFSET_CLK_CTL); - /* configure PLL2_CFG */ - writeq(CLK2_PROTECT | CORE_CLK2_VALID | (unsigned long)index << CORE_PLL2_CFG_SHIFT, - spbu_base + OFFSET_CLK_CTL); - udelay(1); - /* reset over */ - writeq(CORE_CLK2_VALID, spbu_base + OFFSET_CLK_CTL); - retry = 0; - while (retry < MAX_RETRY) { - if (readq(spbu_base + OFFSET_CLK_CTL) & CORE_CLK2_LOCK) - break; - retry++; - udelay(100); - } - if (retry == MAX_RETRY) - return -ETIME; - /* configure over */ - writeq(0, spbu_base + OFFSET_CLK_CTL); - /* select PLL2/PLL2 */ - writeq(CLK_LV1_SEL_MUXA | CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PROTECT, - spbu_base + OFFSET_CLU_LV1_SEL); + node = per_cpu(hard_node_id, policy->cpu); + spbu_base = misc_platform_get_spbu_base(node); + + /* select PLL0/PLL1 */ + writeq(CLK_LV1_SEL_PROTECT, spbu_base + OFFSET_CLU_LV1_SEL); + /* reset PLL2 */ + writeq(CLK2_PROTECT | CORE_CLK2_RESET | CORE_CLK2_VALID, spbu_base + OFFSET_CLK_CTL); + /* configure PLL2_CFG */ + writeq(CLK2_PROTECT | CORE_CLK2_VALID | (unsigned long)index << CORE_PLL2_CFG_SHIFT, + spbu_base + OFFSET_CLK_CTL); + + udelay(1); + /* reset over */ + writeq(CORE_CLK2_VALID, spbu_base + OFFSET_CLK_CTL); + retry = 0; + while (retry < MAX_RETRY) { + if (readq(spbu_base + OFFSET_CLK_CTL) & CORE_CLK2_LOCK) + break; + retry++; + udelay(100); } + + if (retry == MAX_RETRY) + return -ETIME; + + /* configure over */ + writeq(0, spbu_base + OFFSET_CLK_CTL); + /* select PLL2/PLL2 */ + writeq(CLK_LV1_SEL_MUXA | CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PROTECT, + spbu_base + OFFSET_CLU_LV1_SEL); + return 0; } @@ -205,9 +207,6 @@ static unsigned int sunway_cpufreq_get(unsigned int cpu) return sunway_get_rate(policy); } -/* - * Here we notify other drivers of the proposed change and the final change. - */ static int sunway_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) { @@ -218,24 +217,27 @@ static int sunway_cpufreq_target(struct cpufreq_policy *policy, return -ENODEV; /* setting the cpu frequency */ - ret = sunway_set_rate(index); + ret = sunway_set_rate(policy, index); if (ret) return ret; - update_cpu_freq(freq_table[index].frequency); return 0; } static int sunway_cpufreq_init(struct cpufreq_policy *policy) { - cpufreq_generic_init(policy, freq_table, 0); + int cpu, node; - return 0; -} + node = per_cpu(hard_node_id, policy->cpu); -static int sunway_cpufreq_verify(struct cpufreq_policy_data *policy) -{ - return cpufreq_frequency_table_verify(policy, freq_table); + for_each_possible_cpu(cpu) { + if (per_cpu(hard_node_id, cpu) == node) + cpumask_set_cpu(cpu, policy->cpus); + } + + policy->freq_table = freq_table; + + return 0; } static int sunway_cpufreq_exit(struct cpufreq_policy *policy) @@ -251,7 +253,7 @@ static struct cpufreq_driver sunway_cpufreq_driver = { .name = "sunway-cpufreq", .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, .init = sunway_cpufreq_init, - .verify = sunway_cpufreq_verify, + .verify = cpufreq_generic_frequency_table_verify, .target_index = sunway_cpufreq_target, .get = sunway_cpufreq_get, .exit = sunway_cpufreq_exit, @@ -261,14 +263,17 @@ static struct cpufreq_driver sunway_cpufreq_driver = { static int __init cpufreq_init(void) { int i, ret; - unsigned long max_rate = get_cpu_freq() / 1000; + unsigned long max_rate = sunway_max_cpu_freq() / KHZ; /* KHz */ if (!is_in_host()) { pr_warn("cpufreq driver of Sunway platforms is only supported in host mode\n"); return -ENODEV; } +#ifdef CONFIG_PLATFORM_XUELANG fill_freq_table(freq_table); +#endif + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { if (max_rate == freq_table[i].frequency) freq_table[i+1].frequency = CPUFREQ_TABLE_END; -- Gitee From 6dc099b1b74dfff0f3bf0d8be02e48394388baa8 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 31 Dec 2024 17:00:38 +0800 Subject: [PATCH 507/524] sw64: irqchip: add support for PINTC on all NUMA nodes Now, devices located outside NUMA node 0 can also report interrupts to PINTC. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hw_irq.h | 2 - drivers/irqchip/irq-sunway-pintc.c | 106 +++++++++++++++++------------ 2 files changed, 61 insertions(+), 47 deletions(-) diff --git a/arch/sw_64/include/asm/hw_irq.h b/arch/sw_64/include/asm/hw_irq.h index 59418a5fb1d3..b0167e9fd814 100644 --- a/arch/sw_64/include/asm/hw_irq.h +++ b/arch/sw_64/include/asm/hw_irq.h @@ -9,8 +9,6 @@ DECLARE_PER_CPU(unsigned long, irq_pmi_count); #define ACTUAL_NR_IRQS nr_irqs -extern struct irq_domain *mcu_irq_domain; - #ifdef CONFIG_PCI_MSI typedef unsigned int vector_irq_t[PERCPU_MSI_IRQS]; DECLARE_PER_CPU(vector_irq_t, vector_irq); diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index 9d8af2f042ad..8631c7c2fdde 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -67,7 +67,7 @@ #define OFFSET_ADR_INT_CONFIG 0x1580UL /* PINTC version >= 2 */ #define OFFSET_DEVINTWK_INTEN 0x1600UL -#define SW_PINTC_MCU_GSI_BASE 64 +#define PINTC_GSI_BASE(node) (64 + 512 * (node)) #define INTPU_BASE_V1 0x802a00000000 #define INTPU_SIZE_V1 0x1680 @@ -75,6 +75,8 @@ #define MCU_BASE_V1 0x803000000000 #define MCU_SIZE_V1 0x8f00 +#define SUNWAY_MAX_NODES 8 + DECLARE_PER_CPU(unsigned long, hard_node_id); struct pintc_chip_data { @@ -89,13 +91,13 @@ struct pintc_chip_data { raw_spinlock_t mcu_lock; }; -static struct pintc_chip_data *chip_datas[MAX_NUMNODES]; +static struct pintc_chip_data *chip_datas[SUNWAY_MAX_NODES]; static struct pintc_chip_data *pintc_alloc_chip_data(u32 node) { struct pintc_chip_data *chip_data; - if (WARN_ON(node >= MAX_NUMNODES)) + if (WARN_ON(node >= SUNWAY_MAX_NODES)) return NULL; chip_data = kzalloc_node(sizeof(struct pintc_chip_data), @@ -111,7 +113,7 @@ static void pintc_free_chip_data(struct pintc_chip_data *chip_data) if (!chip_data) return; - if (WARN_ON((chip_data->node >= MAX_NUMNODES) || + if (WARN_ON((chip_data->node >= SUNWAY_MAX_NODES) || (chip_datas[chip_data->node] != chip_data))) return; @@ -285,13 +287,14 @@ static int mcu_irq_set_affinity(struct irq_data *irq_data, return assign_mcu_irq_config(chip_data, &targets); } -static struct irq_chip pintc_mcu_chip = { - .name = "MCU-INT", - .irq_enable = mcu_irq_enable, - .irq_disable = mcu_irq_disable, - .irq_mask = mcu_irq_disable, - .irq_unmask = mcu_irq_enable, - .irq_set_affinity = mcu_irq_set_affinity, +static struct irq_chip pintc_mcu_chips[SUNWAY_MAX_NODES] = { + [0 ... SUNWAY_MAX_NODES - 1] = { + .irq_enable = mcu_irq_enable, + .irq_disable = mcu_irq_disable, + .irq_mask = mcu_irq_disable, + .irq_unmask = mcu_irq_enable, + .irq_set_affinity = mcu_irq_set_affinity, + } }; static struct irq_chip pintc_mcu_vt_chip = { @@ -303,6 +306,9 @@ static int pintc_mcu_translate(struct irq_domain *domain, unsigned long *hwirq, unsigned int *type) { + struct pintc_chip_data *chip_data = domain->host_data; + u32 node = chip_data->node; + if (WARN_ON(fwspec->param_count < 1)) return -EINVAL; @@ -315,9 +321,9 @@ static int pintc_mcu_translate(struct irq_domain *domain, /* ACPI */ if (is_fwnode_irqchip(fwspec->fwnode)) { - if (WARN_ON(fwspec->param[0] < SW_PINTC_MCU_GSI_BASE)) + if (WARN_ON(fwspec->param[0] < PINTC_GSI_BASE(node))) return -EINVAL; - *hwirq = fwspec->param[0] - SW_PINTC_MCU_GSI_BASE; + *hwirq = fwspec->param[0] - PINTC_GSI_BASE(node); *type = IRQ_TYPE_NONE; return 0; } @@ -376,13 +382,14 @@ static const struct irq_domain_ops pintc_mcu_domain_ops = { .free = pintc_mcu_free_irqs, }; -struct irq_domain *mcu_irq_domain; -EXPORT_SYMBOL(mcu_irq_domain); +static struct irq_domain *mcu_domains[SUNWAY_MAX_NODES]; static int __init pintc_init_mcu(struct pintc_chip_data *chip_data, struct fwnode_handle *handle) { unsigned int mcu_irq_num = chip_data->mcu_irq_num; + u32 node = chip_data->node; + char *chip_name; if (chip_data->vt) { chip_data->mcu_chip = &pintc_mcu_vt_chip; @@ -392,25 +399,37 @@ static int __init pintc_init_mcu(struct pintc_chip_data *chip_data, * "irq_domain_create_legacy", we have to call * "__irq_domain_add" directly. */ - mcu_irq_domain = __irq_domain_add(handle, mcu_irq_num, + mcu_domains[node] = __irq_domain_add(handle, mcu_irq_num, mcu_irq_num, 0, &pintc_mcu_domain_ops, chip_data); - if (mcu_irq_domain) - irq_domain_associate_many(mcu_irq_domain, + if (mcu_domains[node]) + irq_domain_associate_many(mcu_domains[node], 0, 0, mcu_irq_num); } else { - chip_data->mcu_chip = &pintc_mcu_chip; - mcu_irq_domain = irq_domain_create_linear(handle, mcu_irq_num, + chip_data->mcu_chip = &pintc_mcu_chips[node]; + + chip_name = kzalloc_node(sizeof("MCUX-INT"), GFP_KERNEL, node); + if (!chip_name) + return -ENOMEM; + + snprintf(chip_name, sizeof("MCUX-INT"), "MCU%u-INT", node); + chip_data->mcu_chip->name = chip_name; + + mcu_domains[node] = irq_domain_create_linear(handle, mcu_irq_num, &pintc_mcu_domain_ops, chip_data); + /* Mask all interrupts for now */ writeq(0x0, chip_data->mcu_base + OFFSET_MCU_DVC_INT_EN); - /* When building the root domain, move it to a better location */ - if (mcu_irq_domain) + if (mcu_domains[node]) pintc_mcu_enable(chip_data->pintc_base); + else { + chip_data->mcu_chip->name = NULL; + kfree(chip_name); + } } - if (!mcu_irq_domain) { + if (!mcu_domains[node]) { pr_err("failed to create MCU irq domain\n"); return -ENOMEM; } @@ -424,24 +443,24 @@ static int __init pintc_init_mcu(struct pintc_chip_data *chip_data, return 0; } -/* Currently, only MCU controller on node 0 is supported */ void handle_dev_int(struct pt_regs *regs) { unsigned long stat, val; unsigned int hwirq; + u32 node = __this_cpu_read(hard_node_id); /* Disable global irq of MCU due to some hardware reasons */ - val = pintc_mcu_disable_and_save(chip_datas[0]); + val = pintc_mcu_disable_and_save(chip_datas[node]); - stat = readq(chip_datas[0]->mcu_base + OFFSET_MCU_DVC_INT); + stat = readq(chip_datas[node]->mcu_base + OFFSET_MCU_DVC_INT); while (stat) { hwirq = ffs(stat) - 1; - generic_handle_domain_irq(mcu_irq_domain, hwirq); + generic_handle_domain_irq(mcu_domains[node], hwirq); stat &= ~(1UL << hwirq); } - pintc_mcu_restore(chip_datas[0], val); + pintc_mcu_restore(chip_datas[node], val); } void handle_fault_int(void) @@ -487,13 +506,6 @@ void handle_fault_int(void) static int __init pintc_of_init_mcu(struct pintc_chip_data *chip_data, struct device_node *pintc) { - /* Not yet supported */ - if (chip_data->node > 0) { - pr_info("MCU version [%u] on node [%u] skipped\n", - chip_data->version, chip_data->node); - return 0; - } - return pintc_init_mcu(chip_data, of_node_to_fwnode(pintc)); } @@ -608,6 +620,7 @@ enum sw_pintc_sub_type { static int __init lpc_intc_parse_madt(union acpi_subtable_headers *header, const unsigned long end) { + static bool initialized[SUNWAY_MAX_NODES]; struct acpi_madt_sw_lpc_intc *lpc_intc; lpc_intc = (struct acpi_madt_sw_lpc_intc *)header; @@ -616,13 +629,22 @@ static int __init lpc_intc_parse_madt(union acpi_subtable_headers *header, if (lpc_intc->node > 0) return 0; + if (initialized[lpc_intc->node]) + return 0; + + /* LPC controller is the child of MCU controller */ + if (!mcu_domains[lpc_intc->node]) + return 0; + if ((lpc_intc->version == ACPI_MADT_SW_LPC_INTC_VERSION_NONE) || (lpc_intc->version >= ACPI_MADT_SW_LPC_INTC_VERSION_RESERVED)) { pr_err("invalid LPC-INTC version\n"); return -EINVAL; } - return lpc_intc_acpi_init(mcu_irq_domain, lpc_intc); + initialized[lpc_intc->node] = true; + + return lpc_intc_acpi_init(mcu_domains[lpc_intc->node], lpc_intc); } static bool __init @@ -642,22 +664,16 @@ static int __init pintc_acpi_init_mcu(struct pintc_chip_data *chip_data, struct acpi_madt_sw_sub_pintc *mcu) { struct fwnode_handle *handle; + u32 node = chip_data->node; int ret; - /* Not yet supported */ - if (chip_data->node > 0) { - pr_info("MCU version [%u] on node [%u] skipped\n", - chip_data->version, chip_data->node); - return 0; - } - if (!mcu->status) { pr_info("MCU version [%u] on node [%u] disabled\n", chip_data->version, chip_data->node); return 0; } - if (mcu->gsi_base != SW_PINTC_MCU_GSI_BASE) { + if (mcu->gsi_base != PINTC_GSI_BASE(node)) { pr_err("invalid MCU GSI\n"); return -EINVAL; } @@ -694,7 +710,7 @@ static int __init pintc_acpi_init_mcu(struct pintc_chip_data *chip_data, return 0; out_acpi_free_mcu_domain: - irq_domain_remove(mcu_irq_domain); + irq_domain_remove(mcu_domains[node]); out_acpi_unmap_mcu: iounmap(chip_data->mcu_base); out_acpi_free_fwnode: -- Gitee From 2b28e9729c52845b56ff98023be7d4aab3663a19 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 28 May 2025 14:47:01 +0800 Subject: [PATCH 508/524] sw64: irqchip: update effective affinity for PINTC Since CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK is selected, it is necessary to update effective affinity when setting affinity for PINTC. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sunway-pintc.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-sunway-pintc.c b/drivers/irqchip/irq-sunway-pintc.c index 8631c7c2fdde..9e87b84015f6 100644 --- a/drivers/irqchip/irq-sunway-pintc.c +++ b/drivers/irqchip/irq-sunway-pintc.c @@ -243,9 +243,10 @@ static void update_pintc_mcu_target(struct pintc_chip_data *chip_data, raw_spin_unlock_irqrestore(&chip_data->pintc_lock, flags); } -static int assign_mcu_irq_config(struct pintc_chip_data *chip_data, +static int assign_mcu_irq_config(struct irq_data *irq_data, cpumask_t *targets) { + struct pintc_chip_data *chip_data = irq_data->chip_data; unsigned long dev_int_tar; unsigned int cpu; int rcid; @@ -270,13 +271,14 @@ static int assign_mcu_irq_config(struct pintc_chip_data *chip_data, dev_int_tar = make_pintc_int_target(chip_data->version, rcid); update_pintc_mcu_target(chip_data, dev_int_tar); + irq_data_update_effective_affinity(irq_data, cpumask_of(cpu)); + return 0; } static int mcu_irq_set_affinity(struct irq_data *irq_data, const struct cpumask *dest, bool force) { - struct pintc_chip_data *chip_data = irq_data->chip_data; cpumask_t targets; if (cpumask_any_and(dest, cpu_online_mask) >= nr_cpu_ids) @@ -284,7 +286,7 @@ static int mcu_irq_set_affinity(struct irq_data *irq_data, cpumask_and(&targets, dest, cpu_online_mask); - return assign_mcu_irq_config(chip_data, &targets); + return assign_mcu_irq_config(irq_data, &targets); } static struct irq_chip pintc_mcu_chips[SUNWAY_MAX_NODES] = { -- Gitee From 3116f821f4a0335fa172db5fb3bcfde4db8f83f6 Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Mon, 16 Jun 2025 22:06:52 +0800 Subject: [PATCH 509/524] sw64: adjust the notrace in the code Change "void notrace *" to "void * notrace" to make the syntax more standardized. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/lib/string.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/sw_64/lib/string.c b/arch/sw_64/lib/string.c index 5eed2016dc61..16f5b37a1eca 100644 --- a/arch/sw_64/lib/string.c +++ b/arch/sw_64/lib/string.c @@ -19,14 +19,14 @@ static inline void *____memcpy(void *dest, const void *src, size_t n) return ____memcpy_simd_align(dest, src, n); } -void notrace *memcpy(void *dest, const void *src, size_t n) +void * notrace memcpy(void *dest, const void *src, size_t n) { return ____memcpy(dest, src, n); } EXPORT_SYMBOL(memcpy); /* For backward compatibility with modules. Unused otherwise. */ -void notrace *__memcpy(void *dest, const void *src, size_t n) +void * notrace __memcpy(void *dest, const void *src, size_t n) { return ____memcpy(dest, src, n); } @@ -47,12 +47,12 @@ static inline void *____constant_c_memset(void *s, unsigned long c, size_t n) return ____constant_c_memset_simd_align(s, c, n); } -void notrace *__constant_c_memset(void *s, unsigned long c, size_t n) +void * notrace __constant_c_memset(void *s, unsigned long c, size_t n) { return ____constant_c_memset(s, c, n); } -void notrace *___memset(void *s, int c, size_t n) +void * notrace ___memset(void *s, int c, size_t n) { unsigned long c_ul = (c & 0xff) * 0x0101010101010101UL; @@ -60,7 +60,7 @@ void notrace *___memset(void *s, int c, size_t n) } EXPORT_SYMBOL(___memset); -void notrace *__memset(void *s, int c, size_t n) +void * notrace __memset(void *s, int c, size_t n) { unsigned long c_ul = (c & 0xff) * 0x0101010101010101UL; @@ -68,7 +68,7 @@ void notrace *__memset(void *s, int c, size_t n) } EXPORT_SYMBOL(__memset); -void notrace *memset(void *s, int c, size_t n) +void * notrace memset(void *s, int c, size_t n) { unsigned long c_ul = (c & 0xff) * 0x0101010101010101UL; @@ -76,7 +76,7 @@ void notrace *memset(void *s, int c, size_t n) } EXPORT_SYMBOL(memset); -void notrace *__memsetw(void *dest, unsigned short c, size_t count) +void * notrace __memsetw(void *dest, unsigned short c, size_t count) { unsigned long c_ul = (c & 0xffff) * 0x0001000100010001UL; -- Gitee From 1d6510edebda7b3fb0097b6b2619b17514715fc9 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Tue, 17 Jun 2025 18:46:31 +0800 Subject: [PATCH 510/524] sw64: pci: add base address of devmn There are some registers for configuring PCIe controller in the devmn module on Sunway platform. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 4 +++- drivers/pci/controller/pci-sunway.c | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 6e8d0ed37567..e05b54585e55 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -83,6 +83,8 @@ enum SUNWAY_RC { SUNWAY_RC_SIZE = 0x82500UL, }; +#define SUNWAY_DEVMN_SIZE 0x3480UL + struct pci_dev; struct pci_bus; struct resource; @@ -108,11 +110,11 @@ struct pci_controller { unsigned long sparse_io_base; unsigned long dense_io_base; - /* This one's for the kernel only. It's in KSEG somewhere. */ void __iomem *ep_config_space_base; void __iomem *rc_config_space_base; void __iomem *piu_ior0_base; void __iomem *piu_ior1_base; + void __iomem *devmn_base; unsigned long index; unsigned long node; diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 76514c5695cb..01291d02ae03 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -779,6 +779,7 @@ enum pci_props { PROP_PIU_IOR1_BASE, PROP_RC_INDEX, PROP_PCIE_IO_BASE, + PROP_DEVMN_BASE, PROP_NUM }; @@ -803,7 +804,8 @@ static const char *prop_names[PROP_NUM] = { "sunway,piu-ior0-base", "sunway,piu-ior1-base", "sunway,rc-index", - "sunway,pcie-io-base" + "sunway,pcie-io-base", + "sunway,devmn-base" }; #ifdef CONFIG_NUMA @@ -838,6 +840,9 @@ static int pci_prepare_controller(struct pci_controller *hose, /* Get necessary properties of Root Complex */ for (i = 0; i < PROP_NUM; ++i) { + if ((i == PROP_DEVMN_BASE) && !is_junzhang_v3()) + break; + ret = fwnode_property_read_u64(fwnode, prop_names[i], &props[i]); @@ -869,6 +874,9 @@ static int pci_prepare_controller(struct pci_controller *hose, hose->piu_ior0_base = ioremap(props[PROP_PIU_IOR0_BASE], SUNWAY_PIU_IOR0_SIZE); hose->piu_ior1_base = ioremap(props[PROP_PIU_IOR1_BASE], SUNWAY_PIU_IOR1_SIZE); + if (is_junzhang_v3()) + hose->devmn_base = ioremap(props[PROP_DEVMN_BASE], SUNWAY_DEVMN_SIZE); + hose->first_busno = 0xff; hose->last_busno = 0xff; hose->self_busno = 0xff; -- Gitee From 50e817c378c3f63d652910972051f40cb1c83da4 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 18 Jun 2025 16:50:11 +0800 Subject: [PATCH 511/524] sw64: acpi: select HAVE_ACPI_APEI when ACPI is set Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/include/asm/acpi.h | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 11271d570693..e3ea3f531fc1 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -81,6 +81,7 @@ config SW64 select GPIOLIB if ACPI select HANDLE_DOMAIN_IRQ select HARDIRQS_SW_RESEND + select HAVE_ACPI_APEI if ACPI select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KGDB diff --git a/arch/sw_64/include/asm/acpi.h b/arch/sw_64/include/asm/acpi.h index e386d07d1e73..82982759effa 100644 --- a/arch/sw_64/include/asm/acpi.h +++ b/arch/sw_64/include/asm/acpi.h @@ -15,6 +15,12 @@ extern int acpi_strict; extern int acpi_disabled; extern int acpi_pci_disabled; +/* + * acpi_disable_cmcff is only used in IA-32 Architecture, this + * definition is only for compatibility. + */ +#define acpi_disable_cmcff 1 + /* _ASM_SW64_PDC_H */ #define ACPI_PDC_P_FFH (0x0001) #define ACPI_PDC_C_C1_HALT (0x0002) -- Gitee From 88ab35577503e07ffcecc0c378881141c4a26e4a Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Thu, 26 Jun 2025 09:03:47 +0800 Subject: [PATCH 512/524] sw64: fix for executing syscall(-1) If a task executes syscall(-1), we intercept this early and force r0 to be -ENOSYS so that we don't need to distinguish this scenario from one where the nr is -1 because a tracer wants to skip the system call using ptrace. With the return value set, the return path is the same as the skip case. Signed-off-by: Gu Zitao Reviewed-by: He Sheng --- arch/sw_64/kernel/sys_sw64.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/arch/sw_64/kernel/sys_sw64.c b/arch/sw_64/kernel/sys_sw64.c index c11c02285bf3..8daed9829ec5 100644 --- a/arch/sw_64/kernel/sys_sw64.c +++ b/arch/sw_64/kernel/sys_sw64.c @@ -349,6 +349,23 @@ asmlinkage void noinstr do_entSys(struct pt_regs *regs) nr = regs->regs[0]; if (ti_flags & _TIF_SYSCALL_WORK) { + /* + * The de-facto standard way to skip a system call using ptrace + * is to set the system call to -1 (NO_SYSCALL) and set r0 to a + * suitable error code for consumption by userspace. However, + * this cannot be distinguished from a user-issued syscall(-1) + * and so we must set r0 to -ENOSYS here in case the tracer doesn't + * issue the skip and we fall into trace_exit with r0 preserved. + * + * This is slightly odd because it also means that if a tracer + * sets the system call number to -1 but does not initialise r0, + * then r0 will be preserved for all system calls apart from a + * user-issued syscall(-1). However, requesting a skip and not + * setting the return value is unlikely to do anything sensible + * anyway. + */ + if (nr == NO_SYSCALL) + syscall_set_return_value(current, regs, -ENOSYS, 0); nr = syscall_trace_enter(); if (nr == NO_SYSCALL) goto syscall_out; -- Gitee From 221eeec10598b557afdab6e09b4419462c3dce90 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Thu, 19 Jun 2025 03:51:27 +0000 Subject: [PATCH 513/524] sw64: pci: remove zx200 dma mask reset Remove the unnecessary reset of zx200's dma mask to DMA_BIT_MASK(32). Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/pci/pci.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index deb5d8d052d4..54acde3f5a29 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -236,19 +236,6 @@ static void fixup_root_complex(struct pci_dev *dev) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, fixup_root_complex); -static int setup_bus_dma_cb(struct pci_dev *pdev, void *data) -{ - pdev->dev.bus_dma_limit = DMA_BIT_MASK(32); - return 0; -} - -static void fix_bus_dma_limit(struct pci_dev *dev) -{ - pci_walk_bus(dev->subordinate, setup_bus_dma_cb, NULL); - pr_info("Set zx200 bus_dma_limit to 32-bit\n"); -} -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ZHAOXIN, 0x071f, fix_bus_dma_limit); - #ifdef CONFIG_DCA static void enable_sw_dca(struct pci_dev *dev) { -- Gitee From 15d90873ad82733db9cb00489abe4d9919c65347 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Fri, 27 Jun 2025 08:01:35 +0000 Subject: [PATCH 514/524] sw64: Kconfig: enable HAS_IOPORT by default This patch extends commit fcbfe8121a45("Kconfig: introduce HAS_IOPORT option and select it as necessary") to sw64. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index e3ea3f531fc1..48f9c74e50ea 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -81,6 +81,7 @@ config SW64 select GPIOLIB if ACPI select HANDLE_DOMAIN_IRQ select HARDIRQS_SW_RESEND + select HAS_IOPORT select HAVE_ACPI_APEI if ACPI select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_JUMP_LABEL -- Gitee From fab05d5226a32f22c44af0e85584783416e53646 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Mon, 30 Jun 2025 16:15:44 +0800 Subject: [PATCH 515/524] sw64: kvm: fix perf kvm support This patch: - Register callback for monitoring guest info to enable perf kvm correctly. - For core4, we determine the running mode of the vcpu through the CPU mode bit of the PC rather than PS. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_host.h | 10 ++++++++++ arch/sw_64/kvm/Kconfig | 1 + arch/sw_64/kvm/kvm_core3.c | 2 ++ arch/sw_64/kvm/kvm_core4.c | 2 ++ arch/sw_64/kvm/perf.c | 13 +++++-------- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index 63b344ed0359..cf0009f2d365 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -199,4 +199,14 @@ long kvm_sw64_get_vcb(struct file *filp, unsigned long arg); void update_aptp(unsigned long pgd); void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu); + +/* + * Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event, + * arrived in guest context. + */ +static inline bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu) +{ + return IS_ENABLED(CONFIG_GUEST_PERF_EVENTS) && !!vcpu; +} + #endif /* _ASM_SW64_KVM_HOST_H */ diff --git a/arch/sw_64/kvm/Kconfig b/arch/sw_64/kvm/Kconfig index 5fac5b1d9fb4..124cead28b53 100644 --- a/arch/sw_64/kvm/Kconfig +++ b/arch/sw_64/kvm/Kconfig @@ -32,6 +32,7 @@ config KVM select GENERIC_ALLOCATOR select KVM_GENERIC_DIRTYLOG_READ_PROTECT select HAVE_KVM_DIRTY_RING_ACQ_REL + select GUEST_PERF_EVENTS if PERF_EVENTS help Support for hosting Guest kernels. We don't support KVM with 3-level page tables yet. diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c index 26c27c69ba7b..378831c21139 100644 --- a/arch/sw_64/kvm/kvm_core3.c +++ b/arch/sw_64/kvm/kvm_core3.c @@ -376,6 +376,8 @@ static int __init kvm_core3_init(void) for (i = 0; i < NR_CPUS; i++) last_vpn(i) = VPN_FIRST_VERSION; + kvm_register_perf_callbacks(NULL); + ret = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); if (likely(!ret)) diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c index 4b6376f3c039..98fdbfe88fed 100644 --- a/arch/sw_64/kvm/kvm_core4.c +++ b/arch/sw_64/kvm/kvm_core4.c @@ -220,6 +220,8 @@ static int __init kvm_core4_init(void) for (i = 0; i < NR_CPUS; i++) last_vpn(i) = VPN_FIRST_VERSION; + kvm_register_perf_callbacks(NULL); + ret = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); if (ret) diff --git a/arch/sw_64/kvm/perf.c b/arch/sw_64/kvm/perf.c index 730dd1feeccf..7817f1e199e7 100644 --- a/arch/sw_64/kvm/perf.c +++ b/arch/sw_64/kvm/perf.c @@ -11,17 +11,14 @@ bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) { - return (vcpu->arch.regs.ps & 8) != 0; + if (IS_ENABLED(CONFIG_SUBARCH_C3B)) + return (vcpu->arch.regs.ps & 0x8) == 0; + else + return (vcpu->arch.regs.pc & 0x3) == 0x2; + } unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) { return vcpu->arch.regs.pc; } - - - -static inline bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu) -{ - return IS_ENABLED(CONFIG_GUEST_PERF_EVENTS) && !!vcpu; -} -- Gitee From e368c7bd4b22d81697d59b8c992c9bda9a15394b Mon Sep 17 00:00:00 2001 From: Yu Jiayi Date: Fri, 6 Jun 2025 16:17:36 +0800 Subject: [PATCH 516/524] sw64: delete useless I/O access in VM and emulator RC and PIU components are not used in VM or emulator, so we add judgment to skip the I/O access of RC and PIU components. Signed-off-by: Yu Jiayi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/pci/pci.c | 16 ++++++++++------ drivers/irqchip/irq-sunway-pci-intx.c | 5 ++--- drivers/pci/controller/pci-sunway.c | 13 +++++++------ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c index 54acde3f5a29..48aff446e3a7 100644 --- a/arch/sw_64/pci/pci.c +++ b/arch/sw_64/pci/pci.c @@ -370,8 +370,10 @@ static void sunway_pci_root_bridge_prepare(struct pci_host_bridge *bridge) bridge->swizzle_irq = pci_common_swizzle; bridge->map_irq = sunway_pci_map_irq; - init_busnr = (0xff << 16) + ((last_bus + 1) << 8) + (last_bus); - writel(init_busnr, (hose->rc_config_space_base + RC_PRIMARY_BUS)); + if (!is_guest_or_emul()) { + init_busnr = (0xff << 16) + ((last_bus + 1) << 8) + (last_bus); + writel(init_busnr, (hose->rc_config_space_base + RC_PRIMARY_BUS)); + } hose->first_busno = last_bus + (is_in_host() ? 1 : 0); @@ -399,10 +401,12 @@ void sunway_pci_root_bridge_scan_finish(struct pci_host_bridge *bridge) hose->last_busno = last_bus; hose->busn_space->end = last_bus; - init_busnr = readl(hose->rc_config_space_base + RC_PRIMARY_BUS); - init_busnr &= ~(0xff << 16); - init_busnr |= last_bus << 16; - writel(init_busnr, (hose->rc_config_space_base + RC_PRIMARY_BUS)); + if (!is_guest_or_emul()) { + init_busnr = readl(hose->rc_config_space_base + RC_PRIMARY_BUS); + init_busnr &= ~(0xff << 16); + init_busnr |= last_bus << 16; + writel(init_busnr, (hose->rc_config_space_base + RC_PRIMARY_BUS)); + } pci_bus_update_busn_res_end(bus, last_bus); last_bus++; diff --git a/drivers/irqchip/irq-sunway-pci-intx.c b/drivers/irqchip/irq-sunway-pci-intx.c index bd108d58ff53..6b82872c2346 100644 --- a/drivers/irqchip/irq-sunway-pci-intx.c +++ b/drivers/irqchip/irq-sunway-pci-intx.c @@ -224,11 +224,9 @@ void setup_intx_irqs(struct pci_controller *hose) { unsigned long irq, node, val_node; struct intx_chip_data *chip_data; - void __iomem *piu_ior0_base; int i = 0; node = hose->node; - piu_ior0_base = hose->piu_ior0_base; if (!node_online(node)) val_node = next_node_in(node, node_online_map); @@ -259,7 +257,8 @@ void setup_intx_irqs(struct pci_controller *hose) irq_set_chip_and_handler(irq + 1, &dummy_irq_chip, handle_level_irq); hose->service_irq = irq + 1; - set_pcieport_service_irq(hose); + if (!is_guest_or_emul()) + set_pcieport_service_irq(hose); } void __init sunway_init_pci_intx(void) diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 01291d02ae03..d8e291decf65 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -870,9 +870,13 @@ static int pci_prepare_controller(struct pci_controller *hose, hose->dense_mem_base = props[PROP_PCIE_IO_BASE]; hose->dense_io_base = props[PROP_EP_IO_BASE]; - hose->rc_config_space_base = ioremap(props[PROP_RC_CONFIG_BASE], SUNWAY_RC_SIZE); - hose->piu_ior0_base = ioremap(props[PROP_PIU_IOR0_BASE], SUNWAY_PIU_IOR0_SIZE); - hose->piu_ior1_base = ioremap(props[PROP_PIU_IOR1_BASE], SUNWAY_PIU_IOR1_SIZE); + if (!is_guest_or_emul()) { + hose->rc_config_space_base = ioremap(props[PROP_RC_CONFIG_BASE], SUNWAY_RC_SIZE); + hose->piu_ior0_base = ioremap(props[PROP_PIU_IOR0_BASE], SUNWAY_PIU_IOR0_SIZE); + hose->piu_ior1_base = ioremap(props[PROP_PIU_IOR1_BASE], SUNWAY_PIU_IOR1_SIZE); + hose->piu_ior0 = alloc_piu_saved_data(PIU_IOR0_SAVE_REGS * sizeof(u64)); + hose->piu_ior1 = alloc_piu_saved_data(PIU_IOR1_SAVE_REGS * sizeof(u64)); + } if (is_junzhang_v3()) hose->devmn_base = ioremap(props[PROP_DEVMN_BASE], SUNWAY_DEVMN_SIZE); @@ -883,9 +887,6 @@ static int pci_prepare_controller(struct pci_controller *hose, hose->need_domain_info = 0; - hose->piu_ior0 = alloc_piu_saved_data(PIU_IOR0_SAVE_REGS * sizeof(u64)); - hose->piu_ior1 = alloc_piu_saved_data(PIU_IOR1_SAVE_REGS * sizeof(u64)); - #if IS_ENABLED(CONFIG_PCI_MSI) if (is_in_host()) memset(hose->piu_msiconfig, 0, 256 / 8); /* 256 bits bitmap */ -- Gitee From 241c358da04ba9e341d3c437d759d5305a744604 Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Wed, 2 Jul 2025 16:51:45 +0800 Subject: [PATCH 517/524] sw64: perf unwind: Do not put libunwind-sw_64 in FEATURE_TESTS_BASIC As it is not normally available on x86_64 not being tested on test-all.c but being in FEATURE_TESTS_BASIC ends up implying that those features are present, which leads to trying to link with those libraries and a build failure now that test-all.c is finally again building successfully: /usr/bin/ld: cannot find -lunwind-sw_64 collect2: error: ld returned 1 exit status So remove those features from there and explicitely test them. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/build/Makefile.feature | 2 +- tools/perf/Makefile.config | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index fb290a90b263..2f4a14df3825 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -54,7 +54,6 @@ FEATURE_TESTS_BASIC := \ libtracefs \ libcrypto \ libunwind \ - libunwind-sw_64 \ pthread-attr-setaffinity-np \ pthread-barrier \ reallocarray \ @@ -94,6 +93,7 @@ FEATURE_TESTS_EXTRA := \ libunwind-x86_64 \ libunwind-arm \ libunwind-aarch64 \ + libunwind-sw_64 \ libunwind-debug-frame \ libunwind-debug-frame-arm \ libunwind-debug-frame-aarch64 \ diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 1c2dae9f0152..ff83cc4cc308 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -153,6 +153,7 @@ FEATURE_CHECK_LDFLAGS-libunwind-debug-frame = $(LIBUNWIND_LDFLAGS) $(LIBUNWIND_L FEATURE_CHECK_LDFLAGS-libunwind-arm += -lunwind -lunwind-arm FEATURE_CHECK_LDFLAGS-libunwind-aarch64 += -lunwind -lunwind-aarch64 +FEATURE_CHECK_LDFLAGS-libunwind-sw_64 += -lunwind -lunwind-sw_64 FEATURE_CHECK_LDFLAGS-libunwind-x86 += -lunwind -llzma -lunwind-x86 FEATURE_CHECK_LDFLAGS-libunwind-x86_64 += -lunwind -llzma -lunwind-x86_64 @@ -648,6 +649,8 @@ ifndef NO_LIBUNWIND CFLAGS += -DNO_LIBUNWIND_DEBUG_FRAME_AARCH64 endif endif + + $(call feature_check,libunwind-sw_64) ifeq ($(feature-libunwind-sw_64), 1) $(call detected,CONFIG_LIBUNWIND_SW64) CFLAGS += -DHAVE_LIBUNWIND_SW_64_SUPPORT -- Gitee From 3f3d50d105d2f715ad41a49b6e193a00099e5cf7 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 14 Jul 2025 10:22:14 +0800 Subject: [PATCH 518/524] sw64: fix sw64_rrk_store() wrap handling Make sure we are clearing the rrk area instead of a random area when handling rrk wrap. Fixes commit c1c8fb15df5a ("sw64: improve sw64_rrk"). Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/dup_print.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/dup_print.c b/arch/sw_64/kernel/dup_print.c index fa604abef38c..2d77ccae30d2 100644 --- a/arch/sw_64/kernel/dup_print.c +++ b/arch/sw_64/kernel/dup_print.c @@ -54,7 +54,7 @@ void sw64_rrk_store(const char *text, u16 text_len, u64 ts_nsec, int level, max_offset_allowed = PRINTK_SIZE - text_len - header_len - (newline_first ? 1 : 0); if (unlikely(sw64_printk_offset >= max_offset_allowed)) { sw64_printk_offset = 0; - memset(sw64_printk_buf, 0, PRINTK_SIZE); + memset((void *)KERNEL_PRINTK_BUFF_BASE, 0, PRINTK_SIZE); wrap = true; } sw64_printk_buf = (char *)(KERNEL_PRINTK_BUFF_BASE + sw64_printk_offset); -- Gitee From b7cd930b6d8674971b335000b3ff8a4425f4d770 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 4 Jul 2024 11:31:16 +0800 Subject: [PATCH 519/524] sw64: use wait/complete for smp boot Use wait and complete functions instead of udelay for smp boot synchronization. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 93122a011f66..53aeb4c9792f 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -21,6 +21,8 @@ #include "proto.h" +static DECLARE_COMPLETION(cpu_running); + struct smp_rcb_struct *smp_rcb; EXPORT_SYMBOL(smp_rcb); @@ -135,15 +137,13 @@ void smp_callin(void) save_ktp(); upshift_freq(); cpuid = smp_processor_id(); - local_irq_disable(); + WARN_ON_ONCE(!irqs_disabled()); if (cpu_online(cpuid)) { pr_err("??, cpu 0x%x already present??\n", cpuid); BUG(); } - set_cpu_online(cpuid, true); - /* Set trap vectors. */ trap_init(); @@ -182,6 +182,10 @@ void smp_callin(void) store_cpu_topology(cpuid); numa_add_cpu(cpuid); + set_cpu_online(cpuid, true); + + complete(&cpu_running); + /* Must have completely accurate bogos. */ local_irq_enable(); @@ -205,7 +209,6 @@ static inline void set_secondary_ready(int cpuid) */ static int secondary_cpu_start(int cpuid, struct task_struct *idle) { - unsigned long timeout; /* * Precalculate the target ksp. */ @@ -222,17 +225,12 @@ static int secondary_cpu_start(int cpuid, struct task_struct *idle) #endif /* Wait 10 seconds for secondary cpu. */ - timeout = jiffies + 10*HZ; - while (time_before(jiffies, timeout)) { - if (cpu_online(cpuid)) - goto started; - udelay(10); - barrier(); + if (!wait_for_completion_timeout(&cpu_running, + msecs_to_jiffies(10000))) { + pr_err("SMP: Processor %d failed to start.\n", cpuid); + return -1; } - pr_err("SMP: Processor %d failed to start.\n", cpuid); - return -1; -started: return 0; } -- Gitee From 86588612ea0fa507df1e8455dd08419eae9a2c26 Mon Sep 17 00:00:00 2001 From: Gu Yuchen Date: Tue, 15 Jul 2025 10:06:46 +0800 Subject: [PATCH 520/524] sw64: ftrace: add HAVE_DYNAMIC_FTRACE_WITH_ARGS support Allow for arguments to be passed in to ftrace_regs by default. If this is set, then arguments and stack can be found from the pt_regs. Signed-off-by: Gu Yuchen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/include/asm/ftrace.h | 66 +++++++++++++++ arch/sw_64/kernel/entry-ftrace.S | 140 +++++++++++++++---------------- arch/sw_64/kernel/ftrace.c | 12 +++ 4 files changed, 146 insertions(+), 73 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 48f9c74e50ea..e2513a39938a 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -93,6 +93,7 @@ config SW64 select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_BUGVERBOSE select HAVE_DYNAMIC_FTRACE + select HAVE_DYNAMIC_FTRACE_WITH_ARGS select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS select HAVE_DYNAMIC_FTRACE_WITH_REGS select HAVE_EBPF_JIT diff --git a/arch/sw_64/include/asm/ftrace.h b/arch/sw_64/include/asm/ftrace.h index 33122cd7f91b..eb1b05186705 100644 --- a/arch/sw_64/include/asm/ftrace.h +++ b/arch/sw_64/include/asm/ftrace.h @@ -46,6 +46,70 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) return addr; } +#ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS +struct ftrace_ops; + +struct ftrace_regs { + struct pt_regs regs; +}; + +static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs *fregs) +{ + return &fregs->regs; +} + +static __always_inline unsigned long +ftrace_regs_get_instruction_pointer(const struct ftrace_regs *fregs) +{ + return fregs->regs.regs[28]; +} + + static __always_inline void +ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs, unsigned long ip) +{ + fregs->regs.regs[27] = ip; + fregs->regs.regs[28] = ip; +} + +static __always_inline unsigned long +ftrace_regs_get_stack_pointer(const struct ftrace_regs *fregs) +{ + return fregs->regs.regs[30]; +} + +static __always_inline unsigned long +ftrace_regs_get_argument(struct ftrace_regs *fregs, unsigned int n) +{ + if (n < 6) + return fregs->regs.regs[16+n]; + return 0; +} + +static __always_inline unsigned long +ftrace_regs_get_return_value(const struct ftrace_regs *fregs) +{ + return fregs->regs.regs[0]; +} + +static __always_inline void +ftrace_regs_set_return_value(struct ftrace_regs *fregs, unsigned long ret) +{ + fregs->regs.regs[0] = ret; +} + +static __always_inline void +ftrace_override_function_with_return(struct ftrace_regs *fregs) +{ + fregs->regs.regs[27] = fregs->regs.regs[26]; + fregs->regs.regs[28] = fregs->regs.regs[26]; +} + +int ftrace_regs_query_register_offset(const char *name); + +#define ftrace_graph_func ftrace_graph_func +void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *op, struct ftrace_regs *fregs); + #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS static inline void __arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr) @@ -57,6 +121,8 @@ __arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr) __arch_ftrace_set_direct_caller(&(fregs)->regs, addr) #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ +#endif + #endif /* ifndef __ASSEMBLY__ */ #ifndef __ASSEMBLY__ diff --git a/arch/sw_64/kernel/entry-ftrace.S b/arch/sw_64/kernel/entry-ftrace.S index 9274ca8d9769..ec3677d40477 100644 --- a/arch/sw_64/kernel/entry-ftrace.S +++ b/arch/sw_64/kernel/entry-ftrace.S @@ -75,7 +75,7 @@ #endif .endm - .macro SAVE_PT_REGS + .macro SAVE_PT_REGS allregs=0 /* save $26 & fp of the function before caller */ subl $sp, 0x10, $sp stl $26, 0($sp) @@ -88,15 +88,11 @@ stl $15, 0x8($sp) ldi $15, 0($sp) - /* save pt_regs */ ldi $sp, -PT_REGS_SIZE($sp) stl $0, PT_REGS_R0($sp) +#ifdef CONFIG_FUNCTION_GRAPH_TRACER stl $9, PT_REGS_R9($sp) - stl $10, PT_REGS_R10($sp) - stl $11, PT_REGS_R11($sp) - stl $12, PT_REGS_R12($sp) - stl $13, PT_REGS_R13($sp) - stl $14, PT_REGS_R14($sp) +#endif stl $15, PT_REGS_R15($sp) stl $16, PT_REGS_R16($sp) stl $17, PT_REGS_R17($sp) @@ -112,6 +108,15 @@ stl $27, PT_REGS_R27($sp) stl $28, PT_REGS_R28($sp) stl $29, PT_REGS_GP($sp) + subl $28, MCOUNT_INSN_SIZE, $0 + stl $0, PT_REGS_PC($sp) + .if \allregs + stl $10, PT_REGS_R10($sp) + stl $11, PT_REGS_R11($sp) + stl $12, PT_REGS_R12($sp) + stl $13, PT_REGS_R13($sp) + stl $14, PT_REGS_R14($sp) + .endif ldi $0, PT_REGS_SIZE($sp) stl $0, PT_REGS_R30($sp) @@ -120,15 +125,12 @@ stl $2, PT_REGS_R2($sp) .endm - .macro RESTORE_PT_REGS + .macro RESTORE_PT_REGS allregs=0 /* restore pt_regs */ ldl $0, PT_REGS_R0($sp) +#ifdef CONFIG_FUNCTION_GRAPH_TRACER ldl $9, PT_REGS_R9($sp) - ldl $10, PT_REGS_R10($sp) - ldl $11, PT_REGS_R11($sp) - ldl $12, PT_REGS_R12($sp) - ldl $13, PT_REGS_R13($sp) - ldl $14, PT_REGS_R14($sp) +#endif ldl $15, PT_REGS_R15($sp) ldl $16, PT_REGS_R16($sp) ldl $17, PT_REGS_R17($sp) @@ -144,11 +146,23 @@ ldl $27, PT_REGS_R27($sp) ldl $28, PT_REGS_R28($sp) ldl $29, PT_REGS_GP($sp) + .if \allregs + ldl $10, PT_REGS_R10($sp) + ldl $11, PT_REGS_R11($sp) + ldl $12, PT_REGS_R12($sp) + ldl $13, PT_REGS_R13($sp) + ldl $14, PT_REGS_R14($sp) + .endif ldi $sp, PT_REGS_SIZE($sp) /* only restore $fp */ ldl $15, 0x18($sp) addl $sp, 0x20, $sp + + bne $2, .Ldirect\@ + ret $31, ($28), 1 +.Ldirect\@: + ret $31, ($2), 1 /* direct call */ .endm .macro RESTORE_GRAPH_REG_ARGS @@ -173,6 +187,40 @@ addl $sp, 16, $sp .endm + .macro ftrace_common allregs=0 + br $27, 1f +1: ldgp $29, 0($27) + + subl $28, MCOUNT_INSN_SIZE, $16 + bis $26, $31, $17 + ldi $4, function_trace_op + ldl $18, 0($4) + mov $sp, $19 + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + bis $31, $16, $9 +#endif + ldi $4, current_tracer + ldl $27, 0($4) + +.if \allregs == 0 + .global ftrace_call +ftrace_call: + nop +.endif + +.if \allregs + .global ftrace_regs_call +ftrace_regs_call: + nop +.endif + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + RESTORE_GRAPH_REG_ARGS + call ftrace_graph_caller +#endif + ldl $2, PT_REGS_R2($sp) // load direct call addr + .endm #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* @@ -243,35 +291,9 @@ _mcount: .global ftrace_caller .ent ftrace_caller ftrace_caller: - mcount_enter - br $27, 1f -1: ldgp $29, 0($27) - - subl $28, MCOUNT_INSN_SIZE, $16 - bis $26, $31, $17 - ldl $18, function_trace_op - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - /* - * the graph tracer (specifically, prepare_ftrace_return) needs these - * arguments but for now the function tracer occupies the regs, so we - * save them in callee-saved regs to recover later. - */ - bis $31, $16, $9 -#endif - ldi $4, current_tracer - ldl $27, 0($4) - - .global ftrace_call -ftrace_call: /* tracer(pc, ra); */ - nop - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - RESTORE_GRAPH_ARGS - call ftrace_graph_caller -#endif - mcount_end - ret $31, ($28), 1 + SAVE_PT_REGS allregs=0 + ftrace_common allregs=0 + RESTORE_PT_REGS allregs=0 .end ftrace_caller #else /* !CONFIG_DYNAMIC_FTRACE */ @@ -316,37 +338,9 @@ skip_ftrace: .global ftrace_regs_caller .ent ftrace_regs_caller ftrace_regs_caller: - SAVE_PT_REGS - br $27, 1f -1: ldgp $29, 0($27) - - subl $28, MCOUNT_INSN_SIZE, $16 - stl $16, PT_REGS_PC($sp) - bis $26, $31, $17 - ldi $4, function_trace_op - ldl $18, 0($4) - mov $sp, $19 - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - bis $31, $16, $9 -#endif - ldi $4, current_tracer - ldl $27, 0($4) - - .global ftrace_regs_call -ftrace_regs_call: - nop - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - RESTORE_GRAPH_REG_ARGS - call ftrace_graph_caller -#endif - ldl $2, PT_REGS_R2($sp) // load direct call addr - RESTORE_PT_REGS - bne $2, .Ldirect - ret $31, ($28), 1 -.Ldirect: - ret $31, ($2), 1 + SAVE_PT_REGS allregs=1 + ftrace_common allregs=1 + RESTORE_PT_REGS allregs=1 .end ftrace_regs_caller #endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ diff --git a/arch/sw_64/kernel/ftrace.c b/arch/sw_64/kernel/ftrace.c index 511760ac18c2..84ba10d93c9f 100644 --- a/arch/sw_64/kernel/ftrace.c +++ b/arch/sw_64/kernel/ftrace.c @@ -175,6 +175,17 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, * Turn on/off the call to ftrace_graph_caller() in ftrace_caller() * depending on @enable. */ +#ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS +void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *op, struct ftrace_regs *fregs) +{ + struct pt_regs *regs = arch_ftrace_get_regs(fregs); + unsigned long *parent = (unsigned long *)®s->regs[26]; + unsigned long frame_pointer = regs->regs[15]; + + prepare_ftrace_return(parent, ip, frame_pointer); +} +#else static int ftrace_modify_graph_caller(bool enable) { unsigned long pc = (unsigned long)&ftrace_graph_call; @@ -194,5 +205,6 @@ int ftrace_disable_ftrace_graph_caller(void) { return ftrace_modify_graph_caller(false); } +#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */ #endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ -- Gitee From 90d749ebd8a022847e941e995f6302bd8f7a2522 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Thu, 17 Jul 2025 10:55:50 +0800 Subject: [PATCH 521/524] sw64: fix guest kdump function Using the kdump function to reset cpu in the guest will cause errors, so the temporary solution is to change the non-zero cpu to enter cpu_relax. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/machine_kexec.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/kernel/machine_kexec.c b/arch/sw_64/kernel/machine_kexec.c index 5d54e7cf6823..9547f12a5299 100644 --- a/arch/sw_64/kernel/machine_kexec.c +++ b/arch/sw_64/kernel/machine_kexec.c @@ -155,8 +155,10 @@ static void machine_crash_nonpanic_core(void *unused) atomic_dec(&waiting_for_crash_ipi); while (READ_ONCE(smp_rcb->ready) != 0) mdelay(1); - if (cpu != 0) - reset_cpu(cpu); + if (cpu != 0) { + while (1) + cpu_relax(); + } else machine_kexec(kexec_crash_image); } @@ -209,8 +211,10 @@ void machine_crash_shutdown(struct pt_regs *regs) pr_info("Loading crashdump kernel...\n"); #ifdef CONFIG_SMP WRITE_ONCE(smp_rcb->ready, 0); - if (cpu != 0) - reset_cpu(cpu); + if (cpu != 0) { + while (1) + cpu_relax(); + } #endif } -- Gitee From ff693a866804878592e7396fc634ed6600798db6 Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Tue, 8 Jul 2025 16:31:40 +0800 Subject: [PATCH 522/524] sw64: kvm: enable dirty log gradually in small chunks for core4 When enabling dirty log for the first time, initialize all the bits of the dirty bitmap to 1, and do not write protect the pages. Instead, set write protection in small chunks during subsequent dirty log clearing, thereby reducing the time taken of kvm->mmu_lock. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_host.h | 4 ++++ arch/sw_64/kvm/mmu.c | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index cf0009f2d365..1d475442b795 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -51,6 +51,10 @@ #define KVM_HALT_POLL_NS_DEFAULT 0 #define KVM_IRQCHIP_NUM_PINS 256 + +#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ + KVM_DIRTY_LOG_INITIALLY_SET) + /* KVM Hugepage definitions for sw64 */ #define KVM_NR_PAGE_SIZES 3 #define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9) diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index 92d10fdbd97c..9a89621987b8 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -546,6 +546,14 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, if (change == KVM_MR_FLAGS_ONLY && (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES) && new->flags & KVM_MEM_LOG_DIRTY_PAGES)) { kvm_mark_migration(kvm, 1); + + /* + * Initially-all-set does not require write protecting any page, + * because they're all assumed to be dirty. + */ + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) + return; + kvm_mmu_wp_memory_region(kvm, new->id); } -- Gitee From 8e9b29fc25319101d3e6f919eefd1892c712311f Mon Sep 17 00:00:00 2001 From: Yu Jiayi Date: Wed, 16 Jul 2025 16:49:12 +0800 Subject: [PATCH 523/524] sw64: kvm: fix undefined error with ftrace disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit 847888a23ae2 ("sw64: kvm: fix perf kvm support") introduces calls to some macros and functions when CONFIG_GUEST_PERF_EVENTS=y. However, the head file that defines them is not included when closed ftrace build option. So, we add the introduction of the head file. Signed-off-by: Yu Jiayi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_host.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index 1d475442b795..35c8bf9fd515 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include -- Gitee From 7b57bb5a479b05761aa56a8929b20e19d57ea249 Mon Sep 17 00:00:00 2001 From: Lei Yilong Date: Tue, 17 Jun 2025 14:16:56 +0800 Subject: [PATCH 524/524] =?UTF-8?q?sw64:=20kvm:=20save=20the=20virtual?= =?UTF-8?q?=C2=A0address=20of=20irqs=5Fpending=20in=20vcpucb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The format of virtual address may change in the future. Add an irqs_addr member in vcb to save the virtual address of irqs_pending, so hmcode can get the correct virtual address by accessing the physical address. To distinguish this feature, change the value of feat_vint to 2. Signed-off-by: Lei Yilong Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/vcpu.h | 8 ++++---- arch/sw_64/kvm/kvm_core4.c | 2 ++ arch/sw_64/kvm/sw64.c | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h index b069031add39..38066427afdb 100644 --- a/arch/sw_64/include/asm/vcpu.h +++ b/arch/sw_64/include/asm/vcpu.h @@ -110,10 +110,10 @@ struct vcpucb { unsigned long csr_earg2; unsigned long csr_scratch; unsigned long atc; - unsigned long reserved1[10]; - /* Pending virtual interrupts */ - DECLARE_BITMAP(irqs_pending, CORE4VM_IRQS); - unsigned long reserved2[31]; + unsigned long reserved1[10]; /* USE IN HMCODE */ + DECLARE_BITMAP(irqs_pending, CORE4VM_IRQS); /* Pending virtual interrupts */ + unsigned long irqs_addr; + unsigned long reserved2[30]; }; #endif diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c index 98fdbfe88fed..cea377ac265b 100644 --- a/arch/sw_64/kvm/kvm_core4.c +++ b/arch/sw_64/kvm/kvm_core4.c @@ -97,6 +97,8 @@ long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) set_timer(vcpu, 200000000); vcpu->arch.vcb.migration_mark = 0; } + + vcpu->arch.vcb.irqs_addr = (unsigned long)&vcpu->arch.vcb.irqs_pending; return 0; } diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index 053115c8284c..1cec96017ef9 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -245,7 +245,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) return -EINVAL; feature_vint = (cpuid(GET_FEATURES, 0) & CPU_FEAT_VINT); - smp_rcb->feat_vint = 1; + smp_rcb->feat_vint = 2; return kvm_sw64_init_vm(kvm); } -- Gitee